fork of https://github.com/sourcegraph/zoekt
indexserver: remove all .tmp in cleanup (#427)
Rather than waiting 4 hours, we can remove all tmp files during cleanup.
We can do this now since cleanup runs while holding a global lock on the
index directory.
In practice we have seen disks filling up with monorepos that repeatedly
fail to index.
Test Plan: added a panic into zoekt-git-index during the Close call
before it renames shards into place. This created a bunch of garbage on
disk. After this change, the garbage is cleaned up regularly. Also go
test.
author
Keegan Carruthers-Smith
committer
GitHub
date
(Sep 23, 2022, 7:03 PM +0200)
commit
580b67ec
580b67ec14b7f5de91a60bd1cc9a7f8d9495dc5a
parent
1a338228
1a338228b6c980b147eb4cb4114f7d7bf60dca7b
+11
-18
+6
-5
cmd/zoekt-sourcegraph-indexserver/cleanup.go
+6
-5
cmd/zoekt-sourcegraph-indexserver/cleanup.go
···
152
152
shardsLog(indexDir, "remove", shards)
153
153
}
154
154
155
-
// Remove old .tmp files from crashed indexer runs-- for example, if
156
-
// an indexer OOMs, it will leave around .tmp files, usually in a loop.
157
-
maxAge := now.Add(-4 * time.Hour)
155
+
// Remove .tmp files from crashed indexer runs-- for example, if an indexer
156
+
// OOMs, it will leave around .tmp files, usually in a loop. We can remove
157
+
// the files now since cleanup runs with a global lock (no indexing jobs
158
+
// running at the same time).
158
159
if failures, err := filepath.Glob(filepath.Join(indexDir, "*.tmp")); err != nil {
159
160
log.Printf("Glob: %v", err)
160
161
} else {
···
164
165
log.Printf("Stat(%q): %v", f, err)
165
166
continue
166
167
}
167
-
if !st.IsDir() && st.ModTime().Before(maxAge) {
168
-
log.Printf("removing old tmp file: %s", f)
168
+
if !st.IsDir() {
169
+
log.Printf("removing tmp file: %s", f)
169
170
os.Remove(f)
170
171
}
171
172
}
+5
-13
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
+5
-13
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
···
60
60
repos []string
61
61
index []shard
62
62
trash []shard
63
-
tmps map[string]time.Time
63
+
tmps []string
64
64
65
65
wantIndex []shard
66
66
wantTrash []shard
67
-
wantTmps []string
68
67
}{{
69
68
name: "noop",
70
69
}, {
···
99
98
wantTrash: []shard{mk("bar", 0, now)},
100
99
}, {
101
100
name: "clean old .tmp files",
102
-
tmps: map[string]time.Time{
103
-
"recent.tmp": recent,
104
-
"old.tmp": old,
105
-
},
106
-
wantTmps: []string{"recent.tmp"},
101
+
tmps: []string{"recent.tmp", "old.tmp"},
107
102
}, {
108
103
name: "all",
109
104
repos: []string{"exists", "trashed"},
···
133
128
t.Fatal(err)
134
129
}
135
130
}
136
-
for name, mtime := range tt.tmps {
131
+
for _, name := range tt.tmps {
137
132
path := filepath.Join(dir, name)
138
133
if _, err := os.Create(path); err != nil {
139
-
t.Fatal(err)
140
-
}
141
-
if err := os.Chtimes(path, mtime, mtime); err != nil {
142
134
t.Fatal(err)
143
135
}
144
136
}
···
155
147
if d := cmp.Diff(tt.wantTrash, glob(filepath.Join(dir, ".trash", "*.zoekt"))); d != "" {
156
148
t.Errorf("unexpected trash (-want, +got):\n%s", d)
157
149
}
158
-
if d := cmp.Diff(tt.wantTmps, globBase(filepath.Join(dir, "*.tmp"))); d != "" {
159
-
t.Errorf("unexpected tmps (-want, +got):\n%s", d)
150
+
if tmps := globBase(filepath.Join(dir, "*.tmp")); len(tmps) > 0 {
151
+
t.Errorf("unexpected tmps: %v", tmps)
160
152
}
161
153
162
154
if testing.Verbose() {