fork of https://github.com/sourcegraph/zoekt
build: delete older index versions of shards
When we have indexed v16 of a repo, we need to remove v15 and
before. Otherwise zoekt-webserver will serve both v15 and v16 of a shard =>
duplicate results.
We considered adding this process to the indexserver. However, the cleanup
process happens asynchronously which would leave quite a lot of time where
zoekt-webserver would serve up both formats.
Change-Id: I45705de9088719767ff28c23c1c03a64a46d467c
author
Keegan Carruthers-Smith
date
(Aug 21, 2019, 4:21 PM +0200)
commit
b6bd1c5c
b6bd1c5c555389a615cbafe1e66e453ad580f283
parent
567a6dce
567a6dce367d8ea278f0141605c8edd27071995a
+31
-1
+19
-1
build/builder.go
+19
-1
build/builder.go
···
177
177
178
178
// ShardName returns the name the given index shard.
179
179
func (o *Options) shardName(n int) string {
180
+
return o.shardNameForVersion(zoekt.IndexFormatVersion, n)
181
+
}
182
+
183
+
func (o *Options) shardNameForVersion(version, n int) string {
180
184
abs := url.QueryEscape(o.RepositoryDescription.Name)
181
185
if len(abs) > 200 {
182
186
abs = abs[:200] + hashString(abs)[:8]
183
187
}
184
188
return filepath.Join(o.IndexDir,
185
-
fmt.Sprintf("%s_v%d.%05d.zoekt", abs, zoekt.IndexFormatVersion, n))
189
+
fmt.Sprintf("%s_v%d.%05d.zoekt", abs, version, n))
186
190
}
187
191
188
192
// IndexVersions returns the versions as present in the index, for
···
318
322
319
323
if b.nextShardNum > 0 {
320
324
b.deleteRemainingShards()
325
+
b.deleteOlderVersionShards()
321
326
}
322
327
return b.buildError
323
328
}
···
329
334
name := b.opts.shardName(shard)
330
335
if err := os.Remove(name); os.IsNotExist(err) {
331
336
break
337
+
}
338
+
}
339
+
}
340
+
341
+
// deleteOlderVersionShards removes shards for repo from earlier
342
+
// IndexFormatVersions.
343
+
func (b *Builder) deleteOlderVersionShards() {
344
+
for version := 0; version < zoekt.IndexFormatVersion; version++ {
345
+
for shard := 0; ; shard++ {
346
+
name := b.opts.shardNameForVersion(version, shard)
347
+
if err := os.Remove(name); os.IsNotExist(err) {
348
+
break
349
+
}
332
350
}
333
351
}
334
352
}
+12
build/e2e_test.go
+12
build/e2e_test.go
···
38
38
t.Fatalf("TempDir: %v", err)
39
39
}
40
40
41
+
// Create an older shard to test we delete it
42
+
oldIndex := filepath.Join(dir, fmt.Sprintf("repo_v%d.00000.zoekt", zoekt.IndexFormatVersion-1))
43
+
if f, err := os.Create(oldIndex); err != nil {
44
+
t.Fatal(err)
45
+
} else {
46
+
f.Close()
47
+
}
48
+
41
49
opts := Options{
42
50
IndexDir: dir,
43
51
ShardMax: 1024,
···
65
73
fs, _ := filepath.Glob(dir + "/*")
66
74
if len(fs) <= 1 {
67
75
t.Fatalf("want multiple shards, got %v", fs)
76
+
}
77
+
78
+
if stat, err := os.Stat(oldIndex); !os.IsNotExist(err) {
79
+
t.Fatalf("old index was not removed: %v %v", stat, err)
68
80
}
69
81
70
82
ss, err := shards.NewDirectorySearcher(dir)