fork of https://github.com/sourcegraph/zoekt
merging: convert feature flag to ENV variable (#217)
So far, shard merging was activated by placing a RIP file in the index
directory. However, we have rolled out the feature to the full cluster
and it makes more sense to use an ENV variable instead.
author
Stefan Hengl
committer
GitHub
date
(Dec 1, 2021, 8:35 AM +0100)
commit
13aa1675
13aa167537285c72830b14730c54cd2bc0ae2ea2
parent
56b87f5b
56b87f5bb8539cb32ae1686c6824644595ec8120
+17
-13
+1
-1
build/builder.go
+1
-1
build/builder.go
···
574
574
575
575
for p := range toDelete {
576
576
// Don't delete compound shards, set tombstones instead.
577
-
if zoekt.TombstonesEnabled(filepath.Dir(p)) && strings.HasPrefix(filepath.Base(p), "compound-") {
577
+
if zoekt.ShardMergingEnabled() && strings.HasPrefix(filepath.Base(p), "compound-") {
578
578
if !strings.HasSuffix(p, ".zoekt") {
579
579
continue
580
580
}
+2
-4
cmd/zoekt-sourcegraph-indexserver/cleanup.go
+2
-4
cmd/zoekt-sourcegraph-indexserver/cleanup.go
···
27
27
// that do not exist in indexDir, but do in indexDir/.trash it will move them
28
28
// back into indexDir. Additionally it uses now to remove shards that have
29
29
// been in the trash for 24 hours. It also deletes .tmp files older than 4 hours.
30
-
func cleanup(indexDir string, repos []uint32, now time.Time) {
30
+
func cleanup(indexDir string, repos []uint32, now time.Time, shardMerging bool) {
31
31
start := time.Now()
32
32
trashDir := filepath.Join(indexDir, ".trash")
33
33
if err := os.MkdirAll(trashDir, 0755); err != nil {
···
36
36
37
37
trash := getShards(trashDir)
38
38
index := getShards(indexDir)
39
-
40
-
tombstonesEnabled := zoekt.TombstonesEnabled(indexDir)
41
39
42
40
// trash: Remove old shards and conflicts with index
43
41
minAge := now.Add(-24 * time.Hour)
···
85
83
_ = os.Chtimes(shard.Path, now, now)
86
84
}
87
85
88
-
if tombstonesEnabled {
86
+
if shardMerging {
89
87
// 1 repo can be split across many simple shards but it should only be contained
90
88
// in 1 compound shard. Hence we check that len(shards)==1 and only consider the
91
89
// shard at index 0.
+1
-1
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
+1
-1
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
···
149
149
for _, name := range tt.repos {
150
150
repoIDs = append(repoIDs, fakeID(name))
151
151
}
152
-
cleanup(dir, repoIDs, now)
152
+
cleanup(dir, repoIDs, now, false)
153
153
154
154
if d := cmp.Diff(tt.wantIndex, glob(filepath.Join(dir, "*.zoekt"))); d != "" {
155
155
t.Errorf("unexpected index (-want, +got):\n%s", d)
+7
-3
cmd/zoekt-sourcegraph-indexserver/main.go
+7
-3
cmd/zoekt-sourcegraph-indexserver/main.go
···
141
141
142
142
// Protects the index directory from concurrent access.
143
143
muIndexDir sync.Mutex
144
+
145
+
// If true, shard merging is enabled.
146
+
shardMerging bool
144
147
}
145
148
146
149
var debug = log.New(ioutil.Discard, "", log.LstdFlags)
···
283
286
go func() {
284
287
defer close(cleanupDone)
285
288
s.muIndexDir.Lock()
286
-
cleanup(s.IndexDir, repos.IDs, time.Now())
289
+
cleanup(s.IndexDir, repos.IDs, time.Now(), s.shardMerging)
287
290
s.muIndexDir.Unlock()
288
291
}()
289
292
···
304
307
305
308
go func() {
306
309
for range jitterTicker(s.VacuumInterval, syscall.SIGUSR1) {
307
-
if zoekt.TombstonesEnabled(s.IndexDir) {
310
+
if s.shardMerging {
308
311
s.vacuum()
309
312
}
310
313
}
···
312
315
313
316
go func() {
314
317
for range jitterTicker(s.MergeInterval, syscall.SIGUSR1) {
315
-
if zoekt.TombstonesEnabled(s.IndexDir) {
318
+
if s.shardMerging {
316
319
err := doMerge(s.IndexDir, s.TargetSizeBytes, s.MaxSizeBytes, false)
317
320
if err != nil {
318
321
log.Printf("error during merging: %s", err)
···
780
783
TargetSizeBytes: *targetSize * 1024 * 1024,
781
784
MaxSizeBytes: *maxSize * 1024 * 1024,
782
785
minSizeBytes: *minSize * 1024 * 1024,
786
+
shardMerging: zoekt.ShardMergingEnabled(),
783
787
}
784
788
785
789
if *debugList {
+6
-4
tombstones.go
+6
-4
tombstones.go
···
5
5
"io/ioutil"
6
6
"os"
7
7
"path/filepath"
8
+
"strconv"
8
9
"syscall"
9
10
)
10
11
11
-
// TombstoneEnabled returns true if a file "RIP" is present in dir.
12
-
func TombstonesEnabled(dir string) bool {
13
-
_, err := os.Stat(filepath.Join(dir, "RIP"))
14
-
return err == nil
12
+
// ShardMergingEnabled returns true if SRC_ENABLE_SHARD_MERGING is set to true.
13
+
func ShardMergingEnabled() bool {
14
+
t := os.Getenv("SRC_ENABLE_SHARD_MERGING")
15
+
enabled, _ := strconv.ParseBool(t)
16
+
return enabled
15
17
}
16
18
17
19
var mockRepos []*Repository