fork of https://github.com/sourcegraph/zoekt
refactor shard name logic (#878)
We had 2 copies of this logic in the code and this bit me when I
introduced id-based shards
indexbuilder.go seems an "ok" place to put. We cannot put it in the
shards package because of circular dependencies.
Test plan:
refactor, so relying on CI
author
Stefan Hengl
committer
GitHub
date
(Dec 18, 2024, 4:20 PM +0100)
commit
dc1b23bb
dc1b23bb0da9a0fa91056ad6d5fcf9b8e641f67d
parent
687cafc8
687cafc8f702e6c0efa0b562b35c0eee619a88d8
+68
-33
+1
-14
build/builder.go
+1
-14
build/builder.go
···
21
21
"crypto/sha1"
22
22
"flag"
23
23
"fmt"
24
-
"io"
25
24
"log"
26
25
"net/url"
27
26
"os"
···
334
333
}
335
334
}
336
335
337
-
func hashString(s string) string {
338
-
h := sha1.New()
339
-
_, _ = io.WriteString(h, s)
340
-
return fmt.Sprintf("%x", h.Sum(nil))
341
-
}
342
-
343
336
// ShardName returns the name the given index shard.
344
337
func (o *Options) shardName(n int) string {
345
338
return o.shardNameVersion(zoekt.IndexFormatVersion, n)
346
339
}
347
340
348
341
func (o *Options) shardNameVersion(version, n int) string {
349
-
prefix := url.QueryEscape(cmp.Or(o.ShardPrefix, o.RepositoryDescription.Name))
350
-
if len(prefix) > 200 {
351
-
prefix = prefix[:200] + hashString(prefix)[:8]
352
-
}
353
-
shardName := filepath.Join(o.IndexDir,
354
-
fmt.Sprintf("%s_v%d.%05d.zoekt", prefix, version, n))
355
-
return shardName
342
+
return zoekt.ShardName(o.IndexDir, cmp.Or(o.ShardPrefix, o.RepositoryDescription.Name), version, n)
356
343
}
357
344
358
345
type IndexState string
+1
-2
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
+1
-2
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
···
2
2
3
3
import (
4
4
"fmt"
5
-
"net/url"
6
5
"os"
7
6
"path/filepath"
8
7
"reflect"
···
23
22
return shard{
24
23
RepoID: fakeID(name),
25
24
RepoName: name,
26
-
Path: fmt.Sprintf("%s_v%d.%05d.zoekt", url.QueryEscape(name), 15, n),
25
+
Path: zoekt.ShardName("", name, 15, n),
27
26
ModTime: mtime,
28
27
RepoTombstone: false,
29
28
}
+10
indexbuilder.go
+10
indexbuilder.go
···
583
583
delete(t.trigrams, key)
584
584
}
585
585
}
586
+
587
+
// ShardName returns the name of the shard for the given prefix, version, and
588
+
// shard number.
589
+
func ShardName(indexDir string, prefix string, version, n int) string {
590
+
prefix = url.QueryEscape(prefix)
591
+
if len(prefix) > 200 {
592
+
prefix = prefix[:200] + hashString(prefix)[:8]
593
+
}
594
+
return filepath.Join(indexDir, fmt.Sprintf("%s_v%d.%05d.zoekt", prefix, version, n))
595
+
}
+49
indexbuilder_test.go
+49
indexbuilder_test.go
···
1
+
package zoekt
2
+
3
+
import (
4
+
"strings"
5
+
"testing"
6
+
)
7
+
8
+
func TestShardName(t *testing.T) {
9
+
tests := []struct {
10
+
name string
11
+
indexDir string
12
+
prefix string
13
+
version int
14
+
shardNum int
15
+
expected string
16
+
}{
17
+
{
18
+
name: "short prefix",
19
+
indexDir: "index",
20
+
prefix: "short",
21
+
version: 1,
22
+
shardNum: 42,
23
+
expected: "index/short_v1.00042.zoekt",
24
+
},
25
+
{
26
+
name: "long prefix truncated",
27
+
indexDir: "index",
28
+
prefix: strings.Repeat("a", 300),
29
+
version: 2,
30
+
shardNum: 1,
31
+
expected: "index/" + strings.Repeat("a", 200) + "003ef1ba" + "_v2.00001.zoekt",
32
+
},
33
+
{
34
+
name: "empty indexDir",
35
+
prefix: "short",
36
+
version: 1,
37
+
expected: "short_v1.00000.zoekt",
38
+
},
39
+
}
40
+
41
+
for _, test := range tests {
42
+
t.Run(test.name, func(t *testing.T) {
43
+
actual := ShardName(test.indexDir, test.prefix, test.version, test.shardNum)
44
+
if actual != test.expected {
45
+
t.Errorf("expected %q, got %q", test.expected, actual)
46
+
}
47
+
})
48
+
}
49
+
}
+4
-14
merge.go
+4
-14
merge.go
···
5
5
"fmt"
6
6
"io"
7
7
"log"
8
-
"net/url"
9
8
"os"
10
9
"path/filepath"
11
10
"runtime"
···
170
169
prefix = ib.repoList[0].Name
171
170
}
172
171
173
-
fn := filepath.Join(dstDir, shardName(prefix, ib.indexFormatVersion, 0))
174
-
fnTmp := fn + ".tmp"
175
-
shardNames[fnTmp] = fn
176
-
return builderWriteAll(fnTmp, ib)
172
+
shardName := ShardName(dstDir, prefix, ib.indexFormatVersion, 0)
173
+
shardNameTmp := shardName + ".tmp"
174
+
shardNames[shardNameTmp] = shardName
175
+
return builderWriteAll(shardNameTmp, ib)
177
176
}
178
177
179
178
var ib *IndexBuilder
···
264
263
_, _ = io.WriteString(h, s)
265
264
return fmt.Sprintf("%x", h.Sum(nil))
266
265
}
267
-
268
-
// copied from builder package to avoid circular imports.
269
-
func shardName(prefix string, version, n int) string {
270
-
abs := url.QueryEscape(prefix)
271
-
if len(abs) > 200 {
272
-
abs = abs[:200] + hashString(abs)[:8]
273
-
}
274
-
return fmt.Sprintf("%s_v%d.%05d.zoekt", abs, version, n)
275
-
}
+3
-3
read_test.go
+3
-3
read_test.go
···
370
370
t.Fatal(err)
371
371
}
372
372
373
-
outname := fmt.Sprintf("testdata/backcompat/new_v%d.%05d.zoekt", IndexFormatVersion, 0)
374
-
t.Log("writing new file", outname)
373
+
outName := ShardName("testdata/backcompat", "new", IndexFormatVersion, 0)
374
+
t.Log("writing new file", outName)
375
375
376
-
err = os.WriteFile(outname, buf.Bytes(), 0o644)
376
+
err = os.WriteFile(outName, buf.Bytes(), 0o644)
377
377
if err != nil {
378
378
t.Fatalf("Creating output file: %v", err)
379
379
}