fork of https://github.com/sourcegraph/zoekt
Rename IndexBuilder -> ShardBuilder (#908)
When navigating the code, I've often forgotten the difference between
`NewBuilder` and `NewIndexBuilder`. This rename clarifies that one of these
indexes a whole repo, while the other builds individual shards. Also
`index.NewShardBuilder` sounds better.
author
Julie Tibshirani
committer
GitHub
date
(Feb 2, 2025, 1:04 PM -0800)
commit
261aae37
261aae37dce6a46cdf1eb669d95d314adc0758e7
parent
fc8c2bf8
fc8c2bf80ff74dbfd4941a638025b53c1075182d
+208
-208
+1
-1
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
+1
-1
cmd/zoekt-sourcegraph-indexserver/cleanup_test.go
+3
-3
index/builder.go
+3
-3
index/builder.go
···
1020
1020
}
1021
1021
}
1022
1022
1023
-
func (b *Builder) newShardBuilder() (*IndexBuilder, error) {
1023
+
func (b *Builder) newShardBuilder() (*ShardBuilder, error) {
1024
1024
desc := b.opts.RepositoryDescription
1025
1025
desc.HasSymbols = !b.opts.DisableCTags && b.opts.CTagsPath != ""
1026
1026
desc.SubRepoMap = b.opts.SubRepositories
1027
1027
desc.IndexOptions = b.opts.GetHash()
1028
1028
1029
-
shardBuilder, err := NewIndexBuilder(&desc)
1029
+
shardBuilder, err := NewShardBuilder(&desc)
1030
1030
if err != nil {
1031
1031
return nil, err
1032
1032
}
···
1035
1035
return shardBuilder, nil
1036
1036
}
1037
1037
1038
-
func (b *Builder) writeShard(fn string, ib *IndexBuilder) (*finishedShard, error) {
1038
+
func (b *Builder) writeShard(fn string, ib *ShardBuilder) (*finishedShard, error) {
1039
1039
dir := filepath.Dir(fn)
1040
1040
if err := os.MkdirAll(dir, 0o700); err != nil {
1041
1041
return nil, err
+2
-2
index/eval_test.go
+2
-2
index/eval_test.go
···
202
202
docs = append(docs, ds)
203
203
}
204
204
205
-
b := testIndexBuilderCompound(t, repos, docs)
205
+
b := testShardBuilderCompound(t, repos, docs)
206
206
s := searcherForTest(t, b)
207
207
return s.(*indexData)
208
208
}
···
375
375
376
376
func TestGatherBranches(t *testing.T) {
377
377
content := []byte("dummy")
378
-
b := testIndexBuilder(t, &zoekt.Repository{
378
+
b := testShardBuilder(t, &zoekt.Repository{
379
379
Branches: []zoekt.RepositoryBranch{
380
380
{"foo", "v1"},
381
381
{"foo-2", "v1"},
+109
-109
index/index_test.go
+109
-109
index/index_test.go
···
45
45
}
46
46
}
47
47
48
-
func testIndexBuilder(tb testing.TB, repo *zoekt.Repository, docs ...Document) *IndexBuilder {
48
+
func testShardBuilder(tb testing.TB, repo *zoekt.Repository, docs ...Document) *ShardBuilder {
49
49
tb.Helper()
50
50
51
-
b, err := NewIndexBuilder(repo)
51
+
b, err := NewShardBuilder(repo)
52
52
if err != nil {
53
-
tb.Fatalf("NewIndexBuilder: %v", err)
53
+
tb.Fatalf("NewShardBuilder: %v", err)
54
54
}
55
55
56
56
for i, d := range docs {
···
62
62
return b
63
63
}
64
64
65
-
func testIndexBuilderCompound(t *testing.T, repos []*zoekt.Repository, docs [][]Document) *IndexBuilder {
65
+
func testShardBuilderCompound(t *testing.T, repos []*zoekt.Repository, docs [][]Document) *ShardBuilder {
66
66
t.Helper()
67
67
68
-
b := newIndexBuilder()
68
+
b := newShardBuilder()
69
69
b.indexFormatVersion = NextIndexFormatVersion
70
70
71
71
if len(repos) != len(docs) {
72
-
t.Fatalf("testIndexBuilderCompound: repos must be the same length as docs, got: len(repos)=%d len(docs)=%d", len(repos), len(docs))
72
+
t.Fatalf("testShardBuilderCompound: repos must be the same length as docs, got: len(repos)=%d len(docs)=%d", len(repos), len(docs))
73
73
}
74
74
75
75
for i, repo := range repos {
···
87
87
}
88
88
89
89
func TestBoundary(t *testing.T) {
90
-
b := testIndexBuilder(t, nil,
90
+
b := testShardBuilder(t, nil,
91
91
Document{Name: "f1", Content: []byte("x the")},
92
92
Document{Name: "f1", Content: []byte("reader")})
93
93
res := searchForTest(t, b, &query.Substring{Pattern: "there"})
···
97
97
}
98
98
99
99
func TestDocSectionInvalid(t *testing.T) {
100
-
b, err := NewIndexBuilder(nil)
100
+
b, err := NewShardBuilder(nil)
101
101
if err != nil {
102
-
t.Fatalf("NewIndexBuilder: %v", err)
102
+
t.Fatalf("NewShardBuilder: %v", err)
103
103
}
104
104
doc := Document{
105
105
Name: "f1",
···
123
123
}
124
124
125
125
func TestBasic(t *testing.T) {
126
-
b := testIndexBuilder(t, nil,
126
+
b := testShardBuilder(t, nil,
127
127
Document{
128
128
Name: "f2",
129
129
Content: []byte("to carry water in the no later bla"),
···
166
166
}
167
167
168
168
func TestEmptyIndex(t *testing.T) {
169
-
b := testIndexBuilder(t, nil)
169
+
b := testShardBuilder(t, nil)
170
170
searcher := searcherForTest(t, b)
171
171
172
172
var opts zoekt.SearchOptions
···
201
201
}
202
202
203
203
func TestNewlines(t *testing.T) {
204
-
b := testIndexBuilder(t, nil,
204
+
b := testShardBuilder(t, nil,
205
205
// -----------------------------------------012345-678901-234
206
206
Document{Name: "filename", Content: []byte("line1\nline2\nbla")})
207
207
···
259
259
// single lines.
260
260
func TestQueryNewlines(t *testing.T) {
261
261
text := "line1\nline2\nbla"
262
-
b := testIndexBuilder(t, nil,
262
+
b := testShardBuilder(t, nil,
263
263
Document{Name: "filename", Content: []byte(text)})
264
264
265
265
t.Run("LineMatches", func(t *testing.T) {
···
289
289
290
290
var chunkOpts = zoekt.SearchOptions{ChunkMatches: true}
291
291
292
-
func searchForTest(t *testing.T, b *IndexBuilder, q query.Q, o ...zoekt.SearchOptions) *zoekt.SearchResult {
292
+
func searchForTest(t *testing.T, b *ShardBuilder, q query.Q, o ...zoekt.SearchOptions) *zoekt.SearchResult {
293
293
searcher := searcherForTest(t, b)
294
294
var opts zoekt.SearchOptions
295
295
if len(o) > 0 {
···
303
303
return res
304
304
}
305
305
306
-
func searcherForTest(t testing.TB, b *IndexBuilder) zoekt.Searcher {
306
+
func searcherForTest(t testing.TB, b *ShardBuilder) zoekt.Searcher {
307
307
var buf bytes.Buffer
308
308
if err := b.Write(&buf); err != nil {
309
309
t.Fatal(err)
···
319
319
}
320
320
321
321
func TestCaseFold(t *testing.T) {
322
-
b := testIndexBuilder(t, nil,
322
+
b := testShardBuilder(t, nil,
323
323
Document{Name: "f1", Content: []byte("I love BaNaNAS.")},
324
324
// -----------------------------------012345678901234
325
325
)
···
390
390
391
391
func TestSearchStats(t *testing.T) {
392
392
ctx := context.Background()
393
-
searcher := searcherForTest(t, testIndexBuilder(t, nil,
393
+
searcher := searcherForTest(t, testShardBuilder(t, nil,
394
394
wordsAsSymbols(Document{Name: "f1", Content: []byte("x banana y")}),
395
395
wordsAsSymbols(Document{Name: "f2", Content: []byte("x apple y")}),
396
396
wordsAsSymbols(Document{Name: "f3", Content: []byte("x banana apple y")}),
···
603
603
}
604
604
605
605
func TestAndNegateSearch(t *testing.T) {
606
-
b := testIndexBuilder(t, nil,
606
+
b := testShardBuilder(t, nil,
607
607
Document{Name: "f1", Content: []byte("x banana y")},
608
608
// -----------------------------------0123456789
609
609
Document{Name: "f4", Content: []byte("x banana apple y")})
···
658
658
}
659
659
660
660
func TestNegativeMatchesOnlyShortcut(t *testing.T) {
661
-
b := testIndexBuilder(t, nil,
661
+
b := testShardBuilder(t, nil,
662
662
Document{Name: "f1", Content: []byte("x banana y")},
663
663
Document{Name: "f2", Content: []byte("x appelmoes y")},
664
664
Document{Name: "f3", Content: []byte("x appelmoes y")},
···
694
694
}
695
695
696
696
func TestFileSearch(t *testing.T) {
697
-
b := testIndexBuilder(t, nil,
697
+
b := testShardBuilder(t, nil,
698
698
Document{Name: "banzana", Content: []byte("x orange y")},
699
699
// -------------0123456
700
700
Document{Name: "banana", Content: []byte("x apple y")},
···
781
781
}
782
782
783
783
func TestFileCase(t *testing.T) {
784
-
b := testIndexBuilder(t, nil,
784
+
b := testShardBuilder(t, nil,
785
785
Document{Name: "BANANA", Content: []byte("x orange y")})
786
786
787
787
t.Run("LineMatches", func(t *testing.T) {
···
810
810
}
811
811
812
812
func TestFileRegexpSearchBruteForce(t *testing.T) {
813
-
b := testIndexBuilder(t, nil,
813
+
b := testShardBuilder(t, nil,
814
814
Document{Name: "banzana", Content: []byte("x orange y")},
815
815
Document{Name: "banana", Content: []byte("x apple y")},
816
816
)
···
839
839
}
840
840
841
841
func TestFileRegexpSearchShortString(t *testing.T) {
842
-
b := testIndexBuilder(t, nil,
842
+
b := testShardBuilder(t, nil,
843
843
Document{Name: "banana.py", Content: []byte("x orange y")})
844
844
845
845
t.Run("LineMatches", func(t *testing.T) {
···
868
868
}
869
869
870
870
func TestFileSubstringSearchBruteForce(t *testing.T) {
871
-
b := testIndexBuilder(t, nil,
871
+
b := testShardBuilder(t, nil,
872
872
Document{Name: "BANZANA", Content: []byte("x orange y")},
873
873
Document{Name: "banana", Content: []byte("x apple y")})
874
874
···
893
893
}
894
894
895
895
func TestFileSubstringSearchBruteForceEnd(t *testing.T) {
896
-
b := testIndexBuilder(t, nil,
896
+
b := testShardBuilder(t, nil,
897
897
Document{Name: "BANZANA", Content: []byte("x orange y")},
898
898
Document{Name: "bananaq", Content: []byte("x apple y")})
899
899
···
917
917
}
918
918
919
919
func TestSearchMatchAll(t *testing.T) {
920
-
b := testIndexBuilder(t, nil,
920
+
b := testShardBuilder(t, nil,
921
921
Document{Name: "banzana", Content: []byte("x orange y")},
922
922
Document{Name: "banana", Content: []byte("x apple y")})
923
923
···
939
939
}
940
940
941
941
func TestSearchNewline(t *testing.T) {
942
-
b := testIndexBuilder(t, nil,
942
+
b := testShardBuilder(t, nil,
943
943
Document{Name: "banzana", Content: []byte("abcd\ndefg")})
944
944
945
945
t.Run("LineMatches", func(t *testing.T) {
···
966
966
}
967
967
968
968
func TestSearchMatchAllRegexp(t *testing.T) {
969
-
b := testIndexBuilder(t, nil,
969
+
b := testShardBuilder(t, nil,
970
970
Document{Name: "banzana", Content: []byte("abcd")},
971
971
Document{Name: "banana", Content: []byte("pqrs")})
972
972
···
997
997
998
998
func TestSearchBM25MatchScores(t *testing.T) {
999
999
ctx := context.Background()
1000
-
searcher := searcherForTest(t, testIndexBuilder(t, nil,
1000
+
searcher := searcherForTest(t, testShardBuilder(t, nil,
1001
1001
Document{Name: "f1", Content: []byte("one two three\naaaaaaaaaa\nbbbbbbbb\none two two")},
1002
1002
Document{Name: "f2", Content: []byte("four five six\naaaaaaaaaa\nbbbbbbbb\nfour five five\nsix six")},
1003
1003
wordsAsSymbols(Document{Name: "f3", Content: []byte("public static void main")}),
···
1078
1078
}
1079
1079
1080
1080
func TestFileRestriction(t *testing.T) {
1081
-
b := testIndexBuilder(t, nil,
1081
+
b := testShardBuilder(t, nil,
1082
1082
Document{Name: "banana1", Content: []byte("x orange y")},
1083
1083
Document{Name: "banana2", Content: []byte("x apple y")},
1084
1084
Document{Name: "orange", Content: []byte("x apple z")})
···
1131
1131
}
1132
1132
1133
1133
func TestFileNameBoundary(t *testing.T) {
1134
-
b := testIndexBuilder(t, nil,
1134
+
b := testShardBuilder(t, nil,
1135
1135
Document{Name: "banana2", Content: []byte("x apple y")},
1136
1136
Document{Name: "helpers.go", Content: []byte("x apple y")},
1137
1137
Document{Name: "foo", Content: []byte("x apple y")})
···
1167
1167
docs = append(docs, Document{Name: fmt.Sprintf("f%d", i), Content: []byte("needle")})
1168
1168
}
1169
1169
1170
-
b := testIndexBuilder(t, nil, docs...)
1170
+
b := testShardBuilder(t, nil, docs...)
1171
1171
1172
1172
t.Run("LineMatches", func(t *testing.T) {
1173
1173
sres := searchForTest(t, b, query.NewAnd(
···
1205
1205
}
1206
1206
1207
1207
func TestBranchMask(t *testing.T) {
1208
-
b := testIndexBuilder(t, &zoekt.Repository{
1208
+
b := testShardBuilder(t, &zoekt.Repository{
1209
1209
Branches: []zoekt.RepositoryBranch{
1210
1210
{"master", "v-master"},
1211
1211
{"stable", "v-stable"},
···
1265
1265
s, "v-" + s,
1266
1266
})
1267
1267
}
1268
-
_, err := NewIndexBuilder(r)
1268
+
_, err := NewShardBuilder(r)
1269
1269
if limit == 64 && err != nil {
1270
-
t.Fatalf("NewIndexBuilder: %v", err)
1270
+
t.Fatalf("NewShardBuilder: %v", err)
1271
1271
} else if limit == 65 && err == nil {
1272
-
t.Fatalf("NewIndexBuilder succeeded")
1272
+
t.Fatalf("NewShardBuilder succeeded")
1273
1273
}
1274
1274
}
1275
1275
}
1276
1276
1277
1277
func TestBranchReport(t *testing.T) {
1278
1278
branches := []string{"stable", "master"}
1279
-
b := testIndexBuilder(t, &zoekt.Repository{
1279
+
b := testShardBuilder(t, &zoekt.Repository{
1280
1280
Branches: []zoekt.RepositoryBranch{
1281
1281
{"stable", "vs"},
1282
1282
{"master", "vm"},
···
1314
1314
}
1315
1315
1316
1316
func TestBranchVersions(t *testing.T) {
1317
-
b := testIndexBuilder(t, &zoekt.Repository{
1317
+
b := testShardBuilder(t, &zoekt.Repository{
1318
1318
Branches: []zoekt.RepositoryBranch{
1319
1319
{"stable", "v-stable"},
1320
1320
{"master", "v-master"},
···
1363
1363
content := []byte("needle the bla")
1364
1364
// ----------------01234567890123
1365
1365
1366
-
b := testIndexBuilder(t, nil,
1366
+
b := testShardBuilder(t, nil,
1367
1367
Document{
1368
1368
Name: "f1",
1369
1369
Content: content,
···
1428
1428
content := []byte("needle the bla")
1429
1429
1430
1430
name := "let's play: find the mussel"
1431
-
b := testIndexBuilder(t, nil,
1431
+
b := testShardBuilder(t, nil,
1432
1432
Document{Name: name, Content: content},
1433
1433
Document{Name: "play.txt", Content: content})
1434
1434
···
1469
1469
content := []byte("bla the needle")
1470
1470
// ----------------01234567890123
1471
1471
1472
-
b := testIndexBuilder(t, nil,
1472
+
b := testShardBuilder(t, nil,
1473
1473
Document{Name: "f1", Content: content})
1474
1474
1475
1475
t.Run("LineMatches", func(t *testing.T) {
···
1499
1499
content := []byte("bla the needle")
1500
1500
// ----------------01234567890123
1501
1501
1502
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "bla"},
1502
+
b := testShardBuilder(t, &zoekt.Repository{Name: "bla"},
1503
1503
Document{Name: "f1", Content: content})
1504
1504
1505
1505
t.Run("LineMatches", func(t *testing.T) {
···
1557
1557
1558
1558
func TestMergeMatches(t *testing.T) {
1559
1559
t.Run("LineMatches, adjacent matches", func(t *testing.T) {
1560
-
b := testIndexBuilder(t, nil,
1560
+
b := testShardBuilder(t, nil,
1561
1561
Document{Name: "f1", Content: []byte("blablabla")})
1562
1562
sres := searchForTest(t, b,
1563
1563
&query.Substring{Pattern: "bla"})
···
1572
1572
})
1573
1573
1574
1574
t.Run("LineMatches, overlapping matches", func(t *testing.T) {
1575
-
b := testIndexBuilder(t, nil,
1575
+
b := testShardBuilder(t, nil,
1576
1576
Document{Name: "f1", Content: []byte("hellogoodbye")})
1577
1577
sres := searchForTest(t, b,
1578
1578
&query.And{Children: []query.Q{
···
1591
1591
})
1592
1592
1593
1593
t.Run("ChunkMatches, no overlap", func(t *testing.T) {
1594
-
b := testIndexBuilder(t, nil,
1594
+
b := testShardBuilder(t, nil,
1595
1595
Document{Name: "f1", Content: []byte("blablabla")})
1596
1596
1597
1597
sres := searchForTest(t, b,
···
1608
1608
})
1609
1609
1610
1610
t.Run("ChunkMatches, overlapping matches", func(t *testing.T) {
1611
-
b := testIndexBuilder(t, nil,
1611
+
b := testShardBuilder(t, nil,
1612
1612
Document{Name: "f1", Content: []byte("hellogoodbye")})
1613
1613
sres := searchForTest(t, b,
1614
1614
&query.And{Children: []query.Q{
···
1629
1629
1630
1630
func TestRepoURL(t *testing.T) {
1631
1631
content := []byte("blablabla")
1632
-
b := testIndexBuilder(t, &zoekt.Repository{
1632
+
b := testShardBuilder(t, &zoekt.Repository{
1633
1633
Name: "name",
1634
1634
URL: "URL",
1635
1635
CommitURLTemplate: "commit",
···
1649
1649
1650
1650
func TestRegexpCaseSensitive(t *testing.T) {
1651
1651
content := []byte("bla\nfunc unmarshalGitiles\n")
1652
-
b := testIndexBuilder(t, nil, Document{
1652
+
b := testShardBuilder(t, nil, Document{
1653
1653
Name: "f1",
1654
1654
Content: content,
1655
1655
})
···
1684
1684
func TestRegexpCaseFolding(t *testing.T) {
1685
1685
content := []byte("bla\nfunc unmarshalGitiles\n")
1686
1686
1687
-
b := testIndexBuilder(t, nil,
1687
+
b := testShardBuilder(t, nil,
1688
1688
Document{Name: "f1", Content: content})
1689
1689
res := searchForTest(t, b,
1690
1690
&query.Regexp{
···
1699
1699
1700
1700
func TestCaseRegexp(t *testing.T) {
1701
1701
content := []byte("BLABLABLA")
1702
-
b := testIndexBuilder(t, nil,
1702
+
b := testShardBuilder(t, nil,
1703
1703
Document{Name: "f1", Content: content})
1704
1704
1705
1705
t.Run("LineMatches", func(t *testing.T) {
···
1731
1731
1732
1732
func TestNegativeRegexp(t *testing.T) {
1733
1733
content := []byte("BLABLABLA needle bla")
1734
-
b := testIndexBuilder(t, nil,
1734
+
b := testShardBuilder(t, nil,
1735
1735
Document{Name: "f1", Content: content})
1736
1736
1737
1737
t.Run("LineMatches", func(t *testing.T) {
···
1776
1776
1777
1777
content := []byte("func bla() blubxxxxx")
1778
1778
// ----------------01234567890123456789
1779
-
b := testIndexBuilder(t, nil,
1779
+
b := testShardBuilder(t, nil,
1780
1780
Document{
1781
1781
Name: "f1",
1782
1782
Content: content,
···
1827
1827
content := []byte(prefix +
1828
1828
"func bla() blub")
1829
1829
// ------012345678901234
1830
-
b := testIndexBuilder(t, nil,
1830
+
b := testShardBuilder(t, nil,
1831
1831
Document{
1832
1832
Name: "f1",
1833
1833
Content: content,
···
1875
1875
content := []byte("func bla() blub")
1876
1876
// ----------------012345678901234
1877
1877
1878
-
b := testIndexBuilder(t, nil,
1878
+
b := testShardBuilder(t, nil,
1879
1879
Document{
1880
1880
Name: "f1",
1881
1881
Content: content,
···
1922
1922
func TestNegativeRepo(t *testing.T) {
1923
1923
content := []byte("bla the needle")
1924
1924
// ----------------01234567890123
1925
-
b := testIndexBuilder(t, &zoekt.Repository{
1925
+
b := testShardBuilder(t, &zoekt.Repository{
1926
1926
Name: "bla",
1927
1927
}, Document{Name: "f1", Content: content})
1928
1928
···
1960
1960
Name: "reponame",
1961
1961
Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
1962
1962
}
1963
-
b := testIndexBuilder(t, repo,
1963
+
b := testShardBuilder(t, repo,
1964
1964
Document{Name: "f1", Content: content, Branches: []string{"main", "dev"}},
1965
1965
Document{Name: "f2", Content: content, Branches: []string{"main"}},
1966
1966
Document{Name: "f2", Content: content, Branches: []string{"dev"}},
···
2036
2036
Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
2037
2037
RawConfig: map[string]string{"repoid": "1234"},
2038
2038
}
2039
-
b := testIndexBuilder(t, repo,
2039
+
b := testShardBuilder(t, repo,
2040
2040
Document{Name: "f1", Content: content, Branches: []string{"main", "dev"}},
2041
2041
Document{Name: "f2", Content: content, Branches: []string{"main"}},
2042
2042
Document{Name: "f2", Content: content, Branches: []string{"dev"}},
···
2090
2090
func TestListReposByContent(t *testing.T) {
2091
2091
content := []byte("bla the needle")
2092
2092
2093
-
b := testIndexBuilder(t, &zoekt.Repository{
2093
+
b := testShardBuilder(t, &zoekt.Repository{
2094
2094
Name: "reponame",
2095
2095
},
2096
2096
Document{Name: "f1", Content: content},
···
2121
2121
func TestMetadata(t *testing.T) {
2122
2122
content := []byte("bla the needle")
2123
2123
2124
-
b := testIndexBuilder(t, &zoekt.Repository{
2124
+
b := testShardBuilder(t, &zoekt.Repository{
2125
2125
Name: "reponame",
2126
2126
}, Document{Name: "f1", Content: content},
2127
2127
Document{Name: "f2", Content: content})
···
2143
2143
}
2144
2144
2145
2145
func TestOr(t *testing.T) {
2146
-
b := testIndexBuilder(t, nil,
2146
+
b := testShardBuilder(t, nil,
2147
2147
Document{Name: "f1", Content: []byte("needle")},
2148
2148
Document{Name: "f2", Content: []byte("banana")})
2149
2149
t.Run("LineMatches", func(t *testing.T) {
···
2170
2170
func TestFrequency(t *testing.T) {
2171
2171
content := []byte("sla _Py_HashDouble(double v sla las las shd dot dot")
2172
2172
2173
-
b := testIndexBuilder(t, nil,
2173
+
b := testShardBuilder(t, nil,
2174
2174
Document{
2175
2175
Name: "f1",
2176
2176
Content: content,
···
2199
2199
2200
2200
content := []byte("pqr\nalex")
2201
2201
2202
-
b := testIndexBuilder(t, nil,
2202
+
b := testShardBuilder(t, nil,
2203
2203
Document{
2204
2204
Name: "f1",
2205
2205
Content: content,
···
2234
2234
2235
2235
content := []byte("pqr\nalex")
2236
2236
2237
-
b := testIndexBuilder(t, &zoekt.Repository{
2237
+
b := testShardBuilder(t, &zoekt.Repository{
2238
2238
SubRepoMap: subRepos,
2239
2239
}, Document{
2240
2240
Name: "sub/f1",
···
2258
2258
}
2259
2259
2260
2260
func TestSearchEither(t *testing.T) {
2261
-
b := testIndexBuilder(t, nil,
2261
+
b := testShardBuilder(t, nil,
2262
2262
Document{Name: "f1", Content: []byte("bla needle bla")},
2263
2263
Document{Name: "needle-file-branch", Content: []byte("bla content")})
2264
2264
···
2299
2299
needle := "néédlÉ"
2300
2300
content := []byte("blá blá " + needle + " blâ")
2301
2301
2302
-
b := testIndexBuilder(t, nil,
2302
+
b := testShardBuilder(t, nil,
2303
2303
Document{Name: "f1", Content: content})
2304
2304
2305
2305
t.Run("LineMatches", func(t *testing.T) {
···
2320
2320
needle := "néédlÉ"
2321
2321
content := []byte("blá blá " + needle + " blâ")
2322
2322
2323
-
b := testIndexBuilder(t, nil,
2323
+
b := testShardBuilder(t, nil,
2324
2324
Document{Name: "f1", Content: content})
2325
2325
2326
2326
t.Run("LineMatches", func(t *testing.T) {
···
2361
2361
needle := "nééáádlÉ"
2362
2362
content := []byte("blá blá " + needle + " blâ")
2363
2363
2364
-
b := testIndexBuilder(t, nil,
2364
+
b := testShardBuilder(t, nil,
2365
2365
Document{Name: "f1", Content: content})
2366
2366
2367
2367
t.Run("LineMatches", func(t *testing.T) {
···
2401
2401
" ee" + string([]rune{upper}) + "ee")
2402
2402
2403
2403
t.Run("LineMatches", func(t *testing.T) {
2404
-
b := testIndexBuilder(t, nil,
2404
+
b := testShardBuilder(t, nil,
2405
2405
Document{Name: "f1", Content: []byte(corpus)})
2406
2406
2407
2407
res := searchForTest(t, b, &query.Substring{Pattern: needle, Content: true})
···
2411
2411
})
2412
2412
2413
2413
t.Run("ChunkMatches", func(t *testing.T) {
2414
-
b := testIndexBuilder(t, nil,
2414
+
b := testShardBuilder(t, nil,
2415
2415
Document{Name: "f1", Content: []byte(corpus)})
2416
2416
2417
2417
res := searchForTest(t, b, &query.Substring{Pattern: needle, Content: true}, chunkOpts)
···
2424
2424
func TestUnicodeFileStartOffsets(t *testing.T) {
2425
2425
unicode := "世界"
2426
2426
wat := "waaaaaat"
2427
-
b := testIndexBuilder(t, nil,
2427
+
b := testShardBuilder(t, nil,
2428
2428
Document{
2429
2429
Name: "f1",
2430
2430
Content: []byte(unicode),
···
2447
2447
// 6 bytes.
2448
2448
unicode := "世界"
2449
2449
content := []byte(strings.Repeat(unicode, 100) + needle)
2450
-
b := testIndexBuilder(t, nil,
2450
+
b := testShardBuilder(t, nil,
2451
2451
Document{
2452
2452
Name: "f1",
2453
2453
Content: []byte(strings.Repeat("a", 50)),
···
2476
2476
2477
2477
func TestEstimateDocCount(t *testing.T) {
2478
2478
content := []byte("bla needle bla")
2479
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2479
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2480
2480
Document{Name: "f1", Content: content},
2481
2481
Document{Name: "f2", Content: content},
2482
2482
)
···
2531
2531
2532
2532
// 6 bytes.
2533
2533
unicode := "世界"
2534
-
b := testIndexBuilder(t, nil,
2534
+
b := testShardBuilder(t, nil,
2535
2535
Document{
2536
2536
Name: "f1",
2537
2537
Content: []byte(strings.Repeat(unicode, 100)),
···
2559
2559
}
2560
2560
2561
2561
func TestBuilderStats(t *testing.T) {
2562
-
b := testIndexBuilder(t, nil,
2562
+
b := testShardBuilder(t, nil,
2563
2563
Document{
2564
2564
Name: "f1",
2565
2565
Content: []byte(strings.Repeat("abcd", 1024)),
···
2575
2575
}
2576
2576
2577
2577
func TestIOStats(t *testing.T) {
2578
-
b := testIndexBuilder(t, nil,
2578
+
b := testShardBuilder(t, nil,
2579
2579
Document{
2580
2580
Name: "f1",
2581
2581
Content: []byte(strings.Repeat("abcd", 1024)),
···
2647
2647
}
2648
2648
2649
2649
func TestStartLineAnchor(t *testing.T) {
2650
-
b := testIndexBuilder(t, nil,
2650
+
b := testShardBuilder(t, nil,
2651
2651
Document{
2652
2652
Name: "f1",
2653
2653
Content: []byte(
···
2708
2708
query.NewOr(query.NewAnd(&query.Repo{Regexp: regexp.MustCompile("name")},
2709
2709
query.NewOr(&query.Branch{Pattern: "master"}))))
2710
2710
2711
-
b := testIndexBuilder(t, &zoekt.Repository{
2711
+
b := testShardBuilder(t, &zoekt.Repository{
2712
2712
Name: "name",
2713
2713
Branches: []zoekt.RepositoryBranch{{"master", "master-version"}},
2714
2714
}, Document{
···
2735
2735
2736
2736
func TestAndShort(t *testing.T) {
2737
2737
content := []byte("bla needle at orange bla")
2738
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2738
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2739
2739
Document{Name: "f1", Content: content},
2740
2740
Document{Name: "f2", Content: []byte("xx at xx")},
2741
2741
Document{Name: "f3", Content: []byte("yy orange xx")},
···
2761
2761
2762
2762
func TestNoCollectRegexpSubstring(t *testing.T) {
2763
2763
content := []byte("bla final bla\nfoo final, foo")
2764
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2764
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2765
2765
Document{Name: "f1", Content: content},
2766
2766
)
2767
2767
···
2801
2801
2802
2802
func TestLang(t *testing.T) {
2803
2803
content := []byte("bla needle bla")
2804
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2804
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2805
2805
Document{Name: "f1", Content: content},
2806
2806
Document{Name: "f2", Language: "java", Content: content},
2807
2807
Document{Name: "f3", Language: "cpp", Content: content},
···
2835
2835
2836
2836
func TestLangShortcut(t *testing.T) {
2837
2837
content := []byte("bla needle bla")
2838
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2838
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2839
2839
Document{Name: "f2", Language: "java", Content: content},
2840
2840
Document{Name: "f3", Language: "cpp", Content: content},
2841
2841
)
···
2866
2866
2867
2867
func TestNoTextMatchAtoms(t *testing.T) {
2868
2868
content := []byte("bla needle bla")
2869
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2869
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2870
2870
Document{Name: "f1", Content: content},
2871
2871
Document{Name: "f2", Language: "java", Content: content},
2872
2872
Document{Name: "f3", Language: "cpp", Content: content},
···
2889
2889
2890
2890
func TestNoPositiveAtoms(t *testing.T) {
2891
2891
content := []byte("bla needle bla")
2892
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2892
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2893
2893
Document{Name: "f1", Content: content},
2894
2894
Document{Name: "f2", Content: content},
2895
2895
)
···
2915
2915
content := []byte("start\nbla bla\nend")
2916
2916
// ----------------012345-67890123-456
2917
2917
2918
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2918
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2919
2919
Document{
2920
2920
Name: "f1",
2921
2921
Content: content,
···
2952
2952
content := []byte("start\nbla bla\nend")
2953
2953
// ----------------012345-67890123-456
2954
2954
2955
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2955
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2956
2956
Document{
2957
2957
Name: "f1",
2958
2958
Content: content,
···
2989
2989
content := []byte("bla\nsymblabla\nbla")
2990
2990
// ----------------0123-4567890123-456
2991
2991
2992
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
2992
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
2993
2993
Document{
2994
2994
Name: "f1",
2995
2995
Content: content,
···
3026
3026
content := []byte("bla\nsym\nbla\nsym\nasymb")
3027
3027
// ----------------0123-4567-890123456-78901
3028
3028
3029
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3029
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3030
3030
Document{
3031
3031
Name: "f1",
3032
3032
Content: content,
···
3063
3063
content := []byte("blah\nbla\nbl")
3064
3064
// ----------------01234-5678-90
3065
3065
3066
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3066
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3067
3067
Document{
3068
3068
Name: "f1",
3069
3069
Content: content,
···
3100
3100
content := []byte("abcdef")
3101
3101
// ----------------012345
3102
3102
3103
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3103
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3104
3104
Document{
3105
3105
Name: "f1",
3106
3106
Content: content,
···
3155
3155
},
3156
3156
}
3157
3157
3158
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"}, docs...)
3158
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"}, docs...)
3159
3159
q := &query.Symbol{
3160
3160
Expr: &query.Regexp{Regexp: mustParseRE(".*")},
3161
3161
}
···
3204
3204
// will advance the compressedPostingIterator to beyond the
3205
3205
// end.
3206
3206
content := []byte("abc bcdbcd cdecde abcabc def efg")
3207
-
b := testIndexBuilder(t, nil,
3207
+
b := testShardBuilder(t, nil,
3208
3208
Document{
3209
3209
Name: "f1",
3210
3210
Content: content,
···
3222
3222
3223
3223
func TestDistanceHitIterBailLast(t *testing.T) {
3224
3224
content := []byte("AST AST AST UASH")
3225
-
b := testIndexBuilder(t, nil,
3225
+
b := testShardBuilder(t, nil,
3226
3226
Document{
3227
3227
Name: "f1",
3228
3228
Content: content,
···
3245
3245
3246
3246
func TestDocumentSectionRuneBoundary(t *testing.T) {
3247
3247
content := string([]rune{kelvinCodePoint, kelvinCodePoint, kelvinCodePoint})
3248
-
b, err := NewIndexBuilder(nil)
3248
+
b, err := NewShardBuilder(nil)
3249
3249
if err != nil {
3250
-
t.Fatalf("NewIndexBuilder: %v", err)
3250
+
t.Fatalf("NewShardBuilder: %v", err)
3251
3251
}
3252
3252
3253
3253
for i, sec := range []DocumentSection{
···
3266
3266
3267
3267
func TestUnicodeQuery(t *testing.T) {
3268
3268
content := string([]rune{kelvinCodePoint, kelvinCodePoint, kelvinCodePoint})
3269
-
b := testIndexBuilder(t, nil,
3269
+
b := testShardBuilder(t, nil,
3270
3270
Document{
3271
3271
Name: "f1",
3272
3272
Content: []byte(content),
···
3324
3324
"abc def \x00 abc",
3325
3325
} {
3326
3326
3327
-
b, err := NewIndexBuilder(nil)
3327
+
b, err := NewShardBuilder(nil)
3328
3328
if err != nil {
3329
-
t.Fatalf("NewIndexBuilder: %v", err)
3329
+
t.Fatalf("NewShardBuilder: %v", err)
3330
3330
}
3331
3331
3332
3332
if err := b.Add(Document{
···
3395
3395
}
3396
3396
3397
3397
func TestLineAnd(t *testing.T) {
3398
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3398
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3399
3399
Document{Name: "f1", Content: []byte("apple\nbanana\napple banana chocolate apple pudding banana\ngrape")},
3400
3400
Document{Name: "f2", Content: []byte("apple orange\nbanana")},
3401
3401
Document{Name: "f3", Content: []byte("banana grape")},
···
3431
3431
}
3432
3432
3433
3433
func TestLineAndFileName(t *testing.T) {
3434
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3434
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3435
3435
Document{Name: "f1", Content: []byte("apple banana\ngrape")},
3436
3436
Document{Name: "f2", Content: []byte("apple banana\norange")},
3437
3437
Document{Name: "apple banana", Content: []byte("banana grape")},
···
3467
3467
}
3468
3468
3469
3469
func TestMultiLineRegex(t *testing.T) {
3470
-
b := testIndexBuilder(t, &zoekt.Repository{Name: "reponame"},
3470
+
b := testShardBuilder(t, &zoekt.Repository{Name: "reponame"},
3471
3471
Document{Name: "f1", Content: []byte("apple banana\ngrape")},
3472
3472
Document{Name: "f2", Content: []byte("apple orange")},
3473
3473
Document{Name: "f3", Content: []byte("grape apple")},
···
3511
3511
}
3512
3512
3513
3513
func TestSearchTypeFileName(t *testing.T) {
3514
-
b := testIndexBuilder(t, &zoekt.Repository{
3514
+
b := testShardBuilder(t, &zoekt.Repository{
3515
3515
Name: "reponame",
3516
3516
},
3517
3517
Document{Name: "f1", Content: []byte("bla the needle")},
···
3609
3609
}
3610
3610
3611
3611
func TestSearchTypeLanguage(t *testing.T) {
3612
-
b := testIndexBuilder(t, &zoekt.Repository{
3612
+
b := testShardBuilder(t, &zoekt.Repository{
3613
3613
Name: "reponame",
3614
3614
},
3615
3615
Document{Name: "apex.cls", Content: []byte("public class Car extends Vehicle {")},
···
3721
3721
cmpopts.IgnoreFields(zoekt.RepoStats{}, "IndexBytes"),
3722
3722
}
3723
3723
3724
-
repoListEntries := func(b *IndexBuilder) []zoekt.RepoListEntry {
3724
+
repoListEntries := func(b *ShardBuilder) []zoekt.RepoListEntry {
3725
3725
searcher := searcherForTest(t, b)
3726
3726
indexdata := searcher.(*indexData)
3727
3727
return indexdata.repoListEntry
3728
3728
}
3729
3729
3730
3730
t.Run("one empty repo", func(t *testing.T) {
3731
-
b := testIndexBuilder(t, nil)
3731
+
b := testShardBuilder(t, nil)
3732
3732
got := repoListEntries(b)
3733
3733
want := []zoekt.RepoListEntry{
3734
3734
{
···
3751
3751
})
3752
3752
3753
3753
t.Run("one simple shard", func(t *testing.T) {
3754
-
b := testIndexBuilder(t, nil,
3754
+
b := testShardBuilder(t, nil,
3755
3755
Document{Name: "doc 0", Content: []byte("content 0")},
3756
3756
Document{Name: "doc 1", Content: []byte("content 1")},
3757
3757
)
···
3777
3777
})
3778
3778
3779
3779
t.Run("one compound shard", func(t *testing.T) {
3780
-
b := testIndexBuilderCompound(t,
3780
+
b := testShardBuilderCompound(t,
3781
3781
[]*zoekt.Repository{
3782
3782
{Name: "repo 0"},
3783
3783
{Name: "repo 1"},
···
3827
3827
})
3828
3828
3829
3829
t.Run("compound shard with empty repos", func(t *testing.T) {
3830
-
b := testIndexBuilderCompound(t,
3830
+
b := testShardBuilderCompound(t,
3831
3831
[]*zoekt.Repository{
3832
3832
{Name: "repo 0"},
3833
3833
{Name: "repo 1"},
···
3875
3875
content := []byte("needle the bla")
3876
3876
// ----------------01234567890123
3877
3877
3878
-
b := testIndexBuilder(t, nil,
3878
+
b := testShardBuilder(t, nil,
3879
3879
Document{
3880
3880
Name: "f1",
3881
3881
Content: content,
···
3956
3956
builder.WriteString(fmt.Sprintf("line-%d one one one two two two three three three four four four five five\n", i))
3957
3957
}
3958
3958
3959
-
searcher := searcherForTest(b, testIndexBuilder(b, nil,
3959
+
searcher := searcherForTest(b, testShardBuilder(b, nil,
3960
3960
Document{Name: "f1", Content: []byte(builder.String())},
3961
3961
))
3962
3962
+17
-17
index/indexbuilder.go
index/shard_builder.go
+17
-17
index/indexbuilder.go
index/shard_builder.go
···
163
163
return &dest, runeSecs, nil
164
164
}
165
165
166
-
// IndexBuilder builds a single index shard.
167
-
type IndexBuilder struct {
166
+
// ShardBuilder builds a single index shard.
167
+
type ShardBuilder struct {
168
168
// The version we will write to disk. Sourcegraph Specific. This is to
169
169
// enable feature flagging new format versions.
170
170
indexFormatVersion int
···
254
254
}
255
255
256
256
// ContentSize returns the number of content bytes so far ingested.
257
-
func (b *IndexBuilder) ContentSize() uint32 {
257
+
func (b *ShardBuilder) ContentSize() uint32 {
258
258
// Add the name too so we don't skip building index if we have
259
259
// lots of empty files.
260
260
return b.contentPostings.endByte + b.namePostings.endByte
261
261
}
262
262
263
263
// NumFiles returns the number of files added to this builder
264
-
func (b *IndexBuilder) NumFiles() int {
264
+
func (b *ShardBuilder) NumFiles() int {
265
265
return len(b.contentStrings)
266
266
}
267
267
268
-
// NewIndexBuilder creates a fresh IndexBuilder. The passed in
268
+
// NewShardBuilder creates a fresh ShardBuilder. The passed in
269
269
// Repository contains repo metadata, and may be set to nil.
270
-
func NewIndexBuilder(r *zoekt.Repository) (*IndexBuilder, error) {
271
-
b := newIndexBuilder()
270
+
func NewShardBuilder(r *zoekt.Repository) (*ShardBuilder, error) {
271
+
b := newShardBuilder()
272
272
273
273
if r == nil {
274
274
r = &zoekt.Repository{}
···
279
279
return b, nil
280
280
}
281
281
282
-
func newIndexBuilder() *IndexBuilder {
283
-
return &IndexBuilder{
282
+
func newShardBuilder() *ShardBuilder {
283
+
return &ShardBuilder{
284
284
indexFormatVersion: IndexFormatVersion,
285
285
featureVersion: FeatureVersion,
286
286
···
293
293
}
294
294
}
295
295
296
-
func (b *IndexBuilder) setRepository(desc *zoekt.Repository) error {
296
+
func (b *ShardBuilder) setRepository(desc *zoekt.Repository) error {
297
297
if err := verify(desc); err != nil {
298
298
return err
299
299
}
···
334
334
}
335
335
336
336
// AddFile is a convenience wrapper for Add
337
-
func (b *IndexBuilder) AddFile(name string, content []byte) error {
337
+
func (b *ShardBuilder) AddFile(name string, content []byte) error {
338
338
return b.Add(Document{Name: name, Content: content})
339
339
}
340
340
341
-
func (b *IndexBuilder) populateSubRepoIndices() error {
341
+
func (b *ShardBuilder) populateSubRepoIndices() error {
342
342
if len(b.subRepoIndices) == len(b.repoList) {
343
343
return nil
344
344
}
···
365
365
366
366
const notIndexedMarker = "NOT-INDEXED: "
367
367
368
-
func (b *IndexBuilder) symbolID(sym string) uint32 {
368
+
func (b *ShardBuilder) symbolID(sym string) uint32 {
369
369
if _, ok := b.symIndex[sym]; !ok {
370
370
b.symIndex[sym] = b.symID
371
371
b.symID++
···
373
373
return b.symIndex[sym]
374
374
}
375
375
376
-
func (b *IndexBuilder) symbolKindID(t string) uint32 {
376
+
func (b *ShardBuilder) symbolKindID(t string) uint32 {
377
377
if _, ok := b.symKindIndex[t]; !ok {
378
378
b.symKindIndex[t] = b.symKindID
379
379
b.symKindID++
···
381
381
return b.symKindIndex[t]
382
382
}
383
383
384
-
func (b *IndexBuilder) addSymbols(symbols []*zoekt.Symbol) {
384
+
func (b *ShardBuilder) addSymbols(symbols []*zoekt.Symbol) {
385
385
for _, sym := range symbols {
386
386
b.symMetaData = append(b.symMetaData,
387
387
// This field was removed due to redundancy. To avoid
···
402
402
}
403
403
404
404
// Add a file which only occurs in certain branches.
405
-
func (b *IndexBuilder) Add(doc Document) error {
405
+
func (b *ShardBuilder) Add(doc Document) error {
406
406
hasher := crc64.New(crc64.MakeTable(crc64.ISO))
407
407
408
408
if idx := bytes.IndexByte(doc.Content, 0); idx >= 0 {
···
497
497
return nil
498
498
}
499
499
500
-
func (b *IndexBuilder) branchMask(br string) uint64 {
500
+
func (b *ShardBuilder) branchMask(br string) uint64 {
501
501
for i, b := range b.repoList[len(b.repoList)-1].Branches {
502
502
if b.Name == br {
503
503
return uint64(1) << uint(i)
index/indexbuilder_test.go
index/shard_builder_test.go
index/indexbuilder_test.go
index/shard_builder_test.go
+21
-21
index/merge.go
+21
-21
index/merge.go
···
51
51
return tmpName, dstName, nil
52
52
}
53
53
54
-
func builderWriteAll(fn string, ib *IndexBuilder) error {
54
+
func builderWriteAll(fn string, ib *ShardBuilder) error {
55
55
dir := filepath.Dir(fn)
56
56
if err := os.MkdirAll(dir, 0o700); err != nil {
57
57
return err
···
90
90
return nil
91
91
}
92
92
93
-
func merge(ds ...*indexData) (*IndexBuilder, error) {
93
+
func merge(ds ...*indexData) (*ShardBuilder, error) {
94
94
if len(ds) == 0 {
95
95
return nil, fmt.Errorf("need 1 or more indexData to merge")
96
96
}
···
99
99
return ds[i].repoMetaData[0].GetPriority() > ds[j].repoMetaData[0].GetPriority()
100
100
})
101
101
102
-
ib := newIndexBuilder()
103
-
ib.indexFormatVersion = NextIndexFormatVersion
102
+
sb := newShardBuilder()
103
+
sb.indexFormatVersion = NextIndexFormatVersion
104
104
105
105
for _, d := range ds {
106
106
lastRepoID := -1
···
120
120
// TODO we are losing empty repos on merging since we only get here if
121
121
// there is an associated document.
122
122
123
-
if err := ib.setRepository(&d.repoMetaData[repoID]); err != nil {
123
+
if err := sb.setRepository(&d.repoMetaData[repoID]); err != nil {
124
124
return nil, err
125
125
}
126
126
}
127
127
128
-
if err := addDocument(d, ib, repoID, docID); err != nil {
128
+
if err := addDocument(d, sb, repoID, docID); err != nil {
129
129
return nil, err
130
130
}
131
131
}
132
132
}
133
133
134
-
return ib, nil
134
+
return sb, nil
135
135
}
136
136
137
137
// Explode takes an IndexFile f and creates 1 simple shard per repository
···
142
142
return explode(dstDir, f)
143
143
}
144
144
145
-
type indexBuilderFunc func(ib *IndexBuilder)
145
+
type shardBuilderFunc func(ib *ShardBuilder)
146
146
147
147
// explode offers a richer signature compared to Explode for testing. You
148
148
// probably want to call Explode instead.
149
-
func explode(dstDir string, f IndexFile, ibFuncs ...indexBuilderFunc) (map[string]string, error) {
149
+
func explode(dstDir string, f IndexFile, ibFuncs ...shardBuilderFunc) (map[string]string, error) {
150
150
searcher, err := NewSearcher(f)
151
151
if err != nil {
152
152
return nil, err
···
155
155
156
156
shardNames := make(map[string]string, len(d.repoMetaData))
157
157
158
-
writeShard := func(ib *IndexBuilder) error {
158
+
writeShard := func(ib *ShardBuilder) error {
159
159
if len(ib.repoList) != 1 {
160
-
return fmt.Errorf("expected ib to contain exactly 1 repository")
160
+
return fmt.Errorf("expected sb to contain exactly 1 repository")
161
161
}
162
162
for _, ibFunc := range ibFuncs {
163
163
ibFunc(ib)
···
176
176
return builderWriteAll(shardNameTmp, ib)
177
177
}
178
178
179
-
var ib *IndexBuilder
179
+
var sb *ShardBuilder
180
180
lastRepoID := -1
181
181
for docID := uint32(0); int(docID) < len(d.fileBranchMasks); docID++ {
182
182
repoID := int(d.repos[docID])
···
191
191
}
192
192
lastRepoID = repoID
193
193
194
-
if ib != nil {
195
-
if err := writeShard(ib); err != nil {
194
+
if sb != nil {
195
+
if err := writeShard(sb); err != nil {
196
196
return shardNames, err
197
197
}
198
198
}
199
199
200
-
ib = newIndexBuilder()
201
-
ib.indexFormatVersion = IndexFormatVersion
202
-
if err := ib.setRepository(&d.repoMetaData[repoID]); err != nil {
200
+
sb = newShardBuilder()
201
+
sb.indexFormatVersion = IndexFormatVersion
202
+
if err := sb.setRepository(&d.repoMetaData[repoID]); err != nil {
203
203
return shardNames, err
204
204
}
205
205
}
206
206
207
-
err := addDocument(d, ib, repoID, docID)
207
+
err := addDocument(d, sb, repoID, docID)
208
208
if err != nil {
209
209
return shardNames, err
210
210
}
211
211
}
212
212
213
-
if ib != nil {
214
-
if err := writeShard(ib); err != nil {
213
+
if sb != nil {
214
+
if err := writeShard(sb); err != nil {
215
215
return shardNames, err
216
216
}
217
217
}
···
219
219
return shardNames, nil
220
220
}
221
221
222
-
func addDocument(d *indexData, ib *IndexBuilder, repoID int, docID uint32) error {
222
+
func addDocument(d *indexData, ib *ShardBuilder, repoID int, docID uint32) error {
223
223
doc := Document{
224
224
Name: string(d.fileName(docID)),
225
225
// Content set below since it can return an error
+1
-1
index/merge_test.go
+1
-1
index/merge_test.go
+8
-8
index/read_test.go
+8
-8
index/read_test.go
···
34
34
)
35
35
36
36
func TestReadWrite(t *testing.T) {
37
-
b, err := NewIndexBuilder(nil)
37
+
b, err := NewShardBuilder(nil)
38
38
if err != nil {
39
-
t.Fatalf("NewIndexBuilder: %v", err)
39
+
t.Fatalf("NewShardBuilder: %v", err)
40
40
}
41
41
42
42
if err := b.AddFile("filename", []byte("abcde")); err != nil {
···
79
79
}
80
80
81
81
func TestReadWriteNames(t *testing.T) {
82
-
b, err := NewIndexBuilder(nil)
82
+
b, err := NewShardBuilder(nil)
83
83
if err != nil {
84
-
t.Fatalf("NewIndexBuilder: %v", err)
84
+
t.Fatalf("NewShardBuilder: %v", err)
85
85
}
86
86
87
87
if err := b.AddFile("abCd", []byte("")); err != nil {
···
123
123
}
124
124
125
125
func TestGet(t *testing.T) {
126
-
b, err := NewIndexBuilder(nil)
126
+
b, err := NewShardBuilder(nil)
127
127
if err != nil {
128
-
t.Fatalf("NewIndexBuilder: %v", err)
128
+
t.Fatalf("NewShardBuilder: %v", err)
129
129
}
130
130
131
131
if err := b.AddFile("file_name", []byte("aaa bbbaaa")); err != nil {
···
353
353
354
354
func TestBackwardsCompat(t *testing.T) {
355
355
if *update {
356
-
b, err := NewIndexBuilder(nil)
356
+
b, err := NewShardBuilder(nil)
357
357
if err != nil {
358
-
t.Fatalf("NewIndexBuilder: %v", err)
358
+
t.Fatalf("NewShardBuilder: %v", err)
359
359
}
360
360
361
361
if err := b.AddFile("filename", []byte("abcde")); err != nil {
+2
-2
index/write.go
+2
-2
index/write.go
···
100
100
endRunes.end(w)
101
101
}
102
102
103
-
func (b *IndexBuilder) Write(out io.Writer) error {
103
+
func (b *ShardBuilder) Write(out io.Writer) error {
104
104
next := b.indexFormatVersion == NextIndexFormatVersion
105
105
106
106
buffered := bufio.NewWriterSize(out, 1<<20)
···
211
211
return w.err
212
212
}
213
213
214
-
func (b *IndexBuilder) writeJSON(data interface{}, sec *simpleSection, w *writer) error {
214
+
func (b *ShardBuilder) writeJSON(data interface{}, sec *simpleSection, w *writer) error {
215
215
blob, err := json.Marshal(data)
216
216
if err != nil {
217
217
return err
+1
-1
internal/shards/eval_test.go
+1
-1
internal/shards/eval_test.go
···
14
14
ss := newShardedSearcher(2)
15
15
nextShardNum := 1
16
16
addShard := func(docs ...index.Document) {
17
-
b := testIndexBuilder(t, &zoekt.Repository{ID: 1, Name: "reponame"}, docs...)
17
+
b := testShardBuilder(t, &zoekt.Repository{ID: 1, Name: "reponame"}, docs...)
18
18
shard := searcherForTest(t, b)
19
19
ss.replace(map[string]zoekt.Searcher{fmt.Sprintf("key-%d", nextShardNum): shard})
20
20
nextShardNum++
+22
-22
internal/shards/shards_test.go
+22
-22
internal/shards/shards_test.go
···
206
206
"public": "1",
207
207
"priority": strconv.FormatFloat(priority, 'f', 2, 64),
208
208
}
209
-
b := testIndexBuilder(t, r, docs...)
209
+
b := testShardBuilder(t, r, docs...)
210
210
shard := searcherForTest(t, b)
211
211
ss.replace(map[string]zoekt.Searcher{
212
212
fmt.Sprintf("key-%d", nextShardNum): shard,
···
248
248
"public": "1",
249
249
}
250
250
r.Rank = rank
251
-
b := testIndexBuilder(t, r, docs...)
251
+
b := testShardBuilder(t, r, docs...)
252
252
shard := searcherForTest(t, b)
253
253
ss.replace(map[string]zoekt.Searcher{
254
254
fmt.Sprintf("key-%d", nextShardNum): shard,
···
410
410
}
411
411
412
412
func TestUnloadIndex(t *testing.T) {
413
-
b := testIndexBuilder(t, nil, index.Document{
413
+
b := testShardBuilder(t, nil, index.Document{
414
414
Name: "filename",
415
415
Content: []byte("needle needle needle"),
416
416
})
···
481
481
// Test duplicate removal when ListOptions.Minimal is true and false
482
482
ss := newShardedSearcher(4)
483
483
ss.replace(map[string]zoekt.Searcher{
484
-
"1": searcherForTest(t, testIndexBuilder(t, repos[0], doc)),
485
-
"2": searcherForTest(t, testIndexBuilder(t, repos[0])),
486
-
"3": searcherForTest(t, testIndexBuilder(t, repos[1], doc)),
487
-
"4": searcherForTest(t, testIndexBuilder(t, repos[1])),
484
+
"1": searcherForTest(t, testShardBuilder(t, repos[0], doc)),
485
+
"2": searcherForTest(t, testShardBuilder(t, repos[0])),
486
+
"3": searcherForTest(t, testShardBuilder(t, repos[1], doc)),
487
+
"4": searcherForTest(t, testShardBuilder(t, repos[1])),
488
488
})
489
489
ss.markReady()
490
490
···
609
609
}
610
610
}
611
611
612
-
func testIndexBuilder(t testing.TB, repo *zoekt.Repository, docs ...index.Document) *index.IndexBuilder {
613
-
b, err := index.NewIndexBuilder(repo)
612
+
func testShardBuilder(t testing.TB, repo *zoekt.Repository, docs ...index.Document) *index.ShardBuilder {
613
+
b, err := index.NewShardBuilder(repo)
614
614
if err != nil {
615
-
t.Fatalf("NewIndexBuilder: %v", err)
615
+
t.Fatalf("NewShardBuilder: %v", err)
616
616
}
617
617
618
618
for i, d := range docs {
···
623
623
return b
624
624
}
625
625
626
-
func searcherForTest(t testing.TB, b *index.IndexBuilder) zoekt.Searcher {
626
+
func searcherForTest(t testing.TB, b *index.ShardBuilder) zoekt.Searcher {
627
627
var buf bytes.Buffer
628
628
if err := b.Write(&buf); err != nil {
629
629
t.Fatal(err)
···
649
649
}
650
650
651
651
func testSearcherForRepo(b testing.TB, r *zoekt.Repository, numFiles int) zoekt.Searcher {
652
-
builder := testIndexBuilder(b, r)
652
+
builder := testShardBuilder(b, r)
653
653
654
654
if err := builder.Add(index.Document{
655
655
Name: fmt.Sprintf("%s/filename-%d.go", r.Name, 0),
···
758
758
addShard := func(repo string, rawConfig map[string]string, docs ...index.Document) {
759
759
r := &zoekt.Repository{Name: repo}
760
760
r.RawConfig = rawConfig
761
-
b := testIndexBuilder(t, r, docs...)
761
+
b := testShardBuilder(t, r, docs...)
762
762
shard := searcherForTest(t, b)
763
763
ss.replace(map[string]zoekt.Searcher{fmt.Sprintf("key-%d", nextShardNum): shard})
764
764
nextShardNum++
···
980
980
func TestFileBasedSearch(t *testing.T) {
981
981
cases := []struct {
982
982
name string
983
-
testShardedSearch func(t *testing.T, q query.Q, ib *index.IndexBuilder, useDocumentRanks bool) []zoekt.FileMatch
983
+
testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
984
984
}{
985
985
{"Search", testShardedSearch},
986
986
{"StreamSearch", testShardedStreamSearch},
···
990
990
// -----------0123456789012345678901234567890123456789
991
991
c2 := []byte("In Dutch, ananas means pineapple")
992
992
// -----------0123456789012345678901234567890123456789
993
-
b := testIndexBuilder(t, nil,
993
+
b := testShardBuilder(t, nil,
994
994
index.Document{Name: "f1", Content: c1},
995
995
index.Document{Name: "f2", Content: c2},
996
996
)
···
1020
1020
func TestWordBoundaryRanking(t *testing.T) {
1021
1021
cases := []struct {
1022
1022
name string
1023
-
testShardedSearch func(t *testing.T, q query.Q, ib *index.IndexBuilder, useDocumentRanks bool) []zoekt.FileMatch
1023
+
testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1024
1024
}{
1025
1025
{"Search", testShardedSearch},
1026
1026
{"StreamSearch", testShardedStreamSearch},
1027
1027
}
1028
1028
1029
-
b := testIndexBuilder(t, nil,
1029
+
b := testShardBuilder(t, nil,
1030
1030
index.Document{Name: "f1", Content: []byte("xbytex xbytex")},
1031
1031
index.Document{Name: "f2", Content: []byte("xbytex\nbytex\nbyte bla")},
1032
1032
// -----------------------------------------0123456 789012 34567890
···
1060
1060
func TestAtomCountScore(t *testing.T) {
1061
1061
cases := []struct {
1062
1062
name string
1063
-
testShardedSearch func(t *testing.T, q query.Q, ib *index.IndexBuilder, useDocumentRanks bool) []zoekt.FileMatch
1063
+
testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1064
1064
}{
1065
1065
{"Search", testShardedSearch},
1066
1066
{"StreamSearch", testShardedStreamSearch},
1067
1067
}
1068
1068
1069
-
b := testIndexBuilder(t,
1069
+
b := testShardBuilder(t,
1070
1070
&zoekt.Repository{
1071
1071
Branches: []zoekt.RepositoryBranch{
1072
1072
{Name: "branches", Version: "v1"},
···
1100
1100
}
1101
1101
1102
1102
func TestUseBM25Scoring(t *testing.T) {
1103
-
b := testIndexBuilder(t,
1103
+
b := testShardBuilder(t,
1104
1104
&zoekt.Repository{},
1105
1105
index.Document{Name: "f1", Content: []byte("one two two three")},
1106
1106
index.Document{Name: "f2", Content: []byte("one two one two")},
···
1134
1134
}
1135
1135
}
1136
1136
1137
-
func testShardedStreamSearch(t *testing.T, q query.Q, ib *index.IndexBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1137
+
func testShardedStreamSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1138
1138
ss := newShardedSearcher(1)
1139
1139
searcher := searcherForTest(t, ib)
1140
1140
ss.replace(map[string]zoekt.Searcher{"r1": searcher})
···
1154
1154
return files
1155
1155
}
1156
1156
1157
-
func testShardedSearch(t *testing.T, q query.Q, ib *index.IndexBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1157
+
func testShardedSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1158
1158
ss := newShardedSearcher(1)
1159
1159
searcher := searcherForTest(t, ib)
1160
1160
ss.replace(map[string]zoekt.Searcher{"r1": searcher})
+21
-21
web/e2e_test.go
+21
-21
web/e2e_test.go
···
55
55
return "memSeeker"
56
56
}
57
57
58
-
func searcherForTest(t *testing.T, b *index.IndexBuilder) zoekt.Streamer {
58
+
func searcherForTest(t *testing.T, b *index.ShardBuilder) zoekt.Streamer {
59
59
var buf bytes.Buffer
60
60
if err := b.Write(&buf); err != nil {
61
61
t.Fatal(err)
···
84
84
}
85
85
86
86
func TestBasic(t *testing.T) {
87
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
87
+
b, err := index.NewShardBuilder(&zoekt.Repository{
88
88
Name: "name",
89
89
URL: "repo-url",
90
90
CommitURLTemplate: `{{ URLJoinPath "https://github.com/org/repo/commit/" .Version}}`,
···
93
93
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
94
94
})
95
95
if err != nil {
96
-
t.Fatalf("NewIndexBuilder: %v", err)
96
+
t.Fatalf("NewShardBuilder: %v", err)
97
97
}
98
98
if err := b.Add(index.Document{
99
99
// use a name which requires correct escaping. https://github.com/sourcegraph/zoekt/issues/807
···
150
150
}
151
151
152
152
func TestPrint(t *testing.T) {
153
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
153
+
b, err := index.NewShardBuilder(&zoekt.Repository{
154
154
Name: "name",
155
155
URL: "repo-url",
156
156
CommitURLTemplate: "{{.Version}}",
···
159
159
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
160
160
})
161
161
if err != nil {
162
-
t.Fatalf("NewIndexBuilder: %v", err)
162
+
t.Fatalf("NewShardBuilder: %v", err)
163
163
}
164
164
if err := b.Add(index.Document{
165
165
Name: "f2",
···
203
203
}
204
204
205
205
func TestPrintDefault(t *testing.T) {
206
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
206
+
b, err := index.NewShardBuilder(&zoekt.Repository{
207
207
Name: "name",
208
208
URL: "repo-url",
209
209
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
210
210
})
211
211
if err != nil {
212
-
t.Fatalf("NewIndexBuilder: %v", err)
212
+
t.Fatalf("NewShardBuilder: %v", err)
213
213
}
214
214
if err := b.Add(index.Document{
215
215
Name: "f2",
···
273
273
}
274
274
275
275
func TestFormatJson(t *testing.T) {
276
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
276
+
b, err := index.NewShardBuilder(&zoekt.Repository{
277
277
Name: "name",
278
278
URL: "repo-url",
279
279
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
280
280
})
281
281
if err != nil {
282
-
t.Fatalf("NewIndexBuilder: %v", err)
282
+
t.Fatalf("NewShardBuilder: %v", err)
283
283
}
284
284
if err := b.Add(index.Document{
285
285
Name: "f2",
···
328
328
}
329
329
330
330
func TestContextLines(t *testing.T) {
331
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
331
+
b, err := index.NewShardBuilder(&zoekt.Repository{
332
332
Name: "name",
333
333
URL: "repo-url",
334
334
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
335
335
})
336
336
if err != nil {
337
-
t.Fatalf("NewIndexBuilder: %v", err)
337
+
t.Fatalf("NewShardBuilder: %v", err)
338
338
}
339
339
if err := b.Add(index.Document{
340
340
Name: "f2",
···
652
652
}
653
653
654
654
func TestContextLinesMustBeValid(t *testing.T) {
655
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
655
+
b, err := index.NewShardBuilder(&zoekt.Repository{
656
656
Name: "name",
657
657
URL: "repo-url",
658
658
Branches: []zoekt.RepositoryBranch{{Name: "master", Version: "1234"}},
659
659
})
660
660
if err != nil {
661
-
t.Fatalf("NewIndexBuilder: %v", err)
661
+
t.Fatalf("NewShardBuilder: %v", err)
662
662
}
663
663
if err := b.Add(index.Document{
664
664
Name: "f2",
···
747
747
}
748
748
749
749
func TestHostCustomization(t *testing.T) {
750
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
750
+
b, err := index.NewShardBuilder(&zoekt.Repository{
751
751
Name: "name",
752
752
})
753
753
if err != nil {
754
-
t.Fatalf("NewIndexBuilder: %v", err)
754
+
t.Fatalf("NewShardBuilder: %v", err)
755
755
}
756
756
if err := b.Add(index.Document{
757
757
Name: "file",
···
799
799
}
800
800
801
801
func TestDupResult(t *testing.T) {
802
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
802
+
b, err := index.NewShardBuilder(&zoekt.Repository{
803
803
Name: "name",
804
804
})
805
805
if err != nil {
806
-
t.Fatalf("NewIndexBuilder: %v", err)
806
+
t.Fatalf("NewShardBuilder: %v", err)
807
807
}
808
808
809
809
for i := 0; i < 2; i++ {
···
849
849
}
850
850
851
851
func TestTruncateLine(t *testing.T) {
852
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
852
+
b, err := index.NewShardBuilder(&zoekt.Repository{
853
853
Name: "name",
854
854
})
855
855
if err != nil {
856
-
t.Fatalf("NewIndexBuilder: %v", err)
856
+
t.Fatalf("NewShardBuilder: %v", err)
857
857
}
858
858
859
859
largePadding := bytes.Repeat([]byte{'a'}, 100*1000) // 100kb
···
905
905
}
906
906
907
907
func TestHealthz(t *testing.T) {
908
-
b, err := index.NewIndexBuilder(&zoekt.Repository{
908
+
b, err := index.NewShardBuilder(&zoekt.Repository{
909
909
Name: "name",
910
910
})
911
911
if err != nil {
912
-
t.Fatalf("NewIndexBuilder: %v", err)
912
+
t.Fatalf("NewShardBuilder: %v", err)
913
913
}
914
914
915
915
for i := 0; i < 2; i++ {