fork of https://github.com/sourcegraph/zoekt
change findShard implementation to match on IDs, not names (#260)
author
Geoffrey Gilmore
committer
GitHub
date
(Feb 10, 2022, 12:04 PM -0800)
commit
9d1b56e6
9d1b56e6180fded213637b276ed01130d0e1d162
parent
f6ba702f
f6ba702f99c05f7626a416009a61d62a6870e20f
+230
-1
+1
-1
build/builder.go
+1
-1
build/builder.go
+229
build/builder_test.go
+229
build/builder_test.go
···
6
6
"log"
7
7
"os"
8
8
"path/filepath"
9
+
"strconv"
10
+
"strings"
9
11
"testing"
10
12
11
13
"github.com/google/go-cmp/cmp"
···
230
232
t.Fatalf("content of skipped documents should not count towards shard size thresold")
231
233
}
232
234
}
235
+
236
+
func TestOptions_FindAllShards(t *testing.T) {
237
+
type simpleShard struct {
238
+
Repository zoekt.Repository
239
+
// NumShards is the number of shards that should be created that
240
+
// contain data for "Repository".
241
+
NumShards int
242
+
}
243
+
244
+
tests := []struct {
245
+
name string
246
+
simpleShards []simpleShard
247
+
compoundShards [][]zoekt.Repository
248
+
expectedShardCount int
249
+
expectedRepository zoekt.Repository
250
+
}{
251
+
{
252
+
name: "repository in normal shard",
253
+
simpleShards: []simpleShard{
254
+
{Repository: zoekt.Repository{Name: "repoA", ID: 1}},
255
+
{Repository: zoekt.Repository{Name: "repoB", ID: 2}},
256
+
{Repository: zoekt.Repository{Name: "repoC", ID: 3}},
257
+
},
258
+
expectedShardCount: 1,
259
+
expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
260
+
},
261
+
{
262
+
name: "repository in compound shard",
263
+
compoundShards: [][]zoekt.Repository{
264
+
{
265
+
{Name: "repoA", ID: 1},
266
+
{Name: "repoB", ID: 2},
267
+
{Name: "repoC", ID: 3},
268
+
},
269
+
{
270
+
{Name: "repoD", ID: 4},
271
+
{Name: "repoE", ID: 5},
272
+
{Name: "repoF", ID: 6},
273
+
},
274
+
},
275
+
expectedShardCount: 1,
276
+
expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
277
+
},
278
+
{
279
+
name: "repository split across multiple shards",
280
+
simpleShards: []simpleShard{
281
+
{Repository: zoekt.Repository{Name: "repoA", ID: 1}},
282
+
{Repository: zoekt.Repository{Name: "repoB", ID: 2}, NumShards: 2},
283
+
{Repository: zoekt.Repository{Name: "repoC", ID: 3}},
284
+
},
285
+
expectedShardCount: 2,
286
+
expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
287
+
},
288
+
{
289
+
name: "unknown repository",
290
+
simpleShards: []simpleShard{
291
+
{Repository: zoekt.Repository{Name: "repoA", ID: 1}},
292
+
{Repository: zoekt.Repository{Name: "repoB", ID: 2}},
293
+
{Repository: zoekt.Repository{Name: "repoC", ID: 3}},
294
+
},
295
+
compoundShards: [][]zoekt.Repository{
296
+
{
297
+
{Name: "repoD", ID: 4},
298
+
{Name: "repoE", ID: 5},
299
+
{Name: "repoF", ID: 6},
300
+
},
301
+
},
302
+
expectedShardCount: 0,
303
+
},
304
+
{
305
+
name: "match on ID, not name (compound only)",
306
+
compoundShards: [][]zoekt.Repository{
307
+
{
308
+
{Name: "repoA", ID: 1},
309
+
{Name: "sameName", ID: 2},
310
+
{Name: "sameName", ID: 3},
311
+
},
312
+
{
313
+
{Name: "repoB", ID: 4},
314
+
{Name: "sameName", ID: 5},
315
+
{Name: "sameName", ID: 6},
316
+
},
317
+
},
318
+
expectedShardCount: 1,
319
+
expectedRepository: zoekt.Repository{Name: "sameName", ID: 5},
320
+
},
321
+
}
322
+
for _, tt := range tests {
323
+
t.Run(tt.name, func(t *testing.T) {
324
+
// prepare
325
+
indexDir := t.TempDir()
326
+
327
+
for _, s := range tt.simpleShards {
328
+
createTestShard(t, indexDir, s.Repository, s.NumShards)
329
+
}
330
+
331
+
for _, repositoryGroup := range tt.compoundShards {
332
+
createTestCompoundShard(t, indexDir, repositoryGroup)
333
+
}
334
+
335
+
o := &Options{
336
+
IndexDir: indexDir,
337
+
RepositoryDescription: tt.expectedRepository,
338
+
}
339
+
o.SetDefaults()
340
+
341
+
// run test
342
+
shards := o.FindAllShards()
343
+
344
+
// verify results
345
+
if len(shards) != tt.expectedShardCount {
346
+
t.Fatalf("expected %d shard(s), received %d shard(s)", tt.expectedShardCount, len(shards))
347
+
}
348
+
349
+
if tt.expectedShardCount > 0 {
350
+
for _, s := range shards {
351
+
// all shards should contain the metadata for the desired repository
352
+
repos, _, err := zoekt.ReadMetadataPathAlive(s)
353
+
if err != nil {
354
+
t.Fatalf("reading metadata from shard %q: %s", s, err)
355
+
}
356
+
357
+
foundRepository := false
358
+
for _, r := range repos {
359
+
if r.ID == tt.expectedRepository.ID {
360
+
foundRepository = true
361
+
break
362
+
}
363
+
}
364
+
365
+
if !foundRepository {
366
+
t.Errorf("shard %q doesn't contain metadata for repository %d", s, tt.expectedRepository.ID)
367
+
}
368
+
}
369
+
}
370
+
})
371
+
}
372
+
}
373
+
374
+
func createTestShard(t *testing.T, indexDir string, r zoekt.Repository, numShards int) {
375
+
t.Helper()
376
+
377
+
if err := os.MkdirAll(filepath.Dir(indexDir), 0700); err != nil {
378
+
t.Fatal(err)
379
+
}
380
+
381
+
o := Options{
382
+
IndexDir: indexDir,
383
+
RepositoryDescription: r,
384
+
ShardMax: 75, // create a new shard every 75 bytes
385
+
}
386
+
o.SetDefaults()
387
+
388
+
b, err := NewBuilder(o)
389
+
if err != nil {
390
+
t.Fatalf("NewBuilder: %v", err)
391
+
}
392
+
393
+
if numShards == 0 {
394
+
// We have to make at least 1 shard.
395
+
numShards = 1
396
+
}
397
+
398
+
for i := 0; i < numShards; i++ {
399
+
// Create entries (file + contents) that are ~100 bytes each.
400
+
// This (along with our shardMax setting of 75 bytes) means that each shard
401
+
// will contain at most one of these.
402
+
fileName := strconv.Itoa(i)
403
+
contents := []byte(strings.Repeat("A", 100))
404
+
405
+
err := b.AddFile(fileName, contents)
406
+
if err != nil {
407
+
t.Fatalf("failed to add file %q to builder: %s", fileName, err)
408
+
}
409
+
}
410
+
411
+
if err := b.Finish(); err != nil {
412
+
t.Fatalf("Finish: %v", err)
413
+
}
414
+
}
415
+
416
+
func createTestCompoundShard(t *testing.T, indexDir string, repositories []zoekt.Repository) {
417
+
t.Helper()
418
+
419
+
var shardNames []string
420
+
421
+
for _, r := range repositories {
422
+
// create an isolated scratch space to store normal shards for this repository
423
+
scratchDir := t.TempDir()
424
+
425
+
// create shards that'll be merged later
426
+
createTestShard(t, scratchDir, r, 1)
427
+
428
+
// discover file names for all the normal shards we created
429
+
// note: this only looks in the immediate 'scratchDir' folder and doesn't recurse
430
+
shards, err := filepath.Glob(filepath.Join(scratchDir, "*.zoekt"))
431
+
if err != nil {
432
+
t.Fatalf("while globbing %q to find normal shards: %s", scratchDir, err)
433
+
}
434
+
435
+
shardNames = append(shardNames, shards...)
436
+
}
437
+
438
+
// load the normal shards that we created
439
+
var files []zoekt.IndexFile
440
+
for _, shard := range shardNames {
441
+
f, err := os.Open(shard)
442
+
if err != nil {
443
+
t.Fatalf("opening shard file: %s", err)
444
+
}
445
+
defer f.Close()
446
+
447
+
indexFile, err := zoekt.NewIndexFile(f)
448
+
if err != nil {
449
+
t.Fatalf("creating index file: %s", err)
450
+
}
451
+
defer indexFile.Close()
452
+
453
+
files = append(files, indexFile)
454
+
}
455
+
456
+
// merge all the normal shards into a compound shard
457
+
_, err := zoekt.Merge(indexDir, files...)
458
+
if err != nil {
459
+
t.Fatalf("merging index files into compound shard: %s", err)
460
+
}
461
+
}