fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "log"
23 "os"
24 "path/filepath"
25 "reflect"
26 "runtime"
27 "sort"
28 "strconv"
29 "strings"
30 "testing"
31 "time"
32
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35 "github.com/grafana/regexp"
36 "github.com/sourcegraph/zoekt"
37 "github.com/sourcegraph/zoekt/internal/shards"
38 "github.com/sourcegraph/zoekt/internal/tenant"
39 "github.com/sourcegraph/zoekt/internal/tenant/tenanttest"
40 "github.com/sourcegraph/zoekt/query"
41 "github.com/stretchr/testify/require"
42)
43
44func TestBasic(t *testing.T) {
45 dir := t.TempDir()
46
47 opts := Options{
48 IndexDir: dir,
49 ShardMax: 1024,
50 RepositoryDescription: zoekt.Repository{
51 Name: "repo",
52 },
53 Parallelism: 2,
54 SizeMax: 1 << 20,
55 }
56
57 b, err := NewBuilder(opts)
58 if err != nil {
59 t.Fatalf("NewBuilder: %v", err)
60 }
61
62 for i := 0; i < 4; i++ {
63 s := fmt.Sprintf("%d", i)
64 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1000))); err != nil {
65 t.Fatal(err)
66 }
67 }
68
69 if err := b.Finish(); err != nil {
70 t.Errorf("Finish: %v", err)
71 }
72
73 fs, _ := filepath.Glob(dir + "/*.zoekt")
74 if len(fs) <= 1 {
75 t.Fatalf("want multiple shards, got %v", fs)
76 }
77
78 _, md0, err := zoekt.ReadMetadataPath(fs[0])
79 if err != nil {
80 t.Fatal(err)
81 }
82 for _, f := range fs[1:] {
83 _, md, err := zoekt.ReadMetadataPath(f)
84 if err != nil {
85 t.Fatal(err)
86 }
87 if md.IndexTime != md0.IndexTime {
88 t.Fatalf("wanted identical time stamps but got %v!=%v", md.IndexTime, md0.IndexTime)
89 }
90 if md.ID != md0.ID {
91 t.Fatalf("wanted identical IDs but got %s!=%s", md.ID, md0.ID)
92 }
93 }
94
95 ss, err := shards.NewDirectorySearcher(dir)
96 if err != nil {
97 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
98 }
99 defer ss.Close()
100
101 q, err := query.Parse("111")
102 if err != nil {
103 t.Fatalf("Parse(111): %v", err)
104 }
105
106 var sOpts zoekt.SearchOptions
107 ctx := context.Background()
108 result, err := ss.Search(ctx, q, &sOpts)
109 if err != nil {
110 t.Fatalf("Search(%v): %v", q, err)
111 }
112
113 if len(result.Files) != 1 {
114 t.Errorf("got %v, want 1 file.", result.Files)
115 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
116 t.Errorf("got file %q, want %q", gotFile, wantFile)
117 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo"; gotRepo != wantRepo {
118 t.Errorf("got repo %q, want %q", gotRepo, wantRepo)
119 }
120
121 t.Run("meta file", func(t *testing.T) {
122 // use retryTest to allow for the directory watcher to notice the meta
123 // file
124 retryTest(t, func(fatalf func(format string, args ...interface{})) {
125 // Add a .meta file for each shard with repo.Name set to
126 // "repo-mutated". We do this inside retry helper since we have noticed
127 // some flakiness on github CI.
128 for _, p := range fs {
129 repos, _, err := zoekt.ReadMetadataPath(p)
130 if err != nil {
131 t.Fatal(err)
132 }
133 repos[0].Name = "repo-mutated"
134 b, err := json.Marshal(repos[0])
135 if err != nil {
136 t.Fatal(err)
137 }
138
139 if err := os.WriteFile(p+".meta", b, 0o600); err != nil {
140 t.Fatal(err)
141 }
142 }
143
144 result, err := ss.Search(ctx, q, &sOpts)
145 if err != nil {
146 fatalf("Search(%v): %v", q, err)
147 }
148
149 if len(result.Files) != 1 {
150 fatalf("got %v, want 1 file.", result.Files)
151 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
152 fatalf("got file %q, want %q", gotFile, wantFile)
153 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo-mutated"; gotRepo != wantRepo {
154 fatalf("got repo %q, want %q", gotRepo, wantRepo)
155 }
156 })
157 })
158}
159
160func TestSearchTenant(t *testing.T) {
161 tenanttest.MockEnforce(t)
162
163 dir := t.TempDir()
164
165 ctx1 := tenanttest.NewTestContext()
166 tnt1, err := tenant.FromContext(ctx1)
167 require.NoError(t, err)
168
169 opts := Options{
170 IndexDir: dir,
171 ShardMax: 1024,
172 RepositoryDescription: zoekt.Repository{
173 Name: "repo",
174 RawConfig: map[string]string{"tenantID": strconv.Itoa(tnt1.ID())},
175 },
176 Parallelism: 2,
177 SizeMax: 1 << 20,
178 }
179
180 b, err := NewBuilder(opts)
181 if err != nil {
182 t.Fatalf("NewBuilder: %v", err)
183 }
184
185 for i := 0; i < 4; i++ {
186 s := fmt.Sprintf("%d", i)
187 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1000))); err != nil {
188 t.Fatal(err)
189 }
190 }
191
192 if err := b.Finish(); err != nil {
193 t.Errorf("Finish: %v", err)
194 }
195
196 fs, _ := filepath.Glob(dir + "/*.zoekt")
197 if len(fs) <= 1 {
198 t.Fatalf("want multiple shards, got %v", fs)
199 }
200
201 _, md0, err := zoekt.ReadMetadataPath(fs[0])
202 if err != nil {
203 t.Fatal(err)
204 }
205 for _, f := range fs[1:] {
206 _, md, err := zoekt.ReadMetadataPath(f)
207 if err != nil {
208 t.Fatal(err)
209 }
210 if md.IndexTime != md0.IndexTime {
211 t.Fatalf("wanted identical time stamps but got %v!=%v", md.IndexTime, md0.IndexTime)
212 }
213 if md.ID != md0.ID {
214 t.Fatalf("wanted identical IDs but got %s!=%s", md.ID, md0.ID)
215 }
216 }
217
218 ss, err := shards.NewDirectorySearcher(dir)
219 if err != nil {
220 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
221 }
222 defer ss.Close()
223
224 q, err := query.Parse("111")
225 if err != nil {
226 t.Fatalf("Parse(111): %v", err)
227 }
228
229 var sOpts zoekt.SearchOptions
230
231 // Tenant 1 has access to the repo
232 result, err := ss.Search(ctx1, q, &sOpts)
233 require.NoError(t, err)
234 require.Len(t, result.Files, 1)
235
236 // Tenant 2 does not have access to the repo
237 ctx2 := tenanttest.NewTestContext()
238 result, err = ss.Search(ctx2, q, &sOpts)
239 require.NoError(t, err)
240 require.Len(t, result.Files, 0)
241}
242
243func TestListTenant(t *testing.T) {
244 tenanttest.MockEnforce(t)
245
246 dir := t.TempDir()
247
248 ctx1 := tenanttest.NewTestContext()
249 tnt1, err := tenant.FromContext(ctx1)
250 require.NoError(t, err)
251
252 opts := Options{
253 IndexDir: dir,
254 RepositoryDescription: zoekt.Repository{
255 Name: "repo",
256 RawConfig: map[string]string{"tenantID": strconv.Itoa(tnt1.ID())},
257 },
258 }
259 opts.SetDefaults()
260
261 b, err := NewBuilder(opts)
262 if err != nil {
263 t.Fatalf("NewBuilder: %v", err)
264 }
265 if err := b.Finish(); err != nil {
266 t.Errorf("Finish: %v", err)
267 }
268
269 fs, _ := filepath.Glob(dir + "/*.zoekt")
270 if len(fs) != 1 {
271 t.Fatalf("want a shard, got %v", fs)
272 }
273
274 ss, err := shards.NewDirectorySearcher(dir)
275 if err != nil {
276 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
277 }
278 defer ss.Close()
279
280 // Tenant 1 has access to the repo
281 result, err := ss.List(ctx1, &query.Const{Value: true}, nil)
282 require.NoError(t, err)
283 require.Len(t, result.Repos, 1)
284
285 // Tenant 2 does not have access to the repo
286 ctx2 := tenanttest.NewTestContext()
287 result, err = ss.List(ctx2, &query.Const{Value: true}, nil)
288 require.NoError(t, err)
289 require.Len(t, result.Repos, 0)
290}
291
292// retryTest will retry f until min(t.Deadline(), time.Minute). It returns
293// once f doesn't call fatalf.
294func retryTest(t *testing.T, f func(fatalf func(format string, args ...interface{}))) {
295 t.Helper()
296
297 sleep := 10 * time.Millisecond
298 deadline := time.Now().Add(time.Minute)
299 if d, ok := t.Deadline(); ok && d.Before(deadline) {
300 // give 1s for us to do a final test run
301 deadline = d.Add(-time.Second)
302 }
303
304 for {
305 done := make(chan bool)
306 go func() {
307 defer close(done)
308
309 f(func(format string, args ...interface{}) {
310 runtime.Goexit()
311 })
312
313 done <- true
314 }()
315
316 success := <-done
317 if success {
318 return
319 }
320
321 // each time we increase sleep by 1.5
322 sleep := sleep*2 - sleep/2
323 if time.Now().Add(sleep).After(deadline) {
324 break
325 }
326 time.Sleep(sleep)
327 }
328
329 // final run for the test, using the real t.Fatalf
330 f(t.Fatalf)
331}
332
333func TestLargeFileOption(t *testing.T) {
334 dir := t.TempDir()
335
336 sizeMax := 1000
337 opts := Options{
338 IndexDir: dir,
339 LargeFiles: []string{"F0", "F1", "F2", "!F1"},
340 RepositoryDescription: zoekt.Repository{
341 Name: "repo",
342 },
343 SizeMax: sizeMax,
344 }
345
346 b, err := NewBuilder(opts)
347 if err != nil {
348 t.Fatalf("NewBuilder: %v", err)
349 }
350
351 for i := 0; i < 4; i++ {
352 s := fmt.Sprintf("%d", i)
353 if err := b.AddFile("F"+s, []byte(strings.Repeat("a", sizeMax+1))); err != nil {
354 t.Fatal(err)
355 }
356 }
357
358 if err := b.Finish(); err != nil {
359 t.Errorf("Finish: %v", err)
360 }
361
362 ss, err := shards.NewDirectorySearcher(dir)
363 if err != nil {
364 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
365 }
366
367 q, err := query.Parse("aaa")
368 if err != nil {
369 t.Fatalf("Parse(aaa): %v", err)
370 }
371
372 var sOpts zoekt.SearchOptions
373 ctx := context.Background()
374 result, err := ss.Search(ctx, q, &sOpts)
375 if err != nil {
376 t.Fatalf("Search(%v): %v", q, err)
377 }
378
379 if len(result.Files) != 2 {
380 t.Errorf("got %v files, want 2 files.", len(result.Files))
381 }
382 defer ss.Close()
383}
384
385func TestUpdate(t *testing.T) {
386 dir := t.TempDir()
387
388 opts := Options{
389 IndexDir: dir,
390 ShardMax: 1024,
391 RepositoryDescription: zoekt.Repository{
392 Name: "repo",
393 FileURLTemplate: "url",
394 },
395 Parallelism: 2,
396 SizeMax: 1 << 20,
397 }
398
399 if b, err := NewBuilder(opts); err != nil {
400 t.Fatalf("NewBuilder: %v", err)
401 } else {
402 if err := b.AddFile("F", []byte("hoi")); err != nil {
403 t.Errorf("AddFile: %v", err)
404 }
405 if err := b.Finish(); err != nil {
406 t.Errorf("Finish: %v", err)
407 }
408 }
409 ss, err := shards.NewDirectorySearcher(dir)
410 if err != nil {
411 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
412 }
413
414 ctx := context.Background()
415 repos, err := ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
416 if err != nil {
417 t.Fatalf("List: %v", err)
418 }
419
420 if len(repos.Repos) != 1 {
421 t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
422 }
423
424 fs, err := filepath.Glob(filepath.Join(dir, "*"))
425 if err != nil {
426 t.Fatalf("glob: %v", err)
427 }
428
429 opts.RepositoryDescription = zoekt.Repository{
430 Name: "repo2",
431 FileURLTemplate: "url2",
432 }
433
434 if b, err := NewBuilder(opts); err != nil {
435 t.Fatalf("NewBuilder: %v", err)
436 } else {
437 if err := b.AddFile("F", []byte("hoi")); err != nil {
438 t.Errorf("AddFile: %v", err)
439 }
440 if err := b.Finish(); err != nil {
441 t.Errorf("Finish: %v", err)
442 }
443 }
444
445 // This is ugly, and potentially flaky, but there is no
446 // observable synchronization for the Sharded searcher, so
447 // this is the best we can do.
448 time.Sleep(100 * time.Millisecond)
449
450 ctx = context.Background()
451 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
452 t.Fatalf("List: %v", err)
453 } else if len(repos.Repos) != 2 {
454 t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
455 }
456
457 for _, fn := range fs {
458 log.Printf("removing %s", fn)
459 if err := os.Remove(fn); err != nil {
460 t.Fatalf("Remove(%s): %v", fn, err)
461 }
462 }
463
464 time.Sleep(100 * time.Millisecond)
465
466 ctx = context.Background()
467 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
468 t.Fatalf("List: %v", err)
469 } else if len(repos.Repos) != 1 {
470 var ss []string
471 for _, r := range repos.Repos {
472 ss = append(ss, r.Repository.Name)
473 }
474 t.Errorf("List(repo): got %v, want 1 repo", ss)
475 }
476}
477
478func TestDeleteOldShards(t *testing.T) {
479 dir := t.TempDir()
480
481 opts := Options{
482 IndexDir: dir,
483 ShardMax: 1024,
484 RepositoryDescription: zoekt.Repository{
485 Name: "repo",
486 FileURLTemplate: "url",
487 },
488 SizeMax: 1 << 20,
489 }
490 opts.SetDefaults()
491
492 b, err := NewBuilder(opts)
493 if err != nil {
494 t.Fatalf("NewBuilder: %v", err)
495 }
496 for i := 0; i < 4; i++ {
497 s := fmt.Sprintf("%d\n", i)
498 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
499 t.Errorf("AddFile: %v", err)
500 }
501 }
502 if err := b.Finish(); err != nil {
503 t.Errorf("Finish: %v", err)
504 }
505
506 glob := filepath.Join(dir, "*.zoekt")
507 fs, err := filepath.Glob(glob)
508 if err != nil {
509 t.Fatalf("Glob(%s): %v", glob, err)
510 } else if len(fs) != 4 {
511 t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
512 }
513
514 if fi, err := os.Lstat(fs[0]); err != nil {
515 t.Fatalf("Lstat: %v", err)
516 } else if fi.Mode()&0o666 == 0o600 {
517 // This fails spuriously if your umask is very restrictive.
518 t.Errorf("got mode %o, should respect umask.", fi.Mode())
519 }
520
521 // Do again, without sharding.
522 opts.ShardMax = 1 << 20
523 b, err = NewBuilder(opts)
524 if err != nil {
525 t.Fatalf("NewBuilder: %v", err)
526 }
527 for i := 0; i < 4; i++ {
528 s := fmt.Sprintf("%d\n", i)
529 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
530 t.Fatal(err)
531 }
532 }
533 if err := b.Finish(); err != nil {
534 t.Errorf("Finish: %v", err)
535 }
536
537 fs, err = filepath.Glob(glob)
538 if err != nil {
539 t.Fatalf("Glob(%s): %v", glob, err)
540 } else if len(fs) != 1 {
541 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
542 }
543
544 // Again, but don't index anything; should leave old shards intact.
545 b, err = NewBuilder(opts)
546 if err != nil {
547 t.Fatalf("NewBuilder: %v", err)
548 }
549 if err := b.Finish(); err != nil {
550 t.Errorf("Finish: %v", err)
551 }
552
553 fs, err = filepath.Glob(glob)
554 if err != nil {
555 t.Fatalf("Glob(%s): %v", glob, err)
556 } else if len(fs) != 1 {
557 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
558 }
559}
560
561func TestPartialSuccess(t *testing.T) {
562 dir := t.TempDir()
563
564 opts := Options{
565 IndexDir: dir,
566 ShardMax: 1024,
567 SizeMax: 1 << 20,
568 Parallelism: 1,
569 }
570 opts.RepositoryDescription.Name = "repo"
571 opts.SetDefaults()
572
573 b, err := NewBuilder(opts)
574 if err != nil {
575 t.Fatalf("NewBuilder: %v", err)
576 }
577
578 for i := 0; i < 4; i++ {
579 nm := fmt.Sprintf("F%d", i)
580 _ = b.AddFile(nm, []byte(strings.Repeat("01234567\n", 128)))
581 }
582 b.buildError = fmt.Errorf("any error")
583
584 // No error checking.
585 _ = b.Finish()
586
587 // Finish cleans up temporary files.
588 if fs, err := filepath.Glob(dir + "/*"); err != nil {
589 t.Errorf("glob(%s): %v", dir, err)
590 } else if len(fs) != 0 {
591 t.Errorf("got shards %v, want []", fs)
592 }
593}
594
595type filerankCase struct {
596 name string
597 docs []*zoekt.Document
598 want []int
599}
600
601func testFileRankAspect(t *testing.T, c filerankCase) {
602 var want []*zoekt.Document
603 for _, j := range c.want {
604 want = append(want, c.docs[j])
605 }
606
607 got := make([]*zoekt.Document, len(c.docs))
608 copy(got, c.docs)
609 sortDocuments(got)
610
611 print := func(ds []*zoekt.Document) string {
612 r := ""
613 for _, d := range ds {
614 r += fmt.Sprintf("%v, ", d)
615 }
616 return r
617 }
618 if !reflect.DeepEqual(got, want) {
619 t.Errorf("got docs [%v], want [%v]", print(got), print(want))
620 }
621}
622
623func TestFileRank(t *testing.T) {
624 for _, c := range []filerankCase{{
625 name: "filename",
626 docs: []*zoekt.Document{
627 {
628 Name: "longlonglong",
629 Content: []byte("bla"),
630 },
631 {
632 Name: "short",
633 Content: []byte("bla"),
634 },
635 },
636 want: []int{1, 0},
637 }, {
638 name: "test",
639 docs: []*zoekt.Document{
640 {
641 Name: "foo_test.go",
642 Content: []byte("bla"),
643 },
644 {
645 Name: "longlonglong",
646 Content: []byte("bla"),
647 },
648 },
649 want: []int{1, 0},
650 }, {
651 name: "content",
652 docs: []*zoekt.Document{
653 {
654 Content: []byte("bla"),
655 },
656 {
657 Content: []byte("blablablabla"),
658 },
659 {
660 Content: []byte("blabla"),
661 },
662 },
663 want: []int{0, 2, 1},
664 }, {
665 name: "skipped docs",
666 docs: []*zoekt.Document{
667 {
668 Name: "binary_file",
669 SkipReason: "binary file",
670 },
671 {
672 Name: "some_test.go",
673 Content: []byte("bla"),
674 },
675 {
676 Name: "large_file.go",
677 SkipReason: "too large",
678 },
679 {
680 Name: "file.go",
681 Content: []byte("blabla"),
682 },
683 },
684 want: []int{3, 1, 0, 2},
685 }} {
686 t.Run(c.name, func(t *testing.T) {
687 testFileRankAspect(t, c)
688 })
689 }
690}
691
692func TestEmptyContent(t *testing.T) {
693 dir := t.TempDir()
694
695 opts := Options{
696 IndexDir: dir,
697 RepositoryDescription: zoekt.Repository{
698 Name: "repo",
699 },
700 }
701 opts.SetDefaults()
702
703 b, err := NewBuilder(opts)
704 if err != nil {
705 t.Fatalf("NewBuilder: %v", err)
706 }
707 if err := b.Finish(); err != nil {
708 t.Errorf("Finish: %v", err)
709 }
710
711 fs, _ := filepath.Glob(dir + "/*.zoekt")
712 if len(fs) != 1 {
713 t.Fatalf("want a shard, got %v", fs)
714 }
715
716 ss, err := shards.NewDirectorySearcher(dir)
717 if err != nil {
718 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
719 }
720 defer ss.Close()
721
722 ctx := context.Background()
723 result, err := ss.List(ctx, &query.Const{Value: true}, nil)
724 if err != nil {
725 t.Fatalf("List: %v", err)
726 }
727
728 if len(result.Repos) != 1 || result.Repos[0].Repository.Name != "repo" {
729 t.Errorf("got %+v, want 1 repo.", result.Repos)
730 }
731}
732
733func TestDeltaShards(t *testing.T) {
734 // TODO: Need to write a test for compound shards as well.
735 type step struct {
736 name string
737 documents []zoekt.Document
738 optFn func(t *testing.T, o *Options)
739
740 query string
741 expectedDocuments []zoekt.Document
742 }
743
744 var (
745 fooAtMain = zoekt.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v1")}
746 fooAtMainV2 = zoekt.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v2")}
747
748 fooAtMainAndRelease = zoekt.Document{Name: "foo.go", Branches: []string{"main", "release"}, Content: []byte("common foo-main-and-release")}
749
750 barAtMain = zoekt.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main")}
751 barAtMainV2 = zoekt.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main-v2")}
752 )
753
754 for _, test := range []struct {
755 name string
756 steps []step
757 }{
758 {
759 name: "tombstone older documents",
760 steps: []step{
761 {
762 name: "setup",
763 documents: []zoekt.Document{barAtMain, fooAtMain},
764 query: "common",
765 expectedDocuments: []zoekt.Document{barAtMain, fooAtMain},
766 },
767 {
768 name: "add new version of foo, tombstone older ones",
769 documents: []zoekt.Document{fooAtMainV2},
770 optFn: func(t *testing.T, o *Options) {
771 o.IsDelta = true
772 o.changedOrRemovedFiles = []string{"foo.go"}
773 },
774 query: "common",
775 expectedDocuments: []zoekt.Document{barAtMain, fooAtMainV2},
776 },
777 {
778 name: "add new version of bar, tombstone older ones",
779 documents: []zoekt.Document{barAtMainV2},
780 optFn: func(t *testing.T, o *Options) {
781 o.IsDelta = true
782 o.changedOrRemovedFiles = []string{"bar.go"}
783 },
784 query: "common",
785 expectedDocuments: []zoekt.Document{barAtMainV2, fooAtMainV2},
786 },
787 },
788 },
789 {
790 name: "tombstone older documents even if the latest shard has no documents",
791 steps: []step{
792 {
793 name: "setup",
794 documents: []zoekt.Document{barAtMain, fooAtMain},
795 query: "common",
796 expectedDocuments: []zoekt.Document{barAtMain, fooAtMain},
797 },
798 {
799 // a build with no documents could represent a deletion
800 name: "tombstone older documents",
801 documents: nil,
802 optFn: func(t *testing.T, o *Options) {
803 o.IsDelta = true
804 o.changedOrRemovedFiles = []string{"foo.go"}
805 },
806 query: "common",
807 expectedDocuments: []zoekt.Document{barAtMain},
808 },
809 },
810 },
811 {
812 name: "tombstones affect document across branches",
813 steps: []step{
814 {
815 name: "setup",
816 documents: []zoekt.Document{barAtMain, fooAtMainAndRelease},
817 query: "common",
818 expectedDocuments: []zoekt.Document{barAtMain, fooAtMainAndRelease},
819 },
820 {
821 name: "tombstone foo",
822 documents: nil,
823 optFn: func(t *testing.T, o *Options) {
824 o.IsDelta = true
825 o.changedOrRemovedFiles = []string{"foo.go"}
826 },
827 query: "common",
828 expectedDocuments: []zoekt.Document{barAtMain},
829 },
830 },
831 },
832 } {
833 t.Run(test.name, func(t *testing.T) {
834 indexDir := t.TempDir()
835
836 branchSet := make(map[string]struct{})
837
838 for _, s := range test.steps {
839 for _, d := range s.documents {
840 for _, b := range d.Branches {
841 branchSet[b] = struct{}{}
842 }
843 }
844 }
845
846 for _, step := range test.steps {
847 repository := zoekt.Repository{ID: 1, Name: "repository"}
848
849 for b := range branchSet {
850 repository.Branches = append(repository.Branches, zoekt.RepositoryBranch{Name: b})
851 }
852
853 sort.Slice(repository.Branches, func(i, j int) bool {
854 a, b := repository.Branches[i], repository.Branches[j]
855
856 return a.Name < b.Name
857 })
858
859 buildOpts := Options{
860 IndexDir: indexDir,
861 RepositoryDescription: repository,
862 }
863 buildOpts.SetDefaults()
864
865 if step.optFn != nil {
866 step.optFn(t, &buildOpts)
867 }
868
869 b, err := NewBuilder(buildOpts)
870 if err != nil {
871 t.Fatalf("step %q: NewBuilder: %s", step.name, err)
872 }
873
874 for _, d := range step.documents {
875 err := b.Add(d)
876 if err != nil {
877 t.Fatalf("step %q: adding document %q to builder: %s", step.name, d.Name, err)
878 }
879 }
880
881 // Call b.Finish() multiple times to ensure that it is idempotent
882 for i := 0; i < 3; i++ {
883
884 err = b.Finish()
885 if err != nil {
886 t.Fatalf("step %q: finishing builder (call #%d): %s", step.name, i, err)
887 }
888 }
889
890 err = b.Finish()
891 if err != nil {
892 t.Fatalf("step %q: finishing builder: %s", step.name, err)
893 }
894
895 state, _ := buildOpts.IndexState()
896 if diff := cmp.Diff(IndexStateEqual, state); diff != "" {
897 t.Errorf("unexpected diff in index state (-want +got):\n%s", diff)
898 }
899
900 ss, err := shards.NewDirectorySearcher(indexDir)
901 if err != nil {
902 t.Fatalf("step %q: NewDirectorySearcher(%s): %s", step.name, indexDir, err)
903 }
904 defer ss.Close()
905
906 searchOpts := &zoekt.SearchOptions{Whole: true}
907 q := &query.Substring{Pattern: step.query}
908
909 result, err := ss.Search(context.Background(), q, searchOpts)
910 if err != nil {
911 t.Fatalf("step %q: Search(%q): %s", step.name, step.query, err)
912 }
913
914 var receivedDocuments []zoekt.Document
915 for _, f := range result.Files {
916 receivedDocuments = append(receivedDocuments, zoekt.Document{
917 Name: f.FileName,
918 Content: f.Content,
919 })
920 }
921
922 cmpOpts := []cmp.Option{
923 cmpopts.IgnoreFields(zoekt.Document{}, "Branches"),
924 cmpopts.SortSlices(func(a, b zoekt.Document) bool {
925 if a.Name < b.Name {
926 return true
927 }
928
929 return bytes.Compare(a.Content, b.Content) < 0
930 }),
931 }
932
933 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, cmpOpts...); diff != "" {
934 t.Errorf("step %q: diff in received documents (-want +got):%s\n:", step.name, diff)
935 }
936 }
937 })
938 }
939}