fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package e2e
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "log"
23 "os"
24 "path/filepath"
25 "runtime"
26 "sort"
27 "strconv"
28 "strings"
29 "testing"
30 "time"
31
32 "github.com/google/go-cmp/cmp"
33 "github.com/google/go-cmp/cmp/cmpopts"
34 "github.com/grafana/regexp"
35 "github.com/sourcegraph/zoekt"
36 "github.com/sourcegraph/zoekt/index"
37 "github.com/sourcegraph/zoekt/internal/shards"
38 "github.com/sourcegraph/zoekt/internal/tenant"
39 "github.com/sourcegraph/zoekt/internal/tenant/tenanttest"
40 "github.com/sourcegraph/zoekt/query"
41 "github.com/stretchr/testify/require"
42)
43
44func TestBasicIndexing(t *testing.T) {
45 dir := t.TempDir()
46
47 opts := index.Options{
48 IndexDir: dir,
49 ShardMax: 1024,
50 RepositoryDescription: zoekt.Repository{
51 Name: "repo",
52 },
53 Parallelism: 2,
54 SizeMax: 1 << 20,
55 }
56
57 b, err := index.NewBuilder(opts)
58 if err != nil {
59 t.Fatalf("NewBuilder: %v", err)
60 }
61
62 for i := 0; i < 4; i++ {
63 s := fmt.Sprintf("%d", i)
64 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1000))); err != nil {
65 t.Fatal(err)
66 }
67 }
68
69 if err := b.Finish(); err != nil {
70 t.Errorf("Finish: %v", err)
71 }
72
73 fs, _ := filepath.Glob(dir + "/*.zoekt")
74 if len(fs) <= 1 {
75 t.Fatalf("want multiple shards, got %v", fs)
76 }
77
78 _, md0, err := index.ReadMetadataPath(fs[0])
79 if err != nil {
80 t.Fatal(err)
81 }
82 for _, f := range fs[1:] {
83 _, md, err := index.ReadMetadataPath(f)
84 if err != nil {
85 t.Fatal(err)
86 }
87 if md.IndexTime != md0.IndexTime {
88 t.Fatalf("wanted identical time stamps but got %v!=%v", md.IndexTime, md0.IndexTime)
89 }
90 if md.ID != md0.ID {
91 t.Fatalf("wanted identical IDs but got %s!=%s", md.ID, md0.ID)
92 }
93 }
94
95 ss, err := shards.NewDirectorySearcher(dir)
96 if err != nil {
97 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
98 }
99 defer ss.Close()
100
101 q, err := query.Parse("111")
102 if err != nil {
103 t.Fatalf("Parse(111): %v", err)
104 }
105
106 var sOpts zoekt.SearchOptions
107 ctx := context.Background()
108 result, err := ss.Search(ctx, q, &sOpts)
109 if err != nil {
110 t.Fatalf("Search(%v): %v", q, err)
111 }
112
113 if len(result.Files) != 1 {
114 t.Errorf("got %v, want 1 file.", result.Files)
115 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
116 t.Errorf("got file %q, want %q", gotFile, wantFile)
117 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo"; gotRepo != wantRepo {
118 t.Errorf("got repo %q, want %q", gotRepo, wantRepo)
119 }
120
121 t.Run("meta file", func(t *testing.T) {
122 // use retryTest to allow for the directory watcher to notice the meta
123 // file
124 retryTest(t, func(fatalf func(format string, args ...interface{})) {
125 // Add a .meta file for each shard with repo.Name set to
126 // "repo-mutated". We do this inside retry helper since we have noticed
127 // some flakiness on github CI.
128 for _, p := range fs {
129 repos, _, err := index.ReadMetadataPath(p)
130 if err != nil {
131 t.Fatal(err)
132 }
133 repos[0].Name = "repo-mutated"
134 b, err := json.Marshal(repos[0])
135 if err != nil {
136 t.Fatal(err)
137 }
138
139 if err := os.WriteFile(p+".meta", b, 0o600); err != nil {
140 t.Fatal(err)
141 }
142 }
143
144 result, err := ss.Search(ctx, q, &sOpts)
145 if err != nil {
146 fatalf("Search(%v): %v", q, err)
147 }
148
149 if len(result.Files) != 1 {
150 fatalf("got %v, want 1 file.", result.Files)
151 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
152 fatalf("got file %q, want %q", gotFile, wantFile)
153 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo-mutated"; gotRepo != wantRepo {
154 fatalf("got repo %q, want %q", gotRepo, wantRepo)
155 }
156 })
157 })
158}
159
160func TestSearchTenant(t *testing.T) {
161 tenanttest.MockEnforce(t)
162
163 dir := t.TempDir()
164
165 ctx1 := tenanttest.NewTestContext()
166 tnt1, err := tenant.FromContext(ctx1)
167 require.NoError(t, err)
168
169 opts := index.Options{
170 IndexDir: dir,
171 ShardMax: 1024,
172 RepositoryDescription: zoekt.Repository{
173 Name: "repo",
174 RawConfig: map[string]string{"tenantID": strconv.Itoa(tnt1.ID())},
175 },
176 Parallelism: 2,
177 SizeMax: 1 << 20,
178 }
179
180 b, err := index.NewBuilder(opts)
181 if err != nil {
182 t.Fatalf("NewBuilder: %v", err)
183 }
184
185 for i := 0; i < 4; i++ {
186 s := fmt.Sprintf("%d", i)
187 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1000))); err != nil {
188 t.Fatal(err)
189 }
190 }
191
192 if err := b.Finish(); err != nil {
193 t.Errorf("Finish: %v", err)
194 }
195
196 fs, _ := filepath.Glob(dir + "/*.zoekt")
197 if len(fs) <= 1 {
198 t.Fatalf("want multiple shards, got %v", fs)
199 }
200
201 _, md0, err := index.ReadMetadataPath(fs[0])
202 if err != nil {
203 t.Fatal(err)
204 }
205 for _, f := range fs[1:] {
206 _, md, err := index.ReadMetadataPath(f)
207 if err != nil {
208 t.Fatal(err)
209 }
210 if md.IndexTime != md0.IndexTime {
211 t.Fatalf("wanted identical time stamps but got %v!=%v", md.IndexTime, md0.IndexTime)
212 }
213 if md.ID != md0.ID {
214 t.Fatalf("wanted identical IDs but got %s!=%s", md.ID, md0.ID)
215 }
216 }
217
218 ss, err := shards.NewDirectorySearcher(dir)
219 if err != nil {
220 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
221 }
222 defer ss.Close()
223
224 q, err := query.Parse("111")
225 if err != nil {
226 t.Fatalf("Parse(111): %v", err)
227 }
228
229 var sOpts zoekt.SearchOptions
230
231 // Tenant 1 has access to the repo
232 result, err := ss.Search(ctx1, q, &sOpts)
233 require.NoError(t, err)
234 require.Len(t, result.Files, 1)
235
236 // Tenant 2 does not have access to the repo
237 ctx2 := tenanttest.NewTestContext()
238 result, err = ss.Search(ctx2, q, &sOpts)
239 require.NoError(t, err)
240 require.Len(t, result.Files, 0)
241}
242
243func TestListTenant(t *testing.T) {
244 tenanttest.MockEnforce(t)
245
246 dir := t.TempDir()
247
248 ctx1 := tenanttest.NewTestContext()
249 tnt1, err := tenant.FromContext(ctx1)
250 require.NoError(t, err)
251
252 opts := index.Options{
253 IndexDir: dir,
254 RepositoryDescription: zoekt.Repository{
255 Name: "repo",
256 RawConfig: map[string]string{"tenantID": strconv.Itoa(tnt1.ID())},
257 },
258 }
259 opts.SetDefaults()
260
261 b, err := index.NewBuilder(opts)
262 if err != nil {
263 t.Fatalf("NewBuilder: %v", err)
264 }
265 if err := b.Finish(); err != nil {
266 t.Errorf("Finish: %v", err)
267 }
268
269 fs, _ := filepath.Glob(dir + "/*.zoekt")
270 if len(fs) != 1 {
271 t.Fatalf("want a shard, got %v", fs)
272 }
273
274 ss, err := shards.NewDirectorySearcher(dir)
275 if err != nil {
276 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
277 }
278 defer ss.Close()
279
280 // Tenant 1 has access to the repo
281 result, err := ss.List(ctx1, &query.Const{Value: true}, nil)
282 require.NoError(t, err)
283 require.Len(t, result.Repos, 1)
284
285 // Tenant 2 does not have access to the repo
286 ctx2 := tenanttest.NewTestContext()
287 result, err = ss.List(ctx2, &query.Const{Value: true}, nil)
288 require.NoError(t, err)
289 require.Len(t, result.Repos, 0)
290}
291
292// retryTest will retry f until min(t.Deadline(), time.Minute). It returns
293// once f doesn't call fatalf.
294func retryTest(t *testing.T, f func(fatalf func(format string, args ...interface{}))) {
295 t.Helper()
296
297 sleep := 10 * time.Millisecond
298 deadline := time.Now().Add(time.Minute)
299 if d, ok := t.Deadline(); ok && d.Before(deadline) {
300 // give 1s for us to do a final test run
301 deadline = d.Add(-time.Second)
302 }
303
304 for {
305 done := make(chan bool)
306 go func() {
307 defer close(done)
308
309 f(func(format string, args ...interface{}) {
310 runtime.Goexit()
311 })
312
313 done <- true
314 }()
315
316 success := <-done
317 if success {
318 return
319 }
320
321 // each time we increase sleep by 1.5
322 sleep := sleep*2 - sleep/2
323 if time.Now().Add(sleep).After(deadline) {
324 break
325 }
326 time.Sleep(sleep)
327 }
328
329 // final run for the test, using the real t.Fatalf
330 f(t.Fatalf)
331}
332
333func TestLargeFileOption(t *testing.T) {
334 dir := t.TempDir()
335
336 sizeMax := 1000
337 opts := index.Options{
338 IndexDir: dir,
339 LargeFiles: []string{"F0", "F1", "F2", "!F1"},
340 RepositoryDescription: zoekt.Repository{
341 Name: "repo",
342 },
343 SizeMax: sizeMax,
344 }
345
346 b, err := index.NewBuilder(opts)
347 if err != nil {
348 t.Fatalf("NewBuilder: %v", err)
349 }
350
351 for i := 0; i < 4; i++ {
352 s := fmt.Sprintf("%d", i)
353 if err := b.AddFile("F"+s, []byte(strings.Repeat("a", sizeMax+1))); err != nil {
354 t.Fatal(err)
355 }
356 }
357
358 if err := b.Finish(); err != nil {
359 t.Errorf("Finish: %v", err)
360 }
361
362 ss, err := shards.NewDirectorySearcher(dir)
363 if err != nil {
364 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
365 }
366
367 q, err := query.Parse("aaa")
368 if err != nil {
369 t.Fatalf("Parse(aaa): %v", err)
370 }
371
372 var sOpts zoekt.SearchOptions
373 ctx := context.Background()
374 result, err := ss.Search(ctx, q, &sOpts)
375 if err != nil {
376 t.Fatalf("Search(%v): %v", q, err)
377 }
378
379 if len(result.Files) != 2 {
380 t.Errorf("got %v files, want 2 files.", len(result.Files))
381 }
382 defer ss.Close()
383}
384
385func TestUpdate(t *testing.T) {
386 dir := t.TempDir()
387
388 opts := index.Options{
389 IndexDir: dir,
390 ShardMax: 1024,
391 RepositoryDescription: zoekt.Repository{
392 Name: "repo",
393 FileURLTemplate: "url",
394 },
395 Parallelism: 2,
396 SizeMax: 1 << 20,
397 }
398
399 if b, err := index.NewBuilder(opts); err != nil {
400 t.Fatalf("NewBuilder: %v", err)
401 } else {
402 if err := b.AddFile("F", []byte("hoi")); err != nil {
403 t.Errorf("AddFile: %v", err)
404 }
405 if err := b.Finish(); err != nil {
406 t.Errorf("Finish: %v", err)
407 }
408 }
409 ss, err := shards.NewDirectorySearcher(dir)
410 if err != nil {
411 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
412 }
413
414 ctx := context.Background()
415 repos, err := ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
416 if err != nil {
417 t.Fatalf("List: %v", err)
418 }
419
420 if len(repos.Repos) != 1 {
421 t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
422 }
423
424 fs, err := filepath.Glob(filepath.Join(dir, "*"))
425 if err != nil {
426 t.Fatalf("glob: %v", err)
427 }
428
429 opts.RepositoryDescription = zoekt.Repository{
430 Name: "repo2",
431 FileURLTemplate: "url2",
432 }
433
434 if b, err := index.NewBuilder(opts); err != nil {
435 t.Fatalf("NewBuilder: %v", err)
436 } else {
437 if err := b.AddFile("F", []byte("hoi")); err != nil {
438 t.Errorf("AddFile: %v", err)
439 }
440 if err := b.Finish(); err != nil {
441 t.Errorf("Finish: %v", err)
442 }
443 }
444
445 // This is ugly, and potentially flaky, but there is no
446 // observable synchronization for the Sharded searcher, so
447 // this is the best we can do.
448 time.Sleep(100 * time.Millisecond)
449
450 ctx = context.Background()
451 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
452 t.Fatalf("List: %v", err)
453 } else if len(repos.Repos) != 2 {
454 t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
455 }
456
457 for _, fn := range fs {
458 log.Printf("removing %s", fn)
459 if err := os.Remove(fn); err != nil {
460 t.Fatalf("Remove(%s): %v", fn, err)
461 }
462 }
463
464 time.Sleep(100 * time.Millisecond)
465
466 ctx = context.Background()
467 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
468 t.Fatalf("List: %v", err)
469 } else if len(repos.Repos) != 1 {
470 var ss []string
471 for _, r := range repos.Repos {
472 ss = append(ss, r.Repository.Name)
473 }
474 t.Errorf("List(repo): got %v, want 1 repo", ss)
475 }
476}
477
478func TestDeleteOldShards(t *testing.T) {
479 dir := t.TempDir()
480
481 opts := index.Options{
482 IndexDir: dir,
483 ShardMax: 1024,
484 RepositoryDescription: zoekt.Repository{
485 Name: "repo",
486 FileURLTemplate: "url",
487 },
488 SizeMax: 1 << 20,
489 }
490 opts.SetDefaults()
491
492 b, err := index.NewBuilder(opts)
493 if err != nil {
494 t.Fatalf("NewBuilder: %v", err)
495 }
496 for i := 0; i < 4; i++ {
497 s := fmt.Sprintf("%d\n", i)
498 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
499 t.Errorf("AddFile: %v", err)
500 }
501 }
502 if err := b.Finish(); err != nil {
503 t.Errorf("Finish: %v", err)
504 }
505
506 glob := filepath.Join(dir, "*.zoekt")
507 fs, err := filepath.Glob(glob)
508 if err != nil {
509 t.Fatalf("Glob(%s): %v", glob, err)
510 } else if len(fs) != 4 {
511 t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
512 }
513
514 if fi, err := os.Lstat(fs[0]); err != nil {
515 t.Fatalf("Lstat: %v", err)
516 } else if fi.Mode()&0o666 == 0o600 {
517 // This fails spuriously if your umask is very restrictive.
518 t.Errorf("got mode %o, should respect umask.", fi.Mode())
519 }
520
521 // Do again, without sharding.
522 opts.ShardMax = 1 << 20
523 b, err = index.NewBuilder(opts)
524 if err != nil {
525 t.Fatalf("NewBuilder: %v", err)
526 }
527 for i := 0; i < 4; i++ {
528 s := fmt.Sprintf("%d\n", i)
529 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
530 t.Fatal(err)
531 }
532 }
533 if err := b.Finish(); err != nil {
534 t.Errorf("Finish: %v", err)
535 }
536
537 fs, err = filepath.Glob(glob)
538 if err != nil {
539 t.Fatalf("Glob(%s): %v", glob, err)
540 } else if len(fs) != 1 {
541 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
542 }
543
544 // Again, but don't index anything; should leave old shards intact.
545 b, err = index.NewBuilder(opts)
546 if err != nil {
547 t.Fatalf("NewBuilder: %v", err)
548 }
549 if err := b.Finish(); err != nil {
550 t.Errorf("Finish: %v", err)
551 }
552
553 fs, err = filepath.Glob(glob)
554 if err != nil {
555 t.Fatalf("Glob(%s): %v", glob, err)
556 } else if len(fs) != 1 {
557 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
558 }
559}
560
561func TestEmptyContent(t *testing.T) {
562 dir := t.TempDir()
563
564 opts := index.Options{
565 IndexDir: dir,
566 RepositoryDescription: zoekt.Repository{
567 Name: "repo",
568 },
569 }
570 opts.SetDefaults()
571
572 b, err := index.NewBuilder(opts)
573 if err != nil {
574 t.Fatalf("NewBuilder: %v", err)
575 }
576 if err := b.Finish(); err != nil {
577 t.Errorf("Finish: %v", err)
578 }
579
580 fs, _ := filepath.Glob(dir + "/*.zoekt")
581 if len(fs) != 1 {
582 t.Fatalf("want a shard, got %v", fs)
583 }
584
585 ss, err := shards.NewDirectorySearcher(dir)
586 if err != nil {
587 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
588 }
589 defer ss.Close()
590
591 ctx := context.Background()
592 result, err := ss.List(ctx, &query.Const{Value: true}, nil)
593 if err != nil {
594 t.Fatalf("List: %v", err)
595 }
596
597 if len(result.Repos) != 1 || result.Repos[0].Repository.Name != "repo" {
598 t.Errorf("got %+v, want 1 repo.", result.Repos)
599 }
600}
601
602func TestDeltaShards(t *testing.T) {
603 // TODO: Need to write a test for compound shards as well.
604 type step struct {
605 name string
606 documents []index.Document
607 optFn func(t *testing.T, o *index.Options)
608
609 query string
610 changedFile string
611 expectedDocuments []index.Document
612 }
613
614 var (
615 fooAtMain = index.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v1")}
616 fooAtMainV2 = index.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v2")}
617
618 fooAtMainAndRelease = index.Document{Name: "foo.go", Branches: []string{"main", "release"}, Content: []byte("common foo-main-and-release")}
619
620 barAtMain = index.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main")}
621 barAtMainV2 = index.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main-v2")}
622 )
623
624 for _, test := range []struct {
625 name string
626 steps []step
627 }{
628 {
629 name: "tombstone older documents",
630 steps: []step{
631 {
632 name: "setup",
633 documents: []index.Document{barAtMain, fooAtMain},
634 query: "common",
635 expectedDocuments: []index.Document{barAtMain, fooAtMain},
636 },
637 {
638 name: "add new version of foo, tombstone older ones",
639 documents: []index.Document{fooAtMainV2},
640 optFn: func(t *testing.T, o *index.Options) {
641 o.IsDelta = true
642 },
643 query: "common",
644 changedFile: "foo.go",
645 expectedDocuments: []index.Document{barAtMain, fooAtMainV2},
646 },
647 {
648 name: "add new version of bar, tombstone older ones",
649 documents: []index.Document{barAtMainV2},
650 optFn: func(t *testing.T, o *index.Options) {
651 o.IsDelta = true
652 },
653 query: "common",
654 changedFile: "bar.go",
655 expectedDocuments: []index.Document{barAtMainV2, fooAtMainV2},
656 },
657 }},
658 {
659 name: "tombstone older documents even if the latest shard has no documents",
660 steps: []step{
661 {
662 name: "setup",
663 documents: []index.Document{barAtMain, fooAtMain},
664 query: "common",
665 expectedDocuments: []index.Document{barAtMain, fooAtMain},
666 },
667 {
668 // a build with no documents could represent a deletion
669 name: "tombstone older documents",
670 documents: nil,
671 optFn: func(t *testing.T, o *index.Options) {
672 o.IsDelta = true
673 },
674 query: "common",
675 changedFile: "foo.go",
676 expectedDocuments: []index.Document{barAtMain},
677 },
678 },
679 },
680 {
681 name: "tombstones affect document across branches",
682 steps: []step{
683 {
684 name: "setup",
685 documents: []index.Document{barAtMain, fooAtMainAndRelease},
686 query: "common",
687 expectedDocuments: []index.Document{barAtMain, fooAtMainAndRelease},
688 },
689 {
690 name: "tombstone foo",
691 documents: nil,
692 optFn: func(t *testing.T, o *index.Options) {
693 o.IsDelta = true
694 },
695 query: "common",
696 changedFile: "foo.go",
697 expectedDocuments: []index.Document{barAtMain},
698 },
699 },
700 },
701 } {
702 t.Run(test.name, func(t *testing.T) {
703 indexDir := t.TempDir()
704
705 branchSet := make(map[string]struct{})
706
707 for _, s := range test.steps {
708 for _, d := range s.documents {
709 for _, b := range d.Branches {
710 branchSet[b] = struct{}{}
711 }
712 }
713 }
714
715 for _, step := range test.steps {
716 repository := zoekt.Repository{ID: 1, Name: "repository"}
717
718 for b := range branchSet {
719 repository.Branches = append(repository.Branches, zoekt.RepositoryBranch{Name: b})
720 }
721
722 sort.Slice(repository.Branches, func(i, j int) bool {
723 a, b := repository.Branches[i], repository.Branches[j]
724
725 return a.Name < b.Name
726 })
727
728 buildOpts := index.Options{
729 IndexDir: indexDir,
730 RepositoryDescription: repository,
731 }
732 buildOpts.SetDefaults()
733
734 if step.optFn != nil {
735 step.optFn(t, &buildOpts)
736 }
737
738 b, err := index.NewBuilder(buildOpts)
739 b.MarkFileAsChangedOrRemoved(step.changedFile)
740 if err != nil {
741 t.Fatalf("step %q: NewBuilder: %s", step.name, err)
742 }
743
744 for _, d := range step.documents {
745 err := b.Add(d)
746 if err != nil {
747 t.Fatalf("step %q: adding document %q to builder: %s", step.name, d.Name, err)
748 }
749 }
750
751 // Call b.Finish() multiple times to ensure that it is idempotent
752 for i := 0; i < 3; i++ {
753
754 err = b.Finish()
755 if err != nil {
756 t.Fatalf("step %q: finishing builder (call #%d): %s", step.name, i, err)
757 }
758 }
759
760 err = b.Finish()
761 if err != nil {
762 t.Fatalf("step %q: finishing builder: %s", step.name, err)
763 }
764
765 state, _ := buildOpts.IndexState()
766 if diff := cmp.Diff(index.IndexStateEqual, state); diff != "" {
767 t.Errorf("unexpected diff in index state (-want +got):\n%s", diff)
768 }
769
770 ss, err := shards.NewDirectorySearcher(indexDir)
771 if err != nil {
772 t.Fatalf("step %q: NewDirectorySearcher(%s): %s", step.name, indexDir, err)
773 }
774 defer ss.Close()
775
776 searchOpts := &zoekt.SearchOptions{Whole: true}
777 q := &query.Substring{Pattern: step.query}
778
779 result, err := ss.Search(context.Background(), q, searchOpts)
780 if err != nil {
781 t.Fatalf("step %q: Search(%q): %s", step.name, step.query, err)
782 }
783
784 var receivedDocuments []index.Document
785 for _, f := range result.Files {
786 receivedDocuments = append(receivedDocuments, index.Document{
787 Name: f.FileName,
788 Content: f.Content,
789 })
790 }
791
792 cmpOpts := []cmp.Option{
793 cmpopts.IgnoreFields(index.Document{}, "Branches"),
794 cmpopts.SortSlices(func(a, b index.Document) bool {
795 if a.Name < b.Name {
796 return true
797 }
798
799 return bytes.Compare(a.Content, b.Content) < 0
800 }),
801 }
802
803 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, cmpOpts...); diff != "" {
804 t.Errorf("step %q: diff in received documents (-want +got):%s\n:", step.name, diff)
805 }
806 }
807 })
808 }
809}