fork of https://github.com/sourcegraph/zoekt
1package build
2
3import (
4 "errors"
5 "flag"
6 "io"
7 "log"
8 "os"
9 "path/filepath"
10 "strconv"
11 "strings"
12 "testing"
13 "time"
14
15 "github.com/google/go-cmp/cmp"
16 "github.com/google/go-cmp/cmp/cmpopts"
17
18 "github.com/sourcegraph/zoekt"
19)
20
21var update = flag.Bool("update", false, "update golden file")
22
23// ensure we don't regress on how we build v16
24func TestBuildv16(t *testing.T) {
25 dir := t.TempDir()
26
27 opts := Options{
28 IndexDir: dir,
29 RepositoryDescription: zoekt.Repository{
30 Name: "repo",
31 Source: "./testdata/repo/",
32 },
33 DisableCTags: true,
34 }
35 opts.SetDefaults()
36
37 b, err := NewBuilder(opts)
38 if err != nil {
39 t.Fatal(err)
40 }
41
42 for _, p := range []string{"main.go"} {
43 blob, err := os.ReadFile(filepath.Join("../testdata/repo", p))
44 if err != nil {
45 t.Fatal(err)
46 }
47 if err := b.AddFile(p, blob); err != nil {
48 t.Fatal(err)
49 }
50 }
51
52 wantP := filepath.Join("../testdata/shards", "repo_v16.00000.zoekt")
53
54 // fields indexTime and id depend on time. For this test, we copy the fields from
55 // the old shard.
56 _, wantMetadata, err := zoekt.ReadMetadataPath(wantP)
57 if err != nil {
58 t.Fatal(err)
59 }
60 b.indexTime = wantMetadata.IndexTime
61 b.id = wantMetadata.ID
62
63 if err := b.Finish(); err != nil {
64 t.Fatal(err)
65 }
66
67 gotP := filepath.Join(dir, "repo_v16.00000.zoekt")
68
69 if *update {
70 data, err := os.ReadFile(gotP)
71 if err != nil {
72 t.Fatal(err)
73 }
74 err = os.WriteFile(wantP, data, 0644)
75 if err != nil {
76 t.Fatal(err)
77 }
78 return
79 }
80
81 got, err := os.ReadFile(gotP)
82 if err != nil {
83 t.Fatal(err)
84 }
85 want, err := os.ReadFile(wantP)
86 if err != nil {
87 t.Fatal(err)
88 }
89
90 if d := cmp.Diff(want, got); d != "" {
91 t.Errorf("mismatch (-want +got):\n%s", d)
92 }
93}
94
95func TestFlags(t *testing.T) {
96 cases := []struct {
97 args []string
98 want Options
99 }{{
100 // Defaults
101 args: []string{},
102 want: Options{},
103 }, {
104 args: []string{"-index", "/tmp"},
105 want: Options{
106 IndexDir: "/tmp",
107 },
108 }, {
109 // single large file pattern
110 args: []string{"-large_file", "*.md"},
111 want: Options{
112 LargeFiles: []string{"*.md"},
113 },
114 }, {
115 // multiple large file pattern
116 args: []string{"-large_file", "*.md", "-large_file", "*.yaml"},
117 want: Options{
118 LargeFiles: []string{"*.md", "*.yaml"},
119 },
120 }, {
121 // multiple large file pattern with negated pattern
122 args: []string{"-large_file", "*.md", "-large_file", "!*.yaml"},
123 want: Options{
124 LargeFiles: []string{"*.md", "!*.yaml"},
125 },
126 }, {
127 // multiple large file pattern with escaped character
128 args: []string{"-large_file", "*.md", "-large_file", "\\!*.yaml"},
129 want: Options{
130 LargeFiles: []string{"*.md", "\\!*.yaml"},
131 },
132 }}
133
134 ignored := []cmp.Option{
135 // depends on $PATH setting.
136 cmpopts.IgnoreFields(Options{}, "CTagsPath"),
137 cmpopts.IgnoreFields(Options{}, "changedOrRemovedFiles"),
138 cmpopts.IgnoreFields(zoekt.Repository{}, "priority"),
139 }
140
141 for _, c := range cases {
142 c.want.SetDefaults()
143 // depends on $PATH setting.
144 c.want.CTagsPath = ""
145
146 got := Options{}
147 fs := flag.NewFlagSet("", flag.ContinueOnError)
148 got.Flags(fs)
149 if err := fs.Parse(c.args); err != nil {
150 t.Errorf("failed to parse args %v: %v", c.args, err)
151 } else if d := cmp.Diff(c.want, got, ignored...); d != "" {
152 t.Errorf("mismatch for %v (-want +got):\n%s", c.args, d)
153 }
154 }
155}
156
157func TestIncrementalSkipIndexing(t *testing.T) {
158 cases := []struct {
159 name string
160 want bool
161 opts Options
162 }{{
163 name: "v17-noop",
164 want: true,
165 opts: Options{
166 RepositoryDescription: zoekt.Repository{
167 Name: "repo17",
168 },
169 SizeMax: 2097152,
170 DisableCTags: true,
171 },
172 }, {
173 name: "v16-noop",
174 want: true,
175 opts: Options{
176 RepositoryDescription: zoekt.Repository{
177 Name: "repo",
178 },
179 SizeMax: 2097152,
180 DisableCTags: true,
181 },
182 }, {
183 name: "v17-id",
184 want: false,
185 opts: Options{
186 RepositoryDescription: zoekt.Repository{
187 Name: "repo17",
188 RawConfig: map[string]string{
189 "repoid": "123",
190 },
191 },
192 SizeMax: 2097152,
193 DisableCTags: true,
194 },
195 }, {
196 name: "doesnotexist",
197 want: false,
198 opts: Options{
199 RepositoryDescription: zoekt.Repository{
200 Name: "doesnotexist",
201 },
202 SizeMax: 2097152,
203 DisableCTags: true,
204 },
205 }}
206
207 for _, tc := range cases {
208 t.Run(tc.name, func(t *testing.T) {
209 tc.opts.IndexDir = "../testdata/shards"
210 t.Log(tc.opts.IndexState())
211 got := tc.opts.IncrementalSkipIndexing()
212 if got != tc.want {
213 t.Fatalf("want %v got %v", tc.want, got)
214 }
215 })
216 }
217}
218
219func TestMain(m *testing.M) {
220 flag.Parse()
221 if !testing.Verbose() {
222 log.SetOutput(io.Discard)
223 }
224 os.Exit(m.Run())
225}
226
227func TestDontCountContentOfSkippedFiles(t *testing.T) {
228 b, err := NewBuilder(Options{RepositoryDescription: zoekt.Repository{
229 Name: "foo",
230 }})
231 if err != nil {
232 t.Fatal(err)
233 }
234
235 // content with at least 100 bytes
236 binary := append([]byte("abc def \x00"), make([]byte, 100)...)
237 err = b.Add(zoekt.Document{
238 Name: "f1",
239 Content: binary,
240 })
241 if err != nil {
242 t.Fatal(err)
243 }
244 if len(b.todo) != 1 || b.todo[0].SkipReason == "" {
245 t.Fatalf("document should have been skipped")
246 }
247 if b.todo[0].Content != nil {
248 t.Fatalf("document content should be empty")
249 }
250 if b.size >= 100 {
251 t.Fatalf("content of skipped documents should not count towards shard size thresold")
252 }
253}
254
255func TestOptions_FindAllShards(t *testing.T) {
256 type simpleShard struct {
257 Repository zoekt.Repository
258 // NumShards is the number of shards that should be created that
259 // contain data for "Repository".
260 NumShards int
261 }
262
263 tests := []struct {
264 name string
265 simpleShards []simpleShard
266 compoundShards [][]zoekt.Repository
267 expectedShardCount int
268 expectedRepository zoekt.Repository
269 }{
270 {
271 name: "repository in normal shard",
272 simpleShards: []simpleShard{
273 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
274 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
275 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
276 },
277 expectedShardCount: 1,
278 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
279 },
280 {
281 name: "repository in compound shard",
282 compoundShards: [][]zoekt.Repository{
283 {
284 {Name: "repoA", ID: 1},
285 {Name: "repoB", ID: 2},
286 {Name: "repoC", ID: 3},
287 },
288 {
289 {Name: "repoD", ID: 4},
290 {Name: "repoE", ID: 5},
291 {Name: "repoF", ID: 6},
292 },
293 },
294 expectedShardCount: 1,
295 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
296 },
297 {
298 name: "repository split across multiple shards",
299 simpleShards: []simpleShard{
300 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
301 {Repository: zoekt.Repository{Name: "repoB", ID: 2}, NumShards: 2},
302 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
303 },
304 expectedShardCount: 2,
305 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
306 },
307 {
308 name: "unknown repository",
309 simpleShards: []simpleShard{
310 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
311 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
312 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
313 },
314 compoundShards: [][]zoekt.Repository{
315 {
316 {Name: "repoD", ID: 4},
317 {Name: "repoE", ID: 5},
318 {Name: "repoF", ID: 6},
319 },
320 },
321 expectedShardCount: 0,
322 },
323 {
324 name: "match on ID, not name (compound only)",
325 compoundShards: [][]zoekt.Repository{
326 {
327 {Name: "repoA", ID: 1},
328 {Name: "sameName", ID: 2},
329 {Name: "sameName", ID: 3},
330 },
331 {
332 {Name: "repoB", ID: 4},
333 {Name: "sameName", ID: 5},
334 {Name: "sameName", ID: 6},
335 },
336 },
337 expectedShardCount: 1,
338 expectedRepository: zoekt.Repository{Name: "sameName", ID: 5},
339 },
340 }
341 for _, tt := range tests {
342 t.Run(tt.name, func(t *testing.T) {
343 t.Parallel()
344
345 // prepare
346 indexDir := t.TempDir()
347
348 for _, s := range tt.simpleShards {
349 createTestShard(t, indexDir, s.Repository, s.NumShards)
350 }
351
352 for _, repositoryGroup := range tt.compoundShards {
353 createTestCompoundShard(t, indexDir, repositoryGroup)
354 }
355
356 o := &Options{
357 IndexDir: indexDir,
358 RepositoryDescription: tt.expectedRepository,
359 }
360 o.SetDefaults()
361
362 // run test
363 shards := o.FindAllShards()
364
365 // verify results
366 if len(shards) != tt.expectedShardCount {
367 t.Fatalf("expected %d shard(s), received %d shard(s)", tt.expectedShardCount, len(shards))
368 }
369
370 if tt.expectedShardCount > 0 {
371 for _, s := range shards {
372 // all shards should contain the metadata for the desired repository
373 repos, _, err := zoekt.ReadMetadataPathAlive(s)
374 if err != nil {
375 t.Fatalf("reading metadata from shard %q: %s", s, err)
376 }
377
378 foundRepository := false
379 for _, r := range repos {
380 if r.ID == tt.expectedRepository.ID {
381 foundRepository = true
382 break
383 }
384 }
385
386 if !foundRepository {
387 t.Errorf("shard %q doesn't contain metadata for repository %d", s, tt.expectedRepository.ID)
388 }
389 }
390 }
391 })
392 }
393}
394
395func TestBuilder_BranchNamesEqual(t *testing.T) {
396 for i, test := range []struct {
397 oldBranches []zoekt.RepositoryBranch
398 newBranches []zoekt.RepositoryBranch
399 expected bool
400 }{
401 {
402 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
403 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
404 expected: true,
405 },
406 {
407 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v3"}},
408 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v4"}},
409 expected: true,
410 },
411 {
412 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
413 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v1"}},
414 expected: false,
415 },
416 {
417 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
418 newBranches: []zoekt.RepositoryBranch{{Name: "release", Version: "v1"}},
419 expected: false,
420 },
421 {
422 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
423 newBranches: []zoekt.RepositoryBranch{},
424 expected: false,
425 },
426 {
427 oldBranches: []zoekt.RepositoryBranch{},
428 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
429 expected: false,
430 },
431 } {
432 t.Run(strconv.Itoa(i), func(t *testing.T) {
433 actual := BranchNamesEqual(test.oldBranches, test.newBranches)
434 if test.expected != actual {
435 t.Errorf("expected: %t, got: %t", test.expected, actual)
436 }
437 })
438 }
439}
440
441func TestBuilder_DeltaShardsBuildsShouldErrorOnBranchSet(t *testing.T) {
442 indexDir := t.TempDir()
443
444 repository := zoekt.Repository{
445 Name: "repo",
446 ID: 1,
447 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "bar"}},
448 }
449 createTestShard(t, indexDir, repository, 2)
450
451 repositoryNewBranches := zoekt.Repository{
452 Name: "repo",
453 ID: 1,
454 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "baz"}},
455 }
456
457 o := Options{
458 IndexDir: indexDir,
459 RepositoryDescription: repositoryNewBranches,
460 IsDelta: true,
461 }
462 o.SetDefaults()
463
464 b, err := NewBuilder(o)
465 if err != nil {
466 t.Fatalf("NewBuilder: %v", err)
467 }
468
469 err = b.Finish()
470 if !errors.As(err, &deltaBranchSetError{}) {
471 t.Fatalf("expected error complaning about different branch names, got: %s", err)
472 }
473}
474
475func TestBuilder_DeltaShardsBuildsShouldErrorOnIndexOptionsMismatch(t *testing.T) {
476 repository := zoekt.Repository{
477 Name: "repo",
478 ID: 1,
479 Branches: []zoekt.RepositoryBranch{{Name: "foo"}},
480 }
481
482 for _, test := range []struct {
483 name string
484 options func(options *Options)
485 }{
486 {
487 name: "update option CTagsPath to non default",
488 options: func(options *Options) { options.CTagsPath = "ctags_updated_test/universal-ctags" },
489 },
490 {
491 name: "update option DisableCTags to non default",
492 options: func(options *Options) { options.DisableCTags = true },
493 },
494 {
495 name: "update option SizeMax to non default",
496 options: func(options *Options) { options.SizeMax -= 10 },
497 },
498 {
499 name: "update option LargeFiles to non default",
500 options: func(options *Options) { options.LargeFiles = []string{"-large_file", "*.md", "-large_file", "*.yaml"} },
501 },
502 } {
503 test := test
504
505 t.Run(test.name, func(t *testing.T) {
506 indexDir := t.TempDir()
507
508 // initially use default options
509 createTestShard(t, indexDir, repository, 2)
510
511 o := Options{
512 IndexDir: indexDir,
513 RepositoryDescription: repository,
514 IsDelta: true,
515 }
516 test.options(&o)
517
518 b, err := NewBuilder(o)
519 if err != nil {
520 t.Fatalf("NewBuilder: %v", err)
521 }
522
523 err = b.Finish()
524 if err == nil {
525 t.Fatalf("no error regarding index options mismatch")
526 }
527
528 var optionsMismatchError *deltaIndexOptionsMismatchError
529 if !errors.As(err, &optionsMismatchError) {
530 t.Fatalf("expected error complaining about index options mismatch, got: %s", err)
531 }
532 })
533 }
534}
535
536func TestBuilder_DeltaShardsMetadataInOlderShards(t *testing.T) {
537 olderTime := time.Unix(0, 0)
538 newerTime := time.Unix(10000, 0)
539
540 for _, test := range []struct {
541 name string
542 originalRepository zoekt.Repository
543 updatedRepository zoekt.Repository
544 }{
545 {
546 name: "update commit information",
547 originalRepository: zoekt.Repository{
548 Name: "repo",
549 ID: 1,
550 Branches: []zoekt.RepositoryBranch{
551 {Name: "main", Version: "v1"},
552 {Name: "release", Version: "v1"},
553 },
554 },
555 updatedRepository: zoekt.Repository{
556 Name: "repo",
557 ID: 1,
558 Branches: []zoekt.RepositoryBranch{
559 {Name: "main", Version: "v2"},
560 {Name: "release", Version: "v2"},
561 },
562 },
563 },
564 {
565 name: "update latest commit date (older -> newer)",
566 originalRepository: zoekt.Repository{
567 Name: "repo",
568 ID: 1,
569 Branches: []zoekt.RepositoryBranch{
570 {Name: "main", Version: "v1"},
571 },
572 LatestCommitDate: olderTime,
573 },
574 updatedRepository: zoekt.Repository{
575 Name: "repo",
576 ID: 1,
577 Branches: []zoekt.RepositoryBranch{
578 {Name: "main", Version: "v2"},
579 },
580 LatestCommitDate: newerTime,
581 },
582 },
583 {
584 name: "update latest commit date (even if latest commit date is older - the most recent commits are the source of truth)",
585 originalRepository: zoekt.Repository{
586 Name: "repo",
587 ID: 1,
588 Branches: []zoekt.RepositoryBranch{
589 {Name: "main", Version: "v1"},
590 },
591 LatestCommitDate: newerTime,
592 },
593 updatedRepository: zoekt.Repository{
594 Name: "repo",
595 ID: 1,
596 Branches: []zoekt.RepositoryBranch{
597 {Name: "main", Version: "v2"},
598 },
599 LatestCommitDate: olderTime,
600 },
601 },
602 } {
603 test := test
604
605 t.Run(test.name, func(t *testing.T) {
606 indexDir := t.TempDir()
607
608 createTestShard(t, indexDir, test.originalRepository, 2, func(o *Options) {
609 o.DisableCTags = true
610 })
611
612 shards := createTestShard(t, indexDir, test.updatedRepository, 1, func(o *Options) {
613 o.IsDelta = true
614 o.DisableCTags = true
615 })
616
617 if len(shards) < 3 {
618 t.Fatalf("expected at least 3 shards, got %d (%s)", len(shards), strings.Join(shards, ", "))
619 }
620
621 for _, s := range shards {
622 repositories, _, err := zoekt.ReadMetadataPathAlive(s)
623 if err != nil {
624 t.Fatalf("reading repository metadata from shard %q", s)
625 }
626
627 var foundRepository *zoekt.Repository
628 for _, r := range repositories {
629 if r.ID == test.updatedRepository.ID {
630 foundRepository = r
631 break
632 }
633 }
634
635 if foundRepository == nil {
636 t.Fatalf("repository ID %d not in shard %q", test.updatedRepository.ID, s)
637 }
638
639 diffOptions := []cmp.Option{
640 cmpopts.IgnoreUnexported(zoekt.Repository{}),
641 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
642 cmpopts.EquateEmpty(),
643 }
644
645 if diff := cmp.Diff(&test.updatedRepository, foundRepository, diffOptions...); diff != "" {
646 t.Errorf("shard %q: unexpected diff in repository metadata (-want +got):\n%s", s, diff)
647 }
648 }
649 })
650 }
651}
652
653func TestFindRepositoryMetadata(t *testing.T) {
654 tests := []struct {
655 name string
656 normalShardRepositories []zoekt.Repository
657 compoundShardRepositories []zoekt.Repository
658 input *zoekt.Repository
659 expectedRepository *zoekt.Repository
660 expectedOk bool
661 }{
662 {
663 name: "repository in normal shards",
664 normalShardRepositories: []zoekt.Repository{
665 {Name: "repoA", ID: 1},
666 {Name: "repoB", ID: 2},
667 {Name: "repoC", ID: 3},
668 },
669 compoundShardRepositories: []zoekt.Repository{
670 {Name: "repoD", ID: 4},
671 {Name: "repoE", ID: 5},
672 {Name: "repoF", ID: 6},
673 },
674 input: &zoekt.Repository{Name: "repoB", ID: 2},
675 expectedRepository: &zoekt.Repository{Name: "repoB", ID: 2},
676 expectedOk: true,
677 },
678 {
679 name: "repository in compound shards",
680 normalShardRepositories: []zoekt.Repository{
681 {Name: "repoA", ID: 1},
682 {Name: "repoB", ID: 2},
683 {Name: "repoC", ID: 3},
684 },
685 compoundShardRepositories: []zoekt.Repository{
686 {Name: "repoD", ID: 4},
687 {Name: "repoE", ID: 5},
688 {Name: "repoF", ID: 6},
689 },
690 input: &zoekt.Repository{Name: "repoE", ID: 5},
691 expectedRepository: &zoekt.Repository{Name: "repoE", ID: 5},
692 expectedOk: true,
693 },
694 {
695 name: "repository not in any shard",
696 normalShardRepositories: []zoekt.Repository{
697 {Name: "repoA", ID: 1},
698 {Name: "repoB", ID: 2},
699 {Name: "repoC", ID: 3},
700 },
701 compoundShardRepositories: []zoekt.Repository{
702 {Name: "repoD", ID: 4},
703 {Name: "repoE", ID: 5},
704 {Name: "repoF", ID: 6},
705 },
706 input: &zoekt.Repository{Name: "notPresent", ID: 123},
707 expectedRepository: nil,
708 expectedOk: false,
709 },
710 }
711 for _, tt := range tests {
712 t.Run(tt.name, func(t *testing.T) {
713 // setup
714 indexDir := t.TempDir()
715
716 optFns := []func(o *Options){
717 // ctags aren't important for this test, and the equality checks
718 // for diffing repositories can break due to local configuration
719 func(o *Options) {
720 o.DisableCTags = true
721 },
722 }
723
724 for _, r := range tt.normalShardRepositories {
725 createTestShard(t, indexDir, r, 1, optFns...)
726 }
727
728 if len(tt.compoundShardRepositories) > 0 {
729 createTestCompoundShard(t, indexDir, tt.compoundShardRepositories, optFns...)
730 }
731
732 o := &Options{
733 IndexDir: indexDir,
734 RepositoryDescription: *tt.input,
735 }
736 o.SetDefaults()
737
738 // run test
739 got, _, gotOk, err := o.FindRepositoryMetadata()
740 if err != nil {
741 t.Errorf("received unexpected error: %v", err)
742 return
743 }
744
745 // check outcome
746 compareOptions := []cmp.Option{
747 cmpopts.IgnoreUnexported(zoekt.Repository{}),
748 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
749 cmpopts.EquateEmpty(),
750 }
751
752 if diff := cmp.Diff(tt.expectedRepository, got, compareOptions...); diff != "" {
753 t.Errorf("unexpected difference in repositories (-want +got):\n%s", diff)
754 }
755
756 if tt.expectedOk != gotOk {
757 t.Errorf("unexpected difference in 'ok' value: wanted %t, got %t", tt.expectedOk, gotOk)
758 }
759 })
760 }
761}
762
763func TestIsLowPriority(t *testing.T) {
764 cases := []string{
765 "builder_test.go",
766 "TestQuery.java",
767 "test/mocks.go",
768 "search/vendor/thirdparty.cc",
769 "search/node_modules/search/index.js",
770 "search.min.js",
771 "internal/search.js.map",
772 }
773
774 for _, tt := range cases {
775 t.Run(tt, func(t *testing.T) {
776 if !IsLowPriority(tt) {
777 t.Errorf("expected file '%s' to be low priority", tt)
778 }
779 })
780 }
781
782 negativeCases := []string{
783 "builder.go",
784 "RoutesTrigger.java",
785 "search.js",
786 }
787
788 for _, tt := range negativeCases {
789 t.Run(tt, func(t *testing.T) {
790 if IsLowPriority(tt) {
791 t.Errorf("did not expect file '%s' to be low priority", tt)
792 }
793 })
794 }
795}
796
797func createTestShard(t *testing.T, indexDir string, r zoekt.Repository, numShards int, optFns ...func(options *Options)) []string {
798 t.Helper()
799
800 if err := os.MkdirAll(filepath.Dir(indexDir), 0700); err != nil {
801 t.Fatal(err)
802 }
803
804 o := Options{
805 IndexDir: indexDir,
806 RepositoryDescription: r,
807 ShardMax: 75, // create a new shard every 75 bytes
808 }
809 o.SetDefaults()
810
811 for _, fn := range optFns {
812 fn(&o)
813 }
814
815 b, err := NewBuilder(o)
816 if err != nil {
817 t.Fatalf("NewBuilder: %v", err)
818 }
819
820 if numShards == 0 {
821 // We have to make at least 1 shard.
822 numShards = 1
823 }
824
825 for i := 0; i < numShards; i++ {
826 // Create entries (file + contents) that are ~100 bytes each.
827 // This (along with our shardMax setting of 75 bytes) means that each shard
828 // will contain at most one of these.
829 fileName := strconv.Itoa(i)
830 document := zoekt.Document{Name: fileName, Content: []byte(strings.Repeat("A", 100))}
831 for _, branch := range o.RepositoryDescription.Branches {
832 document.Branches = append(document.Branches, branch.Name)
833 }
834
835 err := b.Add(document)
836 if err != nil {
837 t.Fatalf("failed to add file %q to builder: %s", fileName, err)
838 }
839 }
840
841 if err := b.Finish(); err != nil {
842 t.Fatalf("Finish: %v", err)
843 }
844
845 return o.FindAllShards()
846}
847
848func createTestCompoundShard(t *testing.T, indexDir string, repositories []zoekt.Repository, optFns ...func(options *Options)) {
849 t.Helper()
850
851 var shardNames []string
852
853 for _, r := range repositories {
854 // create an isolated scratch space to store normal shards for this repository
855 scratchDir := t.TempDir()
856
857 // create shards that'll be merged later
858 createTestShard(t, scratchDir, r, 1, optFns...)
859
860 // discover file names for all the normal shards we created
861 // note: this only looks in the immediate 'scratchDir' folder and doesn't recurse
862 shards, err := filepath.Glob(filepath.Join(scratchDir, "*.zoekt"))
863 if err != nil {
864 t.Fatalf("while globbing %q to find normal shards: %s", scratchDir, err)
865 }
866
867 shardNames = append(shardNames, shards...)
868 }
869
870 // load the normal shards that we created
871 var files []zoekt.IndexFile
872 for _, shard := range shardNames {
873 f, err := os.Open(shard)
874 if err != nil {
875 t.Fatalf("opening shard file: %s", err)
876 }
877 defer f.Close()
878
879 indexFile, err := zoekt.NewIndexFile(f)
880 if err != nil {
881 t.Fatalf("creating index file: %s", err)
882 }
883 defer indexFile.Close()
884
885 files = append(files, indexFile)
886 }
887
888 // merge all the simple shards into a compound shard
889 tmpName, dstName, err := zoekt.Merge(indexDir, files...)
890 if err != nil {
891 t.Fatalf("merging index files into compound shard: %s", err)
892 }
893 if err := os.Rename(tmpName, dstName); err != nil {
894 t.Fatal(err)
895 }
896}
897
898func TestIgnoreSizeMax(t *testing.T) {
899
900 for _, test := range []struct {
901 name string
902 largeFiles []string
903 filePaths []string
904 expected bool
905 }{
906 {
907 name: "empty pattern does nothing",
908 largeFiles: []string{""},
909 filePaths: []string{"F0"},
910 expected: false,
911 },
912 {
913 name: "positive match allows",
914 largeFiles: []string{"F0"},
915 filePaths: []string{"F0"},
916 expected: true,
917 },
918 {
919 name: "positive and negative patterns allows",
920 largeFiles: []string{"F?", "!F0"},
921 filePaths: []string{"F1"},
922 expected: true,
923 },
924 {
925 name: "positive and negative patterns disallows",
926 largeFiles: []string{"F?", "!F0"},
927 filePaths: []string{"F0"},
928 expected: false,
929 },
930 {
931 name: "positive escaped pattern allows",
932 largeFiles: []string{"\\!F?"},
933 filePaths: []string{"!F0", "!F1"},
934 expected: true,
935 },
936 {
937 name: "postive escaped pattern does not disallow",
938 largeFiles: []string{"F0", "\\!F?"},
939 filePaths: []string{"F0", "!F0"},
940 expected: true,
941 },
942 {
943 name: "combined meta and literal interpretation disallows",
944 largeFiles: []string{"*F*", "!!F*"},
945 filePaths: []string{"!F0"},
946 expected: false,
947 },
948 {
949 name: "combined meta and literal interpretation allows",
950 largeFiles: []string{"*F*", "!!F*"},
951 filePaths: []string{"F0"},
952 expected: true,
953 },
954 {
955 name: "largeFiles order: positive match overrides previous negative match and allows",
956 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
957 filePaths: []string{"F0"},
958 expected: true,
959 },
960 {
961 name: "largeFiles order: positive match overrides previous negative match and disallows",
962 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
963 filePaths: []string{"F1"},
964 expected: false,
965 },
966 {
967 name: "largeFiles order: negative match overrides previous positive match and allows",
968 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
969 filePaths: []string{"F1"},
970 expected: true,
971 },
972 {
973 name: "largeFiles order: negative match overrides previous positive match and disallows",
974 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
975 filePaths: []string{"F0"},
976 expected: false,
977 },
978 } {
979 t.Run(test.name, func(t *testing.T) {
980 o := Options{
981 LargeFiles: test.largeFiles,
982 }
983
984 for _, filePath := range test.filePaths {
985 ignore := o.IgnoreSizeMax(filePath)
986 if ignore != test.expected {
987 t.Errorf("IgnoreSizeMax() for filepath %v returned unexpected result %v", filePath, ignore)
988 }
989 }
990 })
991 }
992}