fork of https://github.com/sourcegraph/zoekt
1package build
2
3import (
4 "errors"
5 "flag"
6 "io"
7 "log"
8 "os"
9 "path/filepath"
10 "strconv"
11 "strings"
12 "testing"
13 "time"
14
15 "github.com/google/go-cmp/cmp"
16 "github.com/google/go-cmp/cmp/cmpopts"
17
18 "github.com/sourcegraph/zoekt"
19)
20
21var update = flag.Bool("update", false, "update golden file")
22
23// ensure we don't regress on how we build v16
24func TestBuildv16(t *testing.T) {
25 dir := t.TempDir()
26
27 opts := Options{
28 IndexDir: dir,
29 RepositoryDescription: zoekt.Repository{
30 Name: "repo",
31 Source: "./testdata/repo/",
32 },
33 DisableCTags: true,
34 }
35 opts.SetDefaults()
36
37 b, err := NewBuilder(opts)
38 if err != nil {
39 t.Fatal(err)
40 }
41
42 for _, p := range []string{"main.go"} {
43 blob, err := os.ReadFile(filepath.Join("../testdata/repo", p))
44 if err != nil {
45 t.Fatal(err)
46 }
47 if err := b.AddFile(p, blob); err != nil {
48 t.Fatal(err)
49 }
50 }
51
52 wantP := filepath.Join("../testdata/shards", "repo_v16.00000.zoekt")
53
54 // fields indexTime and id depend on time. For this test, we copy the fields from
55 // the old shard.
56 _, wantMetadata, err := zoekt.ReadMetadataPath(wantP)
57 if err != nil {
58 t.Fatal(err)
59 }
60 b.indexTime = wantMetadata.IndexTime
61 b.id = wantMetadata.ID
62
63 if err := b.Finish(); err != nil {
64 t.Fatal(err)
65 }
66
67 gotP := filepath.Join(dir, "repo_v16.00000.zoekt")
68
69 if *update {
70 data, err := os.ReadFile(gotP)
71 if err != nil {
72 t.Fatal(err)
73 }
74 err = os.WriteFile(wantP, data, 0o644)
75 if err != nil {
76 t.Fatal(err)
77 }
78 return
79 }
80
81 got, err := os.ReadFile(gotP)
82 if err != nil {
83 t.Fatal(err)
84 }
85 want, err := os.ReadFile(wantP)
86 if err != nil {
87 t.Fatal(err)
88 }
89
90 if d := cmp.Diff(want, got); d != "" {
91 t.Errorf("mismatch (-want +got):\n%s", d)
92 }
93}
94
95func TestFlags(t *testing.T) {
96 cases := []struct {
97 args []string
98 want Options
99 }{{
100 // Defaults
101 args: []string{},
102 want: Options{},
103 }, {
104 args: []string{"-index", "/tmp"},
105 want: Options{
106 IndexDir: "/tmp",
107 },
108 }, {
109 // single large file pattern
110 args: []string{"-large_file", "*.md"},
111 want: Options{
112 LargeFiles: []string{"*.md"},
113 },
114 }, {
115 // multiple large file pattern
116 args: []string{"-large_file", "*.md", "-large_file", "*.yaml"},
117 want: Options{
118 LargeFiles: []string{"*.md", "*.yaml"},
119 },
120 }, {
121 // multiple large file pattern with negated pattern
122 args: []string{"-large_file", "*.md", "-large_file", "!*.yaml"},
123 want: Options{
124 LargeFiles: []string{"*.md", "!*.yaml"},
125 },
126 }, {
127 // multiple large file pattern with escaped character
128 args: []string{"-large_file", "*.md", "-large_file", "\\!*.yaml"},
129 want: Options{
130 LargeFiles: []string{"*.md", "\\!*.yaml"},
131 },
132 }}
133
134 ignored := []cmp.Option{
135 // depends on $PATH setting.
136 cmpopts.IgnoreFields(Options{}, "CTagsPath"),
137 cmpopts.IgnoreFields(Options{}, "ScipCTagsPath"),
138 cmpopts.IgnoreFields(Options{}, "changedOrRemovedFiles"),
139 cmpopts.IgnoreFields(zoekt.Repository{}, "priority"),
140 }
141
142 for _, c := range cases {
143 c.want.SetDefaults()
144 // depends on $PATH setting.
145 c.want.CTagsPath = ""
146
147 got := Options{}
148 fs := flag.NewFlagSet("", flag.ContinueOnError)
149 got.Flags(fs)
150 if err := fs.Parse(c.args); err != nil {
151 t.Errorf("failed to parse args %v: %v", c.args, err)
152 } else if d := cmp.Diff(c.want, got, ignored...); d != "" {
153 t.Errorf("mismatch for %v (-want +got):\n%s", c.args, d)
154 }
155 }
156}
157
158func TestIncrementalSkipIndexing(t *testing.T) {
159 cases := []struct {
160 name string
161 want bool
162 opts Options
163 }{{
164 name: "v17-noop",
165 want: true,
166 opts: Options{
167 RepositoryDescription: zoekt.Repository{
168 Name: "repo17",
169 },
170 SizeMax: 2097152,
171 DisableCTags: true,
172 },
173 }, {
174 name: "v16-noop",
175 want: true,
176 opts: Options{
177 RepositoryDescription: zoekt.Repository{
178 Name: "repo",
179 },
180 SizeMax: 2097152,
181 DisableCTags: true,
182 },
183 }, {
184 name: "v17-id",
185 want: false,
186 opts: Options{
187 RepositoryDescription: zoekt.Repository{
188 Name: "repo17",
189 RawConfig: map[string]string{
190 "repoid": "123",
191 },
192 },
193 SizeMax: 2097152,
194 DisableCTags: true,
195 },
196 }, {
197 name: "doesnotexist",
198 want: false,
199 opts: Options{
200 RepositoryDescription: zoekt.Repository{
201 Name: "doesnotexist",
202 },
203 SizeMax: 2097152,
204 DisableCTags: true,
205 },
206 }}
207
208 for _, tc := range cases {
209 t.Run(tc.name, func(t *testing.T) {
210 tc.opts.IndexDir = "../testdata/shards"
211 t.Log(tc.opts.IndexState())
212 got := tc.opts.IncrementalSkipIndexing()
213 if got != tc.want {
214 t.Fatalf("want %v got %v", tc.want, got)
215 }
216 })
217 }
218}
219
220func TestMain(m *testing.M) {
221 flag.Parse()
222 if !testing.Verbose() {
223 log.SetOutput(io.Discard)
224 }
225 os.Exit(m.Run())
226}
227
228func TestDontCountContentOfSkippedFiles(t *testing.T) {
229 b, err := NewBuilder(Options{RepositoryDescription: zoekt.Repository{
230 Name: "foo",
231 }})
232 if err != nil {
233 t.Fatal(err)
234 }
235
236 // content with at least 100 bytes
237 binary := append([]byte("abc def \x00"), make([]byte, 100)...)
238 err = b.Add(zoekt.Document{
239 Name: "f1",
240 Content: binary,
241 })
242 if err != nil {
243 t.Fatal(err)
244 }
245 if len(b.todo) != 1 || b.todo[0].SkipReason == "" {
246 t.Fatalf("document should have been skipped")
247 }
248 if b.todo[0].Content != nil {
249 t.Fatalf("document content should be empty")
250 }
251 if b.size >= 100 {
252 t.Fatalf("content of skipped documents should not count towards shard size thresold")
253 }
254}
255
256func TestOptions_FindAllShards(t *testing.T) {
257 type simpleShard struct {
258 Repository zoekt.Repository
259 // NumShards is the number of shards that should be created that
260 // contain data for "Repository".
261 NumShards int
262 }
263
264 tests := []struct {
265 name string
266 simpleShards []simpleShard
267 compoundShards [][]zoekt.Repository
268 expectedShardCount int
269 expectedRepository zoekt.Repository
270 }{
271 {
272 name: "repository in normal shard",
273 simpleShards: []simpleShard{
274 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
275 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
276 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
277 },
278 expectedShardCount: 1,
279 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
280 },
281 {
282 name: "repository in compound shard",
283 compoundShards: [][]zoekt.Repository{
284 {
285 {Name: "repoA", ID: 1},
286 {Name: "repoB", ID: 2},
287 {Name: "repoC", ID: 3},
288 },
289 {
290 {Name: "repoD", ID: 4},
291 {Name: "repoE", ID: 5},
292 {Name: "repoF", ID: 6},
293 },
294 },
295 expectedShardCount: 1,
296 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
297 },
298 {
299 name: "repository split across multiple shards",
300 simpleShards: []simpleShard{
301 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
302 {Repository: zoekt.Repository{Name: "repoB", ID: 2}, NumShards: 2},
303 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
304 },
305 expectedShardCount: 2,
306 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
307 },
308 {
309 name: "unknown repository",
310 simpleShards: []simpleShard{
311 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
312 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
313 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
314 },
315 compoundShards: [][]zoekt.Repository{
316 {
317 {Name: "repoD", ID: 4},
318 {Name: "repoE", ID: 5},
319 {Name: "repoF", ID: 6},
320 },
321 },
322 expectedShardCount: 0,
323 },
324 {
325 name: "match on ID, not name (compound only)",
326 compoundShards: [][]zoekt.Repository{
327 {
328 {Name: "repoA", ID: 1},
329 {Name: "sameName", ID: 2},
330 {Name: "sameName", ID: 3},
331 },
332 {
333 {Name: "repoB", ID: 4},
334 {Name: "sameName", ID: 5},
335 {Name: "sameName", ID: 6},
336 },
337 },
338 expectedShardCount: 1,
339 expectedRepository: zoekt.Repository{Name: "sameName", ID: 5},
340 },
341 }
342 for _, tt := range tests {
343 t.Run(tt.name, func(t *testing.T) {
344 t.Parallel()
345
346 // prepare
347 indexDir := t.TempDir()
348
349 for _, s := range tt.simpleShards {
350 createTestShard(t, indexDir, s.Repository, s.NumShards)
351 }
352
353 for _, repositoryGroup := range tt.compoundShards {
354 createTestCompoundShard(t, indexDir, repositoryGroup)
355 }
356
357 o := &Options{
358 IndexDir: indexDir,
359 RepositoryDescription: tt.expectedRepository,
360 }
361 o.SetDefaults()
362
363 // run test
364 shards := o.FindAllShards()
365
366 // verify results
367 if len(shards) != tt.expectedShardCount {
368 t.Fatalf("expected %d shard(s), received %d shard(s)", tt.expectedShardCount, len(shards))
369 }
370
371 if tt.expectedShardCount > 0 {
372 for _, s := range shards {
373 // all shards should contain the metadata for the desired repository
374 repos, _, err := zoekt.ReadMetadataPathAlive(s)
375 if err != nil {
376 t.Fatalf("reading metadata from shard %q: %s", s, err)
377 }
378
379 foundRepository := false
380 for _, r := range repos {
381 if r.ID == tt.expectedRepository.ID {
382 foundRepository = true
383 break
384 }
385 }
386
387 if !foundRepository {
388 t.Errorf("shard %q doesn't contain metadata for repository %d", s, tt.expectedRepository.ID)
389 }
390 }
391 }
392 })
393 }
394}
395
396func TestBuilder_BranchNamesEqual(t *testing.T) {
397 for i, test := range []struct {
398 oldBranches []zoekt.RepositoryBranch
399 newBranches []zoekt.RepositoryBranch
400 expected bool
401 }{
402 {
403 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
404 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
405 expected: true,
406 },
407 {
408 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v3"}},
409 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v4"}},
410 expected: true,
411 },
412 {
413 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
414 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v1"}},
415 expected: false,
416 },
417 {
418 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
419 newBranches: []zoekt.RepositoryBranch{{Name: "release", Version: "v1"}},
420 expected: false,
421 },
422 {
423 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
424 newBranches: []zoekt.RepositoryBranch{},
425 expected: false,
426 },
427 {
428 oldBranches: []zoekt.RepositoryBranch{},
429 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
430 expected: false,
431 },
432 } {
433 t.Run(strconv.Itoa(i), func(t *testing.T) {
434 actual := BranchNamesEqual(test.oldBranches, test.newBranches)
435 if test.expected != actual {
436 t.Errorf("expected: %t, got: %t", test.expected, actual)
437 }
438 })
439 }
440}
441
442func TestBuilder_DeltaShardsBuildsShouldErrorOnBranchSet(t *testing.T) {
443 indexDir := t.TempDir()
444
445 repository := zoekt.Repository{
446 Name: "repo",
447 ID: 1,
448 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "bar"}},
449 }
450 createTestShard(t, indexDir, repository, 2)
451
452 repositoryNewBranches := zoekt.Repository{
453 Name: "repo",
454 ID: 1,
455 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "baz"}},
456 }
457
458 o := Options{
459 IndexDir: indexDir,
460 RepositoryDescription: repositoryNewBranches,
461 IsDelta: true,
462 }
463 o.SetDefaults()
464
465 b, err := NewBuilder(o)
466 if err != nil {
467 t.Fatalf("NewBuilder: %v", err)
468 }
469
470 err = b.Finish()
471 if !errors.As(err, &deltaBranchSetError{}) {
472 t.Fatalf("expected error complaning about different branch names, got: %s", err)
473 }
474}
475
476func TestBuilder_DeltaShardsBuildsShouldErrorOnIndexOptionsMismatch(t *testing.T) {
477 repository := zoekt.Repository{
478 Name: "repo",
479 ID: 1,
480 Branches: []zoekt.RepositoryBranch{{Name: "foo"}},
481 }
482
483 for _, test := range []struct {
484 name string
485 options func(options *Options)
486 }{
487 {
488 name: "update option CTagsPath to non default",
489 options: func(options *Options) { options.CTagsPath = "ctags_updated_test/universal-ctags" },
490 },
491 {
492 name: "update option DisableCTags to non default",
493 options: func(options *Options) { options.DisableCTags = true },
494 },
495 {
496 name: "update option SizeMax to non default",
497 options: func(options *Options) { options.SizeMax -= 10 },
498 },
499 {
500 name: "update option LargeFiles to non default",
501 options: func(options *Options) { options.LargeFiles = []string{"-large_file", "*.md", "-large_file", "*.yaml"} },
502 },
503 } {
504 test := test
505
506 t.Run(test.name, func(t *testing.T) {
507 indexDir := t.TempDir()
508
509 // initially use default options
510 createTestShard(t, indexDir, repository, 2)
511
512 o := Options{
513 IndexDir: indexDir,
514 RepositoryDescription: repository,
515 IsDelta: true,
516 }
517 test.options(&o)
518
519 b, err := NewBuilder(o)
520 if err != nil {
521 t.Fatalf("NewBuilder: %v", err)
522 }
523
524 err = b.Finish()
525 if err == nil {
526 t.Fatalf("no error regarding index options mismatch")
527 }
528
529 var optionsMismatchError *deltaIndexOptionsMismatchError
530 if !errors.As(err, &optionsMismatchError) {
531 t.Fatalf("expected error complaining about index options mismatch, got: %s", err)
532 }
533 })
534 }
535}
536
537func TestBuilder_DeltaShardsMetadataInOlderShards(t *testing.T) {
538 olderTime := time.Unix(0, 0)
539 newerTime := time.Unix(10000, 0)
540
541 for _, test := range []struct {
542 name string
543 originalRepository zoekt.Repository
544 updatedRepository zoekt.Repository
545 }{
546 {
547 name: "update commit information",
548 originalRepository: zoekt.Repository{
549 Name: "repo",
550 ID: 1,
551 Branches: []zoekt.RepositoryBranch{
552 {Name: "main", Version: "v1"},
553 {Name: "release", Version: "v1"},
554 },
555 },
556 updatedRepository: zoekt.Repository{
557 Name: "repo",
558 ID: 1,
559 Branches: []zoekt.RepositoryBranch{
560 {Name: "main", Version: "v2"},
561 {Name: "release", Version: "v2"},
562 },
563 },
564 },
565 {
566 name: "update latest commit date (older -> newer)",
567 originalRepository: zoekt.Repository{
568 Name: "repo",
569 ID: 1,
570 Branches: []zoekt.RepositoryBranch{
571 {Name: "main", Version: "v1"},
572 },
573 LatestCommitDate: olderTime,
574 },
575 updatedRepository: zoekt.Repository{
576 Name: "repo",
577 ID: 1,
578 Branches: []zoekt.RepositoryBranch{
579 {Name: "main", Version: "v2"},
580 },
581 LatestCommitDate: newerTime,
582 },
583 },
584 {
585 name: "update latest commit date (even if latest commit date is older - the most recent commits are the source of truth)",
586 originalRepository: zoekt.Repository{
587 Name: "repo",
588 ID: 1,
589 Branches: []zoekt.RepositoryBranch{
590 {Name: "main", Version: "v1"},
591 },
592 LatestCommitDate: newerTime,
593 },
594 updatedRepository: zoekt.Repository{
595 Name: "repo",
596 ID: 1,
597 Branches: []zoekt.RepositoryBranch{
598 {Name: "main", Version: "v2"},
599 },
600 LatestCommitDate: olderTime,
601 },
602 },
603 } {
604 test := test
605
606 t.Run(test.name, func(t *testing.T) {
607 indexDir := t.TempDir()
608
609 createTestShard(t, indexDir, test.originalRepository, 2, func(o *Options) {
610 o.DisableCTags = true
611 })
612
613 shards := createTestShard(t, indexDir, test.updatedRepository, 1, func(o *Options) {
614 o.IsDelta = true
615 o.DisableCTags = true
616 })
617
618 if len(shards) < 3 {
619 t.Fatalf("expected at least 3 shards, got %d (%s)", len(shards), strings.Join(shards, ", "))
620 }
621
622 for _, s := range shards {
623 repositories, _, err := zoekt.ReadMetadataPathAlive(s)
624 if err != nil {
625 t.Fatalf("reading repository metadata from shard %q", s)
626 }
627
628 var foundRepository *zoekt.Repository
629 for _, r := range repositories {
630 if r.ID == test.updatedRepository.ID {
631 foundRepository = r
632 break
633 }
634 }
635
636 if foundRepository == nil {
637 t.Fatalf("repository ID %d not in shard %q", test.updatedRepository.ID, s)
638 }
639
640 diffOptions := []cmp.Option{
641 cmpopts.IgnoreUnexported(zoekt.Repository{}),
642 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
643 cmpopts.EquateEmpty(),
644 }
645
646 if diff := cmp.Diff(&test.updatedRepository, foundRepository, diffOptions...); diff != "" {
647 t.Errorf("shard %q: unexpected diff in repository metadata (-want +got):\n%s", s, diff)
648 }
649 }
650 })
651 }
652}
653
654func TestFindRepositoryMetadata(t *testing.T) {
655 tests := []struct {
656 name string
657 normalShardRepositories []zoekt.Repository
658 compoundShardRepositories []zoekt.Repository
659 input *zoekt.Repository
660 expectedRepository *zoekt.Repository
661 expectedOk bool
662 }{
663 {
664 name: "repository in normal shards",
665 normalShardRepositories: []zoekt.Repository{
666 {Name: "repoA", ID: 1},
667 {Name: "repoB", ID: 2},
668 {Name: "repoC", ID: 3},
669 },
670 compoundShardRepositories: []zoekt.Repository{
671 {Name: "repoD", ID: 4},
672 {Name: "repoE", ID: 5},
673 {Name: "repoF", ID: 6},
674 },
675 input: &zoekt.Repository{Name: "repoB", ID: 2},
676 expectedRepository: &zoekt.Repository{Name: "repoB", ID: 2},
677 expectedOk: true,
678 },
679 {
680 name: "repository in compound shards",
681 normalShardRepositories: []zoekt.Repository{
682 {Name: "repoA", ID: 1},
683 {Name: "repoB", ID: 2},
684 {Name: "repoC", ID: 3},
685 },
686 compoundShardRepositories: []zoekt.Repository{
687 {Name: "repoD", ID: 4},
688 {Name: "repoE", ID: 5},
689 {Name: "repoF", ID: 6},
690 },
691 input: &zoekt.Repository{Name: "repoE", ID: 5},
692 expectedRepository: &zoekt.Repository{Name: "repoE", ID: 5},
693 expectedOk: true,
694 },
695 {
696 name: "repository not in any shard",
697 normalShardRepositories: []zoekt.Repository{
698 {Name: "repoA", ID: 1},
699 {Name: "repoB", ID: 2},
700 {Name: "repoC", ID: 3},
701 },
702 compoundShardRepositories: []zoekt.Repository{
703 {Name: "repoD", ID: 4},
704 {Name: "repoE", ID: 5},
705 {Name: "repoF", ID: 6},
706 },
707 input: &zoekt.Repository{Name: "notPresent", ID: 123},
708 expectedRepository: nil,
709 expectedOk: false,
710 },
711 }
712 for _, tt := range tests {
713 t.Run(tt.name, func(t *testing.T) {
714 // setup
715 indexDir := t.TempDir()
716
717 optFns := []func(o *Options){
718 // ctags aren't important for this test, and the equality checks
719 // for diffing repositories can break due to local configuration
720 func(o *Options) {
721 o.DisableCTags = true
722 },
723 }
724
725 for _, r := range tt.normalShardRepositories {
726 createTestShard(t, indexDir, r, 1, optFns...)
727 }
728
729 if len(tt.compoundShardRepositories) > 0 {
730 createTestCompoundShard(t, indexDir, tt.compoundShardRepositories, optFns...)
731 }
732
733 o := &Options{
734 IndexDir: indexDir,
735 RepositoryDescription: *tt.input,
736 }
737 o.SetDefaults()
738
739 // run test
740 got, _, gotOk, err := o.FindRepositoryMetadata()
741 if err != nil {
742 t.Errorf("received unexpected error: %v", err)
743 return
744 }
745
746 // check outcome
747 compareOptions := []cmp.Option{
748 cmpopts.IgnoreUnexported(zoekt.Repository{}),
749 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
750 cmpopts.EquateEmpty(),
751 }
752
753 if diff := cmp.Diff(tt.expectedRepository, got, compareOptions...); diff != "" {
754 t.Errorf("unexpected difference in repositories (-want +got):\n%s", diff)
755 }
756
757 if tt.expectedOk != gotOk {
758 t.Errorf("unexpected difference in 'ok' value: wanted %t, got %t", tt.expectedOk, gotOk)
759 }
760 })
761 }
762}
763
764func TestIsLowPriority(t *testing.T) {
765 cases := []string{
766 "builder_test.go",
767 "test/TestQuery.java",
768 "search/vendor/thirdparty.cc",
769 "search/node_modules/search/index.js",
770 "search.min.js",
771 "internal/search.js.map",
772 }
773
774 for _, tt := range cases {
775 t.Run(tt, func(t *testing.T) {
776 if !IsLowPriority(tt, nil) {
777 t.Errorf("expected file '%s' to be low priority", tt)
778 }
779 })
780 }
781
782 negativeCases := []string{
783 "builder.go",
784 "RoutesTrigger.java",
785 "search.js",
786 }
787
788 for _, tt := range negativeCases {
789 t.Run(tt, func(t *testing.T) {
790 if IsLowPriority(tt, nil) {
791 t.Errorf("did not expect file '%s' to be low priority", tt)
792 }
793 })
794 }
795
796 // Explicitly check that content is important by using the same filename but
797 // different content.
798 normal := "package mock\n\nvar Mock struct {}"
799 generated := "// Code generated by mock\npackage mock\n\nvar Mock struct {}"
800 if IsLowPriority("mock.go", []byte(normal)) {
801 t.Error("expected non-generated content to not be low priority")
802 }
803 if !IsLowPriority("mock.go", []byte(generated)) {
804 t.Error("expected generated content to be low priority")
805 }
806}
807
808func createTestShard(t *testing.T, indexDir string, r zoekt.Repository, numShards int, optFns ...func(options *Options)) []string {
809 t.Helper()
810
811 if err := os.MkdirAll(filepath.Dir(indexDir), 0o700); err != nil {
812 t.Fatal(err)
813 }
814
815 o := Options{
816 IndexDir: indexDir,
817 RepositoryDescription: r,
818 ShardMax: 75, // create a new shard every 75 bytes
819 }
820 o.SetDefaults()
821
822 for _, fn := range optFns {
823 fn(&o)
824 }
825
826 b, err := NewBuilder(o)
827 if err != nil {
828 t.Fatalf("NewBuilder: %v", err)
829 }
830
831 if numShards == 0 {
832 // We have to make at least 1 shard.
833 numShards = 1
834 }
835
836 for i := 0; i < numShards; i++ {
837 // Create entries (file + contents) that are ~100 bytes each.
838 // This (along with our shardMax setting of 75 bytes) means that each shard
839 // will contain at most one of these.
840 fileName := strconv.Itoa(i)
841 document := zoekt.Document{Name: fileName, Content: []byte(strings.Repeat("A", 100))}
842 for _, branch := range o.RepositoryDescription.Branches {
843 document.Branches = append(document.Branches, branch.Name)
844 }
845
846 err := b.Add(document)
847 if err != nil {
848 t.Fatalf("failed to add file %q to builder: %s", fileName, err)
849 }
850 }
851
852 if err := b.Finish(); err != nil {
853 t.Fatalf("Finish: %v", err)
854 }
855
856 return o.FindAllShards()
857}
858
859func createTestCompoundShard(t *testing.T, indexDir string, repositories []zoekt.Repository, optFns ...func(options *Options)) {
860 t.Helper()
861
862 var shardNames []string
863
864 for _, r := range repositories {
865 // create an isolated scratch space to store normal shards for this repository
866 scratchDir := t.TempDir()
867
868 // create shards that'll be merged later
869 createTestShard(t, scratchDir, r, 1, optFns...)
870
871 // discover file names for all the normal shards we created
872 // note: this only looks in the immediate 'scratchDir' folder and doesn't recurse
873 shards, err := filepath.Glob(filepath.Join(scratchDir, "*.zoekt"))
874 if err != nil {
875 t.Fatalf("while globbing %q to find normal shards: %s", scratchDir, err)
876 }
877
878 shardNames = append(shardNames, shards...)
879 }
880
881 // load the normal shards that we created
882 var files []zoekt.IndexFile
883 for _, shard := range shardNames {
884 f, err := os.Open(shard)
885 if err != nil {
886 t.Fatalf("opening shard file: %s", err)
887 }
888 defer f.Close()
889
890 indexFile, err := zoekt.NewIndexFile(f)
891 if err != nil {
892 t.Fatalf("creating index file: %s", err)
893 }
894 defer indexFile.Close()
895
896 files = append(files, indexFile)
897 }
898
899 // merge all the simple shards into a compound shard
900 tmpName, dstName, err := zoekt.Merge(indexDir, files...)
901 if err != nil {
902 t.Fatalf("merging index files into compound shard: %s", err)
903 }
904 if err := os.Rename(tmpName, dstName); err != nil {
905 t.Fatal(err)
906 }
907}
908
909func TestIgnoreSizeMax(t *testing.T) {
910 for _, test := range []struct {
911 name string
912 largeFiles []string
913 filePaths []string
914 expected bool
915 }{
916 {
917 name: "empty pattern does nothing",
918 largeFiles: []string{""},
919 filePaths: []string{"F0"},
920 expected: false,
921 },
922 {
923 name: "positive match allows",
924 largeFiles: []string{"F0"},
925 filePaths: []string{"F0"},
926 expected: true,
927 },
928 {
929 name: "positive and negative patterns allows",
930 largeFiles: []string{"F?", "!F0"},
931 filePaths: []string{"F1"},
932 expected: true,
933 },
934 {
935 name: "positive and negative patterns disallows",
936 largeFiles: []string{"F?", "!F0"},
937 filePaths: []string{"F0"},
938 expected: false,
939 },
940 {
941 name: "positive escaped pattern allows",
942 largeFiles: []string{"\\!F?"},
943 filePaths: []string{"!F0", "!F1"},
944 expected: true,
945 },
946 {
947 name: "postive escaped pattern does not disallow",
948 largeFiles: []string{"F0", "\\!F?"},
949 filePaths: []string{"F0", "!F0"},
950 expected: true,
951 },
952 {
953 name: "combined meta and literal interpretation disallows",
954 largeFiles: []string{"*F*", "!!F*"},
955 filePaths: []string{"!F0"},
956 expected: false,
957 },
958 {
959 name: "combined meta and literal interpretation allows",
960 largeFiles: []string{"*F*", "!!F*"},
961 filePaths: []string{"F0"},
962 expected: true,
963 },
964 {
965 name: "largeFiles order: positive match overrides previous negative match and allows",
966 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
967 filePaths: []string{"F0"},
968 expected: true,
969 },
970 {
971 name: "largeFiles order: positive match overrides previous negative match and disallows",
972 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
973 filePaths: []string{"F1"},
974 expected: false,
975 },
976 {
977 name: "largeFiles order: negative match overrides previous positive match and allows",
978 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
979 filePaths: []string{"F1"},
980 expected: true,
981 },
982 {
983 name: "largeFiles order: negative match overrides previous positive match and disallows",
984 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
985 filePaths: []string{"F0"},
986 expected: false,
987 },
988 } {
989 t.Run(test.name, func(t *testing.T) {
990 o := Options{
991 LargeFiles: test.largeFiles,
992 }
993
994 for _, filePath := range test.filePaths {
995 ignore := o.IgnoreSizeMax(filePath)
996 if ignore != test.expected {
997 t.Errorf("IgnoreSizeMax() for filepath %v returned unexpected result %v", filePath, ignore)
998 }
999 }
1000 })
1001 }
1002}