fork of https://github.com/sourcegraph/zoekt
1package build
2
3import (
4 "errors"
5 "flag"
6 "io"
7 "log"
8 "os"
9 "path/filepath"
10 "strconv"
11 "strings"
12 "testing"
13 "time"
14
15 "github.com/google/go-cmp/cmp"
16 "github.com/google/go-cmp/cmp/cmpopts"
17
18 "github.com/sourcegraph/zoekt"
19)
20
21var update = flag.Bool("update", false, "update golden file")
22
23// ensure we don't regress on how we build v16
24func TestBuildv16(t *testing.T) {
25 dir := t.TempDir()
26
27 opts := Options{
28 IndexDir: dir,
29 RepositoryDescription: zoekt.Repository{
30 Name: "repo",
31 Source: "./testdata/repo/",
32 },
33 DisableCTags: true,
34 }
35 opts.SetDefaults()
36
37 b, err := NewBuilder(opts)
38 if err != nil {
39 t.Fatal(err)
40 }
41
42 for _, p := range []string{"main.go"} {
43 blob, err := os.ReadFile(filepath.Join("../testdata/repo", p))
44 if err != nil {
45 t.Fatal(err)
46 }
47 if err := b.AddFile(p, blob); err != nil {
48 t.Fatal(err)
49 }
50 }
51
52 wantP := filepath.Join("../testdata/shards", "repo_v16.00000.zoekt")
53
54 // fields indexTime and id depend on time. For this test, we copy the fields from
55 // the old shard.
56 _, wantMetadata, err := zoekt.ReadMetadataPath(wantP)
57 if err != nil {
58 t.Fatal(err)
59 }
60 b.indexTime = wantMetadata.IndexTime
61 b.id = wantMetadata.ID
62
63 if err := b.Finish(); err != nil {
64 t.Fatal(err)
65 }
66
67 gotP := filepath.Join(dir, "repo_v16.00000.zoekt")
68
69 if *update {
70 data, err := os.ReadFile(gotP)
71 if err != nil {
72 t.Fatal(err)
73 }
74 err = os.WriteFile(wantP, data, 0644)
75 if err != nil {
76 t.Fatal(err)
77 }
78 return
79 }
80
81 got, err := os.ReadFile(gotP)
82 if err != nil {
83 t.Fatal(err)
84 }
85 want, err := os.ReadFile(wantP)
86 if err != nil {
87 t.Fatal(err)
88 }
89
90 if d := cmp.Diff(want, got); d != "" {
91 t.Errorf("mismatch (-want +got):\n%s", d)
92 }
93}
94
95func TestFlags(t *testing.T) {
96 cases := []struct {
97 args []string
98 want Options
99 }{{
100 // Defaults
101 args: []string{},
102 want: Options{},
103 }, {
104 args: []string{"-index", "/tmp"},
105 want: Options{
106 IndexDir: "/tmp",
107 },
108 }, {
109 // single large file pattern
110 args: []string{"-large_file", "*.md"},
111 want: Options{
112 LargeFiles: []string{"*.md"},
113 },
114 }, {
115 // multiple large file pattern
116 args: []string{"-large_file", "*.md", "-large_file", "*.yaml"},
117 want: Options{
118 LargeFiles: []string{"*.md", "*.yaml"},
119 },
120 }, {
121 // multiple large file pattern with negated pattern
122 args: []string{"-large_file", "*.md", "-large_file", "!*.yaml"},
123 want: Options{
124 LargeFiles: []string{"*.md", "!*.yaml"},
125 },
126 }, {
127 // multiple large file pattern with escaped character
128 args: []string{"-large_file", "*.md", "-large_file", "\\!*.yaml"},
129 want: Options{
130 LargeFiles: []string{"*.md", "\\!*.yaml"},
131 },
132 }}
133
134 ignored := []cmp.Option{
135 // depends on $PATH setting.
136 cmpopts.IgnoreFields(Options{}, "CTagsPath"),
137 cmpopts.IgnoreFields(Options{}, "changedOrRemovedFiles"),
138 cmpopts.IgnoreFields(zoekt.Repository{}, "priority"),
139 }
140
141 for _, c := range cases {
142 c.want.SetDefaults()
143 // depends on $PATH setting.
144 c.want.CTagsPath = ""
145
146 got := Options{}
147 fs := flag.NewFlagSet("", flag.ContinueOnError)
148 got.Flags(fs)
149 if err := fs.Parse(c.args); err != nil {
150 t.Errorf("failed to parse args %v: %v", c.args, err)
151 } else if d := cmp.Diff(c.want, got, ignored...); d != "" {
152 t.Errorf("mismatch for %v (-want +got):\n%s", c.args, d)
153 }
154 }
155}
156
157func TestIncrementalSkipIndexing(t *testing.T) {
158 cases := []struct {
159 name string
160 want bool
161 opts Options
162 }{{
163 name: "v17-noop",
164 want: true,
165 opts: Options{
166 RepositoryDescription: zoekt.Repository{
167 Name: "repo17",
168 },
169 SizeMax: 2097152,
170 DisableCTags: true,
171 },
172 }, {
173 name: "v16-noop",
174 want: true,
175 opts: Options{
176 RepositoryDescription: zoekt.Repository{
177 Name: "repo",
178 },
179 SizeMax: 2097152,
180 DisableCTags: true,
181 },
182 }, {
183 name: "v17-id",
184 want: false,
185 opts: Options{
186 RepositoryDescription: zoekt.Repository{
187 Name: "repo17",
188 RawConfig: map[string]string{
189 "repoid": "123",
190 },
191 },
192 SizeMax: 2097152,
193 DisableCTags: true,
194 },
195 }, {
196 name: "doesnotexist",
197 want: false,
198 opts: Options{
199 RepositoryDescription: zoekt.Repository{
200 Name: "doesnotexist",
201 },
202 SizeMax: 2097152,
203 DisableCTags: true,
204 },
205 }}
206
207 for _, tc := range cases {
208 t.Run(tc.name, func(t *testing.T) {
209 tc.opts.IndexDir = "../testdata/shards"
210 t.Log(tc.opts.IndexState())
211 got := tc.opts.IncrementalSkipIndexing()
212 if got != tc.want {
213 t.Fatalf("want %v got %v", tc.want, got)
214 }
215 })
216 }
217}
218
219func TestMain(m *testing.M) {
220 flag.Parse()
221 if !testing.Verbose() {
222 log.SetOutput(io.Discard)
223 }
224 os.Exit(m.Run())
225}
226
227func TestDontCountContentOfSkippedFiles(t *testing.T) {
228 b, err := NewBuilder(Options{RepositoryDescription: zoekt.Repository{
229 Name: "foo",
230 }})
231 if err != nil {
232 t.Fatal(err)
233 }
234
235 // content with at least 100 bytes
236 binary := append([]byte("abc def \x00"), make([]byte, 100)...)
237 err = b.Add(zoekt.Document{
238 Name: "f1",
239 Content: binary,
240 })
241 if err != nil {
242 t.Fatal(err)
243 }
244 if len(b.todo) != 1 || b.todo[0].SkipReason == "" {
245 t.Fatalf("document should have been skipped")
246 }
247 if b.size >= 100 {
248 t.Fatalf("content of skipped documents should not count towards shard size thresold")
249 }
250}
251
252func TestOptions_FindAllShards(t *testing.T) {
253 type simpleShard struct {
254 Repository zoekt.Repository
255 // NumShards is the number of shards that should be created that
256 // contain data for "Repository".
257 NumShards int
258 }
259
260 tests := []struct {
261 name string
262 simpleShards []simpleShard
263 compoundShards [][]zoekt.Repository
264 expectedShardCount int
265 expectedRepository zoekt.Repository
266 }{
267 {
268 name: "repository in normal shard",
269 simpleShards: []simpleShard{
270 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
271 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
272 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
273 },
274 expectedShardCount: 1,
275 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
276 },
277 {
278 name: "repository in compound shard",
279 compoundShards: [][]zoekt.Repository{
280 {
281 {Name: "repoA", ID: 1},
282 {Name: "repoB", ID: 2},
283 {Name: "repoC", ID: 3},
284 },
285 {
286 {Name: "repoD", ID: 4},
287 {Name: "repoE", ID: 5},
288 {Name: "repoF", ID: 6},
289 },
290 },
291 expectedShardCount: 1,
292 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
293 },
294 {
295 name: "repository split across multiple shards",
296 simpleShards: []simpleShard{
297 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
298 {Repository: zoekt.Repository{Name: "repoB", ID: 2}, NumShards: 2},
299 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
300 },
301 expectedShardCount: 2,
302 expectedRepository: zoekt.Repository{Name: "repoB", ID: 2},
303 },
304 {
305 name: "unknown repository",
306 simpleShards: []simpleShard{
307 {Repository: zoekt.Repository{Name: "repoA", ID: 1}},
308 {Repository: zoekt.Repository{Name: "repoB", ID: 2}},
309 {Repository: zoekt.Repository{Name: "repoC", ID: 3}},
310 },
311 compoundShards: [][]zoekt.Repository{
312 {
313 {Name: "repoD", ID: 4},
314 {Name: "repoE", ID: 5},
315 {Name: "repoF", ID: 6},
316 },
317 },
318 expectedShardCount: 0,
319 },
320 {
321 name: "match on ID, not name (compound only)",
322 compoundShards: [][]zoekt.Repository{
323 {
324 {Name: "repoA", ID: 1},
325 {Name: "sameName", ID: 2},
326 {Name: "sameName", ID: 3},
327 },
328 {
329 {Name: "repoB", ID: 4},
330 {Name: "sameName", ID: 5},
331 {Name: "sameName", ID: 6},
332 },
333 },
334 expectedShardCount: 1,
335 expectedRepository: zoekt.Repository{Name: "sameName", ID: 5},
336 },
337 }
338 for _, tt := range tests {
339 t.Run(tt.name, func(t *testing.T) {
340 t.Parallel()
341
342 // prepare
343 indexDir := t.TempDir()
344
345 for _, s := range tt.simpleShards {
346 createTestShard(t, indexDir, s.Repository, s.NumShards)
347 }
348
349 for _, repositoryGroup := range tt.compoundShards {
350 createTestCompoundShard(t, indexDir, repositoryGroup)
351 }
352
353 o := &Options{
354 IndexDir: indexDir,
355 RepositoryDescription: tt.expectedRepository,
356 }
357 o.SetDefaults()
358
359 // run test
360 shards := o.FindAllShards()
361
362 // verify results
363 if len(shards) != tt.expectedShardCount {
364 t.Fatalf("expected %d shard(s), received %d shard(s)", tt.expectedShardCount, len(shards))
365 }
366
367 if tt.expectedShardCount > 0 {
368 for _, s := range shards {
369 // all shards should contain the metadata for the desired repository
370 repos, _, err := zoekt.ReadMetadataPathAlive(s)
371 if err != nil {
372 t.Fatalf("reading metadata from shard %q: %s", s, err)
373 }
374
375 foundRepository := false
376 for _, r := range repos {
377 if r.ID == tt.expectedRepository.ID {
378 foundRepository = true
379 break
380 }
381 }
382
383 if !foundRepository {
384 t.Errorf("shard %q doesn't contain metadata for repository %d", s, tt.expectedRepository.ID)
385 }
386 }
387 }
388 })
389 }
390}
391
392func TestBuilder_BranchNamesEqual(t *testing.T) {
393 for i, test := range []struct {
394 oldBranches []zoekt.RepositoryBranch
395 newBranches []zoekt.RepositoryBranch
396 expected bool
397 }{
398 {
399 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
400 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v1"}},
401 expected: true,
402 },
403 {
404 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}, {Name: "release", Version: "v3"}},
405 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v4"}},
406 expected: true,
407 },
408 {
409 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
410 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v2"}, {Name: "release", Version: "v1"}},
411 expected: false,
412 },
413 {
414 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
415 newBranches: []zoekt.RepositoryBranch{{Name: "release", Version: "v1"}},
416 expected: false,
417 },
418 {
419 oldBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
420 newBranches: []zoekt.RepositoryBranch{},
421 expected: false,
422 },
423 {
424 oldBranches: []zoekt.RepositoryBranch{},
425 newBranches: []zoekt.RepositoryBranch{{Name: "main", Version: "v1"}},
426 expected: false,
427 },
428 } {
429 t.Run(strconv.Itoa(i), func(t *testing.T) {
430 actual := BranchNamesEqual(test.oldBranches, test.newBranches)
431 if test.expected != actual {
432 t.Errorf("expected: %t, got: %t", test.expected, actual)
433 }
434 })
435 }
436}
437
438func TestBuilder_DeltaShardsBuildsShouldErrorOnBranchSet(t *testing.T) {
439 indexDir := t.TempDir()
440
441 repository := zoekt.Repository{
442 Name: "repo",
443 ID: 1,
444 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "bar"}},
445 }
446 createTestShard(t, indexDir, repository, 2)
447
448 repositoryNewBranches := zoekt.Repository{
449 Name: "repo",
450 ID: 1,
451 Branches: []zoekt.RepositoryBranch{{Name: "foo"}, {Name: "baz"}},
452 }
453
454 o := Options{
455 IndexDir: indexDir,
456 RepositoryDescription: repositoryNewBranches,
457 IsDelta: true,
458 }
459 o.SetDefaults()
460
461 b, err := NewBuilder(o)
462 if err != nil {
463 t.Fatalf("NewBuilder: %v", err)
464 }
465
466 err = b.Finish()
467 if !errors.As(err, &deltaBranchSetError{}) {
468 t.Fatalf("expected error complaning about different branch names, got: %s", err)
469 }
470}
471
472func TestBuilder_DeltaShardsBuildsShouldErrorOnIndexOptionsMismatch(t *testing.T) {
473 repository := zoekt.Repository{
474 Name: "repo",
475 ID: 1,
476 Branches: []zoekt.RepositoryBranch{{Name: "foo"}},
477 }
478
479 for _, test := range []struct {
480 name string
481 options func(options *Options)
482 }{
483 {
484 name: "update option CTagsPath to non default",
485 options: func(options *Options) { options.CTagsPath = "ctags_updated_test" },
486 },
487 {
488 name: "update option DisableCTags to non default",
489 options: func(options *Options) { options.DisableCTags = true },
490 },
491 {
492 name: "update option SizeMax to non default",
493 options: func(options *Options) { options.SizeMax -= 10 },
494 },
495 {
496 name: "update option LargeFiles to non default",
497 options: func(options *Options) { options.LargeFiles = []string{"-large_file", "*.md", "-large_file", "*.yaml"} },
498 },
499 } {
500 test := test
501
502 t.Run(test.name, func(t *testing.T) {
503 indexDir := t.TempDir()
504
505 // initially use default options
506 createTestShard(t, indexDir, repository, 2)
507
508 o := Options{
509 IndexDir: indexDir,
510 RepositoryDescription: repository,
511 IsDelta: true,
512 }
513 test.options(&o)
514
515 b, err := NewBuilder(o)
516 if err != nil {
517 t.Fatalf("NewBuilder: %v", err)
518 }
519
520 err = b.Finish()
521 if err == nil {
522 t.Fatalf("no error regarding index options mismatch")
523 }
524
525 var optionsMismatchError *deltaIndexOptionsMismatchError
526 if !errors.As(err, &optionsMismatchError) {
527 t.Fatalf("expected error complaining about index options mismatch, got: %s", err)
528 }
529 })
530 }
531}
532
533func TestBuilder_DeltaShardsMetadataInOlderShards(t *testing.T) {
534 olderTime := time.Unix(0, 0)
535 newerTime := time.Unix(10000, 0)
536
537 for _, test := range []struct {
538 name string
539 originalRepository zoekt.Repository
540 updatedRepository zoekt.Repository
541 }{
542 {
543 name: "update commit information",
544 originalRepository: zoekt.Repository{
545 Name: "repo",
546 ID: 1,
547 Branches: []zoekt.RepositoryBranch{
548 {Name: "main", Version: "v1"},
549 {Name: "release", Version: "v1"},
550 },
551 },
552 updatedRepository: zoekt.Repository{
553 Name: "repo",
554 ID: 1,
555 Branches: []zoekt.RepositoryBranch{
556 {Name: "main", Version: "v2"},
557 {Name: "release", Version: "v2"},
558 },
559 },
560 },
561 {
562 name: "update latest commit date (older -> newer)",
563 originalRepository: zoekt.Repository{
564 Name: "repo",
565 ID: 1,
566 Branches: []zoekt.RepositoryBranch{
567 {Name: "main", Version: "v1"},
568 },
569 LatestCommitDate: olderTime,
570 },
571 updatedRepository: zoekt.Repository{
572 Name: "repo",
573 ID: 1,
574 Branches: []zoekt.RepositoryBranch{
575 {Name: "main", Version: "v2"},
576 },
577 LatestCommitDate: newerTime,
578 },
579 },
580 {
581 name: "update latest commit date (even if latest commit date is older - the most recent commits are the source of truth)",
582 originalRepository: zoekt.Repository{
583 Name: "repo",
584 ID: 1,
585 Branches: []zoekt.RepositoryBranch{
586 {Name: "main", Version: "v1"},
587 },
588 LatestCommitDate: newerTime,
589 },
590 updatedRepository: zoekt.Repository{
591 Name: "repo",
592 ID: 1,
593 Branches: []zoekt.RepositoryBranch{
594 {Name: "main", Version: "v2"},
595 },
596 LatestCommitDate: olderTime,
597 },
598 },
599 } {
600 test := test
601
602 t.Run(test.name, func(t *testing.T) {
603 indexDir := t.TempDir()
604
605 createTestShard(t, indexDir, test.originalRepository, 2, func(o *Options) {
606 o.DisableCTags = true
607 })
608
609 shards := createTestShard(t, indexDir, test.updatedRepository, 1, func(o *Options) {
610 o.IsDelta = true
611 o.DisableCTags = true
612 })
613
614 if len(shards) < 3 {
615 t.Fatalf("expected at least 3 shards, got %d (%s)", len(shards), strings.Join(shards, ", "))
616 }
617
618 for _, s := range shards {
619 repositories, _, err := zoekt.ReadMetadataPathAlive(s)
620 if err != nil {
621 t.Fatalf("reading repository metadata from shard %q", s)
622 }
623
624 var foundRepository *zoekt.Repository
625 for _, r := range repositories {
626 if r.ID == test.updatedRepository.ID {
627 foundRepository = r
628 break
629 }
630 }
631
632 if foundRepository == nil {
633 t.Fatalf("repository ID %d not in shard %q", test.updatedRepository.ID, s)
634 }
635
636 diffOptions := []cmp.Option{
637 cmpopts.IgnoreUnexported(zoekt.Repository{}),
638 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
639 cmpopts.EquateEmpty(),
640 }
641
642 if diff := cmp.Diff(&test.updatedRepository, foundRepository, diffOptions...); diff != "" {
643 t.Errorf("shard %q: unexpected diff in repository metadata (-want +got):\n%s", s, diff)
644 }
645 }
646 })
647 }
648}
649
650func TestFindRepositoryMetadata(t *testing.T) {
651 tests := []struct {
652 name string
653 normalShardRepositories []zoekt.Repository
654 compoundShardRepositories []zoekt.Repository
655 input *zoekt.Repository
656 expectedRepository *zoekt.Repository
657 expectedOk bool
658 }{
659 {
660 name: "repository in normal shards",
661 normalShardRepositories: []zoekt.Repository{
662 {Name: "repoA", ID: 1},
663 {Name: "repoB", ID: 2},
664 {Name: "repoC", ID: 3},
665 },
666 compoundShardRepositories: []zoekt.Repository{
667 {Name: "repoD", ID: 4},
668 {Name: "repoE", ID: 5},
669 {Name: "repoF", ID: 6},
670 },
671 input: &zoekt.Repository{Name: "repoB", ID: 2},
672 expectedRepository: &zoekt.Repository{Name: "repoB", ID: 2},
673 expectedOk: true,
674 },
675 {
676 name: "repository in compound shards",
677 normalShardRepositories: []zoekt.Repository{
678 {Name: "repoA", ID: 1},
679 {Name: "repoB", ID: 2},
680 {Name: "repoC", ID: 3},
681 },
682 compoundShardRepositories: []zoekt.Repository{
683 {Name: "repoD", ID: 4},
684 {Name: "repoE", ID: 5},
685 {Name: "repoF", ID: 6},
686 },
687 input: &zoekt.Repository{Name: "repoE", ID: 5},
688 expectedRepository: &zoekt.Repository{Name: "repoE", ID: 5},
689 expectedOk: true,
690 },
691 {
692 name: "repository not in any shard",
693 normalShardRepositories: []zoekt.Repository{
694 {Name: "repoA", ID: 1},
695 {Name: "repoB", ID: 2},
696 {Name: "repoC", ID: 3},
697 },
698 compoundShardRepositories: []zoekt.Repository{
699 {Name: "repoD", ID: 4},
700 {Name: "repoE", ID: 5},
701 {Name: "repoF", ID: 6},
702 },
703 input: &zoekt.Repository{Name: "notPresent", ID: 123},
704 expectedRepository: nil,
705 expectedOk: false,
706 },
707 }
708 for _, tt := range tests {
709 t.Run(tt.name, func(t *testing.T) {
710 // setup
711 indexDir := t.TempDir()
712
713 optFns := []func(o *Options){
714 // ctags aren't important for this test, and the equality checks
715 // for diffing repositories can break due to local configuration
716 func(o *Options) {
717 o.DisableCTags = true
718 },
719 }
720
721 for _, r := range tt.normalShardRepositories {
722 createTestShard(t, indexDir, r, 1, optFns...)
723 }
724
725 if len(tt.compoundShardRepositories) > 0 {
726 createTestCompoundShard(t, indexDir, tt.compoundShardRepositories, optFns...)
727 }
728
729 o := &Options{
730 IndexDir: indexDir,
731 RepositoryDescription: *tt.input,
732 }
733 o.SetDefaults()
734
735 // run test
736 got, gotOk, err := o.FindRepositoryMetadata()
737 if err != nil {
738 t.Errorf("received unexpected error: %v", err)
739 return
740 }
741
742 // check outcome
743 compareOptions := []cmp.Option{
744 cmpopts.IgnoreUnexported(zoekt.Repository{}),
745 cmpopts.IgnoreFields(zoekt.Repository{}, "IndexOptions"),
746 cmpopts.EquateEmpty(),
747 }
748
749 if diff := cmp.Diff(tt.expectedRepository, got, compareOptions...); diff != "" {
750 t.Errorf("unexpected difference in repositories (-want +got):\n%s", diff)
751 }
752
753 if tt.expectedOk != gotOk {
754 t.Errorf("unexpected difference in 'ok' value: wanted %t, got %t", tt.expectedOk, gotOk)
755 }
756 })
757 }
758}
759
760func createTestShard(t *testing.T, indexDir string, r zoekt.Repository, numShards int, optFns ...func(options *Options)) []string {
761 t.Helper()
762
763 if err := os.MkdirAll(filepath.Dir(indexDir), 0700); err != nil {
764 t.Fatal(err)
765 }
766
767 o := Options{
768 IndexDir: indexDir,
769 RepositoryDescription: r,
770 ShardMax: 75, // create a new shard every 75 bytes
771 }
772 o.SetDefaults()
773
774 for _, fn := range optFns {
775 fn(&o)
776 }
777
778 b, err := NewBuilder(o)
779 if err != nil {
780 t.Fatalf("NewBuilder: %v", err)
781 }
782
783 if numShards == 0 {
784 // We have to make at least 1 shard.
785 numShards = 1
786 }
787
788 for i := 0; i < numShards; i++ {
789 // Create entries (file + contents) that are ~100 bytes each.
790 // This (along with our shardMax setting of 75 bytes) means that each shard
791 // will contain at most one of these.
792 fileName := strconv.Itoa(i)
793 document := zoekt.Document{Name: fileName, Content: []byte(strings.Repeat("A", 100))}
794 for _, branch := range o.RepositoryDescription.Branches {
795 document.Branches = append(document.Branches, branch.Name)
796 }
797
798 err := b.Add(document)
799 if err != nil {
800 t.Fatalf("failed to add file %q to builder: %s", fileName, err)
801 }
802 }
803
804 if err := b.Finish(); err != nil {
805 t.Fatalf("Finish: %v", err)
806 }
807
808 return o.FindAllShards()
809}
810
811func createTestCompoundShard(t *testing.T, indexDir string, repositories []zoekt.Repository, optFns ...func(options *Options)) {
812 t.Helper()
813
814 var shardNames []string
815
816 for _, r := range repositories {
817 // create an isolated scratch space to store normal shards for this repository
818 scratchDir := t.TempDir()
819
820 // create shards that'll be merged later
821 createTestShard(t, scratchDir, r, 1, optFns...)
822
823 // discover file names for all the normal shards we created
824 // note: this only looks in the immediate 'scratchDir' folder and doesn't recurse
825 shards, err := filepath.Glob(filepath.Join(scratchDir, "*.zoekt"))
826 if err != nil {
827 t.Fatalf("while globbing %q to find normal shards: %s", scratchDir, err)
828 }
829
830 shardNames = append(shardNames, shards...)
831 }
832
833 // load the normal shards that we created
834 var files []zoekt.IndexFile
835 for _, shard := range shardNames {
836 f, err := os.Open(shard)
837 if err != nil {
838 t.Fatalf("opening shard file: %s", err)
839 }
840 defer f.Close()
841
842 indexFile, err := zoekt.NewIndexFile(f)
843 if err != nil {
844 t.Fatalf("creating index file: %s", err)
845 }
846 defer indexFile.Close()
847
848 files = append(files, indexFile)
849 }
850
851 // merge all the simple shards into a compound shard
852 tmpName, dstName, err := zoekt.Merge(indexDir, files...)
853 if err != nil {
854 t.Fatalf("merging index files into compound shard: %s", err)
855 }
856 if err := os.Rename(tmpName, dstName); err != nil {
857 t.Fatal(err)
858 }
859}
860
861func TestIgnoreSizeMax(t *testing.T) {
862
863 for _, test := range []struct {
864 name string
865 largeFiles []string
866 filePaths []string
867 expected bool
868 }{
869 {
870 name: "empty pattern does nothing",
871 largeFiles: []string{""},
872 filePaths: []string{"F0"},
873 expected: false,
874 },
875 {
876 name: "positive match allows",
877 largeFiles: []string{"F0"},
878 filePaths: []string{"F0"},
879 expected: true,
880 },
881 {
882 name: "positive and negative patterns allows",
883 largeFiles: []string{"F?", "!F0"},
884 filePaths: []string{"F1"},
885 expected: true,
886 },
887 {
888 name: "positive and negative patterns disallows",
889 largeFiles: []string{"F?", "!F0"},
890 filePaths: []string{"F0"},
891 expected: false,
892 },
893 {
894 name: "positive escaped pattern allows",
895 largeFiles: []string{"\\!F?"},
896 filePaths: []string{"!F0", "!F1"},
897 expected: true,
898 },
899 {
900 name: "postive escaped pattern does not disallow",
901 largeFiles: []string{"F0", "\\!F?"},
902 filePaths: []string{"F0", "!F0"},
903 expected: true,
904 },
905 {
906 name: "combined meta and literal interpretation disallows",
907 largeFiles: []string{"*F*", "!!F*"},
908 filePaths: []string{"!F0"},
909 expected: false,
910 },
911 {
912 name: "combined meta and literal interpretation allows",
913 largeFiles: []string{"*F*", "!!F*"},
914 filePaths: []string{"F0"},
915 expected: true,
916 },
917 {
918 name: "largeFiles order: positive match overrides previous negative match and allows",
919 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
920 filePaths: []string{"F0"},
921 expected: true,
922 },
923 {
924 name: "largeFiles order: positive match overrides previous negative match and disallows",
925 largeFiles: []string{"F?", "!F0", "!F1", "F0"},
926 filePaths: []string{"F1"},
927 expected: false,
928 },
929 {
930 name: "largeFiles order: negative match overrides previous positive match and allows",
931 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
932 filePaths: []string{"F1"},
933 expected: true,
934 },
935 {
936 name: "largeFiles order: negative match overrides previous positive match and disallows",
937 largeFiles: []string{"F?", "!?0", "F0", "!F0"},
938 filePaths: []string{"F0"},
939 expected: false,
940 },
941 } {
942 t.Run(test.name, func(t *testing.T) {
943 o := Options{
944 LargeFiles: test.largeFiles,
945 }
946
947 for _, filePath := range test.filePaths {
948 ignore := o.IgnoreSizeMax(filePath)
949 if ignore != test.expected {
950 t.Errorf("IgnoreSizeMax() for filepath %v returned unexpected result %v", filePath, ignore)
951 }
952 }
953 })
954 }
955}