fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35 "github.com/sourcegraph/zoekt"
36 "github.com/sourcegraph/zoekt/ignore"
37 "github.com/sourcegraph/zoekt/index"
38 "github.com/sourcegraph/zoekt/query"
39 "github.com/sourcegraph/zoekt/search"
40)
41
42func TestIndexEmptyRepo(t *testing.T) {
43 dir := t.TempDir()
44
45 cmd := exec.Command("git", "init", "-b", "master", "repo")
46 cmd.Dir = dir
47
48 if err := cmd.Run(); err != nil {
49 t.Fatalf("cmd.Run: %v", err)
50 }
51
52 desc := zoekt.Repository{
53 Name: "repo",
54 }
55 opts := Options{
56 RepoDir: filepath.Join(dir, "repo", ".git"),
57 BuildOptions: index.Options{
58 RepositoryDescription: desc,
59 IndexDir: dir,
60 },
61 }
62
63 if _, err := IndexGitRepo(opts); err != nil {
64 t.Fatalf("IndexGitRepo: %v", err)
65 }
66}
67
68func TestIndexNonexistentRepo(t *testing.T) {
69 dir := t.TempDir()
70 desc := zoekt.Repository{
71 Name: "nonexistent",
72 }
73 opts := Options{
74 RepoDir: "does/not/exist",
75 Branches: []string{"main"},
76 BuildOptions: index.Options{
77 RepositoryDescription: desc,
78 IndexDir: dir,
79 },
80 }
81
82 if _, err := IndexGitRepo(opts); err == nil {
83 t.Fatal("expected error, got none")
84 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
85 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
86 }
87}
88
89func TestIndexTinyRepo(t *testing.T) {
90 // Create a repo with one file in it.
91 dir := t.TempDir()
92 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
93
94 repoDir := filepath.Join(dir, "repo")
95 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
96 t.Fatalf("WriteFile: %v", err)
97 }
98 executeCommand(t, repoDir, exec.Command("git", "add", "."))
99 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
100
101 // Test that indexing accepts both the repo directory, and the .git subdirectory.
102 for _, testDir := range []string{"repo", "repo/.git"} {
103 opts := Options{
104 RepoDir: filepath.Join(dir, testDir),
105 Branches: []string{"main"},
106 BuildOptions: index.Options{
107 RepositoryDescription: zoekt.Repository{Name: "repo"},
108 IndexDir: dir,
109 },
110 }
111
112 if _, err := IndexGitRepo(opts); err != nil {
113 t.Fatalf("unexpected error %v", err)
114 }
115
116 searcher, err := search.NewDirectorySearcher(dir)
117 if err != nil {
118 t.Fatal("NewDirectorySearcher", err)
119 }
120
121 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
122 searcher.Close()
123
124 if err != nil {
125 t.Fatal("search failed", err)
126 }
127
128 if len(results.Files) != 1 {
129 t.Fatalf("got search result %v, want 1 file", results.Files)
130 }
131 }
132}
133
134func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
135 cmd.Dir = dir
136 cmd.Env = []string{
137 "GIT_CONFIG_GLOBAL=",
138 "GIT_CONFIG_SYSTEM=",
139 "GIT_COMMITTER_NAME=Kierkegaard",
140 "GIT_COMMITTER_EMAIL=soren@apache.com",
141 "GIT_AUTHOR_NAME=Kierkegaard",
142 "GIT_AUTHOR_EMAIL=soren@apache.com",
143 }
144 if err := cmd.Run(); err != nil {
145 t.Fatalf("cmd.Run: %v", err)
146 }
147 return cmd
148}
149
150func TestIndexDeltaBasic(t *testing.T) {
151 type branchToDocumentMap map[string][]index.Document
152
153 type step struct {
154 name string
155 addedDocuments branchToDocumentMap
156 deletedDocuments branchToDocumentMap
157 optFn func(t *testing.T, options *Options)
158
159 expectedFallbackToNormalBuild bool
160 expectedDocuments []index.Document
161 }
162
163 helloWorld := index.Document{Name: "hello_world.txt", Content: []byte("hello")}
164
165 fruitV1 := index.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
166 fruitV1InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
167 fruitV1WithNewName := index.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
168
169 fruitV2 := index.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
170 fruitV2InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
171
172 fruitV3 := index.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
173 fruitV4 := index.Document{Name: "best_fruit.txt", Content: []byte("apples")}
174
175 foo := index.Document{Name: "foo.txt", Content: []byte("bar")}
176
177 emptySourcegraphIgnore := index.Document{Name: ignore.IgnoreFile}
178 sourcegraphIgnoreWithContent := index.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
179
180 for _, test := range []struct {
181 name string
182 branches []string
183 steps []step
184 }{
185 {
186 name: "modification",
187 branches: []string{"main"},
188 steps: []step{
189 {
190 name: "setup",
191 addedDocuments: branchToDocumentMap{
192 "main": []index.Document{helloWorld, fruitV1},
193 },
194
195 expectedDocuments: []index.Document{helloWorld, fruitV1},
196 },
197 {
198 name: "add newer version of fruits",
199 addedDocuments: branchToDocumentMap{
200 "main": []index.Document{fruitV2},
201 },
202 optFn: func(t *testing.T, o *Options) {
203 o.BuildOptions.IsDelta = true
204 },
205
206 expectedDocuments: []index.Document{helloWorld, fruitV2},
207 },
208 },
209 },
210 {
211 name: "modification only inside nested folder",
212 branches: []string{"main"},
213 steps: []step{
214 {
215 name: "setup",
216 addedDocuments: branchToDocumentMap{
217 "main": []index.Document{foo, fruitV1InFolder},
218 },
219
220 expectedDocuments: []index.Document{foo, fruitV1InFolder},
221 },
222 {
223 name: "add newer version of fruits inside folder",
224 addedDocuments: branchToDocumentMap{
225 "main": []index.Document{fruitV2InFolder},
226 },
227 optFn: func(t *testing.T, o *Options) {
228 o.BuildOptions.IsDelta = true
229 },
230
231 expectedDocuments: []index.Document{foo, fruitV2InFolder},
232 },
233 },
234 },
235 {
236 name: "addition",
237 branches: []string{"main"},
238 steps: []step{
239 {
240 name: "setup",
241 addedDocuments: branchToDocumentMap{
242 "main": []index.Document{helloWorld, fruitV1},
243 },
244
245 expectedDocuments: []index.Document{helloWorld, fruitV1},
246 },
247 {
248 name: "add new file - foo",
249 addedDocuments: branchToDocumentMap{
250 "main": []index.Document{foo},
251 },
252 optFn: func(t *testing.T, o *Options) {
253 o.BuildOptions.IsDelta = true
254 },
255
256 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
257 },
258 },
259 },
260 {
261 name: "deletion",
262 branches: []string{"main"},
263 steps: []step{
264 {
265 name: "setup",
266 addedDocuments: branchToDocumentMap{
267 "main": []index.Document{helloWorld, fruitV1, foo},
268 },
269
270 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
271 },
272 {
273 name: "delete foo file",
274 addedDocuments: nil,
275 deletedDocuments: branchToDocumentMap{
276 "main": []index.Document{foo},
277 },
278
279 optFn: func(t *testing.T, o *Options) {
280 o.BuildOptions.IsDelta = true
281 },
282
283 expectedDocuments: []index.Document{helloWorld, fruitV1},
284 },
285 },
286 },
287 {
288 name: "addition and deletion on only one branch",
289 branches: []string{"main", "release", "dev"},
290 steps: []step{
291 {
292 name: "setup",
293 addedDocuments: branchToDocumentMap{
294 "main": []index.Document{fruitV1},
295 "release": []index.Document{fruitV2},
296 "dev": []index.Document{fruitV3},
297 },
298
299 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
300 },
301 {
302 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
303 addedDocuments: branchToDocumentMap{
304 "dev": []index.Document{fruitV4},
305 },
306 deletedDocuments: branchToDocumentMap{
307 "main": []index.Document{fruitV1},
308 },
309
310 optFn: func(t *testing.T, o *Options) {
311 o.BuildOptions.IsDelta = true
312 },
313
314 expectedDocuments: []index.Document{fruitV2, fruitV4},
315 },
316 },
317 },
318 {
319 name: "rename",
320 branches: []string{"main", "release"},
321 steps: []step{
322 {
323 name: "setup",
324 addedDocuments: branchToDocumentMap{
325 "main": []index.Document{fruitV1},
326 "release": []index.Document{fruitV2},
327 },
328 expectedDocuments: []index.Document{fruitV1, fruitV2},
329 },
330 {
331 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
332 addedDocuments: branchToDocumentMap{
333 "main": []index.Document{fruitV1WithNewName},
334 },
335 deletedDocuments: branchToDocumentMap{
336 "main": []index.Document{fruitV1},
337 },
338
339 optFn: func(t *testing.T, o *Options) {
340 o.BuildOptions.IsDelta = true
341 },
342
343 expectedDocuments: []index.Document{fruitV1WithNewName, fruitV2},
344 },
345 },
346 },
347 {
348 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
349 branches: []string{"main", "dev"},
350 steps: []step{
351 {
352 name: "setup",
353 addedDocuments: branchToDocumentMap{
354 "main": []index.Document{fruitV1},
355 "dev": []index.Document{fruitV2},
356 },
357 expectedDocuments: []index.Document{fruitV1, fruitV2},
358 },
359 {
360 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
361 addedDocuments: branchToDocumentMap{
362 "main": []index.Document{fruitV2},
363 "dev": []index.Document{fruitV3},
364 },
365
366 optFn: func(t *testing.T, o *Options) {
367 o.BuildOptions.IsDelta = true
368 },
369
370 expectedDocuments: []index.Document{fruitV2, fruitV3},
371 },
372 },
373 },
374 {
375 name: "no-op delta builds (reindexing the same commits)",
376 branches: []string{"main", "dev"},
377 steps: []step{
378 {
379 name: "setup",
380 addedDocuments: branchToDocumentMap{
381 "main": []index.Document{fruitV1, foo},
382 "dev": []index.Document{helloWorld},
383 },
384 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
385 },
386 {
387 name: "first no-op (normal build -> delta build)",
388 optFn: func(t *testing.T, o *Options) {
389 o.BuildOptions.IsDelta = true
390 },
391
392 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
393 },
394 {
395 name: "second no-op (delta build -> delta build)",
396 optFn: func(t *testing.T, o *Options) {
397 o.BuildOptions.IsDelta = true
398 },
399
400 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
401 },
402 },
403 },
404 {
405 name: "should fallback to normal build if no prior shards exist",
406 branches: []string{"main"},
407 steps: []step{
408 {
409 name: "attempt delta build on a repository that hasn't been indexed yet",
410 addedDocuments: branchToDocumentMap{
411 "main": []index.Document{helloWorld},
412 },
413 optFn: func(t *testing.T, o *Options) {
414 o.BuildOptions.IsDelta = true
415 },
416
417 expectedFallbackToNormalBuild: true,
418 expectedDocuments: []index.Document{helloWorld},
419 },
420 },
421 },
422 {
423 name: "should fallback to normal build if the set of requested repository branches changes",
424 branches: []string{"main", "release", "dev"},
425 steps: []step{
426 {
427 name: "setup",
428 addedDocuments: branchToDocumentMap{
429 "main": []index.Document{fruitV1},
430 "release": []index.Document{fruitV2},
431 "dev": []index.Document{fruitV3},
432 },
433
434 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
435 },
436 {
437 name: "try delta build after dropping 'main' branch from index ",
438 addedDocuments: branchToDocumentMap{
439 "release": []index.Document{fruitV4},
440 },
441 optFn: func(t *testing.T, o *Options) {
442 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
443 o.BuildOptions.IsDelta = true
444 },
445
446 expectedFallbackToNormalBuild: true,
447 expectedDocuments: []index.Document{fruitV3, fruitV4},
448 },
449 },
450 },
451 {
452 name: "should expand branches correctly when using wildcards in branch names",
453 branches: []string{"release/1", "release/2"},
454 steps: []step{
455 {
456 name: "setup",
457 addedDocuments: branchToDocumentMap{
458 "release/1": []index.Document{fruitV1},
459 "release/2": []index.Document{fruitV2},
460 },
461
462 expectedDocuments: []index.Document{fruitV1, fruitV2},
463 },
464 {
465 name: "try delta build with wildcard in branches",
466 optFn: func(t *testing.T, o *Options) {
467 // use a wildcard here
468 o.Branches = []string{"HEAD", "release/*"}
469 o.BuildOptions.IsDelta = true
470 },
471
472 expectedDocuments: []index.Document{fruitV1, fruitV2},
473 },
474 },
475 },
476 {
477 name: "should fallback to normal build if one or more index options updates requires a full build",
478 branches: []string{"main"},
479 steps: []step{
480 {
481 name: "setup",
482 addedDocuments: branchToDocumentMap{
483 "main": []index.Document{fruitV1},
484 },
485
486 expectedDocuments: []index.Document{fruitV1},
487 },
488 {
489 name: "try delta build after updating Disable CTags index option",
490 addedDocuments: branchToDocumentMap{
491 "main": []index.Document{fruitV2},
492 },
493 optFn: func(t *testing.T, o *Options) {
494 o.BuildOptions.IsDelta = true
495 o.BuildOptions.DisableCTags = true
496 },
497
498 expectedFallbackToNormalBuild: true,
499 expectedDocuments: []index.Document{fruitV2},
500 },
501 {
502 name: "try delta build after reverting Disable CTags index option",
503 addedDocuments: branchToDocumentMap{
504 "main": []index.Document{fruitV3},
505 },
506 optFn: func(t *testing.T, o *Options) {
507 o.BuildOptions.IsDelta = true
508 o.BuildOptions.DisableCTags = false
509 },
510
511 expectedFallbackToNormalBuild: true,
512 expectedDocuments: []index.Document{fruitV3},
513 },
514 },
515 },
516 {
517 name: "should successfully perform multiple delta builds after disabling symbols",
518 branches: []string{"main"},
519 steps: []step{
520 {
521 name: "setup",
522 addedDocuments: branchToDocumentMap{
523 "main": []index.Document{fruitV1},
524 },
525
526 expectedDocuments: []index.Document{fruitV1},
527 },
528 {
529 name: "try delta build after updating Disable CTags index option",
530 addedDocuments: branchToDocumentMap{
531 "main": []index.Document{fruitV2},
532 },
533 optFn: func(t *testing.T, o *Options) {
534 o.BuildOptions.IsDelta = true
535 o.BuildOptions.DisableCTags = true
536 },
537
538 expectedFallbackToNormalBuild: true,
539 expectedDocuments: []index.Document{fruitV2},
540 },
541 {
542 name: "try another delta build while CTags is still disabled",
543 addedDocuments: branchToDocumentMap{
544 "main": []index.Document{fruitV3},
545 },
546 optFn: func(t *testing.T, o *Options) {
547 o.BuildOptions.IsDelta = true
548 o.BuildOptions.DisableCTags = true
549 },
550
551 expectedDocuments: []index.Document{fruitV3},
552 },
553 },
554 },
555 {
556 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
557 branches: []string{"main"},
558 steps: []step{
559 {
560 name: "setup",
561 addedDocuments: branchToDocumentMap{
562 "main": []index.Document{emptySourcegraphIgnore},
563 },
564
565 expectedDocuments: []index.Document{emptySourcegraphIgnore},
566 },
567 {
568 name: "attempt delta build after modifying ignore file",
569 addedDocuments: branchToDocumentMap{
570 "main": []index.Document{sourcegraphIgnoreWithContent},
571 },
572 optFn: func(t *testing.T, o *Options) {
573 o.BuildOptions.IsDelta = true
574 },
575
576 expectedFallbackToNormalBuild: true,
577 expectedDocuments: []index.Document{sourcegraphIgnoreWithContent},
578 },
579 },
580 },
581 {
582 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
583 branches: []string{"main"},
584 steps: []step{
585 {
586 name: "setup: first shard",
587 addedDocuments: branchToDocumentMap{
588 "main": []index.Document{foo},
589 },
590
591 expectedDocuments: []index.Document{foo},
592 },
593 {
594 name: "setup: second shard (delta)",
595 addedDocuments: branchToDocumentMap{
596 "main": []index.Document{fruitV1},
597 },
598 optFn: func(t *testing.T, o *Options) {
599 o.BuildOptions.IsDelta = true
600 },
601
602 expectedDocuments: []index.Document{foo, fruitV1},
603 },
604 {
605 name: "setup: third shard (delta)",
606 addedDocuments: branchToDocumentMap{
607 "main": []index.Document{helloWorld},
608 },
609 optFn: func(t *testing.T, o *Options) {
610 o.BuildOptions.IsDelta = true
611 },
612
613 expectedDocuments: []index.Document{foo, fruitV1, helloWorld},
614 },
615 {
616 name: "attempt another delta build after we already blew past the shard threshold",
617 addedDocuments: branchToDocumentMap{
618 "main": []index.Document{fruitV2InFolder},
619 },
620 optFn: func(t *testing.T, o *Options) {
621 o.DeltaShardNumberFallbackThreshold = 2
622 o.BuildOptions.IsDelta = true
623 },
624
625 expectedFallbackToNormalBuild: true,
626 expectedDocuments: []index.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
627 },
628 },
629 },
630 } {
631 test := test
632
633 t.Run(test.name, func(t *testing.T) {
634 t.Parallel()
635
636 indexDir := t.TempDir()
637 repositoryDir := t.TempDir()
638
639 // setup: initialize the repository and all of its branches
640 runScript(t, repositoryDir, "git init -b master")
641 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
642 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
643
644 for _, b := range test.branches {
645 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
646 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
647 }
648
649 for _, step := range test.steps {
650 t.Run(step.name, func(t *testing.T) {
651 for _, b := range test.branches {
652 // setup: for each branch, process any document deletions / additions and commit those changes
653
654 hadChange := false
655
656 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
657
658 for _, d := range step.deletedDocuments[b] {
659 hadChange = true
660
661 file := filepath.Join(repositoryDir, d.Name)
662
663 err := os.Remove(file)
664 if err != nil {
665 t.Fatalf("deleting file %q: %s", d.Name, err)
666 }
667
668 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
669 }
670
671 for _, d := range step.addedDocuments[b] {
672 hadChange = true
673
674 file := filepath.Join(repositoryDir, d.Name)
675
676 err := os.MkdirAll(filepath.Dir(file), 0o755)
677 if err != nil {
678 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
679 }
680
681 err = os.WriteFile(file, d.Content, 0o644)
682 if err != nil {
683 t.Fatalf("writing file %q: %s", d.Name, err)
684 }
685
686 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
687 }
688
689 if !hadChange {
690 continue
691 }
692
693 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
694 }
695
696 // setup: prepare indexOptions with given overrides
697 buildOptions := index.Options{
698 IndexDir: indexDir,
699 RepositoryDescription: zoekt.Repository{
700 Name: "repository",
701 },
702 IsDelta: false,
703 }
704 buildOptions.SetDefaults()
705
706 branches := append([]string{"HEAD"}, test.branches...)
707
708 options := Options{
709 RepoDir: filepath.Join(repositoryDir, ".git"),
710 BuildOptions: buildOptions,
711 Branches: branches,
712 }
713
714 if step.optFn != nil {
715 step.optFn(t, &options)
716 }
717
718 // setup: prepare spy versions of prepare delta / normal build so that we can observe
719 // whether they were called appropriately
720 deltaBuildCalled := false
721 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
722 deltaBuildCalled = true
723 return prepareDeltaBuild(options, repository)
724 }
725
726 normalBuildCalled := false
727 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
728 normalBuildCalled = true
729 return prepareNormalBuild(options, repository)
730 }
731
732 // run test
733 _, err := indexGitRepo(options, gitIndexConfig{
734 prepareDeltaBuild: prepareDeltaSpy,
735 prepareNormalBuild: prepareNormalSpy,
736 })
737 if err != nil {
738 t.Fatalf("IndexGitRepo: %s", err)
739 }
740
741 if options.BuildOptions.IsDelta != deltaBuildCalled {
742 // We should always try a delta build if we request it in the options.
743 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
744 }
745
746 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
747 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
748 // asked for a delta build in the first place.
749 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
750 }
751
752 // examine outcome: load shards into a searcher instance and run a dummy search query
753 // that returns every document contained in the shards
754 //
755 // then, compare returned set of documents with the expected set for the step and see if they agree
756
757 ss, err := search.NewDirectorySearcher(indexDir)
758 if err != nil {
759 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
760 }
761 defer ss.Close()
762
763 searchOpts := &zoekt.SearchOptions{Whole: true}
764 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
765 if err != nil {
766 t.Fatalf("Search: %s", err)
767 }
768
769 var receivedDocuments []index.Document
770 for _, f := range result.Files {
771 receivedDocuments = append(receivedDocuments, index.Document{
772 Name: f.FileName,
773 Content: f.Content,
774 })
775 }
776
777 for _, docs := range [][]index.Document{step.expectedDocuments, receivedDocuments} {
778 sort.Slice(docs, func(i, j int) bool {
779 a, b := docs[i], docs[j]
780
781 // first compare names, then fallback to contents if the names are equal
782
783 if a.Name < b.Name {
784 return true
785 }
786
787 if a.Name > b.Name {
788 return false
789 }
790
791 return bytes.Compare(a.Content, b.Content) < 0
792 })
793 }
794
795 compareOptions := []cmp.Option{
796 cmpopts.IgnoreFields(index.Document{}, "Branches"),
797 cmpopts.EquateEmpty(),
798 }
799
800 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
801 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
802 }
803 })
804 }
805 })
806 }
807}
808
809func runScript(t *testing.T, cwd string, script string) {
810 t.Helper()
811
812 err := os.MkdirAll(cwd, 0o755)
813 if err != nil {
814 t.Fatalf("ensuring path %q exists: %s", cwd, err)
815 }
816
817 cmd := exec.Command("sh", "-euxc", script)
818 cmd.Dir = cwd
819 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
820
821 if out, err := cmd.CombinedOutput(); err != nil {
822 t.Fatalf("execution error: %v, output %s", err, out)
823 }
824}
825
826func TestSetTemplates_e2e(t *testing.T) {
827 repositoryDir := t.TempDir()
828
829 // setup: initialize the repository and all of its branches
830 runScript(t, repositoryDir, "git init -b master")
831 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
832 desc := zoekt.Repository{}
833 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
834 t.Fatalf("setTemplatesFromConfig: %v", err)
835 }
836
837 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
838 t.Errorf("got %q, want %q", got, want)
839 }
840}
841
842func TestSetTemplates(t *testing.T) {
843 base := "https://example.com/repo/name"
844 version := "VERSION"
845 path := "dir/name.txt"
846 lineNumber := 10
847 cases := []struct {
848 typ string
849 commit string
850 file string
851 line string
852 }{{
853 typ: "gitiles",
854 commit: "https://example.com/repo/name/%2B/VERSION",
855 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
856 line: "#10",
857 }, {
858 typ: "github",
859 commit: "https://example.com/repo/name/commit/VERSION",
860 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
861 line: "#L10",
862 }, {
863 typ: "cgit",
864 commit: "https://example.com/repo/name/commit/?id=VERSION",
865 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
866 line: "#n10",
867 }, {
868 typ: "gitweb",
869 commit: "https://example.com/repo/name;a=commit;h=VERSION",
870 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
871 line: "#l10",
872 }, {
873 typ: "source.bazel.build",
874 commit: "https://example.com/repo/name/%2B/VERSION",
875 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
876 line: ";l=10",
877 }, {
878 typ: "bitbucket-server",
879 commit: "https://example.com/repo/name/commits/VERSION",
880 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
881 line: "#10",
882 }, {
883 typ: "gitlab",
884 commit: "https://example.com/repo/name/-/commit/VERSION",
885 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
886 line: "#L10",
887 }, {
888 typ: "gitea",
889 commit: "https://example.com/repo/name/commit/VERSION",
890 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
891 line: "#L10",
892 }}
893
894 for _, tc := range cases {
895 t.Run(tc.typ, func(t *testing.T) {
896 assertOutput := func(templateText string, want string) {
897 t.Helper()
898
899 tt, err := index.ParseTemplate(templateText)
900 if err != nil {
901 t.Fatal(err)
902 }
903
904 var sb strings.Builder
905 err = tt.Execute(&sb, map[string]any{
906 "Version": version,
907 "Path": path,
908 "LineNumber": lineNumber,
909 })
910 if err != nil {
911 t.Fatal(err)
912 }
913 if got := sb.String(); got != want {
914 t.Fatalf("want: %q\ngot: %q", want, got)
915 }
916 }
917
918 var repo zoekt.Repository
919 u, _ := url.Parse(base)
920 err := setTemplates(&repo, u, tc.typ)
921 if err != nil {
922 t.Fatal(err)
923 }
924 assertOutput(repo.CommitURLTemplate, tc.commit)
925 assertOutput(repo.FileURLTemplate, tc.file)
926 assertOutput(repo.LineFragmentTemplate, tc.line)
927 })
928 }
929}
930
931func BenchmarkPrepareNormalBuild(b *testing.B) {
932 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
933 repoDir := "/path/to/your/repo"
934 repo, err := git.PlainOpen(repoDir)
935 if err != nil {
936 b.Fatalf("Failed to open test repository: %v", err)
937 }
938
939 opts := Options{
940 RepoDir: repoDir,
941 Submodules: false,
942 BranchPrefix: "refs/heads/",
943 Branches: []string{"main"},
944 BuildOptions: index.Options{
945 RepositoryDescription: zoekt.Repository{
946 Name: "test-repo",
947 URL: "https://github.com/example/test-repo",
948 },
949 },
950 }
951
952 b.ReportAllocs()
953
954 repos, branchVersions, err := prepareNormalBuild(opts, repo)
955 if err != nil {
956 b.Fatalf("prepareNormalBuild failed: %v", err)
957 }
958
959 runtime.GC()
960
961 var m runtime.MemStats
962 runtime.ReadMemStats(&m)
963 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
964 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
965
966 if len(repos) == 0 || len(branchVersions) == 0 {
967 b.Fatalf("Unexpected empty results")
968 }
969}