fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35
36 "github.com/sourcegraph/zoekt"
37 "github.com/sourcegraph/zoekt/ignore"
38 "github.com/sourcegraph/zoekt/index"
39 "github.com/sourcegraph/zoekt/query"
40 "github.com/sourcegraph/zoekt/search"
41)
42
43func TestIndexEmptyRepo(t *testing.T) {
44 dir := t.TempDir()
45
46 cmd := exec.Command("git", "init", "-b", "master", "repo")
47 cmd.Dir = dir
48
49 if err := cmd.Run(); err != nil {
50 t.Fatalf("cmd.Run: %v", err)
51 }
52
53 desc := zoekt.Repository{
54 Name: "repo",
55 }
56 opts := Options{
57 RepoDir: filepath.Join(dir, "repo", ".git"),
58 BuildOptions: index.Options{
59 RepositoryDescription: desc,
60 IndexDir: dir,
61 },
62 }
63
64 if _, err := IndexGitRepo(opts); err != nil {
65 t.Fatalf("IndexGitRepo: %v", err)
66 }
67}
68
69func TestIndexNonexistentRepo(t *testing.T) {
70 dir := t.TempDir()
71 desc := zoekt.Repository{
72 Name: "nonexistent",
73 }
74 opts := Options{
75 RepoDir: "does/not/exist",
76 Branches: []string{"main"},
77 BuildOptions: index.Options{
78 RepositoryDescription: desc,
79 IndexDir: dir,
80 },
81 }
82
83 if _, err := IndexGitRepo(opts); err == nil {
84 t.Fatal("expected error, got none")
85 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
86 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
87 }
88}
89
90func TestIndexTinyRepo(t *testing.T) {
91 // Create a repo with one file in it.
92 dir := t.TempDir()
93 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
94
95 repoDir := filepath.Join(dir, "repo")
96 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
97 t.Fatalf("WriteFile: %v", err)
98 }
99 executeCommand(t, repoDir, exec.Command("git", "add", "."))
100 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
101
102 // Test that indexing accepts both the repo directory, and the .git subdirectory.
103 for _, testDir := range []string{"repo", "repo/.git"} {
104 opts := Options{
105 RepoDir: filepath.Join(dir, testDir),
106 Branches: []string{"main"},
107 BuildOptions: index.Options{
108 RepositoryDescription: zoekt.Repository{Name: "repo"},
109 IndexDir: dir,
110 },
111 }
112
113 if _, err := IndexGitRepo(opts); err != nil {
114 t.Fatalf("unexpected error %v", err)
115 }
116
117 searcher, err := search.NewDirectorySearcher(dir)
118 if err != nil {
119 t.Fatal("NewDirectorySearcher", err)
120 }
121
122 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
123 searcher.Close()
124
125 if err != nil {
126 t.Fatal("search failed", err)
127 }
128
129 if len(results.Files) != 1 {
130 t.Fatalf("got search result %v, want 1 file", results.Files)
131 }
132 }
133}
134
135func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
136 cmd.Dir = dir
137 cmd.Env = []string{
138 "GIT_CONFIG_GLOBAL=",
139 "GIT_CONFIG_SYSTEM=",
140 "GIT_COMMITTER_NAME=Kierkegaard",
141 "GIT_COMMITTER_EMAIL=soren@apache.com",
142 "GIT_AUTHOR_NAME=Kierkegaard",
143 "GIT_AUTHOR_EMAIL=soren@apache.com",
144 }
145 if err := cmd.Run(); err != nil {
146 t.Fatalf("cmd.Run: %v", err)
147 }
148 return cmd
149}
150
151func TestIndexDeltaBasic(t *testing.T) {
152 type branchToDocumentMap map[string][]index.Document
153
154 type step struct {
155 name string
156 addedDocuments branchToDocumentMap
157 deletedDocuments branchToDocumentMap
158 optFn func(t *testing.T, options *Options)
159
160 expectedFallbackToNormalBuild bool
161 expectedDocuments []index.Document
162 }
163
164 helloWorld := index.Document{Name: "hello_world.txt", Content: []byte("hello")}
165
166 fruitV1 := index.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
167 fruitV1InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
168 fruitV1WithNewName := index.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
169
170 fruitV2 := index.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
171 fruitV2InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
172
173 fruitV3 := index.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
174 fruitV4 := index.Document{Name: "best_fruit.txt", Content: []byte("apples")}
175
176 foo := index.Document{Name: "foo.txt", Content: []byte("bar")}
177
178 emptySourcegraphIgnore := index.Document{Name: ignore.IgnoreFile}
179 sourcegraphIgnoreWithContent := index.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
180
181 for _, test := range []struct {
182 name string
183 branches []string
184 steps []step
185 }{
186 {
187 name: "modification",
188 branches: []string{"main"},
189 steps: []step{
190 {
191 name: "setup",
192 addedDocuments: branchToDocumentMap{
193 "main": []index.Document{helloWorld, fruitV1},
194 },
195
196 expectedDocuments: []index.Document{helloWorld, fruitV1},
197 },
198 {
199 name: "add newer version of fruits",
200 addedDocuments: branchToDocumentMap{
201 "main": []index.Document{fruitV2},
202 },
203 optFn: func(t *testing.T, o *Options) {
204 o.BuildOptions.IsDelta = true
205 },
206
207 expectedDocuments: []index.Document{helloWorld, fruitV2},
208 },
209 },
210 },
211 {
212 name: "modification only inside nested folder",
213 branches: []string{"main"},
214 steps: []step{
215 {
216 name: "setup",
217 addedDocuments: branchToDocumentMap{
218 "main": []index.Document{foo, fruitV1InFolder},
219 },
220
221 expectedDocuments: []index.Document{foo, fruitV1InFolder},
222 },
223 {
224 name: "add newer version of fruits inside folder",
225 addedDocuments: branchToDocumentMap{
226 "main": []index.Document{fruitV2InFolder},
227 },
228 optFn: func(t *testing.T, o *Options) {
229 o.BuildOptions.IsDelta = true
230 },
231
232 expectedDocuments: []index.Document{foo, fruitV2InFolder},
233 },
234 },
235 },
236 {
237 name: "addition",
238 branches: []string{"main"},
239 steps: []step{
240 {
241 name: "setup",
242 addedDocuments: branchToDocumentMap{
243 "main": []index.Document{helloWorld, fruitV1},
244 },
245
246 expectedDocuments: []index.Document{helloWorld, fruitV1},
247 },
248 {
249 name: "add new file - foo",
250 addedDocuments: branchToDocumentMap{
251 "main": []index.Document{foo},
252 },
253 optFn: func(t *testing.T, o *Options) {
254 o.BuildOptions.IsDelta = true
255 },
256
257 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
258 },
259 },
260 },
261 {
262 name: "deletion",
263 branches: []string{"main"},
264 steps: []step{
265 {
266 name: "setup",
267 addedDocuments: branchToDocumentMap{
268 "main": []index.Document{helloWorld, fruitV1, foo},
269 },
270
271 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
272 },
273 {
274 name: "delete foo file",
275 addedDocuments: nil,
276 deletedDocuments: branchToDocumentMap{
277 "main": []index.Document{foo},
278 },
279
280 optFn: func(t *testing.T, o *Options) {
281 o.BuildOptions.IsDelta = true
282 },
283
284 expectedDocuments: []index.Document{helloWorld, fruitV1},
285 },
286 },
287 },
288 {
289 name: "addition and deletion on only one branch",
290 branches: []string{"main", "release", "dev"},
291 steps: []step{
292 {
293 name: "setup",
294 addedDocuments: branchToDocumentMap{
295 "main": []index.Document{fruitV1},
296 "release": []index.Document{fruitV2},
297 "dev": []index.Document{fruitV3},
298 },
299
300 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
301 },
302 {
303 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
304 addedDocuments: branchToDocumentMap{
305 "dev": []index.Document{fruitV4},
306 },
307 deletedDocuments: branchToDocumentMap{
308 "main": []index.Document{fruitV1},
309 },
310
311 optFn: func(t *testing.T, o *Options) {
312 o.BuildOptions.IsDelta = true
313 },
314
315 expectedDocuments: []index.Document{fruitV2, fruitV4},
316 },
317 },
318 },
319 {
320 name: "rename",
321 branches: []string{"main", "release"},
322 steps: []step{
323 {
324 name: "setup",
325 addedDocuments: branchToDocumentMap{
326 "main": []index.Document{fruitV1},
327 "release": []index.Document{fruitV2},
328 },
329 expectedDocuments: []index.Document{fruitV1, fruitV2},
330 },
331 {
332 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
333 addedDocuments: branchToDocumentMap{
334 "main": []index.Document{fruitV1WithNewName},
335 },
336 deletedDocuments: branchToDocumentMap{
337 "main": []index.Document{fruitV1},
338 },
339
340 optFn: func(t *testing.T, o *Options) {
341 o.BuildOptions.IsDelta = true
342 },
343
344 expectedDocuments: []index.Document{fruitV1WithNewName, fruitV2},
345 },
346 },
347 },
348 {
349 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
350 branches: []string{"main", "dev"},
351 steps: []step{
352 {
353 name: "setup",
354 addedDocuments: branchToDocumentMap{
355 "main": []index.Document{fruitV1},
356 "dev": []index.Document{fruitV2},
357 },
358 expectedDocuments: []index.Document{fruitV1, fruitV2},
359 },
360 {
361 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
362 addedDocuments: branchToDocumentMap{
363 "main": []index.Document{fruitV2},
364 "dev": []index.Document{fruitV3},
365 },
366
367 optFn: func(t *testing.T, o *Options) {
368 o.BuildOptions.IsDelta = true
369 },
370
371 expectedDocuments: []index.Document{fruitV2, fruitV3},
372 },
373 },
374 },
375 {
376 name: "no-op delta builds (reindexing the same commits)",
377 branches: []string{"main", "dev"},
378 steps: []step{
379 {
380 name: "setup",
381 addedDocuments: branchToDocumentMap{
382 "main": []index.Document{fruitV1, foo},
383 "dev": []index.Document{helloWorld},
384 },
385 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
386 },
387 {
388 name: "first no-op (normal build -> delta build)",
389 optFn: func(t *testing.T, o *Options) {
390 o.BuildOptions.IsDelta = true
391 },
392
393 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
394 },
395 {
396 name: "second no-op (delta build -> delta build)",
397 optFn: func(t *testing.T, o *Options) {
398 o.BuildOptions.IsDelta = true
399 },
400
401 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
402 },
403 },
404 },
405 {
406 name: "should fallback to normal build if no prior shards exist",
407 branches: []string{"main"},
408 steps: []step{
409 {
410 name: "attempt delta build on a repository that hasn't been indexed yet",
411 addedDocuments: branchToDocumentMap{
412 "main": []index.Document{helloWorld},
413 },
414 optFn: func(t *testing.T, o *Options) {
415 o.BuildOptions.IsDelta = true
416 },
417
418 expectedFallbackToNormalBuild: true,
419 expectedDocuments: []index.Document{helloWorld},
420 },
421 },
422 },
423 {
424 name: "should fallback to normal build if the set of requested repository branches changes",
425 branches: []string{"main", "release", "dev"},
426 steps: []step{
427 {
428 name: "setup",
429 addedDocuments: branchToDocumentMap{
430 "main": []index.Document{fruitV1},
431 "release": []index.Document{fruitV2},
432 "dev": []index.Document{fruitV3},
433 },
434
435 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
436 },
437 {
438 name: "try delta build after dropping 'main' branch from index ",
439 addedDocuments: branchToDocumentMap{
440 "release": []index.Document{fruitV4},
441 },
442 optFn: func(t *testing.T, o *Options) {
443 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
444 o.BuildOptions.IsDelta = true
445 },
446
447 expectedFallbackToNormalBuild: true,
448 expectedDocuments: []index.Document{fruitV3, fruitV4},
449 },
450 },
451 },
452 {
453 name: "should expand branches correctly when using wildcards in branch names",
454 branches: []string{"release/1", "release/2"},
455 steps: []step{
456 {
457 name: "setup",
458 addedDocuments: branchToDocumentMap{
459 "release/1": []index.Document{fruitV1},
460 "release/2": []index.Document{fruitV2},
461 },
462
463 expectedDocuments: []index.Document{fruitV1, fruitV2},
464 },
465 {
466 name: "try delta build with wildcard in branches",
467 optFn: func(t *testing.T, o *Options) {
468 // use a wildcard here
469 o.Branches = []string{"HEAD", "release/*"}
470 o.BuildOptions.IsDelta = true
471 },
472
473 expectedDocuments: []index.Document{fruitV1, fruitV2},
474 },
475 },
476 },
477 {
478 name: "should fallback to normal build if one or more index options updates requires a full build",
479 branches: []string{"main"},
480 steps: []step{
481 {
482 name: "setup",
483 addedDocuments: branchToDocumentMap{
484 "main": []index.Document{fruitV1},
485 },
486
487 expectedDocuments: []index.Document{fruitV1},
488 },
489 {
490 name: "try delta build after updating Disable CTags index option",
491 addedDocuments: branchToDocumentMap{
492 "main": []index.Document{fruitV2},
493 },
494 optFn: func(t *testing.T, o *Options) {
495 o.BuildOptions.IsDelta = true
496 o.BuildOptions.DisableCTags = true
497 },
498
499 expectedFallbackToNormalBuild: true,
500 expectedDocuments: []index.Document{fruitV2},
501 },
502 {
503 name: "try delta build after reverting Disable CTags index option",
504 addedDocuments: branchToDocumentMap{
505 "main": []index.Document{fruitV3},
506 },
507 optFn: func(t *testing.T, o *Options) {
508 o.BuildOptions.IsDelta = true
509 o.BuildOptions.DisableCTags = false
510 },
511
512 expectedFallbackToNormalBuild: true,
513 expectedDocuments: []index.Document{fruitV3},
514 },
515 },
516 },
517 {
518 name: "should successfully perform multiple delta builds after disabling symbols",
519 branches: []string{"main"},
520 steps: []step{
521 {
522 name: "setup",
523 addedDocuments: branchToDocumentMap{
524 "main": []index.Document{fruitV1},
525 },
526
527 expectedDocuments: []index.Document{fruitV1},
528 },
529 {
530 name: "try delta build after updating Disable CTags index option",
531 addedDocuments: branchToDocumentMap{
532 "main": []index.Document{fruitV2},
533 },
534 optFn: func(t *testing.T, o *Options) {
535 o.BuildOptions.IsDelta = true
536 o.BuildOptions.DisableCTags = true
537 },
538
539 expectedFallbackToNormalBuild: true,
540 expectedDocuments: []index.Document{fruitV2},
541 },
542 {
543 name: "try another delta build while CTags is still disabled",
544 addedDocuments: branchToDocumentMap{
545 "main": []index.Document{fruitV3},
546 },
547 optFn: func(t *testing.T, o *Options) {
548 o.BuildOptions.IsDelta = true
549 o.BuildOptions.DisableCTags = true
550 },
551
552 expectedDocuments: []index.Document{fruitV3},
553 },
554 },
555 },
556 {
557 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
558 branches: []string{"main"},
559 steps: []step{
560 {
561 name: "setup",
562 addedDocuments: branchToDocumentMap{
563 "main": []index.Document{emptySourcegraphIgnore},
564 },
565
566 expectedDocuments: []index.Document{emptySourcegraphIgnore},
567 },
568 {
569 name: "attempt delta build after modifying ignore file",
570 addedDocuments: branchToDocumentMap{
571 "main": []index.Document{sourcegraphIgnoreWithContent},
572 },
573 optFn: func(t *testing.T, o *Options) {
574 o.BuildOptions.IsDelta = true
575 },
576
577 expectedFallbackToNormalBuild: true,
578 expectedDocuments: []index.Document{sourcegraphIgnoreWithContent},
579 },
580 },
581 },
582 {
583 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
584 branches: []string{"main"},
585 steps: []step{
586 {
587 name: "setup: first shard",
588 addedDocuments: branchToDocumentMap{
589 "main": []index.Document{foo},
590 },
591
592 expectedDocuments: []index.Document{foo},
593 },
594 {
595 name: "setup: second shard (delta)",
596 addedDocuments: branchToDocumentMap{
597 "main": []index.Document{fruitV1},
598 },
599 optFn: func(t *testing.T, o *Options) {
600 o.BuildOptions.IsDelta = true
601 },
602
603 expectedDocuments: []index.Document{foo, fruitV1},
604 },
605 {
606 name: "setup: third shard (delta)",
607 addedDocuments: branchToDocumentMap{
608 "main": []index.Document{helloWorld},
609 },
610 optFn: func(t *testing.T, o *Options) {
611 o.BuildOptions.IsDelta = true
612 },
613
614 expectedDocuments: []index.Document{foo, fruitV1, helloWorld},
615 },
616 {
617 name: "attempt another delta build after we already blew past the shard threshold",
618 addedDocuments: branchToDocumentMap{
619 "main": []index.Document{fruitV2InFolder},
620 },
621 optFn: func(t *testing.T, o *Options) {
622 o.DeltaShardNumberFallbackThreshold = 2
623 o.BuildOptions.IsDelta = true
624 },
625
626 expectedFallbackToNormalBuild: true,
627 expectedDocuments: []index.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
628 },
629 },
630 },
631 } {
632 test := test
633
634 t.Run(test.name, func(t *testing.T) {
635 t.Parallel()
636
637 indexDir := t.TempDir()
638 repositoryDir := t.TempDir()
639
640 // setup: initialize the repository and all of its branches
641 runScript(t, repositoryDir, "git init -b master")
642 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
643 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
644
645 for _, b := range test.branches {
646 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
647 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
648 }
649
650 for _, step := range test.steps {
651 t.Run(step.name, func(t *testing.T) {
652 for _, b := range test.branches {
653 // setup: for each branch, process any document deletions / additions and commit those changes
654
655 hadChange := false
656
657 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
658
659 for _, d := range step.deletedDocuments[b] {
660 hadChange = true
661
662 file := filepath.Join(repositoryDir, d.Name)
663
664 err := os.Remove(file)
665 if err != nil {
666 t.Fatalf("deleting file %q: %s", d.Name, err)
667 }
668
669 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
670 }
671
672 for _, d := range step.addedDocuments[b] {
673 hadChange = true
674
675 file := filepath.Join(repositoryDir, d.Name)
676
677 err := os.MkdirAll(filepath.Dir(file), 0o755)
678 if err != nil {
679 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
680 }
681
682 err = os.WriteFile(file, d.Content, 0o644)
683 if err != nil {
684 t.Fatalf("writing file %q: %s", d.Name, err)
685 }
686
687 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
688 }
689
690 if !hadChange {
691 continue
692 }
693
694 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
695 }
696
697 // setup: prepare indexOptions with given overrides
698 buildOptions := index.Options{
699 IndexDir: indexDir,
700 RepositoryDescription: zoekt.Repository{
701 Name: "repository",
702 },
703 IsDelta: false,
704 }
705 buildOptions.SetDefaults()
706
707 branches := append([]string{"HEAD"}, test.branches...)
708
709 options := Options{
710 RepoDir: filepath.Join(repositoryDir, ".git"),
711 BuildOptions: buildOptions,
712 Branches: branches,
713 }
714
715 if step.optFn != nil {
716 step.optFn(t, &options)
717 }
718
719 // setup: prepare spy versions of prepare delta / normal build so that we can observe
720 // whether they were called appropriately
721 deltaBuildCalled := false
722 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
723 deltaBuildCalled = true
724 return prepareDeltaBuild(options, repository)
725 }
726
727 normalBuildCalled := false
728 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
729 normalBuildCalled = true
730 return prepareNormalBuild(options, repository)
731 }
732
733 // run test
734 _, err := indexGitRepo(options, gitIndexConfig{
735 prepareDeltaBuild: prepareDeltaSpy,
736 prepareNormalBuild: prepareNormalSpy,
737 })
738 if err != nil {
739 t.Fatalf("IndexGitRepo: %s", err)
740 }
741
742 if options.BuildOptions.IsDelta != deltaBuildCalled {
743 // We should always try a delta build if we request it in the options.
744 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
745 }
746
747 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
748 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
749 // asked for a delta build in the first place.
750 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
751 }
752
753 // examine outcome: load shards into a searcher instance and run a dummy search query
754 // that returns every document contained in the shards
755 //
756 // then, compare returned set of documents with the expected set for the step and see if they agree
757
758 ss, err := search.NewDirectorySearcher(indexDir)
759 if err != nil {
760 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
761 }
762 defer ss.Close()
763
764 searchOpts := &zoekt.SearchOptions{Whole: true}
765 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
766 if err != nil {
767 t.Fatalf("Search: %s", err)
768 }
769
770 var receivedDocuments []index.Document
771 for _, f := range result.Files {
772 receivedDocuments = append(receivedDocuments, index.Document{
773 Name: f.FileName,
774 Content: f.Content,
775 })
776 }
777
778 for _, docs := range [][]index.Document{step.expectedDocuments, receivedDocuments} {
779 sort.Slice(docs, func(i, j int) bool {
780 a, b := docs[i], docs[j]
781
782 // first compare names, then fallback to contents if the names are equal
783
784 if a.Name < b.Name {
785 return true
786 }
787
788 if a.Name > b.Name {
789 return false
790 }
791
792 return bytes.Compare(a.Content, b.Content) < 0
793 })
794 }
795
796 compareOptions := []cmp.Option{
797 cmpopts.IgnoreFields(index.Document{}, "Branches"),
798 cmpopts.EquateEmpty(),
799 }
800
801 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
802 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
803 }
804 })
805 }
806 })
807 }
808}
809
810func runScript(t *testing.T, cwd string, script string) {
811 t.Helper()
812
813 err := os.MkdirAll(cwd, 0o755)
814 if err != nil {
815 t.Fatalf("ensuring path %q exists: %s", cwd, err)
816 }
817
818 cmd := exec.Command("sh", "-euxc", script)
819 cmd.Dir = cwd
820 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
821
822 if out, err := cmd.CombinedOutput(); err != nil {
823 t.Fatalf("execution error: %v, output %s", err, out)
824 }
825}
826
827func TestSetTemplates_e2e(t *testing.T) {
828 repositoryDir := t.TempDir()
829
830 // setup: initialize the repository and all of its branches
831 runScript(t, repositoryDir, "git init -b master")
832 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
833 desc := zoekt.Repository{}
834 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
835 t.Fatalf("setTemplatesFromConfig: %v", err)
836 }
837
838 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
839 t.Errorf("got %q, want %q", got, want)
840 }
841}
842
843func TestSetTemplates(t *testing.T) {
844 base := "https://example.com/repo/name"
845 version := "VERSION"
846 path := "dir/name.txt"
847 lineNumber := 10
848 cases := []struct {
849 typ string
850 commit string
851 file string
852 line string
853 }{{
854 typ: "gitiles",
855 commit: "https://example.com/repo/name/%2B/VERSION",
856 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
857 line: "#10",
858 }, {
859 typ: "github",
860 commit: "https://example.com/repo/name/commit/VERSION",
861 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
862 line: "#L10",
863 }, {
864 typ: "cgit",
865 commit: "https://example.com/repo/name/commit/?id=VERSION",
866 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
867 line: "#n10",
868 }, {
869 typ: "gitweb",
870 commit: "https://example.com/repo/name;a=commit;h=VERSION",
871 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
872 line: "#l10",
873 }, {
874 typ: "source.bazel.build",
875 commit: "https://example.com/repo/name/%2B/VERSION",
876 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
877 line: ";l=10",
878 }, {
879 typ: "bitbucket-server",
880 commit: "https://example.com/repo/name/commits/VERSION",
881 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
882 line: "#10",
883 }, {
884 typ: "gitlab",
885 commit: "https://example.com/repo/name/-/commit/VERSION",
886 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
887 line: "#L10",
888 }, {
889 typ: "gitea",
890 commit: "https://example.com/repo/name/commit/VERSION",
891 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
892 line: "#L10",
893 }}
894
895 for _, tc := range cases {
896 t.Run(tc.typ, func(t *testing.T) {
897 assertOutput := func(templateText string, want string) {
898 t.Helper()
899
900 tt, err := index.ParseTemplate(templateText)
901 if err != nil {
902 t.Fatal(err)
903 }
904
905 var sb strings.Builder
906 err = tt.Execute(&sb, map[string]any{
907 "Version": version,
908 "Path": path,
909 "LineNumber": lineNumber,
910 })
911 if err != nil {
912 t.Fatal(err)
913 }
914 if got := sb.String(); got != want {
915 t.Fatalf("want: %q\ngot: %q", want, got)
916 }
917 }
918
919 var repo zoekt.Repository
920 u, _ := url.Parse(base)
921 err := setTemplates(&repo, u, tc.typ)
922 if err != nil {
923 t.Fatal(err)
924 }
925 assertOutput(repo.CommitURLTemplate, tc.commit)
926 assertOutput(repo.FileURLTemplate, tc.file)
927 assertOutput(repo.LineFragmentTemplate, tc.line)
928 })
929 }
930}
931
932func BenchmarkPrepareNormalBuild(b *testing.B) {
933 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
934 repoDir := "/path/to/your/repo"
935 repo, err := git.PlainOpen(repoDir)
936 if err != nil {
937 b.Fatalf("Failed to open test repository: %v", err)
938 }
939
940 opts := Options{
941 RepoDir: repoDir,
942 Submodules: false,
943 BranchPrefix: "refs/heads/",
944 Branches: []string{"main"},
945 BuildOptions: index.Options{
946 RepositoryDescription: zoekt.Repository{
947 Name: "test-repo",
948 URL: "https://github.com/example/test-repo",
949 },
950 },
951 }
952
953 b.ReportAllocs()
954
955 repos, branchVersions, err := prepareNormalBuild(opts, repo)
956 if err != nil {
957 b.Fatalf("prepareNormalBuild failed: %v", err)
958 }
959
960 runtime.GC()
961
962 var m runtime.MemStats
963 runtime.ReadMemStats(&m)
964 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
965 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
966
967 if len(repos) == 0 || len(branchVersions) == 0 {
968 b.Fatalf("Unexpected empty results")
969 }
970}