fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35 "github.com/sourcegraph/zoekt"
36 "github.com/sourcegraph/zoekt/ignore"
37 "github.com/sourcegraph/zoekt/index"
38 "github.com/sourcegraph/zoekt/query"
39 "github.com/sourcegraph/zoekt/search"
40)
41
42func TestIndexEmptyRepo(t *testing.T) {
43 dir := t.TempDir()
44
45 cmd := exec.Command("git", "init", "-b", "master", "repo")
46 cmd.Dir = dir
47
48 if err := cmd.Run(); err != nil {
49 t.Fatalf("cmd.Run: %v", err)
50 }
51
52 desc := zoekt.Repository{
53 Name: "repo",
54 }
55 opts := Options{
56 RepoDir: filepath.Join(dir, "repo", ".git"),
57 BuildOptions: index.Options{
58 RepositoryDescription: desc,
59 IndexDir: dir,
60 },
61 }
62
63 if _, err := IndexGitRepo(opts); err != nil {
64 t.Fatalf("IndexGitRepo: %v", err)
65 }
66}
67
68func TestIndexNonexistentRepo(t *testing.T) {
69 dir := t.TempDir()
70 desc := zoekt.Repository{
71 Name: "nonexistent",
72 }
73 opts := Options{
74 RepoDir: "does/not/exist",
75 Branches: []string{"main"},
76 BuildOptions: index.Options{
77 RepositoryDescription: desc,
78 IndexDir: dir,
79 },
80 }
81
82 if _, err := IndexGitRepo(opts); err == nil {
83 t.Fatal("expected error, got none")
84 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
85 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
86 }
87}
88
89func TestIndexTinyRepo(t *testing.T) {
90 // Create a repo with one file in it.
91 dir := t.TempDir()
92 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
93
94 repoDir := filepath.Join(dir, "repo")
95 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
96 t.Fatalf("WriteFile: %v", err)
97 }
98 executeCommand(t, repoDir, exec.Command("git", "add", "."))
99 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
100
101 // Test that indexing accepts both the repo directory, and the .git subdirectory.
102 for _, testDir := range []string{"repo", "repo/.git"} {
103 opts := Options{
104 RepoDir: filepath.Join(dir, testDir),
105 Branches: []string{"main"},
106 BuildOptions: index.Options{
107 RepositoryDescription: zoekt.Repository{Name: "repo"},
108 IndexDir: dir,
109 },
110 }
111
112 if _, err := IndexGitRepo(opts); err != nil {
113 t.Fatalf("unexpected error %v", err)
114 }
115
116 searcher, err := search.NewDirectorySearcher(dir)
117 if err != nil {
118 t.Fatal("NewDirectorySearcher", err)
119 }
120
121 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
122 searcher.Close()
123
124 if err != nil {
125 t.Fatal("search failed", err)
126 }
127
128 if len(results.Files) != 1 {
129 t.Fatalf("got search result %v, want 1 file", results.Files)
130 }
131 }
132}
133
134func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
135 cmd.Dir = dir
136 cmd.Env = []string{
137 "GIT_CONFIG_GLOBAL=",
138 "GIT_CONFIG_SYSTEM=",
139 "GIT_COMMITTER_NAME=Kierkegaard",
140 "GIT_COMMITTER_EMAIL=soren@apache.com",
141 "GIT_AUTHOR_NAME=Kierkegaard",
142 "GIT_AUTHOR_EMAIL=soren@apache.com",
143 }
144 if err := cmd.Run(); err != nil {
145 t.Fatalf("cmd.Run: %v", err)
146 }
147 return cmd
148}
149
150func TestIndexDeltaBasic(t *testing.T) {
151 type branchToDocumentMap map[string][]index.Document
152
153 type step struct {
154 name string
155 addedDocuments branchToDocumentMap
156 deletedDocuments branchToDocumentMap
157 optFn func(t *testing.T, options *Options)
158
159 expectedFallbackToNormalBuild bool
160 expectedDocuments []index.Document
161 }
162
163 helloWorld := index.Document{Name: "hello_world.txt", Content: []byte("hello")}
164
165 fruitV1 := index.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
166 fruitV1InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
167 fruitV1WithNewName := index.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
168
169 fruitV2 := index.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
170 fruitV2InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
171
172 fruitV3 := index.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
173 fruitV4 := index.Document{Name: "best_fruit.txt", Content: []byte("apples")}
174
175 foo := index.Document{Name: "foo.txt", Content: []byte("bar")}
176
177 emptySourcegraphIgnore := index.Document{Name: ignore.IgnoreFile}
178 sourcegraphIgnoreWithContent := index.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
179
180 for _, test := range []struct {
181 name string
182 branches []string
183 steps []step
184 }{
185 {
186 name: "modification",
187 branches: []string{"main"},
188 steps: []step{
189 {
190 name: "setup",
191 addedDocuments: branchToDocumentMap{
192 "main": []index.Document{helloWorld, fruitV1},
193 },
194
195 expectedDocuments: []index.Document{helloWorld, fruitV1},
196 },
197 {
198 name: "add newer version of fruits",
199 addedDocuments: branchToDocumentMap{
200 "main": []index.Document{fruitV2},
201 },
202 optFn: func(t *testing.T, o *Options) {
203 o.BuildOptions.IsDelta = true
204 },
205
206 expectedDocuments: []index.Document{helloWorld, fruitV2},
207 },
208 },
209 },
210 {
211 name: "modification only inside nested folder",
212 branches: []string{"main"},
213 steps: []step{
214 {
215 name: "setup",
216 addedDocuments: branchToDocumentMap{
217 "main": []index.Document{foo, fruitV1InFolder},
218 },
219
220 expectedDocuments: []index.Document{foo, fruitV1InFolder},
221 },
222 {
223 name: "add newer version of fruits inside folder",
224 addedDocuments: branchToDocumentMap{
225 "main": []index.Document{fruitV2InFolder},
226 },
227 optFn: func(t *testing.T, o *Options) {
228 o.BuildOptions.IsDelta = true
229 },
230
231 expectedDocuments: []index.Document{foo, fruitV2InFolder},
232 },
233 },
234 },
235 {
236 name: "addition",
237 branches: []string{"main"},
238 steps: []step{
239 {
240 name: "setup",
241 addedDocuments: branchToDocumentMap{
242 "main": []index.Document{helloWorld, fruitV1},
243 },
244
245 expectedDocuments: []index.Document{helloWorld, fruitV1},
246 },
247 {
248 name: "add new file - foo",
249 addedDocuments: branchToDocumentMap{
250 "main": []index.Document{foo},
251 },
252 optFn: func(t *testing.T, o *Options) {
253 o.BuildOptions.IsDelta = true
254 },
255
256 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
257 },
258 },
259 },
260 {
261 name: "deletion",
262 branches: []string{"main"},
263 steps: []step{
264 {
265 name: "setup",
266 addedDocuments: branchToDocumentMap{
267 "main": []index.Document{helloWorld, fruitV1, foo},
268 },
269
270 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
271 },
272 {
273 name: "delete foo file",
274 addedDocuments: nil,
275 deletedDocuments: branchToDocumentMap{
276 "main": []index.Document{foo},
277 },
278
279 optFn: func(t *testing.T, o *Options) {
280 o.BuildOptions.IsDelta = true
281 },
282
283 expectedDocuments: []index.Document{helloWorld, fruitV1},
284 },
285 },
286 },
287 {
288 name: "addition and deletion on only one branch",
289 branches: []string{"main", "release", "dev"},
290 steps: []step{
291 {
292 name: "setup",
293 addedDocuments: branchToDocumentMap{
294 "main": []index.Document{fruitV1},
295 "release": []index.Document{fruitV2},
296 "dev": []index.Document{fruitV3},
297 },
298
299 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
300 },
301 {
302 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
303 addedDocuments: branchToDocumentMap{
304 "dev": []index.Document{fruitV4},
305 },
306 deletedDocuments: branchToDocumentMap{
307 "main": []index.Document{fruitV1},
308 },
309
310 optFn: func(t *testing.T, o *Options) {
311 o.BuildOptions.IsDelta = true
312 },
313
314 expectedDocuments: []index.Document{fruitV2, fruitV4},
315 },
316 },
317 },
318 {
319 name: "rename",
320 branches: []string{"main", "release"},
321 steps: []step{
322 {
323 name: "setup",
324 addedDocuments: branchToDocumentMap{
325 "main": []index.Document{fruitV1},
326 "release": []index.Document{fruitV2},
327 },
328 expectedDocuments: []index.Document{fruitV1, fruitV2},
329 },
330 {
331 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
332 addedDocuments: branchToDocumentMap{
333 "main": []index.Document{fruitV1WithNewName},
334 },
335 deletedDocuments: branchToDocumentMap{
336 "main": []index.Document{fruitV1},
337 },
338
339 optFn: func(t *testing.T, o *Options) {
340 o.BuildOptions.IsDelta = true
341 },
342
343 expectedDocuments: []index.Document{fruitV1WithNewName, fruitV2},
344 },
345 },
346 },
347 {
348 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
349 branches: []string{"main", "dev"},
350 steps: []step{
351 {
352 name: "setup",
353 addedDocuments: branchToDocumentMap{
354 "main": []index.Document{fruitV1},
355 "dev": []index.Document{fruitV2},
356 },
357 expectedDocuments: []index.Document{fruitV1, fruitV2},
358 },
359 {
360 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
361 addedDocuments: branchToDocumentMap{
362 "main": []index.Document{fruitV2},
363 "dev": []index.Document{fruitV3},
364 },
365
366 optFn: func(t *testing.T, o *Options) {
367 o.BuildOptions.IsDelta = true
368 },
369
370 expectedDocuments: []index.Document{fruitV2, fruitV3},
371 },
372 },
373 },
374 {
375 name: "no-op delta builds (reindexing the same commits)",
376 branches: []string{"main", "dev"},
377 steps: []step{
378 {
379 name: "setup",
380 addedDocuments: branchToDocumentMap{
381 "main": []index.Document{fruitV1, foo},
382 "dev": []index.Document{helloWorld},
383 },
384 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
385 },
386 {
387 name: "first no-op (normal build -> delta build)",
388 optFn: func(t *testing.T, o *Options) {
389 o.BuildOptions.IsDelta = true
390 },
391
392 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
393 },
394 {
395 name: "second no-op (delta build -> delta build)",
396 optFn: func(t *testing.T, o *Options) {
397 o.BuildOptions.IsDelta = true
398 },
399
400 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
401 },
402 },
403 },
404 {
405 name: "should fallback to normal build if no prior shards exist",
406 branches: []string{"main"},
407 steps: []step{
408 {
409 name: "attempt delta build on a repository that hasn't been indexed yet",
410 addedDocuments: branchToDocumentMap{
411 "main": []index.Document{helloWorld},
412 },
413 optFn: func(t *testing.T, o *Options) {
414 o.BuildOptions.IsDelta = true
415 },
416
417 expectedFallbackToNormalBuild: true,
418 expectedDocuments: []index.Document{helloWorld},
419 },
420 },
421 },
422 {
423 name: "should fallback to normal build if the set of requested repository branches changes",
424 branches: []string{"main", "release", "dev"},
425 steps: []step{
426 {
427 name: "setup",
428 addedDocuments: branchToDocumentMap{
429 "main": []index.Document{fruitV1},
430 "release": []index.Document{fruitV2},
431 "dev": []index.Document{fruitV3},
432 },
433
434 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
435 },
436 {
437 name: "try delta build after dropping 'main' branch from index ",
438 addedDocuments: branchToDocumentMap{
439 "release": []index.Document{fruitV4},
440 },
441 optFn: func(t *testing.T, o *Options) {
442 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
443 o.BuildOptions.IsDelta = true
444 },
445
446 expectedFallbackToNormalBuild: true,
447 expectedDocuments: []index.Document{fruitV3, fruitV4},
448 },
449 },
450 },
451 {
452 name: "should fallback to normal build if one or more index options updates requires a full build",
453 branches: []string{"main"},
454 steps: []step{
455 {
456 name: "setup",
457 addedDocuments: branchToDocumentMap{
458 "main": []index.Document{fruitV1},
459 },
460
461 expectedDocuments: []index.Document{fruitV1},
462 },
463 {
464 name: "try delta build after updating Disable CTags index option",
465 addedDocuments: branchToDocumentMap{
466 "main": []index.Document{fruitV2},
467 },
468 optFn: func(t *testing.T, o *Options) {
469 o.BuildOptions.IsDelta = true
470 o.BuildOptions.DisableCTags = true
471 },
472
473 expectedFallbackToNormalBuild: true,
474 expectedDocuments: []index.Document{fruitV2},
475 },
476 {
477 name: "try delta build after reverting Disable CTags index option",
478 addedDocuments: branchToDocumentMap{
479 "main": []index.Document{fruitV3},
480 },
481 optFn: func(t *testing.T, o *Options) {
482 o.BuildOptions.IsDelta = true
483 o.BuildOptions.DisableCTags = false
484 },
485
486 expectedFallbackToNormalBuild: true,
487 expectedDocuments: []index.Document{fruitV3},
488 },
489 },
490 },
491 {
492 name: "should successfully perform multiple delta builds after disabling symbols",
493 branches: []string{"main"},
494 steps: []step{
495 {
496 name: "setup",
497 addedDocuments: branchToDocumentMap{
498 "main": []index.Document{fruitV1},
499 },
500
501 expectedDocuments: []index.Document{fruitV1},
502 },
503 {
504 name: "try delta build after updating Disable CTags index option",
505 addedDocuments: branchToDocumentMap{
506 "main": []index.Document{fruitV2},
507 },
508 optFn: func(t *testing.T, o *Options) {
509 o.BuildOptions.IsDelta = true
510 o.BuildOptions.DisableCTags = true
511 },
512
513 expectedFallbackToNormalBuild: true,
514 expectedDocuments: []index.Document{fruitV2},
515 },
516 {
517 name: "try another delta build while CTags is still disabled",
518 addedDocuments: branchToDocumentMap{
519 "main": []index.Document{fruitV3},
520 },
521 optFn: func(t *testing.T, o *Options) {
522 o.BuildOptions.IsDelta = true
523 o.BuildOptions.DisableCTags = true
524 },
525
526 expectedDocuments: []index.Document{fruitV3},
527 },
528 },
529 },
530 {
531 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
532 branches: []string{"main"},
533 steps: []step{
534 {
535 name: "setup",
536 addedDocuments: branchToDocumentMap{
537 "main": []index.Document{emptySourcegraphIgnore},
538 },
539
540 expectedDocuments: []index.Document{emptySourcegraphIgnore},
541 },
542 {
543 name: "attempt delta build after modifying ignore file",
544 addedDocuments: branchToDocumentMap{
545 "main": []index.Document{sourcegraphIgnoreWithContent},
546 },
547 optFn: func(t *testing.T, o *Options) {
548 o.BuildOptions.IsDelta = true
549 },
550
551 expectedFallbackToNormalBuild: true,
552 expectedDocuments: []index.Document{sourcegraphIgnoreWithContent},
553 },
554 },
555 },
556 {
557 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
558 branches: []string{"main"},
559 steps: []step{
560 {
561 name: "setup: first shard",
562 addedDocuments: branchToDocumentMap{
563 "main": []index.Document{foo},
564 },
565
566 expectedDocuments: []index.Document{foo},
567 },
568 {
569 name: "setup: second shard (delta)",
570 addedDocuments: branchToDocumentMap{
571 "main": []index.Document{fruitV1},
572 },
573 optFn: func(t *testing.T, o *Options) {
574 o.BuildOptions.IsDelta = true
575 },
576
577 expectedDocuments: []index.Document{foo, fruitV1},
578 },
579 {
580 name: "setup: third shard (delta)",
581 addedDocuments: branchToDocumentMap{
582 "main": []index.Document{helloWorld},
583 },
584 optFn: func(t *testing.T, o *Options) {
585 o.BuildOptions.IsDelta = true
586 },
587
588 expectedDocuments: []index.Document{foo, fruitV1, helloWorld},
589 },
590 {
591 name: "attempt another delta build after we already blew past the shard threshold",
592 addedDocuments: branchToDocumentMap{
593 "main": []index.Document{fruitV2InFolder},
594 },
595 optFn: func(t *testing.T, o *Options) {
596 o.DeltaShardNumberFallbackThreshold = 2
597 o.BuildOptions.IsDelta = true
598 },
599
600 expectedFallbackToNormalBuild: true,
601 expectedDocuments: []index.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
602 },
603 },
604 },
605 } {
606 test := test
607
608 t.Run(test.name, func(t *testing.T) {
609 t.Parallel()
610
611 indexDir := t.TempDir()
612 repositoryDir := t.TempDir()
613
614 // setup: initialize the repository and all of its branches
615 runScript(t, repositoryDir, "git init -b master")
616 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
617 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
618
619 for _, b := range test.branches {
620 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
621 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
622 }
623
624 for _, step := range test.steps {
625 t.Run(step.name, func(t *testing.T) {
626 for _, b := range test.branches {
627 // setup: for each branch, process any document deletions / additions and commit those changes
628
629 hadChange := false
630
631 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
632
633 for _, d := range step.deletedDocuments[b] {
634 hadChange = true
635
636 file := filepath.Join(repositoryDir, d.Name)
637
638 err := os.Remove(file)
639 if err != nil {
640 t.Fatalf("deleting file %q: %s", d.Name, err)
641 }
642
643 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
644 }
645
646 for _, d := range step.addedDocuments[b] {
647 hadChange = true
648
649 file := filepath.Join(repositoryDir, d.Name)
650
651 err := os.MkdirAll(filepath.Dir(file), 0o755)
652 if err != nil {
653 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
654 }
655
656 err = os.WriteFile(file, d.Content, 0o644)
657 if err != nil {
658 t.Fatalf("writing file %q: %s", d.Name, err)
659 }
660
661 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
662 }
663
664 if !hadChange {
665 continue
666 }
667
668 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
669 }
670
671 // setup: prepare indexOptions with given overrides
672 buildOptions := index.Options{
673 IndexDir: indexDir,
674 RepositoryDescription: zoekt.Repository{
675 Name: "repository",
676 },
677 IsDelta: false,
678 }
679 buildOptions.SetDefaults()
680
681 branches := append([]string{"HEAD"}, test.branches...)
682
683 options := Options{
684 RepoDir: filepath.Join(repositoryDir, ".git"),
685 BuildOptions: buildOptions,
686 Branches: branches,
687 }
688
689 if step.optFn != nil {
690 step.optFn(t, &options)
691 }
692
693 // setup: prepare spy versions of prepare delta / normal build so that we can observe
694 // whether they were called appropriately
695 deltaBuildCalled := false
696 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
697 deltaBuildCalled = true
698 return prepareDeltaBuild(options, repository)
699 }
700
701 normalBuildCalled := false
702 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
703 normalBuildCalled = true
704 return prepareNormalBuild(options, repository)
705 }
706
707 // run test
708 _, err := indexGitRepo(options, gitIndexConfig{
709 prepareDeltaBuild: prepareDeltaSpy,
710 prepareNormalBuild: prepareNormalSpy,
711 })
712 if err != nil {
713 t.Fatalf("IndexGitRepo: %s", err)
714 }
715
716 if options.BuildOptions.IsDelta != deltaBuildCalled {
717 // We should always try a delta build if we request it in the options.
718 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
719 }
720
721 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
722 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
723 // asked for a delta build in the first place.
724 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
725 }
726
727 // examine outcome: load shards into a searcher instance and run a dummy search query
728 // that returns every document contained in the shards
729 //
730 // then, compare returned set of documents with the expected set for the step and see if they agree
731
732 ss, err := search.NewDirectorySearcher(indexDir)
733 if err != nil {
734 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
735 }
736 defer ss.Close()
737
738 searchOpts := &zoekt.SearchOptions{Whole: true}
739 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
740 if err != nil {
741 t.Fatalf("Search: %s", err)
742 }
743
744 var receivedDocuments []index.Document
745 for _, f := range result.Files {
746 receivedDocuments = append(receivedDocuments, index.Document{
747 Name: f.FileName,
748 Content: f.Content,
749 })
750 }
751
752 for _, docs := range [][]index.Document{step.expectedDocuments, receivedDocuments} {
753 sort.Slice(docs, func(i, j int) bool {
754 a, b := docs[i], docs[j]
755
756 // first compare names, then fallback to contents if the names are equal
757
758 if a.Name < b.Name {
759 return true
760 }
761
762 if a.Name > b.Name {
763 return false
764 }
765
766 return bytes.Compare(a.Content, b.Content) < 0
767 })
768 }
769
770 compareOptions := []cmp.Option{
771 cmpopts.IgnoreFields(index.Document{}, "Branches"),
772 cmpopts.EquateEmpty(),
773 }
774
775 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
776 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
777 }
778 })
779 }
780 })
781 }
782}
783
784func runScript(t *testing.T, cwd string, script string) {
785 t.Helper()
786
787 err := os.MkdirAll(cwd, 0o755)
788 if err != nil {
789 t.Fatalf("ensuring path %q exists: %s", cwd, err)
790 }
791
792 cmd := exec.Command("sh", "-euxc", script)
793 cmd.Dir = cwd
794 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
795
796 if out, err := cmd.CombinedOutput(); err != nil {
797 t.Fatalf("execution error: %v, output %s", err, out)
798 }
799}
800
801func TestSetTemplates_e2e(t *testing.T) {
802 repositoryDir := t.TempDir()
803
804 // setup: initialize the repository and all of its branches
805 runScript(t, repositoryDir, "git init -b master")
806 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
807 desc := zoekt.Repository{}
808 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
809 t.Fatalf("setTemplatesFromConfig: %v", err)
810 }
811
812 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
813 t.Errorf("got %q, want %q", got, want)
814 }
815}
816
817func TestSetTemplates(t *testing.T) {
818 base := "https://example.com/repo/name"
819 version := "VERSION"
820 path := "dir/name.txt"
821 lineNumber := 10
822 cases := []struct {
823 typ string
824 commit string
825 file string
826 line string
827 }{{
828 typ: "gitiles",
829 commit: "https://example.com/repo/name/%2B/VERSION",
830 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
831 line: "#10",
832 }, {
833 typ: "github",
834 commit: "https://example.com/repo/name/commit/VERSION",
835 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
836 line: "#L10",
837 }, {
838 typ: "cgit",
839 commit: "https://example.com/repo/name/commit/?id=VERSION",
840 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
841 line: "#n10",
842 }, {
843 typ: "gitweb",
844 commit: "https://example.com/repo/name;a=commit;h=VERSION",
845 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
846 line: "#l10",
847 }, {
848 typ: "source.bazel.build",
849 commit: "https://example.com/repo/name/%2B/VERSION",
850 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
851 line: ";l=10",
852 }, {
853 typ: "bitbucket-server",
854 commit: "https://example.com/repo/name/commits/VERSION",
855 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
856 line: "#10",
857 }, {
858 typ: "gitlab",
859 commit: "https://example.com/repo/name/-/commit/VERSION",
860 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
861 line: "#L10",
862 }, {
863 typ: "gitea",
864 commit: "https://example.com/repo/name/commit/VERSION",
865 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
866 line: "#L10",
867 }}
868
869 for _, tc := range cases {
870 t.Run(tc.typ, func(t *testing.T) {
871 assertOutput := func(templateText string, want string) {
872 t.Helper()
873
874 tt, err := index.ParseTemplate(templateText)
875 if err != nil {
876 t.Fatal(err)
877 }
878
879 var sb strings.Builder
880 err = tt.Execute(&sb, map[string]any{
881 "Version": version,
882 "Path": path,
883 "LineNumber": lineNumber,
884 })
885 if err != nil {
886 t.Fatal(err)
887 }
888 if got := sb.String(); got != want {
889 t.Fatalf("want: %q\ngot: %q", want, got)
890 }
891 }
892
893 var repo zoekt.Repository
894 u, _ := url.Parse(base)
895 err := setTemplates(&repo, u, tc.typ)
896 if err != nil {
897 t.Fatal(err)
898 }
899 assertOutput(repo.CommitURLTemplate, tc.commit)
900 assertOutput(repo.FileURLTemplate, tc.file)
901 assertOutput(repo.LineFragmentTemplate, tc.line)
902 })
903 }
904}
905
906func BenchmarkPrepareNormalBuild(b *testing.B) {
907 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
908 repoDir := "/path/to/your/repo"
909 repo, err := git.PlainOpen(repoDir)
910 if err != nil {
911 b.Fatalf("Failed to open test repository: %v", err)
912 }
913
914 opts := Options{
915 RepoDir: repoDir,
916 Submodules: false,
917 BranchPrefix: "refs/heads/",
918 Branches: []string{"main"},
919 BuildOptions: index.Options{
920 RepositoryDescription: zoekt.Repository{
921 Name: "test-repo",
922 URL: "https://github.com/example/test-repo",
923 },
924 },
925 }
926
927 b.ReportAllocs()
928
929 repos, branchVersions, err := prepareNormalBuild(opts, repo)
930 if err != nil {
931 b.Fatalf("prepareNormalBuild failed: %v", err)
932 }
933
934 runtime.GC()
935
936 var m runtime.MemStats
937 runtime.ReadMemStats(&m)
938 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
939 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
940
941 if len(repos) == 0 || len(branchVersions) == 0 {
942 b.Fatalf("Unexpected empty results")
943 }
944}