fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "runtime"
26 "sort"
27 "strings"
28 "testing"
29
30 "github.com/go-git/go-git/v5"
31 "github.com/go-git/go-git/v5/plumbing"
32 "github.com/google/go-cmp/cmp"
33 "github.com/google/go-cmp/cmp/cmpopts"
34 "github.com/sourcegraph/zoekt"
35 "github.com/sourcegraph/zoekt/build"
36 "github.com/sourcegraph/zoekt/ignore"
37 "github.com/sourcegraph/zoekt/query"
38 "github.com/sourcegraph/zoekt/shards"
39)
40
41func TestIndexEmptyRepo(t *testing.T) {
42 dir := t.TempDir()
43
44 cmd := exec.Command("git", "init", "-b", "master", "repo")
45 cmd.Dir = dir
46
47 if err := cmd.Run(); err != nil {
48 t.Fatalf("cmd.Run: %v", err)
49 }
50
51 desc := zoekt.Repository{
52 Name: "repo",
53 }
54 opts := Options{
55 RepoDir: filepath.Join(dir, "repo", ".git"),
56 BuildOptions: build.Options{
57 RepositoryDescription: desc,
58 IndexDir: dir,
59 },
60 }
61
62 if _, err := IndexGitRepo(opts); err != nil {
63 t.Fatalf("IndexGitRepo: %v", err)
64 }
65}
66
67func TestIndexDeltaBasic(t *testing.T) {
68 type branchToDocumentMap map[string][]zoekt.Document
69
70 type step struct {
71 name string
72 addedDocuments branchToDocumentMap
73 deletedDocuments branchToDocumentMap
74 optFn func(t *testing.T, options *Options)
75
76 expectedFallbackToNormalBuild bool
77 expectedDocuments []zoekt.Document
78 }
79
80 helloWorld := zoekt.Document{Name: "hello_world.txt", Content: []byte("hello")}
81
82 fruitV1 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
83 fruitV1InFolder := zoekt.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
84 fruitV1WithNewName := zoekt.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
85
86 fruitV2 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
87 fruitV2InFolder := zoekt.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
88
89 fruitV3 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
90 fruitV4 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("apples")}
91
92 foo := zoekt.Document{Name: "foo.txt", Content: []byte("bar")}
93
94 emptySourcegraphIgnore := zoekt.Document{Name: ignore.IgnoreFile}
95 sourcegraphIgnoreWithContent := zoekt.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
96
97 for _, test := range []struct {
98 name string
99 branches []string
100 steps []step
101 }{
102 {
103 name: "modification",
104 branches: []string{"main"},
105 steps: []step{
106 {
107 name: "setup",
108 addedDocuments: branchToDocumentMap{
109 "main": []zoekt.Document{helloWorld, fruitV1},
110 },
111
112 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
113 },
114 {
115 name: "add newer version of fruits",
116 addedDocuments: branchToDocumentMap{
117 "main": []zoekt.Document{fruitV2},
118 },
119 optFn: func(t *testing.T, o *Options) {
120 o.BuildOptions.IsDelta = true
121 },
122
123 expectedDocuments: []zoekt.Document{helloWorld, fruitV2},
124 },
125 },
126 },
127 {
128 name: "modification only inside nested folder",
129 branches: []string{"main"},
130 steps: []step{
131 {
132 name: "setup",
133 addedDocuments: branchToDocumentMap{
134 "main": []zoekt.Document{foo, fruitV1InFolder},
135 },
136
137 expectedDocuments: []zoekt.Document{foo, fruitV1InFolder},
138 },
139 {
140 name: "add newer version of fruits inside folder",
141 addedDocuments: branchToDocumentMap{
142 "main": []zoekt.Document{fruitV2InFolder},
143 },
144 optFn: func(t *testing.T, o *Options) {
145 o.BuildOptions.IsDelta = true
146 },
147
148 expectedDocuments: []zoekt.Document{foo, fruitV2InFolder},
149 },
150 },
151 },
152 {
153 name: "addition",
154 branches: []string{"main"},
155 steps: []step{
156 {
157 name: "setup",
158 addedDocuments: branchToDocumentMap{
159 "main": []zoekt.Document{helloWorld, fruitV1},
160 },
161
162 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
163 },
164 {
165 name: "add new file - foo",
166 addedDocuments: branchToDocumentMap{
167 "main": []zoekt.Document{foo},
168 },
169 optFn: func(t *testing.T, o *Options) {
170 o.BuildOptions.IsDelta = true
171 },
172
173 expectedDocuments: []zoekt.Document{helloWorld, fruitV1, foo},
174 },
175 },
176 },
177 {
178 name: "deletion",
179 branches: []string{"main"},
180 steps: []step{
181 {
182 name: "setup",
183 addedDocuments: branchToDocumentMap{
184 "main": []zoekt.Document{helloWorld, fruitV1, foo},
185 },
186
187 expectedDocuments: []zoekt.Document{helloWorld, fruitV1, foo},
188 },
189 {
190 name: "delete foo file",
191 addedDocuments: nil,
192 deletedDocuments: branchToDocumentMap{
193 "main": []zoekt.Document{foo},
194 },
195
196 optFn: func(t *testing.T, o *Options) {
197 o.BuildOptions.IsDelta = true
198 },
199
200 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
201 },
202 },
203 },
204 {
205 name: "addition and deletion on only one branch",
206 branches: []string{"main", "release", "dev"},
207 steps: []step{
208 {
209 name: "setup",
210 addedDocuments: branchToDocumentMap{
211 "main": []zoekt.Document{fruitV1},
212 "release": []zoekt.Document{fruitV2},
213 "dev": []zoekt.Document{fruitV3},
214 },
215
216 expectedDocuments: []zoekt.Document{fruitV1, fruitV2, fruitV3},
217 },
218 {
219 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
220 addedDocuments: branchToDocumentMap{
221 "dev": []zoekt.Document{fruitV4},
222 },
223 deletedDocuments: branchToDocumentMap{
224 "main": []zoekt.Document{fruitV1},
225 },
226
227 optFn: func(t *testing.T, o *Options) {
228 o.BuildOptions.IsDelta = true
229 },
230
231 expectedDocuments: []zoekt.Document{fruitV2, fruitV4},
232 },
233 },
234 },
235 {
236 name: "rename",
237 branches: []string{"main", "release"},
238 steps: []step{
239 {
240 name: "setup",
241 addedDocuments: branchToDocumentMap{
242 "main": []zoekt.Document{fruitV1},
243 "release": []zoekt.Document{fruitV2},
244 },
245 expectedDocuments: []zoekt.Document{fruitV1, fruitV2},
246 },
247 {
248 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
249 addedDocuments: branchToDocumentMap{
250 "main": []zoekt.Document{fruitV1WithNewName},
251 },
252 deletedDocuments: branchToDocumentMap{
253 "main": []zoekt.Document{fruitV1},
254 },
255
256 optFn: func(t *testing.T, o *Options) {
257 o.BuildOptions.IsDelta = true
258 },
259
260 expectedDocuments: []zoekt.Document{fruitV1WithNewName, fruitV2},
261 },
262 },
263 },
264 {
265 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
266 branches: []string{"main", "dev"},
267 steps: []step{
268 {
269 name: "setup",
270 addedDocuments: branchToDocumentMap{
271 "main": []zoekt.Document{fruitV1},
272 "dev": []zoekt.Document{fruitV2},
273 },
274 expectedDocuments: []zoekt.Document{fruitV1, fruitV2},
275 },
276 {
277 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
278 addedDocuments: branchToDocumentMap{
279 "main": []zoekt.Document{fruitV2},
280 "dev": []zoekt.Document{fruitV3},
281 },
282
283 optFn: func(t *testing.T, o *Options) {
284 o.BuildOptions.IsDelta = true
285 },
286
287 expectedDocuments: []zoekt.Document{fruitV2, fruitV3},
288 },
289 },
290 },
291 {
292 name: "no-op delta builds (reindexing the same commits)",
293 branches: []string{"main", "dev"},
294 steps: []step{
295 {
296 name: "setup",
297 addedDocuments: branchToDocumentMap{
298 "main": []zoekt.Document{fruitV1, foo},
299 "dev": []zoekt.Document{helloWorld},
300 },
301 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
302 },
303 {
304 name: "first no-op (normal build -> delta build)",
305 optFn: func(t *testing.T, o *Options) {
306 o.BuildOptions.IsDelta = true
307 },
308
309 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
310 },
311 {
312 name: "second no-op (delta build -> delta build)",
313 optFn: func(t *testing.T, o *Options) {
314 o.BuildOptions.IsDelta = true
315 },
316
317 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
318 },
319 },
320 },
321 {
322 name: "should fallback to normal build if no prior shards exist",
323 branches: []string{"main"},
324 steps: []step{
325 {
326 name: "attempt delta build on a repository that hasn't been indexed yet",
327 addedDocuments: branchToDocumentMap{
328 "main": []zoekt.Document{helloWorld},
329 },
330 optFn: func(t *testing.T, o *Options) {
331 o.BuildOptions.IsDelta = true
332 },
333
334 expectedFallbackToNormalBuild: true,
335 expectedDocuments: []zoekt.Document{helloWorld},
336 },
337 },
338 },
339 {
340 name: "should fallback to normal build if the set of requested repository branches changes",
341 branches: []string{"main", "release", "dev"},
342 steps: []step{
343 {
344 name: "setup",
345 addedDocuments: branchToDocumentMap{
346 "main": []zoekt.Document{fruitV1},
347 "release": []zoekt.Document{fruitV2},
348 "dev": []zoekt.Document{fruitV3},
349 },
350
351 expectedDocuments: []zoekt.Document{fruitV1, fruitV2, fruitV3},
352 },
353 {
354 name: "try delta build after dropping 'main' branch from index ",
355 addedDocuments: branchToDocumentMap{
356 "release": []zoekt.Document{fruitV4},
357 },
358 optFn: func(t *testing.T, o *Options) {
359 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
360 o.BuildOptions.IsDelta = true
361 },
362
363 expectedFallbackToNormalBuild: true,
364 expectedDocuments: []zoekt.Document{fruitV3, fruitV4},
365 },
366 },
367 },
368 {
369 name: "should fallback to normal build if one or more index options updates requires a full build",
370 branches: []string{"main"},
371 steps: []step{
372 {
373 name: "setup",
374 addedDocuments: branchToDocumentMap{
375 "main": []zoekt.Document{fruitV1},
376 },
377
378 expectedDocuments: []zoekt.Document{fruitV1},
379 },
380 {
381 name: "try delta build after updating Disable CTags index option",
382 addedDocuments: branchToDocumentMap{
383 "main": []zoekt.Document{fruitV2},
384 },
385 optFn: func(t *testing.T, o *Options) {
386 o.BuildOptions.IsDelta = true
387 o.BuildOptions.DisableCTags = true
388 },
389
390 expectedFallbackToNormalBuild: true,
391 expectedDocuments: []zoekt.Document{fruitV2},
392 },
393 {
394 name: "try delta build after reverting Disable CTags index option",
395 addedDocuments: branchToDocumentMap{
396 "main": []zoekt.Document{fruitV3},
397 },
398 optFn: func(t *testing.T, o *Options) {
399 o.BuildOptions.IsDelta = true
400 o.BuildOptions.DisableCTags = false
401 },
402
403 expectedFallbackToNormalBuild: true,
404 expectedDocuments: []zoekt.Document{fruitV3},
405 },
406 },
407 },
408 {
409 name: "should successfully perform multiple delta builds after disabling symbols",
410 branches: []string{"main"},
411 steps: []step{
412 {
413 name: "setup",
414 addedDocuments: branchToDocumentMap{
415 "main": []zoekt.Document{fruitV1},
416 },
417
418 expectedDocuments: []zoekt.Document{fruitV1},
419 },
420 {
421 name: "try delta build after updating Disable CTags index option",
422 addedDocuments: branchToDocumentMap{
423 "main": []zoekt.Document{fruitV2},
424 },
425 optFn: func(t *testing.T, o *Options) {
426 o.BuildOptions.IsDelta = true
427 o.BuildOptions.DisableCTags = true
428 },
429
430 expectedFallbackToNormalBuild: true,
431 expectedDocuments: []zoekt.Document{fruitV2},
432 },
433 {
434 name: "try another delta build while CTags is still disabled",
435 addedDocuments: branchToDocumentMap{
436 "main": []zoekt.Document{fruitV3},
437 },
438 optFn: func(t *testing.T, o *Options) {
439 o.BuildOptions.IsDelta = true
440 o.BuildOptions.DisableCTags = true
441 },
442
443 expectedDocuments: []zoekt.Document{fruitV3},
444 },
445 },
446 },
447 {
448 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
449 branches: []string{"main"},
450 steps: []step{
451 {
452 name: "setup",
453 addedDocuments: branchToDocumentMap{
454 "main": []zoekt.Document{emptySourcegraphIgnore},
455 },
456
457 expectedDocuments: []zoekt.Document{emptySourcegraphIgnore},
458 },
459 {
460 name: "attempt delta build after modifying ignore file",
461 addedDocuments: branchToDocumentMap{
462 "main": []zoekt.Document{sourcegraphIgnoreWithContent},
463 },
464 optFn: func(t *testing.T, o *Options) {
465 o.BuildOptions.IsDelta = true
466 },
467
468 expectedFallbackToNormalBuild: true,
469 expectedDocuments: []zoekt.Document{sourcegraphIgnoreWithContent},
470 },
471 },
472 },
473 {
474 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
475 branches: []string{"main"},
476 steps: []step{
477 {
478 name: "setup: first shard",
479 addedDocuments: branchToDocumentMap{
480 "main": []zoekt.Document{foo},
481 },
482
483 expectedDocuments: []zoekt.Document{foo},
484 },
485 {
486 name: "setup: second shard (delta)",
487 addedDocuments: branchToDocumentMap{
488 "main": []zoekt.Document{fruitV1},
489 },
490 optFn: func(t *testing.T, o *Options) {
491 o.BuildOptions.IsDelta = true
492 },
493
494 expectedDocuments: []zoekt.Document{foo, fruitV1},
495 },
496 {
497 name: "setup: third shard (delta)",
498 addedDocuments: branchToDocumentMap{
499 "main": []zoekt.Document{helloWorld},
500 },
501 optFn: func(t *testing.T, o *Options) {
502 o.BuildOptions.IsDelta = true
503 },
504
505 expectedDocuments: []zoekt.Document{foo, fruitV1, helloWorld},
506 },
507 {
508 name: "attempt another delta build after we already blew past the shard threshold",
509 addedDocuments: branchToDocumentMap{
510 "main": []zoekt.Document{fruitV2InFolder},
511 },
512 optFn: func(t *testing.T, o *Options) {
513 o.DeltaShardNumberFallbackThreshold = 2
514 o.BuildOptions.IsDelta = true
515 },
516
517 expectedFallbackToNormalBuild: true,
518 expectedDocuments: []zoekt.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
519 },
520 },
521 },
522 } {
523 test := test
524
525 t.Run(test.name, func(t *testing.T) {
526 t.Parallel()
527
528 indexDir := t.TempDir()
529 repositoryDir := t.TempDir()
530
531 // setup: initialize the repository and all of its branches
532 runScript(t, repositoryDir, "git init -b master")
533 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
534 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
535
536 for _, b := range test.branches {
537 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
538 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
539 }
540
541 for _, step := range test.steps {
542 t.Run(step.name, func(t *testing.T) {
543 for _, b := range test.branches {
544 // setup: for each branch, process any document deletions / additions and commit those changes
545
546 hadChange := false
547
548 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
549
550 for _, d := range step.deletedDocuments[b] {
551 hadChange = true
552
553 file := filepath.Join(repositoryDir, d.Name)
554
555 err := os.Remove(file)
556 if err != nil {
557 t.Fatalf("deleting file %q: %s", d.Name, err)
558 }
559
560 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
561 }
562
563 for _, d := range step.addedDocuments[b] {
564 hadChange = true
565
566 file := filepath.Join(repositoryDir, d.Name)
567
568 err := os.MkdirAll(filepath.Dir(file), 0o755)
569 if err != nil {
570 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
571 }
572
573 err = os.WriteFile(file, d.Content, 0o644)
574 if err != nil {
575 t.Fatalf("writing file %q: %s", d.Name, err)
576 }
577
578 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
579 }
580
581 if !hadChange {
582 continue
583 }
584
585 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
586 }
587
588 // setup: prepare indexOptions with given overrides
589 buildOptions := build.Options{
590 IndexDir: indexDir,
591 RepositoryDescription: zoekt.Repository{
592 Name: "repository",
593 },
594 IsDelta: false,
595 }
596 buildOptions.SetDefaults()
597
598 branches := append([]string{"HEAD"}, test.branches...)
599
600 options := Options{
601 RepoDir: filepath.Join(repositoryDir, ".git"),
602 BuildOptions: buildOptions,
603 Branches: branches,
604 }
605
606 if step.optFn != nil {
607 step.optFn(t, &options)
608 }
609
610 // setup: prepare spy versions of prepare delta / normal build so that we can observe
611 // whether they were called appropriately
612 deltaBuildCalled := false
613 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
614 deltaBuildCalled = true
615 return prepareDeltaBuild(options, repository)
616 }
617
618 normalBuildCalled := false
619 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
620 normalBuildCalled = true
621 return prepareNormalBuild(options, repository)
622 }
623
624 // run test
625 _, err := indexGitRepo(options, gitIndexConfig{
626 prepareDeltaBuild: prepareDeltaSpy,
627 prepareNormalBuild: prepareNormalSpy,
628 })
629 if err != nil {
630 t.Fatalf("IndexGitRepo: %s", err)
631 }
632
633 if options.BuildOptions.IsDelta != deltaBuildCalled {
634 // We should always try a delta build if we request it in the options.
635 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
636 }
637
638 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
639 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
640 // asked for a delta build in the first place.
641 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
642 }
643
644 // examine outcome: load shards into a searcher instance and run a dummy search query
645 // that returns every document contained in the shards
646 //
647 // then, compare returned set of documents with the expected set for the step and see if they agree
648
649 ss, err := shards.NewDirectorySearcher(indexDir)
650 if err != nil {
651 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
652 }
653 defer ss.Close()
654
655 searchOpts := &zoekt.SearchOptions{Whole: true}
656 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
657 if err != nil {
658 t.Fatalf("Search: %s", err)
659 }
660
661 var receivedDocuments []zoekt.Document
662 for _, f := range result.Files {
663 receivedDocuments = append(receivedDocuments, zoekt.Document{
664 Name: f.FileName,
665 Content: f.Content,
666 })
667 }
668
669 for _, docs := range [][]zoekt.Document{step.expectedDocuments, receivedDocuments} {
670 sort.Slice(docs, func(i, j int) bool {
671 a, b := docs[i], docs[j]
672
673 // first compare names, then fallback to contents if the names are equal
674
675 if a.Name < b.Name {
676 return true
677 }
678
679 if a.Name > b.Name {
680 return false
681 }
682
683 return bytes.Compare(a.Content, b.Content) < 0
684 })
685 }
686
687 compareOptions := []cmp.Option{
688 cmpopts.IgnoreFields(zoekt.Document{}, "Branches"),
689 cmpopts.EquateEmpty(),
690 }
691
692 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
693 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
694 }
695 })
696 }
697 })
698 }
699}
700
701func TestRepoPathRanks(t *testing.T) {
702 pathRanks := repoPathRanks{
703 Paths: map[string]float64{
704 "search.go": 10.23,
705 "internal/index.go": 5.5,
706 "internal/scratch.go": 0.0,
707 "backend/search_test.go": 2.1,
708 },
709 MeanRank: 3.3,
710 }
711 cases := []struct {
712 name string
713 path string
714 rank float64
715 }{
716 {
717 name: "rank for standard file",
718 path: "search.go",
719 rank: 10.23,
720 },
721 {
722 name: "file with rank 0",
723 path: "internal/scratch.go",
724 rank: 0.0,
725 },
726 {
727 name: "rank for test file",
728 path: "backend/search_test.go",
729 rank: 2.1,
730 },
731 {
732 name: "file with missing rank",
733 path: "internal/docs.md",
734 rank: 3.3,
735 },
736 {
737 name: "test file with missing rank",
738 path: "backend/index_test.go",
739 rank: 0.0,
740 },
741 {
742 name: "third-party file with missing rank",
743 path: "node_modules/search/index.js",
744 rank: 0.0,
745 },
746 }
747
748 for _, tt := range cases {
749 t.Run(tt.name, func(t *testing.T) {
750 got := pathRanks.rank(tt.path, nil)
751 if got != tt.rank {
752 t.Errorf("expected file '%s' to have rank %f, but got %f", tt.path, tt.rank, got)
753 }
754 })
755 }
756}
757
758func runScript(t *testing.T, cwd string, script string) {
759 t.Helper()
760
761 err := os.MkdirAll(cwd, 0o755)
762 if err != nil {
763 t.Fatalf("ensuring path %q exists: %s", cwd, err)
764 }
765
766 cmd := exec.Command("sh", "-euxc", script)
767 cmd.Dir = cwd
768 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
769
770 if out, err := cmd.CombinedOutput(); err != nil {
771 t.Fatalf("execution error: %v, output %s", err, out)
772 }
773}
774
775func TestSetTemplates_e2e(t *testing.T) {
776 repositoryDir := t.TempDir()
777
778 // setup: initialize the repository and all of its branches
779 runScript(t, repositoryDir, "git init -b master")
780 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
781 desc := zoekt.Repository{}
782 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
783 t.Fatalf("setTemplatesFromConfig: %v", err)
784 }
785
786 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
787 t.Errorf("got %q, want %q", got, want)
788 }
789}
790
791func TestSetTemplates(t *testing.T) {
792 base := "https://example.com/repo/name"
793 version := "VERSION"
794 path := "dir/name.txt"
795 lineNumber := 10
796 cases := []struct {
797 typ string
798 commit string
799 file string
800 line string
801 }{{
802 typ: "gitiles",
803 commit: "https://example.com/repo/name/%2B/VERSION",
804 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
805 line: "#10",
806 }, {
807 typ: "github",
808 commit: "https://example.com/repo/name/commit/VERSION",
809 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
810 line: "#L10",
811 }, {
812 typ: "cgit",
813 commit: "https://example.com/repo/name/commit/?id=VERSION",
814 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
815 line: "#n10",
816 }, {
817 typ: "gitweb",
818 commit: "https://example.com/repo/name;a=commit;h=VERSION",
819 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
820 line: "#l10",
821 }, {
822 typ: "source.bazel.build",
823 commit: "https://example.com/repo/name/%2B/VERSION",
824 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
825 line: ";l=10",
826 }, {
827 typ: "bitbucket-server",
828 commit: "https://example.com/repo/name/commits/VERSION",
829 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
830 line: "#10",
831 }, {
832 typ: "gitlab",
833 commit: "https://example.com/repo/name/-/commit/VERSION",
834 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
835 line: "#L10",
836 }, {
837 typ: "gitea",
838 commit: "https://example.com/repo/name/commit/VERSION",
839 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
840 line: "#L10",
841 }}
842
843 for _, tc := range cases {
844 t.Run(tc.typ, func(t *testing.T) {
845 assertOutput := func(templateText string, want string) {
846 t.Helper()
847
848 tt, err := zoekt.ParseTemplate(templateText)
849 if err != nil {
850 t.Fatal(err)
851 }
852
853 var sb strings.Builder
854 err = tt.Execute(&sb, map[string]any{
855 "Version": version,
856 "Path": path,
857 "LineNumber": lineNumber,
858 })
859 if err != nil {
860 t.Fatal(err)
861 }
862 if got := sb.String(); got != want {
863 t.Fatalf("want: %q\ngot: %q", want, got)
864 }
865 }
866
867 var repo zoekt.Repository
868 u, _ := url.Parse(base)
869 err := setTemplates(&repo, u, tc.typ)
870 if err != nil {
871 t.Fatal(err)
872 }
873 assertOutput(repo.CommitURLTemplate, tc.commit)
874 assertOutput(repo.FileURLTemplate, tc.file)
875 assertOutput(repo.LineFragmentTemplate, tc.line)
876 })
877 }
878}
879
880func BenchmarkPrepareNormalBuild(b *testing.B) {
881 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
882 repoDir := "/path/to/your/repo"
883 repo, err := git.PlainOpen(repoDir)
884 if err != nil {
885 b.Fatalf("Failed to open test repository: %v", err)
886 }
887
888 opts := Options{
889 RepoDir: repoDir,
890 Submodules: false,
891 BranchPrefix: "refs/heads/",
892 Branches: []string{"main"},
893 BuildOptions: build.Options{
894 RepositoryDescription: zoekt.Repository{
895 Name: "test-repo",
896 URL: "https://github.com/example/test-repo",
897 },
898 },
899 }
900
901 b.ReportAllocs()
902
903 repos, branchVersions, err := prepareNormalBuild(opts, repo)
904 if err != nil {
905 b.Fatalf("prepareNormalBuild failed: %v", err)
906 }
907
908 runtime.GC()
909
910 var m runtime.MemStats
911 runtime.ReadMemStats(&m)
912 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
913 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
914
915 if len(repos) == 0 || len(branchVersions) == 0 {
916 b.Fatalf("Unexpected empty results")
917 }
918}