fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35 "github.com/sourcegraph/zoekt"
36 "github.com/sourcegraph/zoekt/build"
37 "github.com/sourcegraph/zoekt/ignore"
38 "github.com/sourcegraph/zoekt/query"
39 "github.com/sourcegraph/zoekt/shards"
40)
41
42func TestIndexEmptyRepo(t *testing.T) {
43 dir := t.TempDir()
44
45 cmd := exec.Command("git", "init", "-b", "master", "repo")
46 cmd.Dir = dir
47
48 if err := cmd.Run(); err != nil {
49 t.Fatalf("cmd.Run: %v", err)
50 }
51
52 desc := zoekt.Repository{
53 Name: "repo",
54 }
55 opts := Options{
56 RepoDir: filepath.Join(dir, "repo", ".git"),
57 BuildOptions: build.Options{
58 RepositoryDescription: desc,
59 IndexDir: dir,
60 },
61 }
62
63 if _, err := IndexGitRepo(opts); err != nil {
64 t.Fatalf("IndexGitRepo: %v", err)
65 }
66}
67
68func TestIndexNonexistentRepo(t *testing.T) {
69 dir := t.TempDir()
70 desc := zoekt.Repository{
71 Name: "nonexistent",
72 }
73 opts := Options{
74 RepoDir: "does/not/exist",
75 Branches: []string{"main"},
76 BuildOptions: build.Options{
77 RepositoryDescription: desc,
78 IndexDir: dir,
79 },
80 }
81
82 if _, err := IndexGitRepo(opts); err == nil {
83 t.Fatal("expected error, got none")
84 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
85 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
86 }
87}
88
89func TestIndexTinyRepo(t *testing.T) {
90 // Create a repo with one file in it.
91 dir := t.TempDir()
92 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
93
94 repoDir := filepath.Join(dir, "repo")
95 executeCommand(t, repoDir, exec.Command("git", "config", "user.name", "Thomas"))
96 executeCommand(t, repoDir, exec.Command("git", "config", "user.email", "thomas@google.com"))
97
98 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
99 t.Fatalf("WriteFile: %v", err)
100 }
101 executeCommand(t, repoDir, exec.Command("git", "add", "."))
102 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
103
104 // Test that indexing accepts both the repo directory, and the .git subdirectory.
105 for _, testDir := range []string{"repo", "repo/.git"} {
106 opts := Options{
107 RepoDir: filepath.Join(dir, testDir),
108 Branches: []string{"main"},
109 BuildOptions: build.Options{
110 RepositoryDescription: zoekt.Repository{Name: "repo"},
111 IndexDir: dir,
112 },
113 }
114
115 if _, err := IndexGitRepo(opts); err != nil {
116 t.Fatalf("unexpected error %v", err)
117 }
118
119 searcher, err := shards.NewDirectorySearcher(dir)
120 if err != nil {
121 t.Fatal("NewDirectorySearcher", err)
122 }
123
124 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
125 searcher.Close()
126
127 if err != nil {
128 t.Fatal("search failed", err)
129 }
130
131 if len(results.Files) != 1 {
132 t.Fatalf("got search result %v, want 1 file", results.Files)
133 }
134 }
135}
136
137func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
138 cmd.Dir = dir
139 if err := cmd.Run(); err != nil {
140 t.Fatalf("cmd.Run: %v", err)
141 }
142 return cmd
143}
144
145func TestIndexDeltaBasic(t *testing.T) {
146 type branchToDocumentMap map[string][]zoekt.Document
147
148 type step struct {
149 name string
150 addedDocuments branchToDocumentMap
151 deletedDocuments branchToDocumentMap
152 optFn func(t *testing.T, options *Options)
153
154 expectedFallbackToNormalBuild bool
155 expectedDocuments []zoekt.Document
156 }
157
158 helloWorld := zoekt.Document{Name: "hello_world.txt", Content: []byte("hello")}
159
160 fruitV1 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
161 fruitV1InFolder := zoekt.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
162 fruitV1WithNewName := zoekt.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
163
164 fruitV2 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
165 fruitV2InFolder := zoekt.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
166
167 fruitV3 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
168 fruitV4 := zoekt.Document{Name: "best_fruit.txt", Content: []byte("apples")}
169
170 foo := zoekt.Document{Name: "foo.txt", Content: []byte("bar")}
171
172 emptySourcegraphIgnore := zoekt.Document{Name: ignore.IgnoreFile}
173 sourcegraphIgnoreWithContent := zoekt.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
174
175 for _, test := range []struct {
176 name string
177 branches []string
178 steps []step
179 }{
180 {
181 name: "modification",
182 branches: []string{"main"},
183 steps: []step{
184 {
185 name: "setup",
186 addedDocuments: branchToDocumentMap{
187 "main": []zoekt.Document{helloWorld, fruitV1},
188 },
189
190 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
191 },
192 {
193 name: "add newer version of fruits",
194 addedDocuments: branchToDocumentMap{
195 "main": []zoekt.Document{fruitV2},
196 },
197 optFn: func(t *testing.T, o *Options) {
198 o.BuildOptions.IsDelta = true
199 },
200
201 expectedDocuments: []zoekt.Document{helloWorld, fruitV2},
202 },
203 },
204 },
205 {
206 name: "modification only inside nested folder",
207 branches: []string{"main"},
208 steps: []step{
209 {
210 name: "setup",
211 addedDocuments: branchToDocumentMap{
212 "main": []zoekt.Document{foo, fruitV1InFolder},
213 },
214
215 expectedDocuments: []zoekt.Document{foo, fruitV1InFolder},
216 },
217 {
218 name: "add newer version of fruits inside folder",
219 addedDocuments: branchToDocumentMap{
220 "main": []zoekt.Document{fruitV2InFolder},
221 },
222 optFn: func(t *testing.T, o *Options) {
223 o.BuildOptions.IsDelta = true
224 },
225
226 expectedDocuments: []zoekt.Document{foo, fruitV2InFolder},
227 },
228 },
229 },
230 {
231 name: "addition",
232 branches: []string{"main"},
233 steps: []step{
234 {
235 name: "setup",
236 addedDocuments: branchToDocumentMap{
237 "main": []zoekt.Document{helloWorld, fruitV1},
238 },
239
240 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
241 },
242 {
243 name: "add new file - foo",
244 addedDocuments: branchToDocumentMap{
245 "main": []zoekt.Document{foo},
246 },
247 optFn: func(t *testing.T, o *Options) {
248 o.BuildOptions.IsDelta = true
249 },
250
251 expectedDocuments: []zoekt.Document{helloWorld, fruitV1, foo},
252 },
253 },
254 },
255 {
256 name: "deletion",
257 branches: []string{"main"},
258 steps: []step{
259 {
260 name: "setup",
261 addedDocuments: branchToDocumentMap{
262 "main": []zoekt.Document{helloWorld, fruitV1, foo},
263 },
264
265 expectedDocuments: []zoekt.Document{helloWorld, fruitV1, foo},
266 },
267 {
268 name: "delete foo file",
269 addedDocuments: nil,
270 deletedDocuments: branchToDocumentMap{
271 "main": []zoekt.Document{foo},
272 },
273
274 optFn: func(t *testing.T, o *Options) {
275 o.BuildOptions.IsDelta = true
276 },
277
278 expectedDocuments: []zoekt.Document{helloWorld, fruitV1},
279 },
280 },
281 },
282 {
283 name: "addition and deletion on only one branch",
284 branches: []string{"main", "release", "dev"},
285 steps: []step{
286 {
287 name: "setup",
288 addedDocuments: branchToDocumentMap{
289 "main": []zoekt.Document{fruitV1},
290 "release": []zoekt.Document{fruitV2},
291 "dev": []zoekt.Document{fruitV3},
292 },
293
294 expectedDocuments: []zoekt.Document{fruitV1, fruitV2, fruitV3},
295 },
296 {
297 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
298 addedDocuments: branchToDocumentMap{
299 "dev": []zoekt.Document{fruitV4},
300 },
301 deletedDocuments: branchToDocumentMap{
302 "main": []zoekt.Document{fruitV1},
303 },
304
305 optFn: func(t *testing.T, o *Options) {
306 o.BuildOptions.IsDelta = true
307 },
308
309 expectedDocuments: []zoekt.Document{fruitV2, fruitV4},
310 },
311 },
312 },
313 {
314 name: "rename",
315 branches: []string{"main", "release"},
316 steps: []step{
317 {
318 name: "setup",
319 addedDocuments: branchToDocumentMap{
320 "main": []zoekt.Document{fruitV1},
321 "release": []zoekt.Document{fruitV2},
322 },
323 expectedDocuments: []zoekt.Document{fruitV1, fruitV2},
324 },
325 {
326 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
327 addedDocuments: branchToDocumentMap{
328 "main": []zoekt.Document{fruitV1WithNewName},
329 },
330 deletedDocuments: branchToDocumentMap{
331 "main": []zoekt.Document{fruitV1},
332 },
333
334 optFn: func(t *testing.T, o *Options) {
335 o.BuildOptions.IsDelta = true
336 },
337
338 expectedDocuments: []zoekt.Document{fruitV1WithNewName, fruitV2},
339 },
340 },
341 },
342 {
343 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
344 branches: []string{"main", "dev"},
345 steps: []step{
346 {
347 name: "setup",
348 addedDocuments: branchToDocumentMap{
349 "main": []zoekt.Document{fruitV1},
350 "dev": []zoekt.Document{fruitV2},
351 },
352 expectedDocuments: []zoekt.Document{fruitV1, fruitV2},
353 },
354 {
355 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
356 addedDocuments: branchToDocumentMap{
357 "main": []zoekt.Document{fruitV2},
358 "dev": []zoekt.Document{fruitV3},
359 },
360
361 optFn: func(t *testing.T, o *Options) {
362 o.BuildOptions.IsDelta = true
363 },
364
365 expectedDocuments: []zoekt.Document{fruitV2, fruitV3},
366 },
367 },
368 },
369 {
370 name: "no-op delta builds (reindexing the same commits)",
371 branches: []string{"main", "dev"},
372 steps: []step{
373 {
374 name: "setup",
375 addedDocuments: branchToDocumentMap{
376 "main": []zoekt.Document{fruitV1, foo},
377 "dev": []zoekt.Document{helloWorld},
378 },
379 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
380 },
381 {
382 name: "first no-op (normal build -> delta build)",
383 optFn: func(t *testing.T, o *Options) {
384 o.BuildOptions.IsDelta = true
385 },
386
387 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
388 },
389 {
390 name: "second no-op (delta build -> delta build)",
391 optFn: func(t *testing.T, o *Options) {
392 o.BuildOptions.IsDelta = true
393 },
394
395 expectedDocuments: []zoekt.Document{fruitV1, foo, helloWorld},
396 },
397 },
398 },
399 {
400 name: "should fallback to normal build if no prior shards exist",
401 branches: []string{"main"},
402 steps: []step{
403 {
404 name: "attempt delta build on a repository that hasn't been indexed yet",
405 addedDocuments: branchToDocumentMap{
406 "main": []zoekt.Document{helloWorld},
407 },
408 optFn: func(t *testing.T, o *Options) {
409 o.BuildOptions.IsDelta = true
410 },
411
412 expectedFallbackToNormalBuild: true,
413 expectedDocuments: []zoekt.Document{helloWorld},
414 },
415 },
416 },
417 {
418 name: "should fallback to normal build if the set of requested repository branches changes",
419 branches: []string{"main", "release", "dev"},
420 steps: []step{
421 {
422 name: "setup",
423 addedDocuments: branchToDocumentMap{
424 "main": []zoekt.Document{fruitV1},
425 "release": []zoekt.Document{fruitV2},
426 "dev": []zoekt.Document{fruitV3},
427 },
428
429 expectedDocuments: []zoekt.Document{fruitV1, fruitV2, fruitV3},
430 },
431 {
432 name: "try delta build after dropping 'main' branch from index ",
433 addedDocuments: branchToDocumentMap{
434 "release": []zoekt.Document{fruitV4},
435 },
436 optFn: func(t *testing.T, o *Options) {
437 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
438 o.BuildOptions.IsDelta = true
439 },
440
441 expectedFallbackToNormalBuild: true,
442 expectedDocuments: []zoekt.Document{fruitV3, fruitV4},
443 },
444 },
445 },
446 {
447 name: "should fallback to normal build if one or more index options updates requires a full build",
448 branches: []string{"main"},
449 steps: []step{
450 {
451 name: "setup",
452 addedDocuments: branchToDocumentMap{
453 "main": []zoekt.Document{fruitV1},
454 },
455
456 expectedDocuments: []zoekt.Document{fruitV1},
457 },
458 {
459 name: "try delta build after updating Disable CTags index option",
460 addedDocuments: branchToDocumentMap{
461 "main": []zoekt.Document{fruitV2},
462 },
463 optFn: func(t *testing.T, o *Options) {
464 o.BuildOptions.IsDelta = true
465 o.BuildOptions.DisableCTags = true
466 },
467
468 expectedFallbackToNormalBuild: true,
469 expectedDocuments: []zoekt.Document{fruitV2},
470 },
471 {
472 name: "try delta build after reverting Disable CTags index option",
473 addedDocuments: branchToDocumentMap{
474 "main": []zoekt.Document{fruitV3},
475 },
476 optFn: func(t *testing.T, o *Options) {
477 o.BuildOptions.IsDelta = true
478 o.BuildOptions.DisableCTags = false
479 },
480
481 expectedFallbackToNormalBuild: true,
482 expectedDocuments: []zoekt.Document{fruitV3},
483 },
484 },
485 },
486 {
487 name: "should successfully perform multiple delta builds after disabling symbols",
488 branches: []string{"main"},
489 steps: []step{
490 {
491 name: "setup",
492 addedDocuments: branchToDocumentMap{
493 "main": []zoekt.Document{fruitV1},
494 },
495
496 expectedDocuments: []zoekt.Document{fruitV1},
497 },
498 {
499 name: "try delta build after updating Disable CTags index option",
500 addedDocuments: branchToDocumentMap{
501 "main": []zoekt.Document{fruitV2},
502 },
503 optFn: func(t *testing.T, o *Options) {
504 o.BuildOptions.IsDelta = true
505 o.BuildOptions.DisableCTags = true
506 },
507
508 expectedFallbackToNormalBuild: true,
509 expectedDocuments: []zoekt.Document{fruitV2},
510 },
511 {
512 name: "try another delta build while CTags is still disabled",
513 addedDocuments: branchToDocumentMap{
514 "main": []zoekt.Document{fruitV3},
515 },
516 optFn: func(t *testing.T, o *Options) {
517 o.BuildOptions.IsDelta = true
518 o.BuildOptions.DisableCTags = true
519 },
520
521 expectedDocuments: []zoekt.Document{fruitV3},
522 },
523 },
524 },
525 {
526 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
527 branches: []string{"main"},
528 steps: []step{
529 {
530 name: "setup",
531 addedDocuments: branchToDocumentMap{
532 "main": []zoekt.Document{emptySourcegraphIgnore},
533 },
534
535 expectedDocuments: []zoekt.Document{emptySourcegraphIgnore},
536 },
537 {
538 name: "attempt delta build after modifying ignore file",
539 addedDocuments: branchToDocumentMap{
540 "main": []zoekt.Document{sourcegraphIgnoreWithContent},
541 },
542 optFn: func(t *testing.T, o *Options) {
543 o.BuildOptions.IsDelta = true
544 },
545
546 expectedFallbackToNormalBuild: true,
547 expectedDocuments: []zoekt.Document{sourcegraphIgnoreWithContent},
548 },
549 },
550 },
551 {
552 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
553 branches: []string{"main"},
554 steps: []step{
555 {
556 name: "setup: first shard",
557 addedDocuments: branchToDocumentMap{
558 "main": []zoekt.Document{foo},
559 },
560
561 expectedDocuments: []zoekt.Document{foo},
562 },
563 {
564 name: "setup: second shard (delta)",
565 addedDocuments: branchToDocumentMap{
566 "main": []zoekt.Document{fruitV1},
567 },
568 optFn: func(t *testing.T, o *Options) {
569 o.BuildOptions.IsDelta = true
570 },
571
572 expectedDocuments: []zoekt.Document{foo, fruitV1},
573 },
574 {
575 name: "setup: third shard (delta)",
576 addedDocuments: branchToDocumentMap{
577 "main": []zoekt.Document{helloWorld},
578 },
579 optFn: func(t *testing.T, o *Options) {
580 o.BuildOptions.IsDelta = true
581 },
582
583 expectedDocuments: []zoekt.Document{foo, fruitV1, helloWorld},
584 },
585 {
586 name: "attempt another delta build after we already blew past the shard threshold",
587 addedDocuments: branchToDocumentMap{
588 "main": []zoekt.Document{fruitV2InFolder},
589 },
590 optFn: func(t *testing.T, o *Options) {
591 o.DeltaShardNumberFallbackThreshold = 2
592 o.BuildOptions.IsDelta = true
593 },
594
595 expectedFallbackToNormalBuild: true,
596 expectedDocuments: []zoekt.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
597 },
598 },
599 },
600 } {
601 test := test
602
603 t.Run(test.name, func(t *testing.T) {
604 t.Parallel()
605
606 indexDir := t.TempDir()
607 repositoryDir := t.TempDir()
608
609 // setup: initialize the repository and all of its branches
610 runScript(t, repositoryDir, "git init -b master")
611 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
612 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
613
614 for _, b := range test.branches {
615 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
616 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
617 }
618
619 for _, step := range test.steps {
620 t.Run(step.name, func(t *testing.T) {
621 for _, b := range test.branches {
622 // setup: for each branch, process any document deletions / additions and commit those changes
623
624 hadChange := false
625
626 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
627
628 for _, d := range step.deletedDocuments[b] {
629 hadChange = true
630
631 file := filepath.Join(repositoryDir, d.Name)
632
633 err := os.Remove(file)
634 if err != nil {
635 t.Fatalf("deleting file %q: %s", d.Name, err)
636 }
637
638 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
639 }
640
641 for _, d := range step.addedDocuments[b] {
642 hadChange = true
643
644 file := filepath.Join(repositoryDir, d.Name)
645
646 err := os.MkdirAll(filepath.Dir(file), 0o755)
647 if err != nil {
648 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
649 }
650
651 err = os.WriteFile(file, d.Content, 0o644)
652 if err != nil {
653 t.Fatalf("writing file %q: %s", d.Name, err)
654 }
655
656 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
657 }
658
659 if !hadChange {
660 continue
661 }
662
663 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
664 }
665
666 // setup: prepare indexOptions with given overrides
667 buildOptions := build.Options{
668 IndexDir: indexDir,
669 RepositoryDescription: zoekt.Repository{
670 Name: "repository",
671 },
672 IsDelta: false,
673 }
674 buildOptions.SetDefaults()
675
676 branches := append([]string{"HEAD"}, test.branches...)
677
678 options := Options{
679 RepoDir: filepath.Join(repositoryDir, ".git"),
680 BuildOptions: buildOptions,
681 Branches: branches,
682 }
683
684 if step.optFn != nil {
685 step.optFn(t, &options)
686 }
687
688 // setup: prepare spy versions of prepare delta / normal build so that we can observe
689 // whether they were called appropriately
690 deltaBuildCalled := false
691 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
692 deltaBuildCalled = true
693 return prepareDeltaBuild(options, repository)
694 }
695
696 normalBuildCalled := false
697 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
698 normalBuildCalled = true
699 return prepareNormalBuild(options, repository)
700 }
701
702 // run test
703 _, err := indexGitRepo(options, gitIndexConfig{
704 prepareDeltaBuild: prepareDeltaSpy,
705 prepareNormalBuild: prepareNormalSpy,
706 })
707 if err != nil {
708 t.Fatalf("IndexGitRepo: %s", err)
709 }
710
711 if options.BuildOptions.IsDelta != deltaBuildCalled {
712 // We should always try a delta build if we request it in the options.
713 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
714 }
715
716 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
717 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
718 // asked for a delta build in the first place.
719 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
720 }
721
722 // examine outcome: load shards into a searcher instance and run a dummy search query
723 // that returns every document contained in the shards
724 //
725 // then, compare returned set of documents with the expected set for the step and see if they agree
726
727 ss, err := shards.NewDirectorySearcher(indexDir)
728 if err != nil {
729 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
730 }
731 defer ss.Close()
732
733 searchOpts := &zoekt.SearchOptions{Whole: true}
734 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
735 if err != nil {
736 t.Fatalf("Search: %s", err)
737 }
738
739 var receivedDocuments []zoekt.Document
740 for _, f := range result.Files {
741 receivedDocuments = append(receivedDocuments, zoekt.Document{
742 Name: f.FileName,
743 Content: f.Content,
744 })
745 }
746
747 for _, docs := range [][]zoekt.Document{step.expectedDocuments, receivedDocuments} {
748 sort.Slice(docs, func(i, j int) bool {
749 a, b := docs[i], docs[j]
750
751 // first compare names, then fallback to contents if the names are equal
752
753 if a.Name < b.Name {
754 return true
755 }
756
757 if a.Name > b.Name {
758 return false
759 }
760
761 return bytes.Compare(a.Content, b.Content) < 0
762 })
763 }
764
765 compareOptions := []cmp.Option{
766 cmpopts.IgnoreFields(zoekt.Document{}, "Branches"),
767 cmpopts.EquateEmpty(),
768 }
769
770 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
771 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
772 }
773 })
774 }
775 })
776 }
777}
778
779func TestRepoPathRanks(t *testing.T) {
780 pathRanks := repoPathRanks{
781 Paths: map[string]float64{
782 "search.go": 10.23,
783 "internal/index.go": 5.5,
784 "internal/scratch.go": 0.0,
785 "backend/search_test.go": 2.1,
786 },
787 MeanRank: 3.3,
788 }
789 cases := []struct {
790 name string
791 path string
792 rank float64
793 }{
794 {
795 name: "rank for standard file",
796 path: "search.go",
797 rank: 10.23,
798 },
799 {
800 name: "file with rank 0",
801 path: "internal/scratch.go",
802 rank: 0.0,
803 },
804 {
805 name: "rank for test file",
806 path: "backend/search_test.go",
807 rank: 2.1,
808 },
809 {
810 name: "file with missing rank",
811 path: "internal/docs.md",
812 rank: 3.3,
813 },
814 {
815 name: "test file with missing rank",
816 path: "backend/index_test.go",
817 rank: 0.0,
818 },
819 {
820 name: "third-party file with missing rank",
821 path: "node_modules/search/index.js",
822 rank: 0.0,
823 },
824 }
825
826 for _, tt := range cases {
827 t.Run(tt.name, func(t *testing.T) {
828 got := pathRanks.rank(tt.path, nil)
829 if got != tt.rank {
830 t.Errorf("expected file '%s' to have rank %f, but got %f", tt.path, tt.rank, got)
831 }
832 })
833 }
834}
835
836func runScript(t *testing.T, cwd string, script string) {
837 t.Helper()
838
839 err := os.MkdirAll(cwd, 0o755)
840 if err != nil {
841 t.Fatalf("ensuring path %q exists: %s", cwd, err)
842 }
843
844 cmd := exec.Command("sh", "-euxc", script)
845 cmd.Dir = cwd
846 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
847
848 if out, err := cmd.CombinedOutput(); err != nil {
849 t.Fatalf("execution error: %v, output %s", err, out)
850 }
851}
852
853func TestSetTemplates_e2e(t *testing.T) {
854 repositoryDir := t.TempDir()
855
856 // setup: initialize the repository and all of its branches
857 runScript(t, repositoryDir, "git init -b master")
858 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
859 desc := zoekt.Repository{}
860 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
861 t.Fatalf("setTemplatesFromConfig: %v", err)
862 }
863
864 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
865 t.Errorf("got %q, want %q", got, want)
866 }
867}
868
869func TestSetTemplates(t *testing.T) {
870 base := "https://example.com/repo/name"
871 version := "VERSION"
872 path := "dir/name.txt"
873 lineNumber := 10
874 cases := []struct {
875 typ string
876 commit string
877 file string
878 line string
879 }{{
880 typ: "gitiles",
881 commit: "https://example.com/repo/name/%2B/VERSION",
882 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
883 line: "#10",
884 }, {
885 typ: "github",
886 commit: "https://example.com/repo/name/commit/VERSION",
887 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
888 line: "#L10",
889 }, {
890 typ: "cgit",
891 commit: "https://example.com/repo/name/commit/?id=VERSION",
892 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
893 line: "#n10",
894 }, {
895 typ: "gitweb",
896 commit: "https://example.com/repo/name;a=commit;h=VERSION",
897 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
898 line: "#l10",
899 }, {
900 typ: "source.bazel.build",
901 commit: "https://example.com/repo/name/%2B/VERSION",
902 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
903 line: ";l=10",
904 }, {
905 typ: "bitbucket-server",
906 commit: "https://example.com/repo/name/commits/VERSION",
907 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
908 line: "#10",
909 }, {
910 typ: "gitlab",
911 commit: "https://example.com/repo/name/-/commit/VERSION",
912 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
913 line: "#L10",
914 }, {
915 typ: "gitea",
916 commit: "https://example.com/repo/name/commit/VERSION",
917 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
918 line: "#L10",
919 }}
920
921 for _, tc := range cases {
922 t.Run(tc.typ, func(t *testing.T) {
923 assertOutput := func(templateText string, want string) {
924 t.Helper()
925
926 tt, err := zoekt.ParseTemplate(templateText)
927 if err != nil {
928 t.Fatal(err)
929 }
930
931 var sb strings.Builder
932 err = tt.Execute(&sb, map[string]any{
933 "Version": version,
934 "Path": path,
935 "LineNumber": lineNumber,
936 })
937 if err != nil {
938 t.Fatal(err)
939 }
940 if got := sb.String(); got != want {
941 t.Fatalf("want: %q\ngot: %q", want, got)
942 }
943 }
944
945 var repo zoekt.Repository
946 u, _ := url.Parse(base)
947 err := setTemplates(&repo, u, tc.typ)
948 if err != nil {
949 t.Fatal(err)
950 }
951 assertOutput(repo.CommitURLTemplate, tc.commit)
952 assertOutput(repo.FileURLTemplate, tc.file)
953 assertOutput(repo.LineFragmentTemplate, tc.line)
954 })
955 }
956}
957
958func BenchmarkPrepareNormalBuild(b *testing.B) {
959 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
960 repoDir := "/path/to/your/repo"
961 repo, err := git.PlainOpen(repoDir)
962 if err != nil {
963 b.Fatalf("Failed to open test repository: %v", err)
964 }
965
966 opts := Options{
967 RepoDir: repoDir,
968 Submodules: false,
969 BranchPrefix: "refs/heads/",
970 Branches: []string{"main"},
971 BuildOptions: build.Options{
972 RepositoryDescription: zoekt.Repository{
973 Name: "test-repo",
974 URL: "https://github.com/example/test-repo",
975 },
976 },
977 }
978
979 b.ReportAllocs()
980
981 repos, branchVersions, err := prepareNormalBuild(opts, repo)
982 if err != nil {
983 b.Fatalf("prepareNormalBuild failed: %v", err)
984 }
985
986 runtime.GC()
987
988 var m runtime.MemStats
989 runtime.ReadMemStats(&m)
990 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
991 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
992
993 if len(repos) == 0 || len(branchVersions) == 0 {
994 b.Fatalf("Unexpected empty results")
995 }
996}