fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35
36 "github.com/sourcegraph/zoekt"
37 "github.com/sourcegraph/zoekt/ignore"
38 "github.com/sourcegraph/zoekt/index"
39 "github.com/sourcegraph/zoekt/query"
40 "github.com/sourcegraph/zoekt/search"
41)
42
43func TestIndexEmptyRepo(t *testing.T) {
44 dir := t.TempDir()
45
46 cmd := exec.Command("git", "init", "-b", "master", "repo")
47 cmd.Dir = dir
48
49 if err := cmd.Run(); err != nil {
50 t.Fatalf("cmd.Run: %v", err)
51 }
52
53 desc := zoekt.Repository{
54 Name: "repo",
55 }
56 opts := Options{
57 RepoDir: filepath.Join(dir, "repo", ".git"),
58 BuildOptions: index.Options{
59 RepositoryDescription: desc,
60 IndexDir: dir,
61 },
62 }
63
64 if _, err := IndexGitRepo(opts); err != nil {
65 t.Fatalf("IndexGitRepo: %v", err)
66 }
67}
68
69func TestIndexNonexistentRepo(t *testing.T) {
70 dir := t.TempDir()
71 desc := zoekt.Repository{
72 Name: "nonexistent",
73 }
74 opts := Options{
75 RepoDir: "does/not/exist",
76 Branches: []string{"main"},
77 BuildOptions: index.Options{
78 RepositoryDescription: desc,
79 IndexDir: dir,
80 },
81 }
82
83 if _, err := IndexGitRepo(opts); err == nil {
84 t.Fatal("expected error, got none")
85 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
86 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
87 }
88}
89
90func TestIndexTinyRepo(t *testing.T) {
91 // Create a repo with one file in it.
92 dir := t.TempDir()
93 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
94
95 repoDir := filepath.Join(dir, "repo")
96 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
97 t.Fatalf("WriteFile: %v", err)
98 }
99 executeCommand(t, repoDir, exec.Command("git", "add", "."))
100 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
101
102 // Test that indexing accepts both the repo directory, and the .git subdirectory.
103 for _, testDir := range []string{"repo", "repo/.git"} {
104 opts := Options{
105 RepoDir: filepath.Join(dir, testDir),
106 Branches: []string{"main"},
107 BuildOptions: index.Options{
108 RepositoryDescription: zoekt.Repository{Name: "repo"},
109 IndexDir: dir,
110 },
111 }
112
113 if _, err := IndexGitRepo(opts); err != nil {
114 t.Fatalf("unexpected error %v", err)
115 }
116
117 searcher, err := search.NewDirectorySearcher(dir)
118 if err != nil {
119 t.Fatal("NewDirectorySearcher", err)
120 }
121
122 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
123 searcher.Close()
124
125 if err != nil {
126 t.Fatal("search failed", err)
127 }
128
129 if len(results.Files) != 1 {
130 t.Fatalf("got search result %v, want 1 file", results.Files)
131 }
132 }
133}
134
135func TestIndexGitRepo_Worktree(t *testing.T) {
136 _, worktreeDir := initGitWorktree(t, "file1.go", "package main\n\nfunc main() {}\n")
137 indexDir := t.TempDir()
138
139 opts := Options{
140 RepoDir: worktreeDir,
141 Branches: []string{"HEAD"},
142 BuildOptions: index.Options{
143 RepositoryDescription: zoekt.Repository{Name: "repo"},
144 IndexDir: indexDir,
145 },
146 }
147
148 if _, err := IndexGitRepo(opts); err != nil {
149 t.Fatalf("IndexGitRepo(worktree): %v", err)
150 }
151
152 searcher, err := search.NewDirectorySearcher(indexDir)
153 if err != nil {
154 t.Fatal("NewDirectorySearcher", err)
155 }
156 defer searcher.Close()
157
158 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
159 if err != nil {
160 t.Fatal("search failed", err)
161 }
162
163 if len(results.Files) != 1 {
164 t.Fatalf("got search result %v, want 1 file", results.Files)
165 }
166}
167
168func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
169 cmd.Dir = dir
170 cmd.Env = []string{
171 "GIT_CONFIG_GLOBAL=",
172 "GIT_CONFIG_SYSTEM=",
173 "GIT_COMMITTER_NAME=Kierkegaard",
174 "GIT_COMMITTER_EMAIL=soren@apache.com",
175 "GIT_AUTHOR_NAME=Kierkegaard",
176 "GIT_AUTHOR_EMAIL=soren@apache.com",
177 }
178 if err := cmd.Run(); err != nil {
179 t.Fatalf("cmd.Run: %v", err)
180 }
181 return cmd
182}
183
184func initGitWorktree(t *testing.T, fileName, content string) (string, string) {
185 t.Helper()
186
187 dir := t.TempDir()
188 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
189
190 repoDir := filepath.Join(dir, "repo")
191 if err := os.WriteFile(filepath.Join(repoDir, fileName), []byte(content), 0o644); err != nil {
192 t.Fatalf("WriteFile: %v", err)
193 }
194 executeCommand(t, repoDir, exec.Command("git", "config", "remote.origin.url", "git@github.com:sourcegraph/zoekt.git"))
195 executeCommand(t, repoDir, exec.Command("git", "add", "."))
196 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
197
198 worktreeDir := filepath.Join(dir, "wt")
199 executeCommand(t, repoDir, exec.Command("git", "worktree", "add", "-b", "worktree-branch", worktreeDir))
200
201 return repoDir, worktreeDir
202}
203
204func TestIndexDeltaBasic(t *testing.T) {
205 type branchToDocumentMap map[string][]index.Document
206
207 type step struct {
208 name string
209 addedDocuments branchToDocumentMap
210 deletedDocuments branchToDocumentMap
211 optFn func(t *testing.T, options *Options)
212
213 expectedFallbackToNormalBuild bool
214 expectedDocuments []index.Document
215 }
216
217 helloWorld := index.Document{Name: "hello_world.txt", Content: []byte("hello")}
218
219 fruitV1 := index.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
220 fruitV1InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
221 fruitV1WithNewName := index.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
222
223 fruitV2 := index.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
224 fruitV2InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
225
226 fruitV3 := index.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
227 fruitV4 := index.Document{Name: "best_fruit.txt", Content: []byte("apples")}
228
229 foo := index.Document{Name: "foo.txt", Content: []byte("bar")}
230
231 emptySourcegraphIgnore := index.Document{Name: ignore.IgnoreFile}
232 sourcegraphIgnoreWithContent := index.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
233
234 for _, test := range []struct {
235 name string
236 branches []string
237 steps []step
238 }{
239 {
240 name: "modification",
241 branches: []string{"main"},
242 steps: []step{
243 {
244 name: "setup",
245 addedDocuments: branchToDocumentMap{
246 "main": []index.Document{helloWorld, fruitV1},
247 },
248
249 expectedDocuments: []index.Document{helloWorld, fruitV1},
250 },
251 {
252 name: "add newer version of fruits",
253 addedDocuments: branchToDocumentMap{
254 "main": []index.Document{fruitV2},
255 },
256 optFn: func(t *testing.T, o *Options) {
257 o.BuildOptions.IsDelta = true
258 },
259
260 expectedDocuments: []index.Document{helloWorld, fruitV2},
261 },
262 },
263 },
264 {
265 name: "modification only inside nested folder",
266 branches: []string{"main"},
267 steps: []step{
268 {
269 name: "setup",
270 addedDocuments: branchToDocumentMap{
271 "main": []index.Document{foo, fruitV1InFolder},
272 },
273
274 expectedDocuments: []index.Document{foo, fruitV1InFolder},
275 },
276 {
277 name: "add newer version of fruits inside folder",
278 addedDocuments: branchToDocumentMap{
279 "main": []index.Document{fruitV2InFolder},
280 },
281 optFn: func(t *testing.T, o *Options) {
282 o.BuildOptions.IsDelta = true
283 },
284
285 expectedDocuments: []index.Document{foo, fruitV2InFolder},
286 },
287 },
288 },
289 {
290 name: "addition",
291 branches: []string{"main"},
292 steps: []step{
293 {
294 name: "setup",
295 addedDocuments: branchToDocumentMap{
296 "main": []index.Document{helloWorld, fruitV1},
297 },
298
299 expectedDocuments: []index.Document{helloWorld, fruitV1},
300 },
301 {
302 name: "add new file - foo",
303 addedDocuments: branchToDocumentMap{
304 "main": []index.Document{foo},
305 },
306 optFn: func(t *testing.T, o *Options) {
307 o.BuildOptions.IsDelta = true
308 },
309
310 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
311 },
312 },
313 },
314 {
315 name: "deletion",
316 branches: []string{"main"},
317 steps: []step{
318 {
319 name: "setup",
320 addedDocuments: branchToDocumentMap{
321 "main": []index.Document{helloWorld, fruitV1, foo},
322 },
323
324 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
325 },
326 {
327 name: "delete foo file",
328 addedDocuments: nil,
329 deletedDocuments: branchToDocumentMap{
330 "main": []index.Document{foo},
331 },
332
333 optFn: func(t *testing.T, o *Options) {
334 o.BuildOptions.IsDelta = true
335 },
336
337 expectedDocuments: []index.Document{helloWorld, fruitV1},
338 },
339 },
340 },
341 {
342 name: "addition and deletion on only one branch",
343 branches: []string{"main", "release", "dev"},
344 steps: []step{
345 {
346 name: "setup",
347 addedDocuments: branchToDocumentMap{
348 "main": []index.Document{fruitV1},
349 "release": []index.Document{fruitV2},
350 "dev": []index.Document{fruitV3},
351 },
352
353 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
354 },
355 {
356 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
357 addedDocuments: branchToDocumentMap{
358 "dev": []index.Document{fruitV4},
359 },
360 deletedDocuments: branchToDocumentMap{
361 "main": []index.Document{fruitV1},
362 },
363
364 optFn: func(t *testing.T, o *Options) {
365 o.BuildOptions.IsDelta = true
366 },
367
368 expectedDocuments: []index.Document{fruitV2, fruitV4},
369 },
370 },
371 },
372 {
373 name: "rename",
374 branches: []string{"main", "release"},
375 steps: []step{
376 {
377 name: "setup",
378 addedDocuments: branchToDocumentMap{
379 "main": []index.Document{fruitV1},
380 "release": []index.Document{fruitV2},
381 },
382 expectedDocuments: []index.Document{fruitV1, fruitV2},
383 },
384 {
385 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
386 addedDocuments: branchToDocumentMap{
387 "main": []index.Document{fruitV1WithNewName},
388 },
389 deletedDocuments: branchToDocumentMap{
390 "main": []index.Document{fruitV1},
391 },
392
393 optFn: func(t *testing.T, o *Options) {
394 o.BuildOptions.IsDelta = true
395 },
396
397 expectedDocuments: []index.Document{fruitV1WithNewName, fruitV2},
398 },
399 },
400 },
401 {
402 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
403 branches: []string{"main", "dev"},
404 steps: []step{
405 {
406 name: "setup",
407 addedDocuments: branchToDocumentMap{
408 "main": []index.Document{fruitV1},
409 "dev": []index.Document{fruitV2},
410 },
411 expectedDocuments: []index.Document{fruitV1, fruitV2},
412 },
413 {
414 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
415 addedDocuments: branchToDocumentMap{
416 "main": []index.Document{fruitV2},
417 "dev": []index.Document{fruitV3},
418 },
419
420 optFn: func(t *testing.T, o *Options) {
421 o.BuildOptions.IsDelta = true
422 },
423
424 expectedDocuments: []index.Document{fruitV2, fruitV3},
425 },
426 },
427 },
428 {
429 name: "no-op delta builds (reindexing the same commits)",
430 branches: []string{"main", "dev"},
431 steps: []step{
432 {
433 name: "setup",
434 addedDocuments: branchToDocumentMap{
435 "main": []index.Document{fruitV1, foo},
436 "dev": []index.Document{helloWorld},
437 },
438 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
439 },
440 {
441 name: "first no-op (normal build -> delta build)",
442 optFn: func(t *testing.T, o *Options) {
443 o.BuildOptions.IsDelta = true
444 },
445
446 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
447 },
448 {
449 name: "second no-op (delta build -> delta build)",
450 optFn: func(t *testing.T, o *Options) {
451 o.BuildOptions.IsDelta = true
452 },
453
454 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
455 },
456 },
457 },
458 {
459 name: "should fallback to normal build if no prior shards exist",
460 branches: []string{"main"},
461 steps: []step{
462 {
463 name: "attempt delta build on a repository that hasn't been indexed yet",
464 addedDocuments: branchToDocumentMap{
465 "main": []index.Document{helloWorld},
466 },
467 optFn: func(t *testing.T, o *Options) {
468 o.BuildOptions.IsDelta = true
469 },
470
471 expectedFallbackToNormalBuild: true,
472 expectedDocuments: []index.Document{helloWorld},
473 },
474 },
475 },
476 {
477 name: "should fallback to normal build if the set of requested repository branches changes",
478 branches: []string{"main", "release", "dev"},
479 steps: []step{
480 {
481 name: "setup",
482 addedDocuments: branchToDocumentMap{
483 "main": []index.Document{fruitV1},
484 "release": []index.Document{fruitV2},
485 "dev": []index.Document{fruitV3},
486 },
487
488 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
489 },
490 {
491 name: "try delta build after dropping 'main' branch from index ",
492 addedDocuments: branchToDocumentMap{
493 "release": []index.Document{fruitV4},
494 },
495 optFn: func(t *testing.T, o *Options) {
496 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
497 o.BuildOptions.IsDelta = true
498 },
499
500 expectedFallbackToNormalBuild: true,
501 expectedDocuments: []index.Document{fruitV3, fruitV4},
502 },
503 },
504 },
505 {
506 name: "should expand branches correctly when using wildcards in branch names",
507 branches: []string{"release/1", "release/2"},
508 steps: []step{
509 {
510 name: "setup",
511 addedDocuments: branchToDocumentMap{
512 "release/1": []index.Document{fruitV1},
513 "release/2": []index.Document{fruitV2},
514 },
515
516 expectedDocuments: []index.Document{fruitV1, fruitV2},
517 },
518 {
519 name: "try delta build with wildcard in branches",
520 optFn: func(t *testing.T, o *Options) {
521 // use a wildcard here
522 o.Branches = []string{"HEAD", "release/*"}
523 o.BuildOptions.IsDelta = true
524 },
525
526 expectedDocuments: []index.Document{fruitV1, fruitV2},
527 },
528 },
529 },
530 {
531 name: "should fallback to normal build if one or more index options updates requires a full build",
532 branches: []string{"main"},
533 steps: []step{
534 {
535 name: "setup",
536 addedDocuments: branchToDocumentMap{
537 "main": []index.Document{fruitV1},
538 },
539
540 expectedDocuments: []index.Document{fruitV1},
541 },
542 {
543 name: "try delta build after updating Disable CTags index option",
544 addedDocuments: branchToDocumentMap{
545 "main": []index.Document{fruitV2},
546 },
547 optFn: func(t *testing.T, o *Options) {
548 o.BuildOptions.IsDelta = true
549 o.BuildOptions.DisableCTags = true
550 },
551
552 expectedFallbackToNormalBuild: true,
553 expectedDocuments: []index.Document{fruitV2},
554 },
555 {
556 name: "try delta build after reverting Disable CTags index option",
557 addedDocuments: branchToDocumentMap{
558 "main": []index.Document{fruitV3},
559 },
560 optFn: func(t *testing.T, o *Options) {
561 o.BuildOptions.IsDelta = true
562 o.BuildOptions.DisableCTags = false
563 },
564
565 expectedFallbackToNormalBuild: true,
566 expectedDocuments: []index.Document{fruitV3},
567 },
568 },
569 },
570 {
571 name: "should successfully perform multiple delta builds after disabling symbols",
572 branches: []string{"main"},
573 steps: []step{
574 {
575 name: "setup",
576 addedDocuments: branchToDocumentMap{
577 "main": []index.Document{fruitV1},
578 },
579
580 expectedDocuments: []index.Document{fruitV1},
581 },
582 {
583 name: "try delta build after updating Disable CTags index option",
584 addedDocuments: branchToDocumentMap{
585 "main": []index.Document{fruitV2},
586 },
587 optFn: func(t *testing.T, o *Options) {
588 o.BuildOptions.IsDelta = true
589 o.BuildOptions.DisableCTags = true
590 },
591
592 expectedFallbackToNormalBuild: true,
593 expectedDocuments: []index.Document{fruitV2},
594 },
595 {
596 name: "try another delta build while CTags is still disabled",
597 addedDocuments: branchToDocumentMap{
598 "main": []index.Document{fruitV3},
599 },
600 optFn: func(t *testing.T, o *Options) {
601 o.BuildOptions.IsDelta = true
602 o.BuildOptions.DisableCTags = true
603 },
604
605 expectedDocuments: []index.Document{fruitV3},
606 },
607 },
608 },
609 {
610 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
611 branches: []string{"main"},
612 steps: []step{
613 {
614 name: "setup",
615 addedDocuments: branchToDocumentMap{
616 "main": []index.Document{emptySourcegraphIgnore},
617 },
618
619 expectedDocuments: []index.Document{emptySourcegraphIgnore},
620 },
621 {
622 name: "attempt delta build after modifying ignore file",
623 addedDocuments: branchToDocumentMap{
624 "main": []index.Document{sourcegraphIgnoreWithContent},
625 },
626 optFn: func(t *testing.T, o *Options) {
627 o.BuildOptions.IsDelta = true
628 },
629
630 expectedFallbackToNormalBuild: true,
631 expectedDocuments: []index.Document{sourcegraphIgnoreWithContent},
632 },
633 },
634 },
635 {
636 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
637 branches: []string{"main"},
638 steps: []step{
639 {
640 name: "setup: first shard",
641 addedDocuments: branchToDocumentMap{
642 "main": []index.Document{foo},
643 },
644
645 expectedDocuments: []index.Document{foo},
646 },
647 {
648 name: "setup: second shard (delta)",
649 addedDocuments: branchToDocumentMap{
650 "main": []index.Document{fruitV1},
651 },
652 optFn: func(t *testing.T, o *Options) {
653 o.BuildOptions.IsDelta = true
654 },
655
656 expectedDocuments: []index.Document{foo, fruitV1},
657 },
658 {
659 name: "setup: third shard (delta)",
660 addedDocuments: branchToDocumentMap{
661 "main": []index.Document{helloWorld},
662 },
663 optFn: func(t *testing.T, o *Options) {
664 o.BuildOptions.IsDelta = true
665 },
666
667 expectedDocuments: []index.Document{foo, fruitV1, helloWorld},
668 },
669 {
670 name: "attempt another delta build after we already blew past the shard threshold",
671 addedDocuments: branchToDocumentMap{
672 "main": []index.Document{fruitV2InFolder},
673 },
674 optFn: func(t *testing.T, o *Options) {
675 o.DeltaShardNumberFallbackThreshold = 2
676 o.BuildOptions.IsDelta = true
677 },
678
679 expectedFallbackToNormalBuild: true,
680 expectedDocuments: []index.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
681 },
682 },
683 },
684 } {
685 test := test
686
687 t.Run(test.name, func(t *testing.T) {
688 t.Parallel()
689
690 indexDir := t.TempDir()
691 repositoryDir := t.TempDir()
692
693 // setup: initialize the repository and all of its branches
694 runScript(t, repositoryDir, "git init -b master")
695 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
696 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
697
698 for _, b := range test.branches {
699 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
700 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
701 }
702
703 for _, step := range test.steps {
704 t.Run(step.name, func(t *testing.T) {
705 for _, b := range test.branches {
706 // setup: for each branch, process any document deletions / additions and commit those changes
707
708 hadChange := false
709
710 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
711
712 for _, d := range step.deletedDocuments[b] {
713 hadChange = true
714
715 file := filepath.Join(repositoryDir, d.Name)
716
717 err := os.Remove(file)
718 if err != nil {
719 t.Fatalf("deleting file %q: %s", d.Name, err)
720 }
721
722 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
723 }
724
725 for _, d := range step.addedDocuments[b] {
726 hadChange = true
727
728 file := filepath.Join(repositoryDir, d.Name)
729
730 err := os.MkdirAll(filepath.Dir(file), 0o755)
731 if err != nil {
732 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
733 }
734
735 err = os.WriteFile(file, d.Content, 0o644)
736 if err != nil {
737 t.Fatalf("writing file %q: %s", d.Name, err)
738 }
739
740 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
741 }
742
743 if !hadChange {
744 continue
745 }
746
747 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
748 }
749
750 // setup: prepare indexOptions with given overrides
751 buildOptions := index.Options{
752 IndexDir: indexDir,
753 RepositoryDescription: zoekt.Repository{
754 Name: "repository",
755 },
756 IsDelta: false,
757 }
758 buildOptions.SetDefaults()
759
760 branches := append([]string{"HEAD"}, test.branches...)
761
762 options := Options{
763 RepoDir: filepath.Join(repositoryDir, ".git"),
764 BuildOptions: buildOptions,
765 Branches: branches,
766 }
767
768 if step.optFn != nil {
769 step.optFn(t, &options)
770 }
771
772 // setup: prepare spy versions of prepare delta / normal build so that we can observe
773 // whether they were called appropriately
774 deltaBuildCalled := false
775 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
776 deltaBuildCalled = true
777 return prepareDeltaBuild(options, repository)
778 }
779
780 normalBuildCalled := false
781 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
782 normalBuildCalled = true
783 return prepareNormalBuild(options, repository)
784 }
785
786 // run test
787 _, err := indexGitRepo(options, gitIndexConfig{
788 prepareDeltaBuild: prepareDeltaSpy,
789 prepareNormalBuild: prepareNormalSpy,
790 })
791 if err != nil {
792 t.Fatalf("IndexGitRepo: %s", err)
793 }
794
795 if options.BuildOptions.IsDelta != deltaBuildCalled {
796 // We should always try a delta build if we request it in the options.
797 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
798 }
799
800 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
801 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
802 // asked for a delta build in the first place.
803 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
804 }
805
806 // examine outcome: load shards into a searcher instance and run a dummy search query
807 // that returns every document contained in the shards
808 //
809 // then, compare returned set of documents with the expected set for the step and see if they agree
810
811 ss, err := search.NewDirectorySearcher(indexDir)
812 if err != nil {
813 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
814 }
815 defer ss.Close()
816
817 searchOpts := &zoekt.SearchOptions{Whole: true}
818 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
819 if err != nil {
820 t.Fatalf("Search: %s", err)
821 }
822
823 var receivedDocuments []index.Document
824 for _, f := range result.Files {
825 receivedDocuments = append(receivedDocuments, index.Document{
826 Name: f.FileName,
827 Content: f.Content,
828 })
829 }
830
831 for _, docs := range [][]index.Document{step.expectedDocuments, receivedDocuments} {
832 sort.Slice(docs, func(i, j int) bool {
833 a, b := docs[i], docs[j]
834
835 // first compare names, then fallback to contents if the names are equal
836
837 if a.Name < b.Name {
838 return true
839 }
840
841 if a.Name > b.Name {
842 return false
843 }
844
845 return bytes.Compare(a.Content, b.Content) < 0
846 })
847 }
848
849 compareOptions := []cmp.Option{
850 cmpopts.IgnoreFields(index.Document{}, "Branches"),
851 cmpopts.EquateEmpty(),
852 }
853
854 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
855 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
856 }
857 })
858 }
859 })
860 }
861}
862
863func runScript(t *testing.T, cwd string, script string) {
864 t.Helper()
865
866 err := os.MkdirAll(cwd, 0o755)
867 if err != nil {
868 t.Fatalf("ensuring path %q exists: %s", cwd, err)
869 }
870
871 cmd := exec.Command("sh", "-euxc", script)
872 cmd.Dir = cwd
873 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
874
875 if out, err := cmd.CombinedOutput(); err != nil {
876 t.Fatalf("execution error: %v, output %s", err, out)
877 }
878}
879
880func TestSetTemplates_e2e(t *testing.T) {
881 repositoryDir := t.TempDir()
882
883 // setup: initialize the repository and all of its branches
884 runScript(t, repositoryDir, "git init -b master")
885 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
886 desc := zoekt.Repository{}
887 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
888 t.Fatalf("setTemplatesFromConfig: %v", err)
889 }
890
891 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
892 t.Errorf("got %q, want %q", got, want)
893 }
894}
895
896func TestSetTemplates_Worktree(t *testing.T) {
897 _, worktreeDir := initGitWorktree(t, "hello.go", "package main\n")
898 desc := zoekt.Repository{}
899
900 if err := setTemplatesFromConfig(&desc, worktreeDir); err != nil {
901 t.Fatalf("setTemplatesFromConfig(worktree): %v", err)
902 }
903
904 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
905 t.Errorf("got %q, want %q", got, want)
906 }
907}
908
909func TestSetTemplates(t *testing.T) {
910 base := "https://example.com/repo/name"
911 version := "VERSION"
912 path := "dir/name.txt"
913 lineNumber := 10
914 cases := []struct {
915 typ string
916 commit string
917 file string
918 line string
919 }{{
920 typ: "gitiles",
921 commit: "https://example.com/repo/name/%2B/VERSION",
922 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
923 line: "#10",
924 }, {
925 typ: "github",
926 commit: "https://example.com/repo/name/commit/VERSION",
927 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
928 line: "#L10",
929 }, {
930 typ: "cgit",
931 commit: "https://example.com/repo/name/commit/?id=VERSION",
932 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
933 line: "#n10",
934 }, {
935 typ: "gitweb",
936 commit: "https://example.com/repo/name;a=commit;h=VERSION",
937 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
938 line: "#l10",
939 }, {
940 typ: "source.bazel.build",
941 commit: "https://example.com/repo/name/%2B/VERSION",
942 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
943 line: ";l=10",
944 }, {
945 typ: "bitbucket-server",
946 commit: "https://example.com/repo/name/commits/VERSION",
947 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
948 line: "#10",
949 }, {
950 typ: "gitlab",
951 commit: "https://example.com/repo/name/-/commit/VERSION",
952 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
953 line: "#L10",
954 }, {
955 typ: "gitea",
956 commit: "https://example.com/repo/name/commit/VERSION",
957 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
958 line: "#L10",
959 }}
960
961 for _, tc := range cases {
962 t.Run(tc.typ, func(t *testing.T) {
963 assertOutput := func(templateText string, want string) {
964 t.Helper()
965
966 tt, err := index.ParseTemplate(templateText)
967 if err != nil {
968 t.Fatal(err)
969 }
970
971 var sb strings.Builder
972 err = tt.Execute(&sb, map[string]any{
973 "Version": version,
974 "Path": path,
975 "LineNumber": lineNumber,
976 })
977 if err != nil {
978 t.Fatal(err)
979 }
980 if got := sb.String(); got != want {
981 t.Fatalf("want: %q\ngot: %q", want, got)
982 }
983 }
984
985 var repo zoekt.Repository
986 u, _ := url.Parse(base)
987 err := setTemplates(&repo, u, tc.typ)
988 if err != nil {
989 t.Fatal(err)
990 }
991 assertOutput(repo.CommitURLTemplate, tc.commit)
992 assertOutput(repo.FileURLTemplate, tc.file)
993 assertOutput(repo.LineFragmentTemplate, tc.line)
994 })
995 }
996}
997
998func BenchmarkPrepareNormalBuild(b *testing.B) {
999 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
1000 repoDir := "/path/to/your/repo"
1001 repo, err := git.PlainOpen(repoDir)
1002 if err != nil {
1003 b.Fatalf("Failed to open test repository: %v", err)
1004 }
1005
1006 opts := Options{
1007 RepoDir: repoDir,
1008 Submodules: false,
1009 BranchPrefix: "refs/heads/",
1010 Branches: []string{"main"},
1011 BuildOptions: index.Options{
1012 RepositoryDescription: zoekt.Repository{
1013 Name: "test-repo",
1014 URL: "https://github.com/example/test-repo",
1015 },
1016 },
1017 }
1018
1019 b.ReportAllocs()
1020
1021 repos, branchVersions, err := prepareNormalBuild(opts, repo)
1022 if err != nil {
1023 b.Fatalf("prepareNormalBuild failed: %v", err)
1024 }
1025
1026 runtime.GC()
1027
1028 var m runtime.MemStats
1029 runtime.ReadMemStats(&m)
1030 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
1031 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
1032
1033 if len(repos) == 0 || len(branchVersions) == 0 {
1034 b.Fatalf("Unexpected empty results")
1035 }
1036}