fork of https://github.com/sourcegraph/zoekt
1// Copyright 2021 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "errors"
21 "fmt"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/go-git/go-git/v5"
32 "github.com/go-git/go-git/v5/plumbing"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35
36 "github.com/sourcegraph/zoekt"
37 "github.com/sourcegraph/zoekt/ignore"
38 "github.com/sourcegraph/zoekt/index"
39 "github.com/sourcegraph/zoekt/query"
40 "github.com/sourcegraph/zoekt/search"
41)
42
43func TestIndexEmptyRepo(t *testing.T) {
44 dir := t.TempDir()
45
46 cmd := exec.Command("git", "init", "-b", "master", "repo")
47 cmd.Dir = dir
48
49 if err := cmd.Run(); err != nil {
50 t.Fatalf("cmd.Run: %v", err)
51 }
52
53 desc := zoekt.Repository{
54 Name: "repo",
55 }
56 opts := Options{
57 RepoDir: filepath.Join(dir, "repo", ".git"),
58 BuildOptions: index.Options{
59 RepositoryDescription: desc,
60 IndexDir: dir,
61 },
62 }
63
64 if _, err := IndexGitRepo(opts); err != nil {
65 t.Fatalf("IndexGitRepo: %v", err)
66 }
67}
68
69func TestIndexNonexistentRepo(t *testing.T) {
70 dir := t.TempDir()
71 desc := zoekt.Repository{
72 Name: "nonexistent",
73 }
74 opts := Options{
75 RepoDir: "does/not/exist",
76 Branches: []string{"main"},
77 BuildOptions: index.Options{
78 RepositoryDescription: desc,
79 IndexDir: dir,
80 },
81 }
82
83 if _, err := IndexGitRepo(opts); err == nil {
84 t.Fatal("expected error, got none")
85 } else if !errors.Is(err, git.ErrRepositoryNotExists) {
86 t.Fatalf("expected git.ErrRepositoryNotExists, got %v", err)
87 }
88}
89
90func TestIndexTinyRepo(t *testing.T) {
91 // Create a repo with one file in it.
92 dir := t.TempDir()
93 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
94
95 repoDir := filepath.Join(dir, "repo")
96 if err := os.WriteFile(filepath.Join(repoDir, "file1.go"), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil {
97 t.Fatalf("WriteFile: %v", err)
98 }
99 executeCommand(t, repoDir, exec.Command("git", "add", "."))
100 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
101
102 // Test that indexing accepts both the repo directory, and the .git subdirectory.
103 for _, testDir := range []string{"repo", "repo/.git"} {
104 opts := Options{
105 RepoDir: filepath.Join(dir, testDir),
106 Branches: []string{"main"},
107 BuildOptions: index.Options{
108 RepositoryDescription: zoekt.Repository{Name: "repo"},
109 IndexDir: dir,
110 },
111 }
112
113 if _, err := IndexGitRepo(opts); err != nil {
114 t.Fatalf("unexpected error %v", err)
115 }
116
117 searcher, err := search.NewDirectorySearcher(dir)
118 if err != nil {
119 t.Fatal("NewDirectorySearcher", err)
120 }
121
122 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
123 searcher.Close()
124
125 if err != nil {
126 t.Fatal("search failed", err)
127 }
128
129 if len(results.Files) != 1 {
130 t.Fatalf("got search result %v, want 1 file", results.Files)
131 }
132 }
133}
134
135func TestIndexGitRepo_Worktree(t *testing.T) {
136 _, worktreeDir := initGitWorktree(t, "file1.go", "package main\n\nfunc main() {}\n")
137 indexDir := t.TempDir()
138
139 opts := Options{
140 RepoDir: worktreeDir,
141 Branches: []string{"HEAD"},
142 BuildOptions: index.Options{
143 RepositoryDescription: zoekt.Repository{Name: "repo"},
144 IndexDir: indexDir,
145 },
146 }
147
148 if _, err := IndexGitRepo(opts); err != nil {
149 t.Fatalf("IndexGitRepo(worktree): %v", err)
150 }
151
152 searcher, err := search.NewDirectorySearcher(indexDir)
153 if err != nil {
154 t.Fatal("NewDirectorySearcher", err)
155 }
156 defer searcher.Close()
157
158 results, err := searcher.Search(context.Background(), &query.Const{Value: true}, &zoekt.SearchOptions{})
159 if err != nil {
160 t.Fatal("search failed", err)
161 }
162
163 if len(results.Files) != 1 {
164 t.Fatalf("got search result %v, want 1 file", results.Files)
165 }
166}
167
168func TestCatfileFilterSpec(t *testing.T) {
169 for _, tc := range []struct {
170 name string
171 opts Options
172 want string
173 }{
174 {
175 name: "size max",
176 opts: Options{BuildOptions: index.Options{SizeMax: 1 << 20}},
177 want: "blob:limit=1048577",
178 },
179 {
180 name: "large file exception disables filter",
181 opts: Options{BuildOptions: index.Options{SizeMax: 1 << 20, LargeFiles: []string{"*.bin"}}},
182 want: "",
183 },
184 {
185 name: "zero size max disables filter",
186 opts: Options{BuildOptions: index.Options{SizeMax: 0}},
187 want: "",
188 },
189 } {
190 t.Run(tc.name, func(t *testing.T) {
191 if got := catfileFilterSpec(tc.opts); got != tc.want {
192 t.Fatalf("catfileFilterSpec() = %q, want %q", got, tc.want)
193 }
194 })
195 }
196}
197
198func executeCommand(t *testing.T, dir string, cmd *exec.Cmd) *exec.Cmd {
199 cmd.Dir = dir
200 cmd.Env = []string{
201 "GIT_CONFIG_GLOBAL=",
202 "GIT_CONFIG_SYSTEM=",
203 "GIT_COMMITTER_NAME=Kierkegaard",
204 "GIT_COMMITTER_EMAIL=soren@apache.com",
205 "GIT_AUTHOR_NAME=Kierkegaard",
206 "GIT_AUTHOR_EMAIL=soren@apache.com",
207 }
208 if err := cmd.Run(); err != nil {
209 t.Fatalf("cmd.Run: %v", err)
210 }
211 return cmd
212}
213
214func initGitWorktree(t *testing.T, fileName, content string) (string, string) {
215 t.Helper()
216
217 dir := t.TempDir()
218 executeCommand(t, dir, exec.Command("git", "init", "-b", "main", "repo"))
219
220 repoDir := filepath.Join(dir, "repo")
221 if err := os.WriteFile(filepath.Join(repoDir, fileName), []byte(content), 0o644); err != nil {
222 t.Fatalf("WriteFile: %v", err)
223 }
224 executeCommand(t, repoDir, exec.Command("git", "config", "remote.origin.url", "git@github.com:sourcegraph/zoekt.git"))
225 executeCommand(t, repoDir, exec.Command("git", "add", "."))
226 executeCommand(t, repoDir, exec.Command("git", "commit", "-m", "initial commit"))
227
228 worktreeDir := filepath.Join(dir, "wt")
229 executeCommand(t, repoDir, exec.Command("git", "worktree", "add", "-b", "worktree-branch", worktreeDir))
230
231 return repoDir, worktreeDir
232}
233
234func TestIndexDeltaBasic(t *testing.T) {
235 type branchToDocumentMap map[string][]index.Document
236
237 type step struct {
238 name string
239 addedDocuments branchToDocumentMap
240 deletedDocuments branchToDocumentMap
241 optFn func(t *testing.T, options *Options)
242
243 expectedFallbackToNormalBuild bool
244 expectedDocuments []index.Document
245 }
246
247 helloWorld := index.Document{Name: "hello_world.txt", Content: []byte("hello")}
248
249 fruitV1 := index.Document{Name: "best_fruit.txt", Content: []byte("strawberry")}
250 fruitV1InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV1.Content}
251 fruitV1WithNewName := index.Document{Name: "new_fruit.txt", Content: fruitV1.Content}
252
253 fruitV2 := index.Document{Name: "best_fruit.txt", Content: []byte("grapes")}
254 fruitV2InFolder := index.Document{Name: "the_best/best_fruit.txt", Content: fruitV2.Content}
255
256 fruitV3 := index.Document{Name: "best_fruit.txt", Content: []byte("oranges")}
257 fruitV4 := index.Document{Name: "best_fruit.txt", Content: []byte("apples")}
258
259 foo := index.Document{Name: "foo.txt", Content: []byte("bar")}
260
261 emptySourcegraphIgnore := index.Document{Name: ignore.IgnoreFile}
262 sourcegraphIgnoreWithContent := index.Document{Name: ignore.IgnoreFile, Content: []byte("good_content.txt")}
263
264 for _, test := range []struct {
265 name string
266 branches []string
267 steps []step
268 }{
269 {
270 name: "modification",
271 branches: []string{"main"},
272 steps: []step{
273 {
274 name: "setup",
275 addedDocuments: branchToDocumentMap{
276 "main": []index.Document{helloWorld, fruitV1},
277 },
278
279 expectedDocuments: []index.Document{helloWorld, fruitV1},
280 },
281 {
282 name: "add newer version of fruits",
283 addedDocuments: branchToDocumentMap{
284 "main": []index.Document{fruitV2},
285 },
286 optFn: func(t *testing.T, o *Options) {
287 o.BuildOptions.IsDelta = true
288 },
289
290 expectedDocuments: []index.Document{helloWorld, fruitV2},
291 },
292 },
293 },
294 {
295 name: "modification only inside nested folder",
296 branches: []string{"main"},
297 steps: []step{
298 {
299 name: "setup",
300 addedDocuments: branchToDocumentMap{
301 "main": []index.Document{foo, fruitV1InFolder},
302 },
303
304 expectedDocuments: []index.Document{foo, fruitV1InFolder},
305 },
306 {
307 name: "add newer version of fruits inside folder",
308 addedDocuments: branchToDocumentMap{
309 "main": []index.Document{fruitV2InFolder},
310 },
311 optFn: func(t *testing.T, o *Options) {
312 o.BuildOptions.IsDelta = true
313 },
314
315 expectedDocuments: []index.Document{foo, fruitV2InFolder},
316 },
317 },
318 },
319 {
320 name: "addition",
321 branches: []string{"main"},
322 steps: []step{
323 {
324 name: "setup",
325 addedDocuments: branchToDocumentMap{
326 "main": []index.Document{helloWorld, fruitV1},
327 },
328
329 expectedDocuments: []index.Document{helloWorld, fruitV1},
330 },
331 {
332 name: "add new file - foo",
333 addedDocuments: branchToDocumentMap{
334 "main": []index.Document{foo},
335 },
336 optFn: func(t *testing.T, o *Options) {
337 o.BuildOptions.IsDelta = true
338 },
339
340 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
341 },
342 },
343 },
344 {
345 name: "deletion",
346 branches: []string{"main"},
347 steps: []step{
348 {
349 name: "setup",
350 addedDocuments: branchToDocumentMap{
351 "main": []index.Document{helloWorld, fruitV1, foo},
352 },
353
354 expectedDocuments: []index.Document{helloWorld, fruitV1, foo},
355 },
356 {
357 name: "delete foo file",
358 addedDocuments: nil,
359 deletedDocuments: branchToDocumentMap{
360 "main": []index.Document{foo},
361 },
362
363 optFn: func(t *testing.T, o *Options) {
364 o.BuildOptions.IsDelta = true
365 },
366
367 expectedDocuments: []index.Document{helloWorld, fruitV1},
368 },
369 },
370 },
371 {
372 name: "addition and deletion on only one branch",
373 branches: []string{"main", "release", "dev"},
374 steps: []step{
375 {
376 name: "setup",
377 addedDocuments: branchToDocumentMap{
378 "main": []index.Document{fruitV1},
379 "release": []index.Document{fruitV2},
380 "dev": []index.Document{fruitV3},
381 },
382
383 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
384 },
385 {
386 name: "replace fruits v3 with v4 on 'dev', delete fruits on 'main'",
387 addedDocuments: branchToDocumentMap{
388 "dev": []index.Document{fruitV4},
389 },
390 deletedDocuments: branchToDocumentMap{
391 "main": []index.Document{fruitV1},
392 },
393
394 optFn: func(t *testing.T, o *Options) {
395 o.BuildOptions.IsDelta = true
396 },
397
398 expectedDocuments: []index.Document{fruitV2, fruitV4},
399 },
400 },
401 },
402 {
403 name: "rename",
404 branches: []string{"main", "release"},
405 steps: []step{
406 {
407 name: "setup",
408 addedDocuments: branchToDocumentMap{
409 "main": []index.Document{fruitV1},
410 "release": []index.Document{fruitV2},
411 },
412 expectedDocuments: []index.Document{fruitV1, fruitV2},
413 },
414 {
415 name: "rename fruits file on 'main' + ensure that unmodified fruits file on 'release' is still searchable",
416 addedDocuments: branchToDocumentMap{
417 "main": []index.Document{fruitV1WithNewName},
418 },
419 deletedDocuments: branchToDocumentMap{
420 "main": []index.Document{fruitV1},
421 },
422
423 optFn: func(t *testing.T, o *Options) {
424 o.BuildOptions.IsDelta = true
425 },
426
427 expectedDocuments: []index.Document{fruitV1WithNewName, fruitV2},
428 },
429 },
430 },
431 {
432 name: "modification: update one branch with version of document from another branch (a.k.a. Keegan's test)",
433 branches: []string{"main", "dev"},
434 steps: []step{
435 {
436 name: "setup",
437 addedDocuments: branchToDocumentMap{
438 "main": []index.Document{fruitV1},
439 "dev": []index.Document{fruitV2},
440 },
441 expectedDocuments: []index.Document{fruitV1, fruitV2},
442 },
443 {
444 name: "switch main to dev's older version of fruits + bump dev's fruits to new version",
445 addedDocuments: branchToDocumentMap{
446 "main": []index.Document{fruitV2},
447 "dev": []index.Document{fruitV3},
448 },
449
450 optFn: func(t *testing.T, o *Options) {
451 o.BuildOptions.IsDelta = true
452 },
453
454 expectedDocuments: []index.Document{fruitV2, fruitV3},
455 },
456 },
457 },
458 {
459 name: "no-op delta builds (reindexing the same commits)",
460 branches: []string{"main", "dev"},
461 steps: []step{
462 {
463 name: "setup",
464 addedDocuments: branchToDocumentMap{
465 "main": []index.Document{fruitV1, foo},
466 "dev": []index.Document{helloWorld},
467 },
468 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
469 },
470 {
471 name: "first no-op (normal build -> delta build)",
472 optFn: func(t *testing.T, o *Options) {
473 o.BuildOptions.IsDelta = true
474 },
475
476 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
477 },
478 {
479 name: "second no-op (delta build -> delta build)",
480 optFn: func(t *testing.T, o *Options) {
481 o.BuildOptions.IsDelta = true
482 },
483
484 expectedDocuments: []index.Document{fruitV1, foo, helloWorld},
485 },
486 },
487 },
488 {
489 name: "should fallback to normal build if no prior shards exist",
490 branches: []string{"main"},
491 steps: []step{
492 {
493 name: "attempt delta build on a repository that hasn't been indexed yet",
494 addedDocuments: branchToDocumentMap{
495 "main": []index.Document{helloWorld},
496 },
497 optFn: func(t *testing.T, o *Options) {
498 o.BuildOptions.IsDelta = true
499 },
500
501 expectedFallbackToNormalBuild: true,
502 expectedDocuments: []index.Document{helloWorld},
503 },
504 },
505 },
506 {
507 name: "should fallback to normal build if the set of requested repository branches changes",
508 branches: []string{"main", "release", "dev"},
509 steps: []step{
510 {
511 name: "setup",
512 addedDocuments: branchToDocumentMap{
513 "main": []index.Document{fruitV1},
514 "release": []index.Document{fruitV2},
515 "dev": []index.Document{fruitV3},
516 },
517
518 expectedDocuments: []index.Document{fruitV1, fruitV2, fruitV3},
519 },
520 {
521 name: "try delta build after dropping 'main' branch from index ",
522 addedDocuments: branchToDocumentMap{
523 "release": []index.Document{fruitV4},
524 },
525 optFn: func(t *testing.T, o *Options) {
526 o.Branches = []string{"HEAD", "release", "dev"} // a bit of a hack to override it this way, but it gets the job done
527 o.BuildOptions.IsDelta = true
528 },
529
530 expectedFallbackToNormalBuild: true,
531 expectedDocuments: []index.Document{fruitV3, fruitV4},
532 },
533 },
534 },
535 {
536 name: "should expand branches correctly when using wildcards in branch names",
537 branches: []string{"release/1", "release/2"},
538 steps: []step{
539 {
540 name: "setup",
541 addedDocuments: branchToDocumentMap{
542 "release/1": []index.Document{fruitV1},
543 "release/2": []index.Document{fruitV2},
544 },
545
546 expectedDocuments: []index.Document{fruitV1, fruitV2},
547 },
548 {
549 name: "try delta build with wildcard in branches",
550 optFn: func(t *testing.T, o *Options) {
551 // use a wildcard here
552 o.Branches = []string{"HEAD", "release/*"}
553 o.BuildOptions.IsDelta = true
554 },
555
556 expectedDocuments: []index.Document{fruitV1, fruitV2},
557 },
558 },
559 },
560 {
561 name: "should fallback to normal build if one or more index options updates requires a full build",
562 branches: []string{"main"},
563 steps: []step{
564 {
565 name: "setup",
566 addedDocuments: branchToDocumentMap{
567 "main": []index.Document{fruitV1},
568 },
569
570 expectedDocuments: []index.Document{fruitV1},
571 },
572 {
573 name: "try delta build after updating Disable CTags index option",
574 addedDocuments: branchToDocumentMap{
575 "main": []index.Document{fruitV2},
576 },
577 optFn: func(t *testing.T, o *Options) {
578 o.BuildOptions.IsDelta = true
579 o.BuildOptions.DisableCTags = true
580 },
581
582 expectedFallbackToNormalBuild: true,
583 expectedDocuments: []index.Document{fruitV2},
584 },
585 {
586 name: "try delta build after reverting Disable CTags index option",
587 addedDocuments: branchToDocumentMap{
588 "main": []index.Document{fruitV3},
589 },
590 optFn: func(t *testing.T, o *Options) {
591 o.BuildOptions.IsDelta = true
592 o.BuildOptions.DisableCTags = false
593 },
594
595 expectedFallbackToNormalBuild: true,
596 expectedDocuments: []index.Document{fruitV3},
597 },
598 },
599 },
600 {
601 name: "should successfully perform multiple delta builds after disabling symbols",
602 branches: []string{"main"},
603 steps: []step{
604 {
605 name: "setup",
606 addedDocuments: branchToDocumentMap{
607 "main": []index.Document{fruitV1},
608 },
609
610 expectedDocuments: []index.Document{fruitV1},
611 },
612 {
613 name: "try delta build after updating Disable CTags index option",
614 addedDocuments: branchToDocumentMap{
615 "main": []index.Document{fruitV2},
616 },
617 optFn: func(t *testing.T, o *Options) {
618 o.BuildOptions.IsDelta = true
619 o.BuildOptions.DisableCTags = true
620 },
621
622 expectedFallbackToNormalBuild: true,
623 expectedDocuments: []index.Document{fruitV2},
624 },
625 {
626 name: "try another delta build while CTags is still disabled",
627 addedDocuments: branchToDocumentMap{
628 "main": []index.Document{fruitV3},
629 },
630 optFn: func(t *testing.T, o *Options) {
631 o.BuildOptions.IsDelta = true
632 o.BuildOptions.DisableCTags = true
633 },
634
635 expectedDocuments: []index.Document{fruitV3},
636 },
637 },
638 },
639 {
640 name: "should fallback to normal build if repository has unsupported Sourcegraph ignore file",
641 branches: []string{"main"},
642 steps: []step{
643 {
644 name: "setup",
645 addedDocuments: branchToDocumentMap{
646 "main": []index.Document{emptySourcegraphIgnore},
647 },
648
649 expectedDocuments: []index.Document{emptySourcegraphIgnore},
650 },
651 {
652 name: "attempt delta build after modifying ignore file",
653 addedDocuments: branchToDocumentMap{
654 "main": []index.Document{sourcegraphIgnoreWithContent},
655 },
656 optFn: func(t *testing.T, o *Options) {
657 o.BuildOptions.IsDelta = true
658 },
659
660 expectedFallbackToNormalBuild: true,
661 expectedDocuments: []index.Document{sourcegraphIgnoreWithContent},
662 },
663 },
664 },
665 {
666 name: "should fallback to a full, normal build if the repository has more than the specified threshold of shards",
667 branches: []string{"main"},
668 steps: []step{
669 {
670 name: "setup: first shard",
671 addedDocuments: branchToDocumentMap{
672 "main": []index.Document{foo},
673 },
674
675 expectedDocuments: []index.Document{foo},
676 },
677 {
678 name: "setup: second shard (delta)",
679 addedDocuments: branchToDocumentMap{
680 "main": []index.Document{fruitV1},
681 },
682 optFn: func(t *testing.T, o *Options) {
683 o.BuildOptions.IsDelta = true
684 },
685
686 expectedDocuments: []index.Document{foo, fruitV1},
687 },
688 {
689 name: "setup: third shard (delta)",
690 addedDocuments: branchToDocumentMap{
691 "main": []index.Document{helloWorld},
692 },
693 optFn: func(t *testing.T, o *Options) {
694 o.BuildOptions.IsDelta = true
695 },
696
697 expectedDocuments: []index.Document{foo, fruitV1, helloWorld},
698 },
699 {
700 name: "attempt another delta build after we already blew past the shard threshold",
701 addedDocuments: branchToDocumentMap{
702 "main": []index.Document{fruitV2InFolder},
703 },
704 optFn: func(t *testing.T, o *Options) {
705 o.DeltaShardNumberFallbackThreshold = 2
706 o.BuildOptions.IsDelta = true
707 },
708
709 expectedFallbackToNormalBuild: true,
710 expectedDocuments: []index.Document{foo, fruitV1, helloWorld, fruitV2InFolder},
711 },
712 },
713 },
714 } {
715 test := test
716
717 t.Run(test.name, func(t *testing.T) {
718 t.Parallel()
719
720 indexDir := t.TempDir()
721 repositoryDir := t.TempDir()
722
723 // setup: initialize the repository and all of its branches
724 runScript(t, repositoryDir, "git init -b master")
725 runScript(t, repositoryDir, fmt.Sprintf("git config user.email %q", "you@example.com"))
726 runScript(t, repositoryDir, fmt.Sprintf("git config user.name %q", "Your Name"))
727
728 for _, b := range test.branches {
729 runScript(t, repositoryDir, fmt.Sprintf("git checkout -b %q", b))
730 runScript(t, repositoryDir, fmt.Sprintf("git commit --allow-empty -m %q", "empty commit"))
731 }
732
733 for _, step := range test.steps {
734 t.Run(step.name, func(t *testing.T) {
735 for _, b := range test.branches {
736 // setup: for each branch, process any document deletions / additions and commit those changes
737
738 hadChange := false
739
740 runScript(t, repositoryDir, fmt.Sprintf("git checkout %q", b))
741
742 for _, d := range step.deletedDocuments[b] {
743 hadChange = true
744
745 file := filepath.Join(repositoryDir, d.Name)
746
747 err := os.Remove(file)
748 if err != nil {
749 t.Fatalf("deleting file %q: %s", d.Name, err)
750 }
751
752 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
753 }
754
755 for _, d := range step.addedDocuments[b] {
756 hadChange = true
757
758 file := filepath.Join(repositoryDir, d.Name)
759
760 err := os.MkdirAll(filepath.Dir(file), 0o755)
761 if err != nil {
762 t.Fatalf("ensuring that folders exist for file %q: %s", file, err)
763 }
764
765 err = os.WriteFile(file, d.Content, 0o644)
766 if err != nil {
767 t.Fatalf("writing file %q: %s", d.Name, err)
768 }
769
770 runScript(t, repositoryDir, fmt.Sprintf("git add %q", file))
771 }
772
773 if !hadChange {
774 continue
775 }
776
777 runScript(t, repositoryDir, fmt.Sprintf("git commit -m %q", step.name))
778 }
779
780 // setup: prepare indexOptions with given overrides
781 buildOptions := index.Options{
782 IndexDir: indexDir,
783 RepositoryDescription: zoekt.Repository{
784 Name: "repository",
785 },
786 IsDelta: false,
787 }
788 buildOptions.SetDefaults()
789
790 branches := append([]string{"HEAD"}, test.branches...)
791
792 options := Options{
793 RepoDir: filepath.Join(repositoryDir, ".git"),
794 BuildOptions: buildOptions,
795 Branches: branches,
796 }
797
798 if step.optFn != nil {
799 step.optFn(t, &options)
800 }
801
802 // setup: prepare spy versions of prepare delta / normal build so that we can observe
803 // whether they were called appropriately
804 deltaBuildCalled := false
805 prepareDeltaSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, changedOrDeletedPaths []string, err error) {
806 deltaBuildCalled = true
807 return prepareDeltaBuild(options, repository)
808 }
809
810 normalBuildCalled := false
811 prepareNormalSpy := func(options Options, repository *git.Repository) (repos map[fileKey]BlobLocation, branchVersions map[string]map[string]plumbing.Hash, err error) {
812 normalBuildCalled = true
813 return prepareNormalBuild(options, repository)
814 }
815
816 // run test
817 _, err := indexGitRepo(options, gitIndexConfig{
818 prepareDeltaBuild: prepareDeltaSpy,
819 prepareNormalBuild: prepareNormalSpy,
820 })
821 if err != nil {
822 t.Fatalf("IndexGitRepo: %s", err)
823 }
824
825 if options.BuildOptions.IsDelta != deltaBuildCalled {
826 // We should always try a delta build if we request it in the options.
827 t.Fatalf("expected deltaBuildCalled to be %t, got %t", options.BuildOptions.IsDelta, deltaBuildCalled)
828 }
829
830 if options.BuildOptions.IsDelta && (step.expectedFallbackToNormalBuild != normalBuildCalled) {
831 // We only check the normal spy on delta builds because it's only considered a "fallback" if we
832 // asked for a delta build in the first place.
833 t.Fatalf("expected normalBuildCalled to be %t, got %t", step.expectedFallbackToNormalBuild, normalBuildCalled)
834 }
835
836 // examine outcome: load shards into a searcher instance and run a dummy search query
837 // that returns every document contained in the shards
838 //
839 // then, compare returned set of documents with the expected set for the step and see if they agree
840
841 ss, err := search.NewDirectorySearcher(indexDir)
842 if err != nil {
843 t.Fatalf("NewDirectorySearcher(%s): %s", indexDir, err)
844 }
845 defer ss.Close()
846
847 searchOpts := &zoekt.SearchOptions{Whole: true}
848 result, err := ss.Search(context.Background(), &query.Const{Value: true}, searchOpts)
849 if err != nil {
850 t.Fatalf("Search: %s", err)
851 }
852
853 var receivedDocuments []index.Document
854 for _, f := range result.Files {
855 receivedDocuments = append(receivedDocuments, index.Document{
856 Name: f.FileName,
857 Content: f.Content,
858 })
859 }
860
861 for _, docs := range [][]index.Document{step.expectedDocuments, receivedDocuments} {
862 sort.Slice(docs, func(i, j int) bool {
863 a, b := docs[i], docs[j]
864
865 // first compare names, then fallback to contents if the names are equal
866
867 if a.Name < b.Name {
868 return true
869 }
870
871 if a.Name > b.Name {
872 return false
873 }
874
875 return bytes.Compare(a.Content, b.Content) < 0
876 })
877 }
878
879 compareOptions := []cmp.Option{
880 cmpopts.IgnoreFields(index.Document{}, "Branches"),
881 cmpopts.EquateEmpty(),
882 }
883
884 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, compareOptions...); diff != "" {
885 t.Errorf("diff in received documents (-want +got):%s\n:", diff)
886 }
887 })
888 }
889 })
890 }
891}
892
893func runScript(t *testing.T, cwd string, script string) {
894 t.Helper()
895
896 err := os.MkdirAll(cwd, 0o755)
897 if err != nil {
898 t.Fatalf("ensuring path %q exists: %s", cwd, err)
899 }
900
901 cmd := exec.Command("sh", "-euxc", script)
902 cmd.Dir = cwd
903 cmd.Env = append([]string{"GIT_CONFIG_GLOBAL=", "GIT_CONFIG_SYSTEM="}, os.Environ()...)
904
905 if out, err := cmd.CombinedOutput(); err != nil {
906 t.Fatalf("execution error: %v, output %s", err, out)
907 }
908}
909
910func TestSetTemplates_e2e(t *testing.T) {
911 repositoryDir := t.TempDir()
912
913 // setup: initialize the repository and all of its branches
914 runScript(t, repositoryDir, "git init -b master")
915 runScript(t, repositoryDir, "git config remote.origin.url git@github.com:sourcegraph/zoekt.git")
916 desc := zoekt.Repository{}
917 if err := setTemplatesFromConfig(&desc, repositoryDir); err != nil {
918 t.Fatalf("setTemplatesFromConfig: %v", err)
919 }
920
921 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
922 t.Errorf("got %q, want %q", got, want)
923 }
924}
925
926func TestSetTemplates_Worktree(t *testing.T) {
927 _, worktreeDir := initGitWorktree(t, "hello.go", "package main\n")
928 desc := zoekt.Repository{}
929
930 if err := setTemplatesFromConfig(&desc, worktreeDir); err != nil {
931 t.Fatalf("setTemplatesFromConfig(worktree): %v", err)
932 }
933
934 if got, want := desc.FileURLTemplate, `{{URLJoinPath "https://github.com/sourcegraph/zoekt" "blob" .Version .Path}}`; got != want {
935 t.Errorf("got %q, want %q", got, want)
936 }
937}
938
939func TestSetTemplates(t *testing.T) {
940 base := "https://example.com/repo/name"
941 version := "VERSION"
942 path := "dir/name.txt"
943 lineNumber := 10
944 cases := []struct {
945 typ string
946 commit string
947 file string
948 line string
949 }{{
950 typ: "gitiles",
951 commit: "https://example.com/repo/name/%2B/VERSION",
952 file: "https://example.com/repo/name/%2B/VERSION/dir/name.txt",
953 line: "#10",
954 }, {
955 typ: "github",
956 commit: "https://example.com/repo/name/commit/VERSION",
957 file: "https://example.com/repo/name/blob/VERSION/dir/name.txt",
958 line: "#L10",
959 }, {
960 typ: "cgit",
961 commit: "https://example.com/repo/name/commit/?id=VERSION",
962 file: "https://example.com/repo/name/tree/dir/name.txt/?id=VERSION",
963 line: "#n10",
964 }, {
965 typ: "gitweb",
966 commit: "https://example.com/repo/name;a=commit;h=VERSION",
967 file: "https://example.com/repo/name;a=blob;f=dir/name.txt;hb=VERSION",
968 line: "#l10",
969 }, {
970 typ: "source.bazel.build",
971 commit: "https://example.com/repo/name/%2B/VERSION",
972 file: "https://example.com/repo/name/%2B/VERSION:dir/name.txt",
973 line: ";l=10",
974 }, {
975 typ: "bitbucket-server",
976 commit: "https://example.com/repo/name/commits/VERSION",
977 file: "https://example.com/repo/name/dir/name.txt?at=VERSION",
978 line: "#10",
979 }, {
980 typ: "gitlab",
981 commit: "https://example.com/repo/name/-/commit/VERSION",
982 file: "https://example.com/repo/name/-/blob/VERSION/dir/name.txt",
983 line: "#L10",
984 }, {
985 typ: "gitea",
986 commit: "https://example.com/repo/name/commit/VERSION",
987 file: "https://example.com/repo/name/src/commit/VERSION/dir/name.txt?display=source",
988 line: "#L10",
989 }}
990
991 for _, tc := range cases {
992 t.Run(tc.typ, func(t *testing.T) {
993 assertOutput := func(templateText string, want string) {
994 t.Helper()
995
996 tt, err := index.ParseTemplate(templateText)
997 if err != nil {
998 t.Fatal(err)
999 }
1000
1001 var sb strings.Builder
1002 err = tt.Execute(&sb, map[string]any{
1003 "Version": version,
1004 "Path": path,
1005 "LineNumber": lineNumber,
1006 })
1007 if err != nil {
1008 t.Fatal(err)
1009 }
1010 if got := sb.String(); got != want {
1011 t.Fatalf("want: %q\ngot: %q", want, got)
1012 }
1013 }
1014
1015 var repo zoekt.Repository
1016 u, _ := url.Parse(base)
1017 err := setTemplates(&repo, u, tc.typ)
1018 if err != nil {
1019 t.Fatal(err)
1020 }
1021 assertOutput(repo.CommitURLTemplate, tc.commit)
1022 assertOutput(repo.FileURLTemplate, tc.file)
1023 assertOutput(repo.LineFragmentTemplate, tc.line)
1024 })
1025 }
1026}
1027
1028func BenchmarkPrepareNormalBuild(b *testing.B) {
1029 // NOTE: To run the benchmark, download a large repo (like github.com/chromium/chromium/) and change this to its path.
1030 repoDir := "/path/to/your/repo"
1031 repo, err := git.PlainOpen(repoDir)
1032 if err != nil {
1033 b.Fatalf("Failed to open test repository: %v", err)
1034 }
1035
1036 opts := Options{
1037 RepoDir: repoDir,
1038 Submodules: false,
1039 BranchPrefix: "refs/heads/",
1040 Branches: []string{"main"},
1041 BuildOptions: index.Options{
1042 RepositoryDescription: zoekt.Repository{
1043 Name: "test-repo",
1044 URL: "https://github.com/example/test-repo",
1045 },
1046 },
1047 }
1048
1049 b.ReportAllocs()
1050
1051 repos, branchVersions, err := prepareNormalBuild(opts, repo)
1052 if err != nil {
1053 b.Fatalf("prepareNormalBuild failed: %v", err)
1054 }
1055
1056 runtime.GC()
1057
1058 var m runtime.MemStats
1059 runtime.ReadMemStats(&m)
1060 b.ReportMetric(float64(m.HeapInuse), "heap-used-bytes")
1061 b.ReportMetric(float64(m.HeapInuse), "heap-allocated-bytes")
1062
1063 if len(repos) == 0 || len(branchVersions) == 0 {
1064 b.Fatalf("Unexpected empty results")
1065 }
1066}