fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32
33 "github.com/sourcegraph/zoekt"
34 "github.com/sourcegraph/zoekt/ignore"
35 "github.com/sourcegraph/zoekt/index"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/search"
38)
39
40func createSubmoduleRepo(dir string) error {
41 if err := os.MkdirAll(dir, 0o755); err != nil {
42 return err
43 }
44 script := `
45# Fix fatal: transport 'file' not allowed
46export GIT_CONFIG_COUNT=1
47export GIT_CONFIG_KEY_0=protocol.file.allow
48export GIT_CONFIG_VALUE_0=always
49
50mkdir adir bdir
51cd adir
52git init -b master
53mkdir subdir
54echo acont > afile
55echo sub-cont > subdir/sub-file
56git add afile subdir/sub-file
57git config user.email "you@example.com"
58git config user.name "Your Name"
59git commit -am amsg
60
61cd ..
62cd bdir
63git init -b master
64echo bcont > bfile
65ln -s bfile bsymlink
66git add bfile bsymlink
67git config user.email "you@example.com"
68git config user.name "Your Name"
69git commit -am bmsg
70
71cd ../adir
72git submodule add --name bname -- ../bdir bname
73git commit -am bmodmsg
74cat .gitmodules
75cd ..
76mkdir gerrit.googlesource.com
77git clone --bare adir gerrit.googlesource.com/adir.git
78git clone --bare bdir gerrit.googlesource.com/bdir.git
79
80mkdir gerrit.googlesource.com/bogus.git
81mkdir gerrit.googlesource.com/sub
82git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
83
84mkdir -p gerrit.googlesource.com/team/scope/
85cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
86cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
87
88cat << EOF > gerrit.googlesource.com/adir.git/config
89[core]
90 repositoryformatversion = 0
91 filemode = true
92 bare = true
93[remote "origin"]
94 url = http://gerrit.googlesource.com/adir
95[branch "master"]
96 remote = origin
97 merge = refs/heads/master
98EOF
99`
100 cmd := exec.Command("/bin/sh", "-euxc", script)
101 cmd.Dir = dir
102 if out, err := cmd.CombinedOutput(); err != nil {
103 return fmt.Errorf("execution error: %v, output %s", err, out)
104 }
105 return nil
106}
107
108func TestFindGitRepos(t *testing.T) {
109 t.Parallel()
110
111 dir := t.TempDir()
112
113 if err := createSubmoduleRepo(dir); err != nil {
114 t.Error("createSubmoduleRepo", err)
115 }
116 repos, err := FindGitRepos(dir)
117 if err != nil {
118 t.Error("FindGitRepos", err)
119 }
120
121 got := map[string]bool{}
122 for _, r := range repos {
123 p, err := filepath.Rel(dir, r)
124 if err != nil {
125 t.Fatalf("Relative: %v", err)
126 }
127
128 got[p] = true
129 }
130
131 want := map[string]bool{
132 "adir/.git": true,
133 "bdir/.git": true,
134 "gerrit.googlesource.com/adir.git": true,
135 "gerrit.googlesource.com/bdir.git": true,
136 "gerrit.googlesource.com/sub/bdir.git": true,
137 "gerrit.googlesource.com/team/scope/repoa.git": true,
138 "gerrit.googlesource.com/team/scope/repob.git": true,
139 }
140 if !reflect.DeepEqual(got, want) {
141 t.Errorf("got %v want %v", got, want)
142 }
143}
144
145func TestCollectFiles(t *testing.T) {
146 t.Parallel()
147
148 dir := t.TempDir()
149
150 if err := createSubmoduleRepo(dir); err != nil {
151 t.Fatalf("TempDir: %v", err)
152 }
153
154 cache := NewRepoCache(dir)
155
156 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
157 repo, err := cache.Open(aURL)
158 if err != nil {
159 t.Fatalf("Open: %v", err)
160 }
161
162 headRef, err := repo.Head()
163 if err != nil {
164 t.Fatalf("HEAD tree: %v", err)
165 }
166 commit, err := repo.CommitObject(headRef.Hash())
167 if err != nil {
168 t.Fatalf("commit obj HEAD: %v", err)
169 }
170
171 tree, err := repo.TreeObject(commit.TreeHash)
172 if err != nil {
173 t.Fatalf("AsTree: %v", err)
174 }
175
176 rw := NewRepoWalker(repo, aURL.String(), cache)
177 versions, err := rw.CollectFiles(tree, "main", &ignore.Matcher{})
178 if err != nil {
179 t.Fatalf("CollectFiles: %v", err)
180 }
181
182 bnameHash := versions["bname"]
183 if entry, err := tree.FindEntry("bname"); err != nil {
184 t.Fatalf("FindEntry %v", err)
185 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
186 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
187 }
188
189 var paths []string
190 for k := range rw.Files {
191 paths = append(paths, k.FullPath())
192 }
193 sort.Strings(paths)
194
195 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
196 if !reflect.DeepEqual(paths, want) {
197 t.Errorf("got %v, want %v", paths, want)
198 }
199}
200
201func TestSubmoduleIndex(t *testing.T) {
202 t.Parallel()
203
204 dir := t.TempDir()
205
206 if err := createSubmoduleRepo(dir); err != nil {
207 t.Fatalf("createSubmoduleRepo: %v", err)
208 }
209
210 indexDir := t.TempDir()
211
212 buildOpts := index.Options{
213 IndexDir: indexDir,
214 }
215 opts := Options{
216 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
217 BuildOptions: buildOpts,
218 BranchPrefix: "refs/heads/",
219 Branches: []string{"master"},
220 Submodules: true,
221 Incremental: true,
222 RepoCacheDir: dir,
223 }
224 if _, err := IndexGitRepo(opts); err != nil {
225 t.Fatalf("IndexGitRepo: %v", err)
226 }
227
228 searcher, err := search.NewDirectorySearcher(indexDir)
229 if err != nil {
230 t.Fatal("NewDirectorySearcher", err)
231 }
232 defer searcher.Close()
233
234 results, err := searcher.Search(context.Background(),
235 &query.Substring{Pattern: "bcont"},
236 &zoekt.SearchOptions{})
237 if err != nil {
238 t.Fatal("Search", err)
239 }
240
241 if len(results.Files) != 1 {
242 t.Fatalf("got search result %v, want 1 file", results.Files)
243 }
244
245 file := results.Files[0]
246 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
247 t.Errorf("got subrepo name %q, want %q", got, want)
248 }
249 if got, want := file.SubRepositoryPath, "bname"; got != want {
250 t.Errorf("got subrepo path %q, want %q", got, want)
251 }
252
253 subVersion := file.Version
254 if len(subVersion) != 40 {
255 t.Fatalf("got %q, want hex sha1", subVersion)
256 }
257
258 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
259 t.Fatalf("Search('acont'): %v", err)
260 } else if len(results.Files) != 1 {
261 t.Errorf("got %v, want 1 result", results.Files)
262 } else if f := results.Files[0]; f.Version == subVersion {
263 t.Errorf("version in super repo matched version is subrepo.")
264 }
265}
266
267func TestSubmoduleIndexWithoutRepocache(t *testing.T) {
268 t.Parallel()
269
270 dir := t.TempDir()
271
272 if err := createSubmoduleRepo(dir); err != nil {
273 t.Fatalf("createSubmoduleRepo: %v", err)
274 }
275
276 indexDir := t.TempDir()
277
278 buildOpts := index.Options{
279 RepositoryDescription: zoekt.Repository{Name: "adir"},
280 IndexDir: indexDir,
281 }
282 opts := Options{
283 RepoDir: filepath.Join(dir, "adir"),
284 BuildOptions: buildOpts,
285 BranchPrefix: "refs/heads/",
286 Branches: []string{"master"},
287 Submodules: true,
288 Incremental: true,
289 }
290 if _, err := IndexGitRepo(opts); err != nil {
291 t.Fatalf("IndexGitRepo: %v", err)
292 }
293
294 searcher, err := search.NewDirectorySearcher(indexDir)
295 if err != nil {
296 t.Fatal("NewDirectorySearcher", err)
297 }
298 defer searcher.Close()
299
300 results, err := searcher.Search(context.Background(),
301 &query.Substring{Pattern: "bcont"},
302 &zoekt.SearchOptions{})
303 if err != nil {
304 t.Fatal("Search", err)
305 }
306
307 if len(results.Files) != 1 {
308 t.Fatalf("got search result %v, want 1 file", results.Files)
309 }
310
311 file := results.Files[0]
312 if got, want := file.SubRepositoryName, "bname"; got != want {
313 t.Errorf("got subrepo name %q, want %q", got, want)
314 }
315 if got, want := file.SubRepositoryPath, "bname"; got != want {
316 t.Errorf("got subrepo path %q, want %q", got, want)
317 }
318
319 subVersion := file.Version
320 if len(subVersion) != 40 {
321 t.Fatalf("got %q, want hex sha1", subVersion)
322 }
323
324 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
325 t.Fatalf("Search('acont'): %v", err)
326 } else if len(results.Files) != 1 {
327 t.Errorf("got %v, want 1 result", results.Files)
328 } else if f := results.Files[0]; f.Version == subVersion {
329 t.Errorf("version in super repo matched version is subrepo.")
330 }
331}
332
333func createSymlinkRepo(dir string) error {
334 if err := os.MkdirAll(dir, 0o755); err != nil {
335 return err
336 }
337 script := `mkdir adir bdir
338git init -b master
339git config user.email "you@example.com"
340git config user.name "Your Name"
341
342echo acont > adir/afile
343git add adir/afile
344
345echo bcont > bdir/bfile
346git add bdir/bfile
347
348ln -s ./adir/afile asymlink
349git add asymlink
350
351git commit -am amsg
352
353cat << EOF > .git/config
354[core]
355 repositoryformatversion = 0
356 filemode = true
357 bare = true
358[remote "origin"]
359 url = http://codehost.com/arepo
360[branch "master"]
361 remote = origin
362 merge = refs/heads/master
363EOF
364`
365 cmd := exec.Command("/bin/sh", "-euxc", script)
366 cmd.Dir = dir
367 if out, err := cmd.CombinedOutput(); err != nil {
368 return fmt.Errorf("execution error: %v, output %s", err, out)
369 }
370 return nil
371}
372
373func TestSearchSymlinkByContent(t *testing.T) {
374 t.Parallel()
375
376 dir := t.TempDir()
377
378 if err := createSymlinkRepo(dir); err != nil {
379 t.Fatalf("createSubmoduleRepo: %v", err)
380 }
381
382 indexDir := t.TempDir()
383
384 buildOpts := index.Options{
385 IndexDir: indexDir,
386 }
387 opts := Options{
388 RepoDir: filepath.Join(dir),
389 BuildOptions: buildOpts,
390 BranchPrefix: "refs/heads/",
391 Branches: []string{"master"},
392 Submodules: true,
393 Incremental: true,
394 RepoCacheDir: dir,
395 }
396 if _, err := IndexGitRepo(opts); err != nil {
397 t.Fatalf("IndexGitRepo: %v", err)
398 }
399
400 searcher, err := search.NewDirectorySearcher(indexDir)
401 if err != nil {
402 t.Fatal("NewDirectorySearcher", err)
403 }
404 defer searcher.Close()
405
406 // The content of the symlink and the file path the symlink points to both
407 // contain the string "afile". Hence we expect 1 path match and 1 content match.
408 results, err := searcher.Search(context.Background(),
409 &query.Substring{Pattern: "afile"},
410 &zoekt.SearchOptions{})
411 if err != nil {
412 t.Fatal("Search", err)
413 }
414
415 if len(results.Files) != 2 {
416 t.Fatalf("got search result %v, want 2 files", results.Files)
417 }
418
419 got := make([]string, 0, 2)
420 for _, file := range results.Files {
421 got = append(got, file.FileName)
422 }
423 sort.Strings(got)
424
425 want := []string{"adir/afile", "asymlink"}
426
427 if d := cmp.Diff(want, got); d != "" {
428 t.Fatalf("-want, +got %s\n", d)
429 }
430}
431
432func TestAllowMissingBranch(t *testing.T) {
433 t.Parallel()
434
435 dir := t.TempDir()
436
437 if err := createSubmoduleRepo(dir); err != nil {
438 t.Fatalf("createSubmoduleRepo: %v", err)
439 }
440
441 indexDir := t.TempDir()
442
443 buildOpts := index.Options{
444 IndexDir: indexDir,
445 }
446
447 opts := Options{
448 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
449 BuildOptions: buildOpts,
450 BranchPrefix: "refs/heads/",
451 Branches: []string{"master", "nonexist"},
452 Submodules: true,
453 Incremental: true,
454 RepoCacheDir: dir,
455 }
456 if _, err := IndexGitRepo(opts); err == nil {
457 t.Fatalf("IndexGitRepo(nonexist) succeeded")
458 }
459 opts.AllowMissingBranch = true
460 if _, err := IndexGitRepo(opts); err != nil {
461 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
462 }
463}
464
465func createMultibranchRepo(dir string) error {
466 if err := os.MkdirAll(dir, 0o755); err != nil {
467 return err
468 }
469 script := `mkdir repo
470cd repo
471git init -b master
472mkdir subdir
473echo acont > afile
474echo sub-cont > subdir/sub-file
475git add afile subdir/sub-file
476git config user.email "you@example.com"
477git config user.name "Your Name"
478GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
479
480git branch branchdir/a
481
482echo acont >> afile
483git add afile subdir/sub-file
484GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
485
486git branch branchdir/b
487
488git branch c
489
490git update-ref refs/meta/config HEAD
491`
492 cmd := exec.Command("/bin/sh", "-euxc", script)
493 cmd.Dir = dir
494 if out, err := cmd.CombinedOutput(); err != nil {
495 return fmt.Errorf("execution error: %v, output %s", err, out)
496 }
497 return nil
498}
499
500func TestBranchWildcard(t *testing.T) {
501 t.Parallel()
502
503 dir := t.TempDir()
504
505 if err := createMultibranchRepo(dir); err != nil {
506 t.Fatalf("createMultibranchRepo: %v", err)
507 }
508
509 indexDir := t.TempDir()
510
511 buildOpts := index.Options{
512 IndexDir: indexDir,
513 RepositoryDescription: zoekt.Repository{
514 Name: "repo",
515 },
516 }
517 buildOpts.SetDefaults()
518
519 opts := Options{
520 RepoDir: filepath.Join(dir + "/repo"),
521 BuildOptions: buildOpts,
522 BranchPrefix: "refs/heads",
523 Branches: []string{"branchdir/*"},
524 Submodules: true,
525 Incremental: true,
526 }
527 if _, err := IndexGitRepo(opts); err != nil {
528 t.Fatalf("IndexGitRepo: %v", err)
529 }
530
531 searcher, err := search.NewDirectorySearcher(indexDir)
532 if err != nil {
533 t.Fatal("NewDirectorySearcher", err)
534 }
535 defer searcher.Close()
536
537 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
538 t.Fatalf("List(): %v", err)
539 } else if len(rlist.Repos) != 1 {
540 t.Errorf("got %v, want 1 result", rlist.Repos)
541 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
542 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
543 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
544 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
545 }
546}
547
548func TestSkipSubmodules(t *testing.T) {
549 t.Parallel()
550
551 dir := t.TempDir()
552
553 if err := createSubmoduleRepo(dir); err != nil {
554 t.Fatalf("createMultibranchRepo: %v", err)
555 }
556
557 indexDir := t.TempDir()
558
559 buildOpts := index.Options{
560 IndexDir: indexDir,
561 RepositoryDescription: zoekt.Repository{
562 Name: "gerrit.googlesource.com/adir",
563 },
564 }
565 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
566 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
567 t.Fatalf("Rename: %v", err)
568 }
569
570 opts := Options{
571 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
572 BuildOptions: buildOpts,
573 BranchPrefix: "refs/heads",
574 Branches: []string{"master"},
575 Submodules: false,
576 }
577 if _, err := IndexGitRepo(opts); err != nil {
578 t.Fatalf("IndexGitRepo: %v", err)
579 }
580}
581
582func TestFullAndShortRefNames(t *testing.T) {
583 t.Parallel()
584
585 dir := t.TempDir()
586
587 if err := createMultibranchRepo(dir); err != nil {
588 t.Fatalf("createMultibranchRepo: %v", err)
589 }
590
591 indexDir := t.TempDir()
592
593 buildOpts := index.Options{
594 IndexDir: indexDir,
595 RepositoryDescription: zoekt.Repository{
596 Name: "repo",
597 },
598 }
599 buildOpts.SetDefaults()
600
601 opts := Options{
602 RepoDir: filepath.Join(dir + "/repo"),
603 BuildOptions: buildOpts,
604 BranchPrefix: "refs/heads",
605 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
606 Submodules: false,
607 Incremental: false,
608 AllowMissingBranch: false,
609 }
610 if _, err := IndexGitRepo(opts); err != nil {
611 t.Fatalf("IndexGitRepo: %v", err)
612 }
613
614 searcher, err := search.NewDirectorySearcher(indexDir)
615 if err != nil {
616 t.Fatal("NewDirectorySearcher", err)
617 }
618 defer searcher.Close()
619
620 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
621 t.Fatalf("List(): %v", err)
622 } else if len(rlist.Repos) != 1 {
623 t.Errorf("got %v, want 1 result", rlist.Repos)
624 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
625 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
626 }
627}
628
629func TestUniq(t *testing.T) {
630 t.Parallel()
631
632 in := []string{"a", "b", "b", "c", "c"}
633 want := []string{"a", "b", "c"}
634 got := uniq(in)
635 if !reflect.DeepEqual(got, want) {
636 t.Errorf("got %v, want %v", got, want)
637 }
638}
639
640func TestLatestCommit(t *testing.T) {
641 t.Parallel()
642
643 dir := t.TempDir()
644 indexDir := t.TempDir()
645
646 if err := createMultibranchRepo(dir); err != nil {
647 t.Fatalf("createMultibranchRepo: %v", err)
648 }
649
650 buildOpts := index.Options{
651 IndexDir: indexDir,
652 RepositoryDescription: zoekt.Repository{
653 Name: "repo",
654 },
655 }
656 buildOpts.SetDefaults()
657
658 opts := Options{
659 RepoDir: filepath.Join(dir + "/repo"),
660 BuildOptions: buildOpts,
661 BranchPrefix: "refs/heads",
662 Branches: []string{"branchdir/a", "branchdir/b"},
663 }
664 if _, err := IndexGitRepo(opts); err != nil {
665 t.Fatalf("IndexGitRepo: %v", err)
666 }
667
668 searcher, err := search.NewDirectorySearcher(indexDir)
669 if err != nil {
670 t.Fatal("NewDirectorySearcher", err)
671 }
672 defer searcher.Close()
673
674 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
675 if err != nil {
676 t.Fatalf("List(): %v", err)
677 }
678
679 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
680 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
681 }
682}