fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32
33 "github.com/sourcegraph/zoekt"
34 "github.com/sourcegraph/zoekt/ignore"
35 "github.com/sourcegraph/zoekt/index"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/search"
38)
39
40func createSubmoduleRepo(dir string) error {
41 if err := os.MkdirAll(dir, 0o755); err != nil {
42 return err
43 }
44 script := `
45# Fix fatal: transport 'file' not allowed
46export GIT_CONFIG_COUNT=1
47export GIT_CONFIG_KEY_0=protocol.file.allow
48export GIT_CONFIG_VALUE_0=always
49
50mkdir adir bdir
51cd adir
52git init -b master
53mkdir subdir
54echo acont > afile
55echo sub-cont > subdir/sub-file
56git add afile subdir/sub-file
57git config user.email "you@example.com"
58git config user.name "Your Name"
59git commit -am amsg
60
61cd ..
62cd bdir
63git init -b master
64echo bcont > bfile
65ln -s bfile bsymlink
66git add bfile bsymlink
67git config user.email "you@example.com"
68git config user.name "Your Name"
69git commit -am bmsg
70
71cd ../adir
72git submodule add --name bname -- ../bdir bname
73git commit -am bmodmsg
74cat .gitmodules
75cd ..
76mkdir gerrit.googlesource.com
77git clone --bare adir gerrit.googlesource.com/adir.git
78git clone --bare bdir gerrit.googlesource.com/bdir.git
79
80mkdir gerrit.googlesource.com/bogus.git
81mkdir gerrit.googlesource.com/sub
82git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
83
84mkdir -p gerrit.googlesource.com/team/scope/
85cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
86cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
87
88cat << EOF > gerrit.googlesource.com/adir.git/config
89[core]
90 repositoryformatversion = 0
91 filemode = true
92 bare = true
93[remote "origin"]
94 url = http://gerrit.googlesource.com/adir
95[branch "master"]
96 remote = origin
97 merge = refs/heads/master
98EOF
99`
100 cmd := exec.Command("/bin/sh", "-euxc", script)
101 cmd.Dir = dir
102 if out, err := cmd.CombinedOutput(); err != nil {
103 return fmt.Errorf("execution error: %v, output %s", err, out)
104 }
105 return nil
106}
107
108func TestFindGitRepos(t *testing.T) {
109 dir := t.TempDir()
110
111 if err := createSubmoduleRepo(dir); err != nil {
112 t.Error("createSubmoduleRepo", err)
113 }
114 repos, err := FindGitRepos(dir)
115 if err != nil {
116 t.Error("FindGitRepos", err)
117 }
118
119 got := map[string]bool{}
120 for _, r := range repos {
121 p, err := filepath.Rel(dir, r)
122 if err != nil {
123 t.Fatalf("Relative: %v", err)
124 }
125
126 got[p] = true
127 }
128
129 want := map[string]bool{
130 "adir/.git": true,
131 "bdir/.git": true,
132 "gerrit.googlesource.com/adir.git": true,
133 "gerrit.googlesource.com/bdir.git": true,
134 "gerrit.googlesource.com/sub/bdir.git": true,
135 "gerrit.googlesource.com/team/scope/repoa.git": true,
136 "gerrit.googlesource.com/team/scope/repob.git": true,
137 }
138 if !reflect.DeepEqual(got, want) {
139 t.Errorf("got %v want %v", got, want)
140 }
141}
142
143func TestCollectFiles(t *testing.T) {
144 dir := t.TempDir()
145
146 if err := createSubmoduleRepo(dir); err != nil {
147 t.Fatalf("TempDir: %v", err)
148 }
149
150 cache := NewRepoCache(dir)
151
152 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
153 repo, err := cache.Open(aURL)
154 if err != nil {
155 t.Fatalf("Open: %v", err)
156 }
157
158 headRef, err := repo.Head()
159 if err != nil {
160 t.Fatalf("HEAD tree: %v", err)
161 }
162 commit, err := repo.CommitObject(headRef.Hash())
163 if err != nil {
164 t.Fatalf("commit obj HEAD: %v", err)
165 }
166
167 tree, err := repo.TreeObject(commit.TreeHash)
168 if err != nil {
169 t.Fatalf("AsTree: %v", err)
170 }
171
172 rw := NewRepoWalker(repo, aURL.String(), cache)
173 versions, err := rw.CollectFiles(tree, "main", &ignore.Matcher{})
174 if err != nil {
175 t.Fatalf("CollectFiles: %v", err)
176 }
177
178 bnameHash := versions["bname"]
179 if entry, err := tree.FindEntry("bname"); err != nil {
180 t.Fatalf("FindEntry %v", err)
181 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
182 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
183 }
184
185 var paths []string
186 for k := range rw.Files {
187 paths = append(paths, k.FullPath())
188 }
189 sort.Strings(paths)
190
191 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
192 if !reflect.DeepEqual(paths, want) {
193 t.Errorf("got %v, want %v", paths, want)
194 }
195}
196
197func TestSubmoduleIndex(t *testing.T) {
198 dir := t.TempDir()
199
200 if err := createSubmoduleRepo(dir); err != nil {
201 t.Fatalf("createSubmoduleRepo: %v", err)
202 }
203
204 indexDir := t.TempDir()
205
206 buildOpts := index.Options{
207 IndexDir: indexDir,
208 }
209 opts := Options{
210 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
211 BuildOptions: buildOpts,
212 BranchPrefix: "refs/heads/",
213 Branches: []string{"master"},
214 Submodules: true,
215 Incremental: true,
216 RepoCacheDir: dir,
217 }
218 if _, err := IndexGitRepo(opts); err != nil {
219 t.Fatalf("IndexGitRepo: %v", err)
220 }
221
222 searcher, err := search.NewDirectorySearcher(indexDir)
223 if err != nil {
224 t.Fatal("NewDirectorySearcher", err)
225 }
226 defer searcher.Close()
227
228 results, err := searcher.Search(context.Background(),
229 &query.Substring{Pattern: "bcont"},
230 &zoekt.SearchOptions{})
231 if err != nil {
232 t.Fatal("Search", err)
233 }
234
235 if len(results.Files) != 1 {
236 t.Fatalf("got search result %v, want 1 file", results.Files)
237 }
238
239 file := results.Files[0]
240 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
241 t.Errorf("got subrepo name %q, want %q", got, want)
242 }
243 if got, want := file.SubRepositoryPath, "bname"; got != want {
244 t.Errorf("got subrepo path %q, want %q", got, want)
245 }
246
247 subVersion := file.Version
248 if len(subVersion) != 40 {
249 t.Fatalf("got %q, want hex sha1", subVersion)
250 }
251
252 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
253 t.Fatalf("Search('acont'): %v", err)
254 } else if len(results.Files) != 1 {
255 t.Errorf("got %v, want 1 result", results.Files)
256 } else if f := results.Files[0]; f.Version == subVersion {
257 t.Errorf("version in super repo matched version is subrepo.")
258 }
259}
260
261func TestSubmoduleIndexWithoutRepocache(t *testing.T) {
262 dir := t.TempDir()
263
264 if err := createSubmoduleRepo(dir); err != nil {
265 t.Fatalf("createSubmoduleRepo: %v", err)
266 }
267
268 indexDir := t.TempDir()
269
270 buildOpts := index.Options{
271 RepositoryDescription: zoekt.Repository{Name: "adir"},
272 IndexDir: indexDir,
273 }
274 opts := Options{
275 RepoDir: filepath.Join(dir, "adir"),
276 BuildOptions: buildOpts,
277 BranchPrefix: "refs/heads/",
278 Branches: []string{"master"},
279 Submodules: true,
280 Incremental: true,
281 }
282 if _, err := IndexGitRepo(opts); err != nil {
283 t.Fatalf("IndexGitRepo: %v", err)
284 }
285
286 searcher, err := search.NewDirectorySearcher(indexDir)
287 if err != nil {
288 t.Fatal("NewDirectorySearcher", err)
289 }
290 defer searcher.Close()
291
292 results, err := searcher.Search(context.Background(),
293 &query.Substring{Pattern: "bcont"},
294 &zoekt.SearchOptions{})
295 if err != nil {
296 t.Fatal("Search", err)
297 }
298
299 if len(results.Files) != 1 {
300 t.Fatalf("got search result %v, want 1 file", results.Files)
301 }
302
303 file := results.Files[0]
304 if got, want := file.SubRepositoryName, "bname"; got != want {
305 t.Errorf("got subrepo name %q, want %q", got, want)
306 }
307 if got, want := file.SubRepositoryPath, "bname"; got != want {
308 t.Errorf("got subrepo path %q, want %q", got, want)
309 }
310
311 subVersion := file.Version
312 if len(subVersion) != 40 {
313 t.Fatalf("got %q, want hex sha1", subVersion)
314 }
315
316 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
317 t.Fatalf("Search('acont'): %v", err)
318 } else if len(results.Files) != 1 {
319 t.Errorf("got %v, want 1 result", results.Files)
320 } else if f := results.Files[0]; f.Version == subVersion {
321 t.Errorf("version in super repo matched version is subrepo.")
322 }
323}
324
325func createSymlinkRepo(dir string) error {
326 if err := os.MkdirAll(dir, 0o755); err != nil {
327 return err
328 }
329 script := `mkdir adir bdir
330git init -b master
331git config user.email "you@example.com"
332git config user.name "Your Name"
333
334echo acont > adir/afile
335git add adir/afile
336
337echo bcont > bdir/bfile
338git add bdir/bfile
339
340ln -s ./adir/afile asymlink
341git add asymlink
342
343git commit -am amsg
344
345cat << EOF > .git/config
346[core]
347 repositoryformatversion = 0
348 filemode = true
349 bare = true
350[remote "origin"]
351 url = http://codehost.com/arepo
352[branch "master"]
353 remote = origin
354 merge = refs/heads/master
355EOF
356`
357 cmd := exec.Command("/bin/sh", "-euxc", script)
358 cmd.Dir = dir
359 if out, err := cmd.CombinedOutput(); err != nil {
360 return fmt.Errorf("execution error: %v, output %s", err, out)
361 }
362 return nil
363}
364
365func TestSearchSymlinkByContent(t *testing.T) {
366 dir := t.TempDir()
367
368 if err := createSymlinkRepo(dir); err != nil {
369 t.Fatalf("createSubmoduleRepo: %v", err)
370 }
371
372 indexDir := t.TempDir()
373
374 buildOpts := index.Options{
375 IndexDir: indexDir,
376 }
377 opts := Options{
378 RepoDir: filepath.Join(dir),
379 BuildOptions: buildOpts,
380 BranchPrefix: "refs/heads/",
381 Branches: []string{"master"},
382 Submodules: true,
383 Incremental: true,
384 RepoCacheDir: dir,
385 }
386 if _, err := IndexGitRepo(opts); err != nil {
387 t.Fatalf("IndexGitRepo: %v", err)
388 }
389
390 searcher, err := search.NewDirectorySearcher(indexDir)
391 if err != nil {
392 t.Fatal("NewDirectorySearcher", err)
393 }
394 defer searcher.Close()
395
396 // The content of the symlink and the file path the symlink points to both
397 // contain the string "afile". Hence we expect 1 path match and 1 content match.
398 results, err := searcher.Search(context.Background(),
399 &query.Substring{Pattern: "afile"},
400 &zoekt.SearchOptions{})
401 if err != nil {
402 t.Fatal("Search", err)
403 }
404
405 if len(results.Files) != 2 {
406 t.Fatalf("got search result %v, want 2 files", results.Files)
407 }
408
409 got := make([]string, 0, 2)
410 for _, file := range results.Files {
411 got = append(got, file.FileName)
412 }
413 sort.Strings(got)
414
415 want := []string{"adir/afile", "asymlink"}
416
417 if d := cmp.Diff(want, got); d != "" {
418 t.Fatalf("-want, +got %s\n", d)
419 }
420}
421
422func TestAllowMissingBranch(t *testing.T) {
423 dir := t.TempDir()
424
425 if err := createSubmoduleRepo(dir); err != nil {
426 t.Fatalf("createSubmoduleRepo: %v", err)
427 }
428
429 indexDir := t.TempDir()
430
431 buildOpts := index.Options{
432 IndexDir: indexDir,
433 }
434
435 opts := Options{
436 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
437 BuildOptions: buildOpts,
438 BranchPrefix: "refs/heads/",
439 Branches: []string{"master", "nonexist"},
440 Submodules: true,
441 Incremental: true,
442 RepoCacheDir: dir,
443 }
444 if _, err := IndexGitRepo(opts); err == nil {
445 t.Fatalf("IndexGitRepo(nonexist) succeeded")
446 }
447 opts.AllowMissingBranch = true
448 if _, err := IndexGitRepo(opts); err != nil {
449 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
450 }
451}
452
453func createMultibranchRepo(dir string) error {
454 if err := os.MkdirAll(dir, 0o755); err != nil {
455 return err
456 }
457 script := `mkdir repo
458cd repo
459git init -b master
460mkdir subdir
461echo acont > afile
462echo sub-cont > subdir/sub-file
463git add afile subdir/sub-file
464git config user.email "you@example.com"
465git config user.name "Your Name"
466GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
467
468git branch branchdir/a
469
470echo acont >> afile
471git add afile subdir/sub-file
472GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
473
474git branch branchdir/b
475
476git branch c
477
478git update-ref refs/meta/config HEAD
479`
480 cmd := exec.Command("/bin/sh", "-euxc", script)
481 cmd.Dir = dir
482 if out, err := cmd.CombinedOutput(); err != nil {
483 return fmt.Errorf("execution error: %v, output %s", err, out)
484 }
485 return nil
486}
487
488func TestBranchWildcard(t *testing.T) {
489 dir := t.TempDir()
490
491 if err := createMultibranchRepo(dir); err != nil {
492 t.Fatalf("createMultibranchRepo: %v", err)
493 }
494
495 indexDir := t.TempDir()
496
497 buildOpts := index.Options{
498 IndexDir: indexDir,
499 RepositoryDescription: zoekt.Repository{
500 Name: "repo",
501 },
502 }
503 buildOpts.SetDefaults()
504
505 opts := Options{
506 RepoDir: filepath.Join(dir + "/repo"),
507 BuildOptions: buildOpts,
508 BranchPrefix: "refs/heads",
509 Branches: []string{"branchdir/*"},
510 Submodules: true,
511 Incremental: true,
512 }
513 if _, err := IndexGitRepo(opts); err != nil {
514 t.Fatalf("IndexGitRepo: %v", err)
515 }
516
517 searcher, err := search.NewDirectorySearcher(indexDir)
518 if err != nil {
519 t.Fatal("NewDirectorySearcher", err)
520 }
521 defer searcher.Close()
522
523 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
524 t.Fatalf("List(): %v", err)
525 } else if len(rlist.Repos) != 1 {
526 t.Errorf("got %v, want 1 result", rlist.Repos)
527 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
528 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
529 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
530 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
531 }
532}
533
534func TestSkipSubmodules(t *testing.T) {
535 dir := t.TempDir()
536
537 if err := createSubmoduleRepo(dir); err != nil {
538 t.Fatalf("createMultibranchRepo: %v", err)
539 }
540
541 indexDir := t.TempDir()
542
543 buildOpts := index.Options{
544 IndexDir: indexDir,
545 RepositoryDescription: zoekt.Repository{
546 Name: "gerrit.googlesource.com/adir",
547 },
548 }
549 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
550 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
551 t.Fatalf("Rename: %v", err)
552 }
553
554 opts := Options{
555 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
556 BuildOptions: buildOpts,
557 BranchPrefix: "refs/heads",
558 Branches: []string{"master"},
559 Submodules: false,
560 }
561 if _, err := IndexGitRepo(opts); err != nil {
562 t.Fatalf("IndexGitRepo: %v", err)
563 }
564}
565
566func TestFullAndShortRefNames(t *testing.T) {
567 dir := t.TempDir()
568
569 if err := createMultibranchRepo(dir); err != nil {
570 t.Fatalf("createMultibranchRepo: %v", err)
571 }
572
573 indexDir := t.TempDir()
574
575 buildOpts := index.Options{
576 IndexDir: indexDir,
577 RepositoryDescription: zoekt.Repository{
578 Name: "repo",
579 },
580 }
581 buildOpts.SetDefaults()
582
583 opts := Options{
584 RepoDir: filepath.Join(dir + "/repo"),
585 BuildOptions: buildOpts,
586 BranchPrefix: "refs/heads",
587 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
588 Submodules: false,
589 Incremental: false,
590 AllowMissingBranch: false,
591 }
592 if _, err := IndexGitRepo(opts); err != nil {
593 t.Fatalf("IndexGitRepo: %v", err)
594 }
595
596 searcher, err := search.NewDirectorySearcher(indexDir)
597 if err != nil {
598 t.Fatal("NewDirectorySearcher", err)
599 }
600 defer searcher.Close()
601
602 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
603 t.Fatalf("List(): %v", err)
604 } else if len(rlist.Repos) != 1 {
605 t.Errorf("got %v, want 1 result", rlist.Repos)
606 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
607 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
608 }
609}
610
611func TestUniq(t *testing.T) {
612 in := []string{"a", "b", "b", "c", "c"}
613 want := []string{"a", "b", "c"}
614 got := uniq(in)
615 if !reflect.DeepEqual(got, want) {
616 t.Errorf("got %v, want %v", got, want)
617 }
618}
619
620func TestLatestCommit(t *testing.T) {
621 dir := t.TempDir()
622 indexDir := t.TempDir()
623
624 if err := createMultibranchRepo(dir); err != nil {
625 t.Fatalf("createMultibranchRepo: %v", err)
626 }
627
628 buildOpts := index.Options{
629 IndexDir: indexDir,
630 RepositoryDescription: zoekt.Repository{
631 Name: "repo",
632 },
633 }
634 buildOpts.SetDefaults()
635
636 opts := Options{
637 RepoDir: filepath.Join(dir + "/repo"),
638 BuildOptions: buildOpts,
639 BranchPrefix: "refs/heads",
640 Branches: []string{"branchdir/a", "branchdir/b"},
641 }
642 if _, err := IndexGitRepo(opts); err != nil {
643 t.Fatalf("IndexGitRepo: %v", err)
644 }
645
646 searcher, err := search.NewDirectorySearcher(indexDir)
647 if err != nil {
648 t.Fatal("NewDirectorySearcher", err)
649 }
650 defer searcher.Close()
651
652 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
653 if err != nil {
654 t.Fatalf("List(): %v", err)
655 }
656
657 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
658 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
659 }
660}