fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32 "github.com/sourcegraph/zoekt"
33 "github.com/sourcegraph/zoekt/ignore"
34 "github.com/sourcegraph/zoekt/index"
35 "github.com/sourcegraph/zoekt/query"
36 "github.com/sourcegraph/zoekt/search"
37)
38
39func createSubmoduleRepo(dir string) error {
40 if err := os.MkdirAll(dir, 0o755); err != nil {
41 return err
42 }
43 script := `
44# Fix fatal: transport 'file' not allowed
45export GIT_CONFIG_COUNT=1
46export GIT_CONFIG_KEY_0=protocol.file.allow
47export GIT_CONFIG_VALUE_0=always
48
49mkdir adir bdir
50cd adir
51git init -b master
52mkdir subdir
53echo acont > afile
54echo sub-cont > subdir/sub-file
55git add afile subdir/sub-file
56git config user.email "you@example.com"
57git config user.name "Your Name"
58git commit -am amsg
59
60cd ..
61cd bdir
62git init -b master
63echo bcont > bfile
64ln -s bfile bsymlink
65git add bfile bsymlink
66git config user.email "you@example.com"
67git config user.name "Your Name"
68git commit -am bmsg
69
70cd ../adir
71git submodule add --name bname -- ../bdir bname
72git commit -am bmodmsg
73cat .gitmodules
74cd ..
75mkdir gerrit.googlesource.com
76git clone --bare adir gerrit.googlesource.com/adir.git
77git clone --bare bdir gerrit.googlesource.com/bdir.git
78
79mkdir gerrit.googlesource.com/bogus.git
80mkdir gerrit.googlesource.com/sub
81git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
82
83mkdir -p gerrit.googlesource.com/team/scope/
84cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
85cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
86
87cat << EOF > gerrit.googlesource.com/adir.git/config
88[core]
89 repositoryformatversion = 0
90 filemode = true
91 bare = true
92[remote "origin"]
93 url = http://gerrit.googlesource.com/adir
94[branch "master"]
95 remote = origin
96 merge = refs/heads/master
97EOF
98`
99 cmd := exec.Command("/bin/sh", "-euxc", script)
100 cmd.Dir = dir
101 if out, err := cmd.CombinedOutput(); err != nil {
102 return fmt.Errorf("execution error: %v, output %s", err, out)
103 }
104 return nil
105}
106
107func TestFindGitRepos(t *testing.T) {
108 dir := t.TempDir()
109
110 if err := createSubmoduleRepo(dir); err != nil {
111 t.Error("createSubmoduleRepo", err)
112 }
113 repos, err := FindGitRepos(dir)
114 if err != nil {
115 t.Error("FindGitRepos", err)
116 }
117
118 got := map[string]bool{}
119 for _, r := range repos {
120 p, err := filepath.Rel(dir, r)
121 if err != nil {
122 t.Fatalf("Relative: %v", err)
123 }
124
125 got[p] = true
126 }
127
128 want := map[string]bool{
129 "adir/.git": true,
130 "bdir/.git": true,
131 "gerrit.googlesource.com/adir.git": true,
132 "gerrit.googlesource.com/bdir.git": true,
133 "gerrit.googlesource.com/sub/bdir.git": true,
134 "gerrit.googlesource.com/team/scope/repoa.git": true,
135 "gerrit.googlesource.com/team/scope/repob.git": true,
136 }
137 if !reflect.DeepEqual(got, want) {
138 t.Errorf("got %v want %v", got, want)
139 }
140}
141
142func TestCollectFiles(t *testing.T) {
143 dir := t.TempDir()
144
145 if err := createSubmoduleRepo(dir); err != nil {
146 t.Fatalf("TempDir: %v", err)
147 }
148
149 cache := NewRepoCache(dir)
150
151 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
152 repo, err := cache.Open(aURL)
153 if err != nil {
154 t.Fatalf("Open: %v", err)
155 }
156
157 headRef, err := repo.Head()
158 if err != nil {
159 t.Fatalf("HEAD tree: %v", err)
160 }
161 commit, err := repo.CommitObject(headRef.Hash())
162 if err != nil {
163 t.Fatalf("commit obj HEAD: %v", err)
164 }
165
166 tree, err := repo.TreeObject(commit.TreeHash)
167 if err != nil {
168 t.Fatalf("AsTree: %v", err)
169 }
170
171 rw := NewRepoWalker(repo, aURL.String(), cache)
172 versions, err := rw.CollectFiles(tree, "main", &ignore.Matcher{})
173 if err != nil {
174 t.Fatalf("CollectFiles: %v", err)
175 }
176
177 bnameHash := versions["bname"]
178 if entry, err := tree.FindEntry("bname"); err != nil {
179 t.Fatalf("FindEntry %v", err)
180 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
181 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
182 }
183
184 var paths []string
185 for k := range rw.Files {
186 paths = append(paths, k.FullPath())
187 }
188 sort.Strings(paths)
189
190 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
191 if !reflect.DeepEqual(paths, want) {
192 t.Errorf("got %v, want %v", paths, want)
193 }
194}
195
196func TestSubmoduleIndex(t *testing.T) {
197 dir := t.TempDir()
198
199 if err := createSubmoduleRepo(dir); err != nil {
200 t.Fatalf("createSubmoduleRepo: %v", err)
201 }
202
203 indexDir := t.TempDir()
204
205 buildOpts := index.Options{
206 IndexDir: indexDir,
207 }
208 opts := Options{
209 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
210 BuildOptions: buildOpts,
211 BranchPrefix: "refs/heads/",
212 Branches: []string{"master"},
213 Submodules: true,
214 Incremental: true,
215 RepoCacheDir: dir,
216 }
217 if _, err := IndexGitRepo(opts); err != nil {
218 t.Fatalf("IndexGitRepo: %v", err)
219 }
220
221 searcher, err := search.NewDirectorySearcher(indexDir)
222 if err != nil {
223 t.Fatal("NewDirectorySearcher", err)
224 }
225 defer searcher.Close()
226
227 results, err := searcher.Search(context.Background(),
228 &query.Substring{Pattern: "bcont"},
229 &zoekt.SearchOptions{})
230 if err != nil {
231 t.Fatal("Search", err)
232 }
233
234 if len(results.Files) != 1 {
235 t.Fatalf("got search result %v, want 1 file", results.Files)
236 }
237
238 file := results.Files[0]
239 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
240 t.Errorf("got subrepo name %q, want %q", got, want)
241 }
242 if got, want := file.SubRepositoryPath, "bname"; got != want {
243 t.Errorf("got subrepo path %q, want %q", got, want)
244 }
245
246 subVersion := file.Version
247 if len(subVersion) != 40 {
248 t.Fatalf("got %q, want hex sha1", subVersion)
249 }
250
251 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
252 t.Fatalf("Search('acont'): %v", err)
253 } else if len(results.Files) != 1 {
254 t.Errorf("got %v, want 1 result", results.Files)
255 } else if f := results.Files[0]; f.Version == subVersion {
256 t.Errorf("version in super repo matched version is subrepo.")
257 }
258}
259
260func createSymlinkRepo(dir string) error {
261 if err := os.MkdirAll(dir, 0o755); err != nil {
262 return err
263 }
264 script := `mkdir adir bdir
265git init -b master
266git config user.email "you@example.com"
267git config user.name "Your Name"
268
269echo acont > adir/afile
270git add adir/afile
271
272echo bcont > bdir/bfile
273git add bdir/bfile
274
275ln -s ./adir/afile asymlink
276git add asymlink
277
278git commit -am amsg
279
280cat << EOF > .git/config
281[core]
282 repositoryformatversion = 0
283 filemode = true
284 bare = true
285[remote "origin"]
286 url = http://codehost.com/arepo
287[branch "master"]
288 remote = origin
289 merge = refs/heads/master
290EOF
291`
292 cmd := exec.Command("/bin/sh", "-euxc", script)
293 cmd.Dir = dir
294 if out, err := cmd.CombinedOutput(); err != nil {
295 return fmt.Errorf("execution error: %v, output %s", err, out)
296 }
297 return nil
298}
299
300func TestSearchSymlinkByContent(t *testing.T) {
301 dir := t.TempDir()
302
303 if err := createSymlinkRepo(dir); err != nil {
304 t.Fatalf("createSubmoduleRepo: %v", err)
305 }
306
307 indexDir := t.TempDir()
308
309 buildOpts := index.Options{
310 IndexDir: indexDir,
311 }
312 opts := Options{
313 RepoDir: filepath.Join(dir),
314 BuildOptions: buildOpts,
315 BranchPrefix: "refs/heads/",
316 Branches: []string{"master"},
317 Submodules: true,
318 Incremental: true,
319 RepoCacheDir: dir,
320 }
321 if _, err := IndexGitRepo(opts); err != nil {
322 t.Fatalf("IndexGitRepo: %v", err)
323 }
324
325 searcher, err := search.NewDirectorySearcher(indexDir)
326 if err != nil {
327 t.Fatal("NewDirectorySearcher", err)
328 }
329 defer searcher.Close()
330
331 // The content of the symlink and the file path the symlink points to both
332 // contain the string "afile". Hence we expect 1 path match and 1 content match.
333 results, err := searcher.Search(context.Background(),
334 &query.Substring{Pattern: "afile"},
335 &zoekt.SearchOptions{})
336 if err != nil {
337 t.Fatal("Search", err)
338 }
339
340 if len(results.Files) != 2 {
341 t.Fatalf("got search result %v, want 2 files", results.Files)
342 }
343
344 got := make([]string, 0, 2)
345 for _, file := range results.Files {
346 got = append(got, file.FileName)
347 }
348 sort.Strings(got)
349
350 want := []string{"adir/afile", "asymlink"}
351
352 if d := cmp.Diff(want, got); d != "" {
353 t.Fatalf("-want, +got %s\n", d)
354 }
355}
356
357func TestAllowMissingBranch(t *testing.T) {
358 dir := t.TempDir()
359
360 if err := createSubmoduleRepo(dir); err != nil {
361 t.Fatalf("createSubmoduleRepo: %v", err)
362 }
363
364 indexDir := t.TempDir()
365
366 buildOpts := index.Options{
367 IndexDir: indexDir,
368 }
369
370 opts := Options{
371 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
372 BuildOptions: buildOpts,
373 BranchPrefix: "refs/heads/",
374 Branches: []string{"master", "nonexist"},
375 Submodules: true,
376 Incremental: true,
377 RepoCacheDir: dir,
378 }
379 if _, err := IndexGitRepo(opts); err == nil {
380 t.Fatalf("IndexGitRepo(nonexist) succeeded")
381 }
382 opts.AllowMissingBranch = true
383 if _, err := IndexGitRepo(opts); err != nil {
384 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
385 }
386}
387
388func createMultibranchRepo(dir string) error {
389 if err := os.MkdirAll(dir, 0o755); err != nil {
390 return err
391 }
392 script := `mkdir repo
393cd repo
394git init -b master
395mkdir subdir
396echo acont > afile
397echo sub-cont > subdir/sub-file
398git add afile subdir/sub-file
399git config user.email "you@example.com"
400git config user.name "Your Name"
401GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
402
403git branch branchdir/a
404
405echo acont >> afile
406git add afile subdir/sub-file
407GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
408
409git branch branchdir/b
410
411git branch c
412
413git update-ref refs/meta/config HEAD
414`
415 cmd := exec.Command("/bin/sh", "-euxc", script)
416 cmd.Dir = dir
417 if out, err := cmd.CombinedOutput(); err != nil {
418 return fmt.Errorf("execution error: %v, output %s", err, out)
419 }
420 return nil
421}
422
423func TestBranchWildcard(t *testing.T) {
424 dir := t.TempDir()
425
426 if err := createMultibranchRepo(dir); err != nil {
427 t.Fatalf("createMultibranchRepo: %v", err)
428 }
429
430 indexDir := t.TempDir()
431
432 buildOpts := index.Options{
433 IndexDir: indexDir,
434 RepositoryDescription: zoekt.Repository{
435 Name: "repo",
436 },
437 }
438 buildOpts.SetDefaults()
439
440 opts := Options{
441 RepoDir: filepath.Join(dir + "/repo"),
442 BuildOptions: buildOpts,
443 BranchPrefix: "refs/heads",
444 Branches: []string{"branchdir/*"},
445 Submodules: true,
446 Incremental: true,
447 }
448 if _, err := IndexGitRepo(opts); err != nil {
449 t.Fatalf("IndexGitRepo: %v", err)
450 }
451
452 searcher, err := search.NewDirectorySearcher(indexDir)
453 if err != nil {
454 t.Fatal("NewDirectorySearcher", err)
455 }
456 defer searcher.Close()
457
458 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
459 t.Fatalf("List(): %v", err)
460 } else if len(rlist.Repos) != 1 {
461 t.Errorf("got %v, want 1 result", rlist.Repos)
462 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
463 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
464 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
465 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
466 }
467}
468
469func TestSkipSubmodules(t *testing.T) {
470 dir := t.TempDir()
471
472 if err := createSubmoduleRepo(dir); err != nil {
473 t.Fatalf("createMultibranchRepo: %v", err)
474 }
475
476 indexDir := t.TempDir()
477
478 buildOpts := index.Options{
479 IndexDir: indexDir,
480 RepositoryDescription: zoekt.Repository{
481 Name: "gerrit.googlesource.com/adir",
482 },
483 }
484 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
485 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
486 t.Fatalf("Rename: %v", err)
487 }
488
489 opts := Options{
490 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
491 BuildOptions: buildOpts,
492 BranchPrefix: "refs/heads",
493 Branches: []string{"master"},
494 Submodules: false,
495 }
496 if _, err := IndexGitRepo(opts); err != nil {
497 t.Fatalf("IndexGitRepo: %v", err)
498 }
499}
500
501func TestFullAndShortRefNames(t *testing.T) {
502 dir := t.TempDir()
503
504 if err := createMultibranchRepo(dir); err != nil {
505 t.Fatalf("createMultibranchRepo: %v", err)
506 }
507
508 indexDir := t.TempDir()
509
510 buildOpts := index.Options{
511 IndexDir: indexDir,
512 RepositoryDescription: zoekt.Repository{
513 Name: "repo",
514 },
515 }
516 buildOpts.SetDefaults()
517
518 opts := Options{
519 RepoDir: filepath.Join(dir + "/repo"),
520 BuildOptions: buildOpts,
521 BranchPrefix: "refs/heads",
522 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
523 Submodules: false,
524 Incremental: false,
525 AllowMissingBranch: false,
526 }
527 if _, err := IndexGitRepo(opts); err != nil {
528 t.Fatalf("IndexGitRepo: %v", err)
529 }
530
531 searcher, err := search.NewDirectorySearcher(indexDir)
532 if err != nil {
533 t.Fatal("NewDirectorySearcher", err)
534 }
535 defer searcher.Close()
536
537 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
538 t.Fatalf("List(): %v", err)
539 } else if len(rlist.Repos) != 1 {
540 t.Errorf("got %v, want 1 result", rlist.Repos)
541 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
542 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
543 }
544}
545
546func TestUniq(t *testing.T) {
547 in := []string{"a", "b", "b", "c", "c"}
548 want := []string{"a", "b", "c"}
549 got := uniq(in)
550 if !reflect.DeepEqual(got, want) {
551 t.Errorf("got %v, want %v", got, want)
552 }
553}
554
555func TestLatestCommit(t *testing.T) {
556 dir := t.TempDir()
557 indexDir := t.TempDir()
558
559 if err := createMultibranchRepo(dir); err != nil {
560 t.Fatalf("createMultibranchRepo: %v", err)
561 }
562
563 buildOpts := index.Options{
564 IndexDir: indexDir,
565 RepositoryDescription: zoekt.Repository{
566 Name: "repo",
567 },
568 }
569 buildOpts.SetDefaults()
570
571 opts := Options{
572 RepoDir: filepath.Join(dir + "/repo"),
573 BuildOptions: buildOpts,
574 BranchPrefix: "refs/heads",
575 Branches: []string{"branchdir/a", "branchdir/b"},
576 }
577 if _, err := IndexGitRepo(opts); err != nil {
578 t.Fatalf("IndexGitRepo: %v", err)
579 }
580
581 searcher, err := search.NewDirectorySearcher(indexDir)
582 if err != nil {
583 t.Fatal("NewDirectorySearcher", err)
584 }
585 defer searcher.Close()
586
587 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
588 if err != nil {
589 t.Fatalf("List(): %v", err)
590 }
591
592 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
593 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
594 }
595}