fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32 "github.com/sourcegraph/zoekt/ignore"
33
34 "github.com/sourcegraph/zoekt"
35 "github.com/sourcegraph/zoekt/build"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/shards"
38)
39
40func createSubmoduleRepo(dir string) error {
41 if err := os.MkdirAll(dir, 0o755); err != nil {
42 return err
43 }
44 script := `
45# Fix fatal: transport 'file' not allowed
46export GIT_CONFIG_COUNT=1
47export GIT_CONFIG_KEY_0=protocol.file.allow
48export GIT_CONFIG_VALUE_0=always
49
50mkdir adir bdir
51cd adir
52git init -b master
53mkdir subdir
54echo acont > afile
55echo sub-cont > subdir/sub-file
56git add afile subdir/sub-file
57git config user.email "you@example.com"
58git config user.name "Your Name"
59git commit -am amsg
60
61cd ..
62cd bdir
63git init -b master
64echo bcont > bfile
65ln -s bfile bsymlink
66git add bfile bsymlink
67git config user.email "you@example.com"
68git config user.name "Your Name"
69git commit -am bmsg
70
71cd ../adir
72git submodule add --name bname -- ../bdir bname
73git commit -am bmodmsg
74cat .gitmodules
75cd ..
76mkdir gerrit.googlesource.com
77git clone --bare adir gerrit.googlesource.com/adir.git
78git clone --bare bdir gerrit.googlesource.com/bdir.git
79
80mkdir gerrit.googlesource.com/bogus.git
81mkdir gerrit.googlesource.com/sub
82git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
83
84mkdir -p gerrit.googlesource.com/team/scope/
85cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
86cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
87
88cat << EOF > gerrit.googlesource.com/adir.git/config
89[core]
90 repositoryformatversion = 0
91 filemode = true
92 bare = true
93[remote "origin"]
94 url = http://gerrit.googlesource.com/adir
95[branch "master"]
96 remote = origin
97 merge = refs/heads/master
98EOF
99`
100 cmd := exec.Command("/bin/sh", "-euxc", script)
101 cmd.Dir = dir
102 if out, err := cmd.CombinedOutput(); err != nil {
103 return fmt.Errorf("execution error: %v, output %s", err, out)
104 }
105 return nil
106}
107
108func TestFindGitRepos(t *testing.T) {
109 dir := t.TempDir()
110
111 if err := createSubmoduleRepo(dir); err != nil {
112 t.Error("createSubmoduleRepo", err)
113 }
114 repos, err := FindGitRepos(dir)
115 if err != nil {
116 t.Error("FindGitRepos", err)
117 }
118
119 got := map[string]bool{}
120 for _, r := range repos {
121 p, err := filepath.Rel(dir, r)
122 if err != nil {
123 t.Fatalf("Relative: %v", err)
124 }
125
126 got[p] = true
127 }
128
129 want := map[string]bool{
130 "adir/.git": true,
131 "bdir/.git": true,
132 "gerrit.googlesource.com/adir.git": true,
133 "gerrit.googlesource.com/bdir.git": true,
134 "gerrit.googlesource.com/sub/bdir.git": true,
135 "gerrit.googlesource.com/team/scope/repoa.git": true,
136 "gerrit.googlesource.com/team/scope/repob.git": true,
137 }
138 if !reflect.DeepEqual(got, want) {
139 t.Errorf("got %v want %v", got, want)
140 }
141}
142
143func TestCollectFiles(t *testing.T) {
144 dir := t.TempDir()
145
146 if err := createSubmoduleRepo(dir); err != nil {
147 t.Fatalf("TempDir: %v", err)
148 }
149
150 cache := NewRepoCache(dir)
151
152 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
153 repo, err := cache.Open(aURL)
154 if err != nil {
155 t.Fatalf("Open: %v", err)
156 }
157
158 headRef, err := repo.Head()
159 if err != nil {
160 t.Fatalf("HEAD tree: %v", err)
161 }
162 commit, err := repo.CommitObject(headRef.Hash())
163 if err != nil {
164 t.Fatalf("commit obj HEAD: %v", err)
165 }
166
167 tree, err := repo.TreeObject(commit.TreeHash)
168 if err != nil {
169 t.Fatalf("AsTree: %v", err)
170 }
171
172 rw := NewRepoWalker(repo, aURL.String(), cache)
173 versions, err := rw.CollectFiles(tree, "main", &ignore.Matcher{})
174 if err != nil {
175 t.Fatalf("CollectFiles: %v", err)
176 }
177
178 bnameHash := versions["bname"]
179 if entry, err := tree.FindEntry("bname"); err != nil {
180 t.Fatalf("FindEntry %v", err)
181 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
182 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
183 }
184
185 var paths []string
186 for k := range rw.Files {
187 paths = append(paths, k.FullPath())
188 }
189 sort.Strings(paths)
190
191 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
192 if !reflect.DeepEqual(paths, want) {
193 t.Errorf("got %v, want %v", paths, want)
194 }
195}
196
197func TestSubmoduleIndex(t *testing.T) {
198 dir := t.TempDir()
199
200 if err := createSubmoduleRepo(dir); err != nil {
201 t.Fatalf("createSubmoduleRepo: %v", err)
202 }
203
204 indexDir := t.TempDir()
205
206 buildOpts := build.Options{
207 IndexDir: indexDir,
208 }
209 opts := Options{
210 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
211 BuildOptions: buildOpts,
212 BranchPrefix: "refs/heads/",
213 Branches: []string{"master"},
214 Submodules: true,
215 Incremental: true,
216 RepoCacheDir: dir,
217 }
218 if _, err := IndexGitRepo(opts); err != nil {
219 t.Fatalf("IndexGitRepo: %v", err)
220 }
221
222 searcher, err := shards.NewDirectorySearcher(indexDir)
223 if err != nil {
224 t.Fatal("NewDirectorySearcher", err)
225 }
226 defer searcher.Close()
227
228 results, err := searcher.Search(context.Background(),
229 &query.Substring{Pattern: "bcont"},
230 &zoekt.SearchOptions{})
231 if err != nil {
232 t.Fatal("Search", err)
233 }
234
235 if len(results.Files) != 1 {
236 t.Fatalf("got search result %v, want 1 file", results.Files)
237 }
238
239 file := results.Files[0]
240 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
241 t.Errorf("got subrepo name %q, want %q", got, want)
242 }
243 if got, want := file.SubRepositoryPath, "bname"; got != want {
244 t.Errorf("got subrepo path %q, want %q", got, want)
245 }
246
247 subVersion := file.Version
248 if len(subVersion) != 40 {
249 t.Fatalf("got %q, want hex sha1", subVersion)
250 }
251
252 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
253 t.Fatalf("Search('acont'): %v", err)
254 } else if len(results.Files) != 1 {
255 t.Errorf("got %v, want 1 result", results.Files)
256 } else if f := results.Files[0]; f.Version == subVersion {
257 t.Errorf("version in super repo matched version is subrepo.")
258 }
259}
260
261func createSymlinkRepo(dir string) error {
262 if err := os.MkdirAll(dir, 0o755); err != nil {
263 return err
264 }
265 script := `mkdir adir bdir
266git init -b master
267git config user.email "you@example.com"
268git config user.name "Your Name"
269
270echo acont > adir/afile
271git add adir/afile
272
273echo bcont > bdir/bfile
274git add bdir/bfile
275
276ln -s ./adir/afile asymlink
277git add asymlink
278
279git commit -am amsg
280
281cat << EOF > .git/config
282[core]
283 repositoryformatversion = 0
284 filemode = true
285 bare = true
286[remote "origin"]
287 url = http://codehost.com/arepo
288[branch "master"]
289 remote = origin
290 merge = refs/heads/master
291EOF
292`
293 cmd := exec.Command("/bin/sh", "-euxc", script)
294 cmd.Dir = dir
295 if out, err := cmd.CombinedOutput(); err != nil {
296 return fmt.Errorf("execution error: %v, output %s", err, out)
297 }
298 return nil
299}
300
301func TestSearchSymlinkByContent(t *testing.T) {
302 dir := t.TempDir()
303
304 if err := createSymlinkRepo(dir); err != nil {
305 t.Fatalf("createSubmoduleRepo: %v", err)
306 }
307
308 indexDir := t.TempDir()
309
310 buildOpts := build.Options{
311 IndexDir: indexDir,
312 }
313 opts := Options{
314 RepoDir: filepath.Join(dir),
315 BuildOptions: buildOpts,
316 BranchPrefix: "refs/heads/",
317 Branches: []string{"master"},
318 Submodules: true,
319 Incremental: true,
320 RepoCacheDir: dir,
321 }
322 if _, err := IndexGitRepo(opts); err != nil {
323 t.Fatalf("IndexGitRepo: %v", err)
324 }
325
326 searcher, err := shards.NewDirectorySearcher(indexDir)
327 if err != nil {
328 t.Fatal("NewDirectorySearcher", err)
329 }
330 defer searcher.Close()
331
332 // The content of the symlink and the file path the symlink points to both
333 // contain the string "afile". Hence we expect 1 path match and 1 content match.
334 results, err := searcher.Search(context.Background(),
335 &query.Substring{Pattern: "afile"},
336 &zoekt.SearchOptions{})
337 if err != nil {
338 t.Fatal("Search", err)
339 }
340
341 if len(results.Files) != 2 {
342 t.Fatalf("got search result %v, want 2 files", results.Files)
343 }
344
345 got := make([]string, 0, 2)
346 for _, file := range results.Files {
347 got = append(got, file.FileName)
348 }
349 sort.Strings(got)
350
351 want := []string{"adir/afile", "asymlink"}
352
353 if d := cmp.Diff(want, got); d != "" {
354 t.Fatalf("-want, +got %s\n", d)
355 }
356}
357
358func TestAllowMissingBranch(t *testing.T) {
359 dir := t.TempDir()
360
361 if err := createSubmoduleRepo(dir); err != nil {
362 t.Fatalf("createSubmoduleRepo: %v", err)
363 }
364
365 indexDir := t.TempDir()
366
367 buildOpts := build.Options{
368 IndexDir: indexDir,
369 }
370
371 opts := Options{
372 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
373 BuildOptions: buildOpts,
374 BranchPrefix: "refs/heads/",
375 Branches: []string{"master", "nonexist"},
376 Submodules: true,
377 Incremental: true,
378 RepoCacheDir: dir,
379 }
380 if _, err := IndexGitRepo(opts); err == nil {
381 t.Fatalf("IndexGitRepo(nonexist) succeeded")
382 }
383 opts.AllowMissingBranch = true
384 if _, err := IndexGitRepo(opts); err != nil {
385 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
386 }
387}
388
389func createMultibranchRepo(dir string) error {
390 if err := os.MkdirAll(dir, 0o755); err != nil {
391 return err
392 }
393 script := `mkdir repo
394cd repo
395git init -b master
396mkdir subdir
397echo acont > afile
398echo sub-cont > subdir/sub-file
399git add afile subdir/sub-file
400git config user.email "you@example.com"
401git config user.name "Your Name"
402GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
403
404git branch branchdir/a
405
406echo acont >> afile
407git add afile subdir/sub-file
408GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
409
410git branch branchdir/b
411
412git branch c
413
414git update-ref refs/meta/config HEAD
415`
416 cmd := exec.Command("/bin/sh", "-euxc", script)
417 cmd.Dir = dir
418 if out, err := cmd.CombinedOutput(); err != nil {
419 return fmt.Errorf("execution error: %v, output %s", err, out)
420 }
421 return nil
422}
423
424func TestBranchWildcard(t *testing.T) {
425 dir := t.TempDir()
426
427 if err := createMultibranchRepo(dir); err != nil {
428 t.Fatalf("createMultibranchRepo: %v", err)
429 }
430
431 indexDir := t.TempDir()
432
433 buildOpts := build.Options{
434 IndexDir: indexDir,
435 RepositoryDescription: zoekt.Repository{
436 Name: "repo",
437 },
438 }
439 buildOpts.SetDefaults()
440
441 opts := Options{
442 RepoDir: filepath.Join(dir + "/repo"),
443 BuildOptions: buildOpts,
444 BranchPrefix: "refs/heads",
445 Branches: []string{"branchdir/*"},
446 Submodules: true,
447 Incremental: true,
448 }
449 if _, err := IndexGitRepo(opts); err != nil {
450 t.Fatalf("IndexGitRepo: %v", err)
451 }
452
453 searcher, err := shards.NewDirectorySearcher(indexDir)
454 if err != nil {
455 t.Fatal("NewDirectorySearcher", err)
456 }
457 defer searcher.Close()
458
459 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
460 t.Fatalf("List(): %v", err)
461 } else if len(rlist.Repos) != 1 {
462 t.Errorf("got %v, want 1 result", rlist.Repos)
463 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
464 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
465 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
466 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
467 }
468}
469
470func TestSkipSubmodules(t *testing.T) {
471 dir := t.TempDir()
472
473 if err := createSubmoduleRepo(dir); err != nil {
474 t.Fatalf("createMultibranchRepo: %v", err)
475 }
476
477 indexDir := t.TempDir()
478
479 buildOpts := build.Options{
480 IndexDir: indexDir,
481 RepositoryDescription: zoekt.Repository{
482 Name: "gerrit.googlesource.com/adir",
483 },
484 }
485 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
486 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
487 t.Fatalf("Rename: %v", err)
488 }
489
490 opts := Options{
491 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
492 BuildOptions: buildOpts,
493 BranchPrefix: "refs/heads",
494 Branches: []string{"master"},
495 Submodules: false,
496 }
497 if _, err := IndexGitRepo(opts); err != nil {
498 t.Fatalf("IndexGitRepo: %v", err)
499 }
500}
501
502func TestFullAndShortRefNames(t *testing.T) {
503 dir := t.TempDir()
504
505 if err := createMultibranchRepo(dir); err != nil {
506 t.Fatalf("createMultibranchRepo: %v", err)
507 }
508
509 indexDir := t.TempDir()
510
511 buildOpts := build.Options{
512 IndexDir: indexDir,
513 RepositoryDescription: zoekt.Repository{
514 Name: "repo",
515 },
516 }
517 buildOpts.SetDefaults()
518
519 opts := Options{
520 RepoDir: filepath.Join(dir + "/repo"),
521 BuildOptions: buildOpts,
522 BranchPrefix: "refs/heads",
523 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
524 Submodules: false,
525 Incremental: false,
526 AllowMissingBranch: false,
527 }
528 if _, err := IndexGitRepo(opts); err != nil {
529 t.Fatalf("IndexGitRepo: %v", err)
530 }
531
532 searcher, err := shards.NewDirectorySearcher(indexDir)
533 if err != nil {
534 t.Fatal("NewDirectorySearcher", err)
535 }
536 defer searcher.Close()
537
538 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
539 t.Fatalf("List(): %v", err)
540 } else if len(rlist.Repos) != 1 {
541 t.Errorf("got %v, want 1 result", rlist.Repos)
542 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
543 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
544 }
545}
546
547func TestUniq(t *testing.T) {
548 in := []string{"a", "b", "b", "c", "c"}
549 want := []string{"a", "b", "c"}
550 got := uniq(in)
551 if !reflect.DeepEqual(got, want) {
552 t.Errorf("got %v, want %v", got, want)
553 }
554}
555
556func TestLatestCommit(t *testing.T) {
557 dir := t.TempDir()
558 indexDir := t.TempDir()
559
560 if err := createMultibranchRepo(dir); err != nil {
561 t.Fatalf("createMultibranchRepo: %v", err)
562 }
563
564 buildOpts := build.Options{
565 IndexDir: indexDir,
566 RepositoryDescription: zoekt.Repository{
567 Name: "repo",
568 },
569 }
570 buildOpts.SetDefaults()
571
572 opts := Options{
573 RepoDir: filepath.Join(dir + "/repo"),
574 BuildOptions: buildOpts,
575 BranchPrefix: "refs/heads",
576 Branches: []string{"branchdir/a", "branchdir/b"},
577 }
578 if _, err := IndexGitRepo(opts); err != nil {
579 t.Fatalf("IndexGitRepo: %v", err)
580 }
581
582 searcher, err := shards.NewDirectorySearcher(indexDir)
583 if err != nil {
584 t.Fatal("NewDirectorySearcher", err)
585 }
586 defer searcher.Close()
587
588 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
589 if err != nil {
590 t.Fatalf("List(): %v", err)
591 }
592
593 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
594 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
595 }
596}