fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32
33 "github.com/sourcegraph/zoekt"
34 "github.com/sourcegraph/zoekt/build"
35 "github.com/sourcegraph/zoekt/query"
36 "github.com/sourcegraph/zoekt/shards"
37)
38
39func createSubmoduleRepo(dir string) error {
40 if err := os.MkdirAll(dir, 0o755); err != nil {
41 return err
42 }
43 script := `
44# Fix fatal: transport 'file' not allowed
45export GIT_CONFIG_COUNT=1
46export GIT_CONFIG_KEY_0=protocol.file.allow
47export GIT_CONFIG_VALUE_0=always
48
49mkdir adir bdir
50cd adir
51git init -b master
52mkdir subdir
53echo acont > afile
54echo sub-cont > subdir/sub-file
55git add afile subdir/sub-file
56git config user.email "you@example.com"
57git config user.name "Your Name"
58git commit -am amsg
59
60cd ..
61cd bdir
62git init -b master
63echo bcont > bfile
64ln -s bfile bsymlink
65git add bfile bsymlink
66git config user.email "you@example.com"
67git config user.name "Your Name"
68git commit -am bmsg
69
70cd ../adir
71git submodule add --name bname -- ../bdir bname
72git commit -am bmodmsg
73cat .gitmodules
74cd ..
75mkdir gerrit.googlesource.com
76git clone --bare adir gerrit.googlesource.com/adir.git
77git clone --bare bdir gerrit.googlesource.com/bdir.git
78
79mkdir gerrit.googlesource.com/bogus.git
80mkdir gerrit.googlesource.com/sub
81git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
82
83mkdir -p gerrit.googlesource.com/team/scope/
84cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
85cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
86
87cat << EOF > gerrit.googlesource.com/adir.git/config
88[core]
89 repositoryformatversion = 0
90 filemode = true
91 bare = true
92[remote "origin"]
93 url = http://gerrit.googlesource.com/adir
94[branch "master"]
95 remote = origin
96 merge = refs/heads/master
97EOF
98`
99 cmd := exec.Command("/bin/sh", "-euxc", script)
100 cmd.Dir = dir
101 if out, err := cmd.CombinedOutput(); err != nil {
102 return fmt.Errorf("execution error: %v, output %s", err, out)
103 }
104 return nil
105}
106
107func TestFindGitRepos(t *testing.T) {
108 dir := t.TempDir()
109
110 if err := createSubmoduleRepo(dir); err != nil {
111 t.Error("createSubmoduleRepo", err)
112 }
113 repos, err := FindGitRepos(dir)
114 if err != nil {
115 t.Error("FindGitRepos", err)
116 }
117
118 got := map[string]bool{}
119 for _, r := range repos {
120 p, err := filepath.Rel(dir, r)
121 if err != nil {
122 t.Fatalf("Relative: %v", err)
123 }
124
125 got[p] = true
126 }
127
128 want := map[string]bool{
129 "adir/.git": true,
130 "bdir/.git": true,
131 "gerrit.googlesource.com/adir.git": true,
132 "gerrit.googlesource.com/bdir.git": true,
133 "gerrit.googlesource.com/sub/bdir.git": true,
134 "gerrit.googlesource.com/team/scope/repoa.git": true,
135 "gerrit.googlesource.com/team/scope/repob.git": true,
136 }
137 if !reflect.DeepEqual(got, want) {
138 t.Errorf("got %v want %v", got, want)
139 }
140}
141
142func TestTreeToFiles(t *testing.T) {
143 dir := t.TempDir()
144
145 if err := createSubmoduleRepo(dir); err != nil {
146 t.Fatalf("TempDir: %v", err)
147 }
148
149 cache := NewRepoCache(dir)
150
151 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
152 repo, err := cache.Open(aURL)
153 if err != nil {
154 t.Fatalf("Open: %v", err)
155 }
156
157 headRef, err := repo.Head()
158 if err != nil {
159 t.Fatalf("HEAD tree: %v", err)
160 }
161 commit, err := repo.CommitObject(headRef.Hash())
162 if err != nil {
163 t.Fatalf("commit obj HEAD: %v", err)
164 }
165
166 tree, err := repo.TreeObject(commit.TreeHash)
167 if err != nil {
168 t.Fatalf("AsTree: %v", err)
169 }
170
171 files, versions, err := TreeToFiles(repo, tree, aURL.String(), cache)
172 if err != nil {
173 t.Fatalf("TreeToFiles: %v", err)
174 }
175
176 bnameHash := versions["bname"]
177 if entry, err := tree.FindEntry("bname"); err != nil {
178 t.Fatalf("FindEntry %v", err)
179 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
180 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
181 }
182
183 var paths []string
184 for k := range files {
185 paths = append(paths, k.FullPath())
186 }
187 sort.Strings(paths)
188
189 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
190 if !reflect.DeepEqual(paths, want) {
191 t.Errorf("got %v, want %v", paths, want)
192 }
193}
194
195func TestSubmoduleIndex(t *testing.T) {
196 dir := t.TempDir()
197
198 if err := createSubmoduleRepo(dir); err != nil {
199 t.Fatalf("createSubmoduleRepo: %v", err)
200 }
201
202 indexDir := t.TempDir()
203
204 buildOpts := build.Options{
205 IndexDir: indexDir,
206 }
207 opts := Options{
208 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
209 BuildOptions: buildOpts,
210 BranchPrefix: "refs/heads/",
211 Branches: []string{"master"},
212 Submodules: true,
213 Incremental: true,
214 RepoCacheDir: dir,
215 }
216 if err := IndexGitRepo(opts); err != nil {
217 t.Fatalf("IndexGitRepo: %v", err)
218 }
219
220 searcher, err := shards.NewDirectorySearcher(indexDir)
221 if err != nil {
222 t.Fatal("NewDirectorySearcher", err)
223 }
224 defer searcher.Close()
225
226 results, err := searcher.Search(context.Background(),
227 &query.Substring{Pattern: "bcont"},
228 &zoekt.SearchOptions{})
229 if err != nil {
230 t.Fatal("Search", err)
231 }
232
233 if len(results.Files) != 1 {
234 t.Fatalf("got search result %v, want 1 file", results.Files)
235 }
236
237 file := results.Files[0]
238 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
239 t.Errorf("got subrepo name %q, want %q", got, want)
240 }
241 if got, want := file.SubRepositoryPath, "bname"; got != want {
242 t.Errorf("got subrepo path %q, want %q", got, want)
243 }
244
245 subVersion := file.Version
246 if len(subVersion) != 40 {
247 t.Fatalf("got %q, want hex sha1", subVersion)
248 }
249
250 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
251 t.Fatalf("Search('acont'): %v", err)
252 } else if len(results.Files) != 1 {
253 t.Errorf("got %v, want 1 result", results.Files)
254 } else if f := results.Files[0]; f.Version == subVersion {
255 t.Errorf("version in super repo matched version is subrepo.")
256 }
257}
258
259func createSymlinkRepo(dir string) error {
260 if err := os.MkdirAll(dir, 0o755); err != nil {
261 return err
262 }
263 script := `mkdir adir bdir
264git init -b master
265git config user.email "you@example.com"
266git config user.name "Your Name"
267
268echo acont > adir/afile
269git add adir/afile
270
271echo bcont > bdir/bfile
272git add bdir/bfile
273
274ln -s ./adir/afile asymlink
275git add asymlink
276
277git commit -am amsg
278
279cat << EOF > .git/config
280[core]
281 repositoryformatversion = 0
282 filemode = true
283 bare = true
284[remote "origin"]
285 url = http://codehost.com/arepo
286[branch "master"]
287 remote = origin
288 merge = refs/heads/master
289EOF
290`
291 cmd := exec.Command("/bin/sh", "-euxc", script)
292 cmd.Dir = dir
293 if out, err := cmd.CombinedOutput(); err != nil {
294 return fmt.Errorf("execution error: %v, output %s", err, out)
295 }
296 return nil
297}
298
299func TestSearchSymlinkByContent(t *testing.T) {
300 dir := t.TempDir()
301
302 if err := createSymlinkRepo(dir); err != nil {
303 t.Fatalf("createSubmoduleRepo: %v", err)
304 }
305
306 indexDir := t.TempDir()
307
308 buildOpts := build.Options{
309 IndexDir: indexDir,
310 }
311 opts := Options{
312 RepoDir: filepath.Join(dir),
313 BuildOptions: buildOpts,
314 BranchPrefix: "refs/heads/",
315 Branches: []string{"master"},
316 Submodules: true,
317 Incremental: true,
318 RepoCacheDir: dir,
319 }
320 if err := IndexGitRepo(opts); err != nil {
321 t.Fatalf("IndexGitRepo: %v", err)
322 }
323
324 searcher, err := shards.NewDirectorySearcher(indexDir)
325 if err != nil {
326 t.Fatal("NewDirectorySearcher", err)
327 }
328 defer searcher.Close()
329
330 // The content of the symlink and the file path the symlink points to both
331 // contain the string "afile". Hence we expect 1 path match and 1 content match.
332 results, err := searcher.Search(context.Background(),
333 &query.Substring{Pattern: "afile"},
334 &zoekt.SearchOptions{})
335 if err != nil {
336 t.Fatal("Search", err)
337 }
338
339 if len(results.Files) != 2 {
340 t.Fatalf("got search result %v, want 2 files", results.Files)
341 }
342
343 got := make([]string, 0, 2)
344 for _, file := range results.Files {
345 got = append(got, file.FileName)
346 }
347 sort.Strings(got)
348
349 want := []string{"adir/afile", "asymlink"}
350
351 if d := cmp.Diff(want, got); d != "" {
352 t.Fatalf("-want, +got %s\n", d)
353 }
354}
355
356func TestAllowMissingBranch(t *testing.T) {
357 dir := t.TempDir()
358
359 if err := createSubmoduleRepo(dir); err != nil {
360 t.Fatalf("createSubmoduleRepo: %v", err)
361 }
362
363 indexDir := t.TempDir()
364
365 buildOpts := build.Options{
366 IndexDir: indexDir,
367 }
368
369 opts := Options{
370 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
371 BuildOptions: buildOpts,
372 BranchPrefix: "refs/heads/",
373 Branches: []string{"master", "nonexist"},
374 Submodules: true,
375 Incremental: true,
376 RepoCacheDir: dir,
377 }
378 if err := IndexGitRepo(opts); err == nil {
379 t.Fatalf("IndexGitRepo(nonexist) succeeded")
380 }
381 opts.AllowMissingBranch = true
382 if err := IndexGitRepo(opts); err != nil {
383 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
384 }
385}
386
387func createMultibranchRepo(dir string) error {
388 if err := os.MkdirAll(dir, 0o755); err != nil {
389 return err
390 }
391 script := `mkdir repo
392cd repo
393git init -b master
394mkdir subdir
395echo acont > afile
396echo sub-cont > subdir/sub-file
397git add afile subdir/sub-file
398git config user.email "you@example.com"
399git config user.name "Your Name"
400GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
401
402git branch branchdir/a
403
404echo acont >> afile
405git add afile subdir/sub-file
406GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
407
408git branch branchdir/b
409
410git branch c
411
412git update-ref refs/meta/config HEAD
413`
414 cmd := exec.Command("/bin/sh", "-euxc", script)
415 cmd.Dir = dir
416 if out, err := cmd.CombinedOutput(); err != nil {
417 return fmt.Errorf("execution error: %v, output %s", err, out)
418 }
419 return nil
420}
421
422func TestBranchWildcard(t *testing.T) {
423 dir := t.TempDir()
424
425 if err := createMultibranchRepo(dir); err != nil {
426 t.Fatalf("createMultibranchRepo: %v", err)
427 }
428
429 indexDir := t.TempDir()
430
431 buildOpts := build.Options{
432 IndexDir: indexDir,
433 RepositoryDescription: zoekt.Repository{
434 Name: "repo",
435 },
436 }
437 buildOpts.SetDefaults()
438
439 opts := Options{
440 RepoDir: filepath.Join(dir + "/repo"),
441 BuildOptions: buildOpts,
442 BranchPrefix: "refs/heads",
443 Branches: []string{"branchdir/*"},
444 Submodules: true,
445 Incremental: true,
446 }
447 if err := IndexGitRepo(opts); err != nil {
448 t.Fatalf("IndexGitRepo: %v", err)
449 }
450
451 searcher, err := shards.NewDirectorySearcher(indexDir)
452 if err != nil {
453 t.Fatal("NewDirectorySearcher", err)
454 }
455 defer searcher.Close()
456
457 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
458 t.Fatalf("List(): %v", err)
459 } else if len(rlist.Repos) != 1 {
460 t.Errorf("got %v, want 1 result", rlist.Repos)
461 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
462 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
463 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
464 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
465 }
466}
467
468func TestSkipSubmodules(t *testing.T) {
469 dir := t.TempDir()
470
471 if err := createSubmoduleRepo(dir); err != nil {
472 t.Fatalf("createMultibranchRepo: %v", err)
473 }
474
475 indexDir := t.TempDir()
476
477 buildOpts := build.Options{
478 IndexDir: indexDir,
479 RepositoryDescription: zoekt.Repository{
480 Name: "gerrit.googlesource.com/adir",
481 },
482 }
483 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
484 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
485 t.Fatalf("Rename: %v", err)
486 }
487
488 opts := Options{
489 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
490 BuildOptions: buildOpts,
491 BranchPrefix: "refs/heads",
492 Branches: []string{"master"},
493 Submodules: false,
494 }
495 if err := IndexGitRepo(opts); err != nil {
496 t.Fatalf("IndexGitRepo: %v", err)
497 }
498}
499
500func TestFullAndShortRefNames(t *testing.T) {
501 dir := t.TempDir()
502
503 if err := createMultibranchRepo(dir); err != nil {
504 t.Fatalf("createMultibranchRepo: %v", err)
505 }
506
507 indexDir := t.TempDir()
508
509 buildOpts := build.Options{
510 IndexDir: indexDir,
511 RepositoryDescription: zoekt.Repository{
512 Name: "repo",
513 },
514 }
515 buildOpts.SetDefaults()
516
517 opts := Options{
518 RepoDir: filepath.Join(dir + "/repo"),
519 BuildOptions: buildOpts,
520 BranchPrefix: "refs/heads",
521 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
522 Submodules: false,
523 Incremental: false,
524 AllowMissingBranch: false,
525 }
526 if err := IndexGitRepo(opts); err != nil {
527 t.Fatalf("IndexGitRepo: %v", err)
528 }
529
530 searcher, err := shards.NewDirectorySearcher(indexDir)
531 if err != nil {
532 t.Fatal("NewDirectorySearcher", err)
533 }
534 defer searcher.Close()
535
536 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
537 t.Fatalf("List(): %v", err)
538 } else if len(rlist.Repos) != 1 {
539 t.Errorf("got %v, want 1 result", rlist.Repos)
540 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
541 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
542 }
543}
544
545func TestUniq(t *testing.T) {
546 in := []string{"a", "b", "b", "c", "c"}
547 want := []string{"a", "b", "c"}
548 got := uniq(in)
549 if !reflect.DeepEqual(got, want) {
550 t.Errorf("got %v, want %v", got, want)
551 }
552}
553
554func TestLatestCommit(t *testing.T) {
555 dir := t.TempDir()
556 indexDir := t.TempDir()
557
558 if err := createMultibranchRepo(dir); err != nil {
559 t.Fatalf("createMultibranchRepo: %v", err)
560 }
561
562 buildOpts := build.Options{
563 IndexDir: indexDir,
564 RepositoryDescription: zoekt.Repository{
565 Name: "repo",
566 },
567 }
568 buildOpts.SetDefaults()
569
570 opts := Options{
571 RepoDir: filepath.Join(dir + "/repo"),
572 BuildOptions: buildOpts,
573 BranchPrefix: "refs/heads",
574 Branches: []string{"branchdir/a", "branchdir/b"},
575 }
576 if err := IndexGitRepo(opts); err != nil {
577 t.Fatalf("IndexGitRepo: %v", err)
578 }
579
580 searcher, err := shards.NewDirectorySearcher(indexDir)
581 if err != nil {
582 t.Fatal("NewDirectorySearcher", err)
583 }
584 defer searcher.Close()
585
586 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
587 if err != nil {
588 t.Fatalf("List(): %v", err)
589 }
590
591 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
592 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
593 }
594}