fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package gitindex
16
17import (
18 "bytes"
19 "context"
20 "fmt"
21 "net/url"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "reflect"
26 "sort"
27 "testing"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/grafana/regexp"
32
33 "github.com/sourcegraph/zoekt"
34 "github.com/sourcegraph/zoekt/build"
35 "github.com/sourcegraph/zoekt/query"
36 "github.com/sourcegraph/zoekt/shards"
37)
38
39func createSubmoduleRepo(dir string) error {
40 if err := os.MkdirAll(dir, 0o755); err != nil {
41 return err
42 }
43 script := `mkdir adir bdir
44cd adir
45git init -b master
46mkdir subdir
47echo acont > afile
48echo sub-cont > subdir/sub-file
49git add afile subdir/sub-file
50git config user.email "you@example.com"
51git config user.name "Your Name"
52git commit -am amsg
53
54cd ..
55cd bdir
56git init -b master
57echo bcont > bfile
58ln -s bfile bsymlink
59git add bfile bsymlink
60git config user.email "you@example.com"
61git config user.name "Your Name"
62git commit -am bmsg
63
64cd ../adir
65git submodule add --name bname -- ../bdir bname
66git commit -am bmodmsg
67cat .gitmodules
68cd ..
69mkdir gerrit.googlesource.com
70git clone --bare adir gerrit.googlesource.com/adir.git
71git clone --bare bdir gerrit.googlesource.com/bdir.git
72
73mkdir gerrit.googlesource.com/bogus.git
74mkdir gerrit.googlesource.com/sub
75git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
76
77mkdir -p gerrit.googlesource.com/team/scope/
78cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
79cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
80
81cat << EOF > gerrit.googlesource.com/adir.git/config
82[core]
83 repositoryformatversion = 0
84 filemode = true
85 bare = true
86[remote "origin"]
87 url = http://gerrit.googlesource.com/adir
88[branch "master"]
89 remote = origin
90 merge = refs/heads/master
91EOF
92`
93 cmd := exec.Command("/bin/sh", "-euxc", script)
94 cmd.Dir = dir
95 if out, err := cmd.CombinedOutput(); err != nil {
96 return fmt.Errorf("execution error: %v, output %s", err, out)
97 }
98 return nil
99}
100
101func TestFindGitRepos(t *testing.T) {
102 dir := t.TempDir()
103
104 if err := createSubmoduleRepo(dir); err != nil {
105 t.Error("createSubmoduleRepo", err)
106 }
107 repos, err := FindGitRepos(dir)
108 if err != nil {
109 t.Error("FindGitRepos", err)
110 }
111
112 got := map[string]bool{}
113 for _, r := range repos {
114 p, err := filepath.Rel(dir, r)
115 if err != nil {
116 t.Fatalf("Relative: %v", err)
117 }
118
119 got[p] = true
120 }
121
122 want := map[string]bool{
123 "adir/.git": true,
124 "bdir/.git": true,
125 "gerrit.googlesource.com/adir.git": true,
126 "gerrit.googlesource.com/bdir.git": true,
127 "gerrit.googlesource.com/sub/bdir.git": true,
128 "gerrit.googlesource.com/team/scope/repoa.git": true,
129 "gerrit.googlesource.com/team/scope/repob.git": true,
130 }
131 if !reflect.DeepEqual(got, want) {
132 t.Errorf("got %v want %v", got, want)
133 }
134}
135
136func TestTreeToFiles(t *testing.T) {
137 dir := t.TempDir()
138
139 if err := createSubmoduleRepo(dir); err != nil {
140 t.Fatalf("TempDir: %v", err)
141 }
142
143 cache := NewRepoCache(dir)
144
145 aURL, _ := url.Parse("http://gerrit.googlesource.com/adir")
146 repo, err := cache.Open(aURL)
147 if err != nil {
148 t.Fatalf("Open: %v", err)
149 }
150
151 headRef, err := repo.Head()
152 if err != nil {
153 t.Fatalf("HEAD tree: %v", err)
154 }
155 commit, err := repo.CommitObject(headRef.Hash())
156 if err != nil {
157 t.Fatalf("commit obj HEAD: %v", err)
158 }
159
160 tree, err := repo.TreeObject(commit.TreeHash)
161 if err != nil {
162 t.Fatalf("AsTree: %v", err)
163 }
164
165 files, versions, err := TreeToFiles(repo, tree, aURL.String(), cache)
166 if err != nil {
167 t.Fatalf("TreeToFiles: %v", err)
168 }
169
170 bnameHash := versions["bname"]
171 if entry, err := tree.FindEntry("bname"); err != nil {
172 t.Fatalf("FindEntry %v", err)
173 } else if !bytes.Equal(bnameHash[:], entry.Hash[:]) {
174 t.Fatalf("got 'bname' versions %v, want %v", bnameHash, entry.Hash)
175 }
176
177 var paths []string
178 for k := range files {
179 paths = append(paths, k.FullPath())
180 }
181 sort.Strings(paths)
182
183 want := []string{".gitmodules", "afile", "bname/bfile", "bname/bsymlink", "subdir/sub-file"}
184 if !reflect.DeepEqual(paths, want) {
185 t.Errorf("got %v, want %v", paths, want)
186 }
187}
188
189func TestSubmoduleIndex(t *testing.T) {
190 dir := t.TempDir()
191
192 if err := createSubmoduleRepo(dir); err != nil {
193 t.Fatalf("createSubmoduleRepo: %v", err)
194 }
195
196 indexDir := t.TempDir()
197
198 buildOpts := build.Options{
199 IndexDir: indexDir,
200 }
201 opts := Options{
202 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
203 BuildOptions: buildOpts,
204 BranchPrefix: "refs/heads/",
205 Branches: []string{"master"},
206 Submodules: true,
207 Incremental: true,
208 RepoCacheDir: dir,
209 }
210 if err := IndexGitRepo(opts); err != nil {
211 t.Fatalf("IndexGitRepo: %v", err)
212 }
213
214 searcher, err := shards.NewDirectorySearcher(indexDir)
215 if err != nil {
216 t.Fatal("NewDirectorySearcher", err)
217 }
218 defer searcher.Close()
219
220 results, err := searcher.Search(context.Background(),
221 &query.Substring{Pattern: "bcont"},
222 &zoekt.SearchOptions{})
223 if err != nil {
224 t.Fatal("Search", err)
225 }
226
227 if len(results.Files) != 1 {
228 t.Fatalf("got search result %v, want 1 file", results.Files)
229 }
230
231 file := results.Files[0]
232 if got, want := file.SubRepositoryName, "gerrit.googlesource.com/bdir"; got != want {
233 t.Errorf("got subrepo name %q, want %q", got, want)
234 }
235 if got, want := file.SubRepositoryPath, "bname"; got != want {
236 t.Errorf("got subrepo path %q, want %q", got, want)
237 }
238
239 subVersion := file.Version
240 if len(subVersion) != 40 {
241 t.Fatalf("got %q, want hex sha1", subVersion)
242 }
243
244 if results, err := searcher.Search(context.Background(), &query.Substring{Pattern: "acont"}, &zoekt.SearchOptions{}); err != nil {
245 t.Fatalf("Search('acont'): %v", err)
246 } else if len(results.Files) != 1 {
247 t.Errorf("got %v, want 1 result", results.Files)
248 } else if f := results.Files[0]; f.Version == subVersion {
249 t.Errorf("version in super repo matched version is subrepo.")
250 }
251}
252
253func createSymlinkRepo(dir string) error {
254 if err := os.MkdirAll(dir, 0o755); err != nil {
255 return err
256 }
257 script := `mkdir adir bdir
258git init
259git config user.email "you@example.com"
260git config user.name "Your Name"
261
262echo acont > adir/afile
263git add adir/afile
264
265echo bcont > bdir/bfile
266git add bdir/bfile
267
268ln -s ./adir/afile asymlink
269git add asymlink
270
271git commit -am amsg
272
273cat << EOF > .git/config
274[core]
275 repositoryformatversion = 0
276 filemode = true
277 bare = true
278[remote "origin"]
279 url = http://codehost.com/arepo
280[branch "master"]
281 remote = origin
282 merge = refs/heads/master
283EOF
284`
285 cmd := exec.Command("/bin/sh", "-euxc", script)
286 cmd.Dir = dir
287 if out, err := cmd.CombinedOutput(); err != nil {
288 return fmt.Errorf("execution error: %v, output %s", err, out)
289 }
290 return nil
291}
292
293func TestSearchSymlinkByContent(t *testing.T) {
294 dir := t.TempDir()
295
296 if err := createSymlinkRepo(dir); err != nil {
297 t.Fatalf("createSubmoduleRepo: %v", err)
298 }
299
300 indexDir := t.TempDir()
301
302 buildOpts := build.Options{
303 IndexDir: indexDir,
304 }
305 opts := Options{
306 RepoDir: filepath.Join(dir),
307 BuildOptions: buildOpts,
308 BranchPrefix: "refs/heads/",
309 Branches: []string{"master"},
310 Submodules: true,
311 Incremental: true,
312 RepoCacheDir: dir,
313 }
314 if err := IndexGitRepo(opts); err != nil {
315 t.Fatalf("IndexGitRepo: %v", err)
316 }
317
318 searcher, err := shards.NewDirectorySearcher(indexDir)
319 if err != nil {
320 t.Fatal("NewDirectorySearcher", err)
321 }
322 defer searcher.Close()
323
324 // The content of the symlink and the file path the symlink points to both
325 // contain the string "afile". Hence we expect 1 path match and 1 content match.
326 results, err := searcher.Search(context.Background(),
327 &query.Substring{Pattern: "afile"},
328 &zoekt.SearchOptions{})
329 if err != nil {
330 t.Fatal("Search", err)
331 }
332
333 if len(results.Files) != 2 {
334 t.Fatalf("got search result %v, want 2 files", results.Files)
335 }
336
337 got := make([]string, 0, 2)
338 for _, file := range results.Files {
339 got = append(got, file.FileName)
340 }
341 sort.Strings(got)
342
343 want := []string{"adir/afile", "asymlink"}
344
345 if d := cmp.Diff(want, got); d != "" {
346 t.Fatalf("-want, +got %s\n", d)
347 }
348}
349
350func TestAllowMissingBranch(t *testing.T) {
351 dir := t.TempDir()
352
353 if err := createSubmoduleRepo(dir); err != nil {
354 t.Fatalf("createSubmoduleRepo: %v", err)
355 }
356
357 indexDir := t.TempDir()
358
359 buildOpts := build.Options{
360 IndexDir: indexDir,
361 }
362
363 opts := Options{
364 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
365 BuildOptions: buildOpts,
366 BranchPrefix: "refs/heads/",
367 Branches: []string{"master", "nonexist"},
368 Submodules: true,
369 Incremental: true,
370 RepoCacheDir: dir,
371 }
372 if err := IndexGitRepo(opts); err == nil {
373 t.Fatalf("IndexGitRepo(nonexist) succeeded")
374 }
375 opts.AllowMissingBranch = true
376 if err := IndexGitRepo(opts); err != nil {
377 t.Fatalf("IndexGitRepo(nonexist, allow): %v", err)
378 }
379}
380
381func createMultibranchRepo(dir string) error {
382 if err := os.MkdirAll(dir, 0o755); err != nil {
383 return err
384 }
385 script := `mkdir repo
386cd repo
387git init -b master
388mkdir subdir
389echo acont > afile
390echo sub-cont > subdir/sub-file
391git add afile subdir/sub-file
392git config user.email "you@example.com"
393git config user.name "Your Name"
394GIT_COMMITTER_DATE="Mon 5 Oct 2021 11:00:00 +0000" git commit -am amsg
395
396git branch branchdir/a
397
398echo acont >> afile
399git add afile subdir/sub-file
400GIT_COMMITTER_DATE="Tue 6 Oct 2021 12:00:00 +0000" git commit -am amsg
401
402git branch branchdir/b
403
404git branch c
405
406git update-ref refs/meta/config HEAD
407`
408 cmd := exec.Command("/bin/sh", "-euxc", script)
409 cmd.Dir = dir
410 if out, err := cmd.CombinedOutput(); err != nil {
411 return fmt.Errorf("execution error: %v, output %s", err, out)
412 }
413 return nil
414}
415
416func TestBranchWildcard(t *testing.T) {
417 dir := t.TempDir()
418
419 if err := createMultibranchRepo(dir); err != nil {
420 t.Fatalf("createMultibranchRepo: %v", err)
421 }
422
423 indexDir := t.TempDir()
424
425 buildOpts := build.Options{
426 IndexDir: indexDir,
427 RepositoryDescription: zoekt.Repository{
428 Name: "repo",
429 },
430 }
431 buildOpts.SetDefaults()
432
433 opts := Options{
434 RepoDir: filepath.Join(dir + "/repo"),
435 BuildOptions: buildOpts,
436 BranchPrefix: "refs/heads",
437 Branches: []string{"branchdir/*"},
438 Submodules: true,
439 Incremental: true,
440 }
441 if err := IndexGitRepo(opts); err != nil {
442 t.Fatalf("IndexGitRepo: %v", err)
443 }
444
445 searcher, err := shards.NewDirectorySearcher(indexDir)
446 if err != nil {
447 t.Fatal("NewDirectorySearcher", err)
448 }
449 defer searcher.Close()
450
451 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
452 t.Fatalf("List(): %v", err)
453 } else if len(rlist.Repos) != 1 {
454 t.Errorf("got %v, want 1 result", rlist.Repos)
455 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 2 {
456 t.Errorf("got branches %v, want 2", repo.Repository.Branches)
457 } else if repo := rlist.Repos[0]; repo.Stats.Documents != 3 {
458 t.Errorf("got document count %d, want 3", repo.Stats.Documents)
459 }
460}
461
462func TestSkipSubmodules(t *testing.T) {
463 dir := t.TempDir()
464
465 if err := createSubmoduleRepo(dir); err != nil {
466 t.Fatalf("createMultibranchRepo: %v", err)
467 }
468
469 indexDir := t.TempDir()
470
471 buildOpts := build.Options{
472 IndexDir: indexDir,
473 RepositoryDescription: zoekt.Repository{
474 Name: "gerrit.googlesource.com/adir",
475 },
476 }
477 if err := os.Rename(dir+"/gerrit.googlesource.com/bdir.git",
478 dir+"/gerrit.googlesource.com/notexist.git"); err != nil {
479 t.Fatalf("Rename: %v", err)
480 }
481
482 opts := Options{
483 RepoDir: filepath.Join(dir, "gerrit.googlesource.com", "adir.git"),
484 BuildOptions: buildOpts,
485 BranchPrefix: "refs/heads",
486 Branches: []string{"master"},
487 Submodules: false,
488 }
489 if err := IndexGitRepo(opts); err != nil {
490 t.Fatalf("IndexGitRepo: %v", err)
491 }
492}
493
494func TestFullAndShortRefNames(t *testing.T) {
495 dir := t.TempDir()
496
497 if err := createMultibranchRepo(dir); err != nil {
498 t.Fatalf("createMultibranchRepo: %v", err)
499 }
500
501 indexDir := t.TempDir()
502
503 buildOpts := build.Options{
504 IndexDir: indexDir,
505 RepositoryDescription: zoekt.Repository{
506 Name: "repo",
507 },
508 }
509 buildOpts.SetDefaults()
510
511 opts := Options{
512 RepoDir: filepath.Join(dir + "/repo"),
513 BuildOptions: buildOpts,
514 BranchPrefix: "refs/heads",
515 Branches: []string{"refs/heads/master", "branchdir/a", "refs/meta/config"},
516 Submodules: false,
517 Incremental: false,
518 AllowMissingBranch: false,
519 }
520 if err := IndexGitRepo(opts); err != nil {
521 t.Fatalf("IndexGitRepo: %v", err)
522 }
523
524 searcher, err := shards.NewDirectorySearcher(indexDir)
525 if err != nil {
526 t.Fatal("NewDirectorySearcher", err)
527 }
528 defer searcher.Close()
529
530 if rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
531 t.Fatalf("List(): %v", err)
532 } else if len(rlist.Repos) != 1 {
533 t.Errorf("got %v, want 1 result", rlist.Repos)
534 } else if repo := rlist.Repos[0]; len(repo.Repository.Branches) != 3 {
535 t.Errorf("got branches %v, want 3", repo.Repository.Branches)
536 }
537}
538
539func TestUniq(t *testing.T) {
540 in := []string{"a", "b", "b", "c", "c"}
541 want := []string{"a", "b", "c"}
542 got := uniq(in)
543 if !reflect.DeepEqual(got, want) {
544 t.Errorf("got %v, want %v", got, want)
545 }
546}
547
548func TestLatestCommit(t *testing.T) {
549 dir := t.TempDir()
550 indexDir := t.TempDir()
551
552 if err := createMultibranchRepo(dir); err != nil {
553 t.Fatalf("createMultibranchRepo: %v", err)
554 }
555
556 buildOpts := build.Options{
557 IndexDir: indexDir,
558 RepositoryDescription: zoekt.Repository{
559 Name: "repo",
560 },
561 }
562 buildOpts.SetDefaults()
563
564 opts := Options{
565 RepoDir: filepath.Join(dir + "/repo"),
566 BuildOptions: buildOpts,
567 BranchPrefix: "refs/heads",
568 Branches: []string{"branchdir/a", "branchdir/b"},
569 }
570 if err := IndexGitRepo(opts); err != nil {
571 t.Fatalf("IndexGitRepo: %v", err)
572 }
573
574 searcher, err := shards.NewDirectorySearcher(indexDir)
575 if err != nil {
576 t.Fatal("NewDirectorySearcher", err)
577 }
578 defer searcher.Close()
579
580 rlist, err := searcher.List(context.Background(), &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
581 if err != nil {
582 t.Fatalf("List(): %v", err)
583 }
584
585 if want := time.Date(2021, 10, 6, 12, 0, 0, 0, time.UTC); rlist.Repos[0].Repository.LatestCommitDate != want {
586 t.Fatalf("want %s, got %s", want, rlist.Repos[0].Repository.LatestCommitDate)
587 }
588}