fork of https://github.com/sourcegraph/zoekt
index: exclude directories from indexing (#56)
author
Stefan Hengl
committer
GitHub
date
(Sep 8, 2020, 10:00 AM +0200)
commit
f95cecff
f95cecffc54f6811bb255790e87094c7f404b7a2
parent
1373c398
1373c3986c6dee1a593fb61f07729dc7e9b0ff62
+297
-5
+118
gitindex/ignore_test.go
+118
gitindex/ignore_test.go
···
1
+
package gitindex
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"io/ioutil"
7
+
"os"
8
+
"os/exec"
9
+
"path/filepath"
10
+
"reflect"
11
+
"testing"
12
+
13
+
"github.com/google/zoekt/query"
14
+
15
+
"github.com/google/zoekt"
16
+
"github.com/google/zoekt/build"
17
+
"github.com/google/zoekt/shards"
18
+
)
19
+
20
+
func createSourcegraphignoreRepo(dir string) error {
21
+
if err := os.MkdirAll(dir, 0755); err != nil {
22
+
return err
23
+
}
24
+
script := `mkdir repo
25
+
cd repo
26
+
git init
27
+
mkdir subdir
28
+
echo acont > afile
29
+
echo sub-cont > subdir/sub-file
30
+
git add afile subdir/sub-file
31
+
git config user.email "you@example.com"
32
+
git config user.name "Your Name"
33
+
git commit -am amsg
34
+
35
+
git branch branchdir/abranch
36
+
37
+
mkdir .sourcegraph
38
+
echo subdir/ > .sourcegraph/ignore
39
+
git add .sourcegraph/ignore
40
+
git commit -am "ignore subdir/"
41
+
42
+
git update-ref refs/meta/config HEAD
43
+
`
44
+
cmd := exec.Command("/bin/sh", "-euxc", script)
45
+
cmd.Dir = dir
46
+
if out, err := cmd.CombinedOutput(); err != nil {
47
+
return fmt.Errorf("execution error: %v, output %s", err, out)
48
+
}
49
+
return nil
50
+
}
51
+
52
+
func TestIgnore(t *testing.T) {
53
+
dir, err := ioutil.TempDir("", "")
54
+
if err != nil {
55
+
t.Fatalf("TempDir: %v", err)
56
+
}
57
+
defer os.RemoveAll(dir)
58
+
59
+
if err := createSourcegraphignoreRepo(dir); err != nil {
60
+
t.Fatalf("createSourcegraphignoreRepo: %v", err)
61
+
}
62
+
63
+
indexDir, err := ioutil.TempDir("", "")
64
+
if err != nil {
65
+
t.Fatal(err)
66
+
}
67
+
defer os.RemoveAll(indexDir)
68
+
69
+
buildOpts := build.Options{
70
+
IndexDir: indexDir,
71
+
RepositoryDescription: zoekt.Repository{
72
+
Name: "repo",
73
+
},
74
+
}
75
+
buildOpts.SetDefaults()
76
+
77
+
opts := Options{
78
+
RepoDir: filepath.Join(dir + "/repo"),
79
+
BuildOptions: buildOpts,
80
+
BranchPrefix: "refs/heads",
81
+
Branches: []string{"master", "branchdir/*"},
82
+
Submodules: true,
83
+
Incremental: true,
84
+
}
85
+
if err := IndexGitRepo(opts); err != nil {
86
+
t.Fatalf("IndexGitRepo: %v", err)
87
+
}
88
+
89
+
searcher, err := shards.NewDirectorySearcher(indexDir)
90
+
if err != nil {
91
+
t.Fatal("NewDirectorySearcher", err)
92
+
}
93
+
defer searcher.Close()
94
+
95
+
res, err := searcher.Search(context.Background(), &query.Substring{}, &zoekt.SearchOptions{})
96
+
97
+
if len(res.Files) != 3 {
98
+
t.Fatalf("expected 3 file matches")
99
+
}
100
+
for _, match := range res.Files {
101
+
switch match.FileName {
102
+
case "afile":
103
+
if !reflect.DeepEqual(match.Branches, []string{"master", "branchdir/abranch"}) {
104
+
t.Fatalf("expected afile to be present on both branches")
105
+
}
106
+
case "subdir/sub-file":
107
+
if len(match.Branches) != 1 || match.Branches[0] != "branchdir/abranch" {
108
+
t.Fatalf("expected sub-file to be present only on branchdir/abranch")
109
+
}
110
+
case ".sourcegraph/ignore":
111
+
if len(match.Branches) != 1 || match.Branches[0] != "master" {
112
+
t.Fatalf("expected sourcegraphignore to be present only on master")
113
+
}
114
+
default:
115
+
t.Fatalf("match %+v not handled", match)
116
+
}
117
+
}
118
+
}
+24
gitindex/index.go
+24
gitindex/index.go
···
31
31
32
32
"github.com/google/zoekt"
33
33
"github.com/google/zoekt/build"
34
+
"github.com/google/zoekt/ignore"
34
35
35
36
"github.com/go-git/go-git/v5/config"
36
37
"github.com/go-git/go-git/v5/plumbing"
···
413
414
return err
414
415
}
415
416
417
+
ig, err := newIgnoreMatcher(tree)
418
+
if err != nil {
419
+
return err
420
+
}
421
+
416
422
files, subVersions, err := TreeToFiles(repo, tree, opts.BuildOptions.RepositoryDescription.URL, repoCache)
417
423
if err != nil {
418
424
return err
419
425
}
420
426
for k, v := range files {
427
+
if ig.Match(k.Path) {
428
+
continue
429
+
}
421
430
repos[k] = v
422
431
branchMap[k] = append(branchMap[k], b)
423
432
}
···
509
518
}
510
519
}
511
520
return builder.Finish()
521
+
}
522
+
523
+
func newIgnoreMatcher(tree *object.Tree) (*ignore.Matcher, error) {
524
+
ignoreFile, err := tree.File(ignore.IgnoreFile)
525
+
if err == object.ErrFileNotFound {
526
+
return &ignore.Matcher{}, nil
527
+
}
528
+
if err != nil {
529
+
return nil, err
530
+
}
531
+
content, err := ignoreFile.Contents()
532
+
if err != nil {
533
+
return nil, err
534
+
}
535
+
return ignore.ParseIgnoreFile(strings.NewReader(content))
512
536
}
513
537
514
538
func blobContents(blob *object.Blob) ([]byte, error) {
+1
-2
go.mod
+1
-2
go.mod
···
6
6
github.com/fsnotify/fsnotify v1.4.9
7
7
github.com/gfleury/go-bitbucket-v1 v0.0.0-20200312180434-e5170e3280fb
8
8
github.com/go-git/go-git/v5 v5.0.0
9
+
github.com/gobwas/glob v0.2.3
9
10
github.com/golang/protobuf v1.3.5 // indirect
10
11
github.com/google/go-cmp v0.4.0
11
12
github.com/google/go-github/v27 v27.0.6
···
16
17
github.com/keegancsmith/tmpfriend v0.0.0-20180423180255-86e88902a513
17
18
github.com/kylelemons/godebug v1.1.0
18
19
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
19
-
github.com/opentracing-contrib/go-stdlib v1.0.0
20
20
github.com/opentracing/opentracing-go v1.2.0
21
21
github.com/prometheus/client_golang v1.5.1
22
22
github.com/prometheus/procfs v0.0.10 // indirect
23
23
github.com/uber/jaeger-client-go v2.25.0+incompatible
24
24
github.com/uber/jaeger-lib v2.2.0+incompatible
25
25
github.com/xanzy/go-gitlab v0.28.0
26
-
go.uber.org/atomic v1.4.0
27
26
go.uber.org/automaxprocs v1.3.0
28
27
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 // indirect
29
28
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
+1
-3
go.sum
+1
-3
go.sum
···
94
94
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
95
95
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
96
96
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
97
+
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
97
98
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
98
99
github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
99
100
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
···
245
246
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
246
247
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
247
248
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
248
-
github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w=
249
-
github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
250
-
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
251
249
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
252
250
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
253
251
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+66
ignore/ignore.go
+66
ignore/ignore.go
···
1
+
// package ignore provides helpers to support ignore-files similar to .gitignore
2
+
package ignore
3
+
4
+
import (
5
+
"bufio"
6
+
"io"
7
+
"strings"
8
+
9
+
"github.com/gobwas/glob"
10
+
)
11
+
12
+
var (
13
+
lineComment = "#"
14
+
IgnoreFile = ".sourcegraph/ignore"
15
+
)
16
+
17
+
type Matcher struct {
18
+
ignoreList []glob.Glob
19
+
}
20
+
21
+
// ParseIgnoreFile parses an ignore-file according to the following rules
22
+
//
23
+
// - each line represents a glob-pattern relative to the root of the repository
24
+
// - for patterns without any glob-characters, a trailing ** is implicit
25
+
// - lines starting with # are ignored
26
+
// - empty lines are ignored
27
+
func ParseIgnoreFile(r io.Reader) (matcher *Matcher, error error) {
28
+
var patterns []glob.Glob
29
+
scanner := bufio.NewScanner(r)
30
+
for scanner.Scan() {
31
+
line := strings.TrimSpace(scanner.Text())
32
+
// ignore empty lines
33
+
if line == "" {
34
+
continue
35
+
}
36
+
// ignore comments
37
+
if strings.HasPrefix(line, lineComment) {
38
+
continue
39
+
}
40
+
line = strings.TrimPrefix(line, "/")
41
+
// implicit ** for patterns without glob-characters
42
+
if !strings.ContainsAny(line, ".][*?") {
43
+
line += "**"
44
+
}
45
+
// with separators = '/', * becomes path-aware
46
+
pattern, err := glob.Compile(line, '/')
47
+
if err != nil {
48
+
return nil, err
49
+
}
50
+
patterns = append(patterns, pattern)
51
+
}
52
+
return &Matcher{ignoreList: patterns}, scanner.Err()
53
+
}
54
+
55
+
// Match returns true if path has a prefix in common with any item in m.ignoreList
56
+
func (m *Matcher) Match(path string) bool {
57
+
if len(m.ignoreList) == 0 {
58
+
return false
59
+
}
60
+
for _, pattern := range m.ignoreList {
61
+
if pattern.Match(path) {
62
+
return true
63
+
}
64
+
}
65
+
return false
66
+
}
+87
ignore/ignore_test.go
+87
ignore/ignore_test.go
···
1
+
package ignore
2
+
3
+
import (
4
+
"bytes"
5
+
"reflect"
6
+
"strings"
7
+
"testing"
8
+
9
+
"github.com/gobwas/glob"
10
+
)
11
+
12
+
func TestParseIgnoreFile(t *testing.T) {
13
+
tests := []struct {
14
+
ignoreFile []byte
15
+
wantIgnoreList []glob.Glob
16
+
}{
17
+
{
18
+
ignoreFile: []byte("# ignore this \n \n foo\n bar/"),
19
+
wantIgnoreList: []glob.Glob{
20
+
glob.MustCompile("foo**", '/'),
21
+
glob.MustCompile("bar/**", '/')},
22
+
},
23
+
{
24
+
ignoreFile: []byte("/foo/bar \n /qux \n *.go\nfoo.go"),
25
+
wantIgnoreList: []glob.Glob{
26
+
glob.MustCompile("foo/bar**", '/'),
27
+
glob.MustCompile("qux**", '/'),
28
+
glob.MustCompile("*.go", '/'),
29
+
glob.MustCompile("foo.go", '/')},
30
+
},
31
+
}
32
+
33
+
for _, tt := range tests {
34
+
m, err := ParseIgnoreFile(bytes.NewReader(tt.ignoreFile))
35
+
if err != nil {
36
+
t.Error(err)
37
+
}
38
+
if !reflect.DeepEqual(m.ignoreList, tt.wantIgnoreList) {
39
+
t.Errorf("got %v, expected %v", m.ignoreList, tt.wantIgnoreList)
40
+
}
41
+
}
42
+
}
43
+
44
+
func TestIgnoreMatcher(t *testing.T) {
45
+
ignoreFile := `
46
+
dir1/
47
+
*.go
48
+
**/data.*
49
+
`
50
+
ig, err := ParseIgnoreFile(strings.NewReader(ignoreFile))
51
+
if err != nil {
52
+
t.Errorf("error in ignoreFile")
53
+
}
54
+
tests := []struct {
55
+
path string
56
+
wantMatch bool
57
+
}{
58
+
{
59
+
path: "dir1/readme.md",
60
+
wantMatch: true,
61
+
},
62
+
{
63
+
path: "dir1/dir2/readme.md",
64
+
wantMatch: true,
65
+
},
66
+
67
+
{
68
+
path: "foo.go",
69
+
wantMatch: true,
70
+
},
71
+
{
72
+
path: "dir2/foo.go",
73
+
wantMatch: false,
74
+
},
75
+
{
76
+
path: "dir3/data.xyz",
77
+
wantMatch: true,
78
+
},
79
+
}
80
+
for _, tt := range tests {
81
+
t.Run(tt.path, func(t *testing.T) {
82
+
if got := ig.Match(tt.path); got != tt.wantMatch {
83
+
t.Errorf("got %t, expected %t", got, tt.wantMatch)
84
+
}
85
+
})
86
+
}
87
+
}