fork of https://github.com/sourcegraph/zoekt
indexserver: run against local repositories (#127)
This is intended to make manual testing of zoekt-sourcegraph-indexserver
more convenient. It does this by avoiding the need to run the
sourcegraph developer environment, and instead will treat local
repositories as those Sourcegraph would tell Zoekt to index.
Additionally I want to extend it further to make it easy to override
attributes Sourcegraph would return like private/archived/etc.
Tested by
go install ./cmd/zoekt-git-index ./cmd/zoekt-archive-index
go run ./cmd/zoekt-sourcegraph-indexserver \
-index ./index \
-sourcegraph_url ~/src
I then could run zoekt-webserver against the local ./index directory.
author
Keegan Carruthers-Smith
committer
GitHub
date
(Aug 12, 2021, 10:03 AM +0200)
commit
c0312df6
c0312df69e25cbc9dd30adc69e54a53b294120eb
parent
118d8005
118d800528e75d223662f6fed14974753d200c8e
+121
-22
+1
-1
cmd/zoekt-sourcegraph-indexserver/index_test.go
+1
-1
cmd/zoekt-sourcegraph-indexserver/index_test.go
+21
-12
cmd/zoekt-sourcegraph-indexserver/main.go
+21
-12
cmd/zoekt-sourcegraph-indexserver/main.go
···
106
106
// Server is the main functionality of zoekt-sourcegraph-indexserver. It
107
107
// exists to conveniently use all the options passed in via func main.
108
108
type Server struct {
109
-
Sourcegraph *Sourcegraph
109
+
Sourcegraph Sourcegraph
110
110
111
111
// IndexDir is the index directory to use.
112
112
IndexDir string
···
618
618
defaultIndexDir = build.DefaultDir
619
619
}
620
620
621
-
root := flag.String("sourcegraph_url", os.Getenv("SRC_FRONTEND_INTERNAL"), "http://sourcegraph-frontend-internal or http://localhost:3090")
621
+
root := flag.String("sourcegraph_url", os.Getenv("SRC_FRONTEND_INTERNAL"), "http://sourcegraph-frontend-internal or http://localhost:3090. If a path to a directory, we fake the Sourcegraph API and index all repos rooted under path.")
622
622
interval := flag.Duration("interval", time.Minute, "sync with sourcegraph this often")
623
623
index := flag.String("index", defaultIndexDir, "set index directory to use")
624
624
listen := flag.String("listen", ":6072", "listen on this address.")
···
672
672
debug = log.New(os.Stderr, "", log.LstdFlags)
673
673
}
674
674
675
-
client := retryablehttp.NewClient()
676
-
client.Logger = debug
675
+
var sg Sourcegraph
676
+
if rootURL.IsAbs() {
677
+
client := retryablehttp.NewClient()
678
+
client.Logger = debug
679
+
sg = &sourcegraphClient{
680
+
Root: rootURL,
681
+
Client: client,
682
+
Hostname: *hostname,
683
+
}
684
+
} else {
685
+
sg = sourcegraphFake{
686
+
RootDir: rootURL.String(),
687
+
Log: log.New(os.Stderr, "sourcegraph: ", log.LstdFlags),
688
+
}
689
+
}
677
690
678
691
cpuCount := int(math.Round(float64(runtime.GOMAXPROCS(0)) * (*cpuFraction)))
679
692
if cpuCount < 1 {
680
693
cpuCount = 1
681
694
}
682
695
s := &Server{
683
-
Sourcegraph: &Sourcegraph{
684
-
Root: rootURL,
685
-
Client: client,
686
-
Hostname: *hostname,
687
-
},
688
-
IndexDir: *index,
689
-
Interval: *interval,
690
-
CPUCount: cpuCount,
696
+
Sourcegraph: sg,
697
+
IndexDir: *index,
698
+
Interval: *interval,
699
+
CPUCount: cpuCount,
691
700
}
692
701
693
702
if *debugList {
+3
-3
cmd/zoekt-sourcegraph-indexserver/main_test.go
+3
-3
cmd/zoekt-sourcegraph-indexserver/main_test.go
···
22
22
}
23
23
24
24
s := &Server{
25
-
Sourcegraph: &Sourcegraph{
25
+
Sourcegraph: &sourcegraphClient{
26
26
Root: root,
27
27
},
28
28
IndexDir: "/testdata/index",
···
67
67
t.Fatal(err)
68
68
}
69
69
70
-
s := &Sourcegraph{
70
+
s := &sourcegraphClient{
71
71
Root: u,
72
72
Hostname: "test-indexed-search-1",
73
73
Client: retryablehttp.NewClient(),
···
125
125
// We expect waitForFrontend to just work now
126
126
done := make(chan struct{})
127
127
go func() {
128
-
(&Sourcegraph{Root: root}).WaitForFrontend()
128
+
(&sourcegraphClient{Root: root}).WaitForFrontend()
129
129
close(done)
130
130
}()
131
131
+96
-6
cmd/zoekt-sourcegraph-indexserver/sg.go
+96
-6
cmd/zoekt-sourcegraph-indexserver/sg.go
···
5
5
"context"
6
6
"encoding/json"
7
7
"fmt"
8
+
"hash/crc32"
8
9
"io"
9
10
"io/ioutil"
10
11
"log"
11
12
"net/http"
12
13
"net/url"
14
+
"os"
15
+
"os/exec"
13
16
"path"
17
+
"path/filepath"
14
18
"time"
15
19
20
+
"github.com/google/zoekt"
16
21
retryablehttp "github.com/hashicorp/go-retryablehttp"
17
22
)
18
23
19
-
// Sourcegraph contains methods which interact with the Sourcegraph API.
20
-
type Sourcegraph struct {
24
+
type Sourcegraph interface {
25
+
GetIndexOptions(repos ...string) ([]indexOptionsItem, error)
26
+
GetCloneURL(name string) string
27
+
WaitForFrontend()
28
+
ListRepos(ctx context.Context, indexed []string) ([]string, error)
29
+
}
30
+
31
+
// sourcegraphClient contains methods which interact with the sourcegraph API.
32
+
type sourcegraphClient struct {
21
33
// Root is the base URL for the Sourcegraph instance to index. Normally
22
34
// http://sourcegraph-frontend-internal or http://localhost:3090.
23
35
Root *url.URL
···
36
48
Error string
37
49
}
38
50
39
-
func (s *Sourcegraph) GetIndexOptions(repos ...string) ([]indexOptionsItem, error) {
51
+
func (s *sourcegraphClient) GetIndexOptions(repos ...string) ([]indexOptionsItem, error) {
40
52
u := s.Root.ResolveReference(&url.URL{
41
53
Path: "/.internal/search/configuration",
42
54
})
···
71
83
return opts, nil
72
84
}
73
85
74
-
func (s *Sourcegraph) GetCloneURL(name string) string {
86
+
func (s *sourcegraphClient) GetCloneURL(name string) string {
75
87
return s.Root.ResolveReference(&url.URL{Path: path.Join("/.internal/git", name)}).String()
76
88
}
77
89
78
-
func (s *Sourcegraph) WaitForFrontend() {
90
+
func (s *sourcegraphClient) WaitForFrontend() {
79
91
warned := false
80
92
lastWarn := time.Now()
81
93
for {
···
98
110
}
99
111
}
100
112
101
-
func (s *Sourcegraph) ListRepos(ctx context.Context, indexed []string) ([]string, error) {
113
+
func (s *sourcegraphClient) ListRepos(ctx context.Context, indexed []string) ([]string, error) {
102
114
body, err := json.Marshal(&struct {
103
115
Hostname string
104
116
Indexed []string
···
160
172
}
161
173
return nil
162
174
}
175
+
176
+
type sourcegraphFake struct {
177
+
RootDir string
178
+
Log *log.Logger
179
+
}
180
+
181
+
func (sf sourcegraphFake) GetIndexOptions(repos ...string) ([]indexOptionsItem, error) {
182
+
var items []indexOptionsItem
183
+
for _, name := range repos {
184
+
opts, err := sf.getIndexOptions(name)
185
+
if err != nil {
186
+
items = append(items, indexOptionsItem{Error: err.Error()})
187
+
} else {
188
+
items = append(items, indexOptionsItem{IndexOptions: opts})
189
+
}
190
+
}
191
+
return items, nil
192
+
}
193
+
194
+
func (sf sourcegraphFake) getIndexOptions(name string) (IndexOptions, error) {
195
+
dir := filepath.Join(sf.RootDir, filepath.FromSlash(name))
196
+
197
+
opts := IndexOptions{
198
+
// magic at the end is to ensure we get a positive number when casting.
199
+
RepoID: int32(crc32.ChecksumIEEE([]byte(name))%(1<<31-1) + 1),
200
+
Symbols: true,
201
+
}
202
+
203
+
cmd := exec.Command("git", "rev-parse", "HEAD")
204
+
cmd.Dir = dir
205
+
if b, err := cmd.Output(); err != nil {
206
+
return opts, err
207
+
} else {
208
+
head := string(bytes.TrimSpace(b))
209
+
opts.Branches = []zoekt.RepositoryBranch{{
210
+
Name: "HEAD",
211
+
Version: head,
212
+
}}
213
+
}
214
+
215
+
return opts, nil
216
+
}
217
+
218
+
func (sf sourcegraphFake) GetCloneURL(name string) string {
219
+
return filepath.Join(sf.RootDir, filepath.FromSlash(name))
220
+
}
221
+
func (sf sourcegraphFake) WaitForFrontend() {}
222
+
223
+
func (sf sourcegraphFake) ListRepos(ctx context.Context, indexed []string) ([]string, error) {
224
+
var repos []string
225
+
err := filepath.Walk(sf.RootDir, func(path string, fi os.FileInfo, fileErr error) error {
226
+
if fileErr != nil {
227
+
sf.Log.Printf("WARN: ignoring error searching %s: %v", path, fileErr)
228
+
return nil
229
+
}
230
+
if !fi.IsDir() {
231
+
return nil
232
+
}
233
+
234
+
gitdir := filepath.Join(path, ".git")
235
+
if fi, err := os.Stat(gitdir); err != nil || !fi.IsDir() {
236
+
return nil
237
+
}
238
+
239
+
subpath, err := filepath.Rel(sf.RootDir, path)
240
+
if err != nil {
241
+
// According to WalkFunc docs, path is always filepath.Join(root,
242
+
// subpath). So Rel should always work.
243
+
return fmt.Errorf("filepath.Walk returned %s which is not relative to %s: %w", path, sf.RootDir, err)
244
+
}
245
+
246
+
name := filepath.ToSlash(subpath)
247
+
repos = append(repos, name)
248
+
249
+
return filepath.SkipDir
250
+
})
251
+
return repos, err
252
+
}