fork of https://github.com/sourcegraph/zoekt
indexserver: integration test for sourcegraph (#1033)
author
Keegan Carruthers-Smith
committer
GitHub
date
(Mar 31, 2026, 2:44 PM +0200)
commit
817f9753
817f9753f8134e6bf100fb373e9b12bd2bf516c9
parent
13f554bf
13f554bffd45e8735d2da1524e3feaff8342db26
+401
+401
cmd/zoekt-sourcegraph-indexserver/index_integration_test.go
+401
cmd/zoekt-sourcegraph-indexserver/index_integration_test.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"net"
7
+
"os"
8
+
"os/exec"
9
+
"path/filepath"
10
+
"strings"
11
+
"testing"
12
+
"time"
13
+
14
+
"github.com/google/go-cmp/cmp"
15
+
"github.com/sourcegraph/log/logtest"
16
+
"github.com/stretchr/testify/require"
17
+
18
+
"github.com/sourcegraph/zoekt"
19
+
"github.com/sourcegraph/zoekt/gitindex"
20
+
"github.com/sourcegraph/zoekt/query"
21
+
"github.com/sourcegraph/zoekt/search"
22
+
)
23
+
24
+
func TestFetchRepoAndIndex_Integration(t *testing.T) {
25
+
requireGitDaemon(t)
26
+
27
+
for _, tc := range []struct {
28
+
name string
29
+
disableGoGitOptimization bool
30
+
}{
31
+
{name: "optimized repo open"},
32
+
{name: "legacy repo open", disableGoGitOptimization: true},
33
+
} {
34
+
t.Run(tc.name, func(t *testing.T) {
35
+
require := require.New(t)
36
+
37
+
ctx := context.Background()
38
+
fixture := newGitFetchFixture(t)
39
+
40
+
if tc.disableGoGitOptimization {
41
+
t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "true")
42
+
} else {
43
+
t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "false")
44
+
}
45
+
46
+
sg := &recordingSourcegraph{
47
+
opts: IndexOptions{
48
+
RepoID: 123,
49
+
Name: "test/repo",
50
+
CloneURL: fixture.cloneURL,
51
+
Symbols: false,
52
+
Branches: []zoekt.RepositoryBranch{
53
+
{Name: "HEAD", Version: fixture.mainCommit},
54
+
{Name: "dev", Version: fixture.devCommit},
55
+
},
56
+
TenantID: 1,
57
+
},
58
+
}
59
+
60
+
indexDir := t.TempDir()
61
+
server := &Server{
62
+
Sourcegraph: sg,
63
+
IndexDir: indexDir,
64
+
CPUCount: 1,
65
+
IndexConcurrency: 1,
66
+
}
67
+
68
+
result, err := sg.List(ctx, nil)
69
+
require.NoError(err)
70
+
71
+
var args *indexArgs
72
+
result.IterateIndexOptions(func(opts IndexOptions) {
73
+
args = server.indexArgs(opts)
74
+
})
75
+
require.NotNil(args)
76
+
77
+
gitDir := filepath.Join(t.TempDir(), "fetch.git")
78
+
c := gitIndexConfig{
79
+
runCmd: runIntegrationCommand,
80
+
findRepositoryMetadata: func(args *indexArgs) (*zoekt.Repository, *zoekt.IndexMetadata, bool, error) {
81
+
return args.BuildOptions().FindRepositoryMetadata()
82
+
},
83
+
timeout: time.Minute,
84
+
}
85
+
86
+
require.NoError(fetchRepo(ctx, gitDir, args, c, logtest.Scoped(t)))
87
+
assertPartialBareFetch(t, gitDir, fixture)
88
+
89
+
require.NoError(setZoektConfig(ctx, gitDir, args, c))
90
+
91
+
updated, err := gitindex.IndexGitRepo(gitIndexOptionsForTest(args, gitDir))
92
+
require.NoError(err)
93
+
require.True(updated)
94
+
95
+
repository, metadata, ok, err := args.BuildOptions().FindRepositoryMetadata()
96
+
require.NoError(err)
97
+
require.True(ok)
98
+
require.Equal(args.Name, repository.Name)
99
+
require.Equal(args.RepoID, repository.ID)
100
+
require.Equal(args.TenantID, repository.TenantID)
101
+
if diff := cmp.Diff(args.Branches, repository.Branches); diff != "" {
102
+
t.Fatalf("branches mismatch (-want +got):\n%s", diff)
103
+
}
104
+
require.Equal("123", repository.RawConfig["repoid"])
105
+
require.Equal("1", repository.RawConfig["tenantID"])
106
+
107
+
searcher, err := search.NewDirectorySearcher(indexDir)
108
+
require.NoError(err)
109
+
defer searcher.Close()
110
+
111
+
assertSearchContains(t, searcher, "smallneedle", "small.txt")
112
+
assertSearchContains(t, searcher, "devneedle", "dev.txt")
113
+
assertSearchEmpty(t, searcher, "largeneedle")
114
+
115
+
require.NoError(updateIndexStatusOnSourcegraph(c, args, sg))
116
+
require.Len(sg.updates, 1)
117
+
require.Len(sg.updates[0], 1)
118
+
require.Equal(args.RepoID, sg.updates[0][0].RepoID)
119
+
require.Equal(metadata.IndexTime.Unix(), sg.updates[0][0].IndexTimeUnix)
120
+
if diff := cmp.Diff(args.Branches, sg.updates[0][0].Branches); diff != "" {
121
+
t.Fatalf("status branches mismatch (-want +got):\n%s", diff)
122
+
}
123
+
})
124
+
}
125
+
}
126
+
127
+
func requireGitDaemon(t *testing.T) {
128
+
t.Helper()
129
+
130
+
cmd := exec.Command("git", "daemon", "-h")
131
+
cmd.Env = gitTestEnv()
132
+
output, err := cmd.CombinedOutput()
133
+
text := string(output)
134
+
135
+
if strings.Contains(text, "usage: git daemon") {
136
+
return
137
+
}
138
+
139
+
if strings.Contains(text, "git: 'daemon' is not a git command") {
140
+
t.Skipf("skipping integration test: git daemon is unavailable: %s", strings.TrimSpace(text))
141
+
}
142
+
143
+
if err == nil {
144
+
return
145
+
}
146
+
147
+
t.Fatalf("failed to probe git daemon availability: %v\n%s", err, text)
148
+
}
149
+
150
+
type recordingSourcegraph struct {
151
+
opts IndexOptions
152
+
updates [][]indexStatus
153
+
}
154
+
155
+
func (s *recordingSourcegraph) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) {
156
+
return &SourcegraphListResult{
157
+
IDs: []uint32{s.opts.RepoID},
158
+
IterateIndexOptions: func(yield func(IndexOptions)) {
159
+
yield(s.opts)
160
+
},
161
+
}, nil
162
+
}
163
+
164
+
func (s *recordingSourcegraph) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) {
165
+
onSuccess(s.opts)
166
+
}
167
+
168
+
func (s *recordingSourcegraph) UpdateIndexStatus(repositories []indexStatus) error {
169
+
cp := make([]indexStatus, len(repositories))
170
+
copy(cp, repositories)
171
+
s.updates = append(s.updates, cp)
172
+
return nil
173
+
}
174
+
175
+
type gitFetchFixture struct {
176
+
cloneURL string
177
+
mainCommit string
178
+
devCommit string
179
+
bigBlob string
180
+
daemon *gitDaemon
181
+
}
182
+
183
+
func newGitFetchFixture(t *testing.T) *gitFetchFixture {
184
+
t.Helper()
185
+
186
+
root := t.TempDir()
187
+
worktree := filepath.Join(root, "worktree")
188
+
serveRoot := filepath.Join(root, "serve")
189
+
remoteDir := filepath.Join(serveRoot, "repo")
190
+
191
+
require.NoError(t, os.MkdirAll(worktree, 0o755))
192
+
require.NoError(t, os.MkdirAll(serveRoot, 0o755))
193
+
194
+
runGit(t, root, "init", "-b", "main", worktree)
195
+
runGit(t, worktree, "config", "user.name", "Test User")
196
+
runGit(t, worktree, "config", "user.email", "test@example.com")
197
+
198
+
require.NoError(t, os.WriteFile(filepath.Join(worktree, "small.txt"), []byte("smallneedle\n"), 0o644))
199
+
large := strings.Repeat("x", MaxFileSize+1024)
200
+
require.NoError(t, os.WriteFile(filepath.Join(worktree, "big.bin"), []byte("largeneedle\n"+large), 0o644))
201
+
runGit(t, worktree, "add", "small.txt", "big.bin")
202
+
runGit(t, worktree, "commit", "-m", "main commit")
203
+
204
+
mainCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD"))
205
+
bigBlob := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD:big.bin"))
206
+
207
+
runGit(t, worktree, "checkout", "-b", "dev")
208
+
require.NoError(t, os.WriteFile(filepath.Join(worktree, "dev.txt"), []byte("devneedle\n"), 0o644))
209
+
runGit(t, worktree, "add", "dev.txt")
210
+
runGit(t, worktree, "commit", "-m", "dev commit")
211
+
212
+
devCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD"))
213
+
runGit(t, root, "clone", "--bare", worktree, remoteDir)
214
+
runGit(t, remoteDir, "config", "uploadpack.allowFilter", "true")
215
+
runGit(t, remoteDir, "config", "uploadpack.allowAnySHA1InWant", "true")
216
+
217
+
daemon := startGitDaemon(t, serveRoot)
218
+
219
+
return &gitFetchFixture{
220
+
cloneURL: fmt.Sprintf("git://127.0.0.1:%d/repo", daemon.port),
221
+
mainCommit: mainCommit,
222
+
devCommit: devCommit,
223
+
bigBlob: bigBlob,
224
+
daemon: daemon,
225
+
}
226
+
}
227
+
228
+
type gitDaemon struct {
229
+
cmd *exec.Cmd
230
+
logPath string
231
+
port int
232
+
}
233
+
234
+
func startGitDaemon(t *testing.T, serveRoot string) *gitDaemon {
235
+
t.Helper()
236
+
237
+
port := allocatePort(t)
238
+
logFile, err := os.CreateTemp(t.TempDir(), "git-daemon-*.log")
239
+
require.NoError(t, err)
240
+
logPath := logFile.Name()
241
+
cmd := exec.Command("git", "daemon",
242
+
"--verbose",
243
+
"--export-all",
244
+
"--reuseaddr",
245
+
"--base-path="+serveRoot,
246
+
"--listen=127.0.0.1",
247
+
fmt.Sprintf("--port=%d", port),
248
+
serveRoot,
249
+
)
250
+
cmd.Env = gitTestEnv()
251
+
cmd.Stdout = logFile
252
+
cmd.Stderr = logFile
253
+
254
+
require.NoError(t, cmd.Start())
255
+
require.NoError(t, logFile.Close())
256
+
waitForGitDaemon(t, port, logPath)
257
+
258
+
daemon := &gitDaemon{cmd: cmd, logPath: logPath, port: port}
259
+
t.Cleanup(func() {
260
+
if cmd.Process != nil {
261
+
_ = cmd.Process.Kill()
262
+
}
263
+
waitDone := make(chan struct{})
264
+
go func() {
265
+
_ = cmd.Wait()
266
+
close(waitDone)
267
+
}()
268
+
269
+
select {
270
+
case <-waitDone:
271
+
case <-time.After(5 * time.Second):
272
+
}
273
+
})
274
+
275
+
return daemon
276
+
}
277
+
278
+
func waitForGitDaemon(t *testing.T, port int, logPath string) {
279
+
t.Helper()
280
+
281
+
addr := fmt.Sprintf("127.0.0.1:%d", port)
282
+
deadline := time.Now().Add(5 * time.Second)
283
+
284
+
for time.Now().Before(deadline) {
285
+
conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
286
+
if err == nil {
287
+
_ = conn.Close()
288
+
return
289
+
}
290
+
291
+
time.Sleep(50 * time.Millisecond)
292
+
}
293
+
294
+
contents, err := os.ReadFile(logPath)
295
+
if err != nil {
296
+
t.Fatalf("git daemon did not start listening on %s within 5s (failed to read log: %v)", addr, err)
297
+
}
298
+
299
+
t.Fatalf("git daemon did not start listening on %s within 5s\n%s", addr, contents)
300
+
}
301
+
302
+
func allocatePort(t *testing.T) int {
303
+
t.Helper()
304
+
305
+
listener, err := net.Listen("tcp", "127.0.0.1:0")
306
+
require.NoError(t, err)
307
+
defer listener.Close()
308
+
309
+
return listener.Addr().(*net.TCPAddr).Port
310
+
}
311
+
312
+
func gitIndexOptionsForTest(args *indexArgs, repoDir string) gitindex.Options {
313
+
buildOptions := *args.BuildOptions()
314
+
buildOptions.RepositoryDescription.Branches = nil
315
+
316
+
branches := make([]string, 0, len(args.Branches))
317
+
for _, branch := range args.Branches {
318
+
branches = append(branches, branch.Name)
319
+
}
320
+
321
+
return gitindex.Options{
322
+
RepoDir: repoDir,
323
+
Submodules: false,
324
+
Incremental: args.Incremental,
325
+
BuildOptions: buildOptions,
326
+
BranchPrefix: "refs/heads/",
327
+
Branches: branches,
328
+
DeltaShardNumberFallbackThreshold: args.DeltaShardNumberFallbackThreshold,
329
+
}
330
+
}
331
+
332
+
func assertPartialBareFetch(t *testing.T, gitDir string, fixture *gitFetchFixture) {
333
+
t.Helper()
334
+
require := require.New(t)
335
+
336
+
require.Equal(fixture.mainCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "HEAD")))
337
+
require.Equal(fixture.devCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "refs/heads/dev")))
338
+
339
+
promisors, err := filepath.Glob(filepath.Join(gitDir, "objects", "pack", "*.promisor"))
340
+
require.NoError(err)
341
+
require.NotEmpty(promisors)
342
+
343
+
objects := runGitOutput(t, gitDir, "rev-list", "--objects", "--missing=print", "HEAD", "refs/heads/dev")
344
+
require.Contains(objects, fixture.mainCommit)
345
+
require.Contains(objects, fixture.devCommit)
346
+
require.Contains(objects, "?"+fixture.bigBlob)
347
+
}
348
+
349
+
func assertSearchContains(t *testing.T, searcher zoekt.Searcher, pattern string, wantFile string) {
350
+
t.Helper()
351
+
require := require.New(t)
352
+
353
+
result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{})
354
+
require.NoError(err)
355
+
require.Len(result.Files, 1)
356
+
require.Equal(wantFile, result.Files[0].FileName)
357
+
}
358
+
359
+
func assertSearchEmpty(t *testing.T, searcher zoekt.Searcher, pattern string) {
360
+
t.Helper()
361
+
require := require.New(t)
362
+
363
+
result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{})
364
+
require.NoError(err)
365
+
require.Empty(result.Files)
366
+
}
367
+
368
+
func runIntegrationCommand(cmd *exec.Cmd) error {
369
+
cmd.Env = gitTestEnv()
370
+
output, err := cmd.CombinedOutput()
371
+
if err != nil {
372
+
return fmt.Errorf("%s: %w\n%s", strings.Join(cmd.Args, " "), err, output)
373
+
}
374
+
return nil
375
+
}
376
+
377
+
func runGit(t *testing.T, dir string, args ...string) {
378
+
t.Helper()
379
+
_ = runGitOutput(t, dir, args...)
380
+
}
381
+
382
+
func runGitOutput(t *testing.T, dir string, args ...string) string {
383
+
t.Helper()
384
+
385
+
cmd := exec.Command("git", args...)
386
+
cmd.Dir = dir
387
+
cmd.Env = gitTestEnv()
388
+
output, err := cmd.CombinedOutput()
389
+
if err != nil {
390
+
t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output)
391
+
}
392
+
393
+
return string(output)
394
+
}
395
+
396
+
func gitTestEnv() []string {
397
+
return append(os.Environ(),
398
+
"GIT_CONFIG_GLOBAL=",
399
+
"GIT_CONFIG_SYSTEM=",
400
+
)
401
+
}