fork of https://github.com/sourcegraph/zoekt
1package main
2
3import (
4 "context"
5 "fmt"
6 "net"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13
14 "github.com/google/go-cmp/cmp"
15 "github.com/sourcegraph/log/logtest"
16 "github.com/stretchr/testify/require"
17
18 "github.com/sourcegraph/zoekt"
19 "github.com/sourcegraph/zoekt/gitindex"
20 "github.com/sourcegraph/zoekt/query"
21 "github.com/sourcegraph/zoekt/search"
22)
23
24func TestFetchRepoAndIndex_Integration(t *testing.T) {
25 requireGitDaemon(t)
26
27 for _, tc := range []struct {
28 name string
29 disableGoGitOptimization bool
30 }{
31 {name: "optimized repo open"},
32 {name: "legacy repo open", disableGoGitOptimization: true},
33 } {
34 t.Run(tc.name, func(t *testing.T) {
35 require := require.New(t)
36
37 ctx := context.Background()
38 fixture := newGitFetchFixture(t)
39
40 if tc.disableGoGitOptimization {
41 t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "true")
42 } else {
43 t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "false")
44 }
45
46 sg := &recordingSourcegraph{
47 opts: IndexOptions{
48 RepoID: 123,
49 Name: "test/repo",
50 CloneURL: fixture.cloneURL,
51 Symbols: false,
52 Branches: []zoekt.RepositoryBranch{
53 {Name: "HEAD", Version: fixture.mainCommit},
54 {Name: "dev", Version: fixture.devCommit},
55 },
56 TenantID: 1,
57 },
58 }
59
60 indexDir := t.TempDir()
61 server := &Server{
62 Sourcegraph: sg,
63 IndexDir: indexDir,
64 CPUCount: 1,
65 IndexConcurrency: 1,
66 }
67
68 result, err := sg.List(ctx, nil)
69 require.NoError(err)
70
71 var args *indexArgs
72 result.IterateIndexOptions(func(opts IndexOptions) {
73 args = server.indexArgs(opts)
74 })
75 require.NotNil(args)
76
77 gitDir := filepath.Join(t.TempDir(), "fetch.git")
78 c := gitIndexConfig{
79 runCmd: runIntegrationCommand,
80 findRepositoryMetadata: func(args *indexArgs) (*zoekt.Repository, *zoekt.IndexMetadata, bool, error) {
81 return args.BuildOptions().FindRepositoryMetadata()
82 },
83 timeout: time.Minute,
84 }
85
86 require.NoError(fetchRepo(ctx, gitDir, args, c, logtest.Scoped(t)))
87 assertPartialBareFetch(t, gitDir, fixture)
88
89 require.NoError(setZoektConfig(ctx, gitDir, args, c))
90
91 updated, err := gitindex.IndexGitRepo(gitIndexOptionsForTest(args, gitDir))
92 require.NoError(err)
93 require.True(updated)
94
95 repository, metadata, ok, err := args.BuildOptions().FindRepositoryMetadata()
96 require.NoError(err)
97 require.True(ok)
98 require.Equal(args.Name, repository.Name)
99 require.Equal(args.RepoID, repository.ID)
100 require.Equal(args.TenantID, repository.TenantID)
101 if diff := cmp.Diff(args.Branches, repository.Branches); diff != "" {
102 t.Fatalf("branches mismatch (-want +got):\n%s", diff)
103 }
104 require.Equal("123", repository.RawConfig["repoid"])
105 require.Equal("1", repository.RawConfig["tenantID"])
106
107 searcher, err := search.NewDirectorySearcher(indexDir)
108 require.NoError(err)
109 defer searcher.Close()
110
111 assertSearchContains(t, searcher, "smallneedle", "small.txt")
112 assertSearchContains(t, searcher, "devneedle", "dev.txt")
113 assertSearchEmpty(t, searcher, "largeneedle")
114
115 require.NoError(updateIndexStatusOnSourcegraph(c, args, sg))
116 require.Len(sg.updates, 1)
117 require.Len(sg.updates[0], 1)
118 require.Equal(args.RepoID, sg.updates[0][0].RepoID)
119 require.Equal(metadata.IndexTime.Unix(), sg.updates[0][0].IndexTimeUnix)
120 if diff := cmp.Diff(args.Branches, sg.updates[0][0].Branches); diff != "" {
121 t.Fatalf("status branches mismatch (-want +got):\n%s", diff)
122 }
123 })
124 }
125}
126
127func requireGitDaemon(t *testing.T) {
128 t.Helper()
129
130 cmd := exec.Command("git", "daemon", "-h")
131 cmd.Env = gitTestEnv()
132 output, err := cmd.CombinedOutput()
133 text := string(output)
134
135 if strings.Contains(text, "usage: git daemon") {
136 return
137 }
138
139 if strings.Contains(text, "git: 'daemon' is not a git command") {
140 t.Skipf("skipping integration test: git daemon is unavailable: %s", strings.TrimSpace(text))
141 }
142
143 if err == nil {
144 return
145 }
146
147 t.Fatalf("failed to probe git daemon availability: %v\n%s", err, text)
148}
149
150type recordingSourcegraph struct {
151 opts IndexOptions
152 updates [][]indexStatus
153}
154
155func (s *recordingSourcegraph) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) {
156 return &SourcegraphListResult{
157 IDs: []uint32{s.opts.RepoID},
158 IterateIndexOptions: func(yield func(IndexOptions)) {
159 yield(s.opts)
160 },
161 }, nil
162}
163
164func (s *recordingSourcegraph) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) {
165 onSuccess(s.opts)
166}
167
168func (s *recordingSourcegraph) UpdateIndexStatus(repositories []indexStatus) error {
169 cp := make([]indexStatus, len(repositories))
170 copy(cp, repositories)
171 s.updates = append(s.updates, cp)
172 return nil
173}
174
175type gitFetchFixture struct {
176 cloneURL string
177 mainCommit string
178 devCommit string
179 bigBlob string
180 daemon *gitDaemon
181}
182
183func newGitFetchFixture(t *testing.T) *gitFetchFixture {
184 t.Helper()
185
186 root := t.TempDir()
187 worktree := filepath.Join(root, "worktree")
188 serveRoot := filepath.Join(root, "serve")
189 remoteDir := filepath.Join(serveRoot, "repo")
190
191 require.NoError(t, os.MkdirAll(worktree, 0o755))
192 require.NoError(t, os.MkdirAll(serveRoot, 0o755))
193
194 runGit(t, root, "init", "-b", "main", worktree)
195 runGit(t, worktree, "config", "user.name", "Test User")
196 runGit(t, worktree, "config", "user.email", "test@example.com")
197
198 require.NoError(t, os.WriteFile(filepath.Join(worktree, "small.txt"), []byte("smallneedle\n"), 0o644))
199 large := strings.Repeat("x", MaxFileSize+1024)
200 require.NoError(t, os.WriteFile(filepath.Join(worktree, "big.bin"), []byte("largeneedle\n"+large), 0o644))
201 runGit(t, worktree, "add", "small.txt", "big.bin")
202 runGit(t, worktree, "commit", "-m", "main commit")
203
204 mainCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD"))
205 bigBlob := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD:big.bin"))
206
207 runGit(t, worktree, "checkout", "-b", "dev")
208 require.NoError(t, os.WriteFile(filepath.Join(worktree, "dev.txt"), []byte("devneedle\n"), 0o644))
209 runGit(t, worktree, "add", "dev.txt")
210 runGit(t, worktree, "commit", "-m", "dev commit")
211
212 devCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD"))
213 runGit(t, root, "clone", "--bare", worktree, remoteDir)
214 runGit(t, remoteDir, "config", "uploadpack.allowFilter", "true")
215 runGit(t, remoteDir, "config", "uploadpack.allowAnySHA1InWant", "true")
216
217 daemon := startGitDaemon(t, serveRoot)
218
219 return &gitFetchFixture{
220 cloneURL: fmt.Sprintf("git://127.0.0.1:%d/repo", daemon.port),
221 mainCommit: mainCommit,
222 devCommit: devCommit,
223 bigBlob: bigBlob,
224 daemon: daemon,
225 }
226}
227
228type gitDaemon struct {
229 cmd *exec.Cmd
230 logPath string
231 port int
232}
233
234func startGitDaemon(t *testing.T, serveRoot string) *gitDaemon {
235 t.Helper()
236
237 port := allocatePort(t)
238 logFile, err := os.CreateTemp(t.TempDir(), "git-daemon-*.log")
239 require.NoError(t, err)
240 logPath := logFile.Name()
241 cmd := exec.Command("git", "daemon",
242 "--verbose",
243 "--export-all",
244 "--reuseaddr",
245 "--base-path="+serveRoot,
246 "--listen=127.0.0.1",
247 fmt.Sprintf("--port=%d", port),
248 serveRoot,
249 )
250 cmd.Env = gitTestEnv()
251 cmd.Stdout = logFile
252 cmd.Stderr = logFile
253
254 require.NoError(t, cmd.Start())
255 require.NoError(t, logFile.Close())
256 waitForGitDaemon(t, port, logPath)
257
258 daemon := &gitDaemon{cmd: cmd, logPath: logPath, port: port}
259 t.Cleanup(func() {
260 if cmd.Process != nil {
261 _ = cmd.Process.Kill()
262 }
263 waitDone := make(chan struct{})
264 go func() {
265 _ = cmd.Wait()
266 close(waitDone)
267 }()
268
269 select {
270 case <-waitDone:
271 case <-time.After(5 * time.Second):
272 }
273 })
274
275 return daemon
276}
277
278func waitForGitDaemon(t *testing.T, port int, logPath string) {
279 t.Helper()
280
281 addr := fmt.Sprintf("127.0.0.1:%d", port)
282 deadline := time.Now().Add(5 * time.Second)
283
284 for time.Now().Before(deadline) {
285 conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
286 if err == nil {
287 _ = conn.Close()
288 return
289 }
290
291 time.Sleep(50 * time.Millisecond)
292 }
293
294 contents, err := os.ReadFile(logPath)
295 if err != nil {
296 t.Fatalf("git daemon did not start listening on %s within 5s (failed to read log: %v)", addr, err)
297 }
298
299 t.Fatalf("git daemon did not start listening on %s within 5s\n%s", addr, contents)
300}
301
302func allocatePort(t *testing.T) int {
303 t.Helper()
304
305 listener, err := net.Listen("tcp", "127.0.0.1:0")
306 require.NoError(t, err)
307 defer listener.Close()
308
309 return listener.Addr().(*net.TCPAddr).Port
310}
311
312func gitIndexOptionsForTest(args *indexArgs, repoDir string) gitindex.Options {
313 buildOptions := *args.BuildOptions()
314 buildOptions.RepositoryDescription.Branches = nil
315
316 branches := make([]string, 0, len(args.Branches))
317 for _, branch := range args.Branches {
318 branches = append(branches, branch.Name)
319 }
320
321 return gitindex.Options{
322 RepoDir: repoDir,
323 Submodules: false,
324 Incremental: args.Incremental,
325 BuildOptions: buildOptions,
326 BranchPrefix: "refs/heads/",
327 Branches: branches,
328 DeltaShardNumberFallbackThreshold: args.DeltaShardNumberFallbackThreshold,
329 }
330}
331
332func assertPartialBareFetch(t *testing.T, gitDir string, fixture *gitFetchFixture) {
333 t.Helper()
334 require := require.New(t)
335
336 require.Equal(fixture.mainCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "HEAD")))
337 require.Equal(fixture.devCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "refs/heads/dev")))
338
339 promisors, err := filepath.Glob(filepath.Join(gitDir, "objects", "pack", "*.promisor"))
340 require.NoError(err)
341 require.NotEmpty(promisors)
342
343 objects := runGitOutput(t, gitDir, "rev-list", "--objects", "--missing=print", "HEAD", "refs/heads/dev")
344 require.Contains(objects, fixture.mainCommit)
345 require.Contains(objects, fixture.devCommit)
346 require.Contains(objects, "?"+fixture.bigBlob)
347}
348
349func assertSearchContains(t *testing.T, searcher zoekt.Searcher, pattern string, wantFile string) {
350 t.Helper()
351 require := require.New(t)
352
353 result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{})
354 require.NoError(err)
355 require.Len(result.Files, 1)
356 require.Equal(wantFile, result.Files[0].FileName)
357}
358
359func assertSearchEmpty(t *testing.T, searcher zoekt.Searcher, pattern string) {
360 t.Helper()
361 require := require.New(t)
362
363 result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{})
364 require.NoError(err)
365 require.Empty(result.Files)
366}
367
368func runIntegrationCommand(cmd *exec.Cmd) error {
369 cmd.Env = gitTestEnv()
370 output, err := cmd.CombinedOutput()
371 if err != nil {
372 return fmt.Errorf("%s: %w\n%s", strings.Join(cmd.Args, " "), err, output)
373 }
374 return nil
375}
376
377func runGit(t *testing.T, dir string, args ...string) {
378 t.Helper()
379 _ = runGitOutput(t, dir, args...)
380}
381
382func runGitOutput(t *testing.T, dir string, args ...string) string {
383 t.Helper()
384
385 cmd := exec.Command("git", args...)
386 cmd.Dir = dir
387 cmd.Env = gitTestEnv()
388 output, err := cmd.CombinedOutput()
389 if err != nil {
390 t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output)
391 }
392
393 return string(output)
394}
395
396func gitTestEnv() []string {
397 return append(os.Environ(),
398 "GIT_CONFIG_GLOBAL=",
399 "GIT_CONFIG_SYSTEM=",
400 )
401}