fork of https://github.com/sourcegraph/zoekt
0

Configure Feed

Select the types of activity you want to include in your feed.

indexserver: integration test for sourcegraph (#1033)

+401
+401
cmd/zoekt-sourcegraph-indexserver/index_integration_test.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net" 7 + "os" 8 + "os/exec" 9 + "path/filepath" 10 + "strings" 11 + "testing" 12 + "time" 13 + 14 + "github.com/google/go-cmp/cmp" 15 + "github.com/sourcegraph/log/logtest" 16 + "github.com/stretchr/testify/require" 17 + 18 + "github.com/sourcegraph/zoekt" 19 + "github.com/sourcegraph/zoekt/gitindex" 20 + "github.com/sourcegraph/zoekt/query" 21 + "github.com/sourcegraph/zoekt/search" 22 + ) 23 + 24 + func TestFetchRepoAndIndex_Integration(t *testing.T) { 25 + requireGitDaemon(t) 26 + 27 + for _, tc := range []struct { 28 + name string 29 + disableGoGitOptimization bool 30 + }{ 31 + {name: "optimized repo open"}, 32 + {name: "legacy repo open", disableGoGitOptimization: true}, 33 + } { 34 + t.Run(tc.name, func(t *testing.T) { 35 + require := require.New(t) 36 + 37 + ctx := context.Background() 38 + fixture := newGitFetchFixture(t) 39 + 40 + if tc.disableGoGitOptimization { 41 + t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "true") 42 + } else { 43 + t.Setenv("ZOEKT_DISABLE_GOGIT_OPTIMIZATION", "false") 44 + } 45 + 46 + sg := &recordingSourcegraph{ 47 + opts: IndexOptions{ 48 + RepoID: 123, 49 + Name: "test/repo", 50 + CloneURL: fixture.cloneURL, 51 + Symbols: false, 52 + Branches: []zoekt.RepositoryBranch{ 53 + {Name: "HEAD", Version: fixture.mainCommit}, 54 + {Name: "dev", Version: fixture.devCommit}, 55 + }, 56 + TenantID: 1, 57 + }, 58 + } 59 + 60 + indexDir := t.TempDir() 61 + server := &Server{ 62 + Sourcegraph: sg, 63 + IndexDir: indexDir, 64 + CPUCount: 1, 65 + IndexConcurrency: 1, 66 + } 67 + 68 + result, err := sg.List(ctx, nil) 69 + require.NoError(err) 70 + 71 + var args *indexArgs 72 + result.IterateIndexOptions(func(opts IndexOptions) { 73 + args = server.indexArgs(opts) 74 + }) 75 + require.NotNil(args) 76 + 77 + gitDir := filepath.Join(t.TempDir(), "fetch.git") 78 + c := gitIndexConfig{ 79 + runCmd: runIntegrationCommand, 80 + findRepositoryMetadata: func(args *indexArgs) (*zoekt.Repository, *zoekt.IndexMetadata, bool, error) { 81 + return args.BuildOptions().FindRepositoryMetadata() 82 + }, 83 + timeout: time.Minute, 84 + } 85 + 86 + require.NoError(fetchRepo(ctx, gitDir, args, c, logtest.Scoped(t))) 87 + assertPartialBareFetch(t, gitDir, fixture) 88 + 89 + require.NoError(setZoektConfig(ctx, gitDir, args, c)) 90 + 91 + updated, err := gitindex.IndexGitRepo(gitIndexOptionsForTest(args, gitDir)) 92 + require.NoError(err) 93 + require.True(updated) 94 + 95 + repository, metadata, ok, err := args.BuildOptions().FindRepositoryMetadata() 96 + require.NoError(err) 97 + require.True(ok) 98 + require.Equal(args.Name, repository.Name) 99 + require.Equal(args.RepoID, repository.ID) 100 + require.Equal(args.TenantID, repository.TenantID) 101 + if diff := cmp.Diff(args.Branches, repository.Branches); diff != "" { 102 + t.Fatalf("branches mismatch (-want +got):\n%s", diff) 103 + } 104 + require.Equal("123", repository.RawConfig["repoid"]) 105 + require.Equal("1", repository.RawConfig["tenantID"]) 106 + 107 + searcher, err := search.NewDirectorySearcher(indexDir) 108 + require.NoError(err) 109 + defer searcher.Close() 110 + 111 + assertSearchContains(t, searcher, "smallneedle", "small.txt") 112 + assertSearchContains(t, searcher, "devneedle", "dev.txt") 113 + assertSearchEmpty(t, searcher, "largeneedle") 114 + 115 + require.NoError(updateIndexStatusOnSourcegraph(c, args, sg)) 116 + require.Len(sg.updates, 1) 117 + require.Len(sg.updates[0], 1) 118 + require.Equal(args.RepoID, sg.updates[0][0].RepoID) 119 + require.Equal(metadata.IndexTime.Unix(), sg.updates[0][0].IndexTimeUnix) 120 + if diff := cmp.Diff(args.Branches, sg.updates[0][0].Branches); diff != "" { 121 + t.Fatalf("status branches mismatch (-want +got):\n%s", diff) 122 + } 123 + }) 124 + } 125 + } 126 + 127 + func requireGitDaemon(t *testing.T) { 128 + t.Helper() 129 + 130 + cmd := exec.Command("git", "daemon", "-h") 131 + cmd.Env = gitTestEnv() 132 + output, err := cmd.CombinedOutput() 133 + text := string(output) 134 + 135 + if strings.Contains(text, "usage: git daemon") { 136 + return 137 + } 138 + 139 + if strings.Contains(text, "git: 'daemon' is not a git command") { 140 + t.Skipf("skipping integration test: git daemon is unavailable: %s", strings.TrimSpace(text)) 141 + } 142 + 143 + if err == nil { 144 + return 145 + } 146 + 147 + t.Fatalf("failed to probe git daemon availability: %v\n%s", err, text) 148 + } 149 + 150 + type recordingSourcegraph struct { 151 + opts IndexOptions 152 + updates [][]indexStatus 153 + } 154 + 155 + func (s *recordingSourcegraph) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) { 156 + return &SourcegraphListResult{ 157 + IDs: []uint32{s.opts.RepoID}, 158 + IterateIndexOptions: func(yield func(IndexOptions)) { 159 + yield(s.opts) 160 + }, 161 + }, nil 162 + } 163 + 164 + func (s *recordingSourcegraph) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) { 165 + onSuccess(s.opts) 166 + } 167 + 168 + func (s *recordingSourcegraph) UpdateIndexStatus(repositories []indexStatus) error { 169 + cp := make([]indexStatus, len(repositories)) 170 + copy(cp, repositories) 171 + s.updates = append(s.updates, cp) 172 + return nil 173 + } 174 + 175 + type gitFetchFixture struct { 176 + cloneURL string 177 + mainCommit string 178 + devCommit string 179 + bigBlob string 180 + daemon *gitDaemon 181 + } 182 + 183 + func newGitFetchFixture(t *testing.T) *gitFetchFixture { 184 + t.Helper() 185 + 186 + root := t.TempDir() 187 + worktree := filepath.Join(root, "worktree") 188 + serveRoot := filepath.Join(root, "serve") 189 + remoteDir := filepath.Join(serveRoot, "repo") 190 + 191 + require.NoError(t, os.MkdirAll(worktree, 0o755)) 192 + require.NoError(t, os.MkdirAll(serveRoot, 0o755)) 193 + 194 + runGit(t, root, "init", "-b", "main", worktree) 195 + runGit(t, worktree, "config", "user.name", "Test User") 196 + runGit(t, worktree, "config", "user.email", "test@example.com") 197 + 198 + require.NoError(t, os.WriteFile(filepath.Join(worktree, "small.txt"), []byte("smallneedle\n"), 0o644)) 199 + large := strings.Repeat("x", MaxFileSize+1024) 200 + require.NoError(t, os.WriteFile(filepath.Join(worktree, "big.bin"), []byte("largeneedle\n"+large), 0o644)) 201 + runGit(t, worktree, "add", "small.txt", "big.bin") 202 + runGit(t, worktree, "commit", "-m", "main commit") 203 + 204 + mainCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD")) 205 + bigBlob := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD:big.bin")) 206 + 207 + runGit(t, worktree, "checkout", "-b", "dev") 208 + require.NoError(t, os.WriteFile(filepath.Join(worktree, "dev.txt"), []byte("devneedle\n"), 0o644)) 209 + runGit(t, worktree, "add", "dev.txt") 210 + runGit(t, worktree, "commit", "-m", "dev commit") 211 + 212 + devCommit := strings.TrimSpace(runGitOutput(t, worktree, "rev-parse", "HEAD")) 213 + runGit(t, root, "clone", "--bare", worktree, remoteDir) 214 + runGit(t, remoteDir, "config", "uploadpack.allowFilter", "true") 215 + runGit(t, remoteDir, "config", "uploadpack.allowAnySHA1InWant", "true") 216 + 217 + daemon := startGitDaemon(t, serveRoot) 218 + 219 + return &gitFetchFixture{ 220 + cloneURL: fmt.Sprintf("git://127.0.0.1:%d/repo", daemon.port), 221 + mainCommit: mainCommit, 222 + devCommit: devCommit, 223 + bigBlob: bigBlob, 224 + daemon: daemon, 225 + } 226 + } 227 + 228 + type gitDaemon struct { 229 + cmd *exec.Cmd 230 + logPath string 231 + port int 232 + } 233 + 234 + func startGitDaemon(t *testing.T, serveRoot string) *gitDaemon { 235 + t.Helper() 236 + 237 + port := allocatePort(t) 238 + logFile, err := os.CreateTemp(t.TempDir(), "git-daemon-*.log") 239 + require.NoError(t, err) 240 + logPath := logFile.Name() 241 + cmd := exec.Command("git", "daemon", 242 + "--verbose", 243 + "--export-all", 244 + "--reuseaddr", 245 + "--base-path="+serveRoot, 246 + "--listen=127.0.0.1", 247 + fmt.Sprintf("--port=%d", port), 248 + serveRoot, 249 + ) 250 + cmd.Env = gitTestEnv() 251 + cmd.Stdout = logFile 252 + cmd.Stderr = logFile 253 + 254 + require.NoError(t, cmd.Start()) 255 + require.NoError(t, logFile.Close()) 256 + waitForGitDaemon(t, port, logPath) 257 + 258 + daemon := &gitDaemon{cmd: cmd, logPath: logPath, port: port} 259 + t.Cleanup(func() { 260 + if cmd.Process != nil { 261 + _ = cmd.Process.Kill() 262 + } 263 + waitDone := make(chan struct{}) 264 + go func() { 265 + _ = cmd.Wait() 266 + close(waitDone) 267 + }() 268 + 269 + select { 270 + case <-waitDone: 271 + case <-time.After(5 * time.Second): 272 + } 273 + }) 274 + 275 + return daemon 276 + } 277 + 278 + func waitForGitDaemon(t *testing.T, port int, logPath string) { 279 + t.Helper() 280 + 281 + addr := fmt.Sprintf("127.0.0.1:%d", port) 282 + deadline := time.Now().Add(5 * time.Second) 283 + 284 + for time.Now().Before(deadline) { 285 + conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) 286 + if err == nil { 287 + _ = conn.Close() 288 + return 289 + } 290 + 291 + time.Sleep(50 * time.Millisecond) 292 + } 293 + 294 + contents, err := os.ReadFile(logPath) 295 + if err != nil { 296 + t.Fatalf("git daemon did not start listening on %s within 5s (failed to read log: %v)", addr, err) 297 + } 298 + 299 + t.Fatalf("git daemon did not start listening on %s within 5s\n%s", addr, contents) 300 + } 301 + 302 + func allocatePort(t *testing.T) int { 303 + t.Helper() 304 + 305 + listener, err := net.Listen("tcp", "127.0.0.1:0") 306 + require.NoError(t, err) 307 + defer listener.Close() 308 + 309 + return listener.Addr().(*net.TCPAddr).Port 310 + } 311 + 312 + func gitIndexOptionsForTest(args *indexArgs, repoDir string) gitindex.Options { 313 + buildOptions := *args.BuildOptions() 314 + buildOptions.RepositoryDescription.Branches = nil 315 + 316 + branches := make([]string, 0, len(args.Branches)) 317 + for _, branch := range args.Branches { 318 + branches = append(branches, branch.Name) 319 + } 320 + 321 + return gitindex.Options{ 322 + RepoDir: repoDir, 323 + Submodules: false, 324 + Incremental: args.Incremental, 325 + BuildOptions: buildOptions, 326 + BranchPrefix: "refs/heads/", 327 + Branches: branches, 328 + DeltaShardNumberFallbackThreshold: args.DeltaShardNumberFallbackThreshold, 329 + } 330 + } 331 + 332 + func assertPartialBareFetch(t *testing.T, gitDir string, fixture *gitFetchFixture) { 333 + t.Helper() 334 + require := require.New(t) 335 + 336 + require.Equal(fixture.mainCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "HEAD"))) 337 + require.Equal(fixture.devCommit, strings.TrimSpace(runGitOutput(t, gitDir, "rev-parse", "refs/heads/dev"))) 338 + 339 + promisors, err := filepath.Glob(filepath.Join(gitDir, "objects", "pack", "*.promisor")) 340 + require.NoError(err) 341 + require.NotEmpty(promisors) 342 + 343 + objects := runGitOutput(t, gitDir, "rev-list", "--objects", "--missing=print", "HEAD", "refs/heads/dev") 344 + require.Contains(objects, fixture.mainCommit) 345 + require.Contains(objects, fixture.devCommit) 346 + require.Contains(objects, "?"+fixture.bigBlob) 347 + } 348 + 349 + func assertSearchContains(t *testing.T, searcher zoekt.Searcher, pattern string, wantFile string) { 350 + t.Helper() 351 + require := require.New(t) 352 + 353 + result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{}) 354 + require.NoError(err) 355 + require.Len(result.Files, 1) 356 + require.Equal(wantFile, result.Files[0].FileName) 357 + } 358 + 359 + func assertSearchEmpty(t *testing.T, searcher zoekt.Searcher, pattern string) { 360 + t.Helper() 361 + require := require.New(t) 362 + 363 + result, err := searcher.Search(context.Background(), &query.Substring{Pattern: pattern}, &zoekt.SearchOptions{}) 364 + require.NoError(err) 365 + require.Empty(result.Files) 366 + } 367 + 368 + func runIntegrationCommand(cmd *exec.Cmd) error { 369 + cmd.Env = gitTestEnv() 370 + output, err := cmd.CombinedOutput() 371 + if err != nil { 372 + return fmt.Errorf("%s: %w\n%s", strings.Join(cmd.Args, " "), err, output) 373 + } 374 + return nil 375 + } 376 + 377 + func runGit(t *testing.T, dir string, args ...string) { 378 + t.Helper() 379 + _ = runGitOutput(t, dir, args...) 380 + } 381 + 382 + func runGitOutput(t *testing.T, dir string, args ...string) string { 383 + t.Helper() 384 + 385 + cmd := exec.Command("git", args...) 386 + cmd.Dir = dir 387 + cmd.Env = gitTestEnv() 388 + output, err := cmd.CombinedOutput() 389 + if err != nil { 390 + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output) 391 + } 392 + 393 + return string(output) 394 + } 395 + 396 + func gitTestEnv() []string { 397 + return append(os.Environ(), 398 + "GIT_CONFIG_GLOBAL=", 399 + "GIT_CONFIG_SYSTEM=", 400 + ) 401 + }