fork of https://github.com/sourcegraph/zoekt
0

Configure Feed

Select the types of activity you want to include in your feed.

change findShard implementation to match on IDs, not names (#260)

+230 -1
+1 -1
build/builder.go
··· 391 391 continue 392 392 } 393 393 for _, repo := range repos { 394 - if repo.Name == o.RepositoryDescription.Name { 394 + if repo.ID == o.RepositoryDescription.ID { 395 395 return fn 396 396 } 397 397 }
+229
build/builder_test.go
··· 6 6 "log" 7 7 "os" 8 8 "path/filepath" 9 + "strconv" 10 + "strings" 9 11 "testing" 10 12 11 13 "github.com/google/go-cmp/cmp" ··· 230 232 t.Fatalf("content of skipped documents should not count towards shard size thresold") 231 233 } 232 234 } 235 + 236 + func TestOptions_FindAllShards(t *testing.T) { 237 + type simpleShard struct { 238 + Repository zoekt.Repository 239 + // NumShards is the number of shards that should be created that 240 + // contain data for "Repository". 241 + NumShards int 242 + } 243 + 244 + tests := []struct { 245 + name string 246 + simpleShards []simpleShard 247 + compoundShards [][]zoekt.Repository 248 + expectedShardCount int 249 + expectedRepository zoekt.Repository 250 + }{ 251 + { 252 + name: "repository in normal shard", 253 + simpleShards: []simpleShard{ 254 + {Repository: zoekt.Repository{Name: "repoA", ID: 1}}, 255 + {Repository: zoekt.Repository{Name: "repoB", ID: 2}}, 256 + {Repository: zoekt.Repository{Name: "repoC", ID: 3}}, 257 + }, 258 + expectedShardCount: 1, 259 + expectedRepository: zoekt.Repository{Name: "repoB", ID: 2}, 260 + }, 261 + { 262 + name: "repository in compound shard", 263 + compoundShards: [][]zoekt.Repository{ 264 + { 265 + {Name: "repoA", ID: 1}, 266 + {Name: "repoB", ID: 2}, 267 + {Name: "repoC", ID: 3}, 268 + }, 269 + { 270 + {Name: "repoD", ID: 4}, 271 + {Name: "repoE", ID: 5}, 272 + {Name: "repoF", ID: 6}, 273 + }, 274 + }, 275 + expectedShardCount: 1, 276 + expectedRepository: zoekt.Repository{Name: "repoB", ID: 2}, 277 + }, 278 + { 279 + name: "repository split across multiple shards", 280 + simpleShards: []simpleShard{ 281 + {Repository: zoekt.Repository{Name: "repoA", ID: 1}}, 282 + {Repository: zoekt.Repository{Name: "repoB", ID: 2}, NumShards: 2}, 283 + {Repository: zoekt.Repository{Name: "repoC", ID: 3}}, 284 + }, 285 + expectedShardCount: 2, 286 + expectedRepository: zoekt.Repository{Name: "repoB", ID: 2}, 287 + }, 288 + { 289 + name: "unknown repository", 290 + simpleShards: []simpleShard{ 291 + {Repository: zoekt.Repository{Name: "repoA", ID: 1}}, 292 + {Repository: zoekt.Repository{Name: "repoB", ID: 2}}, 293 + {Repository: zoekt.Repository{Name: "repoC", ID: 3}}, 294 + }, 295 + compoundShards: [][]zoekt.Repository{ 296 + { 297 + {Name: "repoD", ID: 4}, 298 + {Name: "repoE", ID: 5}, 299 + {Name: "repoF", ID: 6}, 300 + }, 301 + }, 302 + expectedShardCount: 0, 303 + }, 304 + { 305 + name: "match on ID, not name (compound only)", 306 + compoundShards: [][]zoekt.Repository{ 307 + { 308 + {Name: "repoA", ID: 1}, 309 + {Name: "sameName", ID: 2}, 310 + {Name: "sameName", ID: 3}, 311 + }, 312 + { 313 + {Name: "repoB", ID: 4}, 314 + {Name: "sameName", ID: 5}, 315 + {Name: "sameName", ID: 6}, 316 + }, 317 + }, 318 + expectedShardCount: 1, 319 + expectedRepository: zoekt.Repository{Name: "sameName", ID: 5}, 320 + }, 321 + } 322 + for _, tt := range tests { 323 + t.Run(tt.name, func(t *testing.T) { 324 + // prepare 325 + indexDir := t.TempDir() 326 + 327 + for _, s := range tt.simpleShards { 328 + createTestShard(t, indexDir, s.Repository, s.NumShards) 329 + } 330 + 331 + for _, repositoryGroup := range tt.compoundShards { 332 + createTestCompoundShard(t, indexDir, repositoryGroup) 333 + } 334 + 335 + o := &Options{ 336 + IndexDir: indexDir, 337 + RepositoryDescription: tt.expectedRepository, 338 + } 339 + o.SetDefaults() 340 + 341 + // run test 342 + shards := o.FindAllShards() 343 + 344 + // verify results 345 + if len(shards) != tt.expectedShardCount { 346 + t.Fatalf("expected %d shard(s), received %d shard(s)", tt.expectedShardCount, len(shards)) 347 + } 348 + 349 + if tt.expectedShardCount > 0 { 350 + for _, s := range shards { 351 + // all shards should contain the metadata for the desired repository 352 + repos, _, err := zoekt.ReadMetadataPathAlive(s) 353 + if err != nil { 354 + t.Fatalf("reading metadata from shard %q: %s", s, err) 355 + } 356 + 357 + foundRepository := false 358 + for _, r := range repos { 359 + if r.ID == tt.expectedRepository.ID { 360 + foundRepository = true 361 + break 362 + } 363 + } 364 + 365 + if !foundRepository { 366 + t.Errorf("shard %q doesn't contain metadata for repository %d", s, tt.expectedRepository.ID) 367 + } 368 + } 369 + } 370 + }) 371 + } 372 + } 373 + 374 + func createTestShard(t *testing.T, indexDir string, r zoekt.Repository, numShards int) { 375 + t.Helper() 376 + 377 + if err := os.MkdirAll(filepath.Dir(indexDir), 0700); err != nil { 378 + t.Fatal(err) 379 + } 380 + 381 + o := Options{ 382 + IndexDir: indexDir, 383 + RepositoryDescription: r, 384 + ShardMax: 75, // create a new shard every 75 bytes 385 + } 386 + o.SetDefaults() 387 + 388 + b, err := NewBuilder(o) 389 + if err != nil { 390 + t.Fatalf("NewBuilder: %v", err) 391 + } 392 + 393 + if numShards == 0 { 394 + // We have to make at least 1 shard. 395 + numShards = 1 396 + } 397 + 398 + for i := 0; i < numShards; i++ { 399 + // Create entries (file + contents) that are ~100 bytes each. 400 + // This (along with our shardMax setting of 75 bytes) means that each shard 401 + // will contain at most one of these. 402 + fileName := strconv.Itoa(i) 403 + contents := []byte(strings.Repeat("A", 100)) 404 + 405 + err := b.AddFile(fileName, contents) 406 + if err != nil { 407 + t.Fatalf("failed to add file %q to builder: %s", fileName, err) 408 + } 409 + } 410 + 411 + if err := b.Finish(); err != nil { 412 + t.Fatalf("Finish: %v", err) 413 + } 414 + } 415 + 416 + func createTestCompoundShard(t *testing.T, indexDir string, repositories []zoekt.Repository) { 417 + t.Helper() 418 + 419 + var shardNames []string 420 + 421 + for _, r := range repositories { 422 + // create an isolated scratch space to store normal shards for this repository 423 + scratchDir := t.TempDir() 424 + 425 + // create shards that'll be merged later 426 + createTestShard(t, scratchDir, r, 1) 427 + 428 + // discover file names for all the normal shards we created 429 + // note: this only looks in the immediate 'scratchDir' folder and doesn't recurse 430 + shards, err := filepath.Glob(filepath.Join(scratchDir, "*.zoekt")) 431 + if err != nil { 432 + t.Fatalf("while globbing %q to find normal shards: %s", scratchDir, err) 433 + } 434 + 435 + shardNames = append(shardNames, shards...) 436 + } 437 + 438 + // load the normal shards that we created 439 + var files []zoekt.IndexFile 440 + for _, shard := range shardNames { 441 + f, err := os.Open(shard) 442 + if err != nil { 443 + t.Fatalf("opening shard file: %s", err) 444 + } 445 + defer f.Close() 446 + 447 + indexFile, err := zoekt.NewIndexFile(f) 448 + if err != nil { 449 + t.Fatalf("creating index file: %s", err) 450 + } 451 + defer indexFile.Close() 452 + 453 + files = append(files, indexFile) 454 + } 455 + 456 + // merge all the normal shards into a compound shard 457 + _, err := zoekt.Merge(indexDir, files...) 458 + if err != nil { 459 + t.Fatalf("merging index files into compound shard: %s", err) 460 + } 461 + }