fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package search
16
17import (
18 "bytes"
19 "context"
20 "flag"
21 "fmt"
22 "hash/fnv"
23 "io"
24 "log"
25 "math"
26 "os"
27 "reflect"
28 "runtime"
29 "sort"
30 "strconv"
31 "testing"
32 "testing/quick"
33 "time"
34
35 "github.com/RoaringBitmap/roaring"
36 "github.com/google/go-cmp/cmp"
37 "github.com/google/go-cmp/cmp/cmpopts"
38 "github.com/grafana/regexp"
39
40 "github.com/sourcegraph/zoekt/index"
41
42 "github.com/sourcegraph/zoekt"
43 "github.com/sourcegraph/zoekt/query"
44)
45
46func TestMain(m *testing.M) {
47 flag.Parse()
48 if !testing.Verbose() {
49 log.SetOutput(io.Discard)
50 }
51 os.Exit(m.Run())
52}
53
54type crashSearcher struct{}
55
56func (s *crashSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
57 panic("search")
58}
59
60func (s *crashSearcher) List(ctx context.Context, q query.Q, opts *zoekt.ListOptions) (*zoekt.RepoList, error) {
61 panic("list")
62}
63
64func (s *crashSearcher) Stats() (*zoekt.RepoStats, error) {
65 return &zoekt.RepoStats{}, nil
66}
67
68func (s *crashSearcher) Close() {}
69
70func (s *crashSearcher) String() string { return "crashSearcher" }
71
72func TestCrashResilience(t *testing.T) {
73 out := &bytes.Buffer{}
74 oldOut := log.Writer()
75 log.SetOutput(out)
76 defer log.SetOutput(oldOut)
77
78 ss := newShardedSearcher(2)
79 ss.ranked.Store([]*rankedShard{{Searcher: &crashSearcher{}}})
80
81 var wantCrashes int
82 test := func(t *testing.T) {
83 q := &query.Substring{Pattern: "hoi"}
84 opts := &zoekt.SearchOptions{}
85 if res, err := ss.Search(context.Background(), q, opts); err != nil {
86 t.Fatalf("Search: %v", err)
87 } else if res.Stats.Crashes != wantCrashes {
88 t.Errorf("got stats %#v, want crashes = %d", res.Stats, wantCrashes)
89 }
90
91 if res, err := ss.List(context.Background(), q, nil); err != nil {
92 t.Fatalf("List: %v", err)
93 } else if res.Crashes != wantCrashes {
94 t.Errorf("got result %#v, want crashes = %d", res, wantCrashes)
95 }
96 }
97
98 // Before we are marked as ready we have one extra crash
99 wantCrashes = 2
100 t.Run("loading", test)
101
102 // After marking as ready we should only have the crashSearcher
103 // contributing.
104 ss.markReady()
105 wantCrashes = 1
106 t.Run("ready", test)
107}
108
109type rankSearcher struct {
110 rank uint16
111 repo *zoekt.Repository
112}
113
114func (s *rankSearcher) Close() {
115}
116
117func (s *rankSearcher) String() string {
118 return ""
119}
120
121func (s *rankSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
122 select {
123 case <-ctx.Done():
124 return &zoekt.SearchResult{}, nil
125 default:
126 }
127
128 // Ugly, but without sleep it's too fast, and we can't
129 // simulate the cutoff.
130 time.Sleep(time.Millisecond)
131 return &zoekt.SearchResult{
132 Files: []zoekt.FileMatch{
133 {
134 FileName: fmt.Sprintf("f%d", s.rank),
135 Score: float64(s.rank),
136 },
137 },
138 Stats: zoekt.Stats{
139 MatchCount: 1,
140 },
141 }, nil
142}
143
144func (s *rankSearcher) List(ctx context.Context, q query.Q, opts *zoekt.ListOptions) (*zoekt.RepoList, error) {
145 r := zoekt.Repository{}
146 if s.repo != nil {
147 r = *s.repo
148 }
149 r.Rank = s.rank
150 return &zoekt.RepoList{
151 Repos: []*zoekt.RepoListEntry{
152 {Repository: r},
153 },
154 }, nil
155}
156
157func (s *rankSearcher) Repository() *zoekt.Repository { return s.repo }
158
159func TestOrderByShard(t *testing.T) {
160 ss := newShardedSearcher(1)
161
162 n := 10 * runtime.GOMAXPROCS(0)
163 for i := range n {
164 ss.replace(map[string]zoekt.Searcher{
165 fmt.Sprintf("shard%d", i): &rankSearcher{rank: uint16(i)},
166 })
167 }
168
169 if res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &zoekt.SearchOptions{}); err != nil {
170 t.Errorf("Search: %v", err)
171 } else if len(res.Files) != n {
172 t.Fatalf("empty options: got %d results, want %d", len(res.Files), n)
173 }
174
175 opts := zoekt.SearchOptions{
176 TotalMaxMatchCount: 3,
177 }
178 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &opts)
179 if err != nil {
180 t.Errorf("Search: %v", err)
181 }
182
183 if len(res.Files) < opts.TotalMaxMatchCount {
184 t.Errorf("got %d results, want %d", len(res.Files), opts.TotalMaxMatchCount)
185 }
186 if len(res.Files) == n {
187 t.Errorf("got %d results, want < %d", len(res.Files), n)
188 }
189 for i, f := range res.Files {
190 rev := n - 1 - i
191 want := fmt.Sprintf("f%d", rev)
192 got := f.FileName
193
194 if got != want {
195 t.Logf("%d: got %q, want %q", i, got, want)
196 }
197 }
198}
199
200func TestShardedSearcher_Ranking(t *testing.T) {
201 ss := newShardedSearcher(1)
202
203 var nextShardNum int
204 addShard := func(repo string, priority float64, docs ...index.Document) {
205 r := &zoekt.Repository{ID: hash(repo), Name: repo}
206 r.RawConfig = map[string]string{
207 "public": "1",
208 "priority": strconv.FormatFloat(priority, 'f', 2, 64),
209 }
210 b := testShardBuilder(t, r, docs...)
211 shard := searcherForTest(t, b)
212 ss.replace(map[string]zoekt.Searcher{
213 fmt.Sprintf("key-%d", nextShardNum): shard,
214 })
215 nextShardNum++
216 }
217
218 addShard("weekend-project", 20, index.Document{Name: "f2", Content: []byte("foo bas")})
219 addShard("moderately-popular", 500, index.Document{Name: "f3", Content: []byte("foo bar")})
220 addShard("weekend-project-2", 20, index.Document{Name: "f2", Content: []byte("foo bas")})
221 addShard("super-star", 5000, index.Document{Name: "f1", Content: []byte("foo bar bas")})
222
223 want := []string{
224 "super-star",
225 "moderately-popular",
226 "weekend-project",
227 "weekend-project-2",
228 }
229
230 var have []string
231 for _, s := range ss.getLoaded().shards {
232 for _, r := range s.repos {
233 have = append(have, r.Name)
234 }
235 }
236
237 if !reflect.DeepEqual(want, have) {
238 t.Fatalf("\nwant: %s\nhave: %s", want, have)
239 }
240}
241
242func TestShardedSearcher_DocumentRanking(t *testing.T) {
243 ss := newShardedSearcher(1)
244
245 var nextShardNum int
246 addShard := func(repo string, rank uint16, docs ...index.Document) {
247 r := &zoekt.Repository{ID: hash(repo), Name: repo}
248 r.RawConfig = map[string]string{
249 "public": "1",
250 }
251 r.Rank = rank
252 b := testShardBuilder(t, r, docs...)
253 shard := searcherForTest(t, b)
254 ss.replace(map[string]zoekt.Searcher{
255 fmt.Sprintf("key-%d", nextShardNum): shard,
256 })
257 nextShardNum++
258 }
259
260 addShard("old-project", 1, index.Document{Name: "f1", Content: []byte("foobar")})
261 addShard("recent", 2, index.Document{Name: "f2", Content: []byte("foobaz")})
262 addShard("old-project-2", 1, index.Document{Name: "f3", Content: []byte("foo bar")})
263 addShard("new", 3, index.Document{Name: "f4", Content: []byte("foo baz")},
264 index.Document{Name: "f5", Content: []byte("fooooo")})
265
266 // Run a stream search and gather the results
267 var results []*zoekt.SearchResult
268 opts := &zoekt.SearchOptions{
269 FlushWallTime: 100 * time.Millisecond,
270 }
271
272 err := ss.StreamSearch(context.Background(), &query.Substring{Pattern: "foo"}, opts,
273 zoekt.SenderFunc(func(event *zoekt.SearchResult) {
274 results = append(results, event)
275 }))
276 if err != nil {
277 t.Fatal(err)
278 }
279
280 // There should always be two stream results, first progress-only, then the file results
281 if len(results) != 2 {
282 t.Fatalf("expected 2 streamed results, but got %d", len(results))
283 }
284
285 // The ranking should be determined by whether it's an exact word match,
286 // followed by repository priority
287 want := []string{"f4", "f3", "f5", "f2", "f1"}
288
289 files := results[1].Files
290 got := make([]string, len(files))
291 for i := range files {
292 got[i] = files[i].FileName
293 }
294
295 if !reflect.DeepEqual(got, want) {
296 t.Errorf("got %v, want %v", got, want)
297 }
298}
299
300func TestFilteringShardsByRepoSetOrBranchesReposOrRepoIDs(t *testing.T) {
301 ss := newShardedSearcher(1)
302
303 // namePrefix is so we can create a repo:foo filter and match the same set
304 // of repos.
305 namePrefix := [3]string{"foo", "bar", "baz"}
306
307 repoSetNames := []string{}
308 repoIDs := []uint32{}
309 n := 10 * runtime.GOMAXPROCS(0)
310 for i := range n {
311 shardName := fmt.Sprintf("shard%d", i)
312 repoName := fmt.Sprintf("%s-repository%.3d", namePrefix[i%3], i)
313 repoID := hash(repoName)
314
315 if i%3 == 0 {
316 repoSetNames = append(repoSetNames, repoName)
317 repoIDs = append(repoIDs, repoID)
318 }
319
320 ss.replace(map[string]zoekt.Searcher{
321 shardName: &rankSearcher{
322 repo: &zoekt.Repository{ID: repoID, Name: repoName},
323 rank: uint16(n - i),
324 },
325 })
326 }
327
328 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &zoekt.SearchOptions{})
329 if err != nil {
330 t.Errorf("Search: %v", err)
331 }
332 if len(res.Files) != n {
333 t.Fatalf("no reposet: got %d results, want %d", len(res.Files), n)
334 }
335
336 branchesRepos := &query.BranchesRepos{List: []query.BranchRepos{
337 {Branch: "HEAD", Repos: roaring.New()},
338 }}
339
340 for _, name := range repoSetNames {
341 branchesRepos.List[0].Repos.Add(hash(name))
342 }
343
344 set := query.NewRepoSet(repoSetNames...)
345 sub := &query.Substring{Pattern: "bla"}
346
347 repoIDsQuery := query.NewRepoIDs(repoIDs...)
348 repoQuery := &query.Repo{Regexp: regexp.MustCompile("^foo-.*")}
349
350 queries := []query.Q{
351 query.NewAnd(set, sub),
352 // Test with the same reposet again
353 query.NewAnd(set, sub),
354
355 query.NewAnd(branchesRepos, sub),
356 // Test with the same repoBranches with IDs again
357 query.NewAnd(branchesRepos, sub),
358
359 query.NewAnd(repoIDsQuery, sub),
360 // Test with the same repoIDs again
361 query.NewAnd(repoIDsQuery, sub),
362
363 query.NewAnd(repoQuery, sub),
364 query.NewAnd(repoQuery, sub),
365
366 // List has queries which are just the reposet atoms. We also test twice.
367 set,
368 set,
369 branchesRepos,
370 branchesRepos,
371 repoIDsQuery,
372 repoIDsQuery,
373 repoQuery,
374 repoQuery,
375 }
376
377 for _, q := range queries {
378 res, err = ss.Search(context.Background(), q, &zoekt.SearchOptions{})
379 if err != nil {
380 t.Errorf("Search(%s): %v", q, err)
381 }
382 // Note: Assertion is based on fact that `rankSearcher` always returns a
383 // result and using repoSet will half the number of results
384 if len(res.Files) != len(repoSetNames) {
385 t.Fatalf("%s: got %d results, want %d", q, len(res.Files), len(repoSetNames))
386 }
387 }
388}
389
390func TestFilteringShardsByMeta(t *testing.T) {
391 ss := newShardedSearcher(1)
392
393 // Create repos with different metadata values
394 // We'll create 30 repos total:
395 // - 10 with nickname="project-A"
396 // - 10 with nickname="project-B"
397 // - 10 with no metadata
398 n := 30
399 projectARepos := []string{}
400 projectBRepos := []string{}
401
402 // Common document that will be in all repos
403 doc := index.Document{
404 Name: "common.go",
405 Content: []byte("needle haystack"),
406 }
407
408 for i := range n {
409 shardName := fmt.Sprintf("shard%d", i)
410 repoName := fmt.Sprintf("repository%.3d", i)
411
412 var metadata map[string]string
413 if i < 10 {
414 // First 10 repos have project-A
415 metadata = map[string]string{"nickname": "project-A", "visibility": "public"}
416 projectARepos = append(projectARepos, repoName)
417 } else if i < 20 {
418 // Next 10 repos have project-B
419 metadata = map[string]string{"nickname": "project-B", "visibility": "private"}
420 projectBRepos = append(projectBRepos, repoName)
421 }
422 // Last 10 repos have no metadata
423
424 repo := &zoekt.Repository{
425 ID: uint32(i + 1),
426 Name: repoName,
427 Metadata: metadata,
428 }
429
430 ss.replace(map[string]zoekt.Searcher{
431 shardName: searcherForTest(t, testShardBuilder(t, repo, doc)),
432 })
433 }
434
435 // Test 1: Search without Meta filter - should search all shards
436 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "needle"}, &zoekt.SearchOptions{})
437 if err != nil {
438 t.Fatalf("Search without filter: %v", err)
439 }
440 if len(res.Files) != n {
441 t.Fatalf("no meta filter: got %d results, want %d", len(res.Files), n)
442 }
443
444 sub := &query.Substring{Pattern: "needle"}
445
446 // Helper function to extract unique repo names from search results
447 getRepoNames := func(files []zoekt.FileMatch) []string {
448 repoSet := make(map[string]struct{})
449 for _, f := range files {
450 repoSet[f.Repository] = struct{}{}
451 }
452 repos := make([]string, 0, len(repoSet))
453 for repo := range repoSet {
454 repos = append(repos, repo)
455 }
456 sort.Strings(repos)
457 return repos
458 }
459
460 // Test 2: Filter by nickname="project-A" - should only search 10 shards
461 metaQueryA := &query.Meta{
462 Field: "nickname",
463 Value: regexp.MustCompile("^project-A$"),
464 }
465 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryA, sub), &zoekt.SearchOptions{})
466 if err != nil {
467 t.Fatalf("Search with Meta filter A: %v", err)
468 }
469 gotRepos := getRepoNames(res.Files)
470 wantRepos := append([]string{}, projectARepos...)
471 sort.Strings(wantRepos)
472 if !reflect.DeepEqual(gotRepos, wantRepos) {
473 t.Fatalf("Meta(nickname=project-A):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
474 }
475
476 // Test 3: Filter by nickname="project-B" - should only search 10 shards
477 metaQueryB := &query.Meta{
478 Field: "nickname",
479 Value: regexp.MustCompile("^project-B$"),
480 }
481 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryB, sub), &zoekt.SearchOptions{})
482 if err != nil {
483 t.Fatalf("Search with Meta filter B: %v", err)
484 }
485 gotRepos = getRepoNames(res.Files)
486 wantRepos = append([]string{}, projectBRepos...)
487 sort.Strings(wantRepos)
488 if !reflect.DeepEqual(gotRepos, wantRepos) {
489 t.Fatalf("Meta(nickname=project-B):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
490 }
491
492 // Test 4: Filter by visibility="public" - should only search 10 shards (project-A repos)
493 metaQueryPublic := &query.Meta{
494 Field: "visibility",
495 Value: regexp.MustCompile("^public$"),
496 }
497 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryPublic, sub), &zoekt.SearchOptions{})
498 if err != nil {
499 t.Fatalf("Search with Meta filter public: %v", err)
500 }
501 gotRepos = getRepoNames(res.Files)
502 wantRepos = append([]string{}, projectARepos...)
503 sort.Strings(wantRepos)
504 if !reflect.DeepEqual(gotRepos, wantRepos) {
505 t.Fatalf("Meta(visibility=public):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
506 }
507
508 // Test 5: Filter by non-existent field - should return 0 results
509 metaQueryNonExistent := &query.Meta{
510 Field: "nonexistent_field",
511 Value: regexp.MustCompile(".*"),
512 }
513 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryNonExistent, sub), &zoekt.SearchOptions{})
514 if err != nil {
515 t.Fatalf("Search with Meta filter non-existent: %v", err)
516 }
517 if len(res.Files) != 0 {
518 t.Fatalf("Meta(nonexistent_field): got %d results, want 0", len(res.Files))
519 }
520
521 // Test 6: Filter by regex pattern matching multiple values
522 metaQueryRegex := &query.Meta{
523 Field: "nickname",
524 Value: regexp.MustCompile("project-.*"), // Matches both project-A and project-B
525 }
526 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryRegex, sub), &zoekt.SearchOptions{})
527 if err != nil {
528 t.Fatalf("Search with Meta regex filter: %v", err)
529 }
530 gotRepos = getRepoNames(res.Files)
531 wantRepos = append(append([]string{}, projectARepos...), projectBRepos...)
532 sort.Strings(wantRepos)
533 if !reflect.DeepEqual(gotRepos, wantRepos) {
534 t.Fatalf("Meta(nickname=project-.*):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
535 }
536
537 // Test 7: Test that Meta query alone (without content search) works
538 res, err = ss.Search(context.Background(), metaQueryA, &zoekt.SearchOptions{})
539 if err != nil {
540 t.Fatalf("Search with Meta query alone: %v", err)
541 }
542 gotRepos = getRepoNames(res.Files)
543 wantRepos = append([]string{}, projectARepos...)
544 sort.Strings(wantRepos)
545 if !reflect.DeepEqual(gotRepos, wantRepos) {
546 t.Fatalf("Meta query alone:\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
547 }
548
549 // Test 8: Test with List operation (not just Search)
550 listRes, err := ss.List(context.Background(), metaQueryA, nil)
551 if err != nil {
552 t.Fatalf("List with Meta filter: %v", err)
553 }
554 gotListRepos := make([]string, len(listRes.Repos))
555 for i, r := range listRes.Repos {
556 gotListRepos[i] = r.Repository.Name
557 }
558 sort.Strings(gotListRepos)
559 wantRepos = append([]string{}, projectARepos...)
560 sort.Strings(wantRepos)
561 if !reflect.DeepEqual(gotListRepos, wantRepos) {
562 t.Fatalf("List with Meta(nickname=project-A):\ngot repos: %v\nwant repos: %v", gotListRepos, wantRepos)
563 }
564}
565
566func hash(name string) uint32 {
567 h := fnv.New32()
568 h.Write([]byte(name))
569 return h.Sum32()
570}
571
572type memSeeker struct {
573 data []byte
574}
575
576func (s *memSeeker) Name() string {
577 return "memseeker"
578}
579
580func (s *memSeeker) Close() {}
581func (s *memSeeker) Read(off, sz uint32) ([]byte, error) {
582 return s.data[off : off+sz], nil
583}
584
585func (s *memSeeker) Size() (uint32, error) {
586 return uint32(len(s.data)), nil
587}
588
589func TestUnloadIndex(t *testing.T) {
590 b := testShardBuilder(t, nil, index.Document{
591 Name: "filename",
592 Content: []byte("needle needle needle"),
593 })
594
595 var buf bytes.Buffer
596 if err := b.Write(&buf); err != nil {
597 t.Fatal(err)
598 }
599 indexBytes := buf.Bytes()
600 indexFile := &memSeeker{indexBytes}
601 searcher, err := index.NewSearcher(indexFile)
602 if err != nil {
603 t.Fatalf("NewSearcher: %v", err)
604 }
605
606 ss := newShardedSearcher(2)
607 ss.replace(map[string]zoekt.Searcher{"key": searcher})
608
609 var opts zoekt.SearchOptions
610 q := &query.Substring{Pattern: "needle"}
611 res, err := ss.Search(context.Background(), q, &opts)
612 if err != nil {
613 t.Fatalf("Search(%s): %v", q, err)
614 }
615
616 forbidden := byte(29)
617 for i := range indexBytes {
618 // non-ASCII
619 indexBytes[i] = forbidden
620 }
621
622 for _, f := range res.Files {
623 if bytes.Contains(f.Content, []byte{forbidden}) {
624 t.Errorf("found %d in content %q", forbidden, f.Content)
625 }
626 if bytes.Contains(f.Checksum, []byte{forbidden}) {
627 t.Errorf("found %d in checksum %q", forbidden, f.Checksum)
628 }
629
630 for _, l := range f.LineMatches {
631 if bytes.Contains(l.Line, []byte{forbidden}) {
632 t.Errorf("found %d in line %q", forbidden, l.Line)
633 }
634 }
635 }
636}
637
638func TestShardedSearcher_List(t *testing.T) {
639 repos := []*zoekt.Repository{
640 {
641 ID: 1234,
642 Name: "repo-a",
643 Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
644 RawConfig: map[string]string{"repoid": "1234"},
645 },
646 {
647 Name: "repo-b",
648 Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
649 },
650 }
651
652 doc := index.Document{
653 Name: "foo.go",
654 Content: []byte("bar\nbaz"),
655 Branches: []string{"main", "dev"},
656 }
657
658 // Test duplicate removal when ListOptions.Minimal is true and false
659 ss := newShardedSearcher(4)
660 ss.replace(map[string]zoekt.Searcher{
661 "1": searcherForTest(t, testShardBuilder(t, repos[0], doc)),
662 "2": searcherForTest(t, testShardBuilder(t, repos[0])),
663 "3": searcherForTest(t, testShardBuilder(t, repos[1], doc)),
664 "4": searcherForTest(t, testShardBuilder(t, repos[1])),
665 })
666 ss.markReady()
667
668 stats := zoekt.RepoStats{
669 Shards: 2,
670 Documents: 1,
671 IndexBytes: 196,
672 ContentBytes: 13,
673 NewLinesCount: 1,
674 DefaultBranchNewLinesCount: 1,
675 OtherBranchesNewLinesCount: 1,
676 }
677
678 aggStats := stats
679 aggStats.Add(&aggStats) // since both repos have the exact same stats, this works
680 aggStats.Repos = 2 // Add doesn't populate Repos, this is done in Shards based on the response sizes.
681
682 for _, tc := range []struct {
683 name string
684 opts *zoekt.ListOptions
685 want *zoekt.RepoList
686 }{
687 {
688 name: "nil opts",
689 opts: nil,
690 want: &zoekt.RepoList{
691 Repos: []*zoekt.RepoListEntry{
692 {
693 Repository: *repos[0],
694 Stats: stats,
695 },
696 {
697 Repository: *repos[1],
698 Stats: stats,
699 },
700 },
701 Stats: aggStats,
702 },
703 },
704 {
705 name: "default",
706 opts: &zoekt.ListOptions{},
707 want: &zoekt.RepoList{
708 Repos: []*zoekt.RepoListEntry{
709 {
710 Repository: *repos[0],
711 Stats: stats,
712 },
713 {
714 Repository: *repos[1],
715 Stats: stats,
716 },
717 },
718 Stats: aggStats,
719 },
720 },
721 {
722 name: "field=repos",
723 opts: &zoekt.ListOptions{Field: zoekt.RepoListFieldRepos},
724 want: &zoekt.RepoList{
725 Repos: []*zoekt.RepoListEntry{
726 {
727 Repository: *repos[0],
728 Stats: stats,
729 },
730 {
731 Repository: *repos[1],
732 Stats: stats,
733 },
734 },
735 Stats: aggStats,
736 },
737 },
738 {
739 name: "field=reposmap",
740 opts: &zoekt.ListOptions{Field: zoekt.RepoListFieldReposMap},
741 want: &zoekt.RepoList{
742 Repos: []*zoekt.RepoListEntry{
743 {
744 Repository: *repos[1],
745 Stats: stats,
746 },
747 },
748 ReposMap: zoekt.ReposMap{
749 repos[0].ID: {
750 HasSymbols: repos[0].HasSymbols,
751 Branches: repos[0].Branches,
752 },
753 },
754 Stats: aggStats,
755 },
756 },
757 } {
758 tc := tc
759 t.Run(tc.name, func(t *testing.T) {
760 t.Parallel()
761
762 q := &query.Repo{Regexp: regexp.MustCompile("repo")}
763
764 res, err := ss.List(context.Background(), q, tc.opts)
765 if err != nil {
766 t.Fatalf("List(%v, %s): %v", q, tc.opts, err)
767 }
768
769 sort.Slice(res.Repos, func(i, j int) bool {
770 return res.Repos[i].Repository.Name < res.Repos[j].Repository.Name
771 })
772
773 ignored := []cmp.Option{
774 cmpopts.EquateEmpty(),
775 cmpopts.IgnoreFields(zoekt.MinimalRepoListEntry{}, "IndexTimeUnix"),
776 cmpopts.IgnoreFields(zoekt.RepoListEntry{}, "IndexMetadata"),
777 cmpopts.IgnoreFields(zoekt.RepoStats{}, "IndexBytes"),
778 cmpopts.IgnoreFields(zoekt.Repository{}, "SubRepoMap"),
779 cmpopts.IgnoreFields(zoekt.Repository{}, "priority"),
780 }
781
782 if diff := cmp.Diff(tc.want, res, ignored...); diff != "" {
783 t.Fatalf("mismatch (-want +got):\n%s", diff)
784 }
785 })
786 }
787}
788
789func testShardBuilder(t testing.TB, repo *zoekt.Repository, docs ...index.Document) *index.ShardBuilder {
790 b, err := index.NewShardBuilder(repo)
791 if err != nil {
792 t.Fatalf("NewShardBuilder: %v", err)
793 }
794
795 for i, d := range docs {
796 if err := b.Add(d); err != nil {
797 t.Fatalf("Add %d: %v", i, err)
798 }
799 }
800 return b
801}
802
803func searcherForTest(t testing.TB, b *index.ShardBuilder) zoekt.Searcher {
804 var buf bytes.Buffer
805 if err := b.Write(&buf); err != nil {
806 t.Fatal(err)
807 }
808 f := &memSeeker{buf.Bytes()}
809
810 searcher, err := index.NewSearcher(f)
811 if err != nil {
812 t.Fatalf("NewSearcher: %v", err)
813 }
814
815 return searcher
816}
817
818func reposForTest(n int) (result []*zoekt.Repository) {
819 for i := range n {
820 result = append(result, &zoekt.Repository{
821 ID: uint32(i + 1),
822 Name: fmt.Sprintf("test-repository-%d", i),
823 })
824 }
825 return result
826}
827
828func testSearcherForRepo(b testing.TB, r *zoekt.Repository, numFiles int) zoekt.Searcher {
829 builder := testShardBuilder(b, r)
830
831 if err := builder.Add(index.Document{
832 Name: fmt.Sprintf("%s/filename-%d.go", r.Name, 0),
833 Content: []byte("needle needle needle haystack"),
834 }); err != nil {
835 b.Fatal(err)
836 }
837
838 for i := 1; i < numFiles; i++ {
839 if err := builder.Add(index.Document{
840 Name: fmt.Sprintf("%s/filename-%d.go", r.Name, i),
841 Content: []byte("haystack haystack haystack"),
842 }); err != nil {
843 b.Fatal(err)
844 }
845 }
846
847 return searcherForTest(b, builder)
848}
849
850func BenchmarkShardedSearch(b *testing.B) {
851 ss := newShardedSearcher(int64(runtime.GOMAXPROCS(0)))
852
853 filesPerRepo := 300
854 repos := reposForTest(3000)
855 var repoSetIDs []uint32
856
857 shards := make(map[string]zoekt.Searcher, len(repos))
858 for i, r := range repos {
859 shards[r.Name] = testSearcherForRepo(b, r, filesPerRepo)
860 if i%2 == 0 {
861 repoSetIDs = append(repoSetIDs, r.ID)
862 }
863 }
864
865 ss.replace(shards)
866
867 ctx := context.Background()
868 opts := &zoekt.SearchOptions{}
869
870 needleSub := &query.Substring{Pattern: "needle"}
871 haystackSub := &query.Substring{Pattern: "haystack"}
872 helloworldSub := &query.Substring{Pattern: "helloworld"}
873 haystackCap, err := query.Parse("hay(s(t))(a)ck")
874 if err != nil {
875 b.Fatal(err)
876 }
877
878 haystackNonCap, err := query.Parse("hay(?:s(?:t))(?:a)ck")
879 if err != nil {
880 b.Fatal(err)
881 }
882
883 setAnd := func(q query.Q) func() query.Q {
884 return func() query.Q {
885 return query.NewAnd(query.NewSingleBranchesRepos("head", repoSetIDs...), q)
886 }
887 }
888
889 search := func(b *testing.B, q query.Q, wantFiles int) {
890 b.Helper()
891
892 res, err := ss.Search(ctx, q, opts)
893 if err != nil {
894 b.Fatalf("Search(%s): %v", q, err)
895 }
896 if have := len(res.Files); have != wantFiles {
897 b.Fatalf("wrong number of file results. have=%d, want=%d", have, wantFiles)
898 }
899 }
900
901 benchmarks := []struct {
902 name string
903 q func() query.Q
904 wantFiles int
905 }{
906 {"substring all results", func() query.Q { return haystackSub }, len(repos) * filesPerRepo},
907 {"substring no results", func() query.Q { return helloworldSub }, 0},
908 {"substring some results", func() query.Q { return needleSub }, len(repos)},
909
910 {"regexp all results capture", func() query.Q { return haystackCap }, len(repos) * filesPerRepo},
911 {"regexp all results non-capture", func() query.Q { return haystackNonCap }, len(repos) * filesPerRepo},
912
913 {"substring all results and repo set", setAnd(haystackSub), len(repoSetIDs) * filesPerRepo},
914 {"substring some results and repo set", setAnd(needleSub), len(repoSetIDs)},
915 {"substring no results and repo set", setAnd(helloworldSub), 0},
916 }
917
918 for _, bb := range benchmarks {
919 b.Run(bb.name, func(b *testing.B) {
920 b.ReportAllocs()
921
922 for n := 0; n < b.N; n++ {
923 q := bb.q()
924
925 search(b, q, bb.wantFiles)
926 }
927 })
928 }
929}
930
931func TestRawQuerySearch(t *testing.T) {
932 ss := newShardedSearcher(1)
933
934 var nextShardNum int
935 addShard := func(repo string, rawConfig map[string]string, docs ...index.Document) {
936 r := &zoekt.Repository{Name: repo}
937 r.RawConfig = rawConfig
938 b := testShardBuilder(t, r, docs...)
939 shard := searcherForTest(t, b)
940 ss.replace(map[string]zoekt.Searcher{fmt.Sprintf("key-%d", nextShardNum): shard})
941 nextShardNum++
942 }
943 addShard("public", map[string]string{"public": "1"}, index.Document{Name: "f1", Content: []byte("foo bar bas")})
944 addShard("private_archived", map[string]string{"archived": "1"}, index.Document{Name: "f2", Content: []byte("foo bas")})
945 addShard("public_fork", map[string]string{"public": "1", "fork": "1"}, index.Document{Name: "f3", Content: []byte("foo bar")})
946
947 cases := []struct {
948 pattern string
949 flags query.RawConfig
950 wantRepos []string
951 wantFiles int
952 }{
953 {
954 pattern: "bas",
955 flags: query.RcOnlyPublic,
956 wantRepos: []string{"public"},
957 wantFiles: 1,
958 },
959 {
960 pattern: "foo",
961 flags: query.RcOnlyPublic,
962 wantRepos: []string{"public", "public_fork"},
963 wantFiles: 2,
964 },
965 {
966 pattern: "foo",
967 flags: query.RcOnlyPublic | query.RcNoForks,
968 wantRepos: []string{"public"},
969 wantFiles: 1,
970 },
971 {
972 pattern: "bar",
973 flags: query.RcOnlyForks,
974 wantRepos: []string{"public_fork"},
975 wantFiles: 1,
976 },
977 {
978 pattern: "bas",
979 flags: query.RcNoArchived,
980 wantRepos: []string{"public"},
981 wantFiles: 1,
982 },
983 {
984 pattern: "foo",
985 flags: query.RcNoForks,
986 wantRepos: []string{"public", "private_archived"},
987 wantFiles: 2,
988 },
989 {
990 pattern: "bas",
991 flags: query.RcOnlyArchived,
992 wantRepos: []string{"private_archived"},
993 wantFiles: 1,
994 },
995 {
996 pattern: "foo",
997 flags: query.RcOnlyPrivate,
998 wantRepos: []string{"private_archived"},
999 wantFiles: 1,
1000 },
1001 {
1002 pattern: "foo",
1003 flags: query.RcOnlyPrivate | query.RcNoArchived,
1004 wantRepos: []string{},
1005 wantFiles: 0,
1006 },
1007 }
1008 for _, c := range cases {
1009 t.Run(fmt.Sprintf("pattern:%s", c.pattern), func(t *testing.T) {
1010 q := query.NewAnd(&query.Substring{Pattern: c.pattern}, c.flags)
1011
1012 sr, err := ss.Search(context.Background(), q, &zoekt.SearchOptions{})
1013 if err != nil {
1014 t.Fatal(err)
1015 }
1016
1017 if got := len(sr.Files); got != c.wantFiles {
1018 t.Fatalf("wanted %d, got %d", c.wantFiles, got)
1019 }
1020
1021 if c.wantFiles == 0 {
1022 return
1023 }
1024
1025 gotRepos := make([]string, 0, len(sr.RepoURLs))
1026 for k := range sr.RepoURLs {
1027 gotRepos = append(gotRepos, k)
1028 }
1029 sort.Strings(gotRepos)
1030 sort.Strings(c.wantRepos)
1031 if d := cmp.Diff(c.wantRepos, gotRepos); d != "" {
1032 t.Fatalf("(-want, +got):\n%s", d)
1033 }
1034 })
1035 }
1036}
1037
1038func TestPrioritySlice(t *testing.T) {
1039 p := &prioritySlice{}
1040 for step, oper := range []struct {
1041 isAppend bool
1042 value float64
1043 expectedMax float64
1044 }{
1045 {true, 1, 1},
1046 {true, 3, 3},
1047 {true, 2, 3},
1048 {false, 1, 3},
1049 {false, 3, 2},
1050 {false, 2, math.Inf(-1)},
1051 } {
1052 if oper.isAppend {
1053 p.append(oper.value)
1054 } else {
1055 p.remove(oper.value)
1056 }
1057 max := p.max()
1058 if max != oper.expectedMax {
1059 t.Errorf("%d: got %f, want %f", step, max, oper.expectedMax)
1060 }
1061 }
1062}
1063
1064func TestSendByRepository(t *testing.T) {
1065 wantStats := zoekt.Stats{}
1066 wantStats.ShardsScanned = 1
1067
1068 // n1, n2, n3 are the number of file matches for each of the 3 repositories in this
1069 // test.
1070 f := func(n1, n2, n3 uint8) bool {
1071 sr := createMockSearchResult(n1, n2, n3, wantStats)
1072
1073 mock := &mockSender{}
1074 sendByRepository(sr, &zoekt.SearchOptions{}, mock)
1075
1076 if diff := cmp.Diff(wantStats, mock.stats); diff != "" {
1077 t.Logf("-want,+got\n%s", diff)
1078 return false
1079 }
1080
1081 nonZero := 0
1082 for _, l := range []uint8{n1, n2, n3} {
1083 if l > 0 {
1084 nonZero++
1085 }
1086 }
1087 if l := len(mock.files); l != nonZero {
1088 t.Logf("wanted results from %d repositores, got %d", nonZero, l)
1089 return false
1090 }
1091
1092 gotTotal := 0
1093 for _, fs := range mock.files {
1094 gotTotal += len(fs)
1095 }
1096 wantTotal := int(n1) + int(n2) + int(n3)
1097 if gotTotal != wantTotal {
1098 t.Logf("wanted %d file matches, got %d", wantTotal, gotTotal)
1099 return false
1100 }
1101
1102 for _, fs := range mock.files {
1103 if len(fs) == 0 {
1104 t.Logf("got search result with 0 file matches after split")
1105 return false
1106 }
1107 }
1108 return true
1109 }
1110
1111 if err := quick.Check(f, nil); err != nil {
1112 t.Error(err)
1113 }
1114}
1115
1116type mockSender struct {
1117 stats zoekt.Stats
1118 files [][]zoekt.FileMatch
1119}
1120
1121func (s *mockSender) Send(sr *zoekt.SearchResult) {
1122 s.stats.Add(sr.Stats)
1123 if len(sr.Files) == 0 {
1124 return
1125 }
1126 s.files = append(s.files, sr.Files)
1127}
1128
1129func createMockSearchResult(n1, n2, n3 uint8, stats zoekt.Stats) *zoekt.SearchResult {
1130 sr := &zoekt.SearchResult{RepoURLs: make(map[string]string)}
1131 for i, n := range []uint8{n1, n2, n3} {
1132 if n == 0 {
1133 continue
1134 }
1135 tmp := mkSearchResult(int(n), uint32(i))
1136 sr.Files = append(sr.Files, tmp.Files...)
1137 for k := range tmp.RepoURLs {
1138 sr.RepoURLs[k] = ""
1139 }
1140 }
1141 sr.Stats = stats
1142 return sr
1143}
1144
1145func mkSearchResult(n int, repoID uint32) *zoekt.SearchResult {
1146 if n == 0 {
1147 return &zoekt.SearchResult{}
1148 }
1149 fm := make([]zoekt.FileMatch, 0, n)
1150 for range n {
1151 fm = append(fm, zoekt.FileMatch{Repository: fmt.Sprintf("repo%d", repoID), RepositoryID: repoID})
1152 }
1153
1154 return &zoekt.SearchResult{Files: fm, RepoURLs: map[string]string{fmt.Sprintf("repo%d", repoID): ""}}
1155}
1156
1157func TestFileBasedSearch(t *testing.T) {
1158 cases := []struct {
1159 name string
1160 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1161 }{
1162 {"Search", testShardedSearch},
1163 {"StreamSearch", testShardedStreamSearch},
1164 }
1165
1166 c1 := []byte("I love bananas without skin")
1167 // -----------0123456789012345678901234567890123456789
1168 c2 := []byte("In Dutch, ananas means pineapple")
1169 // -----------0123456789012345678901234567890123456789
1170 b := testShardBuilder(t, nil,
1171 index.Document{Name: "f1", Content: c1},
1172 index.Document{Name: "f2", Content: c2},
1173 )
1174
1175 for _, tt := range cases {
1176 for _, useDocumentRanks := range []bool{false, true} {
1177 t.Run(tt.name, func(t *testing.T) {
1178 matches := tt.testShardedSearch(t, &query.Substring{
1179 CaseSensitive: false,
1180 Pattern: "ananas",
1181 }, b, useDocumentRanks)
1182
1183 if len(matches) != 2 {
1184 t.Fatalf("got %v, want 2 matches", matches)
1185 }
1186 if matches[0].FileName != "f2" || matches[1].FileName != "f1" {
1187 t.Fatalf("got %v, want matches {f1,f2}", matches)
1188 }
1189 if matches[0].LineMatches[0].LineFragments[0].Offset != 10 || matches[1].LineMatches[0].LineFragments[0].Offset != 8 {
1190 t.Fatalf("got %#v, want offsets 10,8", matches)
1191 }
1192 })
1193 }
1194 }
1195}
1196
1197func TestWordBoundaryRanking(t *testing.T) {
1198 cases := []struct {
1199 name string
1200 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1201 }{
1202 {"Search", testShardedSearch},
1203 {"StreamSearch", testShardedStreamSearch},
1204 }
1205
1206 b := testShardBuilder(t, nil,
1207 index.Document{Name: "f1", Content: []byte("xbytex xbytex")},
1208 index.Document{Name: "f2", Content: []byte("xbytex\nbytex\nbyte bla")},
1209 // -----------------------------------------0123456 789012 34567890
1210 index.Document{Name: "f3", Content: []byte("xbytex ybytex")})
1211
1212 for _, tt := range cases {
1213 for _, useDocumentRanks := range []bool{false, true} {
1214 t.Run(tt.name, func(t *testing.T) {
1215 files := tt.testShardedSearch(t, &query.Substring{Pattern: "byte"}, b, useDocumentRanks)
1216
1217 if len(files) != 3 {
1218 t.Fatalf("got %#v, want 3 files", files)
1219 }
1220
1221 file0 := files[0]
1222 if file0.FileName != "f2" || len(file0.LineMatches) != 3 {
1223 t.Fatalf("got file %s, num matches %d (%#v), want 3 matches in file f2", file0.FileName, len(file0.LineMatches), file0)
1224 }
1225
1226 if file0.LineMatches[0].LineFragments[0].Offset != 13 {
1227 t.Fatalf("got first match %#v, want full word match", files[0].LineMatches[0])
1228 }
1229 if file0.LineMatches[1].LineFragments[0].Offset != 7 {
1230 t.Fatalf("got second match %#v, want partial word match", files[0].LineMatches[1])
1231 }
1232 })
1233 }
1234 }
1235}
1236
1237func TestAtomCountScore(t *testing.T) {
1238 cases := []struct {
1239 name string
1240 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1241 }{
1242 {"Search", testShardedSearch},
1243 {"StreamSearch", testShardedStreamSearch},
1244 }
1245
1246 b := testShardBuilder(t,
1247 &zoekt.Repository{
1248 Branches: []zoekt.RepositoryBranch{
1249 {Name: "branches", Version: "v1"},
1250 {Name: "needle", Version: "v2"},
1251 },
1252 },
1253 index.Document{Name: "f1", Content: []byte("needle the bla"), Branches: []string{"branches"}},
1254 index.Document{Name: "needle-file-branch", Content: []byte("needle content"), Branches: []string{"needle"}},
1255 index.Document{Name: "needle-file", Content: []byte("needle content"), Branches: []string{"branches"}})
1256
1257 for _, tt := range cases {
1258 for _, useDocumentRanks := range []bool{false, true} {
1259 t.Run(tt.name, func(t *testing.T) {
1260 files := tt.testShardedSearch(t,
1261 query.NewOr(
1262 &query.Substring{Pattern: "needle"},
1263 &query.Substring{Pattern: "needle", FileName: true},
1264 &query.Branch{Pattern: "needle"},
1265 ), b, useDocumentRanks)
1266 var got []string
1267 for _, f := range files {
1268 got = append(got, f.FileName)
1269 }
1270 want := []string{"needle-file-branch", "needle-file", "f1"}
1271 if !reflect.DeepEqual(got, want) {
1272 t.Errorf("got %v, want %v", got, want)
1273 }
1274 })
1275 }
1276 }
1277}
1278
1279func TestUseBM25Scoring(t *testing.T) {
1280 b := testShardBuilder(t,
1281 &zoekt.Repository{},
1282 index.Document{Name: "f1", Content: []byte("one two two three")},
1283 index.Document{Name: "f2", Content: []byte("one two one two")},
1284 index.Document{Name: "f3", Content: []byte("one three three three")})
1285
1286 ss := newShardedSearcher(1)
1287 searcher := searcherForTest(t, b)
1288 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1289
1290 q := query.NewOr(
1291 &query.Substring{Pattern: "one"},
1292 &query.Substring{Pattern: "three"})
1293
1294 opts := zoekt.SearchOptions{
1295 UseBM25Scoring: true,
1296 }
1297
1298 results, err := ss.Search(context.Background(), q, &opts)
1299 if err != nil {
1300 t.Fatal(err)
1301 }
1302
1303 var got []string
1304 for _, f := range results.Files {
1305 got = append(got, f.FileName)
1306 }
1307
1308 want := []string{"f3", "f1", "f2"}
1309 if !reflect.DeepEqual(got, want) {
1310 t.Errorf("got %v, want %v", got, want)
1311 }
1312}
1313
1314func testShardedStreamSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1315 ss := newShardedSearcher(1)
1316 searcher := searcherForTest(t, ib)
1317 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1318
1319 var files []zoekt.FileMatch
1320 sender := zoekt.SenderFunc(func(result *zoekt.SearchResult) {
1321 files = append(files, result.Files...)
1322 })
1323
1324 opts := zoekt.SearchOptions{}
1325 if useDocumentRanks {
1326 opts.FlushWallTime = 10 * time.Millisecond
1327 }
1328 if err := ss.StreamSearch(context.Background(), q, &opts, sender); err != nil {
1329 t.Fatal(err)
1330 }
1331 return files
1332}
1333
1334func testShardedSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1335 ss := newShardedSearcher(1)
1336 searcher := searcherForTest(t, ib)
1337 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1338
1339 opts := zoekt.SearchOptions{}
1340 if useDocumentRanks {
1341 opts.FlushWallTime = 50 * time.Millisecond
1342 }
1343 sres, _ := ss.Search(context.Background(), q, &opts)
1344 return sres.Files
1345}
1346
1347// Ensure we work on empty shard directories.
1348func TestNewDirectorySearcher_empty(t *testing.T) {
1349 ctx := context.Background()
1350
1351 test := func(t *testing.T, ss zoekt.Streamer) {
1352 res, err := ss.Search(ctx, &query.Const{Value: true}, nil)
1353 if err != nil {
1354 t.Fatal("Search non-nil error", err)
1355 }
1356
1357 if diff := cmp.Diff(&zoekt.SearchResult{}, res, cmpopts.IgnoreFields(zoekt.Stats{}, "Duration", "Wait"), cmpopts.EquateEmpty()); diff != "" {
1358 t.Fatalf("Search had non empty results (-want, +got):\n%s", diff)
1359 }
1360
1361 rl, err := ss.List(ctx, &query.Const{Value: true}, nil)
1362 if err != nil {
1363 t.Fatal("List non-nil error", err)
1364 }
1365 if diff := cmp.Diff(&zoekt.RepoList{}, rl, cmpopts.EquateEmpty()); diff != "" {
1366 t.Fatalf("List had non empty results (-want, +got):\n%s", diff)
1367 }
1368 }
1369
1370 dir := t.TempDir()
1371 t.Run("blocking", func(t *testing.T) {
1372 ss, err := NewDirectorySearcher(dir)
1373 if err != nil {
1374 t.Fatal(err)
1375 }
1376 t.Cleanup(ss.Close)
1377
1378 // We expect crashes to be empty as soon as NewDirectorySearcher returns
1379 // so we can validate straight away.
1380 test(t, ss)
1381 })
1382
1383 t.Run("fast", func(t *testing.T) {
1384 ss, err := NewDirectorySearcherFast(dir)
1385 if err != nil {
1386 t.Fatal(err)
1387 }
1388 t.Cleanup(ss.Close)
1389
1390 deadline := testDeadline(t, 10*time.Second)
1391
1392 // Wait for scanning of directory to be done. We should be returning
1393 // non-zero crashes until then.
1394 waitForPredicate(deadline, 10*time.Millisecond, func() bool {
1395 res, err := ss.Search(ctx, &query.Const{Value: true}, nil)
1396 if err != nil {
1397 t.Fatal(err)
1398 }
1399 return res.Stats.Crashes == 0
1400 })
1401
1402 test(t, ss)
1403 })
1404}
1405
1406// testDeadline returns the deadline for t, but ensures it is no longer than
1407// maxTimeout away.
1408func testDeadline(t *testing.T, maxTimeout time.Duration) time.Time {
1409 deadline := time.Now().Add(maxTimeout)
1410 if d, ok := t.Deadline(); ok && d.Before(deadline) {
1411 // give 1s for us to do a final test run
1412 deadline = d.Add(-time.Second)
1413 }
1414 return deadline
1415}
1416
1417func waitForPredicate(deadline time.Time, tick time.Duration, pred func() bool) bool {
1418 for time.Now().Before(deadline) {
1419 if pred() {
1420 return true
1421 }
1422
1423 time.Sleep(tick)
1424 }
1425
1426 return pred()
1427}