fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package search
16
17import (
18 "bytes"
19 "context"
20 "flag"
21 "fmt"
22 "hash/fnv"
23 "io"
24 "log"
25 "math"
26 "os"
27 "reflect"
28 "runtime"
29 "sort"
30 "strconv"
31 "testing"
32 "testing/quick"
33 "time"
34
35 "github.com/RoaringBitmap/roaring"
36 "github.com/google/go-cmp/cmp"
37 "github.com/google/go-cmp/cmp/cmpopts"
38 "github.com/grafana/regexp"
39 sglog "github.com/sourcegraph/log"
40
41 "github.com/sourcegraph/zoekt/index"
42
43 "github.com/sourcegraph/zoekt"
44 "github.com/sourcegraph/zoekt/query"
45)
46
47func TestMain(m *testing.M) {
48 flag.Parse()
49 if !testing.Verbose() {
50 log.SetOutput(io.Discard)
51 }
52 os.Exit(m.Run())
53}
54
55type crashSearcher struct{}
56
57func (s *crashSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
58 panic("search")
59}
60
61func (s *crashSearcher) List(ctx context.Context, q query.Q, opts *zoekt.ListOptions) (*zoekt.RepoList, error) {
62 panic("list")
63}
64
65func (s *crashSearcher) Stats() (*zoekt.RepoStats, error) {
66 return &zoekt.RepoStats{}, nil
67}
68
69func (s *crashSearcher) Close() {}
70
71func (s *crashSearcher) String() string { return "crashSearcher" }
72
73func TestCrashResilience(t *testing.T) {
74 out := &bytes.Buffer{}
75 oldOut := log.Writer()
76 log.SetOutput(out)
77 defer log.SetOutput(oldOut)
78
79 oldShardRecoveryLogger := shardRecoveryLogger
80 shardRecoveryLogger = sglog.NoOp
81 defer func() { shardRecoveryLogger = oldShardRecoveryLogger }()
82
83 ss := newShardedSearcher(2)
84 ss.ranked.Store([]*rankedShard{{Searcher: &crashSearcher{}}})
85
86 var wantCrashes int
87 test := func(t *testing.T) {
88 q := &query.Substring{Pattern: "hoi"}
89 opts := &zoekt.SearchOptions{}
90 if res, err := ss.Search(context.Background(), q, opts); err != nil {
91 t.Fatalf("Search: %v", err)
92 } else if res.Stats.Crashes != wantCrashes {
93 t.Errorf("got stats %#v, want crashes = %d", res.Stats, wantCrashes)
94 }
95
96 if res, err := ss.List(context.Background(), q, nil); err != nil {
97 t.Fatalf("List: %v", err)
98 } else if res.Crashes != wantCrashes {
99 t.Errorf("got result %#v, want crashes = %d", res, wantCrashes)
100 }
101 }
102
103 // Before we are marked as ready we have one extra crash
104 wantCrashes = 2
105 t.Run("loading", test)
106
107 // After marking as ready we should only have the crashSearcher
108 // contributing.
109 ss.markReady()
110 wantCrashes = 1
111 t.Run("ready", test)
112}
113
114type rankSearcher struct {
115 rank uint16
116 repo *zoekt.Repository
117}
118
119func (s *rankSearcher) Close() {
120}
121
122func (s *rankSearcher) String() string {
123 return ""
124}
125
126func (s *rankSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) {
127 select {
128 case <-ctx.Done():
129 return &zoekt.SearchResult{}, nil
130 default:
131 }
132
133 // Ugly, but without sleep it's too fast, and we can't
134 // simulate the cutoff.
135 time.Sleep(time.Millisecond)
136 return &zoekt.SearchResult{
137 Files: []zoekt.FileMatch{
138 {
139 FileName: fmt.Sprintf("f%d", s.rank),
140 Score: float64(s.rank),
141 },
142 },
143 Stats: zoekt.Stats{
144 MatchCount: 1,
145 },
146 }, nil
147}
148
149func (s *rankSearcher) List(ctx context.Context, q query.Q, opts *zoekt.ListOptions) (*zoekt.RepoList, error) {
150 r := zoekt.Repository{}
151 if s.repo != nil {
152 r = *s.repo
153 }
154 r.Rank = s.rank
155 return &zoekt.RepoList{
156 Repos: []*zoekt.RepoListEntry{
157 {Repository: r},
158 },
159 }, nil
160}
161
162func (s *rankSearcher) Repository() *zoekt.Repository { return s.repo }
163
164func TestOrderByShard(t *testing.T) {
165 ss := newShardedSearcher(1)
166
167 n := 10 * runtime.GOMAXPROCS(0)
168 for i := range n {
169 ss.replace(map[string]zoekt.Searcher{
170 fmt.Sprintf("shard%d", i): &rankSearcher{rank: uint16(i)},
171 })
172 }
173
174 if res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &zoekt.SearchOptions{}); err != nil {
175 t.Errorf("Search: %v", err)
176 } else if len(res.Files) != n {
177 t.Fatalf("empty options: got %d results, want %d", len(res.Files), n)
178 }
179
180 opts := zoekt.SearchOptions{
181 TotalMaxMatchCount: 3,
182 }
183 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &opts)
184 if err != nil {
185 t.Errorf("Search: %v", err)
186 }
187
188 if len(res.Files) < opts.TotalMaxMatchCount {
189 t.Errorf("got %d results, want %d", len(res.Files), opts.TotalMaxMatchCount)
190 }
191 if len(res.Files) == n {
192 t.Errorf("got %d results, want < %d", len(res.Files), n)
193 }
194 for i, f := range res.Files {
195 rev := n - 1 - i
196 want := fmt.Sprintf("f%d", rev)
197 got := f.FileName
198
199 if got != want {
200 t.Logf("%d: got %q, want %q", i, got, want)
201 }
202 }
203}
204
205func TestShardedSearcher_Ranking(t *testing.T) {
206 ss := newShardedSearcher(1)
207
208 var nextShardNum int
209 addShard := func(repo string, priority float64, docs ...index.Document) {
210 r := &zoekt.Repository{ID: hash(repo), Name: repo}
211 r.RawConfig = map[string]string{
212 "public": "1",
213 "priority": strconv.FormatFloat(priority, 'f', 2, 64),
214 }
215 b := testShardBuilder(t, r, docs...)
216 shard := searcherForTest(t, b)
217 ss.replace(map[string]zoekt.Searcher{
218 fmt.Sprintf("key-%d", nextShardNum): shard,
219 })
220 nextShardNum++
221 }
222
223 addShard("weekend-project", 20, index.Document{Name: "f2", Content: []byte("foo bas")})
224 addShard("moderately-popular", 500, index.Document{Name: "f3", Content: []byte("foo bar")})
225 addShard("weekend-project-2", 20, index.Document{Name: "f2", Content: []byte("foo bas")})
226 addShard("super-star", 5000, index.Document{Name: "f1", Content: []byte("foo bar bas")})
227
228 want := []string{
229 "super-star",
230 "moderately-popular",
231 "weekend-project",
232 "weekend-project-2",
233 }
234
235 var have []string
236 for _, s := range ss.getLoaded().shards {
237 for _, r := range s.repos {
238 have = append(have, r.Name)
239 }
240 }
241
242 if !reflect.DeepEqual(want, have) {
243 t.Fatalf("\nwant: %s\nhave: %s", want, have)
244 }
245}
246
247func TestShardedSearcher_DocumentRanking(t *testing.T) {
248 ss := newShardedSearcher(1)
249
250 var nextShardNum int
251 addShard := func(repo string, rank uint16, docs ...index.Document) {
252 r := &zoekt.Repository{ID: hash(repo), Name: repo}
253 r.RawConfig = map[string]string{
254 "public": "1",
255 }
256 r.Rank = rank
257 b := testShardBuilder(t, r, docs...)
258 shard := searcherForTest(t, b)
259 ss.replace(map[string]zoekt.Searcher{
260 fmt.Sprintf("key-%d", nextShardNum): shard,
261 })
262 nextShardNum++
263 }
264
265 addShard("old-project", 1, index.Document{Name: "f1", Content: []byte("foobar")})
266 addShard("recent", 2, index.Document{Name: "f2", Content: []byte("foobaz")})
267 addShard("old-project-2", 1, index.Document{Name: "f3", Content: []byte("foo bar")})
268 addShard("new", 3, index.Document{Name: "f4", Content: []byte("foo baz")},
269 index.Document{Name: "f5", Content: []byte("fooooo")})
270
271 // Run a stream search and gather the results
272 var results []*zoekt.SearchResult
273 opts := &zoekt.SearchOptions{
274 FlushWallTime: 100 * time.Millisecond,
275 }
276
277 err := ss.StreamSearch(context.Background(), &query.Substring{Pattern: "foo"}, opts,
278 zoekt.SenderFunc(func(event *zoekt.SearchResult) {
279 results = append(results, event)
280 }))
281 if err != nil {
282 t.Fatal(err)
283 }
284
285 // There should always be two stream results, first progress-only, then the file results
286 if len(results) != 2 {
287 t.Fatalf("expected 2 streamed results, but got %d", len(results))
288 }
289
290 // The ranking should be determined by whether it's an exact word match,
291 // followed by repository priority
292 want := []string{"f4", "f3", "f5", "f2", "f1"}
293
294 files := results[1].Files
295 got := make([]string, len(files))
296 for i := range files {
297 got[i] = files[i].FileName
298 }
299
300 if !reflect.DeepEqual(got, want) {
301 t.Errorf("got %v, want %v", got, want)
302 }
303}
304
305func TestFilteringShardsByRepoSetOrBranchesReposOrRepoIDs(t *testing.T) {
306 ss := newShardedSearcher(1)
307
308 // namePrefix is so we can create a repo:foo filter and match the same set
309 // of repos.
310 namePrefix := [3]string{"foo", "bar", "baz"}
311
312 repoSetNames := []string{}
313 repoIDs := []uint32{}
314 n := 10 * runtime.GOMAXPROCS(0)
315 for i := range n {
316 shardName := fmt.Sprintf("shard%d", i)
317 repoName := fmt.Sprintf("%s-repository%.3d", namePrefix[i%3], i)
318 repoID := hash(repoName)
319
320 if i%3 == 0 {
321 repoSetNames = append(repoSetNames, repoName)
322 repoIDs = append(repoIDs, repoID)
323 }
324
325 ss.replace(map[string]zoekt.Searcher{
326 shardName: &rankSearcher{
327 repo: &zoekt.Repository{ID: repoID, Name: repoName},
328 rank: uint16(n - i),
329 },
330 })
331 }
332
333 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "bla"}, &zoekt.SearchOptions{})
334 if err != nil {
335 t.Errorf("Search: %v", err)
336 }
337 if len(res.Files) != n {
338 t.Fatalf("no reposet: got %d results, want %d", len(res.Files), n)
339 }
340
341 branchesRepos := &query.BranchesRepos{List: []query.BranchRepos{
342 {Branch: "HEAD", Repos: roaring.New()},
343 }}
344
345 for _, name := range repoSetNames {
346 branchesRepos.List[0].Repos.Add(hash(name))
347 }
348
349 set := query.NewRepoSet(repoSetNames...)
350 sub := &query.Substring{Pattern: "bla"}
351
352 repoIDsQuery := query.NewRepoIDs(repoIDs...)
353 repoQuery := &query.Repo{Regexp: regexp.MustCompile("^foo-.*")}
354
355 queries := []query.Q{
356 query.NewAnd(set, sub),
357 // Test with the same reposet again
358 query.NewAnd(set, sub),
359
360 query.NewAnd(branchesRepos, sub),
361 // Test with the same repoBranches with IDs again
362 query.NewAnd(branchesRepos, sub),
363
364 query.NewAnd(repoIDsQuery, sub),
365 // Test with the same repoIDs again
366 query.NewAnd(repoIDsQuery, sub),
367
368 query.NewAnd(repoQuery, sub),
369 query.NewAnd(repoQuery, sub),
370
371 // List has queries which are just the reposet atoms. We also test twice.
372 set,
373 set,
374 branchesRepos,
375 branchesRepos,
376 repoIDsQuery,
377 repoIDsQuery,
378 repoQuery,
379 repoQuery,
380 }
381
382 for _, q := range queries {
383 res, err = ss.Search(context.Background(), q, &zoekt.SearchOptions{})
384 if err != nil {
385 t.Errorf("Search(%s): %v", q, err)
386 }
387 // Note: Assertion is based on fact that `rankSearcher` always returns a
388 // result and using repoSet will half the number of results
389 if len(res.Files) != len(repoSetNames) {
390 t.Fatalf("%s: got %d results, want %d", q, len(res.Files), len(repoSetNames))
391 }
392 }
393}
394
395func TestFilteringShardsByMeta(t *testing.T) {
396 ss := newShardedSearcher(1)
397
398 // Create repos with different metadata values
399 // We'll create 30 repos total:
400 // - 10 with nickname="project-A"
401 // - 10 with nickname="project-B"
402 // - 10 with no metadata
403 n := 30
404 projectARepos := []string{}
405 projectBRepos := []string{}
406
407 // Common document that will be in all repos
408 doc := index.Document{
409 Name: "common.go",
410 Content: []byte("needle haystack"),
411 }
412
413 for i := range n {
414 shardName := fmt.Sprintf("shard%d", i)
415 repoName := fmt.Sprintf("repository%.3d", i)
416
417 var metadata map[string]string
418 if i < 10 {
419 // First 10 repos have project-A
420 metadata = map[string]string{"nickname": "project-A", "visibility": "public"}
421 projectARepos = append(projectARepos, repoName)
422 } else if i < 20 {
423 // Next 10 repos have project-B
424 metadata = map[string]string{"nickname": "project-B", "visibility": "private"}
425 projectBRepos = append(projectBRepos, repoName)
426 }
427 // Last 10 repos have no metadata
428
429 repo := &zoekt.Repository{
430 ID: uint32(i + 1),
431 Name: repoName,
432 URL: repoName,
433 Metadata: metadata,
434 }
435
436 ss.replace(map[string]zoekt.Searcher{
437 shardName: searcherForTest(t, testShardBuilder(t, repo, doc)),
438 })
439 }
440
441 // Test 1: Search without Meta filter - should search all shards
442 res, err := ss.Search(context.Background(), &query.Substring{Pattern: "needle"}, &zoekt.SearchOptions{})
443 if err != nil {
444 t.Fatalf("Search without filter: %v", err)
445 }
446 if len(res.Files) != n {
447 t.Fatalf("no meta filter: got %d results, want %d", len(res.Files), n)
448 }
449
450 sub := &query.Substring{Pattern: "needle"}
451
452 // Helper function to extract unique repo names from search results
453 getRepoNames := func(files []zoekt.FileMatch) []string {
454 repoSet := make(map[string]struct{})
455 for _, f := range files {
456 repoSet[f.Repository] = struct{}{}
457 }
458 repos := make([]string, 0, len(repoSet))
459 for repo := range repoSet {
460 repos = append(repos, repo)
461 }
462 sort.Strings(repos)
463 return repos
464 }
465
466 // Test 2: Filter by nickname="project-A" - should only search 10 shards
467 metaQueryA := &query.Meta{
468 Field: "nickname",
469 Value: regexp.MustCompile("^project-A$"),
470 }
471 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryA, sub), &zoekt.SearchOptions{})
472 if err != nil {
473 t.Fatalf("Search with Meta filter A: %v", err)
474 }
475 gotRepos := getRepoNames(res.Files)
476 wantRepos := append([]string{}, projectARepos...)
477 sort.Strings(wantRepos)
478 if !reflect.DeepEqual(gotRepos, wantRepos) {
479 t.Fatalf("Meta(nickname=project-A):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
480 }
481
482 // Test 3: Filter by nickname="project-B" - should only search 10 shards
483 metaQueryB := &query.Meta{
484 Field: "nickname",
485 Value: regexp.MustCompile("^project-B$"),
486 }
487 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryB, sub), &zoekt.SearchOptions{})
488 if err != nil {
489 t.Fatalf("Search with Meta filter B: %v", err)
490 }
491 gotRepos = getRepoNames(res.Files)
492 wantRepos = append([]string{}, projectBRepos...)
493 sort.Strings(wantRepos)
494 if !reflect.DeepEqual(gotRepos, wantRepos) {
495 t.Fatalf("Meta(nickname=project-B):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
496 }
497
498 // Test 4: Filter by visibility="public" - should only search 10 shards (project-A repos)
499 metaQueryPublic := &query.Meta{
500 Field: "visibility",
501 Value: regexp.MustCompile("^public$"),
502 }
503 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryPublic, sub), &zoekt.SearchOptions{})
504 if err != nil {
505 t.Fatalf("Search with Meta filter public: %v", err)
506 }
507 gotRepos = getRepoNames(res.Files)
508 wantRepos = append([]string{}, projectARepos...)
509 sort.Strings(wantRepos)
510 if !reflect.DeepEqual(gotRepos, wantRepos) {
511 t.Fatalf("Meta(visibility=public):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
512 }
513
514 // Test 5: Filter by non-existent field - should return 0 results
515 metaQueryNonExistent := &query.Meta{
516 Field: "nonexistent_field",
517 Value: regexp.MustCompile(".*"),
518 }
519 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryNonExistent, sub), &zoekt.SearchOptions{})
520 if err != nil {
521 t.Fatalf("Search with Meta filter non-existent: %v", err)
522 }
523 if len(res.Files) != 0 {
524 t.Fatalf("Meta(nonexistent_field): got %d results, want 0", len(res.Files))
525 }
526
527 // Test 6: Filter by regex pattern matching multiple values
528 metaQueryRegex := &query.Meta{
529 Field: "nickname",
530 Value: regexp.MustCompile("project-.*"), // Matches both project-A and project-B
531 }
532 res, err = ss.Search(context.Background(), query.NewAnd(metaQueryRegex, sub), &zoekt.SearchOptions{})
533 if err != nil {
534 t.Fatalf("Search with Meta regex filter: %v", err)
535 }
536 gotRepos = getRepoNames(res.Files)
537 wantRepos = append(append([]string{}, projectARepos...), projectBRepos...)
538 sort.Strings(wantRepos)
539 if !reflect.DeepEqual(gotRepos, wantRepos) {
540 t.Fatalf("Meta(nickname=project-.*):\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
541 }
542
543 // Test 7: Test that Meta query alone (without content search) works
544 res, err = ss.Search(context.Background(), metaQueryA, &zoekt.SearchOptions{})
545 if err != nil {
546 t.Fatalf("Search with Meta query alone: %v", err)
547 }
548 gotRepos = getRepoNames(res.Files)
549 wantRepos = append([]string{}, projectARepos...)
550 sort.Strings(wantRepos)
551 if !reflect.DeepEqual(gotRepos, wantRepos) {
552 t.Fatalf("Meta query alone:\ngot repos: %v\nwant repos: %v", gotRepos, wantRepos)
553 }
554
555 // Test 8: Test with List operation (not just Search)
556 listRes, err := ss.List(context.Background(), metaQueryA, nil)
557 if err != nil {
558 t.Fatalf("List with Meta filter: %v", err)
559 }
560 gotListRepos := make([]string, len(listRes.Repos))
561 for i, r := range listRes.Repos {
562 gotListRepos[i] = r.Repository.Name
563 }
564 sort.Strings(gotListRepos)
565 wantRepos = append([]string{}, projectARepos...)
566 sort.Strings(wantRepos)
567 if !reflect.DeepEqual(gotListRepos, wantRepos) {
568 t.Fatalf("List with Meta(nickname=project-A):\ngot repos: %v\nwant repos: %v", gotListRepos, wantRepos)
569 }
570}
571
572func hash(name string) uint32 {
573 h := fnv.New32()
574 h.Write([]byte(name))
575 return h.Sum32()
576}
577
578type memSeeker struct {
579 data []byte
580}
581
582func (s *memSeeker) Name() string {
583 return "memseeker"
584}
585
586func (s *memSeeker) Close() {}
587func (s *memSeeker) Read(off, sz uint32) ([]byte, error) {
588 return s.data[off : off+sz], nil
589}
590
591func (s *memSeeker) Size() (uint32, error) {
592 return uint32(len(s.data)), nil
593}
594
595func TestUnloadIndex(t *testing.T) {
596 b := testShardBuilder(t, nil, index.Document{
597 Name: "filename",
598 Content: []byte("needle needle needle"),
599 })
600
601 var buf bytes.Buffer
602 if err := b.Write(&buf); err != nil {
603 t.Fatal(err)
604 }
605 indexBytes := buf.Bytes()
606 indexFile := &memSeeker{indexBytes}
607 searcher, err := index.NewSearcher(indexFile)
608 if err != nil {
609 t.Fatalf("NewSearcher: %v", err)
610 }
611
612 ss := newShardedSearcher(2)
613 ss.replace(map[string]zoekt.Searcher{"key": searcher})
614
615 var opts zoekt.SearchOptions
616 q := &query.Substring{Pattern: "needle"}
617 res, err := ss.Search(context.Background(), q, &opts)
618 if err != nil {
619 t.Fatalf("Search(%s): %v", q, err)
620 }
621
622 forbidden := byte(29)
623 for i := range indexBytes {
624 // non-ASCII
625 indexBytes[i] = forbidden
626 }
627
628 for _, f := range res.Files {
629 if bytes.Contains(f.Content, []byte{forbidden}) {
630 t.Errorf("found %d in content %q", forbidden, f.Content)
631 }
632 if bytes.Contains(f.Checksum, []byte{forbidden}) {
633 t.Errorf("found %d in checksum %q", forbidden, f.Checksum)
634 }
635
636 for _, l := range f.LineMatches {
637 if bytes.Contains(l.Line, []byte{forbidden}) {
638 t.Errorf("found %d in line %q", forbidden, l.Line)
639 }
640 }
641 }
642}
643
644func TestShardedSearcher_List(t *testing.T) {
645 repos := []*zoekt.Repository{
646 {
647 ID: 1234,
648 Name: "repo-a",
649 URL: "url-a",
650 Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
651 RawConfig: map[string]string{"repoid": "1234"},
652 },
653 {
654 Name: "repo-b",
655 URL: "url-b",
656 Branches: []zoekt.RepositoryBranch{{Name: "main"}, {Name: "dev"}},
657 },
658 }
659
660 doc := index.Document{
661 Name: "foo.go",
662 Content: []byte("bar\nbaz"),
663 Branches: []string{"main", "dev"},
664 }
665
666 // Test duplicate removal when ListOptions.Minimal is true and false
667 ss := newShardedSearcher(4)
668 ss.replace(map[string]zoekt.Searcher{
669 "1": searcherForTest(t, testShardBuilder(t, repos[0], doc)),
670 "2": searcherForTest(t, testShardBuilder(t, repos[0])),
671 "3": searcherForTest(t, testShardBuilder(t, repos[1], doc)),
672 "4": searcherForTest(t, testShardBuilder(t, repos[1])),
673 })
674 ss.markReady()
675
676 stats := zoekt.RepoStats{
677 Shards: 2,
678 Documents: 1,
679 IndexBytes: 196,
680 ContentBytes: 13,
681 NewLinesCount: 1,
682 DefaultBranchNewLinesCount: 1,
683 OtherBranchesNewLinesCount: 1,
684 }
685
686 aggStats := stats
687 aggStats.Add(&aggStats) // since both repos have the exact same stats, this works
688 aggStats.Repos = 2 // Add doesn't populate Repos, this is done in Shards based on the response sizes.
689
690 for _, tc := range []struct {
691 name string
692 opts *zoekt.ListOptions
693 want *zoekt.RepoList
694 }{
695 {
696 name: "nil opts",
697 opts: nil,
698 want: &zoekt.RepoList{
699 Repos: []*zoekt.RepoListEntry{
700 {
701 Repository: *repos[0],
702 Stats: stats,
703 },
704 {
705 Repository: *repos[1],
706 Stats: stats,
707 },
708 },
709 Stats: aggStats,
710 },
711 },
712 {
713 name: "default",
714 opts: &zoekt.ListOptions{},
715 want: &zoekt.RepoList{
716 Repos: []*zoekt.RepoListEntry{
717 {
718 Repository: *repos[0],
719 Stats: stats,
720 },
721 {
722 Repository: *repos[1],
723 Stats: stats,
724 },
725 },
726 Stats: aggStats,
727 },
728 },
729 {
730 name: "field=repos",
731 opts: &zoekt.ListOptions{Field: zoekt.RepoListFieldRepos},
732 want: &zoekt.RepoList{
733 Repos: []*zoekt.RepoListEntry{
734 {
735 Repository: *repos[0],
736 Stats: stats,
737 },
738 {
739 Repository: *repos[1],
740 Stats: stats,
741 },
742 },
743 Stats: aggStats,
744 },
745 },
746 {
747 name: "field=reposmap",
748 opts: &zoekt.ListOptions{Field: zoekt.RepoListFieldReposMap},
749 want: &zoekt.RepoList{
750 Repos: []*zoekt.RepoListEntry{
751 {
752 Repository: *repos[1],
753 Stats: stats,
754 },
755 },
756 ReposMap: zoekt.ReposMap{
757 repos[0].ID: {
758 HasSymbols: repos[0].HasSymbols,
759 Branches: repos[0].Branches,
760 },
761 },
762 Stats: aggStats,
763 },
764 },
765 } {
766 tc := tc
767 t.Run(tc.name, func(t *testing.T) {
768 t.Parallel()
769
770 q := &query.Repo{Regexp: regexp.MustCompile("repo")}
771
772 res, err := ss.List(context.Background(), q, tc.opts)
773 if err != nil {
774 t.Fatalf("List(%v, %s): %v", q, tc.opts, err)
775 }
776
777 sort.Slice(res.Repos, func(i, j int) bool {
778 return res.Repos[i].Repository.Name < res.Repos[j].Repository.Name
779 })
780
781 ignored := []cmp.Option{
782 cmpopts.EquateEmpty(),
783 cmpopts.IgnoreFields(zoekt.MinimalRepoListEntry{}, "IndexTimeUnix"),
784 cmpopts.IgnoreFields(zoekt.RepoListEntry{}, "IndexMetadata"),
785 cmpopts.IgnoreFields(zoekt.RepoStats{}, "IndexBytes"),
786 cmpopts.IgnoreFields(zoekt.Repository{}, "SubRepoMap"),
787 cmpopts.IgnoreFields(zoekt.Repository{}, "priority"),
788 }
789
790 if diff := cmp.Diff(tc.want, res, ignored...); diff != "" {
791 t.Fatalf("mismatch (-want +got):\n%s", diff)
792 }
793 })
794 }
795}
796
797func testShardBuilder(t testing.TB, repo *zoekt.Repository, docs ...index.Document) *index.ShardBuilder {
798 b, err := index.NewShardBuilder(repo)
799 if err != nil {
800 t.Fatalf("NewShardBuilder: %v", err)
801 }
802
803 for i, d := range docs {
804 if err := b.Add(d); err != nil {
805 t.Fatalf("Add %d: %v", i, err)
806 }
807 }
808 return b
809}
810
811func searcherForTest(t testing.TB, b *index.ShardBuilder) zoekt.Searcher {
812 var buf bytes.Buffer
813 if err := b.Write(&buf); err != nil {
814 t.Fatal(err)
815 }
816 f := &memSeeker{buf.Bytes()}
817
818 searcher, err := index.NewSearcher(f)
819 if err != nil {
820 t.Fatalf("NewSearcher: %v", err)
821 }
822
823 return searcher
824}
825
826func reposForTest(n int) (result []*zoekt.Repository) {
827 for i := range n {
828 result = append(result, &zoekt.Repository{
829 ID: uint32(i + 1),
830 Name: fmt.Sprintf("test-repository-%d", i),
831 })
832 }
833 return result
834}
835
836func testSearcherForRepo(b testing.TB, r *zoekt.Repository, numFiles int) zoekt.Searcher {
837 builder := testShardBuilder(b, r)
838
839 if err := builder.Add(index.Document{
840 Name: fmt.Sprintf("%s/filename-%d.go", r.Name, 0),
841 Content: []byte("needle needle needle haystack"),
842 }); err != nil {
843 b.Fatal(err)
844 }
845
846 for i := 1; i < numFiles; i++ {
847 if err := builder.Add(index.Document{
848 Name: fmt.Sprintf("%s/filename-%d.go", r.Name, i),
849 Content: []byte("haystack haystack haystack"),
850 }); err != nil {
851 b.Fatal(err)
852 }
853 }
854
855 return searcherForTest(b, builder)
856}
857
858func BenchmarkShardedSearch(b *testing.B) {
859 ss := newShardedSearcher(int64(runtime.GOMAXPROCS(0)))
860
861 filesPerRepo := 300
862 repos := reposForTest(3000)
863 var repoSetIDs []uint32
864
865 shards := make(map[string]zoekt.Searcher, len(repos))
866 for i, r := range repos {
867 shards[r.Name] = testSearcherForRepo(b, r, filesPerRepo)
868 if i%2 == 0 {
869 repoSetIDs = append(repoSetIDs, r.ID)
870 }
871 }
872
873 ss.replace(shards)
874
875 ctx := context.Background()
876 opts := &zoekt.SearchOptions{}
877
878 needleSub := &query.Substring{Pattern: "needle"}
879 haystackSub := &query.Substring{Pattern: "haystack"}
880 helloworldSub := &query.Substring{Pattern: "helloworld"}
881 haystackCap, err := query.Parse("hay(s(t))(a)ck")
882 if err != nil {
883 b.Fatal(err)
884 }
885
886 haystackNonCap, err := query.Parse("hay(?:s(?:t))(?:a)ck")
887 if err != nil {
888 b.Fatal(err)
889 }
890
891 setAnd := func(q query.Q) func() query.Q {
892 return func() query.Q {
893 return query.NewAnd(query.NewSingleBranchesRepos("head", repoSetIDs...), q)
894 }
895 }
896
897 search := func(b *testing.B, q query.Q, wantFiles int) {
898 b.Helper()
899
900 res, err := ss.Search(ctx, q, opts)
901 if err != nil {
902 b.Fatalf("Search(%s): %v", q, err)
903 }
904 if have := len(res.Files); have != wantFiles {
905 b.Fatalf("wrong number of file results. have=%d, want=%d", have, wantFiles)
906 }
907 }
908
909 benchmarks := []struct {
910 name string
911 q func() query.Q
912 wantFiles int
913 }{
914 {"substring all results", func() query.Q { return haystackSub }, len(repos) * filesPerRepo},
915 {"substring no results", func() query.Q { return helloworldSub }, 0},
916 {"substring some results", func() query.Q { return needleSub }, len(repos)},
917
918 {"regexp all results capture", func() query.Q { return haystackCap }, len(repos) * filesPerRepo},
919 {"regexp all results non-capture", func() query.Q { return haystackNonCap }, len(repos) * filesPerRepo},
920
921 {"substring all results and repo set", setAnd(haystackSub), len(repoSetIDs) * filesPerRepo},
922 {"substring some results and repo set", setAnd(needleSub), len(repoSetIDs)},
923 {"substring no results and repo set", setAnd(helloworldSub), 0},
924 }
925
926 for _, bb := range benchmarks {
927 b.Run(bb.name, func(b *testing.B) {
928 b.ReportAllocs()
929
930 for n := 0; n < b.N; n++ {
931 q := bb.q()
932
933 search(b, q, bb.wantFiles)
934 }
935 })
936 }
937}
938
939func TestRawQuerySearch(t *testing.T) {
940 ss := newShardedSearcher(1)
941
942 var nextShardNum int
943 addShard := func(repo string, rawConfig map[string]string, docs ...index.Document) {
944 r := &zoekt.Repository{Name: repo, URL: repo}
945 r.RawConfig = rawConfig
946 b := testShardBuilder(t, r, docs...)
947 shard := searcherForTest(t, b)
948 ss.replace(map[string]zoekt.Searcher{fmt.Sprintf("key-%d", nextShardNum): shard})
949 nextShardNum++
950 }
951 addShard("public", map[string]string{"public": "1"}, index.Document{Name: "f1", Content: []byte("foo bar bas")})
952 addShard("private_archived", map[string]string{"archived": "1"}, index.Document{Name: "f2", Content: []byte("foo bas")})
953 addShard("public_fork", map[string]string{"public": "1", "fork": "1"}, index.Document{Name: "f3", Content: []byte("foo bar")})
954
955 cases := []struct {
956 pattern string
957 flags query.RawConfig
958 wantRepos []string
959 wantFiles int
960 }{
961 {
962 pattern: "bas",
963 flags: query.RcOnlyPublic,
964 wantRepos: []string{"public"},
965 wantFiles: 1,
966 },
967 {
968 pattern: "foo",
969 flags: query.RcOnlyPublic,
970 wantRepos: []string{"public", "public_fork"},
971 wantFiles: 2,
972 },
973 {
974 pattern: "foo",
975 flags: query.RcOnlyPublic | query.RcNoForks,
976 wantRepos: []string{"public"},
977 wantFiles: 1,
978 },
979 {
980 pattern: "bar",
981 flags: query.RcOnlyForks,
982 wantRepos: []string{"public_fork"},
983 wantFiles: 1,
984 },
985 {
986 pattern: "bas",
987 flags: query.RcNoArchived,
988 wantRepos: []string{"public"},
989 wantFiles: 1,
990 },
991 {
992 pattern: "foo",
993 flags: query.RcNoForks,
994 wantRepos: []string{"public", "private_archived"},
995 wantFiles: 2,
996 },
997 {
998 pattern: "bas",
999 flags: query.RcOnlyArchived,
1000 wantRepos: []string{"private_archived"},
1001 wantFiles: 1,
1002 },
1003 {
1004 pattern: "foo",
1005 flags: query.RcOnlyPrivate,
1006 wantRepos: []string{"private_archived"},
1007 wantFiles: 1,
1008 },
1009 {
1010 pattern: "foo",
1011 flags: query.RcOnlyPrivate | query.RcNoArchived,
1012 wantRepos: []string{},
1013 wantFiles: 0,
1014 },
1015 }
1016 for _, c := range cases {
1017 t.Run(fmt.Sprintf("pattern:%s", c.pattern), func(t *testing.T) {
1018 q := query.NewAnd(&query.Substring{Pattern: c.pattern}, c.flags)
1019
1020 sr, err := ss.Search(context.Background(), q, &zoekt.SearchOptions{})
1021 if err != nil {
1022 t.Fatal(err)
1023 }
1024
1025 if got := len(sr.Files); got != c.wantFiles {
1026 t.Fatalf("wanted %d, got %d", c.wantFiles, got)
1027 }
1028
1029 if c.wantFiles == 0 {
1030 return
1031 }
1032
1033 gotRepos := make([]string, 0, len(sr.RepoURLs))
1034 for k := range sr.RepoURLs {
1035 gotRepos = append(gotRepos, k)
1036 }
1037 sort.Strings(gotRepos)
1038 sort.Strings(c.wantRepos)
1039 if d := cmp.Diff(c.wantRepos, gotRepos); d != "" {
1040 t.Fatalf("(-want, +got):\n%s", d)
1041 }
1042 })
1043 }
1044}
1045
1046func TestPrioritySlice(t *testing.T) {
1047 p := &prioritySlice{}
1048 for step, oper := range []struct {
1049 isAppend bool
1050 value float64
1051 expectedMax float64
1052 }{
1053 {true, 1, 1},
1054 {true, 3, 3},
1055 {true, 2, 3},
1056 {false, 1, 3},
1057 {false, 3, 2},
1058 {false, 2, math.Inf(-1)},
1059 } {
1060 if oper.isAppend {
1061 p.append(oper.value)
1062 } else {
1063 p.remove(oper.value)
1064 }
1065 max := p.max()
1066 if max != oper.expectedMax {
1067 t.Errorf("%d: got %f, want %f", step, max, oper.expectedMax)
1068 }
1069 }
1070}
1071
1072func TestSendByRepository(t *testing.T) {
1073 wantStats := zoekt.Stats{}
1074 wantStats.ShardsScanned = 1
1075
1076 // n1, n2, n3 are the number of file matches for each of the 3 repositories in this
1077 // test.
1078 f := func(n1, n2, n3 uint8) bool {
1079 sr := createMockSearchResult(n1, n2, n3, wantStats)
1080
1081 mock := &mockSender{}
1082 sendByRepository(sr, &zoekt.SearchOptions{}, mock)
1083
1084 if diff := cmp.Diff(wantStats, mock.stats); diff != "" {
1085 t.Logf("-want,+got\n%s", diff)
1086 return false
1087 }
1088
1089 nonZero := 0
1090 for _, l := range []uint8{n1, n2, n3} {
1091 if l > 0 {
1092 nonZero++
1093 }
1094 }
1095 if l := len(mock.files); l != nonZero {
1096 t.Logf("wanted results from %d repositores, got %d", nonZero, l)
1097 return false
1098 }
1099
1100 gotTotal := 0
1101 for _, fs := range mock.files {
1102 gotTotal += len(fs)
1103 }
1104 wantTotal := int(n1) + int(n2) + int(n3)
1105 if gotTotal != wantTotal {
1106 t.Logf("wanted %d file matches, got %d", wantTotal, gotTotal)
1107 return false
1108 }
1109
1110 for _, fs := range mock.files {
1111 if len(fs) == 0 {
1112 t.Logf("got search result with 0 file matches after split")
1113 return false
1114 }
1115 }
1116 return true
1117 }
1118
1119 if err := quick.Check(f, nil); err != nil {
1120 t.Error(err)
1121 }
1122}
1123
1124type mockSender struct {
1125 stats zoekt.Stats
1126 files [][]zoekt.FileMatch
1127}
1128
1129func (s *mockSender) Send(sr *zoekt.SearchResult) {
1130 s.stats.Add(sr.Stats)
1131 if len(sr.Files) == 0 {
1132 return
1133 }
1134 s.files = append(s.files, sr.Files)
1135}
1136
1137func createMockSearchResult(n1, n2, n3 uint8, stats zoekt.Stats) *zoekt.SearchResult {
1138 sr := &zoekt.SearchResult{RepoURLs: make(map[string]string)}
1139 for i, n := range []uint8{n1, n2, n3} {
1140 if n == 0 {
1141 continue
1142 }
1143 tmp := mkSearchResult(int(n), uint32(i))
1144 sr.Files = append(sr.Files, tmp.Files...)
1145 for k := range tmp.RepoURLs {
1146 sr.RepoURLs[k] = ""
1147 }
1148 }
1149 sr.Stats = stats
1150 return sr
1151}
1152
1153func mkSearchResult(n int, repoID uint32) *zoekt.SearchResult {
1154 if n == 0 {
1155 return &zoekt.SearchResult{}
1156 }
1157 fm := make([]zoekt.FileMatch, 0, n)
1158 for range n {
1159 fm = append(fm, zoekt.FileMatch{Repository: fmt.Sprintf("repo%d", repoID), RepositoryID: repoID})
1160 }
1161
1162 return &zoekt.SearchResult{Files: fm, RepoURLs: map[string]string{fmt.Sprintf("repo%d", repoID): ""}}
1163}
1164
1165func TestFileBasedSearch(t *testing.T) {
1166 cases := []struct {
1167 name string
1168 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1169 }{
1170 {"Search", testShardedSearch},
1171 {"StreamSearch", testShardedStreamSearch},
1172 }
1173
1174 c1 := []byte("I love bananas without skin")
1175 // -----------0123456789012345678901234567890123456789
1176 c2 := []byte("In Dutch, ananas means pineapple")
1177 // -----------0123456789012345678901234567890123456789
1178 b := testShardBuilder(t, nil,
1179 index.Document{Name: "f1", Content: c1},
1180 index.Document{Name: "f2", Content: c2},
1181 )
1182
1183 for _, tt := range cases {
1184 for _, useDocumentRanks := range []bool{false, true} {
1185 t.Run(tt.name, func(t *testing.T) {
1186 matches := tt.testShardedSearch(t, &query.Substring{
1187 CaseSensitive: false,
1188 Pattern: "ananas",
1189 }, b, useDocumentRanks)
1190
1191 if len(matches) != 2 {
1192 t.Fatalf("got %v, want 2 matches", matches)
1193 }
1194 if matches[0].FileName != "f2" || matches[1].FileName != "f1" {
1195 t.Fatalf("got %v, want matches {f1,f2}", matches)
1196 }
1197 if matches[0].LineMatches[0].LineFragments[0].Offset != 10 || matches[1].LineMatches[0].LineFragments[0].Offset != 8 {
1198 t.Fatalf("got %#v, want offsets 10,8", matches)
1199 }
1200 })
1201 }
1202 }
1203}
1204
1205func TestWordBoundaryRanking(t *testing.T) {
1206 cases := []struct {
1207 name string
1208 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1209 }{
1210 {"Search", testShardedSearch},
1211 {"StreamSearch", testShardedStreamSearch},
1212 }
1213
1214 b := testShardBuilder(t, nil,
1215 index.Document{Name: "f1", Content: []byte("xbytex xbytex")},
1216 index.Document{Name: "f2", Content: []byte("xbytex\nbytex\nbyte bla")},
1217 // -----------------------------------------0123456 789012 34567890
1218 index.Document{Name: "f3", Content: []byte("xbytex ybytex")})
1219
1220 for _, tt := range cases {
1221 for _, useDocumentRanks := range []bool{false, true} {
1222 t.Run(tt.name, func(t *testing.T) {
1223 files := tt.testShardedSearch(t, &query.Substring{Pattern: "byte"}, b, useDocumentRanks)
1224
1225 if len(files) != 3 {
1226 t.Fatalf("got %#v, want 3 files", files)
1227 }
1228
1229 file0 := files[0]
1230 if file0.FileName != "f2" || len(file0.LineMatches) != 3 {
1231 t.Fatalf("got file %s, num matches %d (%#v), want 3 matches in file f2", file0.FileName, len(file0.LineMatches), file0)
1232 }
1233
1234 if file0.LineMatches[0].LineFragments[0].Offset != 13 {
1235 t.Fatalf("got first match %#v, want full word match", files[0].LineMatches[0])
1236 }
1237 if file0.LineMatches[1].LineFragments[0].Offset != 7 {
1238 t.Fatalf("got second match %#v, want partial word match", files[0].LineMatches[1])
1239 }
1240 })
1241 }
1242 }
1243}
1244
1245func TestAtomCountScore(t *testing.T) {
1246 cases := []struct {
1247 name string
1248 testShardedSearch func(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch
1249 }{
1250 {"Search", testShardedSearch},
1251 {"StreamSearch", testShardedStreamSearch},
1252 }
1253
1254 b := testShardBuilder(t,
1255 &zoekt.Repository{
1256 Branches: []zoekt.RepositoryBranch{
1257 {Name: "branches", Version: "v1"},
1258 {Name: "needle", Version: "v2"},
1259 },
1260 },
1261 index.Document{Name: "f1", Content: []byte("needle the bla"), Branches: []string{"branches"}},
1262 index.Document{Name: "needle-file-branch", Content: []byte("needle content"), Branches: []string{"needle"}},
1263 index.Document{Name: "needle-file", Content: []byte("needle content"), Branches: []string{"branches"}})
1264
1265 for _, tt := range cases {
1266 for _, useDocumentRanks := range []bool{false, true} {
1267 t.Run(tt.name, func(t *testing.T) {
1268 files := tt.testShardedSearch(t,
1269 query.NewOr(
1270 &query.Substring{Pattern: "needle"},
1271 &query.Substring{Pattern: "needle", FileName: true},
1272 &query.Branch{Pattern: "needle"},
1273 ), b, useDocumentRanks)
1274 var got []string
1275 for _, f := range files {
1276 got = append(got, f.FileName)
1277 }
1278 want := []string{"needle-file-branch", "needle-file", "f1"}
1279 if !reflect.DeepEqual(got, want) {
1280 t.Errorf("got %v, want %v", got, want)
1281 }
1282 })
1283 }
1284 }
1285}
1286
1287func TestUseBM25Scoring(t *testing.T) {
1288 b := testShardBuilder(t,
1289 &zoekt.Repository{},
1290 index.Document{Name: "f1", Content: []byte("one two two three")},
1291 index.Document{Name: "f2", Content: []byte("one two one two")},
1292 index.Document{Name: "f3", Content: []byte("one three three three")})
1293
1294 ss := newShardedSearcher(1)
1295 searcher := searcherForTest(t, b)
1296 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1297
1298 q := query.NewOr(
1299 &query.Substring{Pattern: "one"},
1300 &query.Substring{Pattern: "three"})
1301
1302 opts := zoekt.SearchOptions{
1303 UseBM25Scoring: true,
1304 }
1305
1306 results, err := ss.Search(context.Background(), q, &opts)
1307 if err != nil {
1308 t.Fatal(err)
1309 }
1310
1311 var got []string
1312 for _, f := range results.Files {
1313 got = append(got, f.FileName)
1314 }
1315
1316 want := []string{"f3", "f1", "f2"}
1317 if !reflect.DeepEqual(got, want) {
1318 t.Errorf("got %v, want %v", got, want)
1319 }
1320}
1321
1322func testShardedStreamSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1323 ss := newShardedSearcher(1)
1324 searcher := searcherForTest(t, ib)
1325 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1326
1327 var files []zoekt.FileMatch
1328 sender := zoekt.SenderFunc(func(result *zoekt.SearchResult) {
1329 files = append(files, result.Files...)
1330 })
1331
1332 opts := zoekt.SearchOptions{}
1333 if useDocumentRanks {
1334 opts.FlushWallTime = 10 * time.Millisecond
1335 }
1336 if err := ss.StreamSearch(context.Background(), q, &opts, sender); err != nil {
1337 t.Fatal(err)
1338 }
1339 return files
1340}
1341
1342func testShardedSearch(t *testing.T, q query.Q, ib *index.ShardBuilder, useDocumentRanks bool) []zoekt.FileMatch {
1343 ss := newShardedSearcher(1)
1344 searcher := searcherForTest(t, ib)
1345 ss.replace(map[string]zoekt.Searcher{"r1": searcher})
1346
1347 opts := zoekt.SearchOptions{}
1348 if useDocumentRanks {
1349 opts.FlushWallTime = 50 * time.Millisecond
1350 }
1351 sres, _ := ss.Search(context.Background(), q, &opts)
1352 return sres.Files
1353}
1354
1355// Ensure we work on empty shard directories.
1356func TestNewDirectorySearcher_empty(t *testing.T) {
1357 ctx := context.Background()
1358
1359 test := func(t *testing.T, ss zoekt.Streamer) {
1360 res, err := ss.Search(ctx, &query.Const{Value: true}, nil)
1361 if err != nil {
1362 t.Fatal("Search non-nil error", err)
1363 }
1364
1365 if diff := cmp.Diff(&zoekt.SearchResult{}, res, cmpopts.IgnoreFields(zoekt.Stats{}, "Duration", "Wait"), cmpopts.EquateEmpty()); diff != "" {
1366 t.Fatalf("Search had non empty results (-want, +got):\n%s", diff)
1367 }
1368
1369 rl, err := ss.List(ctx, &query.Const{Value: true}, nil)
1370 if err != nil {
1371 t.Fatal("List non-nil error", err)
1372 }
1373 if diff := cmp.Diff(&zoekt.RepoList{}, rl, cmpopts.EquateEmpty()); diff != "" {
1374 t.Fatalf("List had non empty results (-want, +got):\n%s", diff)
1375 }
1376 }
1377
1378 dir := t.TempDir()
1379 t.Run("blocking", func(t *testing.T) {
1380 ss, err := NewDirectorySearcher(dir)
1381 if err != nil {
1382 t.Fatal(err)
1383 }
1384 t.Cleanup(ss.Close)
1385
1386 // We expect crashes to be empty as soon as NewDirectorySearcher returns
1387 // so we can validate straight away.
1388 test(t, ss)
1389 })
1390
1391 t.Run("fast", func(t *testing.T) {
1392 ss, err := NewDirectorySearcherFast(dir)
1393 if err != nil {
1394 t.Fatal(err)
1395 }
1396 t.Cleanup(ss.Close)
1397
1398 deadline := testDeadline(t, 10*time.Second)
1399
1400 // Wait for scanning of directory to be done. We should be returning
1401 // non-zero crashes until then.
1402 waitForPredicate(deadline, 10*time.Millisecond, func() bool {
1403 res, err := ss.Search(ctx, &query.Const{Value: true}, nil)
1404 if err != nil {
1405 t.Fatal(err)
1406 }
1407 return res.Stats.Crashes == 0
1408 })
1409
1410 test(t, ss)
1411 })
1412}
1413
1414// testDeadline returns the deadline for t, but ensures it is no longer than
1415// maxTimeout away.
1416func testDeadline(t *testing.T, maxTimeout time.Duration) time.Time {
1417 deadline := time.Now().Add(maxTimeout)
1418 if d, ok := t.Deadline(); ok && d.Before(deadline) {
1419 // give 1s for us to do a final test run
1420 deadline = d.Add(-time.Second)
1421 }
1422 return deadline
1423}
1424
1425func waitForPredicate(deadline time.Time, tick time.Duration, pred func() bool) bool {
1426 for time.Now().Before(deadline) {
1427 if pred() {
1428 return true
1429 }
1430
1431 time.Sleep(tick)
1432 }
1433
1434 return pred()
1435}