fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "log"
23 "os"
24 "path/filepath"
25 "reflect"
26 "runtime"
27 "sort"
28 "strings"
29 "testing"
30 "time"
31
32 "github.com/google/go-cmp/cmp"
33 "github.com/google/go-cmp/cmp/cmpopts"
34 "github.com/grafana/regexp"
35 "github.com/sourcegraph/zoekt"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/shards"
38)
39
40func TestBasic(t *testing.T) {
41 dir := t.TempDir()
42
43 opts := Options{
44 IndexDir: dir,
45 ShardMax: 1024,
46 RepositoryDescription: zoekt.Repository{
47 Name: "repo",
48 },
49 Parallelism: 2,
50 SizeMax: 1 << 20,
51 }
52
53 b, err := NewBuilder(opts)
54 if err != nil {
55 t.Fatalf("NewBuilder: %v", err)
56 }
57
58 for i := 0; i < 4; i++ {
59 s := fmt.Sprintf("%d", i)
60 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1000))); err != nil {
61 t.Fatal(err)
62 }
63 }
64
65 if err := b.Finish(); err != nil {
66 t.Errorf("Finish: %v", err)
67 }
68
69 fs, _ := filepath.Glob(dir + "/*.zoekt")
70 if len(fs) <= 1 {
71 t.Fatalf("want multiple shards, got %v", fs)
72 }
73
74 _, md0, err := zoekt.ReadMetadataPath(fs[0])
75 if err != nil {
76 t.Fatal(err)
77 }
78 for _, f := range fs[1:] {
79 _, md, err := zoekt.ReadMetadataPath(f)
80 if err != nil {
81 t.Fatal(err)
82 }
83 if md.IndexTime != md0.IndexTime {
84 t.Fatalf("wanted identical time stamps but got %v!=%v", md.IndexTime, md0.IndexTime)
85 }
86 if md.ID != md0.ID {
87 t.Fatalf("wanted identical IDs but got %s!=%s", md.ID, md0.ID)
88 }
89 }
90
91 ss, err := shards.NewDirectorySearcher(dir)
92 if err != nil {
93 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
94 }
95 defer ss.Close()
96
97 q, err := query.Parse("111")
98 if err != nil {
99 t.Fatalf("Parse(111): %v", err)
100 }
101
102 var sOpts zoekt.SearchOptions
103 ctx := context.Background()
104 result, err := ss.Search(ctx, q, &sOpts)
105 if err != nil {
106 t.Fatalf("Search(%v): %v", q, err)
107 }
108
109 if len(result.Files) != 1 {
110 t.Errorf("got %v, want 1 file.", result.Files)
111 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
112 t.Errorf("got file %q, want %q", gotFile, wantFile)
113 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo"; gotRepo != wantRepo {
114 t.Errorf("got repo %q, want %q", gotRepo, wantRepo)
115 }
116
117 t.Run("meta file", func(t *testing.T) {
118 // use retryTest to allow for the directory watcher to notice the meta
119 // file
120 retryTest(t, func(fatalf func(format string, args ...interface{})) {
121 // Add a .meta file for each shard with repo.Name set to
122 // "repo-mutated". We do this inside retry helper since we have noticed
123 // some flakiness on github CI.
124 for _, p := range fs {
125 repos, _, err := zoekt.ReadMetadataPath(p)
126 if err != nil {
127 t.Fatal(err)
128 }
129 repos[0].Name = "repo-mutated"
130 b, err := json.Marshal(repos[0])
131 if err != nil {
132 t.Fatal(err)
133 }
134
135 if err := os.WriteFile(p+".meta", b, 0600); err != nil {
136 t.Fatal(err)
137 }
138 }
139
140 result, err := ss.Search(ctx, q, &sOpts)
141 if err != nil {
142 fatalf("Search(%v): %v", q, err)
143 }
144
145 if len(result.Files) != 1 {
146 fatalf("got %v, want 1 file.", result.Files)
147 } else if gotFile, wantFile := result.Files[0].FileName, "F1"; gotFile != wantFile {
148 fatalf("got file %q, want %q", gotFile, wantFile)
149 } else if gotRepo, wantRepo := result.Files[0].Repository, "repo-mutated"; gotRepo != wantRepo {
150 fatalf("got repo %q, want %q", gotRepo, wantRepo)
151 }
152 })
153 })
154}
155
156// retryTest will retry f until min(t.Deadline(), time.Minute). It returns
157// once f doesn't call fatalf.
158func retryTest(t *testing.T, f func(fatalf func(format string, args ...interface{}))) {
159 t.Helper()
160
161 sleep := 10 * time.Millisecond
162 deadline := time.Now().Add(time.Minute)
163 if d, ok := t.Deadline(); ok && d.Before(deadline) {
164 // give 1s for us to do a final test run
165 deadline = d.Add(-time.Second)
166 }
167
168 for {
169 done := make(chan bool)
170 go func() {
171 defer close(done)
172
173 f(func(format string, args ...interface{}) {
174 runtime.Goexit()
175 })
176
177 done <- true
178 }()
179
180 success := <-done
181 if success {
182 return
183 }
184
185 // each time we increase sleep by 1.5
186 sleep := sleep*2 - sleep/2
187 if time.Now().Add(sleep).After(deadline) {
188 break
189 }
190 time.Sleep(sleep)
191 }
192
193 // final run for the test, using the real t.Fatalf
194 f(t.Fatalf)
195}
196
197func TestLargeFileOption(t *testing.T) {
198 dir := t.TempDir()
199
200 sizeMax := 1000
201 opts := Options{
202 IndexDir: dir,
203 LargeFiles: []string{"F0", "F1", "F2", "!F1"},
204 RepositoryDescription: zoekt.Repository{
205 Name: "repo",
206 },
207 SizeMax: sizeMax,
208 }
209
210 b, err := NewBuilder(opts)
211 if err != nil {
212 t.Fatalf("NewBuilder: %v", err)
213 }
214
215 for i := 0; i < 4; i++ {
216 s := fmt.Sprintf("%d", i)
217 if err := b.AddFile("F"+s, []byte(strings.Repeat("a", sizeMax+1))); err != nil {
218 t.Fatal(err)
219 }
220 }
221
222 if err := b.Finish(); err != nil {
223 t.Errorf("Finish: %v", err)
224 }
225
226 ss, err := shards.NewDirectorySearcher(dir)
227 if err != nil {
228 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
229 }
230
231 q, err := query.Parse("aaa")
232 if err != nil {
233 t.Fatalf("Parse(aaa): %v", err)
234 }
235
236 var sOpts zoekt.SearchOptions
237 ctx := context.Background()
238 result, err := ss.Search(ctx, q, &sOpts)
239 if err != nil {
240 t.Fatalf("Search(%v): %v", q, err)
241 }
242
243 if len(result.Files) != 2 {
244 t.Errorf("got %v files, want 2 files.", len(result.Files))
245 }
246 defer ss.Close()
247}
248
249func TestUpdate(t *testing.T) {
250 dir := t.TempDir()
251
252 opts := Options{
253 IndexDir: dir,
254 ShardMax: 1024,
255 RepositoryDescription: zoekt.Repository{
256 Name: "repo",
257 FileURLTemplate: "url",
258 },
259 Parallelism: 2,
260 SizeMax: 1 << 20,
261 }
262
263 if b, err := NewBuilder(opts); err != nil {
264 t.Fatalf("NewBuilder: %v", err)
265 } else {
266 if err := b.AddFile("F", []byte("hoi")); err != nil {
267 t.Errorf("AddFile: %v", err)
268 }
269 if err := b.Finish(); err != nil {
270 t.Errorf("Finish: %v", err)
271 }
272 }
273 ss, err := shards.NewDirectorySearcher(dir)
274 if err != nil {
275 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
276 }
277
278 ctx := context.Background()
279 repos, err := ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil)
280 if err != nil {
281 t.Fatalf("List: %v", err)
282 }
283
284 if len(repos.Repos) != 1 {
285 t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
286 }
287
288 fs, err := filepath.Glob(filepath.Join(dir, "*"))
289 if err != nil {
290 t.Fatalf("glob: %v", err)
291 }
292
293 opts.RepositoryDescription = zoekt.Repository{
294 Name: "repo2",
295 FileURLTemplate: "url2",
296 }
297
298 if b, err := NewBuilder(opts); err != nil {
299 t.Fatalf("NewBuilder: %v", err)
300 } else {
301 if err := b.AddFile("F", []byte("hoi")); err != nil {
302 t.Errorf("AddFile: %v", err)
303 }
304 if err := b.Finish(); err != nil {
305 t.Errorf("Finish: %v", err)
306 }
307 }
308
309 // This is ugly, and potentially flaky, but there is no
310 // observable synchronization for the Sharded searcher, so
311 // this is the best we can do.
312 time.Sleep(100 * time.Millisecond)
313
314 ctx = context.Background()
315 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
316 t.Fatalf("List: %v", err)
317 } else if len(repos.Repos) != 2 {
318 t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
319 }
320
321 for _, fn := range fs {
322 log.Printf("removing %s", fn)
323 if err := os.Remove(fn); err != nil {
324 t.Fatalf("Remove(%s): %v", fn, err)
325 }
326 }
327
328 time.Sleep(100 * time.Millisecond)
329
330 ctx = context.Background()
331 if repos, err = ss.List(ctx, &query.Repo{Regexp: regexp.MustCompile("repo")}, nil); err != nil {
332 t.Fatalf("List: %v", err)
333 } else if len(repos.Repos) != 1 {
334 var ss []string
335 for _, r := range repos.Repos {
336 ss = append(ss, r.Repository.Name)
337 }
338 t.Errorf("List(repo): got %v, want 1 repo", ss)
339 }
340}
341
342func TestDeleteOldShards(t *testing.T) {
343 dir := t.TempDir()
344
345 opts := Options{
346 IndexDir: dir,
347 ShardMax: 1024,
348 RepositoryDescription: zoekt.Repository{
349 Name: "repo",
350 FileURLTemplate: "url",
351 },
352 SizeMax: 1 << 20,
353 }
354 opts.SetDefaults()
355
356 b, err := NewBuilder(opts)
357 if err != nil {
358 t.Fatalf("NewBuilder: %v", err)
359 }
360 for i := 0; i < 4; i++ {
361 s := fmt.Sprintf("%d\n", i)
362 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
363 t.Errorf("AddFile: %v", err)
364 }
365 }
366 if err := b.Finish(); err != nil {
367 t.Errorf("Finish: %v", err)
368 }
369
370 glob := filepath.Join(dir, "*.zoekt")
371 fs, err := filepath.Glob(glob)
372 if err != nil {
373 t.Fatalf("Glob(%s): %v", glob, err)
374 } else if len(fs) != 4 {
375 t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
376 }
377
378 if fi, err := os.Lstat(fs[0]); err != nil {
379 t.Fatalf("Lstat: %v", err)
380 } else if fi.Mode()&0o666 == 0o600 {
381 // This fails spuriously if your umask is very restrictive.
382 t.Errorf("got mode %o, should respect umask.", fi.Mode())
383 }
384
385 // Do again, without sharding.
386 opts.ShardMax = 1 << 20
387 b, err = NewBuilder(opts)
388 if err != nil {
389 t.Fatalf("NewBuilder: %v", err)
390 }
391 for i := 0; i < 4; i++ {
392 s := fmt.Sprintf("%d\n", i)
393 if err := b.AddFile("F"+s, []byte(strings.Repeat(s, 1024/2))); err != nil {
394 t.Fatal(err)
395 }
396 }
397 if err := b.Finish(); err != nil {
398 t.Errorf("Finish: %v", err)
399 }
400
401 fs, err = filepath.Glob(glob)
402 if err != nil {
403 t.Fatalf("Glob(%s): %v", glob, err)
404 } else if len(fs) != 1 {
405 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
406 }
407
408 // Again, but don't index anything; should leave old shards intact.
409 b, err = NewBuilder(opts)
410 if err != nil {
411 t.Fatalf("NewBuilder: %v", err)
412 }
413 if err := b.Finish(); err != nil {
414 t.Errorf("Finish: %v", err)
415 }
416
417 fs, err = filepath.Glob(glob)
418 if err != nil {
419 t.Fatalf("Glob(%s): %v", glob, err)
420 } else if len(fs) != 1 {
421 t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
422 }
423}
424
425func TestPartialSuccess(t *testing.T) {
426 dir := t.TempDir()
427
428 opts := Options{
429 IndexDir: dir,
430 ShardMax: 1024,
431 SizeMax: 1 << 20,
432 Parallelism: 1,
433 }
434 opts.RepositoryDescription.Name = "repo"
435 opts.SetDefaults()
436
437 b, err := NewBuilder(opts)
438 if err != nil {
439 t.Fatalf("NewBuilder: %v", err)
440 }
441
442 for i := 0; i < 4; i++ {
443 nm := fmt.Sprintf("F%d", i)
444 _ = b.AddFile(nm, []byte(strings.Repeat("01234567\n", 128)))
445 }
446 b.buildError = fmt.Errorf("any error")
447
448 // No error checking.
449 _ = b.Finish()
450
451 // Finish cleans up temporary files.
452 if fs, err := filepath.Glob(dir + "/*"); err != nil {
453 t.Errorf("glob(%s): %v", dir, err)
454 } else if len(fs) != 0 {
455 t.Errorf("got shards %v, want []", fs)
456 }
457}
458
459type filerankCase struct {
460 name string
461 docs []*zoekt.Document
462 want []int
463}
464
465func testFileRankAspect(t *testing.T, c filerankCase) {
466 var want []*zoekt.Document
467 for _, j := range c.want {
468 want = append(want, c.docs[j])
469 }
470
471 got := make([]*zoekt.Document, len(c.docs))
472 copy(got, c.docs)
473 sortDocuments(got)
474
475 print := func(ds []*zoekt.Document) string {
476 r := ""
477 for _, d := range ds {
478 r += fmt.Sprintf("%v, ", d)
479 }
480 return r
481 }
482 if !reflect.DeepEqual(got, want) {
483 t.Errorf("got docs [%v], want [%v]", print(got), print(want))
484 }
485}
486
487func TestFileRank(t *testing.T) {
488 for _, c := range []filerankCase{{
489 name: "filename",
490 docs: []*zoekt.Document{
491 {
492 Name: "longlonglong",
493 Content: []byte("bla"),
494 },
495 {
496 Name: "short",
497 Content: []byte("bla"),
498 },
499 },
500 want: []int{1, 0},
501 }, {
502 name: "test",
503 docs: []*zoekt.Document{
504 {
505 Name: "test",
506 Content: []byte("bla"),
507 },
508 {
509 Name: "longlonglong",
510 Content: []byte("bla"),
511 },
512 },
513 want: []int{1, 0},
514 }, {
515 name: "content",
516 docs: []*zoekt.Document{
517 {
518 Content: []byte("bla"),
519 },
520 {
521 Content: []byte("blablablabla"),
522 },
523 {
524 Content: []byte("blabla"),
525 },
526 },
527 want: []int{0, 2, 1},
528 }} {
529 t.Run(c.name, func(t *testing.T) {
530 testFileRankAspect(t, c)
531 })
532 }
533}
534
535func TestEmptyContent(t *testing.T) {
536 dir := t.TempDir()
537
538 opts := Options{
539 IndexDir: dir,
540 RepositoryDescription: zoekt.Repository{
541 Name: "repo",
542 },
543 }
544 opts.SetDefaults()
545
546 b, err := NewBuilder(opts)
547 if err != nil {
548 t.Fatalf("NewBuilder: %v", err)
549 }
550 if err := b.Finish(); err != nil {
551 t.Errorf("Finish: %v", err)
552 }
553
554 fs, _ := filepath.Glob(dir + "/*.zoekt")
555 if len(fs) != 1 {
556 t.Fatalf("want a shard, got %v", fs)
557 }
558
559 ss, err := shards.NewDirectorySearcher(dir)
560 if err != nil {
561 t.Fatalf("NewDirectorySearcher(%s): %v", dir, err)
562 }
563 defer ss.Close()
564
565 ctx := context.Background()
566 result, err := ss.List(ctx, &query.Const{Value: true}, nil)
567 if err != nil {
568 t.Fatalf("List: %v", err)
569 }
570
571 if len(result.Repos) != 1 || result.Repos[0].Repository.Name != "repo" {
572 t.Errorf("got %+v, want 1 repo.", result.Repos)
573 }
574}
575
576func TestDeltaShards(t *testing.T) {
577 // TODO: Need to write a test for compound shards as well.
578 type step struct {
579 name string
580 documents []zoekt.Document
581 optFn func(t *testing.T, o *Options)
582
583 query string
584 expectedDocuments []zoekt.Document
585 }
586
587 var (
588 fooAtMain = zoekt.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v1")}
589 fooAtMainV2 = zoekt.Document{Name: "foo.go", Branches: []string{"main"}, Content: []byte("common foo-main-v2")}
590
591 fooAtMainAndRelease = zoekt.Document{Name: "foo.go", Branches: []string{"main", "release"}, Content: []byte("common foo-main-and-release")}
592
593 barAtMain = zoekt.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main")}
594 barAtMainV2 = zoekt.Document{Name: "bar.go", Branches: []string{"main"}, Content: []byte("common bar-main-v2")}
595 )
596
597 for _, test := range []struct {
598 name string
599 steps []step
600 }{
601 {
602 name: "tombstone older documents",
603 steps: []step{
604 {
605 name: "setup",
606 documents: []zoekt.Document{barAtMain, fooAtMain},
607 query: "common",
608 expectedDocuments: []zoekt.Document{barAtMain, fooAtMain},
609 },
610 {
611 name: "add new version of foo, tombstone older ones",
612 documents: []zoekt.Document{fooAtMainV2},
613 optFn: func(t *testing.T, o *Options) {
614 o.IsDelta = true
615 o.changedOrRemovedFiles = []string{"foo.go"}
616 },
617 query: "common",
618 expectedDocuments: []zoekt.Document{barAtMain, fooAtMainV2},
619 },
620 {
621 name: "add new version of bar, tombstone older ones",
622 documents: []zoekt.Document{barAtMainV2},
623 optFn: func(t *testing.T, o *Options) {
624 o.IsDelta = true
625 o.changedOrRemovedFiles = []string{"bar.go"}
626 },
627 query: "common",
628 expectedDocuments: []zoekt.Document{barAtMainV2, fooAtMainV2},
629 },
630 },
631 },
632 {
633 name: "tombstone older documents even if the latest shard has no documents",
634 steps: []step{
635 {
636 name: "setup",
637 documents: []zoekt.Document{barAtMain, fooAtMain},
638 query: "common",
639 expectedDocuments: []zoekt.Document{barAtMain, fooAtMain},
640 },
641 {
642 // a build with no documents could represent a deletion
643 name: "tombstone older documents",
644 documents: nil,
645 optFn: func(t *testing.T, o *Options) {
646 o.IsDelta = true
647 o.changedOrRemovedFiles = []string{"foo.go"}
648 },
649 query: "common",
650 expectedDocuments: []zoekt.Document{barAtMain},
651 },
652 },
653 },
654 {
655 name: "tombstones affect document across branches",
656 steps: []step{
657 {
658 name: "setup",
659 documents: []zoekt.Document{barAtMain, fooAtMainAndRelease},
660 query: "common",
661 expectedDocuments: []zoekt.Document{barAtMain, fooAtMainAndRelease},
662 },
663 {
664
665 name: "tombstone foo",
666 documents: nil,
667 optFn: func(t *testing.T, o *Options) {
668 o.IsDelta = true
669 o.changedOrRemovedFiles = []string{"foo.go"}
670 },
671 query: "common",
672 expectedDocuments: []zoekt.Document{barAtMain},
673 },
674 },
675 },
676 } {
677 t.Run(test.name, func(t *testing.T) {
678 indexDir := t.TempDir()
679
680 branchSet := make(map[string]struct{})
681
682 for _, s := range test.steps {
683 for _, d := range s.documents {
684 for _, b := range d.Branches {
685 branchSet[b] = struct{}{}
686 }
687 }
688 }
689
690 for _, step := range test.steps {
691 repository := zoekt.Repository{ID: 1, Name: "repository"}
692
693 for b := range branchSet {
694 repository.Branches = append(repository.Branches, zoekt.RepositoryBranch{Name: b})
695 }
696
697 sort.Slice(repository.Branches, func(i, j int) bool {
698 a, b := repository.Branches[i], repository.Branches[j]
699
700 return a.Name < b.Name
701 })
702
703 buildOpts := Options{
704 IndexDir: indexDir,
705 RepositoryDescription: repository,
706 }
707 buildOpts.SetDefaults()
708
709 if step.optFn != nil {
710 step.optFn(t, &buildOpts)
711 }
712
713 b, err := NewBuilder(buildOpts)
714 if err != nil {
715 t.Fatalf("step %q: NewBuilder: %s", step.name, err)
716 }
717
718 for _, d := range step.documents {
719 err := b.Add(d)
720 if err != nil {
721 t.Fatalf("step %q: adding document %q to builder: %s", step.name, d.Name, err)
722 }
723 }
724
725 // Call b.Finish() multiple times to ensure that it is idempotent
726 for i := 0; i < 3; i++ {
727
728 err = b.Finish()
729 if err != nil {
730 t.Fatalf("step %q: finishing builder (call #%d): %s", step.name, i, err)
731 }
732 }
733
734 err = b.Finish()
735 if err != nil {
736 t.Fatalf("step %q: finishing builder: %s", step.name, err)
737 }
738
739 state, _ := buildOpts.IndexState()
740 if diff := cmp.Diff(IndexStateEqual, state); diff != "" {
741 t.Errorf("unexpected diff in index state (-want +got):\n%s", diff)
742 }
743
744 ss, err := shards.NewDirectorySearcher(indexDir)
745 if err != nil {
746 t.Fatalf("step %q: NewDirectorySearcher(%s): %s", step.name, indexDir, err)
747 }
748 defer ss.Close()
749
750 searchOpts := &zoekt.SearchOptions{Whole: true}
751 q := &query.Substring{Pattern: step.query}
752
753 result, err := ss.Search(context.Background(), q, searchOpts)
754 if err != nil {
755 t.Fatalf("step %q: Search(%q): %s", step.name, step.query, err)
756 }
757
758 var receivedDocuments []zoekt.Document
759 for _, f := range result.Files {
760 receivedDocuments = append(receivedDocuments, zoekt.Document{
761 Name: f.FileName,
762 Content: f.Content,
763 })
764 }
765
766 cmpOpts := []cmp.Option{
767 cmpopts.IgnoreFields(zoekt.Document{}, "Branches"),
768 cmpopts.SortSlices(func(a, b zoekt.Document) bool {
769 if a.Name < b.Name {
770 return true
771 }
772
773 return bytes.Compare(a.Content, b.Content) < 0
774 }),
775 }
776
777 if diff := cmp.Diff(step.expectedDocuments, receivedDocuments, cmpOpts...); diff != "" {
778 t.Errorf("step %q: diff in received documents (-want +got):%s\n:", step.name, diff)
779 }
780 }
781 })
782 }
783}