fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package zoekt // import "github.com/sourcegraph/zoekt"
16
17import (
18 "bytes"
19 _ "embed"
20 "encoding/gob"
21 "fmt"
22 "math/rand"
23 "reflect"
24 "testing"
25 "testing/quick"
26 "time"
27
28 "github.com/google/go-cmp/cmp"
29 "github.com/google/go-cmp/cmp/cmpopts"
30 webproto "github.com/sourcegraph/zoekt/grpc/protos/zoekt/webserver/v1"
31 "google.golang.org/protobuf/proto"
32
33 fuzz "github.com/AdaLogics/go-fuzz-headers"
34)
35
36func TestProtoRoundtrip(t *testing.T) {
37 t.Run("FileMatch", func(t *testing.T) {
38 f := func(f1 FileMatch) bool {
39 p1 := f1.ToProto()
40 f2 := FileMatchFromProto(p1)
41 return reflect.DeepEqual(f1, f2)
42 }
43 if err := quick.Check(f, nil); err != nil {
44 t.Fatal(err)
45 }
46 })
47
48 t.Run("ChunkMatch", func(t *testing.T) {
49 f := func(f1 ChunkMatch) bool {
50 p1 := f1.ToProto()
51 f2 := ChunkMatchFromProto(p1)
52 return reflect.DeepEqual(f1, f2)
53 }
54 if err := quick.Check(f, nil); err != nil {
55 t.Fatal(err)
56 }
57 })
58
59 t.Run("Range", func(t *testing.T) {
60 f := func(f1 Range) bool {
61 p1 := f1.ToProto()
62 f2 := RangeFromProto(p1)
63 return reflect.DeepEqual(f1, f2)
64 }
65 if err := quick.Check(f, nil); err != nil {
66 t.Fatal(err)
67 }
68 })
69
70 t.Run("Location", func(t *testing.T) {
71 f := func(f1 Range) bool {
72 p1 := f1.ToProto()
73 f2 := RangeFromProto(p1)
74 return reflect.DeepEqual(f1, f2)
75 }
76 if err := quick.Check(f, nil); err != nil {
77 t.Fatal(err)
78 }
79 })
80
81 t.Run("LineMatch", func(t *testing.T) {
82 f := func(f1 LineMatch) bool {
83 p1 := f1.ToProto()
84 f2 := LineMatchFromProto(p1)
85 return reflect.DeepEqual(f1, f2)
86 }
87 if err := quick.Check(f, nil); err != nil {
88 t.Fatal(err)
89 }
90 })
91
92 t.Run("Symbol", func(t *testing.T) {
93 f := func(f1 *Symbol) bool {
94 p1 := f1.ToProto()
95 f2 := SymbolFromProto(p1)
96 return reflect.DeepEqual(f1, f2)
97 }
98 if err := quick.Check(f, nil); err != nil {
99 t.Fatal(err)
100 }
101 })
102
103 t.Run("FlushReson", func(t *testing.T) {
104 f := func(f1 FlushReason) bool {
105 p1 := f1.ToProto()
106 f2 := FlushReasonFromProto(p1)
107 return reflect.DeepEqual(f1.String(), f2.String())
108 }
109 if err := quick.Check(f, nil); err != nil {
110 t.Fatal(err)
111 }
112 })
113
114 t.Run("Stats", func(t *testing.T) {
115 f := func(f1 Stats) bool {
116 p1 := f1.ToProto()
117 f2 := StatsFromProto(p1)
118 return reflect.DeepEqual(f1, f2)
119 }
120 if err := quick.Check(f, nil); err != nil {
121 t.Fatal(err)
122 }
123 })
124
125 t.Run("Progress", func(t *testing.T) {
126 f := func(f1 Progress) bool {
127 p1 := f1.ToProto()
128 f2 := ProgressFromProto(p1)
129 return reflect.DeepEqual(f1, f2)
130 }
131 if err := quick.Check(f, nil); err != nil {
132 t.Fatal(err)
133 }
134 })
135
136 t.Run("SearchResult", func(t *testing.T) {
137 t.Run("unary", func(t *testing.T) {
138 f := func(f1 *SearchResult) bool {
139 var repoURLs map[string]string
140 var lineFragments map[string]string
141
142 if f1 != nil {
143 repoURLs = f1.RepoURLs
144 lineFragments = f1.LineFragments
145 }
146
147 p1 := f1.ToProto()
148 f2 := SearchResultFromProto(p1, repoURLs, lineFragments)
149
150 return reflect.DeepEqual(f1, f2)
151 }
152 if err := quick.Check(f, nil); err != nil {
153 t.Fatal(err)
154 }
155 })
156
157 t.Run("stream", func(t *testing.T) {
158 f := func(f1 *SearchResult) bool {
159 var repoURLs map[string]string
160 var lineFragments map[string]string
161
162 if f1 != nil {
163 repoURLs = f1.RepoURLs
164 lineFragments = f1.LineFragments
165 }
166
167 p1 := f1.ToStreamProto()
168 f2 := SearchResultFromStreamProto(p1, repoURLs, lineFragments)
169
170 return reflect.DeepEqual(f1, f2)
171 }
172 if err := quick.Check(f, nil); err != nil {
173 t.Fatal(err)
174 }
175 })
176 })
177
178 t.Run("Repository", func(t *testing.T) {
179 f := func(f1 *Repository) bool {
180 p1 := f1.ToProto()
181 f2 := RepositoryFromProto(p1)
182 if diff := cmp.Diff(f1, &f2, cmpopts.IgnoreUnexported(Repository{})); diff != "" {
183 fmt.Printf("got diff: %s", diff)
184 return false
185 }
186 return true
187 }
188 if err := quick.Check(f, nil); err != nil {
189 t.Fatal(err)
190 }
191 })
192
193 t.Run("IndexMetadata", func(t *testing.T) {
194 f := func(f1 *IndexMetadata) bool {
195 p1 := f1.ToProto()
196 f2 := IndexMetadataFromProto(p1)
197 if diff := cmp.Diff(f1, &f2); diff != "" {
198 fmt.Printf("got diff: %s", diff)
199 return false
200 }
201 return true
202 }
203 if err := quick.Check(f, nil); err != nil {
204 t.Fatal(err)
205 }
206 })
207
208 t.Run("RepoStats", func(t *testing.T) {
209 f := func(f1 RepoStats) bool {
210 p1 := f1.ToProto()
211 f2 := RepoStatsFromProto(p1)
212 if diff := cmp.Diff(f1, f2); diff != "" {
213 fmt.Printf("got diff: %s", diff)
214 return false
215 }
216 return true
217 }
218 if err := quick.Check(f, nil); err != nil {
219 t.Fatal(err)
220 }
221 })
222
223 t.Run("RepoListEntry", func(t *testing.T) {
224 r1 := &RepoListEntry{
225 Repository: Repository{
226 ID: 1,
227 Name: "test",
228 URL: "testurl",
229 Source: "testsource",
230 Branches: []RepositoryBranch{{
231 Name: "branch",
232 Version: "version",
233 }},
234 SubRepoMap: map[string]*Repository{
235 "test": {
236 ID: 2,
237 Name: "subrepo",
238 Branches: []RepositoryBranch{},
239 SubRepoMap: map[string]*Repository{},
240 FileTombstones: map[string]struct{}{},
241 },
242 },
243 CommitURLTemplate: "committemplate",
244 FileURLTemplate: "fileurltemplate",
245 LineFragmentTemplate: "linefragmenttemplate",
246 priority: 10,
247 RawConfig: map[string]string{
248 "a": "b",
249 },
250 Rank: 32,
251 IndexOptions: "indexoptions",
252 HasSymbols: true,
253 Tombstone: false,
254 LatestCommitDate: time.Now(),
255 FileTombstones: map[string]struct{}{
256 "test1": {},
257 },
258 },
259 IndexMetadata: IndexMetadata{
260 IndexFormatVersion: 32,
261 IndexFeatureVersion: 42,
262 IndexMinReaderVersion: 52,
263 IndexTime: time.Now(),
264 PlainASCII: true,
265 LanguageMap: map[string]uint16{
266 "go": 1,
267 },
268 ZoektVersion: "32",
269 ID: "52",
270 },
271 Stats: RepoStats{
272 Repos: 3,
273 Shards: 4,
274 Documents: 5,
275 IndexBytes: 6,
276 ContentBytes: 7,
277 NewLinesCount: 8,
278 DefaultBranchNewLinesCount: 9,
279 OtherBranchesNewLinesCount: 10,
280 },
281 }
282
283 p1 := r1.ToProto()
284 r2 := RepoListEntryFromProto(p1)
285 if diff := cmp.Diff(r1, r2, cmpopts.IgnoreUnexported(Repository{})); diff != "" {
286 t.Fatalf("got diff: %s", diff)
287 }
288 })
289
290 t.Run("RepositoryBranch", func(t *testing.T) {
291 f := func(f1 RepositoryBranch) bool {
292 p1 := f1.ToProto()
293 f2 := RepositoryBranchFromProto(p1)
294 if diff := cmp.Diff(f1, f2); diff != "" {
295 fmt.Printf("got diff: %s", diff)
296 return false
297 }
298 return true
299 }
300 if err := quick.Check(f, nil); err != nil {
301 t.Fatal(err)
302 }
303 })
304
305 t.Run("MinimalRepoListEntry", func(t *testing.T) {
306 f := func(f1 MinimalRepoListEntry) bool {
307 p1 := f1.ToProto()
308 f2 := MinimalRepoListEntryFromProto(p1)
309 if diff := cmp.Diff(f1, f2); diff != "" {
310 fmt.Printf("got diff: %s", diff)
311 return false
312 }
313 return true
314 }
315 if err := quick.Check(f, nil); err != nil {
316 t.Fatal(err)
317 }
318 })
319
320 t.Run("ListOptions", func(t *testing.T) {
321 f := func(f1 *ListOptions) bool {
322 p1 := f1.ToProto()
323 f2 := ListOptionsFromProto(p1)
324 if diff := cmp.Diff(f1, f2); diff != "" {
325 fmt.Printf("got diff: %s", diff)
326 return false
327 }
328 return true
329 }
330 if err := quick.Check(f, nil); err != nil {
331 t.Fatal(err)
332 }
333 })
334
335 t.Run("SearchOptions", func(t *testing.T) {
336 f := func(f1 *SearchOptions) bool {
337 if f1 != nil {
338 // Ignore deprecated and unimplemented fields
339 f1.ShardMaxImportantMatch = 0
340 f1.TotalMaxImportantMatch = 0
341 f1.SpanContext = nil
342 }
343 p1 := f1.ToProto()
344 f2 := SearchOptionsFromProto(p1)
345 if diff := cmp.Diff(f1, f2); diff != "" {
346 fmt.Printf("got diff: %s", diff)
347 return false
348 }
349 return true
350 }
351 if err := quick.Check(f, nil); err != nil {
352 t.Fatal(err)
353 }
354 })
355}
356
357func (*IndexMetadata) Generate(r *rand.Rand, _ int) reflect.Value {
358 indexTime := time.Now().Add(time.Duration(r.Int63n(1000)) * time.Hour)
359 var i IndexMetadata
360 i.IndexFormatVersion = gen(i.IndexFormatVersion, r)
361 i.IndexFeatureVersion = gen(i.IndexFeatureVersion, r)
362 i.IndexMinReaderVersion = gen(i.IndexMinReaderVersion, r)
363 i.IndexTime = indexTime
364 i.PlainASCII = gen(i.PlainASCII, r)
365 i.LanguageMap = gen(i.LanguageMap, r)
366 i.ZoektVersion = gen(i.ZoektVersion, r)
367 i.ID = gen(i.ID, r)
368 return reflect.ValueOf(&i)
369}
370
371func (*Repository) Generate(rng *rand.Rand, _ int) reflect.Value {
372 latestCommitDate := time.Now().Add(time.Duration(rng.Int63n(1000)) * time.Hour)
373 var r Repository
374 v := &Repository{
375 ID: gen(r.ID, rng),
376 Name: gen(r.Name, rng),
377 URL: gen(r.URL, rng),
378 Source: gen(r.Source, rng),
379 Branches: gen(r.Branches, rng),
380 SubRepoMap: map[string]*Repository{},
381 CommitURLTemplate: gen(r.CommitURLTemplate, rng),
382 FileURLTemplate: gen(r.FileURLTemplate, rng),
383 LineFragmentTemplate: gen(r.LineFragmentTemplate, rng),
384 priority: gen(r.priority, rng),
385 RawConfig: gen(r.RawConfig, rng),
386 Rank: gen(r.Rank, rng),
387 IndexOptions: gen(r.IndexOptions, rng),
388 HasSymbols: gen(r.HasSymbols, rng),
389 Tombstone: gen(r.Tombstone, rng),
390 LatestCommitDate: latestCommitDate,
391 FileTombstones: gen(r.FileTombstones, rng),
392 }
393 return reflect.ValueOf(v)
394}
395
396func (RepoListField) Generate(rng *rand.Rand, _ int) reflect.Value {
397 switch rng.Int() % 3 {
398 case 0:
399 return reflect.ValueOf(RepoListField(RepoListFieldRepos))
400 case 1:
401 return reflect.ValueOf(RepoListField(RepoListFieldMinimal))
402 default:
403 return reflect.ValueOf(RepoListField(RepoListFieldReposMap))
404 }
405}
406
407func gen[T any](sample T, r *rand.Rand) T {
408 var t T
409 v, _ := quick.Value(reflect.TypeOf(t), r)
410 return v.Interface().(T)
411}
412
413// This is a real search result that is intended to be a reasonable representative
414// for serialization benchmarks.
415// Generated by modifying the code to dump the proto to a file, then running a
416// fairly broadly-matching search.
417var (
418 //go:embed testdata/search_result_1.pb
419 exampleSearchResultBytes []byte
420
421 // The proto struct representation of the search result
422 exampleSearchResultProto = func() *webproto.SearchResponse {
423 sr := new(webproto.SearchResponse)
424 err := proto.Unmarshal(exampleSearchResultBytes, sr)
425 if err != nil {
426 panic(err)
427 }
428 return sr
429 }()
430
431 // The non-proto struct representation of the search result
432 exampleSearchResultGo = SearchResultFromProto(exampleSearchResultProto, nil, nil)
433)
434
435func BenchmarkGobRoundtrip(b *testing.B) {
436 for _, count := range []int{1, 100, 1000, 10000} {
437 b.Run(fmt.Sprintf("count=%d", count), func(b *testing.B) {
438 for i := 0; i < b.N; i++ {
439 var buf bytes.Buffer
440 enc := gob.NewEncoder(&buf)
441
442 for i := 0; i < count; i++ {
443 err := enc.Encode(exampleSearchResultGo)
444 if err != nil {
445 panic(err)
446 }
447
448 }
449
450 dec := gob.NewDecoder(&buf)
451 for i := 0; i < count; i++ {
452 var res SearchResult
453 err := dec.Decode(&res)
454 if err != nil {
455 panic(err)
456 }
457 }
458 }
459 })
460 }
461}
462
463func BenchmarkProtoRoundtrip(b *testing.B) {
464 for _, count := range []int{1, 100, 1000, 10000} {
465 b.Run(fmt.Sprintf("count=%d", count), func(b *testing.B) {
466 for i := 0; i < b.N; i++ {
467 buffers := make([][]byte, 0, count)
468 for i := 0; i < count; i++ {
469 buf, err := proto.Marshal(exampleSearchResultProto)
470 if err != nil {
471 b.Fatal(err)
472 }
473 buffers = append(buffers, buf)
474 }
475
476 for _, buf := range buffers {
477 res := new(webproto.SearchResponse)
478 err := proto.Unmarshal(buf, res)
479 if err != nil {
480 b.Fatal(err)
481 }
482 }
483 }
484 })
485 }
486}
487
488func Fuzz_RepoList_ProtoRoundTrip(f *testing.F) {
489 f.Fuzz(func(t *testing.T, data []byte) {
490 fc := fuzz.NewConsumer(data)
491 fc.AllowUnexportedFields()
492
493 original := &RepoList{}
494 err := fc.GenerateStruct(original)
495 if err != nil {
496 return
497 }
498
499 p := original.ToProto()
500 converted := RepoListFromProto(p)
501
502 opts := []cmp.Option{
503 cmpopts.IgnoreUnexported(Repository{}),
504 cmpopts.EquateEmpty(),
505 }
506
507 if diff := cmp.Diff(original, converted, opts...); diff != "" {
508 t.Fatalf("unexpected diff (-want +got)\n%s", diff)
509 }
510 })
511}