fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package zoekt
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "flag"
22 "fmt"
23 "io/fs"
24 "os"
25 "path"
26 "path/filepath"
27 "reflect"
28 "strconv"
29 "strings"
30 "testing"
31 "testing/quick"
32
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35
36 "github.com/sourcegraph/zoekt/query"
37)
38
39var update = flag.Bool("update", false, "update golden files")
40
41func TestReadWrite(t *testing.T) {
42 b, err := NewIndexBuilder(nil)
43 if err != nil {
44 t.Fatalf("NewIndexBuilder: %v", err)
45 }
46
47 if err := b.AddFile("filename", []byte("abcde")); err != nil {
48 t.Fatalf("AddFile: %v", err)
49 }
50
51 var buf bytes.Buffer
52 if err := b.Write(&buf); err != nil {
53 t.Fatal(err)
54 }
55 f := &memSeeker{buf.Bytes()}
56
57 r := reader{r: f}
58
59 var toc indexTOC
60 err = r.readTOC(&toc)
61
62 if err != nil {
63 t.Errorf("got read error %v", err)
64 }
65 if toc.fileContents.data.sz != 5 {
66 t.Errorf("got contents size %d, want 5", toc.fileContents.data.sz)
67 }
68
69 data, err := r.readIndexData(&toc)
70 if err != nil {
71 t.Fatalf("readIndexData: %v", err)
72 }
73 if got := data.fileName(0); string(got) != "filename" {
74 t.Errorf("got filename %q, want %q", got, "filename")
75 }
76
77 if len(data.ngrams.DumpMap()) != 3 {
78 t.Fatalf("got ngrams %v, want 3 ngrams", data.ngrams)
79 }
80
81 if sec := data.ngrams.Get(stringToNGram("bcq")); sec.sz > 0 {
82 t.Errorf("found ngram bcq (%v) in %v", uint64(stringToNGram("bcq")), data.ngrams)
83 }
84}
85
86func TestReadWriteNames(t *testing.T) {
87 b, err := NewIndexBuilder(nil)
88 if err != nil {
89 t.Fatalf("NewIndexBuilder: %v", err)
90 }
91
92 if err := b.AddFile("abCd", []byte("")); err != nil {
93 t.Fatalf("AddFile: %v", err)
94 }
95
96 var buf bytes.Buffer
97 if err := b.Write(&buf); err != nil {
98 t.Fatal(err)
99 }
100 f := &memSeeker{buf.Bytes()}
101
102 r := reader{r: f}
103
104 var toc indexTOC
105 if err := r.readTOC(&toc); err != nil {
106 t.Errorf("got read error %v", err)
107 }
108 if toc.fileNames.data.sz != 4 {
109 t.Errorf("got contents size %d, want 4", toc.fileNames.data.sz)
110 }
111
112 data, err := r.readIndexData(&toc)
113 if err != nil {
114 t.Fatalf("readIndexData: %v", err)
115 }
116 if !reflect.DeepEqual([]uint32{0, 4}, data.fileNameIndex) {
117 t.Errorf("got index %v, want {0,4}", data.fileNameIndex)
118 }
119 if got := data.fileNameNgrams[stringToNGram("bCd")]; !reflect.DeepEqual(got, []byte{1}) {
120 t.Errorf("got trigram bcd at bits %v, want sz 2", data.fileNameNgrams)
121 }
122}
123
124func loadShard(fn string) (Searcher, error) {
125 f, err := os.Open(fn)
126 if err != nil {
127 return nil, err
128 }
129
130 iFile, err := NewIndexFile(f)
131 if err != nil {
132 return nil, err
133 }
134 s, err := NewSearcher(iFile)
135 if err != nil {
136 iFile.Close()
137 return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
138 }
139
140 return s, nil
141}
142
143func TestReadSearch(t *testing.T) {
144 type out struct {
145 FormatVersion int
146 FeatureVersion int
147 FileMatches [][]FileMatch
148 }
149
150 qs := []query.Q{
151 &query.Substring{Pattern: "func main", Content: true},
152 &query.Regexp{Regexp: mustParseRE("^package"), Content: true},
153 &query.Symbol{Expr: &query.Substring{Pattern: "num"}},
154 &query.Symbol{Expr: &query.Regexp{Regexp: mustParseRE("sage$")}},
155 }
156
157 shards, err := filepath.Glob("testdata/shards/*.zoekt")
158 if err != nil {
159 t.Fatal(err)
160 }
161
162 for _, path := range shards {
163 name := filepath.Base(path)
164 name = strings.TrimSuffix(name, ".zoekt")
165
166 shard, err := loadShard(path)
167 if err != nil {
168 t.Fatalf("error loading shard %s %v", name, err)
169 }
170
171 index, ok := shard.(*indexData)
172 if !ok {
173 t.Fatalf("expected *indexData for %s", name)
174 }
175
176 golden := "testdata/golden/TestReadSearch/" + name + ".golden"
177
178 if *update {
179 got := out{
180 FormatVersion: index.metaData.IndexFormatVersion,
181 FeatureVersion: index.metaData.IndexFeatureVersion,
182 }
183 for _, q := range qs {
184 res, err := shard.Search(context.Background(), q, &SearchOptions{})
185 if err != nil {
186 t.Fatalf("failed search %s on %s during updating: %v", q, name, err)
187 }
188 got.FileMatches = append(got.FileMatches, res.Files)
189 }
190
191 if raw, err := json.MarshalIndent(got, "", " "); err != nil {
192 t.Errorf("failed marshalling search results for %s during updating: %v", name, err)
193 continue
194 } else if err := os.WriteFile(golden, raw, 0644); err != nil {
195 t.Errorf("failed writing search results for %s during updating: %v", name, err)
196 continue
197 }
198 }
199
200 var want out
201 if buf, err := os.ReadFile(golden); err != nil {
202 t.Fatalf("failed reading search results for %s: %v", name, err)
203 } else if err := json.Unmarshal(buf, &want); err != nil {
204 t.Fatalf("failed unmarshalling search results for %s: %v", name, err)
205 }
206
207 if index.metaData.IndexFormatVersion != want.FormatVersion {
208 t.Errorf("got %d index format version, want %d for %s", index.metaData.IndexFormatVersion, want.FormatVersion, name)
209 }
210
211 if index.metaData.IndexFeatureVersion != want.FeatureVersion {
212 t.Errorf("got %d index feature version, want %d for %s", index.metaData.IndexFeatureVersion, want.FeatureVersion, name)
213 }
214
215 for j, q := range qs {
216 res, err := shard.Search(context.Background(), q, &SearchOptions{})
217 if err != nil {
218 t.Fatalf("failed search %s on %s: %v", q, name, err)
219 }
220
221 if len(res.Files) != len(want.FileMatches[j]) {
222 t.Fatalf("got %d file matches for %s on %s, want %d", len(res.Files), q, name, len(want.FileMatches[j]))
223 }
224
225 if len(want.FileMatches[j]) == 0 {
226 continue
227 }
228
229 if d := cmp.Diff(res.Files, want.FileMatches[j]); d != "" {
230 t.Errorf("matches for %s on %s\n%s", q, name, d)
231 }
232 }
233 }
234}
235
236func TestEncodeRawConfig(t *testing.T) {
237 mustParse := func(s string) uint8 {
238 i, err := strconv.ParseInt(s, 2, 8)
239 if err != nil {
240 t.Fatalf("failed to parse %s", s)
241 }
242 return uint8(i)
243 }
244
245 cases := []struct {
246 rawConfig map[string]string
247 want string
248 }{
249 {
250 rawConfig: map[string]string{"public": "1"},
251 want: "101001",
252 },
253 {
254 rawConfig: map[string]string{"fork": "1"},
255 want: "100110",
256 },
257 {
258 rawConfig: map[string]string{"public": "1", "fork": "1"},
259 want: "100101",
260 },
261 {
262 rawConfig: map[string]string{"public": "1", "fork": "1", "archived": "1"},
263 want: "010101",
264 },
265 {
266 rawConfig: map[string]string{},
267 want: "101010",
268 },
269 }
270 for _, c := range cases {
271 t.Run(c.want, func(t *testing.T) {
272 if got := encodeRawConfig(c.rawConfig); got != mustParse(c.want) {
273 t.Fatalf("want %s, got %s", c.want, strconv.FormatInt(int64(got), 2))
274 }
275 })
276 }
277}
278
279func TestBackwardsCompat(t *testing.T) {
280 if *update {
281 b, err := NewIndexBuilder(nil)
282 if err != nil {
283 t.Fatalf("NewIndexBuilder: %v", err)
284 }
285
286 if err := b.AddFile("filename", []byte("abcde")); err != nil {
287 t.Fatalf("AddFile: %v", err)
288 }
289
290 var buf bytes.Buffer
291 if err := b.Write(&buf); err != nil {
292 t.Fatal(err)
293 }
294
295 outname := fmt.Sprintf("testdata/backcompat/new_v%d.%05d.zoekt", IndexFormatVersion, 0)
296 t.Log("writing new file", outname)
297
298 err = os.WriteFile(outname, buf.Bytes(), 0644)
299 if err != nil {
300 t.Fatalf("Creating output file: %v", err)
301 }
302 }
303
304 compatibleFiles, err := fs.Glob(os.DirFS("."), "testdata/backcompat/*.zoekt")
305 if err != nil {
306 t.Fatalf("fs.Glob: %v", err)
307 }
308
309 for _, fname := range compatibleFiles {
310 t.Run(path.Base(fname),
311 func(t *testing.T) {
312 f, err := os.Open(fname)
313 if err != nil {
314 t.Fatal("os.Open", err)
315 }
316 idx, err := NewIndexFile(f)
317 if err != nil {
318 t.Fatal("NewIndexFile", err)
319 }
320 r := reader{r: idx}
321
322 var toc indexTOC
323 err = r.readTOC(&toc)
324
325 if err != nil {
326 t.Errorf("got read error %v", err)
327 }
328 if toc.fileContents.data.sz != 5 {
329 t.Errorf("got contents size %d, want 5", toc.fileContents.data.sz)
330 }
331
332 data, err := r.readIndexData(&toc)
333 if err != nil {
334 t.Fatalf("readIndexData: %v", err)
335 }
336 if got := data.fileName(0); string(got) != "filename" {
337 t.Errorf("got filename %q, want %q", got, "filename")
338 }
339
340 if len(data.ngrams.DumpMap()) != 3 {
341 t.Fatalf("got ngrams %v, want 3 ngrams", data.ngrams)
342 }
343
344 if sec := data.ngrams.Get(stringToNGram("bcq")); sec.sz > 0 {
345 t.Errorf("found ngram bcd in %v", data.ngrams)
346 }
347 },
348 )
349 }
350}
351
352func TestBackfillIDIsDeterministic(t *testing.T) {
353 repo := "github.com/a/b"
354 have1 := backfillID(repo)
355 have2 := backfillID(repo)
356
357 if have1 != have2 {
358 t.Fatalf("%s != %s ", have1, have2)
359 }
360}
361
362func TestEncodeRanks(t *testing.T) {
363 quick.Check(func(ranks [][]float64) bool {
364 buf := bytes.Buffer{}
365 w := &writer{w: &buf}
366
367 if err := encodeRanks(w, ranks); err != nil {
368 return false
369 }
370
371 // In case all rank vectors are empty, IE {{}, {}, ...}, we won't write anything
372 // to w and gob decode will decode this as "nil", which will fail the
373 // comparison even with cmpopts.EquateEmpty().
374 if w.off == 0 {
375 return true
376 }
377
378 d := &indexData{}
379 if err := decodeRanks(buf.Bytes(), &d.ranks); err != nil {
380 t.Fatal(err)
381 }
382
383 if d := cmp.Diff(ranks, d.ranks, cmpopts.EquateEmpty()); d != "" {
384 t.Fatalf("-want, +got:\n%s\n", d)
385 }
386
387 return true
388 }, nil)
389}