fork of https://github.com/sourcegraph/zoekt
Delay line loading a bit.
This is so we can avoid populating Match.Line for less important
results in the future.
author
Han-Wen Nienhuys
date
(Apr 21, 2016, 12:43 PM +0200)
commit
9f447155
9f4471555191474b9ae51a1acc7f4ac601847026
parent
a1e33b78
a1e33b7835c339176b736ca47f3f5bba36b31773
+99
-104
+5
-2
api.go
+5
-2
api.go
···
29
29
30
30
// Match is a match for a single atomic query within a file.
31
31
type Match struct {
32
-
Line []byte
33
-
LineNum int
32
+
// The line in which a match was found.
33
+
Line []byte
34
+
LineStart int
35
+
LineEnd int
36
+
LineNum int
34
37
35
38
// Offset within the line.
36
39
LineOff int
+3
-3
dociter.go
+3
-3
dociter.go
···
86
86
return bytes.Compare(content[m.offset:m.offset+uint32(len(m.substrLowered))], m.substrLowered) == 0
87
87
}
88
88
89
-
func (m *candidateMatch) line(newlines []uint32, content []byte, caseBits []byte) (lineNum, lineOff int, lineContent []byte) {
89
+
func (m *candidateMatch) line(newlines []uint32, fileSize uint32) (lineNum, lineStart, lineEnd int) {
90
90
idx := sort.Search(len(newlines), func(n int) bool {
91
91
return newlines[n] >= m.offset
92
92
})
93
93
94
-
end := len(content)
94
+
end := int(fileSize)
95
95
if idx < len(newlines) {
96
96
end = int(newlines[idx])
97
97
}
···
101
101
start = int(newlines[idx-1] + 1)
102
102
}
103
103
104
-
return idx + 1, int(m.offset) - start, toOriginal(content, caseBits, start, end)
104
+
return idx + 1, start, end
105
105
}
106
106
107
107
func (s *docIterator) next() []*candidateMatch {
+9
-4
eval.go
+9
-4
eval.go
···
295
295
st.caseEvaluated = false
296
296
}
297
297
298
+
var fileStart uint32
299
+
if nextDoc > 0 {
300
+
fileStart = d.fileEnds[nextDoc-1]
301
+
}
298
302
cp := contentProvider{
299
-
reader: d.reader,
300
-
id: d,
301
-
idx: nextDoc,
302
-
stats: &res.Stats,
303
+
reader: d.reader,
304
+
id: d,
305
+
idx: nextDoc,
306
+
stats: &res.Stats,
307
+
fileSize: d.fileEnds[nextDoc] - fileStart,
303
308
}
304
309
305
310
known := make(map[matchTree]bool)
+15
-52
index_test.go
+15
-52
index_test.go
···
116
116
b.AddFile("filename", []byte("line1\nline2\nbla"))
117
117
//----------------------------012345 678901 23456
118
118
119
-
var buf bytes.Buffer
120
-
b.Write(&buf)
121
-
f := &memSeeker{buf.Bytes(), 0}
122
-
123
-
r := reader{r: f}
124
-
125
-
var toc indexTOC
126
-
r.readTOC(&toc)
127
-
data := r.readIndexData(&toc)
128
-
nls := r.readNewlines(data, 0)
129
-
130
-
if want := []uint32{5, 11}; !reflect.DeepEqual(nls, want) {
131
-
t.Errorf("got newlines %v, want %v", nls, want)
132
-
}
133
-
134
-
f = &memSeeker{buf.Bytes(), 0}
135
-
136
-
searcher, err := NewSearcher(f)
119
+
searcher := searcherForTest(t, b)
120
+
sres, err := searcher.Search(&SubstringQuery{Pattern: "ne2"})
137
121
if err != nil {
138
-
t.Fatalf("NewSearcher: %v", err)
122
+
t.Fatal(err)
139
123
}
140
-
sres, err := searcher.Search(&SubstringQuery{Pattern: "ne2"})
141
124
clearScores(sres)
142
125
143
126
matches := sres.Files
···
147
130
{
148
131
Offset: 8,
149
132
Line: []byte("line2"),
133
+
LineStart: 6,
134
+
LineEnd: 11,
150
135
LineNum: 2,
151
136
LineOff: 2,
152
137
MatchLength: 3,
···
232
217
clearScores(sres)
233
218
234
219
matches := sres.Files
235
-
want := []FileMatch{{
236
-
Name: "f2",
237
-
Matches: []Match{{
238
-
Line: c2,
239
-
LineNum: 1,
240
-
LineOff: 10,
241
-
Offset: 10,
242
-
MatchLength: 6,
243
-
}},
244
-
}, {
245
-
Name: "f1",
246
-
Matches: []Match{{
247
-
Offset: 8,
248
-
Line: c1,
249
-
LineNum: 1,
250
-
LineOff: 8,
251
-
MatchLength: 6,
252
-
}},
253
-
}}
254
-
if !reflect.DeepEqual(matches, want) {
255
-
t.Errorf("got matches %#v, want %#v", matches, want)
220
+
if len(matches) != 2 || matches[0].Name != "f2" || matches[1].Name != "f1" {
221
+
t.Fatalf("got %v, want matches {f1,f2}", matches)
222
+
}
223
+
if matches[0].Matches[0].Offset != 10 || matches[1].Matches[0].Offset != 8 {
224
+
t.Fatalf("got %#v, want offsets 10,8", matches)
256
225
}
257
226
}
258
227
···
468
437
t.Fatalf("got %v, want 1 match", matches)
469
438
}
470
439
471
-
got := matches[0].Matches[0]
472
-
want := Match{
473
-
Line: []byte("x apple y"),
474
-
Offset: 2,
475
-
LineNum: 1,
476
-
LineOff: 2,
477
-
MatchLength: 5,
478
-
}
479
-
480
-
if !reflect.DeepEqual(got, want) {
481
-
t.Errorf("got %#v, want %#v", got, want)
440
+
match := matches[0].Matches[0]
441
+
got := string(match.Line)
442
+
want := "x apple y"
443
+
if got != want {
444
+
t.Errorf("got match %#v, want line %q", match, want)
482
445
}
483
446
}
484
447
+21
read_test.go
+21
read_test.go
···
114
114
t.Errorf("got trigram bcd at bits %v, want sz 2", data.fileNameNgrams)
115
115
}
116
116
}
117
+
118
+
func TestReadWriteNewlines(t *testing.T) {
119
+
b := NewIndexBuilder()
120
+
b.AddFile("filename", []byte("line1\nline2\nbla"))
121
+
//----------------------------012345 678901 23456
122
+
123
+
var buf bytes.Buffer
124
+
b.Write(&buf)
125
+
f := &memSeeker{buf.Bytes(), 0}
126
+
127
+
r := reader{r: f}
128
+
129
+
var toc indexTOC
130
+
r.readTOC(&toc)
131
+
data := r.readIndexData(&toc)
132
+
nls := r.readNewlines(data, 0)
133
+
134
+
if want := []uint32{5, 11}; !reflect.DeepEqual(nls, want) {
135
+
t.Errorf("got newlines %v, want %v", nls, want)
136
+
}
137
+
}
+46
-43
search.go
+46
-43
search.go
···
22
22
var _ = log.Println
23
23
24
24
type contentProvider struct {
25
-
reader *reader
26
-
id *indexData
27
-
idx uint32
28
-
stats *Stats
29
-
cb []byte
30
-
data []byte
31
-
nl []uint32
25
+
reader *reader
26
+
id *indexData
27
+
idx uint32
28
+
stats *Stats
29
+
_cb []byte
30
+
_data []byte
31
+
_nl []uint32
32
+
fileSize uint32
33
+
}
34
+
35
+
func (p *contentProvider) newlines() []uint32 {
36
+
if p._nl == nil {
37
+
p._nl = p.reader.readNewlines(p.id, p.idx)
38
+
}
39
+
return p._nl
40
+
}
41
+
42
+
func (p *contentProvider) data(fileName bool) []byte {
43
+
if fileName {
44
+
return p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]]
45
+
}
46
+
47
+
if p._data == nil {
48
+
p._data = p.reader.readContents(p.id, p.idx)
49
+
p.stats.FilesLoaded++
50
+
}
51
+
return p._data
52
+
}
53
+
54
+
func (p *contentProvider) caseBits(fileName bool) []byte {
55
+
if fileName {
56
+
return p.id.fileNameCaseBits[p.id.fileNameCaseBitsIndex[p.idx]:p.id.fileNameCaseBitsIndex[p.idx+1]]
57
+
}
58
+
59
+
if p._cb == nil {
60
+
p._cb = p.reader.readCaseBits(p.id, p.idx)
61
+
}
62
+
return p._cb
32
63
}
33
64
34
65
func (p *contentProvider) caseMatches(m *candidateMatch) bool {
35
-
var cb []byte
36
-
if m.query.FileName {
37
-
cb = p.id.fileNameCaseBits[p.id.fileNameCaseBitsIndex[p.idx]:p.id.fileNameCaseBitsIndex[p.idx+1]]
38
-
} else {
39
-
if p.cb == nil {
40
-
p.cb = p.reader.readCaseBits(p.id, p.idx)
41
-
}
42
-
cb = p.cb
43
-
}
44
-
return m.caseMatches(cb)
66
+
return m.caseMatches(p.caseBits(m.query.FileName))
45
67
}
46
68
47
69
func (p *contentProvider) matchContent(m *candidateMatch) bool {
48
-
var content []byte
49
-
if m.query.FileName {
50
-
content = p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]]
51
-
} else {
52
-
if p.data == nil {
53
-
p.data = p.reader.readContents(p.id, p.idx)
54
-
p.stats.FilesLoaded++
55
-
}
56
-
content = p.data
57
-
}
58
-
return m.matchContent(content)
70
+
return m.matchContent(p.data(m.query.FileName))
59
71
}
60
72
61
73
func (p *contentProvider) fillMatch(m *candidateMatch) Match {
62
74
if m.query.FileName {
63
75
return Match{
64
76
Offset: m.offset,
65
-
Line: p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]],
77
+
Line: p.data(true),
66
78
LineOff: int(m.offset),
67
79
MatchLength: len(m.substrBytes),
68
80
FileName: true,
69
81
}
70
82
}
71
83
72
-
if p.nl == nil {
73
-
p.nl = p.reader.readNewlines(p.id, p.idx)
74
-
}
75
-
if p.data == nil {
76
-
p.data = p.reader.readContents(p.id, p.idx)
77
-
p.stats.FilesLoaded++
78
-
}
79
-
if p.cb == nil {
80
-
p.cb = p.reader.readCaseBits(p.id, p.idx)
81
-
}
82
-
num, off, data := m.line(p.nl, p.data, p.cb)
83
-
84
+
num, start, end := m.line(p.newlines(), p.fileSize)
84
85
finalMatch := Match{
85
86
Offset: m.offset,
86
-
Line: data,
87
+
LineStart: start,
88
+
LineEnd: end,
87
89
LineNum: num,
88
-
LineOff: off,
90
+
LineOff: int(m.offset) - start,
89
91
MatchLength: len(m.substrBytes),
90
92
}
93
+
finalMatch.Line = toOriginal(p.data(false), p.caseBits(false), start, end)
91
94
finalMatch.Score = matchScore(&finalMatch)
92
95
93
96
return finalMatch