fork of https://github.com/sourcegraph/zoekt
Ranking: sort and truncate files while collecting (#543)
Before, the collector aggregated all file matches before flushing, when it
finally sorted and truncated them. Now we sort and limit while collecting
results, instead of at the end. This can help cut down on memory in the case
there are many shard results. (It won't help with `count: all` queries, where
MaxDocDisplayCount is not set.)
Another small benefit is that we obey FlushWallTime more closely. Before, we
might sort a very large number of results after already using up the whole time.
author
Julie Tibshirani
committer
GitHub
date
(Feb 22, 2023, 10:27 AM -0800)
commit
a8188ae8
a8188ae891179adf76a183376b0522fd1069ba8b
parent
b65e3e6b
b65e3e6bceef2587a8a9ecaa4515c886dfbc8837
+9
-11
+1
-2
eval.go
+1
-2
eval.go
···
430
430
}
431
431
})
432
432
433
-
// I am slightly worried about negative interactions with TotalMaxMatchCount
434
-
// so feature flagging this behaviour behind UseDocumentRanks.
433
+
// If document ranking is enabled, then we can rank and truncate the files to save memory.
435
434
if limit := opts.MaxDocDisplayCount; opts.UseDocumentRanks && limit > 0 && limit < len(res.Files) {
436
435
SortFiles(res.Files)
437
436
res.Files = res.Files[:limit]
+8
-9
shards/aggregate.go
+8
-9
shards/aggregate.go
···
34
34
return &collectSender{opts: opts}
35
35
}
36
36
37
+
// Send aggregates the new search result by adding it stats and ranking
38
+
// and truncating its files according to the input SearchOptions.
37
39
func (c *collectSender) Send(r *zoekt.SearchResult) {
38
40
if c.aggregate == nil {
39
41
c.aggregate = &zoekt.SearchResult{
···
47
49
if len(r.Files) > 0 {
48
50
c.aggregate.Files = append(c.aggregate.Files, r.Files...)
49
51
52
+
zoekt.SortFiles(c.aggregate.Files)
53
+
if max := c.opts.MaxDocDisplayCount; max > 0 && max < len(c.aggregate.Files) {
54
+
c.aggregate.Files = c.aggregate.Files[:max]
55
+
}
56
+
50
57
for k, v := range r.RepoURLs {
51
58
c.aggregate.RepoURLs[k] = v
52
59
}
···
64
71
c.aggregate.MaxPendingPriority = r.MaxPendingPriority
65
72
}
66
73
67
-
// Done returns the aggregated result. Before returning them the files are
68
-
// ranked and truncated according to the input SearchOptions.
74
+
// Done returns the aggregated result.
69
75
//
70
76
// If no results are aggregated, ok is false and the result is nil.
71
77
func (c *collectSender) Done() (_ *zoekt.SearchResult, ok bool) {
···
75
81
76
82
agg := c.aggregate
77
83
c.aggregate = nil
78
-
79
-
zoekt.SortFiles(agg.Files)
80
-
81
-
if max := c.opts.MaxDocDisplayCount; max > 0 && len(agg.Files) > max {
82
-
agg.Files = agg.Files[:max]
83
-
}
84
-
85
84
return agg, true
86
85
}
87
86