fork of https://github.com/sourcegraph/zoekt
e2e: recall and visual indication of ranks (#714)
This adds a new field to the golden files "targetRank" which records the
rank of the document we are looking for. Additionally the document is
marked with "**" in the golden files.
Additionally we add a new golden file which contains recall@1, recall@5
and the MRR.
I set the target documents by looking at the existing results and
guessing which was the one we wanted based on memory. In some cases we
no longer had the top document, for example for generate unit test.
Test Plan: go test
author
Keegan Carruthers-Smith
committer
GitHub
date
(Jan 11, 2024, 2:29 PM +0200)
commit
6463096a
6463096a03d2ae198b4da97ae31a3ffd81fe994e
parent
344e33ae
344e33ae3c44c220a48b27e1d2e70e30dcd09a85
+120
-38
+100
-31
internal/e2e/e2e_rank_test.go
+100
-31
internal/e2e/e2e_rank_test.go
···
43
43
"https://github.com/golang/go/tree/go1.21.4",
44
44
"https://github.com/sourcegraph/cody/tree/vscode-v0.14.5",
45
45
}
46
-
queries := []string{
46
+
q := func(query, target string) rankingQuery {
47
+
return rankingQuery{Query: query, Target: target}
48
+
}
49
+
queries := []rankingQuery{
47
50
// golang/go
48
-
"test server",
49
-
"bytes buffer",
50
-
"bufio buffer",
51
+
q("test server", "github.com/golang/go/src/net/http/httptest/server.go"),
52
+
q("bytes buffer", "github.com/golang/go/src/bytes/buffer.go"),
53
+
q("bufio buffer", "github.com/golang/go/src/bufio/scan.go"),
51
54
52
55
// sourcegraph/sourcegraph
53
-
"graphql type User",
54
-
"Get database/user",
55
-
"InternalDoer",
56
-
"Repository metadata Write rbac",
56
+
q("graphql type User", "github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql"),
57
+
q("Get database/user", "github.com/sourcegraph/sourcegraph/internal/database/users.go"),
58
+
q("InternalDoer", "github.com/sourcegraph/sourcegraph/internal/httpcli/client.go"),
59
+
q("Repository metadata Write rbac", "github.com/sourcegraph/sourcegraph/internal/rbac/constants.go"), // unsure if this is the best doc?
57
60
58
61
// cody
59
-
"generate unit test",
60
-
"r:cody sourcegraph url",
62
+
q("generate unit test", "github.com/sourcegraph/cody/lib/shared/src/chat/recipes/generate-test.ts"),
63
+
q("r:cody sourcegraph url", "github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts"),
61
64
}
62
65
63
66
var indexDir string
···
80
83
}
81
84
defer ss.Close()
82
85
83
-
for _, queryStr := range queries {
86
+
var ranks []int
87
+
for _, rq := range queries {
84
88
// normalise queryStr for writing to fs
85
89
name := strings.Map(func(r rune) rune {
86
90
if strings.ContainsRune(" :", r) {
···
92
96
return r
93
97
}
94
98
return -1
95
-
}, queryStr)
99
+
}, rq.Query)
96
100
97
101
t.Run(name, func(t *testing.T) {
98
-
q, err := query.Parse(queryStr)
102
+
q, err := query.Parse(rq.Query)
99
103
if err != nil {
100
104
t.Fatal(err)
101
105
}
···
115
119
t.Fatal(err)
116
120
}
117
121
122
+
ranks = append(ranks, targetRank(rq, result.Files))
123
+
118
124
var gotBuf bytes.Buffer
119
-
marshalMatches(&gotBuf, queryStr, q, result.Files)
120
-
got := gotBuf.Bytes()
125
+
marshalMatches(&gotBuf, rq, q, result.Files)
126
+
assertGolden(t, name, gotBuf.Bytes())
127
+
})
128
+
}
129
+
130
+
t.Run("rank_stats", func(t *testing.T) {
131
+
if len(ranks) != len(queries) {
132
+
t.Skip("not computing rank stats since not all query cases ran")
133
+
}
134
+
135
+
var gotBuf bytes.Buffer
136
+
printf := func(format string, a ...any) {
137
+
_, _ = fmt.Fprintf(&gotBuf, format, a...)
138
+
}
139
+
140
+
printf("queries: %d\n", len(ranks))
121
141
122
-
wantPath := filepath.Join("testdata", name+".txt")
123
-
if *update {
124
-
if err := os.WriteFile(wantPath, got, 0600); err != nil {
125
-
t.Fatal(err)
142
+
for _, recallThreshold := range []int{1, 5} {
143
+
count := 0
144
+
for _, rank := range ranks {
145
+
if rank <= recallThreshold && rank > 0 {
146
+
count++
126
147
}
127
148
}
128
-
want, err := os.ReadFile(wantPath)
129
-
if err != nil {
130
-
t.Fatal(err)
149
+
countp := float64(count) * 100 / float64(len(ranks))
150
+
printf("recall@%d: %d (%.0f%%)\n", recallThreshold, count, countp)
151
+
}
152
+
153
+
// Mean reciprocal rank
154
+
mrr := float64(0)
155
+
for _, rank := range ranks {
156
+
if rank > 0 {
157
+
mrr += 1 / float64(rank)
131
158
}
159
+
}
160
+
mrr /= float64(len(ranks))
161
+
printf("mrr: %f\n", mrr)
132
162
133
-
if d := cmp.Diff(string(want), string(got)); d != "" {
134
-
t.Fatalf("unexpected (-want, +got):\n%s", d)
135
-
}
136
-
})
163
+
assertGolden(t, "rank_stats", gotBuf.Bytes())
164
+
})
165
+
}
166
+
167
+
func assertGolden(t *testing.T, name string, got []byte) {
168
+
t.Helper()
169
+
170
+
wantPath := filepath.Join("testdata", name+".txt")
171
+
if *update {
172
+
if err := os.WriteFile(wantPath, got, 0600); err != nil {
173
+
t.Fatal(err)
174
+
}
175
+
}
176
+
want, err := os.ReadFile(wantPath)
177
+
if err != nil {
178
+
t.Fatal(err)
179
+
}
180
+
181
+
if d := cmp.Diff(string(want), string(got)); d != "" {
182
+
t.Fatalf("unexpected (-want, +got):\n%s", d)
137
183
}
138
184
}
139
185
186
+
type rankingQuery struct {
187
+
Query string
188
+
Target string
189
+
}
190
+
140
191
var tarballCache = "/tmp/zoekt-test-ranking-tarballs-" + os.Getenv("USER")
141
192
var shardCache = "/tmp/zoekt-test-ranking-shards-" + os.Getenv("USER")
142
193
···
166
217
// TODO scip
167
218
// languageMap := make(ctags.LanguageMap)
168
219
// for _, lang := range []string{"kotlin", "rust", "ruby", "go", "python", "javascript", "c_sharp", "scala", "typescript", "zig"} {
169
-
// languageMap[lang] = ctags.ScipCTags
220
+
// languageMap[lang] = ctags.ScipCTags
170
221
// }
171
222
172
223
err := archive.Index(opts, build.Options{
···
213
264
fileMatchesPerSearch = 6
214
265
)
215
266
216
-
func marshalMatches(w io.Writer, queryStr string, q query.Q, files []zoekt.FileMatch) {
217
-
_, _ = fmt.Fprintf(w, "queryString: %s\n", queryStr)
218
-
_, _ = fmt.Fprintf(w, "query: %s\n\n", q)
267
+
func docName(f zoekt.FileMatch) string {
268
+
return f.Repository + "/" + f.FileName
269
+
}
270
+
271
+
func marshalMatches(w io.Writer, rq rankingQuery, q query.Q, files []zoekt.FileMatch) {
272
+
_, _ = fmt.Fprintf(w, "queryString: %s\n", rq.Query)
273
+
_, _ = fmt.Fprintf(w, "query: %s\n", q)
274
+
_, _ = fmt.Fprintf(w, "targetRank: %d\n\n", targetRank(rq, files))
219
275
220
276
files, hiddenFiles := splitAtIndex(files, fileMatchesPerSearch)
221
277
for _, f := range files {
222
-
_, _ = fmt.Fprintf(w, "%s/%s%s\n", f.Repository, f.FileName, addTabIfNonEmpty(f.Debug))
278
+
doc := docName(f)
279
+
if doc == rq.Target {
280
+
doc = "**" + doc + "**"
281
+
}
282
+
_, _ = fmt.Fprintf(w, "%s%s\n", doc, addTabIfNonEmpty(f.Debug))
223
283
224
284
chunks, hidden := splitAtIndex(f.ChunkMatches, chunkMatchesPerFile)
225
285
···
236
296
if len(hiddenFiles) > 0 {
237
297
fmt.Fprintf(w, "hidden %d more file matches\n", len(hiddenFiles))
238
298
}
299
+
}
300
+
301
+
func targetRank(rq rankingQuery, files []zoekt.FileMatch) int {
302
+
for i, f := range files {
303
+
if docName(f) == rq.Target {
304
+
return i + 1
305
+
}
306
+
}
307
+
return -1
239
308
}
240
309
241
310
func splitAtIndex[E any](s []E, idx int) ([]E, []E) {
+2
-1
internal/e2e/testdata/Get_databaseuser.txt
+2
-1
internal/e2e/testdata/Get_databaseuser.txt
···
1
1
queryString: Get database/user
2
2
query: (and case_substr:"Get" substr:"database/user")
3
+
targetRank: 3
3
4
4
5
github.com/sourcegraph/sourcegraph/internal/database/user_emails.go
5
6
161:func (s *userEmailsStore) Get(ctx context.Context, userID int32, email string) (emailCanonicalCase string, verified bool, err error) {
···
13
14
365:func (r *userRoleStore) GetByRoleID(ctx context.Context, opts GetUserRoleOpts) ([]*types.UserRole, error) {
14
15
hidden 8 more line matches
15
16
16
-
github.com/sourcegraph/sourcegraph/internal/database/users.go
17
+
**github.com/sourcegraph/sourcegraph/internal/database/users.go**
17
18
940:func (u *userStore) GetByID(ctx context.Context, id int32) (*types.User, error) {
18
19
947:func (u *userStore) GetByVerifiedEmail(ctx context.Context, email string) (*types.User, error) {
19
20
951:func (u *userStore) GetByUsername(ctx context.Context, username string) (*types.User, error) {
+2
-1
internal/e2e/testdata/InternalDoer.txt
+2
-1
internal/e2e/testdata/InternalDoer.txt
···
1
1
queryString: InternalDoer
2
2
query: case_substr:"InternalDoer"
3
+
targetRank: 1
3
4
4
-
github.com/sourcegraph/sourcegraph/internal/httpcli/client.go
5
+
**github.com/sourcegraph/sourcegraph/internal/httpcli/client.go**
5
6
217:var InternalDoer, _ = InternalClientFactory.Doer()
6
7
215:// InternalDoer is a shared client for internal communication. This is a
7
8
+1
internal/e2e/testdata/Repository_metadata_Write_rbac.txt
+1
internal/e2e/testdata/Repository_metadata_Write_rbac.txt
···
1
1
queryString: Repository metadata Write rbac
2
2
query: (and case_substr:"Repository" substr:"metadata" case_substr:"Write" substr:"rbac")
3
+
targetRank: -1
3
4
4
5
github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/repository_metadata.go
5
6
54:func (r *schemaResolver) AddRepoMetadata(ctx context.Context, args struct {
+2
-1
internal/e2e/testdata/bufio_buffer.txt
+2
-1
internal/e2e/testdata/bufio_buffer.txt
···
1
1
queryString: bufio buffer
2
2
query: (and substr:"bufio" substr:"buffer")
3
+
targetRank: 2
3
4
4
5
github.com/golang/go/src/bytes/buffer.go
5
6
20:type Buffer struct {
···
7
8
472:func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }
8
9
hidden 108 more line matches
9
10
10
-
github.com/golang/go/src/bufio/scan.go
11
+
**github.com/golang/go/src/bufio/scan.go**
11
12
267:func (s *Scanner) Buffer(buf []byte, max int) {
12
13
5:package bufio
13
14
25:// large to fit in the buffer. When a scan stops, the reader may have
+2
-1
internal/e2e/testdata/bytes_buffer.txt
+2
-1
internal/e2e/testdata/bytes_buffer.txt
···
1
1
queryString: bytes buffer
2
2
query: (and substr:"bytes" substr:"buffer")
3
+
targetRank: 1
3
4
4
-
github.com/golang/go/src/bytes/buffer.go
5
+
**github.com/golang/go/src/bytes/buffer.go**
5
6
20:type Buffer struct {
6
7
54:func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
7
8
5:package bytes
+1
internal/e2e/testdata/generate_unit_test.txt
+1
internal/e2e/testdata/generate_unit_test.txt
···
1
1
queryString: generate unit test
2
2
query: (and substr:"generate" substr:"unit" substr:"test")
3
+
targetRank: 11
3
4
4
5
github.com/sourcegraph/sourcegraph/cmd/frontend/internal/insights/resolvers/insight_series_resolver.go
5
6
300:func (j *seriesResolverGenerator) Generate(ctx context.Context, series types.InsightViewSeries, baseResolver baseInsightResolver, filters types.InsightViewFilters, options types.SeriesDisplayOptions) ([]graphqlbackend.InsightSeriesResolver, error) {
+2
-1
internal/e2e/testdata/graphql_type_User.txt
+2
-1
internal/e2e/testdata/graphql_type_User.txt
···
1
1
queryString: graphql type User
2
2
query: (and substr:"graphql" substr:"type" case_substr:"User")
3
+
targetRank: 1
3
4
4
-
github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql
5
+
**github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/schema.graphql**
5
6
6376:type User implements Node & SettingsSubject & Namespace {
6
7
3862: type: GitRefType
7
8
5037: type: GitRefType!
+2
-1
internal/e2e/testdata/r_cody_sourcegraph_url.txt
+2
-1
internal/e2e/testdata/r_cody_sourcegraph_url.txt
···
1
1
queryString: r:cody sourcegraph url
2
2
query: (and repo:cody substr:"sourcegraph" substr:"url")
3
+
targetRank: 1
3
4
4
-
github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts
5
+
**github.com/sourcegraph/cody/lib/shared/src/sourcegraph-api/graphql/client.ts**
5
6
611: const url = buildGraphQLUrl({ request: query, baseUrl: this.config.serverEndpoint })
6
7
626: const url = buildGraphQLUrl({ request: query, baseUrl: this.dotcomUrl.href })
7
8
641: const url = 'http://localhost:49300/.api/testLogging'
+4
internal/e2e/testdata/rank_stats.txt
+4
internal/e2e/testdata/rank_stats.txt
+2
-1
internal/e2e/testdata/test_server.txt
+2
-1
internal/e2e/testdata/test_server.txt
···
1
1
queryString: test server
2
2
query: (and substr:"test" substr:"server")
3
+
targetRank: 1
3
4
4
-
github.com/golang/go/src/net/http/httptest/server.go
5
+
**github.com/golang/go/src/net/http/httptest/server.go**
5
6
26:type Server struct {
6
7
105:func NewServer(handler http.Handler) *Server {
7
8
117:func NewUnstartedServer(handler http.Handler) *Server {