fork of https://github.com/sourcegraph/zoekt
1package main
2
3import (
4 "fmt"
5 "io"
6 "net/http"
7 "net/http/httptest"
8 "strconv"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/google/go-cmp/cmp"
14 "github.com/sourcegraph/log/logtest"
15 "github.com/sourcegraph/zoekt"
16)
17
18func TestQueue(t *testing.T) {
19 backoffDuration := 1 * time.Millisecond
20 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
21
22 for i := 0; i < 100; i++ {
23 queue.AddOrUpdate(mkHEADIndexOptions(i, strconv.Itoa(i)))
24 }
25
26 // Odd numbers are already at the same commit
27 for i := 1; i < 100; i += 2 {
28 queue.SetIndexed(mkHEADIndexOptions(i, strconv.Itoa(i)), indexStateSuccess)
29 }
30
31 // Ensure we process all the even commits first, then odd.
32 want := 0
33 for {
34 opts, ok := queue.Pop()
35 if !ok {
36 break
37 }
38 got, _ := strconv.Atoi(opts.Branches[0].Version)
39 if got != want {
40 t.Fatalf("got %v, want %v", opts, want)
41 }
42 want += 2
43 if want == 100 {
44 // We now switch to processing the odd numbers
45 want = 1
46 }
47 // update current, shouldn't put the job in the queue
48 queue.SetIndexed(opts, indexStateSuccess)
49 }
50 if want != 101 {
51 t.Fatalf("only popped %d items", want)
52 }
53}
54
55func TestQueueFIFO(t *testing.T) {
56 // Tests that the queue fallbacks to FIFO if everything has the same
57 // priority
58 backoffDuration := 1 * time.Millisecond
59 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
60
61 for i := 0; i < 100; i++ {
62 queue.AddOrUpdate(mkHEADIndexOptions(i, strconv.Itoa(i)))
63 }
64
65 want := 0
66 for {
67 opts, ok := queue.Pop()
68 if !ok {
69 break
70 }
71 got, _ := strconv.Atoi(opts.Branches[0].Version)
72 if got != want {
73 t.Fatalf("got %v, want %v", opts, want)
74 }
75 queue.SetIndexed(opts, indexStateSuccess)
76 want++
77 }
78 if want != 100 {
79 t.Fatalf("only popped %d items", want)
80 }
81}
82
83func TestQueue_MaybeRemoveMissing(t *testing.T) {
84 backoffDuration := 1 * time.Millisecond
85 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
86
87 queue.AddOrUpdate(IndexOptions{RepoID: 1, Name: "foo"})
88 queue.AddOrUpdate(IndexOptions{RepoID: 2, Name: "bar"})
89 queue.MaybeRemoveMissing([]uint32{2})
90
91 opts, _ := queue.Pop()
92 if opts.Name != "bar" {
93 t.Fatalf("queue should only contain bar, pop returned %v", opts.Name)
94 }
95 _, ok := queue.Pop()
96 if ok {
97 t.Fatal("queue should be empty")
98 }
99}
100
101func TestQueue_Bump(t *testing.T) {
102 backoffDuration := 1 * time.Millisecond
103 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
104
105 queue.AddOrUpdate(IndexOptions{RepoID: 1, Name: "foo"})
106 queue.AddOrUpdate(IndexOptions{RepoID: 2, Name: "bar"})
107
108 emptyQueue(queue)
109
110 // Bump 2 and 3. 3 doesn't exist, so only 2 should exist.
111 missing := queue.Bump([]uint32{2, 3})
112 if d := cmp.Diff([]uint32{3}, missing); d != "" {
113 t.Errorf("unexpected missing (-want, +got):\n%s", d)
114 }
115
116 want := []IndexOptions{{RepoID: 2, Name: "bar"}}
117 var got []IndexOptions
118 for {
119 opts, ok := queue.Pop()
120 if !ok {
121 break
122 }
123 got = append(got, opts)
124 }
125
126 if d := cmp.Diff(want, got); d != "" {
127 t.Errorf("unexpected items bumped into the queue (-want, +got):\n%s", d)
128 }
129}
130
131func TestQueue_Integration_DebugQueue(t *testing.T) {
132
133 // helper function to normalize the queue's debug output - this makes the test less brittle
134 // + makes it much less annoying to make edits to the expected output in a way that doesn't
135 // materially affect the caller
136 normalizeDebugOutput := func(output string) string {
137 output = strings.TrimSpace(output)
138
139 var outputLines []string
140 for i, line := range strings.Split(output, "\n") {
141 columns := []string{"Position", "Name", "ID", "IsOnQueue", "Age", "Branches"}
142 parts := strings.Fields(line) // Note: splitting on spaces like this would break for repositories that have more than one branch, but it's fine for just this test
143 if len(columns) != len(parts) {
144 t.Fatalf("normalizeDebugOutput: line %d: expected %d columns, got %d columns: %q", i, len(columns), len(parts), line)
145 }
146
147 if i > 0 { // skip past the first line which just contains the column headings
148
149 // The debug output contains time.Durations for tracking the amount of time an indexing job
150 // spent in the queue, but it's not reasonable to assert on this kind of timing minutia.
151 // So, for comparison purposes, we massage the contents of this field in the following manner:
152 //
153 // - "1m30s" -> "*" (for jobs that are still enqueued)
154 // - "-" -> "-" (for jobs that are tracked, but are not currently enqueued)
155
156 if parts[4] != "-" {
157 parts[4] = "*"
158 }
159 }
160
161 outputLines = append(outputLines, strings.Join(parts, " "))
162 }
163
164 return strings.Join(outputLines, "\n")
165 }
166
167 backoffDuration := 1 * time.Millisecond
168 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
169
170 // setup: add two repositories to the queue and pop one of them
171 poppedRepository := mkHEADIndexOptions(0, "popped")
172 queuedRepository := mkHEADIndexOptions(1, "stillQueued")
173
174 queue.AddOrUpdate(poppedRepository)
175 queue.Pop()
176
177 queue.AddOrUpdate(queuedRepository)
178
179 // setup: start test http server that forwards requests to the
180 // queue instance
181 server := httptest.NewServer(http.HandlerFunc(queue.handleDebugQueue))
182 defer server.Close()
183
184 // test: send a request to the queue's debug endpoint
185 response, err := http.Get(server.URL)
186 if err != nil {
187 t.Fatalf(err.Error())
188 }
189
190 defer response.Body.Close()
191 raw, err := io.ReadAll(response.Body)
192 if err != nil {
193 t.Errorf("reading response body: %s", err)
194 }
195
196 actualOutput := normalizeDebugOutput(string(raw))
197
198 expectedOutput := `
199Position Name ID IsOnQueue Age Branches
2000 item-1 1 true * HEAD@stillQueued
2011 item-0 0 false - HEAD@popped
202`
203
204 expectedOutput = normalizeDebugOutput(expectedOutput)
205
206 // verify: ensure that the received output matches what we expect
207 if diff := cmp.Diff(expectedOutput, actualOutput); diff != "" {
208 t.Errorf("unexpected diff in output (-want +got):\n%s", diff)
209 }
210}
211
212func mkHEADIndexOptions(id int, version string) IndexOptions {
213 return IndexOptions{
214 RepoID: uint32(id),
215 Name: fmt.Sprintf("item-%d", id),
216 Branches: []zoekt.RepositoryBranch{{Name: "HEAD", Version: version}},
217 }
218}