fork of https://github.com/sourcegraph/zoekt
1package main
2
3import (
4 "fmt"
5 "io"
6 "net/http"
7 "net/http/httptest"
8 "strconv"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/google/go-cmp/cmp"
14 "github.com/sourcegraph/log/logtest"
15 "github.com/sourcegraph/zoekt"
16)
17
18func TestQueue(t *testing.T) {
19 backoffDuration := 1 * time.Millisecond
20 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
21
22 for i := 0; i < 100; i++ {
23 queue.AddOrUpdate(mkHEADIndexOptions(i, strconv.Itoa(i)))
24 }
25
26 // Odd numbers are already at the same commit
27 for i := 1; i < 100; i += 2 {
28 queue.SetIndexed(mkHEADIndexOptions(i, strconv.Itoa(i)), indexStateSuccess)
29 }
30
31 // Ensure we process all the even commits first, then odd.
32 want := 0
33 for {
34 opts, ok := queue.Pop()
35 if !ok {
36 break
37 }
38 got, _ := strconv.Atoi(opts.Branches[0].Version)
39 if got != want {
40 t.Fatalf("got %v, want %v", opts, want)
41 }
42 want += 2
43 if want == 100 {
44 // We now switch to processing the odd numbers
45 want = 1
46 }
47 // update current, shouldn't put the job in the queue
48 queue.SetIndexed(opts, indexStateSuccess)
49 }
50 if want != 101 {
51 t.Fatalf("only popped %d items", want)
52 }
53}
54
55func TestQueueFIFO(t *testing.T) {
56 // Tests that the queue fallbacks to FIFO if everything has the same
57 // priority
58 backoffDuration := 1 * time.Millisecond
59 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
60
61 for i := 0; i < 100; i++ {
62 queue.AddOrUpdate(mkHEADIndexOptions(i, strconv.Itoa(i)))
63 }
64
65 want := 0
66 for {
67 opts, ok := queue.Pop()
68 if !ok {
69 break
70 }
71 got, _ := strconv.Atoi(opts.Branches[0].Version)
72 if got != want {
73 t.Fatalf("got %v, want %v", opts, want)
74 }
75 queue.SetIndexed(opts, indexStateSuccess)
76 want++
77 }
78 if want != 100 {
79 t.Fatalf("only popped %d items", want)
80 }
81}
82
83func TestQueue_MaybeRemoveMissing(t *testing.T) {
84 backoffDuration := 1 * time.Millisecond
85 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
86
87 queue.AddOrUpdate(IndexOptions{RepoID: 1, Name: "foo"})
88 queue.AddOrUpdate(IndexOptions{RepoID: 2, Name: "bar"})
89 queue.MaybeRemoveMissing([]uint32{2})
90
91 opts, _ := queue.Pop()
92 if opts.Name != "bar" {
93 t.Fatalf("queue should only contain bar, pop returned %v", opts.Name)
94 }
95 _, ok := queue.Pop()
96 if ok {
97 t.Fatal("queue should be empty")
98 }
99}
100
101func TestQueue_Bump(t *testing.T) {
102 backoffDuration := 1 * time.Millisecond
103 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
104
105 queue.AddOrUpdate(IndexOptions{RepoID: 1, Name: "foo"})
106 queue.AddOrUpdate(IndexOptions{RepoID: 2, Name: "bar"})
107
108 emptyQueue(queue)
109
110 // Bump 2 and 3. 3 doesn't exist, so only 2 should exist.
111 missing := queue.Bump([]uint32{2, 3})
112 if d := cmp.Diff([]uint32{3}, missing); d != "" {
113 t.Errorf("unexpected missing (-want, +got):\n%s", d)
114 }
115
116 want := []IndexOptions{{RepoID: 2, Name: "bar"}}
117 var got []IndexOptions
118 for {
119 opts, ok := queue.Pop()
120 if !ok {
121 break
122 }
123 got = append(got, opts)
124 }
125
126 if d := cmp.Diff(want, got); d != "" {
127 t.Errorf("unexpected items bumped into the queue (-want, +got):\n%s", d)
128 }
129}
130
131func TestQueue_Integration_DebugQueue(t *testing.T) {
132 // helper function to normalize the queue's debug output - this makes the test less brittle
133 // + makes it much less annoying to make edits to the expected output in a way that doesn't
134 // materially affect the caller
135 normalizeDebugOutput := func(output string) string {
136 output = strings.TrimSpace(output)
137
138 var outputLines []string
139 for i, line := range strings.Split(output, "\n") {
140 columns := []string{"Position", "Name", "ID", "IsOnQueue", "Age", "Branches"}
141 parts := strings.Fields(line) // Note: splitting on spaces like this would break for repositories that have more than one branch, but it's fine for just this test
142 if len(columns) != len(parts) {
143 t.Fatalf("normalizeDebugOutput: line %d: expected %d columns, got %d columns: %q", i, len(columns), len(parts), line)
144 }
145
146 if i > 0 { // skip past the first line which just contains the column headings
147
148 // The debug output contains time.Durations for tracking the amount of time an indexing job
149 // spent in the queue, but it's not reasonable to assert on this kind of timing minutia.
150 // So, for comparison purposes, we massage the contents of this field in the following manner:
151 //
152 // - "1m30s" -> "*" (for jobs that are still enqueued)
153 // - "-" -> "-" (for jobs that are tracked, but are not currently enqueued)
154
155 if parts[4] != "-" {
156 parts[4] = "*"
157 }
158 }
159
160 outputLines = append(outputLines, strings.Join(parts, " "))
161 }
162
163 return strings.Join(outputLines, "\n")
164 }
165
166 backoffDuration := 1 * time.Millisecond
167 queue := NewQueue(backoffDuration, backoffDuration, logtest.Scoped(t))
168
169 // setup: add two repositories to the queue and pop one of them
170 poppedRepository := mkHEADIndexOptions(0, "popped")
171 queuedRepository := mkHEADIndexOptions(1, "stillQueued")
172
173 queue.AddOrUpdate(poppedRepository)
174 queue.Pop()
175
176 queue.AddOrUpdate(queuedRepository)
177
178 // setup: start test http server that forwards requests to the
179 // queue instance
180 server := httptest.NewServer(http.HandlerFunc(queue.handleDebugQueue))
181 defer server.Close()
182
183 // test: send a request to the queue's debug endpoint
184 response, err := http.Get(server.URL)
185 if err != nil {
186 t.Fatalf(err.Error())
187 }
188
189 defer response.Body.Close()
190 raw, err := io.ReadAll(response.Body)
191 if err != nil {
192 t.Errorf("reading response body: %s", err)
193 }
194
195 actualOutput := normalizeDebugOutput(string(raw))
196
197 expectedOutput := `
198Position Name ID IsOnQueue Age Branches
1990 item-1 1 true * HEAD@stillQueued
2001 item-0 0 false - HEAD@popped
201`
202
203 expectedOutput = normalizeDebugOutput(expectedOutput)
204
205 // verify: ensure that the received output matches what we expect
206 if diff := cmp.Diff(expectedOutput, actualOutput); diff != "" {
207 t.Errorf("unexpected diff in output (-want +got):\n%s", diff)
208 }
209}
210
211func mkHEADIndexOptions(id int, version string) IndexOptions {
212 return IndexOptions{
213 RepoID: uint32(id),
214 Name: fmt.Sprintf("item-%d", id),
215 Branches: []zoekt.RepositoryBranch{{Name: "HEAD", Version: version}},
216 }
217}