+22 -4

cmd/zoekt-sourcegraph-indexserver/main.go

Reviewed

··· 316 316 debug.Printf("updating index queue with %d repositories", len(repos.IDs)) 317 317 318 318 // Stop indexing repos we don't need to track anymore 319 319 - count := s.queue.MaybeRemoveMissing(repos.IDs) 320 320 - metricNumStoppedTrackingTotal.Add(float64(count)) 321 321 - if count > 0 { 322 322 - log.Printf("stopped tracking %d repositories", count) 319 319 + removed := s.queue.MaybeRemoveMissing(repos.IDs) 320 320 + metricNumStoppedTrackingTotal.Add(float64(len(removed))) 321 321 + if len(removed) > 0 { 322 322 + log.Printf("stopped tracking %d repositories: %s", len(removed), formatListUint32(removed, 5)) 323 323 } 324 324 325 325 cleanupDone := make(chan struct{}) ··· 369 369 370 370 // block forever 371 371 select {} 372 372 + } 373 373 + 374 374 + // formatList returns a comma-separated list of the first min(len(v), m) items. 375 375 + func formatListUint32(v []uint32, m int) string { 376 376 + if len(v) < m { 377 377 + m = len(v) 378 378 + } 379 379 + 380 380 + sb := strings.Builder{} 381 381 + for i := 0; i < m; i++ { 382 382 + fmt.Fprintf(&sb, "%d, ", v[i]) 383 383 + } 384 384 + 385 385 + if len(v) > m { 386 386 + sb.WriteString("...") 387 387 + } 388 388 + 389 389 + return strings.TrimRight(sb.String(), ", ") 372 390 } 373 391 374 392 func (s *Server) processQueue() {

+34

cmd/zoekt-sourcegraph-indexserver/main_test.go

Reviewed

··· 3 3 import ( 4 4 "context" 5 5 "flag" 6 6 + "fmt" 6 7 "io" 7 8 "log" 8 9 "net/http" ··· 143 144 t.Fatalf("wanted %t, got %t", true, got) 144 145 } 145 146 } 147 147 + 148 148 + func TestFormatListUint32(t *testing.T) { 149 149 + cases := []struct { 150 150 + in []uint32 151 151 + want string 152 152 + }{ 153 153 + { 154 154 + in: []uint32{42, 8, 3}, 155 155 + want: "42, 8, ...", 156 156 + }, 157 157 + { 158 158 + in: []uint32{42, 8}, 159 159 + want: "42, 8", 160 160 + }, 161 161 + { 162 162 + in: []uint32{42}, 163 163 + want: "42", 164 164 + }, 165 165 + { 166 166 + in: []uint32{}, 167 167 + want: "", 168 168 + }, 169 169 + } 170 170 + 171 171 + for _, tt := range cases { 172 172 + t.Run(fmt.Sprintf("%v", tt.in), func(t *testing.T) { 173 173 + out := formatListUint32(tt.in, 2) 174 174 + if out != tt.want { 175 175 + t.Fatalf("want %s, got %s", tt.want, out) 176 176 + } 177 177 + }) 178 178 + } 179 179 + }

+7 -6

cmd/zoekt-sourcegraph-indexserver/queue.go

Reviewed

··· 263 263 } 264 264 265 265 // MaybeRemoveMissing will remove all queue items not in ids and return the 266 266 - // number of names removed from the queue. It will heuristically not run to 266 266 + // ids of items removed from the queue. It will heuristically not run to 267 267 // conserve resources. 268 268 // 269 269 // In the server's steady state we expect that the list of names is equal to ··· 271 271 // removals. Removal requires memory allocation and coarse locking. To avoid 272 272 // that we use a heuristic which can falsely decide it doesn't need to 273 273 // remove. However, we will converge onto removing items. 274 274 - func (q *Queue) MaybeRemoveMissing(ids []uint32) uint { 274 274 + func (q *Queue) MaybeRemoveMissing(ids []uint32) []uint32 { 275 275 q.mu.Lock() 276 276 sameSize := len(q.items) == len(ids) 277 277 q.mu.Unlock() ··· 279 279 // heuristically skip expensive work 280 280 if sameSize { 281 281 debug.Printf("skipping MaybeRemoveMissing due to same size: %d", len(ids)) 282 282 - return 0 282 282 + return nil 283 283 } 284 284 285 285 set := make(map[uint32]struct{}, len(ids)) ··· 290 290 q.mu.Lock() 291 291 defer q.mu.Unlock() 292 292 293 293 - var count uint 293 293 + var removed []uint32 294 294 for _, item := range q.items { 295 295 if _, ok := set[item.opts.RepoID]; ok { 296 296 continue ··· 303 303 item.indexState = "" 304 304 305 305 delete(q.items, item.opts.RepoID) 306 306 - count++ 306 306 + 307 307 + removed = append(removed, item.opts.RepoID) 307 308 } 308 309 309 310 metricQueueLen.Set(float64(len(q.pq))) 310 311 metricQueueCap.Set(float64(len(q.items))) 311 312 312 312 - return count 313 313 + return removed 313 314 } 314 315 315 316 // getOrAdd returns the item for repoID. If the repoID hasn't been seen before, it