monitoring: run debug server and expose more metrics (#47) · boltless.me/zoekt@0c2b42d

+48 -18

cmd/zoekt-sourcegraph-indexserver/main.go

··· 20 20 "path/filepath" 21 21 "runtime" 22 22 "sort" 23 + "strings" 23 24 "sync" 24 25 "time" 25 26 27 + "github.com/google/zoekt/debugserver" 26 28 "go.uber.org/automaxprocs/maxprocs" 27 29 "golang.org/x/net/trace" 28 30 ··· 31 33 "github.com/keegancsmith/tmpfriend" 32 34 "github.com/prometheus/client_golang/prometheus" 33 35 "github.com/prometheus/client_golang/prometheus/promauto" 34 - "github.com/prometheus/client_golang/prometheus/promhttp" 35 36 ) 36 37 37 38 var ( ··· 52 53 Help: "A histogram of latencies for indexing a repository.", 53 54 Buckets: prometheus.ExponentialBuckets(.1, 10, 7), // 100ms -> 27min 54 55 }, []string{"state"}) // state is an indexState 56 + 57 + metricQueueLen = promauto.NewGaugeVec(prometheus.GaugeOpts{ 58 + Name: "index_queue_len", 59 + Help: "Current length of indexing queue by code host", 60 + }, []string{"codehost"}) 61 + 62 + metricNumIndexed = promauto.NewGaugeVec(prometheus.GaugeOpts{ 63 + Name: "index_num_indexed", 64 + Help: "Number of indexed repos by code host", 65 + }, []string{"codehost"}) 66 + 67 + metricFailedTotal = promauto.NewCounter(prometheus.CounterOpts{ 68 + Name: "index_failed_total", 69 + Help: "Counts failures to index", 70 + }) 71 + 72 + metricIndexedTotal = promauto.NewCounter(prometheus.CounterOpts{ 73 + Name: "index_indexed_total", 74 + Help: "Counts indexings", 75 + }) 55 76 ) 56 77 57 78 type indexState string ··· 121 142 return nil 122 143 } 123 144 145 + func codeHostFromName(repoName string) string { 146 + parts := strings.Split(repoName, "/") 147 + codehost := "unknown" 148 + if len(parts) > 1 { 149 + codehost = parts[0] 150 + } 151 + return codehost 152 + } 153 + 124 154 // Run the sync loop. This blocks forever. 125 155 func (s *Server) Run() { 126 156 removeIncompleteShards(s.IndexDir) ··· 172 202 return 173 203 } 174 204 metricResolveRevisionDuration.WithLabelValues("true").Observe(time.Since(start).Seconds()) 175 - queue.AddOrUpdate(name, commit) 205 + added := queue.AddOrUpdate(name, commit) 206 + if added { 207 + metricQueueLen.WithLabelValues(codeHostFromName(name)).Add(1.0) 208 + } 176 209 }(name) 177 210 } 178 211 sem.Wait() ··· 190 223 time.Sleep(time.Second) 191 224 continue 192 225 } 193 - 226 + metricQueueLen.WithLabelValues(codeHostFromName(name)).Add(-1.0) 194 227 start := time.Now() 195 228 args := s.defaultArgs() 196 229 args.Name = name ··· 213 246 tr := trace.New("index", args.Name) 214 247 215 248 defer func() { 249 + metricIndexedTotal.Inc() 216 250 if err != nil { 217 251 tr.SetError() 218 252 tr.LazyPrintf("error: %v", err) 219 253 state = indexStateFail 254 + metricFailedTotal.Inc() 220 255 } 221 256 tr.LazyPrintf("state: %s", state) 222 257 tr.Finish() ··· 345 380 func listIndexed(indexDir string) []string { 346 381 index := getShards(indexDir) 347 382 repoNames := make([]string, 0, len(index)) 383 + countsByHost := make(map[string]int) 348 384 for name := range index { 349 385 repoNames = append(repoNames, name) 386 + codeHost := codeHostFromName(name) 387 + countsByHost[codeHost] += 1 350 388 } 351 389 sort.Strings(repoNames) 390 + for codeHost, count := range countsByHost { 391 + metricNumIndexed.WithLabelValues(codeHost).Set(float64(count)) 392 + } 352 393 return repoNames 353 394 } 354 395 ··· 577 618 578 619 if *listen != "" { 579 620 go func() { 580 - trace.AuthRequest = func(req *http.Request) (any, sensitive bool) { 581 - return true, true 582 - } 583 - prom := promhttp.Handler() 584 - h := func(w http.ResponseWriter, r *http.Request) { 585 - switch r.URL.Path { 586 - case "/debug/requests": 587 - trace.Traces(w, r) 588 - case "/metrics": 589 - prom.ServeHTTP(w, r) 590 - default: 591 - s.ServeHTTP(w, r) 592 - } 593 - } 621 + mux := http.NewServeMux() 622 + debugserver.AddHandlers(mux, true) 623 + mux.Handle("/", s) 594 624 debug.Printf("serving HTTP on %s", *listen) 595 - log.Fatal(http.ListenAndServe(*listen, http.HandlerFunc(h))) 625 + log.Fatal(http.ListenAndServe(*listen, mux)) 596 626 }() 597 627 } 598 628

+4 -1

cmd/zoekt-sourcegraph-indexserver/queue.go

··· 62 62 63 63 // AddOrUpdate sets which commit to index next for repoName. If repoName is 64 64 // already in the queue, it is updated. 65 - func (q *Queue) AddOrUpdate(repoName, commit string) { 65 + func (q *Queue) AddOrUpdate(repoName, commit string) bool { 66 + added := false 66 67 q.mu.Lock() 67 68 item := q.get(repoName) 68 69 item.latestCommit = commit 69 70 if item.heapIdx < 0 { 71 + added = true 70 72 q.seq++ 71 73 item.seq = q.seq 72 74 heap.Push(&q.pq, item) ··· 74 76 heap.Fix(&q.pq, item.heapIdx) 75 77 } 76 78 q.mu.Unlock() 79 + return added 77 80 } 78 81 79 82 // SetIndexed sets what the currently indexed commit is for repoName.

+2 -22

cmd/zoekt-webserver/main.go

··· 26 26 "io/ioutil" 27 27 "log" 28 28 "net/http" 29 - "net/http/pprof" 30 29 "os" 31 30 "path/filepath" 32 31 "strings" ··· 35 34 36 35 "github.com/google/zoekt" 37 36 "github.com/google/zoekt/build" 37 + "github.com/google/zoekt/debugserver" 38 38 "github.com/google/zoekt/query" 39 39 "github.com/google/zoekt/shards" 40 40 "github.com/google/zoekt/web" 41 41 "github.com/prometheus/client_golang/prometheus" 42 - "github.com/prometheus/client_golang/prometheus/promhttp" 43 42 "go.uber.org/automaxprocs/maxprocs" 44 - "golang.org/x/net/trace" 45 43 ) 46 44 47 45 const logFormat = "2006-01-02T15-04-05.999999999Z07" ··· 207 205 log.Fatal(err) 208 206 } 209 207 210 - handler.Handle("/metrics", promhttp.Handler()) 211 - 212 - if *enablePprof { 213 - // Zoekt is only available in-cluster for Sourcegraph. So only admins 214 - // can reach the endpoint, so we don't need to auth trace 215 - // requests. This allows the admin proxy to work. 216 - trace.AuthRequest = func(req *http.Request) (any, sensitive bool) { 217 - return true, true 218 - } 219 - 220 - handler.HandleFunc("/debug/pprof/", pprof.Index) 221 - handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 222 - handler.HandleFunc("/debug/pprof/profile", pprof.Profile) 223 - handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 224 - handler.HandleFunc("/debug/pprof/trace", pprof.Trace) 225 - handler.HandleFunc("/debug/requests/", trace.Traces) 226 - handler.HandleFunc("/debug/events/", trace.Events) 227 - } 228 - 208 + debugserver.AddHandlers(handler, *enablePprof) 229 209 handler.HandleFunc("/healthz", healthz) 230 210 231 211 watchdogAddr := "http://" + *listen

+44

debugserver/debug.go

··· 1 + package debugserver 2 + 3 + import ( 4 + "net/http" 5 + "net/http/pprof" 6 + 7 + "github.com/prometheus/client_golang/prometheus/promhttp" 8 + "golang.org/x/net/trace" 9 + ) 10 + 11 + func AddHandlers(mux *http.ServeMux, enablePprof bool) { 12 + trace.AuthRequest = func(req *http.Request) (any, sensitive bool) { 13 + return true, true 14 + } 15 + 16 + index := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 + _, _ = w.Write([]byte(` 18 + <a href="vars">Vars</a><br> 19 + <a href="debug/pprof/">PProf</a><br> 20 + <a href="metrics">Metrics</a><br> 21 + <a href="debug/requests">Requests</a><br> 22 + <a href="debug/events">Events</a><br> 23 + `)) 24 + _, _ = w.Write([]byte(` 25 + <br> 26 + <form method="post" action="gc" style="display: inline;"><input type="submit" value="GC"></form> 27 + <form method="post" action="freeosmemory" style="display: inline;"><input type="submit" value="Free OS Memory"></form> 28 + `)) 29 + }) 30 + mux.Handle("/debug", index) 31 + mux.Handle("/vars", http.HandlerFunc(expvarHandler)) 32 + mux.Handle("/gc", http.HandlerFunc(gcHandler)) 33 + mux.Handle("/freeosmemory", http.HandlerFunc(freeOSMemoryHandler)) 34 + if enablePprof { 35 + mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) 36 + mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) 37 + mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) 38 + mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) 39 + mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) 40 + } 41 + mux.Handle("/debug/requests", http.HandlerFunc(trace.Traces)) 42 + mux.Handle("/debug/events", http.HandlerFunc(trace.Events)) 43 + mux.Handle("/metrics", promhttp.Handler()) 44 + }

+48

debugserver/expvar.go

··· 1 + package debugserver 2 + 3 + import ( 4 + "expvar" 5 + "fmt" 6 + "net/http" 7 + "runtime" 8 + "runtime/debug" 9 + "time" 10 + ) 11 + 12 + // expvarHandler is copied from package expvar and exported so that it 13 + // can be mounted on any ServeMux, not just http.DefaultServeMux. 14 + func expvarHandler(w http.ResponseWriter, r *http.Request) { 15 + w.Header().Set("Content-Type", "application/json; charset=utf-8") 16 + fmt.Fprintln(w, "{") 17 + first := true 18 + expvar.Do(func(kv expvar.KeyValue) { 19 + if !first { 20 + fmt.Fprintln(w, ",") 21 + } 22 + first = false 23 + fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) 24 + }) 25 + fmt.Fprintln(w, "\n}") 26 + } 27 + 28 + func gcHandler(w http.ResponseWriter, r *http.Request) { 29 + if r.Method != "POST" { 30 + http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) 31 + return 32 + } 33 + 34 + t0 := time.Now() 35 + runtime.GC() 36 + fmt.Fprintf(w, "GC took %s\n", time.Since(t0)) 37 + } 38 + 39 + func freeOSMemoryHandler(w http.ResponseWriter, r *http.Request) { 40 + if r.Method != "POST" { 41 + http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) 42 + return 43 + } 44 + 45 + t0 := time.Now() 46 + debug.FreeOSMemory() 47 + fmt.Fprintf(w, "FreeOSMemory took %s\n", time.Since(t0)) 48 + }

Configure Feed

Configure Feed