fork of https://github.com/sourcegraph/zoekt
zoekt-webserver: improve /healthz endpoint (#134)
* zoekt-webserver: improve /healthz endpoint
This commit changes the /healthz endpoint to perform a cheap search
rather than always returning 200 OK. It also makes the watchdog use this
endpoint rather than / which is too expensive and locks a lot, hence
explaining the frequent watchdog panics we have in large instances.
* fixup! add test
author
Tomás Senart
committer
GitHub
date
(Aug 30, 2021, 1:37 PM +0200)
commit
b6c9f66d
b6c9f66d5e9387554df25c2a2a22fe05d9f3e0dc
parent
21016c01
21016c015edf6a937d57c96ae1ccf3bf2ecc9f9e
+83
-9
+4
-9
cmd/zoekt-webserver/main.go
+4
-9
cmd/zoekt-webserver/main.go
···
222
222
}
223
223
224
224
debugserver.AddHandlers(handler, *enablePprof)
225
-
handler.HandleFunc("/healthz", healthz)
226
225
227
226
// Sourcegraph: We use environment variables to configure watchdog since
228
227
// they are more convenient than flags in containerized environments.
···
231
230
watchdogTick, _ = time.ParseDuration(v)
232
231
log.Printf("custom ZOEKT_WATCHDOG_TICK=%v", watchdogTick)
233
232
}
233
+
234
234
watchdogErrCount := 3
235
235
if v := os.Getenv("ZOEKT_WATCHDOG_ERRORS"); v != "" {
236
236
watchdogErrCount, _ = strconv.Atoi(v)
237
237
log.Printf("custom ZOEKT_WATCHDOG_ERRORS=%d", watchdogErrCount)
238
238
}
239
+
239
240
watchdogAddr := "http://" + *listen
240
241
if *sslCert != "" || *sslKey != "" {
241
242
watchdogAddr = "https://" + *listen
242
243
}
244
+
watchdogAddr += "/healthz"
245
+
243
246
if watchdogErrCount > 0 && watchdogTick > 0 {
244
247
go watchdog(watchdogTick, watchdogErrCount, watchdogAddr)
245
248
} else {
···
316
319
317
320
log.Printf("shutting down")
318
321
return srv.Shutdown(ctx)
319
-
}
320
-
321
-
// Always returns 200 OK.
322
-
// Used for kubernetes liveness and readiness checks.
323
-
// https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
324
-
func healthz(w http.ResponseWriter, req *http.Request) {
325
-
w.Header().Set("Content-Type", "text/html; charset=utf-8")
326
-
w.Write([]byte("OK"))
327
322
}
328
323
329
324
func watchdogOnce(ctx context.Context, client *http.Client, addr string) error {
+61
web/e2e_test.go
+61
web/e2e_test.go
···
17
17
import (
18
18
"bytes"
19
19
"context"
20
+
"encoding/json"
20
21
"fmt"
21
22
"io/ioutil"
22
23
"log"
23
24
"net/http"
24
25
"net/http/httptest"
26
+
"reflect"
25
27
"strings"
26
28
"testing"
27
29
"time"
···
453
455
t.Fatalf("got %s, want substring %q", result, want)
454
456
}
455
457
}
458
+
459
+
func TestHealthz(t *testing.T) {
460
+
b, err := zoekt.NewIndexBuilder(&zoekt.Repository{
461
+
Name: "name",
462
+
})
463
+
if err != nil {
464
+
t.Fatalf("NewIndexBuilder: %v", err)
465
+
}
466
+
467
+
for i := 0; i < 2; i++ {
468
+
if err := b.Add(zoekt.Document{
469
+
Name: fmt.Sprintf("file%d", i),
470
+
Content: []byte("bla"),
471
+
}); err != nil {
472
+
t.Fatalf("Add: %v", err)
473
+
}
474
+
}
475
+
s := searcherForTest(t, b)
476
+
srv := Server{
477
+
Searcher: s,
478
+
Top: Top,
479
+
HTML: true,
480
+
}
481
+
482
+
mux, err := NewMux(&srv)
483
+
if err != nil {
484
+
t.Fatalf("NewMux: %v", err)
485
+
}
486
+
487
+
ts := httptest.NewServer(mux)
488
+
t.Cleanup(ts.Close)
489
+
490
+
req, err := http.NewRequest("GET", ts.URL+"/healthz", nil)
491
+
if err != nil {
492
+
t.Fatalf("NewRequest: %v", err)
493
+
}
494
+
res, err := http.DefaultClient.Do(req)
495
+
if err != nil {
496
+
t.Fatalf("Do(%v): %v", req, err)
497
+
}
498
+
499
+
t.Cleanup(func() {
500
+
res.Body.Close()
501
+
})
502
+
503
+
if res.StatusCode != http.StatusOK {
504
+
t.Fatalf("want 200 status code, got: %v", res.StatusCode)
505
+
}
506
+
507
+
var result zoekt.SearchResult
508
+
err = json.NewDecoder(res.Body).Decode(&result)
509
+
if err != nil {
510
+
t.Fatalf("json.Decode: %v", err)
511
+
}
512
+
513
+
if reflect.DeepEqual(result, zoekt.SearchResult{}) {
514
+
t.Fatal("empty result in response")
515
+
}
516
+
}
+18
web/server.go
+18
web/server.go
···
16
16
17
17
import (
18
18
"bytes"
19
+
"encoding/json"
19
20
"fmt"
20
21
"html/template"
21
22
"log"
···
178
179
mux.Handle(stream.DefaultSSEPath, stream.Server(traceAwareSearcher{s.Searcher})) // /stream
179
180
}
180
181
182
+
mux.HandleFunc("/healthz", s.serveHealthz)
183
+
181
184
return mux, nil
185
+
}
186
+
187
+
func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
188
+
q := &query.Const{Value: true}
189
+
opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
190
+
191
+
result, err := s.Searcher.Search(r.Context(), q, opts)
192
+
if err != nil {
193
+
http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
194
+
return
195
+
}
196
+
197
+
w.Header().Set("Content-Type", "application/json")
198
+
199
+
_ = json.NewEncoder(w).Encode(result)
182
200
}
183
201
184
202
func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {