fork of https://github.com/sourcegraph/zoekt
zoekt: add metric to track duration of commit fetch operations (#245)
author
Geoffrey Gilmore
committer
GitHub
date
(Jan 19, 2022, 8:59 AM -0800)
commit
b9c6cbce
b9c6cbce232b1625b62fcdeb7561ba027280a4fe
parent
8de66e02
8de66e02fa9b7d09837cc3362f9f5525bb7a14b8
+33
-11
+7
-2
cmd/zoekt-sourcegraph-indexserver/index.go
+7
-2
cmd/zoekt-sourcegraph-indexserver/index.go
···
174
174
175
175
cmd = exec.CommandContext(ctx, "git", fetchArgs...)
176
176
cmd.Stdin = &bytes.Buffer{}
177
-
if err := runCmd(cmd); err != nil {
177
+
178
+
err = runCmd(cmd)
179
+
fetchDuration := time.Since(fetchStart)
180
+
if err != nil {
181
+
metricFetchDuration.WithLabelValues("false", repoNameForMetric(o.Name)).Observe(fetchDuration.Seconds())
178
182
return err
179
183
}
180
184
181
-
debug.Printf("fetched git data for %q (%d commit(s)) in %s", o.Name, len(commits), time.Since(fetchStart))
185
+
metricFetchDuration.WithLabelValues("true", repoNameForMetric(o.Name)).Observe(fetchDuration.Seconds())
186
+
debug.Printf("fetched git data for %q (%d commit(s)) in %s", o.Name, len(commits), fetchDuration)
182
187
183
188
// We then create the relevant refs for each fetched commit.
184
189
for _, b := range o.Branches {
+26
-9
cmd/zoekt-sourcegraph-indexserver/main.go
+26
-9
cmd/zoekt-sourcegraph-indexserver/main.go
···
68
68
"name", // name of the repository that was indexed
69
69
})
70
70
71
+
metricFetchDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
72
+
Name: "index_fetch_seconds",
73
+
Help: "A histogram of latencies for fetching a repository.",
74
+
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 20, 30, 60, 180, 300, 600}, // 50ms -> 10 minutes
75
+
}, []string{
76
+
"success", // true|false
77
+
"name", // the name of the repository that the commits were fetched from
78
+
})
79
+
71
80
metricIndexIncrementalIndexState = promauto.NewCounterVec(prometheus.CounterOpts{
72
81
Name: "index_incremental_index_state",
73
82
Help: "A count of the state on disk vs what we want to build. See zoekt/build.IndexState.",
···
351
360
state, err := s.Index(args)
352
361
s.muIndexDir.Unlock()
353
362
354
-
// Check to see if we want to be able to capture separate indexing metrics for this repository.
355
-
// If we don't, set to a default string to keep the cardinality for the Prometheus metric manageable.
356
-
repoNameForMetric := ""
357
-
if _, ok = reposWithSeparateIndexingMetrics[opts.Name]; ok {
358
-
repoNameForMetric = opts.Name
359
-
}
363
+
elapsed := time.Since(start)
360
364
361
-
metricIndexDuration.WithLabelValues(string(state), repoNameForMetric).Observe(time.Since(start).Seconds())
365
+
metricIndexDuration.WithLabelValues(string(state), repoNameForMetric(opts.Name)).Observe(elapsed.Seconds())
362
366
363
367
if err != nil {
364
368
log.Printf("error indexing %s: %s", args.String(), err)
365
369
}
370
+
366
371
switch state {
367
372
case indexStateSuccess:
368
-
log.Printf("updated index %s in %v", args.String(), time.Since(start))
373
+
log.Printf("updated index %s in %v", args.String(), elapsed)
369
374
case indexStateSuccessMeta:
370
-
log.Printf("updated meta %s in %v", args.String(), time.Since(start))
375
+
log.Printf("updated meta %s in %v", args.String(), elapsed)
371
376
}
372
377
s.queue.SetIndexed(opts, state)
373
378
}
379
+
}
380
+
381
+
// repoNameForMetric returns a normalized version of the given repository name that is
382
+
// suitable for use with Prometheus metrics.
383
+
func repoNameForMetric(repo string) string {
384
+
// Check to see if we want to be able to capture separate indexing metrics for this repository.
385
+
// If we don't, set to a default string to keep the cardinality for the Prometheus metric manageable.
386
+
if _, ok := reposWithSeparateIndexingMetrics[repo]; ok {
387
+
return repo
388
+
}
389
+
390
+
return ""
374
391
}
375
392
376
393
func batched(slice []uint32, size int) <-chan []uint32 {