fork of https://github.com/sourcegraph/zoekt
zoekt: add fgprof for full profiling (#614)
Useful in local testing to capture both on and off cpu time spent.
Should consider shipping this in the webserver as well.
Test Plan: ran zoekt with -full_profile flag and inspected output in
pprof.
author
Keegan Carruthers-Smith
committer
GitHub
date
(Jul 14, 2023, 4:18 PM +0200)
commit
f9d3a0e2
f9d3a0e2e44be9e1a9e993ab1a6ffcde54ea68b6
parent
008a775b
008a775ba82fbd66653e7523457659d1f002ed51
+89
-25
+1
cmd/zoekt/BUILD.bazel
+1
cmd/zoekt/BUILD.bazel
+57
-25
cmd/zoekt/main.go
+57
-25
cmd/zoekt/main.go
···
18
18
"context"
19
19
"flag"
20
20
"fmt"
21
+
"io"
21
22
"log"
22
23
"os"
23
24
"path/filepath"
24
25
"runtime/pprof"
25
26
"time"
26
27
28
+
"github.com/felixge/fgprof"
27
29
"github.com/sourcegraph/zoekt"
28
30
"github.com/sourcegraph/zoekt/query"
29
31
"github.com/sourcegraph/zoekt/shards"
···
76
78
return s, nil
77
79
}
78
80
81
+
func profile(path string, duration time.Duration, start func(io.Writer) (stop func())) func() bool {
82
+
if path == "" {
83
+
return func() bool { return false }
84
+
}
85
+
86
+
f, err := os.Create(path)
87
+
if err != nil {
88
+
log.Fatal(err)
89
+
}
90
+
91
+
t := time.Now()
92
+
stop := start(f)
93
+
94
+
return func() bool {
95
+
if time.Since(t) < duration {
96
+
return true
97
+
}
98
+
stop()
99
+
f.Close()
100
+
return false
101
+
}
102
+
}
103
+
104
+
func startCPUProfile(path string, duration time.Duration) func() bool {
105
+
return profile(path, duration, func(w io.Writer) func() {
106
+
if err := pprof.StartCPUProfile(w); err != nil {
107
+
log.Fatal(err)
108
+
}
109
+
110
+
return pprof.StopCPUProfile
111
+
})
112
+
}
113
+
114
+
func startFullProfile(path string, duration time.Duration) func() bool {
115
+
return profile(path, duration, func(w io.Writer) func() {
116
+
stop := fgprof.Start(w, fgprof.FormatPprof)
117
+
118
+
return func() {
119
+
if err := stop(); err != nil {
120
+
log.Fatal(err)
121
+
}
122
+
}
123
+
})
124
+
}
125
+
79
126
func main() {
80
127
shard := flag.String("shard", "", "search in a specific shard")
81
128
index := flag.String("index_dir",
82
129
filepath.Join(os.Getenv("HOME"), ".zoekt"), "search for index files in `directory`")
83
130
cpuProfile := flag.String("cpu_profile", "", "write cpu profile to `file`")
131
+
fullProfile := flag.String("full_profile", "", "write full profile to `file`")
84
132
profileTime := flag.Duration("profile_time", time.Second, "run this long to gather stats.")
85
133
verbose := flag.Bool("v", false, "print some background data")
86
134
withRepo := flag.Bool("r", false, "print the repo before the file name")
···
124
172
125
173
var sOpts zoekt.SearchOptions
126
174
sres, err := searcher.Search(context.Background(), query, &sOpts)
127
-
if *cpuProfile != "" {
128
-
// If profiling, do it another time so we measure with
129
-
// warm caches.
130
-
f, err := os.Create(*cpuProfile)
131
-
if err != nil {
132
-
log.Fatal(err)
133
-
}
134
-
defer f.Close()
135
-
if *verbose {
136
-
log.Println("Displaying matches...")
137
-
}
138
-
139
-
t := time.Now()
140
-
if err := pprof.StartCPUProfile(f); err != nil {
141
-
log.Fatal(err)
142
-
}
143
-
for {
144
-
sres, _ = searcher.Search(context.Background(), query, &sOpts)
145
-
if time.Since(t) > *profileTime {
146
-
break
147
-
}
148
-
}
149
-
pprof.StopCPUProfile()
175
+
if err != nil {
176
+
log.Fatal(err)
150
177
}
151
178
152
-
if err != nil {
153
-
log.Fatal(err)
179
+
// If profiling, do it another time so we measure with
180
+
// warm caches.
181
+
for run := startCPUProfile(*cpuProfile, *profileTime); run(); {
182
+
sres, _ = searcher.Search(context.Background(), query, &sOpts)
183
+
}
184
+
for run := startFullProfile(*fullProfile, *profileTime); run(); {
185
+
sres, _ = searcher.Search(context.Background(), query, &sOpts)
154
186
}
155
187
156
188
displayMatches(sres.Files, pat, *withRepo, *list)
+22
deps.bzl
+22
deps.bzl
···
170
170
sum = "h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=",
171
171
version = "v2.2.0",
172
172
)
173
+
go_repository(
174
+
name = "com_github_chzyer_logex",
175
+
build_file_proto_mode = "disable_global",
176
+
importpath = "github.com/chzyer/logex",
177
+
sum = "h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=",
178
+
version = "v1.1.10",
179
+
)
173
180
174
181
go_repository(
175
182
name = "com_github_chzyer_readline",
···
177
184
importpath = "github.com/chzyer/readline",
178
185
sum = "h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=",
179
186
version = "v1.5.1",
187
+
)
188
+
go_repository(
189
+
name = "com_github_chzyer_test",
190
+
build_file_proto_mode = "disable_global",
191
+
importpath = "github.com/chzyer/test",
192
+
sum = "h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=",
193
+
version = "v0.0.0-20180213035817-a1ea475d72b1",
180
194
)
181
195
182
196
go_repository(
···
346
360
sum = "h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=",
347
361
version = "v1.1.0",
348
362
)
363
+
go_repository(
364
+
name = "com_github_felixge_fgprof",
365
+
build_file_proto_mode = "disable_global",
366
+
importpath = "github.com/felixge/fgprof",
367
+
sum = "h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=",
368
+
version = "v0.9.3",
369
+
)
370
+
349
371
go_repository(
350
372
name = "com_github_flosch_pongo2_v4",
351
373
build_file_proto_mode = "disable_global",
+1
go.mod
+1
go.mod
···
6
6
github.com/andygrunwald/go-gerrit v0.0.0-20230628115649-c44fe2fbf2ca
7
7
github.com/bmatcuk/doublestar v1.3.4
8
8
github.com/edsrzf/mmap-go v1.1.0
9
+
github.com/felixge/fgprof v0.9.3
9
10
github.com/fsnotify/fsnotify v1.6.0
10
11
github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751
11
12
github.com/go-enry/go-enry/v2 v2.8.4
+8
go.sum
+8
go.sum
···
46
46
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
47
47
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
48
48
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
49
+
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
50
+
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
51
+
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
49
52
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
50
53
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
51
54
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
···
81
84
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
82
85
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
83
86
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
87
+
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
88
+
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
84
89
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
85
90
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
86
91
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
···
155
160
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
156
161
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
157
162
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
163
+
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
158
164
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
159
165
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
160
166
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
···
180
186
github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
181
187
github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
182
188
github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
189
+
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
183
190
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
184
191
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
185
192
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
···
427
434
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
428
435
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
429
436
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
437
+
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
430
438
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
431
439
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
432
440
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=