boltless.me / zoekt
fork of https://github.com/sourcegraph/zoekt
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

ctags: monitor symbol analysis and report stuck documents (#678)

This adds a monitor which will report every minute the progress of
symbol analysis. Additionally, if a document is taking too long to
analyse (10s) we report it.

At first this is just reporting via stdlog. However, once we are
comfortable with thresholds around this we can likely also include a way
to kill analysis for a file.

Test Plan: Adjusted monitorReportStatus to 1s then indexed the
sourcegraph repo and inspected the output

$ go run ./cmd/zoekt-git-index -require_ctags ../sourcegraph/
2023/11/03 16:03:10 attempting to index 14533 total files
2023/11/03 16:03:13 DEBUG: symbol analysis still running for shard statistics: duration=1s symbols=15805 bytes=44288971
2023/11/03 16:03:14 DEBUG: symbol analysis still running for shard statistics: duration=2s symbols=26189 bytes=51564417
2023/11/03 16:03:15 DEBUG: symbol analysis still running for shard statistics: duration=3s symbols=55613 bytes=64748084
2023/11/03 16:03:16 DEBUG: symbol analysis still running for shard statistics: duration=4s symbols=86557 bytes=93771404
2023/11/03 16:03:17 DEBUG: symbol analysis still running for shard statistics: duration=5s symbols=125352 bytes=116319453
2023/11/03 16:03:18 symbol analysis finished for shard statistics: duration=5s symbols=142951 bytes=129180023
2023/11/03 16:03:22 finished shard github.com%2Fsourcegraph%2Fsourcegraph_v16.00000.zoekt: 283983298 index bytes (overhead 2.8), 14533 files processed

I then added a random sleep for a minute in a file to see the stuck
reporting:

$ go run ./cmd/zoekt-git-index -require_ctags ../sourcegraph/
2023/11/03 16:14:57 attempting to index 14533 total files
2023/11/03 16:15:15 WARN: symbol analysis for README.md (3485 bytes) has been running for 14s
2023/11/03 16:15:25 WARN: symbol analysis for README.md (3485 bytes) has been running for 24s
2023/11/03 16:15:45 WARN: symbol analysis for README.md (3485 bytes) has been running for 44s
2023/11/03 16:16:00 DEBUG: symbol analysis still running for shard statistics: duration=1m0s symbols=958 bytes=624329
2023/11/03 16:16:00 symbol analysis for README.md (size 3485 bytes) is done and found 4 symbols
2023/11/03 16:16:06 symbol analysis finished for shard statistics: duration=1m5s symbols=142951 bytes=129180023
2023/11/03 16:16:10 finished shard github.com%2Fsourcegraph%2Fsourcegraph_v16.00000.zoekt: 283983299 index bytes (overhead 2.8), 14533 files processed

author
Keegan Carruthers-Smith committer
GitHub date (Nov 6, 2023, 11:17 AM +0200) commit 0ff0dd58 parent c7e066e9
+117
+117
build/ctags.go
··· 17 17 import ( 18 18 "bytes" 19 19 "fmt" 20 + "log" 20 21 "strings" 22 + "sync" 23 + "time" 21 24 22 25 "github.com/sourcegraph/zoekt" 23 26 "github.com/sourcegraph/zoekt/ctags" ··· 38 41 } 39 42 40 43 func ctagsAddSymbolsParserMap(todo []*zoekt.Document, languageMap ctags.LanguageMap, parserMap ctags.ParserMap) error { 44 + monitor := newMonitor() 45 + defer monitor.Stop() 46 + 41 47 for _, doc := range todo { 42 48 if doc.Symbols != nil { 43 49 continue ··· 59 65 } 60 66 } 61 67 68 + monitor.BeginParsing(doc) 62 69 es, err := parser.Parse(doc.Name, doc.Content) 70 + monitor.EndParsing(es) 71 + 63 72 if err != nil { 64 73 return err 65 74 } ··· 167 176 } 168 177 return out 169 178 } 179 + 180 + // monitorReportStuck is how long we need to be analysing a document before 181 + // reporting it to stdout. 182 + const monitorReportStuck = 10 * time.Second 183 + 184 + // monitorReportStatus is how often we given status updates 185 + const monitorReportStatus = time.Minute 186 + 187 + type monitor struct { 188 + mu sync.Mutex 189 + 190 + lastUpdate time.Time 191 + 192 + start time.Time 193 + totalSize int 194 + totalSymbols int 195 + 196 + currentDocName string 197 + currentDocSize int 198 + currentDocStuckCount int 199 + 200 + done chan struct{} 201 + } 202 + 203 + func newMonitor() *monitor { 204 + start := time.Now() 205 + m := &monitor{ 206 + start: start, 207 + lastUpdate: start, 208 + done: make(chan struct{}), 209 + } 210 + go m.run() 211 + return m 212 + } 213 + 214 + func (m *monitor) BeginParsing(doc *zoekt.Document) { 215 + now := time.Now() 216 + m.mu.Lock() 217 + m.lastUpdate = now 218 + 219 + // set current doc 220 + m.currentDocName = doc.Name 221 + m.currentDocSize = len(doc.Content) 222 + 223 + m.mu.Unlock() 224 + } 225 + 226 + func (m *monitor) EndParsing(entries []*ctags.Entry) { 227 + now := time.Now() 228 + m.mu.Lock() 229 + m.lastUpdate = now 230 + 231 + // update aggregate stats 232 + m.totalSize += m.currentDocSize 233 + m.totalSymbols += len(entries) 234 + 235 + // inform done if we warned about current document 236 + if m.currentDocStuckCount > 0 { 237 + log.Printf("symbol analysis for %s (size %d bytes) is done and found %d symbols", m.currentDocName, m.currentDocSize, len(entries)) 238 + m.currentDocStuckCount = 0 239 + } 240 + 241 + // unset current document 242 + m.currentDocName = "" 243 + m.currentDocSize = 0 244 + 245 + m.mu.Unlock() 246 + } 247 + 248 + func (m *monitor) Stop() { 249 + close(m.done) 250 + } 251 + 252 + func (m *monitor) run() { 253 + stuckTicker := time.NewTicker(monitorReportStuck / 2) // half due to sampling theorem (nyquist) 254 + statusTicker := time.NewTicker(monitorReportStatus) 255 + 256 + defer stuckTicker.Stop() 257 + defer statusTicker.Stop() 258 + 259 + for { 260 + select { 261 + case <-m.done: 262 + now := time.Now() 263 + m.mu.Lock() 264 + log.Printf("symbol analysis finished for shard statistics: duration=%v symbols=%d bytes=%d", now.Sub(m.start).Truncate(time.Second), m.totalSymbols, m.totalSize) 265 + m.mu.Unlock() 266 + return 267 + 268 + case <-stuckTicker.C: 269 + now := time.Now() 270 + m.mu.Lock() 271 + running := now.Sub(m.lastUpdate).Truncate(time.Second) 272 + report := monitorReportStuck * (1 << m.currentDocStuckCount) // double the amount of time each time we report 273 + if m.currentDocName != "" && running >= report { 274 + m.currentDocStuckCount++ 275 + log.Printf("WARN: symbol analysis for %s (%d bytes) has been running for %v", m.currentDocName, m.currentDocSize, running) 276 + } 277 + m.mu.Unlock() 278 + 279 + case <-statusTicker.C: 280 + now := time.Now() 281 + m.mu.Lock() 282 + log.Printf("DEBUG: symbol analysis still running for shard statistics: duration=%v symbols=%d bytes=%d", now.Sub(m.start).Truncate(time.Second), m.totalSymbols, m.totalSize) 283 + m.mu.Unlock() 284 + } 285 + } 286 + }