boltless.me / zoekt
fork of https://github.com/sourcegraph/zoekt
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

ctags: timeout parse requests after a minute (#122)

I believe the most common reason for a repo to not index is making ctags
hang. This commit implements a timeout which will detect and fail
indexing if ctags takes to long to respond for a single file.

We hard fail so we can notice these failures and follow-up to see what
is causing the issue. The main benefit of this change is failing an
index job much sooner (1minute), rather than waiting for indexserver to
kill it after 30 minutes.

author
Keegan Carruthers-Smith committer
GitHub date (Aug 5, 2021, 11:05 AM +0200) commit c8b1e5f5 parent d50ed977
+88 -17
+87 -16
ctags/json.go
··· 15 15 package ctags 16 16 17 17 import ( 18 + "fmt" 18 19 "log" 20 + "os" 19 21 "strings" 20 22 "sync" 23 + "time" 21 24 22 25 goctags "github.com/sourcegraph/go-ctags" 23 26 ) ··· 27 30 type Parser = goctags.Parser 28 31 type Entry = goctags.Entry 29 32 30 - func newProcess(bin string) (Parser, error) { 31 - return goctags.New(goctags.Options{ 32 - Bin: bin, 33 - }) 33 + type parseReq struct { 34 + Name string 35 + Content []byte 36 + } 37 + 38 + type parseResp struct { 39 + Entries []*Entry 40 + Err error 34 41 } 35 42 36 43 type lockedParser struct { 37 - p Parser 38 - l sync.Mutex 44 + mu sync.Mutex 45 + opts goctags.Options 46 + p Parser 47 + send chan<- parseReq 48 + recv <-chan parseResp 39 49 } 40 50 51 + // parseTimeout is how long we wait for a response for parsing a single file 52 + // in ctags. 1 minute is a very conservative timeout which we should only hit 53 + // if ctags hangs. 54 + const parseTimeout = time.Minute 55 + 56 + // Parse wraps go-ctags Parse. It lazily starts the process and adds a timeout 57 + // around parse requests. Additionally it serializes access to the parsing 58 + // process. The timeout is important since we occasionally come across 59 + // documents which hang universal-ctags. 41 60 func (lp *lockedParser) Parse(name string, content []byte) ([]*Entry, error) { 42 - lp.l.Lock() 43 - defer lp.l.Unlock() 44 - return lp.p.Parse(name, content) 61 + lp.mu.Lock() 62 + defer lp.mu.Unlock() 63 + 64 + if lp.p == nil { 65 + p, err := goctags.New(lp.opts) 66 + if err != nil { 67 + return nil, err 68 + } 69 + send := make(chan parseReq) 70 + // buf of 1 so we avoid blocking sends in the parser if we exit early. 71 + recv := make(chan parseResp, 1) 72 + 73 + go func() { 74 + defer close(recv) 75 + for req := range send { 76 + entries, err := p.Parse(req.Name, req.Content) 77 + recv <- parseResp{Entries: entries, Err: err} 78 + } 79 + }() 80 + 81 + lp.p = p 82 + lp.send = send 83 + lp.recv = recv 84 + } 85 + 86 + lp.send <- parseReq{Name: name, Content: content} 87 + 88 + deadline := time.NewTimer(parseTimeout) 89 + defer deadline.Stop() 90 + 91 + select { 92 + case resp := <-lp.recv: 93 + return resp.Entries, resp.Err 94 + case <-deadline.C: 95 + // Error out since ctags hanging is a sign something bad is happening. 96 + lp.close() 97 + return nil, fmt.Errorf("ctags timedout after %s parsing %s", parseTimeout, name) 98 + } 45 99 } 46 100 47 101 func (lp *lockedParser) Close() { 48 - lp.l.Lock() 49 - defer lp.l.Unlock() 102 + lp.mu.Lock() 103 + defer lp.mu.Unlock() 104 + lp.close() 105 + } 106 + 107 + // close assumes lp.mu is held. 108 + func (lp *lockedParser) close() { 109 + if lp.p == nil { 110 + return 111 + } 112 + 50 113 lp.p.Close() 114 + lp.p = nil 115 + close(lp.send) 116 + lp.send = nil 117 + lp.recv = nil 51 118 } 52 119 53 120 // NewParser creates a parser that is implemented by the given 54 121 // universal-ctags binary. The parser is safe for concurrent use. 55 122 func NewParser(bin string) (Parser, error) { 56 123 if strings.Contains(bin, "universal-ctags") { 57 - // todo: restart, parallelization. 58 - proc, err := newProcess(bin) 59 - if err != nil { 60 - return nil, err 124 + opts := goctags.Options{ 125 + Bin: bin, 126 + Info: log.New(os.Stderr, "CTAGS INF: ", log.LstdFlags), 61 127 } 62 - return &lockedParser{p: proc}, nil 128 + if debug { 129 + opts.Debug = log.New(os.Stderr, "CTAGS DBG: ", log.LstdFlags) 130 + } 131 + return &lockedParser{ 132 + opts: opts, 133 + }, nil 63 134 } 64 135 65 136 log.Fatal("not implemented")
+1 -1
ctags/json_test.go
··· 27 27 t.Skip(err) 28 28 } 29 29 30 - p, err := newProcess("universal-ctags") 30 + p, err := NewParser("universal-ctags") 31 31 if err != nil { 32 32 t.Fatal("newProcess", err) 33 33 }