fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 "time"
32
33 "github.com/grafana/regexp"
34 "github.com/sourcegraph/zoekt"
35 zjson "github.com/sourcegraph/zoekt/json"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/rpc"
38 "github.com/sourcegraph/zoekt/stream"
39)
40
41var Funcmap = template.FuncMap{
42 "Inc": func(orig int) int {
43 return orig + 1
44 },
45 "More": func(orig int) int {
46 return orig * 3
47 },
48 "HumanUnit": func(orig int64) string {
49 b := orig
50 suffix := ""
51 if orig > 10*(1<<30) {
52 suffix = "G"
53 b = orig / (1 << 30)
54 } else if orig > 10*(1<<20) {
55 suffix = "M"
56 b = orig / (1 << 20)
57 } else if orig > 10*(1<<10) {
58 suffix = "K"
59 b = orig / (1 << 10)
60 }
61
62 return fmt.Sprintf("%d%s", b, suffix)
63 },
64 "LimitPre": func(limit int, pre string) string {
65 if len(pre) < limit {
66 return pre
67 }
68 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
69 },
70 "LimitPost": func(limit int, post string) string {
71 if len(post) < limit {
72 return post
73 }
74 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
75 },
76}
77
78const defaultNumResults = 50
79
80type Server struct {
81 Searcher zoekt.Streamer
82
83 // Serve HTML interface
84 HTML bool
85
86 // Serve RPC
87 RPC bool
88
89 // If set, show files from the index.
90 Print bool
91
92 // Version string for this server.
93 Version string
94
95 // Depending on the Host header, add a query to the entry
96 // page. For example, when serving on "search.myproject.org"
97 // we could add "r:myproject" automatically. This allows a
98 // single instance to serve as search engine for multiple
99 // domains.
100 HostCustomQueries map[string]string
101
102 // This should contain the following templates: "repolist"
103 // (for the repo search result page), "result" for
104 // the search results, "search" (for the opening page),
105 // "box" for the search query input element and
106 // "print" for the show file functionality.
107 Top *template.Template
108
109 repolist *template.Template
110 search *template.Template
111 result *template.Template
112 print *template.Template
113 about *template.Template
114 robots *template.Template
115
116 startTime time.Time
117
118 templateMu sync.Mutex
119 templateCache map[string]*template.Template
120
121 lastStatsMu sync.Mutex
122 lastStats *zoekt.RepoStats
123 lastStatsTS time.Time
124}
125
126func (s *Server) getTemplate(str string) *template.Template {
127 s.templateMu.Lock()
128 defer s.templateMu.Unlock()
129 t := s.templateCache[str]
130 if t != nil {
131 return t
132 }
133
134 t, err := template.New("cache").Parse(str)
135 if err != nil {
136 log.Printf("template parse error: %v", err)
137 t = template.Must(template.New("empty").Parse(""))
138 }
139 s.templateCache[str] = t
140 return t
141}
142
143func NewMux(s *Server) (*http.ServeMux, error) {
144 s.print = s.Top.Lookup("print")
145 if s.print == nil {
146 return nil, fmt.Errorf("missing template 'print'")
147 }
148
149 for k, v := range map[string]**template.Template{
150 "results": &s.result,
151 "print": &s.print,
152 "search": &s.search,
153 "repolist": &s.repolist,
154 "about": &s.about,
155 "robots": &s.robots,
156 } {
157 *v = s.Top.Lookup(k)
158 if *v == nil {
159 return nil, fmt.Errorf("missing template %q", k)
160 }
161 }
162
163 s.templateCache = map[string]*template.Template{}
164 s.startTime = time.Now()
165
166 mux := http.NewServeMux()
167
168 if s.HTML {
169 mux.HandleFunc("/robots.txt", s.serveRobots)
170 mux.HandleFunc("/search", s.serveSearch)
171 mux.HandleFunc("/", s.serveSearchBox)
172 mux.HandleFunc("/about", s.serveAbout)
173 mux.HandleFunc("/print", s.servePrint)
174 }
175 if s.RPC {
176 mux.Handle(rpc.DefaultRPCPath, rpc.Server(traceAwareSearcher{s.Searcher})) // /rpc
177 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
178 mux.Handle(stream.DefaultSSEPath, stream.Server(traceAwareSearcher{s.Searcher})) // /stream
179 }
180
181 mux.HandleFunc("/healthz", s.serveHealthz)
182
183 return mux, nil
184}
185
186func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
187 q := &query.Const{Value: true}
188 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
189
190 result, err := s.Searcher.Search(r.Context(), q, opts)
191 if err != nil {
192 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
193 return
194 }
195
196 w.Header().Set("Content-Type", "application/json")
197
198 _ = json.NewEncoder(w).Encode(result)
199}
200
201func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
202 result, err := s.serveSearchErr(r)
203 if err != nil {
204 http.Error(w, err.Error(), http.StatusTeapot)
205 return
206 }
207
208 qvals := r.URL.Query()
209 if qvals.Get("format") == "json" {
210 w.Header().Add("Content-Type", "application/json")
211 encoder := json.NewEncoder(w)
212 encoder.Encode(result)
213 return
214 }
215
216 var buf bytes.Buffer
217 if result.Repos != nil {
218 err = s.repolist.Execute(&buf, &result.Repos)
219 } else if result.Result != nil {
220 err = s.result.Execute(&buf, &result.Result)
221 }
222 if err != nil {
223 http.Error(w, err.Error(), http.StatusTeapot)
224 return
225 }
226 w.Write(buf.Bytes())
227}
228
229func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
230 qvals := r.URL.Query()
231
232 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
233
234 queryStr := qvals.Get("q")
235 if queryStr == "" {
236 return nil, fmt.Errorf("no query found")
237 }
238
239 q, err := query.Parse(queryStr)
240 if err != nil {
241 return nil, err
242 }
243
244 // Experimental: The query string and boost exact phrases of it.
245 if phraseBoost, err := strconv.ParseFloat(qvals.Get("phrase-boost"), 64); err == nil {
246 q = query.ExpirementalPhraseBoost(q, queryStr, query.ExperimentalPhraseBoostOptions{
247 Boost: phraseBoost,
248 })
249 }
250
251 repoOnly := true
252 query.VisitAtoms(q, func(q query.Q) {
253 _, ok := q.(*query.Repo)
254 repoOnly = repoOnly && ok
255 })
256 if repoOnly {
257 repos, err := s.serveListReposErr(q, queryStr, r)
258 if err == nil {
259 return &ApiSearchResult{Repos: repos}, nil
260 }
261 return nil, err
262 }
263
264 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
265 repos, err := s.serveListReposErr(q, queryStr, r)
266 if err == nil {
267 return &ApiSearchResult{Repos: repos}, nil
268 }
269 return nil, err
270 }
271
272 numStr := qvals.Get("num")
273
274 num, err := strconv.Atoi(numStr)
275 if err != nil || num <= 0 {
276 num = defaultNumResults
277 }
278
279 sOpts := zoekt.SearchOptions{
280 MaxWallTime: 10 * time.Second,
281 }
282
283 numCtxLines := 0
284 if qvals.Get("format") == "json" {
285 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
286 numCtxLines, err = strconv.Atoi(ctxLinesStr)
287 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
288 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
289 }
290 }
291 }
292 sOpts.NumContextLines = numCtxLines
293
294 sOpts.SetDefaults()
295 sOpts.MaxDocDisplayCount = num
296 sOpts.DebugScore = debugScore
297
298 ctx := r.Context()
299 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
300 return nil, err
301 }
302
303 result, err := s.Searcher.Search(ctx, q, &sOpts)
304 if err != nil {
305 return nil, err
306 }
307
308 fileMatches, err := s.formatResults(result, queryStr, s.Print)
309 if err != nil {
310 return nil, err
311 }
312
313 res := ResultInput{
314 Last: LastInput{
315 Query: queryStr,
316 Num: num,
317 AutoFocus: true,
318 },
319 Stats: result.Stats,
320 Query: q.String(),
321 QueryStr: queryStr,
322 FileMatches: fileMatches,
323 }
324 if res.Stats.Wait < res.Stats.Duration/10 {
325 // Suppress queueing stats if they are neglible.
326 res.Stats.Wait = 0
327 }
328
329 res.Last.Debug = debugScore
330 return &ApiSearchResult{Result: &res}, nil
331}
332
333func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
334 err := s.servePrintErr(w, r)
335 if err != nil {
336 http.Error(w, err.Error(), http.StatusTeapot)
337 }
338}
339
340const statsStaleNess = 30 * time.Second
341
342func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
343 s.lastStatsMu.Lock()
344 stats := s.lastStats
345 if time.Since(s.lastStatsTS) > statsStaleNess {
346 stats = nil
347 }
348 s.lastStatsMu.Unlock()
349
350 if stats != nil {
351 return stats, nil
352 }
353
354 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
355 if err != nil {
356 return nil, err
357 }
358
359 stats = &repos.Stats
360
361 s.lastStatsMu.Lock()
362 s.lastStatsTS = time.Now()
363 s.lastStats = stats
364 s.lastStatsMu.Unlock()
365
366 return stats, nil
367}
368
369func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
370 stats, err := s.fetchStats(r.Context())
371 if err != nil {
372 return err
373 }
374 d := SearchBoxInput{
375 Last: LastInput{
376 Num: defaultNumResults,
377 AutoFocus: true,
378 },
379 Stats: stats,
380 Version: s.Version,
381 Uptime: time.Since(s.startTime),
382 }
383
384 d.Last.Query = r.URL.Query().Get("q")
385 if d.Last.Query == "" {
386 custom := s.HostCustomQueries[r.Host]
387 if custom == "" {
388 host, _, _ := net.SplitHostPort(r.Host)
389 custom = s.HostCustomQueries[host]
390 }
391
392 if custom != "" {
393 d.Last.Query = custom + " "
394 }
395 }
396
397 var buf bytes.Buffer
398 if err := s.search.Execute(&buf, &d); err != nil {
399 return err
400 }
401 _, _ = w.Write(buf.Bytes())
402 return nil
403}
404
405func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
406 if err := s.serveSearchBoxErr(w, r); err != nil {
407 http.Error(w, err.Error(), http.StatusTeapot)
408 }
409}
410
411func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
412 stats, err := s.fetchStats(r.Context())
413 if err != nil {
414 return err
415 }
416
417 d := SearchBoxInput{
418 Stats: stats,
419 Version: s.Version,
420 Uptime: time.Since(s.startTime),
421 }
422
423 var buf bytes.Buffer
424 if err := s.about.Execute(&buf, &d); err != nil {
425 return err
426 }
427 _, _ = w.Write(buf.Bytes())
428 return nil
429}
430
431func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
432 if err := s.serveAboutErr(w, r); err != nil {
433 http.Error(w, err.Error(), http.StatusTeapot)
434 }
435}
436
437func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
438 data := struct{}{}
439 var buf bytes.Buffer
440 if err := s.robots.Execute(&buf, &data); err != nil {
441 return err
442 }
443 _, _ = w.Write(buf.Bytes())
444 return nil
445}
446
447func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
448 if err := s.serveRobotsErr(w, r); err != nil {
449 http.Error(w, err.Error(), http.StatusTeapot)
450 }
451}
452
453func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
454 ctx := r.Context()
455 repos, err := s.Searcher.List(ctx, q, nil)
456 if err != nil {
457 return nil, err
458 }
459
460 qvals := r.URL.Query()
461 order := qvals.Get("order")
462 switch order {
463 case "", "name", "revname":
464 sort.Slice(repos.Repos, func(i, j int) bool {
465 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
466 })
467 case "size", "revsize":
468 sort.Slice(repos.Repos, func(i, j int) bool {
469 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
470 })
471 case "ram", "revram":
472 sort.Slice(repos.Repos, func(i, j int) bool {
473 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
474 })
475 case "time", "revtime":
476 sort.Slice(repos.Repos, func(i, j int) bool {
477 return repos.Repos[i].IndexMetadata.IndexTime.Before(
478 repos.Repos[j].IndexMetadata.IndexTime)
479 })
480 default:
481 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
482 }
483 if strings.HasPrefix(order, "rev") {
484 for i, j := 0, len(repos.Repos)-1; i < j; {
485 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
486 i++
487 j--
488
489 }
490 }
491
492 aggregate := zoekt.RepoStats{
493 Repos: len(repos.Repos),
494 }
495 for _, s := range repos.Repos {
496 aggregate.Add(&s.Stats)
497 }
498
499 numStr := qvals.Get("num")
500 num, err := strconv.Atoi(numStr)
501 if err != nil || num <= 0 {
502 num = 0
503 }
504 if num > 0 {
505 if num > len(repos.Repos) {
506 num = len(repos.Repos)
507 }
508
509 repos.Repos = repos.Repos[:num]
510 }
511
512 res := RepoListInput{
513 Last: LastInput{
514 Query: qStr,
515 Num: num,
516 AutoFocus: true,
517 },
518 Stats: aggregate,
519 }
520
521 for _, r := range repos.Repos {
522 t := s.getTemplate(r.Repository.CommitURLTemplate)
523
524 repo := Repository{
525 Name: r.Repository.Name,
526 URL: r.Repository.URL,
527 IndexTime: r.IndexMetadata.IndexTime,
528 Size: r.Stats.ContentBytes,
529 MemorySize: r.Stats.IndexBytes,
530 Files: int64(r.Stats.Documents),
531 }
532 for _, b := range r.Repository.Branches {
533 var buf bytes.Buffer
534 if err := t.Execute(&buf, b); err != nil {
535 return nil, err
536 }
537 repo.Branches = append(repo.Branches,
538 Branch{
539 Name: b.Name,
540 Version: b.Version,
541 URL: buf.String(),
542 })
543 }
544 res.Repos = append(res.Repos, repo)
545 }
546 return &res, nil
547}
548
549func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
550 qvals := r.URL.Query()
551 fileStr := qvals.Get("f")
552 repoStr := qvals.Get("r")
553 queryStr := qvals.Get("q")
554 numStr := qvals.Get("num")
555 num, err := strconv.Atoi(numStr)
556 if err != nil || num <= 0 {
557 num = defaultNumResults
558 }
559
560 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
561 if err != nil {
562 return err
563 }
564
565 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
566 if err != nil {
567 return err
568 }
569
570 qs := []query.Q{
571 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
572 &query.Repo{Regexp: repoRe},
573 }
574
575 if branchStr := qvals.Get("b"); branchStr != "" {
576 qs = append(qs, &query.Branch{Pattern: branchStr})
577 }
578
579 q := &query.And{Children: qs}
580
581 sOpts := zoekt.SearchOptions{
582 Whole: true,
583 }
584
585 ctx := r.Context()
586 result, err := s.Searcher.Search(ctx, q, &sOpts)
587 if err != nil {
588 return err
589 }
590
591 if len(result.Files) != 1 {
592 var ss []string
593 for _, n := range result.Files {
594 ss = append(ss, n.FileName)
595 }
596 return fmt.Errorf("ambiguous result: %v", ss)
597 }
598
599 f := result.Files[0]
600
601 byteLines := bytes.Split(f.Content, []byte{'\n'})
602 strLines := make([]string, 0, len(byteLines))
603 for _, l := range byteLines {
604 strLines = append(strLines, string(l))
605 }
606
607 d := PrintInput{
608 Name: f.FileName,
609 Repo: f.Repository,
610 Lines: strLines,
611 Last: LastInput{
612 Query: queryStr,
613 Num: num,
614 AutoFocus: false,
615 },
616 }
617
618 var buf bytes.Buffer
619 if err := s.print.Execute(&buf, &d); err != nil {
620 return err
621 }
622
623 _, _ = w.Write(buf.Bytes())
624 return nil
625}