fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 texttemplate "text/template"
32 "time"
33
34 "github.com/grafana/regexp"
35
36 "github.com/sourcegraph/zoekt"
37 "github.com/sourcegraph/zoekt/internal/tenant/systemtenant"
38 zjson "github.com/sourcegraph/zoekt/json"
39 "github.com/sourcegraph/zoekt/query"
40)
41
42var Funcmap = template.FuncMap{
43 "Inc": func(orig int) int {
44 return orig + 1
45 },
46 "More": func(orig int) int {
47 return orig * 3
48 },
49 "HumanUnit": func(orig int64) string {
50 b := orig
51 suffix := ""
52 if orig > 10*(1<<30) {
53 suffix = "G"
54 b = orig / (1 << 30)
55 } else if orig > 10*(1<<20) {
56 suffix = "M"
57 b = orig / (1 << 20)
58 } else if orig > 10*(1<<10) {
59 suffix = "K"
60 b = orig / (1 << 10)
61 }
62
63 return fmt.Sprintf("%d%s", b, suffix)
64 },
65 "LimitPre": func(limit int, pre string) string {
66 if len(pre) < limit {
67 return pre
68 }
69 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
70 },
71 "LimitPost": func(limit int, post string) string {
72 if len(post) < limit {
73 return post
74 }
75 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
76 },
77 "TrimTrailingNewline": func(s string) string {
78 return strings.TrimSuffix(s, "\n")
79 },
80}
81
82const defaultNumResults = 50
83
84type Server struct {
85 Searcher zoekt.Streamer
86
87 // Serve HTML interface
88 HTML bool
89
90 // Serve RPC
91 RPC bool
92
93 // If set, show files from the index.
94 Print bool
95
96 // Version string for this server.
97 Version string
98
99 // Depending on the Host header, add a query to the entry
100 // page. For example, when serving on "search.myproject.org"
101 // we could add "r:myproject" automatically. This allows a
102 // single instance to serve as search engine for multiple
103 // domains.
104 HostCustomQueries map[string]string
105
106 // This should contain the following templates: "repolist"
107 // (for the repo search result page), "result" for
108 // the search results, "search" (for the opening page),
109 // "box" for the search query input element and
110 // "print" for the show file functionality.
111 Top *template.Template
112
113 repolist *template.Template
114 search *template.Template
115 result *template.Template
116 print *template.Template
117 about *template.Template
118 robots *template.Template
119
120 startTime time.Time
121
122 templateMu sync.Mutex
123 templateCache map[string]*template.Template
124 textTemplateCache map[string]*texttemplate.Template
125
126 lastStatsMu sync.Mutex
127 lastStats *zoekt.RepoStats
128 lastStatsTS time.Time
129}
130
131func (s *Server) getTemplate(str string) *template.Template {
132 s.templateMu.Lock()
133 defer s.templateMu.Unlock()
134 t := s.templateCache[str]
135 if t != nil {
136 return t
137 }
138
139 t, err := template.New("cache").Parse(str)
140 if err != nil {
141 log.Printf("html template parse error: %v", err)
142 t = template.Must(template.New("empty").Parse(""))
143 }
144 s.templateCache[str] = t
145 return t
146}
147
148func (s *Server) getTextTemplate(str string) *texttemplate.Template {
149 s.templateMu.Lock()
150 defer s.templateMu.Unlock()
151 t := s.textTemplateCache[str]
152 if t != nil {
153 return t
154 }
155
156 t, err := zoekt.ParseTemplate(str)
157 if err != nil {
158 log.Printf("text template parse error: %v", err)
159 t = texttemplate.Must(texttemplate.New("empty").Parse(""))
160 }
161 s.textTemplateCache[str] = t
162 return t
163}
164
165func NewMux(s *Server) (*http.ServeMux, error) {
166 s.print = s.Top.Lookup("print")
167 if s.print == nil {
168 return nil, fmt.Errorf("missing template 'print'")
169 }
170
171 for k, v := range map[string]**template.Template{
172 "results": &s.result,
173 "print": &s.print,
174 "search": &s.search,
175 "repolist": &s.repolist,
176 "about": &s.about,
177 "robots": &s.robots,
178 } {
179 *v = s.Top.Lookup(k)
180 if *v == nil {
181 return nil, fmt.Errorf("missing template %q", k)
182 }
183 }
184
185 s.templateCache = map[string]*template.Template{}
186 s.textTemplateCache = map[string]*texttemplate.Template{}
187 s.startTime = time.Now()
188
189 mux := http.NewServeMux()
190
191 if s.HTML {
192 mux.HandleFunc("/robots.txt", s.serveRobots)
193 mux.HandleFunc("/search", s.serveSearch)
194 mux.HandleFunc("/", s.serveSearchBox)
195 mux.HandleFunc("/about", s.serveAbout)
196 mux.HandleFunc("/print", s.servePrint)
197 }
198 if s.RPC {
199 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
200 }
201
202 mux.HandleFunc("/healthz", s.serveHealthz)
203
204 return mux, nil
205}
206
207func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
208 q := &query.Const{Value: true}
209 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
210
211 // We need to use WithUnsafeContext here because we want to perform a full
212 // search returning results. The result of this search is not used for anything
213 // other than determining if the server is healthy.
214 result, err := s.Searcher.Search(systemtenant.WithUnsafeContext(r.Context()), q, opts)
215 if err != nil {
216 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
217 return
218 }
219
220 w.Header().Set("Content-Type", "application/json")
221
222 _ = json.NewEncoder(w).Encode(result)
223}
224
225func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
226 result, err := s.serveSearchErr(r)
227 if err != nil {
228 http.Error(w, err.Error(), http.StatusTeapot)
229 return
230 }
231
232 qvals := r.URL.Query()
233 if qvals.Get("format") == "json" {
234 w.Header().Add("Content-Type", "application/json")
235 encoder := json.NewEncoder(w)
236 encoder.Encode(result)
237 return
238 }
239
240 var buf bytes.Buffer
241 if result.Repos != nil {
242 err = s.repolist.Execute(&buf, &result.Repos)
243 } else if result.Result != nil {
244 err = s.result.Execute(&buf, &result.Result)
245 }
246 if err != nil {
247 http.Error(w, err.Error(), http.StatusTeapot)
248 return
249 }
250 w.Write(buf.Bytes())
251}
252
253func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
254 qvals := r.URL.Query()
255
256 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
257
258 queryStr := qvals.Get("q")
259 if queryStr == "" {
260 return nil, fmt.Errorf("no query found")
261 }
262
263 q, err := query.Parse(queryStr)
264 if err != nil {
265 return nil, err
266 }
267
268 repoOnly := true
269 query.VisitAtoms(q, func(q query.Q) {
270 _, ok := q.(*query.Repo)
271 repoOnly = repoOnly && ok
272 })
273 if repoOnly {
274 repos, err := s.serveListReposErr(q, queryStr, r)
275 if err == nil {
276 return &ApiSearchResult{Repos: repos}, nil
277 }
278 return nil, err
279 }
280
281 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
282 repos, err := s.serveListReposErr(q, queryStr, r)
283 if err == nil {
284 return &ApiSearchResult{Repos: repos}, nil
285 }
286 return nil, err
287 }
288
289 numStr := qvals.Get("num")
290
291 num, err := strconv.Atoi(numStr)
292 if err != nil || num <= 0 {
293 num = defaultNumResults
294 }
295
296 sOpts := zoekt.SearchOptions{
297 MaxWallTime: 10 * time.Second,
298 }
299
300 numCtxLines := 0
301 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
302 numCtxLines, err = strconv.Atoi(ctxLinesStr)
303 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
304 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
305 }
306 }
307 sOpts.NumContextLines = numCtxLines
308
309 sOpts.SetDefaults()
310 sOpts.MaxDocDisplayCount = num
311 sOpts.DebugScore = debugScore
312
313 ctx := r.Context()
314 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
315 return nil, err
316 }
317
318 result, err := s.Searcher.Search(ctx, q, &sOpts)
319 if err != nil {
320 return nil, err
321 }
322
323 fileMatches, err := s.formatResults(result, queryStr, s.Print)
324 if err != nil {
325 return nil, err
326 }
327
328 res := ResultInput{
329 Last: LastInput{
330 Query: queryStr,
331 Num: num,
332 Ctx: numCtxLines,
333 AutoFocus: true,
334 },
335 Stats: result.Stats,
336 Query: q.String(),
337 QueryStr: queryStr,
338 FileMatches: fileMatches,
339 }
340 if res.Stats.Wait < res.Stats.Duration/10 {
341 // Suppress queueing stats if they are neglible.
342 res.Stats.Wait = 0
343 }
344
345 res.Last.Debug = debugScore
346 return &ApiSearchResult{Result: &res}, nil
347}
348
349func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
350 err := s.servePrintErr(w, r)
351 if err != nil {
352 http.Error(w, err.Error(), http.StatusTeapot)
353 }
354}
355
356const statsStaleNess = 30 * time.Second
357
358func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
359 s.lastStatsMu.Lock()
360 stats := s.lastStats
361 if time.Since(s.lastStatsTS) > statsStaleNess {
362 stats = nil
363 }
364 s.lastStatsMu.Unlock()
365
366 if stats != nil {
367 return stats, nil
368 }
369
370 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
371 if err != nil {
372 return nil, err
373 }
374
375 stats = &repos.Stats
376
377 s.lastStatsMu.Lock()
378 s.lastStatsTS = time.Now()
379 s.lastStats = stats
380 s.lastStatsMu.Unlock()
381
382 return stats, nil
383}
384
385func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
386 stats, err := s.fetchStats(r.Context())
387 if err != nil {
388 return err
389 }
390 d := SearchBoxInput{
391 Last: LastInput{
392 Num: defaultNumResults,
393 AutoFocus: true,
394 },
395 Stats: stats,
396 Version: s.Version,
397 Uptime: time.Since(s.startTime),
398 }
399
400 d.Last.Query = r.URL.Query().Get("q")
401 if d.Last.Query == "" {
402 custom := s.HostCustomQueries[r.Host]
403 if custom == "" {
404 host, _, _ := net.SplitHostPort(r.Host)
405 custom = s.HostCustomQueries[host]
406 }
407
408 if custom != "" {
409 d.Last.Query = custom + " "
410 }
411 }
412
413 var buf bytes.Buffer
414 if err := s.search.Execute(&buf, &d); err != nil {
415 return err
416 }
417 _, _ = w.Write(buf.Bytes())
418 return nil
419}
420
421func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
422 if err := s.serveSearchBoxErr(w, r); err != nil {
423 http.Error(w, err.Error(), http.StatusTeapot)
424 }
425}
426
427func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
428 stats, err := s.fetchStats(r.Context())
429 if err != nil {
430 return err
431 }
432
433 d := SearchBoxInput{
434 Stats: stats,
435 Version: s.Version,
436 Uptime: time.Since(s.startTime),
437 }
438
439 var buf bytes.Buffer
440 if err := s.about.Execute(&buf, &d); err != nil {
441 return err
442 }
443 _, _ = w.Write(buf.Bytes())
444 return nil
445}
446
447func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
448 if err := s.serveAboutErr(w, r); err != nil {
449 http.Error(w, err.Error(), http.StatusTeapot)
450 }
451}
452
453func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
454 data := struct{}{}
455 var buf bytes.Buffer
456 if err := s.robots.Execute(&buf, &data); err != nil {
457 return err
458 }
459 _, _ = w.Write(buf.Bytes())
460 return nil
461}
462
463func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
464 if err := s.serveRobotsErr(w, r); err != nil {
465 http.Error(w, err.Error(), http.StatusTeapot)
466 }
467}
468
469func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
470 ctx := r.Context()
471 repos, err := s.Searcher.List(ctx, q, nil)
472 if err != nil {
473 return nil, err
474 }
475
476 qvals := r.URL.Query()
477 order := qvals.Get("order")
478 switch order {
479 case "", "name", "revname":
480 sort.Slice(repos.Repos, func(i, j int) bool {
481 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
482 })
483 case "size", "revsize":
484 sort.Slice(repos.Repos, func(i, j int) bool {
485 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
486 })
487 case "ram", "revram":
488 sort.Slice(repos.Repos, func(i, j int) bool {
489 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
490 })
491 case "time", "revtime":
492 sort.Slice(repos.Repos, func(i, j int) bool {
493 return repos.Repos[i].IndexMetadata.IndexTime.Before(
494 repos.Repos[j].IndexMetadata.IndexTime)
495 })
496 default:
497 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
498 }
499 if strings.HasPrefix(order, "rev") {
500 for i, j := 0, len(repos.Repos)-1; i < j; {
501 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
502 i++
503 j--
504
505 }
506 }
507
508 aggregate := zoekt.RepoStats{
509 Repos: len(repos.Repos),
510 }
511 for _, s := range repos.Repos {
512 aggregate.Add(&s.Stats)
513 }
514
515 numStr := qvals.Get("num")
516 num, err := strconv.Atoi(numStr)
517 if err != nil || num <= 0 {
518 num = 0
519 }
520 if num > 0 {
521 if num > len(repos.Repos) {
522 num = len(repos.Repos)
523 }
524
525 repos.Repos = repos.Repos[:num]
526 }
527
528 res := RepoListInput{
529 Last: LastInput{
530 Query: qStr,
531 Num: num,
532 AutoFocus: true,
533 },
534 Stats: aggregate,
535 }
536
537 for _, r := range repos.Repos {
538 t := s.getTextTemplate(r.Repository.CommitURLTemplate)
539
540 repo := Repository{
541 Name: r.Repository.Name,
542 URL: r.Repository.URL,
543 IndexTime: r.IndexMetadata.IndexTime,
544 Size: r.Stats.ContentBytes,
545 MemorySize: r.Stats.IndexBytes,
546 Files: int64(r.Stats.Documents),
547 }
548 for _, b := range r.Repository.Branches {
549 var buf bytes.Buffer
550 if err := t.Execute(&buf, b); err != nil {
551 return nil, err
552 }
553 repo.Branches = append(repo.Branches,
554 Branch{
555 Name: b.Name,
556 Version: b.Version,
557 URL: buf.String(),
558 })
559 }
560 res.Repos = append(res.Repos, repo)
561 }
562 return &res, nil
563}
564
565func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
566 qvals := r.URL.Query()
567 fileStr := qvals.Get("f")
568 repoStr := qvals.Get("r")
569 queryStr := qvals.Get("q")
570 numStr := qvals.Get("num")
571 num, err := strconv.Atoi(numStr)
572 if err != nil || num <= 0 {
573 num = defaultNumResults
574 }
575
576 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
577 if err != nil {
578 return err
579 }
580
581 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
582 if err != nil {
583 return err
584 }
585
586 qs := []query.Q{
587 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
588 &query.Repo{Regexp: repoRe},
589 }
590
591 if branchStr := qvals.Get("b"); branchStr != "" {
592 qs = append(qs, &query.Branch{Pattern: branchStr})
593 }
594
595 q := &query.And{Children: qs}
596
597 sOpts := zoekt.SearchOptions{
598 Whole: true,
599 }
600
601 ctx := r.Context()
602 result, err := s.Searcher.Search(ctx, q, &sOpts)
603 if err != nil {
604 return err
605 }
606
607 if len(result.Files) != 1 {
608 var ss []string
609 for _, n := range result.Files {
610 ss = append(ss, n.FileName)
611 }
612 return fmt.Errorf("ambiguous result: %v", ss)
613 }
614
615 f := result.Files[0]
616
617 byteLines := bytes.Split(f.Content, []byte{'\n'})
618 strLines := make([]string, 0, len(byteLines))
619 for _, l := range byteLines {
620 strLines = append(strLines, string(l))
621 }
622
623 d := PrintInput{
624 Name: f.FileName,
625 Repo: f.Repository,
626 Lines: strLines,
627 Last: LastInput{
628 Query: queryStr,
629 Num: num,
630 AutoFocus: false,
631 },
632 }
633
634 var buf bytes.Buffer
635 if err := s.print.Execute(&buf, &d); err != nil {
636 return err
637 }
638
639 _, _ = w.Write(buf.Bytes())
640 return nil
641}