fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 texttemplate "text/template"
32 "time"
33
34 "github.com/grafana/regexp"
35 "github.com/sourcegraph/zoekt/index"
36 zjson "github.com/sourcegraph/zoekt/internal/json"
37
38 "github.com/sourcegraph/zoekt"
39 "github.com/sourcegraph/zoekt/internal/tenant/systemtenant"
40 "github.com/sourcegraph/zoekt/query"
41)
42
43var Funcmap = template.FuncMap{
44 "Inc": func(orig int) int {
45 return orig + 1
46 },
47 "More": func(orig int) int {
48 return orig * 3
49 },
50 "AddLineNumbers": func(content string, lineNum int, isBefore bool) []lineMatch {
51 return AddLineNumbers(content, lineNum, isBefore)
52 },
53 "HumanUnit": func(orig int64) string {
54 b := orig
55 suffix := ""
56 if orig > 10*(1<<30) {
57 suffix = "G"
58 b = orig / (1 << 30)
59 } else if orig > 10*(1<<20) {
60 suffix = "M"
61 b = orig / (1 << 20)
62 } else if orig > 10*(1<<10) {
63 suffix = "K"
64 b = orig / (1 << 10)
65 }
66
67 return fmt.Sprintf("%d%s", b, suffix)
68 },
69 "LimitPre": func(limit int, pre string) string {
70 if len(pre) < limit {
71 return pre
72 }
73 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
74 },
75 "LimitPost": func(limit int, post string) string {
76 if len(post) < limit {
77 return post
78 }
79 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
80 },
81 "TrimTrailingNewline": func(s string) string {
82 return strings.TrimSuffix(s, "\n")
83 },
84}
85
86// lineMatch represents a line of content with its associated line number
87type lineMatch struct {
88 LineNum int
89 Content string
90}
91
92// AddLineNumbers adds line numbers to the beginning of each line in the given content string.
93// The line numbers are relative to the current line number (lineNum).
94// For 'before' content, numbers will count backwards from lineNum-1.
95// For 'after' content, numbers will count forwards from lineNum+1.
96func AddLineNumbers(content string, lineNum int, isBefore bool) []lineMatch {
97 if content == "" {
98 return nil
99 }
100
101 lines := strings.Split(content, "\n")
102 var result []lineMatch
103
104 for i, line := range lines {
105 if i == len(lines)-1 && line == "" {
106 continue
107 }
108
109 var num int
110 if isBefore {
111 num = lineNum - len(lines) + i
112 } else {
113 num = lineNum + i + 1
114 }
115
116 // Add the line number and content to the result
117 result = append(result, lineMatch{LineNum: num, Content: line})
118 }
119 return result
120}
121
122const defaultNumResults = 50
123
124type Server struct {
125 Searcher zoekt.Streamer
126
127 // Serve HTML interface
128 HTML bool
129
130 // Serve RPC
131 RPC bool
132
133 // If set, show files from the index.
134 Print bool
135
136 // Version string for this server.
137 Version string
138
139 // Depending on the Host header, add a query to the entry
140 // page. For example, when serving on "search.myproject.org"
141 // we could add "r:myproject" automatically. This allows a
142 // single instance to serve as search engine for multiple
143 // domains.
144 HostCustomQueries map[string]string
145
146 // This should contain the following templates: "repolist"
147 // (for the repo search result page), "result" for
148 // the search results, "search" (for the opening page),
149 // "box" for the search query input element and
150 // "print" for the show file functionality.
151 Top *template.Template
152
153 repolist *template.Template
154 search *template.Template
155 result *template.Template
156 print *template.Template
157 about *template.Template
158 robots *template.Template
159
160 startTime time.Time
161
162 templateMu sync.Mutex
163 templateCache map[string]*template.Template
164 textTemplateCache map[string]*texttemplate.Template
165
166 lastStatsMu sync.Mutex
167 lastStats *zoekt.RepoStats
168 lastStatsTS time.Time
169}
170
171func (s *Server) getTemplate(str string) *template.Template {
172 s.templateMu.Lock()
173 defer s.templateMu.Unlock()
174 t := s.templateCache[str]
175 if t != nil {
176 return t
177 }
178
179 t, err := template.New("cache").Parse(str)
180 if err != nil {
181 log.Printf("html template parse error: %v", err)
182 t = template.Must(template.New("empty").Parse(""))
183 }
184 s.templateCache[str] = t
185 return t
186}
187
188func (s *Server) getTextTemplate(str string) *texttemplate.Template {
189 s.templateMu.Lock()
190 defer s.templateMu.Unlock()
191 t := s.textTemplateCache[str]
192 if t != nil {
193 return t
194 }
195
196 t, err := index.ParseTemplate(str)
197 if err != nil {
198 log.Printf("text template parse error: %v", err)
199 t = texttemplate.Must(texttemplate.New("empty").Parse(""))
200 }
201 s.textTemplateCache[str] = t
202 return t
203}
204
205func NewMux(s *Server) (*http.ServeMux, error) {
206 s.print = s.Top.Lookup("print")
207 if s.print == nil {
208 return nil, fmt.Errorf("missing template 'print'")
209 }
210
211 for k, v := range map[string]**template.Template{
212 "results": &s.result,
213 "print": &s.print,
214 "search": &s.search,
215 "repolist": &s.repolist,
216 "about": &s.about,
217 "robots": &s.robots,
218 } {
219 *v = s.Top.Lookup(k)
220 if *v == nil {
221 return nil, fmt.Errorf("missing template %q", k)
222 }
223 }
224
225 s.templateCache = map[string]*template.Template{}
226 s.textTemplateCache = map[string]*texttemplate.Template{}
227 s.startTime = time.Now()
228
229 mux := http.NewServeMux()
230
231 if s.HTML {
232 mux.HandleFunc("/robots.txt", s.serveRobots)
233 mux.HandleFunc("/search", s.serveSearch)
234 mux.HandleFunc("/", s.serveSearchBox)
235 mux.HandleFunc("/about", s.serveAbout)
236 mux.HandleFunc("/print", s.servePrint)
237 }
238 if s.RPC {
239 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
240 }
241
242 mux.HandleFunc("/healthz", s.serveHealthz)
243
244 return mux, nil
245}
246
247func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
248 q := &query.Const{Value: true}
249 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
250
251 // We need to use WithUnsafeContext here because we want to perform a full
252 // search returning results. The result of this search is not used for anything
253 // other than determining if the server is healthy.
254 result, err := s.Searcher.Search(systemtenant.WithUnsafeContext(r.Context()), q, opts)
255 if err != nil {
256 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
257 return
258 }
259
260 w.Header().Set("Content-Type", "application/json")
261
262 _ = json.NewEncoder(w).Encode(result)
263}
264
265func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
266 result, err := s.serveSearchErr(r)
267 if err != nil {
268 http.Error(w, err.Error(), http.StatusTeapot)
269 return
270 }
271
272 qvals := r.URL.Query()
273 if qvals.Get("format") == "json" {
274 w.Header().Add("Content-Type", "application/json")
275 encoder := json.NewEncoder(w)
276 encoder.Encode(result)
277 return
278 }
279
280 var buf bytes.Buffer
281 if result.Repos != nil {
282 err = s.repolist.Execute(&buf, &result.Repos)
283 } else if result.Result != nil {
284 err = s.result.Execute(&buf, &result.Result)
285 }
286 if err != nil {
287 http.Error(w, err.Error(), http.StatusTeapot)
288 return
289 }
290 w.Write(buf.Bytes())
291}
292
293func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
294 qvals := r.URL.Query()
295
296 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
297
298 queryStr := qvals.Get("q")
299 if queryStr == "" {
300 return nil, fmt.Errorf("no query found")
301 }
302
303 q, err := query.Parse(queryStr)
304 if err != nil {
305 return nil, err
306 }
307
308 repoOnly := true
309 query.VisitAtoms(q, func(q query.Q) {
310 _, ok := q.(*query.Repo)
311 repoOnly = repoOnly && ok
312 })
313 if repoOnly {
314 repos, err := s.serveListReposErr(q, queryStr, r)
315 if err == nil {
316 return &ApiSearchResult{Repos: repos}, nil
317 }
318 return nil, err
319 }
320
321 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
322 repos, err := s.serveListReposErr(q, queryStr, r)
323 if err == nil {
324 return &ApiSearchResult{Repos: repos}, nil
325 }
326 return nil, err
327 }
328
329 numStr := qvals.Get("num")
330
331 num, err := strconv.Atoi(numStr)
332 if err != nil || num <= 0 {
333 num = defaultNumResults
334 }
335
336 sOpts := zoekt.SearchOptions{
337 MaxWallTime: 10 * time.Second,
338 }
339
340 numCtxLines := 0
341 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
342 numCtxLines, err = strconv.Atoi(ctxLinesStr)
343 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
344 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
345 }
346 }
347 sOpts.NumContextLines = numCtxLines
348
349 sOpts.SetDefaults()
350 sOpts.MaxDocDisplayCount = num
351 sOpts.DebugScore = debugScore
352
353 ctx := r.Context()
354 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
355 return nil, err
356 }
357
358 result, err := s.Searcher.Search(ctx, q, &sOpts)
359 if err != nil {
360 return nil, err
361 }
362
363 fileMatches, err := s.formatResults(result, queryStr, s.Print)
364 if err != nil {
365 return nil, err
366 }
367
368 res := ResultInput{
369 Last: LastInput{
370 Query: queryStr,
371 Num: num,
372 Ctx: numCtxLines,
373 AutoFocus: true,
374 },
375 Stats: result.Stats,
376 Query: q.String(),
377 QueryStr: queryStr,
378 FileMatches: fileMatches,
379 }
380 if res.Stats.Wait < res.Stats.Duration/10 {
381 // Suppress queueing stats if they are neglible.
382 res.Stats.Wait = 0
383 }
384
385 res.Last.Debug = debugScore
386 return &ApiSearchResult{Result: &res}, nil
387}
388
389func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
390 err := s.servePrintErr(w, r)
391 if err != nil {
392 http.Error(w, err.Error(), http.StatusTeapot)
393 }
394}
395
396const statsStaleNess = 30 * time.Second
397
398func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
399 s.lastStatsMu.Lock()
400 stats := s.lastStats
401 if time.Since(s.lastStatsTS) > statsStaleNess {
402 stats = nil
403 }
404 s.lastStatsMu.Unlock()
405
406 if stats != nil {
407 return stats, nil
408 }
409
410 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
411 if err != nil {
412 return nil, err
413 }
414
415 stats = &repos.Stats
416
417 s.lastStatsMu.Lock()
418 s.lastStatsTS = time.Now()
419 s.lastStats = stats
420 s.lastStatsMu.Unlock()
421
422 return stats, nil
423}
424
425func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
426 stats, err := s.fetchStats(r.Context())
427 if err != nil {
428 return err
429 }
430 d := SearchBoxInput{
431 Last: LastInput{
432 Num: defaultNumResults,
433 AutoFocus: true,
434 },
435 Stats: stats,
436 Version: s.Version,
437 Uptime: time.Since(s.startTime),
438 }
439
440 d.Last.Query = r.URL.Query().Get("q")
441 if d.Last.Query == "" {
442 custom := s.HostCustomQueries[r.Host]
443 if custom == "" {
444 host, _, _ := net.SplitHostPort(r.Host)
445 custom = s.HostCustomQueries[host]
446 }
447
448 if custom != "" {
449 d.Last.Query = custom + " "
450 }
451 }
452
453 var buf bytes.Buffer
454 if err := s.search.Execute(&buf, &d); err != nil {
455 return err
456 }
457 _, _ = w.Write(buf.Bytes())
458 return nil
459}
460
461func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
462 if err := s.serveSearchBoxErr(w, r); err != nil {
463 http.Error(w, err.Error(), http.StatusTeapot)
464 }
465}
466
467func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
468 stats, err := s.fetchStats(r.Context())
469 if err != nil {
470 return err
471 }
472
473 d := SearchBoxInput{
474 Stats: stats,
475 Version: s.Version,
476 Uptime: time.Since(s.startTime),
477 }
478
479 var buf bytes.Buffer
480 if err := s.about.Execute(&buf, &d); err != nil {
481 return err
482 }
483 _, _ = w.Write(buf.Bytes())
484 return nil
485}
486
487func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
488 if err := s.serveAboutErr(w, r); err != nil {
489 http.Error(w, err.Error(), http.StatusTeapot)
490 }
491}
492
493func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
494 data := struct{}{}
495 var buf bytes.Buffer
496 if err := s.robots.Execute(&buf, &data); err != nil {
497 return err
498 }
499 _, _ = w.Write(buf.Bytes())
500 return nil
501}
502
503func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
504 if err := s.serveRobotsErr(w, r); err != nil {
505 http.Error(w, err.Error(), http.StatusTeapot)
506 }
507}
508
509func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
510 ctx := r.Context()
511 repos, err := s.Searcher.List(ctx, q, nil)
512 if err != nil {
513 return nil, err
514 }
515
516 qvals := r.URL.Query()
517 order := qvals.Get("order")
518 switch order {
519 case "", "name", "revname":
520 sort.Slice(repos.Repos, func(i, j int) bool {
521 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
522 })
523 case "size", "revsize":
524 sort.Slice(repos.Repos, func(i, j int) bool {
525 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
526 })
527 case "ram", "revram":
528 sort.Slice(repos.Repos, func(i, j int) bool {
529 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
530 })
531 case "time", "revtime":
532 sort.Slice(repos.Repos, func(i, j int) bool {
533 return repos.Repos[i].IndexMetadata.IndexTime.Before(
534 repos.Repos[j].IndexMetadata.IndexTime)
535 })
536 default:
537 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
538 }
539 if strings.HasPrefix(order, "rev") {
540 for i, j := 0, len(repos.Repos)-1; i < j; {
541 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
542 i++
543 j--
544
545 }
546 }
547
548 aggregate := zoekt.RepoStats{
549 Repos: len(repos.Repos),
550 }
551 for _, s := range repos.Repos {
552 aggregate.Add(&s.Stats)
553 }
554
555 numStr := qvals.Get("num")
556 num, err := strconv.Atoi(numStr)
557 if err != nil || num <= 0 {
558 num = 0
559 }
560 if num > 0 {
561 if num > len(repos.Repos) {
562 num = len(repos.Repos)
563 }
564
565 repos.Repos = repos.Repos[:num]
566 }
567
568 res := RepoListInput{
569 Last: LastInput{
570 Query: qStr,
571 Num: num,
572 AutoFocus: true,
573 },
574 Stats: aggregate,
575 }
576
577 for _, r := range repos.Repos {
578 t := s.getTextTemplate(r.Repository.CommitURLTemplate)
579
580 repo := Repository{
581 Name: r.Repository.Name,
582 URL: r.Repository.URL,
583 IndexTime: r.IndexMetadata.IndexTime,
584 Size: r.Stats.ContentBytes,
585 MemorySize: r.Stats.IndexBytes,
586 Files: int64(r.Stats.Documents),
587 }
588 for _, b := range r.Repository.Branches {
589 var buf bytes.Buffer
590 if err := t.Execute(&buf, b); err != nil {
591 return nil, err
592 }
593 repo.Branches = append(repo.Branches,
594 Branch{
595 Name: b.Name,
596 Version: b.Version,
597 URL: buf.String(),
598 })
599 }
600 res.Repos = append(res.Repos, repo)
601 }
602 return &res, nil
603}
604
605func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
606 qvals := r.URL.Query()
607 fileStr := qvals.Get("f")
608 repoStr := qvals.Get("r")
609 queryStr := qvals.Get("q")
610 numStr := qvals.Get("num")
611 num, err := strconv.Atoi(numStr)
612 if err != nil || num <= 0 {
613 num = defaultNumResults
614 }
615
616 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
617 if err != nil {
618 return err
619 }
620
621 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
622 if err != nil {
623 return err
624 }
625
626 qs := []query.Q{
627 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
628 &query.Repo{Regexp: repoRe},
629 }
630
631 if branchStr := qvals.Get("b"); branchStr != "" {
632 qs = append(qs, &query.Branch{Pattern: branchStr})
633 }
634
635 q := &query.And{Children: qs}
636
637 sOpts := zoekt.SearchOptions{
638 Whole: true,
639 }
640
641 ctx := r.Context()
642 result, err := s.Searcher.Search(ctx, q, &sOpts)
643 if err != nil {
644 return err
645 }
646
647 if len(result.Files) != 1 {
648 var ss []string
649 for _, n := range result.Files {
650 ss = append(ss, n.FileName)
651 }
652 return fmt.Errorf("ambiguous result: %v", ss)
653 }
654
655 f := result.Files[0]
656
657 if qvals.Get("format") == "raw" {
658 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
659 w.Header().Set("X-Content-Type-Options", "nosniff")
660 _, _ = w.Write(f.Content)
661 return nil
662 }
663
664 byteLines := bytes.Split(f.Content, []byte{'\n'})
665 strLines := make([]string, 0, len(byteLines))
666 for _, l := range byteLines {
667 strLines = append(strLines, string(l))
668 }
669
670 d := PrintInput{
671 Name: f.FileName,
672 Repo: f.Repository,
673 Lines: strLines,
674 Last: LastInput{
675 Query: queryStr,
676 Num: num,
677 AutoFocus: false,
678 },
679 }
680
681 var buf bytes.Buffer
682 if err := s.print.Execute(&buf, &d); err != nil {
683 return err
684 }
685
686 _, _ = w.Write(buf.Bytes())
687 return nil
688}