fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 texttemplate "text/template"
32 "time"
33
34 "github.com/grafana/regexp"
35
36 "github.com/sourcegraph/zoekt/index"
37 zjson "github.com/sourcegraph/zoekt/internal/json"
38
39 "github.com/sourcegraph/zoekt"
40 "github.com/sourcegraph/zoekt/internal/tenant/systemtenant"
41 "github.com/sourcegraph/zoekt/query"
42)
43
44var Funcmap = template.FuncMap{
45 "Inc": func(orig int) int {
46 return orig + 1
47 },
48 "More": func(orig int) int {
49 return orig * 3
50 },
51 "AddLineNumbers": func(content string, lineNum int, isBefore bool) []lineMatch {
52 return AddLineNumbers(content, lineNum, isBefore)
53 },
54 "HumanUnit": func(orig int64) string {
55 b := orig
56 suffix := ""
57 if orig > 10*(1<<30) {
58 suffix = "G"
59 b = orig / (1 << 30)
60 } else if orig > 10*(1<<20) {
61 suffix = "M"
62 b = orig / (1 << 20)
63 } else if orig > 10*(1<<10) {
64 suffix = "K"
65 b = orig / (1 << 10)
66 }
67
68 return fmt.Sprintf("%d%s", b, suffix)
69 },
70 "LimitPre": func(limit int, pre string) string {
71 if len(pre) < limit {
72 return pre
73 }
74 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
75 },
76 "LimitPost": func(limit int, post string) string {
77 if len(post) < limit {
78 return post
79 }
80 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
81 },
82 "TrimTrailingNewline": func(s string) string {
83 return strings.TrimSuffix(s, "\n")
84 },
85}
86
87// lineMatch represents a line of content with its associated line number
88type lineMatch struct {
89 LineNum int
90 Content string
91}
92
93// AddLineNumbers adds line numbers to the beginning of each line in the given content string.
94// The line numbers are relative to the current line number (lineNum).
95// For 'before' content, numbers will count backwards from lineNum-1.
96// For 'after' content, numbers will count forwards from lineNum+1.
97func AddLineNumbers(content string, lineNum int, isBefore bool) []lineMatch {
98 if content == "" {
99 return nil
100 }
101
102 lines := strings.Split(content, "\n")
103 var result []lineMatch
104
105 for i, line := range lines {
106 if i == len(lines)-1 && line == "" {
107 continue
108 }
109
110 var num int
111 if isBefore {
112 num = lineNum - len(lines) + i
113 } else {
114 num = lineNum + i + 1
115 }
116
117 // Add the line number and content to the result
118 result = append(result, lineMatch{LineNum: num, Content: line})
119 }
120 return result
121}
122
123const defaultNumResults = 50
124
125type Server struct {
126 Searcher zoekt.Streamer
127
128 // Serve HTML interface
129 HTML bool
130
131 // Serve RPC
132 RPC bool
133
134 // If set, show files from the index.
135 Print bool
136
137 // Version string for this server.
138 Version string
139
140 // Depending on the Host header, add a query to the entry
141 // page. For example, when serving on "search.myproject.org"
142 // we could add "r:myproject" automatically. This allows a
143 // single instance to serve as search engine for multiple
144 // domains.
145 HostCustomQueries map[string]string
146
147 // This should contain the following templates: "repolist"
148 // (for the repo search result page), "result" for
149 // the search results, "search" (for the opening page),
150 // "box" for the search query input element and
151 // "print" for the show file functionality.
152 Top *template.Template
153
154 repolist *template.Template
155 search *template.Template
156 result *template.Template
157 print *template.Template
158 about *template.Template
159 robots *template.Template
160
161 startTime time.Time
162
163 templateMu sync.Mutex
164 templateCache map[string]*template.Template
165 textTemplateCache map[string]*texttemplate.Template
166
167 lastStatsMu sync.Mutex
168 lastStats *zoekt.RepoStats
169 lastStatsTS time.Time
170}
171
172func (s *Server) getTemplate(str string) *template.Template {
173 s.templateMu.Lock()
174 defer s.templateMu.Unlock()
175 t := s.templateCache[str]
176 if t != nil {
177 return t
178 }
179
180 t, err := template.New("cache").Parse(str)
181 if err != nil {
182 log.Printf("html template parse error: %v", err)
183 t = template.Must(template.New("empty").Parse(""))
184 }
185 s.templateCache[str] = t
186 return t
187}
188
189func (s *Server) getTextTemplate(str string) *texttemplate.Template {
190 s.templateMu.Lock()
191 defer s.templateMu.Unlock()
192 t := s.textTemplateCache[str]
193 if t != nil {
194 return t
195 }
196
197 t, err := index.ParseTemplate(str)
198 if err != nil {
199 log.Printf("text template parse error: %v", err)
200 t = texttemplate.Must(texttemplate.New("empty").Parse(""))
201 }
202 s.textTemplateCache[str] = t
203 return t
204}
205
206func NewMux(s *Server) (*http.ServeMux, error) {
207 s.print = s.Top.Lookup("print")
208 if s.print == nil {
209 return nil, fmt.Errorf("missing template 'print'")
210 }
211
212 for k, v := range map[string]**template.Template{
213 "results": &s.result,
214 "print": &s.print,
215 "search": &s.search,
216 "repolist": &s.repolist,
217 "about": &s.about,
218 "robots": &s.robots,
219 } {
220 *v = s.Top.Lookup(k)
221 if *v == nil {
222 return nil, fmt.Errorf("missing template %q", k)
223 }
224 }
225
226 s.templateCache = map[string]*template.Template{}
227 s.textTemplateCache = map[string]*texttemplate.Template{}
228 s.startTime = time.Now()
229
230 mux := http.NewServeMux()
231
232 if s.HTML {
233 mux.HandleFunc("/robots.txt", s.serveRobots)
234 mux.HandleFunc("/search", s.serveSearch)
235 mux.HandleFunc("/", s.serveSearchBox)
236 mux.HandleFunc("/about", s.serveAbout)
237 mux.HandleFunc("/print", s.servePrint)
238 }
239 if s.RPC {
240 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
241 }
242
243 mux.HandleFunc("/healthz", s.serveHealthz)
244
245 return mux, nil
246}
247
248func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
249 q := &query.Const{Value: true}
250 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
251
252 // We need to use WithUnsafeContext here because we want to perform a full
253 // search returning results. The result of this search is not used for anything
254 // other than determining if the server is healthy.
255 result, err := s.Searcher.Search(systemtenant.WithUnsafeContext(r.Context()), q, opts)
256 if err != nil {
257 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
258 return
259 }
260
261 w.Header().Set("Content-Type", "application/json")
262
263 _ = json.NewEncoder(w).Encode(result)
264}
265
266func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
267 result, err := s.serveSearchErr(r)
268 if err != nil {
269 http.Error(w, err.Error(), http.StatusTeapot)
270 return
271 }
272
273 qvals := r.URL.Query()
274 if qvals.Get("format") == "json" {
275 w.Header().Add("Content-Type", "application/json")
276 encoder := json.NewEncoder(w)
277 encoder.Encode(result)
278 return
279 }
280
281 var buf bytes.Buffer
282 if result.Repos != nil {
283 err = s.repolist.Execute(&buf, &result.Repos)
284 } else if result.Result != nil {
285 err = s.result.Execute(&buf, &result.Result)
286 }
287 if err != nil {
288 http.Error(w, err.Error(), http.StatusTeapot)
289 return
290 }
291 w.Write(buf.Bytes())
292}
293
294func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
295 qvals := r.URL.Query()
296
297 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
298
299 queryStr := qvals.Get("q")
300 if queryStr == "" {
301 return nil, fmt.Errorf("no query found")
302 }
303
304 q, err := query.Parse(queryStr)
305 if err != nil {
306 return nil, err
307 }
308
309 repoOnly := true
310 query.VisitAtoms(q, func(q query.Q) {
311 _, ok := q.(*query.Repo)
312 repoOnly = repoOnly && ok
313 })
314 if repoOnly {
315 repos, err := s.serveListReposErr(q, queryStr, r)
316 if err == nil {
317 return &ApiSearchResult{Repos: repos}, nil
318 }
319 return nil, err
320 }
321
322 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
323 repos, err := s.serveListReposErr(q, queryStr, r)
324 if err == nil {
325 return &ApiSearchResult{Repos: repos}, nil
326 }
327 return nil, err
328 }
329
330 numStr := qvals.Get("num")
331
332 num, err := strconv.Atoi(numStr)
333 if err != nil || num <= 0 {
334 num = defaultNumResults
335 }
336
337 sOpts := zoekt.SearchOptions{
338 MaxWallTime: 10 * time.Second,
339 }
340
341 numCtxLines := 0
342 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
343 numCtxLines, err = strconv.Atoi(ctxLinesStr)
344 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
345 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
346 }
347 }
348 sOpts.NumContextLines = numCtxLines
349
350 sOpts.SetDefaults()
351 sOpts.MaxDocDisplayCount = num
352 sOpts.DebugScore = debugScore
353
354 ctx := r.Context()
355 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
356 return nil, err
357 }
358
359 result, err := s.Searcher.Search(ctx, q, &sOpts)
360 if err != nil {
361 return nil, err
362 }
363
364 fileMatches, err := s.formatResults(result, queryStr, s.Print)
365 if err != nil {
366 return nil, err
367 }
368
369 res := ResultInput{
370 Last: LastInput{
371 Query: queryStr,
372 Num: num,
373 Ctx: numCtxLines,
374 AutoFocus: true,
375 },
376 Stats: result.Stats,
377 Query: q.String(),
378 QueryStr: queryStr,
379 FileMatches: fileMatches,
380 }
381 if res.Stats.Wait < res.Stats.Duration/10 {
382 // Suppress queueing stats if they are neglible.
383 res.Stats.Wait = 0
384 }
385
386 res.Last.Debug = debugScore
387 return &ApiSearchResult{Result: &res}, nil
388}
389
390func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
391 err := s.servePrintErr(w, r)
392 if err != nil {
393 http.Error(w, err.Error(), http.StatusTeapot)
394 }
395}
396
397const statsStaleNess = 30 * time.Second
398
399func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
400 s.lastStatsMu.Lock()
401 stats := s.lastStats
402 if time.Since(s.lastStatsTS) > statsStaleNess {
403 stats = nil
404 }
405 s.lastStatsMu.Unlock()
406
407 if stats != nil {
408 return stats, nil
409 }
410
411 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
412 if err != nil {
413 return nil, err
414 }
415
416 stats = &repos.Stats
417
418 s.lastStatsMu.Lock()
419 s.lastStatsTS = time.Now()
420 s.lastStats = stats
421 s.lastStatsMu.Unlock()
422
423 return stats, nil
424}
425
426func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
427 stats, err := s.fetchStats(r.Context())
428 if err != nil {
429 return err
430 }
431 d := SearchBoxInput{
432 Last: LastInput{
433 Num: defaultNumResults,
434 AutoFocus: true,
435 },
436 Stats: stats,
437 Version: s.Version,
438 Uptime: time.Since(s.startTime),
439 }
440
441 d.Last.Query = r.URL.Query().Get("q")
442 if d.Last.Query == "" {
443 custom := s.HostCustomQueries[r.Host]
444 if custom == "" {
445 host, _, _ := net.SplitHostPort(r.Host)
446 custom = s.HostCustomQueries[host]
447 }
448
449 if custom != "" {
450 d.Last.Query = custom + " "
451 }
452 }
453
454 var buf bytes.Buffer
455 if err := s.search.Execute(&buf, &d); err != nil {
456 return err
457 }
458 _, _ = w.Write(buf.Bytes())
459 return nil
460}
461
462func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
463 if err := s.serveSearchBoxErr(w, r); err != nil {
464 http.Error(w, err.Error(), http.StatusTeapot)
465 }
466}
467
468func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
469 stats, err := s.fetchStats(r.Context())
470 if err != nil {
471 return err
472 }
473
474 d := SearchBoxInput{
475 Stats: stats,
476 Version: s.Version,
477 Uptime: time.Since(s.startTime),
478 }
479
480 var buf bytes.Buffer
481 if err := s.about.Execute(&buf, &d); err != nil {
482 return err
483 }
484 _, _ = w.Write(buf.Bytes())
485 return nil
486}
487
488func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
489 if err := s.serveAboutErr(w, r); err != nil {
490 http.Error(w, err.Error(), http.StatusTeapot)
491 }
492}
493
494func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
495 data := struct{}{}
496 var buf bytes.Buffer
497 if err := s.robots.Execute(&buf, &data); err != nil {
498 return err
499 }
500 _, _ = w.Write(buf.Bytes())
501 return nil
502}
503
504func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
505 if err := s.serveRobotsErr(w, r); err != nil {
506 http.Error(w, err.Error(), http.StatusTeapot)
507 }
508}
509
510func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
511 ctx := r.Context()
512 repos, err := s.Searcher.List(ctx, q, nil)
513 if err != nil {
514 return nil, err
515 }
516
517 qvals := r.URL.Query()
518 order := qvals.Get("order")
519 switch order {
520 case "", "name", "revname":
521 sort.Slice(repos.Repos, func(i, j int) bool {
522 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
523 })
524 case "size", "revsize":
525 sort.Slice(repos.Repos, func(i, j int) bool {
526 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
527 })
528 case "ram", "revram":
529 sort.Slice(repos.Repos, func(i, j int) bool {
530 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
531 })
532 case "time", "revtime":
533 sort.Slice(repos.Repos, func(i, j int) bool {
534 return repos.Repos[i].IndexMetadata.IndexTime.Before(
535 repos.Repos[j].IndexMetadata.IndexTime)
536 })
537 default:
538 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
539 }
540 if strings.HasPrefix(order, "rev") {
541 for i, j := 0, len(repos.Repos)-1; i < j; {
542 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
543 i++
544 j--
545
546 }
547 }
548
549 aggregate := zoekt.RepoStats{
550 Repos: len(repos.Repos),
551 }
552 for _, s := range repos.Repos {
553 aggregate.Add(&s.Stats)
554 }
555
556 numStr := qvals.Get("num")
557 num, err := strconv.Atoi(numStr)
558 if err != nil || num <= 0 {
559 num = 0
560 }
561 if num > 0 {
562 if num > len(repos.Repos) {
563 num = len(repos.Repos)
564 }
565
566 repos.Repos = repos.Repos[:num]
567 }
568
569 res := RepoListInput{
570 Last: LastInput{
571 Query: qStr,
572 Num: num,
573 AutoFocus: true,
574 },
575 Stats: aggregate,
576 }
577
578 for _, r := range repos.Repos {
579 t := s.getTextTemplate(r.Repository.CommitURLTemplate)
580
581 repo := Repository{
582 Name: r.Repository.Name,
583 URL: r.Repository.URL,
584 IndexTime: r.IndexMetadata.IndexTime,
585 Size: r.Stats.ContentBytes,
586 MemorySize: r.Stats.IndexBytes,
587 Files: int64(r.Stats.Documents),
588 }
589 for _, b := range r.Repository.Branches {
590 var buf bytes.Buffer
591 if err := t.Execute(&buf, b); err != nil {
592 return nil, err
593 }
594 repo.Branches = append(repo.Branches,
595 Branch{
596 Name: b.Name,
597 Version: b.Version,
598 URL: buf.String(),
599 })
600 }
601 res.Repos = append(res.Repos, repo)
602 }
603 return &res, nil
604}
605
606func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
607 qvals := r.URL.Query()
608 fileStr := qvals.Get("f")
609 repoStr := qvals.Get("r")
610 queryStr := qvals.Get("q")
611 numStr := qvals.Get("num")
612 num, err := strconv.Atoi(numStr)
613 if err != nil || num <= 0 {
614 num = defaultNumResults
615 }
616
617 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
618 if err != nil {
619 return err
620 }
621
622 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
623 if err != nil {
624 return err
625 }
626
627 qs := []query.Q{
628 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
629 &query.Repo{Regexp: repoRe},
630 }
631
632 if branchStr := qvals.Get("b"); branchStr != "" {
633 qs = append(qs, &query.Branch{Pattern: branchStr})
634 }
635
636 q := &query.And{Children: qs}
637
638 sOpts := zoekt.SearchOptions{
639 Whole: true,
640 }
641
642 ctx := r.Context()
643 result, err := s.Searcher.Search(ctx, q, &sOpts)
644 if err != nil {
645 return err
646 }
647
648 if len(result.Files) != 1 {
649 var ss []string
650 for _, n := range result.Files {
651 ss = append(ss, n.FileName)
652 }
653 return fmt.Errorf("ambiguous result: %v", ss)
654 }
655
656 f := result.Files[0]
657
658 if qvals.Get("format") == "raw" {
659 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
660 w.Header().Set("X-Content-Type-Options", "nosniff")
661 _, _ = w.Write(f.Content)
662 return nil
663 }
664
665 byteLines := bytes.Split(f.Content, []byte{'\n'})
666 strLines := make([]string, 0, len(byteLines))
667 for _, l := range byteLines {
668 strLines = append(strLines, string(l))
669 }
670
671 d := PrintInput{
672 Name: f.FileName,
673 Repo: f.Repository,
674 Lines: strLines,
675 Last: LastInput{
676 Query: queryStr,
677 Num: num,
678 AutoFocus: false,
679 },
680 }
681
682 var buf bytes.Buffer
683 if err := s.print.Execute(&buf, &d); err != nil {
684 return err
685 }
686
687 _, _ = w.Write(buf.Bytes())
688 return nil
689}