fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 texttemplate "text/template"
32 "time"
33
34 "github.com/grafana/regexp"
35 "github.com/sourcegraph/zoekt/index"
36 zjson "github.com/sourcegraph/zoekt/internal/json"
37
38 "github.com/sourcegraph/zoekt"
39 "github.com/sourcegraph/zoekt/internal/tenant/systemtenant"
40 "github.com/sourcegraph/zoekt/query"
41)
42
43var Funcmap = template.FuncMap{
44 "Inc": func(orig int) int {
45 return orig + 1
46 },
47 "More": func(orig int) int {
48 return orig * 3
49 },
50 "HumanUnit": func(orig int64) string {
51 b := orig
52 suffix := ""
53 if orig > 10*(1<<30) {
54 suffix = "G"
55 b = orig / (1 << 30)
56 } else if orig > 10*(1<<20) {
57 suffix = "M"
58 b = orig / (1 << 20)
59 } else if orig > 10*(1<<10) {
60 suffix = "K"
61 b = orig / (1 << 10)
62 }
63
64 return fmt.Sprintf("%d%s", b, suffix)
65 },
66 "LimitPre": func(limit int, pre string) string {
67 if len(pre) < limit {
68 return pre
69 }
70 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
71 },
72 "LimitPost": func(limit int, post string) string {
73 if len(post) < limit {
74 return post
75 }
76 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
77 },
78 "TrimTrailingNewline": func(s string) string {
79 return strings.TrimSuffix(s, "\n")
80 },
81}
82
83const defaultNumResults = 50
84
85type Server struct {
86 Searcher zoekt.Streamer
87
88 // Serve HTML interface
89 HTML bool
90
91 // Serve RPC
92 RPC bool
93
94 // If set, show files from the index.
95 Print bool
96
97 // Version string for this server.
98 Version string
99
100 // Depending on the Host header, add a query to the entry
101 // page. For example, when serving on "search.myproject.org"
102 // we could add "r:myproject" automatically. This allows a
103 // single instance to serve as search engine for multiple
104 // domains.
105 HostCustomQueries map[string]string
106
107 // This should contain the following templates: "repolist"
108 // (for the repo search result page), "result" for
109 // the search results, "search" (for the opening page),
110 // "box" for the search query input element and
111 // "print" for the show file functionality.
112 Top *template.Template
113
114 repolist *template.Template
115 search *template.Template
116 result *template.Template
117 print *template.Template
118 about *template.Template
119 robots *template.Template
120
121 startTime time.Time
122
123 templateMu sync.Mutex
124 templateCache map[string]*template.Template
125 textTemplateCache map[string]*texttemplate.Template
126
127 lastStatsMu sync.Mutex
128 lastStats *zoekt.RepoStats
129 lastStatsTS time.Time
130}
131
132func (s *Server) getTemplate(str string) *template.Template {
133 s.templateMu.Lock()
134 defer s.templateMu.Unlock()
135 t := s.templateCache[str]
136 if t != nil {
137 return t
138 }
139
140 t, err := template.New("cache").Parse(str)
141 if err != nil {
142 log.Printf("html template parse error: %v", err)
143 t = template.Must(template.New("empty").Parse(""))
144 }
145 s.templateCache[str] = t
146 return t
147}
148
149func (s *Server) getTextTemplate(str string) *texttemplate.Template {
150 s.templateMu.Lock()
151 defer s.templateMu.Unlock()
152 t := s.textTemplateCache[str]
153 if t != nil {
154 return t
155 }
156
157 t, err := index.ParseTemplate(str)
158 if err != nil {
159 log.Printf("text template parse error: %v", err)
160 t = texttemplate.Must(texttemplate.New("empty").Parse(""))
161 }
162 s.textTemplateCache[str] = t
163 return t
164}
165
166func NewMux(s *Server) (*http.ServeMux, error) {
167 s.print = s.Top.Lookup("print")
168 if s.print == nil {
169 return nil, fmt.Errorf("missing template 'print'")
170 }
171
172 for k, v := range map[string]**template.Template{
173 "results": &s.result,
174 "print": &s.print,
175 "search": &s.search,
176 "repolist": &s.repolist,
177 "about": &s.about,
178 "robots": &s.robots,
179 } {
180 *v = s.Top.Lookup(k)
181 if *v == nil {
182 return nil, fmt.Errorf("missing template %q", k)
183 }
184 }
185
186 s.templateCache = map[string]*template.Template{}
187 s.textTemplateCache = map[string]*texttemplate.Template{}
188 s.startTime = time.Now()
189
190 mux := http.NewServeMux()
191
192 if s.HTML {
193 mux.HandleFunc("/robots.txt", s.serveRobots)
194 mux.HandleFunc("/search", s.serveSearch)
195 mux.HandleFunc("/", s.serveSearchBox)
196 mux.HandleFunc("/about", s.serveAbout)
197 mux.HandleFunc("/print", s.servePrint)
198 }
199 if s.RPC {
200 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
201 }
202
203 mux.HandleFunc("/healthz", s.serveHealthz)
204
205 return mux, nil
206}
207
208func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
209 q := &query.Const{Value: true}
210 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
211
212 // We need to use WithUnsafeContext here because we want to perform a full
213 // search returning results. The result of this search is not used for anything
214 // other than determining if the server is healthy.
215 result, err := s.Searcher.Search(systemtenant.WithUnsafeContext(r.Context()), q, opts)
216 if err != nil {
217 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
218 return
219 }
220
221 w.Header().Set("Content-Type", "application/json")
222
223 _ = json.NewEncoder(w).Encode(result)
224}
225
226func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
227 result, err := s.serveSearchErr(r)
228 if err != nil {
229 http.Error(w, err.Error(), http.StatusTeapot)
230 return
231 }
232
233 qvals := r.URL.Query()
234 if qvals.Get("format") == "json" {
235 w.Header().Add("Content-Type", "application/json")
236 encoder := json.NewEncoder(w)
237 encoder.Encode(result)
238 return
239 }
240
241 var buf bytes.Buffer
242 if result.Repos != nil {
243 err = s.repolist.Execute(&buf, &result.Repos)
244 } else if result.Result != nil {
245 err = s.result.Execute(&buf, &result.Result)
246 }
247 if err != nil {
248 http.Error(w, err.Error(), http.StatusTeapot)
249 return
250 }
251 w.Write(buf.Bytes())
252}
253
254func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
255 qvals := r.URL.Query()
256
257 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
258
259 queryStr := qvals.Get("q")
260 if queryStr == "" {
261 return nil, fmt.Errorf("no query found")
262 }
263
264 q, err := query.Parse(queryStr)
265 if err != nil {
266 return nil, err
267 }
268
269 repoOnly := true
270 query.VisitAtoms(q, func(q query.Q) {
271 _, ok := q.(*query.Repo)
272 repoOnly = repoOnly && ok
273 })
274 if repoOnly {
275 repos, err := s.serveListReposErr(q, queryStr, r)
276 if err == nil {
277 return &ApiSearchResult{Repos: repos}, nil
278 }
279 return nil, err
280 }
281
282 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
283 repos, err := s.serveListReposErr(q, queryStr, r)
284 if err == nil {
285 return &ApiSearchResult{Repos: repos}, nil
286 }
287 return nil, err
288 }
289
290 numStr := qvals.Get("num")
291
292 num, err := strconv.Atoi(numStr)
293 if err != nil || num <= 0 {
294 num = defaultNumResults
295 }
296
297 sOpts := zoekt.SearchOptions{
298 MaxWallTime: 10 * time.Second,
299 }
300
301 numCtxLines := 0
302 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
303 numCtxLines, err = strconv.Atoi(ctxLinesStr)
304 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
305 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
306 }
307 }
308 sOpts.NumContextLines = numCtxLines
309
310 sOpts.SetDefaults()
311 sOpts.MaxDocDisplayCount = num
312 sOpts.DebugScore = debugScore
313
314 ctx := r.Context()
315 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
316 return nil, err
317 }
318
319 result, err := s.Searcher.Search(ctx, q, &sOpts)
320 if err != nil {
321 return nil, err
322 }
323
324 fileMatches, err := s.formatResults(result, queryStr, s.Print)
325 if err != nil {
326 return nil, err
327 }
328
329 res := ResultInput{
330 Last: LastInput{
331 Query: queryStr,
332 Num: num,
333 Ctx: numCtxLines,
334 AutoFocus: true,
335 },
336 Stats: result.Stats,
337 Query: q.String(),
338 QueryStr: queryStr,
339 FileMatches: fileMatches,
340 }
341 if res.Stats.Wait < res.Stats.Duration/10 {
342 // Suppress queueing stats if they are neglible.
343 res.Stats.Wait = 0
344 }
345
346 res.Last.Debug = debugScore
347 return &ApiSearchResult{Result: &res}, nil
348}
349
350func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
351 err := s.servePrintErr(w, r)
352 if err != nil {
353 http.Error(w, err.Error(), http.StatusTeapot)
354 }
355}
356
357const statsStaleNess = 30 * time.Second
358
359func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
360 s.lastStatsMu.Lock()
361 stats := s.lastStats
362 if time.Since(s.lastStatsTS) > statsStaleNess {
363 stats = nil
364 }
365 s.lastStatsMu.Unlock()
366
367 if stats != nil {
368 return stats, nil
369 }
370
371 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
372 if err != nil {
373 return nil, err
374 }
375
376 stats = &repos.Stats
377
378 s.lastStatsMu.Lock()
379 s.lastStatsTS = time.Now()
380 s.lastStats = stats
381 s.lastStatsMu.Unlock()
382
383 return stats, nil
384}
385
386func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
387 stats, err := s.fetchStats(r.Context())
388 if err != nil {
389 return err
390 }
391 d := SearchBoxInput{
392 Last: LastInput{
393 Num: defaultNumResults,
394 AutoFocus: true,
395 },
396 Stats: stats,
397 Version: s.Version,
398 Uptime: time.Since(s.startTime),
399 }
400
401 d.Last.Query = r.URL.Query().Get("q")
402 if d.Last.Query == "" {
403 custom := s.HostCustomQueries[r.Host]
404 if custom == "" {
405 host, _, _ := net.SplitHostPort(r.Host)
406 custom = s.HostCustomQueries[host]
407 }
408
409 if custom != "" {
410 d.Last.Query = custom + " "
411 }
412 }
413
414 var buf bytes.Buffer
415 if err := s.search.Execute(&buf, &d); err != nil {
416 return err
417 }
418 _, _ = w.Write(buf.Bytes())
419 return nil
420}
421
422func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
423 if err := s.serveSearchBoxErr(w, r); err != nil {
424 http.Error(w, err.Error(), http.StatusTeapot)
425 }
426}
427
428func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
429 stats, err := s.fetchStats(r.Context())
430 if err != nil {
431 return err
432 }
433
434 d := SearchBoxInput{
435 Stats: stats,
436 Version: s.Version,
437 Uptime: time.Since(s.startTime),
438 }
439
440 var buf bytes.Buffer
441 if err := s.about.Execute(&buf, &d); err != nil {
442 return err
443 }
444 _, _ = w.Write(buf.Bytes())
445 return nil
446}
447
448func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
449 if err := s.serveAboutErr(w, r); err != nil {
450 http.Error(w, err.Error(), http.StatusTeapot)
451 }
452}
453
454func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
455 data := struct{}{}
456 var buf bytes.Buffer
457 if err := s.robots.Execute(&buf, &data); err != nil {
458 return err
459 }
460 _, _ = w.Write(buf.Bytes())
461 return nil
462}
463
464func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
465 if err := s.serveRobotsErr(w, r); err != nil {
466 http.Error(w, err.Error(), http.StatusTeapot)
467 }
468}
469
470func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
471 ctx := r.Context()
472 repos, err := s.Searcher.List(ctx, q, nil)
473 if err != nil {
474 return nil, err
475 }
476
477 qvals := r.URL.Query()
478 order := qvals.Get("order")
479 switch order {
480 case "", "name", "revname":
481 sort.Slice(repos.Repos, func(i, j int) bool {
482 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
483 })
484 case "size", "revsize":
485 sort.Slice(repos.Repos, func(i, j int) bool {
486 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
487 })
488 case "ram", "revram":
489 sort.Slice(repos.Repos, func(i, j int) bool {
490 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
491 })
492 case "time", "revtime":
493 sort.Slice(repos.Repos, func(i, j int) bool {
494 return repos.Repos[i].IndexMetadata.IndexTime.Before(
495 repos.Repos[j].IndexMetadata.IndexTime)
496 })
497 default:
498 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
499 }
500 if strings.HasPrefix(order, "rev") {
501 for i, j := 0, len(repos.Repos)-1; i < j; {
502 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
503 i++
504 j--
505
506 }
507 }
508
509 aggregate := zoekt.RepoStats{
510 Repos: len(repos.Repos),
511 }
512 for _, s := range repos.Repos {
513 aggregate.Add(&s.Stats)
514 }
515
516 numStr := qvals.Get("num")
517 num, err := strconv.Atoi(numStr)
518 if err != nil || num <= 0 {
519 num = 0
520 }
521 if num > 0 {
522 if num > len(repos.Repos) {
523 num = len(repos.Repos)
524 }
525
526 repos.Repos = repos.Repos[:num]
527 }
528
529 res := RepoListInput{
530 Last: LastInput{
531 Query: qStr,
532 Num: num,
533 AutoFocus: true,
534 },
535 Stats: aggregate,
536 }
537
538 for _, r := range repos.Repos {
539 t := s.getTextTemplate(r.Repository.CommitURLTemplate)
540
541 repo := Repository{
542 Name: r.Repository.Name,
543 URL: r.Repository.URL,
544 IndexTime: r.IndexMetadata.IndexTime,
545 Size: r.Stats.ContentBytes,
546 MemorySize: r.Stats.IndexBytes,
547 Files: int64(r.Stats.Documents),
548 }
549 for _, b := range r.Repository.Branches {
550 var buf bytes.Buffer
551 if err := t.Execute(&buf, b); err != nil {
552 return nil, err
553 }
554 repo.Branches = append(repo.Branches,
555 Branch{
556 Name: b.Name,
557 Version: b.Version,
558 URL: buf.String(),
559 })
560 }
561 res.Repos = append(res.Repos, repo)
562 }
563 return &res, nil
564}
565
566func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
567 qvals := r.URL.Query()
568 fileStr := qvals.Get("f")
569 repoStr := qvals.Get("r")
570 queryStr := qvals.Get("q")
571 numStr := qvals.Get("num")
572 num, err := strconv.Atoi(numStr)
573 if err != nil || num <= 0 {
574 num = defaultNumResults
575 }
576
577 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
578 if err != nil {
579 return err
580 }
581
582 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
583 if err != nil {
584 return err
585 }
586
587 qs := []query.Q{
588 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
589 &query.Repo{Regexp: repoRe},
590 }
591
592 if branchStr := qvals.Get("b"); branchStr != "" {
593 qs = append(qs, &query.Branch{Pattern: branchStr})
594 }
595
596 q := &query.And{Children: qs}
597
598 sOpts := zoekt.SearchOptions{
599 Whole: true,
600 }
601
602 ctx := r.Context()
603 result, err := s.Searcher.Search(ctx, q, &sOpts)
604 if err != nil {
605 return err
606 }
607
608 if len(result.Files) != 1 {
609 var ss []string
610 for _, n := range result.Files {
611 ss = append(ss, n.FileName)
612 }
613 return fmt.Errorf("ambiguous result: %v", ss)
614 }
615
616 f := result.Files[0]
617
618 if qvals.Get("format") == "raw" {
619 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
620 w.Header().Set("X-Content-Type-Options", "nosniff")
621 _, _ = w.Write(f.Content)
622 return nil
623 }
624
625 byteLines := bytes.Split(f.Content, []byte{'\n'})
626 strLines := make([]string, 0, len(byteLines))
627 for _, l := range byteLines {
628 strLines = append(strLines, string(l))
629 }
630
631 d := PrintInput{
632 Name: f.FileName,
633 Repo: f.Repository,
634 Lines: strLines,
635 Last: LastInput{
636 Query: queryStr,
637 Num: num,
638 AutoFocus: false,
639 },
640 }
641
642 var buf bytes.Buffer
643 if err := s.print.Execute(&buf, &d); err != nil {
644 return err
645 }
646
647 _, _ = w.Write(buf.Bytes())
648 return nil
649}