fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 texttemplate "text/template"
32 "time"
33
34 "github.com/grafana/regexp"
35 "github.com/sourcegraph/zoekt"
36 zjson "github.com/sourcegraph/zoekt/json"
37 "github.com/sourcegraph/zoekt/query"
38)
39
40var Funcmap = template.FuncMap{
41 "Inc": func(orig int) int {
42 return orig + 1
43 },
44 "More": func(orig int) int {
45 return orig * 3
46 },
47 "HumanUnit": func(orig int64) string {
48 b := orig
49 suffix := ""
50 if orig > 10*(1<<30) {
51 suffix = "G"
52 b = orig / (1 << 30)
53 } else if orig > 10*(1<<20) {
54 suffix = "M"
55 b = orig / (1 << 20)
56 } else if orig > 10*(1<<10) {
57 suffix = "K"
58 b = orig / (1 << 10)
59 }
60
61 return fmt.Sprintf("%d%s", b, suffix)
62 },
63 "LimitPre": func(limit int, pre string) string {
64 if len(pre) < limit {
65 return pre
66 }
67 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
68 },
69 "LimitPost": func(limit int, post string) string {
70 if len(post) < limit {
71 return post
72 }
73 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
74 },
75 "TrimTrailingNewline": func(s string) string {
76 return strings.TrimSuffix(s, "\n")
77 },
78}
79
80const defaultNumResults = 50
81
82type Server struct {
83 Searcher zoekt.Streamer
84
85 // Serve HTML interface
86 HTML bool
87
88 // Serve RPC
89 RPC bool
90
91 // If set, show files from the index.
92 Print bool
93
94 // Version string for this server.
95 Version string
96
97 // Depending on the Host header, add a query to the entry
98 // page. For example, when serving on "search.myproject.org"
99 // we could add "r:myproject" automatically. This allows a
100 // single instance to serve as search engine for multiple
101 // domains.
102 HostCustomQueries map[string]string
103
104 // This should contain the following templates: "repolist"
105 // (for the repo search result page), "result" for
106 // the search results, "search" (for the opening page),
107 // "box" for the search query input element and
108 // "print" for the show file functionality.
109 Top *template.Template
110
111 repolist *template.Template
112 search *template.Template
113 result *template.Template
114 print *template.Template
115 about *template.Template
116 robots *template.Template
117
118 startTime time.Time
119
120 templateMu sync.Mutex
121 templateCache map[string]*template.Template
122 textTemplateCache map[string]*texttemplate.Template
123
124 lastStatsMu sync.Mutex
125 lastStats *zoekt.RepoStats
126 lastStatsTS time.Time
127}
128
129func (s *Server) getTemplate(str string) *template.Template {
130 s.templateMu.Lock()
131 defer s.templateMu.Unlock()
132 t := s.templateCache[str]
133 if t != nil {
134 return t
135 }
136
137 t, err := template.New("cache").Parse(str)
138 if err != nil {
139 log.Printf("html template parse error: %v", err)
140 t = template.Must(template.New("empty").Parse(""))
141 }
142 s.templateCache[str] = t
143 return t
144}
145
146func (s *Server) getTextTemplate(str string) *texttemplate.Template {
147 s.templateMu.Lock()
148 defer s.templateMu.Unlock()
149 t := s.textTemplateCache[str]
150 if t != nil {
151 return t
152 }
153
154 t, err := zoekt.ParseTemplate(str)
155 if err != nil {
156 log.Printf("text template parse error: %v", err)
157 t = texttemplate.Must(texttemplate.New("empty").Parse(""))
158 }
159 s.textTemplateCache[str] = t
160 return t
161}
162
163func NewMux(s *Server) (*http.ServeMux, error) {
164 s.print = s.Top.Lookup("print")
165 if s.print == nil {
166 return nil, fmt.Errorf("missing template 'print'")
167 }
168
169 for k, v := range map[string]**template.Template{
170 "results": &s.result,
171 "print": &s.print,
172 "search": &s.search,
173 "repolist": &s.repolist,
174 "about": &s.about,
175 "robots": &s.robots,
176 } {
177 *v = s.Top.Lookup(k)
178 if *v == nil {
179 return nil, fmt.Errorf("missing template %q", k)
180 }
181 }
182
183 s.templateCache = map[string]*template.Template{}
184 s.textTemplateCache = map[string]*texttemplate.Template{}
185 s.startTime = time.Now()
186
187 mux := http.NewServeMux()
188
189 if s.HTML {
190 mux.HandleFunc("/robots.txt", s.serveRobots)
191 mux.HandleFunc("/search", s.serveSearch)
192 mux.HandleFunc("/", s.serveSearchBox)
193 mux.HandleFunc("/about", s.serveAbout)
194 mux.HandleFunc("/print", s.servePrint)
195 }
196 if s.RPC {
197 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
198 }
199
200 mux.HandleFunc("/healthz", s.serveHealthz)
201
202 return mux, nil
203}
204
205func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
206 q := &query.Const{Value: true}
207 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
208
209 result, err := s.Searcher.Search(r.Context(), q, opts)
210 if err != nil {
211 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
212 return
213 }
214
215 w.Header().Set("Content-Type", "application/json")
216
217 _ = json.NewEncoder(w).Encode(result)
218}
219
220func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
221 result, err := s.serveSearchErr(r)
222 if err != nil {
223 http.Error(w, err.Error(), http.StatusTeapot)
224 return
225 }
226
227 qvals := r.URL.Query()
228 if qvals.Get("format") == "json" {
229 w.Header().Add("Content-Type", "application/json")
230 encoder := json.NewEncoder(w)
231 encoder.Encode(result)
232 return
233 }
234
235 var buf bytes.Buffer
236 if result.Repos != nil {
237 err = s.repolist.Execute(&buf, &result.Repos)
238 } else if result.Result != nil {
239 err = s.result.Execute(&buf, &result.Result)
240 }
241 if err != nil {
242 http.Error(w, err.Error(), http.StatusTeapot)
243 return
244 }
245 w.Write(buf.Bytes())
246}
247
248func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
249 qvals := r.URL.Query()
250
251 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
252
253 queryStr := qvals.Get("q")
254 if queryStr == "" {
255 return nil, fmt.Errorf("no query found")
256 }
257
258 q, err := query.Parse(queryStr)
259 if err != nil {
260 return nil, err
261 }
262
263 repoOnly := true
264 query.VisitAtoms(q, func(q query.Q) {
265 _, ok := q.(*query.Repo)
266 repoOnly = repoOnly && ok
267 })
268 if repoOnly {
269 repos, err := s.serveListReposErr(q, queryStr, r)
270 if err == nil {
271 return &ApiSearchResult{Repos: repos}, nil
272 }
273 return nil, err
274 }
275
276 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
277 repos, err := s.serveListReposErr(q, queryStr, r)
278 if err == nil {
279 return &ApiSearchResult{Repos: repos}, nil
280 }
281 return nil, err
282 }
283
284 numStr := qvals.Get("num")
285
286 num, err := strconv.Atoi(numStr)
287 if err != nil || num <= 0 {
288 num = defaultNumResults
289 }
290
291 sOpts := zoekt.SearchOptions{
292 MaxWallTime: 10 * time.Second,
293 }
294
295 numCtxLines := 0
296 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
297 numCtxLines, err = strconv.Atoi(ctxLinesStr)
298 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
299 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
300 }
301 }
302 sOpts.NumContextLines = numCtxLines
303
304 sOpts.SetDefaults()
305 sOpts.MaxDocDisplayCount = num
306 sOpts.DebugScore = debugScore
307
308 ctx := r.Context()
309 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
310 return nil, err
311 }
312
313 result, err := s.Searcher.Search(ctx, q, &sOpts)
314 if err != nil {
315 return nil, err
316 }
317
318 fileMatches, err := s.formatResults(result, queryStr, s.Print)
319 if err != nil {
320 return nil, err
321 }
322
323 res := ResultInput{
324 Last: LastInput{
325 Query: queryStr,
326 Num: num,
327 Ctx: numCtxLines,
328 AutoFocus: true,
329 },
330 Stats: result.Stats,
331 Query: q.String(),
332 QueryStr: queryStr,
333 FileMatches: fileMatches,
334 }
335 if res.Stats.Wait < res.Stats.Duration/10 {
336 // Suppress queueing stats if they are neglible.
337 res.Stats.Wait = 0
338 }
339
340 res.Last.Debug = debugScore
341 return &ApiSearchResult{Result: &res}, nil
342}
343
344func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
345 err := s.servePrintErr(w, r)
346 if err != nil {
347 http.Error(w, err.Error(), http.StatusTeapot)
348 }
349}
350
351const statsStaleNess = 30 * time.Second
352
353func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
354 s.lastStatsMu.Lock()
355 stats := s.lastStats
356 if time.Since(s.lastStatsTS) > statsStaleNess {
357 stats = nil
358 }
359 s.lastStatsMu.Unlock()
360
361 if stats != nil {
362 return stats, nil
363 }
364
365 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
366 if err != nil {
367 return nil, err
368 }
369
370 stats = &repos.Stats
371
372 s.lastStatsMu.Lock()
373 s.lastStatsTS = time.Now()
374 s.lastStats = stats
375 s.lastStatsMu.Unlock()
376
377 return stats, nil
378}
379
380func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
381 stats, err := s.fetchStats(r.Context())
382 if err != nil {
383 return err
384 }
385 d := SearchBoxInput{
386 Last: LastInput{
387 Num: defaultNumResults,
388 AutoFocus: true,
389 },
390 Stats: stats,
391 Version: s.Version,
392 Uptime: time.Since(s.startTime),
393 }
394
395 d.Last.Query = r.URL.Query().Get("q")
396 if d.Last.Query == "" {
397 custom := s.HostCustomQueries[r.Host]
398 if custom == "" {
399 host, _, _ := net.SplitHostPort(r.Host)
400 custom = s.HostCustomQueries[host]
401 }
402
403 if custom != "" {
404 d.Last.Query = custom + " "
405 }
406 }
407
408 var buf bytes.Buffer
409 if err := s.search.Execute(&buf, &d); err != nil {
410 return err
411 }
412 _, _ = w.Write(buf.Bytes())
413 return nil
414}
415
416func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
417 if err := s.serveSearchBoxErr(w, r); err != nil {
418 http.Error(w, err.Error(), http.StatusTeapot)
419 }
420}
421
422func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
423 stats, err := s.fetchStats(r.Context())
424 if err != nil {
425 return err
426 }
427
428 d := SearchBoxInput{
429 Stats: stats,
430 Version: s.Version,
431 Uptime: time.Since(s.startTime),
432 }
433
434 var buf bytes.Buffer
435 if err := s.about.Execute(&buf, &d); err != nil {
436 return err
437 }
438 _, _ = w.Write(buf.Bytes())
439 return nil
440}
441
442func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
443 if err := s.serveAboutErr(w, r); err != nil {
444 http.Error(w, err.Error(), http.StatusTeapot)
445 }
446}
447
448func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
449 data := struct{}{}
450 var buf bytes.Buffer
451 if err := s.robots.Execute(&buf, &data); err != nil {
452 return err
453 }
454 _, _ = w.Write(buf.Bytes())
455 return nil
456}
457
458func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
459 if err := s.serveRobotsErr(w, r); err != nil {
460 http.Error(w, err.Error(), http.StatusTeapot)
461 }
462}
463
464func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
465 ctx := r.Context()
466 repos, err := s.Searcher.List(ctx, q, nil)
467 if err != nil {
468 return nil, err
469 }
470
471 qvals := r.URL.Query()
472 order := qvals.Get("order")
473 switch order {
474 case "", "name", "revname":
475 sort.Slice(repos.Repos, func(i, j int) bool {
476 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
477 })
478 case "size", "revsize":
479 sort.Slice(repos.Repos, func(i, j int) bool {
480 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
481 })
482 case "ram", "revram":
483 sort.Slice(repos.Repos, func(i, j int) bool {
484 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
485 })
486 case "time", "revtime":
487 sort.Slice(repos.Repos, func(i, j int) bool {
488 return repos.Repos[i].IndexMetadata.IndexTime.Before(
489 repos.Repos[j].IndexMetadata.IndexTime)
490 })
491 default:
492 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
493 }
494 if strings.HasPrefix(order, "rev") {
495 for i, j := 0, len(repos.Repos)-1; i < j; {
496 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
497 i++
498 j--
499
500 }
501 }
502
503 aggregate := zoekt.RepoStats{
504 Repos: len(repos.Repos),
505 }
506 for _, s := range repos.Repos {
507 aggregate.Add(&s.Stats)
508 }
509
510 numStr := qvals.Get("num")
511 num, err := strconv.Atoi(numStr)
512 if err != nil || num <= 0 {
513 num = 0
514 }
515 if num > 0 {
516 if num > len(repos.Repos) {
517 num = len(repos.Repos)
518 }
519
520 repos.Repos = repos.Repos[:num]
521 }
522
523 res := RepoListInput{
524 Last: LastInput{
525 Query: qStr,
526 Num: num,
527 AutoFocus: true,
528 },
529 Stats: aggregate,
530 }
531
532 for _, r := range repos.Repos {
533 t := s.getTextTemplate(r.Repository.CommitURLTemplate)
534
535 repo := Repository{
536 Name: r.Repository.Name,
537 URL: r.Repository.URL,
538 IndexTime: r.IndexMetadata.IndexTime,
539 Size: r.Stats.ContentBytes,
540 MemorySize: r.Stats.IndexBytes,
541 Files: int64(r.Stats.Documents),
542 }
543 for _, b := range r.Repository.Branches {
544 var buf bytes.Buffer
545 if err := t.Execute(&buf, b); err != nil {
546 return nil, err
547 }
548 repo.Branches = append(repo.Branches,
549 Branch{
550 Name: b.Name,
551 Version: b.Version,
552 URL: buf.String(),
553 })
554 }
555 res.Repos = append(res.Repos, repo)
556 }
557 return &res, nil
558}
559
560func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
561 qvals := r.URL.Query()
562 fileStr := qvals.Get("f")
563 repoStr := qvals.Get("r")
564 queryStr := qvals.Get("q")
565 numStr := qvals.Get("num")
566 num, err := strconv.Atoi(numStr)
567 if err != nil || num <= 0 {
568 num = defaultNumResults
569 }
570
571 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
572 if err != nil {
573 return err
574 }
575
576 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
577 if err != nil {
578 return err
579 }
580
581 qs := []query.Q{
582 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
583 &query.Repo{Regexp: repoRe},
584 }
585
586 if branchStr := qvals.Get("b"); branchStr != "" {
587 qs = append(qs, &query.Branch{Pattern: branchStr})
588 }
589
590 q := &query.And{Children: qs}
591
592 sOpts := zoekt.SearchOptions{
593 Whole: true,
594 }
595
596 ctx := r.Context()
597 result, err := s.Searcher.Search(ctx, q, &sOpts)
598 if err != nil {
599 return err
600 }
601
602 if len(result.Files) != 1 {
603 var ss []string
604 for _, n := range result.Files {
605 ss = append(ss, n.FileName)
606 }
607 return fmt.Errorf("ambiguous result: %v", ss)
608 }
609
610 f := result.Files[0]
611
612 byteLines := bytes.Split(f.Content, []byte{'\n'})
613 strLines := make([]string, 0, len(byteLines))
614 for _, l := range byteLines {
615 strLines = append(strLines, string(l))
616 }
617
618 d := PrintInput{
619 Name: f.FileName,
620 Repo: f.Repository,
621 Lines: strLines,
622 Last: LastInput{
623 Query: queryStr,
624 Num: num,
625 AutoFocus: false,
626 },
627 }
628
629 var buf bytes.Buffer
630 if err := s.print.Execute(&buf, &d); err != nil {
631 return err
632 }
633
634 _, _ = w.Write(buf.Bytes())
635 return nil
636}