fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 "time"
32
33 "github.com/grafana/regexp"
34 "github.com/sourcegraph/zoekt"
35 zjson "github.com/sourcegraph/zoekt/json"
36 "github.com/sourcegraph/zoekt/query"
37 "github.com/sourcegraph/zoekt/rpc"
38 "github.com/sourcegraph/zoekt/stream"
39)
40
41var Funcmap = template.FuncMap{
42 "Inc": func(orig int) int {
43 return orig + 1
44 },
45 "More": func(orig int) int {
46 return orig * 3
47 },
48 "HumanUnit": func(orig int64) string {
49 b := orig
50 suffix := ""
51 if orig > 10*(1<<30) {
52 suffix = "G"
53 b = orig / (1 << 30)
54 } else if orig > 10*(1<<20) {
55 suffix = "M"
56 b = orig / (1 << 20)
57 } else if orig > 10*(1<<10) {
58 suffix = "K"
59 b = orig / (1 << 10)
60 }
61
62 return fmt.Sprintf("%d%s", b, suffix)
63 },
64 "LimitPre": func(limit int, pre string) string {
65 if len(pre) < limit {
66 return pre
67 }
68 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
69 },
70 "LimitPost": func(limit int, post string) string {
71 if len(post) < limit {
72 return post
73 }
74 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
75 },
76 "TrimTrailingNewline": func(s string) string {
77 return strings.TrimSuffix(s, "\n")
78 },
79}
80
81const defaultNumResults = 50
82
83type Server struct {
84 Searcher zoekt.Streamer
85
86 // Serve HTML interface
87 HTML bool
88
89 // Serve RPC
90 RPC bool
91
92 // If set, show files from the index.
93 Print bool
94
95 // Version string for this server.
96 Version string
97
98 // Depending on the Host header, add a query to the entry
99 // page. For example, when serving on "search.myproject.org"
100 // we could add "r:myproject" automatically. This allows a
101 // single instance to serve as search engine for multiple
102 // domains.
103 HostCustomQueries map[string]string
104
105 // This should contain the following templates: "repolist"
106 // (for the repo search result page), "result" for
107 // the search results, "search" (for the opening page),
108 // "box" for the search query input element and
109 // "print" for the show file functionality.
110 Top *template.Template
111
112 repolist *template.Template
113 search *template.Template
114 result *template.Template
115 print *template.Template
116 about *template.Template
117 robots *template.Template
118
119 startTime time.Time
120
121 templateMu sync.Mutex
122 templateCache map[string]*template.Template
123
124 lastStatsMu sync.Mutex
125 lastStats *zoekt.RepoStats
126 lastStatsTS time.Time
127}
128
129func (s *Server) getTemplate(str string) *template.Template {
130 s.templateMu.Lock()
131 defer s.templateMu.Unlock()
132 t := s.templateCache[str]
133 if t != nil {
134 return t
135 }
136
137 t, err := template.New("cache").Parse(str)
138 if err != nil {
139 log.Printf("template parse error: %v", err)
140 t = template.Must(template.New("empty").Parse(""))
141 }
142 s.templateCache[str] = t
143 return t
144}
145
146func NewMux(s *Server) (*http.ServeMux, error) {
147 s.print = s.Top.Lookup("print")
148 if s.print == nil {
149 return nil, fmt.Errorf("missing template 'print'")
150 }
151
152 for k, v := range map[string]**template.Template{
153 "results": &s.result,
154 "print": &s.print,
155 "search": &s.search,
156 "repolist": &s.repolist,
157 "about": &s.about,
158 "robots": &s.robots,
159 } {
160 *v = s.Top.Lookup(k)
161 if *v == nil {
162 return nil, fmt.Errorf("missing template %q", k)
163 }
164 }
165
166 s.templateCache = map[string]*template.Template{}
167 s.startTime = time.Now()
168
169 mux := http.NewServeMux()
170
171 if s.HTML {
172 mux.HandleFunc("/robots.txt", s.serveRobots)
173 mux.HandleFunc("/search", s.serveSearch)
174 mux.HandleFunc("/", s.serveSearchBox)
175 mux.HandleFunc("/about", s.serveAbout)
176 mux.HandleFunc("/print", s.servePrint)
177 }
178 if s.RPC {
179 mux.Handle(rpc.DefaultRPCPath, rpc.Server(traceAwareSearcher{s.Searcher})) // /rpc
180 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
181 mux.Handle(stream.DefaultSSEPath, stream.Server(traceAwareSearcher{s.Searcher})) // /stream
182 }
183
184 mux.HandleFunc("/healthz", s.serveHealthz)
185
186 return mux, nil
187}
188
189func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
190 q := &query.Const{Value: true}
191 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
192
193 result, err := s.Searcher.Search(r.Context(), q, opts)
194 if err != nil {
195 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
196 return
197 }
198
199 w.Header().Set("Content-Type", "application/json")
200
201 _ = json.NewEncoder(w).Encode(result)
202}
203
204func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
205 result, err := s.serveSearchErr(r)
206 if err != nil {
207 http.Error(w, err.Error(), http.StatusTeapot)
208 return
209 }
210
211 qvals := r.URL.Query()
212 if qvals.Get("format") == "json" {
213 w.Header().Add("Content-Type", "application/json")
214 encoder := json.NewEncoder(w)
215 encoder.Encode(result)
216 return
217 }
218
219 var buf bytes.Buffer
220 if result.Repos != nil {
221 err = s.repolist.Execute(&buf, &result.Repos)
222 } else if result.Result != nil {
223 err = s.result.Execute(&buf, &result.Result)
224 }
225 if err != nil {
226 http.Error(w, err.Error(), http.StatusTeapot)
227 return
228 }
229 w.Write(buf.Bytes())
230}
231
232func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
233 qvals := r.URL.Query()
234
235 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
236
237 queryStr := qvals.Get("q")
238 if queryStr == "" {
239 return nil, fmt.Errorf("no query found")
240 }
241
242 q, err := query.Parse(queryStr)
243 if err != nil {
244 return nil, err
245 }
246
247 // Experimental: The query string and boost exact phrases of it.
248 if phraseBoost, err := strconv.ParseFloat(qvals.Get("phrase-boost"), 64); err == nil {
249 q = query.ExpirementalPhraseBoost(q, queryStr, query.ExperimentalPhraseBoostOptions{
250 Boost: phraseBoost,
251 })
252 }
253
254 repoOnly := true
255 query.VisitAtoms(q, func(q query.Q) {
256 _, ok := q.(*query.Repo)
257 repoOnly = repoOnly && ok
258 })
259 if repoOnly {
260 repos, err := s.serveListReposErr(q, queryStr, r)
261 if err == nil {
262 return &ApiSearchResult{Repos: repos}, nil
263 }
264 return nil, err
265 }
266
267 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
268 repos, err := s.serveListReposErr(q, queryStr, r)
269 if err == nil {
270 return &ApiSearchResult{Repos: repos}, nil
271 }
272 return nil, err
273 }
274
275 numStr := qvals.Get("num")
276
277 num, err := strconv.Atoi(numStr)
278 if err != nil || num <= 0 {
279 num = defaultNumResults
280 }
281
282 sOpts := zoekt.SearchOptions{
283 MaxWallTime: 10 * time.Second,
284 }
285
286 numCtxLines := 0
287 if qvals.Get("format") == "json" {
288 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
289 numCtxLines, err = strconv.Atoi(ctxLinesStr)
290 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
291 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
292 }
293 }
294 }
295 sOpts.NumContextLines = numCtxLines
296
297 sOpts.SetDefaults()
298 sOpts.MaxDocDisplayCount = num
299 sOpts.DebugScore = debugScore
300
301 ctx := r.Context()
302 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
303 return nil, err
304 }
305
306 result, err := s.Searcher.Search(ctx, q, &sOpts)
307 if err != nil {
308 return nil, err
309 }
310
311 fileMatches, err := s.formatResults(result, queryStr, s.Print)
312 if err != nil {
313 return nil, err
314 }
315
316 res := ResultInput{
317 Last: LastInput{
318 Query: queryStr,
319 Num: num,
320 AutoFocus: true,
321 },
322 Stats: result.Stats,
323 Query: q.String(),
324 QueryStr: queryStr,
325 FileMatches: fileMatches,
326 }
327 if res.Stats.Wait < res.Stats.Duration/10 {
328 // Suppress queueing stats if they are neglible.
329 res.Stats.Wait = 0
330 }
331
332 res.Last.Debug = debugScore
333 return &ApiSearchResult{Result: &res}, nil
334}
335
336func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
337 err := s.servePrintErr(w, r)
338 if err != nil {
339 http.Error(w, err.Error(), http.StatusTeapot)
340 }
341}
342
343const statsStaleNess = 30 * time.Second
344
345func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
346 s.lastStatsMu.Lock()
347 stats := s.lastStats
348 if time.Since(s.lastStatsTS) > statsStaleNess {
349 stats = nil
350 }
351 s.lastStatsMu.Unlock()
352
353 if stats != nil {
354 return stats, nil
355 }
356
357 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
358 if err != nil {
359 return nil, err
360 }
361
362 stats = &repos.Stats
363
364 s.lastStatsMu.Lock()
365 s.lastStatsTS = time.Now()
366 s.lastStats = stats
367 s.lastStatsMu.Unlock()
368
369 return stats, nil
370}
371
372func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
373 stats, err := s.fetchStats(r.Context())
374 if err != nil {
375 return err
376 }
377 d := SearchBoxInput{
378 Last: LastInput{
379 Num: defaultNumResults,
380 AutoFocus: true,
381 },
382 Stats: stats,
383 Version: s.Version,
384 Uptime: time.Since(s.startTime),
385 }
386
387 d.Last.Query = r.URL.Query().Get("q")
388 if d.Last.Query == "" {
389 custom := s.HostCustomQueries[r.Host]
390 if custom == "" {
391 host, _, _ := net.SplitHostPort(r.Host)
392 custom = s.HostCustomQueries[host]
393 }
394
395 if custom != "" {
396 d.Last.Query = custom + " "
397 }
398 }
399
400 var buf bytes.Buffer
401 if err := s.search.Execute(&buf, &d); err != nil {
402 return err
403 }
404 _, _ = w.Write(buf.Bytes())
405 return nil
406}
407
408func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
409 if err := s.serveSearchBoxErr(w, r); err != nil {
410 http.Error(w, err.Error(), http.StatusTeapot)
411 }
412}
413
414func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
415 stats, err := s.fetchStats(r.Context())
416 if err != nil {
417 return err
418 }
419
420 d := SearchBoxInput{
421 Stats: stats,
422 Version: s.Version,
423 Uptime: time.Since(s.startTime),
424 }
425
426 var buf bytes.Buffer
427 if err := s.about.Execute(&buf, &d); err != nil {
428 return err
429 }
430 _, _ = w.Write(buf.Bytes())
431 return nil
432}
433
434func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
435 if err := s.serveAboutErr(w, r); err != nil {
436 http.Error(w, err.Error(), http.StatusTeapot)
437 }
438}
439
440func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
441 data := struct{}{}
442 var buf bytes.Buffer
443 if err := s.robots.Execute(&buf, &data); err != nil {
444 return err
445 }
446 _, _ = w.Write(buf.Bytes())
447 return nil
448}
449
450func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
451 if err := s.serveRobotsErr(w, r); err != nil {
452 http.Error(w, err.Error(), http.StatusTeapot)
453 }
454}
455
456func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
457 ctx := r.Context()
458 repos, err := s.Searcher.List(ctx, q, nil)
459 if err != nil {
460 return nil, err
461 }
462
463 qvals := r.URL.Query()
464 order := qvals.Get("order")
465 switch order {
466 case "", "name", "revname":
467 sort.Slice(repos.Repos, func(i, j int) bool {
468 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
469 })
470 case "size", "revsize":
471 sort.Slice(repos.Repos, func(i, j int) bool {
472 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
473 })
474 case "ram", "revram":
475 sort.Slice(repos.Repos, func(i, j int) bool {
476 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
477 })
478 case "time", "revtime":
479 sort.Slice(repos.Repos, func(i, j int) bool {
480 return repos.Repos[i].IndexMetadata.IndexTime.Before(
481 repos.Repos[j].IndexMetadata.IndexTime)
482 })
483 default:
484 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
485 }
486 if strings.HasPrefix(order, "rev") {
487 for i, j := 0, len(repos.Repos)-1; i < j; {
488 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
489 i++
490 j--
491
492 }
493 }
494
495 aggregate := zoekt.RepoStats{
496 Repos: len(repos.Repos),
497 }
498 for _, s := range repos.Repos {
499 aggregate.Add(&s.Stats)
500 }
501
502 numStr := qvals.Get("num")
503 num, err := strconv.Atoi(numStr)
504 if err != nil || num <= 0 {
505 num = 0
506 }
507 if num > 0 {
508 if num > len(repos.Repos) {
509 num = len(repos.Repos)
510 }
511
512 repos.Repos = repos.Repos[:num]
513 }
514
515 res := RepoListInput{
516 Last: LastInput{
517 Query: qStr,
518 Num: num,
519 AutoFocus: true,
520 },
521 Stats: aggregate,
522 }
523
524 for _, r := range repos.Repos {
525 t := s.getTemplate(r.Repository.CommitURLTemplate)
526
527 repo := Repository{
528 Name: r.Repository.Name,
529 URL: r.Repository.URL,
530 IndexTime: r.IndexMetadata.IndexTime,
531 Size: r.Stats.ContentBytes,
532 MemorySize: r.Stats.IndexBytes,
533 Files: int64(r.Stats.Documents),
534 }
535 for _, b := range r.Repository.Branches {
536 var buf bytes.Buffer
537 if err := t.Execute(&buf, b); err != nil {
538 return nil, err
539 }
540 repo.Branches = append(repo.Branches,
541 Branch{
542 Name: b.Name,
543 Version: b.Version,
544 URL: buf.String(),
545 })
546 }
547 res.Repos = append(res.Repos, repo)
548 }
549 return &res, nil
550}
551
552func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
553 qvals := r.URL.Query()
554 fileStr := qvals.Get("f")
555 repoStr := qvals.Get("r")
556 queryStr := qvals.Get("q")
557 numStr := qvals.Get("num")
558 num, err := strconv.Atoi(numStr)
559 if err != nil || num <= 0 {
560 num = defaultNumResults
561 }
562
563 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
564 if err != nil {
565 return err
566 }
567
568 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
569 if err != nil {
570 return err
571 }
572
573 qs := []query.Q{
574 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
575 &query.Repo{Regexp: repoRe},
576 }
577
578 if branchStr := qvals.Get("b"); branchStr != "" {
579 qs = append(qs, &query.Branch{Pattern: branchStr})
580 }
581
582 q := &query.And{Children: qs}
583
584 sOpts := zoekt.SearchOptions{
585 Whole: true,
586 }
587
588 ctx := r.Context()
589 result, err := s.Searcher.Search(ctx, q, &sOpts)
590 if err != nil {
591 return err
592 }
593
594 if len(result.Files) != 1 {
595 var ss []string
596 for _, n := range result.Files {
597 ss = append(ss, n.FileName)
598 }
599 return fmt.Errorf("ambiguous result: %v", ss)
600 }
601
602 f := result.Files[0]
603
604 byteLines := bytes.Split(f.Content, []byte{'\n'})
605 strLines := make([]string, 0, len(byteLines))
606 for _, l := range byteLines {
607 strLines = append(strLines, string(l))
608 }
609
610 d := PrintInput{
611 Name: f.FileName,
612 Repo: f.Repository,
613 Lines: strLines,
614 Last: LastInput{
615 Query: queryStr,
616 Num: num,
617 AutoFocus: false,
618 },
619 }
620
621 var buf bytes.Buffer
622 if err := s.print.Execute(&buf, &d); err != nil {
623 return err
624 }
625
626 _, _ = w.Write(buf.Bytes())
627 return nil
628}