fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package web
16
17import (
18 "bytes"
19 "context"
20 "encoding/json"
21 "fmt"
22 "html/template"
23 "log"
24 "net"
25 "net/http"
26 "regexp/syntax"
27 "sort"
28 "strconv"
29 "strings"
30 "sync"
31 "time"
32
33 "github.com/grafana/regexp"
34 "github.com/sourcegraph/zoekt"
35 zjson "github.com/sourcegraph/zoekt/json"
36 "github.com/sourcegraph/zoekt/query"
37)
38
39var Funcmap = template.FuncMap{
40 "Inc": func(orig int) int {
41 return orig + 1
42 },
43 "More": func(orig int) int {
44 return orig * 3
45 },
46 "HumanUnit": func(orig int64) string {
47 b := orig
48 suffix := ""
49 if orig > 10*(1<<30) {
50 suffix = "G"
51 b = orig / (1 << 30)
52 } else if orig > 10*(1<<20) {
53 suffix = "M"
54 b = orig / (1 << 20)
55 } else if orig > 10*(1<<10) {
56 suffix = "K"
57 b = orig / (1 << 10)
58 }
59
60 return fmt.Sprintf("%d%s", b, suffix)
61 },
62 "LimitPre": func(limit int, pre string) string {
63 if len(pre) < limit {
64 return pre
65 }
66 return fmt.Sprintf("...(%d bytes skipped)...%s", len(pre)-limit, pre[len(pre)-limit:])
67 },
68 "LimitPost": func(limit int, post string) string {
69 if len(post) < limit {
70 return post
71 }
72 return fmt.Sprintf("%s...(%d bytes skipped)...", post[:limit], len(post)-limit)
73 },
74 "TrimTrailingNewline": func(s string) string {
75 return strings.TrimSuffix(s, "\n")
76 },
77}
78
79const defaultNumResults = 50
80
81type Server struct {
82 Searcher zoekt.Streamer
83
84 // Serve HTML interface
85 HTML bool
86
87 // Serve RPC
88 RPC bool
89
90 // If set, show files from the index.
91 Print bool
92
93 // Version string for this server.
94 Version string
95
96 // Depending on the Host header, add a query to the entry
97 // page. For example, when serving on "search.myproject.org"
98 // we could add "r:myproject" automatically. This allows a
99 // single instance to serve as search engine for multiple
100 // domains.
101 HostCustomQueries map[string]string
102
103 // This should contain the following templates: "repolist"
104 // (for the repo search result page), "result" for
105 // the search results, "search" (for the opening page),
106 // "box" for the search query input element and
107 // "print" for the show file functionality.
108 Top *template.Template
109
110 repolist *template.Template
111 search *template.Template
112 result *template.Template
113 print *template.Template
114 about *template.Template
115 robots *template.Template
116
117 startTime time.Time
118
119 templateMu sync.Mutex
120 templateCache map[string]*template.Template
121
122 lastStatsMu sync.Mutex
123 lastStats *zoekt.RepoStats
124 lastStatsTS time.Time
125}
126
127func (s *Server) getTemplate(str string) *template.Template {
128 s.templateMu.Lock()
129 defer s.templateMu.Unlock()
130 t := s.templateCache[str]
131 if t != nil {
132 return t
133 }
134
135 t, err := template.New("cache").Parse(str)
136 if err != nil {
137 log.Printf("template parse error: %v", err)
138 t = template.Must(template.New("empty").Parse(""))
139 }
140 s.templateCache[str] = t
141 return t
142}
143
144func NewMux(s *Server) (*http.ServeMux, error) {
145 s.print = s.Top.Lookup("print")
146 if s.print == nil {
147 return nil, fmt.Errorf("missing template 'print'")
148 }
149
150 for k, v := range map[string]**template.Template{
151 "results": &s.result,
152 "print": &s.print,
153 "search": &s.search,
154 "repolist": &s.repolist,
155 "about": &s.about,
156 "robots": &s.robots,
157 } {
158 *v = s.Top.Lookup(k)
159 if *v == nil {
160 return nil, fmt.Errorf("missing template %q", k)
161 }
162 }
163
164 s.templateCache = map[string]*template.Template{}
165 s.startTime = time.Now()
166
167 mux := http.NewServeMux()
168
169 if s.HTML {
170 mux.HandleFunc("/robots.txt", s.serveRobots)
171 mux.HandleFunc("/search", s.serveSearch)
172 mux.HandleFunc("/", s.serveSearchBox)
173 mux.HandleFunc("/about", s.serveAbout)
174 mux.HandleFunc("/print", s.servePrint)
175 }
176 if s.RPC {
177 mux.Handle("/api/", http.StripPrefix("/api", zjson.JSONServer(traceAwareSearcher{s.Searcher})))
178 }
179
180 mux.HandleFunc("/healthz", s.serveHealthz)
181
182 return mux, nil
183}
184
185func (s *Server) serveHealthz(w http.ResponseWriter, r *http.Request) {
186 q := &query.Const{Value: true}
187 opts := &zoekt.SearchOptions{ShardMaxMatchCount: 1, TotalMaxMatchCount: 1, MaxDocDisplayCount: 1}
188
189 result, err := s.Searcher.Search(r.Context(), q, opts)
190 if err != nil {
191 http.Error(w, fmt.Sprintf("not ready: %v", err), http.StatusInternalServerError)
192 return
193 }
194
195 w.Header().Set("Content-Type", "application/json")
196
197 _ = json.NewEncoder(w).Encode(result)
198}
199
200func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request) {
201 result, err := s.serveSearchErr(r)
202 if err != nil {
203 http.Error(w, err.Error(), http.StatusTeapot)
204 return
205 }
206
207 qvals := r.URL.Query()
208 if qvals.Get("format") == "json" {
209 w.Header().Add("Content-Type", "application/json")
210 encoder := json.NewEncoder(w)
211 encoder.Encode(result)
212 return
213 }
214
215 var buf bytes.Buffer
216 if result.Repos != nil {
217 err = s.repolist.Execute(&buf, &result.Repos)
218 } else if result.Result != nil {
219 err = s.result.Execute(&buf, &result.Result)
220 }
221 if err != nil {
222 http.Error(w, err.Error(), http.StatusTeapot)
223 return
224 }
225 w.Write(buf.Bytes())
226}
227
228func (s *Server) serveSearchErr(r *http.Request) (*ApiSearchResult, error) {
229 qvals := r.URL.Query()
230
231 debugScore, _ := strconv.ParseBool(qvals.Get("debug"))
232
233 queryStr := qvals.Get("q")
234 if queryStr == "" {
235 return nil, fmt.Errorf("no query found")
236 }
237
238 q, err := query.Parse(queryStr)
239 if err != nil {
240 return nil, err
241 }
242
243 // Experimental: The query string and boost exact phrases of it.
244 if phraseBoost, err := strconv.ParseFloat(qvals.Get("phrase-boost"), 64); err == nil {
245 q = query.ExpirementalPhraseBoost(q, queryStr, query.ExperimentalPhraseBoostOptions{
246 Boost: phraseBoost,
247 })
248 }
249
250 repoOnly := true
251 query.VisitAtoms(q, func(q query.Q) {
252 _, ok := q.(*query.Repo)
253 repoOnly = repoOnly && ok
254 })
255 if repoOnly {
256 repos, err := s.serveListReposErr(q, queryStr, r)
257 if err == nil {
258 return &ApiSearchResult{Repos: repos}, nil
259 }
260 return nil, err
261 }
262
263 if qt, ok := q.(*query.Type); ok && qt.Type == query.TypeRepo {
264 repos, err := s.serveListReposErr(q, queryStr, r)
265 if err == nil {
266 return &ApiSearchResult{Repos: repos}, nil
267 }
268 return nil, err
269 }
270
271 numStr := qvals.Get("num")
272
273 num, err := strconv.Atoi(numStr)
274 if err != nil || num <= 0 {
275 num = defaultNumResults
276 }
277
278 sOpts := zoekt.SearchOptions{
279 MaxWallTime: 10 * time.Second,
280 }
281
282 numCtxLines := 0
283 if ctxLinesStr := qvals.Get("ctx"); ctxLinesStr != "" {
284 numCtxLines, err = strconv.Atoi(ctxLinesStr)
285 if err != nil || numCtxLines < 0 || numCtxLines > 10 {
286 return nil, fmt.Errorf("Number of context lines must be between 0 and 10")
287 }
288 }
289 sOpts.NumContextLines = numCtxLines
290
291 sOpts.SetDefaults()
292 sOpts.MaxDocDisplayCount = num
293 sOpts.DebugScore = debugScore
294
295 ctx := r.Context()
296 if err := zjson.CalculateDefaultSearchLimits(ctx, q, s.Searcher, &sOpts); err != nil {
297 return nil, err
298 }
299
300 result, err := s.Searcher.Search(ctx, q, &sOpts)
301 if err != nil {
302 return nil, err
303 }
304
305 fileMatches, err := s.formatResults(result, queryStr, s.Print)
306 if err != nil {
307 return nil, err
308 }
309
310 res := ResultInput{
311 Last: LastInput{
312 Query: queryStr,
313 Num: num,
314 Ctx: numCtxLines,
315 AutoFocus: true,
316 },
317 Stats: result.Stats,
318 Query: q.String(),
319 QueryStr: queryStr,
320 FileMatches: fileMatches,
321 }
322 if res.Stats.Wait < res.Stats.Duration/10 {
323 // Suppress queueing stats if they are neglible.
324 res.Stats.Wait = 0
325 }
326
327 res.Last.Debug = debugScore
328 return &ApiSearchResult{Result: &res}, nil
329}
330
331func (s *Server) servePrint(w http.ResponseWriter, r *http.Request) {
332 err := s.servePrintErr(w, r)
333 if err != nil {
334 http.Error(w, err.Error(), http.StatusTeapot)
335 }
336}
337
338const statsStaleNess = 30 * time.Second
339
340func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
341 s.lastStatsMu.Lock()
342 stats := s.lastStats
343 if time.Since(s.lastStatsTS) > statsStaleNess {
344 stats = nil
345 }
346 s.lastStatsMu.Unlock()
347
348 if stats != nil {
349 return stats, nil
350 }
351
352 repos, err := s.Searcher.List(ctx, &query.Const{Value: true}, nil)
353 if err != nil {
354 return nil, err
355 }
356
357 stats = &repos.Stats
358
359 s.lastStatsMu.Lock()
360 s.lastStatsTS = time.Now()
361 s.lastStats = stats
362 s.lastStatsMu.Unlock()
363
364 return stats, nil
365}
366
367func (s *Server) serveSearchBoxErr(w http.ResponseWriter, r *http.Request) error {
368 stats, err := s.fetchStats(r.Context())
369 if err != nil {
370 return err
371 }
372 d := SearchBoxInput{
373 Last: LastInput{
374 Num: defaultNumResults,
375 AutoFocus: true,
376 },
377 Stats: stats,
378 Version: s.Version,
379 Uptime: time.Since(s.startTime),
380 }
381
382 d.Last.Query = r.URL.Query().Get("q")
383 if d.Last.Query == "" {
384 custom := s.HostCustomQueries[r.Host]
385 if custom == "" {
386 host, _, _ := net.SplitHostPort(r.Host)
387 custom = s.HostCustomQueries[host]
388 }
389
390 if custom != "" {
391 d.Last.Query = custom + " "
392 }
393 }
394
395 var buf bytes.Buffer
396 if err := s.search.Execute(&buf, &d); err != nil {
397 return err
398 }
399 _, _ = w.Write(buf.Bytes())
400 return nil
401}
402
403func (s *Server) serveSearchBox(w http.ResponseWriter, r *http.Request) {
404 if err := s.serveSearchBoxErr(w, r); err != nil {
405 http.Error(w, err.Error(), http.StatusTeapot)
406 }
407}
408
409func (s *Server) serveAboutErr(w http.ResponseWriter, r *http.Request) error {
410 stats, err := s.fetchStats(r.Context())
411 if err != nil {
412 return err
413 }
414
415 d := SearchBoxInput{
416 Stats: stats,
417 Version: s.Version,
418 Uptime: time.Since(s.startTime),
419 }
420
421 var buf bytes.Buffer
422 if err := s.about.Execute(&buf, &d); err != nil {
423 return err
424 }
425 _, _ = w.Write(buf.Bytes())
426 return nil
427}
428
429func (s *Server) serveAbout(w http.ResponseWriter, r *http.Request) {
430 if err := s.serveAboutErr(w, r); err != nil {
431 http.Error(w, err.Error(), http.StatusTeapot)
432 }
433}
434
435func (s *Server) serveRobotsErr(w http.ResponseWriter, r *http.Request) error {
436 data := struct{}{}
437 var buf bytes.Buffer
438 if err := s.robots.Execute(&buf, &data); err != nil {
439 return err
440 }
441 _, _ = w.Write(buf.Bytes())
442 return nil
443}
444
445func (s *Server) serveRobots(w http.ResponseWriter, r *http.Request) {
446 if err := s.serveRobotsErr(w, r); err != nil {
447 http.Error(w, err.Error(), http.StatusTeapot)
448 }
449}
450
451func (s *Server) serveListReposErr(q query.Q, qStr string, r *http.Request) (*RepoListInput, error) {
452 ctx := r.Context()
453 repos, err := s.Searcher.List(ctx, q, nil)
454 if err != nil {
455 return nil, err
456 }
457
458 qvals := r.URL.Query()
459 order := qvals.Get("order")
460 switch order {
461 case "", "name", "revname":
462 sort.Slice(repos.Repos, func(i, j int) bool {
463 return strings.Compare(repos.Repos[i].Repository.Name, repos.Repos[j].Repository.Name) < 0
464 })
465 case "size", "revsize":
466 sort.Slice(repos.Repos, func(i, j int) bool {
467 return repos.Repos[i].Stats.ContentBytes < repos.Repos[j].Stats.ContentBytes
468 })
469 case "ram", "revram":
470 sort.Slice(repos.Repos, func(i, j int) bool {
471 return repos.Repos[i].Stats.IndexBytes < repos.Repos[j].Stats.IndexBytes
472 })
473 case "time", "revtime":
474 sort.Slice(repos.Repos, func(i, j int) bool {
475 return repos.Repos[i].IndexMetadata.IndexTime.Before(
476 repos.Repos[j].IndexMetadata.IndexTime)
477 })
478 default:
479 return nil, fmt.Errorf("got unknown sort key %q, allowed [rev]name, [rev]time, [rev]size", order)
480 }
481 if strings.HasPrefix(order, "rev") {
482 for i, j := 0, len(repos.Repos)-1; i < j; {
483 repos.Repos[i], repos.Repos[j] = repos.Repos[j], repos.Repos[i]
484 i++
485 j--
486
487 }
488 }
489
490 aggregate := zoekt.RepoStats{
491 Repos: len(repos.Repos),
492 }
493 for _, s := range repos.Repos {
494 aggregate.Add(&s.Stats)
495 }
496
497 numStr := qvals.Get("num")
498 num, err := strconv.Atoi(numStr)
499 if err != nil || num <= 0 {
500 num = 0
501 }
502 if num > 0 {
503 if num > len(repos.Repos) {
504 num = len(repos.Repos)
505 }
506
507 repos.Repos = repos.Repos[:num]
508 }
509
510 res := RepoListInput{
511 Last: LastInput{
512 Query: qStr,
513 Num: num,
514 AutoFocus: true,
515 },
516 Stats: aggregate,
517 }
518
519 for _, r := range repos.Repos {
520 t := s.getTemplate(r.Repository.CommitURLTemplate)
521
522 repo := Repository{
523 Name: r.Repository.Name,
524 URL: r.Repository.URL,
525 IndexTime: r.IndexMetadata.IndexTime,
526 Size: r.Stats.ContentBytes,
527 MemorySize: r.Stats.IndexBytes,
528 Files: int64(r.Stats.Documents),
529 }
530 for _, b := range r.Repository.Branches {
531 var buf bytes.Buffer
532 if err := t.Execute(&buf, b); err != nil {
533 return nil, err
534 }
535 repo.Branches = append(repo.Branches,
536 Branch{
537 Name: b.Name,
538 Version: b.Version,
539 URL: buf.String(),
540 })
541 }
542 res.Repos = append(res.Repos, repo)
543 }
544 return &res, nil
545}
546
547func (s *Server) servePrintErr(w http.ResponseWriter, r *http.Request) error {
548 qvals := r.URL.Query()
549 fileStr := qvals.Get("f")
550 repoStr := qvals.Get("r")
551 queryStr := qvals.Get("q")
552 numStr := qvals.Get("num")
553 num, err := strconv.Atoi(numStr)
554 if err != nil || num <= 0 {
555 num = defaultNumResults
556 }
557
558 re, err := syntax.Parse("^"+regexp.QuoteMeta(fileStr)+"$", 0)
559 if err != nil {
560 return err
561 }
562
563 repoRe, err := regexp.Compile("^" + regexp.QuoteMeta(repoStr) + "$")
564 if err != nil {
565 return err
566 }
567
568 qs := []query.Q{
569 &query.Regexp{Regexp: re, FileName: true, CaseSensitive: true},
570 &query.Repo{Regexp: repoRe},
571 }
572
573 if branchStr := qvals.Get("b"); branchStr != "" {
574 qs = append(qs, &query.Branch{Pattern: branchStr})
575 }
576
577 q := &query.And{Children: qs}
578
579 sOpts := zoekt.SearchOptions{
580 Whole: true,
581 }
582
583 ctx := r.Context()
584 result, err := s.Searcher.Search(ctx, q, &sOpts)
585 if err != nil {
586 return err
587 }
588
589 if len(result.Files) != 1 {
590 var ss []string
591 for _, n := range result.Files {
592 ss = append(ss, n.FileName)
593 }
594 return fmt.Errorf("ambiguous result: %v", ss)
595 }
596
597 f := result.Files[0]
598
599 byteLines := bytes.Split(f.Content, []byte{'\n'})
600 strLines := make([]string, 0, len(byteLines))
601 for _, l := range byteLines {
602 strLines = append(strLines, string(l))
603 }
604
605 d := PrintInput{
606 Name: f.FileName,
607 Repo: f.Repository,
608 Lines: strLines,
609 Last: LastInput{
610 Query: queryStr,
611 Num: num,
612 AutoFocus: false,
613 },
614 }
615
616 var buf bytes.Buffer
617 if err := s.print.Execute(&buf, &d); err != nil {
618 return err
619 }
620
621 _, _ = w.Write(buf.Bytes())
622 return nil
623}