Add commands. · boltless.me/zoekt@4a00dd1

+158

cmd/index/main.go

··· 1 + // Copyright 2016 Google Inc. All rights reserved. 2 + // 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 + // you may not use this file except in compliance with the License. 5 + // You may obtain a copy of the License at 6 + // 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 + // 9 + // Unless required by applicable law or agreed to in writing, software 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 + // See the License for the specific language governing permissions and 13 + // limitations under the License. 14 + 15 + package main 16 + 17 + import ( 18 + "bytes" 19 + "flag" 20 + "fmt" 21 + "io/ioutil" 22 + "log" 23 + "os" 24 + "path/filepath" 25 + "runtime/pprof" 26 + "sync" 27 + 28 + "github.com/hanwen/codesearch" 29 + ) 30 + 31 + type fileAggregator struct { 32 + chunks chan<- []string 33 + files []string 34 + total int64 35 + shardMax int64 36 + sizeMax int64 37 + } 38 + 39 + func (a *fileAggregator) flush() { 40 + a.chunks <- a.files 41 + a.files = nil 42 + close(a.chunks) 43 + } 44 + 45 + func (a *fileAggregator) add(path string, info os.FileInfo, err error) error { 46 + sz := info.Size() 47 + if sz > a.sizeMax || !info.Mode().IsRegular() { 48 + return nil 49 + } 50 + 51 + a.files = append(a.files, path) 52 + a.total += sz 53 + 54 + if a.total > a.shardMax { 55 + a.chunks <- a.files 56 + a.files = nil 57 + a.total = 0 58 + } 59 + return nil 60 + } 61 + 62 + func main() { 63 + var cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file") 64 + var sizeMax = flag.Int("file_limit", 128*1024, "maximum file size") 65 + var shardLimit = flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard") 66 + var parallelism = flag.Int("parallelism", 4, "maximum number of parallel indexing processes.") 67 + 68 + index := flag.String("index", ".csindex.%05d", "index file to use") 69 + 70 + flag.Parse() 71 + 72 + if *cpuProfile != "" { 73 + f, err := os.Create(*cpuProfile) 74 + if err != nil { 75 + log.Fatal(err) 76 + } 77 + pprof.StartCPUProfile(f) 78 + defer pprof.StopCPUProfile() 79 + } 80 + 81 + chunks := make(chan []string, 10) 82 + agg := fileAggregator{ 83 + chunks: chunks, 84 + sizeMax: int64(*sizeMax), 85 + shardMax: int64(*shardLimit), 86 + } 87 + 88 + shardNum := 0 89 + go func() { 90 + for _, a := range flag.Args() { 91 + if err := filepath.Walk(a, agg.add); err != nil { 92 + log.Fatal(err) 93 + } 94 + } 95 + agg.flush() 96 + }() 97 + 98 + var wg sync.WaitGroup 99 + errors := make(chan error, 10) 100 + throttle := make(chan int, *parallelism) 101 + 102 + for names := range chunks { 103 + fn := fmt.Sprintf(*index, shardNum) 104 + shardNum++ 105 + wg.Add(1) 106 + go func(nm []string) { 107 + throttle <- 1 108 + errors <- buildShard(fn, nm) 109 + <-throttle 110 + wg.Done() 111 + }(names) 112 + } 113 + 114 + go func() { 115 + wg.Wait() 116 + close(errors) 117 + }() 118 + 119 + for err := range errors { 120 + if err != nil { 121 + log.Fatal(err) 122 + } 123 + } 124 + } 125 + 126 + func buildShard(shardName string, files []string) error { 127 + f, err := os.OpenFile( 128 + shardName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600) 129 + if err != nil { 130 + return err 131 + } 132 + 133 + b := codesearch.NewIndexBuilder() 134 + total := 0 135 + for _, a := range files { 136 + c, err := ioutil.ReadFile(a) 137 + if bytes.IndexByte(c, 0) != -1 { 138 + // skip binary 139 + continue 140 + } 141 + total += len(c) 142 + if err != nil { 143 + log.Println(err) 144 + } else { 145 + b.AddFile(a, c) 146 + } 147 + } 148 + 149 + if err := b.Write(f); err != nil { 150 + log.Println("Write", err) 151 + } 152 + if err := f.Close(); err != nil { 153 + log.Println("Write", err) 154 + } 155 + log.Printf("%s: indexed %d bytes\n", shardName, total) 156 + 157 + return nil 158 + }

+62

cmd/search/main.go

··· 1 + // Copyright 2016 Google Inc. All rights reserved. 2 + // 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 + // you may not use this file except in compliance with the License. 5 + // You may obtain a copy of the License at 6 + // 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 + // 9 + // Unless required by applicable law or agreed to in writing, software 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 + // See the License for the specific language governing permissions and 13 + // limitations under the License. 14 + 15 + package main 16 + 17 + import ( 18 + "flag" 19 + "fmt" 20 + "log" 21 + 22 + "github.com/hanwen/codesearch" 23 + ) 24 + 25 + // go1.4 26 + func lastIndex(b string, c byte) int { 27 + for i := len(b) - 1; i >= 0; i-- { 28 + if b[i] == c { 29 + return i 30 + } 31 + } 32 + return -1 33 + } 34 + 35 + const CONTEXT = 20 36 + 37 + func displayMatches(matches []codesearch.Match, pat string) { 38 + for _, m := range matches { 39 + fmt.Printf("%s:%d:%s\n", m.Name, m.LineNum, m.Line) 40 + } 41 + } 42 + 43 + func main() { 44 + index := flag.String("index", ".csindex.*", "index file glob to use") 45 + flag.Parse() 46 + 47 + searcher, err := codesearch.NewShardedSearcher(*index) 48 + if err != nil { 49 + log.Fatal(err) 50 + } 51 + 52 + if len(flag.Args()) == 0 { 53 + log.Fatal("needs argument") 54 + } 55 + pat := flag.Arg(0) 56 + ms, err := searcher.Search(pat) 57 + if err != nil { 58 + log.Fatal(err) 59 + } 60 + 61 + displayMatches(ms, pat) 62 + }

+148

cmd/server/main.go

··· 1 + package main 2 + 3 + import ( 4 + "bytes" 5 + "flag" 6 + "fmt" 7 + "html/template" 8 + "log" 9 + "net/http" 10 + "strconv" 11 + "time" 12 + 13 + "github.com/hanwen/codesearch" 14 + ) 15 + 16 + type httpServer struct { 17 + searcher codesearch.Searcher 18 + } 19 + 20 + func (s *httpServer) serveSearch(w http.ResponseWriter, r *http.Request) { 21 + if err := s.serveSearchErr(w, r); err != nil { 22 + http.Error(w, err.Error(), http.StatusTeapot) 23 + } 24 + } 25 + 26 + func (s *httpServer) serveSearchBox(w http.ResponseWriter, r *http.Request) { 27 + w.Write([]byte(`<html> 28 + <head> 29 + </head> 30 + <body> 31 + <div style="margin: 3em; padding 3em; position: center;"> 32 + <form action="search"> 33 + Search some code: <input type="text" name="q"> Max results: <input type="text" name="num" value="50"><br> 34 + <input type="submit" value="Search"> 35 + </form> 36 + </div> 37 + </body> 38 + </html> 39 + `)) 40 + } 41 + 42 + type MatchLine struct { 43 + LineNum int 44 + Line string 45 + } 46 + 47 + type MatchData struct { 48 + FileName string 49 + Pre string 50 + MatchText string 51 + Post string 52 + LineNum int 53 + } 54 + 55 + type ResultsPage struct { 56 + Query string 57 + MatchCount int 58 + Duration time.Duration 59 + Matches []MatchData 60 + } 61 + 62 + var resultTemplate = template.Must(template.New("page").Parse(`<html> 63 + <head> 64 + <title>Search results</title> 65 + </head> 66 + <body> 67 + Found {{.MatchCount}} results for 68 + <pre style="background: #ffc;">{{.Query}}</pre> in {{.Duration}}: 69 + <p> 70 + {{range .Matches}} 71 + <tt>{{.FileName}}:{{.LineNum}}</tt> 72 + <br> 73 + <div style="background: #eef;"> 74 + <pre>{{.Pre}}<b>{{.MatchText}}</b>{{.Post}}</pre> 75 + </div> 76 + {{end}} 77 + </body> 78 + </html> 79 + `)) 80 + 81 + func (s *httpServer) serveSearchErr(w http.ResponseWriter, r *http.Request) error { 82 + qvals := r.URL.Query() 83 + query := qvals.Get("q") 84 + numStr := qvals.Get("num") 85 + if query == "" { 86 + return fmt.Errorf("no query found") 87 + } 88 + 89 + num, err := strconv.Atoi(numStr) 90 + if err != nil { 91 + num = 50 92 + } 93 + 94 + startT := time.Now() 95 + matches, err := s.searcher.Search(query) 96 + if err != nil { 97 + return err 98 + } 99 + 100 + res := ResultsPage{ 101 + Query: query, 102 + MatchCount: len(matches), 103 + Duration: time.Now().Sub(startT), 104 + } 105 + 106 + if len(matches) > num { 107 + matches = matches[:num] 108 + } 109 + 110 + for _, m := range matches { 111 + res.Matches = append(res.Matches, MatchData{ 112 + FileName: m.Name, 113 + LineNum: m.LineNum, 114 + Pre: m.Line[:m.LineOff], 115 + MatchText: m.Line[m.LineOff : m.LineOff+len(query)], 116 + Post: m.Line[m.LineOff+len(query):], 117 + }) 118 + } 119 + 120 + var buf bytes.Buffer 121 + if err := resultTemplate.Execute(&buf, res); err != nil { 122 + return err 123 + } 124 + 125 + w.Write(buf.Bytes()) 126 + return nil 127 + } 128 + 129 + func main() { 130 + listen := flag.String("listen", ":6070", "address to listen on.") 131 + index := flag.String("index", ".csindex.*", "index file glob to use") 132 + flag.Parse() 133 + 134 + searcher, err := codesearch.NewShardedSearcher(*index) 135 + if err != nil { 136 + log.Fatal(err) 137 + } 138 + 139 + serv := httpServer{ 140 + searcher: searcher, 141 + } 142 + 143 + http.HandleFunc("/search", serv.serveSearch) 144 + http.HandleFunc("/", serv.serveSearchBox) 145 + log.Printf("serving on %s", *listen) 146 + err = http.ListenAndServe(*listen, nil) 147 + log.Printf("ListenAndServe: %v", err) 148 + }

Configure Feed

Configure Feed