indexserver: remove ListRepoNames and GetIndexOptionsName (#185) · boltless.me/zoekt@4180212

+3 -7

cmd/zoekt-sourcegraph-indexserver/index_test.go

··· 29 29 http.Error(w, fmt.Sprintf("got URL %v want %v", got, want), http.StatusBadRequest) 30 30 return 31 31 } 32 - if got, want := r.Form, (url.Values{"repo": []string{"test/repo"}}); !reflect.DeepEqual(got, want) { 32 + if got, want := r.Form, (url.Values{"repoID": []string{"123"}}); !reflect.DeepEqual(got, want) { 33 33 http.Error(w, fmt.Sprintf("got URL %v want %v", got, want), http.StatusBadRequest) 34 34 return 35 35 } ··· 49 49 50 50 cases := map[string]*IndexOptions{ 51 51 `{"Symbols": true, "LargeFiles": ["foo","bar"]}`: { 52 - Name: "test/repo", 53 52 Symbols: true, 54 53 LargeFiles: []string{"foo", "bar"}, 55 54 }, 56 55 57 56 `{"Symbols": false, "LargeFiles": ["foo","bar"]}`: { 58 - Name: "test/repo", 59 57 LargeFiles: []string{"foo", "bar"}, 60 58 }, 61 59 62 - `{}`: {Name: "test/repo"}, 60 + `{}`: {}, 63 61 64 62 `{"Symbols": true}`: { 65 - Name: "test/repo", 66 63 Symbols: true, 67 64 }, 68 65 69 66 `{"RepoID": 123}`: { 70 - Name: "test/repo", 71 67 RepoID: 123, 72 68 }, 73 69 ··· 77 73 for r, want := range cases { 78 74 response = []byte(r) 79 75 80 - got, err := sg.GetIndexOptionsName("test/repo") 76 + got, err := sg.GetIndexOptions(123) 81 77 if err != nil && want != nil { 82 78 t.Fatalf("unexpected error: %v", err) 83 79 }

+25 -46

cmd/zoekt-sourcegraph-indexserver/main.go

··· 89 89 Name: "index_indexing_total", 90 90 Help: "Counts indexings (indexing activity, should be used with rate())", 91 91 }) 92 - 93 - metricsEnqueueRepoForIndex = promauto.NewCounter(prometheus.CounterOpts{ 94 - Name: "enqueue_repo_for_index_total", 95 - Help: "Counts the number of time /enqueueforindex is called", 96 - }) 97 92 ) 98 93 99 94 type indexState string ··· 503 498 <h3>Re-index repository</h3> 504 499 <form action="/" method="post"> 505 500 {{range .Repos}} 506 - <input type="submit" name="repo" value="{{ . }}" /> <br /> 501 + <button type="submit" name="repo" value="{{ .ID }}" />{{ .Name }}</button><br /> 507 502 {{end}} 508 503 </form> 509 504 </body></html> 510 505 `)) 511 506 512 507 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 508 + type Repo struct { 509 + ID uint32 510 + Name string 511 + } 513 512 var data struct { 514 - Repos []string 513 + Repos []Repo 515 514 IndexMsg string 516 515 } 517 516 518 517 if r.Method == "POST" { 519 518 r.ParseForm() 520 - name := r.Form.Get("repo") 521 - data.IndexMsg, _ = s.forceIndex(name) 519 + if id, err := strconv.Atoi(r.Form.Get("repo")); err != nil { 520 + data.IndexMsg = err.Error() 521 + } else { 522 + data.IndexMsg, _ = s.forceIndex(uint32(id)) 523 + } 522 524 } 523 525 524 526 s.queue.Iterate(func(opts *IndexOptions) { 525 - data.Repos = append(data.Repos, opts.Name) 527 + data.Repos = append(data.Repos, Repo{ 528 + ID: opts.RepoID, 529 + Name: opts.Name, 530 + }) 526 531 }) 527 532 528 533 repoTmpl.Execute(w, data) 529 534 } 530 535 531 - // serveEnqueueForIndex is expected to be called by other services in order to 532 - // trigger an index. We expect repo-updater to call this endpoint when a new 533 - // repo has been added to an instance that we wish to index and don't want to 534 - // wait for polling to happen. 535 - func (s *Server) serveEnqueueForIndex(rw http.ResponseWriter, r *http.Request) { 536 - if r.Method != "POST" { 537 - http.Error(rw, "not found", http.StatusNotFound) 538 - return 539 - } 540 - metricsEnqueueRepoForIndex.Inc() 541 - err := r.ParseForm() 542 - if err != nil { 543 - http.Error(rw, "error parsing form", http.StatusBadRequest) 544 - return 545 - } 546 - name := r.Form.Get("repo") 547 - if name == "" { 548 - http.Error(rw, "missing repo", http.StatusBadRequest) 549 - return 550 - } 551 - debug.Printf("enqueueRepoForIndex called with repo: %q", name) 552 - opts, err := s.Sourcegraph.GetIndexOptionsName(name) 553 - if err != nil || opts[0].Error != "" { 554 - http.Error(rw, "fetching index options", http.StatusInternalServerError) 555 - return 556 - } 557 - s.queue.AddOrUpdate(opts[0].IndexOptions) 558 - } 559 - 560 536 // forceIndex will run the index job for repo name now. It will return always 561 537 // return a string explaining what it did, even if it failed. 562 - func (s *Server) forceIndex(name string) (string, error) { 563 - opts, err := s.Sourcegraph.GetIndexOptionsName(name) 538 + func (s *Server) forceIndex(id uint32) (string, error) { 539 + opts, err := s.Sourcegraph.GetIndexOptions(id) 564 540 if err != nil { 565 - return fmt.Sprintf("Indexing %s failed: %v", name, err), err 541 + return fmt.Sprintf("Indexing %d failed: %v", id, err), err 566 542 } 567 543 if errS := opts[0].Error; errS != "" { 568 - return fmt.Sprintf("Indexing %s failed: %s", name, errS), errors.New(errS) 544 + return fmt.Sprintf("Indexing %d failed: %s", id, errS), errors.New(errS) 569 545 } 570 546 571 547 args := s.indexArgs(opts[0].IndexOptions) ··· 690 666 691 667 // non daemon mode for debugging/testing 692 668 debugList := flag.Bool("debug-list", false, "do not start the indexserver, rather list the repositories owned by this indexserver then quit.") 693 - debugIndex := flag.String("debug-index", "", "do not start the indexserver, rather index the repositories then quit.") 669 + debugIndex := flag.String("debug-index", "", "do not start the indexserver, rather index the repository ID then quit.") 694 670 debugShard := flag.String("debug-shard", "", "do not start the indexserver, rather print shard stats then quit.") 695 671 debugMeta := flag.String("debug-meta", "", "do not start the indexserver, rather print shard metadata then quit.") 696 672 debugMerge := flag.String("debug-merge", "", "index dir,compound target size in MiB,simulate(true,false)") ··· 789 765 } 790 766 791 767 if *debugIndex != "" { 792 - msg, err := s.forceIndex(*debugIndex) 768 + id, err := strconv.Atoi(*debugIndex) 769 + if err != nil { 770 + log.Fatal(err) 771 + } 772 + msg, err := s.forceIndex(uint32(id)) 793 773 log.Println(msg) 794 774 if err != nil { 795 775 os.Exit(1) ··· 828 808 mux := http.NewServeMux() 829 809 debugserver.AddHandlers(mux, true) 830 810 mux.Handle("/", s) 831 - mux.HandleFunc("/enqueueforindex", s.serveEnqueueForIndex) 832 811 debug.Printf("serving HTTP on %s", *listen) 833 812 log.Fatal(http.ListenAndServe(*listen, mux)) 834 813 }()

+4 -4

cmd/zoekt-sourcegraph-indexserver/main_test.go

··· 57 57 } 58 58 gotBody = string(b) 59 59 60 - _, err = w.Write([]byte(`{"RepoNames": ["foo", "bar", "baz"]}`)) 60 + _, err = w.Write([]byte(`{"RepoIDs": [1, 2, 3]}`)) 61 61 if err != nil { 62 62 t.Fatal(err) 63 63 } ··· 75 75 Client: retryablehttp.NewClient(), 76 76 } 77 77 78 - gotRepos, err := s.ListRepoNames(context.Background(), []string{"foo", "bam"}) 78 + gotRepos, err := s.ListRepoIDs(context.Background(), []uint32{1, 3}) 79 79 if err != nil { 80 80 t.Fatal(err) 81 81 } 82 82 83 - if want := []string{"foo", "bar", "baz"}; !cmp.Equal(gotRepos, want) { 83 + if want := []uint32{1, 2, 3}; !cmp.Equal(gotRepos, want) { 84 84 t.Errorf("repos mismatch (-want +got):\n%s", cmp.Diff(want, gotRepos)) 85 85 } 86 - if want := `{"Hostname":"test-indexed-search-1","Indexed":["foo","bam"]}`; gotBody != want { 86 + if want := `{"Hostname":"test-indexed-search-1","IndexedIDs":[1,3]}`; gotBody != want { 87 87 t.Errorf("body mismatch (-want +got):\n%s", cmp.Diff(want, gotBody)) 88 88 } 89 89 if want := "/.internal/repos/index"; gotURL.Path != want {

-101

cmd/zoekt-sourcegraph-indexserver/sg.go

··· 25 25 GetIndexOptions(repos ...uint32) ([]indexOptionsItem, error) 26 26 GetCloneURL(name string) string 27 27 ListRepoIDs(ctx context.Context, indexed []uint32) ([]uint32, error) 28 - 29 - // Deprecated. Included to minimize diff sizes. 30 - GetIndexOptionsName(repos ...string) ([]indexOptionsItem, error) 31 - // Deprecated. Included to minimize diff sizes. 32 - ListRepoNames(ctx context.Context, indexed []string) ([]string, error) 33 28 } 34 29 35 30 // sourcegraphClient contains methods which interact with the sourcegraph API. ··· 52 47 Error string 53 48 } 54 49 55 - func (s *sourcegraphClient) GetIndexOptionsName(repos ...string) ([]indexOptionsItem, error) { 56 - u := s.Root.ResolveReference(&url.URL{ 57 - Path: "/.internal/search/configuration", 58 - }) 59 - 60 - resp, err := s.Client.PostForm(u.String(), url.Values{"repo": repos}) 61 - if err != nil { 62 - return nil, err 63 - } 64 - defer resp.Body.Close() 65 - 66 - if resp.StatusCode != http.StatusOK { 67 - b, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024)) 68 - _ = resp.Body.Close() 69 - if err != nil { 70 - return nil, err 71 - } 72 - return nil, &url.Error{ 73 - Op: "Get", 74 - URL: u.String(), 75 - Err: fmt.Errorf("%s: %s", resp.Status, string(b)), 76 - } 77 - } 78 - 79 - opts := make([]indexOptionsItem, len(repos)) 80 - dec := json.NewDecoder(resp.Body) 81 - for i := range opts { 82 - if err := dec.Decode(&opts[i]); err != nil { 83 - return nil, fmt.Errorf("error decoding body: %w", err) 84 - } 85 - // Override Sourcegraph name. This can technically be out of sync with 86 - // somewhat rare races. Once we have switched to ID backed communication 87 - // with Sourcegraph we can fully rely on the name returned. 88 - opts[i].Name = repos[i] 89 - } 90 - 91 - return opts, nil 92 - } 93 - 94 50 func (s *sourcegraphClient) GetIndexOptions(repos ...uint32) ([]indexOptionsItem, error) { 95 51 u := s.Root.ResolveReference(&url.URL{ 96 52 Path: "/.internal/search/configuration", ··· 134 90 return s.Root.ResolveReference(&url.URL{Path: path.Join("/.internal/git", name)}).String() 135 91 } 136 92 137 - func (s *sourcegraphClient) ListRepoNames(ctx context.Context, indexed []string) ([]string, error) { 138 - body, err := json.Marshal(&struct { 139 - Hostname string 140 - Indexed []string 141 - }{ 142 - Hostname: s.Hostname, 143 - Indexed: indexed, 144 - }) 145 - if err != nil { 146 - return nil, err 147 - } 148 - 149 - u := s.Root.ResolveReference(&url.URL{Path: "/.internal/repos/index"}) 150 - resp, err := s.Client.Post(u.String(), "application/json; charset=utf8", bytes.NewReader(body)) 151 - if err != nil { 152 - return nil, err 153 - } 154 - defer resp.Body.Close() 155 - 156 - if resp.StatusCode != http.StatusOK { 157 - return nil, fmt.Errorf("failed to list repositories: status %s", resp.Status) 158 - } 159 - 160 - var data struct { 161 - RepoNames []string 162 - } 163 - err = json.NewDecoder(resp.Body).Decode(&data) 164 - if err != nil { 165 - return nil, err 166 - } 167 - 168 - metricNumAssigned.Set(float64(len(data.RepoNames))) 169 - 170 - return data.RepoNames, nil 171 - } 172 - 173 93 func (s *sourcegraphClient) ListRepoIDs(ctx context.Context, indexed []uint32) ([]uint32, error) { 174 94 body, err := json.Marshal(&struct { 175 95 Hostname string ··· 209 129 type sourcegraphFake struct { 210 130 RootDir string 211 131 Log *log.Logger 212 - } 213 - 214 - func (sf sourcegraphFake) GetIndexOptionsName(repos ...string) ([]indexOptionsItem, error) { 215 - var items []indexOptionsItem 216 - for _, name := range repos { 217 - opts, err := sf.getIndexOptions(name) 218 - if err != nil { 219 - items = append(items, indexOptionsItem{Error: err.Error()}) 220 - } else { 221 - items = append(items, indexOptionsItem{IndexOptions: opts}) 222 - } 223 - } 224 - return items, nil 225 132 } 226 133 227 134 func (sf sourcegraphFake) GetIndexOptions(repos ...uint32) ([]indexOptionsItem, error) { ··· 297 204 var repos []uint32 298 205 err := sf.visitRepos(func(name string) { 299 206 repos = append(repos, fakeID(name)) 300 - }) 301 - return repos, err 302 - } 303 - 304 - func (sf sourcegraphFake) ListRepoNames(ctx context.Context, indexed []string) ([]string, error) { 305 - var repos []string 306 - err := sf.visitRepos(func(name string) { 307 - repos = append(repos, name) 308 207 }) 309 208 return repos, err 310 209 }

Configure Feed

Configure Feed