indexserver: remove unused archive indexing (#124) · boltless.me/zoekt@c3c07ee

+1 -38

cmd/zoekt-sourcegraph-indexserver/index.go

··· 55 55 Archived bool 56 56 } 57 57 58 - // indexArgs represents the arguments we pass to zoekt-archive-index 58 + // indexArgs represents the arguments we pass to zoekt-git-index 59 59 type indexArgs struct { 60 60 IndexOptions 61 61 ··· 171 171 } 172 172 173 173 return opts, nil 174 - } 175 - 176 - func archiveIndex(o *indexArgs, runCmd func(*exec.Cmd) error) error { 177 - // An index should never take longer than an hour. 178 - ctx, cancel := context.WithTimeout(context.Background(), time.Hour) 179 - defer cancel() 180 - 181 - if len(o.Branches) != 1 { 182 - return fmt.Errorf("zoekt-archive-index only supports 1 branch, got %v", o.Branches) 183 - } 184 - 185 - commit := o.Branches[0].Version 186 - args := []string{ 187 - "-name", o.Name, 188 - "-commit", commit, 189 - "-branch", o.Branches[0].Name, 190 - } 191 - 192 - // Even though we check for incremental in this process, we still pass it 193 - // in just in case we regress in how we check in process. We will still 194 - // notice thanks to metrics and increased load on gitserver. 195 - if o.Incremental { 196 - args = append(args, "-incremental") 197 - } 198 - 199 - if o.DownloadLimitMBPS != "" { 200 - args = append(args, "-download-limit-mbps", o.DownloadLimitMBPS) 201 - } 202 - 203 - args = append(args, o.BuildOptions().Args()...) 204 - 205 - args = append(args, o.Root.ResolveReference(&url.URL{Path: fmt.Sprintf("/.internal/git/%s/tar/%s", o.Name, commit)}).String()) 206 - 207 - cmd := exec.CommandContext(ctx, "zoekt-archive-index", args...) 208 - // Prevent prompting 209 - cmd.Stdin = &bytes.Buffer{} 210 - return runCmd(cmd) 211 174 } 212 175 213 176 func gitIndex(o *indexArgs, runCmd func(*exec.Cmd) error) error {

+8 -41

cmd/zoekt-sourcegraph-indexserver/index_test.go

··· 89 89 } 90 90 91 91 cases := []struct { 92 - name string 93 - args indexArgs 94 - wantArchive []string 95 - wantGit []string 92 + name string 93 + args indexArgs 94 + want []string 96 95 }{{ 97 96 name: "minimal", 98 97 args: indexArgs{ ··· 102 101 Branches: []zoekt.RepositoryBranch{{Name: "HEAD", Version: "deadbeef"}}, 103 102 }, 104 103 }, 105 - wantArchive: []string{ 106 - "zoekt-archive-index -name test/repo -commit deadbeef -branch HEAD -disable_ctags http://api.test/.internal/git/test/repo/tar/deadbeef", 107 - }, 108 - wantGit: []string{ 104 + want: []string{ 109 105 "git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git", 110 106 "git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 fetch --depth=1 http://api.test/.internal/git/test/repo deadbeef", 111 107 "git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef", ··· 127 123 RepoID: 123, 128 124 }, 129 125 }, 130 - wantArchive: []string{ 131 - "zoekt-archive-index -name test/repo -commit deadbeef -branch HEAD -disable_ctags http://api.test/.internal/git/test/repo/tar/deadbeef", 132 - }, 133 - wantGit: []string{ 126 + want: []string{ 134 127 "git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git", 135 128 "git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 fetch --depth=1 http://api.test/.internal/git/test/repo deadbeef", 136 129 "git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef", ··· 161 154 }, 162 155 }, 163 156 }, 164 - wantArchive: []string{strings.Join([]string{ 165 - "zoekt-archive-index", 166 - "-name", "test/repo", 167 - "-commit", "deadbeef", 168 - "-branch", "HEAD", 169 - "-incremental", 170 - "-download-limit-mbps", "1000", 171 - "-file_limit", "123", 172 - "-parallelism", "4", 173 - "-index", "/data/index", 174 - "-require_ctags", 175 - "-large_file", "foo", 176 - "-large_file", "bar", 177 - "http://api.test/.internal/git/test/repo/tar/deadbeef", 178 - }, " ")}, 179 - wantGit: []string{ 157 + want: []string{ 180 158 "git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git", 181 159 "git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 fetch --depth=1 http://api.test/.internal/git/test/repo deadbeef feebdaed", 182 160 "git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef", ··· 203 181 return nil 204 182 } 205 183 206 - branches := tc.args.Branches 207 - tc.args.Branches = branches[:1] 208 - if err := archiveIndex(&tc.args, runCmd); err != nil { 209 - t.Fatal(err) 210 - } 211 - if !cmp.Equal(got, tc.wantArchive) { 212 - t.Errorf("archive mismatch (-want +got):\n%s", cmp.Diff(tc.wantArchive, got, splitargs)) 213 - } 214 - 215 - got = nil 216 - tc.args.Branches = branches 217 184 if err := gitIndex(&tc.args, runCmd); err != nil { 218 185 t.Fatal(err) 219 186 } 220 - if !cmp.Equal(got, tc.wantGit) { 221 - t.Errorf("git mismatch (-want +got):\n%s", cmp.Diff(tc.wantGit, got, splitargs)) 187 + if !cmp.Equal(got, tc.want) { 188 + t.Errorf("git mismatch (-want +got):\n%s", cmp.Diff(tc.want, got, splitargs)) 222 189 } 223 190 }) 224 191 }

+2 -17

cmd/zoekt-sourcegraph-indexserver/main.go

··· 126 126 // repository. 127 127 CPUCount int 128 128 129 - // Indexer is the indexer to use. Either archiveIndex (default) or the 130 - // experimental gitIndex. 131 - Indexer func(*indexArgs, func(*exec.Cmd) error) error 132 - 133 129 mu sync.Mutex 134 130 lastListRepos []string 135 131 } ··· 427 423 log.Printf("updating index %s", args.String()) 428 424 429 425 runCmd := func(cmd *exec.Cmd) error { return s.loggedRun(tr, cmd) } 430 - f := s.Indexer 431 - if f == nil && len(args.Branches) > 1 { 432 - f = gitIndex 433 - } 434 - if f == nil { 435 - f = archiveIndex 436 - } 437 426 metricIndexingTotal.Inc() 438 - return indexStateSuccess, f(args, runCmd) 427 + return indexStateSuccess, gitIndex(args, runCmd) 439 428 } 440 429 441 430 func (s *Server) defaultArgs() *indexArgs { ··· 742 731 debugIndex := flag.String("debug-index", "", "do not start the indexserver, rather index the repositories then quit.") 743 732 debugShard := flag.String("debug-shard", "", "do not start the indexserver, rather print shard stats then quit.") 744 733 745 - expGitIndex := flag.Bool("exp-git-index", os.Getenv("DISABLE_GIT_INDEX") == "", "use experimental indexing via shallow clones and zoekt-git-index") 734 + _ = flag.Bool("exp-git-index", true, "DEPRECATED: not read anymore. We always use zoekt-git-index now.") 746 735 747 736 flag.Parse() 748 737 ··· 794 783 Interval: *interval, 795 784 CPUCount: cpuCount, 796 785 Hostname: *hostname, 797 - } 798 - 799 - if *expGitIndex { 800 - s.Indexer = gitIndex 801 786 } 802 787 803 788 if *debugList {

Configure Feed

Configure Feed