fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-mirror-gerrit fetches all repos of a Gerrit host.
16package main
17
18import (
19 "bytes"
20 "context"
21 "flag"
22 "fmt"
23 "io"
24 "log"
25 "net/http"
26 "net/url"
27 "os"
28 "path/filepath"
29 "slices"
30 "strconv"
31 "strings"
32
33 gerrit "github.com/andygrunwald/go-gerrit"
34 git "github.com/go-git/go-git/v5"
35 "github.com/go-git/go-git/v5/config"
36 "github.com/sourcegraph/zoekt/internal/gitindex"
37)
38
39type loggingRT struct {
40 http.RoundTripper
41}
42
43type closeBuffer struct {
44 *bytes.Buffer
45}
46
47func (b *closeBuffer) Close() error { return nil }
48
49const debug = false
50
51func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
52 if debug {
53 log.Println("Req: ", req)
54 }
55 rep, err = rt.RoundTripper.RoundTrip(req)
56 if debug {
57 log.Println("Rep: ", rep, err)
58 }
59 if err == nil {
60 body, _ := io.ReadAll(rep.Body)
61
62 rep.Body.Close()
63 if debug {
64 log.Println("body: ", string(body))
65 }
66 rep.Body = &closeBuffer{bytes.NewBuffer(body)}
67 }
68 return rep, err
69}
70
71func newLoggingClient() *http.Client {
72 return &http.Client{
73 Transport: &loggingRT{
74 RoundTripper: http.DefaultTransport,
75 },
76 }
77}
78
79const qualifiedRepoNameFormat = "qualified"
80const projectRepoNameFormat = "project"
81
82var validRepoNameFormat = []string{qualifiedRepoNameFormat, projectRepoNameFormat}
83
84func validateRepoNameFormat(s string) {
85 if !slices.Contains(validRepoNameFormat, s) {
86 log.Fatal(fmt.Sprintf("repo-name-format must be one of %s", strings.Join(validRepoNameFormat, ", ")))
87 }
88}
89
90func main() {
91
92 dest := flag.String("dest", "", "destination directory")
93 namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
94 repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", ")))
95 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
96 deleteRepos := flag.Bool("delete", false, "delete missing repos")
97 fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch")
98 httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
99 active := flag.Bool("active", false, "mirror only active projects")
100 flag.Parse()
101
102 if len(flag.Args()) < 1 {
103 log.Fatal("must provide URL argument.")
104 }
105 validateRepoNameFormat(*repoNameFormat)
106
107 rootURL, err := url.Parse(flag.Arg(0))
108 if err != nil {
109 log.Fatalf("url.Parse(): %v", err)
110 }
111
112 if *httpCrendentialsPath != "" {
113 creds, err := os.ReadFile(*httpCrendentialsPath)
114 if err != nil {
115 log.Print("Cannot read gerrit http credentials, going Anonymous")
116 } else {
117 splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
118 rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
119 }
120 }
121
122 if *dest == "" {
123 log.Fatal("must set --dest")
124 }
125
126 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
127 if err != nil {
128 log.Fatal(err)
129 }
130
131 ctx := context.Background()
132
133 client, err := gerrit.NewClient(ctx, rootURL.String(), newLoggingClient())
134 if err != nil {
135 log.Fatalf("NewClient(%s): %v", rootURL, err)
136 }
137
138 info, _, err := client.Config.GetServerInfo(ctx)
139 if err != nil {
140 log.Fatalf("GetServerInfo: %v", err)
141 }
142
143 var projectURL string
144 for _, s := range []string{"http", "anonymous http"} {
145 if schemeInfo, ok := info.Download.Schemes[s]; ok {
146 projectURL = schemeInfo.URL
147 if s == "http" && schemeInfo.IsAuthRequired {
148 projectURL = addPassword(projectURL, rootURL.User)
149 // remove "/a/" prefix needed for API call with basic auth but not with git command → cleaner repo name
150 projectURL = strings.Replace(projectURL, "/a/${project}", "/${project}", 1)
151 }
152 break
153 }
154 }
155 if projectURL == "" {
156 log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
157 }
158
159 projects := make(map[string]gerrit.ProjectInfo)
160 skip := 0
161 for {
162 page, _, err := client.Projects.ListProjects(ctx, &gerrit.ProjectOptions{Skip: strconv.Itoa(skip)})
163 if err != nil {
164 log.Fatalf("ListProjects: %v", err)
165 }
166
167 if len(*page) == 0 {
168 break
169 }
170
171 for k, v := range *page {
172 if !*active || "ACTIVE" == v.State {
173 projects[k] = v
174 }
175 skip = skip + 1
176 }
177 }
178
179 for k, v := range projects {
180 if !filter.Include(k) {
181 continue
182 }
183
184 cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
185 if err != nil {
186 log.Fatalf("url.Parse: %v", err)
187 }
188
189 name := filepath.Join(cloneURL.Host, cloneURL.Path)
190 var zoektName string
191 switch *repoNameFormat {
192 case qualifiedRepoNameFormat:
193 zoektName = name
194 case projectRepoNameFormat:
195 zoektName = k
196 }
197 config := map[string]string{
198 "zoekt.name": zoektName,
199 "zoekt.gerrit-project": k,
200 "zoekt.gerrit-host": anonymousURL(rootURL),
201 "zoekt.archived": marshalBool(v.State == "READ_ONLY"),
202 "zoekt.public": marshalBool(v.State != "HIDDEN"),
203 }
204
205 for _, wl := range v.WebLinks {
206 // default gerrit gitiles config is named browse, and does not include
207 // root domain name in it. Cheating.
208 switch wl.Name {
209 case "browse":
210 config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
211 rootURL.Host, wl.URL)
212 config["zoekt.web-url-type"] = "gitiles"
213 default:
214 config["zoekt.web-url"] = wl.URL
215 config["zoekt.web-url-type"] = wl.Name
216 }
217 }
218
219 if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
220 log.Fatalf("CloneRepo: %v", err)
221 } else {
222 fmt.Println(dest)
223 }
224 if *fetchMetaConfig {
225 if err := addMetaConfigFetch(filepath.Join(*dest, name+".git")); err != nil {
226 log.Fatalf("addMetaConfigFetch: %v", err)
227 }
228 }
229 }
230 if *deleteRepos {
231 if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
232 log.Fatalf("deleteStaleRepos: %v", err)
233 }
234 }
235}
236
237func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
238 u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
239 if err != nil {
240 return err
241 }
242
243 names := map[string]struct{}{}
244 for name := range repos {
245 u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
246 if err != nil {
247 return err
248 }
249 names[filepath.Join(u.Host, u.Path)+".git"] = struct{}{}
250 }
251
252 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
253 log.Fatalf("deleteRepos: %v", err)
254 }
255 return nil
256}
257
258func marshalBool(b bool) string {
259 if b {
260 return "1"
261 }
262 return "0"
263}
264
265func anonymousURL(u *url.URL) string {
266 anon := *u
267 anon.User = nil
268 return anon.String()
269}
270
271func addPassword(u string, user *url.Userinfo) string {
272 password, _ := user.Password()
273 username := user.Username()
274 return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1)
275}
276
277func addMetaConfigFetch(repoDir string) error {
278 repo, err := git.PlainOpen(repoDir)
279 if err != nil {
280 return err
281 }
282
283 cfg, err := repo.Config()
284 if err != nil {
285 return err
286 }
287
288 rm := cfg.Remotes["origin"]
289 if rm != nil {
290 configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta-config")
291 if !slices.Contains(rm.Fetch, configRefSpec) {
292 rm.Fetch = append(rm.Fetch, configRefSpec)
293 }
294 }
295 if err := repo.Storer.SetConfig(cfg); err != nil {
296 return err
297 }
298
299 return nil
300}