fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-mirror-gerrit fetches all repos of a Gerrit host.
16package main
17
18import (
19 "bytes"
20 "context"
21 "flag"
22 "fmt"
23 "io"
24 "log"
25 "net/http"
26 "net/url"
27 "os"
28 "path/filepath"
29 "slices"
30 "strconv"
31 "strings"
32
33 gerrit "github.com/andygrunwald/go-gerrit"
34 git "github.com/go-git/go-git/v5"
35 "github.com/go-git/go-git/v5/config"
36
37 "github.com/sourcegraph/zoekt/gitindex"
38)
39
40type loggingRT struct {
41 http.RoundTripper
42}
43
44type closeBuffer struct {
45 *bytes.Buffer
46}
47
48func (b *closeBuffer) Close() error { return nil }
49
50const debug = false
51
52func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
53 if debug {
54 log.Println("Req: ", req)
55 }
56 rep, err = rt.RoundTripper.RoundTrip(req)
57 if debug {
58 log.Println("Rep: ", rep, err)
59 }
60 if err == nil {
61 body, _ := io.ReadAll(rep.Body)
62
63 rep.Body.Close()
64 if debug {
65 log.Println("body: ", string(body))
66 }
67 rep.Body = &closeBuffer{bytes.NewBuffer(body)}
68 }
69 return rep, err
70}
71
72func newLoggingClient() *http.Client {
73 return &http.Client{
74 Transport: &loggingRT{
75 RoundTripper: http.DefaultTransport,
76 },
77 }
78}
79
80const qualifiedRepoNameFormat = "qualified"
81const projectRepoNameFormat = "project"
82
83var validRepoNameFormat = []string{qualifiedRepoNameFormat, projectRepoNameFormat}
84
85func validateRepoNameFormat(s string) {
86 if !slices.Contains(validRepoNameFormat, s) {
87 log.Fatal(fmt.Sprintf("repo-name-format must be one of %s", strings.Join(validRepoNameFormat, ", ")))
88 }
89}
90
91func main() {
92
93 dest := flag.String("dest", "", "destination directory")
94 namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
95 repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", ")))
96 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
97 deleteRepos := flag.Bool("delete", false, "delete missing repos")
98 fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch")
99 httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
100 active := flag.Bool("active", false, "mirror only active projects")
101 flag.Parse()
102
103 if len(flag.Args()) < 1 {
104 log.Fatal("must provide URL argument.")
105 }
106 validateRepoNameFormat(*repoNameFormat)
107
108 rootURL, err := url.Parse(flag.Arg(0))
109 if err != nil {
110 log.Fatalf("url.Parse(): %v", err)
111 }
112
113 if *httpCrendentialsPath != "" {
114 creds, err := os.ReadFile(*httpCrendentialsPath)
115 if err != nil {
116 log.Print("Cannot read gerrit http credentials, going Anonymous")
117 } else {
118 splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
119 rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
120 }
121 }
122
123 if *dest == "" {
124 log.Fatal("must set --dest")
125 }
126
127 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
128 if err != nil {
129 log.Fatal(err)
130 }
131
132 ctx := context.Background()
133
134 client, err := gerrit.NewClient(ctx, rootURL.String(), newLoggingClient())
135 if err != nil {
136 log.Fatalf("NewClient(%s): %v", rootURL, err)
137 }
138
139 info, _, err := client.Config.GetServerInfo(ctx)
140 if err != nil {
141 log.Fatalf("GetServerInfo: %v", err)
142 }
143
144 var projectURL string
145 for _, s := range []string{"http", "anonymous http"} {
146 if schemeInfo, ok := info.Download.Schemes[s]; ok {
147 projectURL = schemeInfo.URL
148 if s == "http" && schemeInfo.IsAuthRequired {
149 projectURL = addPassword(projectURL, rootURL.User)
150 }
151 break
152 }
153 }
154 if projectURL == "" {
155 log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
156 }
157
158 projects := make(map[string]gerrit.ProjectInfo)
159 skip := 0
160 for {
161 page, _, err := client.Projects.ListProjects(ctx, &gerrit.ProjectOptions{Skip: strconv.Itoa(skip)})
162 if err != nil {
163 log.Fatalf("ListProjects: %v", err)
164 }
165
166 if len(*page) == 0 {
167 break
168 }
169
170 for k, v := range *page {
171 if !*active || "ACTIVE" == v.State {
172 projects[k] = v
173 }
174 skip = skip + 1
175 }
176 }
177
178 for k, v := range projects {
179 if !filter.Include(k) {
180 continue
181 }
182
183 cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
184 if err != nil {
185 log.Fatalf("url.Parse: %v", err)
186 }
187
188 name := filepath.Join(cloneURL.Host, cloneURL.Path)
189 var zoektName string
190 switch *repoNameFormat {
191 case qualifiedRepoNameFormat:
192 zoektName = name
193 case projectRepoNameFormat:
194 zoektName = k
195 }
196 config := map[string]string{
197 "zoekt.name": zoektName,
198 "zoekt.gerrit-project": k,
199 "zoekt.gerrit-host": anonymousURL(rootURL),
200 "zoekt.archived": marshalBool(v.State == "READ_ONLY"),
201 "zoekt.public": marshalBool(v.State != "HIDDEN"),
202 }
203
204 for _, wl := range v.WebLinks {
205 // default gerrit gitiles config is named browse, and does not include
206 // root domain name in it. Cheating.
207 switch wl.Name {
208 case "browse":
209 config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
210 rootURL.Host, wl.URL)
211 config["zoekt.web-url-type"] = "gitiles"
212 default:
213 config["zoekt.web-url"] = wl.URL
214 config["zoekt.web-url-type"] = wl.Name
215 }
216 }
217
218 if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
219 log.Fatalf("CloneRepo: %v", err)
220 } else {
221 fmt.Println(dest)
222 }
223 if *fetchMetaConfig {
224 if err := addMetaConfigFetch(filepath.Join(*dest, name+".git")); err != nil {
225 log.Fatalf("addMetaConfigFetch: %v", err)
226 }
227 }
228 }
229 if *deleteRepos {
230 if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
231 log.Fatalf("deleteStaleRepos: %v", err)
232 }
233 }
234}
235
236func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
237 u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
238 if err != nil {
239 return err
240 }
241
242 names := map[string]struct{}{}
243 for name := range repos {
244 u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
245 if err != nil {
246 return err
247 }
248 names[filepath.Join(u.Host, u.Path)+".git"] = struct{}{}
249 }
250
251 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
252 log.Fatalf("deleteRepos: %v", err)
253 }
254 return nil
255}
256
257func marshalBool(b bool) string {
258 if b {
259 return "1"
260 }
261 return "0"
262}
263
264func anonymousURL(u *url.URL) string {
265 anon := *u
266 anon.User = nil
267 return anon.String()
268}
269
270func addPassword(u string, user *url.Userinfo) string {
271 password, _ := user.Password()
272 username := user.Username()
273 return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1)
274}
275
276func addMetaConfigFetch(repoDir string) error {
277 repo, err := git.PlainOpen(repoDir)
278 if err != nil {
279 return err
280 }
281
282 cfg, err := repo.Config()
283 if err != nil {
284 return err
285 }
286
287 rm := cfg.Remotes["origin"]
288 if rm != nil {
289 configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta-config")
290 if !slices.Contains(rm.Fetch, configRefSpec) {
291 rm.Fetch = append(rm.Fetch, configRefSpec)
292 }
293 }
294 if err := repo.Storer.SetConfig(cfg); err != nil {
295 return err
296 }
297
298 return nil
299}