fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-mirror-gerrit fetches all repos of a Gerrit host.
16package main
17
18import (
19 "bytes"
20 "context"
21 "flag"
22 "fmt"
23 "io"
24 "log"
25 "net/http"
26 "net/url"
27 "os"
28 "path/filepath"
29 "slices"
30 "strconv"
31 "strings"
32
33 gerrit "github.com/andygrunwald/go-gerrit"
34 git "github.com/go-git/go-git/v5"
35 "github.com/go-git/go-git/v5/config"
36
37 "github.com/sourcegraph/zoekt/gitindex"
38)
39
40type loggingRT struct {
41 http.RoundTripper
42}
43
44type closeBuffer struct {
45 *bytes.Buffer
46}
47
48func (b *closeBuffer) Close() error { return nil }
49
50const debug = false
51
52func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
53 if debug {
54 log.Println("Req: ", req)
55 }
56 rep, err = rt.RoundTripper.RoundTrip(req)
57 if debug {
58 log.Println("Rep: ", rep, err)
59 }
60 if err == nil {
61 body, _ := io.ReadAll(rep.Body)
62
63 rep.Body.Close()
64 if debug {
65 log.Println("body: ", string(body))
66 }
67 rep.Body = &closeBuffer{bytes.NewBuffer(body)}
68 }
69 return rep, err
70}
71
72func newLoggingClient() *http.Client {
73 return &http.Client{
74 Transport: &loggingRT{
75 RoundTripper: http.DefaultTransport,
76 },
77 }
78}
79
80const qualifiedRepoNameFormat = "qualified"
81const projectRepoNameFormat = "project"
82
83var validRepoNameFormat = []string{qualifiedRepoNameFormat, projectRepoNameFormat}
84
85func validateRepoNameFormat(s string) {
86 if !slices.Contains(validRepoNameFormat, s) {
87 log.Fatal(fmt.Sprintf("repo-name-format must be one of %s", strings.Join(validRepoNameFormat, ", ")))
88 }
89}
90
91func main() {
92
93 dest := flag.String("dest", "", "destination directory")
94 namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
95 repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", ")))
96 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
97 deleteRepos := flag.Bool("delete", false, "delete missing repos")
98 fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch")
99 httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
100 active := flag.Bool("active", false, "mirror only active projects")
101 flag.Parse()
102
103 if len(flag.Args()) < 1 {
104 log.Fatal("must provide URL argument.")
105 }
106 validateRepoNameFormat(*repoNameFormat)
107
108 rootURL, err := url.Parse(flag.Arg(0))
109 if err != nil {
110 log.Fatalf("url.Parse(): %v", err)
111 }
112
113 if *httpCrendentialsPath != "" {
114 creds, err := os.ReadFile(*httpCrendentialsPath)
115 if err != nil {
116 log.Print("Cannot read gerrit http credentials, going Anonymous")
117 } else {
118 splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
119 rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
120 }
121 }
122
123 if *dest == "" {
124 log.Fatal("must set --dest")
125 }
126
127 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
128 if err != nil {
129 log.Fatal(err)
130 }
131
132 ctx := context.Background()
133
134 client, err := gerrit.NewClient(ctx, rootURL.String(), newLoggingClient())
135 if err != nil {
136 log.Fatalf("NewClient(%s): %v", rootURL, err)
137 }
138
139 info, _, err := client.Config.GetServerInfo(ctx)
140 if err != nil {
141 log.Fatalf("GetServerInfo: %v", err)
142 }
143
144 var projectURL string
145 for _, s := range []string{"http", "anonymous http"} {
146 if schemeInfo, ok := info.Download.Schemes[s]; ok {
147 projectURL = schemeInfo.URL
148 if s == "http" && schemeInfo.IsAuthRequired {
149 projectURL = addPassword(projectURL, rootURL.User)
150 // remove "/a/" prefix needed for API call with basic auth but not with git command → cleaner repo name
151 projectURL = strings.Replace(projectURL, "/a/${project}", "/${project}", 1)
152 }
153 break
154 }
155 }
156 if projectURL == "" {
157 log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
158 }
159
160 projects := make(map[string]gerrit.ProjectInfo)
161 skip := 0
162 for {
163 page, _, err := client.Projects.ListProjects(ctx, &gerrit.ProjectOptions{Skip: strconv.Itoa(skip)})
164 if err != nil {
165 log.Fatalf("ListProjects: %v", err)
166 }
167
168 if len(*page) == 0 {
169 break
170 }
171
172 for k, v := range *page {
173 if !*active || "ACTIVE" == v.State {
174 projects[k] = v
175 }
176 skip = skip + 1
177 }
178 }
179
180 for k, v := range projects {
181 if !filter.Include(k) {
182 continue
183 }
184
185 cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
186 if err != nil {
187 log.Fatalf("url.Parse: %v", err)
188 }
189
190 name := filepath.Join(cloneURL.Host, cloneURL.Path)
191 var zoektName string
192 switch *repoNameFormat {
193 case qualifiedRepoNameFormat:
194 zoektName = name
195 case projectRepoNameFormat:
196 zoektName = k
197 }
198 config := map[string]string{
199 "zoekt.name": zoektName,
200 "zoekt.gerrit-project": k,
201 "zoekt.gerrit-host": anonymousURL(rootURL),
202 "zoekt.archived": marshalBool(v.State == "READ_ONLY"),
203 "zoekt.public": marshalBool(v.State != "HIDDEN"),
204 }
205
206 for _, wl := range v.WebLinks {
207 // default gerrit gitiles config is named browse, and does not include
208 // root domain name in it. Cheating.
209 switch wl.Name {
210 case "browse":
211 config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
212 rootURL.Host, wl.URL)
213 config["zoekt.web-url-type"] = "gitiles"
214 default:
215 config["zoekt.web-url"] = wl.URL
216 config["zoekt.web-url-type"] = wl.Name
217 }
218 }
219
220 if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
221 log.Fatalf("CloneRepo: %v", err)
222 } else {
223 fmt.Println(dest)
224 }
225 if *fetchMetaConfig {
226 if err := addMetaConfigFetch(filepath.Join(*dest, name+".git")); err != nil {
227 log.Fatalf("addMetaConfigFetch: %v", err)
228 }
229 }
230 }
231 if *deleteRepos {
232 if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
233 log.Fatalf("deleteStaleRepos: %v", err)
234 }
235 }
236}
237
238func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
239 u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
240 if err != nil {
241 return err
242 }
243
244 names := map[string]struct{}{}
245 for name := range repos {
246 u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
247 if err != nil {
248 return err
249 }
250 names[filepath.Join(u.Host, u.Path)+".git"] = struct{}{}
251 }
252
253 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
254 log.Fatalf("deleteRepos: %v", err)
255 }
256 return nil
257}
258
259func marshalBool(b bool) string {
260 if b {
261 return "1"
262 }
263 return "0"
264}
265
266func anonymousURL(u *url.URL) string {
267 anon := *u
268 anon.User = nil
269 return anon.String()
270}
271
272func addPassword(u string, user *url.Userinfo) string {
273 password, _ := user.Password()
274 username := user.Username()
275 return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1)
276}
277
278func addMetaConfigFetch(repoDir string) error {
279 repo, err := git.PlainOpen(repoDir)
280 if err != nil {
281 return err
282 }
283
284 cfg, err := repo.Config()
285 if err != nil {
286 return err
287 }
288
289 rm := cfg.Remotes["origin"]
290 if rm != nil {
291 configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta-config")
292 if !slices.Contains(rm.Fetch, configRefSpec) {
293 rm.Fetch = append(rm.Fetch, configRefSpec)
294 }
295 }
296 if err := repo.Storer.SetConfig(cfg); err != nil {
297 return err
298 }
299
300 return nil
301}