fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// This binary fetches all repos of a Gerrit host.
16
17package main
18
19import (
20 "bytes"
21 "flag"
22 "fmt"
23 "io"
24 "log"
25 "net/http"
26 "net/url"
27 "os"
28 "path/filepath"
29 "slices"
30 "strconv"
31 "strings"
32
33 gerrit "github.com/andygrunwald/go-gerrit"
34 git "github.com/go-git/go-git/v5"
35 "github.com/go-git/go-git/v5/config"
36 "github.com/sourcegraph/zoekt/gitindex"
37)
38
39type loggingRT struct {
40 http.RoundTripper
41}
42
43type closeBuffer struct {
44 *bytes.Buffer
45}
46
47func (b *closeBuffer) Close() error { return nil }
48
49const debug = false
50
51func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
52 if debug {
53 log.Println("Req: ", req)
54 }
55 rep, err = rt.RoundTripper.RoundTrip(req)
56 if debug {
57 log.Println("Rep: ", rep, err)
58 }
59 if err == nil {
60 body, _ := io.ReadAll(rep.Body)
61
62 rep.Body.Close()
63 if debug {
64 log.Println("body: ", string(body))
65 }
66 rep.Body = &closeBuffer{bytes.NewBuffer(body)}
67 }
68 return rep, err
69}
70
71func newLoggingClient() *http.Client {
72 return &http.Client{
73 Transport: &loggingRT{
74 RoundTripper: http.DefaultTransport,
75 },
76 }
77}
78
79const qualifiedRepoNameFormat = "qualified"
80const projectRepoNameFormat = "project"
81
82var validRepoNameFormat = []string{qualifiedRepoNameFormat, projectRepoNameFormat}
83
84func validateRepoNameFormat(s string) {
85 if !slices.Contains(validRepoNameFormat, s) {
86 log.Fatal(fmt.Sprintf("repo-name-format must be one of %s", strings.Join(validRepoNameFormat, ", ")))
87 }
88}
89
90func main() {
91
92 dest := flag.String("dest", "", "destination directory")
93 namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
94 repoNameFormat := flag.String("repo-name-format", qualifiedRepoNameFormat, fmt.Sprintf("the format of the local repo name in zoekt (valid values: %s)", strings.Join(validRepoNameFormat, ", ")))
95 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
96 deleteRepos := flag.Bool("delete", false, "delete missing repos")
97 fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch")
98 httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
99 active := flag.Bool("active", false, "mirror only active projects")
100 flag.Parse()
101
102 if len(flag.Args()) < 1 {
103 log.Fatal("must provide URL argument.")
104 }
105 validateRepoNameFormat(*repoNameFormat)
106
107 rootURL, err := url.Parse(flag.Arg(0))
108 if err != nil {
109 log.Fatalf("url.Parse(): %v", err)
110 }
111
112 if *httpCrendentialsPath != "" {
113 creds, err := os.ReadFile(*httpCrendentialsPath)
114 if err != nil {
115 log.Print("Cannot read gerrit http credentials, going Anonymous")
116 } else {
117 splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
118 rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
119 }
120 }
121
122 if *dest == "" {
123 log.Fatal("must set --dest")
124 }
125
126 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
127 if err != nil {
128 log.Fatal(err)
129 }
130
131 client, err := gerrit.NewClient(rootURL.String(), newLoggingClient())
132 if err != nil {
133 log.Fatalf("NewClient(%s): %v", rootURL, err)
134 }
135
136 info, _, err := client.Config.GetServerInfo()
137 if err != nil {
138 log.Fatalf("GetServerInfo: %v", err)
139 }
140
141 var projectURL string
142 for _, s := range []string{"http", "anonymous http"} {
143 if schemeInfo, ok := info.Download.Schemes[s]; ok {
144 projectURL = schemeInfo.URL
145 if s == "http" && schemeInfo.IsAuthRequired {
146 projectURL = addPassword(projectURL, rootURL.User)
147 // remove "/a/" prefix needed for API call with basic auth but not with git command → cleaner repo name
148 projectURL = strings.Replace(projectURL, "/a/${project}", "/${project}", 1)
149 }
150 break
151 }
152 }
153 if projectURL == "" {
154 log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
155 }
156
157 projects := make(map[string]gerrit.ProjectInfo)
158 skip := 0
159 for {
160 page, _, err := client.Projects.ListProjects(&gerrit.ProjectOptions{Skip: strconv.Itoa(skip)})
161 if err != nil {
162 log.Fatalf("ListProjects: %v", err)
163 }
164
165 if len(*page) == 0 {
166 break
167 }
168
169 for k, v := range *page {
170 if !*active || "ACTIVE" == v.State {
171 projects[k] = v
172 }
173 skip = skip + 1
174 }
175 }
176
177 for k, v := range projects {
178 if !filter.Include(k) {
179 continue
180 }
181
182 cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
183 if err != nil {
184 log.Fatalf("url.Parse: %v", err)
185 }
186
187 name := filepath.Join(cloneURL.Host, cloneURL.Path)
188 var zoektName string
189 switch *repoNameFormat {
190 case qualifiedRepoNameFormat:
191 zoektName = name
192 case projectRepoNameFormat:
193 zoektName = k
194 }
195 config := map[string]string{
196 "zoekt.name": zoektName,
197 "zoekt.gerrit-project": k,
198 "zoekt.gerrit-host": anonymousURL(rootURL),
199 "zoekt.archived": marshalBool(v.State == "READ_ONLY"),
200 "zoekt.public": marshalBool(v.State != "HIDDEN"),
201 }
202
203 for _, wl := range v.WebLinks {
204 // default gerrit gitiles config is named browse, and does not include
205 // root domain name in it. Cheating.
206 switch wl.Name {
207 case "browse":
208 config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
209 rootURL.Host, wl.URL)
210 config["zoekt.web-url-type"] = "gitiles"
211 default:
212 config["zoekt.web-url"] = wl.URL
213 config["zoekt.web-url-type"] = wl.Name
214 }
215 }
216
217 if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
218 log.Fatalf("CloneRepo: %v", err)
219 } else {
220 fmt.Println(dest)
221 }
222 if *fetchMetaConfig {
223 if err := addMetaConfigFetch(filepath.Join(*dest, name+".git")); err != nil {
224 log.Fatalf("addMetaConfigFetch: %v", err)
225 }
226 }
227 }
228 if *deleteRepos {
229 if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
230 log.Fatalf("deleteStaleRepos: %v", err)
231 }
232 }
233}
234
235func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
236 u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
237 if err != nil {
238 return err
239 }
240
241 names := map[string]struct{}{}
242 for name := range repos {
243 u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
244 if err != nil {
245 return err
246 }
247 names[filepath.Join(u.Host, u.Path)+".git"] = struct{}{}
248 }
249
250 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
251 log.Fatalf("deleteRepos: %v", err)
252 }
253 return nil
254}
255
256func marshalBool(b bool) string {
257 if b {
258 return "1"
259 }
260 return "0"
261}
262
263func anonymousURL(u *url.URL) string {
264 anon := *u
265 anon.User = nil
266 return anon.String()
267}
268
269func addPassword(u string, user *url.Userinfo) string {
270 password, _ := user.Password()
271 username := user.Username()
272 return strings.Replace(u, fmt.Sprintf("://%s@", username), fmt.Sprintf("://%s:%s@", username, password), 1)
273}
274
275func addMetaConfigFetch(repoDir string) error {
276 repo, err := git.PlainOpen(repoDir)
277 if err != nil {
278 return err
279 }
280
281 cfg, err := repo.Config()
282 if err != nil {
283 return err
284 }
285
286 rm := cfg.Remotes["origin"]
287 if rm != nil {
288 configRefSpec := config.RefSpec("+refs/meta/config:refs/heads/meta-config")
289 if !slices.Contains(rm.Fetch, configRefSpec) {
290 rm.Fetch = append(rm.Fetch, configRefSpec)
291 }
292 }
293 if err := repo.Storer.SetConfig(cfg); err != nil {
294 return err
295 }
296
297 return nil
298}