fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// This binary fetches all repos of a Gerrit host.
16
17package main
18
19import (
20 "bytes"
21 "flag"
22 "fmt"
23 "io"
24 "log"
25 "net/http"
26 "net/url"
27 "os"
28 "path/filepath"
29 "strconv"
30 "strings"
31
32 gerrit "github.com/andygrunwald/go-gerrit"
33 "github.com/sourcegraph/zoekt/gitindex"
34)
35
36type loggingRT struct {
37 http.RoundTripper
38}
39
40type closeBuffer struct {
41 *bytes.Buffer
42}
43
44func (b *closeBuffer) Close() error { return nil }
45
46const debug = false
47
48func (rt *loggingRT) RoundTrip(req *http.Request) (rep *http.Response, err error) {
49 if debug {
50 log.Println("Req: ", req)
51 }
52 rep, err = rt.RoundTripper.RoundTrip(req)
53 if debug {
54 log.Println("Rep: ", rep, err)
55 }
56 if err == nil {
57 body, _ := io.ReadAll(rep.Body)
58
59 rep.Body.Close()
60 if debug {
61 log.Println("body: ", string(body))
62 }
63 rep.Body = &closeBuffer{bytes.NewBuffer(body)}
64 }
65 return rep, err
66}
67
68func newLoggingClient() *http.Client {
69 return &http.Client{
70 Transport: &loggingRT{
71 RoundTripper: http.DefaultTransport,
72 },
73 }
74}
75
76func main() {
77 dest := flag.String("dest", "", "destination directory")
78 namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
79 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
80 deleteRepos := flag.Bool("delete", false, "delete missing repos")
81 httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
82 active := flag.Bool("active", false, "mirror only active projects")
83 flag.Parse()
84
85 if len(flag.Args()) < 1 {
86 log.Fatal("must provide URL argument.")
87 }
88
89 rootURL, err := url.Parse(flag.Arg(0))
90 if err != nil {
91 log.Fatalf("url.Parse(): %v", err)
92 }
93
94 if *httpCrendentialsPath != "" {
95 creds, err := os.ReadFile(*httpCrendentialsPath)
96 if err != nil {
97 log.Print("Cannot read gerrit http credentials, going Anonymous")
98 } else {
99 splitCreds := strings.Split(strings.TrimSpace(string(creds)), ":")
100 rootURL.User = url.UserPassword(splitCreds[0], splitCreds[1])
101 }
102 }
103
104 if *dest == "" {
105 log.Fatal("must set --dest")
106 }
107
108 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
109 if err != nil {
110 log.Fatal(err)
111 }
112
113 client, err := gerrit.NewClient(rootURL.String(), newLoggingClient())
114 if err != nil {
115 log.Fatalf("NewClient(%s): %v", rootURL, err)
116 }
117
118 info, _, err := client.Config.GetServerInfo()
119 if err != nil {
120 log.Fatalf("GetServerInfo: %v", err)
121 }
122
123 var projectURL string
124 for _, s := range []string{"http", "anonymous http"} {
125 if schemeInfo, ok := info.Download.Schemes[s]; ok {
126 projectURL = schemeInfo.URL
127 break
128 }
129 }
130 if projectURL == "" {
131 log.Fatalf("project URL is empty, got Schemes %#v", info.Download.Schemes)
132 }
133
134 projects := make(map[string]gerrit.ProjectInfo)
135 skip := 0
136 for {
137 page, _, err := client.Projects.ListProjects(&gerrit.ProjectOptions{Skip: strconv.Itoa(skip)})
138 if err != nil {
139 log.Fatalf("ListProjects: %v", err)
140 }
141
142 if len(*page) == 0 {
143 break
144 }
145
146 for k, v := range *page {
147 if !*active || "ACTIVE" == v.State {
148 projects[k] = v
149 }
150 skip = skip + 1
151 }
152 }
153
154 for k, v := range projects {
155 if !filter.Include(k) {
156 continue
157 }
158
159 cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
160 if err != nil {
161 log.Fatalf("url.Parse: %v", err)
162 }
163
164 name := filepath.Join(cloneURL.Host, cloneURL.Path)
165 config := map[string]string{
166 "zoekt.name": name,
167 "zoekt.gerrit-project": k,
168 "zoekt.gerrit-host": rootURL.String(),
169 "zoekt.archived": marshalBool(v.State == "READ_ONLY"),
170 "zoekt.public": marshalBool(v.State != "HIDDEN"),
171 }
172
173 for _, wl := range v.WebLinks {
174 // default gerrit gitiles config is named browse, and does not include
175 // root domain name in it. Cheating.
176 switch wl.Name {
177 case "browse":
178 config["zoekt.web-url"] = fmt.Sprintf("%s://%s%s", rootURL.Scheme,
179 rootURL.Host, wl.URL)
180 config["zoekt.web-url-type"] = "gitiles"
181 default:
182 config["zoekt.web-url"] = wl.URL
183 config["zoekt.web-url-type"] = wl.Name
184 }
185 }
186
187 if dest, err := gitindex.CloneRepo(*dest, name, cloneURL.String(), config); err != nil {
188 log.Fatalf("CloneRepo: %v", err)
189 } else {
190 fmt.Println(dest)
191 }
192 }
193 if *deleteRepos {
194 if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
195 log.Fatalf("deleteStaleRepos: %v", err)
196 }
197 }
198}
199
200func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
201 u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
202 if err != nil {
203 return err
204 }
205
206 names := map[string]struct{}{}
207 for name := range repos {
208 u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
209 if err != nil {
210 return err
211 }
212 names[filepath.Join(u.Host, u.Path)+".git"] = struct{}{}
213 }
214
215 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
216 log.Fatalf("deleteRepos: %v", err)
217 }
218 return nil
219}
220
221func marshalBool(b bool) string {
222 if b {
223 return "1"
224 }
225 return "0"
226}