fork of https://github.com/sourcegraph/zoekt
1// Licensed under the Apache License, Version 2.0 (the "License");
2// you may not use this file except in compliance with the License.
3// You may obtain a copy of the License at
4//
5// http://www.apache.org/licenses/LICENSE-2.0
6//
7// Unless required by applicable law or agreed to in writing, software
8// distributed under the License is distributed on an "AS IS" BASIS,
9// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10// See the License for the specific language governing permissions and
11// limitations under the License.
12
13// This binary fetches all repos of a project, and of a specific type, in case
14// these are specified, and clones them. By default it fetches and clones all
15// existing repos.
16package main
17
18import (
19 "context"
20 "crypto/tls"
21 "flag"
22 "fmt"
23 "log"
24 "net/http"
25 "net/url"
26 "os"
27 "path/filepath"
28 "strings"
29 "time"
30
31 bitbucketv1 "github.com/gfleury/go-bitbucket-v1"
32
33 "github.com/sourcegraph/zoekt/gitindex"
34)
35
36func main() {
37 dest := flag.String("dest", "", "destination directory")
38 serverUrl := flag.String("url", "", "BitBucket Server url")
39 disableTLS := flag.Bool("disable-tls", false, "disables TLS verification")
40 credentialsFile := flag.String("credentials", ".bitbucket-credentials", "file holding BitBucket Server credentials")
41 project := flag.String("project", "", "project to mirror")
42 deleteRepos := flag.Bool("delete", false, "delete missing repos")
43 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
44 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
45 projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+
46 "Type can be either NORMAl or PERSONAL. Clones projects of both types if not set.")
47 flag.Parse()
48
49 if *serverUrl == "" {
50 log.Fatal("must set --url")
51 }
52
53 rootURL, err := url.Parse(*serverUrl)
54 if err != nil {
55 log.Fatalf("url.Parse(): %v", err)
56 }
57
58 if *dest == "" {
59 log.Fatal("must set --dest")
60 }
61
62 if *projectType != "" && !IsValidProjectType(*projectType) {
63 log.Fatal("type should be either NORMAL or PERSONAL")
64 }
65
66 destDir := filepath.Join(*dest, rootURL.Host)
67 if err := os.MkdirAll(destDir, 0o755); err != nil {
68 log.Fatal(err)
69 }
70
71 username := ""
72 password := ""
73 if *credentialsFile == "" {
74 log.Fatal("must set --credentials")
75 } else {
76 content, err := os.ReadFile(*credentialsFile)
77 if err != nil {
78 log.Fatal(err)
79 }
80 credentials := strings.Fields(string(content))
81 username, password = credentials[0], credentials[1]
82 }
83
84 basicAuth := bitbucketv1.BasicAuth{UserName: username, Password: password}
85 ctx, cancel := context.WithTimeout(context.Background(), 120000*time.Millisecond)
86 ctx = context.WithValue(ctx, bitbucketv1.ContextBasicAuth, basicAuth)
87 defer cancel()
88
89 apiPath, err := url.Parse("/rest")
90 if err != nil {
91 log.Fatal(err)
92 }
93
94 apiBaseURL := rootURL.ResolveReference(apiPath).String()
95
96 var config *bitbucketv1.Configuration
97 if *disableTLS {
98 tr := &http.Transport{
99 TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
100 }
101 httpClient := &http.Client{
102 Transport: tr,
103 }
104 httpClientConfig := func(configs *bitbucketv1.Configuration) {
105 configs.HTTPClient = httpClient
106 }
107 config = bitbucketv1.NewConfiguration(apiBaseURL, httpClientConfig)
108 } else {
109 config = bitbucketv1.NewConfiguration(apiBaseURL)
110 }
111 client := bitbucketv1.NewAPIClient(ctx, config)
112
113 var repos []bitbucketv1.Repository
114
115 if *project != "" {
116 repos, err = getProjectRepos(*client, *project)
117 } else {
118 repos, err = getAllRepos(*client)
119 }
120
121 if err != nil {
122 log.Fatal(err)
123 }
124
125 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
126 if err != nil {
127 log.Fatal(err)
128 }
129
130 trimmed := repos[:0]
131 for _, r := range repos {
132 if filter.Include(r.Slug) && (*projectType == "" || r.Project.Type == *projectType) {
133 trimmed = append(trimmed, r)
134 }
135 }
136 repos = trimmed
137
138 if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil {
139 log.Fatalf("cloneRepos: %v", err)
140 }
141
142 if *deleteRepos {
143 if err := deleteStaleRepos(*dest, filter, repos); err != nil {
144 log.Fatalf("deleteStaleRepos: %v", err)
145 }
146 }
147}
148
149func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []bitbucketv1.Repository) error {
150 var baseURL string
151 if len(repos) > 0 {
152 baseURL = repos[0].Links.Self[0].Href
153 } else {
154 return nil
155 }
156 u, err := url.Parse(baseURL)
157 if err != nil {
158 return err
159 }
160 u.Path = ""
161
162 names := map[string]struct{}{}
163 for _, r := range repos {
164 names[filepath.Join(u.Host, r.Project.Key, r.Slug+".git")] = struct{}{}
165 }
166
167 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
168 log.Fatalf("deleteRepos: %v", err)
169 }
170 return nil
171}
172
173func IsValidProjectType(projectType string) bool {
174 switch projectType {
175 case "NORMAL", "PERSONAL":
176 return true
177 }
178 return false
179}
180
181func getAllRepos(client bitbucketv1.APIClient) ([]bitbucketv1.Repository, error) {
182 var allRepos []bitbucketv1.Repository
183 opts := map[string]interface{}{
184 "limit": 1000,
185 "start": 0,
186 }
187
188 for {
189 resp, err := client.DefaultApi.GetRepositories_19(opts)
190 if err != nil {
191 return nil, err
192 }
193
194 repos, err := bitbucketv1.GetRepositoriesResponse(resp)
195 if err != nil {
196 return nil, err
197 }
198
199 if len(repos) == 0 {
200 break
201 }
202
203 opts["start"] = opts["start"].(int) + opts["limit"].(int)
204
205 allRepos = append(allRepos, repos...)
206 }
207 return allRepos, nil
208}
209
210func getProjectRepos(client bitbucketv1.APIClient, projectName string) ([]bitbucketv1.Repository, error) {
211 var allRepos []bitbucketv1.Repository
212 opts := map[string]interface{}{
213 "limit": 1000,
214 "start": 0,
215 }
216
217 for {
218 resp, err := client.DefaultApi.GetRepositoriesWithOptions(projectName, opts)
219 if err != nil {
220 return nil, err
221 }
222
223 repos, err := bitbucketv1.GetRepositoriesResponse(resp)
224 if err != nil {
225 return nil, err
226 }
227
228 if len(repos) == 0 {
229 break
230 }
231
232 opts["start"] = opts["start"].(int) + opts["limit"].(int)
233
234 allRepos = append(allRepos, repos...)
235 }
236 return allRepos, nil
237}
238
239func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string) error {
240 for _, r := range repos {
241 fullName := filepath.Join(r.Project.Key, r.Slug)
242 config := map[string]string{
243 "zoekt.web-url-type": "bitbucket-server",
244 "zoekt.web-url": r.Links.Self[0].Href,
245 "zoekt.name": filepath.Join(host, fullName),
246 }
247
248 httpsCloneUrl := ""
249 for _, cloneUrl := range r.Links.Clone {
250 // In fact, this is an https url, i.e. there's no separate Name for https.
251 if cloneUrl.Name == "http" {
252 s := strings.Split(cloneUrl.Href, "@")
253 httpsCloneUrl = s[0] + ":" + password + "@" + s[1]
254 }
255 }
256
257 if httpsCloneUrl != "" {
258 dest, err := gitindex.CloneRepo(destDir, fullName, httpsCloneUrl, config)
259 if err != nil {
260 return err
261 }
262 if dest != "" {
263 fmt.Println(dest)
264 }
265 }
266 }
267
268 return nil
269}