fork of https://github.com/sourcegraph/zoekt
1// Licensed under the Apache License, Version 2.0 (the "License");
2// you may not use this file except in compliance with the License.
3// You may obtain a copy of the License at
4//
5// http://www.apache.org/licenses/LICENSE-2.0
6//
7// Unless required by applicable law or agreed to in writing, software
8// distributed under the License is distributed on an "AS IS" BASIS,
9// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10// See the License for the specific language governing permissions and
11// limitations under the License.
12
13// Command zoekt-mirror-bitbucket-server fetches all repos of a bitbucket project,
14// optionally of a specific type, and clones them.
15package main
16
17import (
18 "context"
19 "crypto/tls"
20 "flag"
21 "fmt"
22 "log"
23 "net/http"
24 "net/url"
25 "os"
26 "path/filepath"
27 "strings"
28 "time"
29
30 bitbucketv1 "github.com/gfleury/go-bitbucket-v1"
31
32 "github.com/sourcegraph/zoekt/internal/gitindex"
33)
34
35func main() {
36 dest := flag.String("dest", "", "destination directory")
37 serverUrl := flag.String("url", "", "BitBucket Server url")
38 disableTLS := flag.Bool("disable-tls", false, "disables TLS verification")
39 credentialsFile := flag.String("credentials", ".bitbucket-credentials", "file holding BitBucket Server credentials")
40 project := flag.String("project", "", "project to mirror")
41 deleteRepos := flag.Bool("delete", false, "delete missing repos")
42 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
43 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
44 projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+
45 "Type can be either NORMAl or PERSONAL. Clones projects of both types if not set.")
46 flag.Parse()
47
48 if *serverUrl == "" {
49 log.Fatal("must set --url")
50 }
51
52 rootURL, err := url.Parse(*serverUrl)
53 if err != nil {
54 log.Fatalf("url.Parse(): %v", err)
55 }
56
57 if *dest == "" {
58 log.Fatal("must set --dest")
59 }
60
61 if *projectType != "" && !IsValidProjectType(*projectType) {
62 log.Fatal("type should be either NORMAL or PERSONAL")
63 }
64
65 destDir := filepath.Join(*dest, rootURL.Host)
66 if err := os.MkdirAll(destDir, 0o755); err != nil {
67 log.Fatal(err)
68 }
69
70 username := ""
71 password := ""
72 if *credentialsFile == "" {
73 log.Fatal("must set --credentials")
74 } else {
75 content, err := os.ReadFile(*credentialsFile)
76 if err != nil {
77 log.Fatal(err)
78 }
79 credentials := strings.Fields(string(content))
80 username, password = credentials[0], credentials[1]
81 }
82
83 basicAuth := bitbucketv1.BasicAuth{UserName: username, Password: password}
84 ctx, cancel := context.WithTimeout(context.Background(), 120000*time.Millisecond)
85 ctx = context.WithValue(ctx, bitbucketv1.ContextBasicAuth, basicAuth)
86 defer cancel()
87
88 apiPath, err := url.Parse("/rest")
89 if err != nil {
90 log.Fatal(err)
91 }
92
93 apiBaseURL := rootURL.ResolveReference(apiPath).String()
94
95 var config *bitbucketv1.Configuration
96 if *disableTLS {
97 tr := &http.Transport{
98 TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
99 }
100 httpClient := &http.Client{
101 Transport: tr,
102 }
103 httpClientConfig := func(configs *bitbucketv1.Configuration) {
104 configs.HTTPClient = httpClient
105 }
106 config = bitbucketv1.NewConfiguration(apiBaseURL, httpClientConfig)
107 } else {
108 config = bitbucketv1.NewConfiguration(apiBaseURL)
109 }
110 client := bitbucketv1.NewAPIClient(ctx, config)
111
112 var repos []bitbucketv1.Repository
113
114 if *project != "" {
115 repos, err = getProjectRepos(*client, *project)
116 } else {
117 repos, err = getAllRepos(*client)
118 }
119
120 if err != nil {
121 log.Fatal(err)
122 }
123
124 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
125 if err != nil {
126 log.Fatal(err)
127 }
128
129 trimmed := repos[:0]
130 for _, r := range repos {
131 if filter.Include(r.Slug) && (*projectType == "" || r.Project.Type == *projectType) {
132 trimmed = append(trimmed, r)
133 }
134 }
135 repos = trimmed
136
137 if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil {
138 log.Fatalf("cloneRepos: %v", err)
139 }
140
141 if *deleteRepos {
142 if err := deleteStaleRepos(*dest, filter, repos); err != nil {
143 log.Fatalf("deleteStaleRepos: %v", err)
144 }
145 }
146}
147
148func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []bitbucketv1.Repository) error {
149 var baseURL string
150 if len(repos) > 0 {
151 baseURL = repos[0].Links.Self[0].Href
152 } else {
153 return nil
154 }
155 u, err := url.Parse(baseURL)
156 if err != nil {
157 return err
158 }
159 u.Path = ""
160
161 names := map[string]struct{}{}
162 for _, r := range repos {
163 names[filepath.Join(u.Host, r.Project.Key, r.Slug+".git")] = struct{}{}
164 }
165
166 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
167 log.Fatalf("deleteRepos: %v", err)
168 }
169 return nil
170}
171
172func IsValidProjectType(projectType string) bool {
173 switch projectType {
174 case "NORMAL", "PERSONAL":
175 return true
176 }
177 return false
178}
179
180func getAllRepos(client bitbucketv1.APIClient) ([]bitbucketv1.Repository, error) {
181 var allRepos []bitbucketv1.Repository
182 opts := map[string]any{
183 "limit": 1000,
184 "start": 0,
185 }
186
187 for {
188 resp, err := client.DefaultApi.GetRepositories_19(opts)
189 if err != nil {
190 return nil, err
191 }
192
193 repos, err := bitbucketv1.GetRepositoriesResponse(resp)
194 if err != nil {
195 return nil, err
196 }
197
198 if len(repos) == 0 {
199 break
200 }
201
202 opts["start"] = opts["start"].(int) + opts["limit"].(int)
203
204 allRepos = append(allRepos, repos...)
205 }
206 return allRepos, nil
207}
208
209func getProjectRepos(client bitbucketv1.APIClient, projectName string) ([]bitbucketv1.Repository, error) {
210 var allRepos []bitbucketv1.Repository
211 opts := map[string]any{
212 "limit": 1000,
213 "start": 0,
214 }
215
216 for {
217 resp, err := client.DefaultApi.GetRepositoriesWithOptions(projectName, opts)
218 if err != nil {
219 return nil, err
220 }
221
222 repos, err := bitbucketv1.GetRepositoriesResponse(resp)
223 if err != nil {
224 return nil, err
225 }
226
227 if len(repos) == 0 {
228 break
229 }
230
231 opts["start"] = opts["start"].(int) + opts["limit"].(int)
232
233 allRepos = append(allRepos, repos...)
234 }
235 return allRepos, nil
236}
237
238func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string) error {
239 for _, r := range repos {
240 fullName := filepath.Join(r.Project.Key, r.Slug)
241 config := map[string]string{
242 "zoekt.web-url-type": "bitbucket-server",
243 "zoekt.web-url": r.Links.Self[0].Href,
244 "zoekt.name": filepath.Join(host, fullName),
245 }
246
247 httpsCloneUrl := ""
248 for _, cloneUrl := range r.Links.Clone {
249 // In fact, this is an https url, i.e. there's no separate Name for https.
250 if cloneUrl.Name == "http" {
251 s := strings.Split(cloneUrl.Href, "@")
252 httpsCloneUrl = s[0] + ":" + password + "@" + s[1]
253 }
254 }
255
256 if httpsCloneUrl != "" {
257 dest, err := gitindex.CloneRepo(destDir, fullName, httpsCloneUrl, config)
258 if err != nil {
259 return err
260 }
261 if dest != "" {
262 fmt.Println(dest)
263 }
264 }
265 }
266
267 return nil
268}