fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-mirror-gerrit fetches all repos of a gitea user or organization
16// and clones them. It is strongly recommended to get a personal API token from
17// https://gitea.com/user/settings/applications, save the token in a file, and point
18// the --token option to it.
19package main
20
21import (
22 "flag"
23 "fmt"
24 "log"
25 "net/url"
26 "os"
27 "path/filepath"
28 "strconv"
29 "strings"
30
31 "code.gitea.io/sdk/gitea"
32
33 "github.com/sourcegraph/zoekt/gitindex"
34)
35
36type topicsFlag []string
37
38func (f *topicsFlag) String() string {
39 return strings.Join(*f, ",")
40}
41
42func (f *topicsFlag) Set(value string) error {
43 *f = append(*f, value)
44 return nil
45}
46
47type reposFilters struct {
48 noArchived *bool
49}
50
51func main() {
52 dest := flag.String("dest", "", "destination directory")
53 giteaURL := flag.String("url", "https://gitea.com/", "Gitea url. If not set gitea.com will be used as the host.")
54 org := flag.String("org", "", "organization to mirror")
55 user := flag.String("user", "", "user to mirror")
56 token := flag.String("token",
57 filepath.Join(os.Getenv("HOME"), ".gitea-token"),
58 "file holding API token.")
59 forks := flag.Bool("forks", false, "also mirror forks.")
60 deleteRepos := flag.Bool("delete", false, "delete missing repos")
61 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
62 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
63 topics := topicsFlag{}
64 flag.Var(&topics, "topic", "only clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
65 excludeTopics := topicsFlag{}
66 flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
67 noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
68
69 flag.Parse()
70
71 if *dest == "" {
72 log.Fatal("must set --dest")
73 }
74 if *giteaURL == "" && *org == "" && *user == "" {
75 log.Fatal("must set either --org or --user when gitea.com is used as host")
76 }
77
78 var host string
79 var client *gitea.Client
80 clientOptions := []gitea.ClientOption{}
81
82 destDir := filepath.Join(*dest, host)
83 if err := os.MkdirAll(destDir, 0o755); err != nil {
84 log.Fatal(err)
85 }
86
87 if *token != "" {
88 content, err := os.ReadFile(*token)
89 if err != nil {
90 log.Fatal(err)
91 }
92 clientOptions = append(clientOptions, gitea.SetToken(string(content)))
93 }
94 client, err := gitea.NewClient(*giteaURL, clientOptions...)
95 if err != nil {
96 log.Fatal(err)
97 }
98
99 reposFilters := reposFilters{
100 noArchived: noArchived,
101 }
102 var repos []*gitea.Repository
103 switch {
104 case *org != "":
105 log.Printf("fetch repos for org: %s", *org)
106 repos, err = getOrgRepos(client, *org, reposFilters)
107 case *user != "":
108 log.Printf("fetch repos for user: %s", *user)
109 repos, err = getUserRepos(client, *user, reposFilters)
110 default:
111 log.Printf("no user or org specified, cloning all repos.")
112 repos, err = getUserRepos(client, "", reposFilters)
113 }
114
115 if err != nil {
116 log.Fatal(err)
117 }
118
119 if !*forks {
120 trimmed := []*gitea.Repository{}
121 for _, r := range repos {
122 if r.Fork {
123 continue
124 }
125 trimmed = append(trimmed, r)
126 }
127 repos = trimmed
128 }
129
130 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
131 if err != nil {
132 log.Fatal(err)
133 }
134
135 {
136 trimmed := []*gitea.Repository{}
137 for _, r := range repos {
138 if !filter.Include(r.Name) {
139 log.Println(r.Name)
140 continue
141 }
142 trimmed = append(trimmed, r)
143 }
144 repos = trimmed
145 }
146
147 if err := cloneRepos(destDir, repos); err != nil {
148 log.Fatalf("cloneRepos: %v", err)
149 }
150
151 if *deleteRepos {
152 if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
153 log.Fatalf("deleteStaleRepos: %v", err)
154 }
155 }
156}
157
158func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*gitea.Repository, user string) error {
159 var baseURL string
160 if len(repos) > 0 {
161 baseURL = repos[0].HTMLURL
162 } else {
163 return nil
164 }
165 u, err := url.Parse(baseURL)
166 if err != nil {
167 return err
168 }
169 u.Path = user
170
171 names := map[string]struct{}{}
172 for _, r := range repos {
173 u, err := url.Parse(r.HTMLURL)
174 if err != nil {
175 return err
176 }
177
178 names[filepath.Join(u.Host, u.Path+".git")] = struct{}{}
179 }
180 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
181 log.Fatalf("deleteRepos: %v", err)
182 }
183 return nil
184}
185
186func filterRepositories(repos []*gitea.Repository, noArchived bool) (filteredRepos []*gitea.Repository) {
187 for _, repo := range repos {
188 if noArchived && repo.Archived {
189 continue
190 }
191 filteredRepos = append(filteredRepos, repo)
192 }
193 return
194}
195
196func getOrgRepos(client *gitea.Client, org string, reposFilters reposFilters) ([]*gitea.Repository, error) {
197 var allRepos []*gitea.Repository
198 searchOptions := &gitea.SearchRepoOptions{}
199 // OwnerID
200 organization, _, err := client.GetOrg(org)
201 if err != nil {
202 return nil, err
203 }
204
205 searchOptions.OwnerID = organization.ID
206
207 for {
208 repos, resp, err := client.SearchRepos(*searchOptions)
209 if err != nil {
210 return nil, err
211 }
212 if len(repos) == 0 {
213 break
214 }
215
216 searchOptions.Page = resp.NextPage
217 repos = filterRepositories(repos, *reposFilters.noArchived)
218 allRepos = append(allRepos, repos...)
219 if resp.NextPage == 0 {
220 break
221 }
222 }
223 return allRepos, nil
224}
225
226func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters) ([]*gitea.Repository, error) {
227 var allRepos []*gitea.Repository
228 searchOptions := &gitea.SearchRepoOptions{}
229 u, _, err := client.GetUserInfo(user)
230 if err != nil {
231 return nil, err
232 }
233 searchOptions.OwnerID = u.ID
234 for {
235 repos, resp, err := client.SearchRepos(*searchOptions)
236 if err != nil {
237 return nil, err
238 }
239 if len(repos) == 0 {
240 break
241 }
242 repos = filterRepositories(repos, *reposFilters.noArchived)
243 allRepos = append(allRepos, repos...)
244 searchOptions.Page = resp.NextPage
245 if resp.NextPage == 0 {
246 break
247 }
248 }
249 return allRepos, nil
250}
251
252func cloneRepos(destDir string, repos []*gitea.Repository) error {
253 for _, r := range repos {
254 host, err := url.Parse(r.HTMLURL)
255 if err != nil {
256 return err
257 }
258 log.Printf("cloning %s", r.HTMLURL)
259
260 config := map[string]string{
261 "zoekt.web-url-type": "gitea",
262 "zoekt.web-url": r.HTMLURL,
263 "zoekt.name": filepath.Join(host.Hostname(), r.FullName),
264
265 "zoekt.gitea-stars": strconv.Itoa(r.Stars),
266 "zoekt.gitea-watchers": strconv.Itoa(r.Watchers),
267 "zoekt.gitea-subscribers": strconv.Itoa(r.Watchers), // FIXME: Get repo subscribers from API
268 "zoekt.gitea-forks": strconv.Itoa(r.Forks),
269
270 "zoekt.archived": marshalBool(r.Archived),
271 "zoekt.fork": marshalBool(r.Fork),
272 "zoekt.public": marshalBool(r.Private || r.Internal), // count internal repos as private
273 }
274 dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config)
275 if err != nil {
276 return err
277 }
278 if dest != "" {
279 fmt.Println(dest)
280 }
281
282 }
283
284 return nil
285}
286
287func marshalBool(b bool) string {
288 if b {
289 return "1"
290 }
291 return "0"
292}