fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-mirror-gitea fetches all repos of a gitea user or organization
16// and clones them. It is strongly recommended to get a personal API token from
17// https://gitea.com/user/settings/applications, save the token in a file, and point
18// the --token option to it.
19package main
20
21import (
22 "flag"
23 "fmt"
24 "log"
25 "net/url"
26 "os"
27 "path/filepath"
28 "strconv"
29 "strings"
30
31 "code.gitea.io/sdk/gitea"
32
33 "github.com/sourcegraph/zoekt/gitindex"
34)
35
36type topicsFlag []string
37
38func (f *topicsFlag) String() string {
39 return strings.Join(*f, ",")
40}
41
42func (f *topicsFlag) Set(value string) error {
43 *f = append(*f, value)
44 return nil
45}
46
47type reposFilters struct {
48 noArchived *bool
49}
50
51func main() {
52 dest := flag.String("dest", "", "destination directory")
53 giteaURL := flag.String("url", "https://gitea.com/", "Gitea url. If not set gitea.com will be used as the host.")
54 org := flag.String("org", "", "organization to mirror")
55 user := flag.String("user", "", "user to mirror")
56 token := flag.String("token",
57 filepath.Join(os.Getenv("HOME"), ".gitea-token"),
58 "file holding API token.")
59 forks := flag.Bool("forks", false, "also mirror forks.")
60 deleteRepos := flag.Bool("delete", false, "delete missing repos")
61 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
62 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
63 topics := topicsFlag{}
64 flag.Var(&topics, "topic", "only clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
65 excludeTopics := topicsFlag{}
66 flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
67 noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
68
69 flag.Parse()
70
71 if *dest == "" {
72 log.Fatal("must set --dest")
73 }
74 if *giteaURL == "" && *org == "" && *user == "" {
75 log.Fatal("must set either --org or --user when gitea.com is used as host")
76 }
77
78 var host string
79 var client *gitea.Client
80 clientOptions := []gitea.ClientOption{}
81
82 destDir := filepath.Join(*dest, host)
83 if err := os.MkdirAll(destDir, 0o755); err != nil {
84 log.Fatal(err)
85 }
86
87 if *token != "" {
88 content, err := os.ReadFile(*token)
89 if err != nil {
90 log.Fatal(err)
91 }
92 contentStr := string(content)
93 // Editors tend to insert newlines that make the token invalid, so clean it up
94 contentStr = strings.TrimSpace(contentStr)
95 clientOptions = append(clientOptions, gitea.SetToken(contentStr))
96 }
97 client, err := gitea.NewClient(*giteaURL, clientOptions...)
98 if err != nil {
99 log.Fatal(err)
100 }
101
102 reposFilters := reposFilters{
103 noArchived: noArchived,
104 }
105 var repos []*gitea.Repository
106 switch {
107 case *org != "":
108 log.Printf("fetch repos for org: %s", *org)
109 repos, err = getOrgRepos(client, *org, reposFilters)
110 case *user != "":
111 log.Printf("fetch repos for user: %s", *user)
112 repos, err = getUserRepos(client, *user, reposFilters)
113 default:
114 log.Printf("no user or org specified, cloning all repos.")
115 repos, err = getUserRepos(client, "", reposFilters)
116 }
117
118 if err != nil {
119 log.Fatal(err)
120 }
121
122 if !*forks {
123 trimmed := []*gitea.Repository{}
124 for _, r := range repos {
125 if r.Fork {
126 continue
127 }
128 trimmed = append(trimmed, r)
129 }
130 repos = trimmed
131 }
132
133 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
134 if err != nil {
135 log.Fatal(err)
136 }
137
138 {
139 trimmed := []*gitea.Repository{}
140 for _, r := range repos {
141 if !filter.Include(r.Name) {
142 log.Println(r.Name)
143 continue
144 }
145 trimmed = append(trimmed, r)
146 }
147 repos = trimmed
148 }
149
150 if err := cloneRepos(destDir, repos); err != nil {
151 log.Fatalf("cloneRepos: %v", err)
152 }
153
154 if *deleteRepos {
155 if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
156 log.Fatalf("deleteStaleRepos: %v", err)
157 }
158 }
159}
160
161func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*gitea.Repository, user string) error {
162 var baseURL string
163 if len(repos) > 0 {
164 baseURL = repos[0].HTMLURL
165 } else {
166 return nil
167 }
168 u, err := url.Parse(baseURL)
169 if err != nil {
170 return err
171 }
172 u.Path = user
173
174 names := map[string]struct{}{}
175 for _, r := range repos {
176 u, err := url.Parse(r.HTMLURL)
177 if err != nil {
178 return err
179 }
180
181 names[filepath.Join(u.Host, u.Path+".git")] = struct{}{}
182 }
183 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
184 log.Fatalf("deleteRepos: %v", err)
185 }
186 return nil
187}
188
189func filterRepositories(repos []*gitea.Repository, noArchived bool) (filteredRepos []*gitea.Repository) {
190 for _, repo := range repos {
191 if noArchived && repo.Archived {
192 continue
193 }
194 filteredRepos = append(filteredRepos, repo)
195 }
196 return
197}
198
199func getOrgRepos(client *gitea.Client, org string, reposFilters reposFilters) ([]*gitea.Repository, error) {
200 var allRepos []*gitea.Repository
201 searchOptions := &gitea.SearchRepoOptions{}
202 // OwnerID
203 organization, _, err := client.GetOrg(org)
204 if err != nil {
205 return nil, err
206 }
207
208 searchOptions.OwnerID = organization.ID
209
210 for {
211 repos, resp, err := client.SearchRepos(*searchOptions)
212 if err != nil {
213 return nil, err
214 }
215 if len(repos) == 0 {
216 break
217 }
218
219 searchOptions.Page = resp.NextPage
220 repos = filterRepositories(repos, *reposFilters.noArchived)
221 allRepos = append(allRepos, repos...)
222 if resp.NextPage == 0 {
223 break
224 }
225 }
226 return allRepos, nil
227}
228
229func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters) ([]*gitea.Repository, error) {
230 var allRepos []*gitea.Repository
231 searchOptions := &gitea.SearchRepoOptions{}
232 u, _, err := client.GetUserInfo(user)
233 if err != nil {
234 return nil, err
235 }
236 searchOptions.OwnerID = u.ID
237 for {
238 repos, resp, err := client.SearchRepos(*searchOptions)
239 if err != nil {
240 return nil, err
241 }
242 if len(repos) == 0 {
243 break
244 }
245 repos = filterRepositories(repos, *reposFilters.noArchived)
246 allRepos = append(allRepos, repos...)
247 searchOptions.Page = resp.NextPage
248 if resp.NextPage == 0 {
249 break
250 }
251 }
252 return allRepos, nil
253}
254
255func cloneRepos(destDir string, repos []*gitea.Repository) error {
256 for _, r := range repos {
257 host, err := url.Parse(r.HTMLURL)
258 if err != nil {
259 return err
260 }
261 log.Printf("cloning %s", r.HTMLURL)
262
263 config := map[string]string{
264 "zoekt.web-url-type": "gitea",
265 "zoekt.web-url": r.HTMLURL,
266 "zoekt.name": filepath.Join(host.Hostname(), r.FullName),
267
268 "zoekt.gitea-stars": strconv.Itoa(r.Stars),
269 "zoekt.gitea-watchers": strconv.Itoa(r.Watchers),
270 "zoekt.gitea-subscribers": strconv.Itoa(r.Watchers), // FIXME: Get repo subscribers from API
271 "zoekt.gitea-forks": strconv.Itoa(r.Forks),
272
273 "zoekt.archived": marshalBool(r.Archived),
274 "zoekt.fork": marshalBool(r.Fork),
275 "zoekt.public": marshalBool(r.Private || r.Internal), // count internal repos as private
276 }
277 dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config)
278 if err != nil {
279 return err
280 }
281 if dest != "" {
282 fmt.Println(dest)
283 }
284
285 }
286
287 return nil
288}
289
290func marshalBool(b bool) string {
291 if b {
292 return "1"
293 }
294 return "0"
295}