fork of https://github.com/sourcegraph/zoekt
add zoekt-mirror-gitea (#844)
* add zoekt-mirror-gitea
* * Clean up setting the default
* update note about topic filtering not being implemented as topics are missing from the API
* cleanup some pointers
* cleanup some code syntax
author
techknowlogick
committer
GitHub
date
(Oct 14, 2024, 10:20 AM +0200)
commit
da3626ec
da3626ec7c7cf15d762aa508abefe9896f90b5e0
parent
56878093
5687809315075882a8e7413bdb17b042f3394c02
+308
+292
cmd/zoekt-mirror-gitea/main.go
+292
cmd/zoekt-mirror-gitea/main.go
···
1
+
// Copyright 2016 Google Inc. All rights reserved.
2
+
//
3
+
// Licensed under the Apache License, Version 2.0 (the "License");
4
+
// you may not use this file except in compliance with the License.
5
+
// You may obtain a copy of the License at
6
+
//
7
+
// http://www.apache.org/licenses/LICENSE-2.0
8
+
//
9
+
// Unless required by applicable law or agreed to in writing, software
10
+
// distributed under the License is distributed on an "AS IS" BASIS,
11
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+
// See the License for the specific language governing permissions and
13
+
// limitations under the License.
14
+
15
+
// This binary fetches all repos of a user or organization and clones
16
+
// them. It is strongly recommended to get a personal API token from
17
+
// https://gitea.com/user/settings/applications, save the token in a
18
+
// file, and point the --token option to it.
19
+
package main
20
+
21
+
import (
22
+
"flag"
23
+
"fmt"
24
+
"log"
25
+
"net/url"
26
+
"os"
27
+
"path/filepath"
28
+
"strconv"
29
+
"strings"
30
+
31
+
"code.gitea.io/sdk/gitea"
32
+
33
+
"github.com/sourcegraph/zoekt/gitindex"
34
+
)
35
+
36
+
type topicsFlag []string
37
+
38
+
func (f *topicsFlag) String() string {
39
+
return strings.Join(*f, ",")
40
+
}
41
+
42
+
func (f *topicsFlag) Set(value string) error {
43
+
*f = append(*f, value)
44
+
return nil
45
+
}
46
+
47
+
type reposFilters struct {
48
+
noArchived *bool
49
+
}
50
+
51
+
func main() {
52
+
dest := flag.String("dest", "", "destination directory")
53
+
giteaURL := flag.String("url", "https://gitea.com/", "Gitea url. If not set gitea.com will be used as the host.")
54
+
org := flag.String("org", "", "organization to mirror")
55
+
user := flag.String("user", "", "user to mirror")
56
+
token := flag.String("token",
57
+
filepath.Join(os.Getenv("HOME"), ".gitea-token"),
58
+
"file holding API token.")
59
+
forks := flag.Bool("forks", false, "also mirror forks.")
60
+
deleteRepos := flag.Bool("delete", false, "delete missing repos")
61
+
namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
62
+
excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
63
+
topics := topicsFlag{}
64
+
flag.Var(&topics, "topic", "only clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
65
+
excludeTopics := topicsFlag{}
66
+
flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
67
+
noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
68
+
69
+
flag.Parse()
70
+
71
+
if *dest == "" {
72
+
log.Fatal("must set --dest")
73
+
}
74
+
if *giteaURL == "" && *org == "" && *user == "" {
75
+
log.Fatal("must set either --org or --user when gitea.com is used as host")
76
+
}
77
+
78
+
var host string
79
+
var client *gitea.Client
80
+
clientOptions := []gitea.ClientOption{}
81
+
82
+
destDir := filepath.Join(*dest, host)
83
+
if err := os.MkdirAll(destDir, 0o755); err != nil {
84
+
log.Fatal(err)
85
+
}
86
+
87
+
if *token != "" {
88
+
content, err := os.ReadFile(*token)
89
+
if err != nil {
90
+
log.Fatal(err)
91
+
}
92
+
clientOptions = append(clientOptions, gitea.SetToken(string(content)))
93
+
}
94
+
client, err := gitea.NewClient(*giteaURL, clientOptions...)
95
+
if err != nil {
96
+
log.Fatal(err)
97
+
}
98
+
99
+
reposFilters := reposFilters{
100
+
noArchived: noArchived,
101
+
}
102
+
var repos []*gitea.Repository
103
+
switch {
104
+
case *org != "":
105
+
log.Printf("fetch repos for org: %s", *org)
106
+
repos, err = getOrgRepos(client, *org, reposFilters)
107
+
case *user != "":
108
+
log.Printf("fetch repos for user: %s", *user)
109
+
repos, err = getUserRepos(client, *user, reposFilters)
110
+
default:
111
+
log.Printf("no user or org specified, cloning all repos.")
112
+
repos, err = getUserRepos(client, "", reposFilters)
113
+
}
114
+
115
+
if err != nil {
116
+
log.Fatal(err)
117
+
}
118
+
119
+
if !*forks {
120
+
trimmed := []*gitea.Repository{}
121
+
for _, r := range repos {
122
+
if r.Fork {
123
+
continue
124
+
}
125
+
trimmed = append(trimmed, r)
126
+
}
127
+
repos = trimmed
128
+
}
129
+
130
+
filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
131
+
if err != nil {
132
+
log.Fatal(err)
133
+
}
134
+
135
+
{
136
+
trimmed := []*gitea.Repository{}
137
+
for _, r := range repos {
138
+
if !filter.Include(r.Name) {
139
+
log.Println(r.Name)
140
+
continue
141
+
}
142
+
trimmed = append(trimmed, r)
143
+
}
144
+
repos = trimmed
145
+
}
146
+
147
+
if err := cloneRepos(destDir, repos); err != nil {
148
+
log.Fatalf("cloneRepos: %v", err)
149
+
}
150
+
151
+
if *deleteRepos {
152
+
if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
153
+
log.Fatalf("deleteStaleRepos: %v", err)
154
+
}
155
+
}
156
+
}
157
+
158
+
func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*gitea.Repository, user string) error {
159
+
var baseURL string
160
+
if len(repos) > 0 {
161
+
baseURL = repos[0].HTMLURL
162
+
} else {
163
+
return nil
164
+
}
165
+
u, err := url.Parse(baseURL)
166
+
if err != nil {
167
+
return err
168
+
}
169
+
u.Path = user
170
+
171
+
names := map[string]struct{}{}
172
+
for _, r := range repos {
173
+
u, err := url.Parse(r.HTMLURL)
174
+
if err != nil {
175
+
return err
176
+
}
177
+
178
+
names[filepath.Join(u.Host, u.Path+".git")] = struct{}{}
179
+
}
180
+
if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
181
+
log.Fatalf("deleteRepos: %v", err)
182
+
}
183
+
return nil
184
+
}
185
+
186
+
func filterRepositories(repos []*gitea.Repository, noArchived bool) (filteredRepos []*gitea.Repository) {
187
+
for _, repo := range repos {
188
+
if noArchived && repo.Archived {
189
+
continue
190
+
}
191
+
filteredRepos = append(filteredRepos, repo)
192
+
}
193
+
return
194
+
}
195
+
196
+
func getOrgRepos(client *gitea.Client, org string, reposFilters reposFilters) ([]*gitea.Repository, error) {
197
+
var allRepos []*gitea.Repository
198
+
searchOptions := &gitea.SearchRepoOptions{}
199
+
// OwnerID
200
+
organization, _, err := client.GetOrg(org)
201
+
if err != nil {
202
+
return nil, err
203
+
}
204
+
205
+
searchOptions.OwnerID = organization.ID
206
+
207
+
for {
208
+
repos, resp, err := client.SearchRepos(*searchOptions)
209
+
if err != nil {
210
+
return nil, err
211
+
}
212
+
if len(repos) == 0 {
213
+
break
214
+
}
215
+
216
+
searchOptions.Page = resp.NextPage
217
+
repos = filterRepositories(repos, *reposFilters.noArchived)
218
+
allRepos = append(allRepos, repos...)
219
+
if resp.NextPage == 0 {
220
+
break
221
+
}
222
+
}
223
+
return allRepos, nil
224
+
}
225
+
226
+
func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters) ([]*gitea.Repository, error) {
227
+
var allRepos []*gitea.Repository
228
+
searchOptions := &gitea.SearchRepoOptions{}
229
+
u, _, err := client.GetUserInfo(user)
230
+
if err != nil {
231
+
return nil, err
232
+
}
233
+
searchOptions.OwnerID = u.ID
234
+
for {
235
+
repos, resp, err := client.SearchRepos(*searchOptions)
236
+
if err != nil {
237
+
return nil, err
238
+
}
239
+
if len(repos) == 0 {
240
+
break
241
+
}
242
+
repos = filterRepositories(repos, *reposFilters.noArchived)
243
+
allRepos = append(allRepos, repos...)
244
+
searchOptions.Page = resp.NextPage
245
+
if resp.NextPage == 0 {
246
+
break
247
+
}
248
+
}
249
+
return allRepos, nil
250
+
}
251
+
252
+
func cloneRepos(destDir string, repos []*gitea.Repository) error {
253
+
for _, r := range repos {
254
+
host, err := url.Parse(r.HTMLURL)
255
+
if err != nil {
256
+
return err
257
+
}
258
+
log.Printf("cloning %s", r.HTMLURL)
259
+
260
+
config := map[string]string{
261
+
"zoekt.web-url-type": "gitea",
262
+
"zoekt.web-url": r.HTMLURL,
263
+
"zoekt.name": filepath.Join(host.Hostname(), r.FullName),
264
+
265
+
"zoekt.gitea-stars": strconv.Itoa(r.Stars),
266
+
"zoekt.gitea-watchers": strconv.Itoa(r.Watchers),
267
+
"zoekt.gitea-subscribers": strconv.Itoa(r.Watchers), // FIXME: Get repo subscribers from API
268
+
"zoekt.gitea-forks": strconv.Itoa(r.Forks),
269
+
270
+
"zoekt.archived": marshalBool(r.Archived),
271
+
"zoekt.fork": marshalBool(r.Fork),
272
+
"zoekt.public": marshalBool(r.Private || r.Internal), // count internal repos as private
273
+
}
274
+
dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config)
275
+
if err != nil {
276
+
return err
277
+
}
278
+
if dest != "" {
279
+
fmt.Println(dest)
280
+
}
281
+
282
+
}
283
+
284
+
return nil
285
+
}
286
+
287
+
func marshalBool(b bool) string {
288
+
if b {
289
+
return "1"
290
+
}
291
+
return "0"
292
+
}
+7
go.mod
+7
go.mod
···
56
56
)
57
57
58
58
require (
59
+
github.com/davidmz/go-pageant v1.0.2 // indirect
60
+
github.com/go-fed/httpsig v1.1.0 // indirect
61
+
github.com/hashicorp/go-version v1.6.0 // indirect
62
+
)
63
+
64
+
require (
59
65
cloud.google.com/go v0.115.1 // indirect
60
66
cloud.google.com/go/auth v0.9.3 // indirect
61
67
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
62
68
cloud.google.com/go/compute/metadata v0.5.0 // indirect
69
+
code.gitea.io/sdk/gitea v0.19.0
63
70
dario.cat/mergo v1.0.1 // indirect
64
71
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
65
72
github.com/Microsoft/go-winio v0.6.2 // indirect
+9
go.sum
+9
go.sum
···
14
14
cloud.google.com/go/profiler v0.4.1/go.mod h1:LBrtEX6nbvhv1w/e5CPZmX9ajGG9BGLtGbv56Tg4SHs=
15
15
cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
16
16
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
17
+
code.gitea.io/sdk/gitea v0.19.0 h1:8I6s1s4RHgzxiPHhOQdgim1RWIRcr0LVMbHBjBFXq4Y=
18
+
code.gitea.io/sdk/gitea v0.19.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI=
17
19
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
18
20
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
19
21
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
···
73
75
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
74
76
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
75
77
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
78
+
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
79
+
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
76
80
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
77
81
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
78
82
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
···
108
112
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
109
113
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
110
114
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
115
+
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
116
+
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
111
117
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
112
118
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
113
119
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
···
199
205
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
200
206
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
201
207
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
208
+
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
209
+
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
202
210
github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
203
211
github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
204
212
github.com/hexops/autogold v1.3.1/go.mod h1:sQO+mQUCVfxOKPht+ipDSkJ2SCJ7BNJVHZexsXqWMx4=
···
375
383
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
376
384
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
377
385
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
386
+
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
378
387
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
379
388
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
380
389
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=