fork of https://github.com/sourcegraph/zoekt
0

Configure Feed

Select the types of activity you want to include in your feed.

add zoekt-mirror-gitea (#844)

* add zoekt-mirror-gitea

* * Clean up setting the default
* update note about topic filtering not being implemented as topics are missing from the API
* cleanup some pointers

* cleanup some code syntax

+308
+292
cmd/zoekt-mirror-gitea/main.go
··· 1 + // Copyright 2016 Google Inc. All rights reserved. 2 + // 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 + // you may not use this file except in compliance with the License. 5 + // You may obtain a copy of the License at 6 + // 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 + // 9 + // Unless required by applicable law or agreed to in writing, software 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 + // See the License for the specific language governing permissions and 13 + // limitations under the License. 14 + 15 + // This binary fetches all repos of a user or organization and clones 16 + // them. It is strongly recommended to get a personal API token from 17 + // https://gitea.com/user/settings/applications, save the token in a 18 + // file, and point the --token option to it. 19 + package main 20 + 21 + import ( 22 + "flag" 23 + "fmt" 24 + "log" 25 + "net/url" 26 + "os" 27 + "path/filepath" 28 + "strconv" 29 + "strings" 30 + 31 + "code.gitea.io/sdk/gitea" 32 + 33 + "github.com/sourcegraph/zoekt/gitindex" 34 + ) 35 + 36 + type topicsFlag []string 37 + 38 + func (f *topicsFlag) String() string { 39 + return strings.Join(*f, ",") 40 + } 41 + 42 + func (f *topicsFlag) Set(value string) error { 43 + *f = append(*f, value) 44 + return nil 45 + } 46 + 47 + type reposFilters struct { 48 + noArchived *bool 49 + } 50 + 51 + func main() { 52 + dest := flag.String("dest", "", "destination directory") 53 + giteaURL := flag.String("url", "https://gitea.com/", "Gitea url. If not set gitea.com will be used as the host.") 54 + org := flag.String("org", "", "organization to mirror") 55 + user := flag.String("user", "", "user to mirror") 56 + token := flag.String("token", 57 + filepath.Join(os.Getenv("HOME"), ".gitea-token"), 58 + "file holding API token.") 59 + forks := flag.Bool("forks", false, "also mirror forks.") 60 + deleteRepos := flag.Bool("delete", false, "delete missing repos") 61 + namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.") 62 + excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") 63 + topics := topicsFlag{} 64 + flag.Var(&topics, "topic", "only clone repos whose have one of given topics. You can add multiple topics by setting this more than once.") 65 + excludeTopics := topicsFlag{} 66 + flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.") 67 + noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived") 68 + 69 + flag.Parse() 70 + 71 + if *dest == "" { 72 + log.Fatal("must set --dest") 73 + } 74 + if *giteaURL == "" && *org == "" && *user == "" { 75 + log.Fatal("must set either --org or --user when gitea.com is used as host") 76 + } 77 + 78 + var host string 79 + var client *gitea.Client 80 + clientOptions := []gitea.ClientOption{} 81 + 82 + destDir := filepath.Join(*dest, host) 83 + if err := os.MkdirAll(destDir, 0o755); err != nil { 84 + log.Fatal(err) 85 + } 86 + 87 + if *token != "" { 88 + content, err := os.ReadFile(*token) 89 + if err != nil { 90 + log.Fatal(err) 91 + } 92 + clientOptions = append(clientOptions, gitea.SetToken(string(content))) 93 + } 94 + client, err := gitea.NewClient(*giteaURL, clientOptions...) 95 + if err != nil { 96 + log.Fatal(err) 97 + } 98 + 99 + reposFilters := reposFilters{ 100 + noArchived: noArchived, 101 + } 102 + var repos []*gitea.Repository 103 + switch { 104 + case *org != "": 105 + log.Printf("fetch repos for org: %s", *org) 106 + repos, err = getOrgRepos(client, *org, reposFilters) 107 + case *user != "": 108 + log.Printf("fetch repos for user: %s", *user) 109 + repos, err = getUserRepos(client, *user, reposFilters) 110 + default: 111 + log.Printf("no user or org specified, cloning all repos.") 112 + repos, err = getUserRepos(client, "", reposFilters) 113 + } 114 + 115 + if err != nil { 116 + log.Fatal(err) 117 + } 118 + 119 + if !*forks { 120 + trimmed := []*gitea.Repository{} 121 + for _, r := range repos { 122 + if r.Fork { 123 + continue 124 + } 125 + trimmed = append(trimmed, r) 126 + } 127 + repos = trimmed 128 + } 129 + 130 + filter, err := gitindex.NewFilter(*namePattern, *excludePattern) 131 + if err != nil { 132 + log.Fatal(err) 133 + } 134 + 135 + { 136 + trimmed := []*gitea.Repository{} 137 + for _, r := range repos { 138 + if !filter.Include(r.Name) { 139 + log.Println(r.Name) 140 + continue 141 + } 142 + trimmed = append(trimmed, r) 143 + } 144 + repos = trimmed 145 + } 146 + 147 + if err := cloneRepos(destDir, repos); err != nil { 148 + log.Fatalf("cloneRepos: %v", err) 149 + } 150 + 151 + if *deleteRepos { 152 + if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil { 153 + log.Fatalf("deleteStaleRepos: %v", err) 154 + } 155 + } 156 + } 157 + 158 + func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*gitea.Repository, user string) error { 159 + var baseURL string 160 + if len(repos) > 0 { 161 + baseURL = repos[0].HTMLURL 162 + } else { 163 + return nil 164 + } 165 + u, err := url.Parse(baseURL) 166 + if err != nil { 167 + return err 168 + } 169 + u.Path = user 170 + 171 + names := map[string]struct{}{} 172 + for _, r := range repos { 173 + u, err := url.Parse(r.HTMLURL) 174 + if err != nil { 175 + return err 176 + } 177 + 178 + names[filepath.Join(u.Host, u.Path+".git")] = struct{}{} 179 + } 180 + if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil { 181 + log.Fatalf("deleteRepos: %v", err) 182 + } 183 + return nil 184 + } 185 + 186 + func filterRepositories(repos []*gitea.Repository, noArchived bool) (filteredRepos []*gitea.Repository) { 187 + for _, repo := range repos { 188 + if noArchived && repo.Archived { 189 + continue 190 + } 191 + filteredRepos = append(filteredRepos, repo) 192 + } 193 + return 194 + } 195 + 196 + func getOrgRepos(client *gitea.Client, org string, reposFilters reposFilters) ([]*gitea.Repository, error) { 197 + var allRepos []*gitea.Repository 198 + searchOptions := &gitea.SearchRepoOptions{} 199 + // OwnerID 200 + organization, _, err := client.GetOrg(org) 201 + if err != nil { 202 + return nil, err 203 + } 204 + 205 + searchOptions.OwnerID = organization.ID 206 + 207 + for { 208 + repos, resp, err := client.SearchRepos(*searchOptions) 209 + if err != nil { 210 + return nil, err 211 + } 212 + if len(repos) == 0 { 213 + break 214 + } 215 + 216 + searchOptions.Page = resp.NextPage 217 + repos = filterRepositories(repos, *reposFilters.noArchived) 218 + allRepos = append(allRepos, repos...) 219 + if resp.NextPage == 0 { 220 + break 221 + } 222 + } 223 + return allRepos, nil 224 + } 225 + 226 + func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters) ([]*gitea.Repository, error) { 227 + var allRepos []*gitea.Repository 228 + searchOptions := &gitea.SearchRepoOptions{} 229 + u, _, err := client.GetUserInfo(user) 230 + if err != nil { 231 + return nil, err 232 + } 233 + searchOptions.OwnerID = u.ID 234 + for { 235 + repos, resp, err := client.SearchRepos(*searchOptions) 236 + if err != nil { 237 + return nil, err 238 + } 239 + if len(repos) == 0 { 240 + break 241 + } 242 + repos = filterRepositories(repos, *reposFilters.noArchived) 243 + allRepos = append(allRepos, repos...) 244 + searchOptions.Page = resp.NextPage 245 + if resp.NextPage == 0 { 246 + break 247 + } 248 + } 249 + return allRepos, nil 250 + } 251 + 252 + func cloneRepos(destDir string, repos []*gitea.Repository) error { 253 + for _, r := range repos { 254 + host, err := url.Parse(r.HTMLURL) 255 + if err != nil { 256 + return err 257 + } 258 + log.Printf("cloning %s", r.HTMLURL) 259 + 260 + config := map[string]string{ 261 + "zoekt.web-url-type": "gitea", 262 + "zoekt.web-url": r.HTMLURL, 263 + "zoekt.name": filepath.Join(host.Hostname(), r.FullName), 264 + 265 + "zoekt.gitea-stars": strconv.Itoa(r.Stars), 266 + "zoekt.gitea-watchers": strconv.Itoa(r.Watchers), 267 + "zoekt.gitea-subscribers": strconv.Itoa(r.Watchers), // FIXME: Get repo subscribers from API 268 + "zoekt.gitea-forks": strconv.Itoa(r.Forks), 269 + 270 + "zoekt.archived": marshalBool(r.Archived), 271 + "zoekt.fork": marshalBool(r.Fork), 272 + "zoekt.public": marshalBool(r.Private || r.Internal), // count internal repos as private 273 + } 274 + dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config) 275 + if err != nil { 276 + return err 277 + } 278 + if dest != "" { 279 + fmt.Println(dest) 280 + } 281 + 282 + } 283 + 284 + return nil 285 + } 286 + 287 + func marshalBool(b bool) string { 288 + if b { 289 + return "1" 290 + } 291 + return "0" 292 + }
+7
go.mod
··· 56 56 ) 57 57 58 58 require ( 59 + github.com/davidmz/go-pageant v1.0.2 // indirect 60 + github.com/go-fed/httpsig v1.1.0 // indirect 61 + github.com/hashicorp/go-version v1.6.0 // indirect 62 + ) 63 + 64 + require ( 59 65 cloud.google.com/go v0.115.1 // indirect 60 66 cloud.google.com/go/auth v0.9.3 // indirect 61 67 cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect 62 68 cloud.google.com/go/compute/metadata v0.5.0 // indirect 69 + code.gitea.io/sdk/gitea v0.19.0 63 70 dario.cat/mergo v1.0.1 // indirect 64 71 github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect 65 72 github.com/Microsoft/go-winio v0.6.2 // indirect
+9
go.sum
··· 14 14 cloud.google.com/go/profiler v0.4.1/go.mod h1:LBrtEX6nbvhv1w/e5CPZmX9ajGG9BGLtGbv56Tg4SHs= 15 15 cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= 16 16 cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= 17 + code.gitea.io/sdk/gitea v0.19.0 h1:8I6s1s4RHgzxiPHhOQdgim1RWIRcr0LVMbHBjBFXq4Y= 18 + code.gitea.io/sdk/gitea v0.19.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI= 17 19 dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 18 20 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 19 21 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= ··· 73 75 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 74 76 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 75 77 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 78 + github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= 79 + github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= 76 80 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 77 81 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 78 82 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= ··· 108 112 github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= 109 113 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= 110 114 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 115 + github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= 116 + github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= 111 117 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= 112 118 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= 113 119 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= ··· 199 205 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= 200 206 github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= 201 207 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= 208 + github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= 209 + github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 202 210 github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY= 203 211 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo= 204 212 github.com/hexops/autogold v1.3.1/go.mod h1:sQO+mQUCVfxOKPht+ipDSkJ2SCJ7BNJVHZexsXqWMx4= ··· 375 383 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 376 384 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 377 385 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 386 + golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 378 387 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 379 388 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 380 389 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=