fork of https://github.com/sourcegraph/zoekt
1// Licensed under the Apache License, Version 2.0 (the "License");
2// you may not use this file except in compliance with the License.
3// You may obtain a copy of the License at
4//
5// http://www.apache.org/licenses/LICENSE-2.0
6//
7// Unless required by applicable law or agreed to in writing, software
8// distributed under the License is distributed on an "AS IS" BASIS,
9// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10// See the License for the specific language governing permissions and
11// limitations under the License.
12
13// This binary fetches all repos for a user from gitlab.
14//
15// It is recommended to use a gitlab personal access token:
16// https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This
17// token should be stored in a file and the --token option should be used.
18// In addition, the token should be present in the ~/.netrc of the user running
19// the mirror command. For example, the ~/.netrc may look like:
20//
21// machine gitlab.com
22// login oauth
23// password <personal access token>
24package main
25
26import (
27 "flag"
28 "fmt"
29 "log"
30 "net/url"
31 "os"
32 "path/filepath"
33 "strconv"
34 "strings"
35
36 "github.com/sourcegraph/zoekt/gitindex"
37 gitlab "github.com/xanzy/go-gitlab"
38)
39
40func main() {
41 dest := flag.String("dest", "", "destination directory")
42 gitlabURL := flag.String("url", "https://gitlab.com/api/v4/", "Gitlab URL. If not set https://gitlab.com/api/v4/ will be used")
43 token := flag.String("token",
44 filepath.Join(os.Getenv("HOME"), ".gitlab-token"),
45 "file holding API token.")
46 isMember := flag.Bool("membership", false, "only mirror repos this user is a member of ")
47 isPublic := flag.Bool("public", false, "only mirror public repos")
48 deleteRepos := flag.Bool("delete", false, "delete missing repos")
49 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
50 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
51 flag.Parse()
52
53 if *dest == "" {
54 log.Fatal("must set --dest")
55 }
56
57 var host string
58 rootURL, err := url.Parse(*gitlabURL)
59 if err != nil {
60 log.Fatal(err)
61 }
62 host = rootURL.Host
63
64 destDir := filepath.Join(*dest, host)
65 if err := os.MkdirAll(destDir, 0o755); err != nil {
66 log.Fatal(err)
67 }
68
69 content, err := os.ReadFile(*token)
70 if err != nil {
71 log.Fatal(err)
72 }
73 apiToken := strings.TrimSpace(string(content))
74
75 client, err := gitlab.NewClient(apiToken, gitlab.WithBaseURL(*gitlabURL))
76 if err != nil {
77 log.Fatal(err)
78 }
79
80 opt := &gitlab.ListProjectsOptions{
81 ListOptions: gitlab.ListOptions{
82 PerPage: 100,
83 },
84 Sort: gitlab.String("asc"),
85 OrderBy: gitlab.String("id"),
86 Membership: isMember,
87 }
88 if *isPublic {
89 opt.Visibility = gitlab.Visibility(gitlab.PublicVisibility)
90 }
91
92 var gitlabProjects []*gitlab.Project
93 for {
94 projects, _, err := client.Projects.ListProjects(opt)
95 if err != nil {
96 log.Fatal(err)
97 }
98
99 for _, project := range projects {
100
101 // Skip projects without a default branch - these should be projects
102 // where the repository isn't enabled
103 if project.DefaultBranch == "" {
104 continue
105 }
106
107 gitlabProjects = append(gitlabProjects, project)
108 }
109
110 if len(projects) == 0 {
111 break
112 }
113
114 opt.IDAfter = &projects[len(projects)-1].ID
115 }
116
117 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
118 if err != nil {
119 log.Fatal(err)
120 }
121
122 {
123 trimmed := gitlabProjects[:0]
124 for _, p := range gitlabProjects {
125 if filter.Include(p.NameWithNamespace) {
126 trimmed = append(trimmed, p)
127 }
128 }
129 gitlabProjects = trimmed
130 }
131
132 fetchProjects(destDir, apiToken, gitlabProjects)
133
134 if *deleteRepos {
135 if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil {
136 log.Fatalf("deleteStaleProjects: %v", err)
137 }
138 }
139}
140
141func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gitlab.Project) error {
142 u, err := url.Parse(projects[0].HTTPURLToRepo)
143 u.Path = ""
144 if err != nil {
145 return err
146 }
147
148 names := map[string]struct{}{}
149 for _, p := range projects {
150 u, err := url.Parse(p.HTTPURLToRepo)
151 if err != nil {
152 return err
153 }
154
155 names[filepath.Join(u.Host, u.Path)] = struct{}{}
156 }
157
158 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
159 log.Fatalf("deleteRepos: %v", err)
160 }
161 return nil
162}
163
164func fetchProjects(destDir, token string, projects []*gitlab.Project) {
165 for _, p := range projects {
166 u, err := url.Parse(p.HTTPURLToRepo)
167 if err != nil {
168 log.Printf("Unable to parse project URL: %v", err)
169 continue
170 }
171 config := map[string]string{
172 "zoekt.web-url-type": "gitlab",
173 "zoekt.web-url": p.WebURL,
174 "zoekt.name": filepath.Join(u.Hostname(), p.PathWithNamespace),
175
176 "zoekt.gitlab-stars": strconv.Itoa(p.StarCount),
177 "zoekt.gitlab-forks": strconv.Itoa(p.ForksCount),
178
179 "zoekt.archived": marshalBool(p.Archived),
180 "zoekt.fork": marshalBool(p.ForkedFromProject != nil),
181 "zoekt.public": marshalBool(p.Public),
182 }
183
184 cloneURL := p.HTTPURLToRepo
185 dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)
186 if err != nil {
187 log.Printf("cloneRepos: %v", err)
188 continue
189 }
190 if dest != "" {
191 fmt.Println(dest)
192 }
193 }
194}
195
196func marshalBool(b bool) string {
197 if b {
198 return "1"
199 }
200 return "0"
201}