fork of https://github.com/sourcegraph/zoekt
1// Licensed under the Apache License, Version 2.0 (the "License");
2// you may not use this file except in compliance with the License.
3// You may obtain a copy of the License at
4//
5// http://www.apache.org/licenses/LICENSE-2.0
6//
7// Unless required by applicable law or agreed to in writing, software
8// distributed under the License is distributed on an "AS IS" BASIS,
9// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10// See the License for the specific language governing permissions and
11// limitations under the License.
12
13// Command zoekt-mirror-gitlab fetches all repos for a user from gitlab.
14//
15// It is recommended to use a gitlab personal access token:
16// https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This
17// token should be stored in a file and the --token option should be used.
18// In addition, the token should be present in the ~/.netrc of the user running
19// Command mirror. For example, the ~/.netrc may look like:
20//
21// machine gitlab.com
22// login oauth
23// password <personal access token>
24package main
25
26import (
27 "flag"
28 "fmt"
29 "log"
30 "net/url"
31 "os"
32 "path/filepath"
33 "strconv"
34 "strings"
35 "time"
36
37 gitlab "github.com/xanzy/go-gitlab"
38
39 "github.com/sourcegraph/zoekt/internal/gitindex"
40)
41
42func main() {
43 dest := flag.String("dest", "", "destination directory")
44 gitlabURL := flag.String("url", "https://gitlab.com/api/v4/", "Gitlab URL. If not set https://gitlab.com/api/v4/ will be used")
45 token := flag.String("token",
46 filepath.Join(os.Getenv("HOME"), ".gitlab-token"),
47 "file holding API token.")
48 isMember := flag.Bool("membership", false, "only mirror repos this user is a member of ")
49 isPublic := flag.Bool("public", false, "only mirror public repos")
50 deleteRepos := flag.Bool("delete", false, "delete missing repos")
51 excludeUserRepos := flag.Bool("exclude_user", false, "exclude user repos")
52 namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
53 excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
54 lastActivityAfter := flag.String("last_activity_after", "", "only mirror repos that have been active since this date (format: 2006-01-02).")
55 noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
56
57 flag.Parse()
58
59 if *dest == "" {
60 log.Fatal("must set --dest")
61 }
62
63 var host string
64 rootURL, err := url.Parse(*gitlabURL)
65 if err != nil {
66 log.Fatal(err)
67 }
68 host = rootURL.Host
69
70 destDir := filepath.Join(*dest, host)
71 if err := os.MkdirAll(destDir, 0o755); err != nil {
72 log.Fatal(err)
73 }
74
75 content, err := os.ReadFile(*token)
76 if err != nil {
77 log.Fatal(err)
78 }
79 apiToken := strings.TrimSpace(string(content))
80
81 client, err := gitlab.NewClient(apiToken, gitlab.WithBaseURL(*gitlabURL))
82 if err != nil {
83 log.Fatal(err)
84 }
85
86 opt := &gitlab.ListProjectsOptions{
87 ListOptions: gitlab.ListOptions{
88 PerPage: 100,
89 },
90 Sort: gitlab.String("asc"),
91 OrderBy: gitlab.String("id"),
92 Membership: isMember,
93 }
94 if *isPublic {
95 opt.Visibility = gitlab.Visibility(gitlab.PublicVisibility)
96 }
97
98 if *lastActivityAfter != "" {
99 targetDate, err := time.Parse("2006-01-02", *lastActivityAfter)
100 if err != nil {
101 log.Fatal(err)
102 }
103 opt.LastActivityAfter = gitlab.Time(targetDate)
104 }
105
106 if *noArchived {
107 opt.Archived = gitlab.Bool(false)
108 }
109
110 var gitlabProjects []*gitlab.Project
111 for {
112 projects, _, err := client.Projects.ListProjects(opt)
113 if err != nil {
114 log.Fatal(err)
115 }
116
117 for _, project := range projects {
118
119 // Skip projects without a default branch - these should be projects
120 // where the repository isn't enabled
121 if project.DefaultBranch == "" {
122 continue
123 }
124 if *excludeUserRepos && project.Namespace.Kind == "user" {
125 continue
126 }
127
128 gitlabProjects = append(gitlabProjects, project)
129 }
130
131 if len(projects) == 0 {
132 break
133 }
134
135 opt.IDAfter = &projects[len(projects)-1].ID
136 }
137
138 filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
139 if err != nil {
140 log.Fatal(err)
141 }
142
143 {
144 trimmed := gitlabProjects[:0]
145 for _, p := range gitlabProjects {
146 if filter.Include(p.NameWithNamespace) {
147 trimmed = append(trimmed, p)
148 }
149 }
150 gitlabProjects = trimmed
151 }
152 fetchProjects(destDir, apiToken, gitlabProjects)
153
154 if *deleteRepos {
155 if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil {
156 log.Fatalf("deleteStaleProjects: %v", err)
157 }
158 }
159}
160
161func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gitlab.Project) error {
162 u, err := url.Parse(projects[0].HTTPURLToRepo)
163 u.Path = ""
164 if err != nil {
165 return err
166 }
167
168 names := map[string]struct{}{}
169 for _, p := range projects {
170 u, err := url.Parse(p.HTTPURLToRepo)
171 if err != nil {
172 return err
173 }
174
175 names[filepath.Join(u.Host, u.Path)] = struct{}{}
176 }
177
178 if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
179 log.Fatalf("deleteRepos: %v", err)
180 }
181 return nil
182}
183
184func fetchProjects(destDir, token string, projects []*gitlab.Project) {
185 for _, p := range projects {
186 u, err := url.Parse(p.HTTPURLToRepo)
187 if err != nil {
188 log.Printf("Unable to parse project URL: %v", err)
189 continue
190 }
191 config := map[string]string{
192 "zoekt.web-url-type": "gitlab",
193 "zoekt.web-url": p.WebURL,
194 "zoekt.name": filepath.Join(u.Hostname(), p.PathWithNamespace),
195
196 "zoekt.gitlab-stars": strconv.Itoa(p.StarCount),
197 "zoekt.gitlab-forks": strconv.Itoa(p.ForksCount),
198
199 "zoekt.archived": marshalBool(p.Archived),
200 "zoekt.fork": marshalBool(p.ForkedFromProject != nil),
201 "zoekt.public": marshalBool(p.Visibility == gitlab.PublicVisibility),
202 }
203
204 cloneURL := p.HTTPURLToRepo
205 dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)
206 if err != nil {
207 log.Printf("cloneRepos: %v", err)
208 continue
209 }
210 if dest != "" {
211 fmt.Println(dest)
212 }
213 }
214}
215
216func marshalBool(b bool) string {
217 if b {
218 return "1"
219 }
220 return "0"
221}