fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "log"
22 "math/rand"
23 "net/http"
24 "net/url"
25 "os"
26 "os/exec"
27 "path/filepath"
28 "time"
29
30 "github.com/fsnotify/fsnotify"
31)
32
33type ConfigEntry struct {
34 GithubUser string
35 GithubOrg string
36 BitBucketServerProject string
37 GitHubURL string
38 GitilesURL string
39 CGitURL string
40 BitBucketServerURL string
41 GiteaURL string
42 GiteaUser string
43 GiteaOrg string
44 DisableTLS bool
45 CredentialPath string
46 ProjectType string
47 Name string
48 Exclude string
49 GitLabURL string
50 OnlyPublic bool
51 GerritApiURL string
52 Topics []string
53 ExcludeTopics []string
54 Active bool
55 NoArchived bool
56 KeepDeleted bool
57 GerritFetchMetaConfig bool
58 GerritRepoNameFormat string
59 ExcludeUserRepos bool
60 Forks bool
61}
62
63func randomize(entries []ConfigEntry) []ConfigEntry {
64 perm := rand.Perm(len(entries))
65
66 var shuffled []ConfigEntry
67 for _, i := range perm {
68 shuffled = append(shuffled, entries[i])
69 }
70
71 return shuffled
72}
73
74func isHTTP(u string) bool {
75 asURL, err := url.Parse(u)
76 return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
77}
78
79func readConfigURL(u string) ([]ConfigEntry, error) {
80 var body []byte
81 var readErr error
82
83 if isHTTP(u) {
84 rep, err := http.Get(u)
85 if err != nil {
86 return nil, err
87 }
88 defer rep.Body.Close()
89
90 body, readErr = io.ReadAll(rep.Body)
91 } else {
92 body, readErr = os.ReadFile(u)
93 }
94
95 if readErr != nil {
96 return nil, readErr
97 }
98
99 var result []ConfigEntry
100 if err := json.Unmarshal(body, &result); err != nil {
101 return nil, err
102 }
103 return result, nil
104}
105
106func watchFile(path string) (<-chan struct{}, error) {
107 watcher, err := fsnotify.NewWatcher()
108 if err != nil {
109 return nil, err
110 }
111
112 if err := watcher.Add(filepath.Dir(path)); err != nil {
113 return nil, err
114 }
115
116 out := make(chan struct{}, 1)
117 go func() {
118 var last time.Time
119 for {
120 select {
121 case <-watcher.Events:
122 fi, err := os.Stat(path)
123 if err == nil && fi.ModTime() != last {
124 out <- struct{}{}
125 last = fi.ModTime()
126 }
127 case err := <-watcher.Errors:
128 if err != nil {
129 log.Printf("watcher error: %v", err)
130 }
131 }
132 }
133 }()
134 return out, nil
135}
136
137func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
138 ticker := time.NewTicker(opts.mirrorInterval)
139
140 var watcher <-chan struct{}
141 if !isHTTP(opts.mirrorConfigFile) {
142 var err error
143 watcher, err = watchFile(opts.mirrorConfigFile)
144 if err != nil {
145 log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
146 }
147 }
148
149 var lastCfg []ConfigEntry
150 for {
151 cfg, err := readConfigURL(opts.mirrorConfigFile)
152 if err != nil {
153 log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
154 } else {
155 lastCfg = cfg
156 }
157
158 executeMirror(lastCfg, repoDir, pendingRepos)
159
160 select {
161 case <-watcher:
162 log.Printf("mirror config %s changed", opts.mirrorConfigFile)
163 case <-ticker.C:
164 }
165 }
166}
167
168func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
169 // Randomize the ordering in which we query
170 // things. This is to ensure that quota limits don't
171 // always hit the last one in the list.
172 cfg = randomize(cfg)
173 for _, c := range cfg {
174 var cmd *exec.Cmd
175 if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
176 cmd = exec.Command("zoekt-mirror-github",
177 "-dest", repoDir)
178 if c.GitHubURL != "" {
179 cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
180 }
181 if c.GithubUser != "" {
182 cmd.Args = append(cmd.Args, "-user", c.GithubUser)
183 } else if c.GithubOrg != "" {
184 cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
185 }
186 if c.Name != "" {
187 cmd.Args = append(cmd.Args, "-name", c.Name)
188 }
189 if c.Exclude != "" {
190 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
191 }
192 if c.CredentialPath != "" {
193 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
194 }
195 for _, topic := range c.Topics {
196 cmd.Args = append(cmd.Args, "-topic", topic)
197 }
198 for _, topic := range c.ExcludeTopics {
199 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
200 }
201 if c.NoArchived {
202 cmd.Args = append(cmd.Args, "-no_archived")
203 }
204 if !c.KeepDeleted {
205 cmd.Args = append(cmd.Args, "-delete")
206 }
207 if c.Forks {
208 cmd.Args = append(cmd.Args, "-forks")
209 }
210 } else if c.GitilesURL != "" {
211 cmd = exec.Command("zoekt-mirror-gitiles",
212 "-dest", repoDir, "-name", c.Name)
213 if c.Exclude != "" {
214 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
215 }
216 cmd.Args = append(cmd.Args, c.GitilesURL)
217 } else if c.CGitURL != "" {
218 cmd = exec.Command("zoekt-mirror-gitiles",
219 "-type", "cgit",
220 "-dest", repoDir, "-name", c.Name)
221 if c.Exclude != "" {
222 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
223 }
224 cmd.Args = append(cmd.Args, c.CGitURL)
225 } else if c.BitBucketServerURL != "" {
226 cmd = exec.Command("zoekt-mirror-bitbucket-server",
227 "-dest", repoDir, "-url", c.BitBucketServerURL)
228 if c.BitBucketServerProject != "" {
229 cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
230 }
231 if c.DisableTLS {
232 cmd.Args = append(cmd.Args, "-disable-tls")
233 }
234 if c.ProjectType != "" {
235 cmd.Args = append(cmd.Args, "-type", c.ProjectType)
236 }
237 if c.Name != "" {
238 cmd.Args = append(cmd.Args, "-name", c.Name)
239 }
240 if c.Exclude != "" {
241 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
242 }
243 if c.CredentialPath != "" {
244 cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
245 }
246 if !c.KeepDeleted {
247 cmd.Args = append(cmd.Args, "-delete")
248 }
249 } else if c.GitLabURL != "" {
250 cmd = exec.Command("zoekt-mirror-gitlab",
251 "-dest", repoDir, "-url", c.GitLabURL)
252 if c.Name != "" {
253 cmd.Args = append(cmd.Args, "-name", c.Name)
254 }
255 if c.Exclude != "" {
256 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
257 }
258 if c.OnlyPublic {
259 cmd.Args = append(cmd.Args, "-public")
260 }
261 if c.ExcludeUserRepos {
262 cmd.Args = append(cmd.Args, "-exclude_user")
263 }
264 if c.CredentialPath != "" {
265 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
266 }
267 if c.NoArchived {
268 cmd.Args = append(cmd.Args, "-no_archived")
269 }
270 if !c.KeepDeleted {
271 cmd.Args = append(cmd.Args, "-delete")
272 }
273 } else if c.GerritApiURL != "" {
274 cmd = exec.Command("zoekt-mirror-gerrit",
275 "-dest", repoDir)
276 if c.CredentialPath != "" {
277 cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
278 }
279 if c.Name != "" {
280 cmd.Args = append(cmd.Args, "-name", c.Name)
281 }
282 if c.Exclude != "" {
283 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
284 }
285 if c.Active {
286 cmd.Args = append(cmd.Args, "-active")
287 }
288 if c.GerritFetchMetaConfig {
289 cmd.Args = append(cmd.Args, "-fetch-meta-config")
290 }
291 if c.GerritRepoNameFormat != "" {
292 cmd.Args = append(cmd.Args, "-repo-name-format", c.GerritRepoNameFormat)
293 }
294 if !c.KeepDeleted {
295 cmd.Args = append(cmd.Args, "-delete")
296 }
297 cmd.Args = append(cmd.Args, c.GerritApiURL)
298 } else if c.GiteaURL != "" {
299 cmd = exec.Command("zoekt-mirror-gitea", "-dest", repoDir)
300 if c.GiteaURL != "" {
301 cmd.Args = append(cmd.Args, "-url", c.GiteaURL)
302 }
303 if c.GiteaUser != "" {
304 cmd.Args = append(cmd.Args, "-user", c.GiteaUser)
305 } else if c.GiteaOrg != "" {
306 cmd.Args = append(cmd.Args, "-org", c.GiteaOrg)
307 }
308 if c.Name != "" {
309 cmd.Args = append(cmd.Args, "-name", c.Name)
310 }
311 if c.Exclude != "" {
312 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
313 }
314 if c.CredentialPath != "" {
315 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
316 }
317 for _, topic := range c.Topics {
318 cmd.Args = append(cmd.Args, "-topic", topic)
319 }
320 for _, topic := range c.ExcludeTopics {
321 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
322 }
323 if c.NoArchived {
324 cmd.Args = append(cmd.Args, "-no_archived")
325 }
326 if !c.KeepDeleted {
327 cmd.Args = append(cmd.Args, "-delete")
328 }
329 if c.Forks {
330 cmd.Args = append(cmd.Args, "-forks")
331 }
332 } else {
333 log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c)
334 continue
335 }
336
337 stdout, _ := loggedRun(cmd)
338
339 for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
340 if len(fn) == 0 {
341 continue
342 }
343
344 pendingRepos <- string(fn)
345 }
346
347 }
348}