fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "log"
22 "math/rand"
23 "net/http"
24 "net/url"
25 "os"
26 "os/exec"
27 "path/filepath"
28 "time"
29
30 "github.com/fsnotify/fsnotify"
31)
32
33type ConfigEntry struct {
34 GithubUser string
35 GithubOrg string
36 BitBucketServerProject string
37 GitHubURL string
38 GitilesURL string
39 CGitURL string
40 BitBucketServerURL string
41 DisableTLS bool
42 CredentialPath string
43 ProjectType string
44 Name string
45 Exclude string
46 GitLabURL string
47 OnlyPublic bool
48 GerritApiURL string
49 Topics []string
50 ExcludeTopics []string
51 Active bool
52 NoArchived bool
53 KeepDeleted bool
54 GerritFetchMetaConfig bool
55 GerritRepoNameFormat string
56 ExcludeUserRepos bool
57}
58
59func randomize(entries []ConfigEntry) []ConfigEntry {
60 perm := rand.Perm(len(entries))
61
62 var shuffled []ConfigEntry
63 for _, i := range perm {
64 shuffled = append(shuffled, entries[i])
65 }
66
67 return shuffled
68}
69
70func isHTTP(u string) bool {
71 asURL, err := url.Parse(u)
72 return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
73}
74
75func readConfigURL(u string) ([]ConfigEntry, error) {
76 var body []byte
77 var readErr error
78
79 if isHTTP(u) {
80 rep, err := http.Get(u)
81 if err != nil {
82 return nil, err
83 }
84 defer rep.Body.Close()
85
86 body, readErr = io.ReadAll(rep.Body)
87 } else {
88 body, readErr = os.ReadFile(u)
89 }
90
91 if readErr != nil {
92 return nil, readErr
93 }
94
95 var result []ConfigEntry
96 if err := json.Unmarshal(body, &result); err != nil {
97 return nil, err
98 }
99 return result, nil
100}
101
102func watchFile(path string) (<-chan struct{}, error) {
103 watcher, err := fsnotify.NewWatcher()
104 if err != nil {
105 return nil, err
106 }
107
108 if err := watcher.Add(filepath.Dir(path)); err != nil {
109 return nil, err
110 }
111
112 out := make(chan struct{}, 1)
113 go func() {
114 var last time.Time
115 for {
116 select {
117 case <-watcher.Events:
118 fi, err := os.Stat(path)
119 if err == nil && fi.ModTime() != last {
120 out <- struct{}{}
121 last = fi.ModTime()
122 }
123 case err := <-watcher.Errors:
124 if err != nil {
125 log.Printf("watcher error: %v", err)
126 }
127 }
128 }
129 }()
130 return out, nil
131}
132
133func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
134 ticker := time.NewTicker(opts.mirrorInterval)
135
136 var watcher <-chan struct{}
137 if !isHTTP(opts.mirrorConfigFile) {
138 var err error
139 watcher, err = watchFile(opts.mirrorConfigFile)
140 if err != nil {
141 log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
142 }
143 }
144
145 var lastCfg []ConfigEntry
146 for {
147 cfg, err := readConfigURL(opts.mirrorConfigFile)
148 if err != nil {
149 log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
150 } else {
151 lastCfg = cfg
152 }
153
154 executeMirror(lastCfg, repoDir, pendingRepos)
155
156 select {
157 case <-watcher:
158 log.Printf("mirror config %s changed", opts.mirrorConfigFile)
159 case <-ticker.C:
160 }
161 }
162}
163
164func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
165 // Randomize the ordering in which we query
166 // things. This is to ensure that quota limits don't
167 // always hit the last one in the list.
168 cfg = randomize(cfg)
169 for _, c := range cfg {
170 var cmd *exec.Cmd
171 if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
172 cmd = exec.Command("zoekt-mirror-github",
173 "-dest", repoDir)
174 if c.GitHubURL != "" {
175 cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
176 }
177 if c.GithubUser != "" {
178 cmd.Args = append(cmd.Args, "-user", c.GithubUser)
179 } else if c.GithubOrg != "" {
180 cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
181 }
182 if c.Name != "" {
183 cmd.Args = append(cmd.Args, "-name", c.Name)
184 }
185 if c.Exclude != "" {
186 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
187 }
188 if c.CredentialPath != "" {
189 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
190 }
191 for _, topic := range c.Topics {
192 cmd.Args = append(cmd.Args, "-topic", topic)
193 }
194 for _, topic := range c.ExcludeTopics {
195 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
196 }
197 if c.NoArchived {
198 cmd.Args = append(cmd.Args, "-no_archived")
199 }
200 if !c.KeepDeleted {
201 cmd.Args = append(cmd.Args, "-delete")
202 }
203 } else if c.GitilesURL != "" {
204 cmd = exec.Command("zoekt-mirror-gitiles",
205 "-dest", repoDir, "-name", c.Name)
206 if c.Exclude != "" {
207 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
208 }
209 cmd.Args = append(cmd.Args, c.GitilesURL)
210 } else if c.CGitURL != "" {
211 cmd = exec.Command("zoekt-mirror-gitiles",
212 "-type", "cgit",
213 "-dest", repoDir, "-name", c.Name)
214 if c.Exclude != "" {
215 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
216 }
217 cmd.Args = append(cmd.Args, c.CGitURL)
218 } else if c.BitBucketServerURL != "" {
219 cmd = exec.Command("zoekt-mirror-bitbucket-server",
220 "-dest", repoDir, "-url", c.BitBucketServerURL)
221 if c.BitBucketServerProject != "" {
222 cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
223 }
224 if c.DisableTLS {
225 cmd.Args = append(cmd.Args, "-disable-tls")
226 }
227 if c.ProjectType != "" {
228 cmd.Args = append(cmd.Args, "-type", c.ProjectType)
229 }
230 if c.Name != "" {
231 cmd.Args = append(cmd.Args, "-name", c.Name)
232 }
233 if c.Exclude != "" {
234 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
235 }
236 if c.CredentialPath != "" {
237 cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
238 }
239 if !c.KeepDeleted {
240 cmd.Args = append(cmd.Args, "-delete")
241 }
242 } else if c.GitLabURL != "" {
243 cmd = exec.Command("zoekt-mirror-gitlab",
244 "-dest", repoDir, "-url", c.GitLabURL)
245 if c.Name != "" {
246 cmd.Args = append(cmd.Args, "-name", c.Name)
247 }
248 if c.Exclude != "" {
249 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
250 }
251 if c.OnlyPublic {
252 cmd.Args = append(cmd.Args, "-public")
253 }
254 if c.ExcludeUserRepos {
255 cmd.Args = append(cmd.Args, "-exclude_user")
256 }
257 if c.CredentialPath != "" {
258 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
259 }
260 if c.NoArchived {
261 cmd.Args = append(cmd.Args, "-no_archived")
262 }
263 if !c.KeepDeleted {
264 cmd.Args = append(cmd.Args, "-delete")
265 }
266 } else if c.GerritApiURL != "" {
267 cmd = exec.Command("zoekt-mirror-gerrit",
268 "-dest", repoDir)
269 if c.CredentialPath != "" {
270 cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
271 }
272 if c.Name != "" {
273 cmd.Args = append(cmd.Args, "-name", c.Name)
274 }
275 if c.Exclude != "" {
276 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
277 }
278 if c.Active {
279 cmd.Args = append(cmd.Args, "-active")
280 }
281 if c.GerritFetchMetaConfig {
282 cmd.Args = append(cmd.Args, "-fetch-meta-config")
283 }
284 if c.GerritRepoNameFormat != "" {
285 cmd.Args = append(cmd.Args, "-repo-name-format", c.GerritRepoNameFormat)
286 }
287 if !c.KeepDeleted {
288 cmd.Args = append(cmd.Args, "-delete")
289 }
290 cmd.Args = append(cmd.Args, c.GerritApiURL)
291 } else {
292 log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c)
293 continue
294 }
295
296 stdout, _ := loggedRun(cmd)
297
298 for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
299 if len(fn) == 0 {
300 continue
301 }
302
303 pendingRepos <- string(fn)
304 }
305
306 }
307}