fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "log"
22 "math/rand"
23 "net/http"
24 "net/url"
25 "os"
26 "os/exec"
27 "path/filepath"
28 "time"
29
30 "github.com/fsnotify/fsnotify"
31)
32
33type ConfigEntry struct {
34 GithubUser string
35 GithubOrg string
36 BitBucketServerProject string
37 GitHubURL string
38 GitilesURL string
39 CGitURL string
40 BitBucketServerURL string
41 DisableTLS bool
42 CredentialPath string
43 ProjectType string
44 Name string
45 Exclude string
46 GitLabURL string
47 OnlyPublic bool
48 GerritApiURL string
49 Topics []string
50 ExcludeTopics []string
51 Active bool
52 NoArchived bool
53 GerritFetchMetaConfig bool
54 GerritRepoNameFormat string
55 ExcludeUserRepos bool
56}
57
58func randomize(entries []ConfigEntry) []ConfigEntry {
59 perm := rand.Perm(len(entries))
60
61 var shuffled []ConfigEntry
62 for _, i := range perm {
63 shuffled = append(shuffled, entries[i])
64 }
65
66 return shuffled
67}
68
69func isHTTP(u string) bool {
70 asURL, err := url.Parse(u)
71 return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
72}
73
74func readConfigURL(u string) ([]ConfigEntry, error) {
75 var body []byte
76 var readErr error
77
78 if isHTTP(u) {
79 rep, err := http.Get(u)
80 if err != nil {
81 return nil, err
82 }
83 defer rep.Body.Close()
84
85 body, readErr = io.ReadAll(rep.Body)
86 } else {
87 body, readErr = os.ReadFile(u)
88 }
89
90 if readErr != nil {
91 return nil, readErr
92 }
93
94 var result []ConfigEntry
95 if err := json.Unmarshal(body, &result); err != nil {
96 return nil, err
97 }
98 return result, nil
99}
100
101func watchFile(path string) (<-chan struct{}, error) {
102 watcher, err := fsnotify.NewWatcher()
103 if err != nil {
104 return nil, err
105 }
106
107 if err := watcher.Add(filepath.Dir(path)); err != nil {
108 return nil, err
109 }
110
111 out := make(chan struct{}, 1)
112 go func() {
113 var last time.Time
114 for {
115 select {
116 case <-watcher.Events:
117 fi, err := os.Stat(path)
118 if err == nil && fi.ModTime() != last {
119 out <- struct{}{}
120 last = fi.ModTime()
121 }
122 case err := <-watcher.Errors:
123 if err != nil {
124 log.Printf("watcher error: %v", err)
125 }
126 }
127 }
128 }()
129 return out, nil
130}
131
132func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
133 ticker := time.NewTicker(opts.mirrorInterval)
134
135 var watcher <-chan struct{}
136 if !isHTTP(opts.mirrorConfigFile) {
137 var err error
138 watcher, err = watchFile(opts.mirrorConfigFile)
139 if err != nil {
140 log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
141 }
142 }
143
144 var lastCfg []ConfigEntry
145 for {
146 cfg, err := readConfigURL(opts.mirrorConfigFile)
147 if err != nil {
148 log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
149 } else {
150 lastCfg = cfg
151 }
152
153 executeMirror(lastCfg, repoDir, pendingRepos)
154
155 select {
156 case <-watcher:
157 log.Printf("mirror config %s changed", opts.mirrorConfigFile)
158 case <-ticker.C:
159 }
160 }
161}
162
163func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
164 // Randomize the ordering in which we query
165 // things. This is to ensure that quota limits don't
166 // always hit the last one in the list.
167 cfg = randomize(cfg)
168 for _, c := range cfg {
169 var cmd *exec.Cmd
170 if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
171 cmd = exec.Command("zoekt-mirror-github",
172 "-dest", repoDir, "-delete")
173 if c.GitHubURL != "" {
174 cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
175 }
176 if c.GithubUser != "" {
177 cmd.Args = append(cmd.Args, "-user", c.GithubUser)
178 } else if c.GithubOrg != "" {
179 cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
180 }
181 if c.Name != "" {
182 cmd.Args = append(cmd.Args, "-name", c.Name)
183 }
184 if c.Exclude != "" {
185 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
186 }
187 if c.CredentialPath != "" {
188 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
189 }
190 for _, topic := range c.Topics {
191 cmd.Args = append(cmd.Args, "-topic", topic)
192 }
193 for _, topic := range c.ExcludeTopics {
194 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
195 }
196 if c.NoArchived {
197 cmd.Args = append(cmd.Args, "-no_archived")
198 }
199 } else if c.GitilesURL != "" {
200 cmd = exec.Command("zoekt-mirror-gitiles",
201 "-dest", repoDir, "-name", c.Name)
202 if c.Exclude != "" {
203 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
204 }
205 cmd.Args = append(cmd.Args, c.GitilesURL)
206 } else if c.CGitURL != "" {
207 cmd = exec.Command("zoekt-mirror-gitiles",
208 "-type", "cgit",
209 "-dest", repoDir, "-name", c.Name)
210 if c.Exclude != "" {
211 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
212 }
213 cmd.Args = append(cmd.Args, c.CGitURL)
214 } else if c.BitBucketServerURL != "" {
215 cmd = exec.Command("zoekt-mirror-bitbucket-server",
216 "-dest", repoDir, "-url", c.BitBucketServerURL, "-delete")
217 if c.BitBucketServerProject != "" {
218 cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
219 }
220 if c.DisableTLS {
221 cmd.Args = append(cmd.Args, "-disable-tls")
222 }
223 if c.ProjectType != "" {
224 cmd.Args = append(cmd.Args, "-type", c.ProjectType)
225 }
226 if c.Name != "" {
227 cmd.Args = append(cmd.Args, "-name", c.Name)
228 }
229 if c.Exclude != "" {
230 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
231 }
232 if c.CredentialPath != "" {
233 cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
234 }
235 } else if c.GitLabURL != "" {
236 cmd = exec.Command("zoekt-mirror-gitlab",
237 "-dest", repoDir, "-url", c.GitLabURL)
238 if c.Name != "" {
239 cmd.Args = append(cmd.Args, "-name", c.Name)
240 }
241 if c.Exclude != "" {
242 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
243 }
244 if c.OnlyPublic {
245 cmd.Args = append(cmd.Args, "-public")
246 }
247 if c.ExcludeUserRepos {
248 cmd.Args = append(cmd.Args, "-exclude_user")
249 }
250 if c.CredentialPath != "" {
251 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
252 }
253 } else if c.GerritApiURL != "" {
254 cmd = exec.Command("zoekt-mirror-gerrit",
255 "-dest", repoDir, "-delete")
256 if c.CredentialPath != "" {
257 cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
258 }
259 if c.Name != "" {
260 cmd.Args = append(cmd.Args, "-name", c.Name)
261 }
262 if c.Exclude != "" {
263 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
264 }
265 if c.Active {
266 cmd.Args = append(cmd.Args, "-active")
267 }
268 if c.GerritFetchMetaConfig {
269 cmd.Args = append(cmd.Args, "-fetch-meta-config")
270 }
271 if c.GerritRepoNameFormat != "" {
272 cmd.Args = append(cmd.Args, "-repo-name-format", c.GerritRepoNameFormat)
273 }
274 cmd.Args = append(cmd.Args, c.GerritApiURL)
275 } else {
276 log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c)
277 continue
278 }
279
280 stdout, _ := loggedRun(cmd)
281
282 for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
283 if len(fn) == 0 {
284 continue
285 }
286
287 pendingRepos <- string(fn)
288 }
289
290 }
291}