fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "log"
22 "math/rand"
23 "net/http"
24 "net/url"
25 "os"
26 "os/exec"
27 "path/filepath"
28 "time"
29
30 "github.com/fsnotify/fsnotify"
31)
32
33type ConfigEntry struct {
34 GithubUser string
35 GithubOrg string
36 BitBucketServerProject string
37 GitHubURL string
38 GitilesURL string
39 CGitURL string
40 BitBucketServerURL string
41 DisableTLS bool
42 CredentialPath string
43 ProjectType string
44 Name string
45 Exclude string
46 GitLabURL string
47 OnlyPublic bool
48 GerritApiURL string
49 Topics []string
50 ExcludeTopics []string
51 Active bool
52 NoArchived bool
53 GerritFetchMetaConfig bool
54 GerritRepoNameFormat string
55}
56
57func randomize(entries []ConfigEntry) []ConfigEntry {
58 perm := rand.Perm(len(entries))
59
60 var shuffled []ConfigEntry
61 for _, i := range perm {
62 shuffled = append(shuffled, entries[i])
63 }
64
65 return shuffled
66}
67
68func isHTTP(u string) bool {
69 asURL, err := url.Parse(u)
70 return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
71}
72
73func readConfigURL(u string) ([]ConfigEntry, error) {
74 var body []byte
75 var readErr error
76
77 if isHTTP(u) {
78 rep, err := http.Get(u)
79 if err != nil {
80 return nil, err
81 }
82 defer rep.Body.Close()
83
84 body, readErr = io.ReadAll(rep.Body)
85 } else {
86 body, readErr = os.ReadFile(u)
87 }
88
89 if readErr != nil {
90 return nil, readErr
91 }
92
93 var result []ConfigEntry
94 if err := json.Unmarshal(body, &result); err != nil {
95 return nil, err
96 }
97 return result, nil
98}
99
100func watchFile(path string) (<-chan struct{}, error) {
101 watcher, err := fsnotify.NewWatcher()
102 if err != nil {
103 return nil, err
104 }
105
106 if err := watcher.Add(filepath.Dir(path)); err != nil {
107 return nil, err
108 }
109
110 out := make(chan struct{}, 1)
111 go func() {
112 var last time.Time
113 for {
114 select {
115 case <-watcher.Events:
116 fi, err := os.Stat(path)
117 if err == nil && fi.ModTime() != last {
118 out <- struct{}{}
119 last = fi.ModTime()
120 }
121 case err := <-watcher.Errors:
122 if err != nil {
123 log.Printf("watcher error: %v", err)
124 }
125 }
126 }
127 }()
128 return out, nil
129}
130
131func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
132 ticker := time.NewTicker(opts.mirrorInterval)
133
134 var watcher <-chan struct{}
135 if !isHTTP(opts.mirrorConfigFile) {
136 var err error
137 watcher, err = watchFile(opts.mirrorConfigFile)
138 if err != nil {
139 log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
140 }
141 }
142
143 var lastCfg []ConfigEntry
144 for {
145 cfg, err := readConfigURL(opts.mirrorConfigFile)
146 if err != nil {
147 log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
148 } else {
149 lastCfg = cfg
150 }
151
152 executeMirror(lastCfg, repoDir, pendingRepos)
153
154 select {
155 case <-watcher:
156 log.Printf("mirror config %s changed", opts.mirrorConfigFile)
157 case <-ticker.C:
158 }
159 }
160}
161
162func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
163 // Randomize the ordering in which we query
164 // things. This is to ensure that quota limits don't
165 // always hit the last one in the list.
166 cfg = randomize(cfg)
167 for _, c := range cfg {
168 var cmd *exec.Cmd
169 if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
170 cmd = exec.Command("zoekt-mirror-github",
171 "-dest", repoDir, "-delete")
172 if c.GitHubURL != "" {
173 cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
174 }
175 if c.GithubUser != "" {
176 cmd.Args = append(cmd.Args, "-user", c.GithubUser)
177 } else if c.GithubOrg != "" {
178 cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
179 }
180 if c.Name != "" {
181 cmd.Args = append(cmd.Args, "-name", c.Name)
182 }
183 if c.Exclude != "" {
184 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
185 }
186 if c.CredentialPath != "" {
187 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
188 }
189 for _, topic := range c.Topics {
190 cmd.Args = append(cmd.Args, "-topic", topic)
191 }
192 for _, topic := range c.ExcludeTopics {
193 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
194 }
195 if c.NoArchived {
196 cmd.Args = append(cmd.Args, "-no_archived")
197 }
198 } else if c.GitilesURL != "" {
199 cmd = exec.Command("zoekt-mirror-gitiles",
200 "-dest", repoDir, "-name", c.Name)
201 if c.Exclude != "" {
202 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
203 }
204 cmd.Args = append(cmd.Args, c.GitilesURL)
205 } else if c.CGitURL != "" {
206 cmd = exec.Command("zoekt-mirror-gitiles",
207 "-type", "cgit",
208 "-dest", repoDir, "-name", c.Name)
209 if c.Exclude != "" {
210 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
211 }
212 cmd.Args = append(cmd.Args, c.CGitURL)
213 } else if c.BitBucketServerURL != "" {
214 cmd = exec.Command("zoekt-mirror-bitbucket-server",
215 "-dest", repoDir, "-url", c.BitBucketServerURL, "-delete")
216 if c.BitBucketServerProject != "" {
217 cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
218 }
219 if c.DisableTLS {
220 cmd.Args = append(cmd.Args, "-disable-tls")
221 }
222 if c.ProjectType != "" {
223 cmd.Args = append(cmd.Args, "-type", c.ProjectType)
224 }
225 if c.Name != "" {
226 cmd.Args = append(cmd.Args, "-name", c.Name)
227 }
228 if c.Exclude != "" {
229 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
230 }
231 if c.CredentialPath != "" {
232 cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
233 }
234 } else if c.GitLabURL != "" {
235 cmd = exec.Command("zoekt-mirror-gitlab",
236 "-dest", repoDir, "-url", c.GitLabURL)
237 if c.Name != "" {
238 cmd.Args = append(cmd.Args, "-name", c.Name)
239 }
240 if c.Exclude != "" {
241 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
242 }
243 if c.OnlyPublic {
244 cmd.Args = append(cmd.Args, "-public")
245 }
246 if c.CredentialPath != "" {
247 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
248 }
249 } else if c.GerritApiURL != "" {
250 cmd = exec.Command("zoekt-mirror-gerrit",
251 "-dest", repoDir, "-delete")
252 if c.CredentialPath != "" {
253 cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
254 }
255 if c.Name != "" {
256 cmd.Args = append(cmd.Args, "-name", c.Name)
257 }
258 if c.Exclude != "" {
259 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
260 }
261 if c.Active {
262 cmd.Args = append(cmd.Args, "-active")
263 }
264 if c.GerritFetchMetaConfig {
265 cmd.Args = append(cmd.Args, "-fetch-meta-config")
266 }
267 if c.GerritRepoNameFormat != "" {
268 cmd.Args = append(cmd.Args, "-repo-name-format", c.GerritRepoNameFormat)
269 }
270 cmd.Args = append(cmd.Args, c.GerritApiURL)
271 } else {
272 log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c)
273 continue
274 }
275
276 stdout, _ := loggedRun(cmd)
277
278 for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
279 if len(fn) == 0 {
280 continue
281 }
282
283 pendingRepos <- string(fn)
284 }
285
286 }
287}