fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "log"
22 "math/rand"
23 "net/http"
24 "net/url"
25 "os"
26 "os/exec"
27 "path/filepath"
28 "time"
29
30 "github.com/fsnotify/fsnotify"
31)
32
33type ConfigEntry struct {
34 GithubUser string
35 GithubOrg string
36 BitBucketServerProject string
37 GitHubURL string
38 GitilesURL string
39 CGitURL string
40 BitBucketServerURL string
41 DisableTLS bool
42 CredentialPath string
43 ProjectType string
44 Name string
45 Exclude string
46 GitLabURL string
47 OnlyPublic bool
48 GerritApiURL string
49 Topics []string
50 ExcludeTopics []string
51 Active bool
52 NoArchived bool
53}
54
55func randomize(entries []ConfigEntry) []ConfigEntry {
56 perm := rand.Perm(len(entries))
57
58 var shuffled []ConfigEntry
59 for _, i := range perm {
60 shuffled = append(shuffled, entries[i])
61 }
62
63 return shuffled
64}
65
66func isHTTP(u string) bool {
67 asURL, err := url.Parse(u)
68 return err == nil && (asURL.Scheme == "http" || asURL.Scheme == "https")
69}
70
71func readConfigURL(u string) ([]ConfigEntry, error) {
72 var body []byte
73 var readErr error
74
75 if isHTTP(u) {
76 rep, err := http.Get(u)
77 if err != nil {
78 return nil, err
79 }
80 defer rep.Body.Close()
81
82 body, readErr = io.ReadAll(rep.Body)
83 } else {
84 body, readErr = os.ReadFile(u)
85 }
86
87 if readErr != nil {
88 return nil, readErr
89 }
90
91 var result []ConfigEntry
92 if err := json.Unmarshal(body, &result); err != nil {
93 return nil, err
94 }
95 return result, nil
96}
97
98func watchFile(path string) (<-chan struct{}, error) {
99 watcher, err := fsnotify.NewWatcher()
100 if err != nil {
101 return nil, err
102 }
103
104 if err := watcher.Add(filepath.Dir(path)); err != nil {
105 return nil, err
106 }
107
108 out := make(chan struct{}, 1)
109 go func() {
110 var last time.Time
111 for {
112 select {
113 case <-watcher.Events:
114 fi, err := os.Stat(path)
115 if err == nil && fi.ModTime() != last {
116 out <- struct{}{}
117 last = fi.ModTime()
118 }
119 case err := <-watcher.Errors:
120 if err != nil {
121 log.Printf("watcher error: %v", err)
122 }
123 }
124 }
125 }()
126 return out, nil
127}
128
129func periodicMirrorFile(repoDir string, opts *Options, pendingRepos chan<- string) {
130 ticker := time.NewTicker(opts.mirrorInterval)
131
132 var watcher <-chan struct{}
133 if !isHTTP(opts.mirrorConfigFile) {
134 var err error
135 watcher, err = watchFile(opts.mirrorConfigFile)
136 if err != nil {
137 log.Printf("watchFile(%q): %v", opts.mirrorConfigFile, err)
138 }
139 }
140
141 var lastCfg []ConfigEntry
142 for {
143 cfg, err := readConfigURL(opts.mirrorConfigFile)
144 if err != nil {
145 log.Printf("readConfig(%s): %v", opts.mirrorConfigFile, err)
146 } else {
147 lastCfg = cfg
148 }
149
150 executeMirror(lastCfg, repoDir, pendingRepos)
151
152 select {
153 case <-watcher:
154 log.Printf("mirror config %s changed", opts.mirrorConfigFile)
155 case <-ticker.C:
156 }
157 }
158}
159
160func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string) {
161 // Randomize the ordering in which we query
162 // things. This is to ensure that quota limits don't
163 // always hit the last one in the list.
164 cfg = randomize(cfg)
165 for _, c := range cfg {
166 var cmd *exec.Cmd
167 if c.GitHubURL != "" || c.GithubUser != "" || c.GithubOrg != "" {
168 cmd = exec.Command("zoekt-mirror-github",
169 "-dest", repoDir, "-delete")
170 if c.GitHubURL != "" {
171 cmd.Args = append(cmd.Args, "-url", c.GitHubURL)
172 }
173 if c.GithubUser != "" {
174 cmd.Args = append(cmd.Args, "-user", c.GithubUser)
175 } else if c.GithubOrg != "" {
176 cmd.Args = append(cmd.Args, "-org", c.GithubOrg)
177 }
178 if c.Name != "" {
179 cmd.Args = append(cmd.Args, "-name", c.Name)
180 }
181 if c.Exclude != "" {
182 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
183 }
184 if c.CredentialPath != "" {
185 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
186 }
187 for _, topic := range c.Topics {
188 cmd.Args = append(cmd.Args, "-topic", topic)
189 }
190 for _, topic := range c.ExcludeTopics {
191 cmd.Args = append(cmd.Args, "-exclude_topic", topic)
192 }
193 if c.NoArchived {
194 cmd.Args = append(cmd.Args, "-no_archived")
195 }
196 } else if c.GitilesURL != "" {
197 cmd = exec.Command("zoekt-mirror-gitiles",
198 "-dest", repoDir, "-name", c.Name)
199 if c.Exclude != "" {
200 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
201 }
202 cmd.Args = append(cmd.Args, c.GitilesURL)
203 } else if c.CGitURL != "" {
204 cmd = exec.Command("zoekt-mirror-gitiles",
205 "-type", "cgit",
206 "-dest", repoDir, "-name", c.Name)
207 if c.Exclude != "" {
208 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
209 }
210 cmd.Args = append(cmd.Args, c.CGitURL)
211 } else if c.BitBucketServerURL != "" {
212 cmd = exec.Command("zoekt-mirror-bitbucket-server",
213 "-dest", repoDir, "-url", c.BitBucketServerURL, "-delete")
214 if c.BitBucketServerProject != "" {
215 cmd.Args = append(cmd.Args, "-project", c.BitBucketServerProject)
216 }
217 if c.DisableTLS {
218 cmd.Args = append(cmd.Args, "-disable-tls")
219 }
220 if c.ProjectType != "" {
221 cmd.Args = append(cmd.Args, "-type", c.ProjectType)
222 }
223 if c.Name != "" {
224 cmd.Args = append(cmd.Args, "-name", c.Name)
225 }
226 if c.Exclude != "" {
227 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
228 }
229 if c.CredentialPath != "" {
230 cmd.Args = append(cmd.Args, "-credentials", c.CredentialPath)
231 }
232 } else if c.GitLabURL != "" {
233 cmd = exec.Command("zoekt-mirror-gitlab",
234 "-dest", repoDir, "-url", c.GitLabURL)
235 if c.Name != "" {
236 cmd.Args = append(cmd.Args, "-name", c.Name)
237 }
238 if c.Exclude != "" {
239 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
240 }
241 if c.OnlyPublic {
242 cmd.Args = append(cmd.Args, "-public")
243 }
244 if c.CredentialPath != "" {
245 cmd.Args = append(cmd.Args, "-token", c.CredentialPath)
246 }
247 } else if c.GerritApiURL != "" {
248 cmd = exec.Command("zoekt-mirror-gerrit",
249 "-dest", repoDir, "-delete")
250 if c.CredentialPath != "" {
251 cmd.Args = append(cmd.Args, "-http-credentials", c.CredentialPath)
252 }
253 if c.Name != "" {
254 cmd.Args = append(cmd.Args, "-name", c.Name)
255 }
256 if c.Exclude != "" {
257 cmd.Args = append(cmd.Args, "-exclude", c.Exclude)
258 }
259 if c.Active {
260 cmd.Args = append(cmd.Args, "-active")
261 }
262 cmd.Args = append(cmd.Args, c.GerritApiURL)
263 } else {
264 log.Printf("executeMirror: ignoring config, because it does not contain any valid repository definition: %v", c)
265 continue
266 }
267
268 stdout, _ := loggedRun(cmd)
269
270 for _, fn := range bytes.Split(stdout, []byte{'\n'}) {
271 if len(fn) == 0 {
272 continue
273 }
274
275 pendingRepos <- string(fn)
276 }
277
278 }
279}