fork of https://github.com/sourcegraph/zoekt
1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "encoding/json"
20 "io"
21 "net/http"
22 "net/url"
23 "path"
24)
25
26type Project struct {
27 Name string
28 CloneURL string `json:"clone_url"`
29}
30
31func getGitilesRepos(root *url.URL, filter func(string) bool) (map[string]*crawlTarget, error) {
32 jsRoot := *root
33 jsRoot.RawQuery = "format=JSON"
34 resp, err := http.Get(jsRoot.String())
35 if err != nil {
36 return nil, err
37 }
38 defer resp.Body.Close()
39
40 content, err := io.ReadAll(resp.Body)
41 if err != nil {
42 return nil, err
43 }
44
45 const xssTag = ")]}'\n"
46 content = bytes.TrimPrefix(content, []byte(xssTag))
47
48 m := map[string]*Project{}
49 if err := json.Unmarshal(content, &m); err != nil {
50 return nil, err
51 }
52
53 result := map[string]*crawlTarget{}
54 for k, v := range m {
55 if k == "All-Users" || k == "All-Projects" {
56 continue
57 }
58 if !filter(k) {
59 continue
60 }
61 web := *root
62 web.Path = path.Join(web.Path, v.Name)
63 result[k] = &crawlTarget{
64 cloneURL: v.CloneURL,
65 webURL: web.String(),
66 webURLType: "gitiles",
67 }
68 }
69 return result, nil
70}