fork of https://github.com/sourcegraph/zoekt
zoekt-mirror-gerrit: delete stale repos (#282)
This change adds an option on gerrit mirror to delete repos that don't match mirror criteria anymore.
It also changes the DeleteRepos to handle repository names with multiple "/".
author
df11
committer
GitHub
date
(Mar 17, 2022, 9:32 AM +0100)
commit
9c8ed6f1
9c8ed6f10bc24b3b93686b25bd00461024ddaeb8
parent
acf74e5f
acf74e5fb08ad45a4741c0eeb8e87f7053cd67c6
+72
-17
+1
-1
cmd/zoekt-indexserver/config.go
+1
-1
cmd/zoekt-indexserver/config.go
+28
-1
cmd/zoekt-mirror-gerrit/main.go
+28
-1
cmd/zoekt-mirror-gerrit/main.go
···
76
76
dest := flag.String("dest", "", "destination directory")
77
77
namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
78
78
excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
79
+
deleteRepos := flag.Bool("delete", false, "delete missing repos")
79
80
httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
80
81
active := flag.Bool("active", false, "mirror only active projects")
81
82
flag.Parse()
···
154
155
continue
155
156
}
156
157
157
-
cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, -1))
158
+
cloneURL, err := url.Parse(strings.Replace(projectURL, "${project}", k, 1))
158
159
if err != nil {
159
160
log.Fatalf("url.Parse: %v", err)
160
161
}
···
186
187
fmt.Println(dest)
187
188
}
188
189
}
190
+
if *deleteRepos {
191
+
if err := deleteStaleRepos(*dest, filter, projects, projectURL); err != nil {
192
+
log.Fatalf("deleteStaleRepos: %v", err)
193
+
}
194
+
}
195
+
}
196
+
197
+
func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos map[string]gerrit.ProjectInfo, projectURL string) error {
198
+
u, err := url.Parse(strings.Replace(projectURL, "${project}", "", 1))
199
+
if err != nil {
200
+
return err
201
+
}
202
+
203
+
names := map[string]struct{}{}
204
+
for name, _ := range repos {
205
+
u, err := url.Parse(strings.Replace(projectURL, "${project}", name, 1))
206
+
if err != nil {
207
+
return err
208
+
}
209
+
names[filepath.Join(u.Host, u.Path) + ".git"] = struct{}{}
210
+
}
211
+
212
+
if err := gitindex.DeleteRepos(destDir, u, names, filter); err != nil {
213
+
log.Fatalf("deleteRepos: %v", err)
214
+
}
215
+
return nil
189
216
}
+4
-1
gitindex/delete.go
+4
-1
gitindex/delete.go
···
6
6
"net/url"
7
7
"os"
8
8
"path/filepath"
9
+
"strings"
9
10
)
10
11
11
12
// DeleteRepos deletes stale repos under a specific path in disk. The `names`
···
19
20
var toDelete []string
20
21
for _, p := range paths {
21
22
_, exists := names[p]
22
-
if filter.Include(filepath.Base(p)) && !exists {
23
+
repoName := strings.Replace(p, filepath.Join(urlPrefix.Host, urlPrefix.Path), "", 1)
24
+
repoName = strings.TrimPrefix(repoName, "/")
25
+
if filter.Include(repoName) && !exists {
23
26
toDelete = append(toDelete, p)
24
27
}
25
28
}
+26
-9
gitindex/delete_test.go
+26
-9
gitindex/delete_test.go
···
36
36
}
37
37
38
38
wantBefore := map[string]struct{}{
39
-
"gerrit.googlesource.com/bdir.git": {},
40
-
"gerrit.googlesource.com/sub/bdir.git": {},
41
-
"adir/.git": {},
42
-
"bdir/.git": {},
43
-
"gerrit.googlesource.com/adir.git": {},
39
+
"adir/.git": {},
40
+
"bdir/.git": {},
41
+
"gerrit.googlesource.com/adir.git": {},
42
+
"gerrit.googlesource.com/bdir.git": {},
43
+
"gerrit.googlesource.com/sub/bdir.git": {},
44
+
"gerrit.googlesource.com/team/scope/repoa.git": {},
45
+
"gerrit.googlesource.com/team/scope/repob.git": {},
44
46
}
45
47
46
48
if !reflect.DeepEqual(gotBefore, wantBefore) {
···
59
61
if err != nil {
60
62
t.Fatalf("DeleteRepos: %T", err)
61
63
}
64
+
65
+
bURL, _ := url.Parse("http://gerrit.googlesource.com")
66
+
bURL.Path = ""
67
+
names = map[string]struct{}{
68
+
"gerrit.googlesource.com/adir.git": {},
69
+
"gerrit.googlesource.com/bdir.git": {},
70
+
"gerrit.googlesource.com/team/scope/repob.git": {},
71
+
}
72
+
73
+
err = DeleteRepos(dir, bURL, names, filter)
74
+
if err != nil {
75
+
t.Fatalf("DeleteRepos: %T", err)
76
+
}
77
+
62
78
reposAfter, err := FindGitRepos(dir)
63
79
if err != nil {
64
80
t.Error("FindGitRepos", err)
···
74
90
gotAfter[p] = struct{}{}
75
91
}
76
92
wantAfter := map[string]struct{}{
77
-
"gerrit.googlesource.com/bdir.git": {},
78
-
"adir/.git": {},
79
-
"bdir/.git": {},
80
-
"gerrit.googlesource.com/adir.git": {},
93
+
"adir/.git": {},
94
+
"bdir/.git": {},
95
+
"gerrit.googlesource.com/adir.git": {},
96
+
"gerrit.googlesource.com/bdir.git": {},
97
+
"gerrit.googlesource.com/team/scope/repob.git": {},
81
98
}
82
99
83
100
if !reflect.DeepEqual(gotAfter, wantAfter) {
+2
gitindex/repocache_test.go
+2
gitindex/repocache_test.go
+11
-5
gitindex/tree_test.go
+11
-5
gitindex/tree_test.go
···
74
74
mkdir gerrit.googlesource.com/sub
75
75
git clone --bare bdir gerrit.googlesource.com/sub/bdir.git
76
76
77
+
mkdir -p gerrit.googlesource.com/team/scope/
78
+
cp -r gerrit.googlesource.com/adir.git gerrit.googlesource.com/team/scope/repoa.git
79
+
cp -r gerrit.googlesource.com/bdir.git gerrit.googlesource.com/team/scope/repob.git
80
+
77
81
cat << EOF > gerrit.googlesource.com/adir.git/config
78
82
[core]
79
83
repositoryformatversion = 0
···
120
124
}
121
125
122
126
want := map[string]bool{
123
-
"gerrit.googlesource.com/bdir.git": true,
124
-
"gerrit.googlesource.com/sub/bdir.git": true,
125
-
"adir/.git": true,
126
-
"bdir/.git": true,
127
-
"gerrit.googlesource.com/adir.git": true,
127
+
"adir/.git": true,
128
+
"bdir/.git": true,
129
+
"gerrit.googlesource.com/adir.git": true,
130
+
"gerrit.googlesource.com/bdir.git": true,
131
+
"gerrit.googlesource.com/sub/bdir.git": true,
132
+
"gerrit.googlesource.com/team/scope/repoa.git": true,
133
+
"gerrit.googlesource.com/team/scope/repob.git": true,
128
134
}
129
135
if !reflect.DeepEqual(got, want) {
130
136
t.Errorf("got %v want %v", got, want)