fork of https://github.com/sourcegraph/zoekt
gitlab: Iterate over projects by id instead of offset based pagination (#404)
There is a limit for offset based pagination on the projects API. One
option is to use keyset's but go-gitlab has no support for it. Another
related approach is to iterate over project ids. For this we are going
to:
* Order by 'id'
* Sort in ascending order
* Use ID After to move forward until there are no ids
This helps to iterate the number of projects on a large(r) instance
of GitLab.
author
Holger Freyther
committer
GitHub
date
(Aug 24, 2022, 8:58 AM +0200)
commit
1d4dc3c8
1d4dc3c820c38f03efe9a97c507cafc32c98570e
parent
00761158
00761158b06d4aab579d26d1e0bca5d36e43ccc7
+6
-5
+6
-5
cmd/zoekt-mirror-gitlab/main.go
+6
-5
cmd/zoekt-mirror-gitlab/main.go
···
79
79
80
80
opt := &gitlab.ListProjectsOptions{
81
81
ListOptions: gitlab.ListOptions{
82
-
PerPage: 10,
83
-
Page: 1,
82
+
PerPage: 100,
84
83
},
84
+
Sort: gitlab.String("asc"),
85
+
OrderBy: gitlab.String("id"),
85
86
Membership: isMember,
86
87
}
87
88
if *isPublic {
···
90
91
91
92
var gitlabProjects []*gitlab.Project
92
93
for {
93
-
projects, resp, err := client.Projects.ListProjects(opt)
94
+
projects, _, err := client.Projects.ListProjects(opt)
94
95
if err != nil {
95
96
log.Fatal(err)
96
97
}
···
106
107
gitlabProjects = append(gitlabProjects, project)
107
108
}
108
109
109
-
if resp.CurrentPage >= resp.TotalPages {
110
+
if len(projects) == 0 {
110
111
break
111
112
}
112
113
113
-
opt.Page = resp.NextPage
114
+
opt.IDAfter = &projects[len(projects)-1].ID
114
115
}
115
116
116
117
filter, err := gitindex.NewFilter(*namePattern, *excludePattern)