fork of https://github.com/sourcegraph/zoekt
1package main
2
3import (
4 "bufio"
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "hash/crc32"
10 "log"
11 "math/rand"
12 "net/url"
13 "os"
14 "os/exec"
15 "path"
16 "path/filepath"
17 "strconv"
18 "strings"
19 "time"
20
21 "github.com/go-git/go-git/v5"
22 proto "github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver/protos/sourcegraph/zoekt/configuration/v1"
23 "github.com/sourcegraph/zoekt/ctags"
24 "golang.org/x/net/trace"
25
26 "github.com/sourcegraph/zoekt"
27)
28
29// SourcegraphListResult is the return value of Sourcegraph.List. It is its
30// own type since internally we batch the calculation of index options. This
31// is exposed via IterateIndexOptions.
32//
33// This type has state and is coupled to the Sourcegraph implementation.
34type SourcegraphListResult struct {
35 // IDs is the set of Sourcegraph repository IDs that this replica needs
36 // to index.
37 IDs []uint32
38
39 // IterateIndexOptions best effort resolves the IndexOptions for RepoIDs. If
40 // any repository fails it internally logs. It uses the "config fingerprint"
41 // to reduce the amount of work done. This means we only resolve options for
42 // repositories which have been mutated since the last Sourcegraph.List
43 // call.
44 //
45 // Note: this has a side-effect of setting a the "config fingerprint". The
46 // config fingerprint means we only calculate index options for repositories
47 // that have changed since the last call to IterateIndexOptions. If you want
48 // to force calculation of index options use
49 // Sourcegraph.ForceIterateIndexOptions.
50 //
51 // Note: This should not be called concurrently with the Sourcegraph client.
52 IterateIndexOptions func(func(IndexOptions))
53}
54
55// Sourcegraph represents the Sourcegraph service. It informs the indexserver
56// what to index and which options to use.
57type Sourcegraph interface {
58 // List returns a list of repository IDs to index as well as a facility to
59 // fetch the indexing options.
60 //
61 // Note: The return value is not safe to use concurrently with future calls
62 // to List.
63 List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error)
64
65 // ForceIterateIndexOptions will best-effort calculate the index options for
66 // all repos. For each repo it will call either onSuccess or onError. This
67 // is the forced version of IterateIndexOptions, so will always calculate
68 // options for each id in repos.
69 ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32)
70
71 // GetDocumentRanks returns a map from paths within the given repo to their
72 // rank vectors. Paths are assumed to be ordered by each pairwise component of
73 // the resulting vector, higher ranks coming earlier
74 GetDocumentRanks(ctx context.Context, repoName string) (RepoPathRanks, error)
75
76 // UpdateIndexStatus sends a request to Sourcegraph to confirm that the
77 // given repositories have been indexed.
78 UpdateIndexStatus(repositories []indexStatus) error
79}
80
81type SourcegraphClientOption func(*sourcegraphClient)
82
83// WithBatchSize controls how many repository configurations we request a time.
84// If BatchSize is 0, we default to requesting 10,000 repositories at once.
85func WithBatchSize(batchSize int) SourcegraphClientOption {
86 return func(c *sourcegraphClient) {
87 c.BatchSize = batchSize
88 }
89}
90
91func newSourcegraphClient(rootURL *url.URL, hostname string, grpcClient proto.ZoektConfigurationServiceClient, opts ...SourcegraphClientOption) *sourcegraphClient {
92 client := &sourcegraphClient{
93 Root: rootURL,
94 Hostname: hostname,
95 BatchSize: 0,
96 grpcClient: grpcClient,
97 }
98
99 for _, opt := range opts {
100 opt(client)
101 }
102
103 return client
104}
105
106// sourcegraphClient contains methods which interact with the sourcegraph API.
107type sourcegraphClient struct {
108 // Root is the base URL for the Sourcegraph instance to index. Normally
109 // http://sourcegraph-frontend-internal or http://localhost:3090.
110 Root *url.URL
111
112 // Hostname is the name we advertise to Sourcegraph when asking for the
113 // list of repositories to index.
114 Hostname string
115
116 // BatchSize is how many repository configurations we request at once. If
117 // zero a value of 10000 is used.
118 BatchSize int
119
120 // grpcClient is used to make requests to the Sourcegraph instance if gRPC is enabled.
121 grpcClient proto.ZoektConfigurationServiceClient
122
123 // configFingerprintProto is the last config fingerprint (as GRPC) returned from
124 // Sourcegraph. It can be used for future calls to the configuration
125 // endpoint.
126 //
127 // configFingerprintProto is mutually exclusive with configFingerprint - this field
128 // will only be used if gRPC is enabled.
129 configFingerprintProto *proto.Fingerprint
130
131 // configFingerprintReset tracks when we should zero out the
132 // configFingerprint. We want to periodically do this just in case our
133 // configFingerprint logic is faulty. When it is cleared out, we fallback to
134 // calculating everything.
135 configFingerprintReset time.Time
136}
137
138// GetDocumentRanks asks Sourcegraph for a mapping of file paths to rank
139// vectors.
140func (s *sourcegraphClient) GetDocumentRanks(ctx context.Context, repoName string) (RepoPathRanks, error) {
141 resp, err := s.grpcClient.DocumentRanks(ctx, &proto.DocumentRanksRequest{Repository: repoName})
142 if err != nil {
143 return RepoPathRanks{}, err
144 }
145
146 var out RepoPathRanks
147 out.FromProto(resp)
148
149 return out, nil
150}
151
152func (s *sourcegraphClient) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) {
153 repos, err := s.listRepoIDs(ctx, indexed)
154 if err != nil {
155 return nil, fmt.Errorf("listRepoIDs: %w", err)
156 }
157
158 batchSize := s.BatchSize
159 if batchSize == 0 {
160 batchSize = 10_000
161 }
162
163 // Check if we should recalculate everything.
164 if time.Now().After(s.configFingerprintReset) {
165 // for every 500 repos we wait a minute. 2021-12-15 on sourcegraph.com
166 // this works out to every 100 minutes.
167 next := time.Duration(len(indexed)) * time.Minute / 500
168 if min := 5 * time.Minute; next < min {
169 next = min
170 }
171 next += time.Duration(rand.Int63n(int64(next) / 4)) // jitter
172 s.configFingerprintReset = time.Now().Add(next)
173
174 s.configFingerprintProto = nil
175 }
176
177 // getIndexOptionsFunc is a function that can be used to get the index
178 // options for a set of repos (while properly handling any configuration fingerprint
179 // changes).
180 //
181 // In general, this function provides a consistent fingerprint for each batch call,
182 // and updates the server state with the new fingerprint. If any of the batch calls
183 // fail, the old fingerprint is restored.
184 type getIndexOptionsFunc func(repos ...uint32) ([]indexOptionsItem, error)
185
186 mkGetIndexOptionsFunc := func(tr trace.Trace) getIndexOptionsFunc {
187 startingFingerPrint := s.configFingerprintProto
188 tr.LazyPrintf("fingerprint: %s", startingFingerPrint.String())
189
190 first := true
191 return func(repos ...uint32) ([]indexOptionsItem, error) {
192 options, nextFingerPrint, err := s.getIndexOptions(ctx, startingFingerPrint, repos)
193 if err != nil {
194 first = false
195 s.configFingerprintProto = startingFingerPrint
196
197 return nil, err
198 }
199
200 if first {
201 first = false
202 s.configFingerprintProto = nextFingerPrint
203 tr.LazyPrintf("new fingerprint: %s", nextFingerPrint.String())
204 }
205
206 return options, nil
207 }
208 }
209
210 iterate := func(f func(IndexOptions)) {
211 start := time.Now()
212 tr := trace.New("getIndexOptions", "")
213 tr.LazyPrintf("getting index options for %d repos", len(repos))
214
215 defer func() {
216 metricResolveRevisionsDuration.Observe(time.Since(start).Seconds())
217 tr.Finish()
218 }()
219
220 getIndexOptions := mkGetIndexOptionsFunc(tr)
221
222 // We ask the frontend to get index options in batches.
223 for repos := range batched(repos, batchSize) {
224 start := time.Now()
225 options, err := getIndexOptions(repos...)
226 duration := time.Since(start)
227
228 if err != nil {
229 metricResolveRevisionDuration.WithLabelValues("false").Observe(duration.Seconds())
230 tr.LazyPrintf("failed fetching options batch: %v", err)
231 tr.SetError()
232
233 continue
234 }
235
236 metricResolveRevisionDuration.WithLabelValues("true").Observe(duration.Seconds())
237
238 for _, o := range options {
239 metricGetIndexOptions.Inc()
240
241 if o.Error != "" {
242 metricGetIndexOptionsError.Inc()
243 tr.LazyPrintf("failed fetching options for %v: %v", o.Name, o.Error)
244 tr.SetError()
245
246 continue
247 }
248 f(o.IndexOptions)
249 }
250 }
251 }
252
253 return &SourcegraphListResult{
254 IDs: repos,
255 IterateIndexOptions: iterate,
256 }, nil
257}
258
259func (s *sourcegraphClient) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) {
260 batchSize := s.BatchSize
261 if batchSize == 0 {
262 batchSize = 10_000
263 }
264
265 for repos := range batched(repos, batchSize) {
266 opts, _, err := s.getIndexOptions(context.Background(), nil, repos)
267 if err != nil {
268 for _, id := range repos {
269 onError(id, err)
270 }
271 continue
272 }
273 for _, o := range opts {
274 if o.RepoID > 0 && o.Error != "" {
275 onError(o.RepoID, errors.New(o.Error))
276 }
277 if o.Error == "" {
278 onSuccess(o.IndexOptions)
279 }
280 }
281 }
282}
283
284// indexOptionsItem wraps IndexOptions to also include an error returned by
285// the API.
286type indexOptionsItem struct {
287 IndexOptions
288 Error string
289}
290
291func (o *indexOptionsItem) FromProto(x *proto.ZoektIndexOptions) {
292 branches := make([]zoekt.RepositoryBranch, 0, len(x.Branches))
293 for _, b := range x.GetBranches() {
294 branches = append(branches, zoekt.RepositoryBranch{
295 Name: b.GetName(),
296 Version: b.GetVersion(),
297 })
298 }
299
300 item := indexOptionsItem{}
301 languageMap := make(map[string]ctags.CTagsParserType)
302
303 for _, lang := range x.GetLanguageMap() {
304 languageMap[lang.GetLanguage()] = ctags.CTagsParserType(lang.GetCtags().Number())
305 }
306
307 item.IndexOptions = IndexOptions{
308 RepoID: uint32(x.GetRepoId()),
309 LargeFiles: x.GetLargeFiles(),
310 Symbols: x.GetSymbols(),
311 Branches: branches,
312 Name: x.GetName(),
313
314 Priority: x.GetPriority(),
315
316 DocumentRanksVersion: x.GetDocumentRanksVersion(),
317
318 Public: x.GetPublic(),
319 Fork: x.GetFork(),
320 Archived: x.GetArchived(),
321
322 LanguageMap: languageMap,
323 ShardConcurrency: x.GetShardConcurrency(),
324 }
325
326 item.Error = x.GetError()
327
328 *o = item
329}
330
331func (o *indexOptionsItem) ToProto() *proto.ZoektIndexOptions {
332 branches := make([]*proto.ZoektRepositoryBranch, 0, len(o.Branches))
333 for _, b := range o.Branches {
334 branches = append(branches, &proto.ZoektRepositoryBranch{
335 Name: b.Name,
336 Version: b.Version,
337 })
338 }
339
340 languageMap := make([]*proto.LanguageMapping, 0, len(o.LanguageMap))
341
342 for lang, parser := range o.LanguageMap {
343 languageMap = append(languageMap, &proto.LanguageMapping{
344 Language: lang,
345 Ctags: proto.CTagsParserType(parser),
346 })
347 }
348
349 return &proto.ZoektIndexOptions{
350 RepoId: int32(o.RepoID),
351 LargeFiles: o.LargeFiles,
352 Symbols: o.Symbols,
353 Branches: branches,
354 Name: o.Name,
355
356 Priority: o.Priority,
357
358 DocumentRanksVersion: o.DocumentRanksVersion,
359
360 Public: o.Public,
361 Fork: o.Fork,
362 Archived: o.Archived,
363
364 Error: o.Error,
365
366 LanguageMap: languageMap,
367 ShardConcurrency: o.ShardConcurrency,
368 }
369}
370
371func (s *sourcegraphClient) getIndexOptions(ctx context.Context, fingerprint *proto.Fingerprint, repos []uint32) ([]indexOptionsItem, *proto.Fingerprint, error) {
372 repoIDs := make([]int32, 0, len(repos))
373 for _, id := range repos {
374 repoIDs = append(repoIDs, int32(id))
375 }
376
377 req := proto.SearchConfigurationRequest{
378 RepoIds: repoIDs,
379 Fingerprint: fingerprint,
380 }
381
382 response, err := s.grpcClient.SearchConfiguration(ctx, &req)
383 if err != nil {
384 return nil, nil, err
385 }
386
387 protoItems := response.GetUpdatedOptions()
388 items := make([]indexOptionsItem, 0, len(protoItems))
389 for _, x := range protoItems {
390 var item indexOptionsItem
391 item.FromProto(x)
392 item.IndexOptions.CloneURL = s.getCloneURL(item.Name)
393
394 items = append(items, item)
395 }
396
397 return items, response.GetFingerprint(), nil
398}
399
400func (s *sourcegraphClient) getCloneURL(name string) string {
401 return s.Root.ResolveReference(&url.URL{Path: path.Join("/.internal/git", name)}).String()
402}
403
404func (s *sourcegraphClient) listRepoIDs(ctx context.Context, indexed []uint32) ([]uint32, error) {
405 var request proto.ListRequest
406 request.Hostname = s.Hostname
407 request.IndexedIds = make([]int32, 0, len(indexed))
408 for _, id := range indexed {
409 request.IndexedIds = append(request.IndexedIds, int32(id))
410 }
411
412 response, err := s.grpcClient.List(ctx, &request)
413 if err != nil {
414 return nil, err
415 }
416
417 repoIDs := make([]uint32, 0, len(response.RepoIds))
418 for _, id := range response.RepoIds {
419 repoIDs = append(repoIDs, uint32(id))
420 }
421
422 return repoIDs, nil
423}
424
425type indexStatus struct {
426 RepoID uint32
427 Branches []zoekt.RepositoryBranch
428 IndexTimeUnix int64
429}
430
431type updateIndexStatusRequest struct {
432 Repositories []indexStatus
433}
434
435func (u *updateIndexStatusRequest) ToProto() *proto.UpdateIndexStatusRequest {
436 repositories := make([]*proto.UpdateIndexStatusRequest_Repository, 0, len(u.Repositories))
437
438 for _, repo := range u.Repositories {
439 branches := make([]*proto.ZoektRepositoryBranch, 0, len(repo.Branches))
440
441 for _, branch := range repo.Branches {
442 branches = append(branches, &proto.ZoektRepositoryBranch{
443 Name: branch.Name,
444 Version: branch.Version,
445 })
446 }
447
448 repositories = append(repositories, &proto.UpdateIndexStatusRequest_Repository{
449 RepoId: repo.RepoID,
450 Branches: branches,
451 IndexTimeUnix: repo.IndexTimeUnix,
452 })
453 }
454
455 return &proto.UpdateIndexStatusRequest{
456 Repositories: repositories,
457 }
458}
459
460func (u *updateIndexStatusRequest) FromProto(x *proto.UpdateIndexStatusRequest) {
461 protoRepositories := x.GetRepositories()
462 repositories := make([]indexStatus, 0, len(protoRepositories))
463
464 for _, repo := range x.GetRepositories() {
465 protoBranches := repo.GetBranches()
466 branches := make([]zoekt.RepositoryBranch, 0, len(protoBranches))
467
468 for _, branch := range repo.GetBranches() {
469 branches = append(branches, zoekt.RepositoryBranch{
470 Name: branch.GetName(),
471 Version: branch.GetVersion(),
472 })
473 }
474
475 repositories = append(repositories, indexStatus{
476 RepoID: repo.GetRepoId(),
477 Branches: branches,
478 IndexTimeUnix: repo.GetIndexTimeUnix(),
479 })
480 }
481
482 *u = updateIndexStatusRequest{
483 Repositories: repositories,
484 }
485}
486
487// UpdateIndexStatus sends a request to Sourcegraph to confirm that the given
488// repositories have been indexed.
489func (s *sourcegraphClient) UpdateIndexStatus(repositories []indexStatus) error {
490 r := updateIndexStatusRequest{Repositories: repositories}
491
492 request := r.ToProto()
493 _, err := s.grpcClient.UpdateIndexStatus(context.Background(), request)
494 if err != nil {
495 return fmt.Errorf("failed to update index status: %w", err)
496 }
497
498 return nil
499}
500
501type sourcegraphFake struct {
502 RootDir string
503 Log *log.Logger
504}
505
506// GetDocumentRanks expects a file where each line has the following format:
507// path<tab>rank... where rank is a float64.
508func (sf sourcegraphFake) GetDocumentRanks(ctx context.Context, repoName string) (RepoPathRanks, error) {
509 dir := filepath.Join(sf.RootDir, filepath.FromSlash(repoName))
510
511 fd, err := os.Open(filepath.Join(dir, "SG_DOCUMENT_RANKS"))
512 if err != nil {
513 return RepoPathRanks{}, err
514 }
515
516 ranks := RepoPathRanks{}
517
518 sum := 0.0
519 count := 0
520 scanner := bufio.NewScanner(fd)
521 for scanner.Scan() {
522 s := scanner.Text()
523 pathRanks := strings.Split(s, "\t")
524 if rank, err := strconv.ParseFloat(pathRanks[1], 64); err == nil {
525 ranks.Paths[pathRanks[0]] = rank
526 sum += rank
527 count++
528 }
529 }
530
531 if err := scanner.Err(); err != nil {
532 return RepoPathRanks{}, err
533 }
534
535 ranks.MeanRank = sum / float64(count)
536 return ranks, nil
537}
538
539func (sf sourcegraphFake) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) {
540 repos, err := sf.ListRepoIDs(ctx, indexed)
541 if err != nil {
542 return nil, err
543 }
544
545 iterate := func(f func(IndexOptions)) {
546 opts, err := sf.GetIndexOptions(repos...)
547 if err != nil {
548 sf.Log.Printf("WARN: ignoring GetIndexOptions error: %v", err)
549 }
550 for _, opt := range opts {
551 if opt.Error != "" {
552 sf.Log.Printf("WARN: ignoring GetIndexOptions error for %s: %v", opt.Name, opt.Error)
553 continue
554 }
555 f(opt.IndexOptions)
556 }
557 }
558
559 return &SourcegraphListResult{
560 IDs: repos,
561 IterateIndexOptions: iterate,
562 }, nil
563}
564
565func (sf sourcegraphFake) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) {
566 opts, err := sf.GetIndexOptions(repos...)
567 if err != nil {
568 for _, id := range repos {
569 onError(id, err)
570 }
571 return
572 }
573 for _, o := range opts {
574 if o.RepoID > 0 && o.Error != "" {
575 onError(o.RepoID, errors.New(o.Error))
576 }
577 if o.Error == "" {
578 onSuccess(o.IndexOptions)
579 }
580 }
581}
582
583func (sf sourcegraphFake) GetIndexOptions(repos ...uint32) ([]indexOptionsItem, error) {
584 reposIdx := map[uint32]int{}
585 for i, id := range repos {
586 reposIdx[id] = i
587 }
588
589 items := make([]indexOptionsItem, len(repos))
590 err := sf.visitRepos(func(name string) {
591 idx, ok := reposIdx[sf.id(name)]
592 if !ok {
593 return
594 }
595 opts, err := sf.getIndexOptions(name)
596 if err != nil {
597 items[idx] = indexOptionsItem{Error: err.Error()}
598 } else {
599 items[idx] = indexOptionsItem{IndexOptions: opts}
600 }
601 })
602 if err != nil {
603 return nil, err
604 }
605
606 for i := range items {
607 if items[i].Error == "" && items[i].RepoID == 0 {
608 items[i].Error = "not found"
609 }
610 }
611
612 return items, nil
613}
614
615func (sf sourcegraphFake) getIndexOptions(name string) (IndexOptions, error) {
616 dir := filepath.Join(sf.RootDir, filepath.FromSlash(name))
617 exists := func(p string) bool {
618 _, err := os.Stat(filepath.Join(dir, p))
619 return err == nil
620 }
621 float := func(p string) float64 {
622 b, _ := os.ReadFile(filepath.Join(dir, p))
623 f, _ := strconv.ParseFloat(string(bytes.TrimSpace(b)), 64)
624 return f
625 }
626
627 opts := IndexOptions{
628 RepoID: sf.id(name),
629 Name: name,
630 CloneURL: sf.getCloneURL(name),
631 Symbols: true,
632
633 Public: !exists("SG_PRIVATE"),
634 Fork: exists("SG_FORK"),
635 Archived: exists("SG_ARCHIVED"),
636
637 Priority: float("SG_PRIORITY"),
638 }
639
640 if stat, err := os.Stat(filepath.Join(dir, "SG_DOCUMENT_RANKS")); err == nil {
641 opts.DocumentRanksVersion = stat.ModTime().String()
642 }
643
644 branches, err := sf.getBranches(name)
645 if err != nil {
646 return opts, err
647 }
648 opts.Branches = branches
649
650 return opts, nil
651}
652
653func (sf sourcegraphFake) getBranches(name string) ([]zoekt.RepositoryBranch, error) {
654 dir := filepath.Join(sf.RootDir, filepath.FromSlash(name))
655 repo, err := git.PlainOpen(dir)
656 if err != nil {
657 return nil, err
658 }
659
660 cfg, err := repo.Config()
661 if err != nil {
662 return nil, err
663 }
664
665 sec := cfg.Raw.Section("zoekt")
666 branches := sec.Options.GetAll("branch")
667 if len(branches) == 0 {
668 branches = append(branches, "HEAD")
669 }
670
671 rBranches := make([]zoekt.RepositoryBranch, 0, len(branches))
672 for _, branch := range branches {
673 cmd := exec.Command("git", "rev-parse", branch)
674 cmd.Dir = dir
675 if b, err := cmd.Output(); err != nil {
676 sf.Log.Printf("WARN: Could not get branch %s/%s", name, branch)
677 } else {
678 version := string(bytes.TrimSpace(b))
679 rBranches = append(rBranches, zoekt.RepositoryBranch{
680 Name: branch,
681 Version: version,
682 })
683 }
684 }
685
686 if len(rBranches) == 0 {
687 return nil, fmt.Errorf("WARN: Could not get any branch revisions for repo %s", name)
688 }
689
690 return rBranches, nil
691}
692
693func (sf sourcegraphFake) id(name string) uint32 {
694 // allow overriding the ID.
695 idPath := filepath.Join(sf.RootDir, filepath.FromSlash(name), "SG_ID")
696 if b, _ := os.ReadFile(idPath); len(b) > 0 {
697 id, err := strconv.Atoi(strings.TrimSpace(string(b)))
698 if err == nil {
699 return uint32(id)
700 }
701 }
702 return fakeID(name)
703}
704
705func (sf sourcegraphFake) getCloneURL(name string) string {
706 return filepath.Join(sf.RootDir, filepath.FromSlash(name))
707}
708
709func (sf sourcegraphFake) ListRepoIDs(ctx context.Context, indexed []uint32) ([]uint32, error) {
710 var repos []uint32
711 err := sf.visitRepos(func(name string) {
712 repos = append(repos, sf.id(name))
713 })
714 return repos, err
715}
716
717func (sf sourcegraphFake) visitRepos(visit func(name string)) error {
718 return filepath.Walk(sf.RootDir, func(path string, fi os.FileInfo, fileErr error) error {
719 if fileErr != nil {
720 sf.Log.Printf("WARN: ignoring error searching %s: %v", path, fileErr)
721 return nil
722 }
723 if !fi.IsDir() {
724 return nil
725 }
726
727 gitdir := filepath.Join(path, ".git")
728 if fi, err := os.Stat(gitdir); err != nil || !fi.IsDir() {
729 return nil
730 }
731
732 subpath, err := filepath.Rel(sf.RootDir, path)
733 if err != nil {
734 // According to WalkFunc docs, path is always filepath.Join(root,
735 // subpath). So Rel should always work.
736 return fmt.Errorf("filepath.Walk returned %s which is not relative to %s: %w", path, sf.RootDir, err)
737 }
738
739 name := filepath.ToSlash(subpath)
740 visit(name)
741
742 return filepath.SkipDir
743 })
744}
745
746func (s sourcegraphFake) UpdateIndexStatus(repositories []indexStatus) error {
747 // noop
748 return nil
749}
750
751// fakeID returns a deterministic ID based on name. Used for fakes and tests.
752func fakeID(name string) uint32 {
753 // magic at the end is to ensure we get a positive number when casting.
754 return uint32(crc32.ChecksumIEEE([]byte(name))%(1<<31-1) + 1)
755}
756
757type sourcegraphNop struct{}
758
759func (s sourcegraphNop) List(ctx context.Context, indexed []uint32) (*SourcegraphListResult, error) {
760 return nil, nil
761}
762
763func (s sourcegraphNop) ForceIterateIndexOptions(onSuccess func(IndexOptions), onError func(uint32, error), repos ...uint32) {
764}
765
766func (s sourcegraphNop) GetDocumentRanks(ctx context.Context, repoName string) (RepoPathRanks, error) {
767 return RepoPathRanks{}, nil
768}
769
770func (s sourcegraphNop) UpdateIndexStatus(repositories []indexStatus) error {
771 return nil
772}
773
774type RepoPathRanks struct {
775 MeanRank float64 `json:"mean_reference_count"`
776 Paths map[string]float64 `json:"paths"`
777}
778
779func (r *RepoPathRanks) FromProto(x *proto.DocumentRanksResponse) {
780 protoPaths := x.GetPaths()
781 ranks := make(map[string]float64, len(protoPaths))
782 for filePath, rank := range protoPaths {
783 ranks[filePath] = rank
784 }
785
786 *r = RepoPathRanks{
787 MeanRank: x.GetMeanRank(),
788 Paths: ranks,
789 }
790}
791
792func (r *RepoPathRanks) ToProto() *proto.DocumentRanksResponse {
793 paths := make(map[string]float64, len(r.Paths))
794 for filePath, rank := range r.Paths {
795 paths[filePath] = rank
796 }
797
798 return &proto.DocumentRanksResponse{
799 MeanRank: r.MeanRank,
800 Paths: paths,
801 }
802}