fork of https://github.com/sourcegraph/zoekt
Pull out document creation into its own method (#693)
I experimented with some changes to encourage `go-git` to use less memory. They
didn't pan out, but this intermediate refactor felt useful on its own. It helps
break up the super long `indexGitRepo` method.
author
Julie Tibshirani
committer
GitHub
date
(Nov 15, 2023, 8:00 AM -0800)
commit
3109882b
3109882bd0115b74ace1cfe7db1d33f6abc7fa0d
parent
137eb8f2
137eb8f2261028b05a290f121b9add59c2b34fdc
+45
-36
+45
-36
gitindex/index.go
+45
-36
gitindex/index.go
···
546
546
keys := fileKeys[name]
547
547
548
548
for _, key := range keys {
549
-
brs := branchMap[key]
550
-
blob, err := repos[key].Repo.BlobObject(key.ID)
549
+
doc, err := createDocument(key, repos, branchMap, ranks, opts.BuildOptions)
551
550
if err != nil {
552
551
return err
553
552
}
554
553
555
-
keyFullPath := key.FullPath()
556
-
557
-
if blob.Size > int64(opts.BuildOptions.SizeMax) && !opts.BuildOptions.IgnoreSizeMax(keyFullPath) {
558
-
if err := builder.Add(zoekt.Document{
559
-
SkipReason: fmt.Sprintf("file size %d exceeds maximum size %d", blob.Size, opts.BuildOptions.SizeMax),
560
-
Name: keyFullPath,
561
-
Branches: brs,
562
-
SubRepositoryPath: key.SubRepoPath,
563
-
}); err != nil {
564
-
return err
565
-
}
566
-
continue
567
-
}
568
-
569
-
contents, err := blobContents(blob)
570
-
if err != nil {
571
-
return err
572
-
}
573
-
574
-
var pathRanks []float64
575
-
if len(ranks.Paths) > 0 {
576
-
// If the repository has ranking data, then store the file's rank.
577
-
pathRank := ranks.rank(keyFullPath)
578
-
pathRanks = []float64{pathRank}
579
-
}
580
-
581
-
if err := builder.Add(zoekt.Document{
582
-
SubRepositoryPath: key.SubRepoPath,
583
-
Name: keyFullPath,
584
-
Content: contents,
585
-
Branches: brs,
586
-
Ranks: pathRanks,
587
-
}); err != nil {
588
-
return fmt.Errorf("error adding document with name %s: %w", keyFullPath, err)
554
+
if err := builder.Add(doc); err != nil {
555
+
return fmt.Errorf("error adding document with name %s: %w", key.FullPath(), err)
589
556
}
590
557
}
591
558
}
···
891
858
}
892
859
893
860
return repos, branchMap, branchVersions, nil
861
+
}
862
+
863
+
func createDocument(key fileKey,
864
+
repos map[fileKey]BlobLocation,
865
+
branchMap map[fileKey][]string,
866
+
ranks repoPathRanks,
867
+
opts build.Options,
868
+
) (zoekt.Document, error) {
869
+
blob, err := repos[key].Repo.BlobObject(key.ID)
870
+
if err != nil {
871
+
return zoekt.Document{}, err
872
+
}
873
+
874
+
keyFullPath := key.FullPath()
875
+
if blob.Size > int64(opts.SizeMax) && !opts.IgnoreSizeMax(keyFullPath) {
876
+
return zoekt.Document{
877
+
SkipReason: fmt.Sprintf("file size %d exceeds maximum size %d", blob.Size, opts.SizeMax),
878
+
Name: key.FullPath(),
879
+
Branches: branchMap[key],
880
+
SubRepositoryPath: key.SubRepoPath,
881
+
}, nil
882
+
}
883
+
884
+
contents, err := blobContents(blob)
885
+
if err != nil {
886
+
return zoekt.Document{}, err
887
+
}
888
+
889
+
var pathRanks []float64
890
+
if len(ranks.Paths) > 0 {
891
+
// If the repository has ranking data, then store the file's rank.
892
+
pathRank := ranks.rank(keyFullPath)
893
+
pathRanks = []float64{pathRank}
894
+
}
895
+
896
+
return zoekt.Document{
897
+
SubRepositoryPath: key.SubRepoPath,
898
+
Name: keyFullPath,
899
+
Content: contents,
900
+
Branches: branchMap[key],
901
+
Ranks: pathRanks,
902
+
}, nil
894
903
}
895
904
896
905
func blobContents(blob *object.Blob) ([]byte, error) {