fork of https://github.com/sourcegraph/zoekt
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Command zoekt-git-index indexes a single git repository. It works directly with git
16// repositories and supports git-specific features like branches and submodules.
17package main
18
19import (
20 "flag"
21 "log"
22 "os"
23 "path/filepath"
24 "runtime/pprof"
25 "strings"
26
27 "github.com/dustin/go-humanize"
28 "github.com/sourcegraph/zoekt/cmd"
29 "github.com/sourcegraph/zoekt/internal/ctags"
30 "github.com/sourcegraph/zoekt/internal/gitindex"
31 "github.com/sourcegraph/zoekt/internal/profiler"
32 "go.uber.org/automaxprocs/maxprocs"
33)
34
35func run() int {
36 allowMissing := flag.Bool("allow_missing_branches", false, "allow missing branches.")
37 submodules := flag.Bool("submodules", true, "if set to false, do not recurse into submodules")
38 branchesStr := flag.String("branches", "HEAD", "git branches to index.")
39 branchPrefix := flag.String("prefix", "refs/heads/", "prefix for branch names")
40
41 incremental := flag.Bool("incremental", true, "only index changed repositories")
42 repoCacheDir := flag.String("repo_cache", "", "directory holding bare git repos, named by URL. "+
43 "this is used to find repositories for submodules. "+
44 "It also affects name if the indexed repository is under this directory.")
45 isDelta := flag.Bool("delta", false, "whether we should use delta build")
46 deltaShardNumberFallbackThreshold := flag.Uint64("delta_threshold", 0, "upper limit on the number of preexisting shards that can exist before attempting a delta build (0 to disable fallback behavior)")
47 languageMap := flag.String("language_map", "", "a mapping between a language and its ctags processor (a:0,b:3).")
48
49 cpuProfile := flag.String("cpu_profile", "", "write cpu profile to `file`")
50
51 flag.Parse()
52
53 // Tune GOMAXPROCS to match Linux container CPU quota.
54 _, _ = maxprocs.Set()
55
56 if *cpuProfile != "" {
57 f, err := os.Create(*cpuProfile)
58 if err != nil {
59 log.Fatal("could not create CPU profile: ", err)
60 }
61 defer f.Close() // error handling omitted for example
62 if err := pprof.StartCPUProfile(f); err != nil {
63 log.Fatal("could not start CPU profile: ", err)
64 }
65 defer pprof.StopCPUProfile()
66 }
67
68 if *repoCacheDir != "" {
69 dir, err := filepath.Abs(*repoCacheDir)
70 if err != nil {
71 log.Fatalf("Abs: %v", err)
72 }
73 *repoCacheDir = dir
74 }
75
76 opts := cmd.OptionsFromFlags()
77 opts.IsDelta = *isDelta
78
79 var branches []string
80 if *branchesStr != "" {
81 branches = strings.Split(*branchesStr, ",")
82 }
83
84 gitRepos := map[string]string{}
85 for _, repoDir := range flag.Args() {
86 repoDir, err := filepath.Abs(repoDir)
87 if err != nil {
88 log.Fatal(err)
89 }
90 repoDir = filepath.Clean(repoDir)
91
92 name := strings.TrimSuffix(repoDir, "/.git")
93 if *repoCacheDir != "" && strings.HasPrefix(name, *repoCacheDir) {
94 name = strings.TrimPrefix(name, *repoCacheDir+"/")
95 name = strings.TrimSuffix(name, ".git")
96 } else {
97 name = strings.TrimSuffix(filepath.Base(name), ".git")
98 }
99 gitRepos[repoDir] = name
100 }
101
102 opts.LanguageMap = make(ctags.LanguageMap)
103 for _, mapping := range strings.Split(*languageMap, ",") {
104 m := strings.Split(mapping, ":")
105 if len(m) != 2 {
106 continue
107 }
108 opts.LanguageMap[m[0]] = ctags.StringToParser(m[1])
109 }
110
111 if heapProfileTrigger := os.Getenv("ZOEKT_HEAP_PROFILE_TRIGGER"); heapProfileTrigger != "" {
112 trigger, err := humanize.ParseBytes(heapProfileTrigger)
113 if err != nil {
114 log.Printf("invalid value for ZOEKT_HEAP_PROFILE_TRIGGER: %v", err)
115 } else {
116 opts.HeapProfileTriggerBytes = trigger
117 }
118 }
119
120 profiler.Init("zoekt-git-index")
121 exitStatus := 0
122 for dir, name := range gitRepos {
123 opts.RepositoryDescription.Name = name
124 gitOpts := gitindex.Options{
125 BranchPrefix: *branchPrefix,
126 Incremental: *incremental,
127 Submodules: *submodules,
128 RepoCacheDir: *repoCacheDir,
129 AllowMissingBranch: *allowMissing,
130 BuildOptions: *opts,
131 Branches: branches,
132 RepoDir: dir,
133 DeltaShardNumberFallbackThreshold: *deltaShardNumberFallbackThreshold,
134 }
135
136 if _, err := gitindex.IndexGitRepo(gitOpts); err != nil {
137 log.Printf("indexGitRepo(%s, delta=%t): %v", dir, gitOpts.BuildOptions.IsDelta, err)
138 exitStatus = 1
139 }
140 }
141
142 return exitStatus
143}
144
145func main() {
146 exitStatus := run()
147 os.Exit(exitStatus)
148}