fork of https://github.com/sourcegraph/zoekt
Merge remote-tracking branch 'gerrit/master'
Change-Id: I71a647f5b1efda651e9927f0b3904275c3603cd4
author
Keegan Carruthers-Smith
date
(Jan 21, 2020, 6:31 PM +0200)
commit
c66fea4e
c66fea4e816caf9bdf45010d501a45a49e2d3e09
parent
2f3fa2c3
2f3fa2c3ee1919d37a452c221ffb2a209a0fdfcd
+241
-25
+9
-8
cmd/flags.go
+9
-8
cmd/flags.go
···
38
38
}
39
39
40
40
var (
41
-
sizeMax = flag.Int("file_limit", 128*1024, "maximum file size")
42
-
trigramMax = flag.Int("max_trigram_count", 20000, "maximum number of trigrams per document")
43
-
shardLimit = flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard")
44
-
parallelism = flag.Int("parallelism", 4, "maximum number of parallel indexing processes.")
45
-
indexDir = flag.String("index", build.DefaultDir, "directory for search indices")
46
-
version = flag.Bool("version", false, "Print version number")
41
+
sizeMax = flag.Int("file_limit", 2<<20, "maximum file size")
42
+
trigramMax = flag.Int("max_trigram_count", 20000, "maximum number of trigrams per document")
43
+
shardLimit = flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard")
44
+
parallelism = flag.Int("parallelism", 4, "maximum number of parallel indexing processes.")
45
+
indexDir = flag.String("index", build.DefaultDir, "directory for search indices")
46
+
version = flag.Bool("version", false, "Print version number")
47
+
ctags = flag.Bool("require_ctags", false, "If set, ctags calls must succeed.")
48
+
largeFiles = largeFilesFlag{}
49
+
47
50
disableCTags = flag.Bool("disable_ctags", false, "If set, ctags will not be called.")
48
-
ctags = flag.Bool("require_ctags", false, "If set, ctags calls must succeed.")
49
-
largeFiles = largeFilesFlag{}
50
51
)
51
52
52
53
func init() {
+28
-7
cmd/zoekt-archive-index/archive.go
+28
-7
cmd/zoekt-archive-index/archive.go
···
2
2
3
3
import (
4
4
"archive/tar"
5
+
"bytes"
5
6
"compress/gzip"
6
7
"fmt"
7
8
"io"
···
57
58
return nil
58
59
}
59
60
61
+
func detectContentType(r io.Reader) (string, io.Reader, error) {
62
+
var buf [512]byte
63
+
n, err := io.ReadFull(r, buf[:])
64
+
if err != nil && err != io.ErrUnexpectedEOF {
65
+
return "", nil, err
66
+
}
67
+
68
+
ct := http.DetectContentType(buf[:n])
69
+
70
+
// Return a new reader which merges in the read bytes
71
+
return ct, io.MultiReader(bytes.NewReader(buf[:n]), r), nil
72
+
}
73
+
60
74
// openArchive opens the tar at the URL or filepath u. Also supported is tgz
61
75
// files over http.
62
76
//
···
93
107
}
94
108
closer = resp.Body
95
109
r = body
96
-
if resp.Header.Get("Content-Type") == "application/x-gzip" {
97
-
r, err = gzip.NewReader(r)
98
-
if err != nil {
99
-
resp.Body.Close()
100
-
return nil, err
101
-
}
102
-
}
103
110
} else if u == "-" {
104
111
r = os.Stdin
105
112
} else {
···
109
116
}
110
117
closer = f
111
118
r = f
119
+
}
120
+
121
+
ct, r, err := detectContentType(r)
122
+
if err != nil {
123
+
return nil, err
124
+
}
125
+
if ct == "application/x-gzip" {
126
+
r, err = gzip.NewReader(r)
127
+
if err != nil {
128
+
if closer != nil {
129
+
_ = closer.Close()
130
+
}
131
+
return nil, err
132
+
}
112
133
}
113
134
114
135
return &tarArchive{
+23
-2
cmd/zoekt-archive-index/e2e_test.go
+23
-2
cmd/zoekt-archive-index/e2e_test.go
···
2
2
3
3
import (
4
4
"archive/tar"
5
+
"compress/gzip"
5
6
"context"
6
7
"fmt"
7
8
"io"
···
16
17
"github.com/google/zoekt/shards"
17
18
)
18
19
19
-
func writeArchive(w io.Writer, files map[string]string) error {
20
+
func writeArchive(w io.Writer, format string, files map[string]string) (err error) {
21
+
if format == "tgz" {
22
+
gw := gzip.NewWriter(w)
23
+
defer func() {
24
+
err2 := gw.Close()
25
+
if err == nil {
26
+
err = err2
27
+
}
28
+
}()
29
+
w = gw
30
+
format = "tar"
31
+
}
32
+
20
33
tw := tar.NewWriter(w)
21
34
22
35
for name, body := range files {
···
45
58
// -incremental=true option changing the options between indexes and ensuring
46
59
// the results change as expected.
47
60
func TestIndexIncrementally(t *testing.T) {
61
+
for _, format := range []string{"tar", "tgz"} {
62
+
t.Run(format, func(t *testing.T) {
63
+
testIndexIncrementally(t, format)
64
+
})
65
+
}
66
+
}
67
+
68
+
func testIndexIncrementally(t *testing.T, format string) {
48
69
indexdir, err := ioutil.TempDir("", "TestIndexArg-index")
49
70
if err != nil {
50
71
t.Fatalf("TempDir: %v", err)
···
64
85
files["F"+s] = strings.Repeat("a", fileSize)
65
86
}
66
87
67
-
err = writeArchive(archive, files)
88
+
err = writeArchive(archive, format, files)
68
89
if err != nil {
69
90
t.Fatalf("unable to create archive %v", err)
70
91
}
+28
gitindex/clone.go
+28
gitindex/clone.go
···
21
21
"os/exec"
22
22
"path/filepath"
23
23
"sort"
24
+
25
+
git "gopkg.in/src-d/go-git.v4"
26
+
"gopkg.in/src-d/go-git.v4/config"
24
27
)
25
28
26
29
// CloneRepo clones one repository, adding the given config
···
64
67
return "", err
65
68
}
66
69
70
+
if err := setFetch(repoDest, "origin", "+refs/heads/*:refs/heads/*"); err != nil {
71
+
log.Printf("addFetch: %v", err)
72
+
}
67
73
return repoDest, nil
68
74
}
75
+
76
+
func setFetch(repoDir, remote, refspec string) error {
77
+
repo, err := git.PlainOpen(repoDir)
78
+
if err != nil {
79
+
return err
80
+
}
81
+
82
+
cfg, err := repo.Config()
83
+
if err != nil {
84
+
return err
85
+
}
86
+
87
+
rm := cfg.Remotes[remote]
88
+
if rm != nil {
89
+
rm.Fetch = []config.RefSpec{config.RefSpec(refspec)}
90
+
}
91
+
if err := repo.Storer.SetConfig(cfg); err != nil {
92
+
return err
93
+
}
94
+
95
+
return nil
96
+
}
+63
gitindex/clone_test.go
+63
gitindex/clone_test.go
···
1
+
// Copyright 2019 Google Inc. All rights reserved.
2
+
//
3
+
// Licensed under the Apache License, Version 2.0 (the "License");
4
+
// you may not use this file except in compliance with the License.
5
+
// You may obtain a copy of the License at
6
+
//
7
+
// http://www.apache.org/licenses/LICENSE-2.0
8
+
//
9
+
// Unless required by applicable law or agreed to in writing, software
10
+
// distributed under the License is distributed on an "AS IS" BASIS,
11
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+
// See the License for the specific language governing permissions and
13
+
// limitations under the License.
14
+
15
+
package gitindex
16
+
17
+
import (
18
+
"io/ioutil"
19
+
"os"
20
+
"os/exec"
21
+
"testing"
22
+
23
+
git "gopkg.in/src-d/go-git.v4"
24
+
)
25
+
26
+
func TestSetRemote(t *testing.T) {
27
+
dir, err := ioutil.TempDir("", "")
28
+
if err != nil {
29
+
t.Fatal(err)
30
+
}
31
+
defer os.RemoveAll(dir)
32
+
script := `mkdir orig
33
+
cd orig
34
+
git init
35
+
cd ..
36
+
git clone orig/.git clone.git
37
+
`
38
+
39
+
cmd := exec.Command("/bin/sh", "-euxc", script)
40
+
cmd.Dir = dir
41
+
42
+
if out, err := cmd.CombinedOutput(); err != nil {
43
+
t.Fatalf("execution error: %v, output %s", err, out)
44
+
}
45
+
46
+
r := dir + "/clone.git"
47
+
if err := setFetch(r, "origin", "+refs/heads/*:refs/heads/*"); err != nil {
48
+
t.Fatalf("addFetch: %v", err)
49
+
}
50
+
51
+
repo, err := git.PlainOpen(r)
52
+
if err != nil {
53
+
t.Fatal("PlainOpen", err)
54
+
}
55
+
56
+
rm, err := repo.Remote("origin")
57
+
if err != nil {
58
+
t.Fatal("Remote", err)
59
+
}
60
+
if got, want := rm.Config().Fetch[0].String(), "+refs/heads/*:refs/heads/*"; got != want {
61
+
t.Fatalf("got %q want %q", got, want)
62
+
}
63
+
}
+1
-1
toc.go
+1
-1
toc.go