fork of https://github.com/sourcegraph/zoekt
index: remove shard_prefix option (#947)
We instead rely on the environment which is inheritted to make this
decision. The logic will become a little different in the next PR which
will instead decide layout based on if we are in "workspaces" mode vs
just wanting to enforce tenants.
Note: this is Sourcegraph specific. We want to move to always enforcing
tenant in our environments, even outside of our multitenant
environments.
Test Plan: go test
author
Keegan Carruthers-Smith
committer
GitHub
date
(May 13, 2025, 5:57 PM +0200)
commit
5aa52060
5aa5206067e3d0163e260f8c6d344c65b540b701
parent
490422d1
490422d1adb4b84f023ac1381aa534a7bbdccddc
+22
-87
+2
-7
cmd/zoekt-sourcegraph-indexserver/index.go
+2
-7
cmd/zoekt-sourcegraph-indexserver/index.go
···
21
21
configv1 "github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver/grpc/protos/sourcegraph/zoekt/configuration/v1"
22
22
"github.com/sourcegraph/zoekt/index"
23
23
"github.com/sourcegraph/zoekt/internal/ctags"
24
-
"github.com/sourcegraph/zoekt/internal/tenant"
25
24
)
26
25
27
26
const defaultIndexingTimeout = 1*time.Hour + 30*time.Minute
···
103
102
// BuildOptions returns a index.Options represented by indexArgs. Note: it
104
103
// doesn't set fields like repository/branch.
105
104
func (o *indexArgs) BuildOptions() *index.Options {
106
-
shardPrefix := ""
107
-
if tenant.EnforceTenant() {
108
-
shardPrefix = tenant.SrcPrefix(o.TenantID, o.RepoID)
109
-
}
110
-
111
105
return &index.Options{
112
106
// It is important that this RepositoryDescription exactly matches what
113
107
// the indexer we call will produce. This is to ensure that
···
141
135
142
136
ShardMerging: o.ShardMerging,
143
137
144
-
ShardPrefix: shardPrefix,
138
+
TenantID: o.TenantID,
139
+
RepoID: o.RepoID,
145
140
}
146
141
}
147
142
+4
-71
cmd/zoekt-sourcegraph-indexserver/index_test.go
+4
-71
cmd/zoekt-sourcegraph-indexserver/index_test.go
···
25
25
"github.com/sourcegraph/zoekt"
26
26
configv1 "github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver/grpc/protos/sourcegraph/zoekt/configuration/v1"
27
27
"github.com/sourcegraph/zoekt/internal/ctags"
28
-
"github.com/sourcegraph/zoekt/internal/tenant/tenanttest"
29
28
)
30
29
31
30
func TestIterateIndexOptions_Fingerprint(t *testing.T) {
···
470
469
})
471
470
}
472
471
473
-
func TestIndexTenant(t *testing.T) {
474
-
tenanttest.MockEnforce(t)
475
-
476
-
cases := []struct {
477
-
name string
478
-
args indexArgs
479
-
mockRepositoryMetadata *zoekt.Repository
480
-
want []string
481
-
}{
482
-
{
483
-
name: "prefix",
484
-
args: indexArgs{
485
-
IndexOptions: IndexOptions{
486
-
RepoID: 13,
487
-
Name: "test/repo",
488
-
CloneURL: "http://api.test/.internal/git/test/repo",
489
-
Branches: []zoekt.RepositoryBranch{{Name: "HEAD", Version: "deadbeef"}},
490
-
TenantID: 42,
491
-
},
492
-
},
493
-
want: []string{
494
-
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
495
-
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 42 fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
496
-
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
497
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.archived 0",
498
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.fork 0",
499
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.latestCommitDate 1",
500
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.name test/repo",
501
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.priority 0",
502
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.public 0",
503
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.repoid 13",
504
-
"git -C $TMPDIR/test%2Frepo.git config zoekt.tenantID 42",
505
-
"zoekt-git-index -submodules=false -branches HEAD -disable_ctags -shard_prefix 000000042_000000013 $TMPDIR/test%2Frepo.git",
506
-
},
507
-
},
508
-
}
509
-
510
-
for _, tc := range cases {
511
-
t.Run(tc.name, func(t *testing.T) {
512
-
var got []string
513
-
runCmd := func(c *exec.Cmd) error {
514
-
cmd := strings.Join(c.Args, " ")
515
-
cmd = strings.ReplaceAll(cmd, filepath.Clean(os.TempDir()), "$TMPDIR")
516
-
got = append(got, cmd)
517
-
return nil
518
-
}
519
-
520
-
findRepositoryMetadata := func(args *indexArgs) (repository *zoekt.Repository, metadata *zoekt.IndexMetadata, ok bool, err error) {
521
-
if tc.mockRepositoryMetadata == nil {
522
-
return args.BuildOptions().FindRepositoryMetadata()
523
-
}
524
-
525
-
return tc.mockRepositoryMetadata, &zoekt.IndexMetadata{}, true, nil
526
-
}
527
-
528
-
c := gitIndexConfig{
529
-
runCmd: runCmd,
530
-
findRepositoryMetadata: findRepositoryMetadata,
531
-
}
532
-
533
-
if err := gitIndex(context.Background(), c, &tc.args, sourcegraphNop{}, logtest.Scoped(t)); err != nil {
534
-
t.Fatal(err)
535
-
}
536
-
if !cmp.Equal(got, tc.want) {
537
-
t.Errorf("git mismatch (-want +got):\n%s", cmp.Diff(tc.want, got, splitargs))
538
-
}
539
-
})
540
-
}
541
-
}
542
-
543
472
func TestIndex(t *testing.T) {
544
473
cases := []struct {
545
474
name string
···
687
616
t.Run(tc.name, func(t *testing.T) {
688
617
var got []string
689
618
runCmd := func(c *exec.Cmd) error {
619
+
if c.Env != nil {
620
+
t.Fatal("expected nil Env for command. Tenant Enforcement relies on us inheritting the parent process environment.")
621
+
}
622
+
690
623
cmd := strings.Join(c.Args, " ")
691
624
cmd = strings.ReplaceAll(cmd, filepath.Clean(os.TempDir()), "$TMPDIR")
692
625
got = append(got, cmd)
+16
-9
index/builder.go
+16
-9
index/builder.go
···
17
17
package index
18
18
19
19
import (
20
-
"cmp"
21
20
"crypto/sha1"
22
21
"flag"
23
22
"fmt"
···
46
45
47
46
"github.com/sourcegraph/zoekt"
48
47
"github.com/sourcegraph/zoekt/internal/ctags"
48
+
"github.com/sourcegraph/zoekt/internal/tenant"
49
49
)
50
50
51
51
var DefaultDir = filepath.Join(os.Getenv("HOME"), ".zoekt")
···
121
121
// Note: heap checking is "best effort", and it's possible for the process to OOM without triggering the heap profile.
122
122
HeapProfileTriggerBytes uint64
123
123
124
-
// ShardPrefix is the prefix of the shard. It defaults to the repository name.
125
-
ShardPrefix string
124
+
// TenantID is the ID of the tenant this shard belongs to.
125
+
TenantID int
126
+
127
+
// RepoID is the ID of the repository this shard belongs to.
128
+
RepoID uint32
126
129
}
127
130
128
131
// HashOptions contains only the options in Options that upon modification leads to IndexState of IndexStateMismatch during the next index building.
···
187
190
fs.StringVar(&o.IndexDir, "index", x.IndexDir, "directory for search indices")
188
191
fs.BoolVar(&o.CTagsMustSucceed, "require_ctags", x.CTagsMustSucceed, "If set, ctags calls must succeed.")
189
192
fs.Var(largeFilesFlag{o}, "large_file", "A glob pattern where matching files are to be index regardless of their size. You can add multiple patterns by setting this more than once.")
190
-
fs.StringVar(&o.ShardPrefix, "shard_prefix", x.ShardPrefix, "the prefix of the shard. Defaults to repository name")
191
193
192
194
// Sourcegraph specific
193
195
fs.BoolVar(&o.DisableCTags, "disable_ctags", x.DisableCTags, "If set, ctags will not be called.")
···
235
237
args = append(args, "-shard_merging")
236
238
}
237
239
238
-
if o.ShardPrefix != "" {
239
-
args = append(args, "-shard_prefix", o.ShardPrefix)
240
-
}
241
-
242
240
return args
243
241
}
244
242
···
342
340
}
343
341
344
342
func (o *Options) shardNameVersion(version, n int) string {
345
-
return ShardName(o.IndexDir, cmp.Or(o.ShardPrefix, o.RepositoryDescription.Name), version, n)
343
+
var prefix string
344
+
345
+
// If tenant enforcement is enabled and we have tenant/repo IDs, use those to generate the prefix
346
+
if o.TenantID != 0 && o.RepoID != 0 && tenant.EnforceTenant() {
347
+
prefix = tenant.SrcPrefix(o.TenantID, o.RepoID)
348
+
} else {
349
+
prefix = o.RepositoryDescription.Name
350
+
}
351
+
352
+
return ShardName(o.IndexDir, prefix, version, n)
346
353
}
347
354
348
355
type IndexState string