fork of https://github.com/sourcegraph/zoekt
peterguy/cross platform build (#535)
* Replace unix/syscall `mmap` with `mmap-go`
`mmap-go` is a cross-platform memory-mapping package.
It uses `Mmap` on *Nix systems
and `CreateFileMapping` + `MapViewOfFile` on Windows.
`mmap-go` benchmarks the same as `unix.Mmap`
(see [https://github.com/peterguy/benchmark-mmap](https://github.com/peterguy/benchmark-mmap)).
BONUS! `indexfile_other.go` used straight file reads (`os.File::ReadAt`)
to support the same kind of behavior as mmap on non-*Nix platforms;
`mmap-go` uses Windows APIs to do real memory mapping on Windows,
which is faster by several magnitudes.
Also change the rounding/padding of the memory buffer being allocated.
It was hard-coded to round up to the nearest 4096 bytes
in order to page align for mmap,
but not all operating systems use 4k pages.
Instead of the hard coded value, use `os.Getpagesize()`.
* Build-constrain usages of `unix.Umask`
* build-constrain zoekt webserver for cross-platform
* Fix buffer size in Windows
and refacter to a method that calculates
the correct buffer size for the platform
* use gopsutil v3 for disk space metrics
instead of unix/syscall Statfs
* Build-constrain the Prometheus memory map metrics
Windows does not have the concept of a count of mapped memory areas.
Non-linux operating systems do not use the /sys pseudo-filesystem.
Constrain `mustRegisterMemoryMapMetrics` to linux-only builds.
Other platforms use a no-op function.
* cleanup
- go get -u -t ./..
- remove unused import
* Update cmd/zoekt-webserver/main.go
Co-authored-by: Keegan Carruthers-Smith <keegan.csmith@gmail.com>
---------
Co-authored-by: Keegan Carruthers-Smith <keegan.csmith@gmail.com>
author
Peter Guy
co-author
Keegan Carruthers-Smith
committer
GitHub
date
(Feb 15, 2023, 1:36 PM -0800)
commit
7424ac84
7424ac84bab3ec9ae247339909ffd84e5e3b1338
parent
b8a8461a
b8a8461a9bbff2f15d4272eab5f706217adbeb18
+247
-182
-7
cmd/zoekt-sourcegraph-indexserver/meta.go
-7
cmd/zoekt-sourcegraph-indexserver/meta.go
···
6
6
"os"
7
7
"path/filepath"
8
8
9
-
"golang.org/x/sys/unix"
10
-
11
9
"github.com/sourcegraph/zoekt"
12
10
"github.com/sourcegraph/zoekt/build"
13
11
)
···
112
110
113
111
// respect process umask. build does this.
114
112
var umask os.FileMode
115
-
116
-
func init() {
117
-
umask = os.FileMode(unix.Umask(0))
118
-
unix.Umask(int(umask))
119
-
}
+14
cmd/zoekt-sourcegraph-indexserver/meta_unix.go
+14
cmd/zoekt-sourcegraph-indexserver/meta_unix.go
+6
cmd/zoekt-sourcegraph-indexserver/meta_windows.go
+6
cmd/zoekt-sourcegraph-indexserver/meta_windows.go
+25
-92
cmd/zoekt-webserver/main.go
+25
-92
cmd/zoekt-webserver/main.go
···
31
31
"net/url"
32
32
"os"
33
33
"os/signal"
34
-
"path"
35
34
"path/filepath"
36
35
"runtime"
37
36
"strconv"
38
37
"strings"
39
38
"time"
40
-
41
-
"golang.org/x/sys/unix"
42
39
43
40
"github.com/sourcegraph/mountinfo"
44
41
···
55
52
"github.com/opentracing/opentracing-go"
56
53
"github.com/prometheus/client_golang/prometheus"
57
54
"github.com/prometheus/client_golang/prometheus/promauto"
58
-
"github.com/prometheus/procfs"
55
+
"github.com/shirou/gopsutil/v3/disk"
59
56
sglog "github.com/sourcegraph/log"
60
57
"github.com/uber/jaeger-client-go"
61
58
oteltrace "go.opentelemetry.io/otel/trace"
···
337
334
// times you will read the channel (used as buffer for signal.Notify).
338
335
func shutdownSignalChan(maxReads int) <-chan os.Signal {
339
336
c := make(chan os.Signal, maxReads)
340
-
signal.Notify(c, os.Interrupt) // terminal C-c and goreman
341
-
signal.Notify(c, unix.SIGTERM) // Kubernetes
337
+
signal.Notify(c, os.Interrupt) // terminal C-c and goreman
338
+
signal.Notify(c, PLATFORM_SIGTERM) // Kubernetes
342
339
return c
343
340
}
344
341
···
437
434
}
438
435
}
439
436
437
+
func diskUsage(path string) (*disk.UsageStat, error) {
438
+
duPath := path
439
+
if runtime.GOOS == "windows" {
440
+
duPath = filepath.VolumeName(duPath)
441
+
}
442
+
usage, err := disk.Usage(duPath)
443
+
if err != nil {
444
+
return nil, fmt.Errorf("diskUsage: %w", err)
445
+
}
446
+
return usage, err
447
+
}
448
+
440
449
func mustRegisterDiskMonitor(path string) {
441
450
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
442
451
Name: "src_disk_space_available_bytes",
443
452
Help: "Amount of free space disk space.",
444
453
ConstLabels: prometheus.Labels{"path": path},
445
454
}, func() float64 {
446
-
var stat unix.Statfs_t
447
-
_ = unix.Statfs(path, &stat)
448
-
return float64(stat.Bavail * uint64(stat.Bsize))
455
+
// I know there is no error handling here, and I don't like it
456
+
// but there was no error handling in the previous version
457
+
// that used Statfs, either, so I'm assuming there's no need for it
458
+
usage, _ := diskUsage(path)
459
+
return float64(usage.Free)
449
460
}))
450
461
451
462
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
···
453
464
Help: "Amount of total disk space.",
454
465
ConstLabels: prometheus.Labels{"path": path},
455
466
}, func() float64 {
456
-
var stat unix.Statfs_t
457
-
_ = unix.Statfs(path, &stat)
458
-
return float64(stat.Blocks * uint64(stat.Bsize))
467
+
// I know there is no error handling here, and I don't like it
468
+
// but there was no error handling in the previous version
469
+
// that used Statfs, either, so I'm assuming there's no need for it
470
+
usage, _ := diskUsage(path)
471
+
return float64(usage.Total)
459
472
}))
460
473
}
461
474
···
589
602
Help: "The total number of search requests that zoekt received",
590
603
})
591
604
)
592
-
593
-
func mustRegisterMemoryMapMetrics(logger sglog.Logger) {
594
-
logger = logger.Scoped("memoryMapMetrics", "")
595
-
596
-
// The memory map metrics are collected via /proc, which
597
-
// is only available on linux-based operating systems.
598
-
599
-
if runtime.GOOS != "linux" {
600
-
return
601
-
}
602
-
603
-
// Instantiate shared FS objects for accessing /proc and /proc/self,
604
-
// and skip metrics registration if we're aren't able to instantiate them
605
-
// for whatever reason.
606
-
607
-
fs, err := procfs.NewDefaultFS()
608
-
if err != nil {
609
-
logger.Debug(
610
-
"skipping registration",
611
-
sglog.String("reason", "failed to initialize proc FS"),
612
-
sglog.String("error", err.Error()),
613
-
)
614
-
615
-
return
616
-
}
617
-
618
-
info, err := fs.Self()
619
-
if err != nil {
620
-
logger.Debug(
621
-
"skipping registration",
622
-
sglog.String("path", path.Join(procfs.DefaultMountPoint, "self")),
623
-
sglog.String("reason", "failed to initialize process info object for current process"),
624
-
sglog.String("error", err.Error()),
625
-
)
626
-
627
-
return
628
-
}
629
-
630
-
// Register Prometheus memory map metrics
631
-
632
-
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
633
-
Name: "proc_metrics_memory_map_max_limit",
634
-
Help: "Upper limit on amount of memory mapped regions a process may have.",
635
-
}, func() float64 {
636
-
vm, err := fs.VM()
637
-
if err != nil {
638
-
logger.Debug(
639
-
"failed to read virtual memory statistics for the current process",
640
-
sglog.String("path", path.Join(procfs.DefaultMountPoint, "sys", "vm")),
641
-
sglog.String("error", err.Error()),
642
-
)
643
-
644
-
return 0
645
-
}
646
-
647
-
if vm.MaxMapCount == nil {
648
-
return 0
649
-
}
650
-
651
-
return float64(*vm.MaxMapCount)
652
-
}))
653
-
654
-
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
655
-
Name: "proc_metrics_memory_map_current_count",
656
-
Help: "Amount of memory mapped regions this process is currently using.",
657
-
}, func() float64 {
658
-
procMaps, err := info.ProcMaps()
659
-
if err != nil {
660
-
logger.Debug(
661
-
"failed to read memory mappings for current process",
662
-
sglog.String("path", path.Join(procfs.DefaultMountPoint, "self", "maps")),
663
-
sglog.String("error", err.Error()),
664
-
)
665
-
666
-
return 0
667
-
}
668
-
669
-
return float64(len(procMaps))
670
-
}))
671
-
}
+84
cmd/zoekt-webserver/main_linux.go
+84
cmd/zoekt-webserver/main_linux.go
···
1
+
package main
2
+
3
+
import (
4
+
"github.com/prometheus/client_golang/prometheus"
5
+
"github.com/prometheus/procfs"
6
+
sglog "github.com/sourcegraph/log"
7
+
"path"
8
+
)
9
+
10
+
func mustRegisterMemoryMapMetrics(logger sglog.Logger) {
11
+
logger = logger.Scoped("memoryMapMetrics", "")
12
+
13
+
// The memory map metrics are collected via /proc, which
14
+
// is only available on linux-based operating systems.
15
+
16
+
// Instantiate shared FS objects for accessing /proc and /proc/self,
17
+
// and skip metrics registration if we're aren't able to instantiate them
18
+
// for whatever reason.
19
+
20
+
fs, err := procfs.NewDefaultFS()
21
+
if err != nil {
22
+
logger.Debug(
23
+
"skipping registration",
24
+
sglog.String("reason", "failed to initialize proc FS"),
25
+
sglog.String("error", err.Error()),
26
+
)
27
+
28
+
return
29
+
}
30
+
31
+
info, err := fs.Self()
32
+
if err != nil {
33
+
logger.Debug(
34
+
"skipping registration",
35
+
sglog.String("path", path.Join(procfs.DefaultMountPoint, "self")),
36
+
sglog.String("reason", "failed to initialize process info object for current process"),
37
+
sglog.String("error", err.Error()),
38
+
)
39
+
40
+
return
41
+
}
42
+
43
+
// Register Prometheus memory map metrics
44
+
45
+
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
46
+
Name: "proc_metrics_memory_map_max_limit",
47
+
Help: "Upper limit on amount of memory mapped regions a process may have.",
48
+
}, func() float64 {
49
+
vm, err := fs.VM()
50
+
if err != nil {
51
+
logger.Debug(
52
+
"failed to read virtual memory statistics for the current process",
53
+
sglog.String("path", path.Join(procfs.DefaultMountPoint, "sys", "vm")),
54
+
sglog.String("error", err.Error()),
55
+
)
56
+
57
+
return 0
58
+
}
59
+
60
+
if vm.MaxMapCount == nil {
61
+
return 0
62
+
}
63
+
64
+
return float64(*vm.MaxMapCount)
65
+
}))
66
+
67
+
prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
68
+
Name: "proc_metrics_memory_map_current_count",
69
+
Help: "Amount of memory mapped regions this process is currently using.",
70
+
}, func() float64 {
71
+
procMaps, err := info.ProcMaps()
72
+
if err != nil {
73
+
logger.Debug(
74
+
"failed to read memory mappings for current process",
75
+
sglog.String("path", path.Join(procfs.DefaultMountPoint, "self", "maps")),
76
+
sglog.String("error", err.Error()),
77
+
)
78
+
79
+
return 0
80
+
}
81
+
82
+
return float64(len(procMaps))
83
+
}))
84
+
}
+9
cmd/zoekt-webserver/main_unix.go
+9
cmd/zoekt-webserver/main_unix.go
+18
cmd/zoekt-webserver/main_unsupported.go
+18
cmd/zoekt-webserver/main_unsupported.go
···
1
+
//go:build !linux
2
+
3
+
package main
4
+
5
+
import (
6
+
sglog "github.com/sourcegraph/log"
7
+
)
8
+
9
+
func mustRegisterMemoryMapMetrics(logger sglog.Logger) {
10
+
// The memory map metrics are collected via /proc, which
11
+
// is only available on linux-based operating systems.
12
+
13
+
// as far as I can tell, Windows does not have the same
14
+
// virtual memory statistics as Linux.
15
+
// For example, Windows does not have the concept of
16
+
// a count of memory maps, and a max number of memory maps
17
+
return
18
+
}
+7
cmd/zoekt-webserver/main_windows.go
+7
cmd/zoekt-webserver/main_windows.go
+9
-4
go.mod
+9
-4
go.mod
···
5
5
github.com/RoaringBitmap/roaring v1.2.3
6
6
github.com/andygrunwald/go-gerrit v0.0.0-20230211083816-04e01d7217b2
7
7
github.com/bmatcuk/doublestar v1.3.4
8
+
github.com/edsrzf/mmap-go v1.1.0
8
9
github.com/fsnotify/fsnotify v1.6.0
9
10
github.com/gfleury/go-bitbucket-v1 v0.0.0-20220418082332-711d7d5e805f
10
11
github.com/go-enry/go-enry/v2 v2.8.3
···
25
26
github.com/prometheus/client_golang v1.14.0
26
27
github.com/prometheus/procfs v0.9.0
27
28
github.com/rs/xid v1.4.0
29
+
github.com/shirou/gopsutil/v3 v3.23.1
28
30
github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71
29
31
github.com/sourcegraph/log v0.0.0-20230203201409-49ac5ad3f2ce
30
32
github.com/sourcegraph/mountinfo v0.0.0-20230106004439-7026e28cef67
···
42
44
go.opentelemetry.io/otel/trace v1.13.0
43
45
go.uber.org/atomic v1.10.0
44
46
go.uber.org/automaxprocs v1.5.1
45
-
golang.org/x/net v0.6.0
47
+
golang.org/x/net v0.7.0
46
48
golang.org/x/oauth2 v0.5.0
47
49
golang.org/x/sync v0.1.0
48
50
golang.org/x/sys v0.5.0
···
55
57
cloud.google.com/go/compute/metadata v0.2.3 // indirect
56
58
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
57
59
github.com/Microsoft/go-winio v0.6.0 // indirect
58
-
github.com/ProtonMail/go-crypto v0.0.0-20230201104953-d1d05f4e2bfb // indirect
60
+
github.com/ProtonMail/go-crypto v0.0.0-20230214155104-81033d7f4442 // indirect
59
61
github.com/acomagu/bufpipe v1.0.3 // indirect
60
62
github.com/beorn7/perks v1.0.1 // indirect
61
63
github.com/bits-and-blooms/bitset v1.5.0 // indirect
···
73
75
github.com/go-git/go-billy/v5 v5.4.1 // indirect
74
76
github.com/go-logr/logr v1.2.3 // indirect
75
77
github.com/go-logr/stdr v1.2.2 // indirect
78
+
github.com/go-ole/go-ole v1.2.6 // indirect
76
79
github.com/gogo/protobuf v1.3.2 // indirect
77
80
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
78
81
github.com/golang/protobuf v1.5.2 // indirect
79
82
github.com/google/go-querystring v1.1.0 // indirect
80
83
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
81
84
github.com/google/uuid v1.3.0 // indirect
82
-
github.com/googleapis/enterprise-certificate-proxy v0.2.2 // indirect
85
+
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
83
86
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
84
87
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 // indirect
85
88
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
···
96
99
github.com/moby/sys/mountinfo v0.6.2 // indirect
97
100
github.com/mschoch/smat v0.2.0 // indirect
98
101
github.com/pjbgf/sha1cd v0.2.3 // indirect
102
+
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
99
103
github.com/prometheus/client_model v0.3.0 // indirect
100
104
github.com/prometheus/common v0.39.0 // indirect
101
105
github.com/rogpeppe/go-internal v1.9.0 // indirect
102
106
github.com/sergi/go-diff v1.3.1 // indirect
103
107
github.com/skeema/knownhosts v1.1.0 // indirect
104
108
github.com/xanzy/ssh-agent v0.3.3 // indirect
109
+
github.com/yusufpapurcu/wmi v1.2.2 // indirect
105
110
go.opencensus.io v0.24.0 // indirect
106
111
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect
107
112
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
···
112
117
golang.org/x/text v0.7.0 // indirect
113
118
golang.org/x/time v0.3.0 // indirect
114
119
golang.org/x/tools v0.6.0 // indirect
115
-
google.golang.org/api v0.109.0 // indirect
120
+
google.golang.org/api v0.110.0 // indirect
116
121
google.golang.org/appengine v1.6.7 // indirect
117
122
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
118
123
google.golang.org/grpc v1.53.0 // indirect
+30
go.sum
+30
go.sum
···
57
57
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
58
58
github.com/ProtonMail/go-crypto v0.0.0-20230201104953-d1d05f4e2bfb h1:Vx1Bw/nGULx+FuY7Sw+8ZDpOx9XOdA+mOfo678SqkbU=
59
59
github.com/ProtonMail/go-crypto v0.0.0-20230201104953-d1d05f4e2bfb/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
60
+
github.com/ProtonMail/go-crypto v0.0.0-20230214155104-81033d7f4442 h1:OUJ54Fkd+AQXYmr9eOUxZfWNzpK3/e/KD40qa2rKHS4=
61
+
github.com/ProtonMail/go-crypto v0.0.0-20230214155104-81033d7f4442/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
60
62
github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
61
63
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
62
64
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
···
85
87
github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
86
88
github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
87
89
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
90
+
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
88
91
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
89
92
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
90
93
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
···
123
126
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
124
127
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
125
128
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
129
+
github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
130
+
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
126
131
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
127
132
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
128
133
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
···
181
186
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
182
187
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
183
188
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
189
+
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
190
+
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
184
191
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
185
192
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
186
193
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
···
269
276
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
270
277
github.com/googleapis/enterprise-certificate-proxy v0.2.2 h1:jUqbmxlR+gGPQq/uvQviKpS1bSQecfs2t7o6F14sk9s=
271
278
github.com/googleapis/enterprise-certificate-proxy v0.2.2/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
279
+
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
280
+
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
272
281
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
273
282
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
274
283
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
···
277
286
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
278
287
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA=
279
288
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
289
+
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
280
290
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
281
291
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
282
292
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 h1:1JYBfzqrWPcCclBwxFCPAou9n+q86mfnu7NAeHfte7A=
···
343
353
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
344
354
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
345
355
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
356
+
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
346
357
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
347
358
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
348
359
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
···
404
415
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
405
416
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
406
417
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
418
+
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
419
+
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
420
+
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
421
+
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
407
422
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
408
423
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
409
424
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
···
429
444
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
430
445
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
431
446
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
447
+
github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=
448
+
github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI=
449
+
github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4=
450
+
github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA=
432
451
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
433
452
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
434
453
github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
···
461
480
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
462
481
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
463
482
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
483
+
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
484
+
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
464
485
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
465
486
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
466
487
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
···
493
514
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
494
515
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
495
516
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
517
+
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
518
+
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
496
519
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
497
520
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
498
521
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
···
628
651
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
629
652
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
630
653
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
654
+
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
655
+
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
631
656
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
632
657
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
633
658
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
···
663
688
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
664
689
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
665
690
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
691
+
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
666
692
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
667
693
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
668
694
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
···
684
710
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
685
711
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
686
712
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
713
+
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
687
714
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
688
715
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
689
716
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
···
704
731
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
705
732
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
706
733
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
734
+
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
707
735
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
708
736
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
709
737
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
···
806
834
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
807
835
google.golang.org/api v0.109.0 h1:sW9hgHyX497PP5//NUM7nqfV8D0iDfBApqq7sOh1XR8=
808
836
google.golang.org/api v0.109.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
837
+
google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU=
838
+
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
809
839
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
810
840
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
811
841
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-62
indexfile_other.go
-62
indexfile_other.go
···
1
-
// Copyright 2016 Google Inc. All rights reserved.
2
-
//
3
-
// Licensed under the Apache License, Version 2.0 (the "License");
4
-
// you may not use this file except in compliance with the License.
5
-
// You may obtain a copy of the License at
6
-
//
7
-
// http://www.apache.org/licenses/LICENSE-2.0
8
-
//
9
-
// Unless required by applicable law or agreed to in writing, software
10
-
// distributed under the License is distributed on an "AS IS" BASIS,
11
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
-
// See the License for the specific language governing permissions and
13
-
// limitations under the License.
14
-
15
-
//go:build !linux && !darwin
16
-
// +build !linux,!darwin
17
-
18
-
package zoekt
19
-
20
-
import (
21
-
"fmt"
22
-
"os"
23
-
)
24
-
25
-
// NewIndexFile returns a new index file. The index file takes
26
-
// ownership of the passed in file, and may close it.
27
-
func NewIndexFile(f *os.File) (IndexFile, error) {
28
-
return &indexFileFromOS{f}, nil
29
-
}
30
-
31
-
type indexFileFromOS struct {
32
-
f *os.File
33
-
}
34
-
35
-
func (f *indexFileFromOS) Read(off, sz uint32) ([]byte, error) {
36
-
r := make([]byte, sz)
37
-
_, err := f.f.ReadAt(r, int64(off))
38
-
return r, err
39
-
}
40
-
41
-
func (f indexFileFromOS) Size() (uint32, error) {
42
-
fi, err := f.f.Stat()
43
-
if err != nil {
44
-
return 0, err
45
-
}
46
-
47
-
sz := fi.Size()
48
-
49
-
if sz >= maxUInt32 {
50
-
return 0, fmt.Errorf("overflow")
51
-
}
52
-
53
-
return uint32(sz), nil
54
-
}
55
-
56
-
func (f indexFileFromOS) Close() {
57
-
f.f.Close()
58
-
}
59
-
60
-
func (f indexFileFromOS) Name() string {
61
-
return f.f.Name()
62
-
}
+25
-10
indexfile_unix.go
indexfile.go
+25
-10
indexfile_unix.go
indexfile.go
···
12
12
// See the License for the specific language governing permissions and
13
13
// limitations under the License.
14
14
15
-
//go:build linux || darwin
16
-
17
15
package zoekt
18
16
19
17
import (
20
18
"fmt"
21
19
"log"
22
20
"os"
21
+
"runtime"
23
22
24
-
"golang.org/x/sys/unix"
23
+
// cross-platform memory-mapped file package.
24
+
// Benchmarks the same speed as syscall/unix Mmap
25
+
// see https://github.com/peterguy/benchmark-mmap
26
+
mmap "github.com/edsrzf/mmap-go"
25
27
)
26
28
27
29
type mmapedIndexFile struct {
28
30
name string
29
31
size uint32
30
-
data []byte
32
+
data mmap.MMap
31
33
}
32
34
33
35
func (f *mmapedIndexFile) Read(off, sz uint32) ([]byte, error) {
···
46
48
}
47
49
48
50
func (f *mmapedIndexFile) Close() {
49
-
if err := unix.Munmap(f.data); err != nil {
50
-
log.Printf("WARN failed to Munmap %s: %v", f.name, err)
51
+
if err := f.data.Unmap(); err != nil {
52
+
log.Printf("WARN failed to memory unmap %s: %v", f.name, err)
53
+
}
54
+
}
55
+
56
+
func bufferSize(f *mmapedIndexFile) int {
57
+
// On Unix/Linux, mmap likes to allocate memory in
58
+
// page-sized chunks, so round up to the OS page size.
59
+
// mmap will zero-fill the extra bytes.
60
+
// On Windows, the Windows API CreateFileMapping method
61
+
// requires a buffer the same size as the file.
62
+
bsize := int(f.size)
63
+
if runtime.GOOS != "windows" {
64
+
pagesize := os.Getpagesize() - 1
65
+
bsize = (bsize + pagesize) &^ pagesize
51
66
}
67
+
return bsize
52
68
}
53
69
54
70
// NewIndexFile returns a new index file. The index file takes
···
70
86
size: uint32(sz),
71
87
}
72
88
73
-
rounded := (r.size + 4095) &^ 4095
74
-
r.data, err = unix.Mmap(int(f.Fd()), 0, int(rounded), unix.PROT_READ, unix.MAP_SHARED)
89
+
r.data, err = mmap.MapRegion(f, bufferSize(r), mmap.RDONLY, 0, 0)
75
90
if err != nil {
76
-
return nil, err
91
+
return nil, fmt.Errorf("NewIndexFile: unable to memory map %s: %w", f.Name(), err)
77
92
}
78
93
79
-
return r, err
94
+
return r, nil
80
95
}
-7
tombstones.go
-7
tombstones.go
···
6
6
"os"
7
7
"path/filepath"
8
8
"strconv"
9
-
10
-
"golang.org/x/sys/unix"
11
9
)
12
10
13
11
// ShardMergingEnabled returns true if SRC_ENABLE_SHARD_MERGING is set to true.
···
103
101
104
102
// umask holds the Umask of the current process
105
103
var umask os.FileMode
106
-
107
-
func init() {
108
-
umask = os.FileMode(unix.Umask(0))
109
-
unix.Umask(int(umask))
110
-
}
+14
tombstones_unix.go
+14
tombstones_unix.go