fork of https://github.com/sourcegraph/zoekt
revert use of mmap-go (#706)
We have a suspicion that when we switched to using the mmap-go library
it contributed to an issue where zoekt-webserver stops responding on
some linux versions. Our suspicion has something to do from us
hardcoding the page size to 4k to asking the kernel and how that
interacts with THP. Given we don't actually build for windows anymore,
we are partially reverting the mmap changes from
7424ac84bab3ec9ae247339909ffd84e5e3b1338
Test Plan: go test ./... on darwin. Then CI for linux testing.
author
Keegan Carruthers-Smith
committer
GitHub
date
(Nov 29, 2023, 2:21 PM +0100)
commit
0d03621d
0d03621d45a3dd9a6f2600596a81d50f8d514392
parent
8abf874f
8abf874fe132769de3d440fe89948191e6929551
+72
-28
-1
go.mod
-1
go.mod
···
7
7
github.com/andygrunwald/go-gerrit v0.0.0-20230628115649-c44fe2fbf2ca
8
8
github.com/bmatcuk/doublestar v1.3.4
9
9
github.com/dustin/go-humanize v1.0.1
10
-
github.com/edsrzf/mmap-go v1.1.0
11
10
github.com/felixge/fgprof v0.9.3
12
11
github.com/fsnotify/fsnotify v1.6.0
13
12
github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751
-2
go.sum
-2
go.sum
···
78
78
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
79
79
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
80
80
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
81
-
github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
82
-
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
83
81
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
84
82
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
85
83
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+10
-25
indexfile.go
indexfile_unix.go
+10
-25
indexfile.go
indexfile_unix.go
···
12
12
// See the License for the specific language governing permissions and
13
13
// limitations under the License.
14
14
15
+
//go:build linux || darwin
16
+
15
17
package zoekt
16
18
17
19
import (
18
20
"fmt"
19
21
"log"
20
22
"os"
21
-
"runtime"
22
23
23
-
// cross-platform memory-mapped file package.
24
-
// Benchmarks the same speed as syscall/unix Mmap
25
-
// see https://github.com/peterguy/benchmark-mmap
26
-
mmap "github.com/edsrzf/mmap-go"
24
+
"golang.org/x/sys/unix"
27
25
)
28
26
29
27
type mmapedIndexFile struct {
30
28
name string
31
29
size uint32
32
-
data mmap.MMap
30
+
data []byte
33
31
}
34
32
35
33
func (f *mmapedIndexFile) Read(off, sz uint32) ([]byte, error) {
···
48
46
}
49
47
50
48
func (f *mmapedIndexFile) Close() {
51
-
if err := f.data.Unmap(); err != nil {
52
-
log.Printf("WARN failed to memory unmap %s: %v", f.name, err)
49
+
if err := unix.Munmap(f.data); err != nil {
50
+
log.Printf("WARN failed to Munmap %s: %v", f.name, err)
53
51
}
54
52
}
55
53
56
-
func bufferSize(f *mmapedIndexFile) int {
57
-
// On Unix/Linux, mmap likes to allocate memory in
58
-
// page-sized chunks, so round up to the OS page size.
59
-
// mmap will zero-fill the extra bytes.
60
-
// On Windows, the Windows API CreateFileMapping method
61
-
// requires a buffer the same size as the file.
62
-
bsize := int(f.size)
63
-
if runtime.GOOS != "windows" {
64
-
pagesize := os.Getpagesize() - 1
65
-
bsize = (bsize + pagesize) &^ pagesize
66
-
}
67
-
return bsize
68
-
}
69
-
70
54
// NewIndexFile returns a new index file. The index file takes
71
55
// ownership of the passed in file, and may close it.
72
56
func NewIndexFile(f *os.File) (IndexFile, error) {
···
86
70
size: uint32(sz),
87
71
}
88
72
89
-
r.data, err = mmap.MapRegion(f, bufferSize(r), mmap.RDONLY, 0, 0)
73
+
rounded := (r.size + 4095) &^ 4095
74
+
r.data, err = unix.Mmap(int(f.Fd()), 0, int(rounded), unix.PROT_READ, unix.MAP_SHARED)
90
75
if err != nil {
91
-
return nil, fmt.Errorf("NewIndexFile: unable to memory map %s: %w", f.Name(), err)
76
+
return nil, err
92
77
}
93
78
94
-
return r, nil
79
+
return r, err
95
80
}
+62
indexfile_other.go
+62
indexfile_other.go
···
1
+
// Copyright 2016 Google Inc. All rights reserved.
2
+
//
3
+
// Licensed under the Apache License, Version 2.0 (the "License");
4
+
// you may not use this file except in compliance with the License.
5
+
// You may obtain a copy of the License at
6
+
//
7
+
// http://www.apache.org/licenses/LICENSE-2.0
8
+
//
9
+
// Unless required by applicable law or agreed to in writing, software
10
+
// distributed under the License is distributed on an "AS IS" BASIS,
11
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+
// See the License for the specific language governing permissions and
13
+
// limitations under the License.
14
+
15
+
//go:build !linux && !darwin
16
+
// +build !linux,!darwin
17
+
18
+
package zoekt
19
+
20
+
import (
21
+
"fmt"
22
+
"os"
23
+
)
24
+
25
+
// NewIndexFile returns a new index file. The index file takes
26
+
// ownership of the passed in file, and may close it.
27
+
func NewIndexFile(f *os.File) (IndexFile, error) {
28
+
return &indexFileFromOS{f}, nil
29
+
}
30
+
31
+
type indexFileFromOS struct {
32
+
f *os.File
33
+
}
34
+
35
+
func (f *indexFileFromOS) Read(off, sz uint32) ([]byte, error) {
36
+
r := make([]byte, sz)
37
+
_, err := f.f.ReadAt(r, int64(off))
38
+
return r, err
39
+
}
40
+
41
+
func (f indexFileFromOS) Size() (uint32, error) {
42
+
fi, err := f.f.Stat()
43
+
if err != nil {
44
+
return 0, err
45
+
}
46
+
47
+
sz := fi.Size()
48
+
49
+
if sz >= maxUInt32 {
50
+
return 0, fmt.Errorf("overflow")
51
+
}
52
+
53
+
return uint32(sz), nil
54
+
}
55
+
56
+
func (f indexFileFromOS) Close() {
57
+
f.f.Close()
58
+
}
59
+
60
+
func (f indexFileFromOS) Name() string {
61
+
return f.f.Name()
62
+
}