fork of https://github.com/sourcegraph/zoekt
monitoring: add mount_point_info Prometheus metric (#434)
Co-authored-by: Keegan Carruthers-Smith <keegan.csmith@gmail.com>
author
Geoffrey Gilmore
co-author
Keegan Carruthers-Smith
committer
GitHub
date
(Oct 13, 2022, 9:00 AM -0700)
commit
e091ee33
e091ee338e7d493e0e274916f69d11c17c61f3f0
parent
022743fd
022743fd1ac07e5ee0fbeb88db7324b52944cce9
+469
-3
+4
cmd/zoekt-sourcegraph-indexserver/main.go
+4
cmd/zoekt-sourcegraph-indexserver/main.go
···
33
33
"github.com/prometheus/client_golang/prometheus"
34
34
"github.com/prometheus/client_golang/prometheus/promauto"
35
35
sglog "github.com/sourcegraph/log"
36
+
"github.com/sourcegraph/zoekt/internal/mountinfo"
36
37
"go.uber.org/automaxprocs/maxprocs"
37
38
"golang.org/x/net/trace"
38
39
···
1063
1064
Hostname: conf.hostname,
1064
1065
}
1065
1066
go oc.Run()
1067
+
1068
+
logger := sglog.Scoped("metricsRegistration", "")
1069
+
mountinfo.MustRegisterNewMountPointInfoMetric(logger, map[string]string{"indexDir": conf.index})
1066
1070
1067
1071
s.Run()
1068
1072
return nil
+7
-2
cmd/zoekt-webserver/main.go
+7
-2
cmd/zoekt-webserver/main.go
···
38
38
"github.com/sourcegraph/zoekt"
39
39
"github.com/sourcegraph/zoekt/build"
40
40
"github.com/sourcegraph/zoekt/debugserver"
41
+
"github.com/sourcegraph/zoekt/internal/mountinfo"
41
42
"github.com/sourcegraph/zoekt/internal/profiler"
42
43
"github.com/sourcegraph/zoekt/internal/tracer"
43
44
"github.com/sourcegraph/zoekt/query"
···
180
181
181
182
mustRegisterDiskMonitor(*index)
182
183
183
-
mmapLogger := sglog.Scoped("zoekt_webserver_proc_metrics_memory_map", "")
184
-
mustRegisterMemoryMapMetrics(mmapLogger)
184
+
metricsLogger := sglog.Scoped("metricsRegistration", "")
185
+
186
+
mustRegisterMemoryMapMetrics(metricsLogger)
187
+
mountinfo.MustRegisterNewMountPointInfoMetric(metricsLogger, map[string]string{"indexDir": *index})
185
188
186
189
// Do not block on loading shards so we can become partially available
187
190
// sooner. Otherwise on large instances zoekt can be unavailable on the
···
510
513
)
511
514
512
515
func mustRegisterMemoryMapMetrics(logger sglog.Logger) {
516
+
logger = logger.Scoped("memoryMapMetrics", "")
517
+
513
518
// The memory map metrics are collected via /proc, which
514
519
// is only available on linux-based operating systems.
515
520
-1
indexfile_unix.go
-1
indexfile_unix.go
+214
internal/mountinfo/mountinfo.go
+214
internal/mountinfo/mountinfo.go
···
1
+
package mountinfo
2
+
3
+
import (
4
+
"errors"
5
+
"fmt"
6
+
"os"
7
+
"path/filepath"
8
+
"runtime"
9
+
"strings"
10
+
11
+
"github.com/prometheus/client_golang/prometheus"
12
+
"github.com/prometheus/client_golang/prometheus/promauto"
13
+
sglog "github.com/sourcegraph/log"
14
+
"golang.org/x/sys/unix"
15
+
)
16
+
17
+
// defaultSysMountPoint is the common mount point for the sysfs pseudo-filesystem.
18
+
const defaultSysMountPoint = "/sys"
19
+
20
+
// MustRegisterNewMountPointInfoMetric registers a Prometheus metric named "mount_point_info" that
21
+
// contains the names of the block storage devices that back each of the requested mounts.
22
+
//
23
+
// Mounts is a set of name -> file path mappings (example: {"indexDir": "/home/.zoekt"}).
24
+
//
25
+
// The metric "mount_point_info" has a constant value of 1 and two labels:
26
+
// - mount_name: caller-provided name for the given mount (example: "indexDir")
27
+
// - device: name of the block device that backs the given mount file path (example: "sdb")
28
+
//
29
+
// This metric only works on Linux-based operating systems that have access to the sysfs pseudo-filesystem.
30
+
// On all other operating systems, this metric will not emit any values.
31
+
func MustRegisterNewMountPointInfoMetric(logger sglog.Logger, mounts map[string]string) {
32
+
logger = logger.Scoped("mountPointInfo", "registration logic for mount_point_info Prometheus metric")
33
+
34
+
metric := promauto.NewGaugeVec(prometheus.GaugeOpts{
35
+
Name: "mount_point_info",
36
+
Help: "An info metric with a constant '1' value that contains mount_name, device mappings",
37
+
}, []string{"mount_name", "device"})
38
+
39
+
// This device discovery logic relies on the sysfs pseudo-filesystem, which only exists
40
+
// on linux.
41
+
//
42
+
// See https://en.wikipedia.org/wiki/Sysfs for more information.
43
+
if runtime.GOOS != "linux" {
44
+
return
45
+
}
46
+
47
+
for name, filePath := range mounts {
48
+
// for each <mountName>:<mountFilePath> pairing,
49
+
// discover the name of the block device that stores <mountFilePath>.
50
+
discoveryLogger := logger.Scoped("deviceNameDiscovery", "").With(
51
+
sglog.String("mountName", name),
52
+
sglog.String("mountFilePath", filePath),
53
+
)
54
+
55
+
device, err := discoverDeviceName(discoveryLogger, discoverDeviceNameConfig{}, filePath)
56
+
if err != nil {
57
+
discoveryLogger.Warn("skipping metric registration",
58
+
sglog.String("reason", "failed to discover device name"),
59
+
sglog.Error(err),
60
+
)
61
+
62
+
continue
63
+
}
64
+
65
+
discoveryLogger.Debug("discovered device name",
66
+
sglog.String("deviceName", device),
67
+
)
68
+
69
+
metric.WithLabelValues(name, device).Set(1)
70
+
}
71
+
}
72
+
73
+
type discoverDeviceNameConfig struct {
74
+
// sysfsMountPoint is the location of the sysfs mount point.
75
+
// If empty, defaultSysMountPoint will be used instead.
76
+
sysfsMountPoint string
77
+
78
+
// getDeviceNumber, if non-nil, is the function that will be used to find
79
+
// the number of the block device that stores the specified file.
80
+
// If getDeviceNumber is nil, mountinfo.getDeviceNumber will be used instead.
81
+
getDeviceNumber func(filePath string) (major uint32, minor uint32, err error)
82
+
}
83
+
84
+
// discoverDeviceName returns the name of the block device that filePath is
85
+
// stored on.
86
+
func discoverDeviceName(logger sglog.Logger, config discoverDeviceNameConfig, filePath string) (string, error) {
87
+
// Note: It's quite involved to implement the device discovery logic for
88
+
// every possible kind of storage device (e.x. logical volumes, NFS, etc.) See
89
+
// https://unix.stackexchange.com/a/11312 for more information.
90
+
//
91
+
// As a result, this logic will only work correctly for filePaths that are either:
92
+
// - stored directly on a block device
93
+
// - stored on a block device's partition
94
+
//
95
+
// For all other device types, this logic will either:
96
+
// - return an incorrect device name
97
+
// - return an error
98
+
//
99
+
// This logic was implemented from information gathered from the following sources (amongst others):
100
+
// - "The Linux Programming Interface" by Michael Kerrisk: Chapter 14
101
+
// - "Linux Kernel Development" by Robert Love: Chapters 13, 17
102
+
// - https://man7.org/linux/man-pages/man5/sysfs.5.html
103
+
// - https://en.wikipedia.org/wiki/Sysfs
104
+
// - https://unix.stackexchange.com/a/11312
105
+
// - https://www.kernel.org/doc/ols/2005/ols2005v1-pages-321-334.pdf
106
+
107
+
getDeviceNumber := getDeviceNumber
108
+
if config.getDeviceNumber != nil {
109
+
getDeviceNumber = config.getDeviceNumber
110
+
}
111
+
112
+
sysfsMountPoint := defaultSysMountPoint
113
+
if config.sysfsMountPoint != "" {
114
+
sysfsMountPoint = config.sysfsMountPoint
115
+
}
116
+
117
+
sysfsMountPoint = filepath.Clean(sysfsMountPoint)
118
+
119
+
// the provided sysfs mountpoint could itself be a symlink, so we
120
+
// resolve it immediately so that future file path
121
+
// evaluations / massaging doesn't break
122
+
sysfsMountPoint, err := filepath.EvalSymlinks(sysfsMountPoint)
123
+
if err != nil {
124
+
return "", fmt.Errorf("verifying sysfs mountpoint %q: failed to resolve symlink %w", sysfsMountPoint, err)
125
+
}
126
+
127
+
major, minor, err := getDeviceNumber(filePath)
128
+
if err != nil {
129
+
return "", fmt.Errorf("discovering device number: %w", err)
130
+
}
131
+
132
+
// Represent the number in <major>:<minor> format.
133
+
deviceNumber := fmt.Sprintf("%d:%d", major, minor)
134
+
135
+
logger.Debug(
136
+
"discovered device number",
137
+
sglog.String("deviceNumber", deviceNumber),
138
+
)
139
+
140
+
// /sys/dev/block/<device_number> symlinks to /sys/devices/.../block/.../<deviceName>
141
+
symlink := filepath.Join(sysfsMountPoint, "dev", "block", deviceNumber)
142
+
143
+
devicePath, err := filepath.EvalSymlinks(symlink)
144
+
if err != nil {
145
+
return "", fmt.Errorf("discovering device path: failed to evaluate sysfs symlink %q: %w", symlink, err)
146
+
}
147
+
148
+
devicePath, err = filepath.Abs(devicePath)
149
+
if err != nil {
150
+
return "", fmt.Errorf("discovering device path: failed to massage device path %q to absolute path: %w", devicePath, err)
151
+
}
152
+
153
+
logger.Debug("discovered device path",
154
+
sglog.String("devicePath", devicePath),
155
+
)
156
+
157
+
// Check to see if devicePath points to a disk partition. If so, we need to find the parent
158
+
// device.
159
+
160
+
// massage the sysfs folder name to ensure that it always ends in a '/'
161
+
// so that strings.HasPrefix does what we expect when checking to see if
162
+
// we're still under the /sys sub-folder
163
+
sysFolderPrefix := strings.TrimSuffix(sysfsMountPoint, string(os.PathSeparator))
164
+
sysFolderPrefix = sysFolderPrefix + string(os.PathSeparator)
165
+
166
+
for {
167
+
if !strings.HasPrefix(devicePath, sysFolderPrefix) {
168
+
// ensure that we're still under the /sys/ sub-folder
169
+
return "", fmt.Errorf("validating device path: device path %q isn't a subpath of %q", devicePath, sysFolderPrefix)
170
+
}
171
+
172
+
_, err := os.Stat(filepath.Join(devicePath, "partition"))
173
+
if errors.Is(err, os.ErrNotExist) {
174
+
break
175
+
}
176
+
177
+
parent := filepath.Dir(devicePath)
178
+
179
+
logger.Debug("changing device path",
180
+
sglog.String("reason", "oldDevicePath represents a disk partition"),
181
+
182
+
sglog.String("oldDevicePath", devicePath),
183
+
sglog.String("newDevicePath", parent),
184
+
)
185
+
186
+
devicePath = parent
187
+
}
188
+
189
+
// If this device is a block device, its device path should have a symlink
190
+
// to the block subsystem.
191
+
192
+
subsystemPath, err := filepath.EvalSymlinks(filepath.Join(devicePath, "subsystem"))
193
+
if err != nil {
194
+
return "", fmt.Errorf("validating device path: failed to discover subsystem that device (path %q) is part of: %w", devicePath, err)
195
+
}
196
+
197
+
if filepath.Base(subsystemPath) != "block" {
198
+
return "", fmt.Errorf("validating device path: device (path %q) is not part of the block subsystem", devicePath)
199
+
}
200
+
201
+
device := filepath.Base(devicePath)
202
+
return filepath.Base(device), nil
203
+
}
204
+
205
+
func getDeviceNumber(filePath string) (major uint32, minor uint32, err error) {
206
+
var stat unix.Stat_t
207
+
err = unix.Stat(filePath, &stat)
208
+
if err != nil {
209
+
return 0, 0, fmt.Errorf("failed to stat %q: %w", filePath, err)
210
+
}
211
+
212
+
major, minor = unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))
213
+
return major, minor, nil
214
+
}
+29
internal/mountinfo/mountinfo_linux_test.go
+29
internal/mountinfo/mountinfo_linux_test.go
···
1
+
//go:build linux
2
+
3
+
package mountinfo
4
+
5
+
import (
6
+
"log"
7
+
"os"
8
+
"testing"
9
+
10
+
"github.com/sourcegraph/log/logtest"
11
+
)
12
+
13
+
func Test_DeviceName_SmokeTest(t *testing.T) {
14
+
// A simple smoke test to verify that we can find the storage device
15
+
// for the current working directory.
16
+
logger := logtest.Scoped(t)
17
+
18
+
filePath, err := os.Getwd()
19
+
if err != nil {
20
+
log.Fatalf("getting current working directory: %s", err)
21
+
}
22
+
23
+
device, err := discoverDeviceName(logger, discoverDeviceNameConfig{}, filePath)
24
+
if err != nil {
25
+
t.Fatalf("discovering device name for file path %q: %s", filePath, err)
26
+
}
27
+
28
+
t.Logf("discovered device name %q for file path %q", device, filePath)
29
+
}
+167
internal/mountinfo/mountinfo_test.go
+167
internal/mountinfo/mountinfo_test.go
···
1
+
package mountinfo
2
+
3
+
import (
4
+
"archive/tar"
5
+
"compress/gzip"
6
+
"io"
7
+
"os"
8
+
"path/filepath"
9
+
"testing"
10
+
11
+
"github.com/google/go-cmp/cmp"
12
+
"github.com/sourcegraph/log/logtest"
13
+
)
14
+
15
+
func Test_DeviceName_Snapshots(t *testing.T) {
16
+
// This test uses sysfs snapshots from real linux machines to ensure
17
+
// that the device discovery logic returns the expected device name.
18
+
19
+
for _, test := range []struct {
20
+
name string
21
+
22
+
sysfsTarballFile string
23
+
24
+
deviceMajor uint32
25
+
deviceMinor uint32
26
+
27
+
expectedDeviceName string
28
+
}{
29
+
{
30
+
name: "should find the name of the block device that backs a partition (vda1 -> vda)",
31
+
32
+
// ( lsblk output from the snapshotted machine)
33
+
// ~ # lsblk
34
+
// NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
35
+
// nbd0 43:0 0 0B 0 disk
36
+
// nbd1 43:32 0 0B 0 disk
37
+
// nbd2 43:64 0 0B 0 disk
38
+
// nbd3 43:96 0 0B 0 disk
39
+
// nbd4 43:128 0 0B 0 disk
40
+
// nbd5 43:160 0 0B 0 disk
41
+
// nbd6 43:192 0 0B 0 disk
42
+
// nbd7 43:224 0 0B 0 disk
43
+
// vda 254:0 0 59.6G 0 disk
44
+
// └─vda1 254:1 0 59.6G 0 part /etc/hosts # test targets this partition
45
+
// /etc/hostname
46
+
// /etc/resolv.conf
47
+
// /data/index
48
+
// nbd8 43:256 0 0B 0 disk
49
+
// nbd9 43:288 0 0B 0 disk
50
+
// nbd10 43:320 0 0B 0 disk
51
+
// nbd11 43:352 0 0B 0 disk
52
+
// nbd12 43:384 0 0B 0 disk
53
+
// nbd13 43:416 0 0B 0 disk
54
+
// nbd14 43:448 0 0B 0 disk
55
+
// nbd15 43:480 0 0B 0 disk
56
+
57
+
sysfsTarballFile: "sysfs.vda1.tar.gz",
58
+
59
+
deviceMajor: 254, // points to vda1 partition
60
+
deviceMinor: 1,
61
+
62
+
expectedDeviceName: "vda",
63
+
},
64
+
} {
65
+
test := test
66
+
67
+
t.Run(t.Name(), func(t *testing.T) {
68
+
t.Parallel()
69
+
70
+
// provide a custom sysfs location so that we can point the test
71
+
// at our sysfs snapshot
72
+
mockSysFSDir := filepath.Join(t.TempDir(), "sys")
73
+
74
+
// unpack sysfs tarball
75
+
tarball := filepath.Join("testdata", test.sysfsTarballFile)
76
+
decompressSysFSTarball(t, tarball, mockSysFSDir)
77
+
78
+
logger := logtest.Scoped(t)
79
+
80
+
mockGetDeviceNumber := func(_ string) (major uint32, minor uint32, err error) {
81
+
return test.deviceMajor, test.deviceMinor, nil
82
+
}
83
+
fakeFilePath := "doesn't matter" // the file path itself doesn't matter since we hard-code the device number
84
+
85
+
// execute the test with our injected mocks
86
+
actualDeviceName, err := discoverDeviceName(
87
+
logger,
88
+
discoverDeviceNameConfig{
89
+
sysfsMountPoint: mockSysFSDir,
90
+
getDeviceNumber: mockGetDeviceNumber,
91
+
},
92
+
fakeFilePath,
93
+
)
94
+
95
+
if err != nil {
96
+
t.Fatalf("discovering device name for file path %q: %s", fakeFilePath, err)
97
+
}
98
+
99
+
// verify that the discovered device name is the one that we expect
100
+
101
+
if diff := cmp.Diff(test.expectedDeviceName, actualDeviceName); diff != "" {
102
+
t.Fatalf("recieved unexpected device name (-want +got):\n%s", diff)
103
+
}
104
+
})
105
+
}
106
+
}
107
+
108
+
func decompressSysFSTarball(t *testing.T, tarball, outputFolder string) {
109
+
t.Helper()
110
+
111
+
file, err := os.Open(tarball)
112
+
if err != nil {
113
+
t.Fatalf("opening tarball %q: %s", tarball, err)
114
+
}
115
+
116
+
defer file.Close()
117
+
118
+
gz, err := gzip.NewReader(file)
119
+
if err != nil {
120
+
t.Fatalf("initialzing gzip reader: %s", err)
121
+
}
122
+
123
+
reader := tar.NewReader(gz)
124
+
125
+
for {
126
+
header, err := reader.Next()
127
+
if err == io.EOF {
128
+
break
129
+
}
130
+
131
+
if err != nil {
132
+
t.Fatalf("intializing tar reader: %s", err)
133
+
}
134
+
135
+
outputFile := filepath.Join(outputFolder, header.Name)
136
+
137
+
switch header.Typeflag {
138
+
case tar.TypeDir:
139
+
err := os.MkdirAll(outputFile, os.FileMode(header.Mode))
140
+
if err != nil {
141
+
t.Fatalf("creating directory %q: %s", outputFile, err)
142
+
}
143
+
144
+
case tar.TypeSymlink:
145
+
err := os.Symlink(header.Linkname, outputFile)
146
+
if err != nil {
147
+
t.Fatalf("creating symlink (%q -> %q): %s", outputFile, header.Linkname, err)
148
+
}
149
+
150
+
case tar.TypeReg:
151
+
f, err := os.OpenFile(outputFile, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
152
+
if err != nil {
153
+
t.Fatalf("creating file %q: %s", outputFile, err)
154
+
}
155
+
156
+
_, err = io.Copy(f, reader)
157
+
if err != nil {
158
+
t.Fatalf("writing file %q: %s", outputFile, err)
159
+
}
160
+
161
+
f.Close()
162
+
163
+
default:
164
+
t.Fatalf("encounted unknown file header type (%d) for file %q", header.Typeflag, header.Name)
165
+
}
166
+
}
167
+
}
+48
internal/mountinfo/testdata/snapshot.sh
+48
internal/mountinfo/testdata/snapshot.sh
···
1
+
#!/usr/bin/env bash
2
+
3
+
# Create a tarball of this system's sysfs filesystem + place it in the home directory.
4
+
#
5
+
# (This special logic is necessary since /sys is a pseudo-filesystem that exposes kernel variables.
6
+
# The files in /sys and their sizes will frequently change in between read()'s, which can break naive tar invocations.)
7
+
#
8
+
# Usage: ./snapshot.sh sysfs.tar.gz
9
+
10
+
dst="$PWD/$1"
11
+
tmp=$(mktemp -d -t sysfs_snapshot_XXXXXXX)
12
+
13
+
cleanup() {
14
+
rm -rf "$tmp"
15
+
}
16
+
trap cleanup EXIT
17
+
18
+
set -euxo pipefail
19
+
20
+
find /sys/devices/*/block /sys/dev/block /sys/class/block -print0 | sort -z | while IFS= read -d $'\0' -r file; do
21
+
# create the new file name by stripping the leading
22
+
# /sys and mashing it against the temp folder
23
+
temp_file="${tmp}/${file#*/sys/}"
24
+
25
+
# create equivalent symlink
26
+
if [ -L "$file" ]; then
27
+
cp -d "$file" "$temp_file"
28
+
continue
29
+
fi
30
+
31
+
# create necessary directories
32
+
if [ -d "$file" ]; then
33
+
mkdir -p "$temp_file"
34
+
continue
35
+
fi
36
+
37
+
# skip over any files that we lack permissions to read,
38
+
# we encounter I/O errors when trying to read, or
39
+
# have some other weirdness
40
+
if ! wc -l "$file" >/dev/null 2>&1; then
41
+
continue
42
+
fi
43
+
44
+
cp "$file" "$temp_file"
45
+
done
46
+
47
+
cd "$tmp"
48
+
tar vczf "$dst" .
internal/mountinfo/testdata/sysfs.vda1.tar.gz
internal/mountinfo/testdata/sysfs.vda1.tar.gz
This is a binary file and will not be displayed.