Skip to content

Commit 077fce8

Browse files
committed
nvmeof: add mount cache into nodeserver
adding `mountCache`, which saves the findmnt calls. aslo change in the code any place we call to `findmnt` to call to cache get. Signed-off-by: gadi-didi <gadi.didi@ibm.com>
1 parent 00a8cd0 commit 077fce8

File tree

1 file changed

+46
-8
lines changed

1 file changed

+46
-8
lines changed

internal/nvmeof/nodeserver/nodeserver.go

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ type NodeServer struct {
5252

5353
// node ID of this node server
5454
nodeID string
55+
56+
// mountedDevices is a cache of currently mounted NVMe devices, used to determine
57+
// when it's safe to disconnect a device.
58+
// it is initialized on startup and updated on each stage/unstage operation.
59+
mountCache *nvmeutil.MountCache
5560
}
5661

5762
// ConnectionInfo holds NVMe-oF connection details.
@@ -102,13 +107,25 @@ func NewNodeServer(
102107
volumeLocks: util.NewIDLocker(),
103108
securityKeys: nil, // Initialize lazily when needed
104109
nodeID: nodeID,
110+
mountCache: nvmeutil.NewMountCache(),
105111
}
106112

107113
// Load nvme kernel modules
108114
if err := nvmeInitiator.LoadKernelModules(context.Background()); err != nil {
109115
return nil, fmt.Errorf("failed to load NVMe kernel modules: %w", err)
110116
}
111117

118+
// Initialize mounted devices cache on startup to ensure we have an accurate view
119+
mountedDevices, err := getNVMeMountedDevices(context.Background())
120+
if err != nil {
121+
log.WarningLog(context.Background(),
122+
"failed to get mounted devices on startup: %v (mounted devices cache will be empty)", err)
123+
} else {
124+
for device, mountPoint := range mountedDevices {
125+
ns.mountCache.Add(device, mountPoint)
126+
}
127+
}
128+
112129
return ns, nil
113130
}
114131

@@ -327,7 +344,15 @@ func (ns *NodeServer) NodeUnstageVolume(
327344

328345
stagingTargetPath := getStagingTargetPath(req)
329346

330-
devPath := getDeviceFromStagingPath(ctx, stagingTargetPath)
347+
devPath, exists := ns.mountCache.GetDevice(stagingTargetPath)
348+
if !exists {
349+
log.WarningLog(ctx, "device not in cache for %s, querying system", stagingTargetPath)
350+
// Fallback to findmnt if not in cache
351+
devPath, err = nvmeutil.GetDeviceFromMountpoint(ctx, stagingTargetPath)
352+
if err != nil {
353+
log.ErrorLog(ctx, "failed to get device from mountpoint for %s: %v", stagingTargetPath, err)
354+
}
355+
}
331356

332357
isMnt, err := ns.Mounter.IsMountPoint(stagingTargetPath)
333358
if err != nil {
@@ -365,13 +390,13 @@ func (ns *NodeServer) NodeUnstageVolume(
365390
// Non-fatal - a failed disconnect just means the connection lingers until the
366391
// kernel's ctrl_loss_tmo expires or the next reconnect cycle.
367392
if devPath != "" {
368-
mountedDevices, err := getNVMeMountedDevices(ctx)
369-
if err != nil {
370-
log.WarningLog(ctx, "failed to get mounted devices: %v (skipping disconnect)", err)
371-
} else {
372-
if err := ns.initiator.DisconnectIfLastMount(ctx, devPath, mountedDevices); err != nil {
373-
log.WarningLog(ctx, "failed to disconnect controller for device %s: %v", devPath, err)
374-
}
393+
// Remove from cache
394+
ns.mountCache.RemoveByDevice(devPath)
395+
396+
// Get remaining devices
397+
remainingDevices := ns.mountCache.GetCopyAllDevices()
398+
if err := ns.initiator.DisconnectIfLastMount(ctx, devPath, remainingDevices); err != nil {
399+
log.WarningLog(ctx, "failed to disconnect controller for device %s: %v", devPath, err)
375400
}
376401
}
377402

@@ -489,6 +514,16 @@ func (ns *NodeServer) stageTransaction(
489514
}
490515
transaction.isMounted = true
491516

517+
// Resolve real device and update cache
518+
realDevice, err := nvmeutil.GetDeviceFromMountpoint(ctx, stagingTargetPath)
519+
if err != nil {
520+
log.WarningLog(ctx, "failed to resolve device: %v", err)
521+
// Fallback - try to continue
522+
realDevice = devicePath
523+
}
524+
525+
ns.mountCache.Add(realDevice, stagingTargetPath)
526+
492527
return transaction, nil
493528
}
494529

@@ -519,6 +554,9 @@ func (ns *NodeServer) undoStagingTransaction(
519554
log.ErrorLog(ctx, "failed to remove stagingtargetPath: %s with error: %v", stagingTargetPath, err)
520555
// continue on failure to disconnect the image
521556
}
557+
558+
// remove from cache if we created the staging path
559+
ns.mountCache.RemoveByMountPoint(stagingTargetPath)
522560
}
523561
// disconnect if we connected
524562
if transaction.devicePath != "" {

0 commit comments

Comments
 (0)