Skip to content

Expand nvidia-smi symlink resolution to additional host paths #2611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vishalanarase wants to merge 1 commit into
base: main
Choose a base branch
from
+89 −15
Open

Expand nvidia-smi symlink resolution to additional host paths #2611

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 31 additions & 14 deletions cmd/nvidia-validator/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,15 @@ const (
NVIDIAPEERMEM = "nvidia-peermem"
)

var hostNvidiaSMISearchPaths = []string{
"/opt/bin/nvidia-smi",
"/usr/bin/nvidia-smi",
"/usr/sbin/nvidia-smi",
"/bin/nvidia-smi",
"/sbin/nvidia-smi",
wslNvidiaSMIPath,
}

func main() {
c := cli.Command{}

Expand Down Expand Up @@ -744,19 +753,27 @@ func isDriverManagedByOperator(ctx context.Context) (bool, error) {
}

// resolveHostNvidiaSMI opens and stats nvidia-smi within the mounted host root.
func resolveHostNvidiaSMI(hostRootCtrPath string) (os.FileInfo, error) {
f, err := pathrs.OpenInRoot(hostRootCtrPath, "/usr/bin/nvidia-smi")
if err != nil {
return nil, fmt.Errorf("failed to open 'nvidia-smi' on the host: %w", err)
}
defer f.Close()
// It searches common nvidia-smi locations and returns file info along with the
// resolved path relative to the host root.
func resolveHostNvidiaSMI(hostRootCtrPath string) (os.FileInfo, string, error) {
for _, nvidiaSMIPath := range hostNvidiaSMISearchPaths {
f, err := pathrs.OpenInRoot(hostRootCtrPath, nvidiaSMIPath)
if err != nil {
log.Debugf("failed to open '%s' on the host: %v", nvidiaSMIPath, err)
continue

@tariq1890 tariq1890 Jul 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to the log the error message here? I am thinking that it could be a debug log

@vishalanarase vishalanarase Jul 21, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, Added a l debug before continue so the per-path open failure is visible.

}

fileInfo, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("failed to stat 'nvidia-smi' on the host: %w", err)
fileInfo, err := f.Stat()
_ = f.Close()
if err != nil {
log.Debugf("failed to stat '%s' on the host: %v", nvidiaSMIPath, err)
continue
}

return fileInfo, nvidiaSMIPath, nil
}

return fileInfo, nil
return nil, "", fmt.Errorf("failed to open 'nvidia-smi' on the host")
}

func validateHostDriver(silent bool) error {
Expand All @@ -767,15 +784,15 @@ func validateHostDriver(silent bool) error {
return nil
}

fileInfo, err := resolveHostNvidiaSMI("/host")
fileInfo, nvidiaSMIPath, err := resolveHostNvidiaSMI("/host")
if err != nil {
return err
}
if fileInfo.Size() == 0 {
return fmt.Errorf("empty 'nvidia-smi' file found on the host")
}
command := "chroot"
args := []string{"/host", "nvidia-smi"}
args := []string{"/host", nvidiaSMIPath}

return runCommand(command, args, silent)
}
Expand Down Expand Up @@ -1771,8 +1788,8 @@ func (v *VGPUManager) runValidation(silent bool) (hostDriver bool, err error) {
args := []string{"/run/nvidia/driver", "nvidia-smi"}

// check if driver is pre-installed on the host and use host path for validation
if _, err := resolveHostNvidiaSMI("/host"); err == nil {
args = []string{"/host", "nvidia-smi"}
if _, nvidiaSMIPath, err := resolveHostNvidiaSMI("/host"); err == nil {
args = []string{"/host", nvidiaSMIPath}
hostDriver = true
}

Expand Down
59 changes: 58 additions & 1 deletion cmd/nvidia-validator/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,83 @@ func TestResolveHostNvidiaSMI(t *testing.T) {
testCases := []struct {
description string
contents map[string]string
expectedPath string
expectsError bool
}{
{
description: "nvidia-smi exists in /usr/bin",
contents: map[string]string{
"/usr/bin/nvidia-smi": "fake nvidia-smi",
},
expectedPath: "/usr/bin/nvidia-smi",
},
{
description: "nvidia-smi exists through absolute /usr/bin symlink",
contents: map[string]string{
"/run/current-system/sw/bin/nvidia-smi": "fake nvidia-smi",
"/usr/bin": "symlink=/run/current-system/sw/bin",
},
expectedPath: "/usr/bin/nvidia-smi",
},
{
description: "nvidia-smi exists through relative /usr/bin symlink",
contents: map[string]string{
"/run/current-system/sw/bin/nvidia-smi": "fake nvidia-smi",
"/usr/bin": "symlink=../run/current-system/sw/bin",
},
expectedPath: "/usr/bin/nvidia-smi",
},
{
description: "nvidia-smi exists in /opt/bin",
contents: map[string]string{
"/opt/bin/nvidia-smi": "fake nvidia-smi",
},
expectedPath: "/opt/bin/nvidia-smi",
},
{
description: "nvidia-smi exists in /bin",
contents: map[string]string{
"/bin/nvidia-smi": "fake nvidia-smi",
},
expectedPath: "/bin/nvidia-smi",
},
{
description: "nvidia-smi exists in /usr/sbin",
contents: map[string]string{
"/usr/sbin/nvidia-smi": "fake nvidia-smi",
},
expectedPath: "/usr/sbin/nvidia-smi",
},
{
description: "nvidia-smi exists through absolute /usr/sbin symlink",
contents: map[string]string{
"/run/current-system/sw/bin/nvidia-smi": "fake nvidia-smi",
"/usr/sbin": "symlink=/run/current-system/sw/bin",
},
expectedPath: "/usr/sbin/nvidia-smi",
},
{
description: "nvidia-smi exists in WSL path",
contents: map[string]string{
"/usr/lib/wsl/lib/nvidia-smi": "fake nvidia-smi",
},
expectedPath: "/usr/lib/wsl/lib/nvidia-smi",
},
{
description: "nvidia-smi exists through absolute WSL path symlink",
contents: map[string]string{
"/run/wsl/lib/nvidia-smi": "fake nvidia-smi",
"/usr/lib/wsl/lib": "symlink=/run/wsl/lib",
},
expectedPath: "/usr/lib/wsl/lib/nvidia-smi",
},
{
description: "earlier search path is preferred when multiple exist",
contents: map[string]string{
"/usr/bin/nvidia-smi": "fake nvidia-smi in usr/bin",
"/usr/sbin/nvidia-smi": "fake nvidia-smi in usr/sbin",
},
expectedPath: "/usr/bin/nvidia-smi",
},
{
description: "parent dir is symlink to path not within root",
Expand Down Expand Up @@ -84,13 +140,14 @@ func TestResolveHostNvidiaSMI(t *testing.T) {
require.NoError(t, os.WriteFile(target, []byte(contents), 0600))
}

fileInfo, err := resolveHostNvidiaSMI(hostRoot)
fileInfo, nvidiaSMIPath, err := resolveHostNvidiaSMI(hostRoot)
if tc.expectsError {
require.Error(t, err)
return
}

require.NoError(t, err)
require.Equal(t, tc.expectedPath, nvidiaSMIPath)
require.NotZero(t, fileInfo.Size())
})
}
Expand Down