aitools: categorize install errors

.nextchanges/cli/aitools-install-error-category.md

@@ -0,0 +1 @@
+* `databricks aitools install --output json` now reports an `error_category` for a failed or skipped install (per agent, and at the top level for a failure with no per-agent entry), giving coding agents and CI a stable classification of why an install did not complete. ([#6482](https://github.com/databricks/cli/pull/6482))

acceptance/experimental/aitools/skills/install-output-json-agent-error/output.txt

@@ -0,0 +1,22 @@
+
+=== install --output json records a per-agent error category
+>>> [CLI] aitools install --skills-only --scope=project --agents=claude-code,cursor --output json
+{
+  "scope": "project",
+  "agents": [
+    {
+      "name": "claude-code",
+      "delivery": "skills",
+      "status": "installed"
+    },
+    {
+      "name": "cursor",
+      "delivery": "skip",
+      "status": "skipped",
+      "error_category": "UNSUPPORTED_SCOPE",
+      "message": "does not support project-scoped skills"
+    }
+  ]
+}
+
+Exit code: 1

acceptance/experimental/aitools/skills/install-output-json-agent-error/script

@@ -0,0 +1,9 @@
+# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
+sethome home
+
+title "install --output json records a per-agent error category"
+# claude-code supports project-scoped skills; cursor does not. At --scope=project,
+# claude-code installs while cursor is skipped with an UNSUPPORTED_SCOPE category in
+# its own agents[] entry (the top-level error_category stays unset because the
+# failure is per-agent). cursor was named explicitly, so the run exits non-zero.
+trace $CLI aitools install --skills-only --scope=project --agents=claude-code,cursor --output json

acceptance/experimental/aitools/skills/install-output-json-agent-error/test.toml

@@ -0,0 +1,37 @@
+# Mock server replaces raw.githubusercontent.com for manifest + skill files.
+Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
+Env.DATABRICKS_SKILLS_REF = "test-ref"
+
+Ignore = [
+    "home",
+    ".claude",
+    ".databricks",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+[[Server]]
+Pattern = "GET /test-ref/manifest.json"
+Response.Body = '''
+{
+  "version": "2",
+  "updated_at": "2026-01-01T00:00:00Z",
+  "skills": {
+    "test-stable": {
+      "version": "1.0.0",
+      "description": "Stable test skill",
+      "files": ["SKILL.md"],
+      "repo_dir": "skills"
+    }
+  }
+}
+'''
+
+[[Server]]
+Pattern = "GET /test-ref/skills/test-stable/SKILL.md"
+Response.Body = '''---
+name: test-stable
+---
+
+# Test stable skill
+'''

acceptance/experimental/aitools/skills/install-output-json-agents-skipped/output.txt

@@ -0,0 +1,24 @@
+
+=== install --output json: every named agent skipped, each with its own category
+>>> [CLI] aitools install --skills-only --scope=project --agents=cursor,codex --output json
+{
+  "scope": "project",
+  "agents": [
+    {
+      "name": "cursor",
+      "delivery": "skip",
+      "status": "skipped",
+      "error_category": "UNSUPPORTED_SCOPE",
+      "message": "does not support project-scoped skills"
+    },
+    {
+      "name": "codex",
+      "delivery": "skip",
+      "status": "skipped",
+      "error_category": "UNSUPPORTED_SCOPE",
+      "message": "does not support project-scoped skills"
+    }
+  ]
+}
+
+Exit code: 1

acceptance/experimental/aitools/skills/install-output-json-agents-skipped/script

@@ -0,0 +1,9 @@
+# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
+sethome home
+
+title "install --output json: every named agent skipped, each with its own category"
+# Neither cursor nor codex supports project-scoped skills, so at --scope=project both
+# are skipped with a per-agent UNSUPPORTED_SCOPE category and nothing is installed (no
+# manifest fetch). Both were named explicitly, so the run exits non-zero, yet the
+# top-level error_category stays unset because every failure is per-agent.
+trace $CLI aitools install --skills-only --scope=project --agents=cursor,codex --output json

acceptance/experimental/aitools/skills/install-output-json-agents-skipped/test.toml

@@ -0,0 +1,10 @@
+# Every named agent is skipped for scope before any manifest fetch, so no skills
+# server is needed and nothing is written outside the isolated home.
+Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
+Env.DATABRICKS_SKILLS_REF = "test-ref"
+
+Ignore = [
+    "home",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/experimental/aitools/skills/install-output-json-error/output.txt

@@ -0,0 +1,11 @@
+
+=== install --output json reports a top-level error category and exits non-zero
+>>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json
+{
+  "scope": "global",
+  "agents": [],
+  "error": "skill \"nonexistent\" not found",
+  "error_category": "SKILL_NOT_FOUND"
+}
+
+Exit code: 1

acceptance/experimental/aitools/skills/install-output-json-error/script

@@ -0,0 +1,10 @@
+# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
+sethome home
+
+title "install --output json reports a top-level error category and exits non-zero"
+# A --skills entry absent from the manifest fails before any agent is touched, so the
+# failure has no per-agent entry and surfaces in the top-level error/error_category
+# fields (agents stays empty). In JSON mode progress is silenced and root prints no
+# duplicate "Error:" line, so stdout carries only the JSON document; the command
+# still exits non-zero.
+trace $CLI aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json

acceptance/experimental/aitools/skills/install-output-json-error/test.toml

@@ -0,0 +1,28 @@
+# Mock server replaces raw.githubusercontent.com for manifest + skill files.
+Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
+Env.DATABRICKS_SKILLS_REF = "test-ref"
+
+Ignore = [
+    "home",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# The manifest has one skill; the script asks for a different one so the resolve
+# fails with a skill-not-found error.
+[[Server]]
+Pattern = "GET /test-ref/manifest.json"
+Response.Body = '''
+{
+  "version": "2",
+  "updated_at": "2026-01-01T00:00:00Z",
+  "skills": {
+    "test-stable": {
+      "version": "1.0.0",
+      "description": "Stable test skill",
+      "files": ["SKILL.md"],
+      "repo_dir": "skills"
+    }
+  }
+}
+'''

acceptance/experimental/aitools/skills/install-output-json-experimental-error/output.txt

@@ -0,0 +1,11 @@
+
+=== install --output json categorizes an experimental skill requested without --experimental
+>>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=test-exp --output json
+{
+  "scope": "global",
+  "agents": [],
+  "error": "skill \"test-exp\" is experimental; use --experimental to install",
+  "error_category": "EXPERIMENTAL_SKILL"
+}
+
+Exit code: 1

acceptance/experimental/aitools/skills/install-output-json-experimental-error/script

@@ -0,0 +1,8 @@
+# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
+sethome home
+
+title "install --output json categorizes an experimental skill requested without --experimental"
+# Requesting an experimental skill without --experimental fails during skill
+# resolution, before any agent is touched, so the failure has no per-agent entry
+# and surfaces in the top-level error/error_category fields (EXPERIMENTAL_SKILL).
+trace $CLI aitools install --skills-only --scope=global --agents=claude-code --skills=test-exp --output json

acceptance/experimental/aitools/skills/install-output-json-experimental-error/test.toml

@@ -0,0 +1,28 @@
+# Mock server replaces raw.githubusercontent.com for manifest + skill files.
+Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
+Env.DATABRICKS_SKILLS_REF = "test-ref"
+
+Ignore = [
+    "home",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# The requested skill is experimental (repo_dir=experimental); the script omits
+# --experimental so resolution fails before any skill file is fetched.
+[[Server]]
+Pattern = "GET /test-ref/manifest.json"
+Response.Body = '''
+{
+  "version": "2",
+  "updated_at": "2026-01-01T00:00:00Z",
+  "skills": {
+    "test-exp": {
+      "version": "0.0.1",
+      "description": "Experimental test skill",
+      "files": ["SKILL.md"],
+      "repo_dir": "experimental"
+    }
+  }
+}
+'''

cmd/aitools/categorize.go

@@ -0,0 +1,46 @@
+package aitools
+
+import (
+	"errors"
+
+	"github.com/databricks/cli/libs/aitools/installer"
+	"github.com/databricks/cli/libs/telemetry/protos"
+)
+
+func classifyInstallError(err error) protos.AitoolsErrorCategory {
+	if err == nil {
+		return protos.AitoolsErrorCategoryUnspecified
+	}
+
+	if blocked, ok := errors.AsType[*installer.BlockedError](err); ok {
+		return blockedErrorCategory(blocked)
+	}
+	if skill, ok := errors.AsType[*installer.SkillError](err); ok {
+		return skillErrorCategory(skill)
+	}
+	return protos.AitoolsErrorCategoryUncategorized
+}
+
+func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory {
+	switch e.Reason {
+	case installer.ReasonSkillNotFound:
+		return protos.AitoolsErrorCategorySkillNotFound
+	case installer.ReasonVersionIncompatible:
+		return protos.AitoolsErrorCategoryVersionIncompatible
+	case installer.ReasonExperimentalSkill:
+		return protos.AitoolsErrorCategoryExperimentalSkill
+	default:
+		return protos.AitoolsErrorCategoryUncategorized
+	}
+}
+
+func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory {
+	switch e.Reason {
+	case installer.ReasonCLINotOnPath:
+		return protos.AitoolsErrorCategoryCLINotOnPath
+	case installer.ReasonInstallFailed:
+		return protos.AitoolsErrorCategoryPluginInstallFailed
+	default:
+		return protos.AitoolsErrorCategoryUncategorized
+	}
+}

cmd/aitools/categorize_test.go

@@ -0,0 +1,75 @@
+package aitools
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/databricks/cli/libs/aitools/installer"
+	"github.com/databricks/cli/libs/telemetry/protos"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestClassifyInstallError(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want protos.AitoolsErrorCategory
+	}{
+		{
+			name: "nil is success",
+			err:  nil,
+			want: protos.AitoolsErrorCategoryUnspecified,
+		},
+		{
+			name: "blocked cli not on path",
+			err:  &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath},
+			want: protos.AitoolsErrorCategoryCLINotOnPath,
+		},
+		{
+			name: "blocked install failed",
+			err:  &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed},
+			want: protos.AitoolsErrorCategoryPluginInstallFailed,
+		},
+		{
+			name: "blocked error with unknown reason is uncategorized",
+			err:  &installer.BlockedError{Agent: "codex", Reason: "some-future-reason"},
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+		{
+			name: "wrapped skill not found",
+			err:  fmt.Errorf("resolve failed: %w", &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"}),
+			want: protos.AitoolsErrorCategorySkillNotFound,
+		},
+		{
+			name: "version incompatible",
+			err:  &installer.SkillError{Skill: "databricks", Reason: installer.ReasonVersionIncompatible, Detail: "requires CLI version 0.5 (running 0.4)"},
+			want: protos.AitoolsErrorCategoryVersionIncompatible,
+		},
+		{
+			name: "experimental skill",
+			err:  &installer.SkillError{Skill: "test-exp", Reason: installer.ReasonExperimentalSkill, Detail: "is experimental; use --experimental to install"},
+			want: protos.AitoolsErrorCategoryExperimentalSkill,
+		},
+		{
+			name: "skill error with unknown reason is uncategorized",
+			err:  &installer.SkillError{Skill: "databricks", Reason: "some-future-reason"},
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+		{
+			name: "blocked error joined with another error is still classified",
+			err:  errors.Join(&installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed}, errors.New("other")),
+			want: protos.AitoolsErrorCategoryPluginInstallFailed,
+		},
+		{
+			name: "unrecognized error is uncategorized",
+			err:  errors.New("boom"),
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, classifyInstallError(tc.err))
+		})
+	}
+}