Split dresources resources.yml into one file per resource

.agents/rules/auto-generated-files.md

@@ -15,7 +15,7 @@ globs:
   - "cmd/workspace/**/*.go"
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
-  - "bundle/direct/dresources/*.generated.yml"
+  - "bundle/direct/dresources/configs/*.generated.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
   - "python/databricks/bundles/version.py"
@@ -36,7 +36,7 @@ paths:
   - "cmd/workspace/**/*.go"
   - "internal/genkit/tagging.py"
   - "internal/mocks/**/*.go"
-  - "bundle/direct/dresources/*.generated.yml"
+  - "bundle/direct/dresources/configs/*.generated.yml"
   - "bundle/internal/validation/generated/*.go"
   - "bundle/schema/jsonschema.json"
   - "python/databricks/bundles/version.py"

.agents/skills/bump-sdk/SKILL.md

@@ -9,7 +9,7 @@ allowed-tools: Read, Edit, Write, Bash, Glob, Grep, WebFetch, AskUserQuestion
 
 The SDK version lives in `go.mod` (`github.com/databricks/databricks-sdk-go`) and the pinned spec SHA lives in `.codegen/_openapi_sha`.
 These two move as a pair; everything else in this skill is regenerated from them or is fallout you fix by hand.
-Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `bundle/direct/dresources/resources.generated.yml`, `bundle/terraform_dabs_map/generated.go`, `python/databricks/bundles/**`); regenerate them.
+Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `bundle/direct/dresources/configs/*.generated.yml`, `bundle/terraform_dabs_map/generated.go`, `python/databricks/bundles/**`); regenerate them.
 
 The Python tasks (`pydabs-*`, and the `pydabs-codegen` step inside `generate-check`) all run through `uv`. If one fails because `uv` is missing or because the host's `python3` is too old (e.g. 3.9), install `uv` (`curl -LsSf https://astral.sh/uv/install.sh | sh`) rather than touching the system Python: `uv run` provisions the interpreter each package pins (`>=3.10`, and `==3.13.*` under `python/codegen/`) and downloads it if needed. Do not chase the system Python version.
 
@@ -42,7 +42,7 @@ Run `go build ./...` and fix compile breakages before touching acceptance golden
 Read the SDK's `CHANGELOG.md` at the target version (in the module cache) to enumerate breaking changes before chasing compile errors.
 A removed struct field that the CLI used (e.g. `jobs.AiRuntimeTask.CodeSourcePath`) should have its usage temporarily disabled with a comment noting it returns in a later SDK bump, not deleted outright.
 A new struct field triggers an `exhaustruct` lint failure in `bundle/direct/dresources/*`. Run `./task lint` and (in case of issues) wire the field through `PrepareState` and `RemapState` when it exists on both the input and remote types.
-A field the new spec now annotates as output-only may already be emitted into `resources.generated.yml`, making the manual entry in `resources.yml` redundant; `TestResourcesYMLNoRedundantRules` catches this, so remove the manual entry.
+A field the new spec now annotates as output-only may already be emitted into `<resource_type>.generated.yml`, making the manual entry in `<resource_type>.yml` redundant; `TestResourcesYMLNoRedundantRules` catches this, so remove the manual entry.
 
 **6. Refresh goldens, then VERIFY.**
 

Taskfile.yml

@@ -1050,9 +1050,9 @@ tasks:
       - bundle/direct/dresources/apitypes.yml
       - acceptance/bundle/refschema/out.fields.txt
     generates:
-      - bundle/direct/dresources/resources.generated.yml
+      - bundle/direct/dresources/configs/*.generated.yml
     cmds:
-      - "sh -c 'uv run --script bundle/direct/tools/generate_resources.py .codegen/cli.json bundle/direct/dresources/apitypes.generated.yml bundle/direct/dresources/apitypes.yml acceptance/bundle/refschema/out.fields.txt > bundle/direct/dresources/resources.generated.yml'"
+      - "uv run --script bundle/direct/tools/generate_resources.py .codegen/cli.json bundle/direct/dresources/apitypes.generated.yml bundle/direct/dresources/apitypes.yml acceptance/bundle/refschema/out.fields.txt bundle/direct/dresources/configs"
 
   # pydabs-* tasks are defined in python/Taskfile.yml (included above).
 

acceptance/bundle/empty_string_dropped/gen_empty_config.py

@@ -11,8 +11,8 @@
 
 A field is settable/eligible when out.fields.txt types it as `string` (this
 skips enums, which are named types) with flag ALL or INPUT, and it is not
-output_only (resources.generated.yml), a bundle-framework field, or a known
-terraform-erroring field.
+output_only (<resource_type>.generated.yml), a bundle-framework field, or a
+known terraform-erroring field.
 
 Run from the repo root; writes databricks.yml in the test directory:
   acceptance/bundle/empty_string_dropped/gen_empty_config.py
@@ -24,7 +24,8 @@
 import yaml
 
 FIELDS = Path("acceptance/bundle/refschema/out.fields.txt")
-GENERATED = Path("bundle/direct/dresources/resources.generated.yml")
+CONFIGS = Path("bundle/direct/dresources/configs")
+GENERATED_SUFFIX = ".generated.yml"
 TESTDIR = Path("acceptance/bundle/empty_string_dropped")
 BASE = TESTDIR / "base.yml"
 
@@ -79,14 +80,14 @@ def string_leaf_parents():
 
 def output_only_fields():
     """Map resource type -> {output_only field paths} (user cannot set)."""
-    gen = yaml.safe_load(GENERATED.read_text()) or {}
     result = {}
-    for rtype, spec in (gen.get("resources") or {}).items():
+    for path in sorted(CONFIGS.glob("*" + GENERATED_SUFFIX)):
+        spec = yaml.safe_load(path.read_text()) or {}
         fields = set()
-        for entry in (spec or {}).get("ignore_remote_changes") or []:
+        for entry in spec.get("ignore_remote_changes") or []:
             if str(entry.get("reason", "")).startswith("spec:output_only"):
                 fields.add(entry["field"])
-        result[rtype] = fields
+        result[path.name.removesuffix(GENERATED_SUFFIX)] = fields
     return result
 
 

acceptance/bundle/resources/catalogs/drift/managed_properties/test.toml

@@ -1,5 +1,5 @@
 RecordRequests = true
 
 # Terraform issues a spurious PATCH for enable_predictive_optimization on every
-# deploy, which is outside the scope of backend-default handling in resources.yml.
+# deploy, which is outside the scope of backend-default handling in catalogs.yml.
 EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/schemas/drift/managed_properties/test.toml

@@ -1,5 +1,5 @@
 RecordRequests = true
 
 # Terraform issues a spurious PATCH for enable_predictive_optimization on every
-# deploy, which is outside the scope of backend-default handling in resources.yml.
+# deploy, which is outside the scope of backend-default handling in schemas.yml.
 EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

bundle/configsync/defaults.go

@@ -63,7 +63,7 @@ var serverSideDefaults = map[string]any{
 	// custom_tags and cluster_log_conf are commonly injected by cluster policies
 	// when the user omits them, so they exist only remotely. Syncing them back leaks
 	// one environment's policy values into (often shared) config and breaks deploys in
-	// other environments. TODO: move to backend_defaults in resources.yml once
+	// other environments. TODO: move to backend_defaults in jobs.yml once
 	// configsync filtering is migrated to the direct engine lifecycle metadata.
 	"resources.jobs.*.tasks[*].new_cluster.custom_tags":      backendDefault,
 	"resources.jobs.*.tasks[*].new_cluster.cluster_log_conf": backendDefault,

bundle/direct/bundle_plan_test.go

@@ -81,8 +81,8 @@ resources:
 }
 
 func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) {
-	// Rules mirror the schemas backend_defaults in resources.yml, but the test is
-	// deliberately self-contained so that edits to resources.yml don't break it.
+	// Rules mirror the schemas backend_defaults in schemas.yml, but the test is
+	// deliberately self-contained so that edits to schemas.yml don't break it.
 	// The real wiring is covered by acceptance/bundle/resources/schemas/drift.
 	managedDefaults, err := structpath.ParsePattern("properties['unity.catalog.managed.*.defaults.*']")
 	require.NoError(t, err)
@@ -406,8 +406,8 @@ func bundleWithSkippedJobRun(t *testing.T, remote *dresources.JobRunRemote) *Dep
 }
 
 func TestShouldSkipRemoteAddition(t *testing.T) {
-	// Rules mirror clusters/jobs ignore_remote_additions in resources.yml, but the test is
-	// deliberately self-contained so edits to resources.yml don't break it. The real wiring
+	// Rules mirror clusters/jobs ignore_remote_additions in clusters.yml and jobs.yml, but the
+	// test is deliberately self-contained so edits to those files don't break it. The real wiring
 	// is covered by acceptance/bundle/resources/cluster_policies/*.
 	jobCluster, err := structpath.ParsePattern("job_clusters[*].new_cluster")
 	require.NoError(t, err)

bundle/direct/dresources/README.md

@@ -9,20 +9,25 @@
    An exception could be made if default error message lacks the necessary context.
  - The arguments point to actual struct that will be persisted in state, any changes to it will affect what is stored in state. Usually there is no need to change it, but if there is, there should always be detailed explanation.
  - Each Create/Update/Delete method should correspond to one API call. We persist state right after, so there is minimum chance of having orphaned resources.
- - We should calculate the update type during plan phase. This means it should be configured via resources.yml as much as possible, falling back to OverrideChangeDesc(). The DoUpdate() implementation should be as predictable as possible based on the plan. In particular, avoid reading remote state in DoUpdate() to decide what kind of update to dod.
+ - We should calculate the update type during plan phase. This means it should be configured via the resource's YAML file as much as possible, falling back to OverrideChangeDesc(). The DoUpdate() implementation should be as predictable as possible based on the plan. In particular, avoid reading remote state in DoUpdate() to decide what kind of update to dod.
  - Create/Update/Delete methods should not need to do read requests. They can read state passed to them via \*PlanEntry but that should be reserved for exceptional cases. Most resources should have 1-1 mapping to single SDK/API call.
- - For update with complex logic, ensure that DoUpdate() never results in no-op. If certain fields could not be updated, they should be excluded at plan level in resources.yml.
+ - For update with complex logic, ensure that DoUpdate() never results in no-op. If certain fields could not be updated, they should be excluded at plan level in `<resource_type>.yml`.
 
-## Field classification in resources.yml
+## Field classification in `<resource_type>.yml`
 
-Each field with special plan/deploy behavior must be declared in `resources.yml`. Choose the right category:
+Each field with special plan/deploy behavior must be declared in the YAML file of its resource type, under `configs/`. Both files of a resource type are optional and are omitted when they would be empty:
+
+ - `configs/<resource_type>.yml` (e.g. `configs/jobs.yml`) — hand-written rules.
+ - `configs/<resource_type>.generated.yml` — API field behaviors from the OpenAPI schema. Generated by `./task generate-direct-resources`, do not edit.
+
+Choose the right category:
 
  - **`backend_defaults`**: The backend may fill in a value when the user doesn't specify one. Suppresses the diff when the user's config is nil/empty but remote has a value. Optionally restrict to specific allowed remote values via `values:`. Use for fields the API fills in as defaults (e.g., `format`, `run_if`, `node_type_id`). Link to TF provider suppression comment in the same format as existing entries.
- - **`ignore_remote_changes`**: Ignore changes the remote makes to this field. Use for fields the backend manages (e.g., cloud-provider attributes like `aws_attributes`, `gcp_attributes`) or fields not returned by the update endpoint. Do not zero out such fields in `RemapState` to hide them from diff computation: carry the real remote value through and declare the field here instead, since zeroing discards information and duplicates the suppression logic. For `output_only` fields this rule is often already produced by `resources.generated.yml` from the OpenAPI annotation. Reason codes:
+ - **`ignore_remote_changes`**: Ignore changes the remote makes to this field. Use for fields the backend manages (e.g., cloud-provider attributes like `aws_attributes`, `gcp_attributes`) or fields not returned by the update endpoint. Do not zero out such fields in `RemapState` to hide them from diff computation: carry the real remote value through and declare the field here instead, since zeroing discards information and duplicates the suppression logic. For `output_only` fields this rule is often already produced by `<resource_type>.generated.yml` from the OpenAPI annotation. Reason codes:
    - `output_only` — the field is computed by the backend; the user never sets it
    - `input_only` — accepted on create/update but not returned by GET (e.g., write-only tokens, flags)
    - `managed` — managed by the cloud provider or platform, not by the user config
- - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment in resources.yml explaining why.
+ - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment explaining why.
  - **`recreate_on_changes`**: Changing this field requires delete + create. Use for truly immutable fields (name, type, location). The reason should reference API docs or TF provider.
  - **`updatable_id_fields`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented.
 
@@ -96,7 +101,7 @@ in normal drift detection and is no longer subject to the `missing_in_remote` su
 
 ## OverrideChangeDesc
 
-Use `OverrideChangeDesc` only as a last resort when `resources.yml` settings cannot express the needed logic. Skipping an action with `change.Action = deployplan.Skip` in `OverrideChangeDesc` creates a silent no-op: the plan shows no change even if the user's config differs from remote. Document the skip reason clearly in both the comment and `change.Reason`.
+Use `OverrideChangeDesc` only as a last resort when the `<resource_type>.yml` settings cannot express the needed logic. Skipping an action with `change.Action = deployplan.Skip` in `OverrideChangeDesc` creates a silent no-op: the plan shows no change even if the user's config differs from remote. Document the skip reason clearly in both the comment and `change.Reason`.
 
 ## Nice to have
  - Add link to corresponding API documentation before each method.

bundle/direct/dresources/config.go

@@ -1,8 +1,12 @@
 package dresources
 
 import (
-	_ "embed"
+	"embed"
 	"encoding/json"
+	"fmt"
+	"io/fs"
+	"path"
+	"strings"
 	"sync"
 
 	"github.com/databricks/cli/libs/structs/structpath"
@@ -105,11 +109,21 @@ type Config struct {
 	Resources map[string]ResourceLifecycleConfig `yaml:"resources"`
 }
 
-//go:embed resources.yml
-var resourcesYAML []byte
+// One file per resource type under configs/: <resource_type>.yml holds the
+// hand-written rules and <resource_type>.generated.yml the ones derived from the
+// OpenAPI spec. A resource type without rules has no file. The file name is the
+// resource type, so each file holds the rules directly with no enclosing keys.
+//
+//go:embed configs/*.yml
+var configFS embed.FS
+
+const (
+	configDir = "configs"
+	ymlSuffix = ".yml"
 
-//go:embed resources.generated.yml
-var resourcesGeneratedYAML []byte
+	// generatedSuffix marks a generated file once ymlSuffix is trimmed.
+	generatedSuffix = ".generated"
+)
 
 var empty = ResourceLifecycleConfig{
 	IgnoreRemoteChanges:   nil,
@@ -123,28 +137,50 @@ var empty = ResourceLifecycleConfig{
 	SensitiveFields:       nil,
 }
 
-func mustParseConfig(data []byte) func() *Config {
-	return sync.OnceValue(func() *Config {
-		c := &Config{Resources: nil}
-		if err := yaml.Unmarshal(data, c); err != nil {
+// loadConfigs parses every embedded YAML file into the hand-written or the generated
+// config, keyed by the resource type the file is named after.
+var loadConfigs = sync.OnceValues(func() (*Config, *Config) {
+	handWritten := &Config{Resources: map[string]ResourceLifecycleConfig{}}
+	generated := &Config{Resources: map[string]ResourceLifecycleConfig{}}
+
+	names, err := fs.Glob(configFS, configDir+"/*"+ymlSuffix)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, name := range names {
+		dst, resourceType := handWritten, strings.TrimSuffix(path.Base(name), ymlSuffix)
+		if trimmed, ok := strings.CutSuffix(resourceType, generatedSuffix); ok {
+			dst, resourceType = generated, trimmed
+		}
+
+		data, err := configFS.ReadFile(name)
+		if err != nil {
 			panic(err)
 		}
-		return c
-	})
-}
 
-var loadConfig = mustParseConfig(resourcesYAML)
+		var rc ResourceLifecycleConfig
+		if err := yaml.Unmarshal(data, &rc); err != nil {
+			panic(fmt.Errorf("%s: %w", name, err))
+		}
+
+		dst.Resources[resourceType] = rc
+	}
 
-var loadGeneratedConfig = mustParseConfig(resourcesGeneratedYAML)
+	return handWritten, generated
+})
 
-// MustLoadConfig returns the parsed resources.yml configuration.
+// MustLoadConfig returns the configuration parsed from the configs/<resource_type>.yml files.
 func MustLoadConfig() *Config {
-	return loadConfig()
+	handWritten, _ := loadConfigs()
+	return handWritten
 }
 
-// MustLoadGeneratedConfig returns the parsed resources.generated.yml configuration.
+// MustLoadGeneratedConfig returns the configuration parsed from the
+// configs/<resource_type>.generated.yml files.
 func MustLoadGeneratedConfig() *Config {
-	return loadGeneratedConfig()
+	_, generated := loadConfigs()
+	return generated
 }
 
 // GetResourceConfig returns the lifecycle config for a given resource type.