Do not report cluster fields supplied by a cluster policy as drift

.nextchanges/bundles/cluster-policy-no-drift.md

@@ -0,0 +1 @@
+* On the direct engine, `bundle plan` no longer reports a permanent update on a cluster that uses a cluster policy: when the cluster spec sets `policy_id`, a field present in the remote but absent from the bundle config is not treated as drift. ([#6531](https://github.com/databricks/cli/pull/6531))

acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl

@@ -0,0 +1,71 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    pol:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        custom_tags.CostCenter:
+          type: fixed
+          value: from-policy
+
+  # Every place a cluster spec can appear must be covered by a rule; a gap here is how
+  # https://github.com/databricks/cli/issues/5179 stayed open after a partial fix.
+  clusters:
+    standalone:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      # Use the shared instance pool: a cold cluster boot made this the slowest test in the
+      # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected.
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      policy_id: ${resources.cluster_policies.pol.id}
+
+  jobs:
+    task_cluster:
+      name: test-task-cluster-$UNIQUE_NAME
+      tasks:
+        - task_key: t
+          new_cluster:
+            policy_id: ${resources.cluster_policies.pol.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+          spark_python_task:
+            python_file: ./hello_world.py
+
+    for_each_cluster:
+      name: test-for-each-cluster-$UNIQUE_NAME
+      tasks:
+        - task_key: outer
+          for_each_task:
+            inputs: "[1,2]"
+            task:
+              task_key: inner
+              new_cluster:
+                policy_id: ${resources.cluster_policies.pol.id}
+                spark_version: $DEFAULT_SPARK_VERSION
+                node_type_id: $NODE_TYPE_ID
+                num_workers: 1
+              spark_python_task:
+                python_file: ./hello_world.py
+
+  # A pipeline cluster carries policy_id too, but the Pipelines API does not expand the
+  # policy into the stored spec: it reads back exactly as authored, so there is no addition
+  # to suppress and no ignore_remote_additions rule for pipelines. Kept here to record that.
+  pipelines:
+    pipe:
+      name: test-pipeline-$UNIQUE_NAME
+      clusters:
+        - label: default
+          policy_id: ${resources.cluster_policies.pol.id}
+          node_type_id: $NODE_TYPE_ID
+          num_workers: 1
+      libraries:
+        - file:
+            path: ./hello_world.py

acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt

@@ -0,0 +1,48 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.pol
+Created clusters.standalone
+Created jobs.for_each_cluster
+Created jobs.task_cluster
+Created pipelines.pipe
+Files: 6 uploaded, 0 deleted
+Resources: 5 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Every cluster-spec location converges: the policy tag is not drift anywhere
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged
+
+=== Confirm the tag really is present remotely and was classified, not just absent
+
+>>> [CLI] bundle plan -o json
+json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}";
+json.plan.resources.clusters.standalone.remote_state.custom_tags.CostCenter = "from-policy";
+json.plan.resources.clusters.standalone.changes.custom_tags.action = "skip";
+json.plan.resources.clusters.standalone.changes.custom_tags.reason = "remote_addition";
+json.plan.resources.clusters.standalone.changes.custom_tags.remote.CostCenter = "from-policy";
+json.plan.resources.jobs.for_each_cluster.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy";
+json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.action = "skip";
+json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.reason = "remote_addition";
+json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy";
+json.plan.resources.jobs.task_cluster.remote_state.tasks[0].new_cluster.custom_tags.CostCenter = "from-policy";
+json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.action = "skip";
+json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.reason = "remote_addition";
+json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.remote.CostCenter = "from-policy";
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.pol
+  delete resources.clusters.standalone
+  delete resources.jobs.for_each_cluster
+  delete resources.jobs.task_cluster
+  delete resources.pipelines.pipe
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.pipe
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 5 deleted

acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script

@@ -0,0 +1,14 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+title "Every cluster-spec location converges: the policy tag is not drift anywhere\n"
+trace $CLI bundle plan
+
+title "Confirm the tag really is present remotely and was classified, not just absent\n"
+trace $CLI bundle plan -o json | gron.py | grep custom_tags

acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml

@@ -0,0 +1,10 @@
+# Creating a real cluster made this the slowest test in the cloud suite (~8 min per env), and
+# the per-location classification it checks is fully covered locally. The cheap job-based
+# tests in this directory keep the core behaviour in every cloud run.
+CloudSlow = true
+
+RecordRequests = false
+
+Ignore = [
+    "databricks.yml",
+]

acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl

@@ -0,0 +1,33 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    pol:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        custom_tags.CostCenter:
+          type: fixed
+          value: from-fixed
+
+  jobs:
+    # The config sets the fixed attribute to a value the policy forbids.
+    j:
+      name: test-job-$UNIQUE_NAME
+      job_clusters:
+        - job_cluster_key: small
+          new_cluster:
+            policy_id: ${resources.cluster_policies.pol.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+            custom_tags:
+              CostCenter: not-what-the-policy-says
+      tasks:
+        - task_key: t
+          job_cluster_key: small
+          spark_python_task:
+            python_file: ./hello_world.py

acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/hello_world.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt

@@ -0,0 +1,23 @@
+
+=== Deploy a cluster whose config contradicts a fixed policy value
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Error: cannot create resources.jobs.j: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says")
+
+Files: 5 uploaded, 0 deleted
+
+Exit code: 1
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.pol
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/script

@@ -0,0 +1,9 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy a cluster whose config contradicts a fixed policy value\n"
+errcode trace $CLI bundle deploy

acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    pol:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        spark_version:
+          type: fixed
+          value: $DEFAULT_SPARK_VERSION
+        custom_tags.CostCenter:
+          type: fixed
+          value: policy-supplied
+
+  jobs:
+    # new_cluster deliberately omits spark_version and custom_tags: the policy is
+    # expected to supply both. apply_policy_default_values is NOT set.
+    j:
+      name: test-job-$UNIQUE_NAME
+      job_clusters:
+        - job_cluster_key: small
+          new_cluster:
+            policy_id: ${resources.cluster_policies.pol.id}
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+      tasks:
+        - task_key: t
+          job_cluster_key: small
+          spark_python_task:
+            python_file: ./hello_world.py

acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/hello_world.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt

@@ -0,0 +1,39 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.pol
+Created jobs.j
+Files: 5 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?
+The bundle declares neither, and does not set apply_policy_default_values.
+json.settings.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied";
+json.settings.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12";
+
+=== Is the policy-supplied value reported as drift?
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+=== spark_version is not covered by any backend_defaults rule, so show its verdict
+
+>>> [CLI] bundle plan -o json
+json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}";
+json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied";
+json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.reason = "remote_addition";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.remote.CostCenter = "policy-supplied";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.action = "skip";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.reason = "remote_addition";
+json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.remote = "13.3.x-snapshot-scala2.12";
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.pol
+  delete resources.jobs.j
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 2 deleted

acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script

@@ -0,0 +1,23 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+# Mask the server-assigned policy id (real backend returns a hex id the built-in
+# UUID replacement misses).
+read_id.py pol > /dev/null
+job_id=$(read_id.py j)
+
+title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n"
+echo "The bundle declares neither, and does not set apply_policy_default_values."
+$CLI jobs get "$job_id" | gron.py | grep -E 'new_cluster.(spark_version|custom_tags)'
+
+title "Is the policy-supplied value reported as drift?\n"
+trace $CLI bundle plan
+
+title "spark_version is not covered by any backend_defaults rule, so show its verdict\n"
+trace $CLI bundle plan -o json | gron.py | grep -E 'custom_tags|spark_version'