Merge branch 'main' into lint-model-paths

Signed-off-by: Adegbite Ayoade <tripleaceme@gmail.com>

Author: tripleaceme

.github/dependabot.yml

@@ -4,6 +4,30 @@ updates:
     directory: '/'
     schedule:
       interval: 'weekly'
+    groups:
+      # Storybook add-ons declare a peer dependency on the matching `storybook`
+      # release, so they have to move together. Bump them in a single PR.
+      storybook:
+        patterns:
+          - 'storybook'
+          - '@storybook/*'
+          - '@chromatic-com/storybook'
+          - 'eslint-plugin-storybook'
+        update-types:
+          - 'minor'
+          - 'patch'
+    ignore:
+      # A Storybook major upgrade has to be done by hand as one coordinated
+      # migration (e.g. @storybook/addon-vitest 10 requires vitest 4+), so don't
+      # open per-package major-bump PRs, which break the Storybook build.
+      - dependency-name: 'storybook'
+        update-types: ['version-update:semver-major']
+      - dependency-name: '@storybook/*'
+        update-types: ['version-update:semver-major']
+      - dependency-name: '@chromatic-com/storybook'
+        update-types: ['version-update:semver-major']
+      - dependency-name: 'eslint-plugin-storybook'
+        update-types: ['version-update:semver-major']
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:

.github/scripts/install-prerequisites.sh

@@ -17,6 +17,8 @@ ENGINE_DEPENDENCIES=""
 
 if [ "$ENGINE" == "spark" ]; then
     ENGINE_DEPENDENCIES="default-jdk"
+elif [ "$ENGINE" == "db2" ]; then
+    ENGINE_DEPENDENCIES="libxml2-dev build-essential"
 elif [ "$ENGINE" == "fabric" ]; then
     echo "Installing Microsoft package repository"
 

.github/scripts/wait-for-db.sh

@@ -90,6 +90,20 @@ risingwave_ready() {
     probe_port 4566
 }
 
+db2_ready() {
+    probe_port 50001
+
+    echo "Waiting for Db2 to finish initialising (this can take 2-4 minutes)..."
+    while true; do
+        if docker exec db2 su - db2inst1 -c "db2 connect to TESTDB" > /dev/null 2>&1; then
+            echo "Db2 is accepting connections"
+            break
+        fi
+        echo "Db2 not yet ready; sleeping 15s..."
+        sleep 15
+    done
+}
+
 echo "Waiting for $ENGINE to be ready..."
 
 READINESS_FUNC="${ENGINE}_ready"

.github/workflows/db2.yaml

@@ -0,0 +1,65 @@
+name: db2
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'sqlmesh/core/engine_adapter/db2.py'
+      - 'tests/core/engine_adapter/test_db2.py'
+      - 'tests/core/engine_adapter/integration/test_integration_db2.py'
+      - 'tests/core/engine_adapter/integration/docker/compose.db2.yaml'
+      - '.github/workflows/db2.yaml'
+      - 'pyproject.toml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'sqlmesh/core/engine_adapter/db2.py'
+      - 'tests/core/engine_adapter/test_db2.py'
+      - 'tests/core/engine_adapter/integration/test_integration_db2.py'
+      - 'tests/core/engine_adapter/integration/docker/compose.db2.yaml'
+      - '.github/workflows/db2.yaml'
+      - 'pyproject.toml'
+
+concurrency:
+  group: db2-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  db2-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    env:
+      PYTEST_XDIST_AUTO_NUM_WORKERS: 2
+      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: '1'
+      UV: '1'
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install SQLMesh dev dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          make install-dev
+      - name: Install OS-level dependencies
+        run: ./.github/scripts/install-prerequisites.sh "db2"
+      - name: Run tests
+        run: |
+          source .venv/bin/activate
+          make db2-test
+      - name: Upload test results
+        uses: actions/upload-artifact@v7
+        if: ${{ !cancelled() }}
+        with:
+          name: test-results-docker-db2
+          path: test-results/
+          retention-days: 7

.github/workflows/pr.yaml

@@ -17,6 +17,7 @@ jobs:
       python: ${{ steps.filter.outputs.python }}
       client: ${{ steps.filter.outputs.client }}
       vscode: ${{ steps.filter.outputs.vscode }}
+      lsp: ${{ steps.filter.outputs.lsp }}
       ci: ${{ steps.filter.outputs.ci }}
     steps:
       - uses: actions/checkout@v7
@@ -37,6 +38,8 @@ jobs:
               - 'web/client/**'
             vscode:
               - 'vscode/**'
+            lsp:
+              - 'sqlmesh/lsp/**'
             ci:
               - '.github/**'
               - 'Makefile'
@@ -147,6 +150,10 @@ jobs:
           . ./venv/Scripts/activate
           python.exe -m pip install --upgrade pip
           make install-dev
+      - name: Install DuckLake extension
+        run: |
+          . ./venv/Scripts/activate
+          python -c "import duckdb; duckdb.connect().install_extension('ducklake')"
       - name: Run fast unit tests
         run: |
           . ./venv/Scripts/activate
@@ -264,6 +271,7 @@ jobs:
             trino,
             spark,
             clickhouse,
+            clickhouse-datalake,
             risingwave,
             starrocks,
           ]
@@ -290,6 +298,9 @@ jobs:
         run: |
           source .venv/bin/activate
           make ${{ matrix.engine }}-test
+      - name: Stop DataLake services
+        if: ${{ always() && matrix.engine == 'clickhouse-datalake' }}
+        run: make clickhouse-datalake-down
       - name: Upload test results
         uses: actions/upload-artifact@v7
         if: ${{ !cancelled() }}
@@ -427,10 +438,12 @@ jobs:
       - name: Run CI
         run: pnpm run ci
   test-vscode-e2e:
-    runs-on:
-      labels: [ubuntu-2204-8]
-    # As at 2026-01-12 this job flakes 100% of the time. It needs investigation
-    if: false
+    needs: changes
+    if:
+      needs.changes.outputs.vscode == 'true' || needs.changes.outputs.lsp ==
+      'true' || needs.changes.outputs.ci == 'true' || github.ref ==
+      'refs/heads/main'
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-node@v7
@@ -453,7 +466,9 @@ jobs:
           source .venv/bin/activate
           make install-dev
       - name: Install code-server
-        run: curl -fsSL https://code-server.dev/install.sh | sh
+        # Pinned so that a code-server release cannot change the editor UI the
+        # tests drive without the change showing up here first.
+        run: curl -fsSL https://code-server.dev/install.sh | sh -s -- --version 4.137.0
       - name: Install Playwright browsers
         working-directory: ./vscode/extension
         run: pnpm exec playwright install

.github/workflows/release.yaml

@@ -40,9 +40,13 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v7
         with:
-          python-version: '3.10'
+          # 3.13.17+ bundles pip 26.2.2+, which no longer trusts the compromised eMudhra/emSign root CA.
+          python-version: '3.13'
       - name: Install uv
         uses: astral-sh/setup-uv@v7
+      - name: Check pip includes the eMudhra/emSign CA removal
+        run: |
+          python -c "import sys, pip; v = tuple(int(p) for p in pip.__version__.split('.')[:3]); sys.exit(0 if v >= (26, 2, 2) else f'pip {pip.__version__} predates the eMudhra/emSign CA removal; need >= 26.2.2')"
       - name: Install build dependencies
         run: pip install build twine setuptools_scm
       - name: Publish Python package

Makefile

@@ -1,4 +1,5 @@
 .PHONY: docs
+.PHONY: clickhouse-datalake-up clickhouse-datalake-test clickhouse-datalake-down
 
 ifdef UV
     PIP := uv pip
@@ -132,13 +133,13 @@ engine-up: engine-clickhouse-up engine-mssql-up engine-mysql-up engine-postgres-
 engine-down: engine-clickhouse-down engine-mssql-down engine-mysql-down engine-postgres-down engine-spark-down engine-trino-down
 
 fast-test:
-	pytest -n auto -m "fast and not cicdonly and not isolated" --junitxml=test-results/junit-fast-test.xml && pytest -m "isolated and not slow" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "fast and not cicdonly and not isolated" --junitxml=test-results/junit-fast-test.xml && pytest -m "isolated and not slow and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and fork_isolated and not slow" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and fork_isolated and not slow" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 slow-test:
-	pytest -n auto -m "(fast or slow) and not cicdonly and not isolated" && pytest -m "isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "(fast or slow) and not cicdonly and not isolated" && pytest -m "isolated and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and fork_isolated" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 cicd-test:
-	pytest -n auto -m "(fast or slow) and not pyspark and not isolated" --junitxml=test-results/junit-cicd.xml && pytest -m "pyspark" && pytest -m "isolated and not pyspark" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "(fast or slow) and not pyspark and not isolated" --junitxml=test-results/junit-cicd.xml && pytest -m "pyspark" && pytest -m "isolated and not pyspark and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and not pyspark and fork_isolated" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and not pyspark and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 core-fast-test:
 	pytest -n auto -m "fast and not web and not github and not dbt and not jupyter"
@@ -192,6 +193,21 @@ engine-%-up: engine-%-install engine-docker-%-up
 engine-%-down:
 	docker compose -f ./tests/core/engine_adapter/integration/docker/compose.${*}.yaml down -v
 
+CLICKHOUSE_DATALAKE_COMPOSE = docker compose --project-name sqlmesh-clickhouse-datalake -f ./tests/core/engine_adapter/integration/docker/compose.clickhouse-datalake.yaml
+
+clickhouse-datalake-up:
+	$(CLICKHOUSE_DATALAKE_COMPOSE) up -d --wait --wait-timeout 120 clickhouse minio-init rest
+	$(CLICKHOUSE_DATALAKE_COMPOSE) run --build --rm seed
+
+clickhouse-datalake-test:
+	docker info >/dev/null
+	$(MAKE) engine-clickhouse-install
+	$(MAKE) clickhouse-datalake-up
+	SQLMESH_RUN_CLICKHOUSE_DATALAKE=1 pytest -n 0 -m "docker and clickhouse_datalake" tests/core/engine_adapter/integration/test_integration_clickhouse_datalake.py --junitxml=test-results/junit-clickhouse-datalake.xml
+
+clickhouse-datalake-down:
+	$(CLICKHOUSE_DATALAKE_COMPOSE) down -v --remove-orphans
+
 ##################
 # Docker Engines #
 ##################
@@ -222,6 +238,9 @@ risingwave-test: engine-risingwave-up
 
 starrocks-test: engine-starrocks-up
 	pytest -n auto -m "starrocks" --reruns 3 --junitxml=test-results/junit-starrocks.xml
+	
+db2-test: engine-db2-up
+	pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml
 
 #################
 # Cloud Engines #
@@ -235,7 +254,10 @@ bigquery-test: guard-BIGQUERY_KEYFILE engine-bigquery-install
 	pytest -n auto -m "bigquery" --reruns 3 --junitxml=test-results/junit-bigquery.xml
 
 databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_CONNECT_VERSION engine-databricks-install
-	$(PIP) install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}'
+	# Connect 15.3.x requires numpy<2 and pyarrow>=4 (no upper bound). Cap PyArrow
+	# in the same resolve that downgrades NumPy, or PyArrow 25+ is left installed
+	# and cannot be imported.
+	$(PIP) install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}' 'pyarrow<25'
 	pytest -n auto -m "databricks" --reruns 3 --junitxml=test-results/junit-databricks.xml
 
 redshift-test: guard-REDSHIFT_HOST guard-REDSHIFT_USER guard-REDSHIFT_PASSWORD guard-REDSHIFT_DATABASE engine-redshift-install

docs/concepts/audits.md

@@ -477,7 +477,7 @@ MODEL (
 #### valid_uuid, valid_uuid_non_blocking
 Ensures that all non-NULL rows of a column contain a string with the UUID structure.
 
-UUID structure determined by matching regular expression `'^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'`.
+UUID structure determined by matching regular expression `'^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'`.
 
 This example asserts that all `uuid` values have the UUID structure:
 

docs/guides/configuration.md

@@ -1213,10 +1213,12 @@ Example configuration:
 
 #### Default connections/scheduler
 
-The `default_connection`, `default_test_connection`, and `default_scheduler` keys are used to specify shared defaults across multiple gateways.
+The `default_connection`, `default_test_connection`, and `default_scheduler` keys specify a connection or scheduler for gateways that do not define their own.
 
 For example, you might have a specific connection where your tests should run regardless of which gateway is being used. Instead of duplicating the test connection information in each gateway specification, specify it once in the `default_test_connection` key.
 
+A default is only used when a gateway omits the corresponding key entirely. If a gateway specifies its own `connection`, `test_connection`, or `scheduler`, SQLMesh uses that configuration as written and ignores the default. Fields are not merged, so a gateway connection cannot inherit individual fields such as `type` from `default_connection`. Each gateway connection must be a complete connection configuration.
+
 Example configuration specifying a Postgres default connection, in-memory DuckDB default test connection, and builtin default scheduler:
 
 === "YAML"

docs/guides/connections.md

@@ -84,6 +84,7 @@ default_gateway: local_db
 * [BigQuery](../integrations/engines/bigquery.md)
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
+* [Db2](../integrations/engines/db2.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)