Skip to content

Commit 7651bd6

Browse files
committed
feat(bigquery): add bigquery_upgrade_manifest_biglake_table sample to upgrade a manifest external table to BigLake with automatic metadata caching
Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/query-open-table-format-using-manifest-files) needs a standalone, tested Python code sample demonstrating how to upgrade a manifest external table to BigLake with automatic metadata caching (region tag `bigquery_upgrade_manifest_biglake_table`). Solution: - Add `upgrade_manifest_biglake_table(project_id: str, dataset_id: str, table_id: str, connection_id: str, manifest_uri: str) -> None` in `bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py` (Upgrade a manifest-backed external table to a BigLake table with AUTOMATIC metadata caching and connection binding.) - Add unit tests (`bigquery/biglake/bigquery_upgrade_manifest_biglake_table_test.py`) covering both the success path and API error handling (`GoogleAPICallError`)
1 parent 186be98 commit 7651bd6

5 files changed

Lines changed: 193 additions & 0 deletions

File tree

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START bigquery_upgrade_manifest_biglake_table]
16+
import sys
17+
18+
from google.api_core import exceptions
19+
from google.cloud import bigquery
20+
21+
22+
def upgrade_manifest_biglake_table(
23+
project_id: str,
24+
dataset_id: str,
25+
table_id: str,
26+
connection_id: str,
27+
manifest_uri: str,
28+
) -> None:
29+
"""Upgrades a manifest external table to a BigLake table with metadata caching.
30+
31+
Args:
32+
project_id: Google Cloud project ID (for example, 'example-project').
33+
dataset_id: BigQuery dataset ID containing the external table
34+
(for example, 'example_dataset').
35+
table_id: BigQuery external table ID to upgrade
36+
(for example, 'example_manifest_table').
37+
connection_id: Fully qualified BigQuery Cloud Resource connection ID
38+
(for example, 'example-project.us.example_biglake_conn').
39+
manifest_uri: Cloud Storage URI of the SymLinkTextInputFormat manifest file
40+
(for example, 'gs://example-bucket/manifests/symlink.manifest').
41+
"""
42+
client = bigquery.Client(project=project_id)
43+
table_ref = f"{project_id}.{dataset_id}.{table_id}"
44+
# Upgrading the manifest external table with a Cloud Resource connection and AUTOMATIC metadata
45+
# caching accelerates open-table-format reads and enforces BigLake governance.
46+
sql = f"""
47+
CREATE OR REPLACE EXTERNAL TABLE `{table_ref}`
48+
WITH CONNECTION `{connection_id}`
49+
OPTIONS (
50+
format = 'PARQUET',
51+
uris = ['{manifest_uri}'],
52+
file_set_spec_type = 'NEW_LINE_DELIMITED_MANIFEST',
53+
max_staleness = INTERVAL 1 HOUR,
54+
metadata_cache_mode = 'AUTOMATIC'
55+
);
56+
SELECT
57+
id,
58+
label,
59+
score
60+
FROM `{table_ref}`
61+
ORDER BY id ASC
62+
LIMIT 10;
63+
"""
64+
65+
try:
66+
print(
67+
f"Upgrading manifest external table to BigLake with metadata caching: {table_ref}"
68+
)
69+
query_job = client.query(sql)
70+
rows = list(query_job.result())
71+
except exceptions.NotFound as err:
72+
print(
73+
f"error: manifest external table '{table_id}' or connection '{connection_id}' "
74+
f"not found in dataset '{dataset_id}' for project '{project_id}' "
75+
f"(manifest '{manifest_uri}'): {err.message}",
76+
file=sys.stderr,
77+
)
78+
return
79+
except exceptions.GoogleAPICallError as err:
80+
print(
81+
f"error: failed to upgrade manifest external table '{table_id}' in dataset "
82+
f"'{dataset_id}' for project '{project_id}' (connection '{connection_id}', "
83+
f"manifest '{manifest_uri}'): {err.message}",
84+
file=sys.stderr,
85+
)
86+
return
87+
88+
print(f"Resource ID: {table_ref}")
89+
print(f"Connection ID: {connection_id}")
90+
print(f"Job ID: {query_job.job_id}")
91+
print(f"Manifest URI: {manifest_uri}")
92+
print("Metadata Cache Mode: AUTOMATIC")
93+
print(f"Rows Returned: {len(rows)}")
94+
95+
96+
# [END bigquery_upgrade_manifest_biglake_table]
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from unittest import mock
16+
17+
import bigquery_upgrade_manifest_biglake_table
18+
from google.api_core import exceptions
19+
import pytest
20+
21+
22+
def test_upgrade_manifest_biglake_table_success(
23+
capsys: pytest.CaptureFixture[str],
24+
) -> None:
25+
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
26+
mock_client = mock_client_cls.return_value
27+
mock_job = mock.MagicMock()
28+
mock_job.job_id = "example-job-123"
29+
mock_job.result.return_value = [
30+
mock.MagicMock(),
31+
mock.MagicMock(),
32+
mock.MagicMock(),
33+
]
34+
mock_client.query.return_value = mock_job
35+
36+
bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table(
37+
project_id="example-project",
38+
dataset_id="example_dataset",
39+
table_id="example_manifest_table",
40+
connection_id="example-project.us.example_biglake_conn",
41+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
42+
)
43+
44+
mock_client.query.assert_called_once()
45+
captured = capsys.readouterr()
46+
assert (
47+
"Resource ID: example-project.example_dataset.example_manifest_table"
48+
in captured.out
49+
)
50+
assert "Metadata Cache Mode: AUTOMATIC" in captured.out
51+
assert "Rows Returned: 3" in captured.out
52+
53+
54+
def test_upgrade_manifest_biglake_table_not_found(
55+
capsys: pytest.CaptureFixture[str],
56+
) -> None:
57+
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
58+
mock_client = mock_client_cls.return_value
59+
mock_client.query.side_effect = exceptions.NotFound("Connection not found")
60+
61+
bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table(
62+
project_id="example-project",
63+
dataset_id="example_dataset",
64+
table_id="example_manifest_table",
65+
connection_id="example-project.us.example_biglake_conn",
66+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
67+
)
68+
69+
captured = capsys.readouterr()
70+
assert "error:" in captured.err
71+
assert "example_manifest_table" in captured.err

‎bigquery/biglake/noxfile_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
TEST_CONFIG_OVERRIDE = {
16+
# You can opt out from the test for specific Python versions.
17+
"ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"],
18+
# Enforce type hints on new samples.
19+
"enforce_type_hints": True,
20+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
21+
"pip_version_override": None,
22+
"envs": {},
23+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pytest==9.0.3; python_version >= "3.10"

‎bigquery/biglake/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
google-cloud-bigquery==3.47.0
2+
google-cloud-bigquery-storage==2.43.0

0 commit comments

Comments
 (0)