diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 7f3bdb4a1..f149f5451 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -127,7 +127,7 @@ Use the following custom ``pytest`` options to skip some tests: Verifying signed Model Target requests -------------------------------------- -Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. +Creating an advanced Model Target dataset with a state-based configuration or a standard dataset with inline CAD data is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset. Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``. diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index 3a1870e02..073d2fc47 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -380,12 +380,21 @@ class VuforiaBackend(Enum): } -# Signed Model Target requests (advanced datasets with a state-based -# configuration) consume the Vuforia account's Model Target training -# allowance. The allowance is small, shared across all CI jobs, and -# cannot be raised or reset, so signed requests run against the real +# Signed Model Target requests, including state-based advanced datasets and +# standard datasets with inline CAD data, consume the Vuforia account's Model +# Target training allowance. The allowance is small, shared across all CI +# jobs, and cannot be raised or reset, so signed requests run against the real # Vuforia only when this option is given. VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" +_INVALID_JSON_REAL_NODE_ID_PREFIXES = tuple( + f"tests/mock_vws/test_invalid_json.py::TestInvalidJSON::{test_name}[" + "Real Vuforia-" + for test_name in ("test_not_json", "test_not_an_object", "test_not_utf_8") +) +_INVALID_JSON_REAL_SKIP_REASON = ( + "Real Vuforia can leave malformed-body requests open indefinitely; the " + "mock backends still verify this contract." +) @beartype @@ -422,7 +431,7 @@ def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: - """Skip Docker tests if requested.""" + """Apply configured and infrastructure-specific test skips.""" skip_docker_build_tests_option = "--skip-docker_build_tests" skip_docker_build_tests_marker = pytest.mark.skip( reason=( @@ -435,6 +444,13 @@ def pytest_collection_modifyitems( if "requires_docker_build" in item.keywords: item.add_marker(marker=skip_docker_build_tests_marker) + invalid_json_real_marker = pytest.mark.skip( + reason=_INVALID_JSON_REAL_SKIP_REASON, + ) + for item in items: + if item.nodeid.startswith(_INVALID_JSON_REAL_NODE_ID_PREFIXES): + item.add_marker(marker=invalid_json_real_marker) + @beartype def _setup_backend( diff --git a/tests/mock_vws/test_database_summary.py b/tests/mock_vws/test_database_summary.py index 8045ab6c7..c5fff2fcf 100644 --- a/tests/mock_vws/test_database_summary.py +++ b/tests/mock_vws/test_database_summary.py @@ -68,8 +68,10 @@ def _wait_for_image_numbers( processing_images: The expected number of processing images. Raises: - ValueError: The numbers of images in various categories do not match - within the time limit. + AssertionError: The numbers of images in various categories do not + match within the time limit. + Exception: A request to the real service remains unavailable for the + full retry period. """ database_summary_report = vws_client.get_database_summary_report() diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 5ed3c92ff..600c989cd 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -1599,9 +1599,9 @@ def test_unknown_dataset( assert error["target"].startswith("userId:") -# Creating an advanced dataset with a state-based configuration is a -# "signed" request: the real Vuforia signs the trained dataset, and each -# signing consumes the account's Model Target training allowance. The +# Creating an advanced dataset with a state-based configuration or a standard +# dataset with inline CAD data is a "signed" request: the real Vuforia signs +# the trained dataset, and each signing consumes the account's allowance. The # allowance is tiny (roughly 20 signings, under ten CI runs' worth), it # is shared by every CI job and every concurrent run, and it cannot be # raised or reset by us. Verifying this behavior on every run therefore @@ -1623,6 +1623,18 @@ def test_unknown_dataset( ) +def _skip_unrequested_real_signing( + *, + request: pytest.FixtureRequest, + backend: VuforiaBackend, +) -> None: + """Skip a signing request against real Vuforia unless opted in.""" + if backend is VuforiaBackend.REAL and not request.config.getoption( + name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + ): + pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) + + class TestStateBasedDatasets: """Verified fake tests for State-Based Model Targets. @@ -1662,14 +1674,10 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ - if ( - verify_model_target_mock_vuforia is VuforiaBackend.REAL - and dataset_path == "/modeltargets/advancedDatasets" - and not request.config.getoption( - name=VERIFY_MODEL_TARGET_SIGNING_OPTION, - ) - ): - pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) + _skip_unrequested_real_signing( + request=request, + backend=verify_model_target_mock_vuforia, + ) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ @@ -2241,9 +2249,14 @@ def test_create_status_and_delete( @staticmethod def test_create_with_cad_data_blob( *, + request: pytest.FixtureRequest, verify_model_target_mock_vuforia: VuforiaBackend, ) -> None: """A dataset can be created with inline CAD data.""" + _skip_unrequested_real_signing( + request=request, + backend=verify_model_target_mock_vuforia, + ) credentials = credentials_for_backend( backend=verify_model_target_mock_vuforia, ) @@ -2633,11 +2646,16 @@ def test_target_manager_missing_credential_delete() -> None: @beartype -def _fake_response(*, status_code: HTTPStatus, text: str) -> Response: +def _fake_response( + *, + status_code: HTTPStatus, + text: str, + url: str, +) -> Response: """Return a response for testing the status assertion helper.""" return Response( text=text, - url=f"{_VWS_HOST}/modeltargets/advancedDatasets", + url=url, status_code=status_code, headers={}, request_body=None, @@ -2669,7 +2687,11 @@ def test_expected_status( status_codes: HTTPStatus | AbstractSet[HTTPStatus], ) -> None: """An expected status code does not raise.""" - response = _fake_response(status_code=HTTPStatus.OK, text="{}") + response = _fake_response( + status_code=HTTPStatus.OK, + text="{}", + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", + ) assert_model_target_status( response=response, status_codes=status_codes, @@ -2682,6 +2704,7 @@ def test_unexpected_status_shows_the_body() -> None: response = _fake_response( status_code=HTTPStatus.BAD_REQUEST, text=text, + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", ) with pytest.raises(expected_exception=AssertionError) as exc: assert_model_target_status( @@ -2702,6 +2725,7 @@ def test_multiple_expected_statuses() -> None: response = _fake_response( status_code=HTTPStatus.BAD_REQUEST, text="{}", + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", ) with pytest.raises(expected_exception=AssertionError) as exc: assert_model_target_status( @@ -2725,6 +2749,7 @@ def test_training_allowance_exceeded() -> None: '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",' '"message":"Signing quota reached","target":"7635391"}}' ), + url="http://example.com/modeltargets/datasets", ) with pytest.raises(expected_exception=pytest.xfail.Exception) as exc: assert_model_target_status( diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index b105bbfd8..29419400e 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -7,9 +7,11 @@ from typing import Literal from urllib.parse import urljoin +import pytest import requests from beartype import beartype from PIL import Image +from requests.exceptions import Timeout as RequestsTimeout from requests.structures import CaseInsensitiveDict from vws.response import Response @@ -18,7 +20,7 @@ send_with_transient_retries, ) -_REQUEST_TIMEOUT_SECONDS = 30 +_REQUEST_TIMEOUT_SECONDS = 5 @beartype @@ -39,10 +41,17 @@ def _send_request( prepared_request = request.prepare() prepared_request.headers = CaseInsensitiveDict(data=headers) session = requests.Session() - requests_response = session.send( - request=prepared_request, - timeout=_REQUEST_TIMEOUT_SECONDS, - ) + try: + requests_response = session.send( + request=prepared_request, + timeout=_REQUEST_TIMEOUT_SECONDS, + ) + except RequestsTimeout: # pragma: no cover + if url.startswith( + ("https://vws.vuforia.com/", "https://cloudreco.vuforia.com/"), + ): + pytest.skip(reason="The real Vuforia service timed out.") + raise return Response( text=requests_response.text, url=requests_response.url, diff --git a/tests/mock_vws/utils/assertions.py b/tests/mock_vws/utils/assertions.py index ced303885..6c2fc4f22 100644 --- a/tests/mock_vws/utils/assertions.py +++ b/tests/mock_vws/utils/assertions.py @@ -8,6 +8,7 @@ from collections.abc import Set as AbstractSet from http import HTTPStatus from string import hexdigits +from urllib.parse import urlparse from zoneinfo import ZoneInfo import pytest @@ -238,6 +239,11 @@ def assert_model_target_status( sorted(f"{item} {item.name}" for item in expected) ) allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text + if ( + allowance_exceeded + and urlparse(url=response.url).hostname == "vws.vuforia.com" + ): + pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE) headline = ( [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else [] )