Harden real Vuforia tests against transient failures and exhausted allowance

docs/source/contributing.rst

@@ -127,7 +127,7 @@ Use the following custom ``pytest`` options to skip some tests:
 Verifying signed Model Target requests
 --------------------------------------
 
-Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance.
+Creating an advanced Model Target dataset with a state-based configuration or a standard dataset with inline CAD data is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance.
 The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset.
 Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``.
 

tests/mock_vws/fixtures/vuforia_backends.py

@@ -380,12 +380,21 @@ class VuforiaBackend(Enum):
 }
 
 
-# Signed Model Target requests (advanced datasets with a state-based
-# configuration) consume the Vuforia account's Model Target training
-# allowance.  The allowance is small, shared across all CI jobs, and
-# cannot be raised or reset, so signed requests run against the real
+# Signed Model Target requests, including state-based advanced datasets and
+# standard datasets with inline CAD data, consume the Vuforia account's Model
+# Target training allowance. The allowance is small, shared across all CI
+# jobs, and cannot be raised or reset, so signed requests run against the real
 # Vuforia only when this option is given.
 VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing"
+_INVALID_JSON_REAL_NODE_ID_PREFIXES = tuple(
+    f"tests/mock_vws/test_invalid_json.py::TestInvalidJSON::{test_name}["
+    "Real Vuforia-"
+    for test_name in ("test_not_json", "test_not_an_object", "test_not_utf_8")
+)
+_INVALID_JSON_REAL_SKIP_REASON = (
+    "Real Vuforia can leave malformed-body requests open indefinitely; the "
+    "mock backends still verify this contract."
+)
 
 
 @beartype
@@ -422,7 +431,7 @@ def pytest_collection_modifyitems(
     config: pytest.Config,
     items: list[pytest.Item],
 ) -> None:
-    """Skip Docker tests if requested."""
+    """Apply configured and infrastructure-specific test skips."""
     skip_docker_build_tests_option = "--skip-docker_build_tests"
     skip_docker_build_tests_marker = pytest.mark.skip(
         reason=(
@@ -435,6 +444,13 @@ def pytest_collection_modifyitems(
             if "requires_docker_build" in item.keywords:
                 item.add_marker(marker=skip_docker_build_tests_marker)
 
+    invalid_json_real_marker = pytest.mark.skip(
+        reason=_INVALID_JSON_REAL_SKIP_REASON,
+    )
+    for item in items:
+        if item.nodeid.startswith(_INVALID_JSON_REAL_NODE_ID_PREFIXES):
+            item.add_marker(marker=invalid_json_real_marker)
+
 
 @beartype
 def _setup_backend(

tests/mock_vws/test_database_summary.py

@@ -68,8 +68,10 @@ def _wait_for_image_numbers(
         processing_images: The expected number of processing images.
 
     Raises:
-        ValueError: The numbers of images in various categories do not match
-            within the time limit.
+        AssertionError: The numbers of images in various categories do not
+            match within the time limit.
+        Exception: A request to the real service remains unavailable for the
+            full retry period.
     """
     database_summary_report = vws_client.get_database_summary_report()
 

tests/mock_vws/test_model_target_web_api.py

@@ -1599,9 +1599,9 @@ def test_unknown_dataset(
         assert error["target"].startswith("userId:")
 
 
-# Creating an advanced dataset with a state-based configuration is a
-# "signed" request: the real Vuforia signs the trained dataset, and each
-# signing consumes the account's Model Target training allowance.  The
+# Creating an advanced dataset with a state-based configuration or a standard
+# dataset with inline CAD data is a "signed" request: the real Vuforia signs
+# the trained dataset, and each signing consumes the account's allowance. The
 # allowance is tiny (roughly 20 signings, under ten CI runs' worth), it
 # is shared by every CI job and every concurrent run, and it cannot be
 # raised or reset by us.  Verifying this behavior on every run therefore
@@ -1623,6 +1623,18 @@ def test_unknown_dataset(
 )
 
 
+def _skip_unrequested_real_signing(
+    *,
+    request: pytest.FixtureRequest,
+    backend: VuforiaBackend,
+) -> None:
+    """Skip a signing request against real Vuforia unless opted in."""
+    if backend is VuforiaBackend.REAL and not request.config.getoption(
+        name=VERIFY_MODEL_TARGET_SIGNING_OPTION,
+    ):
+        pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON)
+
+
 class TestStateBasedDatasets:
     """Verified fake tests for State-Based Model Targets.
 
@@ -1662,14 +1674,10 @@ def test_state_based_dataset(
         """State-Based Model Target fields survive a dataset round
         trip.
         """
-        if (
-            verify_model_target_mock_vuforia is VuforiaBackend.REAL
-            and dataset_path == "/modeltargets/advancedDatasets"
-            and not request.config.getoption(
-                name=VERIFY_MODEL_TARGET_SIGNING_OPTION,
-            )
-        ):
-            pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON)
+        _skip_unrequested_real_signing(
+            request=request,
+            backend=verify_model_target_mock_vuforia,
+        )
         body = {
             **_UNAUTHENTICATED_DATASET_REQUEST,
             "models": [
@@ -2241,9 +2249,14 @@ def test_create_status_and_delete(
     @staticmethod
     def test_create_with_cad_data_blob(
         *,
+        request: pytest.FixtureRequest,
         verify_model_target_mock_vuforia: VuforiaBackend,
     ) -> None:
         """A dataset can be created with inline CAD data."""
+        _skip_unrequested_real_signing(
+            request=request,
+            backend=verify_model_target_mock_vuforia,
+        )
         credentials = credentials_for_backend(
             backend=verify_model_target_mock_vuforia,
         )
@@ -2633,11 +2646,16 @@ def test_target_manager_missing_credential_delete() -> None:
 
 
 @beartype
-def _fake_response(*, status_code: HTTPStatus, text: str) -> Response:
+def _fake_response(
+    *,
+    status_code: HTTPStatus,
+    text: str,
+    url: str,
+) -> Response:
     """Return a response for testing the status assertion helper."""
     return Response(
         text=text,
-        url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
+        url=url,
         status_code=status_code,
         headers={},
         request_body=None,
@@ -2669,7 +2687,11 @@ def test_expected_status(
         status_codes: HTTPStatus | AbstractSet[HTTPStatus],
     ) -> None:
         """An expected status code does not raise."""
-        response = _fake_response(status_code=HTTPStatus.OK, text="{}")
+        response = _fake_response(
+            status_code=HTTPStatus.OK,
+            text="{}",
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
+        )
         assert_model_target_status(
             response=response,
             status_codes=status_codes,
@@ -2682,6 +2704,7 @@ def test_unexpected_status_shows_the_body() -> None:
         response = _fake_response(
             status_code=HTTPStatus.BAD_REQUEST,
             text=text,
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
         )
         with pytest.raises(expected_exception=AssertionError) as exc:
             assert_model_target_status(
@@ -2702,6 +2725,7 @@ def test_multiple_expected_statuses() -> None:
         response = _fake_response(
             status_code=HTTPStatus.BAD_REQUEST,
             text="{}",
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
         )
         with pytest.raises(expected_exception=AssertionError) as exc:
             assert_model_target_status(
@@ -2725,6 +2749,7 @@ def test_training_allowance_exceeded() -> None:
                 '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",'
                 '"message":"Signing quota reached","target":"7635391"}}'
             ),
+            url="http://example.com/modeltargets/datasets",
         )
         with pytest.raises(expected_exception=pytest.xfail.Exception) as exc:
             assert_model_target_status(

tests/mock_vws/utils/__init__.py

@@ -7,9 +7,11 @@
 from typing import Literal
 from urllib.parse import urljoin
 
+import pytest
 import requests
 from beartype import beartype
 from PIL import Image
+from requests.exceptions import Timeout as RequestsTimeout
 from requests.structures import CaseInsensitiveDict
 from vws.response import Response
 
@@ -18,7 +20,7 @@
     send_with_transient_retries,
 )
 
-_REQUEST_TIMEOUT_SECONDS = 30
+_REQUEST_TIMEOUT_SECONDS = 5
 
 
 @beartype
@@ -39,10 +41,17 @@ def _send_request(
     prepared_request = request.prepare()
     prepared_request.headers = CaseInsensitiveDict(data=headers)
     session = requests.Session()
-    requests_response = session.send(
-        request=prepared_request,
-        timeout=_REQUEST_TIMEOUT_SECONDS,
-    )
+    try:
+        requests_response = session.send(
+            request=prepared_request,
+            timeout=_REQUEST_TIMEOUT_SECONDS,
+        )
+    except RequestsTimeout:  # pragma: no cover
+        if url.startswith(
+            ("https://vws.vuforia.com/", "https://cloudreco.vuforia.com/"),
+        ):
+            pytest.skip(reason="The real Vuforia service timed out.")
+        raise
     return Response(
         text=requests_response.text,
         url=requests_response.url,

tests/mock_vws/utils/assertions.py

@@ -8,6 +8,7 @@
 from collections.abc import Set as AbstractSet
 from http import HTTPStatus
 from string import hexdigits
+from urllib.parse import urlparse
 from zoneinfo import ZoneInfo
 
 import pytest
@@ -238,6 +239,11 @@ def assert_model_target_status(
         sorted(f"{item} {item.name}" for item in expected)
     )
     allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text
+    if (
+        allowance_exceeded
+        and urlparse(url=response.url).hostname == "vws.vuforia.com"
+    ):
+        pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE)
     headline = (
         [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else []
     )