[hnswlib] Fix SIGILL in stubtest by building without -march=native (#16244)

hnswlib is source-only and compiles with -march=native by default. CI
restores pip's wheel cache across runners with different CPUs, so a
wheel built on one host can hit an illegal instruction on the next.
That is the SIGILL (exit -4) with empty output from #16100.

Add a general `install-environment` key to [tool.stubtest] that sets
environment variables for the pip install step, and use it to pass
hnswlib's own HNSWLIB_NO_NATIVE opt-out. This lets the darwin-only
workaround from #16125 be dropped so hnswlib is tested on Linux again.

Author: ekanshul

CONTRIBUTING.md

@@ -235,6 +235,12 @@ when running stubtest. For example: `mypy-plugins = ["mypy_django_plugin.main"]`
 * `mypy-plugins-config` (default: `{}`): A dictionary mapping plugin names to their
 configuration dictionaries for use by mypy plugins. For example:
 `mypy-plugins-config = {"django-stubs" = {"django_settings_module" = "@tests.django_settings"}}`
+* `install-environment` (default: `{}`): A dictionary of environment variables
+  to set while `pip install`ing the package and its dependencies for stubtest.
+  Useful for packages that read build-time options from the environment, for
+  example to disable CPU-specific compiler flags that do not survive CI's
+  shared wheel cache. For example:
+  `install-environment = { HNSWLIB_NO_NATIVE = "1" }`
 
 `*-dependencies` are usually packages needed to `pip install` the implementation
 distribution.

lib/ts_utils/metadata.py

@@ -53,6 +53,10 @@ def _is_nested_dict(obj: object) -> TypeGuard[dict[str, dict[str, Any]]]:
     return isinstance(obj, dict) and all(isinstance(k, str) and isinstance(v, dict) for k, v in obj.items())
 
 
+def _is_dict_of_strings(obj: object) -> TypeGuard[dict[str, str]]:
+    return isinstance(obj, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in obj.items())
+
+
 @functools.cache
 def get_oldest_supported_python() -> str:
     with PYPROJECT_PATH.open("rb") as config:
@@ -85,6 +89,7 @@ class StubtestSettings:
     stubtest_dependencies: list[str]
     mypy_plugins: list[str]
     mypy_plugins_config: dict[str, dict[str, Any]]
+    install_environment: dict[str, str]
 
     def system_requirements_for_platform(self, platform: str) -> list[str]:
         assert platform in _STUBTEST_PLATFORM_MAPPING, f"Unrecognised platform {platform!r}"
@@ -110,6 +115,7 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     stubtest_dependencies: object = data.get("stubtest-dependencies", [])
     mypy_plugins: object = data.get("mypy-plugins", [])
     mypy_plugins_config: object = data.get("mypy-plugins-config", {})
+    install_environment: object = data.get("install-environment", {})
 
     assert type(skip) is bool
     assert type(ignore_missing_stub) is bool
@@ -124,6 +130,7 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     assert _is_list_of_strings(stubtest_dependencies)
     assert _is_list_of_strings(mypy_plugins)
     assert _is_nested_dict(mypy_plugins_config)
+    assert _is_dict_of_strings(install_environment)
 
     unrecognised_platforms = set(ci_platforms) - _STUBTEST_PLATFORM_MAPPING.keys()
     assert not unrecognised_platforms, f"Unrecognised ci-platforms specified for {distribution!r}: {unrecognised_platforms}"
@@ -152,6 +159,7 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
         stubtest_dependencies=stubtest_dependencies,
         mypy_plugins=mypy_plugins,
         mypy_plugins_config=mypy_plugins_config,
+        install_environment=install_environment,
     )
 
 
@@ -227,6 +235,7 @@ def all_dependencies(self) -> list[Requirement]:
         "stubtest-dependencies",
         "mypy-plugins",
         "mypy-plugins-config",
+        "install-environment",
     }
 }
 _DIST_NAME_RE: Final = re.compile(r"^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$", re.IGNORECASE)

stubs/hnswlib/METADATA.toml

@@ -4,6 +4,8 @@ upstream-repository = "https://github.com/nmslib/hnswlib"
 dependencies = ["numpy>=1.21"]
 
 [tool.stubtest]
-# TODO: stubtest fails on Linux because it gets killed with a SIGILL
-# for unknown reasons. See https://github.com/python/typeshed/issues/16100
-ci-platforms = ["darwin"]
+# hnswlib is source-only and compiles with -march=native by default. CI
+# restores pip's wheel cache across runners with different CPUs, so a wheel
+# built on one machine can SIGILL on the next. See
+# https://github.com/python/typeshed/issues/16100
+install-environment = { HNSWLIB_NO_NATIVE = "1" }

tests/stubtest_third_party.py

@@ -91,8 +91,12 @@ def run_stubtest(dist: Path, *, verbose: bool = False, ci_platforms_only: bool =
             dists_to_install[:] = dists_to_install[1:]
 
         pip_cmd = [pip_exe, "install", *dists_to_install]
+        # Some packages read environment variables at build time, e.g. to
+        # opt out of CPU-specific compiler flags. See `install-environment`
+        # in CONTRIBUTING.md.
+        pip_env = os.environ | stubtest_settings.install_environment
         try:
-            subprocess.run(pip_cmd, check=True, capture_output=True)
+            subprocess.run(pip_cmd, env=pip_env, check=True, capture_output=True)
         except subprocess.CalledProcessError as e:
             print_command_failure("Failed to install", e)
             return False