Add vulnerabilities to the PyPI JSON API

CHANGES/1365.feature

@@ -0,0 +1 @@
+Added a `vulnerabilities` field to the PyPI JSON API, populated from OSV scan reports. Remotes can opt in to scan the new repository version after sync.

docs/user/guides/sync.md

@@ -154,6 +154,16 @@ pulp python remote create \
     --keep-latest-packages 5 
 ```
 
+Set `vulnerabilities` on the remote to scan the new repository version after each successful sync. The scan runs as a follow-up task and does not fail the sync if OSV is unreachable. Results are stored as vulnerability reports and exposed on the JSON API.
+
+```bash
+pulp python remote create \
+    --name 'scanned-remote' \
+    --url 'https://pypi.org/' \
+    --includes '["django==5.2.1"]' \
+    --vulnerabilities
+```
+
 Reference: [Python Remote Usage](site:pulp_python/restapi/#tag/Remotes:-Python)
 
 ### Creating a remote to sync all of PyPI

docs/user/guides/vulnerability_report.md

@@ -81,6 +81,16 @@ The report contains detailed information about each vulnerability, including:
 - **References**: Links to advisories and patches
 - **Repository and Content**: Pulp `RepositoryVersion` and `Content` impacted
 
+## JSON API
+
+The PyPI JSON endpoints (`pypi/<project>/json` and `pypi/<project>/<version>/json`) include a
+`vulnerabilities` array for the selected version. Pulp trims stored OSV reports to the Warehouse
+shape (`id`, `source`, `link`, `aliases`, `details`, `summary`, `fixed_in`, `withdrawn`). The key is
+always present; it is an empty list until a scan has run.
+
+Enable `vulnerabilities` on a remote to scan automatically after sync, or scan a repository version
+manually as shown above.
+
 ## Example Workflow
 
 Here's a complete example of scanning a repository for vulnerabilities:

pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("python", "0024_pythonrepository_error_on_reject"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pythonremote",
+            name="vulnerabilities",
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulp_python/app/models.py

@@ -130,7 +130,11 @@ def content_handler(self, path):
             if not settings.DOMAIN_ENABLED:
                 domain = None
             json_body = python_content_to_json(
-                self.base_path, package_content, version=version, domain=domain
+                self.base_path,
+                package_content,
+                version=version,
+                domain=domain,
+                repository_version=self.publication.repository_version,
             )
             if json_body:
                 return json_response(json_body, headers=headers)
@@ -350,6 +354,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin):
         models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list
     )
     provenance = models.BooleanField(default=False)
+    vulnerabilities = models.BooleanField(default=False)
 
     def get_remote_artifact_url(self, relative_path=None, request=None):
         """Get url for remote_artifact"""

pulp_python/app/osv.py

@@ -0,0 +1,42 @@
+from packaging.version import InvalidVersion, Version
+
+
+def _osv_fixed_in(vuln):
+    """Extract PEP 440 fixed versions from an OSV vulnerability record."""
+    fixed = []
+    seen = set()
+    for affected in vuln.get("affected") or []:
+        for range_ in affected.get("ranges") or []:
+            for event in range_.get("events") or []:
+                if "fixed" not in event:
+                    continue
+                version = event["fixed"]
+                if version in seen:
+                    continue
+                try:
+                    Version(version)
+                except InvalidVersion:
+                    continue
+                seen.add(version)
+                fixed.append(version)
+    return fixed
+
+
+def osv_to_pypi_vulnerabilities(vulns):
+    """Trim OSV vulnerability records to the Warehouse JSON API shape."""
+    seen = {}
+    for vuln in vulns or []:
+        vuln_id = vuln.get("id")
+        if not vuln_id or vuln_id in seen:
+            continue
+        seen[vuln_id] = {
+            "id": vuln_id,
+            "source": "osv",
+            "link": f"https://osv.dev/vulnerability/{vuln_id}",
+            "aliases": vuln.get("aliases") or [],
+            "details": vuln.get("details"),
+            "summary": vuln.get("summary"),
+            "fixed_in": _osv_fixed_in(vuln),
+            "withdrawn": vuln.get("withdrawn"),
+        }
+    return list(seen.values())

pulp_python/app/pypi/serializers.py

@@ -35,6 +35,9 @@ class PackageMetadataSerializer(serializers.Serializer):
     info = serializers.JSONField(help_text=_("Core metadata of the package"))
     releases = serializers.JSONField(help_text=_("List of all the releases of the package"))
     urls = serializers.JSONField()
+    vulnerabilities = serializers.JSONField(
+        help_text=_("Known vulnerabilities for the selected package version."),
+    )
 
 
 class PackageUploadSerializer(serializers.Serializer):

pulp_python/app/serializers.py

@@ -789,6 +789,11 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
         help_text=_("Whether to sync available provenances for Python packages."),
         default=False,
     )
+    vulnerabilities = serializers.BooleanField(
+        required=False,
+        help_text=_("Whether to scan the new repository version for vulnerabilities after a sync."),
+        default=False,
+    )
 
     def validate_includes(self, value):
         """Validates the includes"""
@@ -821,6 +826,7 @@ class Meta:
             "keep_latest_packages",
             "exclude_platforms",
             "provenance",
+            "vulnerabilities",
         )
         model = python_models.PythonRemote
 

pulp_python/app/tasks/__init__.py

@@ -6,5 +6,5 @@
 from .repair import repair  # noqa:F401
 from .sync import sync  # noqa:F401
 from .upload import upload, upload_group  # noqa:F401
-from .vulnerability_report import get_repo_version_content  # noqa:F401
+from .vulnerability_report import dispatch_scan, get_repo_version_content  # noqa:F401
 from .yank import aunyank_package, ayank_package  # noqa:F401

pulp_python/app/tasks/sync.py

@@ -29,6 +29,7 @@
     PythonRemote,
 )
 from pulp_python.app.provenance import Provenance
+from pulp_python.app.tasks.vulnerability_report import dispatch_scan
 from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata
 
 logger = logging.getLogger(__name__)
@@ -56,7 +57,9 @@ def sync(remote_pk, repository_pk, mirror):
         raise SyncError("A remote must have a url attribute to sync.")
 
     first_stage = PythonBanderStage(remote)
-    DeclarativeVersion(first_stage, repository, mirror).create()
+    new_version = DeclarativeVersion(first_stage, repository, mirror).create()
+    if new_version and remote.vulnerabilities:
+        dispatch_scan(repository, new_version)
 
 
 def create_bandersnatch_config(remote):

pulp_python/app/tasks/vulnerability_report.py

@@ -1,5 +1,6 @@
 from pulpcore.plugin.models import RepositoryVersion
 from pulpcore.plugin.sync import sync_to_async_iterable
+from pulpcore.plugin.tasking import check_content, dispatch
 
 from pulp_python.app.models import PythonPackageContent
 
@@ -28,3 +29,13 @@ def _build_osv_data(name, ecosystem, version=None):
     if version:
         osv_data["version"] = version
     return osv_data
+
+
+def dispatch_scan(repository, repository_version):
+    """Dispatch a vulnerability scan for a repository version."""
+    func = f"{get_repo_version_content.__module__}.{get_repo_version_content.__name__}"
+    return dispatch(
+        check_content,
+        shared_resources=[repository],
+        args=[func, [str(repository_version.pk)]],
+    )

pulp_python/app/utils.py

@@ -21,9 +21,11 @@
 from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
 
 from pulpcore.plugin.exceptions import TimeoutException
-from pulpcore.plugin.models import Artifact, Remote
+from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport
 from pulpcore.plugin.util import get_domain
 
+from pulp_python.app.osv import osv_to_pypi_vulnerabilities
+
 log = logging.getLogger(__name__)
 
 
@@ -377,6 +379,7 @@ def python_content_to_json(
         last_serial: int
         releases: Dict
         urls: Dict
+        vulnerabilities: List
 
     Returns None if version is specified but not found within content_query
     """
@@ -407,9 +410,27 @@ def python_content_to_json(
     full_metadata["info"] = python_content_to_info(latest_content[0])
     full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain)
     full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain)
+    full_metadata["vulnerabilities"] = _vulnerabilities_for_content(
+        latest_content, repository_version
+    )
     return full_metadata
 
 
+def _vulnerabilities_for_content(contents, repository_version=None):
+    """Load VulnerabilityReports scanned for this repository version and trim to Warehouse shape."""
+    if not contents or repository_version is None:
+        return []
+    reports = VulnerabilityReport.objects.filter(
+        content_id__in=[c.pk for c in contents],
+        repo_versions=repository_version,
+    )
+    merged = []
+    for vulns in reports.values_list("vulns", flat=True):
+        if vulns:
+            merged.extend(vulns)
+    return osv_to_pypi_vulnerabilities(merged)
+
+
 def latest_content_version(all_content, version):
     """
     Walks through the content list and finds the instances that are the latest version.

pulp_python/app/viewsets.py

@@ -26,7 +26,7 @@
     RepositoryAddRemoveContentSerializer,
     RepositorySyncURLSerializer,
 )
-from pulpcore.plugin.tasking import check_content, dispatch
+from pulpcore.plugin.tasking import dispatch
 from pulpcore.plugin.util import extract_pk
 
 from pulp_python.app import models as python_models
@@ -356,14 +356,7 @@ def scan(self, request, repository_pk, **kwargs):
         Scan a repository version for vulnerabilities.
         """
         repository_version = self.get_object()
-        func = (
-            f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}"
-        )
-        task = dispatch(
-            check_content,
-            shared_resources=[repository_version.repository],
-            args=[func, [repository_version.pk]],
-        )
+        task = tasks.dispatch_scan(repository_version.repository, repository_version)
         return core_viewsets.OperationPostponedResponse(task, request)
 
 

pulp_python/tests/functional/api/test_pypi_apis.py

@@ -324,6 +324,7 @@ def assert_pypi_json(package):
             package["releases"][version],
             "Failed to match version",
         )
+    assert package["vulnerabilities"] == []
 
 
 def assert_download_info(expected, received, message="Failed to match"):
@@ -377,6 +378,8 @@ def test_upload_time_reflects_repo_addition(
     # JSON API
     json_resp = requests.get(urljoin(distro.base_url, "pypi/twine/json"))
     assert json_resp.status_code == 200
-    json_time = datetime.fromisoformat(json_resp.json()["urls"][0]["upload_time"])
+    package = json_resp.json()
+    json_time = datetime.fromisoformat(package["urls"][0]["upload_time"])
     assert json_time > content_created
     assert json_time == simple_time
+    assert package["vulnerabilities"] == []