From f3ed92e380ea07177dcb019454d2b4af8ff15c72 Mon Sep 17 00:00:00 2001 From: TenSt Date: Thu, 27 Aug 2026 17:06:54 +0200 Subject: [PATCH] feat: add HTTP catalog for Python packages and repository metrics Clients can list distinct packages and repository counts over the REST API instead of querying the database. The content list also supports collapsing rebuilds and returns base_version. Closes #1358. Assisted-By: Cursor --- .gitignore | 4 + CHANGES/1358.feature | 1 + CLAUDE.md | 61 +---- docs/index.md | 4 +- docs/user/guides/_SUMMARY.md | 1 + docs/user/guides/catalog.md | 96 ++++++++ pulp_python/app/catalog.py | 166 +++++++++++++ pulp_python/app/serializers.py | 71 ++++++ pulp_python/app/utils.py | 1 - pulp_python/app/versions.py | 17 ++ pulp_python/app/viewsets.py | 183 +++++++++++++- .../tests/functional/api/test_catalog.py | 230 ++++++++++++++++++ pulp_python/tests/unit/test_catalog.py | 27 ++ 13 files changed, 796 insertions(+), 66 deletions(-) create mode 100644 CHANGES/1358.feature create mode 100644 docs/user/guides/catalog.md create mode 100644 pulp_python/app/catalog.py create mode 100644 pulp_python/app/versions.py create mode 100644 pulp_python/tests/functional/api/test_catalog.py create mode 100644 pulp_python/tests/unit/test_catalog.py diff --git a/.gitignore b/.gitignore index 29c94b654..444a89de7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv/ build/ develop-eggs/ dist/ @@ -61,3 +62,6 @@ target/ # PyCharm .idea + +# VS Code / Cursor +.vscode/ diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 000000000..a3a40f519 --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added repository package catalog and metrics endpoints, plus ``collapse_builds`` and ``base_version`` on the Python package content API. Existing installs pick up access policy for the new actions on migrate unless the policy was customized. diff --git a/CLAUDE.md b/CLAUDE.md index 525ea0401..681311eb9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,60 +1 @@ -# CLAUDE.md - -The role of this file is to describe common mistakes and confusion points that agents might encounter as they work in this project. -If you ever encounter something in the project that surprises you, please alert the developer working with you and indicate that this is the case in the CLAUDE.md file to help prevent future agents from having the same issue. - -## Interacting with the developer environment - -Use the `pulp-cli` to interact with the Pulp API. Fallback on `httpie/curl` when the CLI doesn't support the endpoint/options needed. - -```bash -pulp --help -pulp --refresh-api status -pulp python content -t package list --limit 5 -pulp python repository create --name foo -pulp -v python repository sync --name foo --remote foo -pulp task show --wait --href prn:core.task:019c8cae-cc5f-7148-a3de-456d0a9f39a1 -pulp show --href /pulp/api/v3/tasks/019c8cae-cc5f-7148-a3de-456d0a9f39a1/ -``` - -Use the `oci-env` cli to interact with the developer's Pulp instance. It has commands for managing state, running tests, and executing commands against a running Pulp. - -```bash -oci-env --help -oci-env compose ps # check status of the Pulp dev container -oci-env compose up/down/restart # start/stop/restart the Pulp dev container -oci-env poll --attempts 10 --wait 10 # wait till Pulp container finishes booting up -oci-env pstart/pstop/prestart # start/stop/restart the services inside the Pulp container -oci-env generate-client --help # create the client bindings needed for the functional tests! -oci-env test --help # run the functional/unit tests -oci-env pulpcore-manager # run any pulpcore or Django commands -``` - -## Running/Writing tests - -Prefer writing functional tests for new changes/bugfixes and only fallback on unit tests when the change is not easily testable through the API. - -pulp-python functional tests require pulp-python, pulpcore & pulp-file client bindings to be installed. The bindings must be regenerated for any changes to the API spec. - -**Always** use the `oci-env` to run the functional and unit tests. - -## Modifying template_config.yml - -Use the `plugin-template` tool after any changes made to `template_config.yml`. - -```bash -# typically located in the parent directory of pulpcore/plugin -../plugin_template/plugin-template --github -``` - -## Fixing failed backports - -When patchback fails to cherry-pick a PR into an older branch, you need to manually apply the equivalent change. Key things to know: - -- Older branches (e.g. 3.11, 3.12) may use `requirements.txt` for dependencies, while newer branches use `pyproject.toml`. Always check which file the target branch uses before applying changes. -- When creating a PR include `[]` in the PR title (e.g. `[3.12] Raise upperbound for pkginfo requirement`). -- Use `git cherry-pick -x`. - -## Contributing - -When preparing to commit and create a PR you **must** follow our [PR checklist](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) Important to note is the AI attribution requirement in our commit messages. Also, note that our changelog entries are markdown. +CLAUDE.md \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 80c26e434..ff9bb16d1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories @@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful. This documentation falls into two main categories: 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs. -2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each -minor feature, including all fields and options. +2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb2..36699ede5 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 000000000..65a35ddc2 --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,96 @@ +# Browse the package catalog + +Pulp CLI commands for these endpoints are generated from the OpenAPI spec in a separate package; until that is updated, use HTTP. + +The content list (`/pulp/api/v3/content/python/packages/`) returns **one row per distribution file** (wheel, sdist, …). For catalog UIs and automation that need **one row per package name**, plus repository metrics, use the repository package index. + +These endpoints default to the **latest complete repository version**. `{pulp_id}` is the repository UUID. Pass `repository_version` (HREF or PRN) to read a specific version of that repository. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +Pagination `count` is the number of **distinct packages** (`name_normalized`), not files. + +Each row includes both a simple version list and per-version metadata: + +```json +{ + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "versions": ["0.1"], + "latest_releases": [ + { + "version": "0.1", + "release": "", + "created_at": "2026-08-10T10:45:08.099362Z" + } + ] +} +``` + +`set(versions)` is always the same as `set(latest_releases[].version)`. There is one `latest_releases` entry per **logical version** (after stripping a trailing rebuild suffix `\.[a-zA-Z]+-\d+$`), not per wheel or sdist. + +`created_at` is when that logical version entered the repository: the earliest `RepositoryContent.pulp_created` among its files, falling back to the content unit's `pulp_created`. `release` is empty until Python rebuilds are stored. + +### Prefix search + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + name_normalized__istartswith==shelf +``` + +`name_normalized__istartswith` and `name__istartswith` are case-insensitive (`ILIKE`). Prefix search belongs on this index, not on the flat content list. + +## Repository metrics + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/metrics/" +``` + +```json +{ + "package_count": 3, + "version_count": 9, + "build_count": 9 +} +``` + +Counts use Python package content units in that repository version (not filtered by `packagetype`): + +| Field | Identity | +|-------|----------| +| `package_count` | distinct `name_normalized` | +| `version_count` | distinct `(name_normalized, base_version)` after rebuild-suffix strip | +| `build_count` | distinct `(name_normalized, full version)` | + +Until rebuild suffixes exist, `version_count` equals `build_count`. + +## List versions of a package + +Use the existing content API. Pass `packagetype=sdist` for one representative file per PEP version (retry with `packagetype=bdist_wheel` if a release is wheel-only). + +`collapse_builds=true` keeps one unit per logical version (`name_normalized` + `base_version`), the one with the latest `pulp_created`. Do not nest rebuilds on this list. Clients can drain Pulp `next` if the page is full. + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + packagetype==sdist \ + collapse_builds==true \ + repository_version=="${LATEST_VERSION_HREF}" +``` + +Every content row includes `base_version` (stripped version; equal to `version` when there is no suffix). + +## Get one version + +Omit `collapse_builds`. Filter with `name`, `version`, and `packagetype=sdist`: + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + version==0.1 \ + packagetype==sdist +``` diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 000000000..6c35c8f14 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,166 @@ +"""Helpers for repository package catalog, metrics, and rebuild collapse.""" + +from collections import defaultdict + +from django.db.models import CharField, Func, Max, Min, Q, Value +from packaging.version import InvalidVersion, Version + +from pulp_python.app.models import PythonPackageContent + +# POSIX regex for REGEXP_REPLACE. PostgreSQL does not treat ``\d`` as digits. +BUILD_SUFFIX_PG_REGEX = r"\.[a-zA-Z]+-[0-9]+$" + + +def base_version_annotation(field_name="version"): + """SQL expression that strips a trailing rebuild suffix from ``version``. + + PostgreSQL POSIX regex does not treat ``\\d`` as digits, so the SQL pattern + uses ``[0-9]`` while the Python pattern in ``versions.strip_build_suffix`` uses ``\\d``. + Implemented with ``REGEXP_REPLACE`` so it does not depend on Django's + ``RegexpReplace`` (not present in every Django 4.2/5.2 packaging Pulp uses). + """ + return Func( + field_name, + Value(BUILD_SUFFIX_PG_REGEX), + Value(""), + function="REGEXP_REPLACE", + output_field=CharField(), + ) + + +def collapse_python_builds(queryset): + """Keep one content unit per ``(name_normalized, base_version)``. + + ``base_version`` is ``version`` with a trailing rebuild suffix stripped. + The unit with the latest ``pulp_created`` is kept. Callers that want one + row per logical version (not per wheel/sdist) should also filter + ``packagetype``. + """ + return ( + queryset.prefetch_related(None) + .annotate(_collapse_base_version=base_version_annotation()) + .order_by("name_normalized", "_collapse_base_version", "-pulp_created") + .distinct("name_normalized", "_collapse_base_version") + ) + + +def python_packages_in_version(repository_version): + """Python package content contained in ``repository_version``.""" + if repository_version is None: + return PythonPackageContent.objects.none() + return PythonPackageContent.objects.filter(pk__in=repository_version.content) + + +def apply_package_prefix_filters(queryset, name_normalized_prefix=None, name_prefix=None): + """Apply case-insensitive prefix filters used by the package index.""" + if name_normalized_prefix: + queryset = queryset.filter(name_normalized__istartswith=name_normalized_prefix) + if name_prefix: + queryset = queryset.filter(name__istartswith=name_prefix) + return queryset + + +def distinct_package_names_qs(content_qs): + """One row per distinct ``name_normalized``, ordered for stable pagination.""" + return ( + content_qs.order_by() + .values("name_normalized") + .annotate(name=Max("name")) + .order_by("name_normalized") + ) + + +def _version_sort_key(version): + try: + return (0, Version(version)) + except InvalidVersion: + return (1, version) + + +def assemble_package_index(content_qs, name_rows, repository, repository_version): + """Build package-index dicts for ``name_rows``. + + ``created_at`` is the earliest repository-membership time + (``RepositoryContent.pulp_created``) of any file of that logical version + in ``repository_version``, falling back to the content unit's ``pulp_created``. + """ + if not name_rows or repository_version is None: + return [] + + names = [row["name_normalized"] for row in name_rows] + name_by_normalized = {row["name_normalized"]: row["name"] for row in name_rows} + + in_this_version = Q( + version_memberships__repository=repository, + version_memberships__version_added__number__lte=repository_version.number, + ) & ( + Q(version_memberships__version_removed__isnull=True) + | Q(version_memberships__version_removed__number__gt=repository_version.number) + ) + + release_rows = ( + content_qs.filter(name_normalized__in=names) + .annotate(_base_version=base_version_annotation()) + .values("name_normalized", "_base_version") + .annotate( + membership_created=Min( + "version_memberships__pulp_created", + filter=in_this_version, + ), + unit_created=Min("pulp_created"), + ) + ) + + releases_by_name = defaultdict(list) + for rel in release_rows: + releases_by_name[rel["name_normalized"]].append(rel) + + result = [] + for row in name_rows: + normalized = row["name_normalized"] + rels = sorted( + releases_by_name.get(normalized, []), + key=lambda item: _version_sort_key(item["_base_version"]), + ) + versions = [item["_base_version"] for item in rels] + latest_releases = [ + { + "version": item["_base_version"], + "release": "", + "created_at": item["membership_created"] or item["unit_created"], + } + for item in rels + ] + result.append( + { + "name": name_by_normalized[normalized], + "name_normalized": normalized, + "versions": versions, + "latest_releases": latest_releases, + } + ) + return result + + +def repository_metrics(content_qs): + """Distinct package / logical-version / build counts for package content. + + Identity is always ``PythonPackageContent`` (not filtered by packagetype): + + * ``package_count``: distinct ``name_normalized`` + * ``version_count``: distinct ``(name_normalized, base_version)`` + * ``build_count``: distinct ``(name_normalized, version)`` + + Until rebuild suffixes exist, ``version_count`` equals ``build_count``. + """ + content_qs = content_qs.order_by() + return { + "package_count": content_qs.values("name_normalized").distinct().count(), + "version_count": ( + content_qs.annotate(_base_version=base_version_annotation()) + .values("name_normalized", "_base_version") + .distinct() + .count() + ), + "build_count": content_qs.values("name_normalized", "version").distinct().count(), + } diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..f3f607ab9 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -34,6 +34,7 @@ get_project_metadata_from_file, parse_project_metadata, ) +from pulp_python.app.versions import strip_build_suffix log = logging.getLogger(__name__) PYPI_BASE_URL = urljoin(settings.PYPI_API_HOSTNAME, settings.PYPI_PATH_PREFIX) @@ -231,6 +232,16 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa help_text=_("The packages version number."), read_only=True, ) + base_version = serializers.SerializerMethodField( + help_text=_( + "The package version with a trailing rebuild suffix stripped " + r"(matching \.[a-zA-Z]+-\d+$). Equal to version when no suffix is present." + ), + ) + + def get_base_version(self, obj): + return strip_build_suffix(obj.version) + # Version 1.1 classifiers = serializers.JSONField( required=False, @@ -518,6 +529,7 @@ class Meta: "platform", "summary", "version", + "base_version", "classifiers", "download_url", "supported_platform", @@ -636,11 +648,70 @@ class Meta: "packagetype", "name", "version", + "base_version", "sha256", ) model = python_models.PythonPackageContent +class PythonPackageReleaseSerializer(serializers.Serializer): + """One logical version on the repository package index.""" + + version = serializers.CharField( + help_text=_("Logical version key (rebuild suffix stripped)."), + ) + release = serializers.CharField( + help_text=_( + "Rebuild/release qualifier within the version line. Empty until rebuilds exist." + ), + allow_blank=True, + ) + created_at = serializers.DateTimeField( + help_text=_( + "When this logical version entered the repository: the earliest " + "RepositoryContent.pulp_created among files of this version, falling back to " + "the content unit's pulp_created." + ), + ) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package in a repository version (not per wheel/sdist file).""" + + name = serializers.CharField(help_text=_("A representative project name for this package.")) + name_normalized = serializers.CharField( + help_text=_("PEP 503 normalized package name. Index rows are unique on this field."), + ) + versions = serializers.ListField( + child=serializers.CharField(), + help_text=_( + "Distinct logical version keys after rebuild-suffix strip. " + "The set of values matches latest_releases[].version." + ), + ) + latest_releases = PythonPackageReleaseSerializer( + many=True, + help_text=_( + "One object per logical version (not per distribution file). " + "set(versions) === set(latest_releases[].version)." + ), + ) + + +class PythonRepositoryMetricsSerializer(serializers.Serializer): + """Distinct package / version / build counts for a repository version.""" + + package_count = serializers.IntegerField( + help_text=_("Distinct name_normalized values among Python package content units."), + ) + version_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, base_version) pairs after rebuild-suffix strip."), + ) + build_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, full version) pairs among package content units."), + ) + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..091c6b69e 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -26,7 +26,6 @@ log = logging.getLogger(__name__) - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 diff --git a/pulp_python/app/versions.py b/pulp_python/app/versions.py new file mode 100644 index 000000000..83f079d80 --- /dev/null +++ b/pulp_python/app/versions.py @@ -0,0 +1,17 @@ +"""Rebuild-suffix helpers for catalog ``base_version`` (no Django imports). + +CI unit tests run with ``-p no:pulpcore``, so this module must stay importable +before Django apps are loaded. +""" + +import re + +# Rebuild suffix (Lightwell-style): 5.3.18.rhlw-00003 -> 5.3.18. Not hard-coded to "rhlw". +BUILD_SUFFIX_RE = re.compile(r"\.[a-zA-Z]+-\d+$") + + +def strip_build_suffix(version): + """Return ``version`` with a trailing rebuild suffix removed, else unchanged.""" + if not version: + return version + return BUILD_SUFFIX_RE.sub("", version) diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..dbe4c12e1 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -4,7 +4,13 @@ from django.db import transaction from django_filters import CharFilter from django_filters.rest_framework import filters as drf_filters -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiParameter, + extend_schema, + extend_schema_view, + inline_serializer, +) from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name from rest_framework import status @@ -16,7 +22,7 @@ RetrieveModelMixin, ) from rest_framework.response import Response -from rest_framework.serializers import ValidationError +from rest_framework.serializers import IntegerField, URLField, ValidationError from pulpcore.plugin import viewsets as core_viewsets from pulpcore.plugin.actions import ModifyRepositoryActionMixin @@ -32,6 +38,14 @@ from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import ( + apply_package_prefix_filters, + assemble_package_index, + collapse_python_builds, + distinct_package_names_qs, + python_packages_in_version, + repository_metrics, +) class PythonRepositoryViewSet( @@ -64,7 +78,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["retrieve"], + "action": ["retrieve", "packages", "metrics"], "principal": "authenticated", "effect": "allow", "condition": "has_model_or_domain_or_obj_perms:python.view_pythonrepository", @@ -138,6 +152,22 @@ class PythonRepositoryViewSet( "python.pythonrepository_viewer": ["python.view_pythonrepository"], } + def filter_queryset(self, queryset): + """Do not apply the repository FilterSet to package-index query params.""" + if getattr(self, "action", None) in ("packages", "metrics"): + return queryset + return super().filter_queryset(queryset) + + def _requested_repository_version(self, repository): + """Resolve optional ``repository_version`` href/PRN, else latest complete version.""" + href = self.request.query_params.get("repository_version") + if not href: + return repository.latest_version() + repo_version = self.get_resource(href, RepositoryVersion) + if repo_version.repository_id != repository.pk: + raise ValidationError({"repository_version": "Must be a version of this repository."}) + return repo_version + @extend_schema( description="Trigger an asynchronous task to create a new repository version.", summary="Modify Repository Content", @@ -247,6 +277,122 @@ def sync(self, request, pk, **kwargs): ) return core_viewsets.OperationPostponedResponse(result, request) + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in a repository version " + "(latest complete version if repository_version is omitted). " + "Pagination count is the number of distinct packages, not files. " + "Each row includes versions (logical version keys after rebuild-suffix strip) " + "and latest_releases (one object per logical version). " + "created_at is the earliest repository-membership time " + "(RepositoryContent.pulp_created) of any file of that logical version, " + "falling back to the content unit's pulp_created. " + "release is empty until Python rebuilds are stored. " + "set(versions) === set(latest_releases[].version)." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + OpenApiParameter( + name="name_normalized__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Case-insensitive prefix on the PEP 503 normalized package name.", + ), + OpenApiParameter( + name="name__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Case-insensitive prefix on the original package name.", + ), + ], + responses={ + 200: inline_serializer( + name="PaginatedPythonRepositoryPackageList", + fields={ + "count": IntegerField(), + "next": URLField(allow_null=True), + "previous": URLField(allow_null=True), + "results": python_serializers.PythonRepositoryPackageSerializer(many=True), + }, + ) + }, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryPackageSerializer, + ) + def packages(self, request, pk): + """List distinct packages in a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + content_qs = python_packages_in_version(repo_version) + content_qs = apply_package_prefix_filters( + content_qs, + name_normalized_prefix=request.query_params.get("name_normalized__istartswith"), + name_prefix=request.query_params.get("name__istartswith"), + ) + names_qs = distinct_package_names_qs(content_qs) + page = self.paginate_queryset(names_qs) + rows = assemble_package_index( + content_qs, + page if page is not None else list(names_qs), + repository, + repo_version, + ) + serializer = self.get_serializer(rows, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + + @extend_schema( + summary="Repository metrics", + description=( + "Distinct counts for Python package content in a repository version " + "(latest complete version if repository_version is omitted). " + "package_count is distinct name_normalized. version_count is distinct " + "(name_normalized, base_version) after rebuild-suffix strip. build_count is " + "distinct (name_normalized, full version). Counts are not filtered by " + "packagetype. Until rebuild suffixes exist, version_count equals build_count." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + ], + responses={200: python_serializers.PythonRepositoryMetricsSerializer}, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryMetricsSerializer, + ) + def metrics(self, request, pk): + """Return package / version / build counts for a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + serializer = self.get_serializer( + repository_metrics(python_packages_in_version(repo_version)) + ) + return Response(serializer.data) + class PythonBlocklistEntryViewSet( core_viewsets.NamedModelViewSet, @@ -506,6 +652,25 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): field_name="version", help_text="Filter by PEP 440 version specifier (e.g., >=2.4,<3.0 or ~=1.26)", ) + collapse_builds = drf_filters.BooleanFilter( + method="filter_collapse_builds", + help_text=( + "When true, collapse rebuilds of the same logical version: strip a trailing " + r"suffix matching \.[a-zA-Z]+-\d+$ from version, then keep one content unit " + "per (name_normalized, base_version) with the latest pulp_created. " + "Pass packagetype=sdist so wheel and sdist files are not collapsed together. " + "Default false." + ), + ) + + def filter_collapse_builds(self, qs, name, value): + """Documented on the FilterSet; applied in the viewset after ordering. + + DISTINCT ON requires ORDER BY to start with the distinct columns. The + viewset applies collapse after other filter backends so that ordering + cannot break it. + """ + return qs class Meta: model = python_models.PythonPackageContent @@ -539,6 +704,18 @@ class PythonPackageSingleArtifactContentUploadViewSet( minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer filterset_class = PythonPackageContentFilter + def filter_queryset(self, queryset): + """Apply ``collapse_builds`` after other backends so DISTINCT ON stays valid.""" + queryset = super().filter_queryset(queryset) + if getattr(self, "action", "") != "list": + return queryset + raw = self.request.query_params.get("collapse_builds") + if raw is None or raw == "": + return queryset + if str(raw).lower() in ("true", "t", "yes", "y", "1"): + return collapse_python_builds(queryset) + return queryset + DEFAULT_ACCESS_POLICY = { "statements": [ { diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 000000000..974283a55 --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,230 @@ +"""Catalog API tests. + +Generated client methods are unavailable until `oci-env generate-client` is rerun. +""" + +from urllib.parse import urljoin + +import pytest +import requests + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +def _api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +def _content_packages_path(repo_href): + marker = "/api/v3/" + idx = repo_href.find(marker) + assert idx != -1, repo_href + return f"{repo_href[: idx + len(marker)]}content/python/packages/" + + +def _assert_package_row(pkg): + assert pkg["name"] + assert pkg["name_normalized"] + assert set(pkg["versions"]) == {rel["version"] for rel in pkg["latest_releases"]} + for rel in pkg["latest_releases"]: + assert rel["release"] == "" + assert rel["created_at"] + + +@pytest.fixture +def sm_repo(python_repo_with_sync, python_remote_factory): + remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER) + return python_repo_with_sync(remote) + + +@pytest.mark.parallel +def test_package_list_grouping_and_pagination(bindings_cfg, sm_repo): + """Package index is one row per name, and count is distinct packages not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1) + assert data["count"] == 3 + assert len(data["results"]) == 1 + _assert_package_row(data["results"][0]) + + page2 = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1, offset=1) + assert page2["count"] == 3 + assert page2["results"][0]["name_normalized"] != data["results"][0]["name_normalized"] + + all_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=100)["results"] + assert {pkg["name_normalized"] for pkg in all_rows} == {"aiohttp", "celery", "django"} + django = next(pkg for pkg in all_rows if pkg["name_normalized"] == "django") + assert set(django["versions"]) == {"1.10.1", "1.10.2", "1.10.3", "1.10.4"} + # Dual-field contract: one latest_releases entry per logical version, not per wheel/sdist. + assert len(django["latest_releases"]) == 4 + + +@pytest.mark.parallel +def test_package_list_istartswith(bindings_cfg, sm_repo): + """Prefix search is case-insensitive on the package index.""" + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="djan" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="DJAN" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", name__istartswith="Cel") + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "celery" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="shelf" + ) + assert data["count"] == 0 + + +@pytest.mark.parallel +def test_package_list_empty_repository(bindings_cfg, python_repo_factory): + repo = python_repo_factory() + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 0 + assert data["results"] == [] + + +@pytest.mark.parallel +def test_repository_metrics(bindings_cfg, sm_repo, python_repo_factory): + """Metrics count distinct packages / logical versions / builds, not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + assert data["package_count"] == 3 + # aiohttp 3 + celery 2 + Django 4; no rebuild suffixes in fixtures. + assert data["version_count"] == 9 + assert data["build_count"] == 9 + assert data["version_count"] == data["build_count"] + + empty = _api_get(bindings_cfg, f"{python_repo_factory().pulp_href}metrics/") + assert empty == {"package_count": 0, "version_count": 0, "build_count": 0} + + +@pytest.mark.parallel +def test_packages_and_metrics_repository_version(bindings_cfg, sm_repo, python_repo_factory): + """repository_version selects a snapshot; omitted uses the latest complete version.""" + latest_href = sm_repo.latest_version_href + v0_href = f"{sm_repo.pulp_href}versions/0/" + + default_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/") + explicit_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=latest_href + ) + assert default_pkgs["count"] == explicit_pkgs["count"] == 3 + + v0_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=v0_href) + assert v0_pkgs["count"] == 0 + assert v0_pkgs["results"] == [] + + default_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + explicit_metrics = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=latest_href + ) + assert default_metrics == explicit_metrics + v0_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=v0_href) + assert v0_metrics == {"package_count": 0, "version_count": 0, "build_count": 0} + + other = python_repo_factory() + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"repository_version": other.latest_version_href}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + + +@pytest.mark.parallel +def test_collapse_builds_and_base_version(bindings_cfg, sm_repo): + """collapse_builds keeps one unit per logical version; base_version is always present.""" + path = _content_packages_path(sm_repo.pulp_href) + repo_version = sm_repo.latest_version_href + + expanded = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + collapsed = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + # Wheel + sdist per Django version collapse when packagetype is omitted. + assert expanded["count"] == 8 + assert collapsed["count"] == 4 + assert {item["base_version"] for item in collapsed["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + for item in expanded["results"] + collapsed["results"]: + assert item["base_version"] == item["version"] + + sdist_false = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + sdist_true = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + assert sdist_false["count"] == 4 + assert sdist_true["count"] == 4 + assert {item["version"] for item in sdist_true["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + + +@pytest.mark.parallel +def test_package_get_base_version_without_collapse(bindings_cfg, python_repo_with_sync): + """PackageGet uses the content list without collapse_builds; base_version is still present.""" + repo = python_repo_with_sync() + path = _content_packages_path(repo.pulp_href) + data = _api_get( + bindings_cfg, + path, + name="shelf-reader", + version="0.1", + packagetype="sdist", + ) + assert data["count"] == 1 + item = data["results"][0] + assert item["version"] == "0.1" + assert item["base_version"] == "0.1" + assert "collapse_builds" not in item + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + row = pkgs["results"][0] + _assert_package_row(row) + assert row["name_normalized"] == "shelf-reader" + assert row["versions"] == ["0.1"] + assert len(row["latest_releases"]) == 1 diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py new file mode 100644 index 000000000..dcf729b69 --- /dev/null +++ b/pulp_python/tests/unit/test_catalog.py @@ -0,0 +1,27 @@ +"""Unit tests for rebuild-suffix stripping. + +CI runs these with ``pytest -p no:pulpcore``. Import ``versions``, not ``utils``: +``utils`` pulls in Django models and fails collection with AppRegistryNotReady. +""" + +import pytest + +from pulp_python.app.versions import strip_build_suffix + + +@pytest.mark.parametrize( + "version,expected", + [ + ("0.1", "0.1"), + ("5.3.18", "5.3.18"), + ("5.3.18.rhlw-00003", "5.3.18"), + ("1.0.0.abc-1", "1.0.0"), + ("1.0.0.ABC-99", "1.0.0"), + ("1.0.foo-bar", "1.0.foo-bar"), + ("1.0.rhlw-00003.extra", "1.0.rhlw-00003.extra"), + ("", ""), + (None, None), + ], +) +def test_strip_build_suffix(version, expected): + assert strip_build_suffix(version) == expected