pulp-python 3.13.4__tar.gz → 3.14.0__tar.gz

{pulp_python-3.13.4 → pulp_python-3.14.0}/CHANGES.md

@@ -8,9 +8,21 @@
 
 [//]: # (towncrier release notes start)
 
-## 3.13.4 (2025-04-10) {: #3.13.4 }
+## 3.14.0 (2025-04-10) {: #3.14.0 }
 
-#### Misc {: #3.13.4-misc }
+#### Features {: #3.14.0-feature }
+
+- Pull-through caching now respects the include/exclude filters on the upstream remote.
+  [#706](https://github.com/pulp/pulp_python/issues/706)
+- Added support for automatically saving pull-through content to a repository.
+  Requires pulpcore 3.74 or later.
+  [#815](https://github.com/pulp/pulp_python/issues/815)
+
+#### Bugfixes {: #3.14.0-bugfix }
+
+- Fixed a proxy sync regression introduced in 3.13.0.
+
+#### Misc {: #3.14.0-misc }
 
 - [#809](https://github.com/pulp/pulp_python/issues/809)
 
@@ -68,6 +80,14 @@ No significant changes.
 
 ---
 
+## 3.12.6 (2025-02-26) {: #3.12.6 }
+
+#### Misc {: #3.12.6-misc }
+
+- 
+
+---
+
 ## 3.12.5 (2024-10-25) {: #3.12.5 }
 
 #### Bugfixes {: #3.12.5-bugfix }
@@ -142,7 +162,20 @@ No significant changes.
 
 ---
 
-## 3.11.3 (2024-08-21) {: #3.11.3 }
+## 3.11.4 (2025-02-20) {: #3.11.4 }
+
+#### Bugfixes {: #3.11.4-bugfix }
+
+- Fixed the JSONField specification so it doesn't break ruby bindings.
+  See context [here](https://github.com/pulp/pulp_rpm/issues/3639).
+
+#### Misc {: #3.11.4-misc }
+
+- 
+
+---
+
+# ## 3.11.3 (2024-08-21) {: #3.11.3 }
 
 #### Bugfixes {: #3.11.3-bugfix }
 

{pulp_python-3.13.4 → pulp_python-3.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-python
-Version: 3.13.4
+Version: 3.14.0
 Summary: pulp-python plugin for the Pulp Project
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org

{pulp_python-3.13.4 → pulp_python-3.14.0}/pulp_python/app/__init__.py

@@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_python.app"
     label = "python"
-    version = "3.13.4"
+    version = "3.14.0"
     python_package_name = "pulp-python"
     domain_compatible = True
 

{pulp_python-3.13.4 → pulp_python-3.14.0}/pulp_python/app/models.py

@@ -275,6 +275,7 @@ class PythonRepository(Repository, AutoAddObjPermsMixin):
     TYPE = "python"
     CONTENT_TYPES = [PythonPackageContent]
     REMOTE_TYPES = [PythonRemote]
+    PULL_THROUGH_SUPPORTED = True
 
     autopublish = models.BooleanField(default=False)
 

{pulp_python-3.13.4 → pulp_python-3.14.0}/pulp_python/app/pypi/views.py

@@ -1,6 +1,7 @@
+import json
 import logging
-import requests
 
+from aiohttp.client_exceptions import ClientError
 from rest_framework.viewsets import ViewSet
 from rest_framework.response import Response
 from django.core.exceptions import ObjectDoesNotExist
@@ -15,7 +16,8 @@ from django.http.response import (
     Http404,
     HttpResponseForbidden,
     HttpResponseBadRequest,
-    StreamingHttpResponse
+    StreamingHttpResponse,
+    HttpResponse,
 )
 from drf_spectacular.utils import extend_schema
 from dynaconf import settings
@@ -23,11 +25,12 @@ from itertools import chain
 from packaging.utils import canonicalize_name
 from urllib.parse import urljoin, urlparse, urlunsplit
 from pathlib import PurePath
-from pypi_simple import parse_links_stream_response
+from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
 
 from pulpcore.plugin.viewsets import OperationPostponedResponse
 from pulpcore.plugin.tasking import dispatch
 from pulpcore.plugin.util import get_domain
+from pulpcore.plugin.exceptions import TimeoutException
 from pulp_python.app.models import (
     PythonDistribution,
     PythonPackageContent,
@@ -37,7 +40,7 @@ from pulp_python.app.pypi.serializers import (
     SummarySerializer,
     PackageMetadataSerializer,
     PackageUploadSerializer,
-    PackageUploadTaskSerializer
+    PackageUploadTaskSerializer,
 )
 from pulp_python.app.utils import (
     write_simple_index,
@@ -45,6 +48,7 @@ from pulp_python.app.utils import (
     python_content_to_json,
     PYPI_LAST_SERIAL,
     PYPI_SERIAL_CONSTANT,
+    get_remote_package_filter,
 )
 
 from pulp_python.app import tasks
@@ -233,27 +237,36 @@ class SimpleView(PackageUploadMixin, ViewSet):
 
     def pull_through_package_simple(self, package, path, remote):
         """Gets the package's simple page from remote."""
-        def parse_url(link):
-            parsed = urlparse(link.url)
-            digest, _, value = parsed.fragment.partition('=')
+        def parse_package(release_package):
+            parsed = urlparse(release_package.url)
             stripped_url = urlunsplit(chain(parsed[:3], ("", "")))
-            redirect = f'{path}/{link.text}?redirect={stripped_url}'
-            d_url = urljoin(self.base_content_url, redirect)
-            return link.text, d_url, value if digest == 'sha256' else ''
+            redirect_path = f'{path}/{release_package.filename}?redirect={stripped_url}'
+            d_url = urljoin(self.base_content_url, redirect_path)
+            return release_package.filename, d_url, release_package.digests.get("sha256", "")
+
+        rfilter = get_remote_package_filter(remote)
+        if not rfilter.filter_project(package):
+            raise Http404(f"{package} does not exist.")
 
         url = remote.get_remote_artifact_url(f'simple/{package}/')
-        kwargs = {}
-        if proxy_url := remote.proxy_url:
-            if remote.proxy_username or remote.proxy_password:
-                parsed_proxy = urlparse(proxy_url)
-                netloc = f"{remote.proxy_username}:{remote.proxy_password}@{parsed_proxy.netloc}"
-                proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
-            kwargs["proxies"] = {"http": proxy_url, "https": proxy_url}
-
-        response = requests.get(url, stream=True, **kwargs)
-        links = parse_links_stream_response(response)
-        packages = (parse_url(link) for link in links)
-        return StreamingHttpResponse(write_simple_detail(package, packages, streamed=True))
+        remote.headers = remote.headers or []
+        remote.headers.append({"Accept": ACCEPT_JSON_PREFERRED})
+        downloader = remote.get_downloader(url=url, max_retries=1)
+        try:
+            d = downloader.fetch()
+        except ClientError:
+            return HttpResponse(f"Failed to fetch {package} from {remote.url}.", status=502)
+        except TimeoutException:
+            return HttpResponse(f"{remote.url} timed out while fetching {package}.", status=504)
+
+        if d.headers["content-type"] == "application/vnd.pypi.simple.v1+json":
+            page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=remote.url)
+        else:
+            page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=remote.url)
+        packages = [
+            parse_package(p) for p in page.packages if rfilter.filter_release(package, p.version)
+        ]
+        return HttpResponse(write_simple_detail(package, packages))
 
     @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page")
     def retrieve(self, request, path, package):
@@ -262,8 +275,7 @@ class SimpleView(PackageUploadMixin, ViewSet):
         # Should I redirect if the normalized name is different?
         normalized = canonicalize_name(package)
         if self.distribution.remote:
-            if not repo_ver or not content.filter(name__normalize=normalized).exists():
-                return self.pull_through_package_simple(normalized, path, self.distribution.remote)
+            return self.pull_through_package_simple(normalized, path, self.distribution.remote)
         if self.should_redirect(repo_version=repo_ver):
             return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/'))
         packages = (

{pulp_python-3.13.4 → pulp_python-3.14.0}/pulp_python/app/utils.py

@@ -7,7 +7,8 @@ from collections import defaultdict
 from django.conf import settings
 from jinja2 import Template
 from packaging.utils import canonicalize_name
-from packaging.version import parse
+from packaging.requirements import Requirement
+from packaging.version import parse, InvalidVersion
 
 
 PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
@@ -356,3 +357,80 @@ def write_simple_detail(project_name, project_packages, streamed=False):
     detail = Template(simple_detail_template)
     context = {"project_name": project_name, "project_packages": project_packages}
     return detail.stream(**context) if streamed else detail.render(**context)
+
+
+class PackageIncludeFilter:
+    """A special class to help filter Package's based on a remote's include/exclude"""
+
+    def __init__(self, remote):
+        self.remote = remote.cast()
+        self._filter_includes = self._parse_packages(self.remote.includes)
+        self._filter_excludes = self._parse_packages(self.remote.excludes)
+
+    def _parse_packages(self, packages):
+        config = defaultdict(lambda: defaultdict(list))
+        for value in packages:
+            requirement = Requirement(value)
+            requirement.name = canonicalize_name(requirement.name)
+            if requirement.specifier:
+                requirement.specifier.prereleases = True
+                config["range"][requirement.name].append(requirement)
+            else:
+                config["full"][requirement.name].append(requirement)
+        return config
+
+    def filter_project(self, project_name):
+        """Return true/false if project_name would be allowed through remote's filters."""
+        project_name = canonicalize_name(project_name)
+        include_full = self._filter_includes.get("full", {})
+        include_range = self._filter_includes.get("range", {})
+        include = set(include_range.keys()).union(include_full.keys())
+        if include and project_name not in include:
+            return False
+
+        exclude_full = self._filter_excludes.get("full", {})
+        if project_name in exclude_full:
+            return False
+
+        return True
+
+    def filter_release(self, project_name, version):
+        """Returns true/false if release would be allowed through remote's filters."""
+        project_name = canonicalize_name(project_name)
+        if not self.filter_project(project_name):
+            return False
+
+        try:
+            version = parse(version)
+        except InvalidVersion:
+            return False
+
+        include_range = self._filter_includes.get("range", {})
+        if project_name in include_range:
+            for req in include_range[project_name]:
+                if version in req.specifier:
+                    break
+            else:
+                return False
+
+        exclude_range = self._filter_excludes.get("range", {})
+        if project_name in exclude_range:
+            for req in exclude_range[project_name]:
+                if version in req.specifier:
+                    return False
+
+        return True
+
+
+_remote_filters = {}
+
+
+def get_remote_package_filter(remote):
+    if date_filter_tuple := _remote_filters.get(remote.pulp_id):
+        last_update, rfilter = date_filter_tuple
+        if last_update == remote.pulp_last_updated:
+            return rfilter
+
+    rfilter = PackageIncludeFilter(remote)
+    _remote_filters[remote.pulp_id] = (remote.pulp_last_updated, rfilter)
+    return rfilter

pulp_python-3.14.0/pulp_python/tests/functional/api/test_full_mirror.py

@@ -0,0 +1,140 @@
+import pytest
+import requests
+import subprocess
+
+from pulp_python.tests.functional.constants import (
+    PYPI_URL,
+    PYTHON_XS_FIXTURE_CHECKSUMS,
+    PYTHON_SM_PROJECT_SPECIFIER,
+    PYTHON_SM_FIXTURE_RELEASES,
+)
+
+from pypi_simple import ProjectPage
+from packaging.version import parse
+from urllib.parse import urljoin, urlsplit
+from random import sample
+
+
+def test_pull_through_install(
+    python_bindings, python_remote_factory, python_distribution_factory, delete_orphans_pre
+):
+    """Tests that a pull-through distro can be installed from."""
+    remote = python_remote_factory(url=PYPI_URL, includes=[])
+    distro = python_distribution_factory(remote=remote.pulp_href)
+    PACKAGE = "pulpcore-releases"
+
+    # Check if already installed
+    stdout = subprocess.run(("pip", "list"), capture_output=True).stdout.decode("utf-8")
+    if stdout.find(PACKAGE) != -1:
+        subprocess.run(("pip", "uninstall", PACKAGE, "-y"))
+
+    # Perform pull-through install
+    host = urlsplit(distro.base_url).hostname
+    url = f"{distro.base_url}simple/"
+    cmd = ("pip", "install", "--trusted-host", host, "-i", url, PACKAGE)
+    subprocess.run(cmd, check=True)
+
+    stdout = subprocess.run(("pip", "list"), capture_output=True).stdout.decode("utf-8")
+    assert stdout.find(PACKAGE) != -1
+    subprocess.run(("pip", "uninstall", PACKAGE, "-y"))
+    content = python_bindings.ContentPackagesApi.list(name=PACKAGE)
+    assert content.count == 1
+
+
+@pytest.mark.parallel
+def test_pull_through_simple(python_remote_factory, python_distribution_factory, pulp_content_url):
+    """Tests that the simple page is properly modified when requesting a pull-through."""
+    remote = python_remote_factory(url=PYPI_URL, includes=["shelf-reader"])
+    distro = python_distribution_factory(remote=remote.pulp_href)
+
+    url = f"{distro.base_url}simple/shelf-reader/"
+    project_page = ProjectPage.from_response(requests.get(url), "shelf-reader")
+
+    assert len(project_page.packages) == 2
+    for package in project_page.packages:
+        assert package.filename in PYTHON_XS_FIXTURE_CHECKSUMS
+        relative_path = f"{distro.base_path}/{package.filename}?redirect="
+        assert urljoin(pulp_content_url, relative_path) in package.url
+        assert PYTHON_XS_FIXTURE_CHECKSUMS[package.filename] == package.digests["sha256"]
+
+
+@pytest.mark.parallel
+def test_pull_through_filter(python_remote_factory, python_distribution_factory):
+    """Tests that pull-through respects the includes/excludes filter on the remote."""
+    remote = python_remote_factory(url=PYPI_URL, includes=["shelf-reader"])
+    distro = python_distribution_factory(remote=remote.pulp_href)
+
+    r = requests.get(f"{distro.base_url}simple/pulpcore/")
+    assert r.status_code == 404
+    assert r.json() == {"detail": "pulpcore does not exist."}
+
+    r = requests.get(f"{distro.base_url}simple/shelf-reader/")
+    assert r.status_code == 200
+
+    # Test complex include specifiers
+    remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER)
+    distro = python_distribution_factory(remote=remote.pulp_href)
+    for package, releases in PYTHON_SM_FIXTURE_RELEASES.items():
+        url = f"{distro.base_url}simple/{package}/"
+        project_page = ProjectPage.from_response(requests.get(url), package)
+        packages = {p.filename for p in project_page.packages if not parse(p.version).is_prerelease}
+        assert packages == set(releases)
+
+    # Test exclude logic
+    remote = python_remote_factory(includes=[], excludes=["django"])
+    distro = python_distribution_factory(remote=remote.pulp_href)
+
+    r = requests.get(f"{distro.base_url}simple/django/")
+    assert r.status_code == 404
+    assert r.json() == {"detail": "django does not exist."}
+
+    r = requests.get(f"{distro.base_url}simple/pulpcore/")
+    assert r.status_code == 502
+    assert r.text == f"Failed to fetch pulpcore from {remote.url}."
+
+    r = requests.get(f"{distro.base_url}simple/shelf-reader/")
+    assert r.status_code == 200
+
+
+@pytest.mark.parallel
+def test_pull_through_with_repo(
+    python_repo_factory,
+    python_remote_factory,
+    python_distribution_factory,
+    python_bindings,
+    pulpcore_bindings,
+    monitor_task,
+    has_pulp_plugin,
+):
+    """Tests that content is saved to repository."""
+    remote = python_remote_factory(includes=[])
+    repo = python_repo_factory()
+    distro = python_distribution_factory(repository=repo.pulp_href, remote=remote.pulp_href)
+
+    # Perform a download of aiohttp to ensure it's saved to the repo
+    url = urljoin(distro.base_url, "simple/aiohttp/")
+    project_page = ProjectPage.from_response(requests.get(url), "aiohttp")
+    for package in sample(project_page.packages, 3):
+        assert "?redirect=" in package.url
+        r = requests.get(package.url)
+        assert r.status_code == 200
+
+    if has_pulp_plugin("core", max="3.73"):
+        pytest.skip("Pull-through repository save added in 3.74")
+
+    tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.prn)
+    assert tasks.count == 3
+
+    for task in tasks.results:
+        monitor_task(task.pulp_href)
+
+    repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href)
+    assert repo.latest_version_href[-2] == "3"
+    content = python_bindings.ContentPackagesApi.list(repository_version=repo.latest_version_href)
+    assert content.count == 3
+
+    # Check that getting content that is already present doesn't trigger a task
+    r = requests.get(package.url)
+    assert r.status_code == 200
+    tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.prn)
+    assert tasks.count == 3

{pulp_python-3.13.4 → pulp_python-3.14.0}/pulp_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-python
-Version: 3.13.4
+Version: 3.14.0
 Summary: pulp-python plugin for the Pulp Project
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org

{pulp_python-3.13.4 → pulp_python-3.14.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = "pulp-python"
-version = "3.13.4"
+version = "3.14.0"
 description = "pulp-python plugin for the Pulp Project"
 readme = "README.md"
 authors = [
@@ -77,7 +77,7 @@ ignore = [
 [tool.bumpversion]
 # This section is managed by the plugin template. Do not edit manually.
 
-current_version = "3.13.4"
+current_version = "3.14.0"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"

pulp_python-3.13.4/pulp_python/tests/functional/api/test_full_mirror.py

@@ -1,72 +0,0 @@
-import pytest
-import requests
-import subprocess
-
-from pulp_python.tests.functional.constants import (
-    PYPI_URL,
-    PYTHON_XS_FIXTURE_CHECKSUMS,
-)
-
-from pypi_simple import ProjectPage
-from urllib.parse import urljoin, urlsplit
-
-
-def test_pull_through_install(
-    python_bindings, python_remote_factory, python_distribution_factory, delete_orphans_pre
-):
-    """Tests that a pull-through distro can be installed from."""
-    remote = python_remote_factory(url=PYPI_URL)
-    distro = python_distribution_factory(remote=remote.pulp_href)
-    PACKAGE = "pulpcore-releases"
-
-    # Check if already installed
-    stdout = subprocess.run(("pip", "list"), capture_output=True).stdout.decode("utf-8")
-    if stdout.find(PACKAGE) != -1:
-        subprocess.run(("pip", "uninstall", PACKAGE, "-y"))
-
-    # Perform pull-through install
-    host = urlsplit(distro.base_url).hostname
-    url = f"{distro.base_url}simple/"
-    cmd = ("pip", "install", "--trusted-host", host, "-i", url, PACKAGE)
-    subprocess.run(cmd, check=True)
-
-    stdout = subprocess.run(("pip", "list"), capture_output=True).stdout.decode("utf-8")
-    assert stdout.find(PACKAGE) != -1
-    subprocess.run(("pip", "uninstall", PACKAGE, "-y"))
-    content = python_bindings.ContentPackagesApi.list(name=PACKAGE)
-    assert content.count == 1
-
-
-@pytest.mark.parallel
-def test_pull_through_simple(python_remote_factory, python_distribution_factory, pulp_content_url):
-    """Tests that the simple page is properly modified when requesting a pull-through."""
-    remote = python_remote_factory(url=PYPI_URL)
-    distro = python_distribution_factory(remote=remote.pulp_href)
-
-    url = f"{distro.base_url}simple/shelf-reader/"
-    project_page = ProjectPage.from_response(requests.get(url), "shelf-reader")
-
-    assert len(project_page.packages) == 2
-    for package in project_page.packages:
-        assert package.filename in PYTHON_XS_FIXTURE_CHECKSUMS
-        relative_path = f"{distro.base_path}/{package.filename}?redirect="
-        assert urljoin(pulp_content_url, relative_path) in package.url
-        assert PYTHON_XS_FIXTURE_CHECKSUMS[package.filename] == package.digests["sha256"]
-
-
-@pytest.mark.parallel
-def test_pull_through_with_repo(
-    python_repo_with_sync, python_remote_factory, python_distribution_factory
-):
-    """Tests that if content is already in repository, pull-through isn't used."""
-    remote = python_remote_factory()
-    repo = python_repo_with_sync(remote)
-    distro = python_distribution_factory(repository=repo.pulp_href, remote=remote.pulp_href)
-
-    url = urljoin(distro.base_url, "simple/shelf-reader/")
-    project_page = ProjectPage.from_response(requests.get(url), "shelf-reader")
-
-    assert len(project_page.packages) == 2
-    for package in project_page.packages:
-        assert package.filename in PYTHON_XS_FIXTURE_CHECKSUMS
-        assert "?redirect=" not in package.url