pulp-python 3.8.0__tar.gz → 3.9.0__tar.gz

{pulp-python-3.8.0 → pulp-python-3.9.0}/CHANGES.rst

@@ -13,6 +13,35 @@ Changelog
 
 .. towncrier release notes start
 
+3.9.0 (2023-03-17)
+==================
+
+
+Features
+--------
+
+- Added version filter to package list endpoint.
+  `#577 <https://github.com/pulp/pulp_python/issues/577>`__
+- Allow duplicate uploads to return existing packages instead of erring.
+  `#590 <https://github.com/pulp/pulp_python/issues/590>`__
+
+
+Bugfixes
+--------
+
+- Fixed pull-through caching ignoring remote proxy settings.
+  `#553 <https://github.com/pulp/pulp_python/issues/553>`__
+- Changed includes and excludes openapi schema to report as array of strings instead of object.
+  `#576 <https://github.com/pulp/pulp_python/issues/576>`__
+- Fixed syncing ignoring remote proxy.
+  `#581 <https://github.com/pulp/pulp_python/issues/581>`__
+- Fixed duplicate operationID for generated PyPI simple endpoints schema.
+  `#594 <https://github.com/pulp/pulp_python/issues/594>`__
+
+
+----
+
+
 3.8.0 (2022-12-19)
 ==================
 

{pulp-python-3.8.0 → pulp-python-3.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulp-python
-Version: 3.8.0
+Version: 3.9.0
 Summary: pulp-python plugin for the Pulp Project
 Home-page: https://www.pulpproject.org
 Author: Pulp Project Developers

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/app/__init__.py

@@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_python.app"
     label = "python"
-    version = "3.8.0"
+    version = "3.9.0"
     python_package_name = "pulp-python"

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/app/pypi/views.py

@@ -201,12 +201,20 @@ class SimpleView(ViewSet, PackageUploadMixin):
             return link.text, d_url, value if digest == 'sha256' else ''
 
         url = remote.get_remote_artifact_url(f'simple/{package}/')
-        response = requests.get(url, stream=True)
+        kwargs = {}
+        if proxy_url := remote.proxy_url:
+            if remote.proxy_username or remote.proxy_password:
+                parsed_proxy = urlparse(proxy_url)
+                netloc = f"{remote.proxy_username}:{remote.proxy_password}@{parsed_proxy.netloc}"
+                proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
+            kwargs["proxies"] = {"http": proxy_url, "https": proxy_url}
+
+        response = requests.get(url, stream=True, **kwargs)
         links = parse_links_stream_response(response)
         packages = (parse_url(link) for link in links)
         return StreamingHttpResponse(write_simple_detail(package, packages, streamed=True))
 
-    @extend_schema(summary="Get package simple page")
+    @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page")
     def retrieve(self, request, path, package):
         """Retrieves the simple api html page for a package."""
         distro, repo_ver, content = self.get_drvc(path)

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/app/serializers.py

@@ -226,20 +226,21 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
                     "The uploaded artifact's sha256 checksum does not match the one provided"
                 )}
             )
-        sha256 = artifact.sha256
-        if sha256 and python_models.PythonPackageContent.objects.filter(sha256=sha256).exists():
-            raise serializers.ValidationError(detail={"sha256": _('This field must be unique')})
 
         _data = parse_project_metadata(vars(metadata))
         _data['packagetype'] = metadata.packagetype
         _data['version'] = metadata.version
         _data['filename'] = filename
-        _data['sha256'] = sha256
+        _data['sha256'] = artifact.sha256
 
         data.update(_data)
 
         return data
 
+    def retrieve(self, validated_data):
+        content = python_models.PythonPackageContent.objects.filter(sha256=validated_data["sha256"])
+        return content.first()
+
     class Meta:
         fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary',
@@ -278,20 +279,21 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
     A Serializer for PythonRemote.
     """
 
-    includes = serializers.JSONField(
+    includes = serializers.ListField(
+        child=serializers.CharField(allow_blank=False),
         required=False,
-        default=list,
+        allow_empty=True,
         help_text=_(
-            "A JSON list containing project specifiers for Python packages to include."
+            "A list containing project specifiers for Python packages to include."
         ),
     )
-    excludes = serializers.JSONField(
+    excludes = serializers.ListField(
+        child=serializers.CharField(allow_blank=False),
         required=False,
-        default=list,
+        allow_empty=True,
         help_text=_(
-            "A JSON list containing project specifiers for Python packages to exclude."
+            "A list containing project specifiers for Python packages to exclude."
         ),
-
     )
     prereleases = serializers.BooleanField(
         required=False,

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/app/tasks/sync.py

@@ -3,7 +3,7 @@ import logging
 from aiohttp import ClientResponseError, ClientError
 from lxml.etree import LxmlError
 from gettext import gettext as _
-from os import environ
+from os import environ, path
 
 from rest_framework import serializers
 
@@ -26,7 +26,7 @@ from bandersnatch.mirror import Mirror
 from bandersnatch.master import Master
 from bandersnatch.configuration import BandersnatchConfig
 from packaging.requirements import Requirement
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlsplit, urlunsplit
 
 logger = logging.getLogger(__name__)
 
@@ -111,14 +111,22 @@ class PythonBanderStage(Stage):
         """
         If includes is specified, then only sync those,else try to sync all other packages
         """
+        # Prevent bandersnatch from reading actual .netrc file, set to nonexistent file
+        # See discussion on https://github.com/pulp/pulp_python/issues/581
+        environ["NETRC"] = f"{path.curdir}/.fake-netrc"
         # TODO Change Bandersnatch internal API to take proxy settings in from config parameters
-        if self.remote.proxy_url:
-            environ['http_proxy'] = self.remote.proxy_url
-            environ['https_proxy'] = self.remote.proxy_url
+        if proxy_url := self.remote.proxy_url:
+            if self.remote.proxy_username or self.remote.proxy_password:
+                parsed_proxy = urlsplit(proxy_url)
+                creds = f"{self.remote.proxy_username}:{self.remote.proxy_password}"
+                netloc = f"{creds}@{parsed_proxy.netloc}"
+                proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
+            environ['http_proxy'] = proxy_url
+            environ['https_proxy'] = proxy_url
         # Bandersnatch includes leading slash when forming API urls
         url = self.remote.url.rstrip("/")
         # local & global timeouts defaults to 10secs and 5 hours
-        async with PulpMaster(url) as master:
+        async with Master(url) as master:
             deferred_download = self.remote.policy != Remote.IMMEDIATE
             workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
             async with ProgressReport(
@@ -140,18 +148,6 @@ class PythonBanderStage(Stage):
                 await pmirror.synchronize(packages_to_sync)
 
 
-class PulpMaster(Master):
-    """
-    Temporary subclass of bandersnatch.Master until features are in bandersnatch.
-    """
-
-    async def __aenter__(self) -> "Master":
-        """Ensure Pulp does not try to read the .netrc file."""
-        await super().__aenter__()
-        self.session._trust_env = False
-        return self
-
-
 class PulpMirror(Mirror):
     """
     Pulp Mirror Class to perform syncing using Bandersnatch

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/app/viewsets.py

@@ -101,6 +101,7 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter):
             'filename': ['exact', 'in', 'contains'],
             'keywords': ['in', 'contains'],
             'sha256': ['exact', 'in'],
+            'version': ['exact', 'gt', 'lt', 'gte', 'lte']
         }
 
 

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_crud_content_unit.py

@@ -146,7 +146,7 @@ class ContentUnitTestCase(TestCaseUsingBindings, TestHelpersMixin):
         """
         1) upload file
         2) upload the same file again
-        3) this should fail/send an error
+        3) this should return first unit
         """
         delete_orphans()
         response = self.do_upload()
@@ -154,12 +154,8 @@ class ContentUnitTestCase(TestCaseUsingBindings, TestHelpersMixin):
         content_unit = self.content_api.read(created_resources[0])
         self.check_package_data(content_unit.to_dict())
 
-        with self.assertRaises(PulpTaskError) as cm:
-            monitor_task(self.do_upload().task)
-        task_report = cm.exception.task.to_dict()
-        msg = "This field must be unique"
-        self.assertTrue("sha256" in task_report["error"]["description"])
-        self.assertTrue(msg in task_report["error"]["description"])
+        created_resources = monitor_task(self.do_upload().task).created_resources
+        self.assertEqual(content_unit.pulp_href, created_resources[0])
 
     def test_08_upload_same_filename_different_artifact(self):
         """

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_download_content.py

@@ -124,7 +124,7 @@ class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin):
         This test checks that Pulp can fully sync another Python Package repository that is not
         PyPI. This reads the repository's simple page if XMLRPC isn't supported.
         """
-        remote = self._create_remote(includes="", prereleases=True)
+        remote = self._create_remote(includes=[], prereleases=True)
         repo = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY)
 
@@ -138,14 +138,14 @@ class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin):
         # Test using live generated simple pages
         distro = self._create_distribution_from_repo(repo)
 
-        remote = self._create_remote(includes="", url=distro.base_url)
+        remote = self._create_remote(includes=[], url=distro.base_url)
         repo2 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)
 
         # Now test using publication simple pages
         pub = self._create_publication(repo)
         distro2 = self._create_distribution_from_publication(pub)
-        remote = self._create_remote(includes="", url=distro2.base_url, prereleases=True)
+        remote = self._create_remote(includes=[], url=distro2.base_url, prereleases=True)
 
         repo3 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_sync.py

@@ -1,9 +1,10 @@
 # coding=utf-8
 """Tests that sync python plugin repositories."""
+import pytest
 import unittest
 
 from pulp_smash import config
-from pulp_smash.pulp3.bindings import monitor_task, PulpTaskError, delete_orphans
+from pulp_smash.pulp3.bindings import monitor_task, PulpTaskError
 from pulp_smash.pulp3.utils import (
     gen_repo,
     get_added_content_summary,
@@ -582,9 +583,6 @@ class PlatformExcludeTestCase(unittest.TestCase):
         """Destroy class-wide variables per test"""
         cls.remote_api.delete(cls.remote.pulp_href)
         cls.repo_api.delete(cls.repo.pulp_href)
-        # This is the last test case to be ran, delete orphans
-        # TODO: Move this to pytest hook when converting to pytest style
-        delete_orphans()
 
     def test_no_windows_sync(self):
         """Tests that no windows packages are synced"""
@@ -643,6 +641,52 @@ class PlatformExcludeTestCase(unittest.TestCase):
         )
 
 
+@pytest.mark.parallel
+def test_proxy_sync(
+    python_repo,
+    python_repo_api_client,
+    python_remote_factory,
+    python_content_api_client,
+    http_proxy,
+):
+    """Test syncing with a proxy."""
+    body = gen_python_remote(proxy_url=http_proxy.proxy_url)
+    remote = python_remote_factory(**body)
+    sync_resp = python_repo_api_client.sync(python_repo.pulp_href, {"remote": remote.pulp_href})
+    monitor_task(sync_resp.task)
+
+    repo = python_repo_api_client.read(python_repo.pulp_href)
+    assert repo.latest_version_href[-2] == "1"
+
+    content_resp = python_content_api_client.list(repository_version=repo.latest_version_href)
+    assert content_resp.count == 2
+
+
+@pytest.mark.parallel
+def test_proxy_auth_sync(
+    python_repo,
+    python_repo_api_client,
+    python_remote_factory,
+    python_content_api_client,
+    http_proxy_with_auth,
+):
+    """Test syncing with a proxy with auth."""
+    body = gen_python_remote(
+        proxy_url=http_proxy_with_auth.proxy_url,
+        proxy_username=http_proxy_with_auth.username,
+        proxy_password=http_proxy_with_auth.password,
+    )
+    remote = python_remote_factory(**body)
+    sync_resp = python_repo_api_client.sync(python_repo.pulp_href, {"remote": remote.pulp_href})
+    monitor_task(sync_resp.task)
+
+    repo = python_repo_api_client.read(python_repo.pulp_href)
+    assert repo.latest_version_href[-2] == "1"
+
+    content_resp = python_content_api_client.list(repository_version=repo.latest_version_href)
+    assert content_resp.count == 2
+
+
 def sync_to_remote(self, body, create=False, mirror=False):
     """Takes a body and creates/updates a remote object, then it performs a sync"""
     if create:

{pulp-python-3.8.0 → pulp-python-3.9.0}/pulp_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulp-python
-Version: 3.8.0
+Version: 3.9.0
 Summary: pulp-python plugin for the Pulp Project
 Home-page: https://www.pulpproject.org
 Author: Pulp Project Developers

pulp-python-3.9.0/pulp_python.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pulpcore<3.25,>=3.22.0
+pkginfo<1.9.7,>=1.8.2
+bandersnatch<6.2,>=6.1
+pypi-simple<1.0.0,>=0.9.0

pulp-python-3.9.0/requirements.txt

@@ -0,0 +1,4 @@
+pulpcore>=3.22.0,<3.25
+pkginfo>=1.8.2,<1.9.7
+bandersnatch>=6.1,<6.2
+pypi-simple>=0.9.0,<1.0.0

{pulp-python-3.8.0 → pulp-python-3.9.0}/setup.py

@@ -10,7 +10,7 @@ with open("README.md") as f:
 
 setup(
     name="pulp-python",
-    version="3.8.0",
+    version="3.9.0",
     description="pulp-python plugin for the Pulp Project",
     long_description=long_description,
     long_description_content_type="text/markdown",

pulp-python-3.8.0/pulp_python.egg-info/requires.txt

@@ -1,5 +0,0 @@
-pulpcore<3.25,>=3.22.0
-pkginfo~=1.8.2
-packaging~=21.3
-bandersnatch<5.4,>=5.0.0
-pypi-simple~=0.9.0

pulp-python-3.8.0/requirements.txt

@@ -1,5 +0,0 @@
-pulpcore>=3.22.0,<3.25
-pkginfo~=1.8.2
-packaging~=21.3
-bandersnatch>=5.0.0,<5.4
-pypi-simple~=0.9.0