pulp-python 3.7.3__tar.gz → 3.9.0__tar.gz

{pulp-python-3.7.3 → pulp-python-3.9.0}/CHANGES.rst

@@ -13,6 +13,60 @@ Changelog
 
 .. towncrier release notes start
 
+3.9.0 (2023-03-17)
+==================
+
+
+Features
+--------
+
+- Added version filter to package list endpoint.
+  `#577 <https://github.com/pulp/pulp_python/issues/577>`__
+- Allow duplicate uploads to return existing packages instead of erring.
+  `#590 <https://github.com/pulp/pulp_python/issues/590>`__
+
+
+Bugfixes
+--------
+
+- Fixed pull-through caching ignoring remote proxy settings.
+  `#553 <https://github.com/pulp/pulp_python/issues/553>`__
+- Changed includes and excludes openapi schema to report as array of strings instead of object.
+  `#576 <https://github.com/pulp/pulp_python/issues/576>`__
+- Fixed syncing ignoring remote proxy.
+  `#581 <https://github.com/pulp/pulp_python/issues/581>`__
+- Fixed duplicate operationID for generated PyPI simple endpoints schema.
+  `#594 <https://github.com/pulp/pulp_python/issues/594>`__
+
+
+----
+
+
+3.8.0 (2022-12-19)
+==================
+
+
+Bugfixes
+--------
+
+- Fixed syncing failing when using bandersnatch 5.3.0
+  `#554 <https://github.com/pulp/pulp_python/issues/554>`__
+- Prevent .netrc file from being read on syncs.
+  `#566 <https://github.com/pulp/pulp_python/issues/566>`__
+- Fix 500 error when pip installing using object storage.
+  `#572 <https://github.com/pulp/pulp_python/issues/572>`__
+
+
+Improved Documentation
+----------------------
+
+- Documented ``pulp_python`` specific settings.
+  `#571 <https://github.com/pulp/pulp_python/issues/571>`__
+
+
+----
+
+
 3.7.3 (2022-10-06)
 ==================
 
@@ -78,6 +132,20 @@ Misc
 ----
 
 
+3.6.1 (2022-08-19)
+==================
+
+
+Bugfixes
+--------
+
+- Fixed syncing failing when using bandersnatch 5.3.0
+  `#554 <https://github.com/pulp/pulp_python/issues/554>`__
+
+
+----
+
+
 3.6.0 (2021-12-15)
 ==================
 

{pulp-python-3.7.3 → pulp-python-3.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulp-python
-Version: 3.7.3
+Version: 3.9.0
 Summary: pulp-python plugin for the Pulp Project
 Home-page: https://www.pulpproject.org
 Author: Pulp Project Developers

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/__init__.py

@@ -8,5 +8,5 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_python.app"
     label = "python"
-    version = "3.7.3"
+    version = "3.9.0"
     python_package_name = "pulp-python"

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/models.py

@@ -5,15 +5,14 @@ from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.conf import settings
-from yarl import URL
-
 from pulpcore.plugin.models import (
     Content,
     Publication,
     Distribution,
     Remote,
-    Repository
+    Repository,
 )
+from pulpcore.plugin.responses import ArtifactResponse
 
 from pathlib import PurePath
 from .utils import (
@@ -21,7 +20,7 @@ from .utils import (
     parse_project_metadata,
     python_content_to_json,
     PYPI_LAST_SERIAL,
-    PYPI_SERIAL_CONSTANT
+    PYPI_SERIAL_CONSTANT,
 )
 from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version
 
@@ -39,10 +38,12 @@ PACKAGE_TYPES = (
     ("sdist", "sdist"),
 )
 
-PLATFORMS = (("windows", "windows"),
-             ("macos", "macos"),
-             ("freebsd", "freebsd"),
-             ("linux", "linux"))
+PLATFORMS = (
+    ("windows", "windows"),
+    ("macos", "macos"),
+    ("freebsd", "freebsd"),
+    ("linux", "linux"),
+)
 
 
 class PythonDistribution(Distribution):
@@ -50,7 +51,7 @@ class PythonDistribution(Distribution):
     Distribution for 'Python' Content.
     """
 
-    TYPE = 'python'
+    TYPE = "python"
 
     allow_uploads = models.BooleanField(default=True)
 
@@ -76,47 +77,31 @@ class PythonDistribution(Distribution):
             # Temporary fix for PublishedMetadata not being properly served from remote storage
             # https://github.com/pulp/pulp_python/issues/413
             if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
-                try:
-                    publication = self.publication or Publication.objects.filter(
-                        repository_version=self.repository.latest_version()).latest("pulp_created")
-                except ObjectDoesNotExist:
-                    return None
-                rel_path = f"{path}/index.html"
-                try:
-                    ca = publication.published_artifact.select_related(
-                        "content_artifact",
-                        "content_artifact__artifact",
-                    ).get(relative_path=rel_path).content_artifact
-                except ObjectDoesNotExist:
-                    return None
-                file = ca.artifact.file
-                content_disposition = f"attachment%3Bfilename={ca.relative_path}"
-                if settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage":
-                    parameters = {
-                        "content_disposition": content_disposition,
-                        "content_type": "text/html"
-                    }
-                elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage":
-                    parameters = {
-                        "ResponseContentDisposition": content_disposition,
-                        "ResponseContentType": "text/html"
-                    }
-                else:
-                    raise NotImplementedError()
-
-                url = URL(file.storage.url(file.name, parameters=parameters), encoded=True)
-                # Trick the content app to stream the metadata from the remote storage
-                remote = PythonRemote(name="Redirect", url=str(url), policy="streamed")
-                remote.get_remote_artifact_url = lambda *args, **kwargs: str(url)
-                setattr(self, "publication", None)
-                setattr(self, "repository", None)
-                setattr(self, "remote", remote)
-                return None
+                if self.publication or self.repository:
+                    try:
+                        publication = self.publication or Publication.objects.filter(
+                            repository_version=self.repository.latest_version()
+                        ).latest("pulp_created")
+                    except ObjectDoesNotExist:
+                        return None
+                    rel_path = f"{path}/index.html"
+                    try:
+                        ca = (
+                            publication.published_artifact.select_related(
+                                "content_artifact",
+                                "content_artifact__artifact",
+                            )
+                            .get(relative_path=rel_path)
+                            .content_artifact
+                        )
+                    except ObjectDoesNotExist:
+                        return None
+                    headers = {"Content-Type": "text/html"}
+                    return ArtifactResponse(ca.artifact, headers=headers)
 
         if name:
             package_content = PythonPackageContent.objects.filter(
-                pk__in=self.publication.repository_version.content,
-                name__iexact=name
+                pk__in=self.publication.repository_version.content, name__iexact=name
             )
             # TODO Change this value to the Repo's serial value when implemented
             headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
@@ -150,7 +135,7 @@ class PythonPackageContent(Content):
 
     PROTECTED_FROM_RECLAIM = False
 
-    TYPE = 'python'
+    TYPE = "python"
     repo_key_fields = ("filename",)
     # Required metadata
     filename = models.TextField(db_index=True)
@@ -190,9 +175,9 @@ class PythonPackageContent(Content):
         path = PurePath(relative_path)
         metadata = get_project_metadata_from_artifact(path.name, artifact)
         data = parse_project_metadata(vars(metadata))
-        data['packagetype'] = metadata.packagetype
-        data['version'] = metadata.version
-        data['filename'] = path.name
+        data["packagetype"] = metadata.packagetype
+        data["version"] = metadata.version
+        data["filename"] = path.name
         data["sha256"] = artifact.sha256
         return PythonPackageContent(**data)
 
@@ -206,11 +191,11 @@ class PythonPackageContent(Content):
         e.g. <PythonPackageContent: shelf-reader [version] (whl)>
 
         """
-        return '<{obj_name}: {name} [{version}] ({type})>'.format(
+        return "<{obj_name}: {name} [{version}] ({type})>".format(
             obj_name=self._meta.object_name,
             name=self.name,
             version=self.version,
-            type=self.packagetype
+            type=self.packagetype,
         )
 
     class Meta:
@@ -223,7 +208,7 @@ class PythonPublication(Publication):
     A Publication for PythonContent.
     """
 
-    TYPE = 'python'
+    TYPE = "python"
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
@@ -238,16 +223,18 @@ class PythonRemote(Remote):
         prereleases (models.BooleanField): Whether to sync pre-release versions of packages.
     """
 
-    TYPE = 'python'
+    TYPE = "python"
     DEFAULT_DOWNLOAD_CONCURRENCY = 10
     prereleases = models.BooleanField(default=False)
     includes = models.JSONField(default=list)
     excludes = models.JSONField(default=list)
-    package_types = ArrayField(models.CharField(max_length=15, blank=True),
-                               choices=PACKAGE_TYPES, default=list)
+    package_types = ArrayField(
+        models.CharField(max_length=15, blank=True), choices=PACKAGE_TYPES, default=list
+    )
     keep_latest_packages = models.IntegerField(default=0)
-    exclude_platforms = ArrayField(models.CharField(max_length=10, blank=True),
-                                   choices=PLATFORMS, default=list)
+    exclude_platforms = ArrayField(
+        models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list
+    )
 
     def get_remote_artifact_url(self, relative_path=None, request=None):
         """Get url for remote_artifact"""

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/pypi/views.py

@@ -201,12 +201,20 @@ class SimpleView(ViewSet, PackageUploadMixin):
             return link.text, d_url, value if digest == 'sha256' else ''
 
         url = remote.get_remote_artifact_url(f'simple/{package}/')
-        response = requests.get(url, stream=True)
+        kwargs = {}
+        if proxy_url := remote.proxy_url:
+            if remote.proxy_username or remote.proxy_password:
+                parsed_proxy = urlparse(proxy_url)
+                netloc = f"{remote.proxy_username}:{remote.proxy_password}@{parsed_proxy.netloc}"
+                proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
+            kwargs["proxies"] = {"http": proxy_url, "https": proxy_url}
+
+        response = requests.get(url, stream=True, **kwargs)
         links = parse_links_stream_response(response)
         packages = (parse_url(link) for link in links)
         return StreamingHttpResponse(write_simple_detail(package, packages, streamed=True))
 
-    @extend_schema(summary="Get package simple page")
+    @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page")
     def retrieve(self, request, path, package):
         """Retrieves the simple api html page for a package."""
         distro, repo_ver, content = self.get_drvc(path)

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/serializers.py

@@ -226,20 +226,21 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
                     "The uploaded artifact's sha256 checksum does not match the one provided"
                 )}
             )
-        sha256 = artifact.sha256
-        if sha256 and python_models.PythonPackageContent.objects.filter(sha256=sha256).exists():
-            raise serializers.ValidationError(detail={"sha256": _('This field must be unique')})
 
         _data = parse_project_metadata(vars(metadata))
         _data['packagetype'] = metadata.packagetype
         _data['version'] = metadata.version
         _data['filename'] = filename
-        _data['sha256'] = sha256
+        _data['sha256'] = artifact.sha256
 
         data.update(_data)
 
         return data
 
+    def retrieve(self, validated_data):
+        content = python_models.PythonPackageContent.objects.filter(sha256=validated_data["sha256"])
+        return content.first()
+
     class Meta:
         fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary',
@@ -278,20 +279,21 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
     A Serializer for PythonRemote.
     """
 
-    includes = serializers.JSONField(
+    includes = serializers.ListField(
+        child=serializers.CharField(allow_blank=False),
         required=False,
-        default=list,
+        allow_empty=True,
         help_text=_(
-            "A JSON list containing project specifiers for Python packages to include."
+            "A list containing project specifiers for Python packages to include."
         ),
     )
-    excludes = serializers.JSONField(
+    excludes = serializers.ListField(
+        child=serializers.CharField(allow_blank=False),
         required=False,
-        default=list,
+        allow_empty=True,
         help_text=_(
-            "A JSON list containing project specifiers for Python packages to exclude."
+            "A list containing project specifiers for Python packages to exclude."
         ),
-
     )
     prereleases = serializers.BooleanField(
         required=False,

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/tasks/sync.py

@@ -3,7 +3,7 @@ import logging
 from aiohttp import ClientResponseError, ClientError
 from lxml.etree import LxmlError
 from gettext import gettext as _
-from os import environ
+from os import environ, path
 
 from rest_framework import serializers
 
@@ -26,7 +26,7 @@ from bandersnatch.mirror import Mirror
 from bandersnatch.master import Master
 from bandersnatch.configuration import BandersnatchConfig
 from packaging.requirements import Requirement
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlsplit, urlunsplit
 
 logger = logging.getLogger(__name__)
 
@@ -111,14 +111,22 @@ class PythonBanderStage(Stage):
         """
         If includes is specified, then only sync those,else try to sync all other packages
         """
+        # Prevent bandersnatch from reading actual .netrc file, set to nonexistent file
+        # See discussion on https://github.com/pulp/pulp_python/issues/581
+        environ["NETRC"] = f"{path.curdir}/.fake-netrc"
         # TODO Change Bandersnatch internal API to take proxy settings in from config parameters
-        if self.remote.proxy_url:
-            environ['http_proxy'] = self.remote.proxy_url
-            environ['https_proxy'] = self.remote.proxy_url
+        if proxy_url := self.remote.proxy_url:
+            if self.remote.proxy_username or self.remote.proxy_password:
+                parsed_proxy = urlsplit(proxy_url)
+                creds = f"{self.remote.proxy_username}:{self.remote.proxy_password}"
+                netloc = f"{creds}@{parsed_proxy.netloc}"
+                proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
+            environ['http_proxy'] = proxy_url
+            environ['https_proxy'] = proxy_url
         # Bandersnatch includes leading slash when forming API urls
         url = self.remote.url.rstrip("/")
         # local & global timeouts defaults to 10secs and 5 hours
-        async with PulpMaster(url) as master:
+        async with Master(url) as master:
             deferred_download = self.remote.policy != Remote.IMMEDIATE
             workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
             async with ProgressReport(
@@ -140,18 +148,6 @@ class PythonBanderStage(Stage):
                 await pmirror.synchronize(packages_to_sync)
 
 
-class PulpMaster(Master):
-    """
-    Temporary subclass of bandersnatch.Master until features are in bandersnatch.
-    """
-
-    async def __aenter__(self) -> "Master":
-        """Ensure Pulp does not try to read the .netrc file."""
-        await super().__aenter__()
-        self.session._trust_env = False
-        return self
-
-
 class PulpMirror(Mirror):
     """
     Pulp Mirror Class to perform syncing using Bandersnatch

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/app/viewsets.py

@@ -101,6 +101,7 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter):
             'filename': ['exact', 'in', 'contains'],
             'keywords': ['in', 'contains'],
             'sha256': ['exact', 'in'],
+            'version': ['exact', 'gt', 'lt', 'gte', 'lte']
         }
 
 

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_crud_content_unit.py

@@ -146,7 +146,7 @@ class ContentUnitTestCase(TestCaseUsingBindings, TestHelpersMixin):
         """
         1) upload file
         2) upload the same file again
-        3) this should fail/send an error
+        3) this should return first unit
         """
         delete_orphans()
         response = self.do_upload()
@@ -154,12 +154,8 @@ class ContentUnitTestCase(TestCaseUsingBindings, TestHelpersMixin):
         content_unit = self.content_api.read(created_resources[0])
         self.check_package_data(content_unit.to_dict())
 
-        with self.assertRaises(PulpTaskError) as cm:
-            monitor_task(self.do_upload().task)
-        task_report = cm.exception.task.to_dict()
-        msg = "This field must be unique"
-        self.assertTrue("sha256" in task_report["error"]["description"])
-        self.assertTrue(msg in task_report["error"]["description"])
+        created_resources = monitor_task(self.do_upload().task).created_resources
+        self.assertEqual(content_unit.pulp_href, created_resources[0])
 
     def test_08_upload_same_filename_different_artifact(self):
         """

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_download_content.py

@@ -124,7 +124,7 @@ class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin):
         This test checks that Pulp can fully sync another Python Package repository that is not
         PyPI. This reads the repository's simple page if XMLRPC isn't supported.
         """
-        remote = self._create_remote(includes="", prereleases=True)
+        remote = self._create_remote(includes=[], prereleases=True)
         repo = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY)
 
@@ -138,14 +138,14 @@ class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin):
         # Test using live generated simple pages
         distro = self._create_distribution_from_repo(repo)
 
-        remote = self._create_remote(includes="", url=distro.base_url)
+        remote = self._create_remote(includes=[], url=distro.base_url)
         repo2 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)
 
         # Now test using publication simple pages
         pub = self._create_publication(repo)
         distro2 = self._create_distribution_from_publication(pub)
-        remote = self._create_remote(includes="", url=distro2.base_url, prereleases=True)
+        remote = self._create_remote(includes=[], url=distro2.base_url, prereleases=True)
 
         repo3 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python/tests/functional/api/test_sync.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 """Tests that sync python plugin repositories."""
+import pytest
 import unittest
 
 from pulp_smash import config
@@ -640,6 +641,52 @@ class PlatformExcludeTestCase(unittest.TestCase):
         )
 
 
+@pytest.mark.parallel
+def test_proxy_sync(
+    python_repo,
+    python_repo_api_client,
+    python_remote_factory,
+    python_content_api_client,
+    http_proxy,
+):
+    """Test syncing with a proxy."""
+    body = gen_python_remote(proxy_url=http_proxy.proxy_url)
+    remote = python_remote_factory(**body)
+    sync_resp = python_repo_api_client.sync(python_repo.pulp_href, {"remote": remote.pulp_href})
+    monitor_task(sync_resp.task)
+
+    repo = python_repo_api_client.read(python_repo.pulp_href)
+    assert repo.latest_version_href[-2] == "1"
+
+    content_resp = python_content_api_client.list(repository_version=repo.latest_version_href)
+    assert content_resp.count == 2
+
+
+@pytest.mark.parallel
+def test_proxy_auth_sync(
+    python_repo,
+    python_repo_api_client,
+    python_remote_factory,
+    python_content_api_client,
+    http_proxy_with_auth,
+):
+    """Test syncing with a proxy with auth."""
+    body = gen_python_remote(
+        proxy_url=http_proxy_with_auth.proxy_url,
+        proxy_username=http_proxy_with_auth.username,
+        proxy_password=http_proxy_with_auth.password,
+    )
+    remote = python_remote_factory(**body)
+    sync_resp = python_repo_api_client.sync(python_repo.pulp_href, {"remote": remote.pulp_href})
+    monitor_task(sync_resp.task)
+
+    repo = python_repo_api_client.read(python_repo.pulp_href)
+    assert repo.latest_version_href[-2] == "1"
+
+    content_resp = python_content_api_client.list(repository_version=repo.latest_version_href)
+    assert content_resp.count == 2
+
+
 def sync_to_remote(self, body, create=False, mirror=False):
     """Takes a body and creates/updates a remote object, then it performs a sync"""
     if create:

{pulp-python-3.7.3 → pulp-python-3.9.0}/pulp_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulp-python
-Version: 3.7.3
+Version: 3.9.0
 Summary: pulp-python plugin for the Pulp Project
 Home-page: https://www.pulpproject.org
 Author: Pulp Project Developers

pulp-python-3.9.0/pulp_python.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pulpcore<3.25,>=3.22.0
+pkginfo<1.9.7,>=1.8.2
+bandersnatch<6.2,>=6.1
+pypi-simple<1.0.0,>=0.9.0

{pulp-python-3.7.3 → pulp-python-3.9.0}/pyproject.toml

@@ -16,11 +16,12 @@ ignore = [
     "dev_requirements.txt",
     "doc_requirements.txt",
     "docs/**",
-    "flake8.cfg",
     "template_config.yml",
     ".travis/**",
     ".travis.yml",
     "shelf_reader-0.1-py2-none-any.whl",
     ".github/**",
     ".ci/**",
+    "lint_requirements.txt",
+    ".flake8",
 ]

pulp-python-3.9.0/requirements.txt

@@ -0,0 +1,4 @@
+pulpcore>=3.22.0,<3.25
+pkginfo>=1.8.2,<1.9.7
+bandersnatch>=6.1,<6.2
+pypi-simple>=0.9.0,<1.0.0

{pulp-python-3.7.3 → pulp-python-3.9.0}/setup.py

@@ -10,7 +10,7 @@ with open("README.md") as f:
 
 setup(
     name="pulp-python",
-    version="3.7.3",
+    version="3.9.0",
     description="pulp-python plugin for the Pulp Project",
     long_description=long_description,
     long_description_content_type="text/markdown",

pulp-python-3.7.3/pulp_python.egg-info/requires.txt

@@ -1,5 +0,0 @@
-pulpcore<3.25,>=3.17.0
-pkginfo~=1.8.2
-packaging~=21.3
-bandersnatch<5.4,>=5.0.0
-pypi-simple~=0.9.0

pulp-python-3.7.3/requirements.txt

@@ -1,5 +0,0 @@
-pulpcore>=3.17.0,<3.25
-pkginfo~=1.8.2
-packaging~=21.3
-bandersnatch>=5.0.0,<5.4
-pypi-simple~=0.9.0