pulpcore 3.28.18__tar.gz → 3.28.20__tar.gz

{pulpcore-3.28.18 → pulpcore-3.28.20}/CHANGES.rst

@@ -13,6 +13,52 @@ Changelog
 
 .. towncrier release notes start
 
+3.28.20 (2024-01-24)
+====================
+REST API
+--------
+
+Bugfixes
+~~~~~~~~
+
+- Fixed a regular expression used when listing directories.
+  :github:`3673`
+- Taught repair task to be tolerant in the face of any artifact download failure.
+  :github:`4111`
+- Removed a workaround to force close all tcp connections in sync leading to an exhaustion of port
+  numbers and their reuse while in time_wait state.
+  :github:`4452`
+- Fix file:// syncs deleting the original files.
+  :github:`4681`
+
+
+Plugin API
+----------
+
+No significant changes.
+
+
+----
+
+
+3.28.19 (2023-11-06)
+====================
+REST API
+--------
+
+Bugfixes
+~~~~~~~~
+
+- Resolved a sync-time performance regression.
+  :github:`4591`
+
+
+Plugin API
+----------
+
+No significant changes.
+
+
 3.28.18 (2023-10-16)
 ====================
 REST API

{pulpcore-3.28.18 → pulpcore-3.28.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulpcore
-Version: 3.28.18
+Version: 3.28.20
 Summary: Pulp Django Application and Related Modules
 Home-page: https://pulpproject.org
 Author: Pulp Team

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/app/apps.py

@@ -234,7 +234,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.28.18"
+    version = "3.28.20"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/app/management/commands/handle-artifact-checksums.py

@@ -2,8 +2,6 @@ import os
 
 from gettext import gettext as _
 
-from aiohttp.client_exceptions import ClientResponseError
-
 from django.conf import settings
 from django.core.management import BaseCommand, CommandError
 from django.db.models import Q, Sum
@@ -126,7 +124,7 @@ class Command(BaseCommand):
                         downloader = remote.get_downloader(ra)
                         try:
                             dl_result = downloader.fetch()
-                        except ClientResponseError as e:
+                        except Exception as e:
                             self.stdout.write(
                                 _("Redownload failed from '{}': {}.").format(ra.url, str(e))
                             )
@@ -139,8 +137,9 @@ class Command(BaseCommand):
                                 setattr(artifact, checksum, dl_result.artifact_attributes[checksum])
                                 restored = True
                                 break
-                self.stdout.write(_("Deleting unreparable file {}".format(file_path)))
-                artifact.file.delete(save=False)
+                if not restored:
+                    self.stdout.write(_("Deleting unrepairable file {}".format(file_path)))
+                    artifact.file.delete(save=False)
             else:
                 break
         return restored

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/app/models/repository.py

@@ -835,7 +835,9 @@ class RepositoryVersion(BaseModel):
             raise ResourceImmutableError(self)
 
         repo_content = []
-        to_add = set(content.exclude(pk__in=self.content).values_list("pk", flat=True))
+        to_add = set(content.values_list("pk", flat=True)) - set(
+            self.content.values_list("pk", flat=True)
+        )
 
         # Normalize representation if content has already been removed in this version and
         # is re-added: Undo removal by setting version_removed to None.

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/app/tasks/repository.py

@@ -4,7 +4,6 @@ from logging import getLogger
 import asyncio
 import hashlib
 
-from aiohttp.client_exceptions import ClientResponseError
 from asgiref.sync import sync_to_async
 from django.db import transaction
 from rest_framework.serializers import ValidationError
@@ -68,7 +67,7 @@ async def _repair_ca(content_artifact, repaired=None):
             "Artifact {} is unrepairable - no remote source".format(content_artifact.artifact)
         )
         log.warning(
-            "Deleting file for the unreparable artifact {}".format(content_artifact.artifact)
+            "Deleting file for the unrepairable artifact {}".format(content_artifact.artifact)
         )
         await sync_to_async(content_artifact.artifact.file.delete)(save=False)
         return False
@@ -78,7 +77,7 @@ async def _repair_ca(content_artifact, repaired=None):
         downloader = detail_remote.get_downloader(remote_artifact)
         try:
             dl_result = await downloader.run()
-        except ClientResponseError as e:
+        except Exception as e:
             log.warn(_("Redownload failed from '{}': {}.").format(remote_artifact.url, str(e)))
         else:
             if dl_result.artifact_attributes["sha256"] == content_artifact.artifact.sha256:
@@ -91,7 +90,7 @@ async def _repair_ca(content_artifact, repaired=None):
                 if repaired is not None:
                     await repaired.aincrement()
                 return True
-    log.warning("Deleting file for the unreparable artifact {}".format(content_artifact.artifact))
+    log.warning("Deleting file for the unrepairable artifact {}".format(content_artifact.artifact))
     await sync_to_async(content_artifact.artifact.file.delete)(save=False)
     return False
 

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/download/factory.py

@@ -48,10 +48,6 @@ class DownloaderFactory:
     http://aiohttp.readthedocs.io/en/stable/client_quickstart.html#timeouts Behaviorally, it should
     allow for an active download to be arbitrarily long, while still detecting dead or closed
     sessions even when TCPKeepAlive is disabled.
-
-    Also for http and https urls, even though HTTP 1.1 is used, the TCP connection is setup and
-    closed with each request. This is done for compatibility reasons due to various issues related
-    to session continuation implementation in various servers.
     """
 
     def __init__(self, remote, downloader_overrides=None):
@@ -102,7 +98,7 @@ class DownloaderFactory:
         Returns:
             :class:`aiohttp.ClientSession`
         """
-        tcp_conn_opts = {"force_close": True}
+        tcp_conn_opts = {}
 
         sslcontext = None
         if self._remote.ca_cert:
@@ -133,17 +129,17 @@ class DownloaderFactory:
                     headers["User-Agent"] = f"{headers['User-Agent']}, {user_agent_header}"
                 headers.extend(header_dict)
 
-        conn = aiohttp.TCPConnector(**tcp_conn_opts)
-        total = self._remote.total_timeout
-        sock_connect = self._remote.sock_connect_timeout
-        sock_read = self._remote.sock_read_timeout
-        connect = self._remote.connect_timeout
-
         timeout = aiohttp.ClientTimeout(
-            total=total, sock_connect=sock_connect, sock_read=sock_read, connect=connect
+            total=self._remote.total_timeout,
+            sock_connect=self._remote.sock_connect_timeout,
+            sock_read=self._remote.sock_read_timeout,
+            connect=self._remote.connect_timeout,
         )
         return aiohttp.ClientSession(
-            connector=conn, timeout=timeout, headers=headers, requote_redirect_url=False
+            connector=aiohttp.TCPConnector(**tcp_conn_opts),
+            timeout=timeout,
+            headers=headers,
+            requote_redirect_url=False,
         )
 
     def build(self, url, **kwargs):

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/download/file.py

@@ -57,7 +57,7 @@ class FileDownloader(BaseDownloader):
                     break  # the reading is done
                 await self.handle_data(chunk)
             return DownloadResult(
-                path=self._path,
+                path=self.path,
                 artifact_attributes=self.artifact_attributes,
                 url=self.url,
                 headers=None,

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/download/http.py

@@ -71,10 +71,6 @@ class HttpDownloader(BaseDownloader):
     allow for an active download to be arbitrarily long, while still detecting dead or closed
     sessions even when TCPKeepAlive is disabled.
 
-    If a session is not provided, the one created will force TCP connection closure after each
-    request. This is done for compatibility reasons due to various issues related to session
-    continuation implementation in various servers.
-
     `aiohttp.ClientSession` objects allows you to configure options that will apply to all
     downloaders using that session such as auth, timeouts, headers, etc. For more info on these
     options see the `aiohttp.ClientSession` docs for more information:
@@ -165,7 +161,7 @@ class HttpDownloader(BaseDownloader):
             self._close_session_on_finalize = False
         else:
             timeout = aiohttp.ClientTimeout(total=None, sock_connect=600, sock_read=600)
-            conn = aiohttp.TCPConnector({"force_close": True})
+            conn = aiohttp.TCPConnector()
             self.session = aiohttp.ClientSession(
                 connector=conn, timeout=timeout, headers=headers, requote_redirect_url=False
             )

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/tests/functional/gpg_ascii_armor_signing_service.py

@@ -15,7 +15,7 @@ signing_script_string = r"""#!/usr/bin/env bash
 FILE_PATH=$1
 SIGNATURE_PATH="$1.asc"
 
-GPG_KEY_ID="Pulp QE"
+GPG_KEY_ID="pulp-fixture-signing-key"
 
 # Create a detached signature
 gpg --quiet --batch --homedir HOMEDIRHERE --detach-sign --local-user "${GPG_KEY_ID}" \
@@ -87,9 +87,7 @@ def sign_with_ascii_armored_detached_signing_service(signing_script_path, signin
 @pytest.fixture(scope="session")
 def signing_gpg_metadata(signing_gpg_homedir_path):
     """A fixture that returns a GPG instance and related metadata (i.e., fingerprint, keyid)."""
-    private_key_url = (
-        "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/GPG-PRIVATE-KEY-pulp-qe"
-    )
+    private_key_url = "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/GPG-PRIVATE-KEY-fixture-signing"  # noqa: E501
 
     async def download_key():
         async with aiohttp.ClientSession() as session:

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore/tests/unit/stages/test_stages.py

@@ -1,5 +1,6 @@
 import asyncio
 import pytest
+import pytest_asyncio
 
 import mock
 
@@ -9,13 +10,13 @@ from pulpcore.plugin.stages import Stage, EndStage, DeclarativeContent
 pytestmark = pytest.mark.usefixtures("fake_domain")
 
 
-@pytest.fixture
-def in_q():
+@pytest_asyncio.fixture
+async def in_q():
     return asyncio.Queue()
 
 
-@pytest.fixture
-def stage(in_q):
+@pytest_asyncio.fixture
+async def stage(in_q):
     stage = Stage()
     stage._connect(in_q, None)
     return stage

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulpcore
-Version: 3.28.18
+Version: 3.28.20
 Summary: Pulp Django Application and Related Modules
 Home-page: https://pulpproject.org
 Author: Pulp Team

{pulpcore-3.28.18 → pulpcore-3.28.20}/pulpcore.egg-info/requires.txt

@@ -1,10 +1,10 @@
 asyncio-throttle<=1.0.2,>=1.0
-aiohttp<3.8.6,>=3.8.1
+aiohttp<3.9.2,>=3.8.1
 aiodns<=3.0.0,>=3.0
 aiofiles<23.2.0,>=22.1
 backoff<2.2.2,>=2.1.2
 click<=8.1.3,>=8.1.0
-cryptography<41.0.2,>=38.0.1
+cryptography<41.0.8,>=38.0.1
 Django~=4.2.0
 django-filter<=23.2,>=23.1
 django-guid<=3.3.0,>=3.3
@@ -15,7 +15,7 @@ djangorestframework-queryfields<=1.0.0,>=1.0
 drf-access-policy<1.5.1,>=1.1.2
 drf-nested-routers<=0.93.4,>=0.93.4
 drf-spectacular==0.26.2
-dynaconf<3.1.13,>=3.1.12
+dynaconf<3.3,>=3.1.12
 gunicorn<=20.1.0,>=20.1
 jinja2<=3.1.2,>=3.1
 naya<=1.1.1,>=1.1.1
@@ -38,16 +38,16 @@ whitenoise<6.5.0,>=5.0
 yarl<1.9.3,>=1.8
 
 [azure]
-django-storages[azure]>=1.12.2
+django-storages[azure]<=1.13.2,>=1.12.2
 
 [google]
-django-storages[google]>=1.13.2
+django-storages[google]<=1.13.2,>=1.13.2
 
 [prometheus]
 django-prometheus
 
 [s3]
-django-storages[boto3]
+django-storages[boto3]<=1.13.2
 
 [sftp]
-django-storages[sftp]
+django-storages[sftp]<=1.13.2

{pulpcore-3.28.18 → pulpcore-3.28.20}/requirements.txt

@@ -1,10 +1,10 @@
 asyncio-throttle>=1.0,<=1.0.2
-aiohttp>=3.8.1,<3.8.6
+aiohttp>=3.8.1,<3.9.2
 aiodns>=3.0,<=3.0.0
 aiofiles>=22.1,<23.2.0
 backoff>=2.1.2,<2.2.2
 click>=8.1.0,<=8.1.3
-cryptography>=38.0.1,<41.0.2
+cryptography>=38.0.1,<41.0.8
 Django~=4.2.0  # LTS version, switch only if we have a compelling reason to
 django-filter>=23.1,<=23.2
 django-guid>=3.3,<=3.3.0
@@ -15,15 +15,15 @@ djangorestframework-queryfields>=1.0,<=1.0.0
 drf-access-policy>=1.1.2,<1.5.1
 drf-nested-routers>=0.93.4,<=0.93.4
 drf-spectacular==0.26.2  # We monkeypatch this so we need a very narrow requirement string
-dynaconf>=3.1.12,<3.1.13
+dynaconf>=3.1.12,<3.3
 gunicorn>=20.1,<=20.1.0
 jinja2>=3.1,<=3.1.2
 naya>=1.1.1,<=1.1.1
-importlib-metadata>=6.0.1,<=6.0.1  # Pinned to fix opentelemetry dependency solving issues with pip 
+importlib-metadata>=6.0.1,<=6.0.1  # Pinned to fix opentelemetry dependency solving issues with pip
 opentelemetry-distro[otlp]>=0.38b0,<=0.39b0
 opentelemetry-exporter-otlp-proto-http>=1.17.0,<=1.18.0
-opentelemetry-instrumentation-django>=0.38b0,<=0.39b0 
-opentelemetry-instrumentation-wsgi>=0.38b0,<=0.39b0 
+opentelemetry-instrumentation-django>=0.38b0,<=0.39b0
+opentelemetry-instrumentation-wsgi>=0.38b0,<=0.39b0
 pulp-glue>=0.18.0,<0.22
 protobuf>=4.21.1,<4.23.3
 pygtrie>=2.5,<=2.5.0

{pulpcore-3.28.18 → pulpcore-3.28.20}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as requirements:
 
 setup(
     name="pulpcore",
-    version="3.28.18",
+    version="3.28.20",
     description="Pulp Django Application and Related Modules",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -20,10 +20,10 @@ setup(
     python_requires=">=3.8",
     install_requires=requirements,
     extras_require={
-        "sftp": ["django-storages[sftp]"],
-        "s3": ["django-storages[boto3]"],
-        "google": ["django-storages[google]>=1.13.2"],
-        "azure": ["django-storages[azure]>=1.12.2"],
+        "sftp": ["django-storages[sftp]<=1.13.2"],
+        "s3": ["django-storages[boto3]<=1.13.2"],
+        "google": ["django-storages[google]>=1.13.2,<=1.13.2"],
+        "azure": ["django-storages[azure]>=1.12.2,<=1.13.2"],
         "prometheus": ["django-prometheus"],
     },
     include_package_data=True,