pulpcore 3.43.0__tar.gz → 3.43.2__tar.gz

{pulpcore-3.43.0 → pulpcore-3.43.2}/CHANGES.rst

@@ -13,6 +13,61 @@ Changelog
 
 .. towncrier release notes start
 
+3.43.2 (2024-01-09)
+===================
+REST API
+--------
+
+Bugfixes
+~~~~~~~~
+
+- Fix a bug in import/export that could result in a division-by-zero during import.
+  :github:`4777`
+
+
+Plugin API
+----------
+
+No significant changes.
+
+
+Pulp File
+---------
+
+No significant changes.
+
+
+----
+
+
+3.43.1 (2023-12-12)
+===================
+REST API
+--------
+
+Bugfixes
+~~~~~~~~
+
+- Fixed a bug in the repository version repair API for non-default domains.
+  :github:`4776`,
+  :github:`4806`
+
+
+Plugin API
+----------
+
+No significant changes.
+
+
+Pulp File
+---------
+
+No significant changes.
+
+
+----
+
+
 3.43.0 (2023-12-05)
 ===================
 REST API

{pulpcore-3.43.0 → pulpcore-3.43.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulpcore
-Version: 3.43.0
+Version: 3.43.2
 Summary: Pulp Django Application and Related Modules
 Home-page: https://pulpproject.org
 Author: Pulp Team

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.43.0"
+    version = "3.43.2"
     python_package_name = "pulp-file"  # TODO Add python_module_name
     domain_compatible = True

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/app/apps.py

@@ -239,7 +239,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.43.0"
+    version = "3.43.2"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/app/serializers/exporter.py

@@ -1,6 +1,6 @@
 import os
-import re
 from gettext import gettext as _
+import re
 
 from rest_framework import serializers
 from rest_framework.validators import UniqueValidator
@@ -19,6 +19,16 @@ from pulpcore.app.serializers import (
 from pulpcore.constants import FS_EXPORT_CHOICES, FS_EXPORT_METHODS
 
 
+def parse_human_readable_file_size(size: str):
+    # based on https://stackoverflow.com/a/42865957/2002471
+    units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40}
+    size = size.upper()
+    if not re.match(r" ", size):
+        size = re.sub(r"([KMGT]?B)", r" \1", size)
+    number, unit = [string.strip() for string in size.split()]
+    return int(float(number) * units[unit])
+
+
 class ExporterSerializer(ModelSerializer):
     """
     Base serializer for Exporters.
@@ -208,23 +218,13 @@ class PulpExportSerializer(ExportSerializer):
                 )
         return super().validate(data)
 
-    @staticmethod
-    def _parse_size(size):
+    def validate_chunk_size(self, chunk_size):
         try:
-            # based on https://stackoverflow.com/a/42865957/2002471
-            units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40}
-            size = size.upper()
-            if not re.match(r" ", size):
-                size = re.sub(r"([KMGT]?B)", r" \1", size)
-            number, unit = [string.strip() for string in size.split()]
-            return int(float(number) * units[unit])
+            the_size = parse_human_readable_file_size(chunk_size)
         except ValueError:
             raise serializers.ValidationError(
-                _("chunk_size '{}' is not valid (valid units are B/KB/MB/GB/TB)").format(size)
+                _("chunk_size '{}' is not valid (valid units are B/KB/MB/GB/TB)").format(chunk_size)
             )
-
-    def validate_chunk_size(self, chunk_size):
-        the_size = self._parse_size(chunk_size)
         if the_size <= 0:
             raise serializers.ValidationError(
                 _("Chunk size {} is not greater than zero!").format(the_size)

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/app/tasks/export.py

@@ -422,12 +422,10 @@ def pulp_export(exporter_pk, params):
                         os.remove(pathname)
                     raise
             # compute the hashes
-            global_hash = hasher()
             paths = sorted([str(Path(p)) for p in glob(tarfile_fp + ".*")])
             for a_file in paths:
-                a_hash = compute_file_hash(a_file, hasher=hasher(), cumulative_hash=global_hash)
+                a_hash = compute_file_hash(a_file, hasher=hasher())
                 rslts[a_file] = a_hash
-            tarfile_hash = global_hash.hexdigest()
 
         else:
             # write into the file
@@ -450,23 +448,20 @@ def pulp_export(exporter_pk, params):
         # write outputfile/hash info to a file 'next to' the output file(s)
         output_file_info_path = tarfile_fp.replace(".tar", "-toc.json")
         with open(output_file_info_path, "w") as outfile:
-            if the_export.validated_chunk_size:
-                chunk_size = the_export.validated_chunk_size
-            else:
-                chunk_size = 0
-            chunk_toc = {
+            table_of_contents = {
                 "meta": {
-                    "chunk_size": chunk_size,
-                    "file": os.path.basename(tarfile_fp),
-                    "global_hash": tarfile_hash,
                     "checksum_type": checksum_type,
                 },
                 "files": {},
             }
+
+            if the_export.validated_chunk_size:
+                table_of_contents["meta"]["chunk_size"] = the_export.validated_chunk_size
+
             # Build a toc with just filenames (not the path on the exporter-machine)
             for a_path in rslts.keys():
-                chunk_toc["files"][os.path.basename(a_path)] = rslts[a_path]
-            json.dump(chunk_toc, outfile)
+                table_of_contents["files"][os.path.basename(a_path)] = rslts[a_path]
+            json.dump(table_of_contents, outfile)
 
         # store toc info
         toc_hash = compute_file_hash(output_file_info_path)

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/app/tasks/importer.py

@@ -76,12 +76,17 @@ class ChunkedFile(ExitStack):
             raise ValidationError(_("Missing 'files' or 'meta' keys in table-of-contents!"))
 
         toc_dir = os.path.dirname(toc_path)
-        self.chunk_size = int(self.toc["meta"]["chunk_size"])
         # sorting-by-filename is REALLY IMPORTANT here
         # keys are of the form <base-export-name>.00..<base-export-name>.NN,
         # and must be reassembled IN ORDER
         self.chunk_names = sorted(self.toc["files"].keys())
         self.chunk_paths = [os.path.join(toc_dir, chunk_name) for chunk_name in self.chunk_names]
+        self.chunk_size = int(self.toc["meta"].get("chunk_size", 0))
+        if not self.chunk_size:
+            assert (
+                len(self.toc["files"]) == 1
+            ), "chunk_size must exist and be non-zero if more than one chunk exists"
+            self.chunk_size = os.path.getsize(self.chunk_paths[0])
 
     def __enter__(self):
         assert not hasattr(self, "chunks"), "ChunkedFile is not reentrant."

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/app/tasks/repository.py

@@ -1,4 +1,5 @@
 from concurrent.futures import ThreadPoolExecutor
+from contextvars import copy_context
 from gettext import gettext as _
 from logging import getLogger
 import asyncio
@@ -96,6 +97,8 @@ async def _repair_ca(content_artifact, repaired=None):
 
 
 def _verify_artifact(artifact):
+    domain = get_domain()
+    assert domain.pk == artifact.pulp_domain_id
     try:
         # verify files digest
         hasher = hashlib.sha256()
@@ -144,7 +147,7 @@ async def _repair_artifacts_for_content(subset=None, verify_checksums=True):
                     # Should stay in (an) executor so that at least it doesn't completely block
                     # downloads.
                     valid = await loop.run_in_executor(
-                        checksum_executor, _verify_artifact, artifact
+                        checksum_executor, copy_context().run, _verify_artifact, artifact
                     )
                     if not valid:
                         await corrupted.aincrement()

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/tests/functional/api/pulp_file/test_domains.py

@@ -5,6 +5,7 @@ import json
 from pulpcore.app import settings
 from pulpcore.client.pulp_file import ApiException
 from pulpcore.client.pulpcore import ApiException as CoreApiException
+from pulpcore.client.pulpcore import Repair
 from pulpcore.tests.functional.utils import generate_iso, download_file
 
 
@@ -180,6 +181,7 @@ def test_content_upload(
 def test_content_promotion(
     domains_api_client,
     file_repository_api_client,
+    file_repository_version_api_client,
     basic_manifest_path,
     file_remote_factory,
     file_publication_api_client,
@@ -233,6 +235,14 @@ def test_content_promotion(
         assert download.response_obj.status == 200
         assert len(download.body) == 1024
 
+    # Test that a repository version repair operation can be run without error
+    response = file_repository_version_api_client.repair(
+        repo.latest_version_href, Repair(verify_checksums=True)
+    )
+    results = monitor_task(response.task)
+    assert results.state == "completed"
+    assert results.error is None
+
     # Cleanup to delete the domain
     task = file_repository_api_client.delete(repo.pulp_href).task
     monitor_task(task)

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore/tests/functional/api/using_plugin/test_repair.py

@@ -1,29 +1,14 @@
 import pytest
-import os
 
+from django.core.files.storage import default_storage
 from random import sample
 
 from pulpcore.client.pulpcore import Repair
 from pulpcore.client.pulp_file import RepositorySyncURL
 
-from pulpcore.app import settings
-
 from pulpcore.tests.functional.utils import get_files_in_manifest
 
 
-SUPPORTED_STORAGE_FRAMEWORKS = [
-    "django.core.files.storage.FileSystemStorage",
-    "pulpcore.app.models.storage.FileSystem",
-]
-
-pytestmark = pytest.mark.skipif(
-    settings.DEFAULT_FILE_STORAGE not in SUPPORTED_STORAGE_FRAMEWORKS,
-    reason="Cannot simulate bit-rot on this storage platform ({}).".format(
-        settings.DEFAULT_FILE_STORAGE
-    ),
-)
-
-
 @pytest.fixture
 def repository_with_corrupted_artifacts(
     file_repository_api_client,
@@ -42,19 +27,14 @@ def repository_with_corrupted_artifacts(
     # STEP 2: sample artifacts that will be modified on the filesystem later on
     content1, content2 = sample(get_files_in_manifest(remote.url), 2)
 
-    # Modify one artifact on disk.
-    artifact1_path = os.path.join(
-        settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content1[1]).results[0].file
-    )
-    with open(artifact1_path, "r+b") as f:
+    # Modify an artifact
+    artifact1_path = artifacts_api_client.list(sha256=content1[1]).results[0].file
+    with default_storage.open(artifact1_path, "w+b") as f:
         f.write(b"$a bit rot")
 
-    # Delete another one from disk.
-    artifact2_path = os.path.join(
-        settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content2[1]).results[0].file
-    )
-    os.remove(artifact2_path)
-
+    # Delete an artifact
+    artifact2_path = artifacts_api_client.list(sha256=content2[1]).results[0].file
+    default_storage.delete(artifact2_path)
     return repo
 
 

{pulpcore-3.43.0 → pulpcore-3.43.2}/pulpcore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulpcore
-Version: 3.43.0
+Version: 3.43.2
 Summary: Pulp Django Application and Related Modules
 Home-page: https://pulpproject.org
 Author: Pulp Team

{pulpcore-3.43.0 → pulpcore-3.43.2}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as requirements:
 
 setup(
     name="pulpcore",
-    version="3.43.0",
+    version="3.43.2",
     description="Pulp Django Application and Related Modules",
     long_description=long_description,
     long_description_content_type="text/markdown",