pulp-ansible 0.27.0__tar.gz → 0.28.0__tar.gz

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/CHANGES.md

@@ -8,6 +8,30 @@
 
 [//]: # (towncrier release notes start)
 
+## 0.28.0 (2025-08-13) {: #0.28.0 }
+
+#### Features {: #0.28.0-feature }
+
+- Add a database index to collection version lookup by namespace, name, version to speed up common queries.
+
+#### Bugfixes {: #0.28.0-bugfix }
+
+- Fixed the sync pipeline to not manipulate the RepositoryContent table directly when undeprecating collections.
+
+#### Deprecations and Removals {: #0.28.0-removal }
+
+- This release drops support for Python<3.11.
+
+---
+
+## 0.27.1 (2025-07-15) {: #0.27.1 }
+
+#### Bugfixes {: #0.27.1-bugfix }
+
+- Fixed the sync pipeline to not manipulate the RepositoryContent table directly when undeprecating collections.
+
+---
+
 ## 0.27.0 (2025-07-09) {: #0.27.0 }
 
 #### Features {: #0.27.0-feature }
@@ -75,6 +99,14 @@
 
 ---
 
+## 0.24.7 (2025-07-15) {: #0.24.7 }
+
+#### Bugfixes {: #0.24.7-bugfix }
+
+- Fixed the sync pipeline to not manipulate the RepositoryContent table directly when undeprecating collections.
+
+---
+
 ## 0.24.6 (2025-04-23) {: #0.24.6 }
 
 #### Features {: #0.24.6-feature }

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-ansible
-Version: 0.27.0
+Version: 0.28.0
 Summary: Pulp plugin to manage Ansible content, e.g. roles
 Author-email: Pulp Ansible Plugin Project Developers <pulp-dev@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -14,17 +14,15 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Framework :: Django
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Requires-Python: >=3.9
+Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: async_lru<2.1,>=1.0
-Requires-Dist: galaxy_importer<0.5,>=0.4.5
+Requires-Dist: galaxy_importer<0.5,>=0.4.27
 Requires-Dist: GitPython<3.2,>=3.1.24
-Requires-Dist: jsonschema<4.24,>=4.9
-Requires-Dist: Pillow<11.2,>=10.3
-Requires-Dist: pulpcore<3.85,>=3.63.0
-Requires-Dist: PyYAML<7.0,>=5.4.1
+Requires-Dist: jsonschema<4.26,>=4.9
+Requires-Dist: Pillow<11.4,>=10.3
+Requires-Dist: pulpcore<3.100,>=3.63.0
+Requires-Dist: PyYAML<7.0,>=6.0.2
 Requires-Dist: semantic_version<2.11,>=2.9
 Dynamic: license-file
 

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible/app/__init__.py

@@ -8,6 +8,6 @@ class PulpAnsiblePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_ansible.app"
     label = "ansible"
-    version = "0.27.0"
+    version = "0.28.0"
     python_package_name = "pulp-ansible"
     domain_compatible = True

pulp_ansible-0.28.0/pulp_ansible/app/migrations/0065_collectionversion_ansible_col_namespa_ea6f75_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.22 on 2025-07-09 10:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("ansible", "0064_alter_collection_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="collectionversion",
+            index=models.Index(
+                fields=["namespace", "name", "version"], name="ansible_col_namespa_ea6f75_idx"
+            ),
+        ),
+    ]

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible/app/models.py

@@ -215,6 +215,9 @@ class CollectionVersion(Content):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("sha256", "_pulp_domain")
+        indexes = [
+            models.Index(fields=["namespace", "name", "version"]),
+        ]
 
 
 class CollectionVersionMark(Content):

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible/app/tasks/collections.py

@@ -1,4 +1,5 @@
 import asyncio
+import functools
 import hashlib
 import json
 import logging
@@ -8,11 +9,11 @@ from gettext import gettext as _
 from operator import attrgetter
 from urllib.parse import urljoin
 from uuid import uuid4
+import tempfile
 
 import yaml
 from aiohttp.client_exceptions import ClientError, ClientResponseError
 from asgiref.sync import sync_to_async
-from async_lru import alru_cache
 from django.db import transaction
 from django.db.models import Q
 from django.db.utils import IntegrityError
@@ -30,7 +31,6 @@ from pulpcore.plugin.models import (
     ProgressReport,
     PulpTemporaryFile,
     Remote,
-    RepositoryContent,
     RepositoryVersion,
     Task,
 )
@@ -38,6 +38,8 @@ from pulpcore.plugin.stages import (
     ArtifactDownloader,
     ArtifactSaver,
     ContentSaver,
+    ContentAssociation,
+    EndStage,
     DeclarativeArtifact,
     DeclarativeContent,
     DeclarativeVersion,
@@ -47,6 +49,7 @@ from pulpcore.plugin.stages import (
     RemoteArtifactSaver,
     ResolveContentFutures,
     Stage,
+    create_pipeline,
 )
 from pulpcore.plugin.util import get_url, get_domain
 from rest_framework.serializers import ValidationError
@@ -79,6 +82,26 @@ log = logging.getLogger(__name__)
 aget_url = sync_to_async(get_url)
 
 
+def async_cache(f):
+    # A simple implementation for caching an async function with hashable arguments.
+    _cache = {}
+
+    @functools.wraps(f)
+    async def _wrapped_f(*args):
+        try:
+            return _cache[args]
+        except KeyError:
+            result = await f(*args)
+            _cache[args] = result
+            if len(_cache) > 128:
+                # This is a bit ad hoc and will give fifo instead of lru behaviour.
+                # I doubt it even makes a difference here.
+                _cache.pop(next(iter(_cache.keys())))
+            return result
+
+    return _wrapped_f
+
+
 # semantic_version.SimpleSpec interpretes "*" as ">=0.0.0"
 class AnsibleSpec(SimpleSpec):
     def __init__(self, expression):
@@ -197,14 +220,7 @@ def sync(remote_pk, repository_pk, mirror, optimize):
     if not remote.url:
         raise ValueError(_("A CollectionRemote must have a 'url' specified to synchronize."))
 
-    deprecation_before_sync = set()
-    for namespace, name in AnsibleCollectionDeprecated.objects.filter(
-        pk__in=repository.latest_version().content
-    ).values_list("namespace", "name"):
-        deprecation_before_sync.add(f"{namespace}.{name}")
-    first_stage = CollectionSyncFirstStage(
-        remote, repository, is_repo_remote, deprecation_before_sync, optimize
-    )
+    first_stage = CollectionSyncFirstStage(remote, repository, is_repo_remote, optimize)
     if first_stage.should_sync is False:
         log.debug(_("no-op: remote wasn't updated since last sync."))
         return
@@ -218,25 +234,6 @@ def sync(remote_pk, repository_pk, mirror, optimize):
     repository.last_synced_metadata_time = first_stage.last_synced_metadata_time
     repository.save(update_fields=["last_synced_metadata_time"])
 
-    # This goes against all rules!
-    # Content must be added and removed via the dedicated functions, which is impossible after the
-    # repository version was finalized.
-    # TODO Fix this!
-    to_undeprecate = Q()
-    undeprecated = deprecation_before_sync.difference(first_stage.deprecation_after_sync)
-    if undeprecated:
-        for collection in undeprecated:
-            namespace, name = collection.split(".")
-            to_undeprecate |= Q(namespace=namespace, name=name)
-        deprecated = AnsibleCollectionDeprecated.objects.filter(
-            to_undeprecate, pulp_domain=get_domain()
-        )
-        RepositoryContent.objects.filter(
-            repository=repository,
-            content__in=deprecated,
-            version_removed__isnull=True,
-        ).update(version_removed=repo_version)
-
 
 def import_collection(
     temp_file_pk,
@@ -475,13 +472,60 @@ class AnsibleDeclarativeVersion(DeclarativeVersion):
 
         return pipeline
 
+    def create(self):
+        """
+        Perform the work. This is the long-blocking call where all syncing occurs.
+
+        Returns: The created RepositoryVersion or None if it represents no change from the latest.
+        """
+        with tempfile.TemporaryDirectory(dir="."):
+            with self.repository.new_version() as new_version:
+                deprecation_before_sync = {
+                    (namespace, name)
+                    for namespace, name in AnsibleCollectionDeprecated.objects.filter(
+                        pk__in=self.repository.latest_version().content
+                    ).values_list("namespace", "name")
+                }
+                loop = asyncio.get_event_loop()
+                stages = self.pipeline_stages(new_version)
+                stages.append(UndeprecateStage(deprecation_before_sync))
+                stages.append(ContentAssociation(new_version, self.mirror))
+                stages.append(EndStage())
+                pipeline = create_pipeline(stages)
+                loop.run_until_complete(pipeline)
+
+                if deprecation_before_sync:
+                    to_undeprecate = Q()
+                    for namespace, name in deprecation_before_sync:
+                        to_undeprecate |= Q(namespace=namespace, name=name)
+                    new_version.remove_content(
+                        AnsibleCollectionDeprecated.objects.filter(
+                            to_undeprecate, pulp_domain=get_domain()
+                        )
+                    )
+
+        return new_version if new_version.complete else None
+
+
+class UndeprecateStage(Stage):
+    def __init__(self, deprecation_before_sync):
+        self.deprecation_before_sync = deprecation_before_sync
+
+    async def run(self):
+        async for batch in self.batches():
+            for d_content in batch:
+                if isinstance(d_content.content, AnsibleCollectionDeprecated):
+                    key = (d_content.content.namespace, d_content.content.name)
+                    self.deprecation_before_sync.discard(key)
+                await self.put(d_content)
+
 
 class CollectionSyncFirstStage(Stage):
     """
     The first stage of a pulp_ansible sync pipeline.
     """
 
-    def __init__(self, remote, repository, is_repo_remote, deprecation_before_sync, optimize):
+    def __init__(self, remote, repository, is_repo_remote, optimize):
         """
         The first stage of a pulp_ansible sync pipeline.
 
@@ -489,15 +533,12 @@ class CollectionSyncFirstStage(Stage):
             remote (CollectionRemote): The remote data to be used when syncing
             repository (AnsibleRepository): The repository being syncedself.
             is_repo_remote (bool): True if the remote is the repository's remote.
-            deprecation_before_sync (set): Set of deprecations before the sync.
             optimize (boolean): Whether to optimize sync or not.
 
         """
         super().__init__()
         self.remote = remote
         self.repository = repository
-        self.deprecation_before_sync = deprecation_before_sync
-        self.deprecation_after_sync = set()
         self.collection_info = parse_collections_requirements_file(remote.requirements_file)
         self.exclude_info = {}
         self.add_dependents = self.collection_info and self.remote.sync_dependencies
@@ -518,7 +559,7 @@ class CollectionSyncFirstStage(Stage):
             self._should_we_sync()
         )
 
-    @alru_cache(maxsize=128)
+    @async_cache
     async def _get_root_api(self, root):
         """
         Returns the root api path and api version.
@@ -554,7 +595,7 @@ class CollectionSyncFirstStage(Stage):
 
         return endpoint, api_version
 
-    @alru_cache(maxsize=128)
+    @async_cache
     async def _get_paginated_collection_api(self, root):
         """
         Returns the collection api path and api version.
@@ -766,7 +807,6 @@ class CollectionSyncFirstStage(Stage):
                         d_content = DeclarativeContent(
                             content=AnsibleCollectionDeprecated(namespace=namespace, name=name),
                         )
-                        self.deprecation_after_sync.add(f"{namespace}.{name}")
                         await self.put(d_content)
                     tasks.append(
                         loop.create_task(
@@ -801,7 +841,6 @@ class CollectionSyncFirstStage(Stage):
             d_content = DeclarativeContent(
                 content=AnsibleCollectionDeprecated(namespace=namespace, name=name),
             )
-            self.deprecation_after_sync.add(f"{namespace}.{name}")
             await self.put(d_content)
 
         all_versions_of_collection = self._unpaginated_collection_version_metadata[namespace][name]
@@ -924,9 +963,6 @@ class CollectionSyncFirstStage(Stage):
                             namespace=collection["namespace"], name=collection["name"]
                         ),
                     )
-                    self.deprecation_after_sync.add(
-                        f"{collection['namespace']}.{collection['name']}"
-                    )
                     await self.put(d_content)
 
         for collections_in_namespace in self._unpaginated_collection_version_metadata.values():

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible/tests/functional/api/collection/v3/test_deprecation.py

@@ -35,7 +35,6 @@ def test_deprecation(
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         first_distribution.base_path, namespace="testing"
     )
-
     assert collections.data[0].deprecated
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         first_distribution.base_path, namespace="pulp"
@@ -62,15 +61,12 @@ def test_deprecation(
     )
     assert collections.data[0].deprecated
 
-    # Change the deprecated status for the testing collection on the original repo to False
-    result = ansible_bindings.PulpAnsibleApiV3CollectionsApi.update(
-        "k8s_demo_collection", "testing", first_distribution.base_path, {"deprecated": False}
-    )
-    monitor_task(result.task)
+    # Sync again to see if the deprecated state is kept.
+    ansible_sync_factory(second_repo, optimize=False)
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
-        first_distribution.base_path, namespace="testing"
+        second_distribution.base_path, namespace="testing"
     )
-    assert not collections.data[0].deprecated
+    assert collections.data[0].deprecated
 
     # Update the requirements to sync down both collections this time
     requirements = (
@@ -83,9 +79,34 @@ def test_deprecation(
     # Sync the second repo again
     second_repo = ansible_sync_factory(second_repo)
 
+    # Assert the state of deprecated True for testing, False for pulp
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        second_distribution.base_path, namespace="testing"
+    )
+    assert collections.data[0].deprecated
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        second_distribution.base_path, namespace="pulp"
+    )
+    assert not collections.data[0].deprecated
+
+    # Change the deprecated status for the testing collection on the original repo to False
+    monitor_task(
+        ansible_bindings.PulpAnsibleApiV3CollectionsApi.update(
+            "k8s_demo_collection", "testing", first_distribution.base_path, {"deprecated": False}
+        ).task
+    )
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        first_distribution.base_path, namespace="testing"
+    )
+    assert not collections.data[0].deprecated
+
+    # Sync the second repo again
+    second_repo = ansible_sync_factory(second_repo)
+
     # Assert both collections show deprecated=False
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         second_distribution.base_path
     )
+    assert len(collections.data) == 2, collections
     for collection in collections.data:
         assert not collection.deprecated

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-ansible
-Version: 0.27.0
+Version: 0.28.0
 Summary: Pulp plugin to manage Ansible content, e.g. roles
 Author-email: Pulp Ansible Plugin Project Developers <pulp-dev@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -14,17 +14,15 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Framework :: Django
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Requires-Python: >=3.9
+Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: async_lru<2.1,>=1.0
-Requires-Dist: galaxy_importer<0.5,>=0.4.5
+Requires-Dist: galaxy_importer<0.5,>=0.4.27
 Requires-Dist: GitPython<3.2,>=3.1.24
-Requires-Dist: jsonschema<4.24,>=4.9
-Requires-Dist: Pillow<11.2,>=10.3
-Requires-Dist: pulpcore<3.85,>=3.63.0
-Requires-Dist: PyYAML<7.0,>=5.4.1
+Requires-Dist: jsonschema<4.26,>=4.9
+Requires-Dist: Pillow<11.4,>=10.3
+Requires-Dist: pulpcore<3.100,>=3.63.0
+Requires-Dist: PyYAML<7.0,>=6.0.2
 Requires-Dist: semantic_version<2.11,>=2.9
 Dynamic: license-file
 

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pulp_ansible.egg-info/SOURCES.txt

@@ -108,6 +108,7 @@ pulp_ansible/app/migrations/0061_collectionversion_new_tags.py
 pulp_ansible/app/migrations/0062_pivot_collectionversion_tags.py
 pulp_ansible/app/migrations/0063_domain_support.py
 pulp_ansible/app/migrations/0064_alter_collection_unique_together_and_more.py
+pulp_ansible/app/migrations/0065_collectionversion_ansible_col_namespa_ea6f75_idx.py
 pulp_ansible/app/migrations/__init__.py
 pulp_ansible/app/schema/__init__.py
 pulp_ansible/app/schema/copy_config.json

pulp_ansible-0.28.0/pulp_ansible.egg-info/requires.txt

@@ -0,0 +1,7 @@
+galaxy_importer<0.5,>=0.4.27
+GitPython<3.2,>=3.1.24
+jsonschema<4.26,>=4.9
+Pillow<11.4,>=10.3
+pulpcore<3.100,>=3.63.0
+PyYAML<7.0,>=6.0.2
+semantic_version<2.11,>=2.9

{pulp_ansible-0.27.0 → pulp_ansible-0.28.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = "pulp-ansible"
-version = "0.27.0"
+version = "0.28.0"
 description = "Pulp plugin to manage Ansible content, e.g. roles"
 readme = "README.rst"
 authors = [
@@ -20,17 +20,15 @@ classifiers = [
   "Framework :: Django",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.9",
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.11"
 dependencies = [
-  "async_lru>=1.0,<2.1",
-  "galaxy_importer>=0.4.5,<0.5",
+  "galaxy_importer>=0.4.27,<0.5",
   "GitPython>=3.1.24,<3.2",
-  "jsonschema>=4.9,<4.24",
-  "Pillow>=10.3,<11.2",
-  "pulpcore>=3.63.0,<3.85",
-  "PyYAML>=5.4.1,<7.0",
+  "jsonschema>=4.9,<4.26",
+  "Pillow>=10.3,<11.4",
+  "pulpcore>=3.63.0,<3.100",
+  "PyYAML>=6.0.2,<7.0",
   "semantic_version>=2.9,<2.11",
 ]
 
@@ -101,7 +99,7 @@ ignore = [
 [tool.bumpversion]
 # This section is managed by the plugin template. Do not edit manually.
 
-current_version = "0.27.0"
+current_version = "0.28.0"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"

pulp_ansible-0.27.0/pulp_ansible.egg-info/requires.txt

@@ -1,8 +0,0 @@
-async_lru<2.1,>=1.0
-galaxy_importer<0.5,>=0.4.5
-GitPython<3.2,>=3.1.24
-jsonschema<4.24,>=4.9
-Pillow<11.2,>=10.3
-pulpcore<3.85,>=3.63.0
-PyYAML<7.0,>=5.4.1
-semantic_version<2.11,>=2.9