pulp-python 3.15.0__tar.gz → 3.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {pulp_python-3.15.0 → pulp_python-3.16.0}/CHANGES.md +14 -0
  2. {pulp_python-3.15.0 → pulp_python-3.16.0}/PKG-INFO +1 -1
  3. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/__init__.py +1 -1
  4. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/pypi/views.py +2 -2
  5. pulp_python-3.16.0/pulp_python/app/tasks/repair.py +198 -0
  6. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/utils.py +32 -0
  7. pulp_python-3.16.0/pulp_python/tests/functional/api/test_repair.py +228 -0
  8. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/PKG-INFO +1 -1
  9. {pulp_python-3.15.0 → pulp_python-3.16.0}/pyproject.toml +3 -3
  10. pulp_python-3.15.0/pulp_python/app/tasks/repair.py +0 -89
  11. pulp_python-3.15.0/pulp_python/tests/functional/api/test_repair.py +0 -126
  12. {pulp_python-3.15.0 → pulp_python-3.16.0}/COMMITMENT +0 -0
  13. {pulp_python-3.15.0 → pulp_python-3.16.0}/COPYRIGHT +0 -0
  14. {pulp_python-3.15.0 → pulp_python-3.16.0}/LICENSE +0 -0
  15. {pulp_python-3.15.0 → pulp_python-3.16.0}/MANIFEST.in +0 -0
  16. {pulp_python-3.15.0 → pulp_python-3.16.0}/README.md +0 -0
  17. {pulp_python-3.15.0 → pulp_python-3.16.0}/functest_requirements.txt +0 -0
  18. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/__init__.py +0 -0
  19. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/global_access_conditions.py +0 -0
  20. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/management/__init__.py +0 -0
  21. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/management/commands/__init__.py +0 -0
  22. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/management/commands/repair-python-metadata.py +0 -0
  23. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0001_initial.py +0 -0
  24. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +0 -0
  25. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0003_new_sync_filters.py +0 -0
  26. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +0 -0
  27. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +0 -0
  28. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +0 -0
  29. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +0 -0
  30. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +0 -0
  31. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +0 -0
  32. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0010_update_json_field.py +0 -0
  33. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +0 -0
  34. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0012_add_domain.py +0 -0
  35. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0013_add_rbac_permissions.py +0 -0
  36. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/migrations/__init__.py +0 -0
  37. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/modelresource.py +0 -0
  38. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/models.py +0 -0
  39. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/pypi/__init__.py +0 -0
  40. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/pypi/serializers.py +0 -0
  41. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/replica.py +0 -0
  42. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/serializers.py +0 -0
  43. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/settings.py +0 -0
  44. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/tasks/__init__.py +0 -0
  45. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/tasks/publish.py +0 -0
  46. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/tasks/sync.py +0 -0
  47. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/tasks/upload.py +0 -0
  48. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/urls.py +0 -0
  49. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/viewsets.py +0 -0
  50. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/__init__.py +0 -0
  51. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/apache.conf +0 -0
  52. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/nginx.conf +0 -0
  53. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/pytest_plugin.py +0 -0
  54. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/__init__.py +0 -0
  55. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/__init__.py +0 -0
  56. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/__init__.py +0 -0
  57. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_auto_publish.py +0 -0
  58. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_consume_content.py +0 -0
  59. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_content_unit.py +0 -0
  60. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_publications.py +0 -0
  61. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_remotes.py +0 -0
  62. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_domains.py +0 -0
  63. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_download_content.py +0 -0
  64. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_export_import.py +0 -0
  65. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_full_mirror.py +0 -0
  66. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_pypi_apis.py +0 -0
  67. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_rbac.py +0 -0
  68. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_sync.py +0 -0
  69. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/constants.py +0 -0
  70. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/functional/utils.py +0 -0
  71. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/unit/__init__.py +0 -0
  72. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python/tests/unit/test_models.py +0 -0
  73. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/SOURCES.txt +0 -0
  74. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/dependency_links.txt +0 -0
  75. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/entry_points.txt +0 -0
  76. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/requires.txt +0 -0
  77. {pulp_python-3.15.0 → pulp_python-3.16.0}/pulp_python.egg-info/top_level.txt +0 -0
  78. {pulp_python-3.15.0 → pulp_python-3.16.0}/setup.cfg +0 -0
  79. {pulp_python-3.15.0 → pulp_python-3.16.0}/test_requirements.txt +0 -0
  80. {pulp_python-3.15.0 → pulp_python-3.16.0}/unittest_requirements.txt +0 -0
@@ -8,6 +8,20 @@
8
8
 
9
9
  [//]: # (towncrier release notes start)
10
10
 
11
+ ## 3.16.0 (2025-06-10) {: #3.16.0 }
12
+
13
+ #### Features {: #3.16.0-feature }
14
+
15
+ - Added support for on-demand content to `repair_metadata` endpoint.
16
+ [#849](https://github.com/pulp/pulp_python/issues/849)
17
+
18
+ #### Bugfixes {: #3.16.0-bugfix }
19
+
20
+ - Fixed pull-through caching not working for indexes that use relative URLs.
21
+ [#842](https://github.com/pulp/pulp_python/issues/842)
22
+
23
+ ---
24
+
11
25
  ## 3.15.0 (2025-05-13) {: #3.15.0 }
12
26
 
13
27
  #### Features {: #3.15.0-feature }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pulp-python
3
- Version: 3.15.0
3
+ Version: 3.16.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
5
  Author-email: Pulp Team <pulp-list@redhat.com>
6
6
  Project-URL: Homepage, https://pulpproject.org
@@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
10
10
 
11
11
  name = "pulp_python.app"
12
12
  label = "python"
13
- version = "3.15.0"
13
+ version = "3.16.0"
14
14
  python_package_name = "pulp-python"
15
15
  domain_compatible = True
16
16
 
@@ -260,9 +260,9 @@ class SimpleView(PackageUploadMixin, ViewSet):
260
260
  return HttpResponse(f"{remote.url} timed out while fetching {package}.", status=504)
261
261
 
262
262
  if d.headers["content-type"] == "application/vnd.pypi.simple.v1+json":
263
- page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=remote.url)
263
+ page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=url)
264
264
  else:
265
- page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=remote.url)
265
+ page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=url)
266
266
  packages = [
267
267
  parse_package(p) for p in page.packages if rfilter.filter_release(package, p.version)
268
268
  ]
@@ -0,0 +1,198 @@
1
+ import logging
2
+ from collections import defaultdict
3
+ from gettext import gettext as _
4
+ from itertools import groupby
5
+ from uuid import UUID
6
+
7
+ from django.db.models import Prefetch
8
+ from django.db.models.query import QuerySet
9
+ from pulp_python.app.models import PythonPackageContent, PythonRepository
10
+ from pulp_python.app.utils import (
11
+ artifact_to_python_content_data,
12
+ fetch_json_release_metadata,
13
+ parse_metadata,
14
+ )
15
+ from pulpcore.plugin.models import ContentArtifact, ProgressReport
16
+ from pulpcore.plugin.util import get_domain
17
+
18
+ log = logging.getLogger(__name__)
19
+
20
+
21
+ BULK_SIZE = 1000
22
+
23
+
24
+ def repair(repository_pk: UUID) -> None:
25
+ """
26
+ Repairs metadata of all packages for the specified repository.
27
+
28
+ Args:
29
+ repository_pk (UUID): The primary key of the repository to repair.
30
+
31
+ Returns:
32
+ None
33
+ """
34
+ repository = PythonRepository.objects.get(pk=repository_pk)
35
+
36
+ log.info(
37
+ _(
38
+ "Repairing packages' metadata for the latest version of repository {}."
39
+ ).format(repository.name)
40
+ )
41
+ content_set = repository.latest_version().content.values_list("pk", flat=True)
42
+ content = PythonPackageContent.objects.filter(pk__in=content_set)
43
+
44
+ num_repaired, pkgs_not_repaired = repair_metadata(content)
45
+ log.info(
46
+ _(
47
+ "{} packages' metadata repaired. Not repaired packages due to either "
48
+ "inaccessible URL or mismatched sha256: {}."
49
+ ).format(num_repaired, pkgs_not_repaired)
50
+ )
51
+
52
+
53
+ def repair_metadata(content: QuerySet[PythonPackageContent]) -> tuple[int, set[str]]:
54
+ """
55
+ Repairs metadata for a queryset of PythonPackageContent objects
56
+ and updates the progress report.
57
+
58
+ Args:
59
+ content (QuerySet[PythonPackageContent]): The queryset of items to repair.
60
+
61
+ Returns:
62
+ tuple[int, set[str]]: A tuple containing:
63
+ - The number of packages that were repaired.
64
+ - A set of packages' PKs that were not repaired.
65
+ """
66
+ immediate_content = (
67
+ content.filter(contentartifact__artifact__isnull=False)
68
+ .distinct()
69
+ .prefetch_related("_artifacts")
70
+ )
71
+ on_demand_content = (
72
+ content.filter(contentartifact__artifact__isnull=True)
73
+ .distinct()
74
+ .prefetch_related(
75
+ Prefetch(
76
+ "contentartifact_set",
77
+ queryset=ContentArtifact.objects.prefetch_related("remoteartifact_set"),
78
+ )
79
+ )
80
+ .order_by("name", "version")
81
+ )
82
+ domain = get_domain()
83
+
84
+ batch = []
85
+ set_of_update_fields = set()
86
+ total_repaired = 0
87
+ # Keep track of on-demand packages that were not repaired
88
+ pkgs_not_repaired = set()
89
+
90
+ progress_report = ProgressReport(
91
+ message="Repairing packages' metadata",
92
+ code="repair.metadata",
93
+ total=content.count(),
94
+ )
95
+ progress_report.save()
96
+ with progress_report:
97
+ for package in progress_report.iter(
98
+ immediate_content.iterator(chunk_size=BULK_SIZE)
99
+ ):
100
+ new_data = artifact_to_python_content_data(
101
+ package.filename, package._artifacts.get(), domain
102
+ )
103
+ total_repaired += update_package_if_needed(
104
+ package, new_data, batch, set_of_update_fields
105
+ )
106
+
107
+ # For on-demand content, we expect that:
108
+ # 1. PythonPackageContent always has correct name and version
109
+ # 2. RemoteArtifact always has correct sha256
110
+ for (name, version), group in groupby(
111
+ on_demand_content.iterator(chunk_size=BULK_SIZE),
112
+ key=lambda x: (x.name, x.version),
113
+ ):
114
+ group_set = set(group)
115
+ grouped_by_url = defaultdict(list)
116
+
117
+ for package in group_set:
118
+ for ra in package.contentartifact_set.get().remoteartifact_set.all():
119
+ grouped_by_url[ra.remote.url].append((package, ra))
120
+
121
+ # Prioritize the URL that can serve the most packages
122
+ for url, pkg_ra_pairs in sorted(
123
+ grouped_by_url.items(), key=lambda x: len(x[1]), reverse=True
124
+ ):
125
+ if not group_set:
126
+ break # No packages left to repair, move onto the next group
127
+ remotes = set([pkg_ra[1].remote for pkg_ra in pkg_ra_pairs])
128
+ try:
129
+ json_data = fetch_json_release_metadata(name, version, remotes)
130
+ except Exception:
131
+ continue
132
+
133
+ for package, ra in pkg_ra_pairs:
134
+ if package not in group_set:
135
+ continue # Package was already repaired
136
+ # Extract data only for the specific distribution being checked
137
+ dist_data = None
138
+ for dist in json_data["urls"]:
139
+ if ra.sha256 == dist["digests"]["sha256"]:
140
+ dist_data = dist
141
+ break
142
+ if not dist_data:
143
+ continue
144
+
145
+ new_data = parse_metadata(json_data["info"], version, dist_data)
146
+ new_data.pop("url") # url belongs to RemoteArtifact
147
+ total_repaired += update_package_if_needed(
148
+ package, new_data, batch, set_of_update_fields
149
+ )
150
+ group_set.remove(package)
151
+ progress_report.increment()
152
+ # Store and track the unrepaired packages after all URLs are processed
153
+ pkgs_not_repaired.update([p.pk for p in group_set])
154
+ progress_report.increase_by(len(group_set))
155
+
156
+ if batch:
157
+ total_repaired += len(batch)
158
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
159
+
160
+ return total_repaired, pkgs_not_repaired
161
+
162
+
163
+ def update_package_if_needed(
164
+ package: PythonPackageContent,
165
+ new_data: dict,
166
+ batch: list[PythonPackageContent],
167
+ set_of_update_fields: set[str],
168
+ ) -> int:
169
+ """
170
+ Compares the current package data with new data and updates the package
171
+ if needed ("batch" and "set_of_update_fields" are updated in-place).
172
+
173
+ Args:
174
+ package: Package to check and update.
175
+ new_data: A dict of new field values to compare against the package.
176
+ batch: A list of packages that were updated.
177
+ set_of_update_fields: A set of package field names that were updated.
178
+
179
+ Returns:
180
+ The count of repaired packages (increments in multiples of BULK_SIZE only).
181
+ """
182
+ total_repaired = 0
183
+ changed = False
184
+ for field, value in new_data.items():
185
+ if getattr(package, field) != value:
186
+ setattr(package, field, value)
187
+ set_of_update_fields.add(field)
188
+ changed = True
189
+ if changed:
190
+ batch.append(package)
191
+
192
+ if len(batch) == BULK_SIZE:
193
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
194
+ total_repaired += BULK_SIZE
195
+ batch.clear()
196
+ set_of_update_fields.clear()
197
+
198
+ return total_repaired
@@ -9,6 +9,7 @@ from jinja2 import Template
9
9
  from packaging.utils import canonicalize_name
10
10
  from packaging.requirements import Requirement
11
11
  from packaging.version import parse, InvalidVersion
12
+ from pulpcore.plugin.models import Remote
12
13
 
13
14
 
14
15
  PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
@@ -189,6 +190,37 @@ def artifact_to_python_content_data(filename, artifact, domain=None):
189
190
  return data
190
191
 
191
192
 
193
+ def fetch_json_release_metadata(name: str, version: str, remotes: set[Remote]) -> dict:
194
+ """
195
+ Fetches metadata for a specific release from PyPI's JSON API. A release can contain
196
+ multiple distributions. See https://docs.pypi.org/api/json/#get-a-release for more details.
197
+ All remotes should have the same URL.
198
+
199
+ Returns:
200
+ Dict containing "info", "last_serial", "urls", and "vulnerabilities" keys.
201
+ Raises:
202
+ Exception if fetching from all remote URLs fails.
203
+ """
204
+ remote = next(iter(remotes))
205
+ url = remote.get_remote_artifact_url(f"pypi/{name}/{version}/json")
206
+
207
+ result = None
208
+ for remote in remotes:
209
+ downloader = remote.get_downloader(url=url, max_retries=1)
210
+ try:
211
+ result = downloader.fetch()
212
+ break
213
+ except Exception:
214
+ continue
215
+
216
+ if result:
217
+ with open(result.path, "r") as file:
218
+ json_data = json.load(file)
219
+ return json_data
220
+ else:
221
+ raise Exception(f"Failed to fetch {url} from any remote.")
222
+
223
+
192
224
  def python_content_to_json(base_path, content_query, version=None, domain=None):
193
225
  """
194
226
  Converts a QuerySet of PythonPackageContent into the PyPi JSON format
@@ -0,0 +1,228 @@
1
+ import pytest
2
+ import subprocess
3
+ from urllib.parse import urljoin
4
+
5
+ from pulp_python.tests.functional.constants import (
6
+ PYTHON_EGG_FILENAME,
7
+ PYTHON_FIXTURES_URL,
8
+ )
9
+
10
+
11
+ @pytest.fixture
12
+ def create_content_direct(python_bindings):
13
+ def _create(artifact_filename, content_data):
14
+ commands = (
15
+ "from pulpcore.plugin.models import Artifact, ContentArtifact; "
16
+ "from pulpcore.plugin.util import get_url; "
17
+ "from pulp_python.app.models import PythonPackageContent; "
18
+ f"a = Artifact.init_and_validate('{artifact_filename}'); "
19
+ "a.save(); "
20
+ f"c = PythonPackageContent(sha256=a.sha256, **{content_data!r}); "
21
+ "c.save(); "
22
+ f"ca = ContentArtifact(artifact=a, content=c, relative_path=c.filename); "
23
+ "ca.save(); "
24
+ "print(get_url(c))"
25
+ )
26
+ process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
27
+
28
+ assert process.returncode == 0
29
+ content_href = process.stdout.decode().strip()
30
+ return python_bindings.ContentPackagesApi.read(content_href)
31
+
32
+ return _create
33
+
34
+
35
+ @pytest.fixture
36
+ def create_content_remote(python_bindings):
37
+ def _create(content, remote, remote_2=None):
38
+ commands = (
39
+ "from pulpcore.plugin.models import ContentArtifact, RemoteArtifact; "
40
+ "from pulpcore.plugin.util import extract_pk, get_url; "
41
+ "from pulp_python.app.models import PythonPackageContent, PythonRemote; "
42
+ f"c = PythonPackageContent(**{content!r}); "
43
+ "c.save(); "
44
+ f"ca = ContentArtifact(content=c, relative_path=c.filename); "
45
+ "ca.save(); "
46
+ f"r = PythonRemote.objects.get(pk=extract_pk({remote.pulp_href!r})); "
47
+ f"ra = RemoteArtifact(content_artifact=ca, remote=r, sha256=c.sha256); "
48
+ "ra.save(); "
49
+ )
50
+ if remote_2:
51
+ commands += (
52
+ f"r2 = PythonRemote.objects.get(pk=extract_pk({remote_2.pulp_href!r})); "
53
+ f"ra2 = RemoteArtifact(content_artifact=ca, remote=r2, sha256=c.sha256); "
54
+ "ra2.save(); "
55
+ )
56
+ commands += "print(get_url(c))"
57
+ process = subprocess.run(
58
+ ["pulpcore-manager", "shell", "-c", commands], capture_output=True
59
+ )
60
+
61
+ assert process.returncode == 0
62
+ content_href = process.stdout.decode().strip()
63
+ return python_bindings.ContentPackagesApi.read(content_href)
64
+
65
+ return _create
66
+
67
+
68
+ @pytest.fixture
69
+ def move_to_repository(python_bindings, monitor_task):
70
+ def _move(repo_href, content_hrefs):
71
+ body = {"add_content_units": content_hrefs}
72
+ task = monitor_task(python_bindings.RepositoriesPythonApi.modify(repo_href, body).task)
73
+ assert len(task.created_resources) == 1
74
+ return python_bindings.RepositoriesPythonApi.read(repo_href)
75
+
76
+ return _move
77
+
78
+
79
+ def test_metadata_repair_command(
80
+ create_content_direct,
81
+ python_file,
82
+ python_repo,
83
+ move_to_repository,
84
+ python_bindings,
85
+ delete_orphans_pre,
86
+ ):
87
+ """Test pulpcore-manager repair-python-metadata command."""
88
+ data = {
89
+ "name": "shelf-reader",
90
+ "filename": PYTHON_EGG_FILENAME,
91
+ # Wrong metadata
92
+ "version": "0.2",
93
+ "packagetype": "bdist",
94
+ "requires_python": ">=3.8",
95
+ "author": "ME",
96
+ }
97
+ content = create_content_direct(python_file, data)
98
+ for field, wrong_value in data.items():
99
+ if field == "python_version":
100
+ continue
101
+ assert getattr(content, field) == wrong_value
102
+
103
+ move_to_repository(python_repo.pulp_href, [content.pulp_href])
104
+ process = subprocess.run(
105
+ ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href],
106
+ capture_output=True
107
+ )
108
+ assert process.returncode == 0
109
+ output = process.stdout.decode().strip()
110
+ assert output == "1 packages processed, 1 package metadata repaired."
111
+
112
+ content = python_bindings.ContentPackagesApi.read(content.pulp_href)
113
+ assert content.version == "0.1"
114
+ assert content.packagetype == "sdist"
115
+ assert content.requires_python == "" # technically null
116
+ assert content.author == "Austin Macdonald"
117
+
118
+
119
+ def test_metadata_repair_endpoint(
120
+ create_content_direct,
121
+ create_content_remote,
122
+ delete_orphans_pre,
123
+ download_python_file,
124
+ monitor_task,
125
+ move_to_repository,
126
+ python_bindings,
127
+ python_remote_factory,
128
+ python_repo_factory,
129
+ ):
130
+ """
131
+ Test repairing of package metadata via `Repositories.repair_metadata` endpoint.
132
+ """
133
+ # 1. Setup tested data
134
+ # Shared data
135
+ python_remote = python_remote_factory()
136
+ python_remote_bad = python_remote_factory(url="https://fixtures.pulpproject.org/")
137
+ python_repo = python_repo_factory(remote=python_remote)
138
+
139
+ # Immediate content
140
+ scipy_egg_filename = "scipy-1.1.0-cp27-none-win32.whl"
141
+ scipy_egg_url = urljoin(
142
+ urljoin(PYTHON_FIXTURES_URL, "packages/"), scipy_egg_filename
143
+ )
144
+ scipy_file = download_python_file(scipy_egg_filename, scipy_egg_url)
145
+ scipy_data_0 = {
146
+ "filename": scipy_egg_filename,
147
+ "name": "scipy",
148
+ "version": "1.1.0",
149
+ # Wrong metadata
150
+ "author": "ME",
151
+ "packagetype": "bdist",
152
+ "requires_python": ">=3.8",
153
+ }
154
+
155
+ # On-demand content
156
+ celery_data = {
157
+ "filename": "celery-2.4.1.tar.gz",
158
+ "name": "celery",
159
+ "version": "2.4.1",
160
+ "sha256": "c77652ca179d14473975822dbfb1b5dab950c88c171ef6bc2257ddb9066e6790",
161
+ # Wrong metadata
162
+ "author": "ME",
163
+ "packagetype": "bdist",
164
+ "requires_python": ">=3.8",
165
+ }
166
+
167
+ scipy_data_1 = {
168
+ "filename": "scipy-1.1.0.tar.gz",
169
+ "name": "scipy",
170
+ "version": "1.1.0",
171
+ "sha256": "878352408424dffaa695ffedf2f9f92844e116686923ed9aa8626fc30d32cfd1",
172
+ # Wrong metadata
173
+ "author": "ME",
174
+ "packagetype": "bdist",
175
+ "requires_python": ">=3.8",
176
+ }
177
+
178
+ scipy_data_2 = scipy_data_1.copy()
179
+ scipy_data_2["filename"] = "scipy-1.1.0-cp36-none-win32.whl"
180
+ scipy_data_2["sha256"] = (
181
+ "0e9bb7efe5f051ea7212555b290e784b82f21ffd0f655405ac4f87e288b730b3"
182
+ )
183
+
184
+ # 2. Create content
185
+ celery_content = create_content_remote(celery_data, python_remote)
186
+ scipy_content_0 = create_content_direct(scipy_file, scipy_data_0)
187
+ scipy_content_1 = create_content_remote(
188
+ scipy_data_1, python_remote, python_remote_bad
189
+ )
190
+ scipy_content_2 = create_content_remote(scipy_data_2, python_remote_bad)
191
+
192
+ content_hrefs = {}
193
+ for data, content in [
194
+ (celery_data, celery_content),
195
+ (scipy_data_0, scipy_content_0),
196
+ (scipy_data_1, scipy_content_1),
197
+ (scipy_data_2, scipy_content_2),
198
+ ]:
199
+ for field, test_value in data.items():
200
+ assert getattr(content, field) == test_value
201
+ content_hrefs[data["filename"]] = content.pulp_href
202
+ move_to_repository(python_repo.pulp_href, list(content_hrefs.values()))
203
+
204
+ # 3. Repair metadata
205
+ response = python_bindings.RepositoriesPythonApi.repair_metadata(
206
+ python_repo.pulp_href
207
+ )
208
+ monitor_task(response.task)
209
+
210
+ # 4. Check new metadata
211
+ new_metadata = [
212
+ # repaired
213
+ ("celery-2.4.1.tar.gz", "Ask Solem", "sdist", ""),
214
+ (
215
+ "scipy-1.1.0-cp27-none-win32.whl",
216
+ "",
217
+ "bdist_wheel",
218
+ ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
219
+ ),
220
+ ("scipy-1.1.0.tar.gz", "", "sdist", ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"),
221
+ # not repaired
222
+ ("scipy-1.1.0-cp36-none-win32.whl", "ME", "bdist", ">=3.8"),
223
+ ]
224
+ for filename, author, packagetype, requires_python in new_metadata:
225
+ new_content = python_bindings.ContentPackagesApi.read(content_hrefs[filename])
226
+ assert new_content.author == author
227
+ assert new_content.packagetype == packagetype
228
+ assert new_content.requires_python == requires_python
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pulp-python
3
- Version: 3.15.0
3
+ Version: 3.16.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
5
  Author-email: Pulp Team <pulp-list@redhat.com>
6
6
  Project-URL: Homepage, https://pulpproject.org
@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
7
7
 
8
8
  [project]
9
9
  name = "pulp-python"
10
- version = "3.15.0"
10
+ version = "3.16.0"
11
11
  description = "pulp-python plugin for the Pulp Project"
12
12
  readme = "README.md"
13
13
  authors = [
@@ -29,7 +29,7 @@ requires-python = ">=3.9"
29
29
  dependencies = [
30
30
  "pulpcore>=3.49.0,<3.85",
31
31
  "pkginfo>=1.10.0,<1.13.0",
32
- "bandersnatch>=6.3.0,<6.4", # Anything >6.3 requires Python 3.10+
32
+ "bandersnatch>=6.3.0,<6.4", # Anything >=6.4 requires Python 3.10+
33
33
  "pypi-simple>=1.5.0,<2.0",
34
34
  ]
35
35
 
@@ -77,7 +77,7 @@ ignore = [
77
77
  [tool.bumpversion]
78
78
  # This section is managed by the plugin template. Do not edit manually.
79
79
 
80
- current_version = "3.15.0"
80
+ current_version = "3.16.0"
81
81
  commit = false
82
82
  tag = false
83
83
  parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"
@@ -1,89 +0,0 @@
1
- import logging
2
- import uuid
3
- from gettext import gettext as _
4
-
5
- from django.db.models.query import QuerySet
6
- from pulpcore.plugin.models import ProgressReport
7
- from pulpcore.plugin.util import get_domain
8
-
9
- from pulp_python.app.models import PythonPackageContent, PythonRepository
10
- from pulp_python.app.utils import artifact_to_python_content_data
11
-
12
- log = logging.getLogger(__name__)
13
-
14
-
15
- def repair(repository_pk: uuid.UUID) -> None:
16
- """
17
- Repairs metadata of all packages for the specified repository.
18
-
19
- Args:
20
- repository_pk (uuid.UUID): The primary key of the repository to repair.
21
-
22
- Returns:
23
- None
24
- """
25
- repository = PythonRepository.objects.get(pk=repository_pk)
26
-
27
- log.info(
28
- _(
29
- "Repairing packages' metadata for the latest version of repository {}."
30
- ).format(repository.name)
31
- )
32
- content_set = repository.latest_version().content.values_list("pk", flat=True)
33
- content = PythonPackageContent.objects.filter(pk__in=content_set)
34
-
35
- num_repaired = repair_metadata(content)
36
- log.info(_("{} packages' metadata repaired.").format(num_repaired))
37
-
38
-
39
- def repair_metadata(content: QuerySet[PythonPackageContent]) -> int:
40
- """
41
- Repairs metadata for a queryset of PythonPackageContent objects
42
- and updates the progress report.
43
-
44
- Args:
45
- content (QuerySet[PythonPackageContent]): The queryset of items to repair.
46
-
47
- Returns:
48
- int: The number of packages that were repaired.
49
- """
50
- # TODO: Add on_demand content repair
51
- immediate_content = content.filter(contentartifact__artifact__isnull=False)
52
- domain = get_domain()
53
-
54
- batch = []
55
- set_of_update_fields = set()
56
- total_repaired = 0
57
-
58
- progress_report = ProgressReport(
59
- message="Repairing packages' metadata",
60
- code="repair.metadata",
61
- total=immediate_content.count(),
62
- )
63
- progress_report.save()
64
- with progress_report:
65
- for package in progress_report.iter(
66
- immediate_content.prefetch_related("_artifacts").iterator(chunk_size=1000)
67
- ):
68
- new_data = artifact_to_python_content_data(
69
- package.filename, package._artifacts.get(), domain
70
- )
71
- changed = False
72
- for field, value in new_data.items():
73
- if getattr(package, field) != value:
74
- setattr(package, field, value)
75
- set_of_update_fields.add(field)
76
- changed = True
77
- if changed:
78
- batch.append(package)
79
- if len(batch) == 1000:
80
- total_repaired += len(batch)
81
- PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
82
- batch = []
83
- set_of_update_fields.clear()
84
-
85
- if batch:
86
- total_repaired += len(batch)
87
- PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
88
-
89
- return total_repaired
@@ -1,126 +0,0 @@
1
- import pytest
2
- import subprocess
3
- from urllib.parse import urljoin
4
-
5
- from pulp_python.tests.functional.constants import (
6
- PYTHON_EGG_FILENAME,
7
- PYTHON_FIXTURES_URL,
8
- )
9
-
10
-
11
- @pytest.fixture
12
- def create_content_direct(python_bindings):
13
- def _create(artifact_filename, filename, content_data):
14
- commands = (
15
- "from pulpcore.plugin.models import Artifact, ContentArtifact; "
16
- "from pulpcore.plugin.util import get_url; "
17
- "from pulp_python.app.models import PythonPackageContent; "
18
- f"a = Artifact.init_and_validate('{artifact_filename}'); "
19
- "a.save(); "
20
- f"c = PythonPackageContent(sha256=a.sha256, filename={filename!r}, **{content_data!r}); " # noqa: E501
21
- "c.save(); "
22
- f"ca = ContentArtifact(artifact=a, content=c, relative_path={filename!r}); "
23
- "ca.save(); "
24
- "print(get_url(c))"
25
- )
26
- process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
27
-
28
- assert process.returncode == 0
29
- content_href = process.stdout.decode().strip()
30
- return python_bindings.ContentPackagesApi.read(content_href)
31
-
32
- return _create
33
-
34
-
35
- @pytest.fixture
36
- def move_to_repository(python_bindings, monitor_task):
37
- def _move(repo_href, content_hrefs):
38
- body = {"add_content_units": content_hrefs}
39
- task = monitor_task(python_bindings.RepositoriesPythonApi.modify(repo_href, body).task)
40
- assert len(task.created_resources) == 1
41
- return python_bindings.RepositoriesPythonApi.read(repo_href)
42
-
43
- return _move
44
-
45
-
46
- def test_metadata_repair_command(
47
- create_content_direct,
48
- python_file,
49
- python_repo,
50
- move_to_repository,
51
- python_bindings,
52
- delete_orphans_pre,
53
- ):
54
- """Test pulpcore-manager repair-python-metadata command."""
55
- data = {
56
- "name": "shelf-reader",
57
- # Wrong metadata
58
- "version": "0.2",
59
- "packagetype": "bdist",
60
- "requires_python": ">=3.8",
61
- "author": "ME",
62
- }
63
- content = create_content_direct(python_file, PYTHON_EGG_FILENAME, data)
64
- for field, wrong_value in data.items():
65
- if field == "python_version":
66
- continue
67
- assert getattr(content, field) == wrong_value
68
-
69
- move_to_repository(python_repo.pulp_href, [content.pulp_href])
70
- process = subprocess.run(
71
- ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href],
72
- capture_output=True
73
- )
74
- assert process.returncode == 0
75
- output = process.stdout.decode().strip()
76
- assert output == "1 packages processed, 1 package metadata repaired."
77
-
78
- content = python_bindings.ContentPackagesApi.read(content.pulp_href)
79
- assert content.version == "0.1"
80
- assert content.packagetype == "sdist"
81
- assert content.requires_python == "" # technically null
82
- assert content.author == "Austin Macdonald"
83
-
84
-
85
- def test_metadata_repair_endpoint(
86
- create_content_direct,
87
- download_python_file,
88
- monitor_task,
89
- move_to_repository,
90
- python_bindings,
91
- python_repo,
92
- ):
93
- """
94
- Test repairing of package metadata via `Repositories.repair_metadata` endpoint.
95
- """
96
- python_egg_filename = "scipy-1.1.0.tar.gz"
97
- python_egg_url = urljoin(
98
- urljoin(PYTHON_FIXTURES_URL, "packages/"), python_egg_filename
99
- )
100
- python_file = download_python_file(python_egg_filename, python_egg_url)
101
-
102
- data = {
103
- "name": "scipy",
104
- # Wrong metadata
105
- "author": "ME",
106
- "packagetype": "bdist",
107
- "requires_python": ">=3.8",
108
- "version": "0.2",
109
- }
110
- content = create_content_direct(python_file, python_egg_filename, data)
111
- for field, wrong_value in data.items():
112
- if field == "python_version":
113
- continue
114
- assert getattr(content, field) == wrong_value
115
- move_to_repository(python_repo.pulp_href, [content.pulp_href])
116
-
117
- response = python_bindings.RepositoriesPythonApi.repair_metadata(
118
- python_repo.pulp_href
119
- )
120
- monitor_task(response.task)
121
-
122
- content = python_bindings.ContentPackagesApi.read(content.pulp_href)
123
- assert content.version == "1.1.0"
124
- assert content.packagetype == "sdist"
125
- assert content.requires_python == ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
126
- assert content.author == ""
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes