pulp-python 3.14.0__tar.gz → 3.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {pulp_python-3.14.0 → pulp_python-3.16.0}/CHANGES.md +46 -6
  2. {pulp_python-3.14.0 → pulp_python-3.16.0}/PKG-INFO +1 -1
  3. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/__init__.py +1 -1
  4. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/pypi/views.py +2 -2
  5. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/tasks/__init__.py +1 -0
  6. pulp_python-3.16.0/pulp_python/app/tasks/repair.py +198 -0
  7. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/utils.py +32 -0
  8. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/viewsets.py +20 -1
  9. pulp_python-3.16.0/pulp_python/tests/functional/api/test_repair.py +228 -0
  10. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/PKG-INFO +1 -1
  11. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/SOURCES.txt +1 -0
  12. {pulp_python-3.14.0 → pulp_python-3.16.0}/pyproject.toml +3 -3
  13. pulp_python-3.14.0/pulp_python/tests/functional/api/test_repair.py +0 -78
  14. {pulp_python-3.14.0 → pulp_python-3.16.0}/COMMITMENT +0 -0
  15. {pulp_python-3.14.0 → pulp_python-3.16.0}/COPYRIGHT +0 -0
  16. {pulp_python-3.14.0 → pulp_python-3.16.0}/LICENSE +0 -0
  17. {pulp_python-3.14.0 → pulp_python-3.16.0}/MANIFEST.in +0 -0
  18. {pulp_python-3.14.0 → pulp_python-3.16.0}/README.md +0 -0
  19. {pulp_python-3.14.0 → pulp_python-3.16.0}/functest_requirements.txt +0 -0
  20. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/__init__.py +0 -0
  21. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/global_access_conditions.py +0 -0
  22. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/management/__init__.py +0 -0
  23. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/management/commands/__init__.py +0 -0
  24. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/management/commands/repair-python-metadata.py +0 -0
  25. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0001_initial.py +0 -0
  26. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +0 -0
  27. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0003_new_sync_filters.py +0 -0
  28. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +0 -0
  29. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +0 -0
  30. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +0 -0
  31. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +0 -0
  32. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +0 -0
  33. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +0 -0
  34. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0010_update_json_field.py +0 -0
  35. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +0 -0
  36. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0012_add_domain.py +0 -0
  37. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/0013_add_rbac_permissions.py +0 -0
  38. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/migrations/__init__.py +0 -0
  39. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/modelresource.py +0 -0
  40. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/models.py +0 -0
  41. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/pypi/__init__.py +0 -0
  42. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/pypi/serializers.py +0 -0
  43. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/replica.py +0 -0
  44. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/serializers.py +0 -0
  45. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/settings.py +0 -0
  46. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/tasks/publish.py +0 -0
  47. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/tasks/sync.py +0 -0
  48. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/tasks/upload.py +0 -0
  49. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/urls.py +0 -0
  50. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/__init__.py +0 -0
  51. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/apache.conf +0 -0
  52. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/app/webserver_snippets/nginx.conf +0 -0
  53. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/pytest_plugin.py +0 -0
  54. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/__init__.py +0 -0
  55. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/__init__.py +0 -0
  56. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/__init__.py +0 -0
  57. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_auto_publish.py +0 -0
  58. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_consume_content.py +0 -0
  59. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_content_unit.py +0 -0
  60. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_publications.py +0 -0
  61. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_crud_remotes.py +0 -0
  62. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_domains.py +0 -0
  63. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_download_content.py +0 -0
  64. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_export_import.py +0 -0
  65. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_full_mirror.py +0 -0
  66. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_pypi_apis.py +0 -0
  67. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_rbac.py +0 -0
  68. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/api/test_sync.py +0 -0
  69. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/constants.py +0 -0
  70. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/functional/utils.py +0 -0
  71. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/unit/__init__.py +0 -0
  72. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python/tests/unit/test_models.py +0 -0
  73. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/dependency_links.txt +0 -0
  74. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/entry_points.txt +0 -0
  75. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/requires.txt +0 -0
  76. {pulp_python-3.14.0 → pulp_python-3.16.0}/pulp_python.egg-info/top_level.txt +0 -0
  77. {pulp_python-3.14.0 → pulp_python-3.16.0}/setup.cfg +0 -0
  78. {pulp_python-3.14.0 → pulp_python-3.16.0}/test_requirements.txt +0 -0
  79. {pulp_python-3.14.0 → pulp_python-3.16.0}/unittest_requirements.txt +0 -0
@@ -8,6 +8,29 @@
8
8
 
9
9
  [//]: # (towncrier release notes start)
10
10
 
11
+ ## 3.16.0 (2025-06-10) {: #3.16.0 }
12
+
13
+ #### Features {: #3.16.0-feature }
14
+
15
+ - Added support for on-demand content to `repair_metadata` endpoint.
16
+ [#849](https://github.com/pulp/pulp_python/issues/849)
17
+
18
+ #### Bugfixes {: #3.16.0-bugfix }
19
+
20
+ - Fixed pull-through caching not working for indexes that use relative URLs.
21
+ [#842](https://github.com/pulp/pulp_python/issues/842)
22
+
23
+ ---
24
+
25
+ ## 3.15.0 (2025-05-13) {: #3.15.0 }
26
+
27
+ #### Features {: #3.15.0-feature }
28
+
29
+ - Added new `repair_metadata` endpoint to `Repository` for fixing packages' metadata.
30
+ [#805](https://github.com/pulp/pulp_python/issues/805)
31
+
32
+ ---
33
+
11
34
  ## 3.14.0 (2025-04-10) {: #3.14.0 }
12
35
 
13
36
  #### Features {: #3.14.0-feature }
@@ -28,6 +51,20 @@
28
51
 
29
52
  ---
30
53
 
54
+ ## 3.13.5 (2025-04-23) {: #3.13.5 }
55
+
56
+ No significant changes.
57
+
58
+ ---
59
+
60
+ ## 3.13.4 (2025-04-10) {: #3.13.4 }
61
+
62
+ #### Misc {: #3.13.4-misc }
63
+
64
+ - [#809](https://github.com/pulp/pulp_python/issues/809)
65
+
66
+ ---
67
+
31
68
  ## 3.13.3 (2025-04-07) {: #3.13.3 }
32
69
 
33
70
  #### Bugfixes {: #3.13.3-bugfix }
@@ -115,7 +152,7 @@ No significant changes.
115
152
 
116
153
  ---
117
154
 
118
- # ## 3.12.2 (2024-08-21) {: #3.12.2 }
155
+ ## 3.12.2 (2024-08-21) {: #3.12.2 }
119
156
 
120
157
  #### Bugfixes {: #3.12.2-bugfix }
121
158
 
@@ -126,7 +163,6 @@ No significant changes.
126
163
 
127
164
  ## 3.12.1 (2024-06-27) {: #3.12.1 }
128
165
 
129
-
130
166
  #### Bugfixes {: #3.12.1-bugfix }
131
167
 
132
168
  - Fixed the `package_types` filter breaking other remote filters.
@@ -136,7 +172,6 @@ No significant changes.
136
172
 
137
173
  ## 3.12.0 (2024-06-25) {: #3.12.0 }
138
174
 
139
-
140
175
  #### Features {: #3.12.0-feature }
141
176
 
142
177
  - Added RBAC support.
@@ -162,6 +197,12 @@ No significant changes.
162
197
 
163
198
  ---
164
199
 
200
+ ## 3.11.5 (2025-04-15) {: #3.11.5 }
201
+
202
+ No significant changes.
203
+
204
+ ---
205
+
165
206
  ## 3.11.4 (2025-02-20) {: #3.11.4 }
166
207
 
167
208
  #### Bugfixes {: #3.11.4-bugfix }
@@ -175,7 +216,7 @@ No significant changes.
175
216
 
176
217
  ---
177
218
 
178
- # ## 3.11.3 (2024-08-21) {: #3.11.3 }
219
+ ## 3.11.3 (2024-08-21) {: #3.11.3 }
179
220
 
180
221
  #### Bugfixes {: #3.11.3-bugfix }
181
222
 
@@ -188,7 +229,6 @@ No significant changes.
188
229
 
189
230
  ## 3.11.2 (2024-06-27) {: #3.11.2 }
190
231
 
191
-
192
232
  #### Bugfixes {: #3.11.2-bugfix }
193
233
 
194
234
  - Fixed the `package_types` filter breaking other remote filters.
@@ -406,7 +446,7 @@ No significant changes.
406
446
 
407
447
  ---
408
448
 
409
- 3.4.0 (2021-06-17)
449
+ ## 3.4.0 (2021-06-17) {: #3.4.0 }
410
450
 
411
451
  ### Features
412
452
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pulp-python
3
- Version: 3.14.0
3
+ Version: 3.16.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
5
  Author-email: Pulp Team <pulp-list@redhat.com>
6
6
  Project-URL: Homepage, https://pulpproject.org
@@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
10
10
 
11
11
  name = "pulp_python.app"
12
12
  label = "python"
13
- version = "3.14.0"
13
+ version = "3.16.0"
14
14
  python_package_name = "pulp-python"
15
15
  domain_compatible = True
16
16
 
@@ -260,9 +260,9 @@ class SimpleView(PackageUploadMixin, ViewSet):
260
260
  return HttpResponse(f"{remote.url} timed out while fetching {package}.", status=504)
261
261
 
262
262
  if d.headers["content-type"] == "application/vnd.pypi.simple.v1+json":
263
- page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=remote.url)
263
+ page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=url)
264
264
  else:
265
- page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=remote.url)
265
+ page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=url)
266
266
  packages = [
267
267
  parse_package(p) for p in page.packages if rfilter.filter_release(package, p.version)
268
268
  ]
@@ -3,5 +3,6 @@ Asynchronous task definitions.
3
3
  """
4
4
 
5
5
  from .publish import publish # noqa:F401
6
+ from .repair import repair # noqa:F401
6
7
  from .sync import sync # noqa:F401
7
8
  from .upload import upload, upload_group # noqa:F401
@@ -0,0 +1,198 @@
1
+ import logging
2
+ from collections import defaultdict
3
+ from gettext import gettext as _
4
+ from itertools import groupby
5
+ from uuid import UUID
6
+
7
+ from django.db.models import Prefetch
8
+ from django.db.models.query import QuerySet
9
+ from pulp_python.app.models import PythonPackageContent, PythonRepository
10
+ from pulp_python.app.utils import (
11
+ artifact_to_python_content_data,
12
+ fetch_json_release_metadata,
13
+ parse_metadata,
14
+ )
15
+ from pulpcore.plugin.models import ContentArtifact, ProgressReport
16
+ from pulpcore.plugin.util import get_domain
17
+
18
+ log = logging.getLogger(__name__)
19
+
20
+
21
+ BULK_SIZE = 1000
22
+
23
+
24
+ def repair(repository_pk: UUID) -> None:
25
+ """
26
+ Repairs metadata of all packages for the specified repository.
27
+
28
+ Args:
29
+ repository_pk (UUID): The primary key of the repository to repair.
30
+
31
+ Returns:
32
+ None
33
+ """
34
+ repository = PythonRepository.objects.get(pk=repository_pk)
35
+
36
+ log.info(
37
+ _(
38
+ "Repairing packages' metadata for the latest version of repository {}."
39
+ ).format(repository.name)
40
+ )
41
+ content_set = repository.latest_version().content.values_list("pk", flat=True)
42
+ content = PythonPackageContent.objects.filter(pk__in=content_set)
43
+
44
+ num_repaired, pkgs_not_repaired = repair_metadata(content)
45
+ log.info(
46
+ _(
47
+ "{} packages' metadata repaired. Not repaired packages due to either "
48
+ "inaccessible URL or mismatched sha256: {}."
49
+ ).format(num_repaired, pkgs_not_repaired)
50
+ )
51
+
52
+
53
+ def repair_metadata(content: QuerySet[PythonPackageContent]) -> tuple[int, set[str]]:
54
+ """
55
+ Repairs metadata for a queryset of PythonPackageContent objects
56
+ and updates the progress report.
57
+
58
+ Args:
59
+ content (QuerySet[PythonPackageContent]): The queryset of items to repair.
60
+
61
+ Returns:
62
+ tuple[int, set[str]]: A tuple containing:
63
+ - The number of packages that were repaired.
64
+ - A set of packages' PKs that were not repaired.
65
+ """
66
+ immediate_content = (
67
+ content.filter(contentartifact__artifact__isnull=False)
68
+ .distinct()
69
+ .prefetch_related("_artifacts")
70
+ )
71
+ on_demand_content = (
72
+ content.filter(contentartifact__artifact__isnull=True)
73
+ .distinct()
74
+ .prefetch_related(
75
+ Prefetch(
76
+ "contentartifact_set",
77
+ queryset=ContentArtifact.objects.prefetch_related("remoteartifact_set"),
78
+ )
79
+ )
80
+ .order_by("name", "version")
81
+ )
82
+ domain = get_domain()
83
+
84
+ batch = []
85
+ set_of_update_fields = set()
86
+ total_repaired = 0
87
+ # Keep track of on-demand packages that were not repaired
88
+ pkgs_not_repaired = set()
89
+
90
+ progress_report = ProgressReport(
91
+ message="Repairing packages' metadata",
92
+ code="repair.metadata",
93
+ total=content.count(),
94
+ )
95
+ progress_report.save()
96
+ with progress_report:
97
+ for package in progress_report.iter(
98
+ immediate_content.iterator(chunk_size=BULK_SIZE)
99
+ ):
100
+ new_data = artifact_to_python_content_data(
101
+ package.filename, package._artifacts.get(), domain
102
+ )
103
+ total_repaired += update_package_if_needed(
104
+ package, new_data, batch, set_of_update_fields
105
+ )
106
+
107
+ # For on-demand content, we expect that:
108
+ # 1. PythonPackageContent always has correct name and version
109
+ # 2. RemoteArtifact always has correct sha256
110
+ for (name, version), group in groupby(
111
+ on_demand_content.iterator(chunk_size=BULK_SIZE),
112
+ key=lambda x: (x.name, x.version),
113
+ ):
114
+ group_set = set(group)
115
+ grouped_by_url = defaultdict(list)
116
+
117
+ for package in group_set:
118
+ for ra in package.contentartifact_set.get().remoteartifact_set.all():
119
+ grouped_by_url[ra.remote.url].append((package, ra))
120
+
121
+ # Prioritize the URL that can serve the most packages
122
+ for url, pkg_ra_pairs in sorted(
123
+ grouped_by_url.items(), key=lambda x: len(x[1]), reverse=True
124
+ ):
125
+ if not group_set:
126
+ break # No packages left to repair, move onto the next group
127
+ remotes = set([pkg_ra[1].remote for pkg_ra in pkg_ra_pairs])
128
+ try:
129
+ json_data = fetch_json_release_metadata(name, version, remotes)
130
+ except Exception:
131
+ continue
132
+
133
+ for package, ra in pkg_ra_pairs:
134
+ if package not in group_set:
135
+ continue # Package was already repaired
136
+ # Extract data only for the specific distribution being checked
137
+ dist_data = None
138
+ for dist in json_data["urls"]:
139
+ if ra.sha256 == dist["digests"]["sha256"]:
140
+ dist_data = dist
141
+ break
142
+ if not dist_data:
143
+ continue
144
+
145
+ new_data = parse_metadata(json_data["info"], version, dist_data)
146
+ new_data.pop("url") # url belongs to RemoteArtifact
147
+ total_repaired += update_package_if_needed(
148
+ package, new_data, batch, set_of_update_fields
149
+ )
150
+ group_set.remove(package)
151
+ progress_report.increment()
152
+ # Store and track the unrepaired packages after all URLs are processed
153
+ pkgs_not_repaired.update([p.pk for p in group_set])
154
+ progress_report.increase_by(len(group_set))
155
+
156
+ if batch:
157
+ total_repaired += len(batch)
158
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
159
+
160
+ return total_repaired, pkgs_not_repaired
161
+
162
+
163
+ def update_package_if_needed(
164
+ package: PythonPackageContent,
165
+ new_data: dict,
166
+ batch: list[PythonPackageContent],
167
+ set_of_update_fields: set[str],
168
+ ) -> int:
169
+ """
170
+ Compares the current package data with new data and updates the package
171
+ if needed ("batch" and "set_of_update_fields" are updated in-place).
172
+
173
+ Args:
174
+ package: Package to check and update.
175
+ new_data: A dict of new field values to compare against the package.
176
+ batch: A list of packages that were updated.
177
+ set_of_update_fields: A set of package field names that were updated.
178
+
179
+ Returns:
180
+ The count of repaired packages (increments in multiples of BULK_SIZE only).
181
+ """
182
+ total_repaired = 0
183
+ changed = False
184
+ for field, value in new_data.items():
185
+ if getattr(package, field) != value:
186
+ setattr(package, field, value)
187
+ set_of_update_fields.add(field)
188
+ changed = True
189
+ if changed:
190
+ batch.append(package)
191
+
192
+ if len(batch) == BULK_SIZE:
193
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
194
+ total_repaired += BULK_SIZE
195
+ batch.clear()
196
+ set_of_update_fields.clear()
197
+
198
+ return total_repaired
@@ -9,6 +9,7 @@ from jinja2 import Template
9
9
  from packaging.utils import canonicalize_name
10
10
  from packaging.requirements import Requirement
11
11
  from packaging.version import parse, InvalidVersion
12
+ from pulpcore.plugin.models import Remote
12
13
 
13
14
 
14
15
  PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
@@ -189,6 +190,37 @@ def artifact_to_python_content_data(filename, artifact, domain=None):
189
190
  return data
190
191
 
191
192
 
193
+ def fetch_json_release_metadata(name: str, version: str, remotes: set[Remote]) -> dict:
194
+ """
195
+ Fetches metadata for a specific release from PyPI's JSON API. A release can contain
196
+ multiple distributions. See https://docs.pypi.org/api/json/#get-a-release for more details.
197
+ All remotes should have the same URL.
198
+
199
+ Returns:
200
+ Dict containing "info", "last_serial", "urls", and "vulnerabilities" keys.
201
+ Raises:
202
+ Exception if fetching from all remote URLs fails.
203
+ """
204
+ remote = next(iter(remotes))
205
+ url = remote.get_remote_artifact_url(f"pypi/{name}/{version}/json")
206
+
207
+ result = None
208
+ for remote in remotes:
209
+ downloader = remote.get_downloader(url=url, max_retries=1)
210
+ try:
211
+ result = downloader.fetch()
212
+ break
213
+ except Exception:
214
+ continue
215
+
216
+ if result:
217
+ with open(result.path, "r") as file:
218
+ json_data = json.load(file)
219
+ return json_data
220
+ else:
221
+ raise Exception(f"Failed to fetch {url} from any remote.")
222
+
223
+
192
224
  def python_content_to_json(base_path, content_query, version=None, domain=None):
193
225
  """
194
226
  Converts a QuerySet of PythonPackageContent into the PyPi JSON format
@@ -83,7 +83,7 @@ class PythonRepositoryViewSet(
83
83
  ],
84
84
  },
85
85
  {
86
- "action": ["modify"],
86
+ "action": ["modify", "repair_metadata"],
87
87
  "principal": "authenticated",
88
88
  "effect": "allow",
89
89
  "condition": [
@@ -122,6 +122,25 @@ class PythonRepositoryViewSet(
122
122
  "python.pythonrepository_viewer": ["python.view_pythonrepository"],
123
123
  }
124
124
 
125
+ @extend_schema(
126
+ summary="Repair metadata",
127
+ responses={202: AsyncOperationResponseSerializer},
128
+ )
129
+ @action(detail=True, methods=["post"], serializer_class=None)
130
+ def repair_metadata(self, request, pk):
131
+ """
132
+ Trigger an asynchronous task to repair Python metadata. This task will repair metadata
133
+ of all packages for the specified `Repository`, without creating a new `RepositoryVersion`.
134
+ """
135
+ repository = self.get_object()
136
+
137
+ result = dispatch(
138
+ tasks.repair,
139
+ exclusive_resources=[repository],
140
+ kwargs={"repository_pk": str(repository.pk)},
141
+ )
142
+ return core_viewsets.OperationPostponedResponse(result, request)
143
+
125
144
  @extend_schema(
126
145
  summary="Sync from remote",
127
146
  responses={202: AsyncOperationResponseSerializer}
@@ -0,0 +1,228 @@
1
+ import pytest
2
+ import subprocess
3
+ from urllib.parse import urljoin
4
+
5
+ from pulp_python.tests.functional.constants import (
6
+ PYTHON_EGG_FILENAME,
7
+ PYTHON_FIXTURES_URL,
8
+ )
9
+
10
+
11
+ @pytest.fixture
12
+ def create_content_direct(python_bindings):
13
+ def _create(artifact_filename, content_data):
14
+ commands = (
15
+ "from pulpcore.plugin.models import Artifact, ContentArtifact; "
16
+ "from pulpcore.plugin.util import get_url; "
17
+ "from pulp_python.app.models import PythonPackageContent; "
18
+ f"a = Artifact.init_and_validate('{artifact_filename}'); "
19
+ "a.save(); "
20
+ f"c = PythonPackageContent(sha256=a.sha256, **{content_data!r}); "
21
+ "c.save(); "
22
+ f"ca = ContentArtifact(artifact=a, content=c, relative_path=c.filename); "
23
+ "ca.save(); "
24
+ "print(get_url(c))"
25
+ )
26
+ process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
27
+
28
+ assert process.returncode == 0
29
+ content_href = process.stdout.decode().strip()
30
+ return python_bindings.ContentPackagesApi.read(content_href)
31
+
32
+ return _create
33
+
34
+
35
+ @pytest.fixture
36
+ def create_content_remote(python_bindings):
37
+ def _create(content, remote, remote_2=None):
38
+ commands = (
39
+ "from pulpcore.plugin.models import ContentArtifact, RemoteArtifact; "
40
+ "from pulpcore.plugin.util import extract_pk, get_url; "
41
+ "from pulp_python.app.models import PythonPackageContent, PythonRemote; "
42
+ f"c = PythonPackageContent(**{content!r}); "
43
+ "c.save(); "
44
+ f"ca = ContentArtifact(content=c, relative_path=c.filename); "
45
+ "ca.save(); "
46
+ f"r = PythonRemote.objects.get(pk=extract_pk({remote.pulp_href!r})); "
47
+ f"ra = RemoteArtifact(content_artifact=ca, remote=r, sha256=c.sha256); "
48
+ "ra.save(); "
49
+ )
50
+ if remote_2:
51
+ commands += (
52
+ f"r2 = PythonRemote.objects.get(pk=extract_pk({remote_2.pulp_href!r})); "
53
+ f"ra2 = RemoteArtifact(content_artifact=ca, remote=r2, sha256=c.sha256); "
54
+ "ra2.save(); "
55
+ )
56
+ commands += "print(get_url(c))"
57
+ process = subprocess.run(
58
+ ["pulpcore-manager", "shell", "-c", commands], capture_output=True
59
+ )
60
+
61
+ assert process.returncode == 0
62
+ content_href = process.stdout.decode().strip()
63
+ return python_bindings.ContentPackagesApi.read(content_href)
64
+
65
+ return _create
66
+
67
+
68
+ @pytest.fixture
69
+ def move_to_repository(python_bindings, monitor_task):
70
+ def _move(repo_href, content_hrefs):
71
+ body = {"add_content_units": content_hrefs}
72
+ task = monitor_task(python_bindings.RepositoriesPythonApi.modify(repo_href, body).task)
73
+ assert len(task.created_resources) == 1
74
+ return python_bindings.RepositoriesPythonApi.read(repo_href)
75
+
76
+ return _move
77
+
78
+
79
+ def test_metadata_repair_command(
80
+ create_content_direct,
81
+ python_file,
82
+ python_repo,
83
+ move_to_repository,
84
+ python_bindings,
85
+ delete_orphans_pre,
86
+ ):
87
+ """Test pulpcore-manager repair-python-metadata command."""
88
+ data = {
89
+ "name": "shelf-reader",
90
+ "filename": PYTHON_EGG_FILENAME,
91
+ # Wrong metadata
92
+ "version": "0.2",
93
+ "packagetype": "bdist",
94
+ "requires_python": ">=3.8",
95
+ "author": "ME",
96
+ }
97
+ content = create_content_direct(python_file, data)
98
+ for field, wrong_value in data.items():
99
+ if field == "python_version":
100
+ continue
101
+ assert getattr(content, field) == wrong_value
102
+
103
+ move_to_repository(python_repo.pulp_href, [content.pulp_href])
104
+ process = subprocess.run(
105
+ ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href],
106
+ capture_output=True
107
+ )
108
+ assert process.returncode == 0
109
+ output = process.stdout.decode().strip()
110
+ assert output == "1 packages processed, 1 package metadata repaired."
111
+
112
+ content = python_bindings.ContentPackagesApi.read(content.pulp_href)
113
+ assert content.version == "0.1"
114
+ assert content.packagetype == "sdist"
115
+ assert content.requires_python == "" # technically null
116
+ assert content.author == "Austin Macdonald"
117
+
118
+
119
+ def test_metadata_repair_endpoint(
120
+ create_content_direct,
121
+ create_content_remote,
122
+ delete_orphans_pre,
123
+ download_python_file,
124
+ monitor_task,
125
+ move_to_repository,
126
+ python_bindings,
127
+ python_remote_factory,
128
+ python_repo_factory,
129
+ ):
130
+ """
131
+ Test repairing of package metadata via `Repositories.repair_metadata` endpoint.
132
+ """
133
+ # 1. Setup tested data
134
+ # Shared data
135
+ python_remote = python_remote_factory()
136
+ python_remote_bad = python_remote_factory(url="https://fixtures.pulpproject.org/")
137
+ python_repo = python_repo_factory(remote=python_remote)
138
+
139
+ # Immediate content
140
+ scipy_egg_filename = "scipy-1.1.0-cp27-none-win32.whl"
141
+ scipy_egg_url = urljoin(
142
+ urljoin(PYTHON_FIXTURES_URL, "packages/"), scipy_egg_filename
143
+ )
144
+ scipy_file = download_python_file(scipy_egg_filename, scipy_egg_url)
145
+ scipy_data_0 = {
146
+ "filename": scipy_egg_filename,
147
+ "name": "scipy",
148
+ "version": "1.1.0",
149
+ # Wrong metadata
150
+ "author": "ME",
151
+ "packagetype": "bdist",
152
+ "requires_python": ">=3.8",
153
+ }
154
+
155
+ # On-demand content
156
+ celery_data = {
157
+ "filename": "celery-2.4.1.tar.gz",
158
+ "name": "celery",
159
+ "version": "2.4.1",
160
+ "sha256": "c77652ca179d14473975822dbfb1b5dab950c88c171ef6bc2257ddb9066e6790",
161
+ # Wrong metadata
162
+ "author": "ME",
163
+ "packagetype": "bdist",
164
+ "requires_python": ">=3.8",
165
+ }
166
+
167
+ scipy_data_1 = {
168
+ "filename": "scipy-1.1.0.tar.gz",
169
+ "name": "scipy",
170
+ "version": "1.1.0",
171
+ "sha256": "878352408424dffaa695ffedf2f9f92844e116686923ed9aa8626fc30d32cfd1",
172
+ # Wrong metadata
173
+ "author": "ME",
174
+ "packagetype": "bdist",
175
+ "requires_python": ">=3.8",
176
+ }
177
+
178
+ scipy_data_2 = scipy_data_1.copy()
179
+ scipy_data_2["filename"] = "scipy-1.1.0-cp36-none-win32.whl"
180
+ scipy_data_2["sha256"] = (
181
+ "0e9bb7efe5f051ea7212555b290e784b82f21ffd0f655405ac4f87e288b730b3"
182
+ )
183
+
184
+ # 2. Create content
185
+ celery_content = create_content_remote(celery_data, python_remote)
186
+ scipy_content_0 = create_content_direct(scipy_file, scipy_data_0)
187
+ scipy_content_1 = create_content_remote(
188
+ scipy_data_1, python_remote, python_remote_bad
189
+ )
190
+ scipy_content_2 = create_content_remote(scipy_data_2, python_remote_bad)
191
+
192
+ content_hrefs = {}
193
+ for data, content in [
194
+ (celery_data, celery_content),
195
+ (scipy_data_0, scipy_content_0),
196
+ (scipy_data_1, scipy_content_1),
197
+ (scipy_data_2, scipy_content_2),
198
+ ]:
199
+ for field, test_value in data.items():
200
+ assert getattr(content, field) == test_value
201
+ content_hrefs[data["filename"]] = content.pulp_href
202
+ move_to_repository(python_repo.pulp_href, list(content_hrefs.values()))
203
+
204
+ # 3. Repair metadata
205
+ response = python_bindings.RepositoriesPythonApi.repair_metadata(
206
+ python_repo.pulp_href
207
+ )
208
+ monitor_task(response.task)
209
+
210
+ # 4. Check new metadata
211
+ new_metadata = [
212
+ # repaired
213
+ ("celery-2.4.1.tar.gz", "Ask Solem", "sdist", ""),
214
+ (
215
+ "scipy-1.1.0-cp27-none-win32.whl",
216
+ "",
217
+ "bdist_wheel",
218
+ ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
219
+ ),
220
+ ("scipy-1.1.0.tar.gz", "", "sdist", ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"),
221
+ # not repaired
222
+ ("scipy-1.1.0-cp36-none-win32.whl", "ME", "bdist", ">=3.8"),
223
+ ]
224
+ for filename, author, packagetype, requires_python in new_metadata:
225
+ new_content = python_bindings.ContentPackagesApi.read(content_hrefs[filename])
226
+ assert new_content.author == author
227
+ assert new_content.packagetype == packagetype
228
+ assert new_content.requires_python == requires_python
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pulp-python
3
- Version: 3.14.0
3
+ Version: 3.16.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
5
  Author-email: Pulp Team <pulp-list@redhat.com>
6
6
  Project-URL: Homepage, https://pulpproject.org
@@ -48,6 +48,7 @@ pulp_python/app/pypi/serializers.py
48
48
  pulp_python/app/pypi/views.py
49
49
  pulp_python/app/tasks/__init__.py
50
50
  pulp_python/app/tasks/publish.py
51
+ pulp_python/app/tasks/repair.py
51
52
  pulp_python/app/tasks/sync.py
52
53
  pulp_python/app/tasks/upload.py
53
54
  pulp_python/app/webserver_snippets/__init__.py
@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
7
7
 
8
8
  [project]
9
9
  name = "pulp-python"
10
- version = "3.14.0"
10
+ version = "3.16.0"
11
11
  description = "pulp-python plugin for the Pulp Project"
12
12
  readme = "README.md"
13
13
  authors = [
@@ -29,7 +29,7 @@ requires-python = ">=3.9"
29
29
  dependencies = [
30
30
  "pulpcore>=3.49.0,<3.85",
31
31
  "pkginfo>=1.10.0,<1.13.0",
32
- "bandersnatch>=6.3.0,<6.4", # Anything >6.3 requires Python 3.10+
32
+ "bandersnatch>=6.3.0,<6.4", # Anything >=6.4 requires Python 3.10+
33
33
  "pypi-simple>=1.5.0,<2.0",
34
34
  ]
35
35
 
@@ -77,7 +77,7 @@ ignore = [
77
77
  [tool.bumpversion]
78
78
  # This section is managed by the plugin template. Do not edit manually.
79
79
 
80
- current_version = "3.14.0"
80
+ current_version = "3.16.0"
81
81
  commit = false
82
82
  tag = false
83
83
  parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"
@@ -1,78 +0,0 @@
1
- import pytest
2
- import subprocess
3
-
4
- from pulp_python.tests.functional.constants import PYTHON_EGG_FILENAME
5
-
6
-
7
- @pytest.fixture
8
- def create_content_direct(python_bindings):
9
- def _create(artifact_filename, filename, content_data):
10
- commands = (
11
- "from pulpcore.plugin.models import Artifact, ContentArtifact; "
12
- "from pulpcore.plugin.util import get_url; "
13
- "from pulp_python.app.models import PythonPackageContent; "
14
- f"a = Artifact.init_and_validate('{artifact_filename}'); "
15
- "a.save(); "
16
- f"c = PythonPackageContent(sha256=a.sha256, filename={filename!r}, **{content_data!r}); " # noqa: E501
17
- "c.save(); "
18
- f"ca = ContentArtifact(artifact=a, content=c, relative_path={filename!r}); "
19
- "ca.save(); "
20
- "print(get_url(c))"
21
- )
22
- process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
23
-
24
- assert process.returncode == 0
25
- content_href = process.stdout.decode().strip()
26
- return python_bindings.ContentPackagesApi.read(content_href)
27
-
28
- return _create
29
-
30
-
31
- @pytest.fixture
32
- def move_to_repository(python_bindings, monitor_task):
33
- def _move(repo_href, content_hrefs):
34
- body = {"add_content_units": content_hrefs}
35
- task = monitor_task(python_bindings.RepositoriesPythonApi.modify(repo_href, body).task)
36
- assert len(task.created_resources) == 1
37
- return python_bindings.RepositoriesPythonApi.read(repo_href)
38
-
39
- return _move
40
-
41
-
42
- def test_metadata_repair_command(
43
- create_content_direct,
44
- python_file,
45
- python_repo,
46
- move_to_repository,
47
- python_bindings,
48
- delete_orphans_pre,
49
- ):
50
- """Test pulpcore-manager repair-python-metadata command."""
51
- data = {
52
- "name": "shelf-reader",
53
- # Wrong metadata
54
- "version": "0.2",
55
- "packagetype": "bdist",
56
- "requires_python": ">=3.8",
57
- "author": "ME",
58
- }
59
- content = create_content_direct(python_file, PYTHON_EGG_FILENAME, data)
60
- for field, wrong_value in data.items():
61
- if field == "python_version":
62
- continue
63
- assert getattr(content, field) == wrong_value
64
-
65
- move_to_repository(python_repo.pulp_href, [content.pulp_href])
66
- process = subprocess.run(
67
- ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href],
68
- capture_output=True
69
- )
70
- assert process.returncode == 0
71
- output = process.stdout.decode().strip()
72
- assert output == "1 packages processed, 1 package metadata repaired."
73
-
74
- content = python_bindings.ContentPackagesApi.read(content.pulp_href)
75
- assert content.version == "0.1"
76
- assert content.packagetype == "sdist"
77
- assert content.requires_python == "" # technically null
78
- assert content.author == "Austin Macdonald"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes