pulp-python 3.12.4__tar.gz → 3.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pulp-python-3.12.4 → pulp_python-3.13.0}/CHANGES.md +60 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/MANIFEST.in +2 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/PKG-INFO +15 -6
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/__init__.py +1 -1
- pulp_python-3.13.0/pulp_python/app/management/commands/repair-python-metadata.py +118 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0001_initial.py +8 -8
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +0 -1
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/models.py +2 -10
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/pypi/views.py +1 -1
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/serializers.py +2 -10
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/sync.py +11 -41
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/upload.py +2 -10
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/utils.py +43 -5
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/pytest_plugin.py +1 -1
- pulp_python-3.13.0/pulp_python/tests/functional/api/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_content_unit.py +27 -15
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_remotes.py +21 -14
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_domains.py +5 -14
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_full_mirror.py +4 -5
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_rbac.py +16 -8
- pulp_python-3.13.0/pulp_python/tests/functional/api/test_repair.py +78 -0
- pulp_python-3.13.0/pulp_python/tests/unit/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/PKG-INFO +15 -6
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/SOURCES.txt +4 -2
- pulp_python-3.13.0/pulp_python.egg-info/requires.txt +4 -0
- pulp_python-3.13.0/pulp_python.egg-info/top_level.txt +4 -0
- pulp_python-3.13.0/pyproject.toml +124 -0
- pulp-python-3.12.4/pulp_python.egg-info/requires.txt +0 -4
- pulp-python-3.12.4/pulp_python.egg-info/top_level.txt +0 -1
- pulp-python-3.12.4/pyproject.toml +0 -30
- pulp-python-3.12.4/requirements.txt +0 -4
- pulp-python-3.12.4/setup.py +0 -38
- {pulp-python-3.12.4 → pulp_python-3.13.0}/COMMITMENT +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/COPYRIGHT +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/LICENSE +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/README.md +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/functest_requirements.txt +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/global_access_conditions.py +0 -0
- {pulp-python-3.12.4/pulp_python/app/migrations → pulp_python-3.13.0/pulp_python/app/management}/__init__.py +0 -0
- {pulp-python-3.12.4/pulp_python/app/pypi → pulp_python-3.13.0/pulp_python/app/management/commands}/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0003_new_sync_filters.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0010_update_json_field.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0012_add_domain.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0013_add_rbac_permissions.py +0 -0
- {pulp-python-3.12.4/pulp_python/app/webserver_snippets → pulp_python-3.13.0/pulp_python/app/migrations}/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/modelresource.py +0 -0
- {pulp-python-3.12.4/pulp_python/tests/functional → pulp_python-3.13.0/pulp_python/app/pypi}/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/pypi/serializers.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/replica.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/settings.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/publish.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/urls.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/viewsets.py +0 -0
- {pulp-python-3.12.4/pulp_python/tests/functional/api → pulp_python-3.13.0/pulp_python/app/webserver_snippets}/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/webserver_snippets/apache.conf +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/webserver_snippets/nginx.conf +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/__init__.py +0 -0
- {pulp-python-3.12.4/pulp_python/tests/unit → pulp_python-3.13.0/pulp_python/tests/functional}/__init__.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_auto_publish.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_consume_content.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_publications.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_download_content.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_export_import.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_pypi_apis.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_sync.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/constants.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/utils.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/unit/test_models.py +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/dependency_links.txt +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/entry_points.txt +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/setup.cfg +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/test_requirements.txt +0 -0
- {pulp-python-3.12.4 → pulp_python-3.13.0}/unittest_requirements.txt +0 -0
|
@@ -8,6 +8,45 @@
|
|
|
8
8
|
|
|
9
9
|
[//]: # (towncrier release notes start)
|
|
10
10
|
|
|
11
|
+
## 3.13.0 (2025-02-05) {: #3.13.0 }
|
|
12
|
+
|
|
13
|
+
#### Features {: #3.13.0-feature }
|
|
14
|
+
|
|
15
|
+
- Added pulpcore 3.70 compatibility
|
|
16
|
+
|
|
17
|
+
#### Bugfixes {: #3.13.0-bugfix }
|
|
18
|
+
|
|
19
|
+
- Fixed uploads not supporting packages using metadata spec 2.3
|
|
20
|
+
[#682](https://github.com/pulp/pulp_python/issues/682)
|
|
21
|
+
- Fixed the `package_types` filter breaking other remote filters.
|
|
22
|
+
[#691](https://github.com/pulp/pulp_python/issues/691)
|
|
23
|
+
- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names.
|
|
24
|
+
[#716](https://github.com/pulp/pulp_python/issues/716)
|
|
25
|
+
- Fixed replicate failing on upstream on-demand repositories
|
|
26
|
+
[#718](https://github.com/pulp/pulp_python/issues/718)
|
|
27
|
+
- Fixed `requires_python` field not being properly set on package upload.
|
|
28
|
+
|
|
29
|
+
Run the new `pulpcore-manager repair-python-metadata` command with repositories containing affected
|
|
30
|
+
packages to repair their metadata.
|
|
31
|
+
[#773](https://github.com/pulp/pulp_python/issues/773)
|
|
32
|
+
- Fixed the JSONField specification so it doesn't break ruby bindings.
|
|
33
|
+
See context [here](https://github.com/pulp/pulp_rpm/issues/3639).
|
|
34
|
+
|
|
35
|
+
#### Misc {: #3.13.0-misc }
|
|
36
|
+
|
|
37
|
+
- [#774](https://github.com/pulp/pulp_python/issues/774)
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## 3.12.5 (2024-10-25) {: #3.12.5 }
|
|
42
|
+
|
|
43
|
+
#### Bugfixes {: #3.12.5-bugfix }
|
|
44
|
+
|
|
45
|
+
- Fixed the JSONField specification so it doesn't break ruby bindings.
|
|
46
|
+
See context [here](https://github.com/pulp/pulp_rpm/issues/3639).
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
11
50
|
## 3.12.4 (2024-10-14) {: #3.12.4 }
|
|
12
51
|
|
|
13
52
|
#### Bugfixes {: #3.12.4-bugfix }
|
|
@@ -73,6 +112,27 @@
|
|
|
73
112
|
|
|
74
113
|
---
|
|
75
114
|
|
|
115
|
+
## 3.11.3 (2024-08-21) {: #3.11.3 }
|
|
116
|
+
|
|
117
|
+
#### Bugfixes {: #3.11.3-bugfix }
|
|
118
|
+
|
|
119
|
+
- Fixed uploads not supporting packages using metadata spec 2.3
|
|
120
|
+
[#682](https://github.com/pulp/pulp_python/issues/682)
|
|
121
|
+
- Fixed package name normalization issue preventing syncing packages with "." or "_" in their names.
|
|
122
|
+
[#716](https://github.com/pulp/pulp_python/issues/716)
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## 3.11.2 (2024-06-27) {: #3.11.2 }
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
#### Bugfixes {: #3.11.2-bugfix }
|
|
130
|
+
|
|
131
|
+
- Fixed the `package_types` filter breaking other remote filters.
|
|
132
|
+
[#691](https://github.com/pulp/pulp_python/issues/691)
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
76
136
|
## 3.11.1 (2024-04-11) {: #3.11.1 }
|
|
77
137
|
|
|
78
138
|
### Bugfixes
|
|
@@ -3,8 +3,10 @@ include requirements.txt
|
|
|
3
3
|
include pyproject.toml
|
|
4
4
|
include CHANGES.md
|
|
5
5
|
include COMMITMENT
|
|
6
|
+
exclude CONTRIBUTING.md
|
|
6
7
|
include COPYRIGHT
|
|
7
8
|
include functest_requirements.txt
|
|
8
9
|
include test_requirements.txt
|
|
9
10
|
include unittest_requirements.txt
|
|
10
11
|
include pulp_python/app/webserver_snippets/*
|
|
12
|
+
exclude releasing.md
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: pulp-python
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.13.0
|
|
4
4
|
Summary: pulp-python plugin for the Pulp Project
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
Author-email: Pulp Team <pulp-list@redhat.com>
|
|
6
|
+
Project-URL: Homepage, https://pulpproject.org
|
|
7
|
+
Project-URL: Documentation, https://pulpproject.org/pulp_python/
|
|
8
|
+
Project-URL: Repository, https://github.com/pulp/pulp_python
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/pulp/pulp_python/issues
|
|
10
|
+
Project-URL: Changelog, https://pulpproject.org/pulp_python/changes/
|
|
9
11
|
Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)
|
|
10
12
|
Classifier: Operating System :: POSIX :: Linux
|
|
11
13
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -13,9 +15,16 @@ Classifier: Framework :: Django
|
|
|
13
15
|
Classifier: Programming Language :: Python
|
|
14
16
|
Classifier: Programming Language :: Python :: 3
|
|
15
17
|
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
21
|
Requires-Python: >=3.9
|
|
17
22
|
Description-Content-Type: text/markdown
|
|
18
23
|
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pulpcore<3.85,>=3.49.0
|
|
25
|
+
Requires-Dist: pkginfo<1.12.0,>=1.10.0
|
|
26
|
+
Requires-Dist: bandersnatch<7.0,>=6.3
|
|
27
|
+
Requires-Dist: pypi-simple<2.0,>=1.5.0
|
|
19
28
|
|
|
20
29
|
# pulp_python
|
|
21
30
|
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import os
|
|
3
|
+
from django.core.management import BaseCommand, CommandError
|
|
4
|
+
from gettext import gettext as _
|
|
5
|
+
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
|
|
8
|
+
from pulpcore.plugin.util import extract_pk
|
|
9
|
+
from pulp_python.app.models import PythonPackageContent, PythonRepository
|
|
10
|
+
from pulp_python.app.utils import artifact_to_python_content_data
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def repair_metadata(content):
|
|
14
|
+
"""
|
|
15
|
+
Repairs the metadata for the passed in content queryset.
|
|
16
|
+
:param content: The PythonPackageContent queryset.
|
|
17
|
+
Return: number of content units that were repaired
|
|
18
|
+
"""
|
|
19
|
+
# TODO: Add on_demand content repair?
|
|
20
|
+
os.chdir(settings.WORKING_DIRECTORY)
|
|
21
|
+
content = content.select_related("pulp_domain")
|
|
22
|
+
immediate_content = content.filter(contentartifact__artifact__isnull=False)
|
|
23
|
+
batch = []
|
|
24
|
+
set_of_update_fields = set()
|
|
25
|
+
total_repaired = 0
|
|
26
|
+
for package in immediate_content.prefetch_related('_artifacts').iterator(chunk_size=1000):
|
|
27
|
+
new_data = artifact_to_python_content_data(
|
|
28
|
+
package.filename, package._artifacts.get(), package.pulp_domain
|
|
29
|
+
)
|
|
30
|
+
changed = False
|
|
31
|
+
for field, value in new_data.items():
|
|
32
|
+
if getattr(package, field) != value:
|
|
33
|
+
setattr(package, field, value)
|
|
34
|
+
set_of_update_fields.add(field)
|
|
35
|
+
changed = True
|
|
36
|
+
if changed:
|
|
37
|
+
batch.append(package)
|
|
38
|
+
if len(batch) == 1000:
|
|
39
|
+
total_repaired += len(batch)
|
|
40
|
+
PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
|
|
41
|
+
batch = []
|
|
42
|
+
set_of_update_fields.clear()
|
|
43
|
+
|
|
44
|
+
if len(batch) > 0:
|
|
45
|
+
total_repaired += len(batch)
|
|
46
|
+
PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
|
|
47
|
+
|
|
48
|
+
return total_repaired
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def href_prn_list_handler(value):
|
|
52
|
+
"""Common list parsing for a string of hrefs/prns."""
|
|
53
|
+
r = re.compile(
|
|
54
|
+
rf"""
|
|
55
|
+
(?:{settings.API_ROOT}(?:[-_a-zA-Z0-9]+/)?api/v3/repositories/python/python/[-a-f0-9]+/)
|
|
56
|
+
|(?:prn:python\.pythonrepository:[-a-f0-9]+)
|
|
57
|
+
""",
|
|
58
|
+
re.VERBOSE
|
|
59
|
+
)
|
|
60
|
+
values = []
|
|
61
|
+
for v in value.split(","):
|
|
62
|
+
if v:
|
|
63
|
+
if match := r.match(v.strip()):
|
|
64
|
+
values.append(match.group(0))
|
|
65
|
+
else:
|
|
66
|
+
raise CommandError(f"Invalid href/prn: {v}")
|
|
67
|
+
return values
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Command(BaseCommand):
|
|
71
|
+
"""
|
|
72
|
+
Management command to repair metadata of PythonPackageContent.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
help = _("Repair the metadata of PythonPackageContent stored in PythonRepositories")
|
|
76
|
+
|
|
77
|
+
def add_arguments(self, parser):
|
|
78
|
+
"""Set up arguments."""
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--repositories",
|
|
81
|
+
type=href_prn_list_handler,
|
|
82
|
+
required=False,
|
|
83
|
+
help=_(
|
|
84
|
+
"List of PythonRepository hrefs/prns whose content's metadata will be repaired. "
|
|
85
|
+
"Leave blank to include all repositories in all domains. Mutually exclusive "
|
|
86
|
+
"with domain."
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
parser.add_argument(
|
|
90
|
+
"--domain",
|
|
91
|
+
default=None,
|
|
92
|
+
required=False,
|
|
93
|
+
help=_(
|
|
94
|
+
"The pulp domain to gather the repositories from if specified. Mutually"
|
|
95
|
+
" exclusive with repositories."
|
|
96
|
+
),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def handle(self, *args, **options):
|
|
100
|
+
"""Implement the command."""
|
|
101
|
+
domain = options.get("domain")
|
|
102
|
+
repository_hrefs = options.get("repositories")
|
|
103
|
+
if domain and repository_hrefs:
|
|
104
|
+
raise CommandError(_("--domain and --repositories are mutually exclusive"))
|
|
105
|
+
|
|
106
|
+
repositories = PythonRepository.objects.all()
|
|
107
|
+
if repository_hrefs:
|
|
108
|
+
repos_ids = [extract_pk(r) for r in repository_hrefs]
|
|
109
|
+
repositories = repositories.filter(pk__in=repos_ids)
|
|
110
|
+
elif domain:
|
|
111
|
+
repositories = repositories.filter(pulp_domain__name=domain)
|
|
112
|
+
|
|
113
|
+
content_set = set()
|
|
114
|
+
for repository in repositories:
|
|
115
|
+
content_set.update(repository.latest_version().content.values_list("pk", flat=True))
|
|
116
|
+
content = PythonPackageContent.objects.filter(pk__in=content_set)
|
|
117
|
+
num_repaired = repair_metadata(content)
|
|
118
|
+
print(f"{len(content_set)} packages processed, {num_repaired} package metadata repaired.")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Generated by Django
|
|
1
|
+
# Generated by Django 4.2.16 on 2024-12-11 16:42
|
|
2
2
|
|
|
3
3
|
import django.contrib.postgres.fields.jsonb
|
|
4
4
|
from django.db import migrations, models
|
|
@@ -10,14 +10,14 @@ class Migration(migrations.Migration):
|
|
|
10
10
|
initial = True
|
|
11
11
|
|
|
12
12
|
dependencies = [
|
|
13
|
-
('core', '
|
|
13
|
+
('core', '0091_systemid'),
|
|
14
14
|
]
|
|
15
15
|
|
|
16
16
|
operations = [
|
|
17
17
|
migrations.CreateModel(
|
|
18
18
|
name='PythonPublication',
|
|
19
19
|
fields=[
|
|
20
|
-
('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.
|
|
20
|
+
('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.publication')),
|
|
21
21
|
],
|
|
22
22
|
options={
|
|
23
23
|
'default_related_name': '%(app_label)s_%(model_name)s',
|
|
@@ -27,7 +27,7 @@ class Migration(migrations.Migration):
|
|
|
27
27
|
migrations.CreateModel(
|
|
28
28
|
name='PythonRemote',
|
|
29
29
|
fields=[
|
|
30
|
-
('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.
|
|
30
|
+
('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.remote')),
|
|
31
31
|
('prereleases', models.BooleanField(default=False)),
|
|
32
32
|
('includes', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
|
|
33
33
|
('excludes', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
|
|
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
|
|
|
40
40
|
migrations.CreateModel(
|
|
41
41
|
name='PythonRepository',
|
|
42
42
|
fields=[
|
|
43
|
-
('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.
|
|
43
|
+
('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.repository')),
|
|
44
44
|
],
|
|
45
45
|
options={
|
|
46
46
|
'default_related_name': '%(app_label)s_%(model_name)s',
|
|
@@ -50,7 +50,7 @@ class Migration(migrations.Migration):
|
|
|
50
50
|
migrations.CreateModel(
|
|
51
51
|
name='PythonPackageContent',
|
|
52
52
|
fields=[
|
|
53
|
-
('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.
|
|
53
|
+
('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.content')),
|
|
54
54
|
('filename', models.TextField(db_index=True, unique=True)),
|
|
55
55
|
('packagetype', models.TextField(choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')])),
|
|
56
56
|
('name', models.TextField()),
|
|
@@ -85,8 +85,8 @@ class Migration(migrations.Migration):
|
|
|
85
85
|
migrations.CreateModel(
|
|
86
86
|
name='PythonDistribution',
|
|
87
87
|
fields=[
|
|
88
|
-
('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.
|
|
89
|
-
('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.
|
|
88
|
+
('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.basedistribution')),
|
|
89
|
+
('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.publication')),
|
|
90
90
|
],
|
|
91
91
|
options={
|
|
92
92
|
'default_related_name': '%(app_label)s_%(model_name)s',
|
|
@@ -17,9 +17,8 @@ from pulpcore.plugin.responses import ArtifactResponse
|
|
|
17
17
|
|
|
18
18
|
from pathlib import PurePath
|
|
19
19
|
from .utils import (
|
|
20
|
+
artifact_to_python_content_data,
|
|
20
21
|
canonicalize_name,
|
|
21
|
-
get_project_metadata_from_artifact,
|
|
22
|
-
parse_project_metadata,
|
|
23
22
|
python_content_to_json,
|
|
24
23
|
PYPI_LAST_SERIAL,
|
|
25
24
|
PYPI_SERIAL_CONSTANT,
|
|
@@ -189,14 +188,7 @@ class PythonPackageContent(Content):
|
|
|
189
188
|
def init_from_artifact_and_relative_path(artifact, relative_path):
|
|
190
189
|
"""Used when downloading package from pull-through cache."""
|
|
191
190
|
path = PurePath(relative_path)
|
|
192
|
-
|
|
193
|
-
data = parse_project_metadata(vars(metadata))
|
|
194
|
-
data["packagetype"] = metadata.packagetype
|
|
195
|
-
data["version"] = metadata.version
|
|
196
|
-
data["filename"] = path.name
|
|
197
|
-
data["sha256"] = artifact.sha256
|
|
198
|
-
data["pulp_domain_id"] = artifact.pulp_domain_id
|
|
199
|
-
data["_pulp_domain_id"] = artifact.pulp_domain_id
|
|
191
|
+
data = artifact_to_python_content_data(path.name, artifact, domain=get_domain())
|
|
200
192
|
return PythonPackageContent(**data)
|
|
201
193
|
|
|
202
194
|
def __str__(self):
|
|
@@ -23,7 +23,7 @@ from itertools import chain
|
|
|
23
23
|
from packaging.utils import canonicalize_name
|
|
24
24
|
from urllib.parse import urljoin, urlparse, urlunsplit
|
|
25
25
|
from pathlib import PurePath
|
|
26
|
-
from pypi_simple
|
|
26
|
+
from pypi_simple import parse_links_stream_response
|
|
27
27
|
|
|
28
28
|
from pulpcore.plugin.viewsets import OperationPostponedResponse
|
|
29
29
|
from pulpcore.plugin.tasking import dispatch
|
|
@@ -8,7 +8,7 @@ from pulpcore.plugin import serializers as core_serializers
|
|
|
8
8
|
from pulpcore.plugin.util import get_domain
|
|
9
9
|
|
|
10
10
|
from pulp_python.app import models as python_models
|
|
11
|
-
from pulp_python.app.utils import
|
|
11
|
+
from pulp_python.app.utils import artifact_to_python_content_data
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class PythonRepositorySerializer(core_serializers.RepositorySerializer):
|
|
@@ -216,7 +216,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
|
|
|
216
216
|
|
|
217
217
|
artifact = data["artifact"]
|
|
218
218
|
try:
|
|
219
|
-
|
|
219
|
+
_data = artifact_to_python_content_data(filename, artifact, domain=get_domain())
|
|
220
220
|
except ValueError:
|
|
221
221
|
raise serializers.ValidationError(_(
|
|
222
222
|
"Extension on {} is not a valid python extension "
|
|
@@ -230,14 +230,6 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
|
|
|
230
230
|
)}
|
|
231
231
|
)
|
|
232
232
|
|
|
233
|
-
_data = parse_project_metadata(vars(metadata))
|
|
234
|
-
_data['packagetype'] = metadata.packagetype
|
|
235
|
-
_data['version'] = metadata.version
|
|
236
|
-
_data['filename'] = filename
|
|
237
|
-
_data['sha256'] = artifact.sha256
|
|
238
|
-
data["pulp_domain_id"] = artifact.pulp_domain_id
|
|
239
|
-
data["_pulp_domain_id"] = artifact.pulp_domain_id
|
|
240
|
-
|
|
241
233
|
data.update(_data)
|
|
242
234
|
|
|
243
235
|
return data
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import tempfile
|
|
3
|
-
from typing import Optional, Any, AsyncGenerator
|
|
4
2
|
|
|
5
|
-
import aiohttp
|
|
6
3
|
from aiohttp import ClientResponseError, ClientError
|
|
7
4
|
from lxml.etree import LxmlError
|
|
8
5
|
from gettext import gettext as _
|
|
9
|
-
from os import environ
|
|
10
6
|
|
|
11
7
|
from rest_framework import serializers
|
|
12
8
|
|
|
@@ -23,13 +19,13 @@ from pulp_python.app.models import (
|
|
|
23
19
|
PythonRemote,
|
|
24
20
|
)
|
|
25
21
|
from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL
|
|
26
|
-
from pypi_simple import
|
|
22
|
+
from pypi_simple import IndexPage
|
|
27
23
|
|
|
28
24
|
from bandersnatch.mirror import Mirror
|
|
29
25
|
from bandersnatch.master import Master
|
|
30
26
|
from bandersnatch.configuration import BandersnatchConfig
|
|
31
27
|
from packaging.requirements import Requirement
|
|
32
|
-
from urllib.parse import urljoin
|
|
28
|
+
from urllib.parse import urljoin
|
|
33
29
|
|
|
34
30
|
logger = logging.getLogger(__name__)
|
|
35
31
|
|
|
@@ -114,23 +110,14 @@ class PythonBanderStage(Stage):
|
|
|
114
110
|
"""
|
|
115
111
|
If includes is specified, then only sync those,else try to sync all other packages
|
|
116
112
|
"""
|
|
117
|
-
# Prevent bandersnatch from reading actual .netrc file, set to empty file
|
|
118
|
-
# See discussion on https://github.com/pulp/pulp_python/issues/581
|
|
119
|
-
fake_netrc = tempfile.NamedTemporaryFile(dir=".", delete=False)
|
|
120
|
-
environ["NETRC"] = fake_netrc.name
|
|
121
|
-
# TODO Change Bandersnatch internal API to take proxy settings in from config parameters
|
|
122
|
-
if proxy_url := self.remote.proxy_url:
|
|
123
|
-
if self.remote.proxy_username or self.remote.proxy_password:
|
|
124
|
-
parsed_proxy = urlsplit(proxy_url)
|
|
125
|
-
creds = f"{self.remote.proxy_username}:{self.remote.proxy_password}"
|
|
126
|
-
netloc = f"{creds}@{parsed_proxy.netloc}"
|
|
127
|
-
proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
|
|
128
|
-
environ['http_proxy'] = proxy_url
|
|
129
|
-
environ['https_proxy'] = proxy_url
|
|
130
113
|
# Bandersnatch includes leading slash when forming API urls
|
|
131
114
|
url = self.remote.url.rstrip("/")
|
|
132
|
-
|
|
133
|
-
|
|
115
|
+
async with Master(url) as master:
|
|
116
|
+
# Replace the session with the remote's downloader session
|
|
117
|
+
old_session = master.session
|
|
118
|
+
factory = self.remote.download_factory
|
|
119
|
+
master.session = factory._session
|
|
120
|
+
|
|
134
121
|
deferred_download = self.remote.policy != Remote.IMMEDIATE
|
|
135
122
|
workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
|
|
136
123
|
async with ProgressReport(
|
|
@@ -150,25 +137,8 @@ class PythonBanderStage(Stage):
|
|
|
150
137
|
Requirement(pkg).name for pkg in self.remote.includes
|
|
151
138
|
]
|
|
152
139
|
await pmirror.synchronize(packages_to_sync)
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
class PulpMaster(Master):
|
|
156
|
-
"""
|
|
157
|
-
Pulp Master Class for Pulp specific overrides
|
|
158
|
-
"""
|
|
159
|
-
|
|
160
|
-
def __init__(self, *args, tls=True, **kwargs):
|
|
161
|
-
self.tls = tls
|
|
162
|
-
super().__init__(*args, **kwargs)
|
|
163
|
-
|
|
164
|
-
async def get(
|
|
165
|
-
self, path: str, required_serial: Optional[int], **kw: Any
|
|
166
|
-
) -> AsyncGenerator[aiohttp.ClientResponse, None]:
|
|
167
|
-
"""Support tls=false"""
|
|
168
|
-
if not self.tls:
|
|
169
|
-
kw["ssl"] = False
|
|
170
|
-
async for r in super().get(path, required_serial, **kw):
|
|
171
|
-
yield r
|
|
140
|
+
# place back old session so that it is properly closed
|
|
141
|
+
master.session = old_session
|
|
172
142
|
|
|
173
143
|
|
|
174
144
|
class PulpMirror(Mirror):
|
|
@@ -226,7 +196,7 @@ class PulpMirror(Mirror):
|
|
|
226
196
|
downloader = self.python_stage.remote.get_downloader(url=url)
|
|
227
197
|
result = await downloader.run()
|
|
228
198
|
with open(result.path) as f:
|
|
229
|
-
index =
|
|
199
|
+
index = IndexPage.from_html(f.read())
|
|
230
200
|
self.packages_to_sync.update({p: 0 for p in index.projects})
|
|
231
201
|
self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
|
|
232
202
|
|
|
@@ -7,7 +7,7 @@ from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact
|
|
|
7
7
|
from pulpcore.plugin.util import get_domain
|
|
8
8
|
|
|
9
9
|
from pulp_python.app.models import PythonPackageContent, PythonRepository
|
|
10
|
-
from pulp_python.app.utils import
|
|
10
|
+
from pulp_python.app.utils import artifact_to_python_content_data
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def upload(artifact_sha256, filename, repository_pk=None):
|
|
@@ -76,15 +76,7 @@ def create_content(artifact_sha256, filename, domain):
|
|
|
76
76
|
queryset of the new created content
|
|
77
77
|
"""
|
|
78
78
|
artifact = Artifact.objects.get(sha256=artifact_sha256, pulp_domain=domain)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
data = parse_project_metadata(vars(metadata))
|
|
82
|
-
data['packagetype'] = metadata.packagetype
|
|
83
|
-
data['version'] = metadata.version
|
|
84
|
-
data['filename'] = filename
|
|
85
|
-
data['sha256'] = artifact.sha256
|
|
86
|
-
data['pulp_domain'] = domain
|
|
87
|
-
data['_pulp_domain'] = domain
|
|
79
|
+
data = artifact_to_python_content_data(filename, artifact, domain)
|
|
88
80
|
|
|
89
81
|
@transaction.atomic()
|
|
90
82
|
def create():
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import pkginfo
|
|
2
|
+
import re
|
|
2
3
|
import shutil
|
|
3
4
|
import tempfile
|
|
4
5
|
import json
|
|
@@ -51,6 +52,20 @@ DIST_EXTENSIONS = {
|
|
|
51
52
|
".zip": "sdist",
|
|
52
53
|
}
|
|
53
54
|
|
|
55
|
+
DIST_REGEXES = {
|
|
56
|
+
# regex from https://github.com/pypa/pip/blob/18.0/src/pip/_internal/wheel.py#L569
|
|
57
|
+
".whl": re.compile(
|
|
58
|
+
r"""^(?P<name>.+?)-(?P<version>.*?)
|
|
59
|
+
((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
|
|
60
|
+
\.whl|\.dist-info)$""",
|
|
61
|
+
re.VERBOSE
|
|
62
|
+
),
|
|
63
|
+
# regex based on https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#filename-embedded-metadata # noqa: E501
|
|
64
|
+
".egg": re.compile(r"^(?P<name>.+?)-(?P<version>.*?)(-(?P<pyver>.+?(-(?P<plat>.+?))?))?\.egg|\.egg-info$"), # noqa: E501
|
|
65
|
+
# regex based on https://github.com/python/cpython/blob/v3.7.0/Lib/distutils/command/bdist_wininst.py#L292 # noqa: E501
|
|
66
|
+
".exe": re.compile(r"^(?P<name>.+?)-(?P<version>.*?)\.(?P<plat>.+?)(-(?P<pyver>.+?))?\.exe$"),
|
|
67
|
+
}
|
|
68
|
+
|
|
54
69
|
DIST_TYPES = {
|
|
55
70
|
"bdist_wheel": pkginfo.Wheel,
|
|
56
71
|
"bdist_wininst": pkginfo.Distribution,
|
|
@@ -72,6 +87,8 @@ def parse_project_metadata(project):
|
|
|
72
87
|
"""
|
|
73
88
|
package = {}
|
|
74
89
|
package['name'] = project.get('name') or ""
|
|
90
|
+
package['version'] = project.get('version') or ""
|
|
91
|
+
package['packagetype'] = project.get('packagetype') or ""
|
|
75
92
|
package['metadata_version'] = project.get('metadata_version') or ""
|
|
76
93
|
package['summary'] = project.get('summary') or ""
|
|
77
94
|
package['description'] = project.get('description') or ""
|
|
@@ -86,6 +103,7 @@ def parse_project_metadata(project):
|
|
|
86
103
|
package['project_url'] = project.get('project_url') or ""
|
|
87
104
|
package['platform'] = project.get('platform') or ""
|
|
88
105
|
package['supported_platform'] = project.get('supported_platform') or ""
|
|
106
|
+
package['requires_python'] = project.get('requires_python') or ""
|
|
89
107
|
package['requires_dist'] = json.dumps(project.get('requires_dist', []))
|
|
90
108
|
package['provides_dist'] = json.dumps(project.get('provides_dist', []))
|
|
91
109
|
package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', []))
|
|
@@ -93,6 +111,7 @@ def parse_project_metadata(project):
|
|
|
93
111
|
package['classifiers'] = json.dumps(project.get('classifiers', []))
|
|
94
112
|
package['project_urls'] = json.dumps(project.get('project_urls', {}))
|
|
95
113
|
package['description_content_type'] = project.get('description_content_type') or ""
|
|
114
|
+
package['python_version'] = project.get('python_version') or ""
|
|
96
115
|
|
|
97
116
|
return package
|
|
98
117
|
|
|
@@ -113,17 +132,15 @@ def parse_metadata(project, version, distribution):
|
|
|
113
132
|
dictionary: of useful python metadata
|
|
114
133
|
|
|
115
134
|
"""
|
|
116
|
-
package =
|
|
135
|
+
package = parse_project_metadata(project)
|
|
117
136
|
|
|
118
137
|
package['filename'] = distribution.get('filename') or ""
|
|
119
138
|
package['packagetype'] = distribution.get('packagetype') or ""
|
|
120
139
|
package['version'] = version
|
|
121
140
|
package['url'] = distribution.get('url') or ""
|
|
122
141
|
package['sha256'] = distribution.get('digests', {}).get('sha256') or ""
|
|
123
|
-
package['python_version'] = distribution.get('python_version') or
|
|
124
|
-
package['requires_python'] = distribution.get('requires_python') or
|
|
125
|
-
|
|
126
|
-
package.update(parse_project_metadata(project))
|
|
142
|
+
package['python_version'] = distribution.get('python_version') or package.get('python_version')
|
|
143
|
+
package['requires_python'] = distribution.get('requires_python') or package.get('requires_python') # noqa: E501
|
|
127
144
|
|
|
128
145
|
return package
|
|
129
146
|
|
|
@@ -147,9 +164,30 @@ def get_project_metadata_from_artifact(filename, artifact):
|
|
|
147
164
|
temp_file.flush()
|
|
148
165
|
metadata = DIST_TYPES[packagetype](temp_file.name)
|
|
149
166
|
metadata.packagetype = packagetype
|
|
167
|
+
if packagetype == "sdist":
|
|
168
|
+
metadata.python_version = "source"
|
|
169
|
+
else:
|
|
170
|
+
pyver = ""
|
|
171
|
+
regex = DIST_REGEXES[extensions[pkg_type_index]]
|
|
172
|
+
if bdist_name := regex.match(filename):
|
|
173
|
+
pyver = bdist_name.group("pyver") or ""
|
|
174
|
+
metadata.python_version = pyver
|
|
150
175
|
return metadata
|
|
151
176
|
|
|
152
177
|
|
|
178
|
+
def artifact_to_python_content_data(filename, artifact, domain=None):
|
|
179
|
+
"""
|
|
180
|
+
Takes the artifact/filename and returns the metadata needed to create a PythonPackageContent.
|
|
181
|
+
"""
|
|
182
|
+
metadata = get_project_metadata_from_artifact(filename, artifact)
|
|
183
|
+
data = parse_project_metadata(vars(metadata))
|
|
184
|
+
data['sha256'] = artifact.sha256
|
|
185
|
+
data['filename'] = filename
|
|
186
|
+
data['pulp_domain'] = domain or artifact.pulp_domain
|
|
187
|
+
data['_pulp_domain'] = data['pulp_domain']
|
|
188
|
+
return data
|
|
189
|
+
|
|
190
|
+
|
|
153
191
|
def python_content_to_json(base_path, content_query, version=None, domain=None):
|
|
154
192
|
"""
|
|
155
193
|
Converts a QuerySet of PythonPackageContent into the PyPi JSON format
|
|
File without changes
|