pulp-python 3.12.4__tar.gz → 3.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {pulp-python-3.12.4 → pulp_python-3.13.0}/CHANGES.md +60 -0
  2. {pulp-python-3.12.4 → pulp_python-3.13.0}/MANIFEST.in +2 -0
  3. {pulp-python-3.12.4 → pulp_python-3.13.0}/PKG-INFO +15 -6
  4. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/__init__.py +1 -1
  5. pulp_python-3.13.0/pulp_python/app/management/commands/repair-python-metadata.py +118 -0
  6. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0001_initial.py +8 -8
  7. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +0 -1
  8. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/models.py +2 -10
  9. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/pypi/views.py +1 -1
  10. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/serializers.py +2 -10
  11. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/sync.py +11 -41
  12. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/upload.py +2 -10
  13. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/utils.py +43 -5
  14. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/pytest_plugin.py +1 -1
  15. pulp_python-3.13.0/pulp_python/tests/functional/api/__init__.py +0 -0
  16. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_content_unit.py +27 -15
  17. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_remotes.py +21 -14
  18. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_domains.py +5 -14
  19. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_full_mirror.py +4 -5
  20. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_rbac.py +16 -8
  21. pulp_python-3.13.0/pulp_python/tests/functional/api/test_repair.py +78 -0
  22. pulp_python-3.13.0/pulp_python/tests/unit/__init__.py +0 -0
  23. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/PKG-INFO +15 -6
  24. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/SOURCES.txt +4 -2
  25. pulp_python-3.13.0/pulp_python.egg-info/requires.txt +4 -0
  26. pulp_python-3.13.0/pulp_python.egg-info/top_level.txt +4 -0
  27. pulp_python-3.13.0/pyproject.toml +124 -0
  28. pulp-python-3.12.4/pulp_python.egg-info/requires.txt +0 -4
  29. pulp-python-3.12.4/pulp_python.egg-info/top_level.txt +0 -1
  30. pulp-python-3.12.4/pyproject.toml +0 -30
  31. pulp-python-3.12.4/requirements.txt +0 -4
  32. pulp-python-3.12.4/setup.py +0 -38
  33. {pulp-python-3.12.4 → pulp_python-3.13.0}/COMMITMENT +0 -0
  34. {pulp-python-3.12.4 → pulp_python-3.13.0}/COPYRIGHT +0 -0
  35. {pulp-python-3.12.4 → pulp_python-3.13.0}/LICENSE +0 -0
  36. {pulp-python-3.12.4 → pulp_python-3.13.0}/README.md +0 -0
  37. {pulp-python-3.12.4 → pulp_python-3.13.0}/functest_requirements.txt +0 -0
  38. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/__init__.py +0 -0
  39. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/global_access_conditions.py +0 -0
  40. {pulp-python-3.12.4/pulp_python/app/migrations → pulp_python-3.13.0/pulp_python/app/management}/__init__.py +0 -0
  41. {pulp-python-3.12.4/pulp_python/app/pypi → pulp_python-3.13.0/pulp_python/app/management/commands}/__init__.py +0 -0
  42. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +0 -0
  43. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0003_new_sync_filters.py +0 -0
  44. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +0 -0
  45. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +0 -0
  46. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +0 -0
  47. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +0 -0
  48. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +0 -0
  49. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0010_update_json_field.py +0 -0
  50. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +0 -0
  51. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0012_add_domain.py +0 -0
  52. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/migrations/0013_add_rbac_permissions.py +0 -0
  53. {pulp-python-3.12.4/pulp_python/app/webserver_snippets → pulp_python-3.13.0/pulp_python/app/migrations}/__init__.py +0 -0
  54. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/modelresource.py +0 -0
  55. {pulp-python-3.12.4/pulp_python/tests/functional → pulp_python-3.13.0/pulp_python/app/pypi}/__init__.py +0 -0
  56. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/pypi/serializers.py +0 -0
  57. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/replica.py +0 -0
  58. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/settings.py +0 -0
  59. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/__init__.py +0 -0
  60. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/tasks/publish.py +0 -0
  61. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/urls.py +0 -0
  62. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/viewsets.py +0 -0
  63. {pulp-python-3.12.4/pulp_python/tests/functional/api → pulp_python-3.13.0/pulp_python/app/webserver_snippets}/__init__.py +0 -0
  64. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/webserver_snippets/apache.conf +0 -0
  65. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/app/webserver_snippets/nginx.conf +0 -0
  66. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/__init__.py +0 -0
  67. {pulp-python-3.12.4/pulp_python/tests/unit → pulp_python-3.13.0/pulp_python/tests/functional}/__init__.py +0 -0
  68. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_auto_publish.py +0 -0
  69. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_consume_content.py +0 -0
  70. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_crud_publications.py +0 -0
  71. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_download_content.py +0 -0
  72. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_export_import.py +0 -0
  73. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_pypi_apis.py +0 -0
  74. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/api/test_sync.py +0 -0
  75. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/constants.py +0 -0
  76. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/functional/utils.py +0 -0
  77. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python/tests/unit/test_models.py +0 -0
  78. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/dependency_links.txt +0 -0
  79. {pulp-python-3.12.4 → pulp_python-3.13.0}/pulp_python.egg-info/entry_points.txt +0 -0
  80. {pulp-python-3.12.4 → pulp_python-3.13.0}/setup.cfg +0 -0
  81. {pulp-python-3.12.4 → pulp_python-3.13.0}/test_requirements.txt +0 -0
  82. {pulp-python-3.12.4 → pulp_python-3.13.0}/unittest_requirements.txt +0 -0
@@ -8,6 +8,45 @@
8
8
 
9
9
  [//]: # (towncrier release notes start)
10
10
 
11
+ ## 3.13.0 (2025-02-05) {: #3.13.0 }
12
+
13
+ #### Features {: #3.13.0-feature }
14
+
15
+ - Added pulpcore 3.70 compatibility
16
+
17
+ #### Bugfixes {: #3.13.0-bugfix }
18
+
19
+ - Fixed uploads not supporting packages using metadata spec 2.3
20
+ [#682](https://github.com/pulp/pulp_python/issues/682)
21
+ - Fixed the `package_types` filter breaking other remote filters.
22
+ [#691](https://github.com/pulp/pulp_python/issues/691)
23
+ - Fixed package name normalization issue preventing syncing packages with "." or "_" in their names.
24
+ [#716](https://github.com/pulp/pulp_python/issues/716)
25
+ - Fixed replicate failing on upstream on-demand repositories
26
+ [#718](https://github.com/pulp/pulp_python/issues/718)
27
+ - Fixed `requires_python` field not being properly set on package upload.
28
+
29
+ Run the new `pulpcore-manager repair-python-metadata` command with repositories containing affected
30
+ packages to repair their metadata.
31
+ [#773](https://github.com/pulp/pulp_python/issues/773)
32
+ - Fixed the JSONField specification so it doesn't break ruby bindings.
33
+ See context [here](https://github.com/pulp/pulp_rpm/issues/3639).
34
+
35
+ #### Misc {: #3.13.0-misc }
36
+
37
+ - [#774](https://github.com/pulp/pulp_python/issues/774)
38
+
39
+ ---
40
+
41
+ ## 3.12.5 (2024-10-25) {: #3.12.5 }
42
+
43
+ #### Bugfixes {: #3.12.5-bugfix }
44
+
45
+ - Fixed the JSONField specification so it doesn't break ruby bindings.
46
+ See context [here](https://github.com/pulp/pulp_rpm/issues/3639).
47
+
48
+ ---
49
+
11
50
  ## 3.12.4 (2024-10-14) {: #3.12.4 }
12
51
 
13
52
  #### Bugfixes {: #3.12.4-bugfix }
@@ -73,6 +112,27 @@
73
112
 
74
113
  ---
75
114
 
115
+ ## 3.11.3 (2024-08-21) {: #3.11.3 }
116
+
117
+ #### Bugfixes {: #3.11.3-bugfix }
118
+
119
+ - Fixed uploads not supporting packages using metadata spec 2.3
120
+ [#682](https://github.com/pulp/pulp_python/issues/682)
121
+ - Fixed package name normalization issue preventing syncing packages with "." or "_" in their names.
122
+ [#716](https://github.com/pulp/pulp_python/issues/716)
123
+
124
+ ---
125
+
126
+ ## 3.11.2 (2024-06-27) {: #3.11.2 }
127
+
128
+
129
+ #### Bugfixes {: #3.11.2-bugfix }
130
+
131
+ - Fixed the `package_types` filter breaking other remote filters.
132
+ [#691](https://github.com/pulp/pulp_python/issues/691)
133
+
134
+ ---
135
+
76
136
  ## 3.11.1 (2024-04-11) {: #3.11.1 }
77
137
 
78
138
  ### Bugfixes
@@ -3,8 +3,10 @@ include requirements.txt
3
3
  include pyproject.toml
4
4
  include CHANGES.md
5
5
  include COMMITMENT
6
+ exclude CONTRIBUTING.md
6
7
  include COPYRIGHT
7
8
  include functest_requirements.txt
8
9
  include test_requirements.txt
9
10
  include unittest_requirements.txt
10
11
  include pulp_python/app/webserver_snippets/*
12
+ exclude releasing.md
@@ -1,11 +1,13 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: pulp-python
3
- Version: 3.12.4
3
+ Version: 3.13.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
- Home-page: https://www.pulpproject.org
6
- Author: Pulp Project Developers
7
- Author-email: pulp-list@redhat.com
8
- License: GPLv2+
5
+ Author-email: Pulp Team <pulp-list@redhat.com>
6
+ Project-URL: Homepage, https://pulpproject.org
7
+ Project-URL: Documentation, https://pulpproject.org/pulp_python/
8
+ Project-URL: Repository, https://github.com/pulp/pulp_python
9
+ Project-URL: Bug Tracker, https://github.com/pulp/pulp_python/issues
10
+ Project-URL: Changelog, https://pulpproject.org/pulp_python/changes/
9
11
  Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)
10
12
  Classifier: Operating System :: POSIX :: Linux
11
13
  Classifier: Development Status :: 5 - Production/Stable
@@ -13,9 +15,16 @@ Classifier: Framework :: Django
13
15
  Classifier: Programming Language :: Python
14
16
  Classifier: Programming Language :: Python :: 3
15
17
  Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
16
21
  Requires-Python: >=3.9
17
22
  Description-Content-Type: text/markdown
18
23
  License-File: LICENSE
24
+ Requires-Dist: pulpcore<3.85,>=3.49.0
25
+ Requires-Dist: pkginfo<1.12.0,>=1.10.0
26
+ Requires-Dist: bandersnatch<7.0,>=6.3
27
+ Requires-Dist: pypi-simple<2.0,>=1.5.0
19
28
 
20
29
  # pulp_python
21
30
 
@@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
10
10
 
11
11
  name = "pulp_python.app"
12
12
  label = "python"
13
- version = "3.12.4"
13
+ version = "3.13.0"
14
14
  python_package_name = "pulp-python"
15
15
  domain_compatible = True
16
16
 
@@ -0,0 +1,118 @@
1
+ import re
2
+ import os
3
+ from django.core.management import BaseCommand, CommandError
4
+ from gettext import gettext as _
5
+
6
+ from django.conf import settings
7
+
8
+ from pulpcore.plugin.util import extract_pk
9
+ from pulp_python.app.models import PythonPackageContent, PythonRepository
10
+ from pulp_python.app.utils import artifact_to_python_content_data
11
+
12
+
13
+ def repair_metadata(content):
14
+ """
15
+ Repairs the metadata for the passed in content queryset.
16
+ :param content: The PythonPackageContent queryset.
17
+ Return: number of content units that were repaired
18
+ """
19
+ # TODO: Add on_demand content repair?
20
+ os.chdir(settings.WORKING_DIRECTORY)
21
+ content = content.select_related("pulp_domain")
22
+ immediate_content = content.filter(contentartifact__artifact__isnull=False)
23
+ batch = []
24
+ set_of_update_fields = set()
25
+ total_repaired = 0
26
+ for package in immediate_content.prefetch_related('_artifacts').iterator(chunk_size=1000):
27
+ new_data = artifact_to_python_content_data(
28
+ package.filename, package._artifacts.get(), package.pulp_domain
29
+ )
30
+ changed = False
31
+ for field, value in new_data.items():
32
+ if getattr(package, field) != value:
33
+ setattr(package, field, value)
34
+ set_of_update_fields.add(field)
35
+ changed = True
36
+ if changed:
37
+ batch.append(package)
38
+ if len(batch) == 1000:
39
+ total_repaired += len(batch)
40
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
41
+ batch = []
42
+ set_of_update_fields.clear()
43
+
44
+ if len(batch) > 0:
45
+ total_repaired += len(batch)
46
+ PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
47
+
48
+ return total_repaired
49
+
50
+
51
+ def href_prn_list_handler(value):
52
+ """Common list parsing for a string of hrefs/prns."""
53
+ r = re.compile(
54
+ rf"""
55
+ (?:{settings.API_ROOT}(?:[-_a-zA-Z0-9]+/)?api/v3/repositories/python/python/[-a-f0-9]+/)
56
+ |(?:prn:python\.pythonrepository:[-a-f0-9]+)
57
+ """,
58
+ re.VERBOSE
59
+ )
60
+ values = []
61
+ for v in value.split(","):
62
+ if v:
63
+ if match := r.match(v.strip()):
64
+ values.append(match.group(0))
65
+ else:
66
+ raise CommandError(f"Invalid href/prn: {v}")
67
+ return values
68
+
69
+
70
+ class Command(BaseCommand):
71
+ """
72
+ Management command to repair metadata of PythonPackageContent.
73
+ """
74
+
75
+ help = _("Repair the metadata of PythonPackageContent stored in PythonRepositories")
76
+
77
+ def add_arguments(self, parser):
78
+ """Set up arguments."""
79
+ parser.add_argument(
80
+ "--repositories",
81
+ type=href_prn_list_handler,
82
+ required=False,
83
+ help=_(
84
+ "List of PythonRepository hrefs/prns whose content's metadata will be repaired. "
85
+ "Leave blank to include all repositories in all domains. Mutually exclusive "
86
+ "with domain."
87
+ ),
88
+ )
89
+ parser.add_argument(
90
+ "--domain",
91
+ default=None,
92
+ required=False,
93
+ help=_(
94
+ "The pulp domain to gather the repositories from if specified. Mutually"
95
+ " exclusive with repositories."
96
+ ),
97
+ )
98
+
99
+ def handle(self, *args, **options):
100
+ """Implement the command."""
101
+ domain = options.get("domain")
102
+ repository_hrefs = options.get("repositories")
103
+ if domain and repository_hrefs:
104
+ raise CommandError(_("--domain and --repositories are mutually exclusive"))
105
+
106
+ repositories = PythonRepository.objects.all()
107
+ if repository_hrefs:
108
+ repos_ids = [extract_pk(r) for r in repository_hrefs]
109
+ repositories = repositories.filter(pk__in=repos_ids)
110
+ elif domain:
111
+ repositories = repositories.filter(pulp_domain__name=domain)
112
+
113
+ content_set = set()
114
+ for repository in repositories:
115
+ content_set.update(repository.latest_version().content.values_list("pk", flat=True))
116
+ content = PythonPackageContent.objects.filter(pk__in=content_set)
117
+ num_repaired = repair_metadata(content)
118
+ print(f"{len(content_set)} packages processed, {num_repaired} package metadata repaired.")
@@ -1,4 +1,4 @@
1
- # Generated by Django 2.2.3 on 2020-06-18 13:55
1
+ # Generated by Django 4.2.16 on 2024-12-11 16:42
2
2
 
3
3
  import django.contrib.postgres.fields.jsonb
4
4
  from django.db import migrations, models
@@ -10,14 +10,14 @@ class Migration(migrations.Migration):
10
10
  initial = True
11
11
 
12
12
  dependencies = [
13
- ('core', '0032_export_to_chunks'),
13
+ ('core', '0091_systemid'),
14
14
  ]
15
15
 
16
16
  operations = [
17
17
  migrations.CreateModel(
18
18
  name='PythonPublication',
19
19
  fields=[
20
- ('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.Publication')),
20
+ ('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.publication')),
21
21
  ],
22
22
  options={
23
23
  'default_related_name': '%(app_label)s_%(model_name)s',
@@ -27,7 +27,7 @@ class Migration(migrations.Migration):
27
27
  migrations.CreateModel(
28
28
  name='PythonRemote',
29
29
  fields=[
30
- ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.Remote')),
30
+ ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.remote')),
31
31
  ('prereleases', models.BooleanField(default=False)),
32
32
  ('includes', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
33
33
  ('excludes', django.contrib.postgres.fields.jsonb.JSONField(default=list)),
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
40
40
  migrations.CreateModel(
41
41
  name='PythonRepository',
42
42
  fields=[
43
- ('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.Repository')),
43
+ ('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.repository')),
44
44
  ],
45
45
  options={
46
46
  'default_related_name': '%(app_label)s_%(model_name)s',
@@ -50,7 +50,7 @@ class Migration(migrations.Migration):
50
50
  migrations.CreateModel(
51
51
  name='PythonPackageContent',
52
52
  fields=[
53
- ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.Content')),
53
+ ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.content')),
54
54
  ('filename', models.TextField(db_index=True, unique=True)),
55
55
  ('packagetype', models.TextField(choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')])),
56
56
  ('name', models.TextField()),
@@ -85,8 +85,8 @@ class Migration(migrations.Migration):
85
85
  migrations.CreateModel(
86
86
  name='PythonDistribution',
87
87
  fields=[
88
- ('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.BaseDistribution')),
89
- ('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.Publication')),
88
+ ('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.basedistribution')),
89
+ ('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.publication')),
90
90
  ],
91
91
  options={
92
92
  'default_related_name': '%(app_label)s_%(model_name)s',
@@ -48,7 +48,6 @@ class Migration(migrations.Migration):
48
48
  atomic = False
49
49
 
50
50
  dependencies = [
51
- ('core', '0062_add_new_distribution_mastermodel'),
52
51
  ('python', '0003_new_sync_filters'),
53
52
  ]
54
53
 
@@ -17,9 +17,8 @@ from pulpcore.plugin.responses import ArtifactResponse
17
17
 
18
18
  from pathlib import PurePath
19
19
  from .utils import (
20
+ artifact_to_python_content_data,
20
21
  canonicalize_name,
21
- get_project_metadata_from_artifact,
22
- parse_project_metadata,
23
22
  python_content_to_json,
24
23
  PYPI_LAST_SERIAL,
25
24
  PYPI_SERIAL_CONSTANT,
@@ -189,14 +188,7 @@ class PythonPackageContent(Content):
189
188
  def init_from_artifact_and_relative_path(artifact, relative_path):
190
189
  """Used when downloading package from pull-through cache."""
191
190
  path = PurePath(relative_path)
192
- metadata = get_project_metadata_from_artifact(path.name, artifact)
193
- data = parse_project_metadata(vars(metadata))
194
- data["packagetype"] = metadata.packagetype
195
- data["version"] = metadata.version
196
- data["filename"] = path.name
197
- data["sha256"] = artifact.sha256
198
- data["pulp_domain_id"] = artifact.pulp_domain_id
199
- data["_pulp_domain_id"] = artifact.pulp_domain_id
191
+ data = artifact_to_python_content_data(path.name, artifact, domain=get_domain())
200
192
  return PythonPackageContent(**data)
201
193
 
202
194
  def __str__(self):
@@ -23,7 +23,7 @@ from itertools import chain
23
23
  from packaging.utils import canonicalize_name
24
24
  from urllib.parse import urljoin, urlparse, urlunsplit
25
25
  from pathlib import PurePath
26
- from pypi_simple.parse_stream import parse_links_stream_response
26
+ from pypi_simple import parse_links_stream_response
27
27
 
28
28
  from pulpcore.plugin.viewsets import OperationPostponedResponse
29
29
  from pulpcore.plugin.tasking import dispatch
@@ -8,7 +8,7 @@ from pulpcore.plugin import serializers as core_serializers
8
8
  from pulpcore.plugin.util import get_domain
9
9
 
10
10
  from pulp_python.app import models as python_models
11
- from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata
11
+ from pulp_python.app.utils import artifact_to_python_content_data
12
12
 
13
13
 
14
14
  class PythonRepositorySerializer(core_serializers.RepositorySerializer):
@@ -216,7 +216,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
216
216
 
217
217
  artifact = data["artifact"]
218
218
  try:
219
- metadata = get_project_metadata_from_artifact(filename, artifact)
219
+ _data = artifact_to_python_content_data(filename, artifact, domain=get_domain())
220
220
  except ValueError:
221
221
  raise serializers.ValidationError(_(
222
222
  "Extension on {} is not a valid python extension "
@@ -230,14 +230,6 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
230
230
  )}
231
231
  )
232
232
 
233
- _data = parse_project_metadata(vars(metadata))
234
- _data['packagetype'] = metadata.packagetype
235
- _data['version'] = metadata.version
236
- _data['filename'] = filename
237
- _data['sha256'] = artifact.sha256
238
- data["pulp_domain_id"] = artifact.pulp_domain_id
239
- data["_pulp_domain_id"] = artifact.pulp_domain_id
240
-
241
233
  data.update(_data)
242
234
 
243
235
  return data
@@ -1,12 +1,8 @@
1
1
  import logging
2
- import tempfile
3
- from typing import Optional, Any, AsyncGenerator
4
2
 
5
- import aiohttp
6
3
  from aiohttp import ClientResponseError, ClientError
7
4
  from lxml.etree import LxmlError
8
5
  from gettext import gettext as _
9
- from os import environ
10
6
 
11
7
  from rest_framework import serializers
12
8
 
@@ -23,13 +19,13 @@ from pulp_python.app.models import (
23
19
  PythonRemote,
24
20
  )
25
21
  from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL
26
- from pypi_simple import parse_repo_index_page
22
+ from pypi_simple import IndexPage
27
23
 
28
24
  from bandersnatch.mirror import Mirror
29
25
  from bandersnatch.master import Master
30
26
  from bandersnatch.configuration import BandersnatchConfig
31
27
  from packaging.requirements import Requirement
32
- from urllib.parse import urljoin, urlsplit, urlunsplit
28
+ from urllib.parse import urljoin
33
29
 
34
30
  logger = logging.getLogger(__name__)
35
31
 
@@ -114,23 +110,14 @@ class PythonBanderStage(Stage):
114
110
  """
115
111
  If includes is specified, then only sync those,else try to sync all other packages
116
112
  """
117
- # Prevent bandersnatch from reading actual .netrc file, set to empty file
118
- # See discussion on https://github.com/pulp/pulp_python/issues/581
119
- fake_netrc = tempfile.NamedTemporaryFile(dir=".", delete=False)
120
- environ["NETRC"] = fake_netrc.name
121
- # TODO Change Bandersnatch internal API to take proxy settings in from config parameters
122
- if proxy_url := self.remote.proxy_url:
123
- if self.remote.proxy_username or self.remote.proxy_password:
124
- parsed_proxy = urlsplit(proxy_url)
125
- creds = f"{self.remote.proxy_username}:{self.remote.proxy_password}"
126
- netloc = f"{creds}@{parsed_proxy.netloc}"
127
- proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", ""))
128
- environ['http_proxy'] = proxy_url
129
- environ['https_proxy'] = proxy_url
130
113
  # Bandersnatch includes leading slash when forming API urls
131
114
  url = self.remote.url.rstrip("/")
132
- # local & global timeouts defaults to 10secs and 5 hours
133
- async with PulpMaster(url, tls=self.remote.tls_validation) as master:
115
+ async with Master(url) as master:
116
+ # Replace the session with the remote's downloader session
117
+ old_session = master.session
118
+ factory = self.remote.download_factory
119
+ master.session = factory._session
120
+
134
121
  deferred_download = self.remote.policy != Remote.IMMEDIATE
135
122
  workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
136
123
  async with ProgressReport(
@@ -150,25 +137,8 @@ class PythonBanderStage(Stage):
150
137
  Requirement(pkg).name for pkg in self.remote.includes
151
138
  ]
152
139
  await pmirror.synchronize(packages_to_sync)
153
-
154
-
155
- class PulpMaster(Master):
156
- """
157
- Pulp Master Class for Pulp specific overrides
158
- """
159
-
160
- def __init__(self, *args, tls=True, **kwargs):
161
- self.tls = tls
162
- super().__init__(*args, **kwargs)
163
-
164
- async def get(
165
- self, path: str, required_serial: Optional[int], **kw: Any
166
- ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
167
- """Support tls=false"""
168
- if not self.tls:
169
- kw["ssl"] = False
170
- async for r in super().get(path, required_serial, **kw):
171
- yield r
140
+ # place back old session so that it is properly closed
141
+ master.session = old_session
172
142
 
173
143
 
174
144
  class PulpMirror(Mirror):
@@ -226,7 +196,7 @@ class PulpMirror(Mirror):
226
196
  downloader = self.python_stage.remote.get_downloader(url=url)
227
197
  result = await downloader.run()
228
198
  with open(result.path) as f:
229
- index = parse_repo_index_page(f.read())
199
+ index = IndexPage.from_html(f.read())
230
200
  self.packages_to_sync.update({p: 0 for p in index.projects})
231
201
  self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
232
202
 
@@ -7,7 +7,7 @@ from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact
7
7
  from pulpcore.plugin.util import get_domain
8
8
 
9
9
  from pulp_python.app.models import PythonPackageContent, PythonRepository
10
- from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata
10
+ from pulp_python.app.utils import artifact_to_python_content_data
11
11
 
12
12
 
13
13
  def upload(artifact_sha256, filename, repository_pk=None):
@@ -76,15 +76,7 @@ def create_content(artifact_sha256, filename, domain):
76
76
  queryset of the new created content
77
77
  """
78
78
  artifact = Artifact.objects.get(sha256=artifact_sha256, pulp_domain=domain)
79
- metadata = get_project_metadata_from_artifact(filename, artifact)
80
-
81
- data = parse_project_metadata(vars(metadata))
82
- data['packagetype'] = metadata.packagetype
83
- data['version'] = metadata.version
84
- data['filename'] = filename
85
- data['sha256'] = artifact.sha256
86
- data['pulp_domain'] = domain
87
- data['_pulp_domain'] = domain
79
+ data = artifact_to_python_content_data(filename, artifact, domain)
88
80
 
89
81
  @transaction.atomic()
90
82
  def create():
@@ -1,4 +1,5 @@
1
1
  import pkginfo
2
+ import re
2
3
  import shutil
3
4
  import tempfile
4
5
  import json
@@ -51,6 +52,20 @@ DIST_EXTENSIONS = {
51
52
  ".zip": "sdist",
52
53
  }
53
54
 
55
+ DIST_REGEXES = {
56
+ # regex from https://github.com/pypa/pip/blob/18.0/src/pip/_internal/wheel.py#L569
57
+ ".whl": re.compile(
58
+ r"""^(?P<name>.+?)-(?P<version>.*?)
59
+ ((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
60
+ \.whl|\.dist-info)$""",
61
+ re.VERBOSE
62
+ ),
63
+ # regex based on https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#filename-embedded-metadata # noqa: E501
64
+ ".egg": re.compile(r"^(?P<name>.+?)-(?P<version>.*?)(-(?P<pyver>.+?(-(?P<plat>.+?))?))?\.egg|\.egg-info$"), # noqa: E501
65
+ # regex based on https://github.com/python/cpython/blob/v3.7.0/Lib/distutils/command/bdist_wininst.py#L292 # noqa: E501
66
+ ".exe": re.compile(r"^(?P<name>.+?)-(?P<version>.*?)\.(?P<plat>.+?)(-(?P<pyver>.+?))?\.exe$"),
67
+ }
68
+
54
69
  DIST_TYPES = {
55
70
  "bdist_wheel": pkginfo.Wheel,
56
71
  "bdist_wininst": pkginfo.Distribution,
@@ -72,6 +87,8 @@ def parse_project_metadata(project):
72
87
  """
73
88
  package = {}
74
89
  package['name'] = project.get('name') or ""
90
+ package['version'] = project.get('version') or ""
91
+ package['packagetype'] = project.get('packagetype') or ""
75
92
  package['metadata_version'] = project.get('metadata_version') or ""
76
93
  package['summary'] = project.get('summary') or ""
77
94
  package['description'] = project.get('description') or ""
@@ -86,6 +103,7 @@ def parse_project_metadata(project):
86
103
  package['project_url'] = project.get('project_url') or ""
87
104
  package['platform'] = project.get('platform') or ""
88
105
  package['supported_platform'] = project.get('supported_platform') or ""
106
+ package['requires_python'] = project.get('requires_python') or ""
89
107
  package['requires_dist'] = json.dumps(project.get('requires_dist', []))
90
108
  package['provides_dist'] = json.dumps(project.get('provides_dist', []))
91
109
  package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', []))
@@ -93,6 +111,7 @@ def parse_project_metadata(project):
93
111
  package['classifiers'] = json.dumps(project.get('classifiers', []))
94
112
  package['project_urls'] = json.dumps(project.get('project_urls', {}))
95
113
  package['description_content_type'] = project.get('description_content_type') or ""
114
+ package['python_version'] = project.get('python_version') or ""
96
115
 
97
116
  return package
98
117
 
@@ -113,17 +132,15 @@ def parse_metadata(project, version, distribution):
113
132
  dictionary: of useful python metadata
114
133
 
115
134
  """
116
- package = {}
135
+ package = parse_project_metadata(project)
117
136
 
118
137
  package['filename'] = distribution.get('filename') or ""
119
138
  package['packagetype'] = distribution.get('packagetype') or ""
120
139
  package['version'] = version
121
140
  package['url'] = distribution.get('url') or ""
122
141
  package['sha256'] = distribution.get('digests', {}).get('sha256') or ""
123
- package['python_version'] = distribution.get('python_version') or ""
124
- package['requires_python'] = distribution.get('requires_python') or ""
125
-
126
- package.update(parse_project_metadata(project))
142
+ package['python_version'] = distribution.get('python_version') or package.get('python_version')
143
+ package['requires_python'] = distribution.get('requires_python') or package.get('requires_python') # noqa: E501
127
144
 
128
145
  return package
129
146
 
@@ -147,9 +164,30 @@ def get_project_metadata_from_artifact(filename, artifact):
147
164
  temp_file.flush()
148
165
  metadata = DIST_TYPES[packagetype](temp_file.name)
149
166
  metadata.packagetype = packagetype
167
+ if packagetype == "sdist":
168
+ metadata.python_version = "source"
169
+ else:
170
+ pyver = ""
171
+ regex = DIST_REGEXES[extensions[pkg_type_index]]
172
+ if bdist_name := regex.match(filename):
173
+ pyver = bdist_name.group("pyver") or ""
174
+ metadata.python_version = pyver
150
175
  return metadata
151
176
 
152
177
 
178
+ def artifact_to_python_content_data(filename, artifact, domain=None):
179
+ """
180
+ Takes the artifact/filename and returns the metadata needed to create a PythonPackageContent.
181
+ """
182
+ metadata = get_project_metadata_from_artifact(filename, artifact)
183
+ data = parse_project_metadata(vars(metadata))
184
+ data['sha256'] = artifact.sha256
185
+ data['filename'] = filename
186
+ data['pulp_domain'] = domain or artifact.pulp_domain
187
+ data['_pulp_domain'] = data['pulp_domain']
188
+ return data
189
+
190
+
153
191
  def python_content_to_json(base_path, content_query, version=None, domain=None):
154
192
  """
155
193
  Converts a QuerySet of PythonPackageContent into the PyPi JSON format
@@ -145,7 +145,7 @@ def download_python_file(tmp_path, http_get):
145
145
  file_path = tmp_path / relative_path
146
146
  with open(file_path, mode="wb") as f:
147
147
  f.write(http_get(url))
148
- return file_path
148
+ return str(file_path)
149
149
 
150
150
  yield _download_python_file
151
151