pulp-python 3.36.2__tar.gz → 3.37.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {pulp_python-3.36.2 → pulp_python-3.37.0}/CHANGES.md +21 -0
  2. {pulp_python-3.36.2 → pulp_python-3.37.0}/PKG-INFO +3 -4
  3. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/__init__.py +1 -1
  4. pulp_python-3.37.0/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py +16 -0
  5. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/models.py +6 -1
  6. pulp_python-3.37.0/pulp_python/app/osv.py +42 -0
  7. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/pypi/feeds.py +24 -2
  8. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/pypi/serializers.py +3 -0
  9. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/pypi/views.py +1 -1
  10. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/serializers.py +6 -0
  11. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/__init__.py +1 -1
  12. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/publish.py +5 -2
  13. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/sync.py +33 -38
  14. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/vulnerability_report.py +11 -0
  15. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/utils.py +22 -1
  16. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/viewsets.py +2 -9
  17. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_download_content.py +1 -1
  18. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_pypi_apis.py +4 -1
  19. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_pypi_feeds.py +74 -6
  20. pulp_python-3.37.0/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py +171 -0
  21. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_vulnerability_report.py +7 -10
  22. pulp_python-3.37.0/pulp_python/tests/unit/test_feeds.py +52 -0
  23. pulp_python-3.37.0/pulp_python/tests/unit/test_vulnerabilities.py +84 -0
  24. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/PKG-INFO +3 -4
  25. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/SOURCES.txt +6 -1
  26. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/requires.txt +1 -1
  27. {pulp_python-3.36.2 → pulp_python-3.37.0}/pyproject.toml +4 -5
  28. {pulp_python-3.36.2 → pulp_python-3.37.0}/unittest_requirements.txt +1 -0
  29. {pulp_python-3.36.2 → pulp_python-3.37.0}/COMMITMENT +0 -0
  30. {pulp_python-3.36.2 → pulp_python-3.37.0}/COPYRIGHT +0 -0
  31. {pulp_python-3.36.2 → pulp_python-3.37.0}/LICENSE +0 -0
  32. {pulp_python-3.36.2 → pulp_python-3.37.0}/MANIFEST.in +0 -0
  33. {pulp_python-3.36.2 → pulp_python-3.37.0}/README.md +0 -0
  34. {pulp_python-3.36.2 → pulp_python-3.37.0}/functest_requirements.txt +0 -0
  35. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/__init__.py +0 -0
  36. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/cache.py +0 -0
  37. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/exceptions.py +0 -0
  38. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/global_access_conditions.py +0 -0
  39. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/management/__init__.py +0 -0
  40. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/management/commands/__init__.py +0 -0
  41. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/management/commands/repair-python-metadata.py +0 -0
  42. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0001_initial.py +0 -0
  43. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0001_squashed_0010_update_json_field.py +0 -0
  44. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +0 -0
  45. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0003_new_sync_filters.py +0 -0
  46. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +0 -0
  47. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +0 -0
  48. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +0 -0
  49. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +0 -0
  50. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +0 -0
  51. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +0 -0
  52. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0010_update_json_field.py +0 -0
  53. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +0 -0
  54. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0012_add_domain.py +0 -0
  55. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0013_add_rbac_permissions.py +0 -0
  56. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py +0 -0
  57. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0015_alter_pythonpackagecontent_options.py +0 -0
  58. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0016_pythonpackagecontent_metadata_sha256.py +0 -0
  59. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0017_pythonpackagecontent_size.py +0 -0
  60. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0018_packageprovenance.py +0 -0
  61. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0019_create_missing_metadata_artifacts.py +0 -0
  62. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0020_pythonpackagecontent_name_normalized.py +0 -0
  63. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0021_pythonrepository_upload_duplicate_filenames.py +0 -0
  64. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0022_pythonblocklistentry.py +0 -0
  65. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0023_packageyank.py +0 -0
  66. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/0024_pythonrepository_error_on_reject.py +0 -0
  67. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/migrations/__init__.py +0 -0
  68. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/modelresource.py +0 -0
  69. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/provenance.py +0 -0
  70. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/pypi/__init__.py +0 -0
  71. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/replica.py +0 -0
  72. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/settings.py +0 -0
  73. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/repair.py +0 -0
  74. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/upload.py +0 -0
  75. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/tasks/yank.py +0 -0
  76. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/urls.py +0 -0
  77. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/webserver_snippets/__init__.py +0 -0
  78. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/webserver_snippets/apache.conf +0 -0
  79. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/app/webserver_snippets/nginx.conf +0 -0
  80. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/pytest_plugin.py +0 -0
  81. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/__init__.py +0 -0
  82. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/__init__.py +0 -0
  83. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/__init__.py +0 -0
  84. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_attestations.py +0 -0
  85. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_auto_publish.py +0 -0
  86. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_blocklist.py +0 -0
  87. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_consume_content.py +0 -0
  88. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_crud_content_unit.py +0 -0
  89. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_crud_publications.py +0 -0
  90. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_crud_remotes.py +0 -0
  91. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_domains.py +0 -0
  92. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_export_import.py +0 -0
  93. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_full_mirror.py +0 -0
  94. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_pypi_simple_api.py +0 -0
  95. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_rbac.py +0 -0
  96. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_repair.py +0 -0
  97. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_simple_cache.py +0 -0
  98. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_sync.py +0 -0
  99. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_upload.py +0 -0
  100. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_version_specifier_filter.py +0 -0
  101. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/api/test_yank.py +0 -0
  102. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/constants.py +0 -0
  103. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/functional/utils.py +0 -0
  104. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/unit/__init__.py +0 -0
  105. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python/tests/unit/test_models.py +0 -0
  106. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/dependency_links.txt +0 -0
  107. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/entry_points.txt +0 -0
  108. {pulp_python-3.36.2 → pulp_python-3.37.0}/pulp_python.egg-info/top_level.txt +0 -0
  109. {pulp_python-3.36.2 → pulp_python-3.37.0}/setup.cfg +0 -0
  110. {pulp_python-3.36.2 → pulp_python-3.37.0}/test_requirements.txt +0 -0
@@ -8,6 +8,27 @@
8
8
 
9
9
  [//]: # (towncrier release notes start)
10
10
 
11
+ ## 3.37.0 (2026-10-07) {: #3.37.0 }
12
+
13
+ #### Features {: #3.37.0-feature }
14
+
15
+ - Added a `vulnerabilities` field to the PyPI JSON API, populated from OSV scan reports. Remotes can opt in to scan the new repository version after sync.
16
+ [#1360](https://github.com/pulp/pulp_python/issues/1360)
17
+ - Updated minimum required python version to >=3.12.
18
+ - Upgraded Bandersnatch to 8.0. Full-index syncs now list packages from either the JSON or HTML representation of the Simple API.
19
+
20
+ #### Bugfixes {: #3.37.0-bugfix }
21
+
22
+ - Prefer the PyPI Simple API JSON response when clients such as pip and uv advertise JSON alongside HTML.
23
+ [#1370](https://github.com/pulp/pulp_python/issues/1370)
24
+ - RSS feeds now reflect the most recently added file for each release, produce unique guids when content is updated, and support up to 500 entries in `updates.xml`.
25
+ [#1374](https://github.com/pulp/pulp_python/issues/1374)
26
+ - RSS feed GUIDs are now stable when additional platform wheels arrive for the same release. GUIDs only change when a new PEP 427 build tag is introduced, preventing feed readers from showing duplicate entries for multi-architecture builds.
27
+ [#1381](https://github.com/pulp/pulp_python/issues/1381)
28
+ - Fixed the Integrity API evaluating the entire repository's content query before filtering.
29
+
30
+ ---
31
+
11
32
  ## 3.36.2 (2026-09-10) {: #3.36.2 }
12
33
 
13
34
  #### Bugfixes {: #3.36.2-bugfix }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pulp-python
3
- Version: 3.36.2
3
+ Version: 3.37.0
4
4
  Summary: pulp-python plugin for the Pulp Project
5
5
  Author-email: Pulp Team <pulp-list@redhat.com>
6
6
  Project-URL: Homepage, https://pulpproject.org
@@ -14,15 +14,14 @@ Classifier: Development Status :: 5 - Production/Stable
14
14
  Classifier: Framework :: Django
15
15
  Classifier: Programming Language :: Python
16
16
  Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3.11
18
17
  Classifier: Programming Language :: Python :: 3.12
19
18
  Classifier: Programming Language :: Python :: 3.13
20
- Requires-Python: >=3.11
19
+ Requires-Python: >=3.12
21
20
  Description-Content-Type: text/markdown
22
21
  License-File: LICENSE
23
22
  Requires-Dist: pulpcore<3.130,>=3.105.0
24
23
  Requires-Dist: pkginfo<1.13.0,>=1.12.0
25
- Requires-Dist: bandersnatch<6.7,>=6.6.0
24
+ Requires-Dist: bandersnatch<8.1,>=8.0.0
26
25
  Requires-Dist: pypi-simple<2.0,>=1.8.0
27
26
  Requires-Dist: pypi-attestations==0.0.28
28
27
  Dynamic: license-file
@@ -12,7 +12,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
12
12
 
13
13
  name = "pulp_python.app"
14
14
  label = "python"
15
- version = "3.36.2"
15
+ version = "3.37.0"
16
16
  python_package_name = "pulp-python"
17
17
  domain_compatible = True
18
18
 
@@ -0,0 +1,16 @@
1
+ from django.db import migrations, models
2
+
3
+
4
+ class Migration(migrations.Migration):
5
+
6
+ dependencies = [
7
+ ("python", "0024_pythonrepository_error_on_reject"),
8
+ ]
9
+
10
+ operations = [
11
+ migrations.AddField(
12
+ model_name="pythonremote",
13
+ name="vulnerabilities",
14
+ field=models.BooleanField(default=False),
15
+ ),
16
+ ]
@@ -130,7 +130,11 @@ class PythonDistribution(Distribution, AutoAddObjPermsMixin):
130
130
  if not settings.DOMAIN_ENABLED:
131
131
  domain = None
132
132
  json_body = python_content_to_json(
133
- self.base_path, package_content, version=version, domain=domain
133
+ self.base_path,
134
+ package_content,
135
+ version=version,
136
+ domain=domain,
137
+ repository_version=self.publication.repository_version,
134
138
  )
135
139
  if json_body:
136
140
  return json_response(json_body, headers=headers)
@@ -350,6 +354,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin):
350
354
  models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list
351
355
  )
352
356
  provenance = models.BooleanField(default=False)
357
+ vulnerabilities = models.BooleanField(default=False)
353
358
 
354
359
  def get_remote_artifact_url(self, relative_path=None, request=None):
355
360
  """Get url for remote_artifact"""
@@ -0,0 +1,42 @@
1
+ from packaging.version import InvalidVersion, Version
2
+
3
+
4
+ def _osv_fixed_in(vuln):
5
+ """Extract PEP 440 fixed versions from an OSV vulnerability record."""
6
+ fixed = []
7
+ seen = set()
8
+ for affected in vuln.get("affected") or []:
9
+ for range_ in affected.get("ranges") or []:
10
+ for event in range_.get("events") or []:
11
+ if "fixed" not in event:
12
+ continue
13
+ version = event["fixed"]
14
+ if version in seen:
15
+ continue
16
+ try:
17
+ Version(version)
18
+ except InvalidVersion:
19
+ continue
20
+ seen.add(version)
21
+ fixed.append(version)
22
+ return fixed
23
+
24
+
25
+ def osv_to_pypi_vulnerabilities(vulns):
26
+ """Trim OSV vulnerability records to the Warehouse JSON API shape."""
27
+ seen = {}
28
+ for vuln in vulns or []:
29
+ vuln_id = vuln.get("id")
30
+ if not vuln_id or vuln_id in seen:
31
+ continue
32
+ seen[vuln_id] = {
33
+ "id": vuln_id,
34
+ "source": "osv",
35
+ "link": f"https://osv.dev/vulnerability/{vuln_id}",
36
+ "aliases": vuln.get("aliases") or [],
37
+ "details": vuln.get("details"),
38
+ "summary": vuln.get("summary"),
39
+ "fixed_in": _osv_fixed_in(vuln),
40
+ "withdrawn": vuln.get("withdrawn"),
41
+ }
42
+ return list(seen.values())
@@ -2,6 +2,7 @@ import re
2
2
  from email.utils import getaddresses
3
3
  from urllib.parse import urljoin
4
4
 
5
+ from django.contrib.postgres.aggregates import ArrayAgg
5
6
  from django.db.models import F, FilteredRelation, Max, Min, Q
6
7
  from django.http.response import HttpResponse, HttpResponseNotFound
7
8
  from django.utils.decorators import method_decorator
@@ -15,6 +16,8 @@ from rest_framework.viewsets import ViewSet
15
16
  from pulp_python.app.cache import PythonApiCache, find_base_path_cached
16
17
  from pulp_python.app.pypi.views import PyPIMixin, _etag_func
17
18
 
19
+ _WHEEL_BUILD_TAG_RE = re.compile(r"^.+?-.+?-(?P<build>\d[^-]*?)-[^-]+-[^-]+-[^-]+\.whl$")
20
+
18
21
  UPDATES_LIMIT = 500
19
22
  PACKAGES_LIMIT = 40
20
23
  PROJECT_RELEASES_LIMIT = 40
@@ -70,6 +73,7 @@ def iter_releases(content, repo_ver, name_normalized=None, limit=UPDATES_LIMIT):
70
73
  name=Min("name"),
71
74
  summary=Min("summary"),
72
75
  author_email=Min("author_email"),
76
+ filenames=ArrayAgg("filename", distinct=True, ordering="filename"),
73
77
  )
74
78
  .order_by("-added_at", "name_normalized", "version")[:limit]
75
79
  )
@@ -91,14 +95,30 @@ def iter_projects(content, repo_ver, limit=PACKAGES_LIMIT):
91
95
  )
92
96
 
93
97
 
94
- def _item_dict(title, link, description, author_email, pubdate):
98
+ def _build_tag_fragment(filenames):
99
+ """Extract sorted distinct build tags from wheel filenames for GUID stability.
100
+
101
+ Returns a fragment like `#builds=1,2` when build tags are present,
102
+ or an empty string for sdists and wheels without build tags.
103
+ """
104
+ tags = set()
105
+ for fn in filenames or ():
106
+ m = _WHEEL_BUILD_TAG_RE.match(fn)
107
+ if m:
108
+ tags.add(m.group("build"))
109
+ if not tags:
110
+ return ""
111
+ return "#builds=" + ",".join(sorted(tags))
112
+
113
+
114
+ def _item_dict(title, link, description, author_email, pubdate, filenames=()):
95
115
  return {
96
116
  "title": sanitize_xml_text(title),
97
117
  "link": link,
98
118
  "description": sanitize_xml_text(description),
99
119
  "author_email": format_author(author_email),
100
120
  "pubdate": pubdate,
101
- "unique_id": f"{link}#{pubdate.isoformat()}",
121
+ "unique_id": f"{link}{_build_tag_fragment(filenames)}",
102
122
  }
103
123
 
104
124
 
@@ -139,6 +159,7 @@ def render_updates_feed(index_url, releases):
139
159
  description=release["summary"],
140
160
  author_email=release["author_email"],
141
161
  pubdate=release["added_at"],
162
+ filenames=release.get("filenames", ()),
142
163
  )
143
164
  for release in releases
144
165
  ]
@@ -178,6 +199,7 @@ def render_project_releases_feed(index_url, project_name, releases):
178
199
  description=release["summary"],
179
200
  author_email=release["author_email"],
180
201
  pubdate=release["added_at"],
202
+ filenames=release.get("filenames", ()),
181
203
  )
182
204
  for release in releases
183
205
  ]
@@ -35,6 +35,9 @@ class PackageMetadataSerializer(serializers.Serializer):
35
35
  info = serializers.JSONField(help_text=_("Core metadata of the package"))
36
36
  releases = serializers.JSONField(help_text=_("List of all the releases of the package"))
37
37
  urls = serializers.JSONField()
38
+ vulnerabilities = serializers.JSONField(
39
+ help_text=_("Known vulnerabilities for the selected package version."),
40
+ )
38
41
 
39
42
 
40
43
  class PackageUploadSerializer(serializers.Serializer):
@@ -621,7 +621,7 @@ class ProvenanceView(PyPIMixin, ViewSet):
621
621
  def retrieve(self, request, path, package, version, filename):
622
622
  """Gets the provenance for a package."""
623
623
  repo_ver, content = self.get_rvc()
624
- if content:
624
+ if content is not None:
625
625
  package_content = content.filter(
626
626
  name_normalized=package, version=version, filename=filename
627
627
  ).first()
@@ -789,6 +789,11 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
789
789
  help_text=_("Whether to sync available provenances for Python packages."),
790
790
  default=False,
791
791
  )
792
+ vulnerabilities = serializers.BooleanField(
793
+ required=False,
794
+ help_text=_("Whether to scan the new repository version for vulnerabilities after a sync."),
795
+ default=False,
796
+ )
792
797
 
793
798
  def validate_includes(self, value):
794
799
  """Validates the includes"""
@@ -821,6 +826,7 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
821
826
  "keep_latest_packages",
822
827
  "exclude_platforms",
823
828
  "provenance",
829
+ "vulnerabilities",
824
830
  )
825
831
  model = python_models.PythonRemote
826
832
 
@@ -6,5 +6,5 @@ from .publish import publish # noqa:F401
6
6
  from .repair import repair # noqa:F401
7
7
  from .sync import sync # noqa:F401
8
8
  from .upload import upload, upload_group # noqa:F401
9
- from .vulnerability_report import get_repo_version_content # noqa:F401
9
+ from .vulnerability_report import dispatch_scan, get_repo_version_content # noqa:F401
10
10
  from .yank import aunyank_package, ayank_package # noqa:F401
@@ -56,11 +56,14 @@ def write_simple_api(publication):
56
56
  domain = get_domain()
57
57
  simple_dir = "simple/"
58
58
  os.mkdir(simple_dir)
59
+ # Secondary ORDER BY makes DISTINCT ON pick a stable display name when
60
+ # metadata names differ but canonicalize to the same name_normalized
61
+ # (e.g. msg-parser vs msg_parser).
59
62
  project_names = (
60
63
  python_models.PythonPackageContent.objects.filter(
61
64
  pk__in=publication.repository_version.content, _pulp_domain=domain
62
65
  )
63
- .order_by("name_normalized")
66
+ .order_by("name_normalized", "name")
64
67
  .values_list("name", flat=True)
65
68
  .distinct("name_normalized")
66
69
  )
@@ -81,7 +84,7 @@ def write_simple_api(publication):
81
84
  packages = python_models.PythonPackageContent.objects.filter(
82
85
  pk__in=publication.repository_version.content, _pulp_domain=domain
83
86
  )
84
- releases = packages.order_by("name_normalized").values("name", "filename", "sha256")
87
+ releases = packages.order_by("name_normalized", "filename").values("name", "filename", "sha256")
85
88
 
86
89
  ind = 0
87
90
  current_name = canonicalize_name(project_names[ind])
@@ -1,15 +1,14 @@
1
1
  import asyncio
2
+ import json
2
3
  import logging
3
4
  from functools import partial
4
- from urllib.parse import urljoin, urlparse
5
+ from urllib.parse import urlparse
5
6
 
6
- from aiohttp import ClientError, ClientResponseError
7
7
  from bandersnatch.configuration import BandersnatchConfig
8
8
  from bandersnatch.master import Master
9
9
  from bandersnatch.mirror import Mirror
10
- from lxml.etree import LxmlError
11
10
  from packaging.requirements import Requirement
12
- from pypi_simple import IndexPage
11
+ from pypi_simple import ACCEPT_ANY, IndexPage, UnsupportedContentTypeError
13
12
 
14
13
  from pulpcore.plugin.download import HttpDownloader
15
14
  from pulpcore.plugin.exceptions import SyncError
@@ -29,6 +28,7 @@ from pulp_python.app.models import (
29
28
  PythonRemote,
30
29
  )
31
30
  from pulp_python.app.provenance import Provenance
31
+ from pulp_python.app.tasks.vulnerability_report import dispatch_scan
32
32
  from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata
33
33
 
34
34
  logger = logging.getLogger(__name__)
@@ -56,7 +56,9 @@ def sync(remote_pk, repository_pk, mirror):
56
56
  raise SyncError("A remote must have a url attribute to sync.")
57
57
 
58
58
  first_stage = PythonBanderStage(remote)
59
- DeclarativeVersion(first_stage, repository, mirror).create()
59
+ new_version = DeclarativeVersion(first_stage, repository, mirror).create()
60
+ if new_version and remote.vulnerabilities:
61
+ dispatch_scan(repository, new_version)
60
62
 
61
63
 
62
64
  def create_bandersnatch_config(remote):
@@ -169,41 +171,34 @@ class PulpMirror(Mirror):
169
171
 
170
172
  async def determine_packages_to_sync(self):
171
173
  """
172
- Calling this means that includes wasn't specified,
173
- so try to get all of the packages from Mirror (hopefully PyPi)
174
+ Called when includes wasn't specified. List all projects from the remote's simple index.
174
175
  """
175
- number_xmlrpc_attempts = 3
176
- for attempt in range(number_xmlrpc_attempts):
177
- logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url))
178
- try:
179
- if not self.synced_serial:
180
- logger.info("Syncing all packages.")
181
- # First get the current serial, then start to sync.
182
- all_packages = await self.master.all_packages()
183
- self.packages_to_sync.update(all_packages)
184
- self.target_serial = max(
185
- [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()]
186
- )
187
- else:
188
- logger.info("Syncing based on changelog.")
189
- changed_packages = await self.master.changed_packages(self.synced_serial)
190
- self.packages_to_sync.update(changed_packages)
191
- self.target_serial = max(
192
- [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()]
193
- )
194
- break
195
- except (ClientError, ClientResponseError, LxmlError):
196
- # Retry if XMLRPC endpoint failed, server might not support it.
197
- continue
176
+ logger.info("Syncing all packages from %s", self.master.url)
177
+ url = f"{self.remote.url.rstrip('/')}/simple/"
178
+ downloader = self.remote.get_downloader(url=url)
179
+ result = await downloader.run(
180
+ extra_data={"request_kwargs": {"headers": {"Accept": ACCEPT_ANY}}}
181
+ )
182
+
183
+ content_type = result.headers.get("Content-Type", "text/html").partition(";")[0].lower()
184
+ with open(result.path, "rb") as f:
185
+ content = f.read()
186
+
187
+ if content_type == "application/vnd.pypi.simple.v1+json":
188
+ index = IndexPage.from_json_data(json.loads(content))
189
+ elif content_type in {"application/vnd.pypi.simple.v1+html", "text/html"}:
190
+ index = IndexPage.from_html(content)
191
+ else:
192
+ raise UnsupportedContentTypeError(url, content_type)
193
+
194
+ self.packages_to_sync = {project: 0 for project in index.projects}
195
+ last_serial = index.last_serial
196
+ if last_serial is None:
197
+ last_serial = result.headers.get(PYPI_LAST_SERIAL)
198
+ if last_serial is None:
199
+ self.target_serial = self.synced_serial
198
200
  else:
199
- logger.info("Failed to get package list using XMLRPC, trying parse simple page.")
200
- url = urljoin(self.remote.url, "simple/")
201
- downloader = self.remote.get_downloader(url=url)
202
- result = await downloader.run()
203
- with open(result.path) as f:
204
- index = IndexPage.from_html(f.read())
205
- self.packages_to_sync.update({p: 0 for p in index.projects})
206
- self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
201
+ self.target_serial = max(self.synced_serial, int(last_serial))
207
202
 
208
203
  self._filter_packages()
209
204
  if self.target_serial:
@@ -1,5 +1,6 @@
1
1
  from pulpcore.plugin.models import RepositoryVersion
2
2
  from pulpcore.plugin.sync import sync_to_async_iterable
3
+ from pulpcore.plugin.tasking import check_content, dispatch
3
4
 
4
5
  from pulp_python.app.models import PythonPackageContent
5
6
 
@@ -28,3 +29,13 @@ def _build_osv_data(name, ecosystem, version=None):
28
29
  if version:
29
30
  osv_data["version"] = version
30
31
  return osv_data
32
+
33
+
34
+ def dispatch_scan(repository, repository_version):
35
+ """Dispatch a vulnerability scan for a repository version."""
36
+ func = f"{get_repo_version_content.__module__}.{get_repo_version_content.__name__}"
37
+ return dispatch(
38
+ check_content,
39
+ shared_resources=[repository],
40
+ args=[func, [str(repository_version.pk)]],
41
+ )
@@ -21,9 +21,11 @@ from packaging.version import InvalidVersion, parse
21
21
  from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
22
22
 
23
23
  from pulpcore.plugin.exceptions import TimeoutException
24
- from pulpcore.plugin.models import Artifact, Remote
24
+ from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport
25
25
  from pulpcore.plugin.util import get_domain
26
26
 
27
+ from pulp_python.app.osv import osv_to_pypi_vulnerabilities
28
+
27
29
  log = logging.getLogger(__name__)
28
30
 
29
31
 
@@ -377,6 +379,7 @@ def python_content_to_json(
377
379
  last_serial: int
378
380
  releases: Dict
379
381
  urls: Dict
382
+ vulnerabilities: List
380
383
 
381
384
  Returns None if version is specified but not found within content_query
382
385
  """
@@ -407,9 +410,27 @@ def python_content_to_json(
407
410
  full_metadata["info"] = python_content_to_info(latest_content[0])
408
411
  full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain)
409
412
  full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain)
413
+ full_metadata["vulnerabilities"] = _vulnerabilities_for_content(
414
+ latest_content, repository_version
415
+ )
410
416
  return full_metadata
411
417
 
412
418
 
419
+ def _vulnerabilities_for_content(contents, repository_version=None):
420
+ """Load VulnerabilityReports scanned for this repository version and trim to Warehouse shape."""
421
+ if not contents or repository_version is None:
422
+ return []
423
+ reports = VulnerabilityReport.objects.filter(
424
+ content_id__in=[c.pk for c in contents],
425
+ repo_versions=repository_version,
426
+ )
427
+ merged = []
428
+ for vulns in reports.values_list("vulns", flat=True):
429
+ if vulns:
430
+ merged.extend(vulns)
431
+ return osv_to_pypi_vulnerabilities(merged)
432
+
433
+
413
434
  def latest_content_version(all_content, version):
414
435
  """
415
436
  Walks through the content list and finds the instances that are the latest version.
@@ -26,7 +26,7 @@ from pulpcore.plugin.serializers import (
26
26
  RepositoryAddRemoveContentSerializer,
27
27
  RepositorySyncURLSerializer,
28
28
  )
29
- from pulpcore.plugin.tasking import check_content, dispatch
29
+ from pulpcore.plugin.tasking import dispatch
30
30
  from pulpcore.plugin.util import extract_pk
31
31
 
32
32
  from pulp_python.app import models as python_models
@@ -356,14 +356,7 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet):
356
356
  Scan a repository version for vulnerabilities.
357
357
  """
358
358
  repository_version = self.get_object()
359
- func = (
360
- f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}"
361
- )
362
- task = dispatch(
363
- check_content,
364
- shared_resources=[repository_version.repository],
365
- args=[func, [repository_version.pk]],
366
- )
359
+ task = tasks.dispatch_scan(repository_version.repository, repository_version)
367
360
  return core_viewsets.OperationPostponedResponse(task, request)
368
361
 
369
362
 
@@ -48,7 +48,7 @@ def test_full_fixtures_to_pulp_sync(
48
48
  ):
49
49
  """
50
50
  This test checks that Pulp can fully sync another Python Package repository that is not
51
- PyPI. This reads the repository's simple page if XMLRPC isn't supported.
51
+ PyPI. This lists projects from the remote's Simple API index.
52
52
  """
53
53
  # Repository we are syncing from is the fixtures (default url)
54
54
  remote = python_remote_factory(includes=[], prereleases=True)
@@ -324,6 +324,7 @@ def assert_pypi_json(package):
324
324
  package["releases"][version],
325
325
  "Failed to match version",
326
326
  )
327
+ assert package["vulnerabilities"] == []
327
328
 
328
329
 
329
330
  def assert_download_info(expected, received, message="Failed to match"):
@@ -377,6 +378,8 @@ def test_upload_time_reflects_repo_addition(
377
378
  # JSON API
378
379
  json_resp = requests.get(urljoin(distro.base_url, "pypi/twine/json"))
379
380
  assert json_resp.status_code == 200
380
- json_time = datetime.fromisoformat(json_resp.json()["urls"][0]["upload_time"])
381
+ package = json_resp.json()
382
+ json_time = datetime.fromisoformat(package["urls"][0]["upload_time"])
381
383
  assert json_time > content_created
382
384
  assert json_time == simple_time
385
+ assert package["vulnerabilities"] == []
@@ -1,3 +1,5 @@
1
+ import io
2
+ import zipfile
1
3
  from urllib.parse import urljoin, urlsplit
2
4
  from xml.etree import ElementTree as ET
3
5
 
@@ -20,6 +22,23 @@ TWINE_500_WHEEL_FILENAME = "twine-5.0.0-py3-none-any.whl"
20
22
  TWINE_500_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), TWINE_500_WHEEL_FILENAME)
21
23
 
22
24
 
25
+ def _make_build_tagged_wheel(tmp_path, base_wheel_bytes, build_tag, platform="linux_x86_64"):
26
+ """Repackage a wheel with a PEP 427 build tag and platform in the filename."""
27
+ filename = f"shelf_reader-0.1-{build_tag}-py2-none-{platform}.whl"
28
+ path = tmp_path / filename
29
+ with zipfile.ZipFile(io.BytesIO(base_wheel_bytes)) as src, zipfile.ZipFile(path, "w") as dst:
30
+ for item in src.infolist():
31
+ dst.writestr(item, src.read(item.filename))
32
+ # Pulp deduplicates content by sha256, so identical zip bytes with
33
+ # different filenames map to the same content unit. Inject a unique
34
+ # marker so each (build_tag, platform) combination gets its own sha256.
35
+ dst.writestr(
36
+ "shelf_reader/.build_marker",
37
+ f"build={build_tag} platform={platform}\n",
38
+ )
39
+ return str(path), filename
40
+
41
+
23
42
  def _index_url(distro, bindings_cfg):
24
43
  """Build the index URL using the same origin the API client uses."""
25
44
  path = urlsplit(distro.base_url).path
@@ -125,7 +144,7 @@ def test_pinned_version_feeds(
125
144
 
126
145
  item = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))[0]
127
146
  assert item.findtext("link").endswith("pypi/shelf-reader/0.1/json")
128
- assert "pypi/shelf-reader/0.1/json#" in item.findtext("guid")
147
+ assert item.findtext("guid").endswith("pypi/shelf-reader/0.1/json")
129
148
 
130
149
  python_content_factory(TWINE_WHEEL_FILENAME, url=TWINE_WHEEL_URL, repository=repo)
131
150
  update_titles = _titles(_parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg)))
@@ -133,10 +152,10 @@ def test_pinned_version_feeds(
133
152
 
134
153
 
135
154
  @pytest.mark.parallel
136
- def test_new_file_for_existing_version_updates_guid(
155
+ def test_guid_stable_when_adding_file_without_new_build_tag(
137
156
  bindings_cfg, python_content_factory, python_empty_repo_distro
138
157
  ):
139
- """Adding a new file for an existing (name, version) produces a new guid and updated date."""
158
+ """Adding a file without a new build tag keeps the GUID stable but updates pubDate."""
140
159
  repo, distro = python_empty_repo_distro()
141
160
 
142
161
  python_content_factory(PYTHON_EGG_FILENAME, url=PYTHON_EGG_URL, repository=repo)
@@ -145,7 +164,7 @@ def test_new_file_for_existing_version_updates_guid(
145
164
  assert len(items) == 1
146
165
  first_guid = items[0].findtext("guid")
147
166
  first_date = items[0].findtext("pubDate")
148
- assert "pypi/shelf-reader/0.1/json" in first_guid
167
+ assert first_guid.endswith("pypi/shelf-reader/0.1/json")
149
168
 
150
169
  python_content_factory(PYTHON_WHEEL_FILENAME, url=PYTHON_WHEEL_URL, repository=repo)
151
170
 
@@ -154,11 +173,60 @@ def test_new_file_for_existing_version_updates_guid(
154
173
  second_guid = items[0].findtext("guid")
155
174
  second_date = items[0].findtext("pubDate")
156
175
 
157
- assert second_guid != first_guid
176
+ assert second_guid == first_guid
158
177
  assert second_date >= first_date
159
178
 
160
179
  release_items = _parse_items(
161
180
  _get_feed(distro, "rss/project/shelf-reader/releases.xml", bindings_cfg)
162
181
  )
163
182
  assert len(release_items) == 1
164
- assert release_items[0].findtext("guid") == second_guid
183
+ assert release_items[0].findtext("guid") == first_guid
184
+
185
+
186
+ @pytest.mark.parallel
187
+ def test_new_build_tag_changes_guid(
188
+ tmp_path, bindings_cfg, python_bindings, monitor_task, python_empty_repo_distro
189
+ ):
190
+ """A new build tag produces a new GUID; a new arch for the same tag does not."""
191
+ repo, distro = python_empty_repo_distro()
192
+
193
+ base_wheel = requests.get(PYTHON_WHEEL_URL, timeout=30).content
194
+
195
+ # Upload build tag 1 (x86_64)
196
+ path_1, name_1 = _make_build_tagged_wheel(tmp_path, base_wheel, "1")
197
+ task = python_bindings.ContentPackagesApi.create(
198
+ relative_path=name_1, file=path_1, repository=repo.pulp_href
199
+ ).task
200
+ monitor_task(task)
201
+
202
+ items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
203
+ assert len(items) == 1
204
+ guid_after_build1 = items[0].findtext("guid")
205
+ assert "#builds=1" in guid_after_build1
206
+
207
+ # Upload build tag 2 (x86_64) -- different build, GUID must change
208
+ path_2, name_2 = _make_build_tagged_wheel(tmp_path, base_wheel, "2")
209
+ task = python_bindings.ContentPackagesApi.create(
210
+ relative_path=name_2, file=path_2, repository=repo.pulp_href
211
+ ).task
212
+ monitor_task(task)
213
+
214
+ items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
215
+ assert len(items) == 1
216
+ guid_after_build2 = items[0].findtext("guid")
217
+ assert guid_after_build2 != guid_after_build1
218
+ assert "#builds=1,2" in guid_after_build2
219
+
220
+ # Upload build tag 2 again with a different arch -- same build tag, GUID must stay
221
+ path_2_arm, name_2_arm = _make_build_tagged_wheel(
222
+ tmp_path, base_wheel, "2", platform="linux_aarch64"
223
+ )
224
+ task = python_bindings.ContentPackagesApi.create(
225
+ relative_path=name_2_arm, file=path_2_arm, repository=repo.pulp_href
226
+ ).task
227
+ monitor_task(task)
228
+
229
+ items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
230
+ assert len(items) == 1
231
+ guid_after_build2_arm = items[0].findtext("guid")
232
+ assert guid_after_build2_arm == guid_after_build2