pulp-python 3.36.2__py3-none-any.whl → 3.37.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pulp_python/app/__init__.py +1 -1
- pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py +16 -0
- pulp_python/app/models.py +6 -1
- pulp_python/app/osv.py +42 -0
- pulp_python/app/pypi/feeds.py +24 -2
- pulp_python/app/pypi/serializers.py +3 -0
- pulp_python/app/pypi/views.py +1 -1
- pulp_python/app/serializers.py +6 -0
- pulp_python/app/tasks/__init__.py +1 -1
- pulp_python/app/tasks/publish.py +5 -2
- pulp_python/app/tasks/sync.py +33 -38
- pulp_python/app/tasks/vulnerability_report.py +11 -0
- pulp_python/app/utils.py +22 -1
- pulp_python/app/viewsets.py +2 -9
- pulp_python/tests/functional/api/test_download_content.py +1 -1
- pulp_python/tests/functional/api/test_pypi_apis.py +4 -1
- pulp_python/tests/functional/api/test_pypi_feeds.py +74 -6
- pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py +171 -0
- pulp_python/tests/functional/api/test_vulnerability_report.py +7 -10
- pulp_python/tests/unit/test_feeds.py +52 -0
- pulp_python/tests/unit/test_vulnerabilities.py +84 -0
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/METADATA +3 -4
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/RECORD +27 -22
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/WHEEL +0 -0
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/entry_points.txt +0 -0
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/licenses/LICENSE +0 -0
- {pulp_python-3.36.2.dist-info → pulp_python-3.37.0.dist-info}/top_level.txt +0 -0
pulp_python/app/__init__.py
CHANGED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from django.db import migrations, models
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Migration(migrations.Migration):
|
|
5
|
+
|
|
6
|
+
dependencies = [
|
|
7
|
+
("python", "0024_pythonrepository_error_on_reject"),
|
|
8
|
+
]
|
|
9
|
+
|
|
10
|
+
operations = [
|
|
11
|
+
migrations.AddField(
|
|
12
|
+
model_name="pythonremote",
|
|
13
|
+
name="vulnerabilities",
|
|
14
|
+
field=models.BooleanField(default=False),
|
|
15
|
+
),
|
|
16
|
+
]
|
pulp_python/app/models.py
CHANGED
|
@@ -130,7 +130,11 @@ class PythonDistribution(Distribution, AutoAddObjPermsMixin):
|
|
|
130
130
|
if not settings.DOMAIN_ENABLED:
|
|
131
131
|
domain = None
|
|
132
132
|
json_body = python_content_to_json(
|
|
133
|
-
self.base_path,
|
|
133
|
+
self.base_path,
|
|
134
|
+
package_content,
|
|
135
|
+
version=version,
|
|
136
|
+
domain=domain,
|
|
137
|
+
repository_version=self.publication.repository_version,
|
|
134
138
|
)
|
|
135
139
|
if json_body:
|
|
136
140
|
return json_response(json_body, headers=headers)
|
|
@@ -350,6 +354,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin):
|
|
|
350
354
|
models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list
|
|
351
355
|
)
|
|
352
356
|
provenance = models.BooleanField(default=False)
|
|
357
|
+
vulnerabilities = models.BooleanField(default=False)
|
|
353
358
|
|
|
354
359
|
def get_remote_artifact_url(self, relative_path=None, request=None):
|
|
355
360
|
"""Get url for remote_artifact"""
|
pulp_python/app/osv.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from packaging.version import InvalidVersion, Version
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def _osv_fixed_in(vuln):
|
|
5
|
+
"""Extract PEP 440 fixed versions from an OSV vulnerability record."""
|
|
6
|
+
fixed = []
|
|
7
|
+
seen = set()
|
|
8
|
+
for affected in vuln.get("affected") or []:
|
|
9
|
+
for range_ in affected.get("ranges") or []:
|
|
10
|
+
for event in range_.get("events") or []:
|
|
11
|
+
if "fixed" not in event:
|
|
12
|
+
continue
|
|
13
|
+
version = event["fixed"]
|
|
14
|
+
if version in seen:
|
|
15
|
+
continue
|
|
16
|
+
try:
|
|
17
|
+
Version(version)
|
|
18
|
+
except InvalidVersion:
|
|
19
|
+
continue
|
|
20
|
+
seen.add(version)
|
|
21
|
+
fixed.append(version)
|
|
22
|
+
return fixed
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def osv_to_pypi_vulnerabilities(vulns):
|
|
26
|
+
"""Trim OSV vulnerability records to the Warehouse JSON API shape."""
|
|
27
|
+
seen = {}
|
|
28
|
+
for vuln in vulns or []:
|
|
29
|
+
vuln_id = vuln.get("id")
|
|
30
|
+
if not vuln_id or vuln_id in seen:
|
|
31
|
+
continue
|
|
32
|
+
seen[vuln_id] = {
|
|
33
|
+
"id": vuln_id,
|
|
34
|
+
"source": "osv",
|
|
35
|
+
"link": f"https://osv.dev/vulnerability/{vuln_id}",
|
|
36
|
+
"aliases": vuln.get("aliases") or [],
|
|
37
|
+
"details": vuln.get("details"),
|
|
38
|
+
"summary": vuln.get("summary"),
|
|
39
|
+
"fixed_in": _osv_fixed_in(vuln),
|
|
40
|
+
"withdrawn": vuln.get("withdrawn"),
|
|
41
|
+
}
|
|
42
|
+
return list(seen.values())
|
pulp_python/app/pypi/feeds.py
CHANGED
|
@@ -2,6 +2,7 @@ import re
|
|
|
2
2
|
from email.utils import getaddresses
|
|
3
3
|
from urllib.parse import urljoin
|
|
4
4
|
|
|
5
|
+
from django.contrib.postgres.aggregates import ArrayAgg
|
|
5
6
|
from django.db.models import F, FilteredRelation, Max, Min, Q
|
|
6
7
|
from django.http.response import HttpResponse, HttpResponseNotFound
|
|
7
8
|
from django.utils.decorators import method_decorator
|
|
@@ -15,6 +16,8 @@ from rest_framework.viewsets import ViewSet
|
|
|
15
16
|
from pulp_python.app.cache import PythonApiCache, find_base_path_cached
|
|
16
17
|
from pulp_python.app.pypi.views import PyPIMixin, _etag_func
|
|
17
18
|
|
|
19
|
+
_WHEEL_BUILD_TAG_RE = re.compile(r"^.+?-.+?-(?P<build>\d[^-]*?)-[^-]+-[^-]+-[^-]+\.whl$")
|
|
20
|
+
|
|
18
21
|
UPDATES_LIMIT = 500
|
|
19
22
|
PACKAGES_LIMIT = 40
|
|
20
23
|
PROJECT_RELEASES_LIMIT = 40
|
|
@@ -70,6 +73,7 @@ def iter_releases(content, repo_ver, name_normalized=None, limit=UPDATES_LIMIT):
|
|
|
70
73
|
name=Min("name"),
|
|
71
74
|
summary=Min("summary"),
|
|
72
75
|
author_email=Min("author_email"),
|
|
76
|
+
filenames=ArrayAgg("filename", distinct=True, ordering="filename"),
|
|
73
77
|
)
|
|
74
78
|
.order_by("-added_at", "name_normalized", "version")[:limit]
|
|
75
79
|
)
|
|
@@ -91,14 +95,30 @@ def iter_projects(content, repo_ver, limit=PACKAGES_LIMIT):
|
|
|
91
95
|
)
|
|
92
96
|
|
|
93
97
|
|
|
94
|
-
def
|
|
98
|
+
def _build_tag_fragment(filenames):
|
|
99
|
+
"""Extract sorted distinct build tags from wheel filenames for GUID stability.
|
|
100
|
+
|
|
101
|
+
Returns a fragment like `#builds=1,2` when build tags are present,
|
|
102
|
+
or an empty string for sdists and wheels without build tags.
|
|
103
|
+
"""
|
|
104
|
+
tags = set()
|
|
105
|
+
for fn in filenames or ():
|
|
106
|
+
m = _WHEEL_BUILD_TAG_RE.match(fn)
|
|
107
|
+
if m:
|
|
108
|
+
tags.add(m.group("build"))
|
|
109
|
+
if not tags:
|
|
110
|
+
return ""
|
|
111
|
+
return "#builds=" + ",".join(sorted(tags))
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _item_dict(title, link, description, author_email, pubdate, filenames=()):
|
|
95
115
|
return {
|
|
96
116
|
"title": sanitize_xml_text(title),
|
|
97
117
|
"link": link,
|
|
98
118
|
"description": sanitize_xml_text(description),
|
|
99
119
|
"author_email": format_author(author_email),
|
|
100
120
|
"pubdate": pubdate,
|
|
101
|
-
"unique_id": f"{link}
|
|
121
|
+
"unique_id": f"{link}{_build_tag_fragment(filenames)}",
|
|
102
122
|
}
|
|
103
123
|
|
|
104
124
|
|
|
@@ -139,6 +159,7 @@ def render_updates_feed(index_url, releases):
|
|
|
139
159
|
description=release["summary"],
|
|
140
160
|
author_email=release["author_email"],
|
|
141
161
|
pubdate=release["added_at"],
|
|
162
|
+
filenames=release.get("filenames", ()),
|
|
142
163
|
)
|
|
143
164
|
for release in releases
|
|
144
165
|
]
|
|
@@ -178,6 +199,7 @@ def render_project_releases_feed(index_url, project_name, releases):
|
|
|
178
199
|
description=release["summary"],
|
|
179
200
|
author_email=release["author_email"],
|
|
180
201
|
pubdate=release["added_at"],
|
|
202
|
+
filenames=release.get("filenames", ()),
|
|
181
203
|
)
|
|
182
204
|
for release in releases
|
|
183
205
|
]
|
|
@@ -35,6 +35,9 @@ class PackageMetadataSerializer(serializers.Serializer):
|
|
|
35
35
|
info = serializers.JSONField(help_text=_("Core metadata of the package"))
|
|
36
36
|
releases = serializers.JSONField(help_text=_("List of all the releases of the package"))
|
|
37
37
|
urls = serializers.JSONField()
|
|
38
|
+
vulnerabilities = serializers.JSONField(
|
|
39
|
+
help_text=_("Known vulnerabilities for the selected package version."),
|
|
40
|
+
)
|
|
38
41
|
|
|
39
42
|
|
|
40
43
|
class PackageUploadSerializer(serializers.Serializer):
|
pulp_python/app/pypi/views.py
CHANGED
|
@@ -621,7 +621,7 @@ class ProvenanceView(PyPIMixin, ViewSet):
|
|
|
621
621
|
def retrieve(self, request, path, package, version, filename):
|
|
622
622
|
"""Gets the provenance for a package."""
|
|
623
623
|
repo_ver, content = self.get_rvc()
|
|
624
|
-
if content:
|
|
624
|
+
if content is not None:
|
|
625
625
|
package_content = content.filter(
|
|
626
626
|
name_normalized=package, version=version, filename=filename
|
|
627
627
|
).first()
|
pulp_python/app/serializers.py
CHANGED
|
@@ -789,6 +789,11 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
|
|
|
789
789
|
help_text=_("Whether to sync available provenances for Python packages."),
|
|
790
790
|
default=False,
|
|
791
791
|
)
|
|
792
|
+
vulnerabilities = serializers.BooleanField(
|
|
793
|
+
required=False,
|
|
794
|
+
help_text=_("Whether to scan the new repository version for vulnerabilities after a sync."),
|
|
795
|
+
default=False,
|
|
796
|
+
)
|
|
792
797
|
|
|
793
798
|
def validate_includes(self, value):
|
|
794
799
|
"""Validates the includes"""
|
|
@@ -821,6 +826,7 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
|
|
|
821
826
|
"keep_latest_packages",
|
|
822
827
|
"exclude_platforms",
|
|
823
828
|
"provenance",
|
|
829
|
+
"vulnerabilities",
|
|
824
830
|
)
|
|
825
831
|
model = python_models.PythonRemote
|
|
826
832
|
|
|
@@ -6,5 +6,5 @@ from .publish import publish # noqa:F401
|
|
|
6
6
|
from .repair import repair # noqa:F401
|
|
7
7
|
from .sync import sync # noqa:F401
|
|
8
8
|
from .upload import upload, upload_group # noqa:F401
|
|
9
|
-
from .vulnerability_report import get_repo_version_content # noqa:F401
|
|
9
|
+
from .vulnerability_report import dispatch_scan, get_repo_version_content # noqa:F401
|
|
10
10
|
from .yank import aunyank_package, ayank_package # noqa:F401
|
pulp_python/app/tasks/publish.py
CHANGED
|
@@ -56,11 +56,14 @@ def write_simple_api(publication):
|
|
|
56
56
|
domain = get_domain()
|
|
57
57
|
simple_dir = "simple/"
|
|
58
58
|
os.mkdir(simple_dir)
|
|
59
|
+
# Secondary ORDER BY makes DISTINCT ON pick a stable display name when
|
|
60
|
+
# metadata names differ but canonicalize to the same name_normalized
|
|
61
|
+
# (e.g. msg-parser vs msg_parser).
|
|
59
62
|
project_names = (
|
|
60
63
|
python_models.PythonPackageContent.objects.filter(
|
|
61
64
|
pk__in=publication.repository_version.content, _pulp_domain=domain
|
|
62
65
|
)
|
|
63
|
-
.order_by("name_normalized")
|
|
66
|
+
.order_by("name_normalized", "name")
|
|
64
67
|
.values_list("name", flat=True)
|
|
65
68
|
.distinct("name_normalized")
|
|
66
69
|
)
|
|
@@ -81,7 +84,7 @@ def write_simple_api(publication):
|
|
|
81
84
|
packages = python_models.PythonPackageContent.objects.filter(
|
|
82
85
|
pk__in=publication.repository_version.content, _pulp_domain=domain
|
|
83
86
|
)
|
|
84
|
-
releases = packages.order_by("name_normalized").values("name", "filename", "sha256")
|
|
87
|
+
releases = packages.order_by("name_normalized", "filename").values("name", "filename", "sha256")
|
|
85
88
|
|
|
86
89
|
ind = 0
|
|
87
90
|
current_name = canonicalize_name(project_names[ind])
|
pulp_python/app/tasks/sync.py
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import json
|
|
2
3
|
import logging
|
|
3
4
|
from functools import partial
|
|
4
|
-
from urllib.parse import
|
|
5
|
+
from urllib.parse import urlparse
|
|
5
6
|
|
|
6
|
-
from aiohttp import ClientError, ClientResponseError
|
|
7
7
|
from bandersnatch.configuration import BandersnatchConfig
|
|
8
8
|
from bandersnatch.master import Master
|
|
9
9
|
from bandersnatch.mirror import Mirror
|
|
10
|
-
from lxml.etree import LxmlError
|
|
11
10
|
from packaging.requirements import Requirement
|
|
12
|
-
from pypi_simple import IndexPage
|
|
11
|
+
from pypi_simple import ACCEPT_ANY, IndexPage, UnsupportedContentTypeError
|
|
13
12
|
|
|
14
13
|
from pulpcore.plugin.download import HttpDownloader
|
|
15
14
|
from pulpcore.plugin.exceptions import SyncError
|
|
@@ -29,6 +28,7 @@ from pulp_python.app.models import (
|
|
|
29
28
|
PythonRemote,
|
|
30
29
|
)
|
|
31
30
|
from pulp_python.app.provenance import Provenance
|
|
31
|
+
from pulp_python.app.tasks.vulnerability_report import dispatch_scan
|
|
32
32
|
from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata
|
|
33
33
|
|
|
34
34
|
logger = logging.getLogger(__name__)
|
|
@@ -56,7 +56,9 @@ def sync(remote_pk, repository_pk, mirror):
|
|
|
56
56
|
raise SyncError("A remote must have a url attribute to sync.")
|
|
57
57
|
|
|
58
58
|
first_stage = PythonBanderStage(remote)
|
|
59
|
-
DeclarativeVersion(first_stage, repository, mirror).create()
|
|
59
|
+
new_version = DeclarativeVersion(first_stage, repository, mirror).create()
|
|
60
|
+
if new_version and remote.vulnerabilities:
|
|
61
|
+
dispatch_scan(repository, new_version)
|
|
60
62
|
|
|
61
63
|
|
|
62
64
|
def create_bandersnatch_config(remote):
|
|
@@ -169,41 +171,34 @@ class PulpMirror(Mirror):
|
|
|
169
171
|
|
|
170
172
|
async def determine_packages_to_sync(self):
|
|
171
173
|
"""
|
|
172
|
-
|
|
173
|
-
so try to get all of the packages from Mirror (hopefully PyPi)
|
|
174
|
+
Called when includes wasn't specified. List all projects from the remote's simple index.
|
|
174
175
|
"""
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
176
|
+
logger.info("Syncing all packages from %s", self.master.url)
|
|
177
|
+
url = f"{self.remote.url.rstrip('/')}/simple/"
|
|
178
|
+
downloader = self.remote.get_downloader(url=url)
|
|
179
|
+
result = await downloader.run(
|
|
180
|
+
extra_data={"request_kwargs": {"headers": {"Accept": ACCEPT_ANY}}}
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
content_type = result.headers.get("Content-Type", "text/html").partition(";")[0].lower()
|
|
184
|
+
with open(result.path, "rb") as f:
|
|
185
|
+
content = f.read()
|
|
186
|
+
|
|
187
|
+
if content_type == "application/vnd.pypi.simple.v1+json":
|
|
188
|
+
index = IndexPage.from_json_data(json.loads(content))
|
|
189
|
+
elif content_type in {"application/vnd.pypi.simple.v1+html", "text/html"}:
|
|
190
|
+
index = IndexPage.from_html(content)
|
|
191
|
+
else:
|
|
192
|
+
raise UnsupportedContentTypeError(url, content_type)
|
|
193
|
+
|
|
194
|
+
self.packages_to_sync = {project: 0 for project in index.projects}
|
|
195
|
+
last_serial = index.last_serial
|
|
196
|
+
if last_serial is None:
|
|
197
|
+
last_serial = result.headers.get(PYPI_LAST_SERIAL)
|
|
198
|
+
if last_serial is None:
|
|
199
|
+
self.target_serial = self.synced_serial
|
|
198
200
|
else:
|
|
199
|
-
|
|
200
|
-
url = urljoin(self.remote.url, "simple/")
|
|
201
|
-
downloader = self.remote.get_downloader(url=url)
|
|
202
|
-
result = await downloader.run()
|
|
203
|
-
with open(result.path) as f:
|
|
204
|
-
index = IndexPage.from_html(f.read())
|
|
205
|
-
self.packages_to_sync.update({p: 0 for p in index.projects})
|
|
206
|
-
self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
|
|
201
|
+
self.target_serial = max(self.synced_serial, int(last_serial))
|
|
207
202
|
|
|
208
203
|
self._filter_packages()
|
|
209
204
|
if self.target_serial:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from pulpcore.plugin.models import RepositoryVersion
|
|
2
2
|
from pulpcore.plugin.sync import sync_to_async_iterable
|
|
3
|
+
from pulpcore.plugin.tasking import check_content, dispatch
|
|
3
4
|
|
|
4
5
|
from pulp_python.app.models import PythonPackageContent
|
|
5
6
|
|
|
@@ -28,3 +29,13 @@ def _build_osv_data(name, ecosystem, version=None):
|
|
|
28
29
|
if version:
|
|
29
30
|
osv_data["version"] = version
|
|
30
31
|
return osv_data
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def dispatch_scan(repository, repository_version):
|
|
35
|
+
"""Dispatch a vulnerability scan for a repository version."""
|
|
36
|
+
func = f"{get_repo_version_content.__module__}.{get_repo_version_content.__name__}"
|
|
37
|
+
return dispatch(
|
|
38
|
+
check_content,
|
|
39
|
+
shared_resources=[repository],
|
|
40
|
+
args=[func, [str(repository_version.pk)]],
|
|
41
|
+
)
|
pulp_python/app/utils.py
CHANGED
|
@@ -21,9 +21,11 @@ from packaging.version import InvalidVersion, parse
|
|
|
21
21
|
from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
|
|
22
22
|
|
|
23
23
|
from pulpcore.plugin.exceptions import TimeoutException
|
|
24
|
-
from pulpcore.plugin.models import Artifact, Remote
|
|
24
|
+
from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport
|
|
25
25
|
from pulpcore.plugin.util import get_domain
|
|
26
26
|
|
|
27
|
+
from pulp_python.app.osv import osv_to_pypi_vulnerabilities
|
|
28
|
+
|
|
27
29
|
log = logging.getLogger(__name__)
|
|
28
30
|
|
|
29
31
|
|
|
@@ -377,6 +379,7 @@ def python_content_to_json(
|
|
|
377
379
|
last_serial: int
|
|
378
380
|
releases: Dict
|
|
379
381
|
urls: Dict
|
|
382
|
+
vulnerabilities: List
|
|
380
383
|
|
|
381
384
|
Returns None if version is specified but not found within content_query
|
|
382
385
|
"""
|
|
@@ -407,9 +410,27 @@ def python_content_to_json(
|
|
|
407
410
|
full_metadata["info"] = python_content_to_info(latest_content[0])
|
|
408
411
|
full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain)
|
|
409
412
|
full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain)
|
|
413
|
+
full_metadata["vulnerabilities"] = _vulnerabilities_for_content(
|
|
414
|
+
latest_content, repository_version
|
|
415
|
+
)
|
|
410
416
|
return full_metadata
|
|
411
417
|
|
|
412
418
|
|
|
419
|
+
def _vulnerabilities_for_content(contents, repository_version=None):
|
|
420
|
+
"""Load VulnerabilityReports scanned for this repository version and trim to Warehouse shape."""
|
|
421
|
+
if not contents or repository_version is None:
|
|
422
|
+
return []
|
|
423
|
+
reports = VulnerabilityReport.objects.filter(
|
|
424
|
+
content_id__in=[c.pk for c in contents],
|
|
425
|
+
repo_versions=repository_version,
|
|
426
|
+
)
|
|
427
|
+
merged = []
|
|
428
|
+
for vulns in reports.values_list("vulns", flat=True):
|
|
429
|
+
if vulns:
|
|
430
|
+
merged.extend(vulns)
|
|
431
|
+
return osv_to_pypi_vulnerabilities(merged)
|
|
432
|
+
|
|
433
|
+
|
|
413
434
|
def latest_content_version(all_content, version):
|
|
414
435
|
"""
|
|
415
436
|
Walks through the content list and finds the instances that are the latest version.
|
pulp_python/app/viewsets.py
CHANGED
|
@@ -26,7 +26,7 @@ from pulpcore.plugin.serializers import (
|
|
|
26
26
|
RepositoryAddRemoveContentSerializer,
|
|
27
27
|
RepositorySyncURLSerializer,
|
|
28
28
|
)
|
|
29
|
-
from pulpcore.plugin.tasking import
|
|
29
|
+
from pulpcore.plugin.tasking import dispatch
|
|
30
30
|
from pulpcore.plugin.util import extract_pk
|
|
31
31
|
|
|
32
32
|
from pulp_python.app import models as python_models
|
|
@@ -356,14 +356,7 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet):
|
|
|
356
356
|
Scan a repository version for vulnerabilities.
|
|
357
357
|
"""
|
|
358
358
|
repository_version = self.get_object()
|
|
359
|
-
|
|
360
|
-
f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}"
|
|
361
|
-
)
|
|
362
|
-
task = dispatch(
|
|
363
|
-
check_content,
|
|
364
|
-
shared_resources=[repository_version.repository],
|
|
365
|
-
args=[func, [repository_version.pk]],
|
|
366
|
-
)
|
|
359
|
+
task = tasks.dispatch_scan(repository_version.repository, repository_version)
|
|
367
360
|
return core_viewsets.OperationPostponedResponse(task, request)
|
|
368
361
|
|
|
369
362
|
|
|
@@ -48,7 +48,7 @@ def test_full_fixtures_to_pulp_sync(
|
|
|
48
48
|
):
|
|
49
49
|
"""
|
|
50
50
|
This test checks that Pulp can fully sync another Python Package repository that is not
|
|
51
|
-
PyPI. This
|
|
51
|
+
PyPI. This lists projects from the remote's Simple API index.
|
|
52
52
|
"""
|
|
53
53
|
# Repository we are syncing from is the fixtures (default url)
|
|
54
54
|
remote = python_remote_factory(includes=[], prereleases=True)
|
|
@@ -324,6 +324,7 @@ def assert_pypi_json(package):
|
|
|
324
324
|
package["releases"][version],
|
|
325
325
|
"Failed to match version",
|
|
326
326
|
)
|
|
327
|
+
assert package["vulnerabilities"] == []
|
|
327
328
|
|
|
328
329
|
|
|
329
330
|
def assert_download_info(expected, received, message="Failed to match"):
|
|
@@ -377,6 +378,8 @@ def test_upload_time_reflects_repo_addition(
|
|
|
377
378
|
# JSON API
|
|
378
379
|
json_resp = requests.get(urljoin(distro.base_url, "pypi/twine/json"))
|
|
379
380
|
assert json_resp.status_code == 200
|
|
380
|
-
|
|
381
|
+
package = json_resp.json()
|
|
382
|
+
json_time = datetime.fromisoformat(package["urls"][0]["upload_time"])
|
|
381
383
|
assert json_time > content_created
|
|
382
384
|
assert json_time == simple_time
|
|
385
|
+
assert package["vulnerabilities"] == []
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import zipfile
|
|
1
3
|
from urllib.parse import urljoin, urlsplit
|
|
2
4
|
from xml.etree import ElementTree as ET
|
|
3
5
|
|
|
@@ -20,6 +22,23 @@ TWINE_500_WHEEL_FILENAME = "twine-5.0.0-py3-none-any.whl"
|
|
|
20
22
|
TWINE_500_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), TWINE_500_WHEEL_FILENAME)
|
|
21
23
|
|
|
22
24
|
|
|
25
|
+
def _make_build_tagged_wheel(tmp_path, base_wheel_bytes, build_tag, platform="linux_x86_64"):
|
|
26
|
+
"""Repackage a wheel with a PEP 427 build tag and platform in the filename."""
|
|
27
|
+
filename = f"shelf_reader-0.1-{build_tag}-py2-none-{platform}.whl"
|
|
28
|
+
path = tmp_path / filename
|
|
29
|
+
with zipfile.ZipFile(io.BytesIO(base_wheel_bytes)) as src, zipfile.ZipFile(path, "w") as dst:
|
|
30
|
+
for item in src.infolist():
|
|
31
|
+
dst.writestr(item, src.read(item.filename))
|
|
32
|
+
# Pulp deduplicates content by sha256, so identical zip bytes with
|
|
33
|
+
# different filenames map to the same content unit. Inject a unique
|
|
34
|
+
# marker so each (build_tag, platform) combination gets its own sha256.
|
|
35
|
+
dst.writestr(
|
|
36
|
+
"shelf_reader/.build_marker",
|
|
37
|
+
f"build={build_tag} platform={platform}\n",
|
|
38
|
+
)
|
|
39
|
+
return str(path), filename
|
|
40
|
+
|
|
41
|
+
|
|
23
42
|
def _index_url(distro, bindings_cfg):
|
|
24
43
|
"""Build the index URL using the same origin the API client uses."""
|
|
25
44
|
path = urlsplit(distro.base_url).path
|
|
@@ -125,7 +144,7 @@ def test_pinned_version_feeds(
|
|
|
125
144
|
|
|
126
145
|
item = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))[0]
|
|
127
146
|
assert item.findtext("link").endswith("pypi/shelf-reader/0.1/json")
|
|
128
|
-
assert "pypi/shelf-reader/0.1/json
|
|
147
|
+
assert item.findtext("guid").endswith("pypi/shelf-reader/0.1/json")
|
|
129
148
|
|
|
130
149
|
python_content_factory(TWINE_WHEEL_FILENAME, url=TWINE_WHEEL_URL, repository=repo)
|
|
131
150
|
update_titles = _titles(_parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg)))
|
|
@@ -133,10 +152,10 @@ def test_pinned_version_feeds(
|
|
|
133
152
|
|
|
134
153
|
|
|
135
154
|
@pytest.mark.parallel
|
|
136
|
-
def
|
|
155
|
+
def test_guid_stable_when_adding_file_without_new_build_tag(
|
|
137
156
|
bindings_cfg, python_content_factory, python_empty_repo_distro
|
|
138
157
|
):
|
|
139
|
-
"""Adding a
|
|
158
|
+
"""Adding a file without a new build tag keeps the GUID stable but updates pubDate."""
|
|
140
159
|
repo, distro = python_empty_repo_distro()
|
|
141
160
|
|
|
142
161
|
python_content_factory(PYTHON_EGG_FILENAME, url=PYTHON_EGG_URL, repository=repo)
|
|
@@ -145,7 +164,7 @@ def test_new_file_for_existing_version_updates_guid(
|
|
|
145
164
|
assert len(items) == 1
|
|
146
165
|
first_guid = items[0].findtext("guid")
|
|
147
166
|
first_date = items[0].findtext("pubDate")
|
|
148
|
-
assert "pypi/shelf-reader/0.1/json"
|
|
167
|
+
assert first_guid.endswith("pypi/shelf-reader/0.1/json")
|
|
149
168
|
|
|
150
169
|
python_content_factory(PYTHON_WHEEL_FILENAME, url=PYTHON_WHEEL_URL, repository=repo)
|
|
151
170
|
|
|
@@ -154,11 +173,60 @@ def test_new_file_for_existing_version_updates_guid(
|
|
|
154
173
|
second_guid = items[0].findtext("guid")
|
|
155
174
|
second_date = items[0].findtext("pubDate")
|
|
156
175
|
|
|
157
|
-
assert second_guid
|
|
176
|
+
assert second_guid == first_guid
|
|
158
177
|
assert second_date >= first_date
|
|
159
178
|
|
|
160
179
|
release_items = _parse_items(
|
|
161
180
|
_get_feed(distro, "rss/project/shelf-reader/releases.xml", bindings_cfg)
|
|
162
181
|
)
|
|
163
182
|
assert len(release_items) == 1
|
|
164
|
-
assert release_items[0].findtext("guid") ==
|
|
183
|
+
assert release_items[0].findtext("guid") == first_guid
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@pytest.mark.parallel
|
|
187
|
+
def test_new_build_tag_changes_guid(
|
|
188
|
+
tmp_path, bindings_cfg, python_bindings, monitor_task, python_empty_repo_distro
|
|
189
|
+
):
|
|
190
|
+
"""A new build tag produces a new GUID; a new arch for the same tag does not."""
|
|
191
|
+
repo, distro = python_empty_repo_distro()
|
|
192
|
+
|
|
193
|
+
base_wheel = requests.get(PYTHON_WHEEL_URL, timeout=30).content
|
|
194
|
+
|
|
195
|
+
# Upload build tag 1 (x86_64)
|
|
196
|
+
path_1, name_1 = _make_build_tagged_wheel(tmp_path, base_wheel, "1")
|
|
197
|
+
task = python_bindings.ContentPackagesApi.create(
|
|
198
|
+
relative_path=name_1, file=path_1, repository=repo.pulp_href
|
|
199
|
+
).task
|
|
200
|
+
monitor_task(task)
|
|
201
|
+
|
|
202
|
+
items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
|
|
203
|
+
assert len(items) == 1
|
|
204
|
+
guid_after_build1 = items[0].findtext("guid")
|
|
205
|
+
assert "#builds=1" in guid_after_build1
|
|
206
|
+
|
|
207
|
+
# Upload build tag 2 (x86_64) -- different build, GUID must change
|
|
208
|
+
path_2, name_2 = _make_build_tagged_wheel(tmp_path, base_wheel, "2")
|
|
209
|
+
task = python_bindings.ContentPackagesApi.create(
|
|
210
|
+
relative_path=name_2, file=path_2, repository=repo.pulp_href
|
|
211
|
+
).task
|
|
212
|
+
monitor_task(task)
|
|
213
|
+
|
|
214
|
+
items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
|
|
215
|
+
assert len(items) == 1
|
|
216
|
+
guid_after_build2 = items[0].findtext("guid")
|
|
217
|
+
assert guid_after_build2 != guid_after_build1
|
|
218
|
+
assert "#builds=1,2" in guid_after_build2
|
|
219
|
+
|
|
220
|
+
# Upload build tag 2 again with a different arch -- same build tag, GUID must stay
|
|
221
|
+
path_2_arm, name_2_arm = _make_build_tagged_wheel(
|
|
222
|
+
tmp_path, base_wheel, "2", platform="linux_aarch64"
|
|
223
|
+
)
|
|
224
|
+
task = python_bindings.ContentPackagesApi.create(
|
|
225
|
+
relative_path=name_2_arm, file=path_2_arm, repository=repo.pulp_href
|
|
226
|
+
).task
|
|
227
|
+
monitor_task(task)
|
|
228
|
+
|
|
229
|
+
items = _parse_items(_get_feed(distro, "rss/updates.xml", bindings_cfg))
|
|
230
|
+
assert len(items) == 1
|
|
231
|
+
guid_after_build2_arm = items[0].findtext("guid")
|
|
232
|
+
assert guid_after_build2_arm == guid_after_build2
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
from urllib.parse import urljoin, urlsplit
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
from pulp_python.tests.functional.constants import (
|
|
7
|
+
PYPI_URL,
|
|
8
|
+
VULNERABILITY_REPORT_TEST_PACKAGE_NAME,
|
|
9
|
+
VULNERABILITY_REPORT_TEST_PACKAGES,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
WAREHOUSE_VULN_KEYS = {
|
|
13
|
+
"id",
|
|
14
|
+
"source",
|
|
15
|
+
"link",
|
|
16
|
+
"aliases",
|
|
17
|
+
"details",
|
|
18
|
+
"summary",
|
|
19
|
+
"fixed_in",
|
|
20
|
+
"withdrawn",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _index_url(distro, bindings_cfg):
|
|
25
|
+
"""Build the index URL using the same origin the API client uses."""
|
|
26
|
+
path = urlsplit(distro.base_url).path
|
|
27
|
+
if not path.endswith("/"):
|
|
28
|
+
path += "/"
|
|
29
|
+
return bindings_cfg.host.rstrip("/") + path
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _wait_for_child_tasks(pulpcore_bindings, monitor_task, parent_task):
|
|
33
|
+
parent = pulpcore_bindings.TasksApi.read(parent_task.pulp_href)
|
|
34
|
+
children = parent.child_tasks or []
|
|
35
|
+
assert children, "expected a follow-up vulnerability scan task"
|
|
36
|
+
for child in children:
|
|
37
|
+
href = child if isinstance(child, str) else child.pulp_href
|
|
38
|
+
monitor_task(href)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _assert_warehouse_vulnerabilities(package):
|
|
42
|
+
vulns = package["vulnerabilities"]
|
|
43
|
+
assert vulns
|
|
44
|
+
ids = [vuln["id"] for vuln in vulns]
|
|
45
|
+
assert len(ids) == len(set(ids))
|
|
46
|
+
for vuln in vulns:
|
|
47
|
+
assert WAREHOUSE_VULN_KEYS <= vuln.keys()
|
|
48
|
+
assert vuln["source"] == "osv"
|
|
49
|
+
assert vuln["id"]
|
|
50
|
+
assert vuln["link"] == f"https://osv.dev/vulnerability/{vuln['id']}"
|
|
51
|
+
assert isinstance(vuln["aliases"], list)
|
|
52
|
+
assert isinstance(vuln["fixed_in"], list)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.mark.parallel
|
|
56
|
+
def test_pypi_json_vulnerabilities_from_manual_scan(
|
|
57
|
+
bindings_cfg,
|
|
58
|
+
pulpcore_bindings,
|
|
59
|
+
python_bindings,
|
|
60
|
+
python_remote_factory,
|
|
61
|
+
python_repo,
|
|
62
|
+
python_repo_factory,
|
|
63
|
+
python_distribution_factory,
|
|
64
|
+
monitor_task,
|
|
65
|
+
):
|
|
66
|
+
"""A remote without vulnerabilities does not scan; a later scan fills the JSON API."""
|
|
67
|
+
remote = python_remote_factory(url=PYPI_URL, includes=VULNERABILITY_REPORT_TEST_PACKAGES)
|
|
68
|
+
sync_task = monitor_task(
|
|
69
|
+
python_bindings.RepositoriesPythonApi.sync(
|
|
70
|
+
python_repo.pulp_href, dict(remote=remote.pulp_href)
|
|
71
|
+
).task
|
|
72
|
+
)
|
|
73
|
+
assert not sync_task.child_tasks
|
|
74
|
+
|
|
75
|
+
repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href)
|
|
76
|
+
distro = python_distribution_factory(repository=repo)
|
|
77
|
+
name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower()
|
|
78
|
+
index = _index_url(distro, bindings_cfg)
|
|
79
|
+
version_url = urljoin(index, f"pypi/{name}/5.2.1/json")
|
|
80
|
+
project_url = urljoin(index, f"pypi/{name}/json")
|
|
81
|
+
|
|
82
|
+
scan_task = python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href)
|
|
83
|
+
monitor_task(scan_task.task)
|
|
84
|
+
|
|
85
|
+
project = requests.get(project_url).json()
|
|
86
|
+
version = requests.get(version_url).json()
|
|
87
|
+
_assert_warehouse_vulnerabilities(project)
|
|
88
|
+
_assert_warehouse_vulnerabilities(version)
|
|
89
|
+
assert [v["id"] for v in project["vulnerabilities"]] == [
|
|
90
|
+
v["id"] for v in version["vulnerabilities"]
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
packages = python_bindings.ContentPackagesApi.list(
|
|
94
|
+
name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME,
|
|
95
|
+
repository_version=repo.latest_version_href,
|
|
96
|
+
)
|
|
97
|
+
assert packages.count >= 2
|
|
98
|
+
hrefs = []
|
|
99
|
+
for content in packages.results:
|
|
100
|
+
assert content.vuln_report is not None
|
|
101
|
+
report = pulpcore_bindings.VulnReportApi.read(content.vuln_report)
|
|
102
|
+
assert report.vulns
|
|
103
|
+
assert "affected" in report.vulns[0]
|
|
104
|
+
hrefs.append(content.pulp_href)
|
|
105
|
+
|
|
106
|
+
other = python_repo_factory()
|
|
107
|
+
monitor_task(
|
|
108
|
+
python_bindings.RepositoriesPythonApi.modify(
|
|
109
|
+
other.pulp_href, {"add_content_units": hrefs}
|
|
110
|
+
).task
|
|
111
|
+
)
|
|
112
|
+
other = python_bindings.RepositoriesPythonApi.read(other.pulp_href)
|
|
113
|
+
other_distro = python_distribution_factory(repository=other)
|
|
114
|
+
other_json = requests.get(
|
|
115
|
+
urljoin(_index_url(other_distro, bindings_cfg), f"pypi/{name}/json")
|
|
116
|
+
).json()
|
|
117
|
+
assert other_json["vulnerabilities"] == []
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@pytest.mark.parallel
|
|
121
|
+
def test_sync_dispatches_vulnerability_scan(
|
|
122
|
+
bindings_cfg,
|
|
123
|
+
pulpcore_bindings,
|
|
124
|
+
python_bindings,
|
|
125
|
+
python_remote_factory,
|
|
126
|
+
python_repo,
|
|
127
|
+
python_distribution_factory,
|
|
128
|
+
monitor_task,
|
|
129
|
+
):
|
|
130
|
+
"""A remote with vulnerabilities=True scans the new repository version after sync."""
|
|
131
|
+
remote = python_remote_factory(
|
|
132
|
+
url=PYPI_URL,
|
|
133
|
+
includes=VULNERABILITY_REPORT_TEST_PACKAGES,
|
|
134
|
+
vulnerabilities=True,
|
|
135
|
+
)
|
|
136
|
+
sync_task = monitor_task(
|
|
137
|
+
python_bindings.RepositoriesPythonApi.sync(
|
|
138
|
+
python_repo.pulp_href, dict(remote=remote.pulp_href)
|
|
139
|
+
).task
|
|
140
|
+
)
|
|
141
|
+
_wait_for_child_tasks(pulpcore_bindings, monitor_task, sync_task)
|
|
142
|
+
|
|
143
|
+
repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href)
|
|
144
|
+
distro = python_distribution_factory(repository=repo)
|
|
145
|
+
name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower()
|
|
146
|
+
package = requests.get(urljoin(_index_url(distro, bindings_cfg), f"pypi/{name}/json")).json()
|
|
147
|
+
_assert_warehouse_vulnerabilities(package)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@pytest.mark.parallel
|
|
151
|
+
def test_uploaded_package_json_vulnerabilities_empty_until_scan(
|
|
152
|
+
bindings_cfg,
|
|
153
|
+
python_bindings,
|
|
154
|
+
python_content_factory,
|
|
155
|
+
python_repo_factory,
|
|
156
|
+
python_distribution_factory,
|
|
157
|
+
monitor_task,
|
|
158
|
+
):
|
|
159
|
+
"""Uploaded packages have an empty vulnerabilities list until a scan runs."""
|
|
160
|
+
repo = python_repo_factory()
|
|
161
|
+
python_content_factory(repository=repo)
|
|
162
|
+
distro = python_distribution_factory(repository=repo)
|
|
163
|
+
url = urljoin(_index_url(distro, bindings_cfg), "pypi/shelf-reader/json")
|
|
164
|
+
|
|
165
|
+
package = requests.get(url).json()
|
|
166
|
+
assert package["vulnerabilities"] == []
|
|
167
|
+
|
|
168
|
+
repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href)
|
|
169
|
+
monitor_task(python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href).task)
|
|
170
|
+
package = requests.get(url).json()
|
|
171
|
+
assert isinstance(package["vulnerabilities"], list)
|
|
@@ -28,21 +28,18 @@ def test_vulnerability_report(
|
|
|
28
28
|
)
|
|
29
29
|
monitor_task(response.task)
|
|
30
30
|
|
|
31
|
-
# checks
|
|
32
|
-
vulns_list = pulpcore_bindings.VulnReportApi.list()
|
|
33
|
-
assert len(vulns_list.results) > 0
|
|
34
|
-
for results in vulns_list.results:
|
|
35
|
-
assert len(results.vulns) > 0
|
|
36
|
-
for vuln in results.vulns:
|
|
37
|
-
assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in (
|
|
38
|
-
affected["package"]["name"] for affected in vuln["affected"]
|
|
39
|
-
)
|
|
40
|
-
|
|
41
31
|
repo_version = python_bindings.RepositoriesPythonVersionsApi.read(latest_version_href)
|
|
42
32
|
assert repo_version.vuln_report is not None
|
|
43
33
|
|
|
44
34
|
python_packages = python_bindings.ContentPackagesApi.list(
|
|
45
35
|
name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, repository_version=latest_version_href
|
|
46
36
|
)
|
|
37
|
+
assert python_packages.count > 0
|
|
47
38
|
for content in python_packages.results:
|
|
48
39
|
assert content.vuln_report is not None
|
|
40
|
+
report = pulpcore_bindings.VulnReportApi.read(content.vuln_report)
|
|
41
|
+
assert len(report.vulns) > 0
|
|
42
|
+
for vuln in report.vulns:
|
|
43
|
+
assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in (
|
|
44
|
+
affected["package"]["name"] for affected in vuln["affected"]
|
|
45
|
+
)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from pulp_python.app.pypi.feeds import _build_tag_fragment
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@pytest.mark.parametrize(
|
|
7
|
+
"filenames, expected",
|
|
8
|
+
[
|
|
9
|
+
([], ""),
|
|
10
|
+
(["shelf-reader-0.1.tar.gz"], ""),
|
|
11
|
+
(["shelf_reader-0.1-py2-none-any.whl"], ""),
|
|
12
|
+
(
|
|
13
|
+
["docling_parse-7.19.1-1-cp312-cp312-linux_x86_64.whl"],
|
|
14
|
+
"#builds=1",
|
|
15
|
+
),
|
|
16
|
+
(
|
|
17
|
+
[
|
|
18
|
+
"docling_parse-7.19.1-1-cp312-cp312-linux_x86_64.whl",
|
|
19
|
+
"docling_parse-7.19.1-1-cp312-cp312-linux_aarch64.whl",
|
|
20
|
+
"docling_parse-7.19.1-1-cp312-cp312-linux_ppc64le.whl",
|
|
21
|
+
],
|
|
22
|
+
"#builds=1",
|
|
23
|
+
),
|
|
24
|
+
(
|
|
25
|
+
[
|
|
26
|
+
"ctranslate2-4.5.0-1-cp312-cp312-linux_x86_64.whl",
|
|
27
|
+
"ctranslate2-4.5.0-2-cp312-cp312-linux_x86_64.whl",
|
|
28
|
+
],
|
|
29
|
+
"#builds=1,2",
|
|
30
|
+
),
|
|
31
|
+
(
|
|
32
|
+
[
|
|
33
|
+
"foo-1.0-1-cp312-cp312-linux_x86_64.whl",
|
|
34
|
+
"foo-1.0.tar.gz",
|
|
35
|
+
],
|
|
36
|
+
"#builds=1",
|
|
37
|
+
),
|
|
38
|
+
(None, ""),
|
|
39
|
+
],
|
|
40
|
+
ids=[
|
|
41
|
+
"empty",
|
|
42
|
+
"sdist-only",
|
|
43
|
+
"wheel-no-build-tag",
|
|
44
|
+
"single-build-tag",
|
|
45
|
+
"same-build-tag-multi-arch",
|
|
46
|
+
"two-build-tags",
|
|
47
|
+
"mixed-sdist-and-tagged-wheel",
|
|
48
|
+
"none",
|
|
49
|
+
],
|
|
50
|
+
)
|
|
51
|
+
def test_build_tag_fragment(filenames, expected):
|
|
52
|
+
assert _build_tag_fragment(filenames) == expected
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from pulp_python.app.osv import osv_to_pypi_vulnerabilities
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_osv_to_pypi_maps_fixed_versions_and_skips_git_shas():
|
|
5
|
+
vulns = [
|
|
6
|
+
{
|
|
7
|
+
"id": "PYSEC-2020-35",
|
|
8
|
+
"aliases": ["CVE-2020-7471"],
|
|
9
|
+
"details": "SQL injection",
|
|
10
|
+
"summary": None,
|
|
11
|
+
"withdrawn": None,
|
|
12
|
+
"affected": [
|
|
13
|
+
{
|
|
14
|
+
"package": {"name": "django", "ecosystem": "PyPI"},
|
|
15
|
+
"ranges": [
|
|
16
|
+
{
|
|
17
|
+
"type": "ECOSYSTEM",
|
|
18
|
+
"events": [
|
|
19
|
+
{"introduced": "3.0"},
|
|
20
|
+
{"fixed": "3.0.3"},
|
|
21
|
+
{"fixed": "eb31d845323618d688ad429479c6dda973056136"},
|
|
22
|
+
],
|
|
23
|
+
}
|
|
24
|
+
],
|
|
25
|
+
}
|
|
26
|
+
],
|
|
27
|
+
}
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
result = osv_to_pypi_vulnerabilities(vulns)
|
|
31
|
+
|
|
32
|
+
assert result == [
|
|
33
|
+
{
|
|
34
|
+
"id": "PYSEC-2020-35",
|
|
35
|
+
"source": "osv",
|
|
36
|
+
"link": "https://osv.dev/vulnerability/PYSEC-2020-35",
|
|
37
|
+
"aliases": ["CVE-2020-7471"],
|
|
38
|
+
"details": "SQL injection",
|
|
39
|
+
"summary": None,
|
|
40
|
+
"fixed_in": ["3.0.3"],
|
|
41
|
+
"withdrawn": None,
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_osv_to_pypi_deduplicates_by_id():
|
|
47
|
+
vulns = [
|
|
48
|
+
{"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "first"},
|
|
49
|
+
{"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "duplicate"},
|
|
50
|
+
{"id": "GHSA-bbbb", "aliases": [], "details": "other"},
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
result = osv_to_pypi_vulnerabilities(vulns)
|
|
54
|
+
|
|
55
|
+
assert [v["id"] for v in result] == ["GHSA-aaaa", "GHSA-bbbb"]
|
|
56
|
+
assert result[0]["details"] == "first"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_osv_to_pypi_preserves_withdrawn_and_missing_summary():
|
|
60
|
+
vulns = [
|
|
61
|
+
{
|
|
62
|
+
"id": "PYSEC-2022-XXX",
|
|
63
|
+
"aliases": ["CVE-2022-XXXXX"],
|
|
64
|
+
"details": "A long description.",
|
|
65
|
+
"withdrawn": "2022-06-28T16:39:06Z",
|
|
66
|
+
"affected": [
|
|
67
|
+
{
|
|
68
|
+
"ranges": [{"events": [{"fixed": "1.2.3"}]}],
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
result = osv_to_pypi_vulnerabilities(vulns)
|
|
75
|
+
|
|
76
|
+
assert result[0]["summary"] is None
|
|
77
|
+
assert result[0]["withdrawn"] == "2022-06-28T16:39:06Z"
|
|
78
|
+
assert result[0]["fixed_in"] == ["1.2.3"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_osv_to_pypi_empty_and_missing_ids():
|
|
82
|
+
assert osv_to_pypi_vulnerabilities(None) == []
|
|
83
|
+
assert osv_to_pypi_vulnerabilities([]) == []
|
|
84
|
+
assert osv_to_pypi_vulnerabilities([{"details": "no id"}]) == []
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pulp-python
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.37.0
|
|
4
4
|
Summary: pulp-python plugin for the Pulp Project
|
|
5
5
|
Author-email: Pulp Team <pulp-list@redhat.com>
|
|
6
6
|
Project-URL: Homepage, https://pulpproject.org
|
|
@@ -14,15 +14,14 @@ Classifier: Development Status :: 5 - Production/Stable
|
|
|
14
14
|
Classifier: Framework :: Django
|
|
15
15
|
Classifier: Programming Language :: Python
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
-
Requires-Python: >=3.
|
|
19
|
+
Requires-Python: >=3.12
|
|
21
20
|
Description-Content-Type: text/markdown
|
|
22
21
|
License-File: LICENSE
|
|
23
22
|
Requires-Dist: pulpcore<3.130,>=3.105.0
|
|
24
23
|
Requires-Dist: pkginfo<1.13.0,>=1.12.0
|
|
25
|
-
Requires-Dist: bandersnatch<
|
|
24
|
+
Requires-Dist: bandersnatch<8.1,>=8.0.0
|
|
26
25
|
Requires-Dist: pypi-simple<2.0,>=1.8.0
|
|
27
26
|
Requires-Dist: pypi-attestations==0.0.28
|
|
28
27
|
Dynamic: license-file
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
pulp_python/__init__.py,sha256=GIuTLoBTc-07dSLJUh8xrZPRz8x-jJ61pfR0J1IjnzI,65
|
|
2
2
|
pulp_python/pytest_plugin.py,sha256=8Ko-O87JGPC79h8-Ecw2gJLxa6q1Ng8wD6WxbinOnVs,8573
|
|
3
|
-
pulp_python/app/__init__.py,sha256=
|
|
3
|
+
pulp_python/app/__init__.py,sha256=W_Rl5Uwy5-We5S8YHNGyCqhrrgVRWSeSdnIGSy_ak3k,2552
|
|
4
4
|
pulp_python/app/cache.py,sha256=Zl2nb4uKyPYp5a3FBrdRZ_-cXfofNMQmIRaQ6dUZZ1g,1037
|
|
5
5
|
pulp_python/app/exceptions.py,sha256=mPNcWyuzF0XeOPybm-G6oDKtRqvfj4jyp41iaYqcqkA,1314
|
|
6
6
|
pulp_python/app/global_access_conditions.py,sha256=MZJtyoVsr-4hRaty6mKDqh3caOHd5UKJjEWLV2crOLs,1080
|
|
7
7
|
pulp_python/app/modelresource.py,sha256=4SFAdqk6lozi_cZz4uqDIqhqPAZF-7l5jJwPn-xGZFs,1249
|
|
8
|
-
pulp_python/app/models.py,sha256=
|
|
8
|
+
pulp_python/app/models.py,sha256=EJApErZw2LznTtxRQTEq7X9OcOOhzoDgxl1jMfuOEn4,21116
|
|
9
|
+
pulp_python/app/osv.py,sha256=-JIJBtVjEn-KE2qcmVuWCMAhWEGve_uNNxKjccmQwFw,1406
|
|
9
10
|
pulp_python/app/provenance.py,sha256=iyhkuNahHiTDK0Djrd4-PlgErA5SJVI0uQOIPj46tEI,2352
|
|
10
11
|
pulp_python/app/replica.py,sha256=qiWRP7tM_v4yP_XLIQfumfGolru-Jt6ZA0KVb-9g2cA,1882
|
|
11
|
-
pulp_python/app/serializers.py,sha256=
|
|
12
|
+
pulp_python/app/serializers.py,sha256=RKWnTq8uuC_Y9RlPljvqfnYaMYk7dw1PLn3IQe7SGQg,35343
|
|
12
13
|
pulp_python/app/settings.py,sha256=Cyc_p6U0HQjKpyrRL6JFrK3R7RMQJ9MAgNMJCfzPEiA,255
|
|
13
14
|
pulp_python/app/urls.py,sha256=szPHGWrxaWwMZ0QEsftAyFwisEMFfusfDS4lcYEt4nU,2064
|
|
14
|
-
pulp_python/app/utils.py,sha256=
|
|
15
|
-
pulp_python/app/viewsets.py,sha256=
|
|
15
|
+
pulp_python/app/utils.py,sha256=OPj15MXMfEIROo096h0V9j060Kbl1NZygc4PbYsqWvs,28576
|
|
16
|
+
pulp_python/app/viewsets.py,sha256=hFgp-tNM9N3doty1pX-hndZgHarLDtOISqRW5KuAVgA,36154
|
|
16
17
|
pulp_python/app/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
18
|
pulp_python/app/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
19
|
pulp_python/app/management/commands/repair-python-metadata.py,sha256=tHPE4SGF93OpUDIfI7u0Mu5eXaxqd_xQaNWRZVkV-90,4550
|
|
@@ -41,17 +42,18 @@ pulp_python/app/migrations/0021_pythonrepository_upload_duplicate_filenames.py,s
|
|
|
41
42
|
pulp_python/app/migrations/0022_pythonblocklistentry.py,sha256=EbtjZuN65myTAHVWrJYbt4mWgWL7EElEYs1XulHB6Bo,1732
|
|
42
43
|
pulp_python/app/migrations/0023_packageyank.py,sha256=357fqSNK-D1OvI9RynnyMMrXJtdcjoEg5j4GteqiESU,1590
|
|
43
44
|
pulp_python/app/migrations/0024_pythonrepository_error_on_reject.py,sha256=p10wp0OEVA7m-ZpuzlqsJg9dEbf4cF6kow9AfK37JU8,348
|
|
45
|
+
pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py,sha256=IinhUzV3Pw0yueZp9c7MYpopHNSpevRXp_b0sBESSh4,366
|
|
44
46
|
pulp_python/app/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
47
|
pulp_python/app/pypi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
|
-
pulp_python/app/pypi/feeds.py,sha256=
|
|
47
|
-
pulp_python/app/pypi/serializers.py,sha256=
|
|
48
|
-
pulp_python/app/pypi/views.py,sha256=
|
|
49
|
-
pulp_python/app/tasks/__init__.py,sha256=
|
|
50
|
-
pulp_python/app/tasks/publish.py,sha256=
|
|
48
|
+
pulp_python/app/pypi/feeds.py,sha256=bzc_VWJmnNC-GkdC2iWYIQVL-K5hUEa19f5WYidaNS0,9864
|
|
49
|
+
pulp_python/app/pypi/serializers.py,sha256=MgHHprb9xIorjfn0byXpuzRjRuxd9bmKcQpF4vvYFzY,5732
|
|
50
|
+
pulp_python/app/pypi/views.py,sha256=tKGDlsbjv4ni1cFXxOgqqDbeJJBou03MAcvzzarPp2Y,27435
|
|
51
|
+
pulp_python/app/tasks/__init__.py,sha256=tfVSAlgIDvKGgmimCBmSVJXOyLRUGRU6UuqqE6sxWnQ,361
|
|
52
|
+
pulp_python/app/tasks/publish.py,sha256=zrAZCs7sAs11oZ0RdvYSQBEgnOkucYisEnFK1XjvWzY,4541
|
|
51
53
|
pulp_python/app/tasks/repair.py,sha256=suvpn7VLGx_B-mixjQWdiHPDy8FclLQ6IwjcL9PeIpU,12603
|
|
52
|
-
pulp_python/app/tasks/sync.py,sha256=
|
|
54
|
+
pulp_python/app/tasks/sync.py,sha256=umt6Bkt5KAGT6wVe4yGvIxG6bUz3YhlJVpWvQh67gGk,12883
|
|
53
55
|
pulp_python/app/tasks/upload.py,sha256=HBOknlsAb0mlE-2dJsalH_8JUZwrKk4AsVibmVJf2aQ,6102
|
|
54
|
-
pulp_python/app/tasks/vulnerability_report.py,sha256=
|
|
56
|
+
pulp_python/app/tasks/vulnerability_report.py,sha256=ir6f2qQJ_kvKURBrS2BQSOd_yMo9zAobUvzYqv4hNjg,1533
|
|
55
57
|
pulp_python/app/tasks/yank.py,sha256=VU16uLkA2-CUJe7q_u87AdkDH6gqWCwsHyc36wH35rc,2200
|
|
56
58
|
pulp_python/app/webserver_snippets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
57
59
|
pulp_python/app/webserver_snippets/apache.conf,sha256=3frHSl2YV_8pJPscaFxMVo7HmxGJdb8XVmfdLtCxzoA,97
|
|
@@ -69,11 +71,12 @@ pulp_python/tests/functional/api/test_crud_content_unit.py,sha256=WUB4ngHI7fslTn
|
|
|
69
71
|
pulp_python/tests/functional/api/test_crud_publications.py,sha256=3cuSDC9C9M1opt0lh5ObGj8t7gq5jUW-RWj6MrA4Teo,5647
|
|
70
72
|
pulp_python/tests/functional/api/test_crud_remotes.py,sha256=clFFhgG5udyKzWOLwxpmwjhiVz8r5JEPAQHkztcAf9w,5812
|
|
71
73
|
pulp_python/tests/functional/api/test_domains.py,sha256=JtgQq1HwIm2rLCy3ZEq9voBLrGQdmqNE72WIhA-rUts,10418
|
|
72
|
-
pulp_python/tests/functional/api/test_download_content.py,sha256=
|
|
74
|
+
pulp_python/tests/functional/api/test_download_content.py,sha256=L_Q1TFg98Oe-i4iM9vsc1utnzCeHSNn4BE4a15VAk38,4939
|
|
73
75
|
pulp_python/tests/functional/api/test_export_import.py,sha256=rHns9wdaeP-vtfW_qoGHU9EVuJ4YuWE0-rjl_8otAgU,4530
|
|
74
76
|
pulp_python/tests/functional/api/test_full_mirror.py,sha256=v0nSlGKGSycFqZM4opCTHVtbmCfo_Y5y3k_6Rm7eBF8,12058
|
|
75
|
-
pulp_python/tests/functional/api/test_pypi_apis.py,sha256=
|
|
76
|
-
pulp_python/tests/functional/api/test_pypi_feeds.py,sha256=
|
|
77
|
+
pulp_python/tests/functional/api/test_pypi_apis.py,sha256=UZ87mFEQYm7WWKBQQYYFVxtfRTi7prH6hEKoeqwQvPc,13757
|
|
78
|
+
pulp_python/tests/functional/api/test_pypi_feeds.py,sha256=nK0MbpqIlPDQtfZeWyqDc8mKN9qJeWLRQeo0S5wo5GY,9475
|
|
79
|
+
pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py,sha256=uSH2-EflYir85rzG4fjp43x1vw-CBinA5sJ4irzI4I8,5836
|
|
77
80
|
pulp_python/tests/functional/api/test_pypi_simple_api.py,sha256=N01RnpoLbPE1hr3blz60Tmn8gWvixzs9u1Hh3eGfeyg,7336
|
|
78
81
|
pulp_python/tests/functional/api/test_rbac.py,sha256=-xNWqvKKU-v1uC6tCRGBK0aWaff25K9C-AfzQkdHhhI,10513
|
|
79
82
|
pulp_python/tests/functional/api/test_repair.py,sha256=4FR7jx_LA2K-pnRJXKkhq-1uUoWXST1leZ9XdyhGM2U,12909
|
|
@@ -81,13 +84,15 @@ pulp_python/tests/functional/api/test_simple_cache.py,sha256=tql5Isn-ijThFm8CNzc
|
|
|
81
84
|
pulp_python/tests/functional/api/test_sync.py,sha256=TTHR1CpZeRoD97RKxj2pZPsP0OS9ibYKGPOh1WfKISg,13801
|
|
82
85
|
pulp_python/tests/functional/api/test_upload.py,sha256=f21Li9agJ5fdBqErVg1j6v5FDVdazB5mNClmGOjLUq8,5981
|
|
83
86
|
pulp_python/tests/functional/api/test_version_specifier_filter.py,sha256=nmnBbhwxCEmcHOyFbS1uvemar67sONBGM9SsNn79LCc,1063
|
|
84
|
-
pulp_python/tests/functional/api/test_vulnerability_report.py,sha256=
|
|
87
|
+
pulp_python/tests/functional/api/test_vulnerability_report.py,sha256=70KLOZcA4k0rEagZaYHOESZWZbJ_jHlox8heWkmaLMY,1692
|
|
85
88
|
pulp_python/tests/functional/api/test_yank.py,sha256=qyZbAJ6bJO-4LPkVfVxKuWFjypJ8BmBrCnLo0luAhVk,11913
|
|
86
89
|
pulp_python/tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
90
|
+
pulp_python/tests/unit/test_feeds.py,sha256=EKrBBHPfrfUozckWcG9zJQSGJuU2uToofZzfC6oKT64,1400
|
|
87
91
|
pulp_python/tests/unit/test_models.py,sha256=TBI0yKsrdbnJSPeBFfxSqhXK7zaNvR6qg5JehGH3Pds,229
|
|
88
|
-
pulp_python
|
|
89
|
-
pulp_python-3.
|
|
90
|
-
pulp_python-3.
|
|
91
|
-
pulp_python-3.
|
|
92
|
-
pulp_python-3.
|
|
93
|
-
pulp_python-3.
|
|
92
|
+
pulp_python/tests/unit/test_vulnerabilities.py,sha256=mdAbc2scATqWLwnVnLTdOCF76wiFWqcPp_xORUY6drY,2590
|
|
93
|
+
pulp_python-3.37.0.dist-info/licenses/LICENSE,sha256=2ylvL381vKOhdO-w6zkrOxe9lLNBhRQpo9_0EbHC_HM,18046
|
|
94
|
+
pulp_python-3.37.0.dist-info/METADATA,sha256=wu477qGOQR-AwKNzjsLoH0ha3QHurs2HXo3ib3LxKjc,1693
|
|
95
|
+
pulp_python-3.37.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
96
|
+
pulp_python-3.37.0.dist-info/entry_points.txt,sha256=HvqLEXjw_dS5jqAwnE5JiRZFE6f-y5SRtitKLPml2To,115
|
|
97
|
+
pulp_python-3.37.0.dist-info/top_level.txt,sha256=X0hXgXc_bpbiKqVrkt8jD5_QEiQviKbHDwveQcOcJjo,12
|
|
98
|
+
pulp_python-3.37.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|