pulpcore 3.84.0__py3-none-any.whl → 3.85.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pulpcore might be problematic. Click here for more details.
- pulp_certguard/app/__init__.py +1 -1
- pulp_certguard/app/models.py +7 -26
- pulp_certguard/app/serializers.py +0 -2
- pulp_certguard/rhsm/__init__.py +4 -0
- pulp_certguard/rhsm/rhsm_check_path.py +198 -0
- pulp_certguard/tests/unit/certdata.py +249 -0
- pulp_certguard/tests/unit/test_rhsm_check_path.py +213 -0
- pulp_file/app/__init__.py +1 -1
- pulp_file/app/migrations/0001_initial_squashed_0016_add_domain.py +0 -20
- pulp_file/app/migrations/0017_alter_filealternatecontentsource_alternatecontentsource_ptr_and_more.py +1 -1
- pulpcore/app/apps.py +2 -12
- pulpcore/app/entrypoint.py +22 -22
- pulpcore/app/migrations/0001_squashed_0090_char_to_text_field.py +0 -95
- pulpcore/app/migrations/0091_systemid.py +1 -1
- pulp_file/app/migrations/0006_delete_filefilesystemexporter.py → pulpcore/app/migrations/0136_delete_basedistribution.py +3 -3
- pulpcore/app/migrations/0137_appstatus.py +33 -0
- pulpcore/app/migrations/0138_vulnerabilityreport.py +33 -0
- pulpcore/app/models/__init__.py +4 -1
- pulpcore/app/models/publication.py +0 -41
- pulpcore/app/models/status.py +145 -0
- pulpcore/app/models/task.py +1 -0
- pulpcore/app/models/vulnerability_report.py +34 -0
- pulpcore/app/serializers/__init__.py +1 -0
- pulpcore/app/serializers/content.py +13 -1
- pulpcore/app/serializers/repository.py +8 -1
- pulpcore/app/serializers/vulnerability_report.py +27 -0
- pulpcore/app/settings.py +14 -47
- pulpcore/app/tasks/__init__.py +2 -0
- pulpcore/app/tasks/vulnerability_report.py +159 -0
- pulpcore/app/viewsets/__init__.py +1 -0
- pulpcore/app/viewsets/vulnerability_report.py +20 -0
- pulpcore/constants.py +4 -0
- pulpcore/content/__init__.py +23 -22
- pulpcore/content/handler.py +5 -2
- pulpcore/migrations.py +38 -11
- pulpcore/openapi/__init__.py +8 -0
- pulpcore/plugin/models/__init__.py +2 -0
- pulpcore/plugin/serializers/__init__.py +2 -0
- pulpcore/plugin/tasking.py +2 -0
- pulpcore/plugin/viewsets/__init__.py +2 -0
- pulpcore/pytest_plugin.py +21 -21
- pulpcore/tasking/worker.py +38 -35
- pulpcore/tests/functional/api/test_auth.py +18 -3
- pulpcore/tests/functional/api/test_openapi_schema.py +32 -15
- pulpcore/tests/functional/api/using_plugin/test_checkpoint.py +23 -1
- pulpcore/tests/functional/api/using_plugin/test_proxy.py +1 -1
- pulpcore/tests/unit/content/test_heartbeat.py +11 -8
- pulpcore/tests/unit/test_vulnerability_report.py +74 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/METADATA +12 -17
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/RECORD +54 -152
- pulp_certguard/app/utils.py +0 -28
- pulp_certguard/tests/unit/test_models.py +0 -9
- pulp_file/app/migrations/0001_initial.py +0 -59
- pulp_file/app/migrations/0002_file_related_names.py +0 -55
- pulp_file/app/migrations/0003_auto_20191014_1721.py +0 -18
- pulp_file/app/migrations/0004_filefilesystemexporter.py +0 -21
- pulp_file/app/migrations/0005_filerepository.py +0 -24
- pulp_file/app/migrations/0007_filefilesystemexporter.py +0 -25
- pulp_file/app/migrations/0008_add_manifest_field.py +0 -19
- pulp_file/app/migrations/0009_move_data_to_new_master_distribution_model.py +0 -77
- pulp_file/app/migrations/0010_auto_publish.py +0 -23
- pulp_file/app/migrations/0011_fix_auto_publish.py +0 -36
- pulp_file/app/migrations/0012_delete_filefilesystemexporter.py +0 -28
- pulp_file/app/migrations/0013_file_acs.py +0 -24
- pulp_file/app/migrations/0014_new_rbac_permissions.py +0 -33
- pulp_file/app/migrations/0015_allow_null_manifest.py +0 -23
- pulp_file/app/migrations/0016_add_domain.py +0 -25
- pulpcore/app/migrations/0001_initial.py +0 -451
- pulpcore/app/migrations/0002_increase_artifact_size_field.py +0 -18
- pulpcore/app/migrations/0003_remove_upload_completed.py +0 -17
- pulpcore/app/migrations/0004_add_duplicated_reserved_resources.py +0 -45
- pulpcore/app/migrations/0005_progressreport_code.py +0 -19
- pulpcore/app/migrations/0006_repository_plugin_managed.py +0 -18
- pulpcore/app/migrations/0007_delete_progress_proxies.py +0 -19
- pulpcore/app/migrations/0008_published_metadata_as_content.py +0 -44
- pulpcore/app/migrations/0009_remove_task_non_fatal_errors.py +0 -17
- pulpcore/app/migrations/0010_pulp_fields.py +0 -570
- pulpcore/app/migrations/0011_relative_path.py +0 -28
- pulpcore/app/migrations/0012_auto_20191104_2000.py +0 -31
- pulpcore/app/migrations/0013_repository_pulp_type.py +0 -18
- pulpcore/app/migrations/0014_remove_repository_plugin_managed.py +0 -17
- pulpcore/app/migrations/0015_auto_20191112_1426.py +0 -33
- pulpcore/app/migrations/0016_charfield_to_textfield.py +0 -68
- pulpcore/app/migrations/0017_remove_task_parent.py +0 -17
- pulpcore/app/migrations/0018_auto_20191127_2350.py +0 -20
- pulpcore/app/migrations/0019_add_signing_service_model.py +0 -27
- pulpcore/app/migrations/0020_change_publishedartifact_constraints.py +0 -17
- pulpcore/app/migrations/0021_add_signing_service_foreign_key.py +0 -24
- pulpcore/app/migrations/0022_rename_last_version.py +0 -27
- pulpcore/app/migrations/0023_change_exporter_models.py +0 -82
- pulpcore/app/migrations/0024_use_local_storage_for_uploads.py +0 -19
- pulpcore/app/migrations/0025_task_parent_task.py +0 -19
- pulpcore/app/migrations/0026_task_group.py +0 -32
- pulpcore/app/migrations/0027_export_backend.py +0 -31
- pulpcore/app/migrations/0028_import_importer_pulpimporter_pulpimporterrepository.py +0 -85
- pulpcore/app/migrations/0029_export_delete.py +0 -19
- pulpcore/app/migrations/0030_taskgroup_all_tasks_dispatched.py +0 -24
- pulpcore/app/migrations/0031_import_export_validate_params.py +0 -19
- pulpcore/app/migrations/0032_export_to_chunks.py +0 -27
- pulpcore/app/migrations/0033_increase_remote_artifact_size_field.py +0 -18
- pulpcore/app/migrations/0034_groupprogressreport.py +0 -32
- pulpcore/app/migrations/0035_content_upstream_id.py +0 -18
- pulpcore/app/migrations/0036_unprotect_last_export.py +0 -19
- pulpcore/app/migrations/0037_pulptemporaryfile.py +0 -28
- pulpcore/app/migrations/0038_repository_remote.py +0 -19
- pulpcore/app/migrations/0039_change_download_concurrency.py +0 -25
- pulpcore/app/migrations/0040_set_admin_is_staff.py +0 -28
- pulpcore/app/migrations/0041_accesspolicy.py +0 -29
- pulpcore/app/migrations/0042_rbac_for_tasks.py +0 -56
- pulpcore/app/migrations/0043_toc_attribute.py +0 -19
- pulpcore/app/migrations/0044_temp_file_artifact_field.py +0 -20
- pulpcore/app/migrations/0045_accesspolicy_permissions_allow_null.py +0 -19
- pulpcore/app/migrations/0046_task__resource_job_id.py +0 -35
- pulpcore/app/migrations/0047_improve_orphan_cleanup.py +0 -59
- pulpcore/app/migrations/0048_fips_checksums.py +0 -38
- pulpcore/app/migrations/0049_add_file_field_to_uploadchunk.py +0 -24
- pulpcore/app/migrations/0050_namespace_access_policies.py +0 -28
- pulpcore/app/migrations/0051_timeoutfields.py +0 -34
- pulpcore/app/migrations/0052_tasking_logging_cid.py +0 -18
- pulpcore/app/migrations/0053_remote_headers.py +0 -19
- pulpcore/app/migrations/0054_add_public_key.py +0 -104
- pulpcore/app/migrations/0055_label.py +0 -31
- pulpcore/app/migrations/0056_remote_rate_limit.py +0 -18
- pulpcore/app/migrations/0057_add_label_indexes.py +0 -23
- pulpcore/app/migrations/0058_accesspolicy_customized.py +0 -18
- pulpcore/app/migrations/0059_proxy_creds.py +0 -23
- pulpcore/app/migrations/0060_data_migration_proxy_creds.py +0 -45
- pulpcore/app/migrations/0061_call_handle_artifact_checksums_command.py +0 -87
- pulpcore/app/migrations/0062_add_new_distribution_mastermodel.py +0 -36
- pulpcore/app/migrations/0063_repository_retained_versions.py +0 -18
- pulpcore/app/migrations/0064_add_new_style_task_columns.py +0 -109
- pulpcore/app/migrations/0064_repository_user_hidden.py +0 -18
- pulpcore/app/migrations/0065_merge_20210615_1211.py +0 -14
- pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py +0 -24
- pulpcore/app/migrations/0067_add_protect_to_task_reservation.py +0 -19
- pulpcore/app/migrations/0068_add_timestamp_of_interest.py +0 -23
- pulpcore/app/migrations/0069_update_json_fields.py +0 -63
- pulpcore/app/migrations/0070_rename_retained_versions.py +0 -18
- pulpcore/app/migrations/0071_filesystemexport_filesystemexporter.py +0 -35
- pulpcore/app/migrations/0072_add_method_to_filesystem_exporter.py +0 -18
- pulpcore/app/migrations/0073_encrypt_remote_fields.py +0 -139
- pulpcore/app/migrations/0074_acs.py +0 -47
- pulpcore/app/migrations/0075_rbaccontentguard.py +0 -25
- pulpcore/app/migrations/0076_remove_reserved_resource.py +0 -39
- pulpcore/app/migrations/0077_move_remote_url_credentials.py +0 -41
- pulpcore/app/migrations/0078_grouprole_role_userrole.py +0 -70
- pulpcore/app/migrations/0079_rename_permissions_assignment_accesspolicy_creation_hooks.py +0 -18
- pulpcore/app/migrations/0080_proxy_group_model.py +0 -37
- pulpcore/app/migrations/0081_reapplabel_group_permissions.py +0 -59
- pulpcore/app/migrations/0082_add_manage_roles_permissions.py +0 -17
- pulpcore/app/migrations/0083_alter_group_options.py +0 -17
- pulpcore/app/migrations/0084_alter_rbaccontentguard_options.py +0 -17
- pulpcore/app/migrations/0085_contentredirectcontentguard.py +0 -26
- pulpcore/app/migrations/0086_task_json_fields.py +0 -77
- pulpcore/app/migrations/0087_taskschedule.py +0 -34
- pulpcore/app/migrations/0088_accesspolicy_queryset_scoping.py +0 -18
- pulpcore/app/migrations/0089_alter_contentredirectcontentguard_options.py +0 -17
- pulpcore/app/migrations/0090_char_to_text_field.py +0 -79
- pulpcore/tests/unit/migration/test_0077_move_remote_url_credentials.py +0 -35
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/WHEEL +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/entry_points.txt +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/licenses/LICENSE +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/top_level.txt +0 -0
pulpcore/app/models/status.py
CHANGED
|
@@ -6,7 +6,9 @@ from datetime import timedelta
|
|
|
6
6
|
|
|
7
7
|
from django.conf import settings
|
|
8
8
|
from django.contrib.postgres.fields import HStoreField
|
|
9
|
+
from django.contrib.postgres.functions import TransactionNow
|
|
9
10
|
from django.db import models
|
|
11
|
+
from django.db.models import F, Value
|
|
10
12
|
from django.utils import timezone
|
|
11
13
|
|
|
12
14
|
from pulpcore.app.models import BaseModel
|
|
@@ -47,9 +49,138 @@ class AppStatusManager(models.Manager):
|
|
|
47
49
|
return self.filter(last_heartbeat__lt=age_threshold)
|
|
48
50
|
|
|
49
51
|
|
|
52
|
+
class _AppStatusManager(AppStatusManager):
|
|
53
|
+
# This is an intermediate class in order to allow a ZDU.
|
|
54
|
+
# It should be removed from the chain with 3.87.
|
|
55
|
+
def create(self, app_type, **kwargs):
|
|
56
|
+
if app_type == "api":
|
|
57
|
+
old_obj = ApiAppStatus.objects.create(**kwargs)
|
|
58
|
+
elif app_type == "worker":
|
|
59
|
+
from pulpcore.app.models import Worker
|
|
60
|
+
|
|
61
|
+
old_obj = Worker.objects.create(**kwargs)
|
|
62
|
+
else:
|
|
63
|
+
raise NotImplementedError(f"Invalid app_type: {app_type}")
|
|
64
|
+
obj = super().create(app_type=app_type, **kwargs)
|
|
65
|
+
obj._old_status = old_obj
|
|
66
|
+
return obj
|
|
67
|
+
|
|
68
|
+
async def acreate(self, app_type, **kwargs):
|
|
69
|
+
if app_type == "content":
|
|
70
|
+
old_obj = await ContentAppStatus.objects.acreate(**kwargs)
|
|
71
|
+
else:
|
|
72
|
+
raise NotImplementedError(f"Invalid app_type: {app_type}")
|
|
73
|
+
obj = await super().acreate(app_type=app_type, **kwargs)
|
|
74
|
+
obj._old_status = old_obj
|
|
75
|
+
return obj
|
|
76
|
+
|
|
77
|
+
def online(self):
|
|
78
|
+
"""
|
|
79
|
+
Returns a queryset of objects that are online.
|
|
80
|
+
"""
|
|
81
|
+
return self.filter(last_heartbeat__gte=TransactionNow() - F("ttl"))
|
|
82
|
+
|
|
83
|
+
def missing(self):
|
|
84
|
+
"""
|
|
85
|
+
Returns a queryset of workers that are missing.
|
|
86
|
+
"""
|
|
87
|
+
return self.filter(last_heartbeat__lt=TransactionNow() - F("ttl"))
|
|
88
|
+
|
|
89
|
+
def older_than(self, age):
|
|
90
|
+
"""
|
|
91
|
+
Returns a queryset of workers that are older than age.
|
|
92
|
+
"""
|
|
93
|
+
return self.filter(last_heartbeat__lt=TransactionNow() - Value(age))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class AppStatus(BaseModel):
|
|
97
|
+
APP_TYPES = [
|
|
98
|
+
("api", "api"),
|
|
99
|
+
("content", "content"),
|
|
100
|
+
("worker", "worker"),
|
|
101
|
+
]
|
|
102
|
+
_APP_TTL = {
|
|
103
|
+
"api": settings.API_APP_TTL,
|
|
104
|
+
"content": settings.CONTENT_APP_TTL,
|
|
105
|
+
"worker": settings.WORKER_TTL,
|
|
106
|
+
}
|
|
107
|
+
objects = _AppStatusManager()
|
|
108
|
+
|
|
109
|
+
app_type = models.CharField(max_length=10, choices=APP_TYPES)
|
|
110
|
+
name = models.TextField(db_index=True, unique=True)
|
|
111
|
+
versions = HStoreField(default=dict)
|
|
112
|
+
ttl = models.DurationField(null=False)
|
|
113
|
+
last_heartbeat = models.DateTimeField(auto_now=True)
|
|
114
|
+
|
|
115
|
+
def __init__(self, *args, **kwargs):
|
|
116
|
+
super().__init__(*args, **kwargs)
|
|
117
|
+
self.ttl = timedelta(seconds=self._APP_TTL[self.app_type])
|
|
118
|
+
self._old_status = None
|
|
119
|
+
|
|
120
|
+
def delete(self, *args, **kwargs):
|
|
121
|
+
# adelete will call into this, so we should not replicate that one here.
|
|
122
|
+
if self._old_status is not None:
|
|
123
|
+
self._old_status.delete(*args, **kwargs)
|
|
124
|
+
super().delete(*args, **kwargs)
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def online(self) -> bool:
|
|
128
|
+
"""
|
|
129
|
+
To be considered 'online', an app must have a timestamp more recent than ``self.ttl``.
|
|
130
|
+
"""
|
|
131
|
+
age_threshold = timezone.now() - self.ttl
|
|
132
|
+
return self.last_heartbeat >= age_threshold
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def missing(self):
|
|
136
|
+
"""
|
|
137
|
+
Whether an app can be considered 'missing'
|
|
138
|
+
|
|
139
|
+
To be considered 'missing', an App must have a timestamp older than ``self.ttl``.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
bool: True if the app is considered missing, otherwise False
|
|
143
|
+
"""
|
|
144
|
+
return not self.online
|
|
145
|
+
|
|
146
|
+
def save_heartbeat(self):
|
|
147
|
+
"""
|
|
148
|
+
Update the last_heartbeat field to now and save it.
|
|
149
|
+
|
|
150
|
+
Only the last_heartbeat field will be saved. No other changes will be saved.
|
|
151
|
+
|
|
152
|
+
Raises:
|
|
153
|
+
ValueError: When the model instance has never been saved before. This method can
|
|
154
|
+
only update an existing database record.
|
|
155
|
+
"""
|
|
156
|
+
self._old_status.save_heartbeat()
|
|
157
|
+
self.save(update_fields=["last_heartbeat"])
|
|
158
|
+
|
|
159
|
+
async def asave_heartbeat(self):
|
|
160
|
+
"""
|
|
161
|
+
Update the last_heartbeat field to now and save it.
|
|
162
|
+
|
|
163
|
+
Only the last_heartbeat field will be saved. No other changes will be saved.
|
|
164
|
+
|
|
165
|
+
Raises:
|
|
166
|
+
ValueError: When the model instance has never been saved before. This method can
|
|
167
|
+
only update an existing database record.
|
|
168
|
+
"""
|
|
169
|
+
await self._old_status.asave_heartbeat()
|
|
170
|
+
await self.asave(update_fields=["last_heartbeat"])
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def current_task(self):
|
|
174
|
+
"""
|
|
175
|
+
The task this worker is currently executing, if any.
|
|
176
|
+
"""
|
|
177
|
+
return self.tasks.filter(state="running").first()
|
|
178
|
+
|
|
179
|
+
|
|
50
180
|
class BaseAppStatus(BaseModel):
|
|
51
181
|
"""
|
|
52
182
|
Represents an AppStatus.
|
|
183
|
+
Deprecated, to be removed with 3.87.
|
|
53
184
|
|
|
54
185
|
This class is abstract. Subclasses must define `APP_TTL` as a `timedelta`.
|
|
55
186
|
|
|
@@ -103,6 +234,18 @@ class BaseAppStatus(BaseModel):
|
|
|
103
234
|
"""
|
|
104
235
|
self.save(update_fields=["last_heartbeat"])
|
|
105
236
|
|
|
237
|
+
async def asave_heartbeat(self):
|
|
238
|
+
"""
|
|
239
|
+
Update the last_heartbeat field to now and save it.
|
|
240
|
+
|
|
241
|
+
Only the last_heartbeat field will be saved. No other changes will be saved.
|
|
242
|
+
|
|
243
|
+
Raises:
|
|
244
|
+
ValueError: When the model instance has never been saved before. This method can
|
|
245
|
+
only update an existing database record.
|
|
246
|
+
"""
|
|
247
|
+
await self.asave(update_fields=["last_heartbeat"])
|
|
248
|
+
|
|
106
249
|
class Meta:
|
|
107
250
|
abstract = True
|
|
108
251
|
|
|
@@ -110,6 +253,7 @@ class BaseAppStatus(BaseModel):
|
|
|
110
253
|
class ApiAppStatus(BaseAppStatus):
|
|
111
254
|
"""
|
|
112
255
|
Represents a Api App Status
|
|
256
|
+
Deprecated, to be removed with 3.87.
|
|
113
257
|
"""
|
|
114
258
|
|
|
115
259
|
APP_TTL = timedelta(seconds=settings.API_APP_TTL)
|
|
@@ -118,6 +262,7 @@ class ApiAppStatus(BaseAppStatus):
|
|
|
118
262
|
class ContentAppStatus(BaseAppStatus):
|
|
119
263
|
"""
|
|
120
264
|
Represents a Content App Status
|
|
265
|
+
Deprecated, to be removed with 3.87.
|
|
121
266
|
"""
|
|
122
267
|
|
|
123
268
|
APP_TTL = timedelta(seconds=settings.CONTENT_APP_TTL)
|
pulpcore/app/models/task.py
CHANGED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from django.db import models
|
|
2
|
+
|
|
3
|
+
from pulpcore.app.models.base import BaseModel
|
|
4
|
+
from pulpcore.app.util import get_domain_pk
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class VulnerabilityReport(BaseModel):
|
|
8
|
+
"""
|
|
9
|
+
A model for storing vulnerability reports for Content/RepositoryVersion.
|
|
10
|
+
|
|
11
|
+
Fields:
|
|
12
|
+
vulns (models.JSONField): A JSON field containing the list of vulnerabilities found
|
|
13
|
+
in osv.dev.
|
|
14
|
+
|
|
15
|
+
Relations:
|
|
16
|
+
content (models.OneToOneField): The Content object that was scanned for vulnerabilities.
|
|
17
|
+
pulp_domain (models.ForeignKey): The Domain this vulnerability report belongs to,
|
|
18
|
+
providing multi-tenancy isolation.
|
|
19
|
+
repo_versions (models.ManyToManyField): The RepositoryVersion(s) where the scanned
|
|
20
|
+
Content appears. This allows tracking which repository versions contain
|
|
21
|
+
vulnerable content.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
content = models.OneToOneField(
|
|
25
|
+
"Content",
|
|
26
|
+
on_delete=models.CASCADE,
|
|
27
|
+
unique=True,
|
|
28
|
+
)
|
|
29
|
+
vulns = models.JSONField()
|
|
30
|
+
pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.CASCADE)
|
|
31
|
+
repo_versions = models.ManyToManyField("RepositoryVersion", blank=True)
|
|
32
|
+
|
|
33
|
+
class Meta:
|
|
34
|
+
default_related_name = "%(app_label)s_%(model_name)s"
|
|
@@ -5,7 +5,13 @@ from rest_framework import serializers
|
|
|
5
5
|
from rest_framework.validators import UniqueValidator
|
|
6
6
|
|
|
7
7
|
from pulpcore.app import models
|
|
8
|
-
from pulpcore.app.serializers import
|
|
8
|
+
from pulpcore.app.serializers import (
|
|
9
|
+
base,
|
|
10
|
+
fields,
|
|
11
|
+
pulp_labels_validator,
|
|
12
|
+
DetailRelatedField,
|
|
13
|
+
RelatedField,
|
|
14
|
+
)
|
|
9
15
|
from pulpcore.app.util import get_domain
|
|
10
16
|
|
|
11
17
|
|
|
@@ -27,6 +33,11 @@ class NoArtifactContentSerializer(base.ModelSerializer):
|
|
|
27
33
|
view_name_pattern=r"repositories(-.*/.*)-detail",
|
|
28
34
|
queryset=models.Repository.objects.all(),
|
|
29
35
|
)
|
|
36
|
+
vuln_report = RelatedField(
|
|
37
|
+
read_only=True,
|
|
38
|
+
view_name="vuln_report-detail",
|
|
39
|
+
source="core_vulnerabilityreport",
|
|
40
|
+
)
|
|
30
41
|
|
|
31
42
|
def get_artifacts(self, validated_data):
|
|
32
43
|
"""
|
|
@@ -116,6 +127,7 @@ class NoArtifactContentSerializer(base.ModelSerializer):
|
|
|
116
127
|
fields = base.ModelSerializer.Meta.fields + (
|
|
117
128
|
"repository",
|
|
118
129
|
"pulp_labels",
|
|
130
|
+
"vuln_report",
|
|
119
131
|
)
|
|
120
132
|
|
|
121
133
|
|
|
@@ -7,7 +7,7 @@ from rest_framework import fields, serializers
|
|
|
7
7
|
from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer
|
|
8
8
|
|
|
9
9
|
from pulpcore.app import models, settings
|
|
10
|
-
from pulpcore.app.util import get_prn
|
|
10
|
+
from pulpcore.app.util import get_prn, reverse
|
|
11
11
|
from pulpcore.app.serializers import (
|
|
12
12
|
DetailIdentityField,
|
|
13
13
|
DetailRelatedField,
|
|
@@ -490,6 +490,12 @@ class RepositoryVersionSerializer(ModelSerializer, NestedHyperlinkedModelSeriali
|
|
|
490
490
|
source="*",
|
|
491
491
|
read_only=True,
|
|
492
492
|
)
|
|
493
|
+
vuln_report = serializers.SerializerMethodField(
|
|
494
|
+
read_only=True,
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
def get_vuln_report(self, object):
|
|
498
|
+
return f"{reverse('vuln_report-list')}?repository_version={get_prn(object)}"
|
|
493
499
|
|
|
494
500
|
class Meta:
|
|
495
501
|
model = models.RepositoryVersion
|
|
@@ -499,6 +505,7 @@ class RepositoryVersionSerializer(ModelSerializer, NestedHyperlinkedModelSeriali
|
|
|
499
505
|
"repository",
|
|
500
506
|
"base_version",
|
|
501
507
|
"content_summary",
|
|
508
|
+
"vuln_report",
|
|
502
509
|
)
|
|
503
510
|
|
|
504
511
|
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from rest_framework import serializers
|
|
2
|
+
|
|
3
|
+
from pulpcore.app.models import VulnerabilityReport
|
|
4
|
+
from pulpcore.app.serializers import (
|
|
5
|
+
DetailRelatedField,
|
|
6
|
+
IdentityField,
|
|
7
|
+
ModelSerializer,
|
|
8
|
+
RepositoryVersionRelatedField,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class VulnerabilityReportSerializer(ModelSerializer):
|
|
13
|
+
"""
|
|
14
|
+
A serializer for the VulnerabilityReport Model.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
vulns = serializers.JSONField()
|
|
18
|
+
pulp_href = IdentityField(view_name="vuln_report-detail")
|
|
19
|
+
content = DetailRelatedField(
|
|
20
|
+
read_only=True,
|
|
21
|
+
view_name_pattern=r"content(-.*/.*)-detail",
|
|
22
|
+
)
|
|
23
|
+
repo_versions = RepositoryVersionRelatedField(many=True, required=False)
|
|
24
|
+
|
|
25
|
+
class Meta:
|
|
26
|
+
model = VulnerabilityReport
|
|
27
|
+
fields = ModelSerializer.Meta.fields + ("vulns", "repo_versions", "content")
|
pulpcore/app/settings.py
CHANGED
|
@@ -12,16 +12,14 @@ import sys
|
|
|
12
12
|
|
|
13
13
|
from contextlib import suppress
|
|
14
14
|
from importlib import import_module
|
|
15
|
+
from importlib.metadata import entry_points
|
|
15
16
|
from logging import getLogger
|
|
16
17
|
from pathlib import Path
|
|
17
18
|
|
|
18
19
|
from cryptography.fernet import Fernet
|
|
19
|
-
from django.core.files.storage import storages
|
|
20
|
-
from django.conf import global_settings
|
|
21
20
|
from django.core.exceptions import ImproperlyConfigured
|
|
22
21
|
from django.db import connection
|
|
23
|
-
from
|
|
24
|
-
from dynaconf import DjangoDynaconf, Dynaconf, Validator, get_history
|
|
22
|
+
from dynaconf import DjangoDynaconf, Dynaconf, Validator
|
|
25
23
|
|
|
26
24
|
from pulpcore import constants
|
|
27
25
|
|
|
@@ -33,12 +31,6 @@ try:
|
|
|
33
31
|
except ImportError:
|
|
34
32
|
pass
|
|
35
33
|
|
|
36
|
-
if sys.version_info < (3, 10):
|
|
37
|
-
# Python 3.9 has a rather different interface for `entry_points`.
|
|
38
|
-
# Let's use a compatibility version.
|
|
39
|
-
from importlib_metadata import entry_points
|
|
40
|
-
else:
|
|
41
|
-
from importlib.metadata import entry_points
|
|
42
34
|
|
|
43
35
|
# Load settings first pass before applying all the defaults to get a grip on ENABLED_PLUGINS.
|
|
44
36
|
enabled_plugins_validator = Validator(
|
|
@@ -78,16 +70,7 @@ MEDIA_ROOT = str(DEPLOY_ROOT / "media") # Django 3.1 adds support for pathlib.P
|
|
|
78
70
|
STATIC_URL = "/assets/"
|
|
79
71
|
STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/")
|
|
80
72
|
|
|
81
|
-
|
|
82
|
-
# Remove on pulpcore=3.85 or pulpcore=4.0
|
|
83
|
-
|
|
84
|
-
# - What is this?
|
|
85
|
-
# We shouldn't use STORAGES or DEFAULT_FILE_STORAGE directly because those are
|
|
86
|
-
# mutually exclusive by django, which constraints users to use whatever we use.
|
|
87
|
-
# This is a hack/workaround to set Pulp's default while still enabling users to choose
|
|
88
|
-
# the legacy or the new storage setting.
|
|
89
|
-
_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
|
|
90
|
-
_STORAGES = {
|
|
73
|
+
STORAGES = {
|
|
91
74
|
"default": {
|
|
92
75
|
"BACKEND": "pulpcore.app.models.storage.FileSystem",
|
|
93
76
|
},
|
|
@@ -96,10 +79,6 @@ _STORAGES = {
|
|
|
96
79
|
},
|
|
97
80
|
}
|
|
98
81
|
|
|
99
|
-
setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE)
|
|
100
|
-
setattr(global_settings, "STORAGES", _STORAGES)
|
|
101
|
-
# end DEFAULT_FILE_STORAGE deprecation layer
|
|
102
|
-
|
|
103
82
|
REDIRECT_TO_OBJECT_STORAGE = True
|
|
104
83
|
|
|
105
84
|
WORKING_DIRECTORY = DEPLOY_ROOT / "tmp"
|
|
@@ -329,6 +308,7 @@ TASK_PROTECTION_TIME = 0
|
|
|
329
308
|
TMPFILE_PROTECTION_TIME = 0
|
|
330
309
|
|
|
331
310
|
REMOTE_USER_ENVIRON_NAME = "REMOTE_USER"
|
|
311
|
+
REMOTE_USER_OPENAPI_SECURITY_SCHEME = {"type": "mutualTLS"}
|
|
332
312
|
|
|
333
313
|
AUTHENTICATION_JSON_HEADER = ""
|
|
334
314
|
AUTHENTICATION_JSON_HEADER_JQ_FILTER = ""
|
|
@@ -348,6 +328,7 @@ CACHE_SETTINGS = {
|
|
|
348
328
|
REMOTE_CONTENT_FETCH_FAILURE_COOLDOWN = 5 * 60 # 5 minutes
|
|
349
329
|
|
|
350
330
|
SPECTACULAR_SETTINGS = {
|
|
331
|
+
"OAS_VERSION": "3.1.1",
|
|
351
332
|
"SERVE_URLCONF": ROOT_URLCONF,
|
|
352
333
|
"DEFAULT_GENERATOR_CLASS": "pulpcore.openapi.PulpSchemaGenerator",
|
|
353
334
|
"DEFAULT_SCHEMA_CLASS": "pulpcore.openapi.PulpAutoSchema",
|
|
@@ -425,6 +406,9 @@ KAFKA_SASL_PASSWORD = None
|
|
|
425
406
|
# opentelemetry settings
|
|
426
407
|
OTEL_ENABLED = False
|
|
427
408
|
|
|
409
|
+
# VulnerabilityReport settings
|
|
410
|
+
VULN_REPORT_TASK_LIMITER = 10
|
|
411
|
+
|
|
428
412
|
# Replaces asyncio event loop with uvloop
|
|
429
413
|
UVLOOP_ENABLED = False
|
|
430
414
|
|
|
@@ -433,19 +417,17 @@ UVLOOP_ENABLED = False
|
|
|
433
417
|
|
|
434
418
|
# Validators
|
|
435
419
|
|
|
436
|
-
storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE")
|
|
437
420
|
storage_validator = (
|
|
438
421
|
Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False)
|
|
439
|
-
| Validator(
|
|
440
|
-
| Validator(
|
|
441
|
-
| Validator(
|
|
442
|
-
| Validator(
|
|
443
|
-
| Validator(
|
|
422
|
+
| Validator("STORAGES.default.BACKEND", eq="pulpcore.app.models.storage.FileSystem")
|
|
423
|
+
| Validator("STORAGES.default.BACKEND", eq="storages.backends.azure_storage.AzureStorage")
|
|
424
|
+
| Validator("STORAGES.default.BACKEND", eq="storages.backends.s3.S3Storage")
|
|
425
|
+
| Validator("STORAGES.default.BACKEND", eq="storages.backends.s3boto3.S3Boto3Storage")
|
|
426
|
+
| Validator("STORAGES.default.BACKEND", eq="storages.backends.gcloud.GoogleCloudStorage")
|
|
444
427
|
)
|
|
445
428
|
storage_validator.messages["combined"] = (
|
|
446
429
|
"'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure "
|
|
447
|
-
"storage backend configured in STORAGES['default']['BACKEND']
|
|
448
|
-
"(deprecated DEFAULT_FILE_STORAGE)."
|
|
430
|
+
"storage backend configured in STORAGES['default']['BACKEND']."
|
|
449
431
|
)
|
|
450
432
|
|
|
451
433
|
cache_enabled_validator = Validator("CACHE_ENABLED", eq=True)
|
|
@@ -552,21 +534,6 @@ settings = DjangoDynaconf(
|
|
|
552
534
|
post_hooks=(otel_middleware_hook,),
|
|
553
535
|
)
|
|
554
536
|
|
|
555
|
-
# begin compatibility layer for DEFAULT_FILE_STORAGE
|
|
556
|
-
# Remove on pulpcore=3.85 or pulpcore=4.0
|
|
557
|
-
using_deprecated_storage_settings = len(get_history(settings, key="DEFAULT_FILE_STORAGE")) > 1
|
|
558
|
-
if using_deprecated_storage_settings:
|
|
559
|
-
deprecation_logger.warning(
|
|
560
|
-
"[deprecation] DEFAULT_FILE_STORAGE will be removed in pulpcore 3.85. "
|
|
561
|
-
"Learn how to upgrade to STORAGES:\n"
|
|
562
|
-
"https://discourse.pulpproject.org/t/"
|
|
563
|
-
"action-required-upgrade-your-storage-settings-before-pulpcore-3-85/2072/2"
|
|
564
|
-
)
|
|
565
|
-
# Ensures the cached property storage.backends uses the the right value
|
|
566
|
-
storages._backends = settings.STORAGES.copy()
|
|
567
|
-
storages.backends
|
|
568
|
-
# end compatibility layer
|
|
569
|
-
|
|
570
537
|
_logger = getLogger(__name__)
|
|
571
538
|
|
|
572
539
|
|
pulpcore/app/tasks/__init__.py
CHANGED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import aiohttp
|
|
2
|
+
import asyncio
|
|
3
|
+
import importlib
|
|
4
|
+
|
|
5
|
+
from asgiref.sync import sync_to_async
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
|
|
8
|
+
from pulpcore.app.util import get_domain
|
|
9
|
+
from pulpcore.app.models.progress import ProgressReport
|
|
10
|
+
from pulpcore.app.models.vulnerability_report import VulnerabilityReport
|
|
11
|
+
from pulpcore.constants import OSV_QUERY_URL, TASK_STATES
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class VulnerabilityReportScanner:
|
|
15
|
+
"""
|
|
16
|
+
A scanner class to collect vulnerabilities from Pulp Contents using the OSV.dev API.
|
|
17
|
+
|
|
18
|
+
Attributes:
|
|
19
|
+
semaphore (asyncio.Semaphore): Controls concurrent API requests to osv.dev service.
|
|
20
|
+
generator (AsyncGenerator): The async generator with the content data to be scanned for
|
|
21
|
+
vulnerabilities.
|
|
22
|
+
session (aiohttp.ClientSession): HTTP session for making requests to the OSV.dev API.
|
|
23
|
+
Defaults to a new session if not provided.
|
|
24
|
+
created (ProgressReport): Tracks the number of new VulnerabilityReport records created
|
|
25
|
+
during the scanning process.
|
|
26
|
+
updated (ProgressReport): Tracks the number of existing VulnerabilityReport records
|
|
27
|
+
updated during the scanning process.
|
|
28
|
+
total_scanned (ProgressReport): Tracks the total number of content items that have
|
|
29
|
+
been processed for vulnerability scanning.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, semaphore, generator, session=None):
|
|
33
|
+
self.semaphore = semaphore
|
|
34
|
+
self.generator = generator
|
|
35
|
+
self.session = session or aiohttp.ClientSession()
|
|
36
|
+
self.created = ProgressReport(
|
|
37
|
+
message="Vulnerability Reports created", code="created", state=TASK_STATES.RUNNING
|
|
38
|
+
)
|
|
39
|
+
self.updated = ProgressReport(
|
|
40
|
+
message="Vulnerability Reports updated", code="updated", state=TASK_STATES.RUNNING
|
|
41
|
+
)
|
|
42
|
+
self.total_scanned = ProgressReport(
|
|
43
|
+
message="Content scanned", code="total_scanned", state=TASK_STATES.RUNNING
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
async def scan_packages(self):
|
|
47
|
+
"""
|
|
48
|
+
Main entry point for scanning packages for vulnerabilities. Progress is tracked through
|
|
49
|
+
three ProgressReport instances (created, updated, total_scanned).
|
|
50
|
+
"""
|
|
51
|
+
tasks = []
|
|
52
|
+
async for content in self.generator:
|
|
53
|
+
tasks.append(asyncio.create_task(self.scan_package(content)))
|
|
54
|
+
await asyncio.gather(*tasks)
|
|
55
|
+
await self.session.close()
|
|
56
|
+
|
|
57
|
+
if self.created.done > 0:
|
|
58
|
+
self.created.state = TASK_STATES.COMPLETED
|
|
59
|
+
await self.created.asave()
|
|
60
|
+
if self.total_scanned.done > 0:
|
|
61
|
+
self.total_scanned.state = TASK_STATES.COMPLETED
|
|
62
|
+
await self.total_scanned.asave()
|
|
63
|
+
if self.updated.done > 0:
|
|
64
|
+
self.updated.state = TASK_STATES.COMPLETED
|
|
65
|
+
await self.updated.asave()
|
|
66
|
+
|
|
67
|
+
async def scan_package(self, repo_content_osv_data):
|
|
68
|
+
"""
|
|
69
|
+
Makes a request to the osv.dev API and store the results in VulnerabilityReport model.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
repo_content_osv_data (Dict[str, Any]): A dictionary with osv.dev expected request
|
|
73
|
+
data format plus associated Pulp Content and RepoVersion.
|
|
74
|
+
"""
|
|
75
|
+
vulns, content, repo_version = await self._process_reports(repo_content_osv_data)
|
|
76
|
+
await self._save_vulnerability_report(vulns, content, repo_version)
|
|
77
|
+
|
|
78
|
+
async def _process_reports(self, repo_content_osv_data):
|
|
79
|
+
"""
|
|
80
|
+
Process vulnerability scanning data by querying the OSV API and extracting results.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
repo_content_osv_data (Dict[str, Any]): A dictionary containing OSV.dev expected
|
|
84
|
+
request data format plus associated Pulp Content and RepositoryVersion.
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
tuple:
|
|
88
|
+
- vulns (List[Dict]): List of vulnerability (vulns) returned from OSV.dev API
|
|
89
|
+
- pulp_content (Content): The Pulp Content instance that was scanned
|
|
90
|
+
- repo_version (RepositoryVersion or None): The RepositoryVersion where the
|
|
91
|
+
Content is present
|
|
92
|
+
"""
|
|
93
|
+
vulns = []
|
|
94
|
+
async with self.semaphore:
|
|
95
|
+
repo_version = repo_content_osv_data.pop("repo_version", None)
|
|
96
|
+
pulp_content = repo_content_osv_data.pop("content")
|
|
97
|
+
vulnerability_data = await self._query_osv_api(repo_content_osv_data)
|
|
98
|
+
vulns = vulnerability_data.get("vulns", [])
|
|
99
|
+
while next_page_token := vulnerability_data.get("next_page_token"):
|
|
100
|
+
repo_content_osv_data["page_token"] = next_page_token
|
|
101
|
+
vulnerability_data = await self._query_osv_api(repo_content_osv_data)
|
|
102
|
+
vulns.extend(vulnerability_data)
|
|
103
|
+
await self.total_scanned.aincrement()
|
|
104
|
+
return vulns, pulp_content, repo_version
|
|
105
|
+
|
|
106
|
+
async def _query_osv_api(self, osv_data):
|
|
107
|
+
"""
|
|
108
|
+
Make a single HTTP POST request to the OSV.dev vulnerability database API.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
osv_data (Dict[str, Any]): OSV query data dictionary containing package information
|
|
112
|
+
in the format expected by the OSV.dev API.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
Dict[str, Any]: JSON response from the OSV API containing vulnerability data.
|
|
116
|
+
"""
|
|
117
|
+
async with self.session.post(url=OSV_QUERY_URL, json=osv_data) as response:
|
|
118
|
+
try:
|
|
119
|
+
return await response.json()
|
|
120
|
+
except aiohttp.ContentTypeError:
|
|
121
|
+
raise RuntimeError("Vuln report task failed to query osv.dev data.")
|
|
122
|
+
|
|
123
|
+
async def _save_vulnerability_report(self, vulns, content, repo_version):
|
|
124
|
+
"""
|
|
125
|
+
Save vulnerability report to the database.
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
vulns (List): osv.dev vulns output
|
|
129
|
+
content (Content): A Pulp Content instance which vulnerabilities were verified
|
|
130
|
+
repo_version (RepositoryVersion): The RepositoryVersion where Content is present
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
vuln_report, created = await sync_to_async(VulnerabilityReport.objects.update_or_create)(
|
|
134
|
+
vulns=vulns, pulp_domain=get_domain(), content=content
|
|
135
|
+
)
|
|
136
|
+
await vuln_report.repo_versions.aadd(repo_version)
|
|
137
|
+
|
|
138
|
+
if created:
|
|
139
|
+
await self.created.aincrement()
|
|
140
|
+
else:
|
|
141
|
+
await self.updated.aincrement()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def check_content(func, args):
|
|
145
|
+
"""
|
|
146
|
+
Spawn tasks for each async generator value to make the API requests (scan) to osv.dev.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
func (callable | str): A Pulp plugin function that should return an async generator with
|
|
150
|
+
the osv.dev expected request data format plus associated Pulp Content and RepoVersion.
|
|
151
|
+
args (tuple): The positional arguments to pass on to the func.
|
|
152
|
+
"""
|
|
153
|
+
module_name, function_name = func.rsplit(".", 1)
|
|
154
|
+
module = importlib.import_module(module_name)
|
|
155
|
+
func = getattr(module, function_name)
|
|
156
|
+
|
|
157
|
+
semaphore = asyncio.Semaphore(settings.VULN_REPORT_TASK_LIMITER)
|
|
158
|
+
vuln_report_scanner = VulnerabilityReportScanner(semaphore, func(*args))
|
|
159
|
+
await vuln_report_scanner.scan_packages()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from rest_framework.mixins import DestroyModelMixin, ListModelMixin, RetrieveModelMixin
|
|
2
|
+
|
|
3
|
+
from pulpcore.app.models.vulnerability_report import VulnerabilityReport
|
|
4
|
+
from pulpcore.app.serializers.vulnerability_report import VulnerabilityReportSerializer
|
|
5
|
+
from pulpcore.app.viewsets.base import NamedModelViewSet
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class VulnerabilityReportViewSet(
|
|
9
|
+
NamedModelViewSet, ListModelMixin, RetrieveModelMixin, DestroyModelMixin
|
|
10
|
+
):
|
|
11
|
+
|
|
12
|
+
endpoint_name = "vuln_report"
|
|
13
|
+
queryset = VulnerabilityReport.objects.prefetch_related("repo_versions").select_related(
|
|
14
|
+
"content"
|
|
15
|
+
)
|
|
16
|
+
serializer_class = VulnerabilityReportSerializer
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def routable(cls):
|
|
20
|
+
return True
|
pulpcore/constants.py
CHANGED
|
@@ -131,3 +131,7 @@ CHECKPOINT_TS_FORMAT = "%Y%m%dT%H%M%SZ"
|
|
|
131
131
|
# The upper boundary represents an unsigned 32-bit integer and prevents overflow
|
|
132
132
|
ORPHAN_PROTECTION_TIME_LOWER_BOUND = 0
|
|
133
133
|
ORPHAN_PROTECTION_TIME_UPPER_BOUND = 4294967295 # (2^32)-1
|
|
134
|
+
|
|
135
|
+
# VULNERABILITY REPORT CONSTANTS
|
|
136
|
+
# OSV API URL
|
|
137
|
+
OSV_QUERY_URL = "https://api.osv.dev/v1/query"
|