pulpcore 3.84.0__py3-none-any.whl → 3.85.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pulpcore might be problematic. Click here for more details.
- pulp_certguard/app/__init__.py +1 -1
- pulp_certguard/app/models.py +7 -26
- pulp_certguard/app/serializers.py +0 -2
- pulp_certguard/rhsm/__init__.py +4 -0
- pulp_certguard/rhsm/rhsm_check_path.py +198 -0
- pulp_certguard/tests/unit/certdata.py +249 -0
- pulp_certguard/tests/unit/test_rhsm_check_path.py +213 -0
- pulp_file/app/__init__.py +1 -1
- pulp_file/app/migrations/0001_initial_squashed_0016_add_domain.py +0 -20
- pulp_file/app/migrations/0017_alter_filealternatecontentsource_alternatecontentsource_ptr_and_more.py +1 -1
- pulpcore/app/apps.py +2 -12
- pulpcore/app/entrypoint.py +22 -22
- pulpcore/app/migrations/0001_squashed_0090_char_to_text_field.py +0 -95
- pulpcore/app/migrations/0091_systemid.py +1 -1
- pulp_file/app/migrations/0006_delete_filefilesystemexporter.py → pulpcore/app/migrations/0136_delete_basedistribution.py +3 -3
- pulpcore/app/migrations/0137_appstatus.py +33 -0
- pulpcore/app/migrations/0138_vulnerabilityreport.py +33 -0
- pulpcore/app/models/__init__.py +4 -1
- pulpcore/app/models/publication.py +0 -41
- pulpcore/app/models/status.py +145 -0
- pulpcore/app/models/task.py +1 -0
- pulpcore/app/models/vulnerability_report.py +34 -0
- pulpcore/app/serializers/__init__.py +1 -0
- pulpcore/app/serializers/content.py +13 -1
- pulpcore/app/serializers/repository.py +8 -1
- pulpcore/app/serializers/vulnerability_report.py +27 -0
- pulpcore/app/settings.py +14 -47
- pulpcore/app/tasks/__init__.py +2 -0
- pulpcore/app/tasks/vulnerability_report.py +159 -0
- pulpcore/app/viewsets/__init__.py +1 -0
- pulpcore/app/viewsets/vulnerability_report.py +20 -0
- pulpcore/constants.py +4 -0
- pulpcore/content/__init__.py +23 -22
- pulpcore/content/handler.py +5 -2
- pulpcore/migrations.py +38 -11
- pulpcore/openapi/__init__.py +8 -0
- pulpcore/plugin/models/__init__.py +2 -0
- pulpcore/plugin/serializers/__init__.py +2 -0
- pulpcore/plugin/tasking.py +2 -0
- pulpcore/plugin/viewsets/__init__.py +2 -0
- pulpcore/pytest_plugin.py +21 -21
- pulpcore/tasking/worker.py +38 -35
- pulpcore/tests/functional/api/test_auth.py +18 -3
- pulpcore/tests/functional/api/test_openapi_schema.py +32 -15
- pulpcore/tests/functional/api/using_plugin/test_checkpoint.py +23 -1
- pulpcore/tests/functional/api/using_plugin/test_proxy.py +1 -1
- pulpcore/tests/unit/content/test_heartbeat.py +11 -8
- pulpcore/tests/unit/test_vulnerability_report.py +74 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/METADATA +12 -17
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/RECORD +54 -152
- pulp_certguard/app/utils.py +0 -28
- pulp_certguard/tests/unit/test_models.py +0 -9
- pulp_file/app/migrations/0001_initial.py +0 -59
- pulp_file/app/migrations/0002_file_related_names.py +0 -55
- pulp_file/app/migrations/0003_auto_20191014_1721.py +0 -18
- pulp_file/app/migrations/0004_filefilesystemexporter.py +0 -21
- pulp_file/app/migrations/0005_filerepository.py +0 -24
- pulp_file/app/migrations/0007_filefilesystemexporter.py +0 -25
- pulp_file/app/migrations/0008_add_manifest_field.py +0 -19
- pulp_file/app/migrations/0009_move_data_to_new_master_distribution_model.py +0 -77
- pulp_file/app/migrations/0010_auto_publish.py +0 -23
- pulp_file/app/migrations/0011_fix_auto_publish.py +0 -36
- pulp_file/app/migrations/0012_delete_filefilesystemexporter.py +0 -28
- pulp_file/app/migrations/0013_file_acs.py +0 -24
- pulp_file/app/migrations/0014_new_rbac_permissions.py +0 -33
- pulp_file/app/migrations/0015_allow_null_manifest.py +0 -23
- pulp_file/app/migrations/0016_add_domain.py +0 -25
- pulpcore/app/migrations/0001_initial.py +0 -451
- pulpcore/app/migrations/0002_increase_artifact_size_field.py +0 -18
- pulpcore/app/migrations/0003_remove_upload_completed.py +0 -17
- pulpcore/app/migrations/0004_add_duplicated_reserved_resources.py +0 -45
- pulpcore/app/migrations/0005_progressreport_code.py +0 -19
- pulpcore/app/migrations/0006_repository_plugin_managed.py +0 -18
- pulpcore/app/migrations/0007_delete_progress_proxies.py +0 -19
- pulpcore/app/migrations/0008_published_metadata_as_content.py +0 -44
- pulpcore/app/migrations/0009_remove_task_non_fatal_errors.py +0 -17
- pulpcore/app/migrations/0010_pulp_fields.py +0 -570
- pulpcore/app/migrations/0011_relative_path.py +0 -28
- pulpcore/app/migrations/0012_auto_20191104_2000.py +0 -31
- pulpcore/app/migrations/0013_repository_pulp_type.py +0 -18
- pulpcore/app/migrations/0014_remove_repository_plugin_managed.py +0 -17
- pulpcore/app/migrations/0015_auto_20191112_1426.py +0 -33
- pulpcore/app/migrations/0016_charfield_to_textfield.py +0 -68
- pulpcore/app/migrations/0017_remove_task_parent.py +0 -17
- pulpcore/app/migrations/0018_auto_20191127_2350.py +0 -20
- pulpcore/app/migrations/0019_add_signing_service_model.py +0 -27
- pulpcore/app/migrations/0020_change_publishedartifact_constraints.py +0 -17
- pulpcore/app/migrations/0021_add_signing_service_foreign_key.py +0 -24
- pulpcore/app/migrations/0022_rename_last_version.py +0 -27
- pulpcore/app/migrations/0023_change_exporter_models.py +0 -82
- pulpcore/app/migrations/0024_use_local_storage_for_uploads.py +0 -19
- pulpcore/app/migrations/0025_task_parent_task.py +0 -19
- pulpcore/app/migrations/0026_task_group.py +0 -32
- pulpcore/app/migrations/0027_export_backend.py +0 -31
- pulpcore/app/migrations/0028_import_importer_pulpimporter_pulpimporterrepository.py +0 -85
- pulpcore/app/migrations/0029_export_delete.py +0 -19
- pulpcore/app/migrations/0030_taskgroup_all_tasks_dispatched.py +0 -24
- pulpcore/app/migrations/0031_import_export_validate_params.py +0 -19
- pulpcore/app/migrations/0032_export_to_chunks.py +0 -27
- pulpcore/app/migrations/0033_increase_remote_artifact_size_field.py +0 -18
- pulpcore/app/migrations/0034_groupprogressreport.py +0 -32
- pulpcore/app/migrations/0035_content_upstream_id.py +0 -18
- pulpcore/app/migrations/0036_unprotect_last_export.py +0 -19
- pulpcore/app/migrations/0037_pulptemporaryfile.py +0 -28
- pulpcore/app/migrations/0038_repository_remote.py +0 -19
- pulpcore/app/migrations/0039_change_download_concurrency.py +0 -25
- pulpcore/app/migrations/0040_set_admin_is_staff.py +0 -28
- pulpcore/app/migrations/0041_accesspolicy.py +0 -29
- pulpcore/app/migrations/0042_rbac_for_tasks.py +0 -56
- pulpcore/app/migrations/0043_toc_attribute.py +0 -19
- pulpcore/app/migrations/0044_temp_file_artifact_field.py +0 -20
- pulpcore/app/migrations/0045_accesspolicy_permissions_allow_null.py +0 -19
- pulpcore/app/migrations/0046_task__resource_job_id.py +0 -35
- pulpcore/app/migrations/0047_improve_orphan_cleanup.py +0 -59
- pulpcore/app/migrations/0048_fips_checksums.py +0 -38
- pulpcore/app/migrations/0049_add_file_field_to_uploadchunk.py +0 -24
- pulpcore/app/migrations/0050_namespace_access_policies.py +0 -28
- pulpcore/app/migrations/0051_timeoutfields.py +0 -34
- pulpcore/app/migrations/0052_tasking_logging_cid.py +0 -18
- pulpcore/app/migrations/0053_remote_headers.py +0 -19
- pulpcore/app/migrations/0054_add_public_key.py +0 -104
- pulpcore/app/migrations/0055_label.py +0 -31
- pulpcore/app/migrations/0056_remote_rate_limit.py +0 -18
- pulpcore/app/migrations/0057_add_label_indexes.py +0 -23
- pulpcore/app/migrations/0058_accesspolicy_customized.py +0 -18
- pulpcore/app/migrations/0059_proxy_creds.py +0 -23
- pulpcore/app/migrations/0060_data_migration_proxy_creds.py +0 -45
- pulpcore/app/migrations/0061_call_handle_artifact_checksums_command.py +0 -87
- pulpcore/app/migrations/0062_add_new_distribution_mastermodel.py +0 -36
- pulpcore/app/migrations/0063_repository_retained_versions.py +0 -18
- pulpcore/app/migrations/0064_add_new_style_task_columns.py +0 -109
- pulpcore/app/migrations/0064_repository_user_hidden.py +0 -18
- pulpcore/app/migrations/0065_merge_20210615_1211.py +0 -14
- pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py +0 -24
- pulpcore/app/migrations/0067_add_protect_to_task_reservation.py +0 -19
- pulpcore/app/migrations/0068_add_timestamp_of_interest.py +0 -23
- pulpcore/app/migrations/0069_update_json_fields.py +0 -63
- pulpcore/app/migrations/0070_rename_retained_versions.py +0 -18
- pulpcore/app/migrations/0071_filesystemexport_filesystemexporter.py +0 -35
- pulpcore/app/migrations/0072_add_method_to_filesystem_exporter.py +0 -18
- pulpcore/app/migrations/0073_encrypt_remote_fields.py +0 -139
- pulpcore/app/migrations/0074_acs.py +0 -47
- pulpcore/app/migrations/0075_rbaccontentguard.py +0 -25
- pulpcore/app/migrations/0076_remove_reserved_resource.py +0 -39
- pulpcore/app/migrations/0077_move_remote_url_credentials.py +0 -41
- pulpcore/app/migrations/0078_grouprole_role_userrole.py +0 -70
- pulpcore/app/migrations/0079_rename_permissions_assignment_accesspolicy_creation_hooks.py +0 -18
- pulpcore/app/migrations/0080_proxy_group_model.py +0 -37
- pulpcore/app/migrations/0081_reapplabel_group_permissions.py +0 -59
- pulpcore/app/migrations/0082_add_manage_roles_permissions.py +0 -17
- pulpcore/app/migrations/0083_alter_group_options.py +0 -17
- pulpcore/app/migrations/0084_alter_rbaccontentguard_options.py +0 -17
- pulpcore/app/migrations/0085_contentredirectcontentguard.py +0 -26
- pulpcore/app/migrations/0086_task_json_fields.py +0 -77
- pulpcore/app/migrations/0087_taskschedule.py +0 -34
- pulpcore/app/migrations/0088_accesspolicy_queryset_scoping.py +0 -18
- pulpcore/app/migrations/0089_alter_contentredirectcontentguard_options.py +0 -17
- pulpcore/app/migrations/0090_char_to_text_field.py +0 -79
- pulpcore/tests/unit/migration/test_0077_move_remote_url_credentials.py +0 -35
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/WHEEL +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/entry_points.txt +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/licenses/LICENSE +0 -0
- {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/top_level.txt +0 -0
pulpcore/content/__init__.py
CHANGED
|
@@ -7,7 +7,7 @@ import os
|
|
|
7
7
|
|
|
8
8
|
from asgiref.sync import sync_to_async
|
|
9
9
|
from aiohttp import web
|
|
10
|
-
|
|
10
|
+
from gunicorn.arbiter import Arbiter
|
|
11
11
|
import django
|
|
12
12
|
|
|
13
13
|
|
|
@@ -16,12 +16,13 @@ django.setup()
|
|
|
16
16
|
|
|
17
17
|
from django.conf import settings # noqa: E402: module level not at top of file
|
|
18
18
|
from django.db.utils import ( # noqa: E402: module level not at top of file
|
|
19
|
+
IntegrityError,
|
|
19
20
|
InterfaceError,
|
|
20
21
|
DatabaseError,
|
|
21
22
|
)
|
|
22
23
|
|
|
23
24
|
from pulpcore.app.apps import pulp_plugin_configs # noqa: E402: module level not at top of file
|
|
24
|
-
from pulpcore.app.models import
|
|
25
|
+
from pulpcore.app.models import AppStatus # noqa: E402: module level not at top of file
|
|
25
26
|
from pulpcore.app.util import get_worker_name # noqa: E402: module level not at top of file
|
|
26
27
|
|
|
27
28
|
from .handler import Handler # noqa: E402: module level not at top of file
|
|
@@ -48,40 +49,40 @@ CONTENT_MODULE_NAME = "content"
|
|
|
48
49
|
|
|
49
50
|
|
|
50
51
|
async def _heartbeat():
|
|
51
|
-
content_app_status = None
|
|
52
52
|
name = get_worker_name()
|
|
53
53
|
heartbeat_interval = settings.CONTENT_APP_TTL // 4
|
|
54
54
|
msg = "Content App '{name}' heartbeat written, sleeping for '{interarrival}' seconds".format(
|
|
55
55
|
name=name, interarrival=heartbeat_interval
|
|
56
56
|
)
|
|
57
|
-
fail_msg = (
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
).format(name=name, interarrival=heartbeat_interval)
|
|
57
|
+
fail_msg = ("Content App '{name}' failed to write a heartbeat to the database.").format(
|
|
58
|
+
name=name
|
|
59
|
+
)
|
|
61
60
|
versions = {app.label: app.version for app in pulp_plugin_configs()}
|
|
62
61
|
|
|
62
|
+
try:
|
|
63
|
+
app_status = await AppStatus.objects.acreate(
|
|
64
|
+
name=name, app_type="content", versions=versions
|
|
65
|
+
)
|
|
66
|
+
except IntegrityError:
|
|
67
|
+
log.error(f"A content app with name {name} already exists in the database.")
|
|
68
|
+
exit(Arbiter.WORKER_BOOT_ERROR)
|
|
63
69
|
try:
|
|
64
70
|
while True:
|
|
71
|
+
await asyncio.sleep(heartbeat_interval)
|
|
65
72
|
try:
|
|
66
|
-
|
|
67
|
-
name=name, defaults={"versions": versions}
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
if not created:
|
|
71
|
-
await sync_to_async(content_app_status.save_heartbeat)()
|
|
72
|
-
|
|
73
|
-
if content_app_status.versions != versions:
|
|
74
|
-
content_app_status.versions = versions
|
|
75
|
-
await content_app_status.asave(update_fields=["versions"])
|
|
76
|
-
|
|
73
|
+
await app_status.asave_heartbeat()
|
|
77
74
|
log.debug(msg)
|
|
78
75
|
except (InterfaceError, DatabaseError):
|
|
79
76
|
await sync_to_async(Handler._reset_db_connection)()
|
|
80
|
-
|
|
81
|
-
|
|
77
|
+
try:
|
|
78
|
+
await app_status.asave_heartbeat()
|
|
79
|
+
log.debug(msg)
|
|
80
|
+
except (InterfaceError, DatabaseError):
|
|
81
|
+
log.error(fail_msg)
|
|
82
|
+
exit(Arbiter.WORKER_BOOT_ERROR)
|
|
82
83
|
finally:
|
|
83
|
-
if
|
|
84
|
-
await
|
|
84
|
+
if app_status:
|
|
85
|
+
await app_status.adelete()
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
async def _heartbeat_ctx(app):
|
pulpcore/content/handler.py
CHANGED
|
@@ -705,8 +705,11 @@ class Handler:
|
|
|
705
705
|
rel_path = rel_path[len(distro.base_path) :]
|
|
706
706
|
rel_path = rel_path.lstrip("/")
|
|
707
707
|
|
|
708
|
-
if
|
|
709
|
-
|
|
708
|
+
# Check if we need to redirect to add a trailing slash for distro root
|
|
709
|
+
if not path.endswith("/") and (
|
|
710
|
+
rel_path == "" # Distro root
|
|
711
|
+
or (distro.checkpoint and "/" not in rel_path) # Timestamped checkpoint root
|
|
712
|
+
):
|
|
710
713
|
raise HTTPMovedPermanently(f"{request.path}/")
|
|
711
714
|
|
|
712
715
|
original_rel_path = rel_path
|
pulpcore/migrations.py
CHANGED
|
@@ -3,6 +3,8 @@ from packaging.version import parse as parse_version
|
|
|
3
3
|
from django.conf import settings
|
|
4
4
|
from django.utils import timezone
|
|
5
5
|
from django.db.migrations.operations.base import Operation
|
|
6
|
+
from django.db.models import F
|
|
7
|
+
from django.db.models.functions import Now
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class RequireVersion(Operation):
|
|
@@ -22,21 +24,41 @@ class RequireVersion(Operation):
|
|
|
22
24
|
|
|
23
25
|
def database_forwards(self, app_label, schema_editor, from_state, to_state):
|
|
24
26
|
from_state.clear_delayed_apps_cache()
|
|
25
|
-
ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
|
|
26
|
-
ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
|
|
27
|
-
Worker = from_state.apps.get_model("core", "Worker")
|
|
28
27
|
|
|
29
28
|
needed_version = parse_version(self.version)
|
|
30
29
|
errors = []
|
|
30
|
+
found_either_table = False
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
32
|
+
try:
|
|
33
|
+
ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
|
|
34
|
+
ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
|
|
35
|
+
Worker = from_state.apps.get_model("core", "Worker")
|
|
36
|
+
except LookupError:
|
|
37
|
+
pass
|
|
38
|
+
else:
|
|
39
|
+
for worker_class, ttl, class_name in [
|
|
40
|
+
(ApiAppStatus, settings.API_APP_TTL, "api server"),
|
|
41
|
+
(ContentAppStatus, settings.CONTENT_APP_TTL, "content server"),
|
|
42
|
+
(Worker, settings.WORKER_TTL, "pulp worker"),
|
|
43
|
+
]:
|
|
44
|
+
for worker in worker_class.objects.filter(
|
|
45
|
+
last_heartbeat__gte=timezone.now() - timezone.timedelta(seconds=ttl)
|
|
46
|
+
):
|
|
47
|
+
present_version = worker.versions.get(self.plugin)
|
|
48
|
+
if (
|
|
49
|
+
present_version is not None
|
|
50
|
+
and parse_version(present_version) < needed_version
|
|
51
|
+
):
|
|
52
|
+
errors.append(
|
|
53
|
+
f" - '{self.plugin}'='{present_version}' "
|
|
54
|
+
f"with {class_name} '{worker.name}'"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
found_either_table = True
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
AppStatus = from_state.apps.get_model("core", "AppStatus")
|
|
61
|
+
for worker in AppStatus.objects.filter(F("last_heartbeat") + F("ttl") >= Now()):
|
|
40
62
|
present_version = worker.versions.get(self.plugin)
|
|
41
63
|
if present_version is not None and parse_version(present_version) < needed_version:
|
|
42
64
|
errors.append(
|
|
@@ -44,6 +66,11 @@ class RequireVersion(Operation):
|
|
|
44
66
|
f"with {class_name} '{worker.name}'"
|
|
45
67
|
)
|
|
46
68
|
|
|
69
|
+
found_either_table = True
|
|
70
|
+
except LookupError:
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
assert found_either_table
|
|
47
74
|
if errors:
|
|
48
75
|
raise RuntimeError(
|
|
49
76
|
"\n".join(
|
pulpcore/openapi/__init__.py
CHANGED
|
@@ -468,6 +468,14 @@ class BasicAuthenticationScheme(OpenApiAuthenticationExtension):
|
|
|
468
468
|
}
|
|
469
469
|
|
|
470
470
|
|
|
471
|
+
class PulpRemoteUserAuthenticationScheme(OpenApiAuthenticationExtension):
|
|
472
|
+
target_class = "pulpcore.app.authentication.PulpRemoteUserAuthentication"
|
|
473
|
+
name = "remoteUserAuthentication"
|
|
474
|
+
|
|
475
|
+
def get_security_definition(self, auto_schema):
|
|
476
|
+
return settings.REMOTE_USER_OPENAPI_SECURITY_SCHEME
|
|
477
|
+
|
|
478
|
+
|
|
471
479
|
class JSONHeaderRemoteAuthenticationScheme(OpenApiAuthenticationExtension):
|
|
472
480
|
target_class = "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
|
|
473
481
|
name = "json_header_remote_authentication"
|
|
@@ -38,6 +38,7 @@ from pulpcore.app.serializers import (
|
|
|
38
38
|
ValidateFieldsMixin,
|
|
39
39
|
validate_unknown_fields,
|
|
40
40
|
TaskSerializer,
|
|
41
|
+
VulnerabilityReportSerializer,
|
|
41
42
|
)
|
|
42
43
|
|
|
43
44
|
from .content import (
|
|
@@ -87,4 +88,5 @@ __all__ = [
|
|
|
87
88
|
"TaskSerializer",
|
|
88
89
|
"NoArtifactContentUploadSerializer",
|
|
89
90
|
"SingleArtifactContentUploadSerializer",
|
|
91
|
+
"VulnerabilityReportSerializer",
|
|
90
92
|
]
|
pulpcore/plugin/tasking.py
CHANGED
|
@@ -12,11 +12,13 @@ from pulpcore.app.tasks import (
|
|
|
12
12
|
orphan_cleanup,
|
|
13
13
|
reclaim_space,
|
|
14
14
|
)
|
|
15
|
+
from pulpcore.app.tasks.vulnerability_report import check_content
|
|
15
16
|
from pulpcore.app.tasks.repository import add_and_remove
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
__all__ = [
|
|
19
20
|
"ageneral_update",
|
|
21
|
+
"check_content",
|
|
20
22
|
"dispatch",
|
|
21
23
|
"fs_publication_export",
|
|
22
24
|
"fs_repo_version_export",
|
|
@@ -33,6 +33,7 @@ from pulpcore.app.viewsets import (
|
|
|
33
33
|
RolesMixin,
|
|
34
34
|
TaskGroupViewSet,
|
|
35
35
|
TaskViewSet,
|
|
36
|
+
VulnerabilityReportViewSet,
|
|
36
37
|
)
|
|
37
38
|
|
|
38
39
|
from pulpcore.app.viewsets.custom_filters import (
|
|
@@ -89,4 +90,5 @@ __all__ = [
|
|
|
89
90
|
"NoArtifactContentViewSet",
|
|
90
91
|
"NoArtifactContentUploadViewSet",
|
|
91
92
|
"SingleArtifactContentUploadViewSet",
|
|
93
|
+
"VulnerabilityReportViewSet",
|
|
92
94
|
]
|
pulpcore/pytest_plugin.py
CHANGED
|
@@ -608,28 +608,28 @@ def backend_settings_factory(pulp_settings):
|
|
|
608
608
|
"bucket_name",
|
|
609
609
|
]
|
|
610
610
|
keys["storages.backends.azure_storage.AzureStorage"] = [
|
|
611
|
-
"
|
|
612
|
-
"
|
|
613
|
-
"
|
|
614
|
-
"
|
|
615
|
-
"
|
|
616
|
-
"
|
|
617
|
-
"
|
|
611
|
+
"account_name",
|
|
612
|
+
"azure_container",
|
|
613
|
+
"account_key",
|
|
614
|
+
"expiration_secs",
|
|
615
|
+
"overwrite_files",
|
|
616
|
+
"location",
|
|
617
|
+
"connection_string",
|
|
618
618
|
]
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
return
|
|
619
|
+
|
|
620
|
+
def get_installation_storage_option(key, backend):
|
|
621
|
+
value = pulp_settings.STORAGES["default"]["OPTIONS"].get(key)
|
|
622
|
+
# Some FileSystem backend options may be defined in the top settings module
|
|
623
|
+
if backend == "pulpcore.app.models.storage.FileSystem" and not value:
|
|
624
|
+
value = getattr(pulp_settings, key, None)
|
|
625
|
+
return value
|
|
626
|
+
|
|
627
|
+
storage_settings = storage_settings or dict()
|
|
628
|
+
storage_backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
|
|
629
|
+
unset_storage_settings = (k for k in keys[storage_backend] if k not in storage_settings)
|
|
630
|
+
for key in unset_storage_settings:
|
|
631
|
+
storage_settings[key] = get_installation_storage_option(key, storage_backend)
|
|
632
|
+
return storage_backend, storage_settings
|
|
633
633
|
|
|
634
634
|
return _settings_factory
|
|
635
635
|
|
pulpcore/tasking/worker.py
CHANGED
|
@@ -13,7 +13,7 @@ from tempfile import TemporaryDirectory
|
|
|
13
13
|
from packaging.version import parse as parse_version
|
|
14
14
|
|
|
15
15
|
from django.conf import settings
|
|
16
|
-
from django.db import connection
|
|
16
|
+
from django.db import connection, DatabaseError, IntegrityError
|
|
17
17
|
from django.db.models import Case, Count, F, Max, Value, When
|
|
18
18
|
from django.db.models.functions import Random
|
|
19
19
|
from django.utils import timezone
|
|
@@ -29,8 +29,8 @@ from pulpcore.constants import (
|
|
|
29
29
|
)
|
|
30
30
|
from pulpcore.metrics import init_otel_meter
|
|
31
31
|
from pulpcore.app.apps import pulp_plugin_configs
|
|
32
|
-
from pulpcore.app.models import Worker, Task, ApiAppStatus, ContentAppStatus
|
|
33
|
-
from pulpcore.app.util import PGAdvisoryLock
|
|
32
|
+
from pulpcore.app.models import Worker, Task, AppStatus, ApiAppStatus, ContentAppStatus
|
|
33
|
+
from pulpcore.app.util import PGAdvisoryLock
|
|
34
34
|
from pulpcore.exceptions import AdvisoryLockError
|
|
35
35
|
|
|
36
36
|
from pulpcore.tasking.storage import WorkerDirectory
|
|
@@ -73,7 +73,14 @@ class PulpcoreWorker:
|
|
|
73
73
|
self.last_metric_heartbeat = timezone.now()
|
|
74
74
|
self.versions = {app.label: app.version for app in pulp_plugin_configs()}
|
|
75
75
|
self.cursor = connection.cursor()
|
|
76
|
-
|
|
76
|
+
try:
|
|
77
|
+
self.app_status = AppStatus.objects.create(
|
|
78
|
+
name=self.name, app_type="worker", versions=self.versions
|
|
79
|
+
)
|
|
80
|
+
self.worker = self.app_status._old_status
|
|
81
|
+
except IntegrityError:
|
|
82
|
+
_logger.error(f"A worker with name {self.name} already exists in the database.")
|
|
83
|
+
exit(1)
|
|
77
84
|
# This defaults to immediate task cancellation.
|
|
78
85
|
# It will be set into the future on moderately graceful worker shutdown,
|
|
79
86
|
# and set to None for fully graceful shutdown.
|
|
@@ -148,52 +155,48 @@ class PulpcoreWorker:
|
|
|
148
155
|
|
|
149
156
|
def handle_worker_heartbeat(self):
|
|
150
157
|
"""
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
Existing Worker objects are searched for one to update. If an existing one is found, it is
|
|
154
|
-
updated. Otherwise a new Worker entry is created. Logging at the info level is also done.
|
|
158
|
+
Update worker heartbeat records.
|
|
155
159
|
|
|
160
|
+
If the update fails (the record was deleted, the database is unreachable, ...) the worker
|
|
161
|
+
is shut down.
|
|
156
162
|
"""
|
|
157
|
-
worker, created = Worker.objects.get_or_create(
|
|
158
|
-
name=self.name, defaults={"versions": self.versions}
|
|
159
|
-
)
|
|
160
|
-
if not created and worker.versions != self.versions:
|
|
161
|
-
worker.versions = self.versions
|
|
162
|
-
worker.save(update_fields=["versions"])
|
|
163
|
-
|
|
164
|
-
if created:
|
|
165
|
-
_logger.info(_("New worker '{name}' discovered").format(name=self.name))
|
|
166
|
-
elif worker.online is False:
|
|
167
|
-
_logger.info(_("Worker '{name}' is back online.").format(name=self.name))
|
|
168
|
-
|
|
169
|
-
worker.save_heartbeat()
|
|
170
163
|
|
|
171
164
|
msg = "Worker heartbeat from '{name}' at time {timestamp}".format(
|
|
172
|
-
timestamp=
|
|
165
|
+
timestamp=self.app_status.last_heartbeat, name=self.name
|
|
173
166
|
)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
167
|
+
try:
|
|
168
|
+
self.app_status.save_heartbeat()
|
|
169
|
+
_logger.debug(msg)
|
|
170
|
+
except (IntegrityError, DatabaseError):
|
|
171
|
+
# WARNING: Do not attempt to recycle the connection here.
|
|
172
|
+
# The advisory locks are bound to the connection and we must not loose them.
|
|
173
|
+
_logger.error(f"Updating the heartbeat of worker {self.name} failed.")
|
|
174
|
+
# TODO if shutdown_requested, we may need to be more aggressive.
|
|
175
|
+
self.shutdown_requested = True
|
|
176
|
+
self.cancel_task = True
|
|
177
177
|
|
|
178
178
|
def shutdown(self):
|
|
179
|
-
self.
|
|
179
|
+
self.app_status.delete()
|
|
180
180
|
_logger.info(_("Worker %s was shut down."), self.name)
|
|
181
181
|
|
|
182
182
|
def worker_cleanup(self):
|
|
183
|
+
qs = AppStatus.objects.older_than(age=timedelta(days=7))
|
|
184
|
+
for app_worker in qs:
|
|
185
|
+
_logger.info(_("Clean missing %s worker %s."), app_worker.app_type, app_worker.name)
|
|
186
|
+
qs.delete()
|
|
183
187
|
for cls, cls_name in (
|
|
184
188
|
(Worker, "pulp"),
|
|
185
189
|
(ApiAppStatus, "api"),
|
|
186
190
|
(ContentAppStatus, "content"),
|
|
187
191
|
):
|
|
188
192
|
qs = cls.objects.missing(age=timedelta(days=7))
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
qs.delete()
|
|
193
|
+
for app_worker in qs:
|
|
194
|
+
_logger.info(_("Clean missing %s worker %s."), cls_name, app_worker.name)
|
|
195
|
+
qs.delete()
|
|
193
196
|
|
|
194
197
|
def beat(self):
|
|
195
|
-
if self.
|
|
196
|
-
self.
|
|
198
|
+
if self.app_status.last_heartbeat < timezone.now() - self.heartbeat_period:
|
|
199
|
+
self.handle_worker_heartbeat()
|
|
197
200
|
if not self.auxiliary:
|
|
198
201
|
self.worker_cleanup_countdown -= 1
|
|
199
202
|
if self.worker_cleanup_countdown <= 0:
|
|
@@ -217,7 +220,7 @@ class PulpcoreWorker:
|
|
|
217
220
|
Return ``True`` if the task was actually canceled, ``False`` otherwise.
|
|
218
221
|
"""
|
|
219
222
|
# A task is considered abandoned when in running state, but no worker holds its lock
|
|
220
|
-
domain =
|
|
223
|
+
domain = task.pulp_domain
|
|
221
224
|
try:
|
|
222
225
|
task.set_canceling()
|
|
223
226
|
except RuntimeError:
|
|
@@ -244,7 +247,7 @@ class PulpcoreWorker:
|
|
|
244
247
|
return True
|
|
245
248
|
|
|
246
249
|
def is_compatible(self, task):
|
|
247
|
-
domain =
|
|
250
|
+
domain = task.pulp_domain
|
|
248
251
|
unmatched_versions = [
|
|
249
252
|
f"task: {label}>={version} worker: {self.versions.get(label)}"
|
|
250
253
|
for label, version in task.versions.items()
|
|
@@ -427,7 +430,7 @@ class PulpcoreWorker:
|
|
|
427
430
|
task.save(update_fields=["worker"])
|
|
428
431
|
cancel_state = None
|
|
429
432
|
cancel_reason = None
|
|
430
|
-
domain =
|
|
433
|
+
domain = task.pulp_domain
|
|
431
434
|
with TemporaryDirectory(dir=".") as task_working_dir_rel_path:
|
|
432
435
|
task_process = Process(target=perform_task, args=(task.pk, task_working_dir_rel_path))
|
|
433
436
|
task_process.start()
|
|
@@ -13,6 +13,11 @@ from pulpcore.app import settings
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@pytest.mark.parallel
|
|
16
|
+
@pytest.mark.skipif(
|
|
17
|
+
"rest_framework.authentication.BasicAuthentication"
|
|
18
|
+
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
19
|
+
reason="Test can't run unless BasicAuthentication is enabled",
|
|
20
|
+
)
|
|
16
21
|
def test_base_auth_success(pulpcore_bindings, pulp_admin_user):
|
|
17
22
|
"""Perform HTTP basic authentication with valid credentials.
|
|
18
23
|
|
|
@@ -33,6 +38,11 @@ def test_base_auth_success(pulpcore_bindings, pulp_admin_user):
|
|
|
33
38
|
|
|
34
39
|
|
|
35
40
|
@pytest.mark.parallel
|
|
41
|
+
@pytest.mark.skipif(
|
|
42
|
+
"rest_framework.authentication.BasicAuthentication"
|
|
43
|
+
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
44
|
+
reason="Test can't run unless BasicAuthentication is enabled",
|
|
45
|
+
)
|
|
36
46
|
def test_base_auth_failure(pulpcore_bindings, invalid_user):
|
|
37
47
|
"""Perform HTTP basic authentication with invalid credentials.
|
|
38
48
|
|
|
@@ -50,6 +60,11 @@ def test_base_auth_failure(pulpcore_bindings, invalid_user):
|
|
|
50
60
|
|
|
51
61
|
|
|
52
62
|
@pytest.mark.parallel
|
|
63
|
+
@pytest.mark.skipif(
|
|
64
|
+
"rest_framework.authentication.BasicAuthentication"
|
|
65
|
+
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
66
|
+
reason="Test can't run unless BasicAuthentication is enabled",
|
|
67
|
+
)
|
|
53
68
|
def test_base_auth_required(pulpcore_bindings, anonymous_user):
|
|
54
69
|
"""Perform HTTP basic authentication with no credentials.
|
|
55
70
|
|
|
@@ -69,7 +84,7 @@ def test_base_auth_required(pulpcore_bindings, anonymous_user):
|
|
|
69
84
|
@pytest.mark.parallel
|
|
70
85
|
@pytest.mark.skipif(
|
|
71
86
|
"django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
|
|
72
|
-
|
|
87
|
+
or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
|
|
73
88
|
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
74
89
|
reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
|
|
75
90
|
)
|
|
@@ -98,7 +113,7 @@ def test_jq_header_remote_auth(pulpcore_bindings, anonymous_user):
|
|
|
98
113
|
@pytest.mark.parallel
|
|
99
114
|
@pytest.mark.skipif(
|
|
100
115
|
"django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
|
|
101
|
-
|
|
116
|
+
or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
|
|
102
117
|
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
103
118
|
reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
|
|
104
119
|
)
|
|
@@ -127,7 +142,7 @@ def test_jq_header_remote_auth_denied_by_wrong_header(pulpcore_bindings, anonymo
|
|
|
127
142
|
@pytest.mark.parallel
|
|
128
143
|
@pytest.mark.skipif(
|
|
129
144
|
"django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
|
|
130
|
-
|
|
145
|
+
or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
|
|
131
146
|
not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
|
|
132
147
|
reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
|
|
133
148
|
)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import copy
|
|
4
4
|
import json
|
|
5
|
-
import
|
|
5
|
+
from pathlib import Path
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
import jsonschema
|
|
@@ -10,28 +10,31 @@ import jsonschema
|
|
|
10
10
|
from drf_spectacular import validation
|
|
11
11
|
from collections import defaultdict
|
|
12
12
|
|
|
13
|
-
JSON_SCHEMA_SPEC_PATH = os.path.join(
|
|
14
|
-
os.path.dirname(validation.__file__), "openapi_3_0_schema.json"
|
|
15
|
-
)
|
|
16
|
-
|
|
17
13
|
|
|
18
14
|
@pytest.fixture(scope="session")
|
|
19
|
-
def openapi3_schema_spec():
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
15
|
+
def openapi3_schema_spec(pulp_openapi_schema):
|
|
16
|
+
schema_version = pulp_openapi_schema["openapi"]
|
|
17
|
+
if schema_version.startswith("3.0"):
|
|
18
|
+
spec_path = Path(validation.__file__).parent / "openapi_3_0_schema.json"
|
|
19
|
+
elif schema_version.startswith("3.1"):
|
|
20
|
+
spec_path = Path(validation.__file__).parent / "openapi_3_1_schema.json"
|
|
21
|
+
else:
|
|
22
|
+
pytest.fail(f"Unknown OpenAPI schema version [{schema_version}].")
|
|
23
|
+
return json.loads(spec_path.read_text())
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
@pytest.fixture(scope="session")
|
|
27
27
|
def openapi3_schema_with_modified_safe_chars(openapi3_schema_spec):
|
|
28
|
-
|
|
28
|
+
oas_copy = copy.deepcopy(openapi3_schema_spec) # Don't modify the original
|
|
29
29
|
# Making OpenAPI validation to accept paths starting with / and {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
if "3.1.x" in oas_copy["description"]:
|
|
31
|
+
pattern_properties = oas_copy["$defs"]["paths"]["patternProperties"]
|
|
32
|
+
pattern_properties["^/|{"] = pattern_properties.pop("^/")
|
|
33
|
+
else:
|
|
34
|
+
pattern_properties = oas_copy["definitions"]["Paths"]["patternProperties"]
|
|
35
|
+
pattern_properties["^\\/|{"] = pattern_properties.pop("^\\/")
|
|
33
36
|
|
|
34
|
-
return
|
|
37
|
+
return oas_copy
|
|
35
38
|
|
|
36
39
|
|
|
37
40
|
@pytest.mark.parallel
|
|
@@ -68,6 +71,20 @@ def test_no_dup_operation_ids(pulp_openapi_schema):
|
|
|
68
71
|
assert len(dup_ids) == 0, f"Duplicate operationIds found: {dup_ids}"
|
|
69
72
|
|
|
70
73
|
|
|
74
|
+
@pytest.mark.parallel
|
|
75
|
+
def test_remote_user_auth_security_scheme(pulp_settings, pulp_openapi_schema):
|
|
76
|
+
if (
|
|
77
|
+
"pulpcore.app.authentication.PulpRemoteUserAuthentication"
|
|
78
|
+
not in pulp_settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"]
|
|
79
|
+
):
|
|
80
|
+
pytest.skip("Test can't run unless PulpRemoteUserAuthentication is enabled.")
|
|
81
|
+
|
|
82
|
+
expected_security_scheme = pulp_settings.REMOTE_USER_OPENAPI_SECURITY_SCHEME
|
|
83
|
+
security_schemes = pulp_openapi_schema["components"]["securitySchemes"]
|
|
84
|
+
|
|
85
|
+
assert security_schemes["remoteUserAuthentication"] == expected_security_scheme
|
|
86
|
+
|
|
87
|
+
|
|
71
88
|
@pytest.mark.parallel
|
|
72
89
|
def test_external_auth_on_security_scheme(pulp_settings, pulp_openapi_schema):
|
|
73
90
|
if (
|
|
@@ -87,11 +87,17 @@ class TestCheckpointDistribution:
|
|
|
87
87
|
assert Handler._format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
|
|
88
88
|
|
|
89
89
|
@pytest.mark.parallel
|
|
90
|
-
def
|
|
90
|
+
def test_distro_root_no_trailing_slash_is_redirected(
|
|
91
|
+
self,
|
|
92
|
+
setup,
|
|
93
|
+
http_get,
|
|
94
|
+
distribution_base_url,
|
|
95
|
+
):
|
|
91
96
|
"""Test checkpoint listing when path doesn't end with a slash."""
|
|
92
97
|
|
|
93
98
|
pubs, distribution = setup
|
|
94
99
|
|
|
100
|
+
# Test a checkpoint distro listing path
|
|
95
101
|
response = http_get(distribution_base_url(distribution.base_url[:-1])).decode("utf-8")
|
|
96
102
|
checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
|
|
97
103
|
|
|
@@ -99,6 +105,22 @@ class TestCheckpointDistribution:
|
|
|
99
105
|
assert Handler._format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
|
|
100
106
|
assert Handler._format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
|
|
101
107
|
|
|
108
|
+
@pytest.mark.parallel
|
|
109
|
+
def test_timestamped_checkpoint_no_trailing_slash_is_redirected(
|
|
110
|
+
self,
|
|
111
|
+
setup,
|
|
112
|
+
http_get,
|
|
113
|
+
checkpoint_url,
|
|
114
|
+
):
|
|
115
|
+
"""Test a timestamped checkpoint when path doesn't end with a slash."""
|
|
116
|
+
|
|
117
|
+
pubs, distribution = setup
|
|
118
|
+
|
|
119
|
+
pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
|
|
120
|
+
response = http_get(pub_1_url[:-1]).decode("utf-8")
|
|
121
|
+
|
|
122
|
+
assert f"<h1>Index of {urlparse(pub_1_url).path}</h1>" in response
|
|
123
|
+
|
|
102
124
|
@pytest.mark.parallel
|
|
103
125
|
def test_exact_timestamp_is_served(self, setup, http_get, checkpoint_url):
|
|
104
126
|
pubs, distribution = setup
|
|
@@ -144,7 +144,7 @@ def test_sync_https_through_https_proxy(
|
|
|
144
144
|
manifest_path=basic_manifest_path,
|
|
145
145
|
policy="on_demand",
|
|
146
146
|
proxy_url=https_proxy.proxy_url,
|
|
147
|
-
tls_validation=
|
|
147
|
+
tls_validation=False,
|
|
148
148
|
)
|
|
149
149
|
|
|
150
150
|
_run_basic_sync_and_assert(file_bindings, monitor_task, remote_on_demand, file_repo)
|