pulpcore 3.84.0__py3-none-any.whl → 3.85.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pulpcore might be problematic. Click here for more details.

Files changed (163) hide show
  1. pulp_certguard/app/__init__.py +1 -1
  2. pulp_certguard/app/models.py +7 -26
  3. pulp_certguard/app/serializers.py +0 -2
  4. pulp_certguard/rhsm/__init__.py +4 -0
  5. pulp_certguard/rhsm/rhsm_check_path.py +198 -0
  6. pulp_certguard/tests/unit/certdata.py +249 -0
  7. pulp_certguard/tests/unit/test_rhsm_check_path.py +213 -0
  8. pulp_file/app/__init__.py +1 -1
  9. pulp_file/app/migrations/0001_initial_squashed_0016_add_domain.py +0 -20
  10. pulp_file/app/migrations/0017_alter_filealternatecontentsource_alternatecontentsource_ptr_and_more.py +1 -1
  11. pulpcore/app/apps.py +2 -12
  12. pulpcore/app/entrypoint.py +22 -22
  13. pulpcore/app/migrations/0001_squashed_0090_char_to_text_field.py +0 -95
  14. pulpcore/app/migrations/0091_systemid.py +1 -1
  15. pulp_file/app/migrations/0006_delete_filefilesystemexporter.py → pulpcore/app/migrations/0136_delete_basedistribution.py +3 -3
  16. pulpcore/app/migrations/0137_appstatus.py +33 -0
  17. pulpcore/app/migrations/0138_vulnerabilityreport.py +33 -0
  18. pulpcore/app/models/__init__.py +4 -1
  19. pulpcore/app/models/publication.py +0 -41
  20. pulpcore/app/models/status.py +145 -0
  21. pulpcore/app/models/task.py +1 -0
  22. pulpcore/app/models/vulnerability_report.py +34 -0
  23. pulpcore/app/serializers/__init__.py +1 -0
  24. pulpcore/app/serializers/content.py +13 -1
  25. pulpcore/app/serializers/repository.py +8 -1
  26. pulpcore/app/serializers/vulnerability_report.py +27 -0
  27. pulpcore/app/settings.py +14 -47
  28. pulpcore/app/tasks/__init__.py +2 -0
  29. pulpcore/app/tasks/vulnerability_report.py +159 -0
  30. pulpcore/app/viewsets/__init__.py +1 -0
  31. pulpcore/app/viewsets/vulnerability_report.py +20 -0
  32. pulpcore/constants.py +4 -0
  33. pulpcore/content/__init__.py +23 -22
  34. pulpcore/content/handler.py +5 -2
  35. pulpcore/migrations.py +37 -11
  36. pulpcore/openapi/__init__.py +8 -0
  37. pulpcore/plugin/models/__init__.py +2 -0
  38. pulpcore/plugin/serializers/__init__.py +2 -0
  39. pulpcore/plugin/tasking.py +2 -0
  40. pulpcore/plugin/viewsets/__init__.py +2 -0
  41. pulpcore/pytest_plugin.py +21 -21
  42. pulpcore/tasking/worker.py +38 -35
  43. pulpcore/tests/functional/api/test_auth.py +18 -3
  44. pulpcore/tests/functional/api/test_openapi_schema.py +32 -15
  45. pulpcore/tests/functional/api/using_plugin/test_checkpoint.py +23 -1
  46. pulpcore/tests/functional/api/using_plugin/test_proxy.py +1 -1
  47. pulpcore/tests/unit/content/test_heartbeat.py +11 -8
  48. pulpcore/tests/unit/test_vulnerability_report.py +74 -0
  49. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/METADATA +12 -17
  50. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/RECORD +54 -152
  51. pulp_certguard/app/utils.py +0 -28
  52. pulp_certguard/tests/unit/test_models.py +0 -9
  53. pulp_file/app/migrations/0001_initial.py +0 -59
  54. pulp_file/app/migrations/0002_file_related_names.py +0 -55
  55. pulp_file/app/migrations/0003_auto_20191014_1721.py +0 -18
  56. pulp_file/app/migrations/0004_filefilesystemexporter.py +0 -21
  57. pulp_file/app/migrations/0005_filerepository.py +0 -24
  58. pulp_file/app/migrations/0007_filefilesystemexporter.py +0 -25
  59. pulp_file/app/migrations/0008_add_manifest_field.py +0 -19
  60. pulp_file/app/migrations/0009_move_data_to_new_master_distribution_model.py +0 -77
  61. pulp_file/app/migrations/0010_auto_publish.py +0 -23
  62. pulp_file/app/migrations/0011_fix_auto_publish.py +0 -36
  63. pulp_file/app/migrations/0012_delete_filefilesystemexporter.py +0 -28
  64. pulp_file/app/migrations/0013_file_acs.py +0 -24
  65. pulp_file/app/migrations/0014_new_rbac_permissions.py +0 -33
  66. pulp_file/app/migrations/0015_allow_null_manifest.py +0 -23
  67. pulp_file/app/migrations/0016_add_domain.py +0 -25
  68. pulpcore/app/migrations/0001_initial.py +0 -451
  69. pulpcore/app/migrations/0002_increase_artifact_size_field.py +0 -18
  70. pulpcore/app/migrations/0003_remove_upload_completed.py +0 -17
  71. pulpcore/app/migrations/0004_add_duplicated_reserved_resources.py +0 -45
  72. pulpcore/app/migrations/0005_progressreport_code.py +0 -19
  73. pulpcore/app/migrations/0006_repository_plugin_managed.py +0 -18
  74. pulpcore/app/migrations/0007_delete_progress_proxies.py +0 -19
  75. pulpcore/app/migrations/0008_published_metadata_as_content.py +0 -44
  76. pulpcore/app/migrations/0009_remove_task_non_fatal_errors.py +0 -17
  77. pulpcore/app/migrations/0010_pulp_fields.py +0 -570
  78. pulpcore/app/migrations/0011_relative_path.py +0 -28
  79. pulpcore/app/migrations/0012_auto_20191104_2000.py +0 -31
  80. pulpcore/app/migrations/0013_repository_pulp_type.py +0 -18
  81. pulpcore/app/migrations/0014_remove_repository_plugin_managed.py +0 -17
  82. pulpcore/app/migrations/0015_auto_20191112_1426.py +0 -33
  83. pulpcore/app/migrations/0016_charfield_to_textfield.py +0 -68
  84. pulpcore/app/migrations/0017_remove_task_parent.py +0 -17
  85. pulpcore/app/migrations/0018_auto_20191127_2350.py +0 -20
  86. pulpcore/app/migrations/0019_add_signing_service_model.py +0 -27
  87. pulpcore/app/migrations/0020_change_publishedartifact_constraints.py +0 -17
  88. pulpcore/app/migrations/0021_add_signing_service_foreign_key.py +0 -24
  89. pulpcore/app/migrations/0022_rename_last_version.py +0 -27
  90. pulpcore/app/migrations/0023_change_exporter_models.py +0 -82
  91. pulpcore/app/migrations/0024_use_local_storage_for_uploads.py +0 -19
  92. pulpcore/app/migrations/0025_task_parent_task.py +0 -19
  93. pulpcore/app/migrations/0026_task_group.py +0 -32
  94. pulpcore/app/migrations/0027_export_backend.py +0 -31
  95. pulpcore/app/migrations/0028_import_importer_pulpimporter_pulpimporterrepository.py +0 -85
  96. pulpcore/app/migrations/0029_export_delete.py +0 -19
  97. pulpcore/app/migrations/0030_taskgroup_all_tasks_dispatched.py +0 -24
  98. pulpcore/app/migrations/0031_import_export_validate_params.py +0 -19
  99. pulpcore/app/migrations/0032_export_to_chunks.py +0 -27
  100. pulpcore/app/migrations/0033_increase_remote_artifact_size_field.py +0 -18
  101. pulpcore/app/migrations/0034_groupprogressreport.py +0 -32
  102. pulpcore/app/migrations/0035_content_upstream_id.py +0 -18
  103. pulpcore/app/migrations/0036_unprotect_last_export.py +0 -19
  104. pulpcore/app/migrations/0037_pulptemporaryfile.py +0 -28
  105. pulpcore/app/migrations/0038_repository_remote.py +0 -19
  106. pulpcore/app/migrations/0039_change_download_concurrency.py +0 -25
  107. pulpcore/app/migrations/0040_set_admin_is_staff.py +0 -28
  108. pulpcore/app/migrations/0041_accesspolicy.py +0 -29
  109. pulpcore/app/migrations/0042_rbac_for_tasks.py +0 -56
  110. pulpcore/app/migrations/0043_toc_attribute.py +0 -19
  111. pulpcore/app/migrations/0044_temp_file_artifact_field.py +0 -20
  112. pulpcore/app/migrations/0045_accesspolicy_permissions_allow_null.py +0 -19
  113. pulpcore/app/migrations/0046_task__resource_job_id.py +0 -35
  114. pulpcore/app/migrations/0047_improve_orphan_cleanup.py +0 -59
  115. pulpcore/app/migrations/0048_fips_checksums.py +0 -38
  116. pulpcore/app/migrations/0049_add_file_field_to_uploadchunk.py +0 -24
  117. pulpcore/app/migrations/0050_namespace_access_policies.py +0 -28
  118. pulpcore/app/migrations/0051_timeoutfields.py +0 -34
  119. pulpcore/app/migrations/0052_tasking_logging_cid.py +0 -18
  120. pulpcore/app/migrations/0053_remote_headers.py +0 -19
  121. pulpcore/app/migrations/0054_add_public_key.py +0 -104
  122. pulpcore/app/migrations/0055_label.py +0 -31
  123. pulpcore/app/migrations/0056_remote_rate_limit.py +0 -18
  124. pulpcore/app/migrations/0057_add_label_indexes.py +0 -23
  125. pulpcore/app/migrations/0058_accesspolicy_customized.py +0 -18
  126. pulpcore/app/migrations/0059_proxy_creds.py +0 -23
  127. pulpcore/app/migrations/0060_data_migration_proxy_creds.py +0 -45
  128. pulpcore/app/migrations/0061_call_handle_artifact_checksums_command.py +0 -87
  129. pulpcore/app/migrations/0062_add_new_distribution_mastermodel.py +0 -36
  130. pulpcore/app/migrations/0063_repository_retained_versions.py +0 -18
  131. pulpcore/app/migrations/0064_add_new_style_task_columns.py +0 -109
  132. pulpcore/app/migrations/0064_repository_user_hidden.py +0 -18
  133. pulpcore/app/migrations/0065_merge_20210615_1211.py +0 -14
  134. pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py +0 -24
  135. pulpcore/app/migrations/0067_add_protect_to_task_reservation.py +0 -19
  136. pulpcore/app/migrations/0068_add_timestamp_of_interest.py +0 -23
  137. pulpcore/app/migrations/0069_update_json_fields.py +0 -63
  138. pulpcore/app/migrations/0070_rename_retained_versions.py +0 -18
  139. pulpcore/app/migrations/0071_filesystemexport_filesystemexporter.py +0 -35
  140. pulpcore/app/migrations/0072_add_method_to_filesystem_exporter.py +0 -18
  141. pulpcore/app/migrations/0073_encrypt_remote_fields.py +0 -139
  142. pulpcore/app/migrations/0074_acs.py +0 -47
  143. pulpcore/app/migrations/0075_rbaccontentguard.py +0 -25
  144. pulpcore/app/migrations/0076_remove_reserved_resource.py +0 -39
  145. pulpcore/app/migrations/0077_move_remote_url_credentials.py +0 -41
  146. pulpcore/app/migrations/0078_grouprole_role_userrole.py +0 -70
  147. pulpcore/app/migrations/0079_rename_permissions_assignment_accesspolicy_creation_hooks.py +0 -18
  148. pulpcore/app/migrations/0080_proxy_group_model.py +0 -37
  149. pulpcore/app/migrations/0081_reapplabel_group_permissions.py +0 -59
  150. pulpcore/app/migrations/0082_add_manage_roles_permissions.py +0 -17
  151. pulpcore/app/migrations/0083_alter_group_options.py +0 -17
  152. pulpcore/app/migrations/0084_alter_rbaccontentguard_options.py +0 -17
  153. pulpcore/app/migrations/0085_contentredirectcontentguard.py +0 -26
  154. pulpcore/app/migrations/0086_task_json_fields.py +0 -77
  155. pulpcore/app/migrations/0087_taskschedule.py +0 -34
  156. pulpcore/app/migrations/0088_accesspolicy_queryset_scoping.py +0 -18
  157. pulpcore/app/migrations/0089_alter_contentredirectcontentguard_options.py +0 -17
  158. pulpcore/app/migrations/0090_char_to_text_field.py +0 -79
  159. pulpcore/tests/unit/migration/test_0077_move_remote_url_credentials.py +0 -35
  160. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/WHEEL +0 -0
  161. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/entry_points.txt +0 -0
  162. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/licenses/LICENSE +0 -0
  163. {pulpcore-3.84.0.dist-info → pulpcore-3.85.1.dist-info}/top_level.txt +0 -0
@@ -7,7 +7,7 @@ import os
7
7
 
8
8
  from asgiref.sync import sync_to_async
9
9
  from aiohttp import web
10
-
10
+ from gunicorn.arbiter import Arbiter
11
11
  import django
12
12
 
13
13
 
@@ -16,12 +16,13 @@ django.setup()
16
16
 
17
17
  from django.conf import settings # noqa: E402: module level not at top of file
18
18
  from django.db.utils import ( # noqa: E402: module level not at top of file
19
+ IntegrityError,
19
20
  InterfaceError,
20
21
  DatabaseError,
21
22
  )
22
23
 
23
24
  from pulpcore.app.apps import pulp_plugin_configs # noqa: E402: module level not at top of file
24
- from pulpcore.app.models import ContentAppStatus # noqa: E402: module level not at top of file
25
+ from pulpcore.app.models import AppStatus # noqa: E402: module level not at top of file
25
26
  from pulpcore.app.util import get_worker_name # noqa: E402: module level not at top of file
26
27
 
27
28
  from .handler import Handler # noqa: E402: module level not at top of file
@@ -48,40 +49,40 @@ CONTENT_MODULE_NAME = "content"
48
49
 
49
50
 
50
51
  async def _heartbeat():
51
- content_app_status = None
52
52
  name = get_worker_name()
53
53
  heartbeat_interval = settings.CONTENT_APP_TTL // 4
54
54
  msg = "Content App '{name}' heartbeat written, sleeping for '{interarrival}' seconds".format(
55
55
  name=name, interarrival=heartbeat_interval
56
56
  )
57
- fail_msg = (
58
- "Content App '{name}' failed to write a heartbeat to the database, sleeping for "
59
- "'{interarrival}' seconds."
60
- ).format(name=name, interarrival=heartbeat_interval)
57
+ fail_msg = ("Content App '{name}' failed to write a heartbeat to the database.").format(
58
+ name=name
59
+ )
61
60
  versions = {app.label: app.version for app in pulp_plugin_configs()}
62
61
 
62
+ try:
63
+ app_status = await AppStatus.objects.acreate(
64
+ name=name, app_type="content", versions=versions
65
+ )
66
+ except IntegrityError:
67
+ log.error(f"A content app with name {name} already exists in the database.")
68
+ exit(Arbiter.WORKER_BOOT_ERROR)
63
69
  try:
64
70
  while True:
71
+ await asyncio.sleep(heartbeat_interval)
65
72
  try:
66
- content_app_status, created = await ContentAppStatus.objects.aget_or_create(
67
- name=name, defaults={"versions": versions}
68
- )
69
-
70
- if not created:
71
- await sync_to_async(content_app_status.save_heartbeat)()
72
-
73
- if content_app_status.versions != versions:
74
- content_app_status.versions = versions
75
- await content_app_status.asave(update_fields=["versions"])
76
-
73
+ await app_status.asave_heartbeat()
77
74
  log.debug(msg)
78
75
  except (InterfaceError, DatabaseError):
79
76
  await sync_to_async(Handler._reset_db_connection)()
80
- log.info(fail_msg)
81
- await asyncio.sleep(heartbeat_interval)
77
+ try:
78
+ await app_status.asave_heartbeat()
79
+ log.debug(msg)
80
+ except (InterfaceError, DatabaseError):
81
+ log.error(fail_msg)
82
+ exit(Arbiter.WORKER_BOOT_ERROR)
82
83
  finally:
83
- if content_app_status:
84
- await content_app_status.adelete()
84
+ if app_status:
85
+ await app_status.adelete()
85
86
 
86
87
 
87
88
  async def _heartbeat_ctx(app):
@@ -705,8 +705,11 @@ class Handler:
705
705
  rel_path = rel_path[len(distro.base_path) :]
706
706
  rel_path = rel_path.lstrip("/")
707
707
 
708
- if rel_path == "" and not path.endswith("/"):
709
- # The root of a distribution base_path was requested without a slash
708
+ # Check if we need to redirect to add a trailing slash for distro root
709
+ if not path.endswith("/") and (
710
+ rel_path == "" # Distro root
711
+ or (distro.checkpoint and "/" not in rel_path) # Timestamped checkpoint root
712
+ ):
710
713
  raise HTTPMovedPermanently(f"{request.path}/")
711
714
 
712
715
  original_rel_path = rel_path
pulpcore/migrations.py CHANGED
@@ -22,21 +22,44 @@ class RequireVersion(Operation):
22
22
 
23
23
  def database_forwards(self, app_label, schema_editor, from_state, to_state):
24
24
  from_state.clear_delayed_apps_cache()
25
- ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
26
- ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
27
- Worker = from_state.apps.get_model("core", "Worker")
28
25
 
29
26
  needed_version = parse_version(self.version)
30
27
  errors = []
28
+ found_either_table = False
31
29
 
32
- for worker_class, ttl, class_name in [
33
- (ApiAppStatus, settings.API_APP_TTL, "api server"),
34
- (ContentAppStatus, settings.CONTENT_APP_TTL, "content server"),
35
- (Worker, settings.WORKER_TTL, "pulp worker"),
36
- ]:
37
- for worker in worker_class.objects.filter(
38
- last_heartbeat__gte=timezone.now() - timezone.timedelta(seconds=ttl)
39
- ):
30
+ try:
31
+ ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
32
+ ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
33
+ Worker = from_state.apps.get_model("core", "Worker")
34
+ except LookupError:
35
+ pass
36
+ else:
37
+ for worker_class, ttl, class_name in [
38
+ (ApiAppStatus, settings.API_APP_TTL, "api server"),
39
+ (ContentAppStatus, settings.CONTENT_APP_TTL, "content server"),
40
+ (Worker, settings.WORKER_TTL, "pulp worker"),
41
+ ]:
42
+ for worker in worker_class.objects.filter(
43
+ last_heartbeat__gte=timezone.now() - timezone.timedelta(seconds=ttl)
44
+ ):
45
+ present_version = worker.versions.get(self.plugin)
46
+ if (
47
+ present_version is not None
48
+ and parse_version(present_version) < needed_version
49
+ ):
50
+ errors.append(
51
+ f" - '{self.plugin}'='{present_version}' "
52
+ f"with {class_name} '{worker.name}'"
53
+ )
54
+
55
+ found_either_table = True
56
+
57
+ try:
58
+ AppStatus = from_state.apps.get_model("core", "AppStatus")
59
+ except LookupError:
60
+ pass
61
+ else:
62
+ for worker in AppStatus.objects.all():
40
63
  present_version = worker.versions.get(self.plugin)
41
64
  if present_version is not None and parse_version(present_version) < needed_version:
42
65
  errors.append(
@@ -44,6 +67,9 @@ class RequireVersion(Operation):
44
67
  f"with {class_name} '{worker.name}'"
45
68
  )
46
69
 
70
+ found_either_table = True
71
+
72
+ assert found_either_table
47
73
  if errors:
48
74
  raise RuntimeError(
49
75
  "\n".join(
@@ -468,6 +468,14 @@ class BasicAuthenticationScheme(OpenApiAuthenticationExtension):
468
468
  }
469
469
 
470
470
 
471
+ class PulpRemoteUserAuthenticationScheme(OpenApiAuthenticationExtension):
472
+ target_class = "pulpcore.app.authentication.PulpRemoteUserAuthentication"
473
+ name = "remoteUserAuthentication"
474
+
475
+ def get_security_definition(self, auto_schema):
476
+ return settings.REMOTE_USER_OPENAPI_SECURITY_SCHEME
477
+
478
+
471
479
  class JSONHeaderRemoteAuthenticationScheme(OpenApiAuthenticationExtension):
472
480
  target_class = "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
473
481
  name = "json_header_remote_authentication"
@@ -40,6 +40,7 @@ from pulpcore.app.models import (
40
40
  TaskGroup,
41
41
  Upload,
42
42
  UploadChunk,
43
+ VulnerabilityReport,
43
44
  )
44
45
 
45
46
 
@@ -87,4 +88,5 @@ __all__ = [
87
88
  "UploadChunk",
88
89
  "EncryptedTextField",
89
90
  "system_id",
91
+ "VulnerabilityReport",
90
92
  ]
@@ -38,6 +38,7 @@ from pulpcore.app.serializers import (
38
38
  ValidateFieldsMixin,
39
39
  validate_unknown_fields,
40
40
  TaskSerializer,
41
+ VulnerabilityReportSerializer,
41
42
  )
42
43
 
43
44
  from .content import (
@@ -87,4 +88,5 @@ __all__ = [
87
88
  "TaskSerializer",
88
89
  "NoArtifactContentUploadSerializer",
89
90
  "SingleArtifactContentUploadSerializer",
91
+ "VulnerabilityReportSerializer",
90
92
  ]
@@ -12,11 +12,13 @@ from pulpcore.app.tasks import (
12
12
  orphan_cleanup,
13
13
  reclaim_space,
14
14
  )
15
+ from pulpcore.app.tasks.vulnerability_report import check_content
15
16
  from pulpcore.app.tasks.repository import add_and_remove
16
17
 
17
18
 
18
19
  __all__ = [
19
20
  "ageneral_update",
21
+ "check_content",
20
22
  "dispatch",
21
23
  "fs_publication_export",
22
24
  "fs_repo_version_export",
@@ -33,6 +33,7 @@ from pulpcore.app.viewsets import (
33
33
  RolesMixin,
34
34
  TaskGroupViewSet,
35
35
  TaskViewSet,
36
+ VulnerabilityReportViewSet,
36
37
  )
37
38
 
38
39
  from pulpcore.app.viewsets.custom_filters import (
@@ -89,4 +90,5 @@ __all__ = [
89
90
  "NoArtifactContentViewSet",
90
91
  "NoArtifactContentUploadViewSet",
91
92
  "SingleArtifactContentUploadViewSet",
93
+ "VulnerabilityReportViewSet",
92
94
  ]
pulpcore/pytest_plugin.py CHANGED
@@ -608,28 +608,28 @@ def backend_settings_factory(pulp_settings):
608
608
  "bucket_name",
609
609
  ]
610
610
  keys["storages.backends.azure_storage.AzureStorage"] = [
611
- "AZURE_ACCOUNT_NAME",
612
- "AZURE_CONTAINER",
613
- "AZURE_ACCOUNT_KEY",
614
- "AZURE_URL_EXPIRATION_SECS",
615
- "AZURE_OVERWRITE_FILES",
616
- "AZURE_LOCATION",
617
- "AZURE_CONNECTION_STRING",
611
+ "account_name",
612
+ "azure_container",
613
+ "account_key",
614
+ "expiration_secs",
615
+ "overwrite_files",
616
+ "location",
617
+ "connection_string",
618
618
  ]
619
- settings = storage_settings or dict()
620
- backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
621
- not_defined_settings = (k for k in keys[backend] if k not in settings)
622
- # The CI configures s3 with STORAGES and Azure with legacy
623
- # Move all to STORAGES structure on DEFAULT_FILE_STORAGE removal
624
- if backend == "storages.backends.s3boto3.S3Boto3Storage":
625
- storages_dict = getattr(pulp_settings, "STORAGES", {})
626
- storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
627
- for key in not_defined_settings:
628
- settings[key] = storage_options.get(key)
629
- else:
630
- for key in not_defined_settings:
631
- settings[key] = getattr(pulp_settings, key, None)
632
- return backend, settings
619
+
620
+ def get_installation_storage_option(key, backend):
621
+ value = pulp_settings.STORAGES["default"]["OPTIONS"].get(key)
622
+ # Some FileSystem backend options may be defined in the top settings module
623
+ if backend == "pulpcore.app.models.storage.FileSystem" and not value:
624
+ value = getattr(pulp_settings, key, None)
625
+ return value
626
+
627
+ storage_settings = storage_settings or dict()
628
+ storage_backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
629
+ unset_storage_settings = (k for k in keys[storage_backend] if k not in storage_settings)
630
+ for key in unset_storage_settings:
631
+ storage_settings[key] = get_installation_storage_option(key, storage_backend)
632
+ return storage_backend, storage_settings
633
633
 
634
634
  return _settings_factory
635
635
 
@@ -13,7 +13,7 @@ from tempfile import TemporaryDirectory
13
13
  from packaging.version import parse as parse_version
14
14
 
15
15
  from django.conf import settings
16
- from django.db import connection
16
+ from django.db import connection, DatabaseError, IntegrityError
17
17
  from django.db.models import Case, Count, F, Max, Value, When
18
18
  from django.db.models.functions import Random
19
19
  from django.utils import timezone
@@ -29,8 +29,8 @@ from pulpcore.constants import (
29
29
  )
30
30
  from pulpcore.metrics import init_otel_meter
31
31
  from pulpcore.app.apps import pulp_plugin_configs
32
- from pulpcore.app.models import Worker, Task, ApiAppStatus, ContentAppStatus
33
- from pulpcore.app.util import PGAdvisoryLock, get_domain
32
+ from pulpcore.app.models import Worker, Task, AppStatus, ApiAppStatus, ContentAppStatus
33
+ from pulpcore.app.util import PGAdvisoryLock
34
34
  from pulpcore.exceptions import AdvisoryLockError
35
35
 
36
36
  from pulpcore.tasking.storage import WorkerDirectory
@@ -73,7 +73,14 @@ class PulpcoreWorker:
73
73
  self.last_metric_heartbeat = timezone.now()
74
74
  self.versions = {app.label: app.version for app in pulp_plugin_configs()}
75
75
  self.cursor = connection.cursor()
76
- self.worker = self.handle_worker_heartbeat()
76
+ try:
77
+ self.app_status = AppStatus.objects.create(
78
+ name=self.name, app_type="worker", versions=self.versions
79
+ )
80
+ self.worker = self.app_status._old_status
81
+ except IntegrityError:
82
+ _logger.error(f"A worker with name {self.name} already exists in the database.")
83
+ exit(1)
77
84
  # This defaults to immediate task cancellation.
78
85
  # It will be set into the future on moderately graceful worker shutdown,
79
86
  # and set to None for fully graceful shutdown.
@@ -148,52 +155,48 @@ class PulpcoreWorker:
148
155
 
149
156
  def handle_worker_heartbeat(self):
150
157
  """
151
- Create or update worker heartbeat records.
152
-
153
- Existing Worker objects are searched for one to update. If an existing one is found, it is
154
- updated. Otherwise a new Worker entry is created. Logging at the info level is also done.
158
+ Update worker heartbeat records.
155
159
 
160
+ If the update fails (the record was deleted, the database is unreachable, ...) the worker
161
+ is shut down.
156
162
  """
157
- worker, created = Worker.objects.get_or_create(
158
- name=self.name, defaults={"versions": self.versions}
159
- )
160
- if not created and worker.versions != self.versions:
161
- worker.versions = self.versions
162
- worker.save(update_fields=["versions"])
163
-
164
- if created:
165
- _logger.info(_("New worker '{name}' discovered").format(name=self.name))
166
- elif worker.online is False:
167
- _logger.info(_("Worker '{name}' is back online.").format(name=self.name))
168
-
169
- worker.save_heartbeat()
170
163
 
171
164
  msg = "Worker heartbeat from '{name}' at time {timestamp}".format(
172
- timestamp=worker.last_heartbeat, name=self.name
165
+ timestamp=self.app_status.last_heartbeat, name=self.name
173
166
  )
174
- _logger.debug(msg)
175
-
176
- return worker
167
+ try:
168
+ self.app_status.save_heartbeat()
169
+ _logger.debug(msg)
170
+ except (IntegrityError, DatabaseError):
171
+ # WARNING: Do not attempt to recycle the connection here.
172
+ # The advisory locks are bound to the connection and we must not loose them.
173
+ _logger.error(f"Updating the heartbeat of worker {self.name} failed.")
174
+ # TODO if shutdown_requested, we may need to be more aggressive.
175
+ self.shutdown_requested = True
176
+ self.cancel_task = True
177
177
 
178
178
  def shutdown(self):
179
- self.worker.delete()
179
+ self.app_status.delete()
180
180
  _logger.info(_("Worker %s was shut down."), self.name)
181
181
 
182
182
  def worker_cleanup(self):
183
+ qs = AppStatus.objects.older_than(age=timedelta(days=7))
184
+ for app_worker in qs:
185
+ _logger.info(_("Clean missing %s worker %s."), app_worker.app_type, app_worker.name)
186
+ qs.delete()
183
187
  for cls, cls_name in (
184
188
  (Worker, "pulp"),
185
189
  (ApiAppStatus, "api"),
186
190
  (ContentAppStatus, "content"),
187
191
  ):
188
192
  qs = cls.objects.missing(age=timedelta(days=7))
189
- if qs:
190
- for app_worker in qs:
191
- _logger.info(_("Clean missing %s worker %s."), cls_name, app_worker.name)
192
- qs.delete()
193
+ for app_worker in qs:
194
+ _logger.info(_("Clean missing %s worker %s."), cls_name, app_worker.name)
195
+ qs.delete()
193
196
 
194
197
  def beat(self):
195
- if self.worker.last_heartbeat < timezone.now() - self.heartbeat_period:
196
- self.worker = self.handle_worker_heartbeat()
198
+ if self.app_status.last_heartbeat < timezone.now() - self.heartbeat_period:
199
+ self.handle_worker_heartbeat()
197
200
  if not self.auxiliary:
198
201
  self.worker_cleanup_countdown -= 1
199
202
  if self.worker_cleanup_countdown <= 0:
@@ -217,7 +220,7 @@ class PulpcoreWorker:
217
220
  Return ``True`` if the task was actually canceled, ``False`` otherwise.
218
221
  """
219
222
  # A task is considered abandoned when in running state, but no worker holds its lock
220
- domain = get_domain()
223
+ domain = task.pulp_domain
221
224
  try:
222
225
  task.set_canceling()
223
226
  except RuntimeError:
@@ -244,7 +247,7 @@ class PulpcoreWorker:
244
247
  return True
245
248
 
246
249
  def is_compatible(self, task):
247
- domain = get_domain()
250
+ domain = task.pulp_domain
248
251
  unmatched_versions = [
249
252
  f"task: {label}>={version} worker: {self.versions.get(label)}"
250
253
  for label, version in task.versions.items()
@@ -427,7 +430,7 @@ class PulpcoreWorker:
427
430
  task.save(update_fields=["worker"])
428
431
  cancel_state = None
429
432
  cancel_reason = None
430
- domain = get_domain()
433
+ domain = task.pulp_domain
431
434
  with TemporaryDirectory(dir=".") as task_working_dir_rel_path:
432
435
  task_process = Process(target=perform_task, args=(task.pk, task_working_dir_rel_path))
433
436
  task_process.start()
@@ -13,6 +13,11 @@ from pulpcore.app import settings
13
13
 
14
14
 
15
15
  @pytest.mark.parallel
16
+ @pytest.mark.skipif(
17
+ "rest_framework.authentication.BasicAuthentication"
18
+ not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
19
+ reason="Test can't run unless BasicAuthentication is enabled",
20
+ )
16
21
  def test_base_auth_success(pulpcore_bindings, pulp_admin_user):
17
22
  """Perform HTTP basic authentication with valid credentials.
18
23
 
@@ -33,6 +38,11 @@ def test_base_auth_success(pulpcore_bindings, pulp_admin_user):
33
38
 
34
39
 
35
40
  @pytest.mark.parallel
41
+ @pytest.mark.skipif(
42
+ "rest_framework.authentication.BasicAuthentication"
43
+ not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
44
+ reason="Test can't run unless BasicAuthentication is enabled",
45
+ )
36
46
  def test_base_auth_failure(pulpcore_bindings, invalid_user):
37
47
  """Perform HTTP basic authentication with invalid credentials.
38
48
 
@@ -50,6 +60,11 @@ def test_base_auth_failure(pulpcore_bindings, invalid_user):
50
60
 
51
61
 
52
62
  @pytest.mark.parallel
63
+ @pytest.mark.skipif(
64
+ "rest_framework.authentication.BasicAuthentication"
65
+ not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
66
+ reason="Test can't run unless BasicAuthentication is enabled",
67
+ )
53
68
  def test_base_auth_required(pulpcore_bindings, anonymous_user):
54
69
  """Perform HTTP basic authentication with no credentials.
55
70
 
@@ -69,7 +84,7 @@ def test_base_auth_required(pulpcore_bindings, anonymous_user):
69
84
  @pytest.mark.parallel
70
85
  @pytest.mark.skipif(
71
86
  "django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
72
- and "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
87
+ or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
73
88
  not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
74
89
  reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
75
90
  )
@@ -98,7 +113,7 @@ def test_jq_header_remote_auth(pulpcore_bindings, anonymous_user):
98
113
  @pytest.mark.parallel
99
114
  @pytest.mark.skipif(
100
115
  "django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
101
- and "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
116
+ or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
102
117
  not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
103
118
  reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
104
119
  )
@@ -127,7 +142,7 @@ def test_jq_header_remote_auth_denied_by_wrong_header(pulpcore_bindings, anonymo
127
142
  @pytest.mark.parallel
128
143
  @pytest.mark.skipif(
129
144
  "django.contrib.auth.backends.RemoteUserBackend" not in settings.AUTHENTICATION_BACKENDS
130
- and "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
145
+ or "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
131
146
  not in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"],
132
147
  reason="Test can't run unless RemoteUserBackend and JSONHeaderRemoteAuthentication are enabled",
133
148
  )
@@ -2,7 +2,7 @@
2
2
 
3
3
  import copy
4
4
  import json
5
- import os
5
+ from pathlib import Path
6
6
 
7
7
  import pytest
8
8
  import jsonschema
@@ -10,28 +10,31 @@ import jsonschema
10
10
  from drf_spectacular import validation
11
11
  from collections import defaultdict
12
12
 
13
- JSON_SCHEMA_SPEC_PATH = os.path.join(
14
- os.path.dirname(validation.__file__), "openapi_3_0_schema.json"
15
- )
16
-
17
13
 
18
14
  @pytest.fixture(scope="session")
19
- def openapi3_schema_spec():
20
- with open(JSON_SCHEMA_SPEC_PATH) as fh:
21
- openapi3_schema_spec = json.load(fh)
22
-
23
- return openapi3_schema_spec
15
+ def openapi3_schema_spec(pulp_openapi_schema):
16
+ schema_version = pulp_openapi_schema["openapi"]
17
+ if schema_version.startswith("3.0"):
18
+ spec_path = Path(validation.__file__).parent / "openapi_3_0_schema.json"
19
+ elif schema_version.startswith("3.1"):
20
+ spec_path = Path(validation.__file__).parent / "openapi_3_1_schema.json"
21
+ else:
22
+ pytest.fail(f"Unknown OpenAPI schema version [{schema_version}].")
23
+ return json.loads(spec_path.read_text())
24
24
 
25
25
 
26
26
  @pytest.fixture(scope="session")
27
27
  def openapi3_schema_with_modified_safe_chars(openapi3_schema_spec):
28
- openapi3_schema_spec_copy = copy.deepcopy(openapi3_schema_spec) # Don't modify the original
28
+ oas_copy = copy.deepcopy(openapi3_schema_spec) # Don't modify the original
29
29
  # Making OpenAPI validation to accept paths starting with / and {
30
- properties = openapi3_schema_spec_copy["definitions"]["Paths"]["patternProperties"]
31
- properties["^\\/|{"] = properties["^\\/"]
32
- del properties["^\\/"]
30
+ if "3.1.x" in oas_copy["description"]:
31
+ pattern_properties = oas_copy["$defs"]["paths"]["patternProperties"]
32
+ pattern_properties["^/|{"] = pattern_properties.pop("^/")
33
+ else:
34
+ pattern_properties = oas_copy["definitions"]["Paths"]["patternProperties"]
35
+ pattern_properties["^\\/|{"] = pattern_properties.pop("^\\/")
33
36
 
34
- return openapi3_schema_spec_copy
37
+ return oas_copy
35
38
 
36
39
 
37
40
  @pytest.mark.parallel
@@ -68,6 +71,20 @@ def test_no_dup_operation_ids(pulp_openapi_schema):
68
71
  assert len(dup_ids) == 0, f"Duplicate operationIds found: {dup_ids}"
69
72
 
70
73
 
74
+ @pytest.mark.parallel
75
+ def test_remote_user_auth_security_scheme(pulp_settings, pulp_openapi_schema):
76
+ if (
77
+ "pulpcore.app.authentication.PulpRemoteUserAuthentication"
78
+ not in pulp_settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"]
79
+ ):
80
+ pytest.skip("Test can't run unless PulpRemoteUserAuthentication is enabled.")
81
+
82
+ expected_security_scheme = pulp_settings.REMOTE_USER_OPENAPI_SECURITY_SCHEME
83
+ security_schemes = pulp_openapi_schema["components"]["securitySchemes"]
84
+
85
+ assert security_schemes["remoteUserAuthentication"] == expected_security_scheme
86
+
87
+
71
88
  @pytest.mark.parallel
72
89
  def test_external_auth_on_security_scheme(pulp_settings, pulp_openapi_schema):
73
90
  if (
@@ -87,11 +87,17 @@ class TestCheckpointDistribution:
87
87
  assert Handler._format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
88
88
 
89
89
  @pytest.mark.parallel
90
- def test_no_trailing_slash_is_redirected(self, setup, http_get, distribution_base_url):
90
+ def test_distro_root_no_trailing_slash_is_redirected(
91
+ self,
92
+ setup,
93
+ http_get,
94
+ distribution_base_url,
95
+ ):
91
96
  """Test checkpoint listing when path doesn't end with a slash."""
92
97
 
93
98
  pubs, distribution = setup
94
99
 
100
+ # Test a checkpoint distro listing path
95
101
  response = http_get(distribution_base_url(distribution.base_url[:-1])).decode("utf-8")
96
102
  checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
97
103
 
@@ -99,6 +105,22 @@ class TestCheckpointDistribution:
99
105
  assert Handler._format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
100
106
  assert Handler._format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
101
107
 
108
+ @pytest.mark.parallel
109
+ def test_timestamped_checkpoint_no_trailing_slash_is_redirected(
110
+ self,
111
+ setup,
112
+ http_get,
113
+ checkpoint_url,
114
+ ):
115
+ """Test a timestamped checkpoint when path doesn't end with a slash."""
116
+
117
+ pubs, distribution = setup
118
+
119
+ pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
120
+ response = http_get(pub_1_url[:-1]).decode("utf-8")
121
+
122
+ assert f"<h1>Index of {urlparse(pub_1_url).path}</h1>" in response
123
+
102
124
  @pytest.mark.parallel
103
125
  def test_exact_timestamp_is_served(self, setup, http_get, checkpoint_url):
104
126
  pubs, distribution = setup
@@ -144,7 +144,7 @@ def test_sync_https_through_https_proxy(
144
144
  manifest_path=basic_manifest_path,
145
145
  policy="on_demand",
146
146
  proxy_url=https_proxy.proxy_url,
147
- tls_validation="false",
147
+ tls_validation=False,
148
148
  )
149
149
 
150
150
  _run_basic_sync_and_assert(file_bindings, monitor_task, remote_on_demand, file_repo)
@@ -5,29 +5,32 @@ from django.db.utils import InterfaceError, OperationalError
5
5
 
6
6
  from pulpcore.content import _heartbeat
7
7
  from pulpcore.content.handler import Handler
8
- from pulpcore.app.models import ContentAppStatus
8
+ from pulpcore.app.models.status import AppStatusManager
9
9
 
10
10
 
11
11
  class MockException(Exception):
12
12
  pass
13
13
 
14
14
 
15
+ @pytest.mark.parametrize("error_class", [InterfaceError, OperationalError])
15
16
  @pytest.mark.asyncio
16
- async def test_db_connection_interface_error(monkeypatch, settings):
17
+ async def test_db_connection_interface_error(monkeypatch, settings, error_class):
17
18
  """
18
19
  Test that if an InterfaceError or OperationalError is raised,
19
20
  Handler._reset_db_connection() is called
20
21
  """
21
22
 
22
- mock_aget_or_create = AsyncMock()
23
- mock_aget_or_create.side_effect = [InterfaceError(), OperationalError(), MockException()]
24
- monkeypatch.setattr(ContentAppStatus.objects, "aget_or_create", mock_aget_or_create)
23
+ mock_app_status = AsyncMock()
24
+ mock_app_status.asave_heartbeat.side_effect = [error_class(), error_class()]
25
+ mock_acreate = AsyncMock()
26
+ mock_acreate.return_value = mock_app_status
27
+ monkeypatch.setattr(AppStatusManager, "acreate", mock_acreate)
25
28
  mock_reset_db = Mock()
26
29
  monkeypatch.setattr(Handler, "_reset_db_connection", mock_reset_db)
27
30
  settings.CONTENT_APP_TTL = 1
28
31
 
29
- with pytest.raises(MockException):
32
+ with pytest.raises(SystemExit):
30
33
  await _heartbeat()
31
34
 
32
- mock_aget_or_create.assert_called()
33
- mock_reset_db.assert_has_calls([call(), call()])
35
+ mock_app_status.asave_heartbeat.assert_called()
36
+ mock_reset_db.assert_has_calls([call()])