pulpcore 3.84.0__py3-none-any.whl → 3.85.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pulpcore might be problematic. Click here for more details.

Files changed (163) hide show
  1. pulp_certguard/app/__init__.py +1 -1
  2. pulp_certguard/app/models.py +7 -26
  3. pulp_certguard/app/serializers.py +0 -2
  4. pulp_certguard/rhsm/__init__.py +4 -0
  5. pulp_certguard/rhsm/rhsm_check_path.py +198 -0
  6. pulp_certguard/tests/unit/certdata.py +249 -0
  7. pulp_certguard/tests/unit/test_rhsm_check_path.py +213 -0
  8. pulp_file/app/__init__.py +1 -1
  9. pulp_file/app/migrations/0001_initial_squashed_0016_add_domain.py +0 -20
  10. pulp_file/app/migrations/0017_alter_filealternatecontentsource_alternatecontentsource_ptr_and_more.py +1 -1
  11. pulpcore/app/apps.py +2 -12
  12. pulpcore/app/entrypoint.py +22 -22
  13. pulpcore/app/migrations/0001_squashed_0090_char_to_text_field.py +0 -95
  14. pulpcore/app/migrations/0091_systemid.py +1 -1
  15. pulp_file/app/migrations/0006_delete_filefilesystemexporter.py → pulpcore/app/migrations/0136_delete_basedistribution.py +3 -3
  16. pulpcore/app/migrations/0137_appstatus.py +33 -0
  17. pulpcore/app/migrations/0138_vulnerabilityreport.py +33 -0
  18. pulpcore/app/models/__init__.py +4 -1
  19. pulpcore/app/models/publication.py +0 -41
  20. pulpcore/app/models/status.py +145 -0
  21. pulpcore/app/models/task.py +1 -0
  22. pulpcore/app/models/vulnerability_report.py +34 -0
  23. pulpcore/app/serializers/__init__.py +1 -0
  24. pulpcore/app/serializers/content.py +13 -1
  25. pulpcore/app/serializers/repository.py +8 -1
  26. pulpcore/app/serializers/vulnerability_report.py +27 -0
  27. pulpcore/app/settings.py +14 -47
  28. pulpcore/app/tasks/__init__.py +2 -0
  29. pulpcore/app/tasks/vulnerability_report.py +159 -0
  30. pulpcore/app/viewsets/__init__.py +1 -0
  31. pulpcore/app/viewsets/vulnerability_report.py +20 -0
  32. pulpcore/constants.py +4 -0
  33. pulpcore/content/__init__.py +23 -22
  34. pulpcore/content/handler.py +5 -2
  35. pulpcore/migrations.py +38 -11
  36. pulpcore/openapi/__init__.py +8 -0
  37. pulpcore/plugin/models/__init__.py +2 -0
  38. pulpcore/plugin/serializers/__init__.py +2 -0
  39. pulpcore/plugin/tasking.py +2 -0
  40. pulpcore/plugin/viewsets/__init__.py +2 -0
  41. pulpcore/pytest_plugin.py +21 -21
  42. pulpcore/tasking/worker.py +38 -35
  43. pulpcore/tests/functional/api/test_auth.py +18 -3
  44. pulpcore/tests/functional/api/test_openapi_schema.py +32 -15
  45. pulpcore/tests/functional/api/using_plugin/test_checkpoint.py +23 -1
  46. pulpcore/tests/functional/api/using_plugin/test_proxy.py +1 -1
  47. pulpcore/tests/unit/content/test_heartbeat.py +11 -8
  48. pulpcore/tests/unit/test_vulnerability_report.py +74 -0
  49. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/METADATA +12 -17
  50. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/RECORD +54 -152
  51. pulp_certguard/app/utils.py +0 -28
  52. pulp_certguard/tests/unit/test_models.py +0 -9
  53. pulp_file/app/migrations/0001_initial.py +0 -59
  54. pulp_file/app/migrations/0002_file_related_names.py +0 -55
  55. pulp_file/app/migrations/0003_auto_20191014_1721.py +0 -18
  56. pulp_file/app/migrations/0004_filefilesystemexporter.py +0 -21
  57. pulp_file/app/migrations/0005_filerepository.py +0 -24
  58. pulp_file/app/migrations/0007_filefilesystemexporter.py +0 -25
  59. pulp_file/app/migrations/0008_add_manifest_field.py +0 -19
  60. pulp_file/app/migrations/0009_move_data_to_new_master_distribution_model.py +0 -77
  61. pulp_file/app/migrations/0010_auto_publish.py +0 -23
  62. pulp_file/app/migrations/0011_fix_auto_publish.py +0 -36
  63. pulp_file/app/migrations/0012_delete_filefilesystemexporter.py +0 -28
  64. pulp_file/app/migrations/0013_file_acs.py +0 -24
  65. pulp_file/app/migrations/0014_new_rbac_permissions.py +0 -33
  66. pulp_file/app/migrations/0015_allow_null_manifest.py +0 -23
  67. pulp_file/app/migrations/0016_add_domain.py +0 -25
  68. pulpcore/app/migrations/0001_initial.py +0 -451
  69. pulpcore/app/migrations/0002_increase_artifact_size_field.py +0 -18
  70. pulpcore/app/migrations/0003_remove_upload_completed.py +0 -17
  71. pulpcore/app/migrations/0004_add_duplicated_reserved_resources.py +0 -45
  72. pulpcore/app/migrations/0005_progressreport_code.py +0 -19
  73. pulpcore/app/migrations/0006_repository_plugin_managed.py +0 -18
  74. pulpcore/app/migrations/0007_delete_progress_proxies.py +0 -19
  75. pulpcore/app/migrations/0008_published_metadata_as_content.py +0 -44
  76. pulpcore/app/migrations/0009_remove_task_non_fatal_errors.py +0 -17
  77. pulpcore/app/migrations/0010_pulp_fields.py +0 -570
  78. pulpcore/app/migrations/0011_relative_path.py +0 -28
  79. pulpcore/app/migrations/0012_auto_20191104_2000.py +0 -31
  80. pulpcore/app/migrations/0013_repository_pulp_type.py +0 -18
  81. pulpcore/app/migrations/0014_remove_repository_plugin_managed.py +0 -17
  82. pulpcore/app/migrations/0015_auto_20191112_1426.py +0 -33
  83. pulpcore/app/migrations/0016_charfield_to_textfield.py +0 -68
  84. pulpcore/app/migrations/0017_remove_task_parent.py +0 -17
  85. pulpcore/app/migrations/0018_auto_20191127_2350.py +0 -20
  86. pulpcore/app/migrations/0019_add_signing_service_model.py +0 -27
  87. pulpcore/app/migrations/0020_change_publishedartifact_constraints.py +0 -17
  88. pulpcore/app/migrations/0021_add_signing_service_foreign_key.py +0 -24
  89. pulpcore/app/migrations/0022_rename_last_version.py +0 -27
  90. pulpcore/app/migrations/0023_change_exporter_models.py +0 -82
  91. pulpcore/app/migrations/0024_use_local_storage_for_uploads.py +0 -19
  92. pulpcore/app/migrations/0025_task_parent_task.py +0 -19
  93. pulpcore/app/migrations/0026_task_group.py +0 -32
  94. pulpcore/app/migrations/0027_export_backend.py +0 -31
  95. pulpcore/app/migrations/0028_import_importer_pulpimporter_pulpimporterrepository.py +0 -85
  96. pulpcore/app/migrations/0029_export_delete.py +0 -19
  97. pulpcore/app/migrations/0030_taskgroup_all_tasks_dispatched.py +0 -24
  98. pulpcore/app/migrations/0031_import_export_validate_params.py +0 -19
  99. pulpcore/app/migrations/0032_export_to_chunks.py +0 -27
  100. pulpcore/app/migrations/0033_increase_remote_artifact_size_field.py +0 -18
  101. pulpcore/app/migrations/0034_groupprogressreport.py +0 -32
  102. pulpcore/app/migrations/0035_content_upstream_id.py +0 -18
  103. pulpcore/app/migrations/0036_unprotect_last_export.py +0 -19
  104. pulpcore/app/migrations/0037_pulptemporaryfile.py +0 -28
  105. pulpcore/app/migrations/0038_repository_remote.py +0 -19
  106. pulpcore/app/migrations/0039_change_download_concurrency.py +0 -25
  107. pulpcore/app/migrations/0040_set_admin_is_staff.py +0 -28
  108. pulpcore/app/migrations/0041_accesspolicy.py +0 -29
  109. pulpcore/app/migrations/0042_rbac_for_tasks.py +0 -56
  110. pulpcore/app/migrations/0043_toc_attribute.py +0 -19
  111. pulpcore/app/migrations/0044_temp_file_artifact_field.py +0 -20
  112. pulpcore/app/migrations/0045_accesspolicy_permissions_allow_null.py +0 -19
  113. pulpcore/app/migrations/0046_task__resource_job_id.py +0 -35
  114. pulpcore/app/migrations/0047_improve_orphan_cleanup.py +0 -59
  115. pulpcore/app/migrations/0048_fips_checksums.py +0 -38
  116. pulpcore/app/migrations/0049_add_file_field_to_uploadchunk.py +0 -24
  117. pulpcore/app/migrations/0050_namespace_access_policies.py +0 -28
  118. pulpcore/app/migrations/0051_timeoutfields.py +0 -34
  119. pulpcore/app/migrations/0052_tasking_logging_cid.py +0 -18
  120. pulpcore/app/migrations/0053_remote_headers.py +0 -19
  121. pulpcore/app/migrations/0054_add_public_key.py +0 -104
  122. pulpcore/app/migrations/0055_label.py +0 -31
  123. pulpcore/app/migrations/0056_remote_rate_limit.py +0 -18
  124. pulpcore/app/migrations/0057_add_label_indexes.py +0 -23
  125. pulpcore/app/migrations/0058_accesspolicy_customized.py +0 -18
  126. pulpcore/app/migrations/0059_proxy_creds.py +0 -23
  127. pulpcore/app/migrations/0060_data_migration_proxy_creds.py +0 -45
  128. pulpcore/app/migrations/0061_call_handle_artifact_checksums_command.py +0 -87
  129. pulpcore/app/migrations/0062_add_new_distribution_mastermodel.py +0 -36
  130. pulpcore/app/migrations/0063_repository_retained_versions.py +0 -18
  131. pulpcore/app/migrations/0064_add_new_style_task_columns.py +0 -109
  132. pulpcore/app/migrations/0064_repository_user_hidden.py +0 -18
  133. pulpcore/app/migrations/0065_merge_20210615_1211.py +0 -14
  134. pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py +0 -24
  135. pulpcore/app/migrations/0067_add_protect_to_task_reservation.py +0 -19
  136. pulpcore/app/migrations/0068_add_timestamp_of_interest.py +0 -23
  137. pulpcore/app/migrations/0069_update_json_fields.py +0 -63
  138. pulpcore/app/migrations/0070_rename_retained_versions.py +0 -18
  139. pulpcore/app/migrations/0071_filesystemexport_filesystemexporter.py +0 -35
  140. pulpcore/app/migrations/0072_add_method_to_filesystem_exporter.py +0 -18
  141. pulpcore/app/migrations/0073_encrypt_remote_fields.py +0 -139
  142. pulpcore/app/migrations/0074_acs.py +0 -47
  143. pulpcore/app/migrations/0075_rbaccontentguard.py +0 -25
  144. pulpcore/app/migrations/0076_remove_reserved_resource.py +0 -39
  145. pulpcore/app/migrations/0077_move_remote_url_credentials.py +0 -41
  146. pulpcore/app/migrations/0078_grouprole_role_userrole.py +0 -70
  147. pulpcore/app/migrations/0079_rename_permissions_assignment_accesspolicy_creation_hooks.py +0 -18
  148. pulpcore/app/migrations/0080_proxy_group_model.py +0 -37
  149. pulpcore/app/migrations/0081_reapplabel_group_permissions.py +0 -59
  150. pulpcore/app/migrations/0082_add_manage_roles_permissions.py +0 -17
  151. pulpcore/app/migrations/0083_alter_group_options.py +0 -17
  152. pulpcore/app/migrations/0084_alter_rbaccontentguard_options.py +0 -17
  153. pulpcore/app/migrations/0085_contentredirectcontentguard.py +0 -26
  154. pulpcore/app/migrations/0086_task_json_fields.py +0 -77
  155. pulpcore/app/migrations/0087_taskschedule.py +0 -34
  156. pulpcore/app/migrations/0088_accesspolicy_queryset_scoping.py +0 -18
  157. pulpcore/app/migrations/0089_alter_contentredirectcontentguard_options.py +0 -17
  158. pulpcore/app/migrations/0090_char_to_text_field.py +0 -79
  159. pulpcore/tests/unit/migration/test_0077_move_remote_url_credentials.py +0 -35
  160. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/WHEEL +0 -0
  161. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/entry_points.txt +0 -0
  162. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/licenses/LICENSE +0 -0
  163. {pulpcore-3.84.0.dist-info → pulpcore-3.85.0.dist-info}/top_level.txt +0 -0
@@ -6,7 +6,9 @@ from datetime import timedelta
6
6
 
7
7
  from django.conf import settings
8
8
  from django.contrib.postgres.fields import HStoreField
9
+ from django.contrib.postgres.functions import TransactionNow
9
10
  from django.db import models
11
+ from django.db.models import F, Value
10
12
  from django.utils import timezone
11
13
 
12
14
  from pulpcore.app.models import BaseModel
@@ -47,9 +49,138 @@ class AppStatusManager(models.Manager):
47
49
  return self.filter(last_heartbeat__lt=age_threshold)
48
50
 
49
51
 
52
+ class _AppStatusManager(AppStatusManager):
53
+ # This is an intermediate class in order to allow a ZDU.
54
+ # It should be removed from the chain with 3.87.
55
+ def create(self, app_type, **kwargs):
56
+ if app_type == "api":
57
+ old_obj = ApiAppStatus.objects.create(**kwargs)
58
+ elif app_type == "worker":
59
+ from pulpcore.app.models import Worker
60
+
61
+ old_obj = Worker.objects.create(**kwargs)
62
+ else:
63
+ raise NotImplementedError(f"Invalid app_type: {app_type}")
64
+ obj = super().create(app_type=app_type, **kwargs)
65
+ obj._old_status = old_obj
66
+ return obj
67
+
68
+ async def acreate(self, app_type, **kwargs):
69
+ if app_type == "content":
70
+ old_obj = await ContentAppStatus.objects.acreate(**kwargs)
71
+ else:
72
+ raise NotImplementedError(f"Invalid app_type: {app_type}")
73
+ obj = await super().acreate(app_type=app_type, **kwargs)
74
+ obj._old_status = old_obj
75
+ return obj
76
+
77
+ def online(self):
78
+ """
79
+ Returns a queryset of objects that are online.
80
+ """
81
+ return self.filter(last_heartbeat__gte=TransactionNow() - F("ttl"))
82
+
83
+ def missing(self):
84
+ """
85
+ Returns a queryset of workers that are missing.
86
+ """
87
+ return self.filter(last_heartbeat__lt=TransactionNow() - F("ttl"))
88
+
89
+ def older_than(self, age):
90
+ """
91
+ Returns a queryset of workers that are older than age.
92
+ """
93
+ return self.filter(last_heartbeat__lt=TransactionNow() - Value(age))
94
+
95
+
96
+ class AppStatus(BaseModel):
97
+ APP_TYPES = [
98
+ ("api", "api"),
99
+ ("content", "content"),
100
+ ("worker", "worker"),
101
+ ]
102
+ _APP_TTL = {
103
+ "api": settings.API_APP_TTL,
104
+ "content": settings.CONTENT_APP_TTL,
105
+ "worker": settings.WORKER_TTL,
106
+ }
107
+ objects = _AppStatusManager()
108
+
109
+ app_type = models.CharField(max_length=10, choices=APP_TYPES)
110
+ name = models.TextField(db_index=True, unique=True)
111
+ versions = HStoreField(default=dict)
112
+ ttl = models.DurationField(null=False)
113
+ last_heartbeat = models.DateTimeField(auto_now=True)
114
+
115
+ def __init__(self, *args, **kwargs):
116
+ super().__init__(*args, **kwargs)
117
+ self.ttl = timedelta(seconds=self._APP_TTL[self.app_type])
118
+ self._old_status = None
119
+
120
+ def delete(self, *args, **kwargs):
121
+ # adelete will call into this, so we should not replicate that one here.
122
+ if self._old_status is not None:
123
+ self._old_status.delete(*args, **kwargs)
124
+ super().delete(*args, **kwargs)
125
+
126
+ @property
127
+ def online(self) -> bool:
128
+ """
129
+ To be considered 'online', an app must have a timestamp more recent than ``self.ttl``.
130
+ """
131
+ age_threshold = timezone.now() - self.ttl
132
+ return self.last_heartbeat >= age_threshold
133
+
134
+ @property
135
+ def missing(self):
136
+ """
137
+ Whether an app can be considered 'missing'
138
+
139
+ To be considered 'missing', an App must have a timestamp older than ``self.ttl``.
140
+
141
+ Returns:
142
+ bool: True if the app is considered missing, otherwise False
143
+ """
144
+ return not self.online
145
+
146
+ def save_heartbeat(self):
147
+ """
148
+ Update the last_heartbeat field to now and save it.
149
+
150
+ Only the last_heartbeat field will be saved. No other changes will be saved.
151
+
152
+ Raises:
153
+ ValueError: When the model instance has never been saved before. This method can
154
+ only update an existing database record.
155
+ """
156
+ self._old_status.save_heartbeat()
157
+ self.save(update_fields=["last_heartbeat"])
158
+
159
+ async def asave_heartbeat(self):
160
+ """
161
+ Update the last_heartbeat field to now and save it.
162
+
163
+ Only the last_heartbeat field will be saved. No other changes will be saved.
164
+
165
+ Raises:
166
+ ValueError: When the model instance has never been saved before. This method can
167
+ only update an existing database record.
168
+ """
169
+ await self._old_status.asave_heartbeat()
170
+ await self.asave(update_fields=["last_heartbeat"])
171
+
172
+ @property
173
+ def current_task(self):
174
+ """
175
+ The task this worker is currently executing, if any.
176
+ """
177
+ return self.tasks.filter(state="running").first()
178
+
179
+
50
180
  class BaseAppStatus(BaseModel):
51
181
  """
52
182
  Represents an AppStatus.
183
+ Deprecated, to be removed with 3.87.
53
184
 
54
185
  This class is abstract. Subclasses must define `APP_TTL` as a `timedelta`.
55
186
 
@@ -103,6 +234,18 @@ class BaseAppStatus(BaseModel):
103
234
  """
104
235
  self.save(update_fields=["last_heartbeat"])
105
236
 
237
+ async def asave_heartbeat(self):
238
+ """
239
+ Update the last_heartbeat field to now and save it.
240
+
241
+ Only the last_heartbeat field will be saved. No other changes will be saved.
242
+
243
+ Raises:
244
+ ValueError: When the model instance has never been saved before. This method can
245
+ only update an existing database record.
246
+ """
247
+ await self.asave(update_fields=["last_heartbeat"])
248
+
106
249
  class Meta:
107
250
  abstract = True
108
251
 
@@ -110,6 +253,7 @@ class BaseAppStatus(BaseModel):
110
253
  class ApiAppStatus(BaseAppStatus):
111
254
  """
112
255
  Represents a Api App Status
256
+ Deprecated, to be removed with 3.87.
113
257
  """
114
258
 
115
259
  APP_TTL = timedelta(seconds=settings.API_APP_TTL)
@@ -118,6 +262,7 @@ class ApiAppStatus(BaseAppStatus):
118
262
  class ContentAppStatus(BaseAppStatus):
119
263
  """
120
264
  Represents a Content App Status
265
+ Deprecated, to be removed with 3.87.
121
266
  """
122
267
 
123
268
  APP_TTL = timedelta(seconds=settings.CONTENT_APP_TTL)
@@ -33,6 +33,7 @@ _logger = logging.getLogger(__name__)
33
33
  class Worker(BaseAppStatus):
34
34
  """
35
35
  Represents a worker
36
+ Deprecated, to be removed with 3.87.
36
37
  """
37
38
 
38
39
  APP_TTL = timedelta(seconds=settings.WORKER_TTL)
@@ -0,0 +1,34 @@
1
+ from django.db import models
2
+
3
+ from pulpcore.app.models.base import BaseModel
4
+ from pulpcore.app.util import get_domain_pk
5
+
6
+
7
+ class VulnerabilityReport(BaseModel):
8
+ """
9
+ A model for storing vulnerability reports for Content/RepositoryVersion.
10
+
11
+ Fields:
12
+ vulns (models.JSONField): A JSON field containing the list of vulnerabilities found
13
+ in osv.dev.
14
+
15
+ Relations:
16
+ content (models.OneToOneField): The Content object that was scanned for vulnerabilities.
17
+ pulp_domain (models.ForeignKey): The Domain this vulnerability report belongs to,
18
+ providing multi-tenancy isolation.
19
+ repo_versions (models.ManyToManyField): The RepositoryVersion(s) where the scanned
20
+ Content appears. This allows tracking which repository versions contain
21
+ vulnerable content.
22
+ """
23
+
24
+ content = models.OneToOneField(
25
+ "Content",
26
+ on_delete=models.CASCADE,
27
+ unique=True,
28
+ )
29
+ vulns = models.JSONField()
30
+ pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.CASCADE)
31
+ repo_versions = models.ManyToManyField("RepositoryVersion", blank=True)
32
+
33
+ class Meta:
34
+ default_related_name = "%(app_label)s_%(model_name)s"
@@ -121,6 +121,7 @@ from .user import (
121
121
  UserSerializer,
122
122
  )
123
123
  from .replica import UpstreamPulpSerializer
124
+ from .vulnerability_report import VulnerabilityReportSerializer
124
125
  from .openpgp import (
125
126
  OpenPGPDistributionSerializer,
126
127
  OpenPGPKeyringSerializer,
@@ -5,7 +5,13 @@ from rest_framework import serializers
5
5
  from rest_framework.validators import UniqueValidator
6
6
 
7
7
  from pulpcore.app import models
8
- from pulpcore.app.serializers import base, fields, pulp_labels_validator, DetailRelatedField
8
+ from pulpcore.app.serializers import (
9
+ base,
10
+ fields,
11
+ pulp_labels_validator,
12
+ DetailRelatedField,
13
+ RelatedField,
14
+ )
9
15
  from pulpcore.app.util import get_domain
10
16
 
11
17
 
@@ -27,6 +33,11 @@ class NoArtifactContentSerializer(base.ModelSerializer):
27
33
  view_name_pattern=r"repositories(-.*/.*)-detail",
28
34
  queryset=models.Repository.objects.all(),
29
35
  )
36
+ vuln_report = RelatedField(
37
+ read_only=True,
38
+ view_name="vuln_report-detail",
39
+ source="core_vulnerabilityreport",
40
+ )
30
41
 
31
42
  def get_artifacts(self, validated_data):
32
43
  """
@@ -116,6 +127,7 @@ class NoArtifactContentSerializer(base.ModelSerializer):
116
127
  fields = base.ModelSerializer.Meta.fields + (
117
128
  "repository",
118
129
  "pulp_labels",
130
+ "vuln_report",
119
131
  )
120
132
 
121
133
 
@@ -7,7 +7,7 @@ from rest_framework import fields, serializers
7
7
  from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer
8
8
 
9
9
  from pulpcore.app import models, settings
10
- from pulpcore.app.util import get_prn
10
+ from pulpcore.app.util import get_prn, reverse
11
11
  from pulpcore.app.serializers import (
12
12
  DetailIdentityField,
13
13
  DetailRelatedField,
@@ -490,6 +490,12 @@ class RepositoryVersionSerializer(ModelSerializer, NestedHyperlinkedModelSeriali
490
490
  source="*",
491
491
  read_only=True,
492
492
  )
493
+ vuln_report = serializers.SerializerMethodField(
494
+ read_only=True,
495
+ )
496
+
497
+ def get_vuln_report(self, object):
498
+ return f"{reverse('vuln_report-list')}?repository_version={get_prn(object)}"
493
499
 
494
500
  class Meta:
495
501
  model = models.RepositoryVersion
@@ -499,6 +505,7 @@ class RepositoryVersionSerializer(ModelSerializer, NestedHyperlinkedModelSeriali
499
505
  "repository",
500
506
  "base_version",
501
507
  "content_summary",
508
+ "vuln_report",
502
509
  )
503
510
 
504
511
 
@@ -0,0 +1,27 @@
1
+ from rest_framework import serializers
2
+
3
+ from pulpcore.app.models import VulnerabilityReport
4
+ from pulpcore.app.serializers import (
5
+ DetailRelatedField,
6
+ IdentityField,
7
+ ModelSerializer,
8
+ RepositoryVersionRelatedField,
9
+ )
10
+
11
+
12
+ class VulnerabilityReportSerializer(ModelSerializer):
13
+ """
14
+ A serializer for the VulnerabilityReport Model.
15
+ """
16
+
17
+ vulns = serializers.JSONField()
18
+ pulp_href = IdentityField(view_name="vuln_report-detail")
19
+ content = DetailRelatedField(
20
+ read_only=True,
21
+ view_name_pattern=r"content(-.*/.*)-detail",
22
+ )
23
+ repo_versions = RepositoryVersionRelatedField(many=True, required=False)
24
+
25
+ class Meta:
26
+ model = VulnerabilityReport
27
+ fields = ModelSerializer.Meta.fields + ("vulns", "repo_versions", "content")
pulpcore/app/settings.py CHANGED
@@ -12,16 +12,14 @@ import sys
12
12
 
13
13
  from contextlib import suppress
14
14
  from importlib import import_module
15
+ from importlib.metadata import entry_points
15
16
  from logging import getLogger
16
17
  from pathlib import Path
17
18
 
18
19
  from cryptography.fernet import Fernet
19
- from django.core.files.storage import storages
20
- from django.conf import global_settings
21
20
  from django.core.exceptions import ImproperlyConfigured
22
21
  from django.db import connection
23
- from pulpcore.app.loggers import deprecation_logger
24
- from dynaconf import DjangoDynaconf, Dynaconf, Validator, get_history
22
+ from dynaconf import DjangoDynaconf, Dynaconf, Validator
25
23
 
26
24
  from pulpcore import constants
27
25
 
@@ -33,12 +31,6 @@ try:
33
31
  except ImportError:
34
32
  pass
35
33
 
36
- if sys.version_info < (3, 10):
37
- # Python 3.9 has a rather different interface for `entry_points`.
38
- # Let's use a compatibility version.
39
- from importlib_metadata import entry_points
40
- else:
41
- from importlib.metadata import entry_points
42
34
 
43
35
  # Load settings first pass before applying all the defaults to get a grip on ENABLED_PLUGINS.
44
36
  enabled_plugins_validator = Validator(
@@ -78,16 +70,7 @@ MEDIA_ROOT = str(DEPLOY_ROOT / "media") # Django 3.1 adds support for pathlib.P
78
70
  STATIC_URL = "/assets/"
79
71
  STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/")
80
72
 
81
- # begin compatibility layer for DEFAULT_FILE_STORAGE
82
- # Remove on pulpcore=3.85 or pulpcore=4.0
83
-
84
- # - What is this?
85
- # We shouldn't use STORAGES or DEFAULT_FILE_STORAGE directly because those are
86
- # mutually exclusive by django, which constraints users to use whatever we use.
87
- # This is a hack/workaround to set Pulp's default while still enabling users to choose
88
- # the legacy or the new storage setting.
89
- _DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
90
- _STORAGES = {
73
+ STORAGES = {
91
74
  "default": {
92
75
  "BACKEND": "pulpcore.app.models.storage.FileSystem",
93
76
  },
@@ -96,10 +79,6 @@ _STORAGES = {
96
79
  },
97
80
  }
98
81
 
99
- setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE)
100
- setattr(global_settings, "STORAGES", _STORAGES)
101
- # end DEFAULT_FILE_STORAGE deprecation layer
102
-
103
82
  REDIRECT_TO_OBJECT_STORAGE = True
104
83
 
105
84
  WORKING_DIRECTORY = DEPLOY_ROOT / "tmp"
@@ -329,6 +308,7 @@ TASK_PROTECTION_TIME = 0
329
308
  TMPFILE_PROTECTION_TIME = 0
330
309
 
331
310
  REMOTE_USER_ENVIRON_NAME = "REMOTE_USER"
311
+ REMOTE_USER_OPENAPI_SECURITY_SCHEME = {"type": "mutualTLS"}
332
312
 
333
313
  AUTHENTICATION_JSON_HEADER = ""
334
314
  AUTHENTICATION_JSON_HEADER_JQ_FILTER = ""
@@ -348,6 +328,7 @@ CACHE_SETTINGS = {
348
328
  REMOTE_CONTENT_FETCH_FAILURE_COOLDOWN = 5 * 60 # 5 minutes
349
329
 
350
330
  SPECTACULAR_SETTINGS = {
331
+ "OAS_VERSION": "3.1.1",
351
332
  "SERVE_URLCONF": ROOT_URLCONF,
352
333
  "DEFAULT_GENERATOR_CLASS": "pulpcore.openapi.PulpSchemaGenerator",
353
334
  "DEFAULT_SCHEMA_CLASS": "pulpcore.openapi.PulpAutoSchema",
@@ -425,6 +406,9 @@ KAFKA_SASL_PASSWORD = None
425
406
  # opentelemetry settings
426
407
  OTEL_ENABLED = False
427
408
 
409
+ # VulnerabilityReport settings
410
+ VULN_REPORT_TASK_LIMITER = 10
411
+
428
412
  # Replaces asyncio event loop with uvloop
429
413
  UVLOOP_ENABLED = False
430
414
 
@@ -433,19 +417,17 @@ UVLOOP_ENABLED = False
433
417
 
434
418
  # Validators
435
419
 
436
- storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE")
437
420
  storage_validator = (
438
421
  Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False)
439
- | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem")
440
- | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage")
441
- | Validator(*storage_keys, eq="storages.backends.s3.S3Storage")
442
- | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage")
443
- | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage")
422
+ | Validator("STORAGES.default.BACKEND", eq="pulpcore.app.models.storage.FileSystem")
423
+ | Validator("STORAGES.default.BACKEND", eq="storages.backends.azure_storage.AzureStorage")
424
+ | Validator("STORAGES.default.BACKEND", eq="storages.backends.s3.S3Storage")
425
+ | Validator("STORAGES.default.BACKEND", eq="storages.backends.s3boto3.S3Boto3Storage")
426
+ | Validator("STORAGES.default.BACKEND", eq="storages.backends.gcloud.GoogleCloudStorage")
444
427
  )
445
428
  storage_validator.messages["combined"] = (
446
429
  "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure "
447
- "storage backend configured in STORAGES['default']['BACKEND'] "
448
- "(deprecated DEFAULT_FILE_STORAGE)."
430
+ "storage backend configured in STORAGES['default']['BACKEND']."
449
431
  )
450
432
 
451
433
  cache_enabled_validator = Validator("CACHE_ENABLED", eq=True)
@@ -552,21 +534,6 @@ settings = DjangoDynaconf(
552
534
  post_hooks=(otel_middleware_hook,),
553
535
  )
554
536
 
555
- # begin compatibility layer for DEFAULT_FILE_STORAGE
556
- # Remove on pulpcore=3.85 or pulpcore=4.0
557
- using_deprecated_storage_settings = len(get_history(settings, key="DEFAULT_FILE_STORAGE")) > 1
558
- if using_deprecated_storage_settings:
559
- deprecation_logger.warning(
560
- "[deprecation] DEFAULT_FILE_STORAGE will be removed in pulpcore 3.85. "
561
- "Learn how to upgrade to STORAGES:\n"
562
- "https://discourse.pulpproject.org/t/"
563
- "action-required-upgrade-your-storage-settings-before-pulpcore-3-85/2072/2"
564
- )
565
- # Ensures the cached property storage.backends uses the the right value
566
- storages._backends = settings.STORAGES.copy()
567
- storages.backends
568
- # end compatibility layer
569
-
570
537
  _logger = getLogger(__name__)
571
538
 
572
539
 
@@ -25,3 +25,5 @@ from .replica import replicate_distributions
25
25
  from .repository import repair_all_artifacts
26
26
 
27
27
  from .analytics import post_analytics
28
+
29
+ from .vulnerability_report import check_content
@@ -0,0 +1,159 @@
1
+ import aiohttp
2
+ import asyncio
3
+ import importlib
4
+
5
+ from asgiref.sync import sync_to_async
6
+ from django.conf import settings
7
+
8
+ from pulpcore.app.util import get_domain
9
+ from pulpcore.app.models.progress import ProgressReport
10
+ from pulpcore.app.models.vulnerability_report import VulnerabilityReport
11
+ from pulpcore.constants import OSV_QUERY_URL, TASK_STATES
12
+
13
+
14
+ class VulnerabilityReportScanner:
15
+ """
16
+ A scanner class to collect vulnerabilities from Pulp Contents using the OSV.dev API.
17
+
18
+ Attributes:
19
+ semaphore (asyncio.Semaphore): Controls concurrent API requests to osv.dev service.
20
+ generator (AsyncGenerator): The async generator with the content data to be scanned for
21
+ vulnerabilities.
22
+ session (aiohttp.ClientSession): HTTP session for making requests to the OSV.dev API.
23
+ Defaults to a new session if not provided.
24
+ created (ProgressReport): Tracks the number of new VulnerabilityReport records created
25
+ during the scanning process.
26
+ updated (ProgressReport): Tracks the number of existing VulnerabilityReport records
27
+ updated during the scanning process.
28
+ total_scanned (ProgressReport): Tracks the total number of content items that have
29
+ been processed for vulnerability scanning.
30
+ """
31
+
32
+ def __init__(self, semaphore, generator, session=None):
33
+ self.semaphore = semaphore
34
+ self.generator = generator
35
+ self.session = session or aiohttp.ClientSession()
36
+ self.created = ProgressReport(
37
+ message="Vulnerability Reports created", code="created", state=TASK_STATES.RUNNING
38
+ )
39
+ self.updated = ProgressReport(
40
+ message="Vulnerability Reports updated", code="updated", state=TASK_STATES.RUNNING
41
+ )
42
+ self.total_scanned = ProgressReport(
43
+ message="Content scanned", code="total_scanned", state=TASK_STATES.RUNNING
44
+ )
45
+
46
+ async def scan_packages(self):
47
+ """
48
+ Main entry point for scanning packages for vulnerabilities. Progress is tracked through
49
+ three ProgressReport instances (created, updated, total_scanned).
50
+ """
51
+ tasks = []
52
+ async for content in self.generator:
53
+ tasks.append(asyncio.create_task(self.scan_package(content)))
54
+ await asyncio.gather(*tasks)
55
+ await self.session.close()
56
+
57
+ if self.created.done > 0:
58
+ self.created.state = TASK_STATES.COMPLETED
59
+ await self.created.asave()
60
+ if self.total_scanned.done > 0:
61
+ self.total_scanned.state = TASK_STATES.COMPLETED
62
+ await self.total_scanned.asave()
63
+ if self.updated.done > 0:
64
+ self.updated.state = TASK_STATES.COMPLETED
65
+ await self.updated.asave()
66
+
67
+ async def scan_package(self, repo_content_osv_data):
68
+ """
69
+ Makes a request to the osv.dev API and store the results in VulnerabilityReport model.
70
+
71
+ Args:
72
+ repo_content_osv_data (Dict[str, Any]): A dictionary with osv.dev expected request
73
+ data format plus associated Pulp Content and RepoVersion.
74
+ """
75
+ vulns, content, repo_version = await self._process_reports(repo_content_osv_data)
76
+ await self._save_vulnerability_report(vulns, content, repo_version)
77
+
78
+ async def _process_reports(self, repo_content_osv_data):
79
+ """
80
+ Process vulnerability scanning data by querying the OSV API and extracting results.
81
+
82
+ Args:
83
+ repo_content_osv_data (Dict[str, Any]): A dictionary containing OSV.dev expected
84
+ request data format plus associated Pulp Content and RepositoryVersion.
85
+
86
+ Returns:
87
+ tuple:
88
+ - vulns (List[Dict]): List of vulnerability (vulns) returned from OSV.dev API
89
+ - pulp_content (Content): The Pulp Content instance that was scanned
90
+ - repo_version (RepositoryVersion or None): The RepositoryVersion where the
91
+ Content is present
92
+ """
93
+ vulns = []
94
+ async with self.semaphore:
95
+ repo_version = repo_content_osv_data.pop("repo_version", None)
96
+ pulp_content = repo_content_osv_data.pop("content")
97
+ vulnerability_data = await self._query_osv_api(repo_content_osv_data)
98
+ vulns = vulnerability_data.get("vulns", [])
99
+ while next_page_token := vulnerability_data.get("next_page_token"):
100
+ repo_content_osv_data["page_token"] = next_page_token
101
+ vulnerability_data = await self._query_osv_api(repo_content_osv_data)
102
+ vulns.extend(vulnerability_data)
103
+ await self.total_scanned.aincrement()
104
+ return vulns, pulp_content, repo_version
105
+
106
+ async def _query_osv_api(self, osv_data):
107
+ """
108
+ Make a single HTTP POST request to the OSV.dev vulnerability database API.
109
+
110
+ Args:
111
+ osv_data (Dict[str, Any]): OSV query data dictionary containing package information
112
+ in the format expected by the OSV.dev API.
113
+
114
+ Returns:
115
+ Dict[str, Any]: JSON response from the OSV API containing vulnerability data.
116
+ """
117
+ async with self.session.post(url=OSV_QUERY_URL, json=osv_data) as response:
118
+ try:
119
+ return await response.json()
120
+ except aiohttp.ContentTypeError:
121
+ raise RuntimeError("Vuln report task failed to query osv.dev data.")
122
+
123
+ async def _save_vulnerability_report(self, vulns, content, repo_version):
124
+ """
125
+ Save vulnerability report to the database.
126
+
127
+ Args:
128
+ vulns (List): osv.dev vulns output
129
+ content (Content): A Pulp Content instance which vulnerabilities were verified
130
+ repo_version (RepositoryVersion): The RepositoryVersion where Content is present
131
+ """
132
+
133
+ vuln_report, created = await sync_to_async(VulnerabilityReport.objects.update_or_create)(
134
+ vulns=vulns, pulp_domain=get_domain(), content=content
135
+ )
136
+ await vuln_report.repo_versions.aadd(repo_version)
137
+
138
+ if created:
139
+ await self.created.aincrement()
140
+ else:
141
+ await self.updated.aincrement()
142
+
143
+
144
+ async def check_content(func, args):
145
+ """
146
+ Spawn tasks for each async generator value to make the API requests (scan) to osv.dev.
147
+
148
+ Args:
149
+ func (callable | str): A Pulp plugin function that should return an async generator with
150
+ the osv.dev expected request data format plus associated Pulp Content and RepoVersion.
151
+ args (tuple): The positional arguments to pass on to the func.
152
+ """
153
+ module_name, function_name = func.rsplit(".", 1)
154
+ module = importlib.import_module(module_name)
155
+ func = getattr(module, function_name)
156
+
157
+ semaphore = asyncio.Semaphore(settings.VULN_REPORT_TASK_LIMITER)
158
+ vuln_report_scanner = VulnerabilityReportScanner(semaphore, func(*args))
159
+ await vuln_report_scanner.scan_packages()
@@ -82,6 +82,7 @@ from .user import (
82
82
  UserRoleViewSet,
83
83
  )
84
84
  from .replica import UpstreamPulpViewSet
85
+ from .vulnerability_report import VulnerabilityReportViewSet
85
86
  from .openpgp import (
86
87
  OpenPGPDistributionViewSet,
87
88
  OpenPGPKeyringViewSet,
@@ -0,0 +1,20 @@
1
+ from rest_framework.mixins import DestroyModelMixin, ListModelMixin, RetrieveModelMixin
2
+
3
+ from pulpcore.app.models.vulnerability_report import VulnerabilityReport
4
+ from pulpcore.app.serializers.vulnerability_report import VulnerabilityReportSerializer
5
+ from pulpcore.app.viewsets.base import NamedModelViewSet
6
+
7
+
8
+ class VulnerabilityReportViewSet(
9
+ NamedModelViewSet, ListModelMixin, RetrieveModelMixin, DestroyModelMixin
10
+ ):
11
+
12
+ endpoint_name = "vuln_report"
13
+ queryset = VulnerabilityReport.objects.prefetch_related("repo_versions").select_related(
14
+ "content"
15
+ )
16
+ serializer_class = VulnerabilityReportSerializer
17
+
18
+ @classmethod
19
+ def routable(cls):
20
+ return True
pulpcore/constants.py CHANGED
@@ -131,3 +131,7 @@ CHECKPOINT_TS_FORMAT = "%Y%m%dT%H%M%SZ"
131
131
  # The upper boundary represents an unsigned 32-bit integer and prevents overflow
132
132
  ORPHAN_PROTECTION_TIME_LOWER_BOUND = 0
133
133
  ORPHAN_PROTECTION_TIME_UPPER_BOUND = 4294967295 # (2^32)-1
134
+
135
+ # VULNERABILITY REPORT CONSTANTS
136
+ # OSV API URL
137
+ OSV_QUERY_URL = "https://api.osv.dev/v1/query"