pulpcore 3.85.1__py3-none-any.whl → 3.87.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pulpcore might be problematic. Click here for more details.
- pulp_certguard/app/__init__.py +1 -1
- pulp_file/app/__init__.py +1 -1
- pulpcore/app/apps.py +1 -1
- pulpcore/app/management/commands/optimizemigration.py +84 -0
- pulpcore/app/management/commands/remove-plugin.py +2 -5
- pulpcore/app/migrations/0139_task_app_lock.py +19 -0
- pulpcore/app/migrations/0140_require_appstatus_zdu.py +15 -0
- pulpcore/app/migrations/0141_alter_appstatus_name.py +18 -0
- pulpcore/app/models/status.py +20 -25
- pulpcore/app/models/task.py +6 -1
- pulpcore/app/serializers/status.py +18 -8
- pulpcore/app/serializers/task.py +1 -29
- pulpcore/app/settings.py +12 -0
- pulpcore/app/tasks/analytics.py +3 -4
- pulpcore/app/tasks/importer.py +2 -2
- pulpcore/app/views/status.py +5 -6
- pulpcore/app/viewsets/task.py +14 -5
- pulpcore/middleware.py +37 -10
- pulpcore/openapi/__init__.py +1 -1
- pulpcore/plugin/repo_version_utils.py +8 -2
- pulpcore/pytest_plugin.py +6 -4
- pulpcore/tasking/tasks.py +16 -15
- pulpcore/tasking/worker.py +59 -41
- pulpcore/tests/functional/api/test_tasking.py +14 -51
- pulpcore/tests/functional/api/test_workers.py +1 -2
- pulpcore/tests/unit/content/test_heartbeat.py +2 -1
- pulpcore/tests/unit/models/test_repository.py +84 -1
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/METADATA +3 -3
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/RECORD +33 -29
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/WHEEL +0 -0
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/entry_points.txt +0 -0
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/licenses/LICENSE +0 -0
- {pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/top_level.txt +0 -0
pulp_certguard/app/__init__.py
CHANGED
pulp_file/app/__init__.py
CHANGED
pulpcore/app/apps.py
CHANGED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from gettext import gettext as _
|
|
3
|
+
|
|
4
|
+
from django.db import connection
|
|
5
|
+
from django.db.migrations.loader import MigrationLoader
|
|
6
|
+
from django.db.migrations.migration import SwappableTuple
|
|
7
|
+
from django.db.migrations.optimizer import MigrationOptimizer
|
|
8
|
+
from django.db.migrations.writer import MigrationWriter
|
|
9
|
+
from django.conf import settings
|
|
10
|
+
from django.core.management import BaseCommand
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def print_stats(migration):
|
|
14
|
+
operations = migration.operations
|
|
15
|
+
migration_types = defaultdict(int)
|
|
16
|
+
for operation in operations:
|
|
17
|
+
migration_types[operation.__class__.__name__] += 1
|
|
18
|
+
for key, value in migration_types.items():
|
|
19
|
+
print(f"{value: 4} {key}")
|
|
20
|
+
print("---")
|
|
21
|
+
print(_("Total: {count}").format(count=len(operations)))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Command(BaseCommand):
|
|
25
|
+
"""
|
|
26
|
+
Django management command to optimize a migration.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
help = _("Optimize a migration.")
|
|
30
|
+
|
|
31
|
+
def add_arguments(self, parser):
|
|
32
|
+
parser.add_argument("--dry-run", action="store_true", help=_("Don't change anything."))
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--stat", action="store_true", help=_("Print statistics about operations.")
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument("app-label", help=_("App label of the migrations to optimize."))
|
|
37
|
+
parser.add_argument("migration", help=_("Prefix of the migration to optimize."))
|
|
38
|
+
|
|
39
|
+
def handle(self, *args, **options):
|
|
40
|
+
dry_run = options.get("dry_run", False)
|
|
41
|
+
stat = options.get("stat", False)
|
|
42
|
+
app_label = options["app-label"]
|
|
43
|
+
migration_prefix = options["migration"]
|
|
44
|
+
|
|
45
|
+
loader = MigrationLoader(connection)
|
|
46
|
+
|
|
47
|
+
migration = loader.get_migration_by_prefix(app_label, migration_prefix)
|
|
48
|
+
|
|
49
|
+
print(_("Optimizing migration {}").format((migration.app_label, migration.name)))
|
|
50
|
+
if stat:
|
|
51
|
+
print(_("=== Old Migration Summary ==="))
|
|
52
|
+
print_stats(migration)
|
|
53
|
+
|
|
54
|
+
new_dependencies = []
|
|
55
|
+
for dependency in migration.dependencies:
|
|
56
|
+
if (
|
|
57
|
+
isinstance(dependency, SwappableTuple)
|
|
58
|
+
and settings.AUTH_USER_MODEL == dependency.setting
|
|
59
|
+
):
|
|
60
|
+
new_dependencies.append(("__setting__", "AUTH_USER_MODEL"))
|
|
61
|
+
else:
|
|
62
|
+
new_dependencies.append(dependency)
|
|
63
|
+
|
|
64
|
+
optimizer = MigrationOptimizer()
|
|
65
|
+
new_operations = optimizer.optimize(migration.operations, app_label)
|
|
66
|
+
|
|
67
|
+
if new_operations != migration.operations:
|
|
68
|
+
print(
|
|
69
|
+
_("Changed from {old_count} to {new_count} operations.").format(
|
|
70
|
+
old_count=len(migration.operations), new_count=len(new_operations)
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
if stat:
|
|
74
|
+
print(_("=== New Migration Summary ==="))
|
|
75
|
+
print_stats(migration)
|
|
76
|
+
|
|
77
|
+
migration.operations = new_operations
|
|
78
|
+
migration.dependencies = new_dependencies
|
|
79
|
+
if not dry_run:
|
|
80
|
+
writer = MigrationWriter(migration)
|
|
81
|
+
with open(writer.path, "w") as output_file:
|
|
82
|
+
output_file.write(writer.as_string())
|
|
83
|
+
else:
|
|
84
|
+
print(_("No optimizations found."))
|
|
@@ -11,7 +11,7 @@ from django.contrib.contenttypes.models import ContentType
|
|
|
11
11
|
from django.core.management import BaseCommand, call_command, CommandError
|
|
12
12
|
|
|
13
13
|
from pulpcore.app.apps import pulp_plugin_configs
|
|
14
|
-
from pulpcore.app.models import AccessPolicy,
|
|
14
|
+
from pulpcore.app.models import AccessPolicy, AppStatus
|
|
15
15
|
from pulpcore.app.models.role import Role
|
|
16
16
|
from pulpcore.app.util import get_view_urlpattern
|
|
17
17
|
|
|
@@ -53,10 +53,7 @@ class Command(BaseCommand):
|
|
|
53
53
|
"Checking if Pulp services are running, it can take up to {}s...".format(waiting_time)
|
|
54
54
|
)
|
|
55
55
|
while is_pulp_running and (time.time() - check_started) < waiting_time:
|
|
56
|
-
is_pulp_running = (
|
|
57
|
-
ContentAppStatus.objects.online().exists()
|
|
58
|
-
or Worker.objects.online_workers().exists()
|
|
59
|
-
)
|
|
56
|
+
is_pulp_running = AppStatus.objects.online().exists()
|
|
60
57
|
time.sleep(2)
|
|
61
58
|
|
|
62
59
|
if is_pulp_running:
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-08-07 11:43
|
|
2
|
+
|
|
3
|
+
from django.db import migrations, models
|
|
4
|
+
import django.db.models.deletion
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Migration(migrations.Migration):
|
|
8
|
+
|
|
9
|
+
dependencies = [
|
|
10
|
+
('core', '0138_vulnerabilityreport'),
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
operations = [
|
|
14
|
+
migrations.AddField(
|
|
15
|
+
model_name='task',
|
|
16
|
+
name='app_lock',
|
|
17
|
+
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tasks', to='core.appstatus'),
|
|
18
|
+
),
|
|
19
|
+
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-08-12 15:20
|
|
2
|
+
|
|
3
|
+
from django.db import migrations
|
|
4
|
+
from pulpcore.migrations import RequireVersion
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Migration(migrations.Migration):
|
|
8
|
+
|
|
9
|
+
dependencies = [
|
|
10
|
+
('core', '0139_task_app_lock'),
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
operations = [
|
|
14
|
+
RequireVersion("core", "3.85"),
|
|
15
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-08-27 10:56
|
|
2
|
+
|
|
3
|
+
from django.db import migrations, models
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Migration(migrations.Migration):
|
|
7
|
+
|
|
8
|
+
dependencies = [
|
|
9
|
+
('core', '0140_require_appstatus_zdu'),
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
operations = [
|
|
13
|
+
migrations.AlterField(
|
|
14
|
+
model_name='appstatus',
|
|
15
|
+
name='name',
|
|
16
|
+
field=models.TextField(),
|
|
17
|
+
),
|
|
18
|
+
]
|
pulpcore/app/models/status.py
CHANGED
|
@@ -15,6 +15,7 @@ from pulpcore.app.models import BaseModel
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class AppStatusManager(models.Manager):
|
|
18
|
+
# This should be replaced with 3.87.
|
|
18
19
|
def online(self):
|
|
19
20
|
"""
|
|
20
21
|
Returns a queryset of objects that are online.
|
|
@@ -51,29 +52,32 @@ class AppStatusManager(models.Manager):
|
|
|
51
52
|
|
|
52
53
|
class _AppStatusManager(AppStatusManager):
|
|
53
54
|
# This is an intermediate class in order to allow a ZDU.
|
|
54
|
-
# It should be
|
|
55
|
+
# It should be made the real thing with 3.87.
|
|
56
|
+
def __init__(self):
|
|
57
|
+
super().__init__()
|
|
58
|
+
self._current_app_status = None
|
|
59
|
+
|
|
55
60
|
def create(self, app_type, **kwargs):
|
|
56
|
-
if
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
from pulpcore.app.models import Worker
|
|
60
|
-
|
|
61
|
-
old_obj = Worker.objects.create(**kwargs)
|
|
62
|
-
else:
|
|
63
|
-
raise NotImplementedError(f"Invalid app_type: {app_type}")
|
|
61
|
+
if self._current_app_status is not None:
|
|
62
|
+
raise RuntimeError("There is already an app status in this process.")
|
|
63
|
+
|
|
64
64
|
obj = super().create(app_type=app_type, **kwargs)
|
|
65
|
-
|
|
65
|
+
self._current_app_status = obj
|
|
66
66
|
return obj
|
|
67
67
|
|
|
68
68
|
async def acreate(self, app_type, **kwargs):
|
|
69
|
-
if
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
raise NotImplementedError(f"Invalid app_type: {app_type}")
|
|
69
|
+
if self._current_app_status is not None:
|
|
70
|
+
raise RuntimeError("There is already an app status in this process.")
|
|
71
|
+
|
|
73
72
|
obj = await super().acreate(app_type=app_type, **kwargs)
|
|
74
|
-
|
|
73
|
+
self._current_app_status = obj
|
|
75
74
|
return obj
|
|
76
75
|
|
|
76
|
+
def current(self):
|
|
77
|
+
if self._current_app_status is None:
|
|
78
|
+
raise RuntimeError("There is no current app status.")
|
|
79
|
+
return self._current_app_status
|
|
80
|
+
|
|
77
81
|
def online(self):
|
|
78
82
|
"""
|
|
79
83
|
Returns a queryset of objects that are online.
|
|
@@ -107,7 +111,7 @@ class AppStatus(BaseModel):
|
|
|
107
111
|
objects = _AppStatusManager()
|
|
108
112
|
|
|
109
113
|
app_type = models.CharField(max_length=10, choices=APP_TYPES)
|
|
110
|
-
name = models.TextField(
|
|
114
|
+
name = models.TextField()
|
|
111
115
|
versions = HStoreField(default=dict)
|
|
112
116
|
ttl = models.DurationField(null=False)
|
|
113
117
|
last_heartbeat = models.DateTimeField(auto_now=True)
|
|
@@ -115,13 +119,6 @@ class AppStatus(BaseModel):
|
|
|
115
119
|
def __init__(self, *args, **kwargs):
|
|
116
120
|
super().__init__(*args, **kwargs)
|
|
117
121
|
self.ttl = timedelta(seconds=self._APP_TTL[self.app_type])
|
|
118
|
-
self._old_status = None
|
|
119
|
-
|
|
120
|
-
def delete(self, *args, **kwargs):
|
|
121
|
-
# adelete will call into this, so we should not replicate that one here.
|
|
122
|
-
if self._old_status is not None:
|
|
123
|
-
self._old_status.delete(*args, **kwargs)
|
|
124
|
-
super().delete(*args, **kwargs)
|
|
125
122
|
|
|
126
123
|
@property
|
|
127
124
|
def online(self) -> bool:
|
|
@@ -153,7 +150,6 @@ class AppStatus(BaseModel):
|
|
|
153
150
|
ValueError: When the model instance has never been saved before. This method can
|
|
154
151
|
only update an existing database record.
|
|
155
152
|
"""
|
|
156
|
-
self._old_status.save_heartbeat()
|
|
157
153
|
self.save(update_fields=["last_heartbeat"])
|
|
158
154
|
|
|
159
155
|
async def asave_heartbeat(self):
|
|
@@ -166,7 +162,6 @@ class AppStatus(BaseModel):
|
|
|
166
162
|
ValueError: When the model instance has never been saved before. This method can
|
|
167
163
|
only update an existing database record.
|
|
168
164
|
"""
|
|
169
|
-
await self._old_status.asave_heartbeat()
|
|
170
165
|
await self.asave(update_fields=["last_heartbeat"])
|
|
171
166
|
|
|
172
167
|
@property
|
pulpcore/app/models/task.py
CHANGED
|
@@ -116,7 +116,8 @@ class Task(BaseModel, AutoAddObjPermsMixin):
|
|
|
116
116
|
Defaults to `True`.
|
|
117
117
|
|
|
118
118
|
Relations:
|
|
119
|
-
|
|
119
|
+
app_lock (AppStatus): The app holding the lock on this task.
|
|
120
|
+
Warning: This is not yet implemented/enforced.
|
|
120
121
|
parent (models.ForeignKey): Task that spawned this task (if any)
|
|
121
122
|
worker (models.ForeignKey): The worker that this task is in
|
|
122
123
|
pulp_domain (models.ForeignKey): The domain the Task is a part of
|
|
@@ -138,6 +139,10 @@ class Task(BaseModel, AutoAddObjPermsMixin):
|
|
|
138
139
|
enc_kwargs = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
|
|
139
140
|
|
|
140
141
|
worker = models.ForeignKey("Worker", null=True, related_name="tasks", on_delete=models.SET_NULL)
|
|
142
|
+
# This field is supposed to replace the session advisory locks to protect tasks.
|
|
143
|
+
app_lock = models.ForeignKey(
|
|
144
|
+
"AppStatus", null=True, related_name="tasks", on_delete=models.SET_NULL
|
|
145
|
+
)
|
|
141
146
|
|
|
142
147
|
parent_task = models.ForeignKey(
|
|
143
148
|
"Task", null=True, related_name="child_tasks", on_delete=models.SET_NULL
|
|
@@ -2,11 +2,21 @@ from gettext import gettext as _
|
|
|
2
2
|
|
|
3
3
|
from rest_framework import serializers
|
|
4
4
|
|
|
5
|
-
from pulpcore.app.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
)
|
|
5
|
+
from pulpcore.app.models import AppStatus
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AppStatusSerializer(serializers.ModelSerializer):
|
|
9
|
+
name = serializers.CharField(help_text=_("The name of the worker."), read_only=True)
|
|
10
|
+
last_heartbeat = serializers.DateTimeField(
|
|
11
|
+
help_text=_("Timestamp of the last time the worker talked to the service."), read_only=True
|
|
12
|
+
)
|
|
13
|
+
versions = serializers.HStoreField(
|
|
14
|
+
help_text=_("Versions of the components installed."), read_only=True
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
class Meta:
|
|
18
|
+
model = AppStatus
|
|
19
|
+
fields = ("name", "last_heartbeat", "versions")
|
|
10
20
|
|
|
11
21
|
|
|
12
22
|
class VersionSerializer(serializers.Serializer):
|
|
@@ -88,7 +98,7 @@ class StatusSerializer(serializers.Serializer):
|
|
|
88
98
|
|
|
89
99
|
versions = VersionSerializer(help_text=_("Version information of Pulp components"), many=True)
|
|
90
100
|
|
|
91
|
-
online_workers =
|
|
101
|
+
online_workers = AppStatusSerializer(
|
|
92
102
|
help_text=_(
|
|
93
103
|
"List of online workers known to the application. An online worker is actively "
|
|
94
104
|
"heartbeating and can respond to new work."
|
|
@@ -96,7 +106,7 @@ class StatusSerializer(serializers.Serializer):
|
|
|
96
106
|
many=True,
|
|
97
107
|
)
|
|
98
108
|
|
|
99
|
-
online_api_apps =
|
|
109
|
+
online_api_apps = AppStatusSerializer(
|
|
100
110
|
help_text=_(
|
|
101
111
|
"List of online api apps known to the application. An online api app "
|
|
102
112
|
"is actively heartbeating and can serve the rest api to clients."
|
|
@@ -104,7 +114,7 @@ class StatusSerializer(serializers.Serializer):
|
|
|
104
114
|
many=True,
|
|
105
115
|
)
|
|
106
116
|
|
|
107
|
-
online_content_apps =
|
|
117
|
+
online_content_apps = AppStatusSerializer(
|
|
108
118
|
help_text=_(
|
|
109
119
|
"List of online content apps known to the application. An online content app "
|
|
110
120
|
"is actively heartbeating and can serve data to clients."
|
pulpcore/app/serializers/task.py
CHANGED
|
@@ -198,34 +198,6 @@ class TaskCancelSerializer(serializers.Serializer):
|
|
|
198
198
|
fields = ("state",)
|
|
199
199
|
|
|
200
200
|
|
|
201
|
-
class ApiAppStatusSerializer(ModelSerializer):
|
|
202
|
-
name = serializers.CharField(help_text=_("The name of the worker."), read_only=True)
|
|
203
|
-
last_heartbeat = serializers.DateTimeField(
|
|
204
|
-
help_text=_("Timestamp of the last time the worker talked to the service."), read_only=True
|
|
205
|
-
)
|
|
206
|
-
versions = serializers.HStoreField(
|
|
207
|
-
help_text=_("Versions of the components installed."), read_only=True
|
|
208
|
-
)
|
|
209
|
-
|
|
210
|
-
class Meta:
|
|
211
|
-
model = models.ApiAppStatus
|
|
212
|
-
fields = ("name", "last_heartbeat", "versions")
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
class ContentAppStatusSerializer(ModelSerializer):
|
|
216
|
-
name = serializers.CharField(help_text=_("The name of the worker."), read_only=True)
|
|
217
|
-
last_heartbeat = serializers.DateTimeField(
|
|
218
|
-
help_text=_("Timestamp of the last time the worker talked to the service."), read_only=True
|
|
219
|
-
)
|
|
220
|
-
versions = serializers.HStoreField(
|
|
221
|
-
help_text=_("Versions of the components installed."), read_only=True
|
|
222
|
-
)
|
|
223
|
-
|
|
224
|
-
class Meta:
|
|
225
|
-
model = models.ContentAppStatus
|
|
226
|
-
fields = ("name", "last_heartbeat", "versions")
|
|
227
|
-
|
|
228
|
-
|
|
229
201
|
class WorkerSerializer(ModelSerializer):
|
|
230
202
|
pulp_href = IdentityField(view_name="workers-detail")
|
|
231
203
|
|
|
@@ -246,7 +218,7 @@ class WorkerSerializer(ModelSerializer):
|
|
|
246
218
|
)
|
|
247
219
|
|
|
248
220
|
class Meta:
|
|
249
|
-
model = models.
|
|
221
|
+
model = models.AppStatus
|
|
250
222
|
fields = ModelSerializer.Meta.fields + (
|
|
251
223
|
"name",
|
|
252
224
|
"last_heartbeat",
|
pulpcore/app/settings.py
CHANGED
|
@@ -405,6 +405,7 @@ KAFKA_SASL_PASSWORD = None
|
|
|
405
405
|
|
|
406
406
|
# opentelemetry settings
|
|
407
407
|
OTEL_ENABLED = False
|
|
408
|
+
OTEL_PULP_API_HISTOGRAM_BUCKETS = []
|
|
408
409
|
|
|
409
410
|
# VulnerabilityReport settings
|
|
410
411
|
VULN_REPORT_TASK_LIMITER = 10
|
|
@@ -504,6 +505,16 @@ authentication_json_header_openapi_security_scheme_validator = Validator(
|
|
|
504
505
|
messages={"is_type_of": "{name} must be a dictionary."},
|
|
505
506
|
)
|
|
506
507
|
|
|
508
|
+
otel_pulp_api_histogram_buckets_validator = Validator(
|
|
509
|
+
"OTEL_PULP_API_HISTOGRAM_BUCKETS",
|
|
510
|
+
is_type_of=list,
|
|
511
|
+
condition=lambda v: all([isinstance(value, float) for value in v]),
|
|
512
|
+
messages={
|
|
513
|
+
"is_type_of": "{name} must be a list.",
|
|
514
|
+
"condition": "All buckets must be declared as a float value",
|
|
515
|
+
},
|
|
516
|
+
)
|
|
517
|
+
|
|
507
518
|
|
|
508
519
|
def otel_middleware_hook(settings):
|
|
509
520
|
data = {"dynaconf_merge": True}
|
|
@@ -530,6 +541,7 @@ settings = DjangoDynaconf(
|
|
|
530
541
|
unknown_algs_validator,
|
|
531
542
|
json_header_auth_validator,
|
|
532
543
|
authentication_json_header_openapi_security_scheme_validator,
|
|
544
|
+
otel_pulp_api_histogram_buckets_validator,
|
|
533
545
|
],
|
|
534
546
|
post_hooks=(otel_middleware_hook,),
|
|
535
547
|
)
|
pulpcore/app/tasks/analytics.py
CHANGED
|
@@ -19,8 +19,7 @@ from google.protobuf.json_format import MessageToJson
|
|
|
19
19
|
from pulpcore.app.apps import pulp_plugin_configs
|
|
20
20
|
from pulpcore.app.models import SystemID, Group, Domain, AccessPolicy
|
|
21
21
|
from pulpcore.app.models.role import Role
|
|
22
|
-
from pulpcore.app.models.status import
|
|
23
|
-
from pulpcore.app.models.task import Worker
|
|
22
|
+
from pulpcore.app.models.status import AppStatus
|
|
24
23
|
from pulpcore.app.protobuf.analytics_pb2 import Analytics
|
|
25
24
|
|
|
26
25
|
|
|
@@ -79,13 +78,13 @@ async def _versions_data(analytics):
|
|
|
79
78
|
|
|
80
79
|
|
|
81
80
|
async def _online_content_apps_data(analytics):
|
|
82
|
-
online_content_apps_qs =
|
|
81
|
+
online_content_apps_qs = AppStatus.objects.online().filter(app_type="content")
|
|
83
82
|
analytics.online_content_apps.processes = await online_content_apps_qs.acount()
|
|
84
83
|
analytics.online_content_apps.hosts = await _num_hosts(online_content_apps_qs)
|
|
85
84
|
|
|
86
85
|
|
|
87
86
|
async def _online_workers_data(analytics):
|
|
88
|
-
online_workers_qs =
|
|
87
|
+
online_workers_qs = AppStatus.objects.online().filter(app_type="worker")
|
|
89
88
|
analytics.online_workers.processes = await online_workers_qs.acount()
|
|
90
89
|
analytics.online_workers.hosts = await _num_hosts(online_workers_qs)
|
|
91
90
|
|
pulpcore/app/tasks/importer.py
CHANGED
|
@@ -18,6 +18,7 @@ from tablib import Dataset
|
|
|
18
18
|
from pulpcore.exceptions.plugin import MissingPlugin
|
|
19
19
|
from pulpcore.app.apps import get_plugin_config
|
|
20
20
|
from pulpcore.app.models import (
|
|
21
|
+
AppStatus,
|
|
21
22
|
Artifact,
|
|
22
23
|
Content,
|
|
23
24
|
CreatedResource,
|
|
@@ -28,7 +29,6 @@ from pulpcore.app.models import (
|
|
|
28
29
|
Repository,
|
|
29
30
|
Task,
|
|
30
31
|
TaskGroup,
|
|
31
|
-
Worker,
|
|
32
32
|
)
|
|
33
33
|
from pulpcore.app.modelresource import (
|
|
34
34
|
ArtifactResource,
|
|
@@ -508,7 +508,7 @@ def pulp_import(importer_pk, path, toc, create_repositories):
|
|
|
508
508
|
# By default (setting is not-set), import will continue to use 100% of the available
|
|
509
509
|
# workers.
|
|
510
510
|
import_workers_percent = int(settings.get("IMPORT_WORKERS_PERCENT", 100))
|
|
511
|
-
total_workers =
|
|
511
|
+
total_workers = AppStatus.objects.online().filter(app_type="worker").count()
|
|
512
512
|
import_workers = max(1, int(total_workers * (import_workers_percent / 100.0)))
|
|
513
513
|
|
|
514
514
|
with open(os.path.join(temp_dir, REPO_FILE), "r") as repo_data_file:
|
pulpcore/app/views/status.py
CHANGED
|
@@ -11,8 +11,7 @@ from collections import namedtuple
|
|
|
11
11
|
|
|
12
12
|
from pulpcore.app.apps import pulp_plugin_configs
|
|
13
13
|
from pulpcore.app.models.content import Artifact
|
|
14
|
-
from pulpcore.app.models.status import
|
|
15
|
-
from pulpcore.app.models.task import Worker
|
|
14
|
+
from pulpcore.app.models.status import AppStatus
|
|
16
15
|
from pulpcore.app.serializers.status import StatusSerializer
|
|
17
16
|
from pulpcore.app.redis_connection import get_redis_connection
|
|
18
17
|
from pulpcore.app.util import get_domain
|
|
@@ -79,9 +78,9 @@ class StatusView(APIView):
|
|
|
79
78
|
|
|
80
79
|
db_status = {"connected": self._get_db_conn_status()}
|
|
81
80
|
|
|
82
|
-
online_workers =
|
|
83
|
-
online_api_apps =
|
|
84
|
-
online_content_apps =
|
|
81
|
+
online_workers = AppStatus.objects.online().filter(app_type="worker")
|
|
82
|
+
online_api_apps = AppStatus.objects.online().filter(app_type="api")
|
|
83
|
+
online_content_apps = AppStatus.objects.online().filter(app_type="content")
|
|
85
84
|
|
|
86
85
|
content_settings = {
|
|
87
86
|
"content_origin": settings.CONTENT_ORIGIN,
|
|
@@ -113,7 +112,7 @@ class StatusView(APIView):
|
|
|
113
112
|
bool: True if there's a db connection. False otherwise.
|
|
114
113
|
"""
|
|
115
114
|
try:
|
|
116
|
-
|
|
115
|
+
AppStatus.objects.count()
|
|
117
116
|
except Exception:
|
|
118
117
|
_logger.exception(_("Cannot connect to database during status check."))
|
|
119
118
|
return False
|
pulpcore/app/viewsets/task.py
CHANGED
|
@@ -12,11 +12,11 @@ from rest_framework.serializers import DictField, URLField, ValidationError
|
|
|
12
12
|
|
|
13
13
|
from pulpcore.filters import BaseFilterSet
|
|
14
14
|
from pulpcore.app.models import (
|
|
15
|
+
AppStatus,
|
|
15
16
|
ProfileArtifact,
|
|
16
17
|
Task,
|
|
17
18
|
TaskGroup,
|
|
18
19
|
TaskSchedule,
|
|
19
|
-
Worker,
|
|
20
20
|
CreatedResource,
|
|
21
21
|
RepositoryVersion,
|
|
22
22
|
)
|
|
@@ -48,6 +48,7 @@ from pulpcore.app.role_util import get_objects_for_user
|
|
|
48
48
|
|
|
49
49
|
class TaskFilter(BaseFilterSet):
|
|
50
50
|
created_resources = CreatedResourcesFilter()
|
|
51
|
+
worker = filters.CharFilter(method="worker_filter")
|
|
51
52
|
# Non model field filters
|
|
52
53
|
reserved_resources = ReservedResourcesFilter(exclusive=True, shared=True)
|
|
53
54
|
reserved_resources__in = ReservedResourcesInFilter(exclusive=True, shared=True)
|
|
@@ -56,6 +57,13 @@ class TaskFilter(BaseFilterSet):
|
|
|
56
57
|
shared_resources = ReservedResourcesFilter(exclusive=False, shared=True)
|
|
57
58
|
shared_resources__in = ReservedResourcesInFilter(exclusive=False, shared=True)
|
|
58
59
|
|
|
60
|
+
def worker_filter(self, queryset, name, value):
|
|
61
|
+
# The worker field on tasks is no longer used.
|
|
62
|
+
if value is None:
|
|
63
|
+
return queryset
|
|
64
|
+
else:
|
|
65
|
+
return queryset.none()
|
|
66
|
+
|
|
59
67
|
class Meta:
|
|
60
68
|
model = Task
|
|
61
69
|
fields = {
|
|
@@ -354,14 +362,14 @@ class WorkerFilter(BaseFilterSet):
|
|
|
354
362
|
missing = filters.BooleanFilter(method="filter_missing")
|
|
355
363
|
|
|
356
364
|
class Meta:
|
|
357
|
-
model =
|
|
365
|
+
model = AppStatus
|
|
358
366
|
fields = {
|
|
359
367
|
"name": NAME_FILTER_OPTIONS,
|
|
360
368
|
"last_heartbeat": DATETIME_FILTER_OPTIONS,
|
|
361
369
|
}
|
|
362
370
|
|
|
363
371
|
def filter_online(self, queryset, name, value):
|
|
364
|
-
online_workers =
|
|
372
|
+
online_workers = AppStatus.objects.online()
|
|
365
373
|
|
|
366
374
|
if value:
|
|
367
375
|
return queryset.filter(pk__in=online_workers)
|
|
@@ -369,7 +377,7 @@ class WorkerFilter(BaseFilterSet):
|
|
|
369
377
|
return queryset.exclude(pk__in=online_workers)
|
|
370
378
|
|
|
371
379
|
def filter_missing(self, queryset, name, value):
|
|
372
|
-
missing_workers =
|
|
380
|
+
missing_workers = AppStatus.objects.missing()
|
|
373
381
|
|
|
374
382
|
if value:
|
|
375
383
|
return queryset.filter(pk__in=missing_workers)
|
|
@@ -378,7 +386,8 @@ class WorkerFilter(BaseFilterSet):
|
|
|
378
386
|
|
|
379
387
|
|
|
380
388
|
class WorkerViewSet(NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin):
|
|
381
|
-
queryset =
|
|
389
|
+
queryset = AppStatus.objects.filter(app_type="worker")
|
|
390
|
+
pulp_model_alias = "Worker"
|
|
382
391
|
serializer_class = WorkerSerializer
|
|
383
392
|
endpoint_name = "workers"
|
|
384
393
|
http_method_names = ["get", "options"]
|
pulpcore/middleware.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import time
|
|
2
|
+
import re
|
|
3
|
+
|
|
2
4
|
from contextvars import ContextVar
|
|
5
|
+
from os import environ
|
|
3
6
|
|
|
4
7
|
from django.http.response import Http404
|
|
5
8
|
from django.conf import settings
|
|
@@ -101,15 +104,32 @@ class APIRootRewriteMiddleware:
|
|
|
101
104
|
class DjangoMetricsMiddleware:
|
|
102
105
|
def __init__(self, get_response):
|
|
103
106
|
self.meter = init_otel_meter("pulp-api")
|
|
104
|
-
self.
|
|
105
|
-
name="api.request_duration",
|
|
106
|
-
description="Tracks the duration of HTTP requests",
|
|
107
|
-
unit="ms",
|
|
108
|
-
)
|
|
107
|
+
self._set_histogram(self.meter)
|
|
109
108
|
|
|
110
109
|
self.get_response = get_response
|
|
111
110
|
|
|
111
|
+
def _excluded_urls(self, url):
|
|
112
|
+
|
|
113
|
+
excluded_urls = environ.get(
|
|
114
|
+
"OTEL_PYTHON_EXCLUDED_URLS", environ.get("OTEL_PYTHON_DJANGO_EXCLUDED_URLS", "")
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
if excluded_urls:
|
|
118
|
+
excluded_urls_list = [excluded_url.strip() for excluded_url in excluded_urls.split(",")]
|
|
119
|
+
else:
|
|
120
|
+
return False
|
|
121
|
+
|
|
122
|
+
exclusion_pattern = "|".join(excluded_urls_list)
|
|
123
|
+
|
|
124
|
+
if re.search(exclusion_pattern, url):
|
|
125
|
+
return True
|
|
126
|
+
|
|
127
|
+
return False
|
|
128
|
+
|
|
112
129
|
def __call__(self, request):
|
|
130
|
+
if self._excluded_urls(request.build_absolute_uri("?")):
|
|
131
|
+
return self.get_response(request)
|
|
132
|
+
|
|
113
133
|
start_time = time.time()
|
|
114
134
|
response = self.get_response(request)
|
|
115
135
|
end_time = time.time()
|
|
@@ -122,11 +142,18 @@ class DjangoMetricsMiddleware:
|
|
|
122
142
|
return response
|
|
123
143
|
|
|
124
144
|
def _set_histogram(self, meter):
|
|
125
|
-
|
|
126
|
-
name
|
|
127
|
-
description
|
|
128
|
-
unit
|
|
129
|
-
|
|
145
|
+
create_histogram_kwargs = {
|
|
146
|
+
"name": "api.request_duration",
|
|
147
|
+
"description": "Tracks the duration of HTTP requests",
|
|
148
|
+
"unit": "ms",
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if settings.OTEL_PULP_API_HISTOGRAM_BUCKETS:
|
|
152
|
+
create_histogram_kwargs["explicit_bucket_boundaries_advisory"] = (
|
|
153
|
+
settings.OTEL_PULP_API_HISTOGRAM_BUCKETS
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
self.request_duration_histogram = meter.create_histogram(**create_histogram_kwargs)
|
|
130
157
|
|
|
131
158
|
def _process_attributes(self, request, response):
|
|
132
159
|
return {
|
pulpcore/openapi/__init__.py
CHANGED
|
@@ -260,7 +260,7 @@ class PulpSchemaGenerator(SchemaGenerator):
|
|
|
260
260
|
prefix (str): Optional prefix to add to the slug
|
|
261
261
|
pulp_model_alias (str): Optional model name to use instead of model.__name__
|
|
262
262
|
Returns:
|
|
263
|
-
str:
|
|
263
|
+
str: '{model_name_snake_case}_href'
|
|
264
264
|
"""
|
|
265
265
|
app_label = model._meta.app_label
|
|
266
266
|
model_name = pulp_model_alias or model.__name__
|