pulpcore 3.89.1__py3-none-any.whl → 3.90.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pulpcore might be problematic. Click here for more details.
- pulp_certguard/app/__init__.py +1 -1
- pulp_file/app/__init__.py +1 -1
- pulp_file/tests/functional/api/test_filesystem_export.py +220 -0
- pulp_file/tests/functional/api/test_pulp_export.py +103 -3
- pulpcore/app/apps.py +1 -1
- pulpcore/app/importexport.py +18 -2
- pulpcore/app/management/commands/shell.py +8 -0
- pulpcore/app/migrations/0144_delete_old_appstatus.py +28 -0
- pulpcore/app/migrations/0145_domainize_import_export.py +53 -0
- pulpcore/app/modelresource.py +61 -21
- pulpcore/app/models/__init__.py +2 -5
- pulpcore/app/models/exporter.py +7 -1
- pulpcore/app/models/fields.py +0 -1
- pulpcore/app/models/importer.py +8 -1
- pulpcore/app/models/repository.py +16 -0
- pulpcore/app/models/status.py +8 -138
- pulpcore/app/models/task.py +15 -25
- pulpcore/app/serializers/domain.py +1 -1
- pulpcore/app/serializers/exporter.py +4 -4
- pulpcore/app/serializers/importer.py +2 -2
- pulpcore/app/serializers/task.py +11 -8
- pulpcore/app/tasks/importer.py +44 -10
- pulpcore/app/tasks/repository.py +27 -0
- pulpcore/app/viewsets/base.py +18 -14
- pulpcore/app/viewsets/domain.py +1 -1
- pulpcore/app/viewsets/exporter.py +1 -8
- pulpcore/app/viewsets/importer.py +1 -6
- pulpcore/app/viewsets/task.py +0 -1
- pulpcore/openapi/__init__.py +16 -2
- pulpcore/plugin/tasking.py +4 -2
- pulpcore/tasking/tasks.py +245 -127
- pulpcore/tasking/worker.py +6 -17
- pulpcore/tests/functional/api/test_crud_domains.py +7 -0
- pulpcore/tests/functional/api/test_tasking.py +2 -2
- pulpcore/tests/functional/api/using_plugin/test_crud_repos.py +9 -2
- pulpcore/tests/unit/content/test_handler.py +43 -0
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/METADATA +7 -7
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/RECORD +42 -38
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/WHEEL +0 -0
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/entry_points.txt +0 -0
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/licenses/LICENSE +0 -0
- {pulpcore-3.89.1.dist-info → pulpcore-3.90.0.dist-info}/top_level.txt +0 -0
pulp_certguard/app/__init__.py
CHANGED
pulp_file/app/__init__.py
CHANGED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import pytest
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
from pulpcore.client.pulpcore.exceptions import ApiException, BadRequestException
|
|
6
|
+
from pulpcore.app import settings
|
|
7
|
+
from pulpcore.constants import TASK_STATES
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
pytestmark = [
|
|
11
|
+
pytest.mark.skipif(
|
|
12
|
+
"/tmp" not in settings.ALLOWED_EXPORT_PATHS,
|
|
13
|
+
reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS "
|
|
14
|
+
f"({settings.ALLOWED_EXPORT_PATHS}).",
|
|
15
|
+
),
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.fixture
|
|
20
|
+
def fs_exporter_factory(
|
|
21
|
+
tmpdir,
|
|
22
|
+
pulpcore_bindings,
|
|
23
|
+
gen_object_with_cleanup,
|
|
24
|
+
add_to_filesystem_cleanup,
|
|
25
|
+
):
|
|
26
|
+
def _fs_exporter_factory(method="write", pulp_domain=None):
|
|
27
|
+
name = str(uuid.uuid4())
|
|
28
|
+
path = "{}/{}/".format(tmpdir, name)
|
|
29
|
+
body = {
|
|
30
|
+
"name": name,
|
|
31
|
+
"path": path,
|
|
32
|
+
"method": method,
|
|
33
|
+
}
|
|
34
|
+
kwargs = {}
|
|
35
|
+
if pulp_domain:
|
|
36
|
+
kwargs["pulp_domain"] = pulp_domain
|
|
37
|
+
exporter = gen_object_with_cleanup(pulpcore_bindings.ExportersFilesystemApi, body, **kwargs)
|
|
38
|
+
add_to_filesystem_cleanup(path)
|
|
39
|
+
assert exporter.name == name
|
|
40
|
+
assert exporter.path == path
|
|
41
|
+
assert exporter.method == method
|
|
42
|
+
return exporter
|
|
43
|
+
|
|
44
|
+
return _fs_exporter_factory
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.fixture
|
|
48
|
+
def fs_export_factory(pulpcore_bindings, monitor_task):
|
|
49
|
+
def _fs_export_factory(exporter, body):
|
|
50
|
+
task = monitor_task(
|
|
51
|
+
pulpcore_bindings.ExportersFilesystemExportsApi.create(
|
|
52
|
+
exporter.pulp_href, body or {}
|
|
53
|
+
).task
|
|
54
|
+
)
|
|
55
|
+
assert len(task.created_resources) == 1
|
|
56
|
+
export = pulpcore_bindings.ExportersFilesystemExportsApi.read(task.created_resources[0])
|
|
57
|
+
for report in task.progress_reports:
|
|
58
|
+
assert report.state == TASK_STATES.COMPLETED
|
|
59
|
+
return export
|
|
60
|
+
|
|
61
|
+
return _fs_export_factory
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.fixture
|
|
65
|
+
def pub_and_repo(
|
|
66
|
+
file_repository_factory,
|
|
67
|
+
file_bindings,
|
|
68
|
+
gen_object_with_cleanup,
|
|
69
|
+
random_artifact_factory,
|
|
70
|
+
monitor_task,
|
|
71
|
+
):
|
|
72
|
+
def _pub_and_repo(pulp_domain=None):
|
|
73
|
+
random_artifact = random_artifact_factory(pulp_domain=pulp_domain)
|
|
74
|
+
repository = file_repository_factory(pulp_domain=pulp_domain)
|
|
75
|
+
kwargs = {}
|
|
76
|
+
if pulp_domain:
|
|
77
|
+
kwargs["pulp_domain"] = pulp_domain
|
|
78
|
+
for i in range(2):
|
|
79
|
+
monitor_task(
|
|
80
|
+
file_bindings.ContentFilesApi.create(
|
|
81
|
+
artifact=random_artifact.pulp_href,
|
|
82
|
+
relative_path=f"{i}.dat",
|
|
83
|
+
repository=repository.pulp_href,
|
|
84
|
+
**kwargs,
|
|
85
|
+
).task
|
|
86
|
+
)
|
|
87
|
+
publish_data = file_bindings.FileFilePublication(repository=repository.pulp_href)
|
|
88
|
+
publication = gen_object_with_cleanup(
|
|
89
|
+
file_bindings.PublicationsFileApi, publish_data, **kwargs
|
|
90
|
+
)
|
|
91
|
+
return publication, repository
|
|
92
|
+
|
|
93
|
+
return _pub_and_repo
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@pytest.mark.parallel
|
|
97
|
+
def test_crud_fsexporter(fs_exporter_factory, pulpcore_bindings, monitor_task):
|
|
98
|
+
# READ
|
|
99
|
+
exporter = fs_exporter_factory()
|
|
100
|
+
exporter_read = pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href)
|
|
101
|
+
assert exporter_read.name == exporter.name
|
|
102
|
+
assert exporter_read.path == exporter.path
|
|
103
|
+
|
|
104
|
+
# UPDATE
|
|
105
|
+
body = {"path": "/tmp/{}".format(str(uuid.uuid4()))}
|
|
106
|
+
result = pulpcore_bindings.ExportersFilesystemApi.partial_update(exporter.pulp_href, body)
|
|
107
|
+
monitor_task(result.task)
|
|
108
|
+
exporter_read = pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href)
|
|
109
|
+
assert exporter_read.path != exporter.path
|
|
110
|
+
assert exporter_read.path == body["path"]
|
|
111
|
+
|
|
112
|
+
# LIST
|
|
113
|
+
exporters = pulpcore_bindings.ExportersFilesystemApi.list(name=exporter.name).results
|
|
114
|
+
assert exporter.name in [e.name for e in exporters]
|
|
115
|
+
|
|
116
|
+
# DELETE
|
|
117
|
+
result = pulpcore_bindings.ExportersFilesystemApi.delete(exporter.pulp_href)
|
|
118
|
+
monitor_task(result.task)
|
|
119
|
+
with pytest.raises(ApiException):
|
|
120
|
+
pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@pytest.mark.parallel
|
|
124
|
+
def test_fsexport(pulpcore_bindings, fs_exporter_factory, fs_export_factory, pub_and_repo):
|
|
125
|
+
exporter = fs_exporter_factory()
|
|
126
|
+
(publication, _) = pub_and_repo()
|
|
127
|
+
# Test export
|
|
128
|
+
body = {"publication": publication.pulp_href}
|
|
129
|
+
export = fs_export_factory(exporter, body=body)
|
|
130
|
+
|
|
131
|
+
# Test list and delete
|
|
132
|
+
exports = pulpcore_bindings.ExportersPulpExportsApi.list(exporter.pulp_href).results
|
|
133
|
+
assert len(exports) == 1
|
|
134
|
+
pulpcore_bindings.ExportersPulpExportsApi.delete(export.pulp_href)
|
|
135
|
+
exports = pulpcore_bindings.ExportersPulpExportsApi.list(exporter.pulp_href).results
|
|
136
|
+
assert len(exports) == 0
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@pytest.mark.parallel
|
|
140
|
+
def test_fsexport_by_version(
|
|
141
|
+
fs_exporter_factory,
|
|
142
|
+
fs_export_factory,
|
|
143
|
+
pub_and_repo,
|
|
144
|
+
):
|
|
145
|
+
(publication, repository) = pub_and_repo()
|
|
146
|
+
latest = repository.latest_version_href
|
|
147
|
+
zeroth = latest.replace("/2/", "/0/")
|
|
148
|
+
|
|
149
|
+
# export by version
|
|
150
|
+
exporter = fs_exporter_factory()
|
|
151
|
+
body = {"repository_version": latest}
|
|
152
|
+
fs_export_factory(exporter, body=body)
|
|
153
|
+
|
|
154
|
+
# export by version with start_version
|
|
155
|
+
exporter = fs_exporter_factory()
|
|
156
|
+
body = {"repository_version": latest, "start_repository_version": zeroth}
|
|
157
|
+
fs_export_factory(exporter, body=body)
|
|
158
|
+
|
|
159
|
+
# export by publication with start_version
|
|
160
|
+
exporter = fs_exporter_factory()
|
|
161
|
+
body = {"publication": publication.pulp_href, "start_repository_version": zeroth}
|
|
162
|
+
fs_export_factory(exporter, body=body)
|
|
163
|
+
|
|
164
|
+
# negative: specify publication and version
|
|
165
|
+
with pytest.raises(BadRequestException) as e:
|
|
166
|
+
exporter = fs_exporter_factory()
|
|
167
|
+
body = {"publication": publication.pulp_href, "repository_version": zeroth}
|
|
168
|
+
fs_export_factory(exporter, body=body)
|
|
169
|
+
assert e.value.status == 400
|
|
170
|
+
assert json.loads(e.value.body) == {
|
|
171
|
+
"non_field_errors": [
|
|
172
|
+
"publication or repository_version must either be supplied but not both."
|
|
173
|
+
]
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.")
|
|
178
|
+
@pytest.mark.parallel
|
|
179
|
+
def test_fsexport_cross_domain(
|
|
180
|
+
fs_exporter_factory,
|
|
181
|
+
fs_export_factory,
|
|
182
|
+
gen_object_with_cleanup,
|
|
183
|
+
pulpcore_bindings,
|
|
184
|
+
pub_and_repo,
|
|
185
|
+
):
|
|
186
|
+
|
|
187
|
+
entities = [{}, {}]
|
|
188
|
+
for e in entities:
|
|
189
|
+
body = {
|
|
190
|
+
"name": str(uuid.uuid4()),
|
|
191
|
+
"storage_class": "pulpcore.app.models.storage.FileSystem",
|
|
192
|
+
"storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"},
|
|
193
|
+
}
|
|
194
|
+
e["domain"] = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body)
|
|
195
|
+
(e["publication"], e["repository"]) = pub_and_repo(pulp_domain=e["domain"].name)
|
|
196
|
+
e["exporter"] = fs_exporter_factory(pulp_domain=e["domain"].name)
|
|
197
|
+
body = {"publication": e["publication"].pulp_href}
|
|
198
|
+
e["export"] = fs_export_factory(e["exporter"], body=body)
|
|
199
|
+
|
|
200
|
+
latest = entities[0]["repository"].latest_version_href
|
|
201
|
+
zeroth = latest.replace("/2/", "/0/")
|
|
202
|
+
|
|
203
|
+
with pytest.raises(BadRequestException) as e:
|
|
204
|
+
body = {"publication": entities[0]["publication"].pulp_href}
|
|
205
|
+
fs_export_factory(entities[1]["exporter"], body=body)
|
|
206
|
+
|
|
207
|
+
with pytest.raises(BadRequestException) as e:
|
|
208
|
+
body = {"repository_version": latest}
|
|
209
|
+
fs_export_factory(entities[1]["exporter"], body=body)
|
|
210
|
+
|
|
211
|
+
with pytest.raises(BadRequestException) as e:
|
|
212
|
+
body = {"repository_version": latest, "start_repository_version": zeroth}
|
|
213
|
+
fs_export_factory(entities[1]["exporter"], body=body)
|
|
214
|
+
|
|
215
|
+
with pytest.raises(BadRequestException) as e:
|
|
216
|
+
body = {
|
|
217
|
+
"publication": entities[0]["publication"].pulp_href,
|
|
218
|
+
"start_repository_version": zeroth,
|
|
219
|
+
}
|
|
220
|
+
fs_export_factory(entities[1]["exporter"], body=body)
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import pytest
|
|
2
3
|
import uuid
|
|
3
4
|
|
|
4
5
|
from pulpcore.client.pulpcore.exceptions import ApiException
|
|
6
|
+
from pulpcore.client.pulpcore.exceptions import BadRequestException
|
|
5
7
|
from pulpcore.app import settings
|
|
6
8
|
from pulpcore.constants import TASK_STATES
|
|
7
9
|
|
|
8
10
|
|
|
9
11
|
pytestmark = [
|
|
10
|
-
pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export."),
|
|
11
12
|
pytest.mark.skipif(
|
|
12
13
|
"/tmp" not in settings.ALLOWED_EXPORT_PATHS,
|
|
13
14
|
reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS "
|
|
@@ -23,7 +24,10 @@ def pulp_exporter_factory(
|
|
|
23
24
|
gen_object_with_cleanup,
|
|
24
25
|
add_to_filesystem_cleanup,
|
|
25
26
|
):
|
|
26
|
-
def _pulp_exporter_factory(
|
|
27
|
+
def _pulp_exporter_factory(
|
|
28
|
+
repositories=None,
|
|
29
|
+
pulp_domain=None,
|
|
30
|
+
):
|
|
27
31
|
if repositories is None:
|
|
28
32
|
repositories = []
|
|
29
33
|
name = str(uuid.uuid4())
|
|
@@ -33,7 +37,11 @@ def pulp_exporter_factory(
|
|
|
33
37
|
"path": path,
|
|
34
38
|
"repositories": [r.pulp_href for r in repositories],
|
|
35
39
|
}
|
|
36
|
-
|
|
40
|
+
kwargs = {}
|
|
41
|
+
if pulp_domain:
|
|
42
|
+
kwargs["pulp_domain"] = pulp_domain.name
|
|
43
|
+
|
|
44
|
+
exporter = gen_object_with_cleanup(pulpcore_bindings.ExportersPulpApi, body, **kwargs)
|
|
37
45
|
add_to_filesystem_cleanup(path)
|
|
38
46
|
assert exporter.name == name
|
|
39
47
|
assert exporter.path == path
|
|
@@ -291,3 +299,95 @@ def test_export_incremental(
|
|
|
291
299
|
with pytest.raises(ApiException):
|
|
292
300
|
body = {"start_versions": [file_repo.latest_version_href], "full": False}
|
|
293
301
|
pulp_export_factory(exporter, body)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# Test that cross-domain attempts fail
|
|
305
|
+
# Exporter: repository, last-export : create, update
|
|
306
|
+
# Export: versions, start_versions
|
|
307
|
+
@pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.")
|
|
308
|
+
@pytest.mark.parallel
|
|
309
|
+
def test_cross_domain_exporter(
|
|
310
|
+
basic_manifest_path,
|
|
311
|
+
file_bindings,
|
|
312
|
+
file_remote_factory,
|
|
313
|
+
gen_object_with_cleanup,
|
|
314
|
+
pulpcore_bindings,
|
|
315
|
+
pulp_export_factory,
|
|
316
|
+
pulp_exporter_factory,
|
|
317
|
+
monitor_task,
|
|
318
|
+
):
|
|
319
|
+
# Create two domains
|
|
320
|
+
# In each, create and sync a repository, create and export an exporter
|
|
321
|
+
# Attempt to create an exporter using the *other domain's* repo
|
|
322
|
+
# Attempt to update the exporter using the *other domain's* repo and last_export
|
|
323
|
+
# Use the exporter and attempt to export the *other domain's* repo-versions
|
|
324
|
+
|
|
325
|
+
entities = [{}, {}]
|
|
326
|
+
for e in entities:
|
|
327
|
+
body = {
|
|
328
|
+
"name": str(uuid.uuid4()),
|
|
329
|
+
"storage_class": "pulpcore.app.models.storage.FileSystem",
|
|
330
|
+
"storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"},
|
|
331
|
+
}
|
|
332
|
+
e["domain"] = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body)
|
|
333
|
+
remote = file_remote_factory(
|
|
334
|
+
manifest_path=basic_manifest_path, policy="immediate", pulp_domain=e["domain"].name
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
repo_body = {"name": str(uuid.uuid4()), "remote": remote.pulp_href}
|
|
338
|
+
e["repository"] = gen_object_with_cleanup(
|
|
339
|
+
file_bindings.RepositoriesFileApi, repo_body, pulp_domain=e["domain"].name
|
|
340
|
+
)
|
|
341
|
+
task = file_bindings.RepositoriesFileApi.sync(e["repository"].pulp_href, {}).task
|
|
342
|
+
monitor_task(task)
|
|
343
|
+
e["repository"] = file_bindings.RepositoriesFileApi.read(e["repository"].pulp_href)
|
|
344
|
+
e["exporter"] = pulp_exporter_factory(
|
|
345
|
+
repositories=[e["repository"]], pulp_domain=e["domain"]
|
|
346
|
+
)
|
|
347
|
+
e["export"] = pulp_export_factory(e["exporter"])
|
|
348
|
+
|
|
349
|
+
target_domain = entities[1]["domain"]
|
|
350
|
+
# cross-create
|
|
351
|
+
with pytest.raises(BadRequestException) as e:
|
|
352
|
+
pulp_exporter_factory(repositories=[entities[0]["repository"]], pulp_domain=target_domain)
|
|
353
|
+
assert e.value.status == 400
|
|
354
|
+
assert json.loads(e.value.body) == {
|
|
355
|
+
"non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."]
|
|
356
|
+
}
|
|
357
|
+
# cross-update
|
|
358
|
+
body = {"repositories": [entities[0]["repository"].pulp_href]}
|
|
359
|
+
with pytest.raises(BadRequestException) as e:
|
|
360
|
+
pulpcore_bindings.ExportersPulpApi.partial_update(entities[1]["exporter"].pulp_href, body)
|
|
361
|
+
assert e.value.status == 400
|
|
362
|
+
assert json.loads(e.value.body) == {
|
|
363
|
+
"non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."]
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
body = {"last_export": entities[0]["export"].pulp_href}
|
|
367
|
+
with pytest.raises(BadRequestException) as e:
|
|
368
|
+
pulpcore_bindings.ExportersPulpApi.partial_update(entities[1]["exporter"].pulp_href, body)
|
|
369
|
+
assert e.value.status == 400
|
|
370
|
+
assert json.loads(e.value.body) == {
|
|
371
|
+
"non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."]
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
# cross-export
|
|
375
|
+
with pytest.raises(BadRequestException) as e:
|
|
376
|
+
latest_v = entities[0]["repository"].latest_version_href
|
|
377
|
+
zero_v = latest_v.replace("/1/", "/0/")
|
|
378
|
+
body = {
|
|
379
|
+
"start_versions": [latest_v],
|
|
380
|
+
"versions": [zero_v],
|
|
381
|
+
"full": False,
|
|
382
|
+
}
|
|
383
|
+
pulp_export_factory(entities[1]["exporter"], body)
|
|
384
|
+
assert e.value.status == 400
|
|
385
|
+
msgs = json.loads(e.value.body)
|
|
386
|
+
assert "versions" in msgs
|
|
387
|
+
assert "start_versions" in msgs
|
|
388
|
+
assert msgs["versions"] == [
|
|
389
|
+
"Requested RepositoryVersions must belong to the Repositories named by the Exporter!"
|
|
390
|
+
]
|
|
391
|
+
assert msgs["start_versions"] == [
|
|
392
|
+
"Requested RepositoryVersions must belong to the Repositories named by the Exporter!"
|
|
393
|
+
]
|
pulpcore/app/apps.py
CHANGED
pulpcore/app/importexport.py
CHANGED
|
@@ -130,14 +130,30 @@ def export_artifacts(export, artifact_pks):
|
|
|
130
130
|
temp_file.write(artifact.file.read())
|
|
131
131
|
temp_file.flush()
|
|
132
132
|
artifact.file.close()
|
|
133
|
-
|
|
133
|
+
# If we're domain-enabled, replace our domain-pk with "DOMAIN" in
|
|
134
|
+
# the tarfile
|
|
135
|
+
if settings.DOMAIN_ENABLED:
|
|
136
|
+
tarfile_loc = artifact.file.name.replace(
|
|
137
|
+
str(artifact.pulp_domain_id), "DOMAIN"
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
tarfile_loc = artifact.file.name
|
|
141
|
+
export.tarfile.add(temp_file.name, tarfile_loc)
|
|
134
142
|
else:
|
|
135
143
|
for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE):
|
|
136
144
|
batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE]
|
|
137
145
|
batch_qs = Artifact.objects.filter(pk__in=batch).only("file")
|
|
138
146
|
|
|
139
147
|
for artifact in pb.iter(batch_qs.iterator()):
|
|
140
|
-
|
|
148
|
+
# If we're domain-enabled, replace our domain-pk with "DOMAIN" in
|
|
149
|
+
# the tarfile
|
|
150
|
+
if settings.DOMAIN_ENABLED:
|
|
151
|
+
tarfile_loc = artifact.file.name.replace(
|
|
152
|
+
str(artifact.pulp_domain_id), "DOMAIN"
|
|
153
|
+
)
|
|
154
|
+
else:
|
|
155
|
+
tarfile_loc = artifact.file.name
|
|
156
|
+
export.tarfile.add(artifact.file.path, tarfile_loc)
|
|
141
157
|
|
|
142
158
|
resource = ArtifactResource()
|
|
143
159
|
resource.queryset = Artifact.objects.filter(pk__in=artifact_pks)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-09-09 08:24
|
|
2
|
+
|
|
3
|
+
from django.db import migrations
|
|
4
|
+
from pulpcore.migrations import RequireVersion
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Migration(migrations.Migration):
|
|
8
|
+
|
|
9
|
+
dependencies = [
|
|
10
|
+
('core', '0143_require_app_lock_zdu'),
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
operations = [
|
|
14
|
+
RequireVersion("core", "3.87"),
|
|
15
|
+
migrations.DeleteModel(
|
|
16
|
+
name='ApiAppStatus',
|
|
17
|
+
),
|
|
18
|
+
migrations.DeleteModel(
|
|
19
|
+
name='ContentAppStatus',
|
|
20
|
+
),
|
|
21
|
+
migrations.RemoveField(
|
|
22
|
+
model_name='task',
|
|
23
|
+
name='worker',
|
|
24
|
+
),
|
|
25
|
+
migrations.DeleteModel(
|
|
26
|
+
name='Worker',
|
|
27
|
+
),
|
|
28
|
+
]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-08-21 21:21
|
|
2
|
+
|
|
3
|
+
from django.db import migrations, models
|
|
4
|
+
import django.db.models.deletion
|
|
5
|
+
import pulpcore.app.util
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Migration(migrations.Migration):
|
|
9
|
+
|
|
10
|
+
dependencies = [
|
|
11
|
+
("core", "0144_delete_old_appstatus"),
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
operations = [
|
|
15
|
+
migrations.AddField(
|
|
16
|
+
model_name="export",
|
|
17
|
+
name="pulp_domain",
|
|
18
|
+
field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"),
|
|
19
|
+
),
|
|
20
|
+
migrations.AddField(
|
|
21
|
+
model_name="exporter",
|
|
22
|
+
name="pulp_domain",
|
|
23
|
+
field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"),
|
|
24
|
+
),
|
|
25
|
+
migrations.AlterUniqueTogether(
|
|
26
|
+
name="exporter",
|
|
27
|
+
unique_together={("name", "pulp_domain")},
|
|
28
|
+
),
|
|
29
|
+
migrations.AddField(
|
|
30
|
+
model_name="import",
|
|
31
|
+
name="pulp_domain",
|
|
32
|
+
field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"),
|
|
33
|
+
),
|
|
34
|
+
migrations.AddField(
|
|
35
|
+
model_name="importer",
|
|
36
|
+
name="pulp_domain",
|
|
37
|
+
field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"),
|
|
38
|
+
),
|
|
39
|
+
migrations.AlterUniqueTogether(
|
|
40
|
+
name="importer",
|
|
41
|
+
unique_together={("name", "pulp_domain")},
|
|
42
|
+
),
|
|
43
|
+
migrations.AlterField(
|
|
44
|
+
model_name="exporter",
|
|
45
|
+
name="name",
|
|
46
|
+
field=models.TextField(),
|
|
47
|
+
),
|
|
48
|
+
migrations.AlterField(
|
|
49
|
+
model_name="importer",
|
|
50
|
+
name="name",
|
|
51
|
+
field=models.TextField(),
|
|
52
|
+
),
|
|
53
|
+
]
|
pulpcore/app/modelresource.py
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from django.conf import settings
|
|
1
4
|
from import_export import fields
|
|
2
5
|
from import_export.widgets import ForeignKeyWidget
|
|
3
6
|
from logging import getLogger
|
|
@@ -8,12 +11,16 @@ from pulpcore.app.models.content import (
|
|
|
8
11
|
ContentArtifact,
|
|
9
12
|
)
|
|
10
13
|
from pulpcore.app.models.repository import Repository
|
|
14
|
+
from pulpcore.app.util import get_domain_pk, get_domain
|
|
11
15
|
from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS
|
|
12
16
|
from pulpcore.plugin.importexport import QueryModelResource
|
|
13
17
|
|
|
14
|
-
|
|
15
18
|
log = getLogger(__name__)
|
|
16
19
|
|
|
20
|
+
domain_artifact_file_regex = re.compile(
|
|
21
|
+
r"^artifact/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
|
|
22
|
+
)
|
|
23
|
+
|
|
17
24
|
|
|
18
25
|
#
|
|
19
26
|
# Artifact and Repository are different from other import-export entities, in that they are not
|
|
@@ -31,6 +38,25 @@ class ArtifactResource(QueryModelResource):
|
|
|
31
38
|
kwargs: args passed along from the import() call.
|
|
32
39
|
|
|
33
40
|
"""
|
|
41
|
+
# IF we're domain-enabled *AND NOT IMPORTING INTO DEFAULT*:
|
|
42
|
+
# REPLACE "their" domain-id with ours, if there is one.
|
|
43
|
+
# If not, INSERT "our" domain-id into the path in the right place.
|
|
44
|
+
# Otherwise:
|
|
45
|
+
# REMOVE their domain-id if there is one.
|
|
46
|
+
# Do this before letting QMR run, since it will replace "their" domain-id with "ours"
|
|
47
|
+
upstream_domain_enabled = re.match(domain_artifact_file_regex, row["file"])
|
|
48
|
+
into_default = "default" == get_domain().name
|
|
49
|
+
domain = str(get_domain_pk())
|
|
50
|
+
|
|
51
|
+
if settings.DOMAIN_ENABLED and not into_default: # Replace "their" domain-id with "ours"
|
|
52
|
+
if upstream_domain_enabled:
|
|
53
|
+
row["file"] = row["file"].replace(row["pulp_domain"], domain)
|
|
54
|
+
else: # Add in our domain-id to the path
|
|
55
|
+
row["file"] = row["file"].replace("artifact", f"artifact/{domain}")
|
|
56
|
+
else: # Strip domain-id out of the artifact-file *if there is one there*
|
|
57
|
+
if upstream_domain_enabled:
|
|
58
|
+
row["file"] = row["file"].replace(f'artifact/{row["pulp_domain"]}/', "artifact/")
|
|
59
|
+
|
|
34
60
|
super().before_import_row(row, **kwargs)
|
|
35
61
|
|
|
36
62
|
# the export converts None to blank strings but sha384 and sha512 have unique constraints
|
|
@@ -39,14 +65,15 @@ class ArtifactResource(QueryModelResource):
|
|
|
39
65
|
if row[checksum] == "":
|
|
40
66
|
row[checksum] = None
|
|
41
67
|
|
|
68
|
+
def set_up_queryset(self):
|
|
69
|
+
"""
|
|
70
|
+
:return: Artifacts for a specific domain
|
|
71
|
+
"""
|
|
72
|
+
return Artifact.objects.filter(pulp_domain_id=get_domain_pk())
|
|
73
|
+
|
|
42
74
|
class Meta:
|
|
43
75
|
model = Artifact
|
|
44
|
-
exclude = (
|
|
45
|
-
"pulp_id",
|
|
46
|
-
"pulp_created",
|
|
47
|
-
"pulp_last_updated",
|
|
48
|
-
"timestamp_of_interest",
|
|
49
|
-
)
|
|
76
|
+
exclude = QueryModelResource.Meta.exclude + ("timestamp_of_interest",)
|
|
50
77
|
import_id_fields = (
|
|
51
78
|
"pulp_domain",
|
|
52
79
|
"sha256",
|
|
@@ -54,16 +81,20 @@ class ArtifactResource(QueryModelResource):
|
|
|
54
81
|
|
|
55
82
|
|
|
56
83
|
class RepositoryResource(QueryModelResource):
|
|
84
|
+
|
|
85
|
+
def set_up_queryset(self):
|
|
86
|
+
"""
|
|
87
|
+
:return: Repositories for a specific domain
|
|
88
|
+
"""
|
|
89
|
+
return Repository.objects.filter(pulp_domain_id=get_domain_pk())
|
|
90
|
+
|
|
57
91
|
class Meta:
|
|
58
92
|
model = Repository
|
|
59
93
|
import_id_fields = (
|
|
60
94
|
"pulp_domain",
|
|
61
95
|
"name",
|
|
62
96
|
)
|
|
63
|
-
exclude = (
|
|
64
|
-
"pulp_id",
|
|
65
|
-
"pulp_created",
|
|
66
|
-
"pulp_last_updated",
|
|
97
|
+
exclude = QueryModelResource.Meta.exclude + (
|
|
67
98
|
"content",
|
|
68
99
|
"next_version",
|
|
69
100
|
"repository_ptr",
|
|
@@ -72,9 +103,18 @@ class RepositoryResource(QueryModelResource):
|
|
|
72
103
|
)
|
|
73
104
|
|
|
74
105
|
|
|
106
|
+
class ArtifactDomainForeignKeyWidget(ForeignKeyWidget):
|
|
107
|
+
def get_queryset(self, value, row, *args, **kwargs):
|
|
108
|
+
qs = self.model.objects.filter(sha256=row["artifact"], pulp_domain_id=get_domain_pk())
|
|
109
|
+
return qs
|
|
110
|
+
|
|
111
|
+
def render(self, value, obj=None, **kwargs):
|
|
112
|
+
return value.sha256
|
|
113
|
+
|
|
114
|
+
|
|
75
115
|
class ContentArtifactResource(QueryModelResource):
|
|
76
116
|
"""
|
|
77
|
-
Handles import/export of the ContentArtifact model
|
|
117
|
+
Handles import/export of the ContentArtifact model
|
|
78
118
|
|
|
79
119
|
ContentArtifact is different from other import-export entities because it has no 'natural key'
|
|
80
120
|
other than a pulp_id, which aren't shared across instances. We do some magic to link up
|
|
@@ -85,7 +125,9 @@ class ContentArtifactResource(QueryModelResource):
|
|
|
85
125
|
"""
|
|
86
126
|
|
|
87
127
|
artifact = fields.Field(
|
|
88
|
-
column_name="artifact",
|
|
128
|
+
column_name="artifact",
|
|
129
|
+
attribute="artifact",
|
|
130
|
+
widget=ArtifactDomainForeignKeyWidget(Artifact, "sha256"),
|
|
89
131
|
)
|
|
90
132
|
linked_content = {}
|
|
91
133
|
|
|
@@ -121,18 +163,21 @@ class ContentArtifactResource(QueryModelResource):
|
|
|
121
163
|
for content_ids in self.content_mapping.values():
|
|
122
164
|
content_pks |= set(content_ids)
|
|
123
165
|
|
|
124
|
-
|
|
166
|
+
qs = (
|
|
125
167
|
ContentArtifact.objects.filter(content__in=content_pks)
|
|
126
168
|
.order_by("content", "relative_path")
|
|
127
169
|
.select_related("artifact")
|
|
128
170
|
)
|
|
171
|
+
return qs
|
|
129
172
|
|
|
130
173
|
def dehydrate_content(self, content_artifact):
|
|
131
174
|
return str(content_artifact.content_id)
|
|
132
175
|
|
|
133
176
|
def fetch_linked_content(self):
|
|
134
177
|
linked_content = {}
|
|
135
|
-
c_qs = Content.objects.filter(
|
|
178
|
+
c_qs = Content.objects.filter(
|
|
179
|
+
upstream_id__isnull=False, pulp_domain_id=get_domain_pk()
|
|
180
|
+
).values("upstream_id", "pulp_id")
|
|
136
181
|
for c in c_qs.iterator():
|
|
137
182
|
linked_content[str(c["upstream_id"])] = str(c["pulp_id"])
|
|
138
183
|
|
|
@@ -144,9 +189,4 @@ class ContentArtifactResource(QueryModelResource):
|
|
|
144
189
|
"content",
|
|
145
190
|
"relative_path",
|
|
146
191
|
)
|
|
147
|
-
exclude = (
|
|
148
|
-
"pulp_created",
|
|
149
|
-
"pulp_last_updated",
|
|
150
|
-
"_artifacts",
|
|
151
|
-
"pulp_id",
|
|
152
|
-
)
|
|
192
|
+
exclude = QueryModelResource.Meta.exclude + ("_artifacts",)
|