feincms3-api 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- feincms3_api/__init__.py +1 -0
- feincms3_api/adapters.py +303 -0
- feincms3_api/admin.py +93 -0
- feincms3_api/api.py +66 -0
- feincms3_api/apps.py +70 -0
- feincms3_api/auth.py +29 -0
- feincms3_api/cabinet.py +78 -0
- feincms3_api/codecs.py +495 -0
- feincms3_api/conf.py +31 -0
- feincms3_api/content.py +312 -0
- feincms3_api/errors.py +53 -0
- feincms3_api/forms.py +36 -0
- feincms3_api/formsets.py +135 -0
- feincms3_api/http.py +195 -0
- feincms3_api/incubator/__init__.py +6 -0
- feincms3_api/incubator/autotranslate/__init__.py +0 -0
- feincms3_api/incubator/autotranslate/apps.py +47 -0
- feincms3_api/incubator/autotranslate/backends/__init__.py +78 -0
- feincms3_api/incubator/autotranslate/backends/anthropic.py +98 -0
- feincms3_api/incubator/autotranslate/conf.py +59 -0
- feincms3_api/incubator/autotranslate/datasets.py +428 -0
- feincms3_api/incubator/autotranslate/fields.py +213 -0
- feincms3_api/incubator/autotranslate/links.py +122 -0
- feincms3_api/incubator/autotranslate/loader.py +1250 -0
- feincms3_api/incubator/autotranslate/management/__init__.py +0 -0
- feincms3_api/incubator/autotranslate/management/commands/__init__.py +0 -0
- feincms3_api/incubator/autotranslate/management/commands/f3autotranslate.py +162 -0
- feincms3_api/incubator/autotranslate/management/commands/f3autotranslateload.py +92 -0
- feincms3_api/incubator/autotranslate/review.py +104 -0
- feincms3_api/incubator/autotranslate/strings.py +313 -0
- feincms3_api/incubator/autotranslate/translate.py +260 -0
- feincms3_api/kinds.py +86 -0
- feincms3_api/management/__init__.py +0 -0
- feincms3_api/management/commands/__init__.py +0 -0
- feincms3_api/management/commands/f3api_cleanup.py +18 -0
- feincms3_api/management/commands/f3api_schema.py +32 -0
- feincms3_api/management/commands/f3api_token.py +39 -0
- feincms3_api/migrations/0001_initial.py +91 -0
- feincms3_api/migrations/0002_upload.py +61 -0
- feincms3_api/migrations/__init__.py +0 -0
- feincms3_api/models.py +189 -0
- feincms3_api/paths.py +43 -0
- feincms3_api/reading.py +120 -0
- feincms3_api/references.py +123 -0
- feincms3_api/resources.py +580 -0
- feincms3_api/schema.py +80 -0
- feincms3_api/snapshot.py +110 -0
- feincms3_api/templates/admin/feincms3_api/apitoken/created.html +19 -0
- feincms3_api/writing.py +255 -0
- feincms3_api-0.0.1.dist-info/METADATA +69 -0
- feincms3_api-0.0.1.dist-info/RECORD +53 -0
- feincms3_api-0.0.1.dist-info/WHEEL +4 -0
- feincms3_api-0.0.1.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,1250 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Loading translated objects as inactive copies
|
|
3
|
+
|
|
4
|
+
Django's deserialization does the reading; what ``loaddata`` would do next,
|
|
5
|
+
saving with ``raw=True`` under the original primary keys, is left out.
|
|
6
|
+
Instead every object gets a fresh primary key, its references are mapped,
|
|
7
|
+
its own fields rewritten, and it goes through ``full_clean()`` and
|
|
8
|
+
``save()``. Nothing but inserts, and replacing content with ``--replace``.
|
|
9
|
+
|
|
10
|
+
The same decisions answer ``--plan``, which only needs the dumps and the
|
|
11
|
+
target database, and nothing is written then.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import dataclasses
|
|
15
|
+
import json
|
|
16
|
+
from functools import cached_property
|
|
17
|
+
|
|
18
|
+
from django.apps import apps
|
|
19
|
+
from django.core import serializers
|
|
20
|
+
from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
|
|
21
|
+
from django.core.management.base import CommandError
|
|
22
|
+
from django.db import DatabaseError, transaction
|
|
23
|
+
from django.urls import NoReverseMatch, reverse
|
|
24
|
+
from django.utils.text import slugify
|
|
25
|
+
|
|
26
|
+
from feincms3_api.incubator.autotranslate.conf import first_language
|
|
27
|
+
from feincms3_api.incubator.autotranslate.datasets import (
|
|
28
|
+
_describe,
|
|
29
|
+
has_field,
|
|
30
|
+
has_unique_slug,
|
|
31
|
+
label,
|
|
32
|
+
plugin_models,
|
|
33
|
+
registry as default_registry,
|
|
34
|
+
translated_fields,
|
|
35
|
+
)
|
|
36
|
+
from feincms3_api.incubator.autotranslate.fields import (
|
|
37
|
+
HTML,
|
|
38
|
+
LINK,
|
|
39
|
+
field_specs,
|
|
40
|
+
json_proxies,
|
|
41
|
+
)
|
|
42
|
+
from feincms3_api.incubator.autotranslate.links import (
|
|
43
|
+
host_of,
|
|
44
|
+
map_html,
|
|
45
|
+
page_location,
|
|
46
|
+
path_scope,
|
|
47
|
+
split_url,
|
|
48
|
+
)
|
|
49
|
+
from feincms3_api.incubator.autotranslate.strings import (
|
|
50
|
+
OWNER,
|
|
51
|
+
REGIONS,
|
|
52
|
+
DumpError,
|
|
53
|
+
all_objects,
|
|
54
|
+
group_units,
|
|
55
|
+
source_language,
|
|
56
|
+
unit_key,
|
|
57
|
+
unit_strings,
|
|
58
|
+
)
|
|
59
|
+
from feincms3_api.paths import iter_path
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
CREATE = "create"
|
|
63
|
+
REPLACE = "replace"
|
|
64
|
+
SKIP = "skip"
|
|
65
|
+
FAIL = "fail"
|
|
66
|
+
|
|
67
|
+
LOG_STATUS = {CREATE: "created", REPLACE: "replaced", SKIP: "skipped", FAIL: "failed"}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Failed(Exception):
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def admin_url(obj):
|
|
75
|
+
try:
|
|
76
|
+
return reverse(
|
|
77
|
+
f"admin:{obj._meta.app_label}_{obj._meta.model_name}_change",
|
|
78
|
+
args=[obj.pk],
|
|
79
|
+
)
|
|
80
|
+
except NoReverseMatch:
|
|
81
|
+
return ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def location(obj):
|
|
85
|
+
"""Path or slug, whatever says best where an object is"""
|
|
86
|
+
if path := getattr(obj, "path", None):
|
|
87
|
+
return path
|
|
88
|
+
return getattr(obj, "slug", None) or ""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _proxy_class(obj):
|
|
92
|
+
"""
|
|
93
|
+
The proxy class of a typed plugin (django-json-schema-editor,
|
|
94
|
+
feincms3-forms), whose ``save()`` needs its ``TYPE``
|
|
95
|
+
"""
|
|
96
|
+
model = type(obj)
|
|
97
|
+
if not has_field(model, "type"):
|
|
98
|
+
return model
|
|
99
|
+
for proxy in json_proxies(model):
|
|
100
|
+
if obj.type == proxy.TYPE:
|
|
101
|
+
return proxy
|
|
102
|
+
stack = list(model.__subclasses__())
|
|
103
|
+
while stack:
|
|
104
|
+
sub = stack.pop()
|
|
105
|
+
if sub._meta.proxy and sub.__dict__.get("TYPE") == obj.type:
|
|
106
|
+
return sub
|
|
107
|
+
stack.extend(sub.__subclasses__())
|
|
108
|
+
return model
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _reset_pk(obj):
|
|
112
|
+
for parent, link in obj._meta.parents.items():
|
|
113
|
+
setattr(obj, parent._meta.pk.attname, None)
|
|
114
|
+
if link:
|
|
115
|
+
setattr(obj, link.attname, None)
|
|
116
|
+
obj.pk = None
|
|
117
|
+
obj._state.adding = True
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _foreign_keys(obj):
|
|
121
|
+
return [
|
|
122
|
+
field
|
|
123
|
+
for field in obj._meta.concrete_fields
|
|
124
|
+
if field.is_relation and (field.many_to_one or field.one_to_one)
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _is_slug_clash(exc):
|
|
129
|
+
if not hasattr(exc, "error_dict"):
|
|
130
|
+
return False
|
|
131
|
+
for name in ("slug", "path"):
|
|
132
|
+
if any(error.code == "unique" for error in exc.error_dict.get(name, ())):
|
|
133
|
+
return True
|
|
134
|
+
return any(
|
|
135
|
+
error.code in {"unique", "unique_together"}
|
|
136
|
+
and "slug" in (error.params or {}).get("unique_check", ())
|
|
137
|
+
for error in exc.error_dict.get(NON_FIELD_ERRORS, ())
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _message(exc):
|
|
142
|
+
if isinstance(exc, ValidationError):
|
|
143
|
+
if hasattr(exc, "error_dict"):
|
|
144
|
+
return "; ".join(
|
|
145
|
+
f"{field}: {' '.join(messages)}"
|
|
146
|
+
for field, messages in exc.message_dict.items()
|
|
147
|
+
)
|
|
148
|
+
return " ".join(exc.messages)
|
|
149
|
+
return str(exc)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@dataclasses.dataclass(eq=False)
|
|
153
|
+
class Entry:
|
|
154
|
+
unit: object
|
|
155
|
+
model: type
|
|
156
|
+
config: object
|
|
157
|
+
deserialized: list
|
|
158
|
+
status: str | None = None
|
|
159
|
+
reason: str = ""
|
|
160
|
+
action: str = ""
|
|
161
|
+
original: object = None
|
|
162
|
+
existing: object = None
|
|
163
|
+
# The object receiving the content: the copy, the existing translation,
|
|
164
|
+
# or an object with a region per language itself.
|
|
165
|
+
target: object = None
|
|
166
|
+
# Set for pages whose parent isn't part of the run.
|
|
167
|
+
parent: object = None
|
|
168
|
+
notes: list = dataclasses.field(default_factory=list)
|
|
169
|
+
saved: list = dataclasses.field(default_factory=list)
|
|
170
|
+
deferred: list = dataclasses.field(default_factory=list)
|
|
171
|
+
child_copies: dict = dataclasses.field(default_factory=dict)
|
|
172
|
+
done: bool = False
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def key(self):
|
|
176
|
+
return self.unit.key
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def is_regions(self):
|
|
180
|
+
return self.unit.kind == REGIONS
|
|
181
|
+
|
|
182
|
+
def fail(self, reason):
|
|
183
|
+
self.action = FAIL
|
|
184
|
+
self.reason = reason
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class Run:
|
|
188
|
+
def __init__(
|
|
189
|
+
self,
|
|
190
|
+
objects,
|
|
191
|
+
*,
|
|
192
|
+
target,
|
|
193
|
+
statuses=None,
|
|
194
|
+
replace=False,
|
|
195
|
+
activate=False,
|
|
196
|
+
into=None,
|
|
197
|
+
parent=None,
|
|
198
|
+
registry=None,
|
|
199
|
+
):
|
|
200
|
+
self.registry = registry or default_registry()
|
|
201
|
+
try:
|
|
202
|
+
self.units = group_units(objects, self.registry)
|
|
203
|
+
self.source = source_language(self.units)
|
|
204
|
+
except DumpError as exc:
|
|
205
|
+
raise CommandError(str(exc)) from exc
|
|
206
|
+
self.target = target
|
|
207
|
+
if self.target == self.source:
|
|
208
|
+
raise CommandError(
|
|
209
|
+
f"The target language is the source language ({self.source})."
|
|
210
|
+
f" Copying without translating is out of scope."
|
|
211
|
+
)
|
|
212
|
+
self.statuses = statuses
|
|
213
|
+
self.dry_run = False
|
|
214
|
+
self.replace = replace
|
|
215
|
+
self.activate = activate
|
|
216
|
+
self.into = into or {}
|
|
217
|
+
self.parent_pk = parent
|
|
218
|
+
self.problems = {
|
|
219
|
+
"references": [],
|
|
220
|
+
"regions": [],
|
|
221
|
+
"translated_fields": [],
|
|
222
|
+
"live": [],
|
|
223
|
+
"static_paths": [],
|
|
224
|
+
}
|
|
225
|
+
self._translation_cache = {}
|
|
226
|
+
self._object_cache = {}
|
|
227
|
+
|
|
228
|
+
deserialized = list(
|
|
229
|
+
serializers.deserialize("json", json.dumps(objects), ignorenonexistent=True)
|
|
230
|
+
)
|
|
231
|
+
by_key = {
|
|
232
|
+
unit_key(raw["model"], raw["pk"]): ds
|
|
233
|
+
for raw, ds in zip(objects, deserialized)
|
|
234
|
+
}
|
|
235
|
+
self.entries = {}
|
|
236
|
+
# (label, pk) of a plugin or related object -> its entry
|
|
237
|
+
self.child_entries = {}
|
|
238
|
+
for unit in self.units.values():
|
|
239
|
+
config = self.registry.translatable.get(
|
|
240
|
+
unit.label
|
|
241
|
+
) or self.registry.language_regions.get(unit.label)
|
|
242
|
+
entry = Entry(
|
|
243
|
+
unit=unit,
|
|
244
|
+
model=apps.get_model(unit.label),
|
|
245
|
+
config=config,
|
|
246
|
+
deserialized=[
|
|
247
|
+
by_key[unit_key(r["model"], r["pk"])] for r in unit.objects()
|
|
248
|
+
],
|
|
249
|
+
)
|
|
250
|
+
if statuses is not None:
|
|
251
|
+
state = statuses.get(unit.key, {})
|
|
252
|
+
entry.status = state.get("status", "untranslated")
|
|
253
|
+
entry.reason = state.get("reason", "")
|
|
254
|
+
self.entries[unit.key] = entry
|
|
255
|
+
for raw in unit.children:
|
|
256
|
+
self.child_entries[(raw["model"], raw["pk"])] = entry
|
|
257
|
+
|
|
258
|
+
# Lookups ################################################################
|
|
259
|
+
|
|
260
|
+
def _get(self, model, pk):
|
|
261
|
+
key = (label(model), pk)
|
|
262
|
+
if key not in self._object_cache:
|
|
263
|
+
self._object_cache[key] = model._base_manager.filter(pk=pk).first()
|
|
264
|
+
return self._object_cache[key]
|
|
265
|
+
|
|
266
|
+
def _primary_pk(self, obj):
|
|
267
|
+
"""The first-language object ``translation_of`` points at"""
|
|
268
|
+
if obj.language_code == first_language():
|
|
269
|
+
return obj.pk
|
|
270
|
+
return obj.translation_of_id
|
|
271
|
+
|
|
272
|
+
def translation(self, model, obj):
|
|
273
|
+
"""The existing translation of ``obj`` in the target language"""
|
|
274
|
+
key = (label(model), obj.pk)
|
|
275
|
+
if key in self._translation_cache:
|
|
276
|
+
return self._translation_cache[key]
|
|
277
|
+
result = None
|
|
278
|
+
if has_field(model, "translation_of") and has_field(model, "language_code"):
|
|
279
|
+
primary = self._primary_pk(obj)
|
|
280
|
+
if primary is None:
|
|
281
|
+
result = None
|
|
282
|
+
elif self.target == first_language():
|
|
283
|
+
result = (
|
|
284
|
+
model._default_manager.filter(pk=primary).first()
|
|
285
|
+
if primary != obj.pk
|
|
286
|
+
else None
|
|
287
|
+
)
|
|
288
|
+
else:
|
|
289
|
+
result = model._default_manager.filter(
|
|
290
|
+
language_code=self.target, translation_of=primary
|
|
291
|
+
).first()
|
|
292
|
+
if (
|
|
293
|
+
result is None
|
|
294
|
+
and self._is_tree(model)
|
|
295
|
+
and getattr(obj, "parent_id", True) is None
|
|
296
|
+
and getattr(obj, "language_code", None) == self.source
|
|
297
|
+
):
|
|
298
|
+
result = self._target_root(model, obj)
|
|
299
|
+
self._translation_cache[key] = result
|
|
300
|
+
return result
|
|
301
|
+
|
|
302
|
+
def _is_tree(self, model):
|
|
303
|
+
config = self.registry.translatable.get(label(model))
|
|
304
|
+
return bool(config and config.tree)
|
|
305
|
+
|
|
306
|
+
def _scope(self, model, obj):
|
|
307
|
+
return {
|
|
308
|
+
field: getattr(obj, field)
|
|
309
|
+
for field in path_scope(model)
|
|
310
|
+
if field != "language_code"
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
def _target_root(self, model, source_root):
|
|
314
|
+
roots = list(
|
|
315
|
+
model._default_manager.filter(
|
|
316
|
+
parent=None,
|
|
317
|
+
language_code=self.target,
|
|
318
|
+
**self._scope(model, source_root),
|
|
319
|
+
)[:2]
|
|
320
|
+
)
|
|
321
|
+
return roots[0] if len(roots) == 1 else None
|
|
322
|
+
|
|
323
|
+
# Decisions ##############################################################
|
|
324
|
+
|
|
325
|
+
def decide(self):
|
|
326
|
+
"""Decide what happens to each unit; raise if the run can't go on"""
|
|
327
|
+
stops = []
|
|
328
|
+
for entry in self.entries.values():
|
|
329
|
+
if entry.is_regions:
|
|
330
|
+
self._decide_regions(entry)
|
|
331
|
+
else:
|
|
332
|
+
self._decide_object(entry)
|
|
333
|
+
|
|
334
|
+
for entry in self.entries.values():
|
|
335
|
+
if entry.action == CREATE and entry.config.tree:
|
|
336
|
+
try:
|
|
337
|
+
self._decide_parent(entry)
|
|
338
|
+
except Failed as exc:
|
|
339
|
+
stops.append(str(exc))
|
|
340
|
+
if stops:
|
|
341
|
+
raise CommandError("\n".join(stops))
|
|
342
|
+
|
|
343
|
+
if self.statuses is not None:
|
|
344
|
+
for entry in self.entries.values():
|
|
345
|
+
if entry.action not in {CREATE, REPLACE}:
|
|
346
|
+
continue
|
|
347
|
+
if entry.status == "failed":
|
|
348
|
+
entry.fail(f"Translating failed: {entry.reason}")
|
|
349
|
+
elif entry.status != "translated":
|
|
350
|
+
entry.fail(
|
|
351
|
+
"It wasn't translated, because the plan skipped it."
|
|
352
|
+
" Run the same selection again to translate it."
|
|
353
|
+
)
|
|
354
|
+
self._propagate_failures()
|
|
355
|
+
|
|
356
|
+
def _propagate_failures(self):
|
|
357
|
+
changed = True
|
|
358
|
+
while changed:
|
|
359
|
+
changed = False
|
|
360
|
+
for entry in self.entries.values():
|
|
361
|
+
if entry.action != CREATE or not entry.config.tree:
|
|
362
|
+
continue
|
|
363
|
+
parent = self.entries.get(
|
|
364
|
+
unit_key(entry.unit.label, entry.original.parent_id)
|
|
365
|
+
)
|
|
366
|
+
if parent and parent.action == FAIL:
|
|
367
|
+
entry.fail(f"Its parent {parent.original} failed.")
|
|
368
|
+
changed = True
|
|
369
|
+
|
|
370
|
+
def _decide_regions(self, entry):
|
|
371
|
+
obj = self._get(entry.model, entry.unit.pk)
|
|
372
|
+
entry.original = obj
|
|
373
|
+
if obj is None:
|
|
374
|
+
entry.fail("It doesn't exist in this database.")
|
|
375
|
+
return
|
|
376
|
+
keys = {region.key for region in obj.regions}
|
|
377
|
+
if missing := {self.source, self.target} - keys:
|
|
378
|
+
entry.fail(f"It has no region for {', '.join(sorted(missing))}.")
|
|
379
|
+
return
|
|
380
|
+
entry.target = obj
|
|
381
|
+
if self._has_target_region(obj):
|
|
382
|
+
if self.replace:
|
|
383
|
+
entry.action = REPLACE
|
|
384
|
+
else:
|
|
385
|
+
entry.action = SKIP
|
|
386
|
+
entry.reason = f"Its {self.target} region has plugins already."
|
|
387
|
+
else:
|
|
388
|
+
entry.action = CREATE
|
|
389
|
+
|
|
390
|
+
def _has_target_region(self, obj):
|
|
391
|
+
return any(
|
|
392
|
+
plugin._default_manager.filter(parent=obj, region=self.target).exists()
|
|
393
|
+
for plugin in plugin_models(type(obj))
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def _decide_object(self, entry):
|
|
397
|
+
model = entry.model
|
|
398
|
+
original = self._get(model, entry.unit.pk)
|
|
399
|
+
entry.original = original
|
|
400
|
+
if original is None:
|
|
401
|
+
entry.fail(
|
|
402
|
+
"The original doesn't exist in this database; was the dump"
|
|
403
|
+
" made from a different one?"
|
|
404
|
+
)
|
|
405
|
+
return
|
|
406
|
+
|
|
407
|
+
if (into := self.into.get((label(model), original.pk))) is not None:
|
|
408
|
+
existing = self._get(model, into)
|
|
409
|
+
if existing is None:
|
|
410
|
+
entry.fail(f"--into names {label(model)} {into}, which doesn't exist.")
|
|
411
|
+
return
|
|
412
|
+
else:
|
|
413
|
+
existing = self.translation(model, original)
|
|
414
|
+
if existing is not None and existing.pk == original.pk:
|
|
415
|
+
existing = None
|
|
416
|
+
|
|
417
|
+
if existing is not None:
|
|
418
|
+
entry.existing = existing
|
|
419
|
+
if self.replace:
|
|
420
|
+
entry.action = REPLACE
|
|
421
|
+
entry.target = existing
|
|
422
|
+
else:
|
|
423
|
+
entry.action = SKIP
|
|
424
|
+
entry.reason = f"Translated already: {_describe(existing)}."
|
|
425
|
+
return
|
|
426
|
+
|
|
427
|
+
if clash := self._application_clash(original):
|
|
428
|
+
entry.action = SKIP
|
|
429
|
+
entry.reason = (
|
|
430
|
+
f"The target language has an application page of this type"
|
|
431
|
+
f" already: {_describe(clash)}."
|
|
432
|
+
)
|
|
433
|
+
return
|
|
434
|
+
entry.action = CREATE
|
|
435
|
+
|
|
436
|
+
def _application_clash(self, page):
|
|
437
|
+
try:
|
|
438
|
+
from feincms3.applications import ApplicationType # noqa: PLC0415
|
|
439
|
+
except ImportError: # pragma: no cover
|
|
440
|
+
return None
|
|
441
|
+
type = getattr(page, "type", None)
|
|
442
|
+
if not isinstance(type, ApplicationType) or not has_field(
|
|
443
|
+
page.__class__, "app_namespace"
|
|
444
|
+
):
|
|
445
|
+
return None
|
|
446
|
+
return (
|
|
447
|
+
page._clash_candidates()
|
|
448
|
+
.filter(
|
|
449
|
+
app_namespace=type.app_namespace(page),
|
|
450
|
+
language_code=self.target,
|
|
451
|
+
)
|
|
452
|
+
.first()
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
def _decide_parent(self, entry):
|
|
456
|
+
original = entry.original
|
|
457
|
+
model = entry.model
|
|
458
|
+
if unit_key(entry.unit.label, original.parent_id) in self.entries:
|
|
459
|
+
return
|
|
460
|
+
if self.parent_pk is not None:
|
|
461
|
+
entry.parent = self._get(model, self.parent_pk)
|
|
462
|
+
if entry.parent is None:
|
|
463
|
+
raise Failed(f"--parent {self.parent_pk} doesn't exist.")
|
|
464
|
+
return
|
|
465
|
+
parent = self._get(model, original.parent_id)
|
|
466
|
+
if counterpart := self.translation(model, parent):
|
|
467
|
+
entry.parent = counterpart
|
|
468
|
+
return
|
|
469
|
+
raise Failed(
|
|
470
|
+
f"{_describe(original)} needs a {self.target} counterpart of its"
|
|
471
|
+
f" parent {_describe(parent)}. Translate that page first, or pass"
|
|
472
|
+
f" --parent <pk>."
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
# Plan ###################################################################
|
|
476
|
+
|
|
477
|
+
def plan(self):
|
|
478
|
+
self.decide()
|
|
479
|
+
objects = {}
|
|
480
|
+
chars = 0
|
|
481
|
+
for entry in self.entries.values():
|
|
482
|
+
item = {"action": entry.action}
|
|
483
|
+
if entry.reason:
|
|
484
|
+
item["reason"] = entry.reason
|
|
485
|
+
if entry.action in {CREATE, REPLACE}:
|
|
486
|
+
strings, _html = unit_strings(entry.unit)
|
|
487
|
+
item["characters"] = sum(len(text) for text in strings.values())
|
|
488
|
+
chars += item["characters"]
|
|
489
|
+
objects[entry.key] = item
|
|
490
|
+
plan = {
|
|
491
|
+
"feincms3_autotranslate_plan": 1,
|
|
492
|
+
"source": self.source,
|
|
493
|
+
"target": self.target,
|
|
494
|
+
"replace": self.replace,
|
|
495
|
+
"objects": objects,
|
|
496
|
+
}
|
|
497
|
+
count = dict.fromkeys((CREATE, REPLACE, SKIP, FAIL), 0)
|
|
498
|
+
for item in objects.values():
|
|
499
|
+
count[item["action"]] += 1
|
|
500
|
+
summary = (
|
|
501
|
+
f"To translate: {count[CREATE] + count[REPLACE]} objects"
|
|
502
|
+
f" ({count[CREATE]} to create, {count[REPLACE]} to replace),"
|
|
503
|
+
f" {chars} characters. Skipped: {count[SKIP]}. Failed: {count[FAIL]}."
|
|
504
|
+
)
|
|
505
|
+
return plan, summary
|
|
506
|
+
|
|
507
|
+
# Loading ################################################################
|
|
508
|
+
|
|
509
|
+
def load(self, *, dry_run=False):
|
|
510
|
+
self.decide()
|
|
511
|
+
order = self._order()
|
|
512
|
+
self.dry_run = dry_run
|
|
513
|
+
with transaction.atomic():
|
|
514
|
+
for entry in order:
|
|
515
|
+
if entry.action not in {CREATE, REPLACE}:
|
|
516
|
+
continue
|
|
517
|
+
try:
|
|
518
|
+
with transaction.atomic():
|
|
519
|
+
self._save(entry)
|
|
520
|
+
except (Failed, ValidationError, DatabaseError) as exc:
|
|
521
|
+
entry.fail(_message(exc))
|
|
522
|
+
entry.target = entry.original if entry.is_regions else None
|
|
523
|
+
entry.saved = []
|
|
524
|
+
entry.deferred = []
|
|
525
|
+
entry.child_copies = {}
|
|
526
|
+
self._propagate_failures()
|
|
527
|
+
else:
|
|
528
|
+
entry.done = True
|
|
529
|
+
self._fill_deferred()
|
|
530
|
+
self._map_text_references()
|
|
531
|
+
self._collect_problems()
|
|
532
|
+
log = self._log()
|
|
533
|
+
if dry_run:
|
|
534
|
+
transaction.set_rollback(True)
|
|
535
|
+
return log
|
|
536
|
+
|
|
537
|
+
def _order(self):
|
|
538
|
+
"""Units in an order where everything a unit needs exists before it"""
|
|
539
|
+
deps = {}
|
|
540
|
+
for entry in self.entries.values():
|
|
541
|
+
deps[entry] = set()
|
|
542
|
+
if entry.action not in {CREATE, REPLACE}:
|
|
543
|
+
continue
|
|
544
|
+
for ds in entry.deserialized:
|
|
545
|
+
is_owner = ds is entry.deserialized[0]
|
|
546
|
+
for field in _foreign_keys(ds.object):
|
|
547
|
+
value = getattr(ds.object, field.attname)
|
|
548
|
+
if value is None or field.name == "translation_of":
|
|
549
|
+
continue
|
|
550
|
+
hard = (
|
|
551
|
+
not field.null
|
|
552
|
+
or field.remote_field.parent_link
|
|
553
|
+
or (is_owner and entry.config.tree and field.name == "parent")
|
|
554
|
+
)
|
|
555
|
+
if not hard:
|
|
556
|
+
continue
|
|
557
|
+
related = field.related_model._meta.concrete_model
|
|
558
|
+
dep = self.entries.get(unit_key(label(related), value))
|
|
559
|
+
dep = dep or self.child_entries.get((label(related), value))
|
|
560
|
+
if dep is not None and dep is not entry and dep.action == CREATE:
|
|
561
|
+
deps[entry].add(dep)
|
|
562
|
+
|
|
563
|
+
order, state = [], {}
|
|
564
|
+
|
|
565
|
+
def visit(entry, path):
|
|
566
|
+
if state.get(entry) == "done":
|
|
567
|
+
return
|
|
568
|
+
if state.get(entry) == "visiting":
|
|
569
|
+
cycle = " -> ".join(e.key for e in [*path, entry])
|
|
570
|
+
raise CommandError(
|
|
571
|
+
f"These objects need each other to exist first: {cycle}."
|
|
572
|
+
f" Nothing has been loaded."
|
|
573
|
+
)
|
|
574
|
+
state[entry] = "visiting"
|
|
575
|
+
for dep in sorted(deps[entry], key=lambda e: e.key):
|
|
576
|
+
visit(dep, [*path, entry])
|
|
577
|
+
state[entry] = "done"
|
|
578
|
+
order.append(entry)
|
|
579
|
+
|
|
580
|
+
for entry in self.entries.values():
|
|
581
|
+
visit(entry, [])
|
|
582
|
+
return order
|
|
583
|
+
|
|
584
|
+
def counterpart(self, model, pk):
|
|
585
|
+
"""
|
|
586
|
+
``(pk, state)`` of what a reference to ``model`` ``pk`` becomes
|
|
587
|
+
|
|
588
|
+
``state`` is ``"copy"`` (an object of this run), ``"translation"``
|
|
589
|
+
(an existing translation), ``"pending"`` (an object of this run which
|
|
590
|
+
isn't saved yet) or ``"same"`` (the reference stays).
|
|
591
|
+
"""
|
|
592
|
+
model = model._meta.concrete_model
|
|
593
|
+
key = unit_key(label(model), pk)
|
|
594
|
+
if entry := self.entries.get(key):
|
|
595
|
+
if entry.is_regions:
|
|
596
|
+
return pk, "self"
|
|
597
|
+
if entry.target is not None and entry.target.pk is not None:
|
|
598
|
+
return entry.target.pk, "copy"
|
|
599
|
+
if entry.action == CREATE and not entry.done:
|
|
600
|
+
return None, "pending"
|
|
601
|
+
elif entry := self.child_entries.get((label(model), pk)):
|
|
602
|
+
if (new := entry.child_copies.get((label(model), pk))) is not None:
|
|
603
|
+
return new, "copy"
|
|
604
|
+
if entry.action in {CREATE, REPLACE} and not entry.done:
|
|
605
|
+
return None, "pending"
|
|
606
|
+
return pk, "same"
|
|
607
|
+
if (obj := self._get(model, pk)) and (
|
|
608
|
+
translation := self.translation(model, obj)
|
|
609
|
+
):
|
|
610
|
+
return translation.pk, "translation"
|
|
611
|
+
return pk, "same"
|
|
612
|
+
|
|
613
|
+
def _map_foreign_keys(self, entry, obj, *, skip=()):
|
|
614
|
+
"""Map the references of ``obj``, return the names of those changed"""
|
|
615
|
+
changed = set()
|
|
616
|
+
for field in _foreign_keys(obj):
|
|
617
|
+
if field.name in skip or field.name == "translation_of":
|
|
618
|
+
continue
|
|
619
|
+
if field.remote_field.parent_link:
|
|
620
|
+
continue
|
|
621
|
+
value = getattr(obj, field.attname)
|
|
622
|
+
if value is None:
|
|
623
|
+
continue
|
|
624
|
+
new, state = self.counterpart(field.related_model, value)
|
|
625
|
+
if state == "pending":
|
|
626
|
+
if field.null:
|
|
627
|
+
setattr(obj, field.attname, None)
|
|
628
|
+
entry.deferred.append((obj, field, value))
|
|
629
|
+
changed.add(field.name)
|
|
630
|
+
else:
|
|
631
|
+
self._reference_left(
|
|
632
|
+
entry, obj, field.name, field.related_model, value
|
|
633
|
+
)
|
|
634
|
+
elif state == "same":
|
|
635
|
+
self._reference_left(entry, obj, field.name, field.related_model, value)
|
|
636
|
+
else:
|
|
637
|
+
setattr(obj, field.attname, new)
|
|
638
|
+
if new != value:
|
|
639
|
+
changed.add(field.name)
|
|
640
|
+
return changed
|
|
641
|
+
|
|
642
|
+
def _reference_left(self, entry, obj, field_name, model, pk):
|
|
643
|
+
model = model._meta.concrete_model
|
|
644
|
+
target = self._get(model, pk)
|
|
645
|
+
if target is None:
|
|
646
|
+
return
|
|
647
|
+
# The text is put together in report(), once the copy is saved
|
|
648
|
+
where = (entry, f"{label(type(obj))}.{field_name}")
|
|
649
|
+
language = getattr(target, "language_code", None)
|
|
650
|
+
if language and language != self.target:
|
|
651
|
+
self.problems["references"].append((*where, _describe(target)))
|
|
652
|
+
if label(
|
|
653
|
+
model
|
|
654
|
+
) in self.registry.language_regions and not self._has_target_region(target):
|
|
655
|
+
self.problems["regions"].append(
|
|
656
|
+
(
|
|
657
|
+
*where,
|
|
658
|
+
f"{_describe(target)}, which has nothing in its {self.target} region",
|
|
659
|
+
)
|
|
660
|
+
)
|
|
661
|
+
for name, columns in translated_fields(model).items():
|
|
662
|
+
source_col = columns.get(self.source)
|
|
663
|
+
target_col = columns.get(self.target)
|
|
664
|
+
if (
|
|
665
|
+
source_col
|
|
666
|
+
and target_col
|
|
667
|
+
and getattr(target, source_col)
|
|
668
|
+
and not getattr(target, target_col)
|
|
669
|
+
):
|
|
670
|
+
self.problems["translated_fields"].append(
|
|
671
|
+
(
|
|
672
|
+
*where,
|
|
673
|
+
f"{_describe(target)}: {name} has no {self.target} text",
|
|
674
|
+
)
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
def _save(self, entry):
|
|
678
|
+
owner_ds, *children = entry.deserialized
|
|
679
|
+
if entry.is_regions:
|
|
680
|
+
if entry.action == REPLACE:
|
|
681
|
+
for plugin in plugin_models(entry.model):
|
|
682
|
+
plugin._default_manager.filter(
|
|
683
|
+
parent=entry.target, region=self.target
|
|
684
|
+
).delete()
|
|
685
|
+
elif entry.action == CREATE:
|
|
686
|
+
self._create_owner(entry, owner_ds)
|
|
687
|
+
else:
|
|
688
|
+
self._replace_owner(entry, owner_ds)
|
|
689
|
+
|
|
690
|
+
for ds in children:
|
|
691
|
+
obj = ds.object
|
|
692
|
+
old_pk = obj.pk
|
|
693
|
+
_reset_pk(obj)
|
|
694
|
+
if entry.is_regions:
|
|
695
|
+
obj.region = self.target
|
|
696
|
+
changed = self._map_foreign_keys(entry, obj)
|
|
697
|
+
obj.__class__ = _proxy_class(obj)
|
|
698
|
+
# Only what the run wrote is validated: a value copied as it is
|
|
699
|
+
# can't be fixed here, and the original has it already.
|
|
700
|
+
changed |= {
|
|
701
|
+
spec.field
|
|
702
|
+
for spec in field_specs(type(obj))
|
|
703
|
+
if getattr(obj, spec.field) not in ("", None)
|
|
704
|
+
}
|
|
705
|
+
if entry.is_regions:
|
|
706
|
+
changed.add("region")
|
|
707
|
+
obj.full_clean(
|
|
708
|
+
exclude=[f.name for f in obj._meta.fields if f.name not in changed]
|
|
709
|
+
)
|
|
710
|
+
obj.save()
|
|
711
|
+
entry.child_copies[(label(obj._meta.concrete_model), old_pk)] = obj.pk
|
|
712
|
+
entry.saved.append(ds)
|
|
713
|
+
|
|
714
|
+
def _rewrite_owner(self, entry, obj):
|
|
715
|
+
original = entry.original
|
|
716
|
+
model = entry.model
|
|
717
|
+
obj.language_code = self.target
|
|
718
|
+
if has_field(model, "is_active"):
|
|
719
|
+
obj.is_active = self.activate
|
|
720
|
+
for name, value in (entry.config.offline or {}).items():
|
|
721
|
+
setattr(obj, name, value)
|
|
722
|
+
if has_field(model, "translation_of"):
|
|
723
|
+
primary = self._primary_pk(original)
|
|
724
|
+
obj.translation_of_id = None if self.target == first_language() else primary
|
|
725
|
+
if entry.config.tree:
|
|
726
|
+
parent = entry.parent
|
|
727
|
+
if parent is None:
|
|
728
|
+
parent_entry = self.entries[unit_key(label(model), original.parent_id)]
|
|
729
|
+
parent = parent_entry.target or parent_entry.existing
|
|
730
|
+
if parent is None or parent.pk is None:
|
|
731
|
+
raise Failed("Its parent has no counterpart.")
|
|
732
|
+
obj.parent_id = parent.pk
|
|
733
|
+
# The parent instance has to be current for path computation.
|
|
734
|
+
obj.parent = model._default_manager.get(pk=parent.pk)
|
|
735
|
+
if getattr(obj, "static_path", False):
|
|
736
|
+
obj.static_path = False
|
|
737
|
+
self.problems["static_paths"].append(original)
|
|
738
|
+
|
|
739
|
+
def _create_owner(self, entry, ds):
|
|
740
|
+
obj = ds.object
|
|
741
|
+
_reset_pk(obj)
|
|
742
|
+
self._rewrite_owner(entry, obj)
|
|
743
|
+
entry.target = obj
|
|
744
|
+
self._map_foreign_keys(entry, obj, skip={"parent"} if entry.config.tree else ())
|
|
745
|
+
self._clean_with_slug(entry, obj)
|
|
746
|
+
obj.save()
|
|
747
|
+
entry.saved.append(ds)
|
|
748
|
+
|
|
749
|
+
def _clean_with_slug(self, entry, obj):
|
|
750
|
+
model = entry.model
|
|
751
|
+
base = None
|
|
752
|
+
if has_field(model, "slug") and (
|
|
753
|
+
title := getattr(obj, entry.config.title_field, "")
|
|
754
|
+
):
|
|
755
|
+
base = slugify(title)
|
|
756
|
+
max_length = model._meta.get_field("slug").max_length
|
|
757
|
+
if max_length:
|
|
758
|
+
base = base[:max_length].strip("-")
|
|
759
|
+
if not base:
|
|
760
|
+
obj.full_clean()
|
|
761
|
+
return
|
|
762
|
+
for number in range(1, 101):
|
|
763
|
+
suffix = "" if number == 1 else f"-{number}"
|
|
764
|
+
obj.slug = f"{base[: (max_length or len(base) + len(suffix)) - len(suffix)]}{suffix}"
|
|
765
|
+
try:
|
|
766
|
+
obj.full_clean()
|
|
767
|
+
except ValidationError as exc:
|
|
768
|
+
if not _is_slug_clash(exc):
|
|
769
|
+
raise
|
|
770
|
+
if not has_field(model, "translation_of"):
|
|
771
|
+
raise Failed(
|
|
772
|
+
f"The slug {obj.slug!r} is taken. Without translation_of"
|
|
773
|
+
f" that's the only hint of an earlier copy; delete it, or"
|
|
774
|
+
f" name it with --into {label(model)}:{entry.original.pk}=<pk>."
|
|
775
|
+
) from exc
|
|
776
|
+
continue
|
|
777
|
+
if number > 1:
|
|
778
|
+
entry.notes.append(f"The slug {base!r} was taken, it's {obj.slug!r}.")
|
|
779
|
+
return
|
|
780
|
+
raise Failed(f"No free slug for {base!r}.")
|
|
781
|
+
|
|
782
|
+
def _replace_owner(self, entry, ds):
|
|
783
|
+
translated = ds.object
|
|
784
|
+
existing = entry.existing
|
|
785
|
+
names = {spec.field for spec in field_specs(entry.model)}
|
|
786
|
+
# Only the content: references, slug, path, parent, position, menu
|
|
787
|
+
# and is_active are what an editor decided about the copy.
|
|
788
|
+
for name in names:
|
|
789
|
+
setattr(existing, name, getattr(translated, name))
|
|
790
|
+
existing.full_clean()
|
|
791
|
+
existing.save()
|
|
792
|
+
for plugin in plugin_models(entry.model):
|
|
793
|
+
plugin._default_manager.filter(parent=existing).delete()
|
|
794
|
+
for rel_model, name in entry.config.related:
|
|
795
|
+
rel_model._default_manager.filter(**{name: existing}).delete()
|
|
796
|
+
# The translated object stands in for the existing one from now on,
|
|
797
|
+
# so that the text references of its fields are mapped below.
|
|
798
|
+
entry.saved.append(_Saved(existing, ds.m2m_data))
|
|
799
|
+
|
|
800
|
+
def _fill_deferred(self):
|
|
801
|
+
for entry in self.entries.values():
|
|
802
|
+
for obj, field, value in entry.deferred:
|
|
803
|
+
new, state = self.counterpart(field.related_model, value)
|
|
804
|
+
if state in {"copy", "translation"}:
|
|
805
|
+
type(obj)._base_manager.filter(pk=obj.pk).update(
|
|
806
|
+
**{field.attname: new}
|
|
807
|
+
)
|
|
808
|
+
else:
|
|
809
|
+
self._reference_left(
|
|
810
|
+
entry, obj, field.name, field.related_model, value
|
|
811
|
+
)
|
|
812
|
+
for ds in entry.saved:
|
|
813
|
+
for name, pks in (ds.m2m_data or {}).items():
|
|
814
|
+
field = ds.object._meta.get_field(name)
|
|
815
|
+
mapped = []
|
|
816
|
+
for pk in pks:
|
|
817
|
+
new, state = self.counterpart(field.related_model, pk)
|
|
818
|
+
if state in {"copy", "translation", "self"}:
|
|
819
|
+
mapped.append(new)
|
|
820
|
+
else:
|
|
821
|
+
self._reference_left(
|
|
822
|
+
entry, ds.object, name, field.related_model, pk
|
|
823
|
+
)
|
|
824
|
+
mapped.append(pk)
|
|
825
|
+
getattr(ds.object, name).set(mapped)
|
|
826
|
+
|
|
827
|
+
# Text references ########################################################
|
|
828
|
+
|
|
829
|
+
@cached_property
|
|
830
|
+
def _flat_urls(self):
|
|
831
|
+
"""Paths of the source-language objects without a tree"""
|
|
832
|
+
result = {}
|
|
833
|
+
for config in self.registry.translatable.values():
|
|
834
|
+
if config.tree or not hasattr(config.model, "get_absolute_url"):
|
|
835
|
+
continue
|
|
836
|
+
for obj in config.model._default_manager.filter(language_code=self.source):
|
|
837
|
+
if parts := split_url(_absolute_url(obj)):
|
|
838
|
+
result.setdefault(parts[1], (config.model, obj.pk))
|
|
839
|
+
return result
|
|
840
|
+
|
|
841
|
+
def _resolve_path(self, path, context):
|
|
842
|
+
for config in self.registry.translatable.values():
|
|
843
|
+
if not config.tree:
|
|
844
|
+
continue
|
|
845
|
+
model = config.model
|
|
846
|
+
scope = {
|
|
847
|
+
field: getattr(context, field)
|
|
848
|
+
for field in path_scope(model)
|
|
849
|
+
if hasattr(context, field)
|
|
850
|
+
}
|
|
851
|
+
candidates = {path, f"{path}/"} if not path.endswith("/") else {path}
|
|
852
|
+
if page := model._default_manager.filter(
|
|
853
|
+
path__in=candidates, **scope
|
|
854
|
+
).first():
|
|
855
|
+
return model, page.pk
|
|
856
|
+
return self._flat_urls.get(path)
|
|
857
|
+
|
|
858
|
+
def _public_url(self, obj):
|
|
859
|
+
"""The URL of an object once it's active"""
|
|
860
|
+
if self._is_tree(type(obj)):
|
|
861
|
+
return page_location(obj)
|
|
862
|
+
restore = getattr(obj, "is_active", None)
|
|
863
|
+
if restore is not None:
|
|
864
|
+
obj.is_active = True
|
|
865
|
+
try:
|
|
866
|
+
url = obj.get_absolute_url()
|
|
867
|
+
except Exception: # noqa: BLE001
|
|
868
|
+
return None
|
|
869
|
+
finally:
|
|
870
|
+
if restore is not None:
|
|
871
|
+
obj.is_active = restore
|
|
872
|
+
parts = split_url(url) if url else None
|
|
873
|
+
if not parts or parts[1] in {"", "/"} or parts[1].startswith(reverse_admin()):
|
|
874
|
+
return None
|
|
875
|
+
return host_of(parts[0]), parts[1]
|
|
876
|
+
|
|
877
|
+
def _map_url(self, entry, obj, field_name, url):
|
|
878
|
+
parts = split_url(url)
|
|
879
|
+
if not parts:
|
|
880
|
+
return url
|
|
881
|
+
origin, path, rest = parts
|
|
882
|
+
resolved = self._resolve_path(path, entry.original)
|
|
883
|
+
if resolved is None:
|
|
884
|
+
return url
|
|
885
|
+
model, pk = resolved
|
|
886
|
+
new, state = self.counterpart(model, pk)
|
|
887
|
+
if state not in {"copy", "translation"}:
|
|
888
|
+
self._reference_left(entry, obj, field_name, model, pk)
|
|
889
|
+
return url
|
|
890
|
+
location = self._public_url(
|
|
891
|
+
self._get(model, new) or model._base_manager.get(pk=new)
|
|
892
|
+
)
|
|
893
|
+
if location is None:
|
|
894
|
+
self._reference_left(entry, obj, field_name, model, pk)
|
|
895
|
+
return url
|
|
896
|
+
host, new_path = location
|
|
897
|
+
own_host = (
|
|
898
|
+
(self._public_url(entry.target) or (None, None))[0]
|
|
899
|
+
if entry.target
|
|
900
|
+
else None
|
|
901
|
+
)
|
|
902
|
+
if host and host != own_host:
|
|
903
|
+
return f"https://{host}{new_path}{rest}"
|
|
904
|
+
if host is None and origin:
|
|
905
|
+
return f"{origin}{new_path}{rest}"
|
|
906
|
+
return f"{new_path}{rest}"
|
|
907
|
+
|
|
908
|
+
def _map_text_references(self):
|
|
909
|
+
for entry in self.entries.values():
|
|
910
|
+
if not entry.done:
|
|
911
|
+
continue
|
|
912
|
+
for ds in entry.saved:
|
|
913
|
+
obj = ds.object
|
|
914
|
+
self._object_cache.pop((label(obj._meta.concrete_model), obj.pk), None)
|
|
915
|
+
specs = field_specs(type(obj))
|
|
916
|
+
fields = {spec.field: getattr(obj, spec.field) for spec in specs}
|
|
917
|
+
changed = set()
|
|
918
|
+
for spec in specs:
|
|
919
|
+
if spec.kind not in {HTML, LINK}:
|
|
920
|
+
continue
|
|
921
|
+
for _concrete, container, key in iter_path(fields, spec.path):
|
|
922
|
+
value = container[key]
|
|
923
|
+
if not isinstance(value, str) or not value:
|
|
924
|
+
continue
|
|
925
|
+
new = (
|
|
926
|
+
self._map_html(entry, obj, spec.field, value)
|
|
927
|
+
if spec.kind == HTML
|
|
928
|
+
else self._map_url(entry, obj, spec.field, value)
|
|
929
|
+
)
|
|
930
|
+
if new != value:
|
|
931
|
+
container[key] = new
|
|
932
|
+
changed.add(spec.field)
|
|
933
|
+
changed |= self._map_json_foreign_keys(entry, obj, fields)
|
|
934
|
+
if changed:
|
|
935
|
+
for name in changed:
|
|
936
|
+
setattr(obj, name, fields[name])
|
|
937
|
+
type(obj)._base_manager.filter(pk=obj.pk).update(
|
|
938
|
+
**{name: fields[name] for name in changed}
|
|
939
|
+
)
|
|
940
|
+
|
|
941
|
+
def _map_html(self, entry, obj, field_name, text):
|
|
942
|
+
return map_html(text, lambda url: self._map_url(entry, obj, field_name, url))
|
|
943
|
+
|
|
944
|
+
def _map_json_foreign_keys(self, entry, obj, fields):
|
|
945
|
+
paths_by_model = getattr(type(obj), "_proxy_types_foreign_key_paths", {}).get(
|
|
946
|
+
getattr(obj, "type", None), {}
|
|
947
|
+
)
|
|
948
|
+
if not paths_by_model:
|
|
949
|
+
return set()
|
|
950
|
+
fields.setdefault("data", obj.data)
|
|
951
|
+
changed = set()
|
|
952
|
+
for model_label, paths in paths_by_model.items():
|
|
953
|
+
model = apps.get_model(model_label)
|
|
954
|
+
for path in paths:
|
|
955
|
+
for _concrete, container, key in iter_path(
|
|
956
|
+
{"data": fields["data"]}, f"data.{_jmes_to_path(path)}"
|
|
957
|
+
):
|
|
958
|
+
value = container[key]
|
|
959
|
+
if value in (None, ""):
|
|
960
|
+
continue
|
|
961
|
+
new, state = self.counterpart(model, value)
|
|
962
|
+
if state in {"copy", "translation"} and new != value:
|
|
963
|
+
container[key] = (
|
|
964
|
+
type(value)(new) if isinstance(value, str) else new
|
|
965
|
+
)
|
|
966
|
+
changed.add("data")
|
|
967
|
+
elif state == "same":
|
|
968
|
+
self._reference_left(entry, obj, f"data.{path}", model, value)
|
|
969
|
+
return changed
|
|
970
|
+
|
|
971
|
+
# Reporting ##############################################################
|
|
972
|
+
|
|
973
|
+
def _collect_problems(self):
|
|
974
|
+
for entry in self.entries.values():
|
|
975
|
+
if not entry.is_regions or entry.action not in {CREATE, REPLACE}:
|
|
976
|
+
continue
|
|
977
|
+
if not entry.done:
|
|
978
|
+
continue
|
|
979
|
+
for config in self.registry.translatable.values():
|
|
980
|
+
for plugin in plugin_models(config.model):
|
|
981
|
+
for field in _foreign_keys(plugin._meta.model):
|
|
982
|
+
if field.related_model is not entry.model:
|
|
983
|
+
continue
|
|
984
|
+
filters = {field.name: entry.target}
|
|
985
|
+
filters["parent__language_code"] = self.target
|
|
986
|
+
if has_field(config.model, "is_active"):
|
|
987
|
+
filters["parent__is_active"] = True
|
|
988
|
+
for p in plugin._default_manager.filter(
|
|
989
|
+
**filters
|
|
990
|
+
).select_related("parent"):
|
|
991
|
+
self.problems["live"].append(
|
|
992
|
+
f"{_describe(entry.target)} is live on {_describe(p.parent)}"
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
def _log(self):
|
|
996
|
+
objects = []
|
|
997
|
+
for entry in self.entries.values():
|
|
998
|
+
item = {
|
|
999
|
+
"model": label(entry.model),
|
|
1000
|
+
"original": entry.unit.pk,
|
|
1001
|
+
"status": LOG_STATUS[entry.action],
|
|
1002
|
+
}
|
|
1003
|
+
if entry.is_regions:
|
|
1004
|
+
item["region"] = self.target
|
|
1005
|
+
target = (
|
|
1006
|
+
entry.target if entry.action in {CREATE, REPLACE} else entry.existing
|
|
1007
|
+
)
|
|
1008
|
+
if target is not None and target.pk is not None and not self.dry_run:
|
|
1009
|
+
item["copy"] = target.pk
|
|
1010
|
+
if target is not None and target.pk is not None:
|
|
1011
|
+
item["location"] = location(target)
|
|
1012
|
+
if not self.dry_run or entry.action == SKIP:
|
|
1013
|
+
item["admin_url"] = admin_url(target)
|
|
1014
|
+
if entry.reason:
|
|
1015
|
+
item["reason"] = entry.reason
|
|
1016
|
+
objects.append(item)
|
|
1017
|
+
return {
|
|
1018
|
+
"feincms3_autotranslate_log": 1,
|
|
1019
|
+
"dry_run": self.dry_run,
|
|
1020
|
+
"activate": self.activate,
|
|
1021
|
+
"source": self.source,
|
|
1022
|
+
"target": self.target,
|
|
1023
|
+
"objects": objects,
|
|
1024
|
+
# Shared objects with a region per language, live on loading
|
|
1025
|
+
"live": sorted(set(self.problems["live"])),
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
def report(self):
|
|
1029
|
+
lines = []
|
|
1030
|
+
|
|
1031
|
+
def section(title, items):
|
|
1032
|
+
if items:
|
|
1033
|
+
lines.append(f"\n{title}")
|
|
1034
|
+
lines.extend(f" {item}" for item in items)
|
|
1035
|
+
|
|
1036
|
+
section(
|
|
1037
|
+
"GOES LIVE AT ONCE (shared objects with a region per language):",
|
|
1038
|
+
sorted(set(self.problems["live"])),
|
|
1039
|
+
)
|
|
1040
|
+
warnings = []
|
|
1041
|
+
if self.activate and any(
|
|
1042
|
+
e.action == CREATE and has_field(e.model, "is_active")
|
|
1043
|
+
for e in self.entries.values()
|
|
1044
|
+
):
|
|
1045
|
+
warnings.append(
|
|
1046
|
+
"--activate: the new copies are active and visible as soon as"
|
|
1047
|
+
" they're loaded. Translations which existed already keep their"
|
|
1048
|
+
" is_active, and offline={...} of the datasets still applies."
|
|
1049
|
+
)
|
|
1050
|
+
for config in self.registry.translatable.values():
|
|
1051
|
+
model = config.model
|
|
1052
|
+
if not any(e.model is model for e in self.entries.values()):
|
|
1053
|
+
continue
|
|
1054
|
+
if not has_field(model, "translation_of"):
|
|
1055
|
+
warnings.append(
|
|
1056
|
+
f"{label(model)} has no translation_of: earlier copies can't"
|
|
1057
|
+
f" be recognised, running again creates another copy."
|
|
1058
|
+
+ (
|
|
1059
|
+
""
|
|
1060
|
+
if has_unique_slug(model)
|
|
1061
|
+
else " It has no unique slug either, so not even a clash"
|
|
1062
|
+
" would show a duplicate."
|
|
1063
|
+
)
|
|
1064
|
+
)
|
|
1065
|
+
if self.dry_run:
|
|
1066
|
+
warnings.append(
|
|
1067
|
+
"Signal handlers ran in this dry run too; only the database"
|
|
1068
|
+
" writes have been rolled back."
|
|
1069
|
+
)
|
|
1070
|
+
section("Warnings:", warnings)
|
|
1071
|
+
|
|
1072
|
+
def describe(entry):
|
|
1073
|
+
target = entry.target or entry.existing
|
|
1074
|
+
text = f"{label(entry.model)} {entry.original or entry.unit.pk}"
|
|
1075
|
+
if entry.original is not None and (where := location(entry.original)):
|
|
1076
|
+
text += f" {where}"
|
|
1077
|
+
if target is not None and target.pk is not None and not entry.is_regions:
|
|
1078
|
+
text += f" -> {location(target)}"
|
|
1079
|
+
# The copies of a dry run are gone, and their pks with them
|
|
1080
|
+
if not self.dry_run or entry.action == SKIP:
|
|
1081
|
+
text += f" {admin_url(target)}"
|
|
1082
|
+
if entry.is_regions:
|
|
1083
|
+
text += f" (region {self.target})"
|
|
1084
|
+
return text
|
|
1085
|
+
|
|
1086
|
+
for action, title in [
|
|
1087
|
+
(
|
|
1088
|
+
CREATE,
|
|
1089
|
+
"Created:" if not self.dry_run else "To create:",
|
|
1090
|
+
),
|
|
1091
|
+
(
|
|
1092
|
+
REPLACE,
|
|
1093
|
+
"Replaced:" if not self.dry_run else "To replace:",
|
|
1094
|
+
),
|
|
1095
|
+
]:
|
|
1096
|
+
section(
|
|
1097
|
+
title,
|
|
1098
|
+
[
|
|
1099
|
+
describe(e) + "".join(f"\n {note}" for note in e.notes)
|
|
1100
|
+
for e in self.entries.values()
|
|
1101
|
+
if e.action == action
|
|
1102
|
+
],
|
|
1103
|
+
)
|
|
1104
|
+
section(
|
|
1105
|
+
"Skipped:",
|
|
1106
|
+
[
|
|
1107
|
+
f"{describe(e)}: {e.reason}"
|
|
1108
|
+
for e in self.entries.values()
|
|
1109
|
+
if e.action == SKIP
|
|
1110
|
+
],
|
|
1111
|
+
)
|
|
1112
|
+
section(
|
|
1113
|
+
"FAILED:",
|
|
1114
|
+
[
|
|
1115
|
+
f"{describe(e)}: {e.reason}"
|
|
1116
|
+
for e in self.entries.values()
|
|
1117
|
+
if e.action == FAIL
|
|
1118
|
+
],
|
|
1119
|
+
)
|
|
1120
|
+
section(
|
|
1121
|
+
"Static paths, set one again where it's wanted:",
|
|
1122
|
+
[f"{_describe(page)}" for page in self.problems["static_paths"]],
|
|
1123
|
+
)
|
|
1124
|
+
section(
|
|
1125
|
+
f"References left in another language than {self.target}:",
|
|
1126
|
+
self._references("references"),
|
|
1127
|
+
)
|
|
1128
|
+
section(
|
|
1129
|
+
f"Referenced objects with a region per language, without {self.target} plugins:",
|
|
1130
|
+
self._references("regions"),
|
|
1131
|
+
)
|
|
1132
|
+
section(
|
|
1133
|
+
f"Referenced objects with empty {self.target} fields:",
|
|
1134
|
+
self._references("translated_fields"),
|
|
1135
|
+
)
|
|
1136
|
+
return "\n".join(lines).strip() + "\n"
|
|
1137
|
+
|
|
1138
|
+
def _references(self, key):
|
|
1139
|
+
"""Problems found on references, of the objects which were loaded"""
|
|
1140
|
+
lines = set()
|
|
1141
|
+
for entry, field, text in self.problems[key]:
|
|
1142
|
+
if entry.action == FAIL:
|
|
1143
|
+
continue
|
|
1144
|
+
target = entry.target if entry.target and entry.target.pk else None
|
|
1145
|
+
lines.add(f"{_describe(target or entry.original)}: {field} -> {text}")
|
|
1146
|
+
return sorted(lines)
|
|
1147
|
+
|
|
1148
|
+
@property
|
|
1149
|
+
def failed(self):
|
|
1150
|
+
return [e for e in self.entries.values() if e.action == FAIL]
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
@dataclasses.dataclass
|
|
1154
|
+
class _Saved:
|
|
1155
|
+
object: object
|
|
1156
|
+
m2m_data: dict
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
def _absolute_url(obj):
|
|
1160
|
+
try:
|
|
1161
|
+
return obj.get_absolute_url()
|
|
1162
|
+
except Exception: # noqa: BLE001
|
|
1163
|
+
# Whatever breaks here, the object is then just not linkable.
|
|
1164
|
+
return None
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def reverse_admin():
|
|
1168
|
+
try:
|
|
1169
|
+
return reverse("admin:index")
|
|
1170
|
+
except NoReverseMatch:
|
|
1171
|
+
return "/\0"
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _jmes_to_path(path):
|
|
1175
|
+
"""``items[*].page`` -> ``items[].page``; only this subset is supported"""
|
|
1176
|
+
return path.replace("[*]", "[]")
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
# Reading the input files ####################################################
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
def read_translated(path_or_data):
|
|
1183
|
+
if isinstance(path_or_data, dict):
|
|
1184
|
+
data = path_or_data
|
|
1185
|
+
else:
|
|
1186
|
+
with open(path_or_data, encoding="utf-8") as f:
|
|
1187
|
+
data = json.load(f)
|
|
1188
|
+
if data.get("feincms3_autotranslate") != 1:
|
|
1189
|
+
raise CommandError(
|
|
1190
|
+
f"{path_or_data} isn't a translated file. Dumps go through"
|
|
1191
|
+
f" f3autotranslate first (or f3autotranslateload --plan)."
|
|
1192
|
+
)
|
|
1193
|
+
return data
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def parse_into(values):
|
|
1197
|
+
"""``label:pk=pk`` pairs and load logs into ``{(label, pk): pk}``"""
|
|
1198
|
+
result = {}
|
|
1199
|
+
for value in values or ():
|
|
1200
|
+
if ":" in value and "=" in value and not value.endswith(".json"):
|
|
1201
|
+
model, _sep, pair = value.partition(":")
|
|
1202
|
+
original, _sep, copy = pair.partition("=")
|
|
1203
|
+
try:
|
|
1204
|
+
result[(model.lower(), int(original))] = int(copy)
|
|
1205
|
+
except ValueError:
|
|
1206
|
+
raise CommandError(f"Invalid --into {value!r}.") from None
|
|
1207
|
+
continue
|
|
1208
|
+
with open(value, encoding="utf-8") as f:
|
|
1209
|
+
log = json.load(f)
|
|
1210
|
+
if log.get("feincms3_autotranslate_log") != 1:
|
|
1211
|
+
raise CommandError(f"{value} isn't a load log.")
|
|
1212
|
+
for item in log["objects"]:
|
|
1213
|
+
if item.get("copy") is not None and "region" not in item:
|
|
1214
|
+
result[(item["model"], item["original"])] = item["copy"]
|
|
1215
|
+
return result
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
def _all_objects(dumps):
|
|
1219
|
+
try:
|
|
1220
|
+
return all_objects(dumps)
|
|
1221
|
+
except DumpError as exc:
|
|
1222
|
+
raise CommandError(str(exc)) from exc
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def run_from_dumps(dumps, *, target, **kwargs):
|
|
1226
|
+
return Run(_all_objects(dumps), target=target, **kwargs)
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
def run_from_translated(data, **kwargs):
|
|
1230
|
+
statuses = data.get("units", {})
|
|
1231
|
+
return Run(
|
|
1232
|
+
_all_objects(data["dumps"]),
|
|
1233
|
+
target=data["target"],
|
|
1234
|
+
statuses=statuses,
|
|
1235
|
+
**kwargs,
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
__all__ = [
|
|
1240
|
+
"CREATE",
|
|
1241
|
+
"FAIL",
|
|
1242
|
+
"OWNER",
|
|
1243
|
+
"REPLACE",
|
|
1244
|
+
"SKIP",
|
|
1245
|
+
"Run",
|
|
1246
|
+
"parse_into",
|
|
1247
|
+
"read_translated",
|
|
1248
|
+
"run_from_dumps",
|
|
1249
|
+
"run_from_translated",
|
|
1250
|
+
]
|