feincms3-api 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. feincms3_api/__init__.py +1 -0
  2. feincms3_api/adapters.py +303 -0
  3. feincms3_api/admin.py +93 -0
  4. feincms3_api/api.py +66 -0
  5. feincms3_api/apps.py +70 -0
  6. feincms3_api/auth.py +29 -0
  7. feincms3_api/cabinet.py +78 -0
  8. feincms3_api/codecs.py +495 -0
  9. feincms3_api/conf.py +31 -0
  10. feincms3_api/content.py +312 -0
  11. feincms3_api/errors.py +53 -0
  12. feincms3_api/forms.py +36 -0
  13. feincms3_api/formsets.py +135 -0
  14. feincms3_api/http.py +195 -0
  15. feincms3_api/incubator/__init__.py +6 -0
  16. feincms3_api/incubator/autotranslate/__init__.py +0 -0
  17. feincms3_api/incubator/autotranslate/apps.py +47 -0
  18. feincms3_api/incubator/autotranslate/backends/__init__.py +78 -0
  19. feincms3_api/incubator/autotranslate/backends/anthropic.py +98 -0
  20. feincms3_api/incubator/autotranslate/conf.py +59 -0
  21. feincms3_api/incubator/autotranslate/datasets.py +428 -0
  22. feincms3_api/incubator/autotranslate/fields.py +213 -0
  23. feincms3_api/incubator/autotranslate/links.py +122 -0
  24. feincms3_api/incubator/autotranslate/loader.py +1250 -0
  25. feincms3_api/incubator/autotranslate/management/__init__.py +0 -0
  26. feincms3_api/incubator/autotranslate/management/commands/__init__.py +0 -0
  27. feincms3_api/incubator/autotranslate/management/commands/f3autotranslate.py +162 -0
  28. feincms3_api/incubator/autotranslate/management/commands/f3autotranslateload.py +92 -0
  29. feincms3_api/incubator/autotranslate/review.py +104 -0
  30. feincms3_api/incubator/autotranslate/strings.py +313 -0
  31. feincms3_api/incubator/autotranslate/translate.py +260 -0
  32. feincms3_api/kinds.py +86 -0
  33. feincms3_api/management/__init__.py +0 -0
  34. feincms3_api/management/commands/__init__.py +0 -0
  35. feincms3_api/management/commands/f3api_cleanup.py +18 -0
  36. feincms3_api/management/commands/f3api_schema.py +32 -0
  37. feincms3_api/management/commands/f3api_token.py +39 -0
  38. feincms3_api/migrations/0001_initial.py +91 -0
  39. feincms3_api/migrations/0002_upload.py +61 -0
  40. feincms3_api/migrations/__init__.py +0 -0
  41. feincms3_api/models.py +189 -0
  42. feincms3_api/paths.py +43 -0
  43. feincms3_api/reading.py +120 -0
  44. feincms3_api/references.py +123 -0
  45. feincms3_api/resources.py +580 -0
  46. feincms3_api/schema.py +80 -0
  47. feincms3_api/snapshot.py +110 -0
  48. feincms3_api/templates/admin/feincms3_api/apitoken/created.html +19 -0
  49. feincms3_api/writing.py +255 -0
  50. feincms3_api-0.0.1.dist-info/METADATA +69 -0
  51. feincms3_api-0.0.1.dist-info/RECORD +53 -0
  52. feincms3_api-0.0.1.dist-info/WHEEL +4 -0
  53. feincms3_api-0.0.1.dist-info/licenses/LICENSE +29 -0
@@ -0,0 +1,1250 @@
1
+ """
2
+ Loading translated objects as inactive copies
3
+
4
+ Django's deserialization does the reading; what ``loaddata`` would do next,
5
+ saving with ``raw=True`` under the original primary keys, is left out.
6
+ Instead every object gets a fresh primary key, its references are mapped,
7
+ its own fields rewritten, and it goes through ``full_clean()`` and
8
+ ``save()``. Nothing but inserts, and replacing content with ``--replace``.
9
+
10
+ The same decisions answer ``--plan``, which only needs the dumps and the
11
+ target database, and nothing is written then.
12
+ """
13
+
14
+ import dataclasses
15
+ import json
16
+ from functools import cached_property
17
+
18
+ from django.apps import apps
19
+ from django.core import serializers
20
+ from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
21
+ from django.core.management.base import CommandError
22
+ from django.db import DatabaseError, transaction
23
+ from django.urls import NoReverseMatch, reverse
24
+ from django.utils.text import slugify
25
+
26
+ from feincms3_api.incubator.autotranslate.conf import first_language
27
+ from feincms3_api.incubator.autotranslate.datasets import (
28
+ _describe,
29
+ has_field,
30
+ has_unique_slug,
31
+ label,
32
+ plugin_models,
33
+ registry as default_registry,
34
+ translated_fields,
35
+ )
36
+ from feincms3_api.incubator.autotranslate.fields import (
37
+ HTML,
38
+ LINK,
39
+ field_specs,
40
+ json_proxies,
41
+ )
42
+ from feincms3_api.incubator.autotranslate.links import (
43
+ host_of,
44
+ map_html,
45
+ page_location,
46
+ path_scope,
47
+ split_url,
48
+ )
49
+ from feincms3_api.incubator.autotranslate.strings import (
50
+ OWNER,
51
+ REGIONS,
52
+ DumpError,
53
+ all_objects,
54
+ group_units,
55
+ source_language,
56
+ unit_key,
57
+ unit_strings,
58
+ )
59
+ from feincms3_api.paths import iter_path
60
+
61
+
62
+ CREATE = "create"
63
+ REPLACE = "replace"
64
+ SKIP = "skip"
65
+ FAIL = "fail"
66
+
67
+ LOG_STATUS = {CREATE: "created", REPLACE: "replaced", SKIP: "skipped", FAIL: "failed"}
68
+
69
+
70
+ class Failed(Exception):
71
+ pass
72
+
73
+
74
+ def admin_url(obj):
75
+ try:
76
+ return reverse(
77
+ f"admin:{obj._meta.app_label}_{obj._meta.model_name}_change",
78
+ args=[obj.pk],
79
+ )
80
+ except NoReverseMatch:
81
+ return ""
82
+
83
+
84
+ def location(obj):
85
+ """Path or slug, whatever says best where an object is"""
86
+ if path := getattr(obj, "path", None):
87
+ return path
88
+ return getattr(obj, "slug", None) or ""
89
+
90
+
91
+ def _proxy_class(obj):
92
+ """
93
+ The proxy class of a typed plugin (django-json-schema-editor,
94
+ feincms3-forms), whose ``save()`` needs its ``TYPE``
95
+ """
96
+ model = type(obj)
97
+ if not has_field(model, "type"):
98
+ return model
99
+ for proxy in json_proxies(model):
100
+ if obj.type == proxy.TYPE:
101
+ return proxy
102
+ stack = list(model.__subclasses__())
103
+ while stack:
104
+ sub = stack.pop()
105
+ if sub._meta.proxy and sub.__dict__.get("TYPE") == obj.type:
106
+ return sub
107
+ stack.extend(sub.__subclasses__())
108
+ return model
109
+
110
+
111
+ def _reset_pk(obj):
112
+ for parent, link in obj._meta.parents.items():
113
+ setattr(obj, parent._meta.pk.attname, None)
114
+ if link:
115
+ setattr(obj, link.attname, None)
116
+ obj.pk = None
117
+ obj._state.adding = True
118
+
119
+
120
+ def _foreign_keys(obj):
121
+ return [
122
+ field
123
+ for field in obj._meta.concrete_fields
124
+ if field.is_relation and (field.many_to_one or field.one_to_one)
125
+ ]
126
+
127
+
128
+ def _is_slug_clash(exc):
129
+ if not hasattr(exc, "error_dict"):
130
+ return False
131
+ for name in ("slug", "path"):
132
+ if any(error.code == "unique" for error in exc.error_dict.get(name, ())):
133
+ return True
134
+ return any(
135
+ error.code in {"unique", "unique_together"}
136
+ and "slug" in (error.params or {}).get("unique_check", ())
137
+ for error in exc.error_dict.get(NON_FIELD_ERRORS, ())
138
+ )
139
+
140
+
141
+ def _message(exc):
142
+ if isinstance(exc, ValidationError):
143
+ if hasattr(exc, "error_dict"):
144
+ return "; ".join(
145
+ f"{field}: {' '.join(messages)}"
146
+ for field, messages in exc.message_dict.items()
147
+ )
148
+ return " ".join(exc.messages)
149
+ return str(exc)
150
+
151
+
152
+ @dataclasses.dataclass(eq=False)
153
+ class Entry:
154
+ unit: object
155
+ model: type
156
+ config: object
157
+ deserialized: list
158
+ status: str | None = None
159
+ reason: str = ""
160
+ action: str = ""
161
+ original: object = None
162
+ existing: object = None
163
+ # The object receiving the content: the copy, the existing translation,
164
+ # or an object with a region per language itself.
165
+ target: object = None
166
+ # Set for pages whose parent isn't part of the run.
167
+ parent: object = None
168
+ notes: list = dataclasses.field(default_factory=list)
169
+ saved: list = dataclasses.field(default_factory=list)
170
+ deferred: list = dataclasses.field(default_factory=list)
171
+ child_copies: dict = dataclasses.field(default_factory=dict)
172
+ done: bool = False
173
+
174
+ @property
175
+ def key(self):
176
+ return self.unit.key
177
+
178
+ @property
179
+ def is_regions(self):
180
+ return self.unit.kind == REGIONS
181
+
182
+ def fail(self, reason):
183
+ self.action = FAIL
184
+ self.reason = reason
185
+
186
+
187
+ class Run:
188
+ def __init__(
189
+ self,
190
+ objects,
191
+ *,
192
+ target,
193
+ statuses=None,
194
+ replace=False,
195
+ activate=False,
196
+ into=None,
197
+ parent=None,
198
+ registry=None,
199
+ ):
200
+ self.registry = registry or default_registry()
201
+ try:
202
+ self.units = group_units(objects, self.registry)
203
+ self.source = source_language(self.units)
204
+ except DumpError as exc:
205
+ raise CommandError(str(exc)) from exc
206
+ self.target = target
207
+ if self.target == self.source:
208
+ raise CommandError(
209
+ f"The target language is the source language ({self.source})."
210
+ f" Copying without translating is out of scope."
211
+ )
212
+ self.statuses = statuses
213
+ self.dry_run = False
214
+ self.replace = replace
215
+ self.activate = activate
216
+ self.into = into or {}
217
+ self.parent_pk = parent
218
+ self.problems = {
219
+ "references": [],
220
+ "regions": [],
221
+ "translated_fields": [],
222
+ "live": [],
223
+ "static_paths": [],
224
+ }
225
+ self._translation_cache = {}
226
+ self._object_cache = {}
227
+
228
+ deserialized = list(
229
+ serializers.deserialize("json", json.dumps(objects), ignorenonexistent=True)
230
+ )
231
+ by_key = {
232
+ unit_key(raw["model"], raw["pk"]): ds
233
+ for raw, ds in zip(objects, deserialized)
234
+ }
235
+ self.entries = {}
236
+ # (label, pk) of a plugin or related object -> its entry
237
+ self.child_entries = {}
238
+ for unit in self.units.values():
239
+ config = self.registry.translatable.get(
240
+ unit.label
241
+ ) or self.registry.language_regions.get(unit.label)
242
+ entry = Entry(
243
+ unit=unit,
244
+ model=apps.get_model(unit.label),
245
+ config=config,
246
+ deserialized=[
247
+ by_key[unit_key(r["model"], r["pk"])] for r in unit.objects()
248
+ ],
249
+ )
250
+ if statuses is not None:
251
+ state = statuses.get(unit.key, {})
252
+ entry.status = state.get("status", "untranslated")
253
+ entry.reason = state.get("reason", "")
254
+ self.entries[unit.key] = entry
255
+ for raw in unit.children:
256
+ self.child_entries[(raw["model"], raw["pk"])] = entry
257
+
258
+ # Lookups ################################################################
259
+
260
+ def _get(self, model, pk):
261
+ key = (label(model), pk)
262
+ if key not in self._object_cache:
263
+ self._object_cache[key] = model._base_manager.filter(pk=pk).first()
264
+ return self._object_cache[key]
265
+
266
+ def _primary_pk(self, obj):
267
+ """The first-language object ``translation_of`` points at"""
268
+ if obj.language_code == first_language():
269
+ return obj.pk
270
+ return obj.translation_of_id
271
+
272
+ def translation(self, model, obj):
273
+ """The existing translation of ``obj`` in the target language"""
274
+ key = (label(model), obj.pk)
275
+ if key in self._translation_cache:
276
+ return self._translation_cache[key]
277
+ result = None
278
+ if has_field(model, "translation_of") and has_field(model, "language_code"):
279
+ primary = self._primary_pk(obj)
280
+ if primary is None:
281
+ result = None
282
+ elif self.target == first_language():
283
+ result = (
284
+ model._default_manager.filter(pk=primary).first()
285
+ if primary != obj.pk
286
+ else None
287
+ )
288
+ else:
289
+ result = model._default_manager.filter(
290
+ language_code=self.target, translation_of=primary
291
+ ).first()
292
+ if (
293
+ result is None
294
+ and self._is_tree(model)
295
+ and getattr(obj, "parent_id", True) is None
296
+ and getattr(obj, "language_code", None) == self.source
297
+ ):
298
+ result = self._target_root(model, obj)
299
+ self._translation_cache[key] = result
300
+ return result
301
+
302
+ def _is_tree(self, model):
303
+ config = self.registry.translatable.get(label(model))
304
+ return bool(config and config.tree)
305
+
306
+ def _scope(self, model, obj):
307
+ return {
308
+ field: getattr(obj, field)
309
+ for field in path_scope(model)
310
+ if field != "language_code"
311
+ }
312
+
313
+ def _target_root(self, model, source_root):
314
+ roots = list(
315
+ model._default_manager.filter(
316
+ parent=None,
317
+ language_code=self.target,
318
+ **self._scope(model, source_root),
319
+ )[:2]
320
+ )
321
+ return roots[0] if len(roots) == 1 else None
322
+
323
+ # Decisions ##############################################################
324
+
325
+ def decide(self):
326
+ """Decide what happens to each unit; raise if the run can't go on"""
327
+ stops = []
328
+ for entry in self.entries.values():
329
+ if entry.is_regions:
330
+ self._decide_regions(entry)
331
+ else:
332
+ self._decide_object(entry)
333
+
334
+ for entry in self.entries.values():
335
+ if entry.action == CREATE and entry.config.tree:
336
+ try:
337
+ self._decide_parent(entry)
338
+ except Failed as exc:
339
+ stops.append(str(exc))
340
+ if stops:
341
+ raise CommandError("\n".join(stops))
342
+
343
+ if self.statuses is not None:
344
+ for entry in self.entries.values():
345
+ if entry.action not in {CREATE, REPLACE}:
346
+ continue
347
+ if entry.status == "failed":
348
+ entry.fail(f"Translating failed: {entry.reason}")
349
+ elif entry.status != "translated":
350
+ entry.fail(
351
+ "It wasn't translated, because the plan skipped it."
352
+ " Run the same selection again to translate it."
353
+ )
354
+ self._propagate_failures()
355
+
356
+ def _propagate_failures(self):
357
+ changed = True
358
+ while changed:
359
+ changed = False
360
+ for entry in self.entries.values():
361
+ if entry.action != CREATE or not entry.config.tree:
362
+ continue
363
+ parent = self.entries.get(
364
+ unit_key(entry.unit.label, entry.original.parent_id)
365
+ )
366
+ if parent and parent.action == FAIL:
367
+ entry.fail(f"Its parent {parent.original} failed.")
368
+ changed = True
369
+
370
+ def _decide_regions(self, entry):
371
+ obj = self._get(entry.model, entry.unit.pk)
372
+ entry.original = obj
373
+ if obj is None:
374
+ entry.fail("It doesn't exist in this database.")
375
+ return
376
+ keys = {region.key for region in obj.regions}
377
+ if missing := {self.source, self.target} - keys:
378
+ entry.fail(f"It has no region for {', '.join(sorted(missing))}.")
379
+ return
380
+ entry.target = obj
381
+ if self._has_target_region(obj):
382
+ if self.replace:
383
+ entry.action = REPLACE
384
+ else:
385
+ entry.action = SKIP
386
+ entry.reason = f"Its {self.target} region has plugins already."
387
+ else:
388
+ entry.action = CREATE
389
+
390
+ def _has_target_region(self, obj):
391
+ return any(
392
+ plugin._default_manager.filter(parent=obj, region=self.target).exists()
393
+ for plugin in plugin_models(type(obj))
394
+ )
395
+
396
+ def _decide_object(self, entry):
397
+ model = entry.model
398
+ original = self._get(model, entry.unit.pk)
399
+ entry.original = original
400
+ if original is None:
401
+ entry.fail(
402
+ "The original doesn't exist in this database; was the dump"
403
+ " made from a different one?"
404
+ )
405
+ return
406
+
407
+ if (into := self.into.get((label(model), original.pk))) is not None:
408
+ existing = self._get(model, into)
409
+ if existing is None:
410
+ entry.fail(f"--into names {label(model)} {into}, which doesn't exist.")
411
+ return
412
+ else:
413
+ existing = self.translation(model, original)
414
+ if existing is not None and existing.pk == original.pk:
415
+ existing = None
416
+
417
+ if existing is not None:
418
+ entry.existing = existing
419
+ if self.replace:
420
+ entry.action = REPLACE
421
+ entry.target = existing
422
+ else:
423
+ entry.action = SKIP
424
+ entry.reason = f"Translated already: {_describe(existing)}."
425
+ return
426
+
427
+ if clash := self._application_clash(original):
428
+ entry.action = SKIP
429
+ entry.reason = (
430
+ f"The target language has an application page of this type"
431
+ f" already: {_describe(clash)}."
432
+ )
433
+ return
434
+ entry.action = CREATE
435
+
436
+ def _application_clash(self, page):
437
+ try:
438
+ from feincms3.applications import ApplicationType # noqa: PLC0415
439
+ except ImportError: # pragma: no cover
440
+ return None
441
+ type = getattr(page, "type", None)
442
+ if not isinstance(type, ApplicationType) or not has_field(
443
+ page.__class__, "app_namespace"
444
+ ):
445
+ return None
446
+ return (
447
+ page._clash_candidates()
448
+ .filter(
449
+ app_namespace=type.app_namespace(page),
450
+ language_code=self.target,
451
+ )
452
+ .first()
453
+ )
454
+
455
+ def _decide_parent(self, entry):
456
+ original = entry.original
457
+ model = entry.model
458
+ if unit_key(entry.unit.label, original.parent_id) in self.entries:
459
+ return
460
+ if self.parent_pk is not None:
461
+ entry.parent = self._get(model, self.parent_pk)
462
+ if entry.parent is None:
463
+ raise Failed(f"--parent {self.parent_pk} doesn't exist.")
464
+ return
465
+ parent = self._get(model, original.parent_id)
466
+ if counterpart := self.translation(model, parent):
467
+ entry.parent = counterpart
468
+ return
469
+ raise Failed(
470
+ f"{_describe(original)} needs a {self.target} counterpart of its"
471
+ f" parent {_describe(parent)}. Translate that page first, or pass"
472
+ f" --parent <pk>."
473
+ )
474
+
475
+ # Plan ###################################################################
476
+
477
+ def plan(self):
478
+ self.decide()
479
+ objects = {}
480
+ chars = 0
481
+ for entry in self.entries.values():
482
+ item = {"action": entry.action}
483
+ if entry.reason:
484
+ item["reason"] = entry.reason
485
+ if entry.action in {CREATE, REPLACE}:
486
+ strings, _html = unit_strings(entry.unit)
487
+ item["characters"] = sum(len(text) for text in strings.values())
488
+ chars += item["characters"]
489
+ objects[entry.key] = item
490
+ plan = {
491
+ "feincms3_autotranslate_plan": 1,
492
+ "source": self.source,
493
+ "target": self.target,
494
+ "replace": self.replace,
495
+ "objects": objects,
496
+ }
497
+ count = dict.fromkeys((CREATE, REPLACE, SKIP, FAIL), 0)
498
+ for item in objects.values():
499
+ count[item["action"]] += 1
500
+ summary = (
501
+ f"To translate: {count[CREATE] + count[REPLACE]} objects"
502
+ f" ({count[CREATE]} to create, {count[REPLACE]} to replace),"
503
+ f" {chars} characters. Skipped: {count[SKIP]}. Failed: {count[FAIL]}."
504
+ )
505
+ return plan, summary
506
+
507
+ # Loading ################################################################
508
+
509
+ def load(self, *, dry_run=False):
510
+ self.decide()
511
+ order = self._order()
512
+ self.dry_run = dry_run
513
+ with transaction.atomic():
514
+ for entry in order:
515
+ if entry.action not in {CREATE, REPLACE}:
516
+ continue
517
+ try:
518
+ with transaction.atomic():
519
+ self._save(entry)
520
+ except (Failed, ValidationError, DatabaseError) as exc:
521
+ entry.fail(_message(exc))
522
+ entry.target = entry.original if entry.is_regions else None
523
+ entry.saved = []
524
+ entry.deferred = []
525
+ entry.child_copies = {}
526
+ self._propagate_failures()
527
+ else:
528
+ entry.done = True
529
+ self._fill_deferred()
530
+ self._map_text_references()
531
+ self._collect_problems()
532
+ log = self._log()
533
+ if dry_run:
534
+ transaction.set_rollback(True)
535
+ return log
536
+
537
+ def _order(self):
538
+ """Units in an order where everything a unit needs exists before it"""
539
+ deps = {}
540
+ for entry in self.entries.values():
541
+ deps[entry] = set()
542
+ if entry.action not in {CREATE, REPLACE}:
543
+ continue
544
+ for ds in entry.deserialized:
545
+ is_owner = ds is entry.deserialized[0]
546
+ for field in _foreign_keys(ds.object):
547
+ value = getattr(ds.object, field.attname)
548
+ if value is None or field.name == "translation_of":
549
+ continue
550
+ hard = (
551
+ not field.null
552
+ or field.remote_field.parent_link
553
+ or (is_owner and entry.config.tree and field.name == "parent")
554
+ )
555
+ if not hard:
556
+ continue
557
+ related = field.related_model._meta.concrete_model
558
+ dep = self.entries.get(unit_key(label(related), value))
559
+ dep = dep or self.child_entries.get((label(related), value))
560
+ if dep is not None and dep is not entry and dep.action == CREATE:
561
+ deps[entry].add(dep)
562
+
563
+ order, state = [], {}
564
+
565
+ def visit(entry, path):
566
+ if state.get(entry) == "done":
567
+ return
568
+ if state.get(entry) == "visiting":
569
+ cycle = " -> ".join(e.key for e in [*path, entry])
570
+ raise CommandError(
571
+ f"These objects need each other to exist first: {cycle}."
572
+ f" Nothing has been loaded."
573
+ )
574
+ state[entry] = "visiting"
575
+ for dep in sorted(deps[entry], key=lambda e: e.key):
576
+ visit(dep, [*path, entry])
577
+ state[entry] = "done"
578
+ order.append(entry)
579
+
580
+ for entry in self.entries.values():
581
+ visit(entry, [])
582
+ return order
583
+
584
+ def counterpart(self, model, pk):
585
+ """
586
+ ``(pk, state)`` of what a reference to ``model`` ``pk`` becomes
587
+
588
+ ``state`` is ``"copy"`` (an object of this run), ``"translation"``
589
+ (an existing translation), ``"pending"`` (an object of this run which
590
+ isn't saved yet) or ``"same"`` (the reference stays).
591
+ """
592
+ model = model._meta.concrete_model
593
+ key = unit_key(label(model), pk)
594
+ if entry := self.entries.get(key):
595
+ if entry.is_regions:
596
+ return pk, "self"
597
+ if entry.target is not None and entry.target.pk is not None:
598
+ return entry.target.pk, "copy"
599
+ if entry.action == CREATE and not entry.done:
600
+ return None, "pending"
601
+ elif entry := self.child_entries.get((label(model), pk)):
602
+ if (new := entry.child_copies.get((label(model), pk))) is not None:
603
+ return new, "copy"
604
+ if entry.action in {CREATE, REPLACE} and not entry.done:
605
+ return None, "pending"
606
+ return pk, "same"
607
+ if (obj := self._get(model, pk)) and (
608
+ translation := self.translation(model, obj)
609
+ ):
610
+ return translation.pk, "translation"
611
+ return pk, "same"
612
+
613
+ def _map_foreign_keys(self, entry, obj, *, skip=()):
614
+ """Map the references of ``obj``, return the names of those changed"""
615
+ changed = set()
616
+ for field in _foreign_keys(obj):
617
+ if field.name in skip or field.name == "translation_of":
618
+ continue
619
+ if field.remote_field.parent_link:
620
+ continue
621
+ value = getattr(obj, field.attname)
622
+ if value is None:
623
+ continue
624
+ new, state = self.counterpart(field.related_model, value)
625
+ if state == "pending":
626
+ if field.null:
627
+ setattr(obj, field.attname, None)
628
+ entry.deferred.append((obj, field, value))
629
+ changed.add(field.name)
630
+ else:
631
+ self._reference_left(
632
+ entry, obj, field.name, field.related_model, value
633
+ )
634
+ elif state == "same":
635
+ self._reference_left(entry, obj, field.name, field.related_model, value)
636
+ else:
637
+ setattr(obj, field.attname, new)
638
+ if new != value:
639
+ changed.add(field.name)
640
+ return changed
641
+
642
+ def _reference_left(self, entry, obj, field_name, model, pk):
643
+ model = model._meta.concrete_model
644
+ target = self._get(model, pk)
645
+ if target is None:
646
+ return
647
+ # The text is put together in report(), once the copy is saved
648
+ where = (entry, f"{label(type(obj))}.{field_name}")
649
+ language = getattr(target, "language_code", None)
650
+ if language and language != self.target:
651
+ self.problems["references"].append((*where, _describe(target)))
652
+ if label(
653
+ model
654
+ ) in self.registry.language_regions and not self._has_target_region(target):
655
+ self.problems["regions"].append(
656
+ (
657
+ *where,
658
+ f"{_describe(target)}, which has nothing in its {self.target} region",
659
+ )
660
+ )
661
+ for name, columns in translated_fields(model).items():
662
+ source_col = columns.get(self.source)
663
+ target_col = columns.get(self.target)
664
+ if (
665
+ source_col
666
+ and target_col
667
+ and getattr(target, source_col)
668
+ and not getattr(target, target_col)
669
+ ):
670
+ self.problems["translated_fields"].append(
671
+ (
672
+ *where,
673
+ f"{_describe(target)}: {name} has no {self.target} text",
674
+ )
675
+ )
676
+
677
+ def _save(self, entry):
678
+ owner_ds, *children = entry.deserialized
679
+ if entry.is_regions:
680
+ if entry.action == REPLACE:
681
+ for plugin in plugin_models(entry.model):
682
+ plugin._default_manager.filter(
683
+ parent=entry.target, region=self.target
684
+ ).delete()
685
+ elif entry.action == CREATE:
686
+ self._create_owner(entry, owner_ds)
687
+ else:
688
+ self._replace_owner(entry, owner_ds)
689
+
690
+ for ds in children:
691
+ obj = ds.object
692
+ old_pk = obj.pk
693
+ _reset_pk(obj)
694
+ if entry.is_regions:
695
+ obj.region = self.target
696
+ changed = self._map_foreign_keys(entry, obj)
697
+ obj.__class__ = _proxy_class(obj)
698
+ # Only what the run wrote is validated: a value copied as it is
699
+ # can't be fixed here, and the original has it already.
700
+ changed |= {
701
+ spec.field
702
+ for spec in field_specs(type(obj))
703
+ if getattr(obj, spec.field) not in ("", None)
704
+ }
705
+ if entry.is_regions:
706
+ changed.add("region")
707
+ obj.full_clean(
708
+ exclude=[f.name for f in obj._meta.fields if f.name not in changed]
709
+ )
710
+ obj.save()
711
+ entry.child_copies[(label(obj._meta.concrete_model), old_pk)] = obj.pk
712
+ entry.saved.append(ds)
713
+
714
+ def _rewrite_owner(self, entry, obj):
715
+ original = entry.original
716
+ model = entry.model
717
+ obj.language_code = self.target
718
+ if has_field(model, "is_active"):
719
+ obj.is_active = self.activate
720
+ for name, value in (entry.config.offline or {}).items():
721
+ setattr(obj, name, value)
722
+ if has_field(model, "translation_of"):
723
+ primary = self._primary_pk(original)
724
+ obj.translation_of_id = None if self.target == first_language() else primary
725
+ if entry.config.tree:
726
+ parent = entry.parent
727
+ if parent is None:
728
+ parent_entry = self.entries[unit_key(label(model), original.parent_id)]
729
+ parent = parent_entry.target or parent_entry.existing
730
+ if parent is None or parent.pk is None:
731
+ raise Failed("Its parent has no counterpart.")
732
+ obj.parent_id = parent.pk
733
+ # The parent instance has to be current for path computation.
734
+ obj.parent = model._default_manager.get(pk=parent.pk)
735
+ if getattr(obj, "static_path", False):
736
+ obj.static_path = False
737
+ self.problems["static_paths"].append(original)
738
+
739
+ def _create_owner(self, entry, ds):
740
+ obj = ds.object
741
+ _reset_pk(obj)
742
+ self._rewrite_owner(entry, obj)
743
+ entry.target = obj
744
+ self._map_foreign_keys(entry, obj, skip={"parent"} if entry.config.tree else ())
745
+ self._clean_with_slug(entry, obj)
746
+ obj.save()
747
+ entry.saved.append(ds)
748
+
749
+ def _clean_with_slug(self, entry, obj):
750
+ model = entry.model
751
+ base = None
752
+ if has_field(model, "slug") and (
753
+ title := getattr(obj, entry.config.title_field, "")
754
+ ):
755
+ base = slugify(title)
756
+ max_length = model._meta.get_field("slug").max_length
757
+ if max_length:
758
+ base = base[:max_length].strip("-")
759
+ if not base:
760
+ obj.full_clean()
761
+ return
762
+ for number in range(1, 101):
763
+ suffix = "" if number == 1 else f"-{number}"
764
+ obj.slug = f"{base[: (max_length or len(base) + len(suffix)) - len(suffix)]}{suffix}"
765
+ try:
766
+ obj.full_clean()
767
+ except ValidationError as exc:
768
+ if not _is_slug_clash(exc):
769
+ raise
770
+ if not has_field(model, "translation_of"):
771
+ raise Failed(
772
+ f"The slug {obj.slug!r} is taken. Without translation_of"
773
+ f" that's the only hint of an earlier copy; delete it, or"
774
+ f" name it with --into {label(model)}:{entry.original.pk}=<pk>."
775
+ ) from exc
776
+ continue
777
+ if number > 1:
778
+ entry.notes.append(f"The slug {base!r} was taken, it's {obj.slug!r}.")
779
+ return
780
+ raise Failed(f"No free slug for {base!r}.")
781
+
782
+ def _replace_owner(self, entry, ds):
783
+ translated = ds.object
784
+ existing = entry.existing
785
+ names = {spec.field for spec in field_specs(entry.model)}
786
+ # Only the content: references, slug, path, parent, position, menu
787
+ # and is_active are what an editor decided about the copy.
788
+ for name in names:
789
+ setattr(existing, name, getattr(translated, name))
790
+ existing.full_clean()
791
+ existing.save()
792
+ for plugin in plugin_models(entry.model):
793
+ plugin._default_manager.filter(parent=existing).delete()
794
+ for rel_model, name in entry.config.related:
795
+ rel_model._default_manager.filter(**{name: existing}).delete()
796
+ # The translated object stands in for the existing one from now on,
797
+ # so that the text references of its fields are mapped below.
798
+ entry.saved.append(_Saved(existing, ds.m2m_data))
799
+
800
+ def _fill_deferred(self):
801
+ for entry in self.entries.values():
802
+ for obj, field, value in entry.deferred:
803
+ new, state = self.counterpart(field.related_model, value)
804
+ if state in {"copy", "translation"}:
805
+ type(obj)._base_manager.filter(pk=obj.pk).update(
806
+ **{field.attname: new}
807
+ )
808
+ else:
809
+ self._reference_left(
810
+ entry, obj, field.name, field.related_model, value
811
+ )
812
+ for ds in entry.saved:
813
+ for name, pks in (ds.m2m_data or {}).items():
814
+ field = ds.object._meta.get_field(name)
815
+ mapped = []
816
+ for pk in pks:
817
+ new, state = self.counterpart(field.related_model, pk)
818
+ if state in {"copy", "translation", "self"}:
819
+ mapped.append(new)
820
+ else:
821
+ self._reference_left(
822
+ entry, ds.object, name, field.related_model, pk
823
+ )
824
+ mapped.append(pk)
825
+ getattr(ds.object, name).set(mapped)
826
+
827
+ # Text references ########################################################
828
+
829
+ @cached_property
830
+ def _flat_urls(self):
831
+ """Paths of the source-language objects without a tree"""
832
+ result = {}
833
+ for config in self.registry.translatable.values():
834
+ if config.tree or not hasattr(config.model, "get_absolute_url"):
835
+ continue
836
+ for obj in config.model._default_manager.filter(language_code=self.source):
837
+ if parts := split_url(_absolute_url(obj)):
838
+ result.setdefault(parts[1], (config.model, obj.pk))
839
+ return result
840
+
841
+ def _resolve_path(self, path, context):
842
+ for config in self.registry.translatable.values():
843
+ if not config.tree:
844
+ continue
845
+ model = config.model
846
+ scope = {
847
+ field: getattr(context, field)
848
+ for field in path_scope(model)
849
+ if hasattr(context, field)
850
+ }
851
+ candidates = {path, f"{path}/"} if not path.endswith("/") else {path}
852
+ if page := model._default_manager.filter(
853
+ path__in=candidates, **scope
854
+ ).first():
855
+ return model, page.pk
856
+ return self._flat_urls.get(path)
857
+
858
+ def _public_url(self, obj):
859
+ """The URL of an object once it's active"""
860
+ if self._is_tree(type(obj)):
861
+ return page_location(obj)
862
+ restore = getattr(obj, "is_active", None)
863
+ if restore is not None:
864
+ obj.is_active = True
865
+ try:
866
+ url = obj.get_absolute_url()
867
+ except Exception: # noqa: BLE001
868
+ return None
869
+ finally:
870
+ if restore is not None:
871
+ obj.is_active = restore
872
+ parts = split_url(url) if url else None
873
+ if not parts or parts[1] in {"", "/"} or parts[1].startswith(reverse_admin()):
874
+ return None
875
+ return host_of(parts[0]), parts[1]
876
+
877
+ def _map_url(self, entry, obj, field_name, url):
878
+ parts = split_url(url)
879
+ if not parts:
880
+ return url
881
+ origin, path, rest = parts
882
+ resolved = self._resolve_path(path, entry.original)
883
+ if resolved is None:
884
+ return url
885
+ model, pk = resolved
886
+ new, state = self.counterpart(model, pk)
887
+ if state not in {"copy", "translation"}:
888
+ self._reference_left(entry, obj, field_name, model, pk)
889
+ return url
890
+ location = self._public_url(
891
+ self._get(model, new) or model._base_manager.get(pk=new)
892
+ )
893
+ if location is None:
894
+ self._reference_left(entry, obj, field_name, model, pk)
895
+ return url
896
+ host, new_path = location
897
+ own_host = (
898
+ (self._public_url(entry.target) or (None, None))[0]
899
+ if entry.target
900
+ else None
901
+ )
902
+ if host and host != own_host:
903
+ return f"https://{host}{new_path}{rest}"
904
+ if host is None and origin:
905
+ return f"{origin}{new_path}{rest}"
906
+ return f"{new_path}{rest}"
907
+
908
+ def _map_text_references(self):
909
+ for entry in self.entries.values():
910
+ if not entry.done:
911
+ continue
912
+ for ds in entry.saved:
913
+ obj = ds.object
914
+ self._object_cache.pop((label(obj._meta.concrete_model), obj.pk), None)
915
+ specs = field_specs(type(obj))
916
+ fields = {spec.field: getattr(obj, spec.field) for spec in specs}
917
+ changed = set()
918
+ for spec in specs:
919
+ if spec.kind not in {HTML, LINK}:
920
+ continue
921
+ for _concrete, container, key in iter_path(fields, spec.path):
922
+ value = container[key]
923
+ if not isinstance(value, str) or not value:
924
+ continue
925
+ new = (
926
+ self._map_html(entry, obj, spec.field, value)
927
+ if spec.kind == HTML
928
+ else self._map_url(entry, obj, spec.field, value)
929
+ )
930
+ if new != value:
931
+ container[key] = new
932
+ changed.add(spec.field)
933
+ changed |= self._map_json_foreign_keys(entry, obj, fields)
934
+ if changed:
935
+ for name in changed:
936
+ setattr(obj, name, fields[name])
937
+ type(obj)._base_manager.filter(pk=obj.pk).update(
938
+ **{name: fields[name] for name in changed}
939
+ )
940
+
941
+ def _map_html(self, entry, obj, field_name, text):
942
+ return map_html(text, lambda url: self._map_url(entry, obj, field_name, url))
943
+
944
+ def _map_json_foreign_keys(self, entry, obj, fields):
945
+ paths_by_model = getattr(type(obj), "_proxy_types_foreign_key_paths", {}).get(
946
+ getattr(obj, "type", None), {}
947
+ )
948
+ if not paths_by_model:
949
+ return set()
950
+ fields.setdefault("data", obj.data)
951
+ changed = set()
952
+ for model_label, paths in paths_by_model.items():
953
+ model = apps.get_model(model_label)
954
+ for path in paths:
955
+ for _concrete, container, key in iter_path(
956
+ {"data": fields["data"]}, f"data.{_jmes_to_path(path)}"
957
+ ):
958
+ value = container[key]
959
+ if value in (None, ""):
960
+ continue
961
+ new, state = self.counterpart(model, value)
962
+ if state in {"copy", "translation"} and new != value:
963
+ container[key] = (
964
+ type(value)(new) if isinstance(value, str) else new
965
+ )
966
+ changed.add("data")
967
+ elif state == "same":
968
+ self._reference_left(entry, obj, f"data.{path}", model, value)
969
+ return changed
970
+
971
+ # Reporting ##############################################################
972
+
973
+ def _collect_problems(self):
974
+ for entry in self.entries.values():
975
+ if not entry.is_regions or entry.action not in {CREATE, REPLACE}:
976
+ continue
977
+ if not entry.done:
978
+ continue
979
+ for config in self.registry.translatable.values():
980
+ for plugin in plugin_models(config.model):
981
+ for field in _foreign_keys(plugin._meta.model):
982
+ if field.related_model is not entry.model:
983
+ continue
984
+ filters = {field.name: entry.target}
985
+ filters["parent__language_code"] = self.target
986
+ if has_field(config.model, "is_active"):
987
+ filters["parent__is_active"] = True
988
+ for p in plugin._default_manager.filter(
989
+ **filters
990
+ ).select_related("parent"):
991
+ self.problems["live"].append(
992
+ f"{_describe(entry.target)} is live on {_describe(p.parent)}"
993
+ )
994
+
995
+ def _log(self):
996
+ objects = []
997
+ for entry in self.entries.values():
998
+ item = {
999
+ "model": label(entry.model),
1000
+ "original": entry.unit.pk,
1001
+ "status": LOG_STATUS[entry.action],
1002
+ }
1003
+ if entry.is_regions:
1004
+ item["region"] = self.target
1005
+ target = (
1006
+ entry.target if entry.action in {CREATE, REPLACE} else entry.existing
1007
+ )
1008
+ if target is not None and target.pk is not None and not self.dry_run:
1009
+ item["copy"] = target.pk
1010
+ if target is not None and target.pk is not None:
1011
+ item["location"] = location(target)
1012
+ if not self.dry_run or entry.action == SKIP:
1013
+ item["admin_url"] = admin_url(target)
1014
+ if entry.reason:
1015
+ item["reason"] = entry.reason
1016
+ objects.append(item)
1017
+ return {
1018
+ "feincms3_autotranslate_log": 1,
1019
+ "dry_run": self.dry_run,
1020
+ "activate": self.activate,
1021
+ "source": self.source,
1022
+ "target": self.target,
1023
+ "objects": objects,
1024
+ # Shared objects with a region per language, live on loading
1025
+ "live": sorted(set(self.problems["live"])),
1026
+ }
1027
+
1028
+ def report(self):
1029
+ lines = []
1030
+
1031
+ def section(title, items):
1032
+ if items:
1033
+ lines.append(f"\n{title}")
1034
+ lines.extend(f" {item}" for item in items)
1035
+
1036
+ section(
1037
+ "GOES LIVE AT ONCE (shared objects with a region per language):",
1038
+ sorted(set(self.problems["live"])),
1039
+ )
1040
+ warnings = []
1041
+ if self.activate and any(
1042
+ e.action == CREATE and has_field(e.model, "is_active")
1043
+ for e in self.entries.values()
1044
+ ):
1045
+ warnings.append(
1046
+ "--activate: the new copies are active and visible as soon as"
1047
+ " they're loaded. Translations which existed already keep their"
1048
+ " is_active, and offline={...} of the datasets still applies."
1049
+ )
1050
+ for config in self.registry.translatable.values():
1051
+ model = config.model
1052
+ if not any(e.model is model for e in self.entries.values()):
1053
+ continue
1054
+ if not has_field(model, "translation_of"):
1055
+ warnings.append(
1056
+ f"{label(model)} has no translation_of: earlier copies can't"
1057
+ f" be recognised, running again creates another copy."
1058
+ + (
1059
+ ""
1060
+ if has_unique_slug(model)
1061
+ else " It has no unique slug either, so not even a clash"
1062
+ " would show a duplicate."
1063
+ )
1064
+ )
1065
+ if self.dry_run:
1066
+ warnings.append(
1067
+ "Signal handlers ran in this dry run too; only the database"
1068
+ " writes have been rolled back."
1069
+ )
1070
+ section("Warnings:", warnings)
1071
+
1072
+ def describe(entry):
1073
+ target = entry.target or entry.existing
1074
+ text = f"{label(entry.model)} {entry.original or entry.unit.pk}"
1075
+ if entry.original is not None and (where := location(entry.original)):
1076
+ text += f" {where}"
1077
+ if target is not None and target.pk is not None and not entry.is_regions:
1078
+ text += f" -> {location(target)}"
1079
+ # The copies of a dry run are gone, and their pks with them
1080
+ if not self.dry_run or entry.action == SKIP:
1081
+ text += f" {admin_url(target)}"
1082
+ if entry.is_regions:
1083
+ text += f" (region {self.target})"
1084
+ return text
1085
+
1086
+ for action, title in [
1087
+ (
1088
+ CREATE,
1089
+ "Created:" if not self.dry_run else "To create:",
1090
+ ),
1091
+ (
1092
+ REPLACE,
1093
+ "Replaced:" if not self.dry_run else "To replace:",
1094
+ ),
1095
+ ]:
1096
+ section(
1097
+ title,
1098
+ [
1099
+ describe(e) + "".join(f"\n {note}" for note in e.notes)
1100
+ for e in self.entries.values()
1101
+ if e.action == action
1102
+ ],
1103
+ )
1104
+ section(
1105
+ "Skipped:",
1106
+ [
1107
+ f"{describe(e)}: {e.reason}"
1108
+ for e in self.entries.values()
1109
+ if e.action == SKIP
1110
+ ],
1111
+ )
1112
+ section(
1113
+ "FAILED:",
1114
+ [
1115
+ f"{describe(e)}: {e.reason}"
1116
+ for e in self.entries.values()
1117
+ if e.action == FAIL
1118
+ ],
1119
+ )
1120
+ section(
1121
+ "Static paths, set one again where it's wanted:",
1122
+ [f"{_describe(page)}" for page in self.problems["static_paths"]],
1123
+ )
1124
+ section(
1125
+ f"References left in another language than {self.target}:",
1126
+ self._references("references"),
1127
+ )
1128
+ section(
1129
+ f"Referenced objects with a region per language, without {self.target} plugins:",
1130
+ self._references("regions"),
1131
+ )
1132
+ section(
1133
+ f"Referenced objects with empty {self.target} fields:",
1134
+ self._references("translated_fields"),
1135
+ )
1136
+ return "\n".join(lines).strip() + "\n"
1137
+
1138
+ def _references(self, key):
1139
+ """Problems found on references, of the objects which were loaded"""
1140
+ lines = set()
1141
+ for entry, field, text in self.problems[key]:
1142
+ if entry.action == FAIL:
1143
+ continue
1144
+ target = entry.target if entry.target and entry.target.pk else None
1145
+ lines.add(f"{_describe(target or entry.original)}: {field} -> {text}")
1146
+ return sorted(lines)
1147
+
1148
+ @property
1149
+ def failed(self):
1150
+ return [e for e in self.entries.values() if e.action == FAIL]
1151
+
1152
+
1153
+ @dataclasses.dataclass
1154
+ class _Saved:
1155
+ object: object
1156
+ m2m_data: dict
1157
+
1158
+
1159
+ def _absolute_url(obj):
1160
+ try:
1161
+ return obj.get_absolute_url()
1162
+ except Exception: # noqa: BLE001
1163
+ # Whatever breaks here, the object is then just not linkable.
1164
+ return None
1165
+
1166
+
1167
+ def reverse_admin():
1168
+ try:
1169
+ return reverse("admin:index")
1170
+ except NoReverseMatch:
1171
+ return "/\0"
1172
+
1173
+
1174
+ def _jmes_to_path(path):
1175
+ """``items[*].page`` -> ``items[].page``; only this subset is supported"""
1176
+ return path.replace("[*]", "[]")
1177
+
1178
+
1179
+ # Reading the input files ####################################################
1180
+
1181
+
1182
+ def read_translated(path_or_data):
1183
+ if isinstance(path_or_data, dict):
1184
+ data = path_or_data
1185
+ else:
1186
+ with open(path_or_data, encoding="utf-8") as f:
1187
+ data = json.load(f)
1188
+ if data.get("feincms3_autotranslate") != 1:
1189
+ raise CommandError(
1190
+ f"{path_or_data} isn't a translated file. Dumps go through"
1191
+ f" f3autotranslate first (or f3autotranslateload --plan)."
1192
+ )
1193
+ return data
1194
+
1195
+
1196
+ def parse_into(values):
1197
+ """``label:pk=pk`` pairs and load logs into ``{(label, pk): pk}``"""
1198
+ result = {}
1199
+ for value in values or ():
1200
+ if ":" in value and "=" in value and not value.endswith(".json"):
1201
+ model, _sep, pair = value.partition(":")
1202
+ original, _sep, copy = pair.partition("=")
1203
+ try:
1204
+ result[(model.lower(), int(original))] = int(copy)
1205
+ except ValueError:
1206
+ raise CommandError(f"Invalid --into {value!r}.") from None
1207
+ continue
1208
+ with open(value, encoding="utf-8") as f:
1209
+ log = json.load(f)
1210
+ if log.get("feincms3_autotranslate_log") != 1:
1211
+ raise CommandError(f"{value} isn't a load log.")
1212
+ for item in log["objects"]:
1213
+ if item.get("copy") is not None and "region" not in item:
1214
+ result[(item["model"], item["original"])] = item["copy"]
1215
+ return result
1216
+
1217
+
1218
+ def _all_objects(dumps):
1219
+ try:
1220
+ return all_objects(dumps)
1221
+ except DumpError as exc:
1222
+ raise CommandError(str(exc)) from exc
1223
+
1224
+
1225
+ def run_from_dumps(dumps, *, target, **kwargs):
1226
+ return Run(_all_objects(dumps), target=target, **kwargs)
1227
+
1228
+
1229
+ def run_from_translated(data, **kwargs):
1230
+ statuses = data.get("units", {})
1231
+ return Run(
1232
+ _all_objects(data["dumps"]),
1233
+ target=data["target"],
1234
+ statuses=statuses,
1235
+ **kwargs,
1236
+ )
1237
+
1238
+
1239
+ __all__ = [
1240
+ "CREATE",
1241
+ "FAIL",
1242
+ "OWNER",
1243
+ "REPLACE",
1244
+ "SKIP",
1245
+ "Run",
1246
+ "parse_into",
1247
+ "read_translated",
1248
+ "run_from_dumps",
1249
+ "run_from_translated",
1250
+ ]