django-postgres-denormalize 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_postgres_denormalize-1.0.0.dist-info/METADATA +154 -0
- django_postgres_denormalize-1.0.0.dist-info/RECORD +15 -0
- django_postgres_denormalize-1.0.0.dist-info/WHEEL +5 -0
- django_postgres_denormalize-1.0.0.dist-info/licenses/LICENSE +23 -0
- django_postgres_denormalize-1.0.0.dist-info/top_level.txt +1 -0
- postgres_denormalize/__init__.py +36 -0
- postgres_denormalize/apps.py +18 -0
- postgres_denormalize/batch.py +130 -0
- postgres_denormalize/chains.py +121 -0
- postgres_denormalize/checks.py +381 -0
- postgres_denormalize/conf.py +170 -0
- postgres_denormalize/fields.py +716 -0
- postgres_denormalize/operations.py +139 -0
- postgres_denormalize/py.typed +0 -0
- postgres_denormalize/triggers.py +597 -0
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The triggers that keep every denormalized field equal to the value at the end of its lookup path.
|
|
3
|
+
|
|
4
|
+
Three kinds of trigger keep each field correct:
|
|
5
|
+
|
|
6
|
+
- ``DenormalizeSetTrigger``, named ``denormalize_set``, is a ``BEFORE INSERT OR UPDATE`` row trigger on the leaf's
|
|
7
|
+
table. It computes every denormalized field of the row that is being written, so the column is correct no matter
|
|
8
|
+
which code writes the row: forms, services that build instances directly, the admin, factories, ``loaddata``, raw SQL,
|
|
9
|
+
and the previous release, which is still serving during a rolling deployment.
|
|
10
|
+
- ``DenormalizeCascadeTrigger``, named ``denormalize_cascade_<app>_<leaf>``, is an ``AFTER UPDATE`` row trigger on the
|
|
11
|
+
table of each model that a leaf's lookup paths cross. Moving a project to another tenant changes the tenant that
|
|
12
|
+
every task under it should have, and renaming a project changes the name that its tasks copy. Neither writes anything
|
|
13
|
+
to the tasks' table, so ``denormalize_set`` never fires for the tasks. This trigger rewrites them.
|
|
14
|
+
- ``DenormalizeFillTrigger``, named ``denormalize_fill_<app>_<leaf>``, is an ``AFTER INSERT`` row trigger on the same
|
|
15
|
+
tables. A transaction can insert a task before its project, because Django defers the foreign key checks to the
|
|
16
|
+
commit. ``denormalize_set`` then finds no project for the task, and this trigger fills the task's copies when the
|
|
17
|
+
project is inserted.
|
|
18
|
+
|
|
19
|
+
``PostgresDenormalizeConfig.ready()`` calls ``register_triggers``, which derives every kind of trigger from the fields
|
|
20
|
+
and registers them with pgtrigger. pgtrigger adds every registered trigger to the migration state of its model, so
|
|
21
|
+
``makemigrations`` writes the trigger into a migration exactly as it writes a trigger that is listed in
|
|
22
|
+
``Meta.triggers``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
|
|
27
|
+
import pgtrigger
|
|
28
|
+
import pgtrigger.utils
|
|
29
|
+
from django.apps import apps
|
|
30
|
+
from django.core.exceptions import FieldDoesNotExist
|
|
31
|
+
|
|
32
|
+
from postgres_denormalize import conf
|
|
33
|
+
from postgres_denormalize.fields import get_denormalized_fields
|
|
34
|
+
|
|
35
|
+
SET_TRIGGER_NAME = 'denormalize_set'
|
|
36
|
+
|
|
37
|
+
CASCADE_TRIGGER_PREFIX = 'denormalize_cascade_'
|
|
38
|
+
|
|
39
|
+
FILL_TRIGGER_PREFIX = 'denormalize_fill_'
|
|
40
|
+
|
|
41
|
+
# The transaction-local Postgres setting in which ``defer_cascades()`` lists the cascade triggers that it skips, as a
|
|
42
|
+
# comma-separated list of their Postgres identifiers. While ``DEFERRABLE_CASCADES`` is on, the WHEN condition of every
|
|
43
|
+
# cascade trigger reads this setting.
|
|
44
|
+
DEFERRED_CASCADES_SETTING = 'postgres_denormalize.deferred_cascades'
|
|
45
|
+
|
|
46
|
+
# pgtrigger refuses longer trigger names: it adds the prefix `pgtrigger_` and a hash suffix to form a 63-character
|
|
47
|
+
# Postgres identifier.
|
|
48
|
+
MAX_TRIGGER_NAME_LENGTH = 47
|
|
49
|
+
|
|
50
|
+
# The alias prefix for the tables of the hops above the hop that changed. The cascade trigger walks those hops to read
|
|
51
|
+
# the value that the leaves get. The prefix differs from the `hop` prefix that the same statement uses for the hops
|
|
52
|
+
# below, so each alias names one table.
|
|
53
|
+
UPPER_HOP_PREFIX = 'upper'
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def quote(name):
|
|
57
|
+
"""
|
|
58
|
+
Quote a table or column name, the way Django's Postgres backend quotes it in the statements that create the table.
|
|
59
|
+
|
|
60
|
+
Postgres folds a name that is not quoted to lowercase. A table or column whose name has an uppercase letter, or is a
|
|
61
|
+
reserved word, would therefore not be found when the trigger runs. The aliases that the triggers make up themselves,
|
|
62
|
+
such as ``hop0`` and ``leaf``, are lowercase and not reserved, so they are not quoted.
|
|
63
|
+
|
|
64
|
+
The name is quoted by pgtrigger's helper, which needs no database connection. The default database of a project can
|
|
65
|
+
be one that cannot quote names, such as Django's dummy backend when the Postgres database has another alias.
|
|
66
|
+
"""
|
|
67
|
+
return pgtrigger.utils.quote(name)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def hop_alias(index, prefix='hop'):
|
|
71
|
+
"""
|
|
72
|
+
Name the SQL alias of the table at position ``index`` of a lookup path.
|
|
73
|
+
|
|
74
|
+
``hop0`` is the table that the first hop points at, and each following alias is the table that the next hop points
|
|
75
|
+
at. The source column is a column of the last alias, which ``value_alias`` names.
|
|
76
|
+
"""
|
|
77
|
+
return f'{prefix}{index}'
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def value_alias(chain, prefix='hop'):
|
|
81
|
+
"""
|
|
82
|
+
Name the alias of the table that has the source column at the end of the lookup path.
|
|
83
|
+
"""
|
|
84
|
+
return hop_alias(len(chain) - 2, prefix)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def build_hop_joins(chain, prefix='hop', join='JOIN'):
|
|
88
|
+
"""
|
|
89
|
+
Build the ``FROM`` and ``JOIN`` lines that walk a lookup path, with one alias for each table that the path reaches
|
|
90
|
+
after its first hop.
|
|
91
|
+
|
|
92
|
+
Both kinds of trigger build their joins with this function, and the cascade trigger also uses it for a part of the
|
|
93
|
+
path. A lookup that reads a value passes ``join='LEFT JOIN'``, so an empty hop along the path yields NULL instead of
|
|
94
|
+
no row.
|
|
95
|
+
|
|
96
|
+
Each table is joined on the column that the hop before it points at. That is the primary key, unless the hop is
|
|
97
|
+
declared with ``to_field``.
|
|
98
|
+
"""
|
|
99
|
+
lines = [f'FROM {quote(chain[0].related_model._meta.db_table)} {hop_alias(0, prefix)}']
|
|
100
|
+
for index, hop in enumerate(chain[1:-1], start=1):
|
|
101
|
+
alias = hop_alias(index, prefix)
|
|
102
|
+
lines.append(
|
|
103
|
+
f'{join} {quote(hop.related_model._meta.db_table)} {alias} '
|
|
104
|
+
f'ON {alias}.{quote(hop.target_field.column)} = {hop_alias(index - 1, prefix)}.{quote(hop.column)}'
|
|
105
|
+
)
|
|
106
|
+
return lines
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def resolvable_fields(model):
|
|
110
|
+
"""
|
|
111
|
+
Return ``(field, chain)`` for every denormalized field of ``model`` whose lookup path resolves, in declaration
|
|
112
|
+
order.
|
|
113
|
+
|
|
114
|
+
A field whose path does not resolve is skipped, so no trigger is compiled from it. The developer then sees the
|
|
115
|
+
system check error that the field reports, and not a traceback from pgtrigger.
|
|
116
|
+
"""
|
|
117
|
+
resolvable = []
|
|
118
|
+
for field in get_denormalized_fields(model):
|
|
119
|
+
try:
|
|
120
|
+
chain = field.resolve_chain(model)
|
|
121
|
+
except FieldDoesNotExist, ValueError:
|
|
122
|
+
continue
|
|
123
|
+
resolvable.append((field, chain))
|
|
124
|
+
return resolvable
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _path_key(hops):
|
|
128
|
+
"""
|
|
129
|
+
Identify a sequence of hops, so that lookup paths that start with the same hops can share one statement.
|
|
130
|
+
"""
|
|
131
|
+
return tuple((hop.model._meta.label, hop.name) for hop in hops)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def group_by_ancestor_row(resolvable):
|
|
135
|
+
"""
|
|
136
|
+
Group ``(field, chain)`` pairs by the ancestor row that they read from, in the order in which the pairs are given.
|
|
137
|
+
|
|
138
|
+
Two fields read the same row when their lookup paths cross the same hops and differ only in their source column, as
|
|
139
|
+
``project__tenant`` and ``project__name`` do. One lookup then serves both fields.
|
|
140
|
+
"""
|
|
141
|
+
groups = {}
|
|
142
|
+
for field, chain in resolvable:
|
|
143
|
+
groups.setdefault(_path_key(chain[:-1]), []).append((field, chain))
|
|
144
|
+
return list(groups.values())
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def build_skip_unchanged_sql(resolvable):
|
|
148
|
+
"""
|
|
149
|
+
Build the statement that ends the trigger early for an UPDATE that cannot change any denormalized field.
|
|
150
|
+
|
|
151
|
+
Every field is computed from the leaf's own first hops. An UPDATE that changes none of them therefore reads the same
|
|
152
|
+
ancestor rows as before. Django assigns each denormalized field to itself on every save, so such an UPDATE also
|
|
153
|
+
changes no copy, and the trigger returns without looking anything up. An UPDATE that writes a copy still recomputes
|
|
154
|
+
every field. That happens when a cascade trigger rewrites a copy, or when ``QuerySet.update()`` assigns one. The
|
|
155
|
+
trigger therefore always overwrites a value that code assigns.
|
|
156
|
+
|
|
157
|
+
The condition is nested inside ``TG_OP = 'UPDATE'`` rather than joined to it with ``AND``, because SQL does not
|
|
158
|
+
promise to evaluate ``AND`` from left to right, and ``OLD`` is NULL on an INSERT.
|
|
159
|
+
"""
|
|
160
|
+
columns = []
|
|
161
|
+
for field, chain in resolvable:
|
|
162
|
+
for column in (chain[0].column, field.column):
|
|
163
|
+
if column not in columns:
|
|
164
|
+
columns.append(column)
|
|
165
|
+
unchanged = '\n AND '.join(
|
|
166
|
+
f'OLD.{quote(column)} IS NOT DISTINCT FROM NEW.{quote(column)}' for column in columns
|
|
167
|
+
)
|
|
168
|
+
return '\n'.join(
|
|
169
|
+
[
|
|
170
|
+
"IF TG_OP = 'UPDATE' THEN",
|
|
171
|
+
f' IF {unchanged} THEN',
|
|
172
|
+
' RETURN NEW;',
|
|
173
|
+
' END IF;',
|
|
174
|
+
'END IF;',
|
|
175
|
+
]
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def ancestor_variable(index):
|
|
180
|
+
"""
|
|
181
|
+
Name the record variable into which the lookup at position ``index`` reads its ancestor row.
|
|
182
|
+
"""
|
|
183
|
+
return f'ancestor{index}'
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def build_lookup_sql(index, group):
|
|
187
|
+
"""
|
|
188
|
+
Build the statements that read one ancestor row and copy its columns onto the row that is being written.
|
|
189
|
+
|
|
190
|
+
The row is read into a variable of the generic type ``record``. Postgres resolves a ``DECLARE`` that names a
|
|
191
|
+
column's type, such as ``%TYPE``, when it creates the function, so such a function cannot be created before the
|
|
192
|
+
table exists. A ``record`` names no table, and Postgres resolves a statement in the function body when the
|
|
193
|
+
statement first runs. The function can therefore be created before the tables that it reads. Tools that copy a
|
|
194
|
+
schema rely on that, because they create every function before any table.
|
|
195
|
+
|
|
196
|
+
``FOUND`` tells the two outcomes of the lookup apart. A row that is found is copied as it is, including NULL values.
|
|
197
|
+
No row is found when the transaction inserts a row before the row that its first hop points at. Django creates
|
|
198
|
+
every foreign key as ``DEFERRABLE INITIALLY DEFERRED``, so Postgres checks the constraint only at commit, and any
|
|
199
|
+
code that inserts the rows in that order can do so, not only a bulk load such as ``loaddata``. The value that the
|
|
200
|
+
insert supplies is then kept. When the ancestor row is inserted later in the transaction, the fill trigger on its
|
|
201
|
+
table rewrites the copy.
|
|
202
|
+
"""
|
|
203
|
+
_field, chain = group[0]
|
|
204
|
+
first = chain[0]
|
|
205
|
+
variable = ancestor_variable(index)
|
|
206
|
+
selected = ', '.join(
|
|
207
|
+
f'{value_alias(field_chain)}.{quote(field_chain[-1].column)} AS value{position}'
|
|
208
|
+
for position, (_field, field_chain) in enumerate(group)
|
|
209
|
+
)
|
|
210
|
+
lines = [
|
|
211
|
+
f'SELECT {selected}',
|
|
212
|
+
f'INTO {variable}',
|
|
213
|
+
*build_hop_joins(chain, join='LEFT JOIN'),
|
|
214
|
+
f'WHERE {hop_alias(0)}.{quote(first.target_field.column)} = NEW.{quote(first.column)};',
|
|
215
|
+
'IF FOUND THEN',
|
|
216
|
+
*(
|
|
217
|
+
f' NEW.{quote(field.column)} := {variable}.value{position};'
|
|
218
|
+
for position, (field, _chain) in enumerate(group)
|
|
219
|
+
),
|
|
220
|
+
'END IF;',
|
|
221
|
+
]
|
|
222
|
+
return '\n'.join(lines)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class DenormalizeSetTrigger(pgtrigger.Trigger):
|
|
226
|
+
"""
|
|
227
|
+
Computes every denormalized field of a leaf row before the row is inserted or updated.
|
|
228
|
+
|
|
229
|
+
``register_triggers`` creates one instance per leaf model. Two instances are equal when they have the same class and
|
|
230
|
+
the same name. ``AppConfig.ready()`` can run more than once in a test process, and pgtrigger's registry raises
|
|
231
|
+
``KeyError`` when a second instance with the same name is registered that is not equal to the first.
|
|
232
|
+
"""
|
|
233
|
+
|
|
234
|
+
name = SET_TRIGGER_NAME
|
|
235
|
+
when = pgtrigger.Before
|
|
236
|
+
operation = pgtrigger.Insert | pgtrigger.Update
|
|
237
|
+
|
|
238
|
+
def __eq__(self, other):
|
|
239
|
+
return type(other) is type(self) and other.name == self.name
|
|
240
|
+
|
|
241
|
+
def __hash__(self):
|
|
242
|
+
return hash((type(self), self.name))
|
|
243
|
+
|
|
244
|
+
def get_declare(self, model):
|
|
245
|
+
groups = group_by_ancestor_row(resolvable_fields(model))
|
|
246
|
+
return [(ancestor_variable(index), 'record') for index in range(len(groups))]
|
|
247
|
+
|
|
248
|
+
def get_func(self, model):
|
|
249
|
+
resolvable = resolvable_fields(model)
|
|
250
|
+
if not resolvable:
|
|
251
|
+
return 'RETURN NEW;'
|
|
252
|
+
lookups = [build_lookup_sql(index, group) for index, group in enumerate(group_by_ancestor_row(resolvable))]
|
|
253
|
+
return '\n'.join([build_skip_unchanged_sql(resolvable), *lookups, 'RETURN NEW;'])
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def build_moved_value_sql(chain, hop_index):
|
|
257
|
+
"""
|
|
258
|
+
Build the SQL expression for the value that the leaves below a changed column get, read from the new row of the
|
|
259
|
+
column's owner.
|
|
260
|
+
|
|
261
|
+
When the changed column is the source column, the value is that column of the new row. Otherwise it is a scalar
|
|
262
|
+
subquery that walks the remaining hops from the new row, under the ``upper`` aliases. The subquery uses left joins,
|
|
263
|
+
so that an empty hop along the path yields NULL.
|
|
264
|
+
"""
|
|
265
|
+
if hop_index == len(chain) - 1:
|
|
266
|
+
return f'NEW.{quote(chain[hop_index].column)}'
|
|
267
|
+
upper = chain[hop_index:]
|
|
268
|
+
lines = [
|
|
269
|
+
f'(SELECT {value_alias(upper, UPPER_HOP_PREFIX)}.{quote(upper[-1].column)}',
|
|
270
|
+
*(f' {line}' for line in build_hop_joins(upper, UPPER_HOP_PREFIX, join='LEFT JOIN')),
|
|
271
|
+
f' WHERE {hop_alias(0, UPPER_HOP_PREFIX)}.{quote(upper[0].target_field.column)}'
|
|
272
|
+
f' = NEW.{quote(upper[0].column)})',
|
|
273
|
+
]
|
|
274
|
+
return '\n'.join(lines)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def build_cascade_update_sql(leaf, assignments):
|
|
278
|
+
"""
|
|
279
|
+
Build the statement that rewrites denormalized fields on the leaf rows below a row that an UPDATE changed or an
|
|
280
|
+
INSERT added.
|
|
281
|
+
|
|
282
|
+
``assignments`` lists ``(field, chain, hop_index)`` triples: each field to rewrite, its resolved lookup path, and
|
|
283
|
+
the position in that path of the column that belongs to the trigger's table. Every triple reaches that table through
|
|
284
|
+
the same hops, so one statement writes all of them, and a leaf row whose copies all change is written once rather
|
|
285
|
+
than once per field. ``NEW`` is the row that the UPDATE or INSERT wrote. The leaf rows below it are the rows whose
|
|
286
|
+
first hop leads to that row.
|
|
287
|
+
When the changed column is the second segment of the path, the first hop points at the row directly. Otherwise the
|
|
288
|
+
statement joins the tables of the hops in between, under the ``hop`` aliases.
|
|
289
|
+
|
|
290
|
+
The final ``WHERE`` condition keeps only leaf rows in which some copy differs from the value that is about to be
|
|
291
|
+
written. Without it, moving a task to another project *of the same tenant* would rewrite every comment under the
|
|
292
|
+
task with the tenant that the comment already has, and a history table that tracks the comments would get a row for
|
|
293
|
+
each of them.
|
|
294
|
+
|
|
295
|
+
Leaf rows that a soft-delete library has only marked as deleted are rewritten too. If they kept the old value,
|
|
296
|
+
restoring one would bring back a value that differs from its lookup path.
|
|
297
|
+
"""
|
|
298
|
+
values = [(field, build_moved_value_sql(chain, hop_index)) for field, chain, hop_index in assignments]
|
|
299
|
+
|
|
300
|
+
_field, chain, hop_index = assignments[0]
|
|
301
|
+
# The column of the new row that the hop below it points at: the primary key, unless that hop has a to_field.
|
|
302
|
+
owner_key_column = quote(chain[hop_index - 1].target_field.column)
|
|
303
|
+
if hop_index == 1:
|
|
304
|
+
joins = []
|
|
305
|
+
reaches_new_row = f'leaf.{quote(chain[0].column)} = NEW.{owner_key_column}'
|
|
306
|
+
else:
|
|
307
|
+
below = chain[:hop_index]
|
|
308
|
+
joins = build_hop_joins(below)
|
|
309
|
+
reaches_new_row = (
|
|
310
|
+
f'leaf.{quote(chain[0].column)} = {hop_alias(0)}.{quote(chain[0].target_field.column)}\n'
|
|
311
|
+
f' AND {hop_alias(hop_index - 2)}.{quote(below[-1].column)} = NEW.{owner_key_column}'
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
assigned = ',\n '.join(f'{quote(field.column)} = {value}' for field, value in values)
|
|
315
|
+
differs = [f'leaf.{quote(field.column)} IS DISTINCT FROM {value}' for field, value in values]
|
|
316
|
+
differs_sql = differs[0] if len(differs) == 1 else '(' + '\n OR '.join(differs) + ')'
|
|
317
|
+
|
|
318
|
+
lines = [
|
|
319
|
+
f'UPDATE {quote(leaf._meta.db_table)} leaf',
|
|
320
|
+
f'SET {assigned}',
|
|
321
|
+
*joins,
|
|
322
|
+
f'WHERE {reaches_new_row}',
|
|
323
|
+
f' AND {differs_sql};',
|
|
324
|
+
]
|
|
325
|
+
return '\n'.join(lines)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def build_changed_columns_sql(columns):
|
|
329
|
+
"""
|
|
330
|
+
Build the condition that is true for a row in which any of ``columns`` changed.
|
|
331
|
+
"""
|
|
332
|
+
return ' OR '.join(f'OLD.{quote(column)} IS DISTINCT FROM NEW.{quote(column)}' for column in columns)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def build_not_deferred_sql(pgid):
|
|
336
|
+
"""
|
|
337
|
+
Build the condition that is true unless the current transaction lists the trigger ``pgid`` in
|
|
338
|
+
``DEFERRED_CASCADES_SETTING``.
|
|
339
|
+
|
|
340
|
+
``current_setting`` returns NULL for a setting that this session never set, and an empty string after a transaction
|
|
341
|
+
that set it has ended. ``string_to_array`` turns NULL into NULL, which would make the whole condition NULL, and the
|
|
342
|
+
trigger would never fire. ``COALESCE`` therefore treats NULL as not deferred.
|
|
343
|
+
"""
|
|
344
|
+
listed = f"'{pgid}' = ANY(string_to_array(current_setting('{DEFERRED_CASCADES_SETTING}', true), ','))"
|
|
345
|
+
return f'NOT COALESCE({listed}, false)'
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def cascade_trigger_name(leaf_label, prefix=CASCADE_TRIGGER_PREFIX):
|
|
349
|
+
"""
|
|
350
|
+
Name the cascade trigger for the leaf ``leaf_label``: ``denormalize_cascade_<app_label>_<model_name>``. The fill
|
|
351
|
+
trigger passes its own ``prefix``.
|
|
352
|
+
|
|
353
|
+
When the app label contains an underscore, two labels can give the same name: ``a_b.C`` and ``a.B_C`` both give
|
|
354
|
+
``<prefix>a_b_c``. A hash of the label then follows the name, so the name is unique, and it still depends only on
|
|
355
|
+
the label. When the name would exceed pgtrigger's limit, it is cut to the length that fits and followed by the hash
|
|
356
|
+
as well.
|
|
357
|
+
"""
|
|
358
|
+
app_label, _, model_name = leaf_label.partition('.')
|
|
359
|
+
name = f'{prefix}{app_label.lower()}_{model_name.lower()}'
|
|
360
|
+
if '_' not in app_label and len(name) <= MAX_TRIGGER_NAME_LENGTH:
|
|
361
|
+
return name
|
|
362
|
+
digest = hashlib.sha1(leaf_label.lower().encode(), usedforsecurity=False).hexdigest()[:8]
|
|
363
|
+
return f'{name[: MAX_TRIGGER_NAME_LENGTH - len(digest) - 1]}_{digest}'
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class OwnerTrigger(pgtrigger.Trigger):
|
|
367
|
+
"""
|
|
368
|
+
The base class of the triggers on the table of an owner: a model that a leaf's lookup paths cross after the first
|
|
369
|
+
hop.
|
|
370
|
+
|
|
371
|
+
Each instance serves exactly one leaf, named by ``leaf_label``, and its body writes only that leaf's table. A model
|
|
372
|
+
that the lookup paths of several leaves cross has one instance per leaf. Each trigger body therefore follows from
|
|
373
|
+
the declarations of its own leaf and from nothing else. Adding a field to a leaf changes that leaf's triggers and
|
|
374
|
+
does not add new ones.
|
|
375
|
+
|
|
376
|
+
The owner can be a proxy model, when a hop points at the proxy. Its fields, and therefore the hops, belong to the
|
|
377
|
+
concrete model, so a hop is matched to the owner by the concrete model.
|
|
378
|
+
|
|
379
|
+
Two instances are equal when they have the same class and serve the same leaf, for the same reason as for
|
|
380
|
+
``DenormalizeSetTrigger``.
|
|
381
|
+
"""
|
|
382
|
+
|
|
383
|
+
name_prefix = CASCADE_TRIGGER_PREFIX
|
|
384
|
+
|
|
385
|
+
def __init__(self, *, leaf_label, **kwargs):
|
|
386
|
+
self.leaf_label = leaf_label
|
|
387
|
+
super().__init__(name=cascade_trigger_name(leaf_label, self.name_prefix), **kwargs)
|
|
388
|
+
|
|
389
|
+
def __eq__(self, other):
|
|
390
|
+
return type(other) is type(self) and other.leaf_label == self.leaf_label
|
|
391
|
+
|
|
392
|
+
def __hash__(self):
|
|
393
|
+
return hash((type(self), self.leaf_label))
|
|
394
|
+
|
|
395
|
+
def _leaf_and_assignment_groups(self, model):
|
|
396
|
+
"""
|
|
397
|
+
Return the leaf, and its ``(field, chain, hop_index)`` triples grouped by the hops that lead from the leaf to
|
|
398
|
+
``model``.
|
|
399
|
+
|
|
400
|
+
A triple names a denormalized field whose lookup path has a segment that belongs to ``model``, and the position
|
|
401
|
+
of that segment.
|
|
402
|
+
"""
|
|
403
|
+
# The leaf is looked up through the registry that `model` belongs to. When makemigrations compiles the trigger,
|
|
404
|
+
# that is the registry of the migration state.
|
|
405
|
+
leaf = model._meta.apps.get_model(self.leaf_label)
|
|
406
|
+
table_label = model._meta.concrete_model._meta.label
|
|
407
|
+
groups = {}
|
|
408
|
+
for field, chain in resolvable_fields(leaf):
|
|
409
|
+
for index, hop in enumerate(chain):
|
|
410
|
+
if index and hop.model._meta.concrete_model._meta.label == table_label:
|
|
411
|
+
groups.setdefault(_path_key(chain[:index]), []).append((field, chain, index))
|
|
412
|
+
return leaf, list(groups.values())
|
|
413
|
+
|
|
414
|
+
@staticmethod
|
|
415
|
+
def _owned_columns(group):
|
|
416
|
+
columns = []
|
|
417
|
+
for _field, chain, index in group:
|
|
418
|
+
if chain[index].column not in columns:
|
|
419
|
+
columns.append(chain[index].column)
|
|
420
|
+
return columns
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
class DenormalizeCascadeTrigger(OwnerTrigger):
|
|
424
|
+
"""
|
|
425
|
+
Rewrites a leaf's denormalized fields after an UPDATE on the table of a model that the leaf's lookup paths cross.
|
|
426
|
+
|
|
427
|
+
The trigger is row-level. Its ``WHEN`` condition tests every column of its table that one of the leaf's lookup paths
|
|
428
|
+
reads, so Postgres calls its function only for a row in which one of those columns changed. Every other UPDATE on
|
|
429
|
+
the table costs nothing beyond evaluating the condition. The price is that an UPDATE that changes many rows at once
|
|
430
|
+
runs the function once per row, and each run is an indexed UPDATE on the leaf. Frequent bulk updates of such columns
|
|
431
|
+
are therefore a reason not to denormalize them, unless the batch runs inside ``ignore_cascades()`` or
|
|
432
|
+
``defer_cascades()``.
|
|
433
|
+
|
|
434
|
+
While ``DEFERRABLE_CASCADES`` is on, which is the default, the condition also requires that the current transaction
|
|
435
|
+
does not list the trigger in ``DEFERRED_CASCADES_SETTING``. ``defer_cascades()`` lists it there, so Postgres does
|
|
436
|
+
not call the function at all for the rows that a batch changes. When the setting is off, the condition only tests
|
|
437
|
+
the columns. Changing the setting changes the compiled condition of every cascade trigger, so ``makemigrations``
|
|
438
|
+
recreates all of them.
|
|
439
|
+
"""
|
|
440
|
+
|
|
441
|
+
when = pgtrigger.After
|
|
442
|
+
operation = pgtrigger.Update
|
|
443
|
+
|
|
444
|
+
def get_condition(self, model):
|
|
445
|
+
_leaf, groups = self._leaf_and_assignment_groups(model)
|
|
446
|
+
columns = []
|
|
447
|
+
for group in groups:
|
|
448
|
+
columns.extend(column for column in self._owned_columns(group) if column not in columns)
|
|
449
|
+
changed = build_changed_columns_sql(columns)
|
|
450
|
+
if not conf.get_setting('DEFERRABLE_CASCADES'):
|
|
451
|
+
return pgtrigger.Condition(changed)
|
|
452
|
+
# pgtrigger adds parentheses around a condition only when it does not already start with a parenthesis, so the
|
|
453
|
+
# whole condition is wrapped here, and not just the test on the columns.
|
|
454
|
+
return pgtrigger.Condition(f'(({changed}) AND {build_not_deferred_sql(self.get_pgid(model))})')
|
|
455
|
+
|
|
456
|
+
def get_func(self, model):
|
|
457
|
+
leaf, groups = self._leaf_and_assignment_groups(model)
|
|
458
|
+
all_columns = {column for group in groups for column in self._owned_columns(group)}
|
|
459
|
+
statements = []
|
|
460
|
+
for group in groups:
|
|
461
|
+
statement = build_cascade_update_sql(leaf, group)
|
|
462
|
+
columns = self._owned_columns(group)
|
|
463
|
+
# The WHEN condition already guarantees that one of this table's columns changed. Only when several groups
|
|
464
|
+
# share the trigger does each group need its own test, so that it does not rewrite rows whose copies cannot
|
|
465
|
+
# differ.
|
|
466
|
+
if set(columns) != all_columns:
|
|
467
|
+
indented = '\n'.join(f' {line}' for line in statement.split('\n'))
|
|
468
|
+
statement = f'IF {build_changed_columns_sql(columns)} THEN\n{indented}\nEND IF;'
|
|
469
|
+
statements.append(statement)
|
|
470
|
+
return '\n'.join([*statements, 'RETURN NULL;'])
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
class DenormalizeFillTrigger(OwnerTrigger):
|
|
474
|
+
"""
|
|
475
|
+
Rewrites a leaf's denormalized fields after an INSERT on the table of a model that the leaf's lookup paths cross.
|
|
476
|
+
|
|
477
|
+
Django creates every foreign key as ``DEFERRABLE INITIALLY DEFERRED``, so a transaction can insert a leaf row before
|
|
478
|
+
the ancestor row that its path reaches. The set trigger then finds no ancestor row and keeps the value that the
|
|
479
|
+
insert supplies. This trigger, named ``denormalize_fill_<app>_<leaf>``, corrects that value when the ancestor row is
|
|
480
|
+
inserted. Its function runs the same statements as the cascade trigger, one for each group of copies. A new row has
|
|
481
|
+
no leaf rows below it in the usual order of inserts, so each statement is one indexed lookup that writes nothing.
|
|
482
|
+
|
|
483
|
+
The trigger has no ``WHEN`` condition, because every column of a new row is new, and the batch helpers do not skip
|
|
484
|
+
it.
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
name_prefix = FILL_TRIGGER_PREFIX
|
|
488
|
+
when = pgtrigger.After
|
|
489
|
+
operation = pgtrigger.Insert
|
|
490
|
+
|
|
491
|
+
def get_func(self, model):
|
|
492
|
+
leaf, groups = self._leaf_and_assignment_groups(model)
|
|
493
|
+
return '\n'.join([*(build_cascade_update_sql(leaf, group) for group in groups), 'RETURN NULL;'])
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def owners_of(resolvable):
|
|
497
|
+
"""
|
|
498
|
+
Return the owners of a leaf's triggers, in label order: one model for each table that the lookup paths in
|
|
499
|
+
``resolvable`` cross after the first hop.
|
|
500
|
+
|
|
501
|
+
The owner of a table is the model that the hop before it points at. When the hops of the leaf reach one table both
|
|
502
|
+
through a proxy model and through the concrete model, or through several proxy models, the first proxy model in
|
|
503
|
+
label order owns the triggers. A table gets one trigger of each kind per leaf, and a project that points a hop at a
|
|
504
|
+
proxy model wants the triggers in the proxy model's app.
|
|
505
|
+
"""
|
|
506
|
+
candidates = {}
|
|
507
|
+
for _field, chain in resolvable:
|
|
508
|
+
for index in range(1, len(chain)):
|
|
509
|
+
owner = chain[index - 1].related_model
|
|
510
|
+
candidates.setdefault(owner._meta.concrete_model._meta.label, set()).add(owner)
|
|
511
|
+
owners = [
|
|
512
|
+
min(models, key=lambda model: (not model._meta.proxy, model._meta.label)) for models in candidates.values()
|
|
513
|
+
]
|
|
514
|
+
return sorted(owners, key=lambda model: model._meta.label)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def register_triggers():
|
|
518
|
+
"""
|
|
519
|
+
Register the three kinds of trigger for every denormalized field in the live model registry.
|
|
520
|
+
|
|
521
|
+
Every leaf model gets one ``DenormalizeSetTrigger``. Every model that the leaf's lookup paths cross after the first
|
|
522
|
+
hop gets one ``DenormalizeCascadeTrigger`` and one ``DenormalizeFillTrigger`` for that leaf. The leaves are visited
|
|
523
|
+
in label order, so the triggers are added to the migration state, and to the ``makemigrations`` output, in the same
|
|
524
|
+
order on every run.
|
|
525
|
+
|
|
526
|
+
pgtrigger appends a registered trigger to ``Meta.triggers`` and to the migration state of its model only while
|
|
527
|
+
``PGTRIGGER_MIGRATIONS`` is on. System check ``postgres_denormalize.E301`` requires that.
|
|
528
|
+
|
|
529
|
+
Calling this function a second time changes nothing. The trigger classes compare equal by value, and pgtrigger's
|
|
530
|
+
registry ignores a trigger that is equal to a trigger that it already has under the same name.
|
|
531
|
+
"""
|
|
532
|
+
for leaf in sorted(apps.get_models(), key=lambda model: model._meta.label):
|
|
533
|
+
register_triggers_for(leaf)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def register_triggers_for(leaf):
|
|
537
|
+
"""
|
|
538
|
+
Register the three kinds of trigger for the denormalized fields of one leaf model.
|
|
539
|
+
|
|
540
|
+
A field whose lookup path does not resolve is skipped, and the field reports it as a system check error. When none
|
|
541
|
+
of the leaf's fields resolves, the leaf gets no trigger at all.
|
|
542
|
+
"""
|
|
543
|
+
resolvable = resolvable_fields(leaf)
|
|
544
|
+
if not resolvable:
|
|
545
|
+
return
|
|
546
|
+
DenormalizeSetTrigger().register(leaf)
|
|
547
|
+
for owner in owners_of(resolvable):
|
|
548
|
+
DenormalizeCascadeTrigger(leaf_label=leaf._meta.label).register(owner)
|
|
549
|
+
DenormalizeFillTrigger(leaf_label=leaf._meta.label).register(owner)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def cascade_triggers_on(owners):
|
|
553
|
+
"""
|
|
554
|
+
Return ``(owner, leaf, trigger)`` for every cascade trigger on the tables of the models in ``owners``, in label
|
|
555
|
+
order.
|
|
556
|
+
|
|
557
|
+
``owner`` is the model that the trigger is registered on. That is a proxy model when a hop points at the proxy, and
|
|
558
|
+
the caller can name either the proxy or the concrete model.
|
|
559
|
+
|
|
560
|
+
The triggers are read from pgtrigger's registry, where ``register_triggers`` put them, so no lookup path is resolved
|
|
561
|
+
again. A model that has no cascade trigger raises ``ValueError``. No lookup path reads one of its columns after a
|
|
562
|
+
first hop, so naming it is a mistake. Usually a leaf was named instead of its owner.
|
|
563
|
+
"""
|
|
564
|
+
registered = [
|
|
565
|
+
(model, trigger) for model, trigger in pgtrigger.registered() if isinstance(trigger, DenormalizeCascadeTrigger)
|
|
566
|
+
]
|
|
567
|
+
found = []
|
|
568
|
+
for owner in sorted(set(owners), key=lambda model: model._meta.label):
|
|
569
|
+
table_label = owner._meta.concrete_model._meta.label
|
|
570
|
+
on_owner = [
|
|
571
|
+
(model, apps.get_model(trigger.leaf_label), trigger)
|
|
572
|
+
for model, trigger in sorted(registered, key=lambda pair: pair[1].leaf_label)
|
|
573
|
+
if model._meta.concrete_model._meta.label == table_label
|
|
574
|
+
]
|
|
575
|
+
if not on_owner:
|
|
576
|
+
raise ValueError(
|
|
577
|
+
f'{owner._meta.label} has no cascade trigger: no lookup path reads one of its columns after a first '
|
|
578
|
+
f'hop. Name the models whose rows the batch updates, not the leaves that copy from them.'
|
|
579
|
+
)
|
|
580
|
+
found.extend(on_owner)
|
|
581
|
+
# A caller that names both a proxy model and its concrete model finds the same triggers twice.
|
|
582
|
+
return list(dict.fromkeys(found))
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
__all__ = [
|
|
586
|
+
'DEFERRED_CASCADES_SETTING',
|
|
587
|
+
'DenormalizeCascadeTrigger',
|
|
588
|
+
'DenormalizeFillTrigger',
|
|
589
|
+
'DenormalizeSetTrigger',
|
|
590
|
+
'OwnerTrigger',
|
|
591
|
+
'cascade_trigger_name',
|
|
592
|
+
'cascade_triggers_on',
|
|
593
|
+
'owners_of',
|
|
594
|
+
'register_triggers',
|
|
595
|
+
'register_triggers_for',
|
|
596
|
+
'resolvable_fields',
|
|
597
|
+
]
|