django-bulk-hooks 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-bulk-hooks might be problematic. Click here for more details.
- django_bulk_hooks/operations/bulk_executor.py +281 -2
- django_bulk_hooks/operations/mti_handler.py +374 -4
- django_bulk_hooks/operations/mti_plans.py +87 -0
- {django_bulk_hooks-0.2.1.dist-info → django_bulk_hooks-0.2.2.dist-info}/METADATA +1 -1
- {django_bulk_hooks-0.2.1.dist-info → django_bulk_hooks-0.2.2.dist-info}/RECORD +7 -6
- {django_bulk_hooks-0.2.1.dist-info → django_bulk_hooks-0.2.2.dist-info}/LICENSE +0 -0
- {django_bulk_hooks-0.2.1.dist-info → django_bulk_hooks-0.2.2.dist-info}/WHEEL +0 -0
|
@@ -6,6 +6,7 @@ This service coordinates bulk database operations with validation and MTI handli
|
|
|
6
6
|
|
|
7
7
|
import logging
|
|
8
8
|
from django.db import transaction
|
|
9
|
+
from django.db.models import AutoField
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
@@ -65,7 +66,21 @@ class BulkExecutor:
|
|
|
65
66
|
if not objs:
|
|
66
67
|
return objs
|
|
67
68
|
|
|
68
|
-
#
|
|
69
|
+
# Check if this is an MTI model and route accordingly
|
|
70
|
+
if self.mti_handler.is_mti_model():
|
|
71
|
+
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk create")
|
|
72
|
+
# Build execution plan
|
|
73
|
+
plan = self.mti_handler.build_create_plan(
|
|
74
|
+
objs,
|
|
75
|
+
batch_size=batch_size,
|
|
76
|
+
update_conflicts=update_conflicts,
|
|
77
|
+
update_fields=update_fields,
|
|
78
|
+
unique_fields=unique_fields,
|
|
79
|
+
)
|
|
80
|
+
# Execute the plan
|
|
81
|
+
return self._execute_mti_create_plan(plan)
|
|
82
|
+
|
|
83
|
+
# Non-MTI model - use Django's native bulk_create
|
|
69
84
|
return self._execute_bulk_create(
|
|
70
85
|
objs,
|
|
71
86
|
batch_size,
|
|
@@ -124,13 +139,277 @@ class BulkExecutor:
|
|
|
124
139
|
if not objs:
|
|
125
140
|
return 0
|
|
126
141
|
|
|
127
|
-
#
|
|
142
|
+
# Check if this is an MTI model and route accordingly
|
|
143
|
+
if self.mti_handler.is_mti_model():
|
|
144
|
+
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk update")
|
|
145
|
+
# Build execution plan
|
|
146
|
+
plan = self.mti_handler.build_update_plan(objs, fields, batch_size=batch_size)
|
|
147
|
+
# Execute the plan
|
|
148
|
+
return self._execute_mti_update_plan(plan)
|
|
149
|
+
|
|
150
|
+
# Non-MTI model - use Django's native bulk_update
|
|
128
151
|
# Validation already done by coordinator
|
|
129
152
|
from django.db.models import QuerySet
|
|
130
153
|
|
|
131
154
|
base_qs = QuerySet(model=self.model_cls, using=self.queryset.db)
|
|
132
155
|
return base_qs.bulk_update(objs, fields, batch_size=batch_size)
|
|
133
156
|
|
|
157
|
+
# ==================== MTI PLAN EXECUTION ====================
|
|
158
|
+
|
|
159
|
+
def _execute_mti_create_plan(self, plan):
|
|
160
|
+
"""
|
|
161
|
+
Execute an MTI create plan.
|
|
162
|
+
|
|
163
|
+
This is where ALL database operations happen for MTI bulk_create.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
plan: MTICreatePlan object from MTIHandler
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
List of created objects with PKs assigned
|
|
170
|
+
"""
|
|
171
|
+
from django.db import transaction
|
|
172
|
+
from django.db.models import QuerySet as BaseQuerySet
|
|
173
|
+
|
|
174
|
+
if not plan:
|
|
175
|
+
return []
|
|
176
|
+
|
|
177
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
178
|
+
# Step 1: Create all parent objects level by level
|
|
179
|
+
parent_instances_map = {} # Maps original obj id() -> {model: parent_instance}
|
|
180
|
+
|
|
181
|
+
for parent_level in plan.parent_levels:
|
|
182
|
+
# Bulk create parents for this level
|
|
183
|
+
bulk_kwargs = {"batch_size": len(parent_level.objects)}
|
|
184
|
+
|
|
185
|
+
if parent_level.update_conflicts:
|
|
186
|
+
bulk_kwargs["update_conflicts"] = True
|
|
187
|
+
bulk_kwargs["unique_fields"] = parent_level.unique_fields
|
|
188
|
+
bulk_kwargs["update_fields"] = parent_level.update_fields
|
|
189
|
+
|
|
190
|
+
# Use base QuerySet to avoid recursion
|
|
191
|
+
base_qs = BaseQuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
192
|
+
created_parents = base_qs.bulk_create(parent_level.objects, **bulk_kwargs)
|
|
193
|
+
|
|
194
|
+
# Copy generated fields back to parent objects
|
|
195
|
+
for created_parent, parent_obj in zip(created_parents, parent_level.objects):
|
|
196
|
+
for field in parent_level.model_class._meta.local_fields:
|
|
197
|
+
created_value = getattr(created_parent, field.name, None)
|
|
198
|
+
if created_value is not None:
|
|
199
|
+
setattr(parent_obj, field.name, created_value)
|
|
200
|
+
|
|
201
|
+
parent_obj._state.adding = False
|
|
202
|
+
parent_obj._state.db = self.queryset.db
|
|
203
|
+
|
|
204
|
+
# Map parents back to original objects
|
|
205
|
+
for parent_obj in parent_level.objects:
|
|
206
|
+
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
207
|
+
if orig_obj_id not in parent_instances_map:
|
|
208
|
+
parent_instances_map[orig_obj_id] = {}
|
|
209
|
+
parent_instances_map[orig_obj_id][parent_level.model_class] = parent_obj
|
|
210
|
+
|
|
211
|
+
# Step 2: Add parent links to child objects
|
|
212
|
+
for child_obj, orig_obj in zip(plan.child_objects, plan.original_objects):
|
|
213
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
214
|
+
|
|
215
|
+
for parent_model, parent_instance in parent_instances.items():
|
|
216
|
+
parent_link = plan.child_model._meta.get_ancestor_link(parent_model)
|
|
217
|
+
if parent_link:
|
|
218
|
+
setattr(child_obj, parent_link.attname, parent_instance.pk)
|
|
219
|
+
setattr(child_obj, parent_link.name, parent_instance)
|
|
220
|
+
|
|
221
|
+
# Step 3: Bulk create child objects using _batched_insert (to bypass MTI check)
|
|
222
|
+
base_qs = BaseQuerySet(model=plan.child_model, using=self.queryset.db)
|
|
223
|
+
base_qs._prepare_for_bulk_create(plan.child_objects)
|
|
224
|
+
|
|
225
|
+
# Partition objects by PK status
|
|
226
|
+
objs_without_pk, objs_with_pk = [], []
|
|
227
|
+
for obj in plan.child_objects:
|
|
228
|
+
if obj._is_pk_set():
|
|
229
|
+
objs_with_pk.append(obj)
|
|
230
|
+
else:
|
|
231
|
+
objs_without_pk.append(obj)
|
|
232
|
+
|
|
233
|
+
# Get fields for insert
|
|
234
|
+
opts = plan.child_model._meta
|
|
235
|
+
fields = [f for f in opts.local_fields if not f.generated]
|
|
236
|
+
|
|
237
|
+
# Execute bulk insert
|
|
238
|
+
if objs_with_pk:
|
|
239
|
+
returned_columns = base_qs._batched_insert(
|
|
240
|
+
objs_with_pk,
|
|
241
|
+
fields,
|
|
242
|
+
batch_size=len(objs_with_pk),
|
|
243
|
+
)
|
|
244
|
+
if returned_columns:
|
|
245
|
+
for obj, results in zip(objs_with_pk, returned_columns):
|
|
246
|
+
if hasattr(opts, "db_returning_fields") and hasattr(opts, "pk"):
|
|
247
|
+
for result, field in zip(results, opts.db_returning_fields):
|
|
248
|
+
if field != opts.pk:
|
|
249
|
+
setattr(obj, field.attname, result)
|
|
250
|
+
obj._state.adding = False
|
|
251
|
+
obj._state.db = self.queryset.db
|
|
252
|
+
else:
|
|
253
|
+
for obj in objs_with_pk:
|
|
254
|
+
obj._state.adding = False
|
|
255
|
+
obj._state.db = self.queryset.db
|
|
256
|
+
|
|
257
|
+
if objs_without_pk:
|
|
258
|
+
filtered_fields = [
|
|
259
|
+
f for f in fields
|
|
260
|
+
if not isinstance(f, AutoField) and not f.primary_key
|
|
261
|
+
]
|
|
262
|
+
returned_columns = base_qs._batched_insert(
|
|
263
|
+
objs_without_pk,
|
|
264
|
+
filtered_fields,
|
|
265
|
+
batch_size=len(objs_without_pk),
|
|
266
|
+
)
|
|
267
|
+
if returned_columns:
|
|
268
|
+
for obj, results in zip(objs_without_pk, returned_columns):
|
|
269
|
+
if hasattr(opts, "db_returning_fields"):
|
|
270
|
+
for result, field in zip(results, opts.db_returning_fields):
|
|
271
|
+
setattr(obj, field.attname, result)
|
|
272
|
+
obj._state.adding = False
|
|
273
|
+
obj._state.db = self.queryset.db
|
|
274
|
+
else:
|
|
275
|
+
for obj in objs_without_pk:
|
|
276
|
+
obj._state.adding = False
|
|
277
|
+
obj._state.db = self.queryset.db
|
|
278
|
+
|
|
279
|
+
created_children = plan.child_objects
|
|
280
|
+
|
|
281
|
+
# Step 4: Copy PKs and auto-generated fields back to original objects
|
|
282
|
+
pk_field_name = plan.child_model._meta.pk.name
|
|
283
|
+
|
|
284
|
+
for orig_obj, child_obj in zip(plan.original_objects, created_children):
|
|
285
|
+
# Copy PK
|
|
286
|
+
child_pk = getattr(child_obj, pk_field_name)
|
|
287
|
+
setattr(orig_obj, pk_field_name, child_pk)
|
|
288
|
+
|
|
289
|
+
# Copy auto-generated fields from all levels
|
|
290
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
291
|
+
|
|
292
|
+
for model_class in plan.inheritance_chain:
|
|
293
|
+
# Get source object for this level
|
|
294
|
+
if model_class in parent_instances:
|
|
295
|
+
source_obj = parent_instances[model_class]
|
|
296
|
+
elif model_class == plan.child_model:
|
|
297
|
+
source_obj = child_obj
|
|
298
|
+
else:
|
|
299
|
+
continue
|
|
300
|
+
|
|
301
|
+
# Copy auto-generated field values
|
|
302
|
+
for field in model_class._meta.local_fields:
|
|
303
|
+
if field.name == pk_field_name:
|
|
304
|
+
continue
|
|
305
|
+
|
|
306
|
+
# Skip parent link fields
|
|
307
|
+
if hasattr(field, 'remote_field') and field.remote_field:
|
|
308
|
+
parent_link = plan.child_model._meta.get_ancestor_link(model_class)
|
|
309
|
+
if parent_link and field.name == parent_link.name:
|
|
310
|
+
continue
|
|
311
|
+
|
|
312
|
+
# Copy auto_now_add, auto_now, and db_returning fields
|
|
313
|
+
if (getattr(field, 'auto_now_add', False) or
|
|
314
|
+
getattr(field, 'auto_now', False) or
|
|
315
|
+
getattr(field, 'db_returning', False)):
|
|
316
|
+
source_value = getattr(source_obj, field.name, None)
|
|
317
|
+
if source_value is not None:
|
|
318
|
+
setattr(orig_obj, field.name, source_value)
|
|
319
|
+
|
|
320
|
+
# Update object state
|
|
321
|
+
orig_obj._state.adding = False
|
|
322
|
+
orig_obj._state.db = self.queryset.db
|
|
323
|
+
|
|
324
|
+
return plan.original_objects
|
|
325
|
+
|
|
326
|
+
def _execute_mti_update_plan(self, plan):
|
|
327
|
+
"""
|
|
328
|
+
Execute an MTI update plan.
|
|
329
|
+
|
|
330
|
+
Updates each table in the inheritance chain using CASE/WHEN for bulk updates.
|
|
331
|
+
|
|
332
|
+
Args:
|
|
333
|
+
plan: MTIUpdatePlan object from MTIHandler
|
|
334
|
+
|
|
335
|
+
Returns:
|
|
336
|
+
Number of objects updated
|
|
337
|
+
"""
|
|
338
|
+
from django.db import transaction
|
|
339
|
+
from django.db.models import Case, Value, When, QuerySet as BaseQuerySet
|
|
340
|
+
|
|
341
|
+
if not plan:
|
|
342
|
+
return 0
|
|
343
|
+
|
|
344
|
+
total_updated = 0
|
|
345
|
+
|
|
346
|
+
# Get PKs for filtering
|
|
347
|
+
root_pks = [
|
|
348
|
+
getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
349
|
+
for obj in plan.objects
|
|
350
|
+
if getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
351
|
+
]
|
|
352
|
+
|
|
353
|
+
if not root_pks:
|
|
354
|
+
return 0
|
|
355
|
+
|
|
356
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
357
|
+
# Update each table in the chain
|
|
358
|
+
for field_group in plan.field_groups:
|
|
359
|
+
if not field_group.fields:
|
|
360
|
+
continue
|
|
361
|
+
|
|
362
|
+
base_qs = BaseQuerySet(model=field_group.model_class, using=self.queryset.db)
|
|
363
|
+
|
|
364
|
+
# Check if records exist
|
|
365
|
+
existing_count = base_qs.filter(**{f"{field_group.filter_field}__in": root_pks}).count()
|
|
366
|
+
if existing_count == 0:
|
|
367
|
+
continue
|
|
368
|
+
|
|
369
|
+
# Build CASE statements for bulk update
|
|
370
|
+
case_statements = {}
|
|
371
|
+
for field_name in field_group.fields:
|
|
372
|
+
field = field_group.model_class._meta.get_field(field_name)
|
|
373
|
+
|
|
374
|
+
# Use column name for FK fields
|
|
375
|
+
if getattr(field, 'is_relation', False) and hasattr(field, 'attname'):
|
|
376
|
+
db_field_name = field.attname
|
|
377
|
+
target_field = field.target_field
|
|
378
|
+
else:
|
|
379
|
+
db_field_name = field_name
|
|
380
|
+
target_field = field
|
|
381
|
+
|
|
382
|
+
when_statements = []
|
|
383
|
+
for pk, obj in zip(root_pks, plan.objects):
|
|
384
|
+
obj_pk = getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
385
|
+
if obj_pk is None:
|
|
386
|
+
continue
|
|
387
|
+
|
|
388
|
+
value = getattr(obj, db_field_name)
|
|
389
|
+
when_statements.append(
|
|
390
|
+
When(
|
|
391
|
+
**{field_group.filter_field: pk},
|
|
392
|
+
then=Value(value, output_field=target_field),
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
if when_statements:
|
|
397
|
+
case_statements[db_field_name] = Case(
|
|
398
|
+
*when_statements, output_field=target_field
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
# Execute bulk update
|
|
402
|
+
if case_statements:
|
|
403
|
+
try:
|
|
404
|
+
updated_count = base_qs.filter(
|
|
405
|
+
**{f"{field_group.filter_field}__in": root_pks}
|
|
406
|
+
).update(**case_statements)
|
|
407
|
+
total_updated += updated_count
|
|
408
|
+
except Exception as e:
|
|
409
|
+
logger.error(f"MTI bulk update failed for {field_group.model_class.__name__}: {e}")
|
|
410
|
+
|
|
411
|
+
return total_updated
|
|
412
|
+
|
|
134
413
|
def delete_queryset(self):
|
|
135
414
|
"""
|
|
136
415
|
Execute delete on the queryset.
|
|
@@ -1,20 +1,30 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Multi-table inheritance (MTI) handler service.
|
|
3
3
|
|
|
4
|
-
Handles detection and
|
|
4
|
+
Handles detection and planning for multi-table inheritance operations.
|
|
5
|
+
|
|
6
|
+
This handler is PURE LOGIC - it does not execute database operations.
|
|
7
|
+
It returns plans (data structures) that the BulkExecutor executes.
|
|
5
8
|
"""
|
|
6
9
|
|
|
7
10
|
import logging
|
|
11
|
+
from django.db.models import AutoField
|
|
8
12
|
|
|
9
13
|
logger = logging.getLogger(__name__)
|
|
10
14
|
|
|
11
15
|
|
|
12
16
|
class MTIHandler:
|
|
13
17
|
"""
|
|
14
|
-
Handles multi-table inheritance (MTI)
|
|
18
|
+
Handles multi-table inheritance (MTI) operation planning.
|
|
15
19
|
|
|
16
|
-
This service detects MTI models and
|
|
17
|
-
|
|
20
|
+
This service detects MTI models and builds execution plans.
|
|
21
|
+
It does NOT execute database operations - that's the BulkExecutor's job.
|
|
22
|
+
|
|
23
|
+
Responsibilities:
|
|
24
|
+
- Detect MTI models
|
|
25
|
+
- Build inheritance chains
|
|
26
|
+
- Create parent/child instances (in-memory only)
|
|
27
|
+
- Return execution plans
|
|
18
28
|
"""
|
|
19
29
|
|
|
20
30
|
def __init__(self, model_cls):
|
|
@@ -101,3 +111,363 @@ class MTIHandler:
|
|
|
101
111
|
list: Field objects defined on this model
|
|
102
112
|
"""
|
|
103
113
|
return list(model_cls._meta.local_fields)
|
|
114
|
+
|
|
115
|
+
# ==================== MTI BULK CREATE PLANNING ====================
|
|
116
|
+
|
|
117
|
+
def build_create_plan(
|
|
118
|
+
self,
|
|
119
|
+
objs,
|
|
120
|
+
batch_size=None,
|
|
121
|
+
update_conflicts=False,
|
|
122
|
+
unique_fields=None,
|
|
123
|
+
update_fields=None,
|
|
124
|
+
):
|
|
125
|
+
"""
|
|
126
|
+
Build an execution plan for bulk creating MTI model instances.
|
|
127
|
+
|
|
128
|
+
This method does NOT execute any database operations.
|
|
129
|
+
It returns a plan that the BulkExecutor will execute.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
objs: List of model instances to create
|
|
133
|
+
batch_size: Number of objects per batch
|
|
134
|
+
update_conflicts: Enable UPSERT on conflict
|
|
135
|
+
unique_fields: Fields for conflict detection
|
|
136
|
+
update_fields: Fields to update on conflict
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
MTICreatePlan object
|
|
140
|
+
"""
|
|
141
|
+
from django_bulk_hooks.operations.mti_plans import MTICreatePlan, ParentLevel
|
|
142
|
+
|
|
143
|
+
if not objs:
|
|
144
|
+
return None
|
|
145
|
+
|
|
146
|
+
inheritance_chain = self.get_inheritance_chain()
|
|
147
|
+
if len(inheritance_chain) <= 1:
|
|
148
|
+
raise ValueError("build_create_plan called on non-MTI model")
|
|
149
|
+
|
|
150
|
+
batch_size = batch_size or len(objs)
|
|
151
|
+
|
|
152
|
+
# Build parent levels
|
|
153
|
+
parent_levels = self._build_parent_levels(
|
|
154
|
+
objs,
|
|
155
|
+
inheritance_chain,
|
|
156
|
+
update_conflicts=update_conflicts,
|
|
157
|
+
unique_fields=unique_fields,
|
|
158
|
+
update_fields=update_fields,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Build child object templates (without parent links - executor adds them)
|
|
162
|
+
child_objects = []
|
|
163
|
+
for obj in objs:
|
|
164
|
+
child_obj = self._create_child_instance_template(obj, inheritance_chain[-1])
|
|
165
|
+
child_objects.append(child_obj)
|
|
166
|
+
|
|
167
|
+
return MTICreatePlan(
|
|
168
|
+
inheritance_chain=inheritance_chain,
|
|
169
|
+
parent_levels=parent_levels,
|
|
170
|
+
child_objects=child_objects,
|
|
171
|
+
child_model=inheritance_chain[-1],
|
|
172
|
+
original_objects=objs,
|
|
173
|
+
batch_size=batch_size,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
def _build_parent_levels(
|
|
177
|
+
self,
|
|
178
|
+
objs,
|
|
179
|
+
inheritance_chain,
|
|
180
|
+
update_conflicts=False,
|
|
181
|
+
unique_fields=None,
|
|
182
|
+
update_fields=None,
|
|
183
|
+
):
|
|
184
|
+
"""
|
|
185
|
+
Build parent level objects for each level in the inheritance chain.
|
|
186
|
+
|
|
187
|
+
This is pure in-memory object creation - no DB operations.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
List of ParentLevel objects
|
|
191
|
+
"""
|
|
192
|
+
from django_bulk_hooks.operations.mti_plans import ParentLevel
|
|
193
|
+
|
|
194
|
+
parent_levels = []
|
|
195
|
+
parent_instances_map = {} # Maps obj id() -> {model_class: parent_instance}
|
|
196
|
+
|
|
197
|
+
for level_idx, model_class in enumerate(inheritance_chain[:-1]):
|
|
198
|
+
parent_objs_for_level = []
|
|
199
|
+
|
|
200
|
+
for obj in objs:
|
|
201
|
+
# Get current parent from previous level
|
|
202
|
+
current_parent = None
|
|
203
|
+
if level_idx > 0:
|
|
204
|
+
prev_parents = parent_instances_map.get(id(obj), {})
|
|
205
|
+
current_parent = prev_parents.get(inheritance_chain[level_idx - 1])
|
|
206
|
+
|
|
207
|
+
# Create parent instance
|
|
208
|
+
parent_obj = self._create_parent_instance(obj, model_class, current_parent)
|
|
209
|
+
parent_objs_for_level.append(parent_obj)
|
|
210
|
+
|
|
211
|
+
# Store in map
|
|
212
|
+
if id(obj) not in parent_instances_map:
|
|
213
|
+
parent_instances_map[id(obj)] = {}
|
|
214
|
+
parent_instances_map[id(obj)][model_class] = parent_obj
|
|
215
|
+
|
|
216
|
+
# Determine upsert parameters for this level
|
|
217
|
+
level_update_conflicts = False
|
|
218
|
+
level_unique_fields = []
|
|
219
|
+
level_update_fields = []
|
|
220
|
+
|
|
221
|
+
if update_conflicts and unique_fields:
|
|
222
|
+
# Filter unique_fields and update_fields to only those in this model
|
|
223
|
+
model_fields_by_name = {f.name: f for f in model_class._meta.local_fields}
|
|
224
|
+
|
|
225
|
+
# Normalize unique fields
|
|
226
|
+
normalized_unique = []
|
|
227
|
+
for uf in unique_fields or []:
|
|
228
|
+
if uf in model_fields_by_name:
|
|
229
|
+
normalized_unique.append(uf)
|
|
230
|
+
elif uf.endswith("_id") and uf[:-3] in model_fields_by_name:
|
|
231
|
+
normalized_unique.append(uf[:-3])
|
|
232
|
+
|
|
233
|
+
# Check if this model has a matching constraint
|
|
234
|
+
if normalized_unique and self._has_matching_constraint(model_class, normalized_unique):
|
|
235
|
+
# Filter update fields
|
|
236
|
+
filtered_updates = [
|
|
237
|
+
uf for uf in (update_fields or []) if uf in model_fields_by_name
|
|
238
|
+
]
|
|
239
|
+
|
|
240
|
+
if filtered_updates:
|
|
241
|
+
level_update_conflicts = True
|
|
242
|
+
level_unique_fields = normalized_unique
|
|
243
|
+
level_update_fields = filtered_updates
|
|
244
|
+
|
|
245
|
+
# Create parent level
|
|
246
|
+
parent_level = ParentLevel(
|
|
247
|
+
model_class=model_class,
|
|
248
|
+
objects=parent_objs_for_level,
|
|
249
|
+
original_object_map={id(p): id(o) for p, o in zip(parent_objs_for_level, objs)},
|
|
250
|
+
update_conflicts=level_update_conflicts,
|
|
251
|
+
unique_fields=level_unique_fields,
|
|
252
|
+
update_fields=level_update_fields,
|
|
253
|
+
)
|
|
254
|
+
parent_levels.append(parent_level)
|
|
255
|
+
|
|
256
|
+
return parent_levels
|
|
257
|
+
|
|
258
|
+
def _has_matching_constraint(self, model_class, normalized_unique):
|
|
259
|
+
"""Check if model has a unique constraint matching the given fields."""
|
|
260
|
+
try:
|
|
261
|
+
from django.db.models import UniqueConstraint
|
|
262
|
+
constraint_field_sets = [
|
|
263
|
+
tuple(c.fields) for c in model_class._meta.constraints
|
|
264
|
+
if isinstance(c, UniqueConstraint)
|
|
265
|
+
]
|
|
266
|
+
except Exception:
|
|
267
|
+
constraint_field_sets = []
|
|
268
|
+
|
|
269
|
+
# Check unique_together
|
|
270
|
+
ut = getattr(model_class._meta, "unique_together", ()) or ()
|
|
271
|
+
if isinstance(ut, tuple) and ut and not isinstance(ut[0], (list, tuple)):
|
|
272
|
+
ut = (ut,)
|
|
273
|
+
ut_field_sets = [tuple(group) for group in ut]
|
|
274
|
+
|
|
275
|
+
# Compare as sets
|
|
276
|
+
provided_set = set(normalized_unique)
|
|
277
|
+
for group in constraint_field_sets + ut_field_sets:
|
|
278
|
+
if provided_set == set(group):
|
|
279
|
+
return True
|
|
280
|
+
return False
|
|
281
|
+
|
|
282
|
+
def _create_parent_instance(self, source_obj, parent_model, current_parent):
|
|
283
|
+
"""
|
|
284
|
+
Create a parent instance from source object (in-memory only).
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
source_obj: Original object with data
|
|
288
|
+
parent_model: Parent model class to create instance of
|
|
289
|
+
current_parent: Parent instance from previous level (if any)
|
|
290
|
+
|
|
291
|
+
Returns:
|
|
292
|
+
Parent model instance (not saved)
|
|
293
|
+
"""
|
|
294
|
+
parent_obj = parent_model()
|
|
295
|
+
|
|
296
|
+
# Copy field values from source
|
|
297
|
+
for field in parent_model._meta.local_fields:
|
|
298
|
+
if hasattr(source_obj, field.name):
|
|
299
|
+
value = getattr(source_obj, field.name, None)
|
|
300
|
+
if value is not None:
|
|
301
|
+
if (field.is_relation and not field.many_to_many and
|
|
302
|
+
not field.one_to_many):
|
|
303
|
+
# Handle FK fields
|
|
304
|
+
if hasattr(value, "pk") and value.pk is not None:
|
|
305
|
+
setattr(parent_obj, field.attname, value.pk)
|
|
306
|
+
else:
|
|
307
|
+
setattr(parent_obj, field.attname, value)
|
|
308
|
+
else:
|
|
309
|
+
setattr(parent_obj, field.name, value)
|
|
310
|
+
|
|
311
|
+
# Link to parent if exists
|
|
312
|
+
if current_parent is not None:
|
|
313
|
+
for field in parent_model._meta.local_fields:
|
|
314
|
+
if (hasattr(field, "remote_field") and field.remote_field and
|
|
315
|
+
field.remote_field.model == current_parent.__class__):
|
|
316
|
+
setattr(parent_obj, field.name, current_parent)
|
|
317
|
+
break
|
|
318
|
+
|
|
319
|
+
# Copy object state
|
|
320
|
+
if hasattr(source_obj, '_state') and hasattr(parent_obj, '_state'):
|
|
321
|
+
parent_obj._state.adding = source_obj._state.adding
|
|
322
|
+
if hasattr(source_obj._state, 'db'):
|
|
323
|
+
parent_obj._state.db = source_obj._state.db
|
|
324
|
+
|
|
325
|
+
# Handle auto_now_add and auto_now fields
|
|
326
|
+
for field in parent_model._meta.local_fields:
|
|
327
|
+
if getattr(field, 'auto_now_add', False):
|
|
328
|
+
if getattr(parent_obj, field.name) is None:
|
|
329
|
+
field.pre_save(parent_obj, add=True)
|
|
330
|
+
setattr(parent_obj, field.attname, field.value_from_object(parent_obj))
|
|
331
|
+
elif getattr(field, 'auto_now', False):
|
|
332
|
+
field.pre_save(parent_obj, add=True)
|
|
333
|
+
|
|
334
|
+
return parent_obj
|
|
335
|
+
|
|
336
|
+
def _create_child_instance_template(self, source_obj, child_model):
|
|
337
|
+
"""
|
|
338
|
+
Create a child instance template (in-memory only, without parent links).
|
|
339
|
+
|
|
340
|
+
The executor will add parent links after creating parent objects.
|
|
341
|
+
|
|
342
|
+
Args:
|
|
343
|
+
source_obj: Original object with data
|
|
344
|
+
child_model: Child model class
|
|
345
|
+
|
|
346
|
+
Returns:
|
|
347
|
+
Child model instance (not saved, no parent links)
|
|
348
|
+
"""
|
|
349
|
+
child_obj = child_model()
|
|
350
|
+
|
|
351
|
+
# Copy field values (excluding AutoField and parent links)
|
|
352
|
+
for field in child_model._meta.local_fields:
|
|
353
|
+
if isinstance(field, AutoField):
|
|
354
|
+
continue
|
|
355
|
+
|
|
356
|
+
# Skip parent link fields - executor will set these
|
|
357
|
+
if field.is_relation and hasattr(field, 'related_model'):
|
|
358
|
+
# Check if this field is a parent link
|
|
359
|
+
if child_model._meta.get_ancestor_link(field.related_model) == field:
|
|
360
|
+
continue
|
|
361
|
+
|
|
362
|
+
if hasattr(source_obj, field.name):
|
|
363
|
+
value = getattr(source_obj, field.name, None)
|
|
364
|
+
if value is not None:
|
|
365
|
+
if (field.is_relation and not field.many_to_many and
|
|
366
|
+
not field.one_to_many):
|
|
367
|
+
if hasattr(value, "pk") and value.pk is not None:
|
|
368
|
+
setattr(child_obj, field.attname, value.pk)
|
|
369
|
+
else:
|
|
370
|
+
setattr(child_obj, field.attname, value)
|
|
371
|
+
else:
|
|
372
|
+
setattr(child_obj, field.name, value)
|
|
373
|
+
|
|
374
|
+
# Copy object state
|
|
375
|
+
if hasattr(source_obj, '_state') and hasattr(child_obj, '_state'):
|
|
376
|
+
child_obj._state.adding = source_obj._state.adding
|
|
377
|
+
if hasattr(source_obj._state, 'db'):
|
|
378
|
+
child_obj._state.db = source_obj._state.db
|
|
379
|
+
|
|
380
|
+
# Handle auto_now_add and auto_now fields
|
|
381
|
+
for field in child_model._meta.local_fields:
|
|
382
|
+
if getattr(field, 'auto_now_add', False):
|
|
383
|
+
if getattr(child_obj, field.name) is None:
|
|
384
|
+
field.pre_save(child_obj, add=True)
|
|
385
|
+
setattr(child_obj, field.attname, field.value_from_object(child_obj))
|
|
386
|
+
elif getattr(field, 'auto_now', False):
|
|
387
|
+
field.pre_save(child_obj, add=True)
|
|
388
|
+
|
|
389
|
+
return child_obj
|
|
390
|
+
|
|
391
|
+
# ==================== MTI BULK UPDATE PLANNING ====================
|
|
392
|
+
|
|
393
|
+
def build_update_plan(self, objs, fields, batch_size=None):
|
|
394
|
+
"""
|
|
395
|
+
Build an execution plan for bulk updating MTI model instances.
|
|
396
|
+
|
|
397
|
+
This method does NOT execute any database operations.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
objs: List of model instances to update
|
|
401
|
+
fields: List of field names to update
|
|
402
|
+
batch_size: Number of objects per batch
|
|
403
|
+
|
|
404
|
+
Returns:
|
|
405
|
+
MTIUpdatePlan object
|
|
406
|
+
"""
|
|
407
|
+
from django_bulk_hooks.operations.mti_plans import MTIUpdatePlan, ModelFieldGroup
|
|
408
|
+
|
|
409
|
+
if not objs:
|
|
410
|
+
return None
|
|
411
|
+
|
|
412
|
+
inheritance_chain = self.get_inheritance_chain()
|
|
413
|
+
if len(inheritance_chain) <= 1:
|
|
414
|
+
raise ValueError("build_update_plan called on non-MTI model")
|
|
415
|
+
|
|
416
|
+
batch_size = batch_size or len(objs)
|
|
417
|
+
|
|
418
|
+
# Handle auto_now fields
|
|
419
|
+
for obj in objs:
|
|
420
|
+
for model in inheritance_chain:
|
|
421
|
+
for field in model._meta.local_fields:
|
|
422
|
+
if getattr(field, 'auto_now', False):
|
|
423
|
+
field.pre_save(obj, add=False)
|
|
424
|
+
|
|
425
|
+
# Add auto_now fields to update list
|
|
426
|
+
auto_now_fields = set()
|
|
427
|
+
for model in inheritance_chain:
|
|
428
|
+
for field in model._meta.local_fields:
|
|
429
|
+
if getattr(field, 'auto_now', False):
|
|
430
|
+
auto_now_fields.add(field.name)
|
|
431
|
+
|
|
432
|
+
all_fields = list(fields) + list(auto_now_fields)
|
|
433
|
+
|
|
434
|
+
# Group fields by model
|
|
435
|
+
field_groups = []
|
|
436
|
+
for model_idx, model in enumerate(inheritance_chain):
|
|
437
|
+
model_fields = []
|
|
438
|
+
|
|
439
|
+
for field_name in all_fields:
|
|
440
|
+
try:
|
|
441
|
+
field = self.model_cls._meta.get_field(field_name)
|
|
442
|
+
if field in model._meta.local_fields:
|
|
443
|
+
# Skip auto_now_add fields for updates
|
|
444
|
+
if not getattr(field, 'auto_now_add', False):
|
|
445
|
+
model_fields.append(field_name)
|
|
446
|
+
except Exception:
|
|
447
|
+
continue
|
|
448
|
+
|
|
449
|
+
if model_fields:
|
|
450
|
+
# Determine filter field
|
|
451
|
+
if model_idx == 0:
|
|
452
|
+
filter_field = "pk"
|
|
453
|
+
else:
|
|
454
|
+
# Find parent link
|
|
455
|
+
parent_link = None
|
|
456
|
+
for parent_model in inheritance_chain:
|
|
457
|
+
if parent_model in model._meta.parents:
|
|
458
|
+
parent_link = model._meta.parents[parent_model]
|
|
459
|
+
break
|
|
460
|
+
filter_field = parent_link.attname if parent_link else "pk"
|
|
461
|
+
|
|
462
|
+
field_groups.append(ModelFieldGroup(
|
|
463
|
+
model_class=model,
|
|
464
|
+
fields=model_fields,
|
|
465
|
+
filter_field=filter_field,
|
|
466
|
+
))
|
|
467
|
+
|
|
468
|
+
return MTIUpdatePlan(
|
|
469
|
+
inheritance_chain=inheritance_chain,
|
|
470
|
+
field_groups=field_groups,
|
|
471
|
+
objects=objs,
|
|
472
|
+
batch_size=batch_size,
|
|
473
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MTI operation plans - Data structures for multi-table inheritance operations.
|
|
3
|
+
|
|
4
|
+
These are pure data structures returned by MTIHandler to be executed by BulkExecutor.
|
|
5
|
+
This separates planning (logic) from execution (database operations).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Dict, List, Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class ParentLevel:
|
|
14
|
+
"""
|
|
15
|
+
Represents one level in the parent hierarchy for MTI bulk create.
|
|
16
|
+
|
|
17
|
+
Attributes:
|
|
18
|
+
model_class: The parent model class for this level
|
|
19
|
+
objects: List of parent instances to create
|
|
20
|
+
original_object_map: Maps parent instance id() -> original object id()
|
|
21
|
+
update_conflicts: Whether to enable UPSERT for this level
|
|
22
|
+
unique_fields: Fields for conflict detection (if update_conflicts=True)
|
|
23
|
+
update_fields: Fields to update on conflict (if update_conflicts=True)
|
|
24
|
+
"""
|
|
25
|
+
model_class: Any
|
|
26
|
+
objects: List[Any]
|
|
27
|
+
original_object_map: Dict[int, int] = field(default_factory=dict)
|
|
28
|
+
update_conflicts: bool = False
|
|
29
|
+
unique_fields: List[str] = field(default_factory=list)
|
|
30
|
+
update_fields: List[str] = field(default_factory=list)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class MTICreatePlan:
|
|
35
|
+
"""
|
|
36
|
+
Plan for executing bulk_create on an MTI model.
|
|
37
|
+
|
|
38
|
+
This plan describes WHAT to create, not HOW to create it.
|
|
39
|
+
The executor is responsible for executing this plan.
|
|
40
|
+
|
|
41
|
+
Attributes:
|
|
42
|
+
inheritance_chain: List of model classes from root to child
|
|
43
|
+
parent_levels: List of ParentLevel objects, one per parent model
|
|
44
|
+
child_objects: List of child instances to create (not yet with parent links)
|
|
45
|
+
child_model: The child model class
|
|
46
|
+
original_objects: Original objects provided by user
|
|
47
|
+
batch_size: Batch size for operations
|
|
48
|
+
"""
|
|
49
|
+
inheritance_chain: List[Any]
|
|
50
|
+
parent_levels: List[ParentLevel]
|
|
51
|
+
child_objects: List[Any]
|
|
52
|
+
child_model: Any
|
|
53
|
+
original_objects: List[Any]
|
|
54
|
+
batch_size: int = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class ModelFieldGroup:
|
|
59
|
+
"""
|
|
60
|
+
Represents fields to update for one model in the inheritance chain.
|
|
61
|
+
|
|
62
|
+
Attributes:
|
|
63
|
+
model_class: The model class
|
|
64
|
+
fields: List of field names to update on this model
|
|
65
|
+
filter_field: Field to use for filtering (e.g., 'pk' or parent link attname)
|
|
66
|
+
"""
|
|
67
|
+
model_class: Any
|
|
68
|
+
fields: List[str]
|
|
69
|
+
filter_field: str = "pk"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class MTIUpdatePlan:
|
|
74
|
+
"""
|
|
75
|
+
Plan for executing bulk_update on an MTI model.
|
|
76
|
+
|
|
77
|
+
Attributes:
|
|
78
|
+
inheritance_chain: List of model classes from root to child
|
|
79
|
+
field_groups: List of ModelFieldGroup objects
|
|
80
|
+
objects: Objects to update
|
|
81
|
+
batch_size: Batch size for operations
|
|
82
|
+
"""
|
|
83
|
+
inheritance_chain: List[Any]
|
|
84
|
+
field_groups: List[ModelFieldGroup]
|
|
85
|
+
objects: List[Any]
|
|
86
|
+
batch_size: int = None
|
|
87
|
+
|
|
@@ -14,12 +14,13 @@ django_bulk_hooks/manager.py,sha256=3mFzB0ZzHHeXWdKGObZD_H0NlskHJc8uYBF69KKdAXU,
|
|
|
14
14
|
django_bulk_hooks/models.py,sha256=62tn5wL55EjJVOsZofMluhEJB8bH7CzBvH0vd214_RY,2570
|
|
15
15
|
django_bulk_hooks/operations/__init__.py,sha256=5L5NnwiFw8Yn5WO6-38eGdCYBkA0URpwyDcAdeYfc5w,550
|
|
16
16
|
django_bulk_hooks/operations/analyzer.py,sha256=S9qcLRM_VBR6Cy_ObUq0Mok8bp07ALLPDF_S0Yypi2k,6507
|
|
17
|
-
django_bulk_hooks/operations/bulk_executor.py,sha256=
|
|
17
|
+
django_bulk_hooks/operations/bulk_executor.py,sha256=PuRVS5OlOysZ3qEHMsadr06rZt5CoZL6tgzqBAvDQxY,17825
|
|
18
18
|
django_bulk_hooks/operations/coordinator.py,sha256=HMJyvntKXo4aAOwElrvS0F05zoOllfPvYakdAr6JCkk,12326
|
|
19
|
-
django_bulk_hooks/operations/mti_handler.py,sha256=
|
|
19
|
+
django_bulk_hooks/operations/mti_handler.py,sha256=eIH-tImMqcWR5lLQr6Ca-HeVYta-UkXk5X5fcpS885Y,18245
|
|
20
|
+
django_bulk_hooks/operations/mti_plans.py,sha256=fHUYbrUAHq8UXqxgAD43oHdTxOnEkmpxoOD4Qrzfqk8,2878
|
|
20
21
|
django_bulk_hooks/queryset.py,sha256=ody4MXrRREL27Ts2ey1UpS0tb5Dxnw-6kN3unxPQ3zY,5860
|
|
21
22
|
django_bulk_hooks/registry.py,sha256=UPerNhtVz_9tKZqrYSZD2LhjAcs4F6hVUuk8L5oOeHc,8821
|
|
22
|
-
django_bulk_hooks-0.2.
|
|
23
|
-
django_bulk_hooks-0.2.
|
|
24
|
-
django_bulk_hooks-0.2.
|
|
25
|
-
django_bulk_hooks-0.2.
|
|
23
|
+
django_bulk_hooks-0.2.2.dist-info/LICENSE,sha256=dguKIcbDGeZD-vXWdLyErPUALYOvtX_fO4Zjhq481uk,1088
|
|
24
|
+
django_bulk_hooks-0.2.2.dist-info/METADATA,sha256=m8XX5tJbiTgwP9fMCajuhOEvDJRxoqJqiiyzTUpdY50,9264
|
|
25
|
+
django_bulk_hooks-0.2.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
26
|
+
django_bulk_hooks-0.2.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|