django-bulk-hooks 0.2.41__py3-none-any.whl → 0.2.42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-bulk-hooks might be problematic. Click here for more details.
- django_bulk_hooks/dispatcher.py +242 -255
- django_bulk_hooks/factory.py +541 -563
- django_bulk_hooks/handler.py +106 -114
- django_bulk_hooks/operations/bulk_executor.py +512 -576
- django_bulk_hooks/operations/mti_handler.py +5 -4
- django_bulk_hooks/queryset.py +188 -191
- django_bulk_hooks/registry.py +277 -298
- {django_bulk_hooks-0.2.41.dist-info → django_bulk_hooks-0.2.42.dist-info}/METADATA +1 -1
- {django_bulk_hooks-0.2.41.dist-info → django_bulk_hooks-0.2.42.dist-info}/RECORD +11 -11
- {django_bulk_hooks-0.2.41.dist-info → django_bulk_hooks-0.2.42.dist-info}/LICENSE +0 -0
- {django_bulk_hooks-0.2.41.dist-info → django_bulk_hooks-0.2.42.dist-info}/WHEEL +0 -0
|
@@ -1,576 +1,512 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Bulk executor service for database operations.
|
|
3
|
-
|
|
4
|
-
This service coordinates bulk database operations with validation and MTI handling.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
import logging
|
|
8
|
-
|
|
9
|
-
from django.db import transaction
|
|
10
|
-
from django.db.models import AutoField, ForeignKey, Case, When, Value
|
|
11
|
-
from django.db.models.functions import Cast
|
|
12
|
-
|
|
13
|
-
logger = logging.getLogger(__name__)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class BulkExecutor:
|
|
17
|
-
"""
|
|
18
|
-
Executes bulk database operations.
|
|
19
|
-
|
|
20
|
-
This service coordinates validation, MTI handling, and actual database
|
|
21
|
-
operations. It's the only service that directly calls Django ORM methods.
|
|
22
|
-
|
|
23
|
-
Dependencies are explicitly injected via constructor.
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
def __init__(self, queryset, analyzer, mti_handler, record_classifier):
|
|
27
|
-
"""
|
|
28
|
-
Initialize bulk executor with explicit dependencies.
|
|
29
|
-
|
|
30
|
-
Args:
|
|
31
|
-
queryset: Django QuerySet instance
|
|
32
|
-
analyzer: ModelAnalyzer instance (replaces validator + field_tracker)
|
|
33
|
-
mti_handler: MTIHandler instance
|
|
34
|
-
record_classifier: RecordClassifier instance
|
|
35
|
-
"""
|
|
36
|
-
self.queryset = queryset
|
|
37
|
-
self.analyzer = analyzer
|
|
38
|
-
self.mti_handler = mti_handler
|
|
39
|
-
self.record_classifier = record_classifier
|
|
40
|
-
self.model_cls = queryset.model
|
|
41
|
-
|
|
42
|
-
def bulk_create(
|
|
43
|
-
self,
|
|
44
|
-
objs,
|
|
45
|
-
batch_size=None,
|
|
46
|
-
ignore_conflicts=False,
|
|
47
|
-
update_conflicts=False,
|
|
48
|
-
update_fields=None,
|
|
49
|
-
unique_fields=None,
|
|
50
|
-
**kwargs,
|
|
51
|
-
):
|
|
52
|
-
"""
|
|
53
|
-
Execute bulk create operation.
|
|
54
|
-
|
|
55
|
-
NOTE: Coordinator is responsible for validation before calling this method.
|
|
56
|
-
This executor trusts that inputs have already been validated.
|
|
57
|
-
|
|
58
|
-
Args:
|
|
59
|
-
objs: List of model instances to create (pre-validated)
|
|
60
|
-
batch_size: Number of objects to create per batch
|
|
61
|
-
ignore_conflicts: Whether to ignore conflicts
|
|
62
|
-
update_conflicts: Whether to update on conflict
|
|
63
|
-
update_fields: Fields to update on conflict
|
|
64
|
-
unique_fields: Fields to use for conflict detection
|
|
65
|
-
**kwargs: Additional arguments
|
|
66
|
-
|
|
67
|
-
Returns:
|
|
68
|
-
List of created objects
|
|
69
|
-
"""
|
|
70
|
-
if not objs:
|
|
71
|
-
return objs
|
|
72
|
-
|
|
73
|
-
# Check if this is an MTI model and route accordingly
|
|
74
|
-
if self.mti_handler.is_mti_model():
|
|
75
|
-
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk create")
|
|
76
|
-
|
|
77
|
-
# Classify records using the classifier service
|
|
78
|
-
existing_record_ids = set()
|
|
79
|
-
existing_pks_map = {}
|
|
80
|
-
if update_conflicts and unique_fields:
|
|
81
|
-
existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields)
|
|
82
|
-
|
|
83
|
-
# Build execution plan with classification results
|
|
84
|
-
plan = self.mti_handler.build_create_plan(
|
|
85
|
-
objs,
|
|
86
|
-
batch_size=batch_size,
|
|
87
|
-
update_conflicts=update_conflicts,
|
|
88
|
-
update_fields=update_fields,
|
|
89
|
-
unique_fields=unique_fields,
|
|
90
|
-
existing_record_ids=existing_record_ids,
|
|
91
|
-
existing_pks_map=existing_pks_map,
|
|
92
|
-
)
|
|
93
|
-
# Execute the plan
|
|
94
|
-
return self._execute_mti_create_plan(plan)
|
|
95
|
-
|
|
96
|
-
# Non-MTI model - use Django's native bulk_create
|
|
97
|
-
return self._execute_bulk_create(
|
|
98
|
-
objs,
|
|
99
|
-
batch_size,
|
|
100
|
-
ignore_conflicts,
|
|
101
|
-
update_conflicts,
|
|
102
|
-
update_fields,
|
|
103
|
-
unique_fields,
|
|
104
|
-
**kwargs,
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
def _execute_bulk_create(
|
|
108
|
-
self,
|
|
109
|
-
objs,
|
|
110
|
-
batch_size=None,
|
|
111
|
-
ignore_conflicts=False,
|
|
112
|
-
update_conflicts=False,
|
|
113
|
-
update_fields=None,
|
|
114
|
-
unique_fields=None,
|
|
115
|
-
**kwargs,
|
|
116
|
-
):
|
|
117
|
-
"""
|
|
118
|
-
Execute the actual Django bulk_create.
|
|
119
|
-
|
|
120
|
-
This is the only method that directly calls Django ORM.
|
|
121
|
-
We must call the base Django QuerySet to avoid recursion.
|
|
122
|
-
"""
|
|
123
|
-
from django.db.models import QuerySet
|
|
124
|
-
|
|
125
|
-
# Create a base Django queryset (not our HookQuerySet)
|
|
126
|
-
base_qs = QuerySet(model=self.model_cls, using=self.queryset.db)
|
|
127
|
-
|
|
128
|
-
return base_qs.bulk_create(
|
|
129
|
-
objs,
|
|
130
|
-
batch_size=batch_size,
|
|
131
|
-
ignore_conflicts=ignore_conflicts,
|
|
132
|
-
update_conflicts=update_conflicts,
|
|
133
|
-
update_fields=update_fields,
|
|
134
|
-
unique_fields=unique_fields,
|
|
135
|
-
)
|
|
136
|
-
|
|
137
|
-
def bulk_update(self, objs, fields, batch_size=None):
|
|
138
|
-
"""
|
|
139
|
-
Execute bulk update operation.
|
|
140
|
-
|
|
141
|
-
NOTE: Coordinator is responsible for validation before calling this method.
|
|
142
|
-
This executor trusts that inputs have already been validated.
|
|
143
|
-
|
|
144
|
-
Args:
|
|
145
|
-
objs: List of model instances to update (pre-validated)
|
|
146
|
-
fields: List of field names to update
|
|
147
|
-
batch_size: Number of objects to update per batch
|
|
148
|
-
|
|
149
|
-
Returns:
|
|
150
|
-
Number of objects updated
|
|
151
|
-
"""
|
|
152
|
-
if not objs:
|
|
153
|
-
return 0
|
|
154
|
-
|
|
155
|
-
# Check if this is an MTI model and route accordingly
|
|
156
|
-
if self.mti_handler.is_mti_model():
|
|
157
|
-
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk update")
|
|
158
|
-
# Build execution plan
|
|
159
|
-
plan = self.mti_handler.build_update_plan(objs, fields, batch_size=batch_size)
|
|
160
|
-
# Execute the plan
|
|
161
|
-
return self._execute_mti_update_plan(plan)
|
|
162
|
-
|
|
163
|
-
# Non-MTI model - use Django's native bulk_update
|
|
164
|
-
# Validation already done by coordinator
|
|
165
|
-
from django.db.models import QuerySet
|
|
166
|
-
|
|
167
|
-
base_qs = QuerySet(model=self.model_cls, using=self.queryset.db)
|
|
168
|
-
return base_qs.bulk_update(objs, fields, batch_size=batch_size)
|
|
169
|
-
|
|
170
|
-
# ==================== MTI PLAN EXECUTION ====================
|
|
171
|
-
|
|
172
|
-
def _execute_mti_create_plan(self, plan):
|
|
173
|
-
"""
|
|
174
|
-
Execute an MTI create plan.
|
|
175
|
-
|
|
176
|
-
This is where ALL database operations happen for MTI bulk_create.
|
|
177
|
-
Handles both new records (INSERT) and existing records (UPDATE) for upsert.
|
|
178
|
-
|
|
179
|
-
Args:
|
|
180
|
-
plan: MTICreatePlan object from MTIHandler
|
|
181
|
-
|
|
182
|
-
Returns:
|
|
183
|
-
List of created/updated objects with PKs assigned
|
|
184
|
-
"""
|
|
185
|
-
from django.db.models import QuerySet as BaseQuerySet
|
|
186
|
-
|
|
187
|
-
if not plan:
|
|
188
|
-
return []
|
|
189
|
-
|
|
190
|
-
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
191
|
-
# Step 1: Create/Update all parent objects level by level
|
|
192
|
-
parent_instances_map = {} # Maps original obj id() -> {model: parent_instance}
|
|
193
|
-
|
|
194
|
-
for parent_level in plan.parent_levels:
|
|
195
|
-
# Separate new and existing parent objects
|
|
196
|
-
new_parents = []
|
|
197
|
-
existing_parents = []
|
|
198
|
-
|
|
199
|
-
for parent_obj in parent_level.objects:
|
|
200
|
-
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
201
|
-
if orig_obj_id in plan.existing_record_ids:
|
|
202
|
-
existing_parents.append(parent_obj)
|
|
203
|
-
else:
|
|
204
|
-
new_parents.append(parent_obj)
|
|
205
|
-
|
|
206
|
-
# Bulk create new parents
|
|
207
|
-
if new_parents:
|
|
208
|
-
bulk_kwargs = {"batch_size": len(new_parents)}
|
|
209
|
-
|
|
210
|
-
if parent_level.update_conflicts:
|
|
211
|
-
bulk_kwargs["update_conflicts"] = True
|
|
212
|
-
bulk_kwargs["unique_fields"] = parent_level.unique_fields
|
|
213
|
-
bulk_kwargs["update_fields"] = parent_level.update_fields
|
|
214
|
-
|
|
215
|
-
# Use base QuerySet to avoid recursion
|
|
216
|
-
base_qs = BaseQuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
217
|
-
created_parents = base_qs.bulk_create(new_parents, **bulk_kwargs)
|
|
218
|
-
|
|
219
|
-
# Copy generated fields back to parent objects
|
|
220
|
-
for created_parent, parent_obj in zip(created_parents, new_parents):
|
|
221
|
-
for field in parent_level.model_class._meta.local_fields:
|
|
222
|
-
created_value = getattr(created_parent, field.name, None)
|
|
223
|
-
if created_value is not None:
|
|
224
|
-
setattr(parent_obj, field.name, created_value)
|
|
225
|
-
|
|
226
|
-
parent_obj._state.adding = False
|
|
227
|
-
parent_obj._state.db = self.queryset.db
|
|
228
|
-
|
|
229
|
-
# Update existing parents
|
|
230
|
-
if existing_parents and parent_level.update_fields:
|
|
231
|
-
# Filter update fields to only those that exist in this parent model
|
|
232
|
-
parent_model_fields = {field.name for field in parent_level.model_class._meta.local_fields}
|
|
233
|
-
filtered_update_fields = [field for field in parent_level.update_fields if field in parent_model_fields]
|
|
234
|
-
|
|
235
|
-
if filtered_update_fields:
|
|
236
|
-
base_qs = BaseQuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
237
|
-
base_qs.bulk_update(existing_parents, filtered_update_fields)
|
|
238
|
-
|
|
239
|
-
# Mark as not adding
|
|
240
|
-
for parent_obj in existing_parents:
|
|
241
|
-
parent_obj._state.adding = False
|
|
242
|
-
parent_obj._state.db = self.queryset.db
|
|
243
|
-
|
|
244
|
-
# Map parents back to original objects
|
|
245
|
-
for parent_obj in parent_level.objects:
|
|
246
|
-
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
247
|
-
if orig_obj_id not in parent_instances_map:
|
|
248
|
-
parent_instances_map[orig_obj_id] = {}
|
|
249
|
-
parent_instances_map[orig_obj_id][parent_level.model_class] = parent_obj
|
|
250
|
-
|
|
251
|
-
# Step 2: Add parent links to child objects and separate new/existing
|
|
252
|
-
new_child_objects = []
|
|
253
|
-
existing_child_objects = []
|
|
254
|
-
|
|
255
|
-
for child_obj, orig_obj in zip(plan.child_objects, plan.original_objects):
|
|
256
|
-
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
257
|
-
|
|
258
|
-
# Set parent links
|
|
259
|
-
for parent_model, parent_instance in parent_instances.items():
|
|
260
|
-
parent_link = plan.child_model._meta.get_ancestor_link(parent_model)
|
|
261
|
-
if parent_link:
|
|
262
|
-
setattr(child_obj, parent_link.attname, parent_instance.pk)
|
|
263
|
-
setattr(child_obj, parent_link.name, parent_instance)
|
|
264
|
-
|
|
265
|
-
# Classify as new or existing
|
|
266
|
-
if id(orig_obj) in plan.existing_record_ids:
|
|
267
|
-
# For existing records, set the PK on child object
|
|
268
|
-
pk_value = getattr(orig_obj, "pk", None)
|
|
269
|
-
if pk_value:
|
|
270
|
-
child_obj.pk = pk_value
|
|
271
|
-
child_obj.id = pk_value
|
|
272
|
-
existing_child_objects.append(child_obj)
|
|
273
|
-
else:
|
|
274
|
-
new_child_objects.append(child_obj)
|
|
275
|
-
|
|
276
|
-
# Step 3: Bulk create new child objects using _batched_insert (to bypass MTI check)
|
|
277
|
-
if new_child_objects:
|
|
278
|
-
base_qs = BaseQuerySet(model=plan.child_model, using=self.queryset.db)
|
|
279
|
-
base_qs._prepare_for_bulk_create(new_child_objects)
|
|
280
|
-
|
|
281
|
-
# Partition objects by PK status
|
|
282
|
-
objs_without_pk, objs_with_pk = [], []
|
|
283
|
-
for obj in new_child_objects:
|
|
284
|
-
if obj._is_pk_set():
|
|
285
|
-
objs_with_pk.append(obj)
|
|
286
|
-
else:
|
|
287
|
-
objs_without_pk.append(obj)
|
|
288
|
-
|
|
289
|
-
# Get fields for insert
|
|
290
|
-
opts = plan.child_model._meta
|
|
291
|
-
fields = [f for f in opts.local_fields if not f.generated]
|
|
292
|
-
|
|
293
|
-
# Execute bulk insert
|
|
294
|
-
if objs_with_pk:
|
|
295
|
-
returned_columns = base_qs._batched_insert(
|
|
296
|
-
objs_with_pk,
|
|
297
|
-
fields,
|
|
298
|
-
batch_size=len(objs_with_pk),
|
|
299
|
-
)
|
|
300
|
-
if returned_columns:
|
|
301
|
-
for obj, results in zip(objs_with_pk, returned_columns):
|
|
302
|
-
if hasattr(opts, "db_returning_fields") and hasattr(opts, "pk"):
|
|
303
|
-
for result, field in zip(results, opts.db_returning_fields):
|
|
304
|
-
if field != opts.pk:
|
|
305
|
-
setattr(obj, field.attname, result)
|
|
306
|
-
obj._state.adding = False
|
|
307
|
-
obj._state.db = self.queryset.db
|
|
308
|
-
else:
|
|
309
|
-
for obj in objs_with_pk:
|
|
310
|
-
obj._state.adding = False
|
|
311
|
-
obj._state.db = self.queryset.db
|
|
312
|
-
|
|
313
|
-
if objs_without_pk:
|
|
314
|
-
filtered_fields = [f for f in fields if not isinstance(f, AutoField) and not f.primary_key]
|
|
315
|
-
returned_columns = base_qs._batched_insert(
|
|
316
|
-
objs_without_pk,
|
|
317
|
-
filtered_fields,
|
|
318
|
-
batch_size=len(objs_without_pk),
|
|
319
|
-
)
|
|
320
|
-
if returned_columns:
|
|
321
|
-
for obj, results in zip(objs_without_pk, returned_columns):
|
|
322
|
-
if hasattr(opts, "db_returning_fields"):
|
|
323
|
-
for result, field in zip(results, opts.db_returning_fields):
|
|
324
|
-
setattr(obj, field.attname, result)
|
|
325
|
-
obj._state.adding = False
|
|
326
|
-
obj._state.db = self.queryset.db
|
|
327
|
-
else:
|
|
328
|
-
for obj in objs_without_pk:
|
|
329
|
-
obj._state.adding = False
|
|
330
|
-
obj._state.db = self.queryset.db
|
|
331
|
-
|
|
332
|
-
# Step 3.5: Update existing child objects
|
|
333
|
-
if existing_child_objects and plan.update_fields:
|
|
334
|
-
# Filter update fields to only those that exist in the child model
|
|
335
|
-
child_model_fields = {field.name for field in plan.child_model._meta.local_fields}
|
|
336
|
-
filtered_child_update_fields = [field for field in plan.update_fields if field in child_model_fields]
|
|
337
|
-
|
|
338
|
-
if filtered_child_update_fields:
|
|
339
|
-
base_qs = BaseQuerySet(model=plan.child_model, using=self.queryset.db)
|
|
340
|
-
base_qs.bulk_update(existing_child_objects, filtered_child_update_fields)
|
|
341
|
-
|
|
342
|
-
# Mark as not adding
|
|
343
|
-
for child_obj in existing_child_objects:
|
|
344
|
-
child_obj._state.adding = False
|
|
345
|
-
child_obj._state.db = self.queryset.db
|
|
346
|
-
|
|
347
|
-
# Combine all children for final processing
|
|
348
|
-
created_children = new_child_objects + existing_child_objects
|
|
349
|
-
|
|
350
|
-
# Step 4: Copy PKs and auto-generated fields back to original objects
|
|
351
|
-
pk_field_name = plan.child_model._meta.pk.name
|
|
352
|
-
|
|
353
|
-
for orig_obj, child_obj in zip(plan.original_objects, created_children):
|
|
354
|
-
# Copy PK
|
|
355
|
-
child_pk = getattr(child_obj, pk_field_name)
|
|
356
|
-
setattr(orig_obj, pk_field_name, child_pk)
|
|
357
|
-
|
|
358
|
-
# Copy auto-generated fields from all levels
|
|
359
|
-
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
360
|
-
|
|
361
|
-
for model_class in plan.inheritance_chain:
|
|
362
|
-
# Get source object for this level
|
|
363
|
-
if model_class in parent_instances:
|
|
364
|
-
source_obj = parent_instances[model_class]
|
|
365
|
-
elif model_class == plan.child_model:
|
|
366
|
-
source_obj = child_obj
|
|
367
|
-
else:
|
|
368
|
-
continue
|
|
369
|
-
|
|
370
|
-
# Copy auto-generated field values
|
|
371
|
-
for field in model_class._meta.local_fields:
|
|
372
|
-
if field.name == pk_field_name:
|
|
373
|
-
continue
|
|
374
|
-
|
|
375
|
-
# Skip parent link fields
|
|
376
|
-
if hasattr(field, "remote_field") and field.remote_field:
|
|
377
|
-
parent_link = plan.child_model._meta.get_ancestor_link(model_class)
|
|
378
|
-
if parent_link and field.name == parent_link.name:
|
|
379
|
-
continue
|
|
380
|
-
|
|
381
|
-
# Copy auto_now_add, auto_now, and db_returning fields
|
|
382
|
-
if (
|
|
383
|
-
getattr(field, "auto_now_add", False)
|
|
384
|
-
or getattr(field, "auto_now", False)
|
|
385
|
-
or getattr(field, "db_returning", False)
|
|
386
|
-
):
|
|
387
|
-
source_value = getattr(source_obj, field.name, None)
|
|
388
|
-
if source_value is not None:
|
|
389
|
-
setattr(orig_obj, field.name, source_value)
|
|
390
|
-
|
|
391
|
-
# Update object state
|
|
392
|
-
orig_obj._state.adding = False
|
|
393
|
-
orig_obj._state.db = self.queryset.db
|
|
394
|
-
|
|
395
|
-
return plan.original_objects
|
|
396
|
-
|
|
397
|
-
def _execute_mti_update_plan(self, plan):
|
|
398
|
-
"""
|
|
399
|
-
Execute an MTI update plan.
|
|
400
|
-
|
|
401
|
-
Updates each table in the inheritance chain using CASE/WHEN for bulk updates.
|
|
402
|
-
|
|
403
|
-
Args:
|
|
404
|
-
plan: MTIUpdatePlan object from MTIHandler
|
|
405
|
-
|
|
406
|
-
Returns:
|
|
407
|
-
Number of objects updated
|
|
408
|
-
"""
|
|
409
|
-
from django.db.models import Case
|
|
410
|
-
from django.db.models import QuerySet as BaseQuerySet
|
|
411
|
-
from django.db.models import Value
|
|
412
|
-
from django.db.models import When
|
|
413
|
-
|
|
414
|
-
if not plan:
|
|
415
|
-
return 0
|
|
416
|
-
|
|
417
|
-
total_updated = 0
|
|
418
|
-
|
|
419
|
-
# Get PKs for filtering
|
|
420
|
-
root_pks = [
|
|
421
|
-
getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
422
|
-
for obj in plan.objects
|
|
423
|
-
if getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
424
|
-
]
|
|
425
|
-
|
|
426
|
-
if not root_pks:
|
|
427
|
-
return 0
|
|
428
|
-
|
|
429
|
-
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
430
|
-
# Update each table in the chain
|
|
431
|
-
for field_group in plan.field_groups:
|
|
432
|
-
if not field_group.fields:
|
|
433
|
-
continue
|
|
434
|
-
|
|
435
|
-
base_qs = BaseQuerySet(model=field_group.model_class, using=self.queryset.db)
|
|
436
|
-
|
|
437
|
-
# Check if records exist
|
|
438
|
-
existing_count = base_qs.filter(**{f"{field_group.filter_field}__in": root_pks}).count()
|
|
439
|
-
if existing_count == 0:
|
|
440
|
-
continue
|
|
441
|
-
|
|
442
|
-
# Build CASE statements for bulk update
|
|
443
|
-
case_statements = {}
|
|
444
|
-
for field_name in field_group.fields:
|
|
445
|
-
field = field_group.model_class._meta.get_field(field_name)
|
|
446
|
-
when_statements = []
|
|
447
|
-
|
|
448
|
-
# Determine the correct output field for type casting
|
|
449
|
-
# For ForeignKey fields, use the target field to ensure correct SQL types
|
|
450
|
-
is_fk = isinstance(field, ForeignKey)
|
|
451
|
-
case_output_field = field.target_field if is_fk else field
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
),
|
|
514
|
-
)
|
|
515
|
-
|
|
516
|
-
if when_statements:
|
|
517
|
-
case_statements[field_name] = Case(*when_statements, output_field=case_output_field)
|
|
518
|
-
logger.info(
|
|
519
|
-
f"DEBUG Case Statement - Field: {field_name}, "
|
|
520
|
-
f"CaseOutputField: {type(case_output_field).__name__}, "
|
|
521
|
-
f"NumWhenStatements: {len(when_statements)}"
|
|
522
|
-
)
|
|
523
|
-
|
|
524
|
-
# DEBUG: Log the actual When statements to see what's being generated
|
|
525
|
-
for i, when_stmt in enumerate(when_statements):
|
|
526
|
-
logger.info(
|
|
527
|
-
f"DEBUG When Statement {i} - Field: {field_name}, "
|
|
528
|
-
f"WhenCondition: {when_stmt.condition}, "
|
|
529
|
-
f"WhenThen: {when_stmt.result}, "
|
|
530
|
-
f"WhenThenType: {type(when_stmt.result).__name__}"
|
|
531
|
-
)
|
|
532
|
-
|
|
533
|
-
# DEBUG: Check if the When result has output_field
|
|
534
|
-
if hasattr(when_stmt.result, 'output_field'):
|
|
535
|
-
logger.info(
|
|
536
|
-
f"DEBUG When Result OutputField - Field: {field_name}, "
|
|
537
|
-
f"WhenIndex: {i}, "
|
|
538
|
-
f"ResultOutputField: {type(when_stmt.result.output_field).__name__}"
|
|
539
|
-
)
|
|
540
|
-
else:
|
|
541
|
-
logger.info(
|
|
542
|
-
f"DEBUG When Result No OutputField - Field: {field_name}, "
|
|
543
|
-
f"WhenIndex: {i}, "
|
|
544
|
-
f"ResultType: {type(when_stmt.result).__name__}"
|
|
545
|
-
)
|
|
546
|
-
|
|
547
|
-
# Execute bulk update
|
|
548
|
-
if case_statements:
|
|
549
|
-
try:
|
|
550
|
-
updated_count = base_qs.filter(
|
|
551
|
-
**{f"{field_group.filter_field}__in": root_pks},
|
|
552
|
-
).update(**case_statements)
|
|
553
|
-
total_updated += updated_count
|
|
554
|
-
except Exception as e:
|
|
555
|
-
logger.error(f"MTI bulk update failed for {field_group.model_class.__name__}: {e}")
|
|
556
|
-
|
|
557
|
-
return total_updated
|
|
558
|
-
|
|
559
|
-
def delete_queryset(self):
|
|
560
|
-
"""
|
|
561
|
-
Execute delete on the queryset.
|
|
562
|
-
|
|
563
|
-
NOTE: Coordinator is responsible for validation before calling this method.
|
|
564
|
-
This executor trusts that inputs have already been validated.
|
|
565
|
-
|
|
566
|
-
Returns:
|
|
567
|
-
Tuple of (count, details dict)
|
|
568
|
-
"""
|
|
569
|
-
if not self.queryset:
|
|
570
|
-
return 0, {}
|
|
571
|
-
|
|
572
|
-
# Execute delete via QuerySet
|
|
573
|
-
# Validation already done by coordinator
|
|
574
|
-
from django.db.models import QuerySet
|
|
575
|
-
|
|
576
|
-
return QuerySet.delete(self.queryset)
|
|
1
|
+
"""
|
|
2
|
+
Bulk executor service for database operations.
|
|
3
|
+
|
|
4
|
+
This service coordinates bulk database operations with validation and MTI handling.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
from django.db import transaction
|
|
10
|
+
from django.db.models import AutoField, ForeignKey, Case, When, Value
|
|
11
|
+
from django.db.models.functions import Cast
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BulkExecutor:
|
|
17
|
+
"""
|
|
18
|
+
Executes bulk database operations.
|
|
19
|
+
|
|
20
|
+
This service coordinates validation, MTI handling, and actual database
|
|
21
|
+
operations. It's the only service that directly calls Django ORM methods.
|
|
22
|
+
|
|
23
|
+
Dependencies are explicitly injected via constructor.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, queryset, analyzer, mti_handler, record_classifier):
|
|
27
|
+
"""
|
|
28
|
+
Initialize bulk executor with explicit dependencies.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
queryset: Django QuerySet instance
|
|
32
|
+
analyzer: ModelAnalyzer instance (replaces validator + field_tracker)
|
|
33
|
+
mti_handler: MTIHandler instance
|
|
34
|
+
record_classifier: RecordClassifier instance
|
|
35
|
+
"""
|
|
36
|
+
self.queryset = queryset
|
|
37
|
+
self.analyzer = analyzer
|
|
38
|
+
self.mti_handler = mti_handler
|
|
39
|
+
self.record_classifier = record_classifier
|
|
40
|
+
self.model_cls = queryset.model
|
|
41
|
+
|
|
42
|
+
def bulk_create(
|
|
43
|
+
self,
|
|
44
|
+
objs,
|
|
45
|
+
batch_size=None,
|
|
46
|
+
ignore_conflicts=False,
|
|
47
|
+
update_conflicts=False,
|
|
48
|
+
update_fields=None,
|
|
49
|
+
unique_fields=None,
|
|
50
|
+
**kwargs,
|
|
51
|
+
):
|
|
52
|
+
"""
|
|
53
|
+
Execute bulk create operation.
|
|
54
|
+
|
|
55
|
+
NOTE: Coordinator is responsible for validation before calling this method.
|
|
56
|
+
This executor trusts that inputs have already been validated.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
objs: List of model instances to create (pre-validated)
|
|
60
|
+
batch_size: Number of objects to create per batch
|
|
61
|
+
ignore_conflicts: Whether to ignore conflicts
|
|
62
|
+
update_conflicts: Whether to update on conflict
|
|
63
|
+
update_fields: Fields to update on conflict
|
|
64
|
+
unique_fields: Fields to use for conflict detection
|
|
65
|
+
**kwargs: Additional arguments
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
List of created objects
|
|
69
|
+
"""
|
|
70
|
+
if not objs:
|
|
71
|
+
return objs
|
|
72
|
+
|
|
73
|
+
# Check if this is an MTI model and route accordingly
|
|
74
|
+
if self.mti_handler.is_mti_model():
|
|
75
|
+
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk create")
|
|
76
|
+
|
|
77
|
+
# Classify records using the classifier service
|
|
78
|
+
existing_record_ids = set()
|
|
79
|
+
existing_pks_map = {}
|
|
80
|
+
if update_conflicts and unique_fields:
|
|
81
|
+
existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields)
|
|
82
|
+
|
|
83
|
+
# Build execution plan with classification results
|
|
84
|
+
plan = self.mti_handler.build_create_plan(
|
|
85
|
+
objs,
|
|
86
|
+
batch_size=batch_size,
|
|
87
|
+
update_conflicts=update_conflicts,
|
|
88
|
+
update_fields=update_fields,
|
|
89
|
+
unique_fields=unique_fields,
|
|
90
|
+
existing_record_ids=existing_record_ids,
|
|
91
|
+
existing_pks_map=existing_pks_map,
|
|
92
|
+
)
|
|
93
|
+
# Execute the plan
|
|
94
|
+
return self._execute_mti_create_plan(plan)
|
|
95
|
+
|
|
96
|
+
# Non-MTI model - use Django's native bulk_create
|
|
97
|
+
return self._execute_bulk_create(
|
|
98
|
+
objs,
|
|
99
|
+
batch_size,
|
|
100
|
+
ignore_conflicts,
|
|
101
|
+
update_conflicts,
|
|
102
|
+
update_fields,
|
|
103
|
+
unique_fields,
|
|
104
|
+
**kwargs,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def _execute_bulk_create(
|
|
108
|
+
self,
|
|
109
|
+
objs,
|
|
110
|
+
batch_size=None,
|
|
111
|
+
ignore_conflicts=False,
|
|
112
|
+
update_conflicts=False,
|
|
113
|
+
update_fields=None,
|
|
114
|
+
unique_fields=None,
|
|
115
|
+
**kwargs,
|
|
116
|
+
):
|
|
117
|
+
"""
|
|
118
|
+
Execute the actual Django bulk_create.
|
|
119
|
+
|
|
120
|
+
This is the only method that directly calls Django ORM.
|
|
121
|
+
We must call the base Django QuerySet to avoid recursion.
|
|
122
|
+
"""
|
|
123
|
+
from django.db.models import QuerySet
|
|
124
|
+
|
|
125
|
+
# Create a base Django queryset (not our HookQuerySet)
|
|
126
|
+
base_qs = QuerySet(model=self.model_cls, using=self.queryset.db)
|
|
127
|
+
|
|
128
|
+
return base_qs.bulk_create(
|
|
129
|
+
objs,
|
|
130
|
+
batch_size=batch_size,
|
|
131
|
+
ignore_conflicts=ignore_conflicts,
|
|
132
|
+
update_conflicts=update_conflicts,
|
|
133
|
+
update_fields=update_fields,
|
|
134
|
+
unique_fields=unique_fields,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
def bulk_update(self, objs, fields, batch_size=None):
|
|
138
|
+
"""
|
|
139
|
+
Execute bulk update operation.
|
|
140
|
+
|
|
141
|
+
NOTE: Coordinator is responsible for validation before calling this method.
|
|
142
|
+
This executor trusts that inputs have already been validated.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
objs: List of model instances to update (pre-validated)
|
|
146
|
+
fields: List of field names to update
|
|
147
|
+
batch_size: Number of objects to update per batch
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
Number of objects updated
|
|
151
|
+
"""
|
|
152
|
+
if not objs:
|
|
153
|
+
return 0
|
|
154
|
+
|
|
155
|
+
# Check if this is an MTI model and route accordingly
|
|
156
|
+
if self.mti_handler.is_mti_model():
|
|
157
|
+
logger.info(f"Detected MTI model {self.model_cls.__name__}, using MTI bulk update")
|
|
158
|
+
# Build execution plan
|
|
159
|
+
plan = self.mti_handler.build_update_plan(objs, fields, batch_size=batch_size)
|
|
160
|
+
# Execute the plan
|
|
161
|
+
return self._execute_mti_update_plan(plan)
|
|
162
|
+
|
|
163
|
+
# Non-MTI model - use Django's native bulk_update
|
|
164
|
+
# Validation already done by coordinator
|
|
165
|
+
from django.db.models import QuerySet
|
|
166
|
+
|
|
167
|
+
base_qs = QuerySet(model=self.model_cls, using=self.queryset.db)
|
|
168
|
+
return base_qs.bulk_update(objs, fields, batch_size=batch_size)
|
|
169
|
+
|
|
170
|
+
# ==================== MTI PLAN EXECUTION ====================
|
|
171
|
+
|
|
172
|
+
def _execute_mti_create_plan(self, plan):
|
|
173
|
+
"""
|
|
174
|
+
Execute an MTI create plan.
|
|
175
|
+
|
|
176
|
+
This is where ALL database operations happen for MTI bulk_create.
|
|
177
|
+
Handles both new records (INSERT) and existing records (UPDATE) for upsert.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
plan: MTICreatePlan object from MTIHandler
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
List of created/updated objects with PKs assigned
|
|
184
|
+
"""
|
|
185
|
+
from django.db.models import QuerySet as BaseQuerySet
|
|
186
|
+
|
|
187
|
+
if not plan:
|
|
188
|
+
return []
|
|
189
|
+
|
|
190
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
191
|
+
# Step 1: Create/Update all parent objects level by level
|
|
192
|
+
parent_instances_map = {} # Maps original obj id() -> {model: parent_instance}
|
|
193
|
+
|
|
194
|
+
for parent_level in plan.parent_levels:
|
|
195
|
+
# Separate new and existing parent objects
|
|
196
|
+
new_parents = []
|
|
197
|
+
existing_parents = []
|
|
198
|
+
|
|
199
|
+
for parent_obj in parent_level.objects:
|
|
200
|
+
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
201
|
+
if orig_obj_id in plan.existing_record_ids:
|
|
202
|
+
existing_parents.append(parent_obj)
|
|
203
|
+
else:
|
|
204
|
+
new_parents.append(parent_obj)
|
|
205
|
+
|
|
206
|
+
# Bulk create new parents
|
|
207
|
+
if new_parents:
|
|
208
|
+
bulk_kwargs = {"batch_size": len(new_parents)}
|
|
209
|
+
|
|
210
|
+
if parent_level.update_conflicts:
|
|
211
|
+
bulk_kwargs["update_conflicts"] = True
|
|
212
|
+
bulk_kwargs["unique_fields"] = parent_level.unique_fields
|
|
213
|
+
bulk_kwargs["update_fields"] = parent_level.update_fields
|
|
214
|
+
|
|
215
|
+
# Use base QuerySet to avoid recursion
|
|
216
|
+
base_qs = BaseQuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
217
|
+
created_parents = base_qs.bulk_create(new_parents, **bulk_kwargs)
|
|
218
|
+
|
|
219
|
+
# Copy generated fields back to parent objects
|
|
220
|
+
for created_parent, parent_obj in zip(created_parents, new_parents):
|
|
221
|
+
for field in parent_level.model_class._meta.local_fields:
|
|
222
|
+
created_value = getattr(created_parent, field.name, None)
|
|
223
|
+
if created_value is not None:
|
|
224
|
+
setattr(parent_obj, field.name, created_value)
|
|
225
|
+
|
|
226
|
+
parent_obj._state.adding = False
|
|
227
|
+
parent_obj._state.db = self.queryset.db
|
|
228
|
+
|
|
229
|
+
# Update existing parents
|
|
230
|
+
if existing_parents and parent_level.update_fields:
|
|
231
|
+
# Filter update fields to only those that exist in this parent model
|
|
232
|
+
parent_model_fields = {field.name for field in parent_level.model_class._meta.local_fields}
|
|
233
|
+
filtered_update_fields = [field for field in parent_level.update_fields if field in parent_model_fields]
|
|
234
|
+
|
|
235
|
+
if filtered_update_fields:
|
|
236
|
+
base_qs = BaseQuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
237
|
+
base_qs.bulk_update(existing_parents, filtered_update_fields)
|
|
238
|
+
|
|
239
|
+
# Mark as not adding
|
|
240
|
+
for parent_obj in existing_parents:
|
|
241
|
+
parent_obj._state.adding = False
|
|
242
|
+
parent_obj._state.db = self.queryset.db
|
|
243
|
+
|
|
244
|
+
# Map parents back to original objects
|
|
245
|
+
for parent_obj in parent_level.objects:
|
|
246
|
+
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
247
|
+
if orig_obj_id not in parent_instances_map:
|
|
248
|
+
parent_instances_map[orig_obj_id] = {}
|
|
249
|
+
parent_instances_map[orig_obj_id][parent_level.model_class] = parent_obj
|
|
250
|
+
|
|
251
|
+
# Step 2: Add parent links to child objects and separate new/existing
|
|
252
|
+
new_child_objects = []
|
|
253
|
+
existing_child_objects = []
|
|
254
|
+
|
|
255
|
+
for child_obj, orig_obj in zip(plan.child_objects, plan.original_objects):
|
|
256
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
257
|
+
|
|
258
|
+
# Set parent links
|
|
259
|
+
for parent_model, parent_instance in parent_instances.items():
|
|
260
|
+
parent_link = plan.child_model._meta.get_ancestor_link(parent_model)
|
|
261
|
+
if parent_link:
|
|
262
|
+
setattr(child_obj, parent_link.attname, parent_instance.pk)
|
|
263
|
+
setattr(child_obj, parent_link.name, parent_instance)
|
|
264
|
+
|
|
265
|
+
# Classify as new or existing
|
|
266
|
+
if id(orig_obj) in plan.existing_record_ids:
|
|
267
|
+
# For existing records, set the PK on child object
|
|
268
|
+
pk_value = getattr(orig_obj, "pk", None)
|
|
269
|
+
if pk_value:
|
|
270
|
+
child_obj.pk = pk_value
|
|
271
|
+
child_obj.id = pk_value
|
|
272
|
+
existing_child_objects.append(child_obj)
|
|
273
|
+
else:
|
|
274
|
+
new_child_objects.append(child_obj)
|
|
275
|
+
|
|
276
|
+
# Step 3: Bulk create new child objects using _batched_insert (to bypass MTI check)
|
|
277
|
+
if new_child_objects:
|
|
278
|
+
base_qs = BaseQuerySet(model=plan.child_model, using=self.queryset.db)
|
|
279
|
+
base_qs._prepare_for_bulk_create(new_child_objects)
|
|
280
|
+
|
|
281
|
+
# Partition objects by PK status
|
|
282
|
+
objs_without_pk, objs_with_pk = [], []
|
|
283
|
+
for obj in new_child_objects:
|
|
284
|
+
if obj._is_pk_set():
|
|
285
|
+
objs_with_pk.append(obj)
|
|
286
|
+
else:
|
|
287
|
+
objs_without_pk.append(obj)
|
|
288
|
+
|
|
289
|
+
# Get fields for insert
|
|
290
|
+
opts = plan.child_model._meta
|
|
291
|
+
fields = [f for f in opts.local_fields if not f.generated]
|
|
292
|
+
|
|
293
|
+
# Execute bulk insert
|
|
294
|
+
if objs_with_pk:
|
|
295
|
+
returned_columns = base_qs._batched_insert(
|
|
296
|
+
objs_with_pk,
|
|
297
|
+
fields,
|
|
298
|
+
batch_size=len(objs_with_pk),
|
|
299
|
+
)
|
|
300
|
+
if returned_columns:
|
|
301
|
+
for obj, results in zip(objs_with_pk, returned_columns):
|
|
302
|
+
if hasattr(opts, "db_returning_fields") and hasattr(opts, "pk"):
|
|
303
|
+
for result, field in zip(results, opts.db_returning_fields):
|
|
304
|
+
if field != opts.pk:
|
|
305
|
+
setattr(obj, field.attname, result)
|
|
306
|
+
obj._state.adding = False
|
|
307
|
+
obj._state.db = self.queryset.db
|
|
308
|
+
else:
|
|
309
|
+
for obj in objs_with_pk:
|
|
310
|
+
obj._state.adding = False
|
|
311
|
+
obj._state.db = self.queryset.db
|
|
312
|
+
|
|
313
|
+
if objs_without_pk:
|
|
314
|
+
filtered_fields = [f for f in fields if not isinstance(f, AutoField) and not f.primary_key]
|
|
315
|
+
returned_columns = base_qs._batched_insert(
|
|
316
|
+
objs_without_pk,
|
|
317
|
+
filtered_fields,
|
|
318
|
+
batch_size=len(objs_without_pk),
|
|
319
|
+
)
|
|
320
|
+
if returned_columns:
|
|
321
|
+
for obj, results in zip(objs_without_pk, returned_columns):
|
|
322
|
+
if hasattr(opts, "db_returning_fields"):
|
|
323
|
+
for result, field in zip(results, opts.db_returning_fields):
|
|
324
|
+
setattr(obj, field.attname, result)
|
|
325
|
+
obj._state.adding = False
|
|
326
|
+
obj._state.db = self.queryset.db
|
|
327
|
+
else:
|
|
328
|
+
for obj in objs_without_pk:
|
|
329
|
+
obj._state.adding = False
|
|
330
|
+
obj._state.db = self.queryset.db
|
|
331
|
+
|
|
332
|
+
# Step 3.5: Update existing child objects
|
|
333
|
+
if existing_child_objects and plan.update_fields:
|
|
334
|
+
# Filter update fields to only those that exist in the child model
|
|
335
|
+
child_model_fields = {field.name for field in plan.child_model._meta.local_fields}
|
|
336
|
+
filtered_child_update_fields = [field for field in plan.update_fields if field in child_model_fields]
|
|
337
|
+
|
|
338
|
+
if filtered_child_update_fields:
|
|
339
|
+
base_qs = BaseQuerySet(model=plan.child_model, using=self.queryset.db)
|
|
340
|
+
base_qs.bulk_update(existing_child_objects, filtered_child_update_fields)
|
|
341
|
+
|
|
342
|
+
# Mark as not adding
|
|
343
|
+
for child_obj in existing_child_objects:
|
|
344
|
+
child_obj._state.adding = False
|
|
345
|
+
child_obj._state.db = self.queryset.db
|
|
346
|
+
|
|
347
|
+
# Combine all children for final processing
|
|
348
|
+
created_children = new_child_objects + existing_child_objects
|
|
349
|
+
|
|
350
|
+
# Step 4: Copy PKs and auto-generated fields back to original objects
|
|
351
|
+
pk_field_name = plan.child_model._meta.pk.name
|
|
352
|
+
|
|
353
|
+
for orig_obj, child_obj in zip(plan.original_objects, created_children):
|
|
354
|
+
# Copy PK
|
|
355
|
+
child_pk = getattr(child_obj, pk_field_name)
|
|
356
|
+
setattr(orig_obj, pk_field_name, child_pk)
|
|
357
|
+
|
|
358
|
+
# Copy auto-generated fields from all levels
|
|
359
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
360
|
+
|
|
361
|
+
for model_class in plan.inheritance_chain:
|
|
362
|
+
# Get source object for this level
|
|
363
|
+
if model_class in parent_instances:
|
|
364
|
+
source_obj = parent_instances[model_class]
|
|
365
|
+
elif model_class == plan.child_model:
|
|
366
|
+
source_obj = child_obj
|
|
367
|
+
else:
|
|
368
|
+
continue
|
|
369
|
+
|
|
370
|
+
# Copy auto-generated field values
|
|
371
|
+
for field in model_class._meta.local_fields:
|
|
372
|
+
if field.name == pk_field_name:
|
|
373
|
+
continue
|
|
374
|
+
|
|
375
|
+
# Skip parent link fields
|
|
376
|
+
if hasattr(field, "remote_field") and field.remote_field:
|
|
377
|
+
parent_link = plan.child_model._meta.get_ancestor_link(model_class)
|
|
378
|
+
if parent_link and field.name == parent_link.name:
|
|
379
|
+
continue
|
|
380
|
+
|
|
381
|
+
# Copy auto_now_add, auto_now, and db_returning fields
|
|
382
|
+
if (
|
|
383
|
+
getattr(field, "auto_now_add", False)
|
|
384
|
+
or getattr(field, "auto_now", False)
|
|
385
|
+
or getattr(field, "db_returning", False)
|
|
386
|
+
):
|
|
387
|
+
source_value = getattr(source_obj, field.name, None)
|
|
388
|
+
if source_value is not None:
|
|
389
|
+
setattr(orig_obj, field.name, source_value)
|
|
390
|
+
|
|
391
|
+
# Update object state
|
|
392
|
+
orig_obj._state.adding = False
|
|
393
|
+
orig_obj._state.db = self.queryset.db
|
|
394
|
+
|
|
395
|
+
return plan.original_objects
|
|
396
|
+
|
|
397
|
+
def _execute_mti_update_plan(self, plan):
|
|
398
|
+
"""
|
|
399
|
+
Execute an MTI update plan.
|
|
400
|
+
|
|
401
|
+
Updates each table in the inheritance chain using CASE/WHEN for bulk updates.
|
|
402
|
+
|
|
403
|
+
Args:
|
|
404
|
+
plan: MTIUpdatePlan object from MTIHandler
|
|
405
|
+
|
|
406
|
+
Returns:
|
|
407
|
+
Number of objects updated
|
|
408
|
+
"""
|
|
409
|
+
from django.db.models import Case
|
|
410
|
+
from django.db.models import QuerySet as BaseQuerySet
|
|
411
|
+
from django.db.models import Value
|
|
412
|
+
from django.db.models import When
|
|
413
|
+
|
|
414
|
+
if not plan:
|
|
415
|
+
return 0
|
|
416
|
+
|
|
417
|
+
total_updated = 0
|
|
418
|
+
|
|
419
|
+
# Get PKs for filtering
|
|
420
|
+
root_pks = [
|
|
421
|
+
getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
422
|
+
for obj in plan.objects
|
|
423
|
+
if getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
424
|
+
]
|
|
425
|
+
|
|
426
|
+
if not root_pks:
|
|
427
|
+
return 0
|
|
428
|
+
|
|
429
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
430
|
+
# Update each table in the chain
|
|
431
|
+
for field_group in plan.field_groups:
|
|
432
|
+
if not field_group.fields:
|
|
433
|
+
continue
|
|
434
|
+
|
|
435
|
+
base_qs = BaseQuerySet(model=field_group.model_class, using=self.queryset.db)
|
|
436
|
+
|
|
437
|
+
# Check if records exist
|
|
438
|
+
existing_count = base_qs.filter(**{f"{field_group.filter_field}__in": root_pks}).count()
|
|
439
|
+
if existing_count == 0:
|
|
440
|
+
continue
|
|
441
|
+
|
|
442
|
+
# Build CASE statements for bulk update
|
|
443
|
+
case_statements = {}
|
|
444
|
+
for field_name in field_group.fields:
|
|
445
|
+
field = field_group.model_class._meta.get_field(field_name)
|
|
446
|
+
when_statements = []
|
|
447
|
+
|
|
448
|
+
# Determine the correct output field for type casting
|
|
449
|
+
# For ForeignKey fields, use the target field to ensure correct SQL types
|
|
450
|
+
is_fk = isinstance(field, ForeignKey)
|
|
451
|
+
case_output_field = field.target_field if is_fk else field
|
|
452
|
+
|
|
453
|
+
for pk, obj in zip(root_pks, plan.objects):
|
|
454
|
+
obj_pk = getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
455
|
+
if obj_pk is None:
|
|
456
|
+
continue
|
|
457
|
+
|
|
458
|
+
# Get the field value - handle ForeignKey fields specially
|
|
459
|
+
value = getattr(obj, field.attname, None) if is_fk else getattr(obj, field_name)
|
|
460
|
+
|
|
461
|
+
# Handle NULL values specially for ForeignKey fields
|
|
462
|
+
if is_fk and value is None:
|
|
463
|
+
# For ForeignKey fields with None values, use Cast to ensure proper NULL type
|
|
464
|
+
# PostgreSQL needs explicit type casting for NULL values in CASE statements
|
|
465
|
+
when_statements.append(
|
|
466
|
+
When(
|
|
467
|
+
**{field_group.filter_field: pk},
|
|
468
|
+
then=Cast(Value(None), output_field=case_output_field),
|
|
469
|
+
),
|
|
470
|
+
)
|
|
471
|
+
else:
|
|
472
|
+
# For non-None values or non-FK fields, use Value with output_field
|
|
473
|
+
when_statements.append(
|
|
474
|
+
When(
|
|
475
|
+
**{field_group.filter_field: pk},
|
|
476
|
+
then=Value(value, output_field=case_output_field),
|
|
477
|
+
),
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
if when_statements:
|
|
481
|
+
case_statements[field_name] = Case(*when_statements, output_field=case_output_field)
|
|
482
|
+
|
|
483
|
+
# Execute bulk update
|
|
484
|
+
if case_statements:
|
|
485
|
+
try:
|
|
486
|
+
updated_count = base_qs.filter(
|
|
487
|
+
**{f"{field_group.filter_field}__in": root_pks},
|
|
488
|
+
).update(**case_statements)
|
|
489
|
+
total_updated += updated_count
|
|
490
|
+
except Exception as e:
|
|
491
|
+
logger.error(f"MTI bulk update failed for {field_group.model_class.__name__}: {e}")
|
|
492
|
+
|
|
493
|
+
return total_updated
|
|
494
|
+
|
|
495
|
+
def delete_queryset(self):
|
|
496
|
+
"""
|
|
497
|
+
Execute delete on the queryset.
|
|
498
|
+
|
|
499
|
+
NOTE: Coordinator is responsible for validation before calling this method.
|
|
500
|
+
This executor trusts that inputs have already been validated.
|
|
501
|
+
|
|
502
|
+
Returns:
|
|
503
|
+
Tuple of (count, details dict)
|
|
504
|
+
"""
|
|
505
|
+
if not self.queryset:
|
|
506
|
+
return 0, {}
|
|
507
|
+
|
|
508
|
+
# Execute delete via QuerySet
|
|
509
|
+
# Validation already done by coordinator
|
|
510
|
+
from django.db.models import QuerySet
|
|
511
|
+
|
|
512
|
+
return QuerySet.delete(self.queryset)
|