flask-deck 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flask_deck/__init__.py +184 -0
- flask_deck/adapters/__init__.py +23 -0
- flask_deck/adapters/sqlalchemy.py +272 -0
- flask_deck/audit.py +363 -0
- flask_deck/auth.py +462 -0
- flask_deck/branding.py +217 -0
- flask_deck/charts.py +514 -0
- flask_deck/cli.py +135 -0
- flask_deck/config.py +81 -0
- flask_deck/dashboard.py +416 -0
- flask_deck/deck.py +4153 -0
- flask_deck/demo.py +1066 -0
- flask_deck/events.py +309 -0
- flask_deck/exceptions.py +34 -0
- flask_deck/exporter.py +240 -0
- flask_deck/formatters.py +386 -0
- flask_deck/i18n.py +399 -0
- flask_deck/icons.py +92 -0
- flask_deck/importer.py +612 -0
- flask_deck/introspection.py +376 -0
- flask_deck/jobs.py +382 -0
- flask_deck/mfa.py +670 -0
- flask_deck/model_view.py +1097 -0
- flask_deck/py.typed +0 -0
- flask_deck/query.py +1220 -0
- flask_deck/schema_graph.py +137 -0
- flask_deck/security.py +373 -0
- flask_deck/sql_console.py +391 -0
- flask_deck/sso.py +293 -0
- flask_deck/static/css/deck.css +1333 -0
- flask_deck/static/js/deck.js +1433 -0
- flask_deck/storage.py +669 -0
- flask_deck/templates/flask_deck/audit_list.html +117 -0
- flask_deck/templates/flask_deck/base.html +454 -0
- flask_deck/templates/flask_deck/custom_page.html +12 -0
- flask_deck/templates/flask_deck/dashboard.html +32 -0
- flask_deck/templates/flask_deck/diagnostics.html +80 -0
- flask_deck/templates/flask_deck/error.html +15 -0
- flask_deck/templates/flask_deck/import_wizard.html +143 -0
- flask_deck/templates/flask_deck/jobs_list.html +95 -0
- flask_deck/templates/flask_deck/login.html +93 -0
- flask_deck/templates/flask_deck/login_mfa.html +59 -0
- flask_deck/templates/flask_deck/mfa_setup.html +121 -0
- flask_deck/templates/flask_deck/model_detail.html +124 -0
- flask_deck/templates/flask_deck/model_form.html +389 -0
- flask_deck/templates/flask_deck/model_list.html +489 -0
- flask_deck/templates/flask_deck/partials/drawer_detail.html +93 -0
- flask_deck/templates/flask_deck/partials/widget_grid.html +207 -0
- flask_deck/templates/flask_deck/rbac_matrix.html +59 -0
- flask_deck/templates/flask_deck/record_history.html +102 -0
- flask_deck/templates/flask_deck/schema_inspector.html +83 -0
- flask_deck/templates/flask_deck/sql_console.html +87 -0
- flask_deck-0.3.0.dist-info/METADATA +382 -0
- flask_deck-0.3.0.dist-info/RECORD +57 -0
- flask_deck-0.3.0.dist-info/WHEEL +4 -0
- flask_deck-0.3.0.dist-info/entry_points.txt +5 -0
- flask_deck-0.3.0.dist-info/licenses/LICENSE +21 -0
flask_deck/importer.py
ADDED
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import io
|
|
5
|
+
import json
|
|
6
|
+
import secrets
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections import OrderedDict
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
import sqlalchemy as sa
|
|
15
|
+
from sqlalchemy.exc import SQLAlchemyError
|
|
16
|
+
from sqlalchemy.orm import Session
|
|
17
|
+
|
|
18
|
+
from flask_deck.adapters.sqlalchemy import extract_pk
|
|
19
|
+
from flask_deck.audit import serialize_audit_value
|
|
20
|
+
from flask_deck.auth import AuthProvider, DeckUser
|
|
21
|
+
from flask_deck.exceptions import ImportLimitError
|
|
22
|
+
from flask_deck.model_view import ModelView, coerce_form_value
|
|
23
|
+
from flask_deck.query import soft_delete_predicate
|
|
24
|
+
from flask_deck.security import check_json_depth
|
|
25
|
+
|
|
26
|
+
RowStatus = Literal["create", "update", "unchanged", "error"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True, slots=True)
|
|
30
|
+
class RowDiff:
|
|
31
|
+
"""Dry-run outcome for a single imported row."""
|
|
32
|
+
|
|
33
|
+
row_number: int
|
|
34
|
+
status: RowStatus
|
|
35
|
+
key_display: str
|
|
36
|
+
changes: dict[str, tuple[Any, Any]] = field(default_factory=dict)
|
|
37
|
+
errors: tuple[str, ...] = ()
|
|
38
|
+
coerced_data: dict[str, Any] = field(default_factory=dict)
|
|
39
|
+
existing_pk: Any = None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True, slots=True)
|
|
43
|
+
class DryRunReport:
|
|
44
|
+
"""Complete savepoint-verified preview of a pending CSV/JSON import."""
|
|
45
|
+
|
|
46
|
+
token: str
|
|
47
|
+
model_slug: str
|
|
48
|
+
filename: str
|
|
49
|
+
format: str
|
|
50
|
+
upsert_key: str | None
|
|
51
|
+
headers: tuple[str, ...]
|
|
52
|
+
mapped_columns: dict[str, str]
|
|
53
|
+
rows: tuple[RowDiff, ...]
|
|
54
|
+
creates: int
|
|
55
|
+
updates: int
|
|
56
|
+
unchanged: int
|
|
57
|
+
errors: int
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def total_rows(self) -> int:
|
|
61
|
+
return len(self.rows)
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def applicable_count(self) -> int:
|
|
65
|
+
return self.creates + self.updates
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(slots=True)
|
|
69
|
+
class _CachedPreview:
|
|
70
|
+
report: DryRunReport
|
|
71
|
+
user_id: str
|
|
72
|
+
session_token: str
|
|
73
|
+
created_at: float
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ImportPreviewStore:
|
|
77
|
+
"""Thread-safe bounded, user/session-bound, single-use TTL store for dry-run previews."""
|
|
78
|
+
|
|
79
|
+
__slots__ = ("_entries", "_lock", "_max_entries", "_ttl")
|
|
80
|
+
|
|
81
|
+
def __init__(self, *, ttl_seconds: int = 900, max_entries: int = 64) -> None:
|
|
82
|
+
self._ttl = max(60, ttl_seconds)
|
|
83
|
+
self._max_entries = max(4, max_entries)
|
|
84
|
+
self._entries: OrderedDict[str, _CachedPreview] = OrderedDict()
|
|
85
|
+
self._lock = threading.Lock()
|
|
86
|
+
|
|
87
|
+
def _prune(self, now: float) -> None:
|
|
88
|
+
expired = [k for k, v in self._entries.items() if (now - v.created_at) > self._ttl]
|
|
89
|
+
for k in expired:
|
|
90
|
+
self._entries.pop(k, None)
|
|
91
|
+
while len(self._entries) > self._max_entries:
|
|
92
|
+
self._entries.popitem(last=False)
|
|
93
|
+
|
|
94
|
+
def put(self, report: DryRunReport, *, user_id: str, session_token: str) -> None:
|
|
95
|
+
now = time.monotonic()
|
|
96
|
+
with self._lock:
|
|
97
|
+
self._prune(now)
|
|
98
|
+
self._entries[report.token] = _CachedPreview(
|
|
99
|
+
report=report,
|
|
100
|
+
user_id=user_id,
|
|
101
|
+
session_token=session_token,
|
|
102
|
+
created_at=now,
|
|
103
|
+
)
|
|
104
|
+
self._entries.move_to_end(report.token)
|
|
105
|
+
self._prune(now)
|
|
106
|
+
|
|
107
|
+
def consume(
|
|
108
|
+
self,
|
|
109
|
+
token: str,
|
|
110
|
+
*,
|
|
111
|
+
user_id: str,
|
|
112
|
+
session_token: str,
|
|
113
|
+
model_slug: str,
|
|
114
|
+
) -> DryRunReport | None:
|
|
115
|
+
"""Atomically pop a preview token only if user, session, model, and TTL all match."""
|
|
116
|
+
now = time.monotonic()
|
|
117
|
+
with self._lock:
|
|
118
|
+
self._prune(now)
|
|
119
|
+
item = self._entries.get(token)
|
|
120
|
+
if item is None:
|
|
121
|
+
return None
|
|
122
|
+
if (
|
|
123
|
+
item.user_id != user_id
|
|
124
|
+
or item.session_token != session_token
|
|
125
|
+
or item.report.model_slug != model_slug
|
|
126
|
+
):
|
|
127
|
+
return None
|
|
128
|
+
self._entries.pop(token, None)
|
|
129
|
+
return item.report
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def parse_import_payload(
|
|
133
|
+
raw_bytes: bytes,
|
|
134
|
+
*,
|
|
135
|
+
filename: str,
|
|
136
|
+
max_bytes: int,
|
|
137
|
+
max_rows: int,
|
|
138
|
+
max_columns: int,
|
|
139
|
+
max_json_depth: int,
|
|
140
|
+
) -> tuple[str, tuple[str, ...], list[dict[str, Any]]]:
|
|
141
|
+
"""Parse bounded CSV or JSON upload into (format, headers, row_dicts)."""
|
|
142
|
+
if len(raw_bytes) > max_bytes:
|
|
143
|
+
raise ImportLimitError(
|
|
144
|
+
f"Upload exceeds maximum allowed size ({max_bytes // (1024 * 1024)} MiB)."
|
|
145
|
+
)
|
|
146
|
+
if not raw_bytes.strip():
|
|
147
|
+
raise ImportLimitError("Uploaded file is empty.")
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
text = raw_bytes.decode("utf-8-sig")
|
|
151
|
+
except UnicodeDecodeError as exc:
|
|
152
|
+
raise ImportLimitError("File must be valid UTF-8 text.") from exc
|
|
153
|
+
|
|
154
|
+
lower_name = filename.lower()
|
|
155
|
+
is_json = lower_name.endswith(".json") or text.lstrip().startswith("[")
|
|
156
|
+
|
|
157
|
+
if is_json:
|
|
158
|
+
try:
|
|
159
|
+
parsed = json.loads(text)
|
|
160
|
+
except json.JSONDecodeError as exc:
|
|
161
|
+
raise ImportLimitError(f"Invalid JSON syntax: {exc.msg} (line {exc.lineno}).") from exc
|
|
162
|
+
if not isinstance(parsed, list):
|
|
163
|
+
raise ImportLimitError("JSON import root must be an array of objects.")
|
|
164
|
+
if len(parsed) > max_rows:
|
|
165
|
+
raise ImportLimitError(f"Import exceeds maximum row limit ({max_rows}).")
|
|
166
|
+
if not check_json_depth(parsed, max_json_depth):
|
|
167
|
+
raise ImportLimitError(f"JSON nesting exceeds maximum depth ({max_json_depth}).")
|
|
168
|
+
|
|
169
|
+
headers_order: list[str] = []
|
|
170
|
+
seen_headers: set[str] = set()
|
|
171
|
+
rows: list[dict[str, Any]] = []
|
|
172
|
+
for idx, item in enumerate(parsed, start=1):
|
|
173
|
+
if not isinstance(item, dict):
|
|
174
|
+
raise ImportLimitError(f"JSON item #{idx} must be an object.")
|
|
175
|
+
for k in item:
|
|
176
|
+
ks = str(k)
|
|
177
|
+
if ks not in seen_headers:
|
|
178
|
+
seen_headers.add(ks)
|
|
179
|
+
headers_order.append(ks)
|
|
180
|
+
if len(headers_order) > max_columns:
|
|
181
|
+
raise ImportLimitError(
|
|
182
|
+
f"Import exceeds maximum column limit ({max_columns})."
|
|
183
|
+
)
|
|
184
|
+
rows.append({str(k): v for k, v in item.items()})
|
|
185
|
+
return "json", tuple(headers_order), rows
|
|
186
|
+
|
|
187
|
+
# CSV parsing with safe delimiter sniffing (, ; \t)
|
|
188
|
+
sample = text[:4096]
|
|
189
|
+
delimiter = ","
|
|
190
|
+
try:
|
|
191
|
+
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
|
192
|
+
delimiter = dialect.delimiter
|
|
193
|
+
except csv.Error:
|
|
194
|
+
delimiter = ","
|
|
195
|
+
|
|
196
|
+
reader = csv.DictReader(io.StringIO(text), delimiter=delimiter)
|
|
197
|
+
if not reader.fieldnames:
|
|
198
|
+
raise ImportLimitError("CSV file is missing a header row.")
|
|
199
|
+
headers = tuple(h.strip() for h in reader.fieldnames if h is not None)
|
|
200
|
+
if len(headers) > max_columns:
|
|
201
|
+
raise ImportLimitError(f"CSV exceeds maximum column limit ({max_columns}).")
|
|
202
|
+
|
|
203
|
+
rows = []
|
|
204
|
+
for raw_row in reader:
|
|
205
|
+
if len(rows) >= max_rows:
|
|
206
|
+
raise ImportLimitError(f"CSV exceeds maximum row limit ({max_rows}).")
|
|
207
|
+
clean_row = {
|
|
208
|
+
(k.strip() if k is not None else ""): v for k, v in raw_row.items() if k is not None
|
|
209
|
+
}
|
|
210
|
+
if any(str(v or "").strip() != "" for v in clean_row.values()):
|
|
211
|
+
rows.append(clean_row)
|
|
212
|
+
return "csv", headers, rows
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def auto_map_headers(
|
|
216
|
+
headers: Sequence[str],
|
|
217
|
+
allowed_fields: Sequence[str],
|
|
218
|
+
overrides: Mapping[str, str] | None = None,
|
|
219
|
+
) -> dict[str, str]:
|
|
220
|
+
"""Map uploaded column headers to strictly allowed writable model fields."""
|
|
221
|
+
allowed_set = set(allowed_fields)
|
|
222
|
+
norm_lookup = {f.lower().replace("_", ""): f for f in allowed_fields}
|
|
223
|
+
mapping: dict[str, str] = {}
|
|
224
|
+
for header in headers:
|
|
225
|
+
if overrides and header in overrides:
|
|
226
|
+
chosen = overrides[header].strip()
|
|
227
|
+
if chosen in allowed_set:
|
|
228
|
+
mapping[header] = chosen
|
|
229
|
+
continue
|
|
230
|
+
if header in allowed_set:
|
|
231
|
+
mapping[header] = header
|
|
232
|
+
continue
|
|
233
|
+
norm = header.strip().lower().replace(" ", "").replace("-", "").replace("_", "")
|
|
234
|
+
if norm in norm_lookup:
|
|
235
|
+
mapping[header] = norm_lookup[norm]
|
|
236
|
+
return mapping
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _batch_load_existing(
|
|
240
|
+
session: Session,
|
|
241
|
+
view: ModelView,
|
|
242
|
+
user: DeckUser | None,
|
|
243
|
+
key_col: str,
|
|
244
|
+
typed_keys: Sequence[Any],
|
|
245
|
+
*,
|
|
246
|
+
batch_size: int = 500,
|
|
247
|
+
) -> dict[Any, Any]:
|
|
248
|
+
"""Preload active matching records in chunked IN(...) batches to eliminate O(N) lookups."""
|
|
249
|
+
if not typed_keys:
|
|
250
|
+
return {}
|
|
251
|
+
col_attr = getattr(view.model, key_col)
|
|
252
|
+
unique_keys = list(dict.fromkeys(typed_keys))
|
|
253
|
+
sd_pred = soft_delete_predicate(view.meta, view.soft_delete_column, trashed=False)
|
|
254
|
+
found: dict[Any, Any] = {}
|
|
255
|
+
for start in range(0, len(unique_keys), batch_size):
|
|
256
|
+
chunk = unique_keys[start : start + batch_size]
|
|
257
|
+
stmt = view.scope_query(sa.select(view.model), user).where(col_attr.in_(chunk))
|
|
258
|
+
if sd_pred is not None:
|
|
259
|
+
stmt = stmt.where(sd_pred)
|
|
260
|
+
for obj in session.scalars(stmt).all():
|
|
261
|
+
found[getattr(obj, key_col)] = obj
|
|
262
|
+
return found
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _batch_load_by_pk_tuples(
|
|
266
|
+
session: Session,
|
|
267
|
+
view: ModelView,
|
|
268
|
+
user: DeckUser | None,
|
|
269
|
+
pk_tuples: Sequence[tuple[Any, ...]],
|
|
270
|
+
*,
|
|
271
|
+
batch_size: int = 500,
|
|
272
|
+
) -> dict[tuple[Any, ...], Any]:
|
|
273
|
+
"""Preload active records by single or composite primary key tuples."""
|
|
274
|
+
if not pk_tuples:
|
|
275
|
+
return {}
|
|
276
|
+
unique_pks = list(dict.fromkeys(pk_tuples))
|
|
277
|
+
sd_pred = soft_delete_predicate(view.meta, view.soft_delete_column, trashed=False)
|
|
278
|
+
pk_cols = [getattr(view.model, pk.name) for pk in view.meta.primary_keys]
|
|
279
|
+
found: dict[tuple[Any, ...], Any] = {}
|
|
280
|
+
for start in range(0, len(unique_pks), batch_size):
|
|
281
|
+
chunk = unique_pks[start : start + batch_size]
|
|
282
|
+
stmt = view.scope_query(sa.select(view.model), user)
|
|
283
|
+
if sd_pred is not None:
|
|
284
|
+
stmt = stmt.where(sd_pred)
|
|
285
|
+
if len(pk_cols) == 1:
|
|
286
|
+
stmt = stmt.where(pk_cols[0].in_([t[0] for t in chunk]))
|
|
287
|
+
else:
|
|
288
|
+
stmt = stmt.where(sa.tuple_(*pk_cols).in_(chunk))
|
|
289
|
+
for obj in session.scalars(stmt).all():
|
|
290
|
+
found[extract_pk(obj, view.meta)] = obj
|
|
291
|
+
return found
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def run_dry_run_import(
|
|
295
|
+
session: Session,
|
|
296
|
+
view: ModelView,
|
|
297
|
+
*,
|
|
298
|
+
raw_bytes: bytes,
|
|
299
|
+
filename: str,
|
|
300
|
+
auth: AuthProvider,
|
|
301
|
+
user: DeckUser | None,
|
|
302
|
+
upsert_key: str | None = None,
|
|
303
|
+
column_overrides: Mapping[str, str] | None = None,
|
|
304
|
+
max_bytes: int = 5 * 1024 * 1024,
|
|
305
|
+
max_rows: int = 5000,
|
|
306
|
+
max_columns: int = 100,
|
|
307
|
+
max_json_depth: int = 12,
|
|
308
|
+
) -> DryRunReport:
|
|
309
|
+
"""Validate and diff an import inside a guaranteed-rollback SQLAlchemy SAVEPOINT."""
|
|
310
|
+
fmt, headers, raw_rows = parse_import_payload(
|
|
311
|
+
raw_bytes,
|
|
312
|
+
filename=filename,
|
|
313
|
+
max_bytes=max_bytes,
|
|
314
|
+
max_rows=max_rows,
|
|
315
|
+
max_columns=max_columns,
|
|
316
|
+
max_json_depth=max_json_depth,
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
hidden = auth.hidden_fields_for(user, view.meta.slug)
|
|
320
|
+
readonly = auth.readonly_fields_for(user, view.meta.slug)
|
|
321
|
+
masked = auth.masked_fields_for(user, view.meta.slug)
|
|
322
|
+
writable_fields = tuple(
|
|
323
|
+
c
|
|
324
|
+
for c in view.resolved_import_columns
|
|
325
|
+
if c not in hidden and c not in readonly and c not in masked and c not in view.meta.sensitive_fields
|
|
326
|
+
)
|
|
327
|
+
# Allow matching upsert_key even on PK or unique columns if not hidden/masked/sensitive.
|
|
328
|
+
all_importable = list(writable_fields)
|
|
329
|
+
if (
|
|
330
|
+
upsert_key
|
|
331
|
+
and upsert_key in view.meta.fields_by_name
|
|
332
|
+
and upsert_key not in hidden
|
|
333
|
+
and upsert_key not in masked
|
|
334
|
+
and upsert_key not in view.meta.sensitive_fields
|
|
335
|
+
):
|
|
336
|
+
if upsert_key not in all_importable:
|
|
337
|
+
all_importable.append(upsert_key)
|
|
338
|
+
else:
|
|
339
|
+
upsert_key = None
|
|
340
|
+
|
|
341
|
+
create_writable = set(writable_fields)
|
|
342
|
+
if upsert_key:
|
|
343
|
+
uf = view.meta.fields_by_name[upsert_key]
|
|
344
|
+
if not uf.autoincrement:
|
|
345
|
+
create_writable.add(upsert_key)
|
|
346
|
+
|
|
347
|
+
mapped = auto_map_headers(headers, all_importable, overrides=column_overrides)
|
|
348
|
+
if not mapped:
|
|
349
|
+
raise ImportLimitError("No uploaded columns matched any writable fields on this model.")
|
|
350
|
+
|
|
351
|
+
# Coerce rows first & collect upsert lookup keys for batched DB preload.
|
|
352
|
+
parsed_entries: list[tuple[int, dict[str, Any], list[str], Any]] = []
|
|
353
|
+
lookup_keys: list[Any] = []
|
|
354
|
+
for idx, raw_row in enumerate(raw_rows, start=1):
|
|
355
|
+
row_data: dict[str, Any] = {}
|
|
356
|
+
row_errors: list[str] = []
|
|
357
|
+
for src_header, dst_field in mapped.items():
|
|
358
|
+
fmeta = view.meta.fields_by_name[dst_field]
|
|
359
|
+
raw_val = raw_row.get(src_header)
|
|
360
|
+
try:
|
|
361
|
+
coerced = coerce_form_value(raw_val, fmeta, max_json_depth=max_json_depth)
|
|
362
|
+
row_data[dst_field] = coerced
|
|
363
|
+
except ValueError as exc:
|
|
364
|
+
row_errors.append(f"{dst_field}: {exc}")
|
|
365
|
+
|
|
366
|
+
key_val = row_data.get(upsert_key) if upsert_key else None
|
|
367
|
+
if upsert_key and key_val is not None:
|
|
368
|
+
lookup_keys.append(key_val)
|
|
369
|
+
parsed_entries.append((idx, row_data, row_errors, key_val))
|
|
370
|
+
|
|
371
|
+
existing_map = (
|
|
372
|
+
_batch_load_existing(session, view, user, upsert_key, lookup_keys)
|
|
373
|
+
if upsert_key
|
|
374
|
+
else {}
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
validator_fn = view.get_form_validator()
|
|
378
|
+
diffs: list[RowDiff] = []
|
|
379
|
+
creates = updates = unchanged = error_count = 0
|
|
380
|
+
|
|
381
|
+
# Outer savepoint guarantees ZERO mutations persist during dry-run even on exceptions!
|
|
382
|
+
outer_sp = session.begin_nested()
|
|
383
|
+
try:
|
|
384
|
+
for row_num, row_data, row_errors, key_val in parsed_entries:
|
|
385
|
+
key_display = str(key_val) if key_val is not None else f"Row #{row_num}"
|
|
386
|
+
if row_errors:
|
|
387
|
+
error_count += 1
|
|
388
|
+
diffs.append(
|
|
389
|
+
RowDiff(
|
|
390
|
+
row_number=row_num,
|
|
391
|
+
status="error",
|
|
392
|
+
key_display=key_display,
|
|
393
|
+
errors=tuple(row_errors),
|
|
394
|
+
)
|
|
395
|
+
)
|
|
396
|
+
continue
|
|
397
|
+
|
|
398
|
+
existing_obj = existing_map.get(key_val) if (upsert_key and key_val is not None) else None
|
|
399
|
+
|
|
400
|
+
if existing_obj is not None:
|
|
401
|
+
# Compute field-level diff against existing record.
|
|
402
|
+
changes: dict[str, tuple[Any, Any]] = {}
|
|
403
|
+
for fname, new_val in row_data.items():
|
|
404
|
+
if fname not in writable_fields:
|
|
405
|
+
continue
|
|
406
|
+
old_val = getattr(existing_obj, fname, None)
|
|
407
|
+
if serialize_audit_value(old_val) != serialize_audit_value(new_val):
|
|
408
|
+
changes[fname] = (
|
|
409
|
+
serialize_audit_value(old_val),
|
|
410
|
+
serialize_audit_value(new_val),
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
custom_errs = dict(view.validate(row_data, obj=existing_obj))
|
|
414
|
+
if validator_fn is not None:
|
|
415
|
+
merged_payload = {**view.extract_snapshot(existing_obj), **row_data}
|
|
416
|
+
for fk, fmsgs in validator_fn(merged_payload, existing_obj, False).items():
|
|
417
|
+
custom_errs.setdefault(fk, []).extend(fmsgs)
|
|
418
|
+
if custom_errs:
|
|
419
|
+
flat = tuple(f"{f}: {', '.join(m)}" for f, m in custom_errs.items())
|
|
420
|
+
error_count += 1
|
|
421
|
+
diffs.append(
|
|
422
|
+
RowDiff(
|
|
423
|
+
row_number=row_num,
|
|
424
|
+
status="error",
|
|
425
|
+
key_display=key_display,
|
|
426
|
+
errors=flat,
|
|
427
|
+
)
|
|
428
|
+
)
|
|
429
|
+
continue
|
|
430
|
+
|
|
431
|
+
if not changes:
|
|
432
|
+
unchanged += 1
|
|
433
|
+
diffs.append(
|
|
434
|
+
RowDiff(
|
|
435
|
+
row_number=row_num,
|
|
436
|
+
status="unchanged",
|
|
437
|
+
key_display=key_display,
|
|
438
|
+
existing_pk=extract_pk(existing_obj, view.meta),
|
|
439
|
+
)
|
|
440
|
+
)
|
|
441
|
+
continue
|
|
442
|
+
|
|
443
|
+
# Test update inside inner savepoint (released on success so savepoints never stack!)
|
|
444
|
+
row_sp = session.begin_nested()
|
|
445
|
+
try:
|
|
446
|
+
for fname in changes:
|
|
447
|
+
setattr(existing_obj, fname, row_data[fname])
|
|
448
|
+
session.flush()
|
|
449
|
+
row_sp.commit()
|
|
450
|
+
updates += 1
|
|
451
|
+
diffs.append(
|
|
452
|
+
RowDiff(
|
|
453
|
+
row_number=row_num,
|
|
454
|
+
status="update",
|
|
455
|
+
key_display=key_display,
|
|
456
|
+
changes=changes,
|
|
457
|
+
coerced_data={k: row_data[k] for k in changes},
|
|
458
|
+
existing_pk=extract_pk(existing_obj, view.meta),
|
|
459
|
+
)
|
|
460
|
+
)
|
|
461
|
+
except SQLAlchemyError as exc:
|
|
462
|
+
row_sp.rollback()
|
|
463
|
+
error_count += 1
|
|
464
|
+
diffs.append(
|
|
465
|
+
RowDiff(
|
|
466
|
+
row_number=row_num,
|
|
467
|
+
status="error",
|
|
468
|
+
key_display=key_display,
|
|
469
|
+
errors=(f"Database constraint error: {exc.__class__.__name__}",),
|
|
470
|
+
)
|
|
471
|
+
)
|
|
472
|
+
continue
|
|
473
|
+
|
|
474
|
+
# Create path: verify required non-nullable fields without defaults.
|
|
475
|
+
missing_req: list[str] = []
|
|
476
|
+
for fmeta in view.meta.fields:
|
|
477
|
+
if (
|
|
478
|
+
not fmeta.nullable
|
|
479
|
+
and not fmeta.has_default
|
|
480
|
+
and not fmeta.autoincrement
|
|
481
|
+
and not fmeta.is_sensitive
|
|
482
|
+
and fmeta.name not in row_data
|
|
483
|
+
):
|
|
484
|
+
missing_req.append(f"{fmeta.name}: required column missing.")
|
|
485
|
+
if missing_req:
|
|
486
|
+
error_count += 1
|
|
487
|
+
diffs.append(
|
|
488
|
+
RowDiff(
|
|
489
|
+
row_number=row_num,
|
|
490
|
+
status="error",
|
|
491
|
+
key_display=key_display,
|
|
492
|
+
errors=tuple(missing_req),
|
|
493
|
+
)
|
|
494
|
+
)
|
|
495
|
+
continue
|
|
496
|
+
|
|
497
|
+
custom_errs = dict(view.validate(row_data, obj=None))
|
|
498
|
+
if validator_fn is not None:
|
|
499
|
+
for fk, fmsgs in validator_fn(row_data, None, True).items():
|
|
500
|
+
custom_errs.setdefault(fk, []).extend(fmsgs)
|
|
501
|
+
if custom_errs:
|
|
502
|
+
flat = tuple(f"{f}: {', '.join(m)}" for f, m in custom_errs.items())
|
|
503
|
+
error_count += 1
|
|
504
|
+
diffs.append(
|
|
505
|
+
RowDiff(
|
|
506
|
+
row_number=row_num,
|
|
507
|
+
status="error",
|
|
508
|
+
key_display=key_display,
|
|
509
|
+
errors=flat,
|
|
510
|
+
)
|
|
511
|
+
)
|
|
512
|
+
continue
|
|
513
|
+
|
|
514
|
+
row_sp = session.begin_nested()
|
|
515
|
+
try:
|
|
516
|
+
new_obj = view.model()
|
|
517
|
+
for fname, val in row_data.items():
|
|
518
|
+
if fname in create_writable:
|
|
519
|
+
setattr(new_obj, fname, val)
|
|
520
|
+
session.add(new_obj)
|
|
521
|
+
session.flush()
|
|
522
|
+
row_sp.commit()
|
|
523
|
+
creates += 1
|
|
524
|
+
create_changes = {
|
|
525
|
+
fname: (None, serialize_audit_value(val))
|
|
526
|
+
for fname, val in row_data.items()
|
|
527
|
+
if fname in create_writable
|
|
528
|
+
}
|
|
529
|
+
diffs.append(
|
|
530
|
+
RowDiff(
|
|
531
|
+
row_number=row_num,
|
|
532
|
+
status="create",
|
|
533
|
+
key_display=key_display,
|
|
534
|
+
changes=create_changes,
|
|
535
|
+
coerced_data={k: row_data[k] for k in create_changes},
|
|
536
|
+
)
|
|
537
|
+
)
|
|
538
|
+
except SQLAlchemyError as exc:
|
|
539
|
+
row_sp.rollback()
|
|
540
|
+
error_count += 1
|
|
541
|
+
diffs.append(
|
|
542
|
+
RowDiff(
|
|
543
|
+
row_number=row_num,
|
|
544
|
+
status="error",
|
|
545
|
+
key_display=key_display,
|
|
546
|
+
errors=(f"Database constraint error: {exc.__class__.__name__}",),
|
|
547
|
+
)
|
|
548
|
+
)
|
|
549
|
+
finally:
|
|
550
|
+
# Unconditionally roll back the dry-run savepoint so zero changes leak!
|
|
551
|
+
if outer_sp.is_active:
|
|
552
|
+
outer_sp.rollback()
|
|
553
|
+
session.expire_all()
|
|
554
|
+
|
|
555
|
+
return DryRunReport(
|
|
556
|
+
token=secrets.token_urlsafe(24),
|
|
557
|
+
model_slug=view.meta.slug,
|
|
558
|
+
filename=filename,
|
|
559
|
+
format=fmt,
|
|
560
|
+
upsert_key=upsert_key,
|
|
561
|
+
headers=headers,
|
|
562
|
+
mapped_columns=mapped,
|
|
563
|
+
rows=tuple(diffs),
|
|
564
|
+
creates=creates,
|
|
565
|
+
updates=updates,
|
|
566
|
+
unchanged=unchanged,
|
|
567
|
+
errors=error_count,
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def commit_dry_run_import(
|
|
572
|
+
session: Session,
|
|
573
|
+
view: ModelView,
|
|
574
|
+
report: DryRunReport,
|
|
575
|
+
*,
|
|
576
|
+
user: DeckUser | None,
|
|
577
|
+
skip_errors: bool = False,
|
|
578
|
+
) -> tuple[int, int]:
|
|
579
|
+
"""Atomically apply a previously validated DryRunReport to the database."""
|
|
580
|
+
if report.errors > 0 and not skip_errors:
|
|
581
|
+
raise ImportLimitError(
|
|
582
|
+
f"Import contains {report.errors} errored row(s). Fix errors or enable 'Skip errored rows'."
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
update_pks: list[tuple[Any, ...]] = []
|
|
586
|
+
for r in report.rows:
|
|
587
|
+
if r.status == "update" and r.existing_pk is not None:
|
|
588
|
+
pk_t = r.existing_pk if isinstance(r.existing_pk, tuple) else (r.existing_pk,)
|
|
589
|
+
update_pks.append(pk_t)
|
|
590
|
+
existing_by_pk = _batch_load_by_pk_tuples(session, view, user, update_pks)
|
|
591
|
+
|
|
592
|
+
created = 0
|
|
593
|
+
updated = 0
|
|
594
|
+
for row in report.rows:
|
|
595
|
+
if row.status == "create":
|
|
596
|
+
obj = view.model()
|
|
597
|
+
for fname, val in row.coerced_data.items():
|
|
598
|
+
setattr(obj, fname, val)
|
|
599
|
+
view.on_model_change(obj, is_created=True, user=user)
|
|
600
|
+
session.add(obj)
|
|
601
|
+
created += 1
|
|
602
|
+
elif row.status == "update" and row.existing_pk is not None:
|
|
603
|
+
pk_key = row.existing_pk if isinstance(row.existing_pk, tuple) else (row.existing_pk,)
|
|
604
|
+
if pk_key in existing_by_pk:
|
|
605
|
+
obj = existing_by_pk[pk_key]
|
|
606
|
+
for fname, val in row.coerced_data.items():
|
|
607
|
+
setattr(obj, fname, val)
|
|
608
|
+
view.on_model_change(obj, is_created=False, user=user)
|
|
609
|
+
updated += 1
|
|
610
|
+
|
|
611
|
+
session.commit()
|
|
612
|
+
return created, updated
|