flask-deck 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flask_deck/__init__.py +184 -0
- flask_deck/adapters/__init__.py +23 -0
- flask_deck/adapters/sqlalchemy.py +272 -0
- flask_deck/audit.py +363 -0
- flask_deck/auth.py +462 -0
- flask_deck/branding.py +217 -0
- flask_deck/charts.py +514 -0
- flask_deck/cli.py +135 -0
- flask_deck/config.py +81 -0
- flask_deck/dashboard.py +416 -0
- flask_deck/deck.py +4153 -0
- flask_deck/demo.py +1066 -0
- flask_deck/events.py +309 -0
- flask_deck/exceptions.py +34 -0
- flask_deck/exporter.py +240 -0
- flask_deck/formatters.py +386 -0
- flask_deck/i18n.py +399 -0
- flask_deck/icons.py +92 -0
- flask_deck/importer.py +612 -0
- flask_deck/introspection.py +376 -0
- flask_deck/jobs.py +382 -0
- flask_deck/mfa.py +670 -0
- flask_deck/model_view.py +1097 -0
- flask_deck/py.typed +0 -0
- flask_deck/query.py +1220 -0
- flask_deck/schema_graph.py +137 -0
- flask_deck/security.py +373 -0
- flask_deck/sql_console.py +391 -0
- flask_deck/sso.py +293 -0
- flask_deck/static/css/deck.css +1333 -0
- flask_deck/static/js/deck.js +1433 -0
- flask_deck/storage.py +669 -0
- flask_deck/templates/flask_deck/audit_list.html +117 -0
- flask_deck/templates/flask_deck/base.html +454 -0
- flask_deck/templates/flask_deck/custom_page.html +12 -0
- flask_deck/templates/flask_deck/dashboard.html +32 -0
- flask_deck/templates/flask_deck/diagnostics.html +80 -0
- flask_deck/templates/flask_deck/error.html +15 -0
- flask_deck/templates/flask_deck/import_wizard.html +143 -0
- flask_deck/templates/flask_deck/jobs_list.html +95 -0
- flask_deck/templates/flask_deck/login.html +93 -0
- flask_deck/templates/flask_deck/login_mfa.html +59 -0
- flask_deck/templates/flask_deck/mfa_setup.html +121 -0
- flask_deck/templates/flask_deck/model_detail.html +124 -0
- flask_deck/templates/flask_deck/model_form.html +389 -0
- flask_deck/templates/flask_deck/model_list.html +489 -0
- flask_deck/templates/flask_deck/partials/drawer_detail.html +93 -0
- flask_deck/templates/flask_deck/partials/widget_grid.html +207 -0
- flask_deck/templates/flask_deck/rbac_matrix.html +59 -0
- flask_deck/templates/flask_deck/record_history.html +102 -0
- flask_deck/templates/flask_deck/schema_inspector.html +83 -0
- flask_deck/templates/flask_deck/sql_console.html +87 -0
- flask_deck-0.3.0.dist-info/METADATA +382 -0
- flask_deck-0.3.0.dist-info/RECORD +57 -0
- flask_deck-0.3.0.dist-info/WHEEL +4 -0
- flask_deck-0.3.0.dist-info/entry_points.txt +5 -0
- flask_deck-0.3.0.dist-info/licenses/LICENSE +21 -0
flask_deck/query.py
ADDED
|
@@ -0,0 +1,1220 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import contextlib
|
|
5
|
+
import enum
|
|
6
|
+
import hashlib
|
|
7
|
+
import hmac
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
import weakref
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from datetime import date, datetime, timedelta, timezone
|
|
16
|
+
from datetime import time as dt_time
|
|
17
|
+
from decimal import Decimal, InvalidOperation
|
|
18
|
+
from typing import Any, Literal
|
|
19
|
+
|
|
20
|
+
import sqlalchemy as sa
|
|
21
|
+
from flask import g, has_request_context
|
|
22
|
+
from sqlalchemy.engine import Engine
|
|
23
|
+
from sqlalchemy.orm import Session, defer, joinedload, selectinload
|
|
24
|
+
|
|
25
|
+
from flask_deck.formatters import normalize_tone
|
|
26
|
+
from flask_deck.introspection import FieldMeta, ModelMeta
|
|
27
|
+
from flask_deck.security import check_json_depth, escape_like
|
|
28
|
+
|
|
29
|
+
_TELEMETRY_G_KEY = "_flask_deck_sql_telemetry"
|
|
30
|
+
_INSTRUMENTED_ENGINES: weakref.WeakSet[Engine] = weakref.WeakSet()
|
|
31
|
+
|
|
32
|
+
_JSON_PATH_RE = re.compile(r"^[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){0,4}$")
|
|
33
|
+
|
|
34
|
+
PaginationMode = Literal["offset", "cursor"]
|
|
35
|
+
CountMode = Literal["exact", "estimated", "none"]
|
|
36
|
+
|
|
37
|
+
_FILTER_OP_SUFFIXES: tuple[tuple[str, str], ...] = (
|
|
38
|
+
("__array_contains", "array_contains"),
|
|
39
|
+
("__json_contains", "json_contains"),
|
|
40
|
+
("__startswith", "startswith"),
|
|
41
|
+
("__array_any", "array_any"),
|
|
42
|
+
("__contains", "contains"),
|
|
43
|
+
("__has_key", "has_key"),
|
|
44
|
+
("__preset", "preset"),
|
|
45
|
+
("__null", "null"),
|
|
46
|
+
("__path", "path"),
|
|
47
|
+
("__neq", "neq"),
|
|
48
|
+
("__gte", "gte"),
|
|
49
|
+
("__lte", "lte"),
|
|
50
|
+
("__gt", "gt"),
|
|
51
|
+
("__lt", "lt"),
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(slots=True)
|
|
56
|
+
class RequestSQLTelemetry:
|
|
57
|
+
"""Request-scoped SQL query count and execution duration tracker."""
|
|
58
|
+
|
|
59
|
+
query_count: int = 0
|
|
60
|
+
sql_ms: float = 0.0
|
|
61
|
+
started_at: float = field(default_factory=time.perf_counter)
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def total_ms(self) -> float:
|
|
65
|
+
return (time.perf_counter() - self.started_at) * 1000.0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def begin_request_telemetry() -> RequestSQLTelemetry:
|
|
69
|
+
tel = RequestSQLTelemetry()
|
|
70
|
+
setattr(g, _TELEMETRY_G_KEY, tel)
|
|
71
|
+
return tel
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_request_telemetry() -> RequestSQLTelemetry | None:
|
|
75
|
+
if not has_request_context():
|
|
76
|
+
return None
|
|
77
|
+
tel: RequestSQLTelemetry | None = getattr(g, _TELEMETRY_G_KEY, None)
|
|
78
|
+
return tel
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def attach_sql_telemetry(engine: Engine) -> None:
|
|
82
|
+
"""Attach a lightweight zero-overhead SQL listener once per SQLAlchemy Engine."""
|
|
83
|
+
if engine in _INSTRUMENTED_ENGINES:
|
|
84
|
+
return
|
|
85
|
+
|
|
86
|
+
@sa.event.listens_for(engine, "before_cursor_execute")
|
|
87
|
+
def _before_cursor_execute(
|
|
88
|
+
conn: Any,
|
|
89
|
+
cursor: Any,
|
|
90
|
+
statement: str,
|
|
91
|
+
parameters: Any,
|
|
92
|
+
context: Any,
|
|
93
|
+
executemany: bool,
|
|
94
|
+
) -> None:
|
|
95
|
+
if has_request_context() and getattr(g, _TELEMETRY_G_KEY, None) is not None:
|
|
96
|
+
context._deck_q_start = time.perf_counter()
|
|
97
|
+
|
|
98
|
+
@sa.event.listens_for(engine, "after_cursor_execute")
|
|
99
|
+
def _after_cursor_execute(
|
|
100
|
+
conn: Any,
|
|
101
|
+
cursor: Any,
|
|
102
|
+
statement: str,
|
|
103
|
+
parameters: Any,
|
|
104
|
+
context: Any,
|
|
105
|
+
executemany: bool,
|
|
106
|
+
) -> None:
|
|
107
|
+
if not has_request_context():
|
|
108
|
+
return
|
|
109
|
+
tel: RequestSQLTelemetry | None = getattr(g, _TELEMETRY_G_KEY, None)
|
|
110
|
+
if tel is None:
|
|
111
|
+
return
|
|
112
|
+
tel.query_count += 1
|
|
113
|
+
q_start = getattr(context, "_deck_q_start", None)
|
|
114
|
+
if q_start is not None:
|
|
115
|
+
tel.sql_ms += (time.perf_counter() - q_start) * 1000.0
|
|
116
|
+
|
|
117
|
+
_INSTRUMENTED_ENGINES.add(engine)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True, slots=True)
|
|
121
|
+
class QuickTab:
|
|
122
|
+
"""Declarative quick-filter tab displayed above model list grids."""
|
|
123
|
+
|
|
124
|
+
key: str
|
|
125
|
+
label: str
|
|
126
|
+
filters: Mapping[str, Any] = field(default_factory=dict)
|
|
127
|
+
tone: str = "neutral"
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def normalized_tone(self) -> str:
|
|
131
|
+
return normalize_tone(self.tone)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True, slots=True)
|
|
135
|
+
class PageResult:
|
|
136
|
+
"""Paginated query result supporting both Offset and HMAC Keyset Cursor modes."""
|
|
137
|
+
|
|
138
|
+
items: list[Any]
|
|
139
|
+
total: int
|
|
140
|
+
page: int
|
|
141
|
+
page_size: int
|
|
142
|
+
pages: int
|
|
143
|
+
tab_counts: dict[str, int]
|
|
144
|
+
active_filters: dict[str, str]
|
|
145
|
+
pagination_mode: PaginationMode = "offset"
|
|
146
|
+
next_cursor: str | None = None
|
|
147
|
+
prev_cursor: str | None = None
|
|
148
|
+
has_next: bool = False
|
|
149
|
+
has_prev: bool = False
|
|
150
|
+
total_is_estimated: bool = False
|
|
151
|
+
total_known: bool = True
|
|
152
|
+
trashed_count: int = 0
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def iter_filter_items(
|
|
156
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]],
|
|
157
|
+
) -> list[tuple[str, str]]:
|
|
158
|
+
"""Normalize filter parameters (single-value Mapping or multi-value pairs) into pairs."""
|
|
159
|
+
if isinstance(filter_params, Mapping):
|
|
160
|
+
return [(str(k), str(v)) for k, v in filter_params.items()]
|
|
161
|
+
return [(str(k), str(v)) for k, v in filter_params]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def parse_filter_key(
|
|
165
|
+
raw_key: str,
|
|
166
|
+
known_fields: frozenset[str] | set[str] | None = None,
|
|
167
|
+
) -> tuple[str, str]:
|
|
168
|
+
"""Split a query filter key like `price__gte` into `(field_name, op)`."""
|
|
169
|
+
if known_fields is not None and raw_key in known_fields:
|
|
170
|
+
return raw_key, "eq"
|
|
171
|
+
for suffix, op in _FILTER_OP_SUFFIXES:
|
|
172
|
+
if raw_key.endswith(suffix):
|
|
173
|
+
base = raw_key[: -len(suffix)]
|
|
174
|
+
if known_fields is None or base in known_fields:
|
|
175
|
+
return base, op
|
|
176
|
+
return raw_key, "eq"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _serialize_cursor_val(val: Any) -> Any:
|
|
180
|
+
if val is None or isinstance(val, (int, float, bool, str)):
|
|
181
|
+
return val
|
|
182
|
+
if isinstance(val, enum.Enum):
|
|
183
|
+
return val.value
|
|
184
|
+
if isinstance(val, (datetime, date, dt_time)):
|
|
185
|
+
return val.isoformat()
|
|
186
|
+
if isinstance(val, (Decimal, uuid.UUID)):
|
|
187
|
+
return str(val)
|
|
188
|
+
return str(val)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def encode_cursor(payload: Mapping[str, Any], *, secret: bytes) -> str:
|
|
192
|
+
"""Encode and HMAC-SHA256 sign an opaque keyset pagination cursor."""
|
|
193
|
+
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()
|
|
194
|
+
b64 = base64.urlsafe_b64encode(body).decode("ascii").rstrip("=")
|
|
195
|
+
sig = hmac.new(secret or b"flask-deck-cursor", body, hashlib.sha256).hexdigest()[:24]
|
|
196
|
+
return f"{b64}.{sig}"
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def decode_cursor(token: str | None, *, secret: bytes) -> dict[str, Any] | None:
|
|
200
|
+
"""Verify HMAC signature and decode an opaque cursor token (fail-closed -> None)."""
|
|
201
|
+
if not token or "." not in token or len(token) > 4096:
|
|
202
|
+
return None
|
|
203
|
+
b64_part, provided_sig = token.rsplit(".", 1)
|
|
204
|
+
pad = "=" * (-len(b64_part) % 4)
|
|
205
|
+
try:
|
|
206
|
+
body = base64.urlsafe_b64decode(b64_part + pad)
|
|
207
|
+
except Exception:
|
|
208
|
+
return None
|
|
209
|
+
expected_sig = hmac.new(secret or b"flask-deck-cursor", body, hashlib.sha256).hexdigest()[:24]
|
|
210
|
+
if not hmac.compare_digest(expected_sig, provided_sig):
|
|
211
|
+
return None
|
|
212
|
+
try:
|
|
213
|
+
parsed = json.loads(body.decode("utf-8"))
|
|
214
|
+
except Exception:
|
|
215
|
+
return None
|
|
216
|
+
return parsed if isinstance(parsed, dict) else None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _coerce_filter_value(
|
|
220
|
+
raw: Any,
|
|
221
|
+
fmeta: FieldMeta,
|
|
222
|
+
*,
|
|
223
|
+
allow_empty_string: bool = False,
|
|
224
|
+
) -> Any:
|
|
225
|
+
if raw is None:
|
|
226
|
+
return None
|
|
227
|
+
if raw == "":
|
|
228
|
+
if allow_empty_string and fmeta.field_type in {"string", "text", "email", "url"}:
|
|
229
|
+
return ""
|
|
230
|
+
return None
|
|
231
|
+
if fmeta.enum_class is not None:
|
|
232
|
+
if isinstance(raw, fmeta.enum_class):
|
|
233
|
+
return raw
|
|
234
|
+
text = str(raw)
|
|
235
|
+
for member in fmeta.enum_class:
|
|
236
|
+
if str(member.value) == text or member.name == text:
|
|
237
|
+
return member
|
|
238
|
+
return None
|
|
239
|
+
if fmeta.field_type == "boolean":
|
|
240
|
+
if isinstance(raw, bool):
|
|
241
|
+
return raw
|
|
242
|
+
lower = str(raw).strip().lower()
|
|
243
|
+
if lower in {"1", "true", "yes", "on"}:
|
|
244
|
+
return True
|
|
245
|
+
if lower in {"0", "false", "no", "off"}:
|
|
246
|
+
return False
|
|
247
|
+
return None
|
|
248
|
+
if fmeta.field_type == "integer":
|
|
249
|
+
try:
|
|
250
|
+
return int(raw)
|
|
251
|
+
except (TypeError, ValueError):
|
|
252
|
+
return None
|
|
253
|
+
if fmeta.field_type == "float":
|
|
254
|
+
try:
|
|
255
|
+
return float(raw)
|
|
256
|
+
except (TypeError, ValueError):
|
|
257
|
+
return None
|
|
258
|
+
if fmeta.field_type == "decimal":
|
|
259
|
+
try:
|
|
260
|
+
return Decimal(str(raw))
|
|
261
|
+
except (InvalidOperation, ValueError):
|
|
262
|
+
return None
|
|
263
|
+
if fmeta.field_type == "uuid":
|
|
264
|
+
try:
|
|
265
|
+
return raw if isinstance(raw, uuid.UUID) else uuid.UUID(str(raw))
|
|
266
|
+
except (TypeError, ValueError):
|
|
267
|
+
return None
|
|
268
|
+
if fmeta.field_type == "date":
|
|
269
|
+
if isinstance(raw, date) and not isinstance(raw, datetime):
|
|
270
|
+
return raw
|
|
271
|
+
try:
|
|
272
|
+
return date.fromisoformat(str(raw)[:10])
|
|
273
|
+
except ValueError:
|
|
274
|
+
return None
|
|
275
|
+
if fmeta.field_type == "time":
|
|
276
|
+
if isinstance(raw, dt_time):
|
|
277
|
+
return raw
|
|
278
|
+
try:
|
|
279
|
+
return dt_time.fromisoformat(str(raw))
|
|
280
|
+
except ValueError:
|
|
281
|
+
return None
|
|
282
|
+
if fmeta.field_type == "datetime":
|
|
283
|
+
if isinstance(raw, datetime):
|
|
284
|
+
return raw
|
|
285
|
+
try:
|
|
286
|
+
dt = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
|
|
287
|
+
return dt
|
|
288
|
+
except ValueError:
|
|
289
|
+
try:
|
|
290
|
+
d = date.fromisoformat(str(raw)[:10])
|
|
291
|
+
return datetime(d.year, d.month, d.day, tzinfo=timezone.utc)
|
|
292
|
+
except ValueError:
|
|
293
|
+
return None
|
|
294
|
+
return str(raw)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _coerce_array_item(raw: str, item_type: str | None) -> Any:
|
|
298
|
+
text = raw.strip()
|
|
299
|
+
if item_type == "integer":
|
|
300
|
+
try:
|
|
301
|
+
return int(text)
|
|
302
|
+
except ValueError:
|
|
303
|
+
return None
|
|
304
|
+
if item_type == "float":
|
|
305
|
+
try:
|
|
306
|
+
return float(text)
|
|
307
|
+
except ValueError:
|
|
308
|
+
return None
|
|
309
|
+
if item_type == "boolean":
|
|
310
|
+
lower = text.lower()
|
|
311
|
+
if lower in {"1", "true", "yes"}:
|
|
312
|
+
return True
|
|
313
|
+
if lower in {"0", "false", "no"}:
|
|
314
|
+
return False
|
|
315
|
+
return None
|
|
316
|
+
return text
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _pg_json_text_accessor(col: Any, parts: list[str]) -> Any:
|
|
320
|
+
"""Return text accessor compatible with both postgresql.JSONB (.astext) and generic sa.JSON (.as_string)."""
|
|
321
|
+
indexed = col[parts] if len(parts) > 1 else col[parts[0]]
|
|
322
|
+
return indexed.astext if hasattr(indexed, "astext") else indexed.as_string()
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _build_json_path_condition(
|
|
326
|
+
col: Any,
|
|
327
|
+
fmeta: FieldMeta,
|
|
328
|
+
raw_value: str,
|
|
329
|
+
*,
|
|
330
|
+
dialect_name: str,
|
|
331
|
+
) -> sa.ColumnElement[bool] | None:
|
|
332
|
+
if "=" not in raw_value:
|
|
333
|
+
return None
|
|
334
|
+
raw_path, expected = raw_value.split("=", 1)
|
|
335
|
+
clean_path = raw_path.strip()
|
|
336
|
+
if not _JSON_PATH_RE.fullmatch(clean_path):
|
|
337
|
+
return None
|
|
338
|
+
parts = clean_path.split(".")
|
|
339
|
+
target = expected.strip()
|
|
340
|
+
if dialect_name == "postgresql":
|
|
341
|
+
accessor = _pg_json_text_accessor(col, parts)
|
|
342
|
+
expr: sa.ColumnElement[bool] = accessor == target
|
|
343
|
+
return expr
|
|
344
|
+
# SQLite / MySQL / generic JSON extract
|
|
345
|
+
json_path = f"$.{clean_path}"
|
|
346
|
+
extracted = sa.func.json_extract(col, json_path)
|
|
347
|
+
if dialect_name in {"mysql", "mariadb"}:
|
|
348
|
+
mysql_expr: sa.ColumnElement[bool] = sa.func.json_unquote(extracted) == target
|
|
349
|
+
return mysql_expr
|
|
350
|
+
# Handle JSON string vs numeric/boolean scalar matching cleanly in SQLite
|
|
351
|
+
sqlite_expr: sa.ColumnElement[bool] = sa.cast(extracted, sa.String) == target
|
|
352
|
+
if target.lower() in {"true", "false"}:
|
|
353
|
+
bool_int = 1 if target.lower() == "true" else 0
|
|
354
|
+
return sa.or_(sqlite_expr, extracted == bool_int)
|
|
355
|
+
return sqlite_expr
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _build_json_has_key_condition(
|
|
359
|
+
col: Any,
|
|
360
|
+
fmeta: FieldMeta,
|
|
361
|
+
raw_key: str,
|
|
362
|
+
*,
|
|
363
|
+
dialect_name: str,
|
|
364
|
+
) -> sa.ColumnElement[bool] | None:
|
|
365
|
+
clean_key = raw_key.strip()
|
|
366
|
+
if not _JSON_PATH_RE.fullmatch(clean_key):
|
|
367
|
+
return None
|
|
368
|
+
if dialect_name == "postgresql" and fmeta.is_jsonb and "." not in clean_key and hasattr(col, "has_key"):
|
|
369
|
+
pg_expr: sa.ColumnElement[bool] = col.has_key(clean_key)
|
|
370
|
+
return pg_expr
|
|
371
|
+
if dialect_name == "postgresql":
|
|
372
|
+
parts = clean_key.split(".")
|
|
373
|
+
accessor = _pg_json_text_accessor(col, parts)
|
|
374
|
+
pg_null_expr: sa.ColumnElement[bool] = accessor.is_not(None)
|
|
375
|
+
return pg_null_expr
|
|
376
|
+
if dialect_name in {"mysql", "mariadb"}:
|
|
377
|
+
mysql_expr: sa.ColumnElement[bool] = sa.func.json_contains_path(col, "one", f"$.{clean_key}") == 1
|
|
378
|
+
return mysql_expr
|
|
379
|
+
# SQLite json_type returns NULL when path does not exist, non-NULL even if JSON value is null!
|
|
380
|
+
sqlite_expr: sa.ColumnElement[bool] = sa.func.json_type(col, f"$.{clean_key}").is_not(None)
|
|
381
|
+
return sqlite_expr
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _flatten_json_contains(
|
|
385
|
+
prefix: str,
|
|
386
|
+
mapping: Mapping[str, Any],
|
|
387
|
+
out: list[tuple[str, Any]],
|
|
388
|
+
) -> bool:
|
|
389
|
+
for k, v in mapping.items():
|
|
390
|
+
if not isinstance(k, str) or not _JSON_PATH_RE.fullmatch(k):
|
|
391
|
+
return False
|
|
392
|
+
path = f"{prefix}.{k}" if prefix else k
|
|
393
|
+
if not _JSON_PATH_RE.fullmatch(path):
|
|
394
|
+
return False
|
|
395
|
+
if isinstance(v, dict):
|
|
396
|
+
if not v or not _flatten_json_contains(path, v, out):
|
|
397
|
+
return False
|
|
398
|
+
elif isinstance(v, list):
|
|
399
|
+
return False
|
|
400
|
+
else:
|
|
401
|
+
out.append((path, v))
|
|
402
|
+
return True
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _build_json_contains_condition(
|
|
406
|
+
col: Any,
|
|
407
|
+
fmeta: FieldMeta,
|
|
408
|
+
raw_json: str,
|
|
409
|
+
*,
|
|
410
|
+
dialect_name: str,
|
|
411
|
+
) -> sa.ColumnElement[bool] | None:
|
|
412
|
+
try:
|
|
413
|
+
parsed = json.loads(raw_json)
|
|
414
|
+
except Exception:
|
|
415
|
+
return None
|
|
416
|
+
if not check_json_depth(parsed, max_depth=4):
|
|
417
|
+
return None
|
|
418
|
+
if dialect_name == "postgresql" and fmeta.is_jsonb and hasattr(col, "contains"):
|
|
419
|
+
pg_expr: sa.ColumnElement[bool] = col.contains(parsed)
|
|
420
|
+
return pg_expr
|
|
421
|
+
if isinstance(parsed, dict) and parsed:
|
|
422
|
+
flat_pairs: list[tuple[str, Any]] = []
|
|
423
|
+
if not _flatten_json_contains("", parsed, flat_pairs):
|
|
424
|
+
return None
|
|
425
|
+
preds: list[sa.ColumnElement[bool]] = []
|
|
426
|
+
for path, v in flat_pairs:
|
|
427
|
+
if v is None:
|
|
428
|
+
if dialect_name == "postgresql":
|
|
429
|
+
preds.append(
|
|
430
|
+
sa.and_(
|
|
431
|
+
_build_json_has_key_condition(col, fmeta, path, dialect_name=dialect_name)
|
|
432
|
+
or sa.true(),
|
|
433
|
+
_pg_json_text_accessor(col, path.split(".")).is_(None),
|
|
434
|
+
)
|
|
435
|
+
)
|
|
436
|
+
else:
|
|
437
|
+
preds.append(sa.func.json_type(col, f"$.{path}") == "null")
|
|
438
|
+
continue
|
|
439
|
+
val_str = "true" if v is True else ("false" if v is False else str(v))
|
|
440
|
+
sub = _build_json_path_condition(col, fmeta, f"{path}={val_str}", dialect_name=dialect_name)
|
|
441
|
+
if sub is not None:
|
|
442
|
+
preds.append(sub)
|
|
443
|
+
if preds:
|
|
444
|
+
return sa.and_(*preds) if len(preds) > 1 else preds[0]
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def build_filter_condition(
|
|
449
|
+
meta: ModelMeta,
|
|
450
|
+
field_name: str,
|
|
451
|
+
raw_value: Any,
|
|
452
|
+
*,
|
|
453
|
+
op: str = "eq",
|
|
454
|
+
dialect_name: str = "",
|
|
455
|
+
) -> sa.ColumnElement[bool] | None:
|
|
456
|
+
"""Build a safe parameterized SQLAlchemy filter expression for a single field."""
|
|
457
|
+
fmeta = meta.fields_by_name.get(field_name)
|
|
458
|
+
if fmeta is None:
|
|
459
|
+
return None
|
|
460
|
+
col: Any = getattr(meta.model, field_name)
|
|
461
|
+
|
|
462
|
+
if op == "null":
|
|
463
|
+
if str(raw_value).lower() in {"1", "true", "null"}:
|
|
464
|
+
return col.is_(None) # type: ignore[no-any-return]
|
|
465
|
+
if str(raw_value).lower() in {"0", "false", "notnull"}:
|
|
466
|
+
return col.is_not(None) # type: ignore[no-any-return]
|
|
467
|
+
return None
|
|
468
|
+
|
|
469
|
+
# JSON / JSONB operators
|
|
470
|
+
if fmeta.field_type == "json":
|
|
471
|
+
text_val = str(raw_value).strip()
|
|
472
|
+
if not text_val:
|
|
473
|
+
return None
|
|
474
|
+
if op == "has_key":
|
|
475
|
+
return _build_json_has_key_condition(col, fmeta, text_val, dialect_name=dialect_name)
|
|
476
|
+
if op == "path":
|
|
477
|
+
return _build_json_path_condition(col, fmeta, text_val, dialect_name=dialect_name)
|
|
478
|
+
if op == "json_contains":
|
|
479
|
+
return _build_json_contains_condition(col, fmeta, text_val, dialect_name=dialect_name)
|
|
480
|
+
if op == "contains":
|
|
481
|
+
escaped = f"%{escape_like(text_val)}%"
|
|
482
|
+
return sa.cast(col, sa.String).ilike(escaped, escape="\\")
|
|
483
|
+
return None
|
|
484
|
+
|
|
485
|
+
# PostgreSQL ARRAY operators (compatible with both postgresql.ARRAY and generic sa.ARRAY)
|
|
486
|
+
if fmeta.field_type == "array":
|
|
487
|
+
raw_text = str(raw_value).strip()
|
|
488
|
+
if not raw_text:
|
|
489
|
+
return None
|
|
490
|
+
if op in {"array_contains", "contains", "eq"}:
|
|
491
|
+
item = _coerce_array_item(raw_text, fmeta.array_item_type)
|
|
492
|
+
if item is None:
|
|
493
|
+
return None
|
|
494
|
+
if dialect_name == "postgresql":
|
|
495
|
+
if hasattr(col, "contains"):
|
|
496
|
+
try:
|
|
497
|
+
pg_arr: sa.ColumnElement[bool] = col.contains([item])
|
|
498
|
+
return pg_arr
|
|
499
|
+
except NotImplementedError:
|
|
500
|
+
pass
|
|
501
|
+
pg_any: sa.ColumnElement[bool] = sa.any_(col) == item
|
|
502
|
+
return pg_any
|
|
503
|
+
return sa.cast(col, sa.String).ilike(f"%{escape_like(str(item))}%", escape="\\")
|
|
504
|
+
if op == "array_any":
|
|
505
|
+
items = [
|
|
506
|
+
coerced
|
|
507
|
+
for part in raw_text.split(",")
|
|
508
|
+
if part.strip()
|
|
509
|
+
and (coerced := _coerce_array_item(part, fmeta.array_item_type)) is not None
|
|
510
|
+
]
|
|
511
|
+
if not items:
|
|
512
|
+
return None
|
|
513
|
+
if dialect_name == "postgresql":
|
|
514
|
+
if hasattr(col, "overlap"):
|
|
515
|
+
try:
|
|
516
|
+
pg_ov: sa.ColumnElement[bool] = col.overlap(items)
|
|
517
|
+
return pg_ov
|
|
518
|
+
except NotImplementedError:
|
|
519
|
+
pass
|
|
520
|
+
return sa.or_(*(sa.any_(col) == it for it in items))
|
|
521
|
+
return sa.or_(
|
|
522
|
+
*(sa.cast(col, sa.String).ilike(f"%{escape_like(str(it))}%", escape="\\") for it in items)
|
|
523
|
+
)
|
|
524
|
+
return None
|
|
525
|
+
|
|
526
|
+
if op == "preset" and fmeta.field_type in {"datetime", "date"}:
|
|
527
|
+
now = datetime.now(timezone.utc)
|
|
528
|
+
preset = str(raw_value).lower()
|
|
529
|
+
start_dt: datetime | None = None
|
|
530
|
+
if preset == "today":
|
|
531
|
+
start_dt = datetime(now.year, now.month, now.day, tzinfo=timezone.utc)
|
|
532
|
+
elif preset == "7d":
|
|
533
|
+
start_dt = now - timedelta(days=7)
|
|
534
|
+
elif preset == "30d":
|
|
535
|
+
start_dt = now - timedelta(days=30)
|
|
536
|
+
elif preset == "month":
|
|
537
|
+
start_dt = datetime(now.year, now.month, 1, tzinfo=timezone.utc)
|
|
538
|
+
if start_dt is None:
|
|
539
|
+
return None
|
|
540
|
+
bound: datetime | date = start_dt.date() if fmeta.field_type == "date" else start_dt
|
|
541
|
+
preset_expr: sa.ColumnElement[bool] = col >= bound
|
|
542
|
+
return preset_expr
|
|
543
|
+
|
|
544
|
+
if op in {"contains", "startswith"}:
|
|
545
|
+
clean_term = str(raw_value).strip()
|
|
546
|
+
if not clean_term:
|
|
547
|
+
return None
|
|
548
|
+
escaped = f"%{escape_like(clean_term)}%" if op == "contains" else f"{escape_like(clean_term)}%"
|
|
549
|
+
target_col = col if fmeta.field_type in {"string", "text", "email", "url"} else sa.cast(col, sa.String)
|
|
550
|
+
like_expr: sa.ColumnElement[bool] = target_col.ilike(escaped, escape="\\")
|
|
551
|
+
return like_expr
|
|
552
|
+
|
|
553
|
+
if op in {"gt", "gte", "lt", "lte"}:
|
|
554
|
+
coerced = _coerce_filter_value(raw_value, fmeta)
|
|
555
|
+
if coerced is None:
|
|
556
|
+
return None
|
|
557
|
+
if op == "lte" and fmeta.field_type == "datetime" and isinstance(raw_value, str) and len(raw_value) <= 10:
|
|
558
|
+
coerced = coerced + timedelta(days=1) - timedelta(microseconds=1)
|
|
559
|
+
if op == "gt":
|
|
560
|
+
gt_expr: sa.ColumnElement[bool] = col > coerced
|
|
561
|
+
return gt_expr
|
|
562
|
+
if op == "gte":
|
|
563
|
+
gte_expr: sa.ColumnElement[bool] = col >= coerced
|
|
564
|
+
return gte_expr
|
|
565
|
+
if op == "lt":
|
|
566
|
+
lt_expr: sa.ColumnElement[bool] = col < coerced
|
|
567
|
+
return lt_expr
|
|
568
|
+
lte_expr: sa.ColumnElement[bool] = col <= coerced
|
|
569
|
+
return lte_expr
|
|
570
|
+
|
|
571
|
+
if op == "neq":
|
|
572
|
+
coerced = _coerce_filter_value(raw_value, fmeta)
|
|
573
|
+
if coerced is None:
|
|
574
|
+
return None
|
|
575
|
+
neq_expr: sa.ColumnElement[bool] = col != coerced
|
|
576
|
+
return neq_expr
|
|
577
|
+
|
|
578
|
+
# Comma-separated multi-select support for enum/string/integer equality
|
|
579
|
+
if isinstance(raw_value, str) and "," in raw_value and fmeta.field_type in {"enum", "string", "integer"}:
|
|
580
|
+
parts = [_coerce_filter_value(p.strip(), fmeta) for p in raw_value.split(",")]
|
|
581
|
+
valid = [p for p in parts if p is not None]
|
|
582
|
+
if not valid:
|
|
583
|
+
return None
|
|
584
|
+
in_expr: sa.ColumnElement[bool] = col.in_(valid)
|
|
585
|
+
return in_expr
|
|
586
|
+
|
|
587
|
+
coerced = _coerce_filter_value(raw_value, fmeta)
|
|
588
|
+
if coerced is None:
|
|
589
|
+
return None
|
|
590
|
+
eq_expr: sa.ColumnElement[bool] = col == coerced
|
|
591
|
+
return eq_expr
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def build_search_clause(
|
|
595
|
+
meta: ModelMeta,
|
|
596
|
+
searchable_columns: Sequence[str],
|
|
597
|
+
term: str,
|
|
598
|
+
) -> tuple[sa.ColumnElement[bool] | None, list[Any]]:
|
|
599
|
+
"""Build safe escaped LIKE search clause across scalar and dotted FK columns."""
|
|
600
|
+
clean = term.strip()
|
|
601
|
+
if not clean or not searchable_columns:
|
|
602
|
+
return None, []
|
|
603
|
+
|
|
604
|
+
escaped = f"%{escape_like(clean)}%"
|
|
605
|
+
predicates: list[sa.ColumnElement[bool]] = []
|
|
606
|
+
joins: list[Any] = []
|
|
607
|
+
seen_joins: set[str] = set()
|
|
608
|
+
|
|
609
|
+
for col_spec in searchable_columns:
|
|
610
|
+
if "." in col_spec:
|
|
611
|
+
rel_name, target_col_name = col_spec.split(".", 1)
|
|
612
|
+
rel_meta = meta.relationships_by_name.get(rel_name)
|
|
613
|
+
if rel_meta is None or rel_meta.direction != "MANYTOONE":
|
|
614
|
+
continue
|
|
615
|
+
target_col = getattr(rel_meta.target_model, target_col_name, None)
|
|
616
|
+
if target_col is None:
|
|
617
|
+
continue
|
|
618
|
+
if rel_name not in seen_joins:
|
|
619
|
+
joins.append(getattr(meta.model, rel_name))
|
|
620
|
+
seen_joins.add(rel_name)
|
|
621
|
+
predicates.append(sa.cast(target_col, sa.String).ilike(escaped, escape="\\"))
|
|
622
|
+
continue
|
|
623
|
+
|
|
624
|
+
fmeta = meta.fields_by_name.get(col_spec)
|
|
625
|
+
if fmeta is None or fmeta.is_sensitive:
|
|
626
|
+
continue
|
|
627
|
+
col = getattr(meta.model, col_spec)
|
|
628
|
+
if fmeta.field_type in {"string", "text", "email", "url"}:
|
|
629
|
+
predicates.append(col.ilike(escaped, escape="\\"))
|
|
630
|
+
elif fmeta.field_type == "integer":
|
|
631
|
+
with contextlib.suppress(ValueError):
|
|
632
|
+
predicates.append(col == int(clean))
|
|
633
|
+
elif fmeta.field_type == "uuid":
|
|
634
|
+
try:
|
|
635
|
+
predicates.append(col == uuid.UUID(clean))
|
|
636
|
+
except ValueError:
|
|
637
|
+
predicates.append(sa.cast(col, sa.String).ilike(escaped, escape="\\"))
|
|
638
|
+
elif fmeta.field_type == "enum":
|
|
639
|
+
predicates.append(sa.cast(col, sa.String).ilike(escaped, escape="\\"))
|
|
640
|
+
|
|
641
|
+
if not predicates:
|
|
642
|
+
return None, []
|
|
643
|
+
return sa.or_(*predicates), joins
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def apply_list_load_options(
|
|
647
|
+
stmt: sa.Select[Any],
|
|
648
|
+
meta: ModelMeta,
|
|
649
|
+
displayed_columns: Sequence[str],
|
|
650
|
+
*,
|
|
651
|
+
defer_unshown_heavy: bool = True,
|
|
652
|
+
) -> sa.Select[Any]:
|
|
653
|
+
"""Attach automatic joinedload/selectinload for displayed relations and defer heavy unshown blobs."""
|
|
654
|
+
shown_set = set(displayed_columns)
|
|
655
|
+
options: list[Any] = []
|
|
656
|
+
loaded_rels: set[str] = set()
|
|
657
|
+
|
|
658
|
+
for col_name in displayed_columns:
|
|
659
|
+
rel_meta = meta.relationships_by_name.get(col_name)
|
|
660
|
+
if rel_meta is None:
|
|
661
|
+
rel_meta = meta.fk_column_to_rel.get(col_name)
|
|
662
|
+
if rel_meta is None or rel_meta.name in loaded_rels:
|
|
663
|
+
continue
|
|
664
|
+
rel_attr = getattr(meta.model, rel_meta.name)
|
|
665
|
+
if rel_meta.direction == "MANYTOONE":
|
|
666
|
+
options.append(joinedload(rel_attr))
|
|
667
|
+
else:
|
|
668
|
+
options.append(selectinload(rel_attr))
|
|
669
|
+
loaded_rels.add(rel_meta.name)
|
|
670
|
+
|
|
671
|
+
if defer_unshown_heavy:
|
|
672
|
+
pk_names = {pk.name for pk in meta.primary_keys}
|
|
673
|
+
for heavy_col in meta.heavy_fields:
|
|
674
|
+
if heavy_col not in shown_set and heavy_col not in pk_names:
|
|
675
|
+
options.append(defer(getattr(meta.model, heavy_col)))
|
|
676
|
+
|
|
677
|
+
if options:
|
|
678
|
+
stmt = stmt.options(*options)
|
|
679
|
+
return stmt
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def soft_delete_predicate(
|
|
683
|
+
meta: ModelMeta,
|
|
684
|
+
soft_delete_column: str | None,
|
|
685
|
+
*,
|
|
686
|
+
trashed: bool = False,
|
|
687
|
+
) -> sa.ColumnElement[bool] | None:
|
|
688
|
+
"""Return active (non-deleted) or trashed predicate for a soft-delete column."""
|
|
689
|
+
if not soft_delete_column:
|
|
690
|
+
return None
|
|
691
|
+
fmeta = meta.fields_by_name.get(soft_delete_column)
|
|
692
|
+
if fmeta is None:
|
|
693
|
+
return None
|
|
694
|
+
col: Any = getattr(meta.model, soft_delete_column)
|
|
695
|
+
if fmeta.field_type == "boolean":
|
|
696
|
+
expr: sa.ColumnElement[bool] = col.is_(True) if trashed else sa.or_(col.is_(False), col.is_(None))
|
|
697
|
+
return expr
|
|
698
|
+
expr_dt: sa.ColumnElement[bool] = col.is_not(None) if trashed else col.is_(None)
|
|
699
|
+
return expr_dt
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def build_pg_reltuples_statement(table_name: str, *, schema: str | None = None) -> sa.TextClause:
|
|
703
|
+
"""Build parameterized O(1) PostgreSQL catalog row-estimate statement (schema-aware)."""
|
|
704
|
+
return sa.text(
|
|
705
|
+
"SELECT CAST(c.reltuples AS BIGINT) "
|
|
706
|
+
"FROM pg_class c "
|
|
707
|
+
"JOIN pg_namespace n ON n.oid = c.relnamespace "
|
|
708
|
+
"WHERE c.relname = :tbl AND n.nspname = COALESCE(:sch, current_schema()) "
|
|
709
|
+
"LIMIT 1"
|
|
710
|
+
).bindparams(tbl=table_name, sch=schema)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def estimate_table_rows(session: Session, meta: ModelMeta) -> int | None:
|
|
714
|
+
"""Return O(1) estimated row count on PostgreSQL via pg_class.reltuples (None if unanalyzed)."""
|
|
715
|
+
bind = session.get_bind()
|
|
716
|
+
dialect_name = bind.dialect.name if bind is not None else ""
|
|
717
|
+
if dialect_name != "postgresql":
|
|
718
|
+
return None
|
|
719
|
+
try:
|
|
720
|
+
with session.begin_nested():
|
|
721
|
+
val = session.execute(
|
|
722
|
+
build_pg_reltuples_statement(meta.table_name, schema=meta.schema)
|
|
723
|
+
).scalar()
|
|
724
|
+
if val is None:
|
|
725
|
+
return None
|
|
726
|
+
est = int(val)
|
|
727
|
+
# pg_class.reltuples == -1 means table has never been analyzed/vacuumed -> fallback to COUNT(*)
|
|
728
|
+
return est if est >= 0 else None
|
|
729
|
+
except Exception:
|
|
730
|
+
return None
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def compute_quick_tab_counts(
|
|
734
|
+
session: Session,
|
|
735
|
+
meta: ModelMeta,
|
|
736
|
+
base_where: Sequence[sa.ColumnElement[bool]],
|
|
737
|
+
quick_tabs: Sequence[QuickTab],
|
|
738
|
+
joins: Sequence[Any] = (),
|
|
739
|
+
*,
|
|
740
|
+
dialect_name: str = "",
|
|
741
|
+
soft_delete_column: str | None = None,
|
|
742
|
+
) -> tuple[dict[str, int], int]:
|
|
743
|
+
"""Compute badge counts for ALL QuickTabs (plus Trash count) in ONE SQL query."""
|
|
744
|
+
if not quick_tabs and not soft_delete_column:
|
|
745
|
+
return {}, 0
|
|
746
|
+
|
|
747
|
+
active_sd = soft_delete_predicate(meta, soft_delete_column, trashed=False)
|
|
748
|
+
trashed_sd = soft_delete_predicate(meta, soft_delete_column, trashed=True)
|
|
749
|
+
known = frozenset(meta.fields_by_name.keys())
|
|
750
|
+
|
|
751
|
+
agg_exprs: list[Any] = []
|
|
752
|
+
for tab in quick_tabs:
|
|
753
|
+
tab_conds: list[sa.ColumnElement[bool]] = []
|
|
754
|
+
if active_sd is not None:
|
|
755
|
+
tab_conds.append(active_sd)
|
|
756
|
+
for raw_fname, fval in tab.filters.items():
|
|
757
|
+
fname, fop = parse_filter_key(raw_fname, known)
|
|
758
|
+
cond = build_filter_condition(meta, fname, fval, op=fop, dialect_name=dialect_name)
|
|
759
|
+
if cond is not None:
|
|
760
|
+
tab_conds.append(cond)
|
|
761
|
+
if tab_conds:
|
|
762
|
+
combined = sa.and_(*tab_conds) if len(tab_conds) > 1 else tab_conds[0]
|
|
763
|
+
agg_exprs.append(sa.func.count(sa.case((combined, 1))).label(tab.key))
|
|
764
|
+
else:
|
|
765
|
+
agg_exprs.append(sa.func.count().label(tab.key))
|
|
766
|
+
|
|
767
|
+
if trashed_sd is not None:
|
|
768
|
+
agg_exprs.append(sa.func.count(sa.case((trashed_sd, 1))).label("__trashed"))
|
|
769
|
+
|
|
770
|
+
stmt = sa.select(*agg_exprs).select_from(meta.model)
|
|
771
|
+
for rel_attr in joins:
|
|
772
|
+
stmt = stmt.outerjoin(rel_attr)
|
|
773
|
+
if base_where:
|
|
774
|
+
stmt = stmt.where(*base_where)
|
|
775
|
+
|
|
776
|
+
row = session.execute(stmt).first()
|
|
777
|
+
if row is None:
|
|
778
|
+
return {tab.key: 0 for tab in quick_tabs}, 0
|
|
779
|
+
counts = {tab.key: int(row[idx] or 0) for idx, tab in enumerate(quick_tabs)}
|
|
780
|
+
trashed_cnt = int(row[len(quick_tabs)] or 0) if trashed_sd is not None else 0
|
|
781
|
+
return counts, trashed_cnt
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _build_keyset_predicate(
|
|
785
|
+
meta: ModelMeta,
|
|
786
|
+
*,
|
|
787
|
+
sort_field: FieldMeta,
|
|
788
|
+
pk_field: FieldMeta,
|
|
789
|
+
sort_val: Any,
|
|
790
|
+
pk_val: Any,
|
|
791
|
+
effective_dir: str,
|
|
792
|
+
) -> sa.ColumnElement[bool]:
|
|
793
|
+
sort_col: Any = getattr(meta.model, sort_field.name)
|
|
794
|
+
pk_col: Any = getattr(meta.model, pk_field.name)
|
|
795
|
+
if sort_field.name == pk_field.name:
|
|
796
|
+
expr_single: sa.ColumnElement[bool] = (
|
|
797
|
+
(pk_col < pk_val) if effective_dir == "desc" else (pk_col > pk_val)
|
|
798
|
+
)
|
|
799
|
+
return expr_single
|
|
800
|
+
|
|
801
|
+
if sort_val is None:
|
|
802
|
+
pk_cmp: sa.ColumnElement[bool] = (pk_col < pk_val) if effective_dir == "desc" else (pk_col > pk_val)
|
|
803
|
+
if effective_dir == "desc":
|
|
804
|
+
return sa.and_(sort_col.is_(None), pk_cmp)
|
|
805
|
+
return sa.or_(sort_col.is_not(None), sa.and_(sort_col.is_(None), pk_cmp))
|
|
806
|
+
|
|
807
|
+
if effective_dir == "desc":
|
|
808
|
+
return sa.or_(
|
|
809
|
+
sort_col < sort_val,
|
|
810
|
+
sort_col.is_(None),
|
|
811
|
+
sa.and_(sort_col == sort_val, pk_col < pk_val),
|
|
812
|
+
)
|
|
813
|
+
return sa.or_(
|
|
814
|
+
sort_col > sort_val,
|
|
815
|
+
sa.and_(sort_col == sort_val, pk_col > pk_val),
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def execute_list_query(
|
|
820
|
+
session: Session,
|
|
821
|
+
meta: ModelMeta,
|
|
822
|
+
*,
|
|
823
|
+
base_stmt: sa.Select[Any],
|
|
824
|
+
displayed_columns: Sequence[str],
|
|
825
|
+
searchable_columns: Sequence[str],
|
|
826
|
+
filterable_columns: frozenset[str],
|
|
827
|
+
sortable_columns: frozenset[str],
|
|
828
|
+
quick_tabs: Sequence[QuickTab],
|
|
829
|
+
active_tab_key: str | None,
|
|
830
|
+
search_term: str,
|
|
831
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]],
|
|
832
|
+
sort_by: str | None,
|
|
833
|
+
sort_dir: str,
|
|
834
|
+
default_sort: tuple[str, str] | None,
|
|
835
|
+
page: int,
|
|
836
|
+
page_size: int,
|
|
837
|
+
max_page_size: int,
|
|
838
|
+
pagination_mode: PaginationMode = "offset",
|
|
839
|
+
count_mode: CountMode = "exact",
|
|
840
|
+
after_cursor: str | None = None,
|
|
841
|
+
before_cursor: str | None = None,
|
|
842
|
+
cursor_secret: bytes = b"",
|
|
843
|
+
soft_delete_column: str | None = None,
|
|
844
|
+
trashed: bool = False,
|
|
845
|
+
) -> PageResult:
|
|
846
|
+
"""Execute an optimized, N+1-free, SQLi-safe paginated list query (Offset or Keyset Cursor)."""
|
|
847
|
+
safe_page = max(1, page)
|
|
848
|
+
safe_size = max(1, min(page_size, max_page_size))
|
|
849
|
+
|
|
850
|
+
bind = session.get_bind()
|
|
851
|
+
dialect_name = bind.dialect.name if bind is not None else ""
|
|
852
|
+
|
|
853
|
+
base_conditions: list[sa.ColumnElement[bool]] = list(base_stmt._where_criteria)
|
|
854
|
+
joins: list[Any] = []
|
|
855
|
+
|
|
856
|
+
search_clause, search_joins = build_search_clause(meta, searchable_columns, search_term)
|
|
857
|
+
if search_clause is not None:
|
|
858
|
+
base_conditions.append(search_clause)
|
|
859
|
+
joins.extend(search_joins)
|
|
860
|
+
|
|
861
|
+
# Single-pass conditional aggregation for ALL quick tabs + optional Trash count (1 SQL query!).
|
|
862
|
+
tab_counts, trashed_count = compute_quick_tab_counts(
|
|
863
|
+
session,
|
|
864
|
+
meta,
|
|
865
|
+
base_conditions,
|
|
866
|
+
quick_tabs,
|
|
867
|
+
joins=joins,
|
|
868
|
+
dialect_name=dialect_name,
|
|
869
|
+
soft_delete_column=soft_delete_column,
|
|
870
|
+
)
|
|
871
|
+
|
|
872
|
+
where_clauses: list[sa.ColumnElement[bool]] = list(base_conditions)
|
|
873
|
+
sd_pred = soft_delete_predicate(meta, soft_delete_column, trashed=trashed)
|
|
874
|
+
if sd_pred is not None:
|
|
875
|
+
where_clauses.append(sd_pred)
|
|
876
|
+
|
|
877
|
+
known_fields = frozenset(meta.fields_by_name.keys())
|
|
878
|
+
active_filters: dict[str, str] = {}
|
|
879
|
+
applied_filter_pairs: list[tuple[str, str]] = []
|
|
880
|
+
if active_tab_key and not trashed:
|
|
881
|
+
for tab in quick_tabs:
|
|
882
|
+
if tab.key == active_tab_key:
|
|
883
|
+
for raw_fname, fval in tab.filters.items():
|
|
884
|
+
fname, fop = parse_filter_key(raw_fname, known_fields)
|
|
885
|
+
cond = build_filter_condition(meta, fname, fval, op=fop, dialect_name=dialect_name)
|
|
886
|
+
if cond is not None:
|
|
887
|
+
where_clauses.append(cond)
|
|
888
|
+
break
|
|
889
|
+
|
|
890
|
+
for raw_key, raw_val in iter_filter_items(filter_params):
|
|
891
|
+
if not raw_val:
|
|
892
|
+
continue
|
|
893
|
+
fname, op = parse_filter_key(raw_key, known_fields)
|
|
894
|
+
if fname not in filterable_columns:
|
|
895
|
+
continue
|
|
896
|
+
cond = build_filter_condition(meta, fname, raw_val, op=op, dialect_name=dialect_name)
|
|
897
|
+
if cond is not None:
|
|
898
|
+
where_clauses.append(cond)
|
|
899
|
+
active_filters[raw_key] = raw_val
|
|
900
|
+
applied_filter_pairs.append((raw_key, raw_val))
|
|
901
|
+
|
|
902
|
+
# Strictly allowlisted sorting (zero SQL injection surface).
|
|
903
|
+
chosen_sort = sort_by if (sort_by and sort_by in sortable_columns) else None
|
|
904
|
+
chosen_dir = "desc" if sort_dir.lower() == "desc" else "asc"
|
|
905
|
+
if chosen_sort is None and default_sort is not None:
|
|
906
|
+
cand_col, cand_dir = default_sort
|
|
907
|
+
if cand_col in sortable_columns:
|
|
908
|
+
chosen_sort = cand_col
|
|
909
|
+
chosen_dir = "desc" if cand_dir.lower() == "desc" else "asc"
|
|
910
|
+
if chosen_sort is None:
|
|
911
|
+
chosen_sort = meta.primary_keys[0].name
|
|
912
|
+
chosen_dir = "desc"
|
|
913
|
+
|
|
914
|
+
sort_field = meta.fields_by_name[chosen_sort]
|
|
915
|
+
pk_field = meta.primary_keys[0]
|
|
916
|
+
sort_col: Any = getattr(meta.model, chosen_sort)
|
|
917
|
+
pk_col: Any = getattr(meta.model, pk_field.name)
|
|
918
|
+
|
|
919
|
+
use_cursor_mode = pagination_mode == "cursor" and len(meta.primary_keys) == 1
|
|
920
|
+
|
|
921
|
+
# Resolve row count strategy (only skip COUNT when cursor mode is actually active)
|
|
922
|
+
total = 0
|
|
923
|
+
total_is_estimated = False
|
|
924
|
+
total_known = True
|
|
925
|
+
|
|
926
|
+
if use_cursor_mode and count_mode == "none":
|
|
927
|
+
total_known = False
|
|
928
|
+
elif trashed and not active_filters and soft_delete_column:
|
|
929
|
+
total = trashed_count
|
|
930
|
+
elif not trashed and not active_filters and active_tab_key and active_tab_key in tab_counts:
|
|
931
|
+
total = tab_counts[active_tab_key]
|
|
932
|
+
elif (
|
|
933
|
+
not trashed
|
|
934
|
+
and not active_filters
|
|
935
|
+
and not active_tab_key
|
|
936
|
+
and quick_tabs
|
|
937
|
+
and not quick_tabs[0].filters
|
|
938
|
+
):
|
|
939
|
+
total = tab_counts[quick_tabs[0].key]
|
|
940
|
+
elif count_mode == "estimated" and not where_clauses:
|
|
941
|
+
est = estimate_table_rows(session, meta)
|
|
942
|
+
if est is not None:
|
|
943
|
+
total = est
|
|
944
|
+
total_is_estimated = True
|
|
945
|
+
else:
|
|
946
|
+
count_stmt = sa.select(sa.func.count()).select_from(meta.model)
|
|
947
|
+
total = int(session.execute(count_stmt).scalar_one() or 0)
|
|
948
|
+
else:
|
|
949
|
+
count_stmt = sa.select(sa.func.count()).select_from(meta.model)
|
|
950
|
+
for rel_attr in joins:
|
|
951
|
+
count_stmt = count_stmt.outerjoin(rel_attr)
|
|
952
|
+
if where_clauses:
|
|
953
|
+
count_stmt = count_stmt.where(*where_clauses)
|
|
954
|
+
total = int(session.execute(count_stmt).scalar_one() or 0)
|
|
955
|
+
|
|
956
|
+
# Keyset Cursor Mode (zero OFFSET, signed opaque cursors, deterministic PG/SQLite NULL order)
|
|
957
|
+
if use_cursor_mode:
|
|
958
|
+
slice_digest = hashlib.sha256(
|
|
959
|
+
json.dumps(
|
|
960
|
+
[
|
|
961
|
+
meta.slug,
|
|
962
|
+
active_tab_key or "",
|
|
963
|
+
search_term.strip(),
|
|
964
|
+
sorted(applied_filter_pairs),
|
|
965
|
+
bool(trashed),
|
|
966
|
+
],
|
|
967
|
+
separators=(",", ":"),
|
|
968
|
+
).encode("utf-8")
|
|
969
|
+
).hexdigest()[:12]
|
|
970
|
+
|
|
971
|
+
going_back = bool(before_cursor and not after_cursor)
|
|
972
|
+
raw_token = before_cursor if going_back else after_cursor
|
|
973
|
+
decoded = decode_cursor(raw_token, secret=cursor_secret) if raw_token else None
|
|
974
|
+
|
|
975
|
+
cursor_where: list[sa.ColumnElement[bool]] = list(where_clauses)
|
|
976
|
+
has_valid_cursor = False
|
|
977
|
+
effective_dir = chosen_dir
|
|
978
|
+
if (
|
|
979
|
+
decoded is not None
|
|
980
|
+
and decoded.get("m", meta.slug) == meta.slug
|
|
981
|
+
and decoded.get("h", slice_digest) == slice_digest
|
|
982
|
+
and decoded.get("s") == chosen_sort
|
|
983
|
+
and decoded.get("d") == chosen_dir
|
|
984
|
+
and "pk" in decoded
|
|
985
|
+
):
|
|
986
|
+
pk_val = _coerce_filter_value(decoded.get("pk"), pk_field, allow_empty_string=True)
|
|
987
|
+
raw_sv = decoded.get("val")
|
|
988
|
+
sort_val = (
|
|
989
|
+
None
|
|
990
|
+
if raw_sv is None
|
|
991
|
+
else _coerce_filter_value(raw_sv, sort_field, allow_empty_string=True)
|
|
992
|
+
)
|
|
993
|
+
if pk_val is not None and (raw_sv is None or sort_val is not None):
|
|
994
|
+
effective_dir = ("asc" if chosen_dir == "desc" else "desc") if going_back else chosen_dir
|
|
995
|
+
cursor_where.append(
|
|
996
|
+
_build_keyset_predicate(
|
|
997
|
+
meta,
|
|
998
|
+
sort_field=sort_field,
|
|
999
|
+
pk_field=pk_field,
|
|
1000
|
+
sort_val=sort_val,
|
|
1001
|
+
pk_val=pk_val,
|
|
1002
|
+
effective_dir=effective_dir,
|
|
1003
|
+
)
|
|
1004
|
+
)
|
|
1005
|
+
has_valid_cursor = True
|
|
1006
|
+
|
|
1007
|
+
def _run_cursor_fetch(
|
|
1008
|
+
clauses: Sequence[sa.ColumnElement[bool]],
|
|
1009
|
+
direction: str,
|
|
1010
|
+
) -> list[Any]:
|
|
1011
|
+
stmt = sa.select(meta.model)
|
|
1012
|
+
for rel_attr in joins:
|
|
1013
|
+
stmt = stmt.outerjoin(rel_attr)
|
|
1014
|
+
if clauses:
|
|
1015
|
+
stmt = stmt.where(*clauses)
|
|
1016
|
+
# Explicit NULL placement guarantees SQLite & PostgreSQL match _build_keyset_predicate
|
|
1017
|
+
order_clauses: list[Any] = [
|
|
1018
|
+
sort_col.desc().nulls_last()
|
|
1019
|
+
if direction == "desc"
|
|
1020
|
+
else sort_col.asc().nulls_first()
|
|
1021
|
+
]
|
|
1022
|
+
if chosen_sort != pk_field.name:
|
|
1023
|
+
order_clauses.append(pk_col.desc() if direction == "desc" else pk_col.asc())
|
|
1024
|
+
stmt = stmt.order_by(*order_clauses)
|
|
1025
|
+
stmt = apply_list_load_options(stmt, meta, displayed_columns)
|
|
1026
|
+
stmt = stmt.limit(safe_size + 1)
|
|
1027
|
+
return list(session.scalars(stmt).unique().all())
|
|
1028
|
+
|
|
1029
|
+
fetched = _run_cursor_fetch(cursor_where, effective_dir)
|
|
1030
|
+
has_more = len(fetched) > safe_size
|
|
1031
|
+
items = fetched[:safe_size]
|
|
1032
|
+
|
|
1033
|
+
if going_back and has_valid_cursor:
|
|
1034
|
+
if not items:
|
|
1035
|
+
# Rows preceding cursor were deleted; fall back cleanly to page 1 so user is never stranded.
|
|
1036
|
+
fetched = _run_cursor_fetch(where_clauses, chosen_dir)
|
|
1037
|
+
has_more = len(fetched) > safe_size
|
|
1038
|
+
items = fetched[:safe_size]
|
|
1039
|
+
has_next = has_more
|
|
1040
|
+
has_prev = False
|
|
1041
|
+
else:
|
|
1042
|
+
items.reverse()
|
|
1043
|
+
has_prev = has_more
|
|
1044
|
+
has_next = True
|
|
1045
|
+
else:
|
|
1046
|
+
has_next = has_more
|
|
1047
|
+
has_prev = has_valid_cursor and bool(items)
|
|
1048
|
+
|
|
1049
|
+
def _make_tok(obj: Any) -> str:
|
|
1050
|
+
return encode_cursor(
|
|
1051
|
+
{
|
|
1052
|
+
"v": 2,
|
|
1053
|
+
"m": meta.slug,
|
|
1054
|
+
"h": slice_digest,
|
|
1055
|
+
"s": chosen_sort,
|
|
1056
|
+
"d": chosen_dir,
|
|
1057
|
+
"val": _serialize_cursor_val(getattr(obj, chosen_sort, None)),
|
|
1058
|
+
"pk": _serialize_cursor_val(getattr(obj, pk_field.name, None)),
|
|
1059
|
+
},
|
|
1060
|
+
secret=cursor_secret,
|
|
1061
|
+
)
|
|
1062
|
+
|
|
1063
|
+
next_tok = _make_tok(items[-1]) if (items and has_next) else None
|
|
1064
|
+
prev_tok = _make_tok(items[0]) if (items and has_prev) else None
|
|
1065
|
+
pages = max(1, (total + safe_size - 1) // safe_size) if total_known else 1
|
|
1066
|
+
return PageResult(
|
|
1067
|
+
items=items,
|
|
1068
|
+
total=total,
|
|
1069
|
+
page=1,
|
|
1070
|
+
page_size=safe_size,
|
|
1071
|
+
pages=pages,
|
|
1072
|
+
tab_counts=tab_counts,
|
|
1073
|
+
active_filters=active_filters,
|
|
1074
|
+
pagination_mode="cursor",
|
|
1075
|
+
next_cursor=next_tok,
|
|
1076
|
+
prev_cursor=prev_tok,
|
|
1077
|
+
has_next=has_next,
|
|
1078
|
+
has_prev=has_prev,
|
|
1079
|
+
total_is_estimated=total_is_estimated,
|
|
1080
|
+
total_known=total_known,
|
|
1081
|
+
trashed_count=trashed_count,
|
|
1082
|
+
)
|
|
1083
|
+
|
|
1084
|
+
# Classic Offset Pagination
|
|
1085
|
+
pages = max(1, (total + safe_size - 1) // safe_size)
|
|
1086
|
+
if safe_page > pages:
|
|
1087
|
+
safe_page = pages
|
|
1088
|
+
|
|
1089
|
+
data_stmt = sa.select(meta.model)
|
|
1090
|
+
for rel_attr in joins:
|
|
1091
|
+
data_stmt = data_stmt.outerjoin(rel_attr)
|
|
1092
|
+
if where_clauses:
|
|
1093
|
+
data_stmt = data_stmt.where(*where_clauses)
|
|
1094
|
+
|
|
1095
|
+
order_expr = sort_col.desc() if chosen_dir == "desc" else sort_col.asc()
|
|
1096
|
+
offset_orders: list[Any] = [order_expr]
|
|
1097
|
+
for pk_f in meta.primary_keys:
|
|
1098
|
+
if pk_f.name != chosen_sort:
|
|
1099
|
+
pcol = getattr(meta.model, pk_f.name)
|
|
1100
|
+
offset_orders.append(pcol.desc() if chosen_dir == "desc" else pcol.asc())
|
|
1101
|
+
data_stmt = data_stmt.order_by(*offset_orders)
|
|
1102
|
+
data_stmt = apply_list_load_options(data_stmt, meta, displayed_columns)
|
|
1103
|
+
data_stmt = data_stmt.limit(safe_size).offset((safe_page - 1) * safe_size)
|
|
1104
|
+
|
|
1105
|
+
items = list(session.scalars(data_stmt).unique().all())
|
|
1106
|
+
return PageResult(
|
|
1107
|
+
items=items,
|
|
1108
|
+
total=total,
|
|
1109
|
+
page=safe_page,
|
|
1110
|
+
page_size=safe_size,
|
|
1111
|
+
pages=pages,
|
|
1112
|
+
tab_counts=tab_counts,
|
|
1113
|
+
active_filters=active_filters,
|
|
1114
|
+
pagination_mode="offset",
|
|
1115
|
+
has_next=safe_page < pages,
|
|
1116
|
+
has_prev=safe_page > 1,
|
|
1117
|
+
total_is_estimated=total_is_estimated,
|
|
1118
|
+
total_known=total_known,
|
|
1119
|
+
trashed_count=trashed_count,
|
|
1120
|
+
)
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def compute_page_aggregates(
|
|
1124
|
+
items: Sequence[Any],
|
|
1125
|
+
column_aggregates: Mapping[str, str],
|
|
1126
|
+
) -> dict[str, tuple[str, float]]:
|
|
1127
|
+
"""Compute page-level numeric aggregates (sum, avg, min, max) without extra SQL queries."""
|
|
1128
|
+
if not items or not column_aggregates:
|
|
1129
|
+
return {}
|
|
1130
|
+
out: dict[str, tuple[str, float]] = {}
|
|
1131
|
+
for col, raw_op in column_aggregates.items():
|
|
1132
|
+
op = raw_op.strip().lower()
|
|
1133
|
+
if op not in {"sum", "avg", "min", "max"}:
|
|
1134
|
+
continue
|
|
1135
|
+
vals: list[float] = []
|
|
1136
|
+
for obj in items:
|
|
1137
|
+
raw = getattr(obj, col, None)
|
|
1138
|
+
if raw is None or isinstance(raw, bool):
|
|
1139
|
+
continue
|
|
1140
|
+
with contextlib.suppress(TypeError, ValueError):
|
|
1141
|
+
vals.append(float(raw))
|
|
1142
|
+
if not vals:
|
|
1143
|
+
continue
|
|
1144
|
+
if op == "sum":
|
|
1145
|
+
agg = sum(vals)
|
|
1146
|
+
elif op == "avg":
|
|
1147
|
+
agg = sum(vals) / len(vals)
|
|
1148
|
+
elif op == "min":
|
|
1149
|
+
agg = min(vals)
|
|
1150
|
+
else:
|
|
1151
|
+
agg = max(vals)
|
|
1152
|
+
out[col] = (op.upper(), agg)
|
|
1153
|
+
return out
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
def execute_kanban_query(
|
|
1157
|
+
session: Session,
|
|
1158
|
+
meta: ModelMeta,
|
|
1159
|
+
*,
|
|
1160
|
+
base_stmt: sa.Select[Any],
|
|
1161
|
+
kanban_column: str,
|
|
1162
|
+
card_fields: Sequence[str],
|
|
1163
|
+
searchable_columns: Sequence[str],
|
|
1164
|
+
filterable_columns: frozenset[str],
|
|
1165
|
+
search_term: str,
|
|
1166
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]],
|
|
1167
|
+
limit: int = 120,
|
|
1168
|
+
soft_delete_column: str | None = None,
|
|
1169
|
+
) -> list[dict[str, Any]]:
|
|
1170
|
+
"""Fetch bounded, eager-loaded records grouped by enum choices for Kanban Board view."""
|
|
1171
|
+
fmeta = meta.fields_by_name.get(kanban_column)
|
|
1172
|
+
if fmeta is None or not fmeta.choices:
|
|
1173
|
+
return []
|
|
1174
|
+
|
|
1175
|
+
bind = session.get_bind()
|
|
1176
|
+
dialect_name = bind.dialect.name if bind is not None else ""
|
|
1177
|
+
|
|
1178
|
+
where_clauses: list[sa.ColumnElement[bool]] = list(base_stmt._where_criteria)
|
|
1179
|
+
sd_pred = soft_delete_predicate(meta, soft_delete_column, trashed=False)
|
|
1180
|
+
if sd_pred is not None:
|
|
1181
|
+
where_clauses.append(sd_pred)
|
|
1182
|
+
|
|
1183
|
+
joins: list[Any] = []
|
|
1184
|
+
search_clause, search_joins = build_search_clause(meta, searchable_columns, search_term)
|
|
1185
|
+
if search_clause is not None:
|
|
1186
|
+
where_clauses.append(search_clause)
|
|
1187
|
+
joins.extend(search_joins)
|
|
1188
|
+
|
|
1189
|
+
known_fields = frozenset(meta.fields_by_name.keys())
|
|
1190
|
+
for raw_key, raw_val in iter_filter_items(filter_params):
|
|
1191
|
+
if not raw_val:
|
|
1192
|
+
continue
|
|
1193
|
+
fname, op = parse_filter_key(raw_key, known_fields)
|
|
1194
|
+
if fname not in filterable_columns:
|
|
1195
|
+
continue
|
|
1196
|
+
cond = build_filter_condition(meta, fname, raw_val, op=op, dialect_name=dialect_name)
|
|
1197
|
+
if cond is not None:
|
|
1198
|
+
where_clauses.append(cond)
|
|
1199
|
+
|
|
1200
|
+
stmt = sa.select(meta.model)
|
|
1201
|
+
for rel_attr in joins:
|
|
1202
|
+
stmt = stmt.outerjoin(rel_attr)
|
|
1203
|
+
if where_clauses:
|
|
1204
|
+
stmt = stmt.where(*where_clauses)
|
|
1205
|
+
stmt = stmt.order_by(getattr(meta.model, meta.primary_keys[0].name).desc())
|
|
1206
|
+
stmt = apply_list_load_options(stmt, meta, card_fields)
|
|
1207
|
+
stmt = stmt.limit(max(1, min(limit, 250)))
|
|
1208
|
+
records = list(session.scalars(stmt).unique().all())
|
|
1209
|
+
|
|
1210
|
+
buckets: dict[str, list[Any]] = {val: [] for val, _ in fmeta.choices}
|
|
1211
|
+
for obj in records:
|
|
1212
|
+
raw_state = getattr(obj, kanban_column, None)
|
|
1213
|
+
key = str(raw_state.value) if isinstance(raw_state, enum.Enum) else str(raw_state or "")
|
|
1214
|
+
if key in buckets:
|
|
1215
|
+
buckets[key].append(obj)
|
|
1216
|
+
|
|
1217
|
+
return [
|
|
1218
|
+
{"key": val, "label": lbl, "records": buckets[val]}
|
|
1219
|
+
for val, lbl in fmeta.choices
|
|
1220
|
+
]
|