flask-deck 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flask_deck/__init__.py +184 -0
- flask_deck/adapters/__init__.py +23 -0
- flask_deck/adapters/sqlalchemy.py +272 -0
- flask_deck/audit.py +363 -0
- flask_deck/auth.py +462 -0
- flask_deck/branding.py +217 -0
- flask_deck/charts.py +514 -0
- flask_deck/cli.py +135 -0
- flask_deck/config.py +81 -0
- flask_deck/dashboard.py +416 -0
- flask_deck/deck.py +4153 -0
- flask_deck/demo.py +1066 -0
- flask_deck/events.py +309 -0
- flask_deck/exceptions.py +34 -0
- flask_deck/exporter.py +240 -0
- flask_deck/formatters.py +386 -0
- flask_deck/i18n.py +399 -0
- flask_deck/icons.py +92 -0
- flask_deck/importer.py +612 -0
- flask_deck/introspection.py +376 -0
- flask_deck/jobs.py +382 -0
- flask_deck/mfa.py +670 -0
- flask_deck/model_view.py +1097 -0
- flask_deck/py.typed +0 -0
- flask_deck/query.py +1220 -0
- flask_deck/schema_graph.py +137 -0
- flask_deck/security.py +373 -0
- flask_deck/sql_console.py +391 -0
- flask_deck/sso.py +293 -0
- flask_deck/static/css/deck.css +1333 -0
- flask_deck/static/js/deck.js +1433 -0
- flask_deck/storage.py +669 -0
- flask_deck/templates/flask_deck/audit_list.html +117 -0
- flask_deck/templates/flask_deck/base.html +454 -0
- flask_deck/templates/flask_deck/custom_page.html +12 -0
- flask_deck/templates/flask_deck/dashboard.html +32 -0
- flask_deck/templates/flask_deck/diagnostics.html +80 -0
- flask_deck/templates/flask_deck/error.html +15 -0
- flask_deck/templates/flask_deck/import_wizard.html +143 -0
- flask_deck/templates/flask_deck/jobs_list.html +95 -0
- flask_deck/templates/flask_deck/login.html +93 -0
- flask_deck/templates/flask_deck/login_mfa.html +59 -0
- flask_deck/templates/flask_deck/mfa_setup.html +121 -0
- flask_deck/templates/flask_deck/model_detail.html +124 -0
- flask_deck/templates/flask_deck/model_form.html +389 -0
- flask_deck/templates/flask_deck/model_list.html +489 -0
- flask_deck/templates/flask_deck/partials/drawer_detail.html +93 -0
- flask_deck/templates/flask_deck/partials/widget_grid.html +207 -0
- flask_deck/templates/flask_deck/rbac_matrix.html +59 -0
- flask_deck/templates/flask_deck/record_history.html +102 -0
- flask_deck/templates/flask_deck/schema_inspector.html +83 -0
- flask_deck/templates/flask_deck/sql_console.html +87 -0
- flask_deck-0.3.0.dist-info/METADATA +382 -0
- flask_deck-0.3.0.dist-info/RECORD +57 -0
- flask_deck-0.3.0.dist-info/WHEEL +4 -0
- flask_deck-0.3.0.dist-info/entry_points.txt +5 -0
- flask_deck-0.3.0.dist-info/licenses/LICENSE +21 -0
flask_deck/events.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import hmac
|
|
5
|
+
import http.client
|
|
6
|
+
import ipaddress
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import socket
|
|
10
|
+
import ssl
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.request
|
|
13
|
+
from collections.abc import Callable, Sequence
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
from typing import Any
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger("flask_deck.events")
|
|
20
|
+
|
|
21
|
+
_IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True, slots=True)
|
|
25
|
+
class DeckEvent:
|
|
26
|
+
"""Structured operational lifecycle event emitted by Flask-Deck."""
|
|
27
|
+
|
|
28
|
+
name: str
|
|
29
|
+
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
30
|
+
actor_id: str = "system"
|
|
31
|
+
actor_name: str = "System"
|
|
32
|
+
model_slug: str = ""
|
|
33
|
+
record_pk: str = ""
|
|
34
|
+
payload: dict[str, Any] = field(default_factory=dict)
|
|
35
|
+
|
|
36
|
+
def to_dict(self) -> dict[str, Any]:
|
|
37
|
+
return {
|
|
38
|
+
"event": self.name,
|
|
39
|
+
"timestamp": self.timestamp.isoformat(),
|
|
40
|
+
"actor_id": self.actor_id,
|
|
41
|
+
"actor_name": self.actor_name,
|
|
42
|
+
"model_slug": self.model_slug,
|
|
43
|
+
"record_pk": self.record_pk,
|
|
44
|
+
"payload": self.payload,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
EventListener = Callable[[DeckEvent], None]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _is_non_public_ip(ip_obj: _IPAddress) -> bool:
|
|
52
|
+
"""Return True if an IP (including IPv4-mapped / 6to4 / Teredo / CGNAT) is non-public."""
|
|
53
|
+
if isinstance(ip_obj, ipaddress.IPv6Address):
|
|
54
|
+
if ip_obj.ipv4_mapped is not None and _is_non_public_ip(ip_obj.ipv4_mapped):
|
|
55
|
+
return True
|
|
56
|
+
if ip_obj.sixtofour is not None and _is_non_public_ip(ip_obj.sixtofour):
|
|
57
|
+
return True
|
|
58
|
+
if ip_obj.teredo is not None:
|
|
59
|
+
server, client = ip_obj.teredo
|
|
60
|
+
if _is_non_public_ip(server) or _is_non_public_ip(client):
|
|
61
|
+
return True
|
|
62
|
+
return (
|
|
63
|
+
not ip_obj.is_global
|
|
64
|
+
or ip_obj.is_private
|
|
65
|
+
or ip_obj.is_loopback
|
|
66
|
+
or ip_obj.is_link_local
|
|
67
|
+
or ip_obj.is_multicast
|
|
68
|
+
or ip_obj.is_reserved
|
|
69
|
+
or ip_obj.is_unspecified
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _resolve_and_validate_url(
|
|
74
|
+
url: str,
|
|
75
|
+
*,
|
|
76
|
+
allow_http: bool = False,
|
|
77
|
+
allow_private_ips: bool = False,
|
|
78
|
+
) -> tuple[bool, str, str | None]:
|
|
79
|
+
"""Validate URL and return ``(ok, reason, pinned_ip)`` to prevent DNS rebinding."""
|
|
80
|
+
parsed = urlsplit(url.strip())
|
|
81
|
+
scheme = parsed.scheme.lower()
|
|
82
|
+
if scheme != "https" and not (allow_http and scheme == "http"):
|
|
83
|
+
return False, "Webhook URL must use https://", None
|
|
84
|
+
host = parsed.hostname
|
|
85
|
+
if not host:
|
|
86
|
+
return False, "Webhook URL is missing hostname", None
|
|
87
|
+
if host.lower() in {"localhost", "metadata.google.internal", "169.254.169.254"} and not allow_private_ips:
|
|
88
|
+
return False, f"Blocked restricted webhook host: {host}", None
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
port = parsed.port or (443 if scheme == "https" else 80)
|
|
92
|
+
except ValueError as exc:
|
|
93
|
+
return False, f"Invalid webhook port: {exc}", None
|
|
94
|
+
|
|
95
|
+
# Direct IP literal check
|
|
96
|
+
try:
|
|
97
|
+
ip_obj = ipaddress.ip_address(host.split("%", 1)[0])
|
|
98
|
+
if not allow_private_ips and _is_non_public_ip(ip_obj):
|
|
99
|
+
return False, f"Blocked non-public IP address: {host}", None
|
|
100
|
+
return True, "", str(ip_obj)
|
|
101
|
+
except ValueError:
|
|
102
|
+
pass
|
|
103
|
+
|
|
104
|
+
if allow_private_ips:
|
|
105
|
+
return True, "", None
|
|
106
|
+
|
|
107
|
+
try:
|
|
108
|
+
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
109
|
+
except (socket.gaierror, OSError) as exc:
|
|
110
|
+
return False, f"DNS resolution failed for {host}: {exc}", None
|
|
111
|
+
|
|
112
|
+
if not infos:
|
|
113
|
+
return False, f"No IP addresses resolved for {host}", None
|
|
114
|
+
|
|
115
|
+
pinned: str | None = None
|
|
116
|
+
for _family, _type, _proto, _canon, sockaddr in infos:
|
|
117
|
+
raw_ip = str(sockaddr[0]).split("%", 1)[0]
|
|
118
|
+
ip_obj = ipaddress.ip_address(raw_ip)
|
|
119
|
+
if _is_non_public_ip(ip_obj):
|
|
120
|
+
return False, f"Blocked DNS resolution to non-public IP: {raw_ip}", None
|
|
121
|
+
if pinned is None:
|
|
122
|
+
pinned = raw_ip
|
|
123
|
+
|
|
124
|
+
return True, "", pinned
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def is_ssrf_safe_url(
|
|
128
|
+
url: str,
|
|
129
|
+
*,
|
|
130
|
+
allow_http: bool = False,
|
|
131
|
+
allow_private_ips: bool = False,
|
|
132
|
+
) -> tuple[bool, str]:
|
|
133
|
+
"""Validate webhook destination URL and verify DNS resolves only to globally routable IPs."""
|
|
134
|
+
ok, reason, _pinned = _resolve_and_validate_url(
|
|
135
|
+
url,
|
|
136
|
+
allow_http=allow_http,
|
|
137
|
+
allow_private_ips=allow_private_ips,
|
|
138
|
+
)
|
|
139
|
+
return ok, reason
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
143
|
+
"""Refuse all 3xx redirects so webhooks cannot be bounced to internal/metadata IPs."""
|
|
144
|
+
|
|
145
|
+
def redirect_request(
|
|
146
|
+
self,
|
|
147
|
+
req: urllib.request.Request,
|
|
148
|
+
fp: Any,
|
|
149
|
+
code: int,
|
|
150
|
+
msg: str,
|
|
151
|
+
headers: Any,
|
|
152
|
+
newurl: str,
|
|
153
|
+
) -> urllib.request.Request | None:
|
|
154
|
+
raise urllib.error.HTTPError(
|
|
155
|
+
req.full_url,
|
|
156
|
+
code,
|
|
157
|
+
f"SSRF guard blocked HTTP {code} redirect to {newurl}",
|
|
158
|
+
headers,
|
|
159
|
+
fp,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _send_pinned_request(
|
|
164
|
+
url: str,
|
|
165
|
+
*,
|
|
166
|
+
pinned_ip: str | None,
|
|
167
|
+
body: bytes,
|
|
168
|
+
headers: dict[str, str],
|
|
169
|
+
timeout: float,
|
|
170
|
+
) -> None:
|
|
171
|
+
"""Send POST request refusing redirects and connecting directly to the validated IP."""
|
|
172
|
+
parsed = urlsplit(url)
|
|
173
|
+
scheme = parsed.scheme.lower()
|
|
174
|
+
host = parsed.hostname or ""
|
|
175
|
+
port = parsed.port or (443 if scheme == "https" else 80)
|
|
176
|
+
path = parsed.path or "/"
|
|
177
|
+
if parsed.query:
|
|
178
|
+
path = f"{path}?{parsed.query}"
|
|
179
|
+
|
|
180
|
+
if pinned_ip is None:
|
|
181
|
+
opener = urllib.request.build_opener(_NoRedirectHandler())
|
|
182
|
+
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
183
|
+
with opener.open(req, timeout=timeout) as resp:
|
|
184
|
+
resp.read(1024)
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
# Connect straight to pinned_ip while preserving Host header and TLS SNI/hostname check.
|
|
188
|
+
conn: http.client.HTTPConnection
|
|
189
|
+
if scheme == "https":
|
|
190
|
+
ctx = ssl.create_default_context()
|
|
191
|
+
|
|
192
|
+
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
193
|
+
def connect(self) -> None:
|
|
194
|
+
raw_sock = socket.create_connection((pinned_ip, port), self.timeout)
|
|
195
|
+
self.sock = ctx.wrap_socket(raw_sock, server_hostname=host)
|
|
196
|
+
|
|
197
|
+
conn = _PinnedHTTPSConnection(host, port=port, timeout=timeout, context=ctx)
|
|
198
|
+
else:
|
|
199
|
+
conn = http.client.HTTPConnection(pinned_ip, port=port, timeout=timeout)
|
|
200
|
+
|
|
201
|
+
req_headers = dict(headers)
|
|
202
|
+
default_port = 443 if scheme == "https" else 80
|
|
203
|
+
req_headers.setdefault("Host", host if port == default_port else f"{host}:{port}")
|
|
204
|
+
try:
|
|
205
|
+
conn.request("POST", path, body=body, headers=req_headers)
|
|
206
|
+
resp = conn.getresponse()
|
|
207
|
+
resp.read(1024)
|
|
208
|
+
if 300 <= resp.status < 400:
|
|
209
|
+
raise ValueError(f"SSRF guard blocked webhook HTTP {resp.status} redirect")
|
|
210
|
+
finally:
|
|
211
|
+
conn.close()
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def sign_webhook_payload(body: bytes, secret: str) -> str:
|
|
215
|
+
"""Compute HMAC-SHA256 webhook signature header value."""
|
|
216
|
+
mac = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
|
217
|
+
return f"sha256={mac}"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class WebhookSink:
|
|
221
|
+
"""SSRF-hardened, HMAC-SHA256 signed HTTP webhook sink for DeckEventBus."""
|
|
222
|
+
|
|
223
|
+
__slots__ = (
|
|
224
|
+
"_allow_http",
|
|
225
|
+
"_allow_private_ips",
|
|
226
|
+
"_events",
|
|
227
|
+
"_secret",
|
|
228
|
+
"_timeout",
|
|
229
|
+
"_transport",
|
|
230
|
+
"_url",
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def __init__(
|
|
234
|
+
self,
|
|
235
|
+
url: str,
|
|
236
|
+
*,
|
|
237
|
+
secret: str,
|
|
238
|
+
events: Sequence[str] = ("*",),
|
|
239
|
+
timeout: float = 5.0,
|
|
240
|
+
allow_http: bool = False,
|
|
241
|
+
allow_private_ips: bool = False,
|
|
242
|
+
transport: Callable[[str, bytes, dict[str, str], float], int] | None = None,
|
|
243
|
+
) -> None:
|
|
244
|
+
if not secret or len(secret) < 16:
|
|
245
|
+
raise ValueError("WebhookSink secret must be at least 16 characters")
|
|
246
|
+
self._url = url.strip()
|
|
247
|
+
self._secret = secret
|
|
248
|
+
self._events = frozenset(events)
|
|
249
|
+
self._timeout = max(0.5, float(timeout))
|
|
250
|
+
self._allow_http = allow_http
|
|
251
|
+
self._allow_private_ips = allow_private_ips
|
|
252
|
+
self._transport = transport
|
|
253
|
+
|
|
254
|
+
def matches(self, event_name: str) -> bool:
|
|
255
|
+
return "*" in self._events or event_name in self._events
|
|
256
|
+
|
|
257
|
+
def __call__(self, event: DeckEvent) -> None:
|
|
258
|
+
if not self.matches(event.name):
|
|
259
|
+
return
|
|
260
|
+
safe, reason, pinned_ip = _resolve_and_validate_url(
|
|
261
|
+
self._url,
|
|
262
|
+
allow_http=self._allow_http,
|
|
263
|
+
allow_private_ips=self._allow_private_ips,
|
|
264
|
+
)
|
|
265
|
+
if not safe:
|
|
266
|
+
raise ValueError(f"SSRF guard blocked webhook delivery: {reason}")
|
|
267
|
+
|
|
268
|
+
body = json.dumps(event.to_dict(), default=str, separators=(",", ":")).encode("utf-8")
|
|
269
|
+
headers = {
|
|
270
|
+
"Content-Type": "application/json",
|
|
271
|
+
"User-Agent": "flask-deck-webhook/0.3",
|
|
272
|
+
"X-Deck-Event": event.name,
|
|
273
|
+
"X-Deck-Signature": sign_webhook_payload(body, self._secret),
|
|
274
|
+
}
|
|
275
|
+
if self._transport is not None:
|
|
276
|
+
self._transport(self._url, body, headers, self._timeout)
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
_send_pinned_request(
|
|
280
|
+
self._url,
|
|
281
|
+
pinned_ip=pinned_ip,
|
|
282
|
+
body=body,
|
|
283
|
+
headers=headers,
|
|
284
|
+
timeout=self._timeout,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class DeckEventBus:
|
|
289
|
+
"""In-process fault-isolated lifecycle event dispatcher."""
|
|
290
|
+
|
|
291
|
+
__slots__ = ("_listeners",)
|
|
292
|
+
|
|
293
|
+
def __init__(self) -> None:
|
|
294
|
+
self._listeners: dict[str, list[EventListener]] = {}
|
|
295
|
+
|
|
296
|
+
def subscribe(self, event_name: str, listener: EventListener) -> None:
|
|
297
|
+
self._listeners.setdefault(event_name, []).append(listener)
|
|
298
|
+
|
|
299
|
+
def emit(self, event: DeckEvent) -> list[str]:
|
|
300
|
+
"""Dispatch event to matching listeners; isolate errors so mutations never break."""
|
|
301
|
+
errors: list[str] = []
|
|
302
|
+
targets = list(self._listeners.get(event.name, ())) + list(self._listeners.get("*", ()))
|
|
303
|
+
for fn in targets:
|
|
304
|
+
try:
|
|
305
|
+
fn(event)
|
|
306
|
+
except Exception as exc:
|
|
307
|
+
logger.warning("Deck event listener error for %s: %s", event.name, exc)
|
|
308
|
+
errors.append(str(exc))
|
|
309
|
+
return errors
|
flask_deck/exceptions.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DeckError(Exception):
|
|
5
|
+
"""Base exception for all flask-deck errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SecurityConfigurationError(DeckError):
|
|
9
|
+
"""Raised when flask-deck is started in an insecure production configuration."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PermissionDeniedError(DeckError):
|
|
13
|
+
"""Raised when a user lacks permission to perform an operation."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ReadOnlyError(DeckError):
|
|
17
|
+
"""Raised when a mutating operation is attempted while read_only mode is enabled."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class InvalidPKError(DeckError):
|
|
21
|
+
"""Raised when a primary key parameter cannot be decoded or coerced."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ValidationError(DeckError):
|
|
25
|
+
"""Raised when form or record validation fails."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, errors: dict[str, list[str]]) -> None:
|
|
28
|
+
self.errors = errors
|
|
29
|
+
summary = "; ".join(f"{field}: {', '.join(msgs)}" for field, msgs in errors.items())
|
|
30
|
+
super().__init__(summary or "Validation failed")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ImportLimitError(DeckError):
|
|
34
|
+
"""Raised when an import payload exceeds configured byte, row, or structure limits."""
|
flask_deck/exporter.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import enum
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
8
|
+
from datetime import date, datetime, time
|
|
9
|
+
from decimal import Decimal
|
|
10
|
+
from typing import Any
|
|
11
|
+
from uuid import UUID
|
|
12
|
+
|
|
13
|
+
import sqlalchemy as sa
|
|
14
|
+
from flask import Response, stream_with_context
|
|
15
|
+
from sqlalchemy.orm import Session
|
|
16
|
+
|
|
17
|
+
from flask_deck.auth import AuthProvider, DeckUser
|
|
18
|
+
from flask_deck.model_view import ModelView
|
|
19
|
+
from flask_deck.query import (
|
|
20
|
+
build_filter_condition,
|
|
21
|
+
build_search_clause,
|
|
22
|
+
iter_filter_items,
|
|
23
|
+
parse_filter_key,
|
|
24
|
+
soft_delete_predicate,
|
|
25
|
+
)
|
|
26
|
+
from flask_deck.security import sanitize_csv_cell
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _serialize_export_value(value: Any) -> Any:
|
|
30
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
31
|
+
return value
|
|
32
|
+
if isinstance(value, enum.Enum):
|
|
33
|
+
return value.value
|
|
34
|
+
if isinstance(value, (datetime, date, time)):
|
|
35
|
+
return value.isoformat()
|
|
36
|
+
if isinstance(value, (Decimal, UUID)):
|
|
37
|
+
return str(value)
|
|
38
|
+
if isinstance(value, (dict, list)):
|
|
39
|
+
return value
|
|
40
|
+
return str(value)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _build_export_statement(
|
|
44
|
+
view: ModelView,
|
|
45
|
+
*,
|
|
46
|
+
auth: AuthProvider,
|
|
47
|
+
user: DeckUser | None,
|
|
48
|
+
search_term: str,
|
|
49
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]],
|
|
50
|
+
active_tab_key: str | None,
|
|
51
|
+
selected_pks: Sequence[tuple[Any, ...]] = (),
|
|
52
|
+
dialect_name: str = "",
|
|
53
|
+
trashed: bool = False,
|
|
54
|
+
) -> sa.Select[Any]:
|
|
55
|
+
meta = view.meta
|
|
56
|
+
stmt = view.scope_query(sa.select(view.model), user)
|
|
57
|
+
sd_pred = soft_delete_predicate(meta, view.soft_delete_column, trashed=trashed)
|
|
58
|
+
if sd_pred is not None:
|
|
59
|
+
stmt = stmt.where(sd_pred)
|
|
60
|
+
|
|
61
|
+
restricted = auth.hidden_fields_for(user, meta.slug) | auth.masked_fields_for(user, meta.slug)
|
|
62
|
+
allowed_searchable = tuple(
|
|
63
|
+
c for c in view.resolved_searchable if c.split(".", 1)[0] not in restricted
|
|
64
|
+
)
|
|
65
|
+
allowed_filterable = view.resolved_filterable - restricted
|
|
66
|
+
known_fields = frozenset(meta.fields_by_name.keys())
|
|
67
|
+
|
|
68
|
+
search_clause, search_joins = build_search_clause(meta, allowed_searchable, search_term)
|
|
69
|
+
for rel_attr in search_joins:
|
|
70
|
+
stmt = stmt.outerjoin(rel_attr)
|
|
71
|
+
if search_clause is not None:
|
|
72
|
+
stmt = stmt.where(search_clause)
|
|
73
|
+
|
|
74
|
+
if active_tab_key and not trashed:
|
|
75
|
+
for tab in view.quick_tabs:
|
|
76
|
+
if tab.key == active_tab_key:
|
|
77
|
+
for raw_fname, fval in tab.filters.items():
|
|
78
|
+
fname, fop = parse_filter_key(raw_fname, known_fields)
|
|
79
|
+
cond = build_filter_condition(meta, fname, fval, op=fop, dialect_name=dialect_name)
|
|
80
|
+
if cond is not None:
|
|
81
|
+
stmt = stmt.where(cond)
|
|
82
|
+
break
|
|
83
|
+
|
|
84
|
+
for raw_key, raw_val in iter_filter_items(filter_params):
|
|
85
|
+
if not raw_val:
|
|
86
|
+
continue
|
|
87
|
+
fname, op = parse_filter_key(raw_key, known_fields)
|
|
88
|
+
if fname not in allowed_filterable:
|
|
89
|
+
continue
|
|
90
|
+
cond = build_filter_condition(meta, fname, raw_val, op=op, dialect_name=dialect_name)
|
|
91
|
+
if cond is not None:
|
|
92
|
+
stmt = stmt.where(cond)
|
|
93
|
+
|
|
94
|
+
if selected_pks:
|
|
95
|
+
if len(meta.primary_keys) == 1:
|
|
96
|
+
pk_col = getattr(view.model, meta.primary_keys[0].name)
|
|
97
|
+
stmt = stmt.where(pk_col.in_([pk[0] for pk in selected_pks]))
|
|
98
|
+
else:
|
|
99
|
+
pk_cols = [getattr(view.model, pk.name) for pk in meta.primary_keys]
|
|
100
|
+
stmt = stmt.where(sa.tuple_(*pk_cols).in_(selected_pks))
|
|
101
|
+
|
|
102
|
+
return stmt.order_by(getattr(view.model, meta.primary_keys[0].name).asc())
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def export_columns_for_user(
|
|
106
|
+
view: ModelView,
|
|
107
|
+
auth: AuthProvider,
|
|
108
|
+
user: DeckUser | None,
|
|
109
|
+
) -> tuple[str, ...]:
|
|
110
|
+
hidden = auth.hidden_fields_for(user, view.meta.slug)
|
|
111
|
+
return tuple(
|
|
112
|
+
col
|
|
113
|
+
for col in view.resolved_export_columns
|
|
114
|
+
if col not in hidden and col not in view.meta.sensitive_fields
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def iter_export_chunks(
|
|
119
|
+
session: Session,
|
|
120
|
+
view: ModelView,
|
|
121
|
+
*,
|
|
122
|
+
fmt: str,
|
|
123
|
+
auth: AuthProvider,
|
|
124
|
+
user: DeckUser | None,
|
|
125
|
+
search_term: str = "",
|
|
126
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]] = (),
|
|
127
|
+
active_tab_key: str | None = None,
|
|
128
|
+
selected_pks: Sequence[tuple[Any, ...]] = (),
|
|
129
|
+
sanitize_csv: bool = True,
|
|
130
|
+
chunk_size: int = 500,
|
|
131
|
+
trashed: bool = False,
|
|
132
|
+
) -> tuple[Iterator[str], str, str]:
|
|
133
|
+
"""Yield formatted export chunks in O(1) memory without requiring a Flask request context."""
|
|
134
|
+
cols = export_columns_for_user(view, auth, user)
|
|
135
|
+
masked = auth.masked_fields_for(user, view.meta.slug)
|
|
136
|
+
bind = session.get_bind()
|
|
137
|
+
dialect_name = bind.dialect.name if bind is not None else ""
|
|
138
|
+
stmt = _build_export_statement(
|
|
139
|
+
view,
|
|
140
|
+
auth=auth,
|
|
141
|
+
user=user,
|
|
142
|
+
search_term=search_term,
|
|
143
|
+
filter_params=filter_params,
|
|
144
|
+
active_tab_key=active_tab_key,
|
|
145
|
+
selected_pks=selected_pks,
|
|
146
|
+
dialect_name=dialect_name,
|
|
147
|
+
trashed=trashed,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def _row_dict(obj: Any) -> dict[str, Any]:
|
|
151
|
+
out: dict[str, Any] = {}
|
|
152
|
+
for col in cols:
|
|
153
|
+
val = getattr(obj, col, None)
|
|
154
|
+
if col in masked:
|
|
155
|
+
val = auth.mask_field_value(user, view.meta.slug, col, val)
|
|
156
|
+
out[col] = _serialize_export_value(val)
|
|
157
|
+
return out
|
|
158
|
+
|
|
159
|
+
fmt_clean = fmt.lower()
|
|
160
|
+
if fmt_clean == "csv":
|
|
161
|
+
|
|
162
|
+
def _gen_csv() -> Iterator[str]:
|
|
163
|
+
buf = io.StringIO()
|
|
164
|
+
writer = csv.writer(buf)
|
|
165
|
+
writer.writerow(cols)
|
|
166
|
+
yield buf.getvalue()
|
|
167
|
+
buf.seek(0)
|
|
168
|
+
buf.truncate(0)
|
|
169
|
+
|
|
170
|
+
for obj in session.scalars(stmt).yield_per(chunk_size):
|
|
171
|
+
row_data = _row_dict(obj)
|
|
172
|
+
cells = [
|
|
173
|
+
sanitize_csv_cell(
|
|
174
|
+
json.dumps(row_data[c], separators=(",", ":"))
|
|
175
|
+
if isinstance(row_data[c], (dict, list))
|
|
176
|
+
else row_data[c],
|
|
177
|
+
enabled=sanitize_csv,
|
|
178
|
+
)
|
|
179
|
+
for c in cols
|
|
180
|
+
]
|
|
181
|
+
writer.writerow(cells)
|
|
182
|
+
yield buf.getvalue()
|
|
183
|
+
buf.seek(0)
|
|
184
|
+
buf.truncate(0)
|
|
185
|
+
|
|
186
|
+
return _gen_csv(), "text/csv; charset=utf-8", f"{view.meta.slug}.csv"
|
|
187
|
+
|
|
188
|
+
if fmt_clean == "jsonl":
|
|
189
|
+
|
|
190
|
+
def _gen_jsonl() -> Iterator[str]:
|
|
191
|
+
for obj in session.scalars(stmt).yield_per(chunk_size):
|
|
192
|
+
yield json.dumps(_row_dict(obj), separators=(",", ":")) + "\n"
|
|
193
|
+
|
|
194
|
+
return _gen_jsonl(), "application/x-ndjson; charset=utf-8", f"{view.meta.slug}.jsonl"
|
|
195
|
+
|
|
196
|
+
def _gen_json() -> Iterator[str]:
|
|
197
|
+
yield "[\n"
|
|
198
|
+
first = True
|
|
199
|
+
for obj in session.scalars(stmt).yield_per(chunk_size):
|
|
200
|
+
prefix = "" if first else ",\n"
|
|
201
|
+
first = False
|
|
202
|
+
yield prefix + " " + json.dumps(_row_dict(obj), separators=(",", ":"))
|
|
203
|
+
yield "\n]\n"
|
|
204
|
+
|
|
205
|
+
return _gen_json(), "application/json; charset=utf-8", f"{view.meta.slug}.json"
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def stream_export_response(
|
|
209
|
+
session: Session,
|
|
210
|
+
view: ModelView,
|
|
211
|
+
*,
|
|
212
|
+
fmt: str,
|
|
213
|
+
auth: AuthProvider,
|
|
214
|
+
user: DeckUser | None,
|
|
215
|
+
search_term: str = "",
|
|
216
|
+
filter_params: Mapping[str, str] | Sequence[tuple[str, str]] = (),
|
|
217
|
+
active_tab_key: str | None = None,
|
|
218
|
+
selected_pks: Sequence[tuple[Any, ...]] = (),
|
|
219
|
+
sanitize_csv: bool = True,
|
|
220
|
+
chunk_size: int = 500,
|
|
221
|
+
trashed: bool = False,
|
|
222
|
+
) -> Response:
|
|
223
|
+
"""Stream filtered model records in O(1) memory as CSV, JSON, or JSONL."""
|
|
224
|
+
gen, mimetype, filename = iter_export_chunks(
|
|
225
|
+
session,
|
|
226
|
+
view,
|
|
227
|
+
fmt=fmt,
|
|
228
|
+
auth=auth,
|
|
229
|
+
user=user,
|
|
230
|
+
search_term=search_term,
|
|
231
|
+
filter_params=filter_params,
|
|
232
|
+
active_tab_key=active_tab_key,
|
|
233
|
+
selected_pks=selected_pks,
|
|
234
|
+
sanitize_csv=sanitize_csv,
|
|
235
|
+
chunk_size=chunk_size,
|
|
236
|
+
trashed=trashed,
|
|
237
|
+
)
|
|
238
|
+
resp = Response(stream_with_context(gen), mimetype=mimetype)
|
|
239
|
+
resp.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
|
240
|
+
return resp
|