teaql 0.2.7__tar.gz → 0.2.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {teaql-0.2.7 → teaql-0.2.8}/PKG-INFO +1 -1
- {teaql-0.2.7 → teaql-0.2.8}/pyproject.toml +1 -1
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/dynamic_search.py +2 -2
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/meta.py +10 -1
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/data_service/__init__.py +14 -0
- teaql-0.2.8/src/teaql/provider/mysql/dialect.py +13 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/mysql/transport.py +47 -2
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/postgres/dialect.py +1 -1
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/postgres/transport.py +46 -2
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/audit.py +10 -8
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/context.py +36 -10
- teaql-0.2.8/src/teaql/runtime/log_privacy.py +195 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/sql/dialect.py +58 -22
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/sql/executor.py +193 -92
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/sql/types.py +62 -5
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql.egg-info/PKG-INFO +1 -1
- teaql-0.2.7/src/teaql/provider/mysql/dialect.py +0 -31
- teaql-0.2.7/src/teaql/runtime/log_privacy.py +0 -93
- {teaql-0.2.7 → teaql-0.2.8}/LICENSE +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/README.md +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/setup.cfg +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/entity.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/eval.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/expr.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/graph.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/list.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/mutation.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/query.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/safe_expression.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/value.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/core/xls.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/mysql/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/postgres/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/redis/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/redis/remote.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/redis/store.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/sqlite/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/sqlite/dialect.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/sqlite/transport.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/provider/tfp_client/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/py.typed +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/_schema_capability.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/builtin-messages-v1.json +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/env.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/i18n.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/module.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/opentelemetry.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/store.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/telemetry.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/tools.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/runtime/wire_fields.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/sql/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/web_integration/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql/web_integration/fastapi/__init__.py +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql.egg-info/SOURCES.txt +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql.egg-info/dependency_links.txt +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql.egg-info/requires.txt +0 -0
- {teaql-0.2.7 → teaql-0.2.8}/src/teaql.egg-info/top_level.txt +0 -0
|
@@ -17,8 +17,8 @@ def _reject_json_constant(_):
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def _warning(warning):
|
|
20
|
-
_LOG.warning('%s entity=%s clause=%s fieldPath
|
|
21
|
-
warning['entity'], warning['clause']
|
|
20
|
+
_LOG.warning('%s entity=%s clause=%s fieldPath=<omitted>', warning['code'],
|
|
21
|
+
warning['entity'], warning['clause'])
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def normalize_dynamic_search(source, entity, models, warn=_warning, max_clauses=100):
|
|
@@ -6,6 +6,13 @@ class PropertyDescriptor:
|
|
|
6
6
|
self._is_id = False
|
|
7
7
|
self._is_version = False
|
|
8
8
|
self.nullable = True
|
|
9
|
+
self.log_policy_val = 'unknown'
|
|
10
|
+
def log_policy(self, policy):
|
|
11
|
+
"""Trusted schema policy; incoming requests cannot override it."""
|
|
12
|
+
if policy not in ('plain', 'masked', 'credential', 'unknown'):
|
|
13
|
+
raise ValueError('invalid SQL parameter log policy')
|
|
14
|
+
self.log_policy_val = policy
|
|
15
|
+
return self
|
|
9
16
|
def column_name(self, name):
|
|
10
17
|
self.column_name_val = name
|
|
11
18
|
return self
|
|
@@ -46,6 +53,7 @@ class EntityDescriptor:
|
|
|
46
53
|
self.properties = []
|
|
47
54
|
self.relations = []
|
|
48
55
|
self.audit_mask_fields_val = []
|
|
56
|
+
self.audit_mask_fields_declared = False
|
|
49
57
|
self.audit_value_max_len_val = None
|
|
50
58
|
def table_name(self, name):
|
|
51
59
|
self.table_name_val = name
|
|
@@ -65,7 +73,8 @@ class EntityDescriptor:
|
|
|
65
73
|
return next((prop for prop in self.properties if prop.name == name), None)
|
|
66
74
|
|
|
67
75
|
def audit_mask_fields(self, fields):
|
|
68
|
-
self.audit_mask_fields_val = list(fields)
|
|
76
|
+
self.audit_mask_fields_val = list(fields) if fields is not None else []
|
|
77
|
+
self.audit_mask_fields_declared = fields is not None
|
|
69
78
|
return self
|
|
70
79
|
|
|
71
80
|
def audit_value_max_len(self, max_len):
|
|
@@ -32,6 +32,15 @@ class QueryRequest:
|
|
|
32
32
|
_comment: Optional[str] = None
|
|
33
33
|
_purpose: Optional[str] = None
|
|
34
34
|
|
|
35
|
+
def __post_init__(self) -> None:
|
|
36
|
+
# Older generated wrappers store intent on SelectQuery but construct
|
|
37
|
+
# QueryRequest(query) without forwarding it. Keep the explicit request
|
|
38
|
+
# values authoritative while preserving their diagnostic intent.
|
|
39
|
+
if self._comment is None:
|
|
40
|
+
self._comment = getattr(self.query, 'comment_text', None)
|
|
41
|
+
if self._purpose is None:
|
|
42
|
+
self._purpose = getattr(self.query, 'purpose_text', None)
|
|
43
|
+
|
|
35
44
|
def comment(self, text: str) -> 'QueryRequest':
|
|
36
45
|
if not text:
|
|
37
46
|
raise ValueError("comment cannot be empty")
|
|
@@ -71,6 +80,11 @@ class ExecutionMetadata:
|
|
|
71
80
|
audit_reason: Optional[str] = None
|
|
72
81
|
backend_request_id: Optional[str] = None
|
|
73
82
|
debug_query: Optional[str] = None
|
|
83
|
+
database_kind: Any = None
|
|
84
|
+
parameter_log_policies: Optional[List[str]] = None
|
|
85
|
+
sql_origin: Optional[str] = None
|
|
86
|
+
# Statement/cursor termination only, not transaction commit.
|
|
87
|
+
execution_outcome: Optional[str] = None
|
|
74
88
|
|
|
75
89
|
|
|
76
90
|
@dataclass
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from teaql.sql.dialect import SqlDialect, quote_identifier_if_needed
|
|
2
|
+
from teaql.sql.types import DatabaseKind
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class MysqlDialect(SqlDialect):
|
|
6
|
+
def kind(self) -> DatabaseKind:
|
|
7
|
+
return DatabaseKind.MySql
|
|
8
|
+
|
|
9
|
+
def quote_ident(self, ident: str) -> str:
|
|
10
|
+
return quote_identifier_if_needed(ident, "`")
|
|
11
|
+
|
|
12
|
+
def placeholder(self, index: int) -> str:
|
|
13
|
+
return "%s"
|
|
@@ -3,11 +3,11 @@ import aiomysql
|
|
|
3
3
|
from decimal import Decimal
|
|
4
4
|
from datetime import date, datetime, timezone
|
|
5
5
|
from typing import List, Dict, Any, Optional, AsyncIterator
|
|
6
|
-
from teaql.sql.executor import
|
|
6
|
+
from teaql.sql.executor import SqlTransactionTransport, SqlTransactionTransportTx
|
|
7
7
|
from teaql.sql.types import CompiledQuery
|
|
8
8
|
from teaql.core.value import Value, DataType, Timestamp
|
|
9
9
|
|
|
10
|
-
class MysqlTransport(
|
|
10
|
+
class MysqlTransport(SqlTransactionTransport):
|
|
11
11
|
def __init__(self, db_url: str):
|
|
12
12
|
self.db_url = db_url
|
|
13
13
|
|
|
@@ -99,3 +99,48 @@ class MysqlTransport(SqlTransport):
|
|
|
99
99
|
return cur.rowcount, cur.lastrowid
|
|
100
100
|
finally:
|
|
101
101
|
conn.close()
|
|
102
|
+
|
|
103
|
+
async def begin_sql(self) -> SqlTransactionTransportTx:
|
|
104
|
+
user, password, host, port, db = self._parse_url()
|
|
105
|
+
conn = await aiomysql.connect(
|
|
106
|
+
host=host, port=port, user=user, password=password, db=db,
|
|
107
|
+
cursorclass=aiomysql.DictCursor, autocommit=False,
|
|
108
|
+
)
|
|
109
|
+
try:
|
|
110
|
+
await conn.begin()
|
|
111
|
+
return _MysqlTransaction(conn, self)
|
|
112
|
+
except BaseException:
|
|
113
|
+
conn.close()
|
|
114
|
+
raise
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class _MysqlTransaction(SqlTransactionTransportTx):
|
|
118
|
+
def __init__(self, conn, owner: MysqlTransport):
|
|
119
|
+
self._conn = conn
|
|
120
|
+
self._owner = owner
|
|
121
|
+
|
|
122
|
+
async def fetch_all_sql(self, query: CompiledQuery) -> List[Dict[str, Any]]:
|
|
123
|
+
async with self._conn.cursor() as cursor:
|
|
124
|
+
await cursor.execute(query.sql_with_comment(), self._owner._bind_values(query.params))
|
|
125
|
+
rows = await cursor.fetchall()
|
|
126
|
+
return [
|
|
127
|
+
{key: self._owner._decode_value(value) for key, value in row.items()}
|
|
128
|
+
for row in rows
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
async def execute_sql(self, query: CompiledQuery) -> tuple[int, int]:
|
|
132
|
+
async with self._conn.cursor() as cursor:
|
|
133
|
+
await cursor.execute(query.sql_with_comment(), self._owner._bind_values(query.params))
|
|
134
|
+
return cursor.rowcount, cursor.lastrowid
|
|
135
|
+
|
|
136
|
+
async def commit_sql(self) -> None:
|
|
137
|
+
try:
|
|
138
|
+
await self._conn.commit()
|
|
139
|
+
finally:
|
|
140
|
+
self._conn.close()
|
|
141
|
+
|
|
142
|
+
async def rollback_sql(self) -> None:
|
|
143
|
+
try:
|
|
144
|
+
await self._conn.rollback()
|
|
145
|
+
finally:
|
|
146
|
+
self._conn.close()
|
|
@@ -5,7 +5,7 @@ from teaql.core.meta import PropertyDescriptor
|
|
|
5
5
|
|
|
6
6
|
class PostgresDialect(SqlDialect):
|
|
7
7
|
def kind(self) -> DatabaseKind:
|
|
8
|
-
return DatabaseKind.
|
|
8
|
+
return DatabaseKind.PostgreSql
|
|
9
9
|
|
|
10
10
|
def quote_ident(self, ident: str) -> str:
|
|
11
11
|
return quote_identifier_if_needed(ident, '"')
|
|
@@ -3,11 +3,11 @@ import asyncpg
|
|
|
3
3
|
from decimal import Decimal
|
|
4
4
|
from datetime import date, datetime, timezone
|
|
5
5
|
from typing import List, Dict, Any, Optional, AsyncIterator
|
|
6
|
-
from teaql.sql.executor import
|
|
6
|
+
from teaql.sql.executor import SqlTransactionTransport, SqlTransactionTransportTx
|
|
7
7
|
from teaql.sql.types import CompiledQuery
|
|
8
8
|
from teaql.core.value import Value, DataType, Timestamp
|
|
9
9
|
|
|
10
|
-
class PostgresTransport(
|
|
10
|
+
class PostgresTransport(SqlTransactionTransport):
|
|
11
11
|
def __init__(self, db_url: str):
|
|
12
12
|
self.db_url = db_url
|
|
13
13
|
|
|
@@ -88,3 +88,47 @@ class PostgresTransport(SqlTransport):
|
|
|
88
88
|
return affected_rows, 0
|
|
89
89
|
finally:
|
|
90
90
|
await conn.close()
|
|
91
|
+
|
|
92
|
+
async def begin_sql(self) -> SqlTransactionTransportTx:
|
|
93
|
+
conn = await asyncpg.connect(self.db_url)
|
|
94
|
+
try:
|
|
95
|
+
transaction = conn.transaction()
|
|
96
|
+
await transaction.start()
|
|
97
|
+
return _PostgresTransaction(conn, transaction, self)
|
|
98
|
+
except BaseException:
|
|
99
|
+
await conn.close()
|
|
100
|
+
raise
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class _PostgresTransaction(SqlTransactionTransportTx):
|
|
104
|
+
def __init__(self, conn, transaction, owner: PostgresTransport):
|
|
105
|
+
self._conn = conn
|
|
106
|
+
self._transaction = transaction
|
|
107
|
+
self._owner = owner
|
|
108
|
+
|
|
109
|
+
async def fetch_all_sql(self, query: CompiledQuery) -> List[Dict[str, Any]]:
|
|
110
|
+
rows = await self._conn.fetch(query.sql_with_comment(), *self._owner._bind_values(query.params))
|
|
111
|
+
return [
|
|
112
|
+
{key: self._owner._decode_value(value) for key, value in row.items()}
|
|
113
|
+
for row in rows
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
async def execute_sql(self, query: CompiledQuery) -> tuple[int, int]:
|
|
117
|
+
status = await self._conn.execute(query.sql_with_comment(), *self._owner._bind_values(query.params))
|
|
118
|
+
if status.startswith("INSERT "):
|
|
119
|
+
return int(status.split()[-1]), 0
|
|
120
|
+
if status.startswith(("UPDATE ", "DELETE ")):
|
|
121
|
+
return int(status.split()[-1]), 0
|
|
122
|
+
return 0, 0
|
|
123
|
+
|
|
124
|
+
async def commit_sql(self) -> None:
|
|
125
|
+
try:
|
|
126
|
+
await self._transaction.commit()
|
|
127
|
+
finally:
|
|
128
|
+
await self._conn.close()
|
|
129
|
+
|
|
130
|
+
async def rollback_sql(self) -> None:
|
|
131
|
+
try:
|
|
132
|
+
await self._transaction.rollback()
|
|
133
|
+
finally:
|
|
134
|
+
await self._conn.close()
|
|
@@ -35,22 +35,23 @@ class RawAuditEvent:
|
|
|
35
35
|
fields = []
|
|
36
36
|
for change in self.changes:
|
|
37
37
|
value = None if change.new_value is None else str(getattr(change.new_value, "val", change.new_value))
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
credential = (credential_name(change.field)
|
|
39
|
+
or payload_has_credentials(change.new_value)
|
|
40
|
+
or payload_has_credentials(change.old_value))
|
|
41
|
+
masked = credential or (change.field in mask_fields and not allow)
|
|
42
42
|
if masked:
|
|
43
43
|
secrets.extend(value_strings(change.old_value))
|
|
44
44
|
secrets.extend(value_strings(change.new_value))
|
|
45
45
|
if value is not None and masked:
|
|
46
|
-
value = REDACTED
|
|
46
|
+
value = REDACTED if credential else _mask(value)
|
|
47
47
|
truncated = value is not None and max_length is not None and len(value) > max_length
|
|
48
48
|
if truncated:
|
|
49
49
|
value = "*" * max_length if max_length <= 3 else value[:max_length - 3] + "..."
|
|
50
50
|
fields.append(SafeAuditField(change.field, value, masked, truncated))
|
|
51
|
+
intent_values = secrets + value_strings(self.entity_id)
|
|
51
52
|
return SafeAuditEvent(
|
|
52
|
-
self.kind, self.entity, self.entity_id, scrub(tuple(fields), secrets), scrub(self.trace_chain,
|
|
53
|
-
scrub(self.actor,
|
|
53
|
+
self.kind, self.entity, self.entity_id, scrub(tuple(fields), secrets), scrub(self.trace_chain, intent_values),
|
|
54
|
+
scrub(self.actor, intent_values), self.category,
|
|
54
55
|
)
|
|
55
56
|
|
|
56
57
|
|
|
@@ -74,7 +75,8 @@ class SafeAuditEvent:
|
|
|
74
75
|
|
|
75
76
|
|
|
76
77
|
def _mask(value: str) -> str:
|
|
77
|
-
|
|
78
|
+
# Unicode scalar length, ASCII digits: the same contract as Rust and Go.
|
|
79
|
+
if len(value) < 8 or (value.isascii() and value.isdigit()):
|
|
78
80
|
return "*" * len(value)
|
|
79
81
|
return value[:2] + "*" * (len(value) - 4) + value[-2:]
|
|
80
82
|
|
|
@@ -705,6 +705,10 @@ class UserContext:
|
|
|
705
705
|
return self.get_resource("locale") or Locale.ENGLISH
|
|
706
706
|
|
|
707
707
|
def record_metadata_log(self, metadata: Any):
|
|
708
|
+
self._record_metadata_log(metadata)
|
|
709
|
+
|
|
710
|
+
def _record_metadata_log(self, metadata: Any, *, intent_source=None, intent_values=()):
|
|
711
|
+
"""Internal statement plumbing: source bindings never reach sinks/buffers."""
|
|
708
712
|
op = SqlLogOperation.Select
|
|
709
713
|
op_str = str(getattr(metadata, 'operation', '')).lower()
|
|
710
714
|
if 'insert' in op_str: op = SqlLogOperation.Insert
|
|
@@ -726,6 +730,10 @@ class UserContext:
|
|
|
726
730
|
params=list(getattr(metadata, 'parameters', [])),
|
|
727
731
|
debug_sql=getattr(metadata, 'debug_query', '') or '',
|
|
728
732
|
pretty_sql=getattr(metadata, 'debug_query', '') or '',
|
|
733
|
+
database_kind=getattr(metadata, 'database_kind', None),
|
|
734
|
+
parameter_log_policies=getattr(metadata, 'parameter_log_policies', None),
|
|
735
|
+
sql_origin=getattr(metadata, 'sql_origin', None),
|
|
736
|
+
execution_outcome=getattr(metadata, 'execution_outcome', None),
|
|
729
737
|
started_at=started_at,
|
|
730
738
|
ended_at=ended_at,
|
|
731
739
|
elapsed=ended_at - started_at,
|
|
@@ -740,13 +748,11 @@ class UserContext:
|
|
|
740
748
|
entry.result_summary = f"{entry.affected_rows} rows affected"
|
|
741
749
|
|
|
742
750
|
from .log_privacy import sql_log_projection
|
|
743
|
-
entry = sql_log_projection(entry)
|
|
751
|
+
entry = sql_log_projection(entry, _intent_source=intent_source, _intent_values=intent_values)
|
|
744
752
|
logs = self.sql_logs()
|
|
745
753
|
logs.append(entry)
|
|
746
754
|
self._resources["sql_logs"] = logs
|
|
747
|
-
|
|
748
|
-
if sink is not None:
|
|
749
|
-
sink.write(entry)
|
|
755
|
+
self._write_diagnostic_sql_log(entry)
|
|
750
756
|
buf = self.get_resource("UnifiedLogBuffer")
|
|
751
757
|
if buf:
|
|
752
758
|
buf.entries.append(UnifiedLogEntry(
|
|
@@ -760,7 +766,8 @@ class UserContext:
|
|
|
760
766
|
if not self.sql_log_options().enabled_for(operation):
|
|
761
767
|
return
|
|
762
768
|
|
|
763
|
-
|
|
769
|
+
# Projection renders only safe values. Never materialize plaintext first.
|
|
770
|
+
debug_sql = ''
|
|
764
771
|
|
|
765
772
|
entry = SqlLogEntry(
|
|
766
773
|
operation=operation,
|
|
@@ -772,6 +779,9 @@ class UserContext:
|
|
|
772
779
|
params=getattr(query, 'params', []),
|
|
773
780
|
debug_sql=debug_sql,
|
|
774
781
|
pretty_sql=debug_sql,
|
|
782
|
+
database_kind=getattr(query, 'database_kind', None),
|
|
783
|
+
parameter_log_policies=getattr(query, 'parameter_log_policies', None),
|
|
784
|
+
sql_origin=getattr(query, 'sql_origin', None),
|
|
775
785
|
started_at=started_at,
|
|
776
786
|
ended_at=ended_at,
|
|
777
787
|
elapsed=elapsed,
|
|
@@ -791,9 +801,7 @@ class UserContext:
|
|
|
791
801
|
logs = self.sql_logs()
|
|
792
802
|
logs.append(entry)
|
|
793
803
|
self._resources["sql_logs"] = logs
|
|
794
|
-
|
|
795
|
-
if sink is not None:
|
|
796
|
-
sink.write(entry)
|
|
804
|
+
self._write_diagnostic_sql_log(entry)
|
|
797
805
|
|
|
798
806
|
buf = self.get_resource("UnifiedLogBuffer")
|
|
799
807
|
if buf:
|
|
@@ -804,6 +812,17 @@ class UserContext:
|
|
|
804
812
|
payload=LogPayload.Sql(entry)
|
|
805
813
|
))
|
|
806
814
|
|
|
815
|
+
def _write_diagnostic_sql_log(self, entry: 'SqlLogEntry'):
|
|
816
|
+
sink = self.get_resource("diagnostic_sql_log_sink")
|
|
817
|
+
if sink is None:
|
|
818
|
+
return
|
|
819
|
+
try:
|
|
820
|
+
sink.write(entry)
|
|
821
|
+
except Exception:
|
|
822
|
+
# An optional diagnostic destination must not change SQL outcomes.
|
|
823
|
+
# Do not print the exception: custom sinks may include raw values.
|
|
824
|
+
pass
|
|
825
|
+
|
|
807
826
|
def register_executor(self, executor: Any):
|
|
808
827
|
self.insert_resource("executor", executor)
|
|
809
828
|
|
|
@@ -1035,6 +1054,13 @@ class SqlLogEntry:
|
|
|
1035
1054
|
result_type: Optional[str]
|
|
1036
1055
|
affected_rows: Optional[int]
|
|
1037
1056
|
result_summary: str
|
|
1057
|
+
database_kind: Any = None
|
|
1058
|
+
parameter_log_policies: Optional[List[str]] = None
|
|
1059
|
+
sql_origin: Optional[str] = None
|
|
1060
|
+
masked_parameters: Optional[List[bool]] = None
|
|
1061
|
+
log_mode: Optional[str] = None
|
|
1062
|
+
omission_reason: Optional[str] = None
|
|
1063
|
+
execution_outcome: Optional[str] = None
|
|
1038
1064
|
|
|
1039
1065
|
class DiagnosticSqlLogSink:
|
|
1040
1066
|
"""Policy-projected SQL destination; the text sink is installed by default."""
|
|
@@ -1051,9 +1077,9 @@ class TextDiagnosticSqlLogSink(DiagnosticSqlLogSink):
|
|
|
1051
1077
|
elapsed_us = int(entry.elapsed.total_seconds() * 1_000_000) if entry.elapsed else 0
|
|
1052
1078
|
self._writer(
|
|
1053
1079
|
f"[TeaQL SQL][{entry.operation.name.lower()}][{elapsed_us}us] "
|
|
1054
|
-
f"{entry.result_summary} comment={entry.comment!r} purpose={entry.purpose!r} "
|
|
1080
|
+
f"{entry.result_summary} outcome={entry.execution_outcome or 'unknown'} comment={entry.comment!r} purpose={entry.purpose!r} "
|
|
1055
1081
|
f"auditReason={entry.audit_reason!r} tracePath={entry.trace_path!r}\n"
|
|
1056
|
-
f"
|
|
1082
|
+
f"SQL omission reason: {entry.omission_reason or 'none'}\n"
|
|
1057
1083
|
f"Debug SQL: {entry.debug_sql}"
|
|
1058
1084
|
)
|
|
1059
1085
|
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Log-only projections. Never modify execution parameters or stored values."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import weakref
|
|
7
|
+
from copy import deepcopy
|
|
8
|
+
from dataclasses import fields, is_dataclass, replace
|
|
9
|
+
from functools import lru_cache
|
|
10
|
+
|
|
11
|
+
PLAINTEXT_ENV = "TEAQL_ALLOW_SENSITIVE_PLAINTEXT_LOGS"
|
|
12
|
+
PLAINTEXT_ACK = "I_UNDERSTAND_SENSITIVE_DATA_MAY_BE_WRITTEN_TO_DISK"
|
|
13
|
+
REDACTED = "[REDACTED]"
|
|
14
|
+
SQL_REDACTED = "[REDACTED SQL; NOT REPLAYABLE]"
|
|
15
|
+
DEBUG_LABEL = "-- TeaQL DEBUG PLAINTEXT; EXPLICIT OPT-IN\n"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def label_debug_sql(sql):
|
|
19
|
+
return sql if not sql or sql.startswith(DEBUG_LABEL) else DEBUG_LABEL + sql
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@lru_cache(maxsize=1)
|
|
23
|
+
def _warn_plaintext():
|
|
24
|
+
logging.getLogger("teaql.runtime").warning(
|
|
25
|
+
"Sensitive plaintext logging enabled: application data may be written to disk. "
|
|
26
|
+
"Authentication secrets remain redacted."
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def plaintext_enabled():
|
|
31
|
+
enabled = os.environ.get(PLAINTEXT_ENV) == PLAINTEXT_ACK
|
|
32
|
+
if enabled:
|
|
33
|
+
_warn_plaintext()
|
|
34
|
+
return enabled
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def credential_name(name):
|
|
38
|
+
normalized = re.sub(r"[^a-z0-9]", "", str(name).lower())
|
|
39
|
+
return any(word in normalized for word in (
|
|
40
|
+
"password", "passwd", "passphrase", "privatekey", "secret", "accesstoken",
|
|
41
|
+
"refreshtoken", "idtoken", "apikey", "authorization", "credential",
|
|
42
|
+
"sessiontoken", "magiclinktoken",
|
|
43
|
+
))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def payload_has_credentials(value):
|
|
47
|
+
value = getattr(value, "val", value)
|
|
48
|
+
if isinstance(value, dict):
|
|
49
|
+
return any(credential_name(k) or payload_has_credentials(v) for k, v in value.items())
|
|
50
|
+
if isinstance(value, (tuple, list)):
|
|
51
|
+
return any(payload_has_credentials(v) for v in value)
|
|
52
|
+
return False
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def value_strings(value):
|
|
56
|
+
value = getattr(value, "val", value)
|
|
57
|
+
if isinstance(value, dict):
|
|
58
|
+
return [s for v in value.values() for s in value_strings(v)]
|
|
59
|
+
if isinstance(value, (tuple, list)):
|
|
60
|
+
return [s for v in value for s in value_strings(v)]
|
|
61
|
+
return [] if value is None or value == "" else [str(value)]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def scrub(value, secrets, hide_all=False):
|
|
65
|
+
"""Copy annotations, removing known values even when embedded in intent."""
|
|
66
|
+
if isinstance(value, str):
|
|
67
|
+
if hide_all:
|
|
68
|
+
return REDACTED if value else value
|
|
69
|
+
for secret in sorted(set(secrets), key=len, reverse=True):
|
|
70
|
+
value = value.replace(secret, REDACTED)
|
|
71
|
+
return value
|
|
72
|
+
if isinstance(value, list):
|
|
73
|
+
return [scrub(v, secrets, hide_all) for v in value]
|
|
74
|
+
if isinstance(value, tuple):
|
|
75
|
+
return tuple(scrub(v, secrets, hide_all) for v in value)
|
|
76
|
+
if isinstance(value, dict):
|
|
77
|
+
return {k: scrub(v, secrets, hide_all) for k, v in value.items()}
|
|
78
|
+
if is_dataclass(value):
|
|
79
|
+
return replace(value, **{f.name: scrub(getattr(value, f.name), secrets, hide_all)
|
|
80
|
+
for f in fields(value) if f.init})
|
|
81
|
+
return value
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_projections = {}
|
|
85
|
+
|
|
86
|
+
def _remember_projection(projected, allow, alternative=None):
|
|
87
|
+
key = id(projected)
|
|
88
|
+
_projections[key] = (weakref.ref(projected, lambda _: _projections.pop(key, None)),
|
|
89
|
+
allow, deepcopy(projected), alternative)
|
|
90
|
+
return projected
|
|
91
|
+
|
|
92
|
+
def _business_mask(value):
|
|
93
|
+
from .audit import _mask
|
|
94
|
+
value = getattr(value, 'val', value)
|
|
95
|
+
if value is None:
|
|
96
|
+
return None
|
|
97
|
+
if isinstance(value, (tuple, list)):
|
|
98
|
+
return [_business_mask(child) for child in value]
|
|
99
|
+
if isinstance(value, dict):
|
|
100
|
+
return REDACTED
|
|
101
|
+
return _mask(str(value))
|
|
102
|
+
|
|
103
|
+
def _binding_policies(source):
|
|
104
|
+
supplied = source.parameter_log_policies
|
|
105
|
+
valid = supplied is not None and len(supplied) == len(source.params)
|
|
106
|
+
credentials = credential_name(source.sql) and (source.sql_origin != 'generated' or not valid)
|
|
107
|
+
policies = []
|
|
108
|
+
for index, value in enumerate(source.params):
|
|
109
|
+
policy = supplied[index] if valid else 'unknown'
|
|
110
|
+
if credentials or payload_has_credentials(value):
|
|
111
|
+
policy = 'credential'
|
|
112
|
+
policies.append(policy if policy in ('plain', 'masked', 'credential') else 'unknown')
|
|
113
|
+
return policies
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _is_masked(policy, allow):
|
|
117
|
+
return policy in ('credential', 'unknown') or (not allow and policy != 'plain')
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def sql_log_projection(entry, *, _intent_source=None, _intent_values=()):
|
|
121
|
+
"""Source bindings are call-local runtime plumbing, never stored on a log entry."""
|
|
122
|
+
allow = plaintext_enabled() and entry.log_mode != 'masked'
|
|
123
|
+
prior = _projections.get(id(entry))
|
|
124
|
+
# Log entries are mutable: reuse only an unchanged projection. Weak references
|
|
125
|
+
# keep this idempotence bookkeeping from retaining a long-running log history.
|
|
126
|
+
if prior and prior[0]() is entry and prior[2] == entry:
|
|
127
|
+
if not prior[1] or allow:
|
|
128
|
+
return entry
|
|
129
|
+
# Entries are mutable. Do not expose the cached safe alternative itself.
|
|
130
|
+
if prior[3] is not None:
|
|
131
|
+
return _remember_projection(deepcopy(prior[3]), False)
|
|
132
|
+
projected = _project_with_policy(entry, allow, _intent_source, _intent_values)
|
|
133
|
+
alternative = _project_with_policy(entry, False, _intent_source, _intent_values) if allow else None
|
|
134
|
+
return _remember_projection(projected, allow, alternative)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _project_with_policy(entry, allow, intent_source, intent_values):
|
|
138
|
+
from teaql.sql.types import DatabaseKind, render_log_sql, _sql_literal
|
|
139
|
+
from teaql.core.value import Value
|
|
140
|
+
supplied = entry.parameter_log_policies
|
|
141
|
+
valid = supplied is None or len(supplied) == len(entry.params)
|
|
142
|
+
# Compiler-owned bindings already identify individual credential fields. A
|
|
143
|
+
# credential column elsewhere in the projection must not hide ordinary binds.
|
|
144
|
+
# Unclassified/custom statements retain the conservative whole-SQL fallback.
|
|
145
|
+
credentials = credential_name(entry.sql) and (
|
|
146
|
+
entry.sql_origin != 'generated' or supplied is None or not valid)
|
|
147
|
+
policies = _binding_policies(entry)
|
|
148
|
+
masked = [_is_masked(policy, allow) for policy in policies]
|
|
149
|
+
safe_values = [(_business_mask(value) if policies[index] == 'masked' else REDACTED)
|
|
150
|
+
if masked[index] else deepcopy(value) for index, value in enumerate(entry.params)]
|
|
151
|
+
secrets = [text for index, value in enumerate(entry.params) if masked[index]
|
|
152
|
+
for text in value_strings(value)]
|
|
153
|
+
if intent_source is not None:
|
|
154
|
+
source_policies = _binding_policies(intent_source)
|
|
155
|
+
secrets.extend(text for index, value in enumerate(intent_source.params)
|
|
156
|
+
if _is_masked(source_policies[index], allow) for text in value_strings(value))
|
|
157
|
+
intent_secrets = secrets + [text for value in intent_values for text in value_strings(value)]
|
|
158
|
+
# A copied/changed debug record has lost its reliable private alternative.
|
|
159
|
+
# Its inherited intent may mention values absent from its own SQL bindings.
|
|
160
|
+
unknown_debug_intent = not allow and entry.log_mode == 'debug-plaintext' and intent_source is None
|
|
161
|
+
def intent(value):
|
|
162
|
+
return scrub(value, intent_secrets, hide_all=unknown_debug_intent)
|
|
163
|
+
bare = re.sub(r'\$[0-9]+', '?', entry.sql)
|
|
164
|
+
unsafe = ((not allow or credentials) and entry.sql_origin != 'generated' and
|
|
165
|
+
bool(re.search(r"['\"`$]|--|/\*|\b\d+\b|:[A-Za-z_]", bare)))
|
|
166
|
+
reason = 'untrusted-literal-sql' if unsafe else 'policy-count-mismatch' if not valid else None
|
|
167
|
+
rendered = SQL_REDACTED
|
|
168
|
+
kind = entry.database_kind or (DatabaseKind.PostgreSql if re.search(r'\$[0-9]+', entry.sql)
|
|
169
|
+
else DatabaseKind.MySql if '%s' in entry.sql else DatabaseKind.Sqlite)
|
|
170
|
+
if reason is None:
|
|
171
|
+
try:
|
|
172
|
+
def literal(index):
|
|
173
|
+
value = safe_values[index]
|
|
174
|
+
value = value if isinstance(value, Value) else Value.from_any(value)
|
|
175
|
+
return _sql_literal(value, kind) + (' /* masked */' if masked[index] else '')
|
|
176
|
+
rendered = render_log_sql(entry.sql, safe_values, kind, literal)
|
|
177
|
+
prefix = (('-- TeaQL DEBUG PLAINTEXT; EXPLICIT OPT-IN; PARTIALLY MASKED; NOT REPLAYABLE\n'
|
|
178
|
+
if any(masked) else DEBUG_LABEL) if allow else '-- TeaQL MASKED; NOT REPLAYABLE\n')
|
|
179
|
+
rendered = prefix + rendered
|
|
180
|
+
except Exception:
|
|
181
|
+
# Renderer failures must not echo source SQL, values, or exception text.
|
|
182
|
+
rendered = SQL_REDACTED
|
|
183
|
+
reason = 'unsupported-or-mismatched-bindings'
|
|
184
|
+
projected = replace(entry, sql=SQL_REDACTED if unsafe else entry.sql
|
|
185
|
+
if entry.sql_origin == 'generated' else scrub(entry.sql, secrets),
|
|
186
|
+
params=safe_values, parameter_log_policies=policies, masked_parameters=masked,
|
|
187
|
+
debug_sql=rendered, pretty_sql=rendered, omission_reason=reason,
|
|
188
|
+
log_mode='debug-plaintext' if allow else 'masked',
|
|
189
|
+
comment=intent(entry.comment), purpose=intent(entry.purpose),
|
|
190
|
+
audit_reason=intent(entry.audit_reason),
|
|
191
|
+
result_summary=(f'{entry.result_count} rows returned' if entry.result_count is not None
|
|
192
|
+
else f'{entry.affected_rows} rows affected' if entry.affected_rows is not None
|
|
193
|
+
else scrub(entry.result_summary, secrets, hide_all=unknown_debug_intent)),
|
|
194
|
+
trace_path=intent(entry.trace_path))
|
|
195
|
+
return projected
|