atlan-application-sdk-api 3.51.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- application_sdk/__init__.py +7 -0
- application_sdk/_build_info.py +62 -0
- application_sdk/_context_errors.py +65 -0
- application_sdk/_install.py +130 -0
- application_sdk/_logging.py +171 -0
- application_sdk/_runtime/__init__.py +29 -0
- application_sdk/_runtime/offload.py +1020 -0
- application_sdk/_runtime/progress.py +757 -0
- application_sdk/clients/__init__.py +63 -0
- application_sdk/clients/_interface.py +58 -0
- application_sdk/clients/_sql_failure_classifier.py +112 -0
- application_sdk/clients/base.py +565 -0
- application_sdk/clients/http_errors.py +28 -0
- application_sdk/clients/models.py +49 -0
- application_sdk/clients/sql.py +1218 -0
- application_sdk/clients/sql_errors.py +192 -0
- application_sdk/clients/sql_typecasters.py +431 -0
- application_sdk/clients/ssl_utils.py +247 -0
- application_sdk/common/__init__.py +53 -0
- application_sdk/common/_generated_tree.py +321 -0
- application_sdk/common/_listing.py +192 -0
- application_sdk/common/aws_utils.py +522 -0
- application_sdk/common/aws_utils_errors.py +88 -0
- application_sdk/common/build_identity.py +120 -0
- application_sdk/common/concurrency.py +45 -0
- application_sdk/common/errors.py +99 -0
- application_sdk/common/filter_matching.py +261 -0
- application_sdk/common/logger.py +11 -0
- application_sdk/common/models.py +31 -0
- application_sdk/common/sql_filters.py +719 -0
- application_sdk/common/sql_filters_errors.py +17 -0
- application_sdk/common/task_queue.py +431 -0
- application_sdk/common/types.py +51 -0
- application_sdk/common/utils.py +87 -0
- application_sdk/constants.py +1120 -0
- application_sdk/contracts/__init__.py +158 -0
- application_sdk/contracts/base.py +1178 -0
- application_sdk/contracts/events.py +251 -0
- application_sdk/contracts/types.py +801 -0
- application_sdk/contracts/types_errors.py +17 -0
- application_sdk/credentials/__init__.py +326 -0
- application_sdk/credentials/errors.py +226 -0
- application_sdk/credentials/extra.py +114 -0
- application_sdk/credentials/ingress.py +283 -0
- application_sdk/credentials/ref.py +376 -0
- application_sdk/credentials/spec.py +182 -0
- application_sdk/credentials/types.py +339 -0
- application_sdk/credentials/utils.py +257 -0
- application_sdk/errors/__init__.py +224 -0
- application_sdk/errors/base.py +504 -0
- application_sdk/errors/categories.py +118 -0
- application_sdk/errors/http.py +81 -0
- application_sdk/errors/leaves.py +526 -0
- application_sdk/errors/wire.py +124 -0
- application_sdk/handler/__init__.py +97 -0
- application_sdk/handler/_warmup.py +198 -0
- application_sdk/handler/asgi.py +252 -0
- application_sdk/handler/base.py +281 -0
- application_sdk/handler/context.py +212 -0
- application_sdk/handler/contracts.py +1166 -0
- application_sdk/handler/manifest.py +81 -0
- application_sdk/handler/metadata_routes.py +699 -0
- application_sdk/handler/object_routes.py +418 -0
- application_sdk/handler/request_contract.py +91 -0
- application_sdk/handler/routes.py +1118 -0
- application_sdk/handler/service_errors.py +37 -0
- application_sdk/server/__init__.py +15 -0
- application_sdk/server/fastapi/models.py +338 -0
- application_sdk/templates/__init__.py +52 -0
- application_sdk/templates/contracts/__init__.py +111 -0
- application_sdk/templates/contracts/base_metadata_extraction.py +32 -0
- application_sdk/templates/contracts/incremental_sql.py +563 -0
- application_sdk/templates/contracts/sql_metadata.py +824 -0
- application_sdk/templates/contracts/sql_query.py +105 -0
- application_sdk/version.py +40 -0
- atlan_application_sdk_api-3.51.0.dist-info/METADATA +230 -0
- atlan_application_sdk_api-3.51.0.dist-info/RECORD +78 -0
- atlan_application_sdk_api-3.51.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Typed error leaves for the SQL client family."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import ClassVar
|
|
7
|
+
|
|
8
|
+
from application_sdk.errors.categories import Audience
|
|
9
|
+
from application_sdk.errors.leaves import (
|
|
10
|
+
AppTimeoutError,
|
|
11
|
+
AuthError,
|
|
12
|
+
InternalError,
|
|
13
|
+
InvalidInputError,
|
|
14
|
+
SourceUnavailableError,
|
|
15
|
+
UnimplementedError,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(kw_only=True)
|
|
20
|
+
class SqlClientConfigError(InternalError):
|
|
21
|
+
"""DB_CONFIG is not set on this SQL client subclass."""
|
|
22
|
+
|
|
23
|
+
code: ClassVar[str] = "INTERNAL_SQL_DB_CONFIG_NOT_SET"
|
|
24
|
+
message: str = "DB_CONFIG is not configured for this SQL client"
|
|
25
|
+
component: str | None = "sql_client"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(kw_only=True)
|
|
29
|
+
class SqlClientAuthFailedError(AuthError):
|
|
30
|
+
"""SQL database authentication failed."""
|
|
31
|
+
|
|
32
|
+
code: ClassVar[str] = "AUTH_SQL_CLIENT_FAILED"
|
|
33
|
+
message: str = "SQL client authentication failed"
|
|
34
|
+
auth_method: str | None = "sql"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(kw_only=True)
|
|
38
|
+
class SqlClientSourceUnavailableError(SourceUnavailableError):
|
|
39
|
+
"""The SQL client could not reach the source while connecting.
|
|
40
|
+
|
|
41
|
+
DNS failure, refused or filtered connection, TLS handshake failure: the
|
|
42
|
+
credentials were never checked, so this is not an authentication problem
|
|
43
|
+
and an account unlock will not fix it.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
code: ClassVar[str] = "SOURCE_UNAVAILABLE_SQL_CLIENT"
|
|
47
|
+
message: str = "SQL client could not reach the source"
|
|
48
|
+
suggested_action: str | None = (
|
|
49
|
+
"Check DNS resolution, firewall and private-link rules, TLS "
|
|
50
|
+
"configuration, and that the source is accepting connections from "
|
|
51
|
+
"the worker network."
|
|
52
|
+
)
|
|
53
|
+
source_type: str | None = "sql"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(kw_only=True)
|
|
57
|
+
class SqlClientConnectTimeoutError(AppTimeoutError):
|
|
58
|
+
"""The SQL client's connect or login ran out of time waiting on the source.
|
|
59
|
+
|
|
60
|
+
``AppTimeoutError`` defaults to ``APP_OWNER`` because a timeout's locus is
|
|
61
|
+
usually ambiguous. Here it is not: what times out is the driver's or the
|
|
62
|
+
socket's own timeout, or ``load()``'s connect budget, spent building the
|
|
63
|
+
connection string (an STS AssumeRole for IAM-role auth) and connecting —
|
|
64
|
+
so the clock ran out waiting for the customer's source or its auth path.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
code: ClassVar[str] = "TIMEOUT_SQL_CLIENT_CONNECT"
|
|
68
|
+
audience: ClassVar[Audience] = Audience.USER
|
|
69
|
+
message: str = "SQL client timed out connecting to the source"
|
|
70
|
+
suggested_action: str | None = (
|
|
71
|
+
"Check that the source is up and reachable from the worker network, "
|
|
72
|
+
"and that the connection's login timeout suits the source."
|
|
73
|
+
)
|
|
74
|
+
operation: str | None = "sql_client_connect"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(kw_only=True)
|
|
78
|
+
class SqlClientConnectUnclassifiedError(InternalError):
|
|
79
|
+
"""The SQL client failed to connect for a reason the classifier does not know.
|
|
80
|
+
|
|
81
|
+
Raised instead of guessing at authentication. ``classification_pending``
|
|
82
|
+
flags it so the driver error can be added to the shared SQL failure
|
|
83
|
+
classifier rather than read at face value.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
code: ClassVar[str] = "INTERNAL_SQL_CLIENT_CONNECT"
|
|
87
|
+
message: str = "SQL client failed to connect for an unclassified reason"
|
|
88
|
+
component: str | None = "sql_client"
|
|
89
|
+
classification_pending: bool = True
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(kw_only=True)
|
|
93
|
+
class SqlCredentialsParseError(InvalidInputError):
|
|
94
|
+
"""SQL credentials could not be parsed or are missing required fields."""
|
|
95
|
+
|
|
96
|
+
code: ClassVar[str] = "INVALID_INPUT_CREDENTIALS_PARSE"
|
|
97
|
+
message: str = "SQL credentials parse error"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(kw_only=True)
|
|
101
|
+
class SqlAwsCredentialsError(AuthError):
|
|
102
|
+
"""AWS IAM credentials required for RDS authentication are missing."""
|
|
103
|
+
|
|
104
|
+
code: ClassVar[str] = "AUTH_AWS_CREDENTIALS"
|
|
105
|
+
message: str = "AWS IAM credentials are missing or incomplete"
|
|
106
|
+
auth_method: str | None = "aws_iam"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(kw_only=True)
|
|
110
|
+
class MissingSqlParamError(InvalidInputError):
|
|
111
|
+
"""A required SQL connection parameter is missing."""
|
|
112
|
+
|
|
113
|
+
code: ClassVar[str] = "INVALID_INPUT_SQL_MISSING_PARAM"
|
|
114
|
+
message: str = "Required SQL connection parameter is missing"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(kw_only=True)
|
|
118
|
+
class EngineNotInitializedError(InternalError):
|
|
119
|
+
"""SQL engine is not initialized; call load() before running queries."""
|
|
120
|
+
|
|
121
|
+
code: ClassVar[str] = "INTERNAL_SQL_ENGINE_NOT_INITIALIZED"
|
|
122
|
+
message: str = "Engine is not initialized. Call load() first."
|
|
123
|
+
component: str | None = "sql_engine"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass(kw_only=True)
|
|
127
|
+
class SqlBindParameterError(InternalError):
|
|
128
|
+
"""``run_query`` was given bind parameters it cannot apply.
|
|
129
|
+
|
|
130
|
+
A name that is not an identifier, or one the query never references as
|
|
131
|
+
``:name``. Running anyway would execute the query without that value, so a
|
|
132
|
+
filter meant to narrow the result would silently not apply.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
code: ClassVar[str] = "INTERNAL_SQL_BIND_PARAMETER"
|
|
136
|
+
message: str = "SQL bind parameters do not match the query"
|
|
137
|
+
component: str | None = "sql_client"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@dataclass(kw_only=True)
|
|
141
|
+
class UnsupportedSqlCursorError(UnimplementedError):
|
|
142
|
+
"""The current SQL engine or connection does not support cursors."""
|
|
143
|
+
|
|
144
|
+
code: ClassVar[str] = "UNIMPLEMENTED_SQL_CURSOR_TYPE"
|
|
145
|
+
message: str = "Cursor is not supported by this SQL engine"
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass(kw_only=True)
|
|
149
|
+
class InvalidSqlEngineTypeError(InternalError):
|
|
150
|
+
"""The engine is a raw connection string, not a SQLAlchemy engine object."""
|
|
151
|
+
|
|
152
|
+
code: ClassVar[str] = "INTERNAL_SQL_INVALID_ENGINE_TYPE"
|
|
153
|
+
message: str = (
|
|
154
|
+
"Engine should be an SQLAlchemy engine object, not a connection string"
|
|
155
|
+
)
|
|
156
|
+
component: str | None = "sql_engine"
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@dataclass(kw_only=True)
|
|
160
|
+
class SqlPandasResultError(InternalError):
|
|
161
|
+
"""A SQL read operation failed to produce a valid pandas DataFrame result.
|
|
162
|
+
|
|
163
|
+
Raised in two distinct situations, which differ in retryability:
|
|
164
|
+
|
|
165
|
+
- **Invariant violation** (the read path returned a non-DataFrame). A bug in
|
|
166
|
+
our code; retrying cannot help. Keeps the ``INTERNAL`` default
|
|
167
|
+
``retryable=False``.
|
|
168
|
+
- **Catch-and-rewrap of an unclassified driver exception** at the query
|
|
169
|
+
boundary. Here the SDK has *not* established what failed — a transient
|
|
170
|
+
source-side condition (concurrent DDL, lock contention, a dropped
|
|
171
|
+
connection) is wrapped identically to a permanent one. Those sites pass
|
|
172
|
+
``retryable=True`` explicitly, which does not assert "this will succeed on
|
|
173
|
+
retry"; it declines to override the activity's own declared Temporal retry
|
|
174
|
+
policy for an error we never classified. Asserting ``retryable=False``
|
|
175
|
+
there marks the failure ``non_retryable`` on the wire and makes the
|
|
176
|
+
activity's declared ``maximum_attempts`` dead code (CONAT-747).
|
|
177
|
+
|
|
178
|
+
Errors that *are* classified never reach either site — the enclosing
|
|
179
|
+
``except AppError: raise`` clauses re-raise them with their own retryability.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
code: ClassVar[str] = "INTERNAL_SQL_PANDAS"
|
|
183
|
+
message: str = "Error reading data from SQL into a pandas DataFrame"
|
|
184
|
+
component: str | None = "sql_pandas"
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(kw_only=True)
|
|
188
|
+
class AbstractClientLoadError(UnimplementedError):
|
|
189
|
+
"""load() called on a ClientInterface subclass that has not implemented it."""
|
|
190
|
+
|
|
191
|
+
code: ClassVar[str] = "UNIMPLEMENTED_CLIENT_LOAD"
|
|
192
|
+
message: str = "load method is not implemented"
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Tolerant UTF-8 text decoders for SQL DBAPI cursors.
|
|
2
|
+
|
|
3
|
+
Long-lived warehouses accumulate query text from heterogeneous client encodings
|
|
4
|
+
(SSMS, Excel, Word paste, terminals, etc.). When the SDK pulls that text via a
|
|
5
|
+
strict UTF-8 cursor decoder, a single non-UTF-8 byte (commonly ``0x96`` —
|
|
6
|
+
Windows-1252 en-dash) raises ``UnicodeDecodeError`` and aborts the entire
|
|
7
|
+
batch, breaking query-history miners.
|
|
8
|
+
|
|
9
|
+
The intended semantics have not changed since WARE-970: decode UTF-8, and where
|
|
10
|
+
the bytes are not valid UTF-8 under any reading, emit ``U+FFFD`` rather than
|
|
11
|
+
kill the batch. What CONAT-767 changed is only the *injection point* — the
|
|
12
|
+
tolerance was registered at a layer psycopg2 never reaches (see
|
|
13
|
+
:func:`install_tolerant_connection_decoder`), so production kept crashing while
|
|
14
|
+
the code read as fixed. Nothing here infers or guesses an encoding: the wire
|
|
15
|
+
encoding is declared by the connection, and ``errors="replace"`` is a total
|
|
16
|
+
function over that already-fixed decoding. No correctly-encoded byte can be
|
|
17
|
+
dropped.
|
|
18
|
+
|
|
19
|
+
Scope (psycopg2)
|
|
20
|
+
----------------
|
|
21
|
+
The psycopg2 fix covers connections whose server reports its client encoding as
|
|
22
|
+
``UNICODE`` — Redshift (a PostgreSQL 8.0.2 fork, and PG 8.0 spelled UTF-8
|
|
23
|
+
``UNICODE``) and other pre-8.1-era forks. Connections that report literally
|
|
24
|
+
``UTF8`` — which is every PostgreSQL from 8.1 onward — take psycopg2's C fast
|
|
25
|
+
decoder (``PyUnicode_DecodeUTF8``, strict), which no Python-level hook can
|
|
26
|
+
intercept. **The original crash class therefore remains open for plain
|
|
27
|
+
PostgreSQL query-history miners on psycopg2.** Closing it would require owning
|
|
28
|
+
``client_encoding``, which is a separate decision (see below). psycopg3 is
|
|
29
|
+
covered on all servers via its loader API.
|
|
30
|
+
|
|
31
|
+
Recorded decisions
|
|
32
|
+
------------------
|
|
33
|
+
``errors="replace"``, not ``surrogateescape``. ``surrogateescape`` would
|
|
34
|
+
preserve the original byte and make the loss reversible, which is genuinely
|
|
35
|
+
attractive. It is rejected because decoded text flows on to parquet and JSON,
|
|
36
|
+
where lone surrogates are not serializable — a deferred hard failure in the
|
|
37
|
+
writer is worse than a visible ``U+FFFD`` in the data. ``replace`` is lossy on
|
|
38
|
+
purpose, and the log signal below is what makes the loss visible.
|
|
39
|
+
|
|
40
|
+
Not ``client_encoding=latin1``. That stops the crash but mojibakes valid UTF-8
|
|
41
|
+
(``0xC3 0xA9`` → ``é`` instead of ``é``) — a strictly worse fidelity trade.
|
|
42
|
+
|
|
43
|
+
Not ``client_encoding=SQL_ASCII`` either, though it is the strongest
|
|
44
|
+
alternative and deserves an explicit rejection. Under ``SQL_ASCII`` psycopg2
|
|
45
|
+
maps to the ``ascii`` codec, so per-value decoding could move into a typecaster:
|
|
46
|
+
that would give per-value control, natural logging, and would close the ``UTF8``
|
|
47
|
+
gap above uniformly. The cost is that the SDK would then own *outgoing* encoding
|
|
48
|
+
for every non-ASCII identifier and parameter on every connector that inherits
|
|
49
|
+
this, on connections it did not configure — a much larger blast radius than the
|
|
50
|
+
defect being fixed, and one that fails in the write direction rather than the
|
|
51
|
+
read direction. Rejected on that basis, not on merit.
|
|
52
|
+
|
|
53
|
+
Observability
|
|
54
|
+
-------------
|
|
55
|
+
The replacement path logs a rate-limited WARNING with the offending byte, its
|
|
56
|
+
offset, and a redacted excerpt (identifiers and literals masked — no query text
|
|
57
|
+
reaches the logs). One stray byte in a large query and a column that was
|
|
58
|
+
Windows-1252 all along are both silent in the data; the log line is what
|
|
59
|
+
distinguishes them, and what makes "did this run take the tolerant path?"
|
|
60
|
+
answerable after the fact instead of inferred.
|
|
61
|
+
|
|
62
|
+
Tracking: CONAT-767 (injection point), WARE-970 (original intent).
|
|
63
|
+
|
|
64
|
+
References
|
|
65
|
+
----------
|
|
66
|
+
- psycopg2 typecasters: https://www.psycopg.org/docs/extensions.html#psycopg2.extensions.new_type
|
|
67
|
+
- psycopg3 loaders: https://www.psycopg.org/psycopg3/docs/advanced/adapt.html
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
from __future__ import annotations
|
|
71
|
+
|
|
72
|
+
import codecs
|
|
73
|
+
import threading
|
|
74
|
+
from typing import Any
|
|
75
|
+
|
|
76
|
+
from application_sdk._logging import get_logger
|
|
77
|
+
|
|
78
|
+
logger = get_logger(__name__)
|
|
79
|
+
|
|
80
|
+
# Name of the private codec registered by ``_ensure_tolerant_codec``. It is an
|
|
81
|
+
# ordinary UTF-8 codec whose *decoder* replaces invalid bytes instead of
|
|
82
|
+
# raising; the encoder is stock strict UTF-8, so outgoing SQL is unchanged.
|
|
83
|
+
_TOLERANT_CODEC_NAME = "atlan_tolerant_utf8"
|
|
84
|
+
|
|
85
|
+
# Bytes shown either side of the offending byte in the redacted log excerpt.
|
|
86
|
+
_EXCERPT_WINDOW = 8
|
|
87
|
+
|
|
88
|
+
# Cumulative count of *values* that needed replacement in this process, and the
|
|
89
|
+
# lock guarding it. Only touched on the (rare, by construction) failure path.
|
|
90
|
+
_replacement_total = 0
|
|
91
|
+
_replacement_lock = threading.Lock()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# psycopg3 text-ish PostgreSQL OIDs we want to override. These match the OIDs
|
|
95
|
+
# psycopg3 itself registers ``TextLoader``/``TextBinaryLoader`` for: text(25),
|
|
96
|
+
# varchar(1043), bpchar(1042), name(19), char(18), unknown(705).
|
|
97
|
+
# Sourced from ``psycopg.postgres.types`` for the canonical names.
|
|
98
|
+
#
|
|
99
|
+
# We also register for OID 0 (``InvalidOid``). Some Postgres-compatible proxies
|
|
100
|
+
# — notably PgBouncer in transaction-pooling mode, Azure Flex Server's built-in
|
|
101
|
+
# pooler, and AWS RDS Proxy — can return OID 0 on prepared-statement metadata
|
|
102
|
+
# (e.g. ``pg_catalog.version()``), which psycopg3 otherwise hands to the caller
|
|
103
|
+
# as raw bytes. SQLAlchemy's Postgres dialect then runs a string regex against
|
|
104
|
+
# those bytes in ``_get_server_version_info`` and raises ``TypeError: cannot
|
|
105
|
+
# use a string pattern on a bytes-like object`` before any query can run.
|
|
106
|
+
_PSYCOPG3_TEXT_OIDS: tuple[int, ...] = (0, 18, 19, 25, 705, 1042, 1043)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _redacted_excerpt(data: bytes, offset: int) -> str:
|
|
110
|
+
"""Render the bytes around ``offset`` without leaking query text.
|
|
111
|
+
|
|
112
|
+
ASCII letters collapse to ``a``/``A`` and digits to ``0``, so identifiers,
|
|
113
|
+
literals and any customer-identifying value are destroyed while the *shape*
|
|
114
|
+
survives. Punctuation is structural and kept. Only the bracketed offending
|
|
115
|
+
byte is rendered as its exact ``\\xNN`` value — it is already unrecoverable
|
|
116
|
+
from the decoded text, so revealing it leaks nothing.
|
|
117
|
+
|
|
118
|
+
Every *other* non-ASCII byte renders as the fixed token ``\\x??``: a valid
|
|
119
|
+
UTF-8 multi-byte sequence next to the bad byte is customer content (``é`` in
|
|
120
|
+
a name), so its value must not reach the log — but the *count* of high bytes
|
|
121
|
+
is kept, because that is the tell for a wholesale Windows-1252 column: one
|
|
122
|
+
lone ``[\\x96]`` in masked ASCII is a stray byte, whereas a run of ``\\x??``
|
|
123
|
+
neighbours means the column was never UTF-8.
|
|
124
|
+
"""
|
|
125
|
+
start = max(0, offset - _EXCERPT_WINDOW)
|
|
126
|
+
end = min(len(data), offset + _EXCERPT_WINDOW + 1)
|
|
127
|
+
|
|
128
|
+
rendered: list[str] = []
|
|
129
|
+
for index in range(start, end):
|
|
130
|
+
byte = data[index]
|
|
131
|
+
if index == offset:
|
|
132
|
+
rendered.append(f"[\\x{byte:02x}]")
|
|
133
|
+
elif 0x41 <= byte <= 0x5A: # A-Z
|
|
134
|
+
rendered.append("A")
|
|
135
|
+
elif 0x61 <= byte <= 0x7A: # a-z
|
|
136
|
+
rendered.append("a")
|
|
137
|
+
elif 0x30 <= byte <= 0x39: # 0-9
|
|
138
|
+
rendered.append("0")
|
|
139
|
+
elif byte in (0x09, 0x0A, 0x0D, 0x20): # whitespace
|
|
140
|
+
rendered.append(" ")
|
|
141
|
+
elif byte < 0x80: # ASCII punctuation — structural, not content
|
|
142
|
+
rendered.append(chr(byte))
|
|
143
|
+
else: # non-ASCII neighbour — value is content; only the count matters
|
|
144
|
+
rendered.append("\\x??")
|
|
145
|
+
|
|
146
|
+
prefix = "…" if start > 0 else ""
|
|
147
|
+
suffix = "…" if end < len(data) else ""
|
|
148
|
+
return f"{prefix}{''.join(rendered)}{suffix}"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _note_replacement(data: bytes, error: UnicodeDecodeError, decoded: str) -> None:
|
|
152
|
+
"""Emit a rate-limited WARNING that the tolerant decoder dropped a byte.
|
|
153
|
+
|
|
154
|
+
Rate limiting is per *process* on powers of two (1st, 2nd, 4th, 8th, …
|
|
155
|
+
value): loud on first occurrence, so a single stray byte is never silent,
|
|
156
|
+
and log-bounded when an entire column turns out to be Windows-1252. Per
|
|
157
|
+
*connection* would be the nicer granularity but is not reachable here —
|
|
158
|
+
psycopg2 resolves the codec by name from a module-level dict and calls the
|
|
159
|
+
decoder with the bytes as the sole argument, so no connection identity
|
|
160
|
+
reaches this frame.
|
|
161
|
+
"""
|
|
162
|
+
global _replacement_total
|
|
163
|
+
|
|
164
|
+
with _replacement_lock:
|
|
165
|
+
_replacement_total += 1
|
|
166
|
+
total = _replacement_total
|
|
167
|
+
|
|
168
|
+
if total & (total - 1): # not a power of two — stay quiet
|
|
169
|
+
return
|
|
170
|
+
|
|
171
|
+
logger.warning(
|
|
172
|
+
"Tolerant UTF-8 decoder replaced invalid byte 0x%02x at offset %d of a "
|
|
173
|
+
"%d-byte value with U+FFFD (%d replacement char(s) in this value; %d "
|
|
174
|
+
"value(s) affected in this process so far). The source is not valid "
|
|
175
|
+
"UTF-8 and the original byte is unrecoverable. Redacted context: %s",
|
|
176
|
+
data[error.start],
|
|
177
|
+
error.start,
|
|
178
|
+
len(data),
|
|
179
|
+
decoded.count("�"),
|
|
180
|
+
total,
|
|
181
|
+
_redacted_excerpt(data, error.start),
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _tolerant_utf8_decode(data: Any, errors: str = "strict") -> tuple[str, int]:
|
|
186
|
+
"""Codec-protocol decoder that never raises on malformed UTF-8.
|
|
187
|
+
|
|
188
|
+
``errors`` is accepted for codec-protocol compatibility and deliberately
|
|
189
|
+
ignored: psycopg2 invokes the connection decoder with the bytes as the only
|
|
190
|
+
argument, so honouring the default ``"strict"`` is exactly the crash we are
|
|
191
|
+
here to prevent.
|
|
192
|
+
|
|
193
|
+
Decoding is attempted strictly first. That keeps the hot path byte-for-byte
|
|
194
|
+
the cost of the previous strict decoder, and it is what yields the offset of
|
|
195
|
+
the offending byte for :func:`_note_replacement` — the tolerant re-decode
|
|
196
|
+
only runs on the rare value that actually needs it.
|
|
197
|
+
"""
|
|
198
|
+
if isinstance(data, (memoryview, bytearray)):
|
|
199
|
+
data = bytes(data)
|
|
200
|
+
try:
|
|
201
|
+
return codecs.utf_8_decode(data, "strict", True)
|
|
202
|
+
except UnicodeDecodeError as error:
|
|
203
|
+
result = codecs.utf_8_decode(data, "replace", True)
|
|
204
|
+
_note_replacement(data, error, result[0])
|
|
205
|
+
return result # conformance: ignore[E007] _note_replacement emits the rate-limited WARNING for this exception before we return
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# Stock strict UTF-8 encoder, tolerant decoder. psycopg2 resolves *both* the
|
|
209
|
+
# encoder and the decoder from the same codec name, so the encoder must stay
|
|
210
|
+
# byte-identical to UTF-8 or we would corrupt outgoing SQL.
|
|
211
|
+
_TOLERANT_CODEC_INFO = codecs.CodecInfo(
|
|
212
|
+
name=_TOLERANT_CODEC_NAME,
|
|
213
|
+
encode=codecs.utf_8_encode,
|
|
214
|
+
decode=_tolerant_utf8_decode,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _decode_tolerant_utf8(data: Any) -> str:
|
|
219
|
+
"""Decode a bytes-like value as UTF-8, replacing invalid bytes with ``�``.
|
|
220
|
+
|
|
221
|
+
Accepts ``bytes``, ``bytearray``, or ``memoryview`` so it can be plugged
|
|
222
|
+
into either psycopg2 or psycopg3 callbacks without further conversion.
|
|
223
|
+
Shares :func:`_tolerant_utf8_decode`, so the psycopg3 path emits the same
|
|
224
|
+
rate-limited WARNING on the replacement path as the psycopg2 path does.
|
|
225
|
+
"""
|
|
226
|
+
if data is None:
|
|
227
|
+
return None # type: ignore[return-value] # mirrors driver semantics for SQL NULL
|
|
228
|
+
return _tolerant_utf8_decode(data)[0]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _ensure_tolerant_codec() -> None:
|
|
232
|
+
"""Register :data:`_TOLERANT_CODEC_NAME` with the codecs registry, once.
|
|
233
|
+
|
|
234
|
+
``codecs.register`` cannot be undone, so the search function is installed at
|
|
235
|
+
most once per process and only answers to our private codec name.
|
|
236
|
+
"""
|
|
237
|
+
try:
|
|
238
|
+
codecs.lookup(_TOLERANT_CODEC_NAME)
|
|
239
|
+
return
|
|
240
|
+
except LookupError: # conformance: ignore[E002] existence probe: LookupError *is* the "not registered yet" answer, and registering is the next statement
|
|
241
|
+
pass
|
|
242
|
+
|
|
243
|
+
def _search(name: str) -> codecs.CodecInfo | None:
|
|
244
|
+
return _TOLERANT_CODEC_INFO if name == _TOLERANT_CODEC_NAME else None
|
|
245
|
+
|
|
246
|
+
codecs.register(_search)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def install_tolerant_connection_decoder() -> bool:
|
|
250
|
+
"""Make psycopg2's *connection-level* text decode tolerant.
|
|
251
|
+
|
|
252
|
+
Why a typecaster cannot do this. psycopg2 decodes wire bytes to ``str``
|
|
253
|
+
**before** it calls any Python typecaster — unconditionally, for every
|
|
254
|
+
encoding::
|
|
255
|
+
|
|
256
|
+
/* typecast.c :: typecast_cast */
|
|
257
|
+
else if (self->pcast) {
|
|
258
|
+
s = conn_decode(((cursorObject *)curs)->conn, str, len);
|
|
259
|
+
res = PyObject_CallFunctionObjArgs(self->pcast, s, curs, NULL);
|
|
260
|
+
|
|
261
|
+
so a Python typecaster registered over the text OIDs can never see an
|
|
262
|
+
invalid byte — ``conn_decode`` has already raised ``UnicodeDecodeError``.
|
|
263
|
+
psycopg2's own source says as much: "it is about impossible to create a
|
|
264
|
+
python typecaster on a binary type."
|
|
265
|
+
|
|
266
|
+
``conn_decode`` uses the connection's ``pydecoder`` whenever no C fast codec
|
|
267
|
+
applies, and calls it with the bytes as the sole argument (i.e. strict).
|
|
268
|
+
``pydecoder`` is resolved at connect time from
|
|
269
|
+
``psycopg2.extensions.encodings``, mapping the server-reported encoding name
|
|
270
|
+
to a Python codec name — which makes that dict the only Python-reachable
|
|
271
|
+
seam. It is the very dict the C layer reads (``psycopgmodule.c`` publishes
|
|
272
|
+
its ``psycoEncodings`` as the module's ``encodings`` attribute), so it must
|
|
273
|
+
be mutated **in place**; rebinding ``ext.encodings`` would have no effect.
|
|
274
|
+
|
|
275
|
+
Redshift reports its client encoding as ``UNICODE`` (not ``UTF8``), so it
|
|
276
|
+
takes exactly this path. Note the deliberate asymmetry: a connection
|
|
277
|
+
reporting literally ``UTF8`` gets psycopg2's C fast decoder
|
|
278
|
+
(``PyUnicode_DecodeUTF8``, strict) and bypasses ``pydecoder`` entirely, so
|
|
279
|
+
remapping is a no-op there and plain PostgreSQL ≥ 8.1 stays uncovered — see
|
|
280
|
+
the module docstring.
|
|
281
|
+
|
|
282
|
+
Must run *before* connections are opened; a per-connection ``connect``
|
|
283
|
+
listener is already too late, because ``pydecoder`` is bound during connect.
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
True if at least one mapping was rewritten, False if psycopg2 is absent
|
|
287
|
+
or every UTF-8 entry was already tolerant.
|
|
288
|
+
"""
|
|
289
|
+
try:
|
|
290
|
+
import psycopg2.extensions as ext # type: ignore[import-not-found] # noqa: PLC0415
|
|
291
|
+
except ImportError: # conformance: ignore[E008] optional dep psycopg2 not installed; driver-detection probe
|
|
292
|
+
return False # conformance: ignore[E007] driver-detection probe; ImportError means optional dep absent, not an error
|
|
293
|
+
|
|
294
|
+
encodings = getattr(ext, "encodings", None)
|
|
295
|
+
if not isinstance(encodings, dict):
|
|
296
|
+
return False
|
|
297
|
+
|
|
298
|
+
_ensure_tolerant_codec()
|
|
299
|
+
|
|
300
|
+
# This rewrite is process-global and there is no per-connection alternative:
|
|
301
|
+
# ``pydecoder`` is not settable from Python, and psycopg2 resolves it from
|
|
302
|
+
# this one dict. Within a connector worker — a single-purpose process whose
|
|
303
|
+
# only psycopg2 user is the SDK — that is the intended scope, and it is
|
|
304
|
+
# strictly more available than the status quo: valid UTF-8 is unchanged, and
|
|
305
|
+
# malformed bytes surface as a logged U+FFFD instead of killing the batch.
|
|
306
|
+
rewritten = False
|
|
307
|
+
for pg_name, py_name in list(encodings.items()):
|
|
308
|
+
if py_name == _TOLERANT_CODEC_NAME:
|
|
309
|
+
continue
|
|
310
|
+
# Name prefilter before ``codecs.lookup``: psycopg2 ships 92 entries and
|
|
311
|
+
# looking each one up would import every codec in the map (EUC_JP, BIG5,
|
|
312
|
+
# KOI8, …) at install time. Only utf-8 spellings can pass, and the
|
|
313
|
+
# lookup below still decides — this narrows work, not correctness.
|
|
314
|
+
if "utf" not in py_name.lower():
|
|
315
|
+
continue
|
|
316
|
+
try:
|
|
317
|
+
if codecs.lookup(py_name).name != "utf-8":
|
|
318
|
+
continue
|
|
319
|
+
except (
|
|
320
|
+
LookupError,
|
|
321
|
+
TypeError,
|
|
322
|
+
): # conformance: ignore[E014] an unresolvable or non-string codec name is definitionally not utf-8, which is the only question asked here
|
|
323
|
+
continue
|
|
324
|
+
encodings[pg_name] = _TOLERANT_CODEC_NAME
|
|
325
|
+
rewritten = True
|
|
326
|
+
|
|
327
|
+
if rewritten:
|
|
328
|
+
logger.debug(
|
|
329
|
+
"Installed tolerant UTF-8 connection decoder for psycopg2 encodings"
|
|
330
|
+
)
|
|
331
|
+
return rewritten
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _attach_psycopg3(dbapi_connection: Any) -> bool:
|
|
335
|
+
"""Register a tolerant text Loader on a psycopg (v3) connection.
|
|
336
|
+
|
|
337
|
+
Returns True on success, False if the driver isn't available.
|
|
338
|
+
"""
|
|
339
|
+
try:
|
|
340
|
+
from psycopg.adapt import ( # type: ignore[import-not-found] # noqa: PLC0415
|
|
341
|
+
Loader,
|
|
342
|
+
)
|
|
343
|
+
except ImportError: # conformance: ignore[E008] optional dep psycopg not installed; driver-detection probe
|
|
344
|
+
return False # conformance: ignore[E007] driver-detection probe; ImportError means optional dep absent, not an error
|
|
345
|
+
|
|
346
|
+
class _TolerantTextLoader(Loader):
|
|
347
|
+
def load(self, data: Any) -> str:
|
|
348
|
+
return _decode_tolerant_utf8(data)
|
|
349
|
+
|
|
350
|
+
adapters = getattr(dbapi_connection, "adapters", None)
|
|
351
|
+
if adapters is None:
|
|
352
|
+
return False
|
|
353
|
+
for oid in _PSYCOPG3_TEXT_OIDS:
|
|
354
|
+
adapters.register_loader(oid, _TolerantTextLoader)
|
|
355
|
+
return True
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def install_tolerant_text_decoder_hook(engine: Any) -> None:
|
|
359
|
+
"""Wire :func:`attach_tolerant_text_decoder` into a SQLAlchemy engine.
|
|
360
|
+
|
|
361
|
+
Registers a ``connect`` event listener on ``engine`` (sync or
|
|
362
|
+
``AsyncEngine.sync_engine``) so every newly-checked-out DBAPI connection
|
|
363
|
+
gets a tolerant UTF-8 text decoder installed. This is the canonical
|
|
364
|
+
integration point for ``BaseSQLClient`` / ``AsyncBaseSQLClient``.
|
|
365
|
+
|
|
366
|
+
Designed to be a single, easily-monkeypatchable seam so unit tests that
|
|
367
|
+
mock ``create_engine`` to return a ``MagicMock`` aren't forced to also mock
|
|
368
|
+
SQLAlchemy's event registry.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
engine: A SQLAlchemy ``Engine`` (for sync clients) or the
|
|
372
|
+
``sync_engine`` exposed by ``AsyncEngine`` (for async clients).
|
|
373
|
+
"""
|
|
374
|
+
from sqlalchemy import event # noqa: PLC0415 — optional dep: sqlalchemy
|
|
375
|
+
from sqlalchemy.exc import ( # noqa: PLC0415 — optional dep: sqlalchemy
|
|
376
|
+
InvalidRequestError,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
# Runs here, not in the listener below: psycopg2 binds the connection's
|
|
380
|
+
# decoder while connecting, so by the time the ``connect`` event fires the
|
|
381
|
+
# strict decoder is already in place for that connection.
|
|
382
|
+
install_tolerant_connection_decoder()
|
|
383
|
+
|
|
384
|
+
def _on_connect(dbapi_connection: Any, _connection_record: Any) -> None:
|
|
385
|
+
attach_tolerant_text_decoder(dbapi_connection)
|
|
386
|
+
|
|
387
|
+
try:
|
|
388
|
+
event.listen(engine, "connect", _on_connect)
|
|
389
|
+
except InvalidRequestError:
|
|
390
|
+
# The engine target doesn't support the ``connect`` event — typically a
|
|
391
|
+
# MagicMock in unit tests. Worst case is fall-back to the driver's
|
|
392
|
+
# default strict decoder, which is the pre-WARE-970 behavior.
|
|
393
|
+
logger.debug(
|
|
394
|
+
"Skipping tolerant-decoder hook: engine %r doesn't expose 'connect' event",
|
|
395
|
+
engine,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def attach_tolerant_text_decoder(dbapi_connection: Any) -> bool:
|
|
400
|
+
"""Attach a tolerant UTF-8 text decoder to a DBAPI connection.
|
|
401
|
+
|
|
402
|
+
Only psycopg3 needs per-connection work. psycopg2 is handled once per
|
|
403
|
+
process by :func:`install_tolerant_connection_decoder`, because psycopg2
|
|
404
|
+
decodes wire bytes to ``str`` before dispatching to any Python typecaster —
|
|
405
|
+
a per-connection typecaster over the text OIDs is unreachable for malformed
|
|
406
|
+
input by construction, so this returns False for psycopg2 rather than
|
|
407
|
+
registering one.
|
|
408
|
+
|
|
409
|
+
No-ops (returning False) on any non-psycopg driver — by design, since this
|
|
410
|
+
is a psycopg-family fix.
|
|
411
|
+
|
|
412
|
+
Args:
|
|
413
|
+
dbapi_connection: The raw DBAPI connection handed to the SQLAlchemy
|
|
414
|
+
``connect`` event listener (i.e. ``dbapi_connection`` arg, not the
|
|
415
|
+
wrapping ``ConnectionRecord``).
|
|
416
|
+
|
|
417
|
+
Returns:
|
|
418
|
+
True if a decoder was attached, False otherwise.
|
|
419
|
+
"""
|
|
420
|
+
module = type(dbapi_connection).__module__ or ""
|
|
421
|
+
try:
|
|
422
|
+
if module.startswith("psycopg.") or module == "psycopg":
|
|
423
|
+
return _attach_psycopg3(dbapi_connection)
|
|
424
|
+
except Exception: # pragma: no cover — defensive: never fail load() over this
|
|
425
|
+
logger.warning(
|
|
426
|
+
"Failed to attach tolerant UTF-8 decoder to %s; falling back to driver default",
|
|
427
|
+
module,
|
|
428
|
+
exc_info=True,
|
|
429
|
+
)
|
|
430
|
+
return False
|
|
431
|
+
return False
|