atlan-application-sdk-api 3.51.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. application_sdk/__init__.py +7 -0
  2. application_sdk/_build_info.py +62 -0
  3. application_sdk/_context_errors.py +65 -0
  4. application_sdk/_install.py +130 -0
  5. application_sdk/_logging.py +171 -0
  6. application_sdk/_runtime/__init__.py +29 -0
  7. application_sdk/_runtime/offload.py +1020 -0
  8. application_sdk/_runtime/progress.py +757 -0
  9. application_sdk/clients/__init__.py +63 -0
  10. application_sdk/clients/_interface.py +58 -0
  11. application_sdk/clients/_sql_failure_classifier.py +112 -0
  12. application_sdk/clients/base.py +565 -0
  13. application_sdk/clients/http_errors.py +28 -0
  14. application_sdk/clients/models.py +49 -0
  15. application_sdk/clients/sql.py +1218 -0
  16. application_sdk/clients/sql_errors.py +192 -0
  17. application_sdk/clients/sql_typecasters.py +431 -0
  18. application_sdk/clients/ssl_utils.py +247 -0
  19. application_sdk/common/__init__.py +53 -0
  20. application_sdk/common/_generated_tree.py +321 -0
  21. application_sdk/common/_listing.py +192 -0
  22. application_sdk/common/aws_utils.py +522 -0
  23. application_sdk/common/aws_utils_errors.py +88 -0
  24. application_sdk/common/build_identity.py +120 -0
  25. application_sdk/common/concurrency.py +45 -0
  26. application_sdk/common/errors.py +99 -0
  27. application_sdk/common/filter_matching.py +261 -0
  28. application_sdk/common/logger.py +11 -0
  29. application_sdk/common/models.py +31 -0
  30. application_sdk/common/sql_filters.py +719 -0
  31. application_sdk/common/sql_filters_errors.py +17 -0
  32. application_sdk/common/task_queue.py +431 -0
  33. application_sdk/common/types.py +51 -0
  34. application_sdk/common/utils.py +87 -0
  35. application_sdk/constants.py +1120 -0
  36. application_sdk/contracts/__init__.py +158 -0
  37. application_sdk/contracts/base.py +1178 -0
  38. application_sdk/contracts/events.py +251 -0
  39. application_sdk/contracts/types.py +801 -0
  40. application_sdk/contracts/types_errors.py +17 -0
  41. application_sdk/credentials/__init__.py +326 -0
  42. application_sdk/credentials/errors.py +226 -0
  43. application_sdk/credentials/extra.py +114 -0
  44. application_sdk/credentials/ingress.py +283 -0
  45. application_sdk/credentials/ref.py +376 -0
  46. application_sdk/credentials/spec.py +182 -0
  47. application_sdk/credentials/types.py +339 -0
  48. application_sdk/credentials/utils.py +257 -0
  49. application_sdk/errors/__init__.py +224 -0
  50. application_sdk/errors/base.py +504 -0
  51. application_sdk/errors/categories.py +118 -0
  52. application_sdk/errors/http.py +81 -0
  53. application_sdk/errors/leaves.py +526 -0
  54. application_sdk/errors/wire.py +124 -0
  55. application_sdk/handler/__init__.py +97 -0
  56. application_sdk/handler/_warmup.py +198 -0
  57. application_sdk/handler/asgi.py +252 -0
  58. application_sdk/handler/base.py +281 -0
  59. application_sdk/handler/context.py +212 -0
  60. application_sdk/handler/contracts.py +1166 -0
  61. application_sdk/handler/manifest.py +81 -0
  62. application_sdk/handler/metadata_routes.py +699 -0
  63. application_sdk/handler/object_routes.py +418 -0
  64. application_sdk/handler/request_contract.py +91 -0
  65. application_sdk/handler/routes.py +1118 -0
  66. application_sdk/handler/service_errors.py +37 -0
  67. application_sdk/server/__init__.py +15 -0
  68. application_sdk/server/fastapi/models.py +338 -0
  69. application_sdk/templates/__init__.py +52 -0
  70. application_sdk/templates/contracts/__init__.py +111 -0
  71. application_sdk/templates/contracts/base_metadata_extraction.py +32 -0
  72. application_sdk/templates/contracts/incremental_sql.py +563 -0
  73. application_sdk/templates/contracts/sql_metadata.py +824 -0
  74. application_sdk/templates/contracts/sql_query.py +105 -0
  75. application_sdk/version.py +40 -0
  76. atlan_application_sdk_api-3.51.0.dist-info/METADATA +230 -0
  77. atlan_application_sdk_api-3.51.0.dist-info/RECORD +78 -0
  78. atlan_application_sdk_api-3.51.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,192 @@
1
+ """Typed error leaves for the SQL client family."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import ClassVar
7
+
8
+ from application_sdk.errors.categories import Audience
9
+ from application_sdk.errors.leaves import (
10
+ AppTimeoutError,
11
+ AuthError,
12
+ InternalError,
13
+ InvalidInputError,
14
+ SourceUnavailableError,
15
+ UnimplementedError,
16
+ )
17
+
18
+
19
+ @dataclass(kw_only=True)
20
+ class SqlClientConfigError(InternalError):
21
+ """DB_CONFIG is not set on this SQL client subclass."""
22
+
23
+ code: ClassVar[str] = "INTERNAL_SQL_DB_CONFIG_NOT_SET"
24
+ message: str = "DB_CONFIG is not configured for this SQL client"
25
+ component: str | None = "sql_client"
26
+
27
+
28
+ @dataclass(kw_only=True)
29
+ class SqlClientAuthFailedError(AuthError):
30
+ """SQL database authentication failed."""
31
+
32
+ code: ClassVar[str] = "AUTH_SQL_CLIENT_FAILED"
33
+ message: str = "SQL client authentication failed"
34
+ auth_method: str | None = "sql"
35
+
36
+
37
+ @dataclass(kw_only=True)
38
+ class SqlClientSourceUnavailableError(SourceUnavailableError):
39
+ """The SQL client could not reach the source while connecting.
40
+
41
+ DNS failure, refused or filtered connection, TLS handshake failure: the
42
+ credentials were never checked, so this is not an authentication problem
43
+ and an account unlock will not fix it.
44
+ """
45
+
46
+ code: ClassVar[str] = "SOURCE_UNAVAILABLE_SQL_CLIENT"
47
+ message: str = "SQL client could not reach the source"
48
+ suggested_action: str | None = (
49
+ "Check DNS resolution, firewall and private-link rules, TLS "
50
+ "configuration, and that the source is accepting connections from "
51
+ "the worker network."
52
+ )
53
+ source_type: str | None = "sql"
54
+
55
+
56
+ @dataclass(kw_only=True)
57
+ class SqlClientConnectTimeoutError(AppTimeoutError):
58
+ """The SQL client's connect or login ran out of time waiting on the source.
59
+
60
+ ``AppTimeoutError`` defaults to ``APP_OWNER`` because a timeout's locus is
61
+ usually ambiguous. Here it is not: what times out is the driver's or the
62
+ socket's own timeout, or ``load()``'s connect budget, spent building the
63
+ connection string (an STS AssumeRole for IAM-role auth) and connecting —
64
+ so the clock ran out waiting for the customer's source or its auth path.
65
+ """
66
+
67
+ code: ClassVar[str] = "TIMEOUT_SQL_CLIENT_CONNECT"
68
+ audience: ClassVar[Audience] = Audience.USER
69
+ message: str = "SQL client timed out connecting to the source"
70
+ suggested_action: str | None = (
71
+ "Check that the source is up and reachable from the worker network, "
72
+ "and that the connection's login timeout suits the source."
73
+ )
74
+ operation: str | None = "sql_client_connect"
75
+
76
+
77
+ @dataclass(kw_only=True)
78
+ class SqlClientConnectUnclassifiedError(InternalError):
79
+ """The SQL client failed to connect for a reason the classifier does not know.
80
+
81
+ Raised instead of guessing at authentication. ``classification_pending``
82
+ flags it so the driver error can be added to the shared SQL failure
83
+ classifier rather than read at face value.
84
+ """
85
+
86
+ code: ClassVar[str] = "INTERNAL_SQL_CLIENT_CONNECT"
87
+ message: str = "SQL client failed to connect for an unclassified reason"
88
+ component: str | None = "sql_client"
89
+ classification_pending: bool = True
90
+
91
+
92
+ @dataclass(kw_only=True)
93
+ class SqlCredentialsParseError(InvalidInputError):
94
+ """SQL credentials could not be parsed or are missing required fields."""
95
+
96
+ code: ClassVar[str] = "INVALID_INPUT_CREDENTIALS_PARSE"
97
+ message: str = "SQL credentials parse error"
98
+
99
+
100
+ @dataclass(kw_only=True)
101
+ class SqlAwsCredentialsError(AuthError):
102
+ """AWS IAM credentials required for RDS authentication are missing."""
103
+
104
+ code: ClassVar[str] = "AUTH_AWS_CREDENTIALS"
105
+ message: str = "AWS IAM credentials are missing or incomplete"
106
+ auth_method: str | None = "aws_iam"
107
+
108
+
109
+ @dataclass(kw_only=True)
110
+ class MissingSqlParamError(InvalidInputError):
111
+ """A required SQL connection parameter is missing."""
112
+
113
+ code: ClassVar[str] = "INVALID_INPUT_SQL_MISSING_PARAM"
114
+ message: str = "Required SQL connection parameter is missing"
115
+
116
+
117
+ @dataclass(kw_only=True)
118
+ class EngineNotInitializedError(InternalError):
119
+ """SQL engine is not initialized; call load() before running queries."""
120
+
121
+ code: ClassVar[str] = "INTERNAL_SQL_ENGINE_NOT_INITIALIZED"
122
+ message: str = "Engine is not initialized. Call load() first."
123
+ component: str | None = "sql_engine"
124
+
125
+
126
+ @dataclass(kw_only=True)
127
+ class SqlBindParameterError(InternalError):
128
+ """``run_query`` was given bind parameters it cannot apply.
129
+
130
+ A name that is not an identifier, or one the query never references as
131
+ ``:name``. Running anyway would execute the query without that value, so a
132
+ filter meant to narrow the result would silently not apply.
133
+ """
134
+
135
+ code: ClassVar[str] = "INTERNAL_SQL_BIND_PARAMETER"
136
+ message: str = "SQL bind parameters do not match the query"
137
+ component: str | None = "sql_client"
138
+
139
+
140
+ @dataclass(kw_only=True)
141
+ class UnsupportedSqlCursorError(UnimplementedError):
142
+ """The current SQL engine or connection does not support cursors."""
143
+
144
+ code: ClassVar[str] = "UNIMPLEMENTED_SQL_CURSOR_TYPE"
145
+ message: str = "Cursor is not supported by this SQL engine"
146
+
147
+
148
+ @dataclass(kw_only=True)
149
+ class InvalidSqlEngineTypeError(InternalError):
150
+ """The engine is a raw connection string, not a SQLAlchemy engine object."""
151
+
152
+ code: ClassVar[str] = "INTERNAL_SQL_INVALID_ENGINE_TYPE"
153
+ message: str = (
154
+ "Engine should be an SQLAlchemy engine object, not a connection string"
155
+ )
156
+ component: str | None = "sql_engine"
157
+
158
+
159
+ @dataclass(kw_only=True)
160
+ class SqlPandasResultError(InternalError):
161
+ """A SQL read operation failed to produce a valid pandas DataFrame result.
162
+
163
+ Raised in two distinct situations, which differ in retryability:
164
+
165
+ - **Invariant violation** (the read path returned a non-DataFrame). A bug in
166
+ our code; retrying cannot help. Keeps the ``INTERNAL`` default
167
+ ``retryable=False``.
168
+ - **Catch-and-rewrap of an unclassified driver exception** at the query
169
+ boundary. Here the SDK has *not* established what failed — a transient
170
+ source-side condition (concurrent DDL, lock contention, a dropped
171
+ connection) is wrapped identically to a permanent one. Those sites pass
172
+ ``retryable=True`` explicitly, which does not assert "this will succeed on
173
+ retry"; it declines to override the activity's own declared Temporal retry
174
+ policy for an error we never classified. Asserting ``retryable=False``
175
+ there marks the failure ``non_retryable`` on the wire and makes the
176
+ activity's declared ``maximum_attempts`` dead code (CONAT-747).
177
+
178
+ Errors that *are* classified never reach either site — the enclosing
179
+ ``except AppError: raise`` clauses re-raise them with their own retryability.
180
+ """
181
+
182
+ code: ClassVar[str] = "INTERNAL_SQL_PANDAS"
183
+ message: str = "Error reading data from SQL into a pandas DataFrame"
184
+ component: str | None = "sql_pandas"
185
+
186
+
187
+ @dataclass(kw_only=True)
188
+ class AbstractClientLoadError(UnimplementedError):
189
+ """load() called on a ClientInterface subclass that has not implemented it."""
190
+
191
+ code: ClassVar[str] = "UNIMPLEMENTED_CLIENT_LOAD"
192
+ message: str = "load method is not implemented"
@@ -0,0 +1,431 @@
1
+ """Tolerant UTF-8 text decoders for SQL DBAPI cursors.
2
+
3
+ Long-lived warehouses accumulate query text from heterogeneous client encodings
4
+ (SSMS, Excel, Word paste, terminals, etc.). When the SDK pulls that text via a
5
+ strict UTF-8 cursor decoder, a single non-UTF-8 byte (commonly ``0x96`` —
6
+ Windows-1252 en-dash) raises ``UnicodeDecodeError`` and aborts the entire
7
+ batch, breaking query-history miners.
8
+
9
+ The intended semantics have not changed since WARE-970: decode UTF-8, and where
10
+ the bytes are not valid UTF-8 under any reading, emit ``U+FFFD`` rather than
11
+ kill the batch. What CONAT-767 changed is only the *injection point* — the
12
+ tolerance was registered at a layer psycopg2 never reaches (see
13
+ :func:`install_tolerant_connection_decoder`), so production kept crashing while
14
+ the code read as fixed. Nothing here infers or guesses an encoding: the wire
15
+ encoding is declared by the connection, and ``errors="replace"`` is a total
16
+ function over that already-fixed decoding. No correctly-encoded byte can be
17
+ dropped.
18
+
19
+ Scope (psycopg2)
20
+ ----------------
21
+ The psycopg2 fix covers connections whose server reports its client encoding as
22
+ ``UNICODE`` — Redshift (a PostgreSQL 8.0.2 fork, and PG 8.0 spelled UTF-8
23
+ ``UNICODE``) and other pre-8.1-era forks. Connections that report literally
24
+ ``UTF8`` — which is every PostgreSQL from 8.1 onward — take psycopg2's C fast
25
+ decoder (``PyUnicode_DecodeUTF8``, strict), which no Python-level hook can
26
+ intercept. **The original crash class therefore remains open for plain
27
+ PostgreSQL query-history miners on psycopg2.** Closing it would require owning
28
+ ``client_encoding``, which is a separate decision (see below). psycopg3 is
29
+ covered on all servers via its loader API.
30
+
31
+ Recorded decisions
32
+ ------------------
33
+ ``errors="replace"``, not ``surrogateescape``. ``surrogateescape`` would
34
+ preserve the original byte and make the loss reversible, which is genuinely
35
+ attractive. It is rejected because decoded text flows on to parquet and JSON,
36
+ where lone surrogates are not serializable — a deferred hard failure in the
37
+ writer is worse than a visible ``U+FFFD`` in the data. ``replace`` is lossy on
38
+ purpose, and the log signal below is what makes the loss visible.
39
+
40
+ Not ``client_encoding=latin1``. That stops the crash but mojibakes valid UTF-8
41
+ (``0xC3 0xA9`` → ``é`` instead of ``é``) — a strictly worse fidelity trade.
42
+
43
+ Not ``client_encoding=SQL_ASCII`` either, though it is the strongest
44
+ alternative and deserves an explicit rejection. Under ``SQL_ASCII`` psycopg2
45
+ maps to the ``ascii`` codec, so per-value decoding could move into a typecaster:
46
+ that would give per-value control, natural logging, and would close the ``UTF8``
47
+ gap above uniformly. The cost is that the SDK would then own *outgoing* encoding
48
+ for every non-ASCII identifier and parameter on every connector that inherits
49
+ this, on connections it did not configure — a much larger blast radius than the
50
+ defect being fixed, and one that fails in the write direction rather than the
51
+ read direction. Rejected on that basis, not on merit.
52
+
53
+ Observability
54
+ -------------
55
+ The replacement path logs a rate-limited WARNING with the offending byte, its
56
+ offset, and a redacted excerpt (identifiers and literals masked — no query text
57
+ reaches the logs). One stray byte in a large query and a column that was
58
+ Windows-1252 all along are both silent in the data; the log line is what
59
+ distinguishes them, and what makes "did this run take the tolerant path?"
60
+ answerable after the fact instead of inferred.
61
+
62
+ Tracking: CONAT-767 (injection point), WARE-970 (original intent).
63
+
64
+ References
65
+ ----------
66
+ - psycopg2 typecasters: https://www.psycopg.org/docs/extensions.html#psycopg2.extensions.new_type
67
+ - psycopg3 loaders: https://www.psycopg.org/psycopg3/docs/advanced/adapt.html
68
+ """
69
+
70
+ from __future__ import annotations
71
+
72
+ import codecs
73
+ import threading
74
+ from typing import Any
75
+
76
+ from application_sdk._logging import get_logger
77
+
78
+ logger = get_logger(__name__)
79
+
80
+ # Name of the private codec registered by ``_ensure_tolerant_codec``. It is an
81
+ # ordinary UTF-8 codec whose *decoder* replaces invalid bytes instead of
82
+ # raising; the encoder is stock strict UTF-8, so outgoing SQL is unchanged.
83
+ _TOLERANT_CODEC_NAME = "atlan_tolerant_utf8"
84
+
85
+ # Bytes shown either side of the offending byte in the redacted log excerpt.
86
+ _EXCERPT_WINDOW = 8
87
+
88
+ # Cumulative count of *values* that needed replacement in this process, and the
89
+ # lock guarding it. Only touched on the (rare, by construction) failure path.
90
+ _replacement_total = 0
91
+ _replacement_lock = threading.Lock()
92
+
93
+
94
+ # psycopg3 text-ish PostgreSQL OIDs we want to override. These match the OIDs
95
+ # psycopg3 itself registers ``TextLoader``/``TextBinaryLoader`` for: text(25),
96
+ # varchar(1043), bpchar(1042), name(19), char(18), unknown(705).
97
+ # Sourced from ``psycopg.postgres.types`` for the canonical names.
98
+ #
99
+ # We also register for OID 0 (``InvalidOid``). Some Postgres-compatible proxies
100
+ # — notably PgBouncer in transaction-pooling mode, Azure Flex Server's built-in
101
+ # pooler, and AWS RDS Proxy — can return OID 0 on prepared-statement metadata
102
+ # (e.g. ``pg_catalog.version()``), which psycopg3 otherwise hands to the caller
103
+ # as raw bytes. SQLAlchemy's Postgres dialect then runs a string regex against
104
+ # those bytes in ``_get_server_version_info`` and raises ``TypeError: cannot
105
+ # use a string pattern on a bytes-like object`` before any query can run.
106
+ _PSYCOPG3_TEXT_OIDS: tuple[int, ...] = (0, 18, 19, 25, 705, 1042, 1043)
107
+
108
+
109
+ def _redacted_excerpt(data: bytes, offset: int) -> str:
110
+ """Render the bytes around ``offset`` without leaking query text.
111
+
112
+ ASCII letters collapse to ``a``/``A`` and digits to ``0``, so identifiers,
113
+ literals and any customer-identifying value are destroyed while the *shape*
114
+ survives. Punctuation is structural and kept. Only the bracketed offending
115
+ byte is rendered as its exact ``\\xNN`` value — it is already unrecoverable
116
+ from the decoded text, so revealing it leaks nothing.
117
+
118
+ Every *other* non-ASCII byte renders as the fixed token ``\\x??``: a valid
119
+ UTF-8 multi-byte sequence next to the bad byte is customer content (``é`` in
120
+ a name), so its value must not reach the log — but the *count* of high bytes
121
+ is kept, because that is the tell for a wholesale Windows-1252 column: one
122
+ lone ``[\\x96]`` in masked ASCII is a stray byte, whereas a run of ``\\x??``
123
+ neighbours means the column was never UTF-8.
124
+ """
125
+ start = max(0, offset - _EXCERPT_WINDOW)
126
+ end = min(len(data), offset + _EXCERPT_WINDOW + 1)
127
+
128
+ rendered: list[str] = []
129
+ for index in range(start, end):
130
+ byte = data[index]
131
+ if index == offset:
132
+ rendered.append(f"[\\x{byte:02x}]")
133
+ elif 0x41 <= byte <= 0x5A: # A-Z
134
+ rendered.append("A")
135
+ elif 0x61 <= byte <= 0x7A: # a-z
136
+ rendered.append("a")
137
+ elif 0x30 <= byte <= 0x39: # 0-9
138
+ rendered.append("0")
139
+ elif byte in (0x09, 0x0A, 0x0D, 0x20): # whitespace
140
+ rendered.append(" ")
141
+ elif byte < 0x80: # ASCII punctuation — structural, not content
142
+ rendered.append(chr(byte))
143
+ else: # non-ASCII neighbour — value is content; only the count matters
144
+ rendered.append("\\x??")
145
+
146
+ prefix = "…" if start > 0 else ""
147
+ suffix = "…" if end < len(data) else ""
148
+ return f"{prefix}{''.join(rendered)}{suffix}"
149
+
150
+
151
+ def _note_replacement(data: bytes, error: UnicodeDecodeError, decoded: str) -> None:
152
+ """Emit a rate-limited WARNING that the tolerant decoder dropped a byte.
153
+
154
+ Rate limiting is per *process* on powers of two (1st, 2nd, 4th, 8th, …
155
+ value): loud on first occurrence, so a single stray byte is never silent,
156
+ and log-bounded when an entire column turns out to be Windows-1252. Per
157
+ *connection* would be the nicer granularity but is not reachable here —
158
+ psycopg2 resolves the codec by name from a module-level dict and calls the
159
+ decoder with the bytes as the sole argument, so no connection identity
160
+ reaches this frame.
161
+ """
162
+ global _replacement_total
163
+
164
+ with _replacement_lock:
165
+ _replacement_total += 1
166
+ total = _replacement_total
167
+
168
+ if total & (total - 1): # not a power of two — stay quiet
169
+ return
170
+
171
+ logger.warning(
172
+ "Tolerant UTF-8 decoder replaced invalid byte 0x%02x at offset %d of a "
173
+ "%d-byte value with U+FFFD (%d replacement char(s) in this value; %d "
174
+ "value(s) affected in this process so far). The source is not valid "
175
+ "UTF-8 and the original byte is unrecoverable. Redacted context: %s",
176
+ data[error.start],
177
+ error.start,
178
+ len(data),
179
+ decoded.count("�"),
180
+ total,
181
+ _redacted_excerpt(data, error.start),
182
+ )
183
+
184
+
185
+ def _tolerant_utf8_decode(data: Any, errors: str = "strict") -> tuple[str, int]:
186
+ """Codec-protocol decoder that never raises on malformed UTF-8.
187
+
188
+ ``errors`` is accepted for codec-protocol compatibility and deliberately
189
+ ignored: psycopg2 invokes the connection decoder with the bytes as the only
190
+ argument, so honouring the default ``"strict"`` is exactly the crash we are
191
+ here to prevent.
192
+
193
+ Decoding is attempted strictly first. That keeps the hot path byte-for-byte
194
+ the cost of the previous strict decoder, and it is what yields the offset of
195
+ the offending byte for :func:`_note_replacement` — the tolerant re-decode
196
+ only runs on the rare value that actually needs it.
197
+ """
198
+ if isinstance(data, (memoryview, bytearray)):
199
+ data = bytes(data)
200
+ try:
201
+ return codecs.utf_8_decode(data, "strict", True)
202
+ except UnicodeDecodeError as error:
203
+ result = codecs.utf_8_decode(data, "replace", True)
204
+ _note_replacement(data, error, result[0])
205
+ return result # conformance: ignore[E007] _note_replacement emits the rate-limited WARNING for this exception before we return
206
+
207
+
208
+ # Stock strict UTF-8 encoder, tolerant decoder. psycopg2 resolves *both* the
209
+ # encoder and the decoder from the same codec name, so the encoder must stay
210
+ # byte-identical to UTF-8 or we would corrupt outgoing SQL.
211
+ _TOLERANT_CODEC_INFO = codecs.CodecInfo(
212
+ name=_TOLERANT_CODEC_NAME,
213
+ encode=codecs.utf_8_encode,
214
+ decode=_tolerant_utf8_decode,
215
+ )
216
+
217
+
218
+ def _decode_tolerant_utf8(data: Any) -> str:
219
+ """Decode a bytes-like value as UTF-8, replacing invalid bytes with ``�``.
220
+
221
+ Accepts ``bytes``, ``bytearray``, or ``memoryview`` so it can be plugged
222
+ into either psycopg2 or psycopg3 callbacks without further conversion.
223
+ Shares :func:`_tolerant_utf8_decode`, so the psycopg3 path emits the same
224
+ rate-limited WARNING on the replacement path as the psycopg2 path does.
225
+ """
226
+ if data is None:
227
+ return None # type: ignore[return-value] # mirrors driver semantics for SQL NULL
228
+ return _tolerant_utf8_decode(data)[0]
229
+
230
+
231
+ def _ensure_tolerant_codec() -> None:
232
+ """Register :data:`_TOLERANT_CODEC_NAME` with the codecs registry, once.
233
+
234
+ ``codecs.register`` cannot be undone, so the search function is installed at
235
+ most once per process and only answers to our private codec name.
236
+ """
237
+ try:
238
+ codecs.lookup(_TOLERANT_CODEC_NAME)
239
+ return
240
+ except LookupError: # conformance: ignore[E002] existence probe: LookupError *is* the "not registered yet" answer, and registering is the next statement
241
+ pass
242
+
243
+ def _search(name: str) -> codecs.CodecInfo | None:
244
+ return _TOLERANT_CODEC_INFO if name == _TOLERANT_CODEC_NAME else None
245
+
246
+ codecs.register(_search)
247
+
248
+
249
+ def install_tolerant_connection_decoder() -> bool:
250
+ """Make psycopg2's *connection-level* text decode tolerant.
251
+
252
+ Why a typecaster cannot do this. psycopg2 decodes wire bytes to ``str``
253
+ **before** it calls any Python typecaster — unconditionally, for every
254
+ encoding::
255
+
256
+ /* typecast.c :: typecast_cast */
257
+ else if (self->pcast) {
258
+ s = conn_decode(((cursorObject *)curs)->conn, str, len);
259
+ res = PyObject_CallFunctionObjArgs(self->pcast, s, curs, NULL);
260
+
261
+ so a Python typecaster registered over the text OIDs can never see an
262
+ invalid byte — ``conn_decode`` has already raised ``UnicodeDecodeError``.
263
+ psycopg2's own source says as much: "it is about impossible to create a
264
+ python typecaster on a binary type."
265
+
266
+ ``conn_decode`` uses the connection's ``pydecoder`` whenever no C fast codec
267
+ applies, and calls it with the bytes as the sole argument (i.e. strict).
268
+ ``pydecoder`` is resolved at connect time from
269
+ ``psycopg2.extensions.encodings``, mapping the server-reported encoding name
270
+ to a Python codec name — which makes that dict the only Python-reachable
271
+ seam. It is the very dict the C layer reads (``psycopgmodule.c`` publishes
272
+ its ``psycoEncodings`` as the module's ``encodings`` attribute), so it must
273
+ be mutated **in place**; rebinding ``ext.encodings`` would have no effect.
274
+
275
+ Redshift reports its client encoding as ``UNICODE`` (not ``UTF8``), so it
276
+ takes exactly this path. Note the deliberate asymmetry: a connection
277
+ reporting literally ``UTF8`` gets psycopg2's C fast decoder
278
+ (``PyUnicode_DecodeUTF8``, strict) and bypasses ``pydecoder`` entirely, so
279
+ remapping is a no-op there and plain PostgreSQL ≥ 8.1 stays uncovered — see
280
+ the module docstring.
281
+
282
+ Must run *before* connections are opened; a per-connection ``connect``
283
+ listener is already too late, because ``pydecoder`` is bound during connect.
284
+
285
+ Returns:
286
+ True if at least one mapping was rewritten, False if psycopg2 is absent
287
+ or every UTF-8 entry was already tolerant.
288
+ """
289
+ try:
290
+ import psycopg2.extensions as ext # type: ignore[import-not-found] # noqa: PLC0415
291
+ except ImportError: # conformance: ignore[E008] optional dep psycopg2 not installed; driver-detection probe
292
+ return False # conformance: ignore[E007] driver-detection probe; ImportError means optional dep absent, not an error
293
+
294
+ encodings = getattr(ext, "encodings", None)
295
+ if not isinstance(encodings, dict):
296
+ return False
297
+
298
+ _ensure_tolerant_codec()
299
+
300
+ # This rewrite is process-global and there is no per-connection alternative:
301
+ # ``pydecoder`` is not settable from Python, and psycopg2 resolves it from
302
+ # this one dict. Within a connector worker — a single-purpose process whose
303
+ # only psycopg2 user is the SDK — that is the intended scope, and it is
304
+ # strictly more available than the status quo: valid UTF-8 is unchanged, and
305
+ # malformed bytes surface as a logged U+FFFD instead of killing the batch.
306
+ rewritten = False
307
+ for pg_name, py_name in list(encodings.items()):
308
+ if py_name == _TOLERANT_CODEC_NAME:
309
+ continue
310
+ # Name prefilter before ``codecs.lookup``: psycopg2 ships 92 entries and
311
+ # looking each one up would import every codec in the map (EUC_JP, BIG5,
312
+ # KOI8, …) at install time. Only utf-8 spellings can pass, and the
313
+ # lookup below still decides — this narrows work, not correctness.
314
+ if "utf" not in py_name.lower():
315
+ continue
316
+ try:
317
+ if codecs.lookup(py_name).name != "utf-8":
318
+ continue
319
+ except (
320
+ LookupError,
321
+ TypeError,
322
+ ): # conformance: ignore[E014] an unresolvable or non-string codec name is definitionally not utf-8, which is the only question asked here
323
+ continue
324
+ encodings[pg_name] = _TOLERANT_CODEC_NAME
325
+ rewritten = True
326
+
327
+ if rewritten:
328
+ logger.debug(
329
+ "Installed tolerant UTF-8 connection decoder for psycopg2 encodings"
330
+ )
331
+ return rewritten
332
+
333
+
334
+ def _attach_psycopg3(dbapi_connection: Any) -> bool:
335
+ """Register a tolerant text Loader on a psycopg (v3) connection.
336
+
337
+ Returns True on success, False if the driver isn't available.
338
+ """
339
+ try:
340
+ from psycopg.adapt import ( # type: ignore[import-not-found] # noqa: PLC0415
341
+ Loader,
342
+ )
343
+ except ImportError: # conformance: ignore[E008] optional dep psycopg not installed; driver-detection probe
344
+ return False # conformance: ignore[E007] driver-detection probe; ImportError means optional dep absent, not an error
345
+
346
+ class _TolerantTextLoader(Loader):
347
+ def load(self, data: Any) -> str:
348
+ return _decode_tolerant_utf8(data)
349
+
350
+ adapters = getattr(dbapi_connection, "adapters", None)
351
+ if adapters is None:
352
+ return False
353
+ for oid in _PSYCOPG3_TEXT_OIDS:
354
+ adapters.register_loader(oid, _TolerantTextLoader)
355
+ return True
356
+
357
+
358
+ def install_tolerant_text_decoder_hook(engine: Any) -> None:
359
+ """Wire :func:`attach_tolerant_text_decoder` into a SQLAlchemy engine.
360
+
361
+ Registers a ``connect`` event listener on ``engine`` (sync or
362
+ ``AsyncEngine.sync_engine``) so every newly-checked-out DBAPI connection
363
+ gets a tolerant UTF-8 text decoder installed. This is the canonical
364
+ integration point for ``BaseSQLClient`` / ``AsyncBaseSQLClient``.
365
+
366
+ Designed to be a single, easily-monkeypatchable seam so unit tests that
367
+ mock ``create_engine`` to return a ``MagicMock`` aren't forced to also mock
368
+ SQLAlchemy's event registry.
369
+
370
+ Args:
371
+ engine: A SQLAlchemy ``Engine`` (for sync clients) or the
372
+ ``sync_engine`` exposed by ``AsyncEngine`` (for async clients).
373
+ """
374
+ from sqlalchemy import event # noqa: PLC0415 — optional dep: sqlalchemy
375
+ from sqlalchemy.exc import ( # noqa: PLC0415 — optional dep: sqlalchemy
376
+ InvalidRequestError,
377
+ )
378
+
379
+ # Runs here, not in the listener below: psycopg2 binds the connection's
380
+ # decoder while connecting, so by the time the ``connect`` event fires the
381
+ # strict decoder is already in place for that connection.
382
+ install_tolerant_connection_decoder()
383
+
384
+ def _on_connect(dbapi_connection: Any, _connection_record: Any) -> None:
385
+ attach_tolerant_text_decoder(dbapi_connection)
386
+
387
+ try:
388
+ event.listen(engine, "connect", _on_connect)
389
+ except InvalidRequestError:
390
+ # The engine target doesn't support the ``connect`` event — typically a
391
+ # MagicMock in unit tests. Worst case is fall-back to the driver's
392
+ # default strict decoder, which is the pre-WARE-970 behavior.
393
+ logger.debug(
394
+ "Skipping tolerant-decoder hook: engine %r doesn't expose 'connect' event",
395
+ engine,
396
+ )
397
+
398
+
399
+ def attach_tolerant_text_decoder(dbapi_connection: Any) -> bool:
400
+ """Attach a tolerant UTF-8 text decoder to a DBAPI connection.
401
+
402
+ Only psycopg3 needs per-connection work. psycopg2 is handled once per
403
+ process by :func:`install_tolerant_connection_decoder`, because psycopg2
404
+ decodes wire bytes to ``str`` before dispatching to any Python typecaster —
405
+ a per-connection typecaster over the text OIDs is unreachable for malformed
406
+ input by construction, so this returns False for psycopg2 rather than
407
+ registering one.
408
+
409
+ No-ops (returning False) on any non-psycopg driver — by design, since this
410
+ is a psycopg-family fix.
411
+
412
+ Args:
413
+ dbapi_connection: The raw DBAPI connection handed to the SQLAlchemy
414
+ ``connect`` event listener (i.e. ``dbapi_connection`` arg, not the
415
+ wrapping ``ConnectionRecord``).
416
+
417
+ Returns:
418
+ True if a decoder was attached, False otherwise.
419
+ """
420
+ module = type(dbapi_connection).__module__ or ""
421
+ try:
422
+ if module.startswith("psycopg.") or module == "psycopg":
423
+ return _attach_psycopg3(dbapi_connection)
424
+ except Exception: # pragma: no cover — defensive: never fail load() over this
425
+ logger.warning(
426
+ "Failed to attach tolerant UTF-8 decoder to %s; falling back to driver default",
427
+ module,
428
+ exc_info=True,
429
+ )
430
+ return False
431
+ return False