structured-data-models-client 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm_client/__init__.py +70 -0
- sdm_client/_version.py +5 -0
- sdm_client/adapters/__init__.py +9 -0
- sdm_client/adapters/relational.py +870 -0
- sdm_client/adapters/tabular.py +580 -0
- sdm_client/base.py +300 -0
- sdm_client/client.py +1416 -0
- sdm_client/core/__init__.py +5 -0
- sdm_client/core/databricks_serving.py +611 -0
- sdm_client/core/databricks_user_agent.py +76 -0
- sdm_client/core/dates.py +278 -0
- sdm_client/core/transport.py +898 -0
- sdm_client/core/warnings.py +51 -0
- sdm_client/decompose.py +421 -0
- sdm_client/errors.py +177 -0
- sdm_client/materializers/__init__.py +10 -0
- sdm_client/materializers/relational.py +1786 -0
- sdm_client/materializers/sources.py +331 -0
- sdm_client/materializers/tabular.py +339 -0
- sdm_client/models.py +251 -0
- sdm_client/py.typed +0 -0
- sdm_client/query.py +647 -0
- sdm_client/request.py +1127 -0
- sdm_client/result.py +328 -0
- sdm_client/schema.py +3187 -0
- sdm_client/wire/__init__.py +46 -0
- sdm_client/wire/answers.py +80 -0
- sdm_client/wire/dtypes.py +701 -0
- sdm_client/wire/keys.py +363 -0
- sdm_client/wire/tables.py +50 -0
- structured_data_models_client-1.0.0.dist-info/METADATA +99 -0
- structured_data_models_client-1.0.0.dist-info/RECORD +36 -0
- structured_data_models_client-1.0.0.dist-info/WHEEL +5 -0
- structured_data_models_client-1.0.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models_client-1.0.0.dist-info/licenses/NOTICE +10 -0
- structured_data_models_client-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,701 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
|
|
2
|
+
# All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import math
|
|
8
|
+
from collections.abc import Iterable, Mapping
|
|
9
|
+
from decimal import Decimal
|
|
10
|
+
from numbers import Integral
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import pandas as pd
|
|
15
|
+
|
|
16
|
+
from sdm_client.errors import StructuredDataError
|
|
17
|
+
|
|
18
|
+
JSON_SAFE_INT_MAX = 9007199254740991
|
|
19
|
+
JSON_SAFE_INT_MIN = -9007199254740991
|
|
20
|
+
|
|
21
|
+
# The range the wire's `int64` represents. Beyond it a value cannot be sent
|
|
22
|
+
# under that dtype whatever its Python type, so the declaration and the cells
|
|
23
|
+
# would disagree.
|
|
24
|
+
INT64_MAX = 2**63 - 1
|
|
25
|
+
INT64_MIN = -(2**63)
|
|
26
|
+
|
|
27
|
+
# Ordered from narrowest to widest; anything outside this map (timestamps,
|
|
28
|
+
# strings) has no numeric widening and falls back to 'string'.
|
|
29
|
+
_NUMERIC_WIDTH = {
|
|
30
|
+
'bool': 0,
|
|
31
|
+
'int32': 1,
|
|
32
|
+
'int64': 2,
|
|
33
|
+
'float32': 3,
|
|
34
|
+
'float64': 4,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# `infer_tfm_dtype`'s answer for a column that carries no type information at
|
|
38
|
+
# all, so that it contributes nothing to a widening. Distinct from every real
|
|
39
|
+
# wire dtype, and never emitted into a payload.
|
|
40
|
+
UNTYPED = ''
|
|
41
|
+
|
|
42
|
+
# What `pandas.api.types.infer_dtype` reports for the contents of an `object`
|
|
43
|
+
# column, mapped to the wire dtype that represents those values. `object` is a
|
|
44
|
+
# container, not a type: classifying it by the container alone declares numbers
|
|
45
|
+
# as strings, which changes what the model is asked to predict on.
|
|
46
|
+
_OBJECT_CONTENT_DTYPES = {
|
|
47
|
+
'integer': 'int64',
|
|
48
|
+
'floating': 'float64',
|
|
49
|
+
'mixed-integer-float': 'float64',
|
|
50
|
+
'boolean': 'bool',
|
|
51
|
+
'date': 'timestamp[us]',
|
|
52
|
+
'datetime': 'timestamp[us]',
|
|
53
|
+
'datetime64': 'timestamp[us]',
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _holds_only_scalars(dtype: Any) -> bool:
|
|
58
|
+
r"""Whether a column of ``dtype`` cannot contain a sequence.
|
|
59
|
+
|
|
60
|
+
Asked of the dtype rather than by elimination: ``is_object_dtype`` alone
|
|
61
|
+
would call an Arrow-backed list column scalar, and that is what
|
|
62
|
+
``pd.read_parquet(dtype_backend='pyarrow')`` gives for one.
|
|
63
|
+
"""
|
|
64
|
+
arrow_type = getattr(dtype, 'pyarrow_dtype', None)
|
|
65
|
+
if arrow_type is not None:
|
|
66
|
+
return _arrow_holds_only_scalars(arrow_type)
|
|
67
|
+
return bool(
|
|
68
|
+
pd.api.types.is_numeric_dtype(dtype)
|
|
69
|
+
or pd.api.types.is_bool_dtype(dtype)
|
|
70
|
+
or pd.api.types.is_datetime64_any_dtype(dtype)
|
|
71
|
+
or pd.api.types.is_timedelta64_dtype(dtype)
|
|
72
|
+
or isinstance(dtype, pd.StringDtype)
|
|
73
|
+
or dtype == 'str'
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _arrow_holds_only_scalars(arrow_type: Any) -> bool:
|
|
78
|
+
r"""The same question of an Arrow type, through its wrappers.
|
|
79
|
+
|
|
80
|
+
Answered by naming what is *safe to skip* rather than what must be walked.
|
|
81
|
+
Listing the unsafe types is how a list view, then Arrow binary, then a
|
|
82
|
+
binary view each got through in turn. Every new Arrow type defaulted to
|
|
83
|
+
the fast path, and the fast path is the one that can be wrong. This way an
|
|
84
|
+
unfamiliar type is walked: slower, and correct.
|
|
85
|
+
|
|
86
|
+
Types that wrap another, including dictionaries, run-end encodings, and
|
|
87
|
+
extensions, are unwrapped first because a dictionary of lists is still a
|
|
88
|
+
column of lists.
|
|
89
|
+
"""
|
|
90
|
+
import pyarrow as pa
|
|
91
|
+
|
|
92
|
+
while True:
|
|
93
|
+
if isinstance(arrow_type, pa.BaseExtensionType):
|
|
94
|
+
arrow_type = arrow_type.storage_type
|
|
95
|
+
continue
|
|
96
|
+
if pa.types.is_dictionary(arrow_type) or pa.types.is_run_end_encoded(
|
|
97
|
+
arrow_type
|
|
98
|
+
):
|
|
99
|
+
arrow_type = arrow_type.value_type
|
|
100
|
+
continue
|
|
101
|
+
break
|
|
102
|
+
|
|
103
|
+
# Each of these gives pandas a scalar cell: a number, a `str`, a
|
|
104
|
+
# `Timestamp`, a `timedelta`, `None`. Two are deliberately absent. Binary,
|
|
105
|
+
# because pandas hands those back as `bytes`. And `month_day_nano_interval`,
|
|
106
|
+
# because its cell is a `MonthDayNano` (a NamedTuple and thus a `tuple`),
|
|
107
|
+
# which the cell rule refuses, correctly: it has no scalar form on the wire.
|
|
108
|
+
checks = (
|
|
109
|
+
'is_integer',
|
|
110
|
+
'is_floating',
|
|
111
|
+
'is_decimal',
|
|
112
|
+
'is_boolean',
|
|
113
|
+
'is_string',
|
|
114
|
+
'is_large_string',
|
|
115
|
+
'is_string_view',
|
|
116
|
+
'is_date',
|
|
117
|
+
'is_time',
|
|
118
|
+
'is_timestamp',
|
|
119
|
+
'is_duration',
|
|
120
|
+
'is_null',
|
|
121
|
+
)
|
|
122
|
+
return any(
|
|
123
|
+
getattr(pa.types, name)(arrow_type)
|
|
124
|
+
for name in checks
|
|
125
|
+
if hasattr(pa.types, name)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def require_scalar_cells(series: pd.Series, described: str) -> None:
|
|
130
|
+
r"""Refuse a column whose cells are lists or mappings.
|
|
131
|
+
|
|
132
|
+
The contract carries a `stringlist` dtype, and says what it is for: array
|
|
133
|
+
cells exist "so that the stringlist dtype can carry JSON arrays **without
|
|
134
|
+
permitting array-valued prediction scalars**": temporal link prediction
|
|
135
|
+
labels, which no model reachable from here serves.
|
|
136
|
+
|
|
137
|
+
Nothing in this client sends one. `pandas.api.types.infer_dtype` reports a
|
|
138
|
+
list column as `mixed`, which falls through to `string`. `serialize_cell`
|
|
139
|
+
ends in `str(value)`. A column of lists used to travel as the *repr* of
|
|
140
|
+
a Python list, `"['a', 'b']"`, as an ordinary category. That is silent, and
|
|
141
|
+
the model is asked about text that was never in the data.
|
|
142
|
+
|
|
143
|
+
Raised here rather than only where a schema is built, because a value can
|
|
144
|
+
reach the wire without one: the rows being scored are not part of the
|
|
145
|
+
schema, and a caller may supply a schema of their own.
|
|
146
|
+
|
|
147
|
+
A warehouse `binary` column arrives as one `bytes` per row. It is refused
|
|
148
|
+
for having no cell form on the wire, which is not the mistake a nested
|
|
149
|
+
column is, so it is told apart.
|
|
150
|
+
"""
|
|
151
|
+
# Skipped only for a dtype that cannot hold a sequence, which is most of
|
|
152
|
+
# them on the hot path for every table in every request. Asked the other way
|
|
153
|
+
# round on purpose: `is_object_dtype` alone would let an Arrow-backed list
|
|
154
|
+
# column through, and that is what `pd.read_parquet(dtype_backend='pyarrow')`
|
|
155
|
+
# gives for one.
|
|
156
|
+
if _holds_only_scalars(series.dtype):
|
|
157
|
+
return
|
|
158
|
+
for value in series:
|
|
159
|
+
if not is_container(value):
|
|
160
|
+
continue
|
|
161
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
162
|
+
raise StructuredDataError(
|
|
163
|
+
f'{described} holds raw bytes ({type(value).__name__}), and '
|
|
164
|
+
f'the contract has no cell that carries them. It is already '
|
|
165
|
+
f'one value per row, so what it needs is a reading rather '
|
|
166
|
+
f'than a flattening: decode the column to text, or leave it '
|
|
167
|
+
f'out of the schema',
|
|
168
|
+
code='UNSUPPORTED_REQUEST',
|
|
169
|
+
)
|
|
170
|
+
shape = 'a mapping' if isinstance(value, Mapping) else 'a container'
|
|
171
|
+
raise StructuredDataError(
|
|
172
|
+
f'{described} holds {shape} ({type(value).__name__}), and a '
|
|
173
|
+
f'nested column is not something this client can send. '
|
|
174
|
+
f'Multi-label targets are not supported by these models; for a '
|
|
175
|
+
f'feature, give one value per row',
|
|
176
|
+
code='UNSUPPORTED_REQUEST',
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def is_container(value: Any) -> bool:
|
|
181
|
+
r"""Whether ``value`` holds other values rather than being one.
|
|
182
|
+
|
|
183
|
+
Asked as a property, not as a list of types. Enumerating them is how a
|
|
184
|
+
tuple, then a mapping, then a `range`, then a nested Series each turned out
|
|
185
|
+
to be missing. Every one of them would have been written by
|
|
186
|
+
:func:`serialize_cell`'s `str()` fallback as its *repr*: `"[1, 2]"`,
|
|
187
|
+
`"{'a': 1}"`, `"range(0, 2)"`.
|
|
188
|
+
|
|
189
|
+
A `str` is iterable and is exactly what a text column holds, so it is the
|
|
190
|
+
one exception. Legitimate scalar values, including numbers, `Decimal`,
|
|
191
|
+
`UUID`, and timestamps, are not iterable.
|
|
192
|
+
"""
|
|
193
|
+
if isinstance(value, str):
|
|
194
|
+
return False
|
|
195
|
+
return isinstance(value, (Iterable, Mapping, bytes, memoryview))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def infer_tfm_dtype(series: pd.Series) -> str:
|
|
199
|
+
r"""The wire dtype that represents ``series``' values.
|
|
200
|
+
|
|
201
|
+
An ``object`` column is classified by what it holds, not by the fact that
|
|
202
|
+
it is ``object``: ``pd.read_sql`` over an all-null integer column, a frame
|
|
203
|
+
built from records, and a column that has been through ``fillna`` are all
|
|
204
|
+
``object`` while holding perfectly ordinary numbers.
|
|
205
|
+
|
|
206
|
+
An ``object`` column with no non-null values is reported as
|
|
207
|
+
:data:`UNTYPED` rather than guessed at, so that a frame which happens to be
|
|
208
|
+
missing a feature entirely cannot re-type the same column in another frame.
|
|
209
|
+
|
|
210
|
+
Every integer width is reported as ``int64``, narrow ones included. The
|
|
211
|
+
two model paths used to disagree here: one reported the true width and the
|
|
212
|
+
other always reported ``int64``. Identical data was described two
|
|
213
|
+
ways. The server decodes both through ``int()`` and cannot tell them
|
|
214
|
+
apart, so the choice is about which description to standardize on, and
|
|
215
|
+
``int64`` is the one that is never wrong: every narrower value fits it,
|
|
216
|
+
while the reverse is not true of a column that is widened after its dtype
|
|
217
|
+
is taken.
|
|
218
|
+
|
|
219
|
+
Being consistent matters more than being narrow. The server requires a
|
|
220
|
+
task's declared target dtype to equal the schema's dtype for that column
|
|
221
|
+
exactly, and answers 422 when they differ, so a second rule anywhere is a
|
|
222
|
+
rejected request rather than a cosmetic difference.
|
|
223
|
+
"""
|
|
224
|
+
dtype = series.dtype
|
|
225
|
+
if pd.api.types.is_bool_dtype(dtype):
|
|
226
|
+
return 'bool'
|
|
227
|
+
if pd.api.types.is_datetime64_any_dtype(dtype):
|
|
228
|
+
return 'timestamp[us]'
|
|
229
|
+
if pd.api.types.is_integer_dtype(dtype):
|
|
230
|
+
return _whole_numbers_dtype(series, dtype)
|
|
231
|
+
if pd.api.types.is_float_dtype(dtype):
|
|
232
|
+
return 'float64' if dtype.itemsize > 4 else 'float32'
|
|
233
|
+
decimal_type = arrow_decimal_type(dtype)
|
|
234
|
+
if decimal_type is not None:
|
|
235
|
+
return _decimal_dtype(series, decimal_type)
|
|
236
|
+
if pd.api.types.is_object_dtype(dtype):
|
|
237
|
+
content = pd.api.types.infer_dtype(series, skipna=True)
|
|
238
|
+
if content == 'empty':
|
|
239
|
+
return UNTYPED
|
|
240
|
+
if content == 'decimal':
|
|
241
|
+
return _decimal_dtype(series)
|
|
242
|
+
if content == 'integer':
|
|
243
|
+
return _whole_numbers_dtype(series, dtype)
|
|
244
|
+
return _OBJECT_CONTENT_DTYPES.get(content, 'string')
|
|
245
|
+
return 'string'
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
#: Digits an int64 always holds. 18 nines is 999999999999999999, below
|
|
249
|
+
#: INT64_MAX; the 19th digit is where it starts depending on the value.
|
|
250
|
+
_INT64_SAFE_PRECISION = 18
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def arrow_decimal_type(dtype: Any) -> Any | None:
|
|
254
|
+
r"""``dtype``'s Arrow decimal type, or ``None`` if it is not one.
|
|
255
|
+
|
|
256
|
+
A warehouse ``DECIMAL`` reaches us two ways. Read through pandas it is an
|
|
257
|
+
``object`` column of :class:`~decimal.Decimal`. Read through Arrow, which
|
|
258
|
+
the connectors now prefer because it keeps a wide integer exact, it is an
|
|
259
|
+
``ArrowDtype``, which satisfies none of the pandas dtype
|
|
260
|
+
predicates and so used to be announced as ``string``. An id column
|
|
261
|
+
announced as a string stops being an id to the model.
|
|
262
|
+
|
|
263
|
+
Read off the dtype rather than by importing pyarrow: this package depends
|
|
264
|
+
on pandas and numpy only, and a client should not gain a build dependency
|
|
265
|
+
to name a column.
|
|
266
|
+
"""
|
|
267
|
+
arrow_type = getattr(dtype, 'pyarrow_dtype', None)
|
|
268
|
+
if arrow_type is None or not str(arrow_type).startswith('decimal'):
|
|
269
|
+
return None
|
|
270
|
+
if not isinstance(getattr(arrow_type, 'scale', None), int):
|
|
271
|
+
return None
|
|
272
|
+
return arrow_type
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _whole_numbers_dtype(series: pd.Series, dtype: Any) -> str:
|
|
276
|
+
r"""Use ``int64`` for fitting whole numbers and ``string`` otherwise.
|
|
277
|
+
|
|
278
|
+
The rule that a whole number too wide for an ``int64`` travels as its
|
|
279
|
+
digits was applied to a decimal column and not to the integer column
|
|
280
|
+
beside it. A ``uint64`` or a Python ``int`` past the range was announced as
|
|
281
|
+
``int64``, a dtype the values provably cannot hold, and the request
|
|
282
|
+
failed at serialization instead of being described correctly here.
|
|
283
|
+
|
|
284
|
+
Only the two that can exceed are read. Every signed numpy width fits an
|
|
285
|
+
``int64`` by construction, and that is the hot path.
|
|
286
|
+
"""
|
|
287
|
+
if not _may_hold_more_than_an_int64(dtype):
|
|
288
|
+
return 'int64'
|
|
289
|
+
for value in series:
|
|
290
|
+
if _is_missing(value):
|
|
291
|
+
continue
|
|
292
|
+
try:
|
|
293
|
+
if not INT64_MIN <= int(value) <= INT64_MAX:
|
|
294
|
+
return 'string'
|
|
295
|
+
except (TypeError, ValueError, OverflowError):
|
|
296
|
+
return 'string'
|
|
297
|
+
return 'int64'
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _may_hold_more_than_an_int64(dtype: Any) -> bool:
|
|
301
|
+
"""Whether a column of ``dtype`` can carry a value an ``int64`` cannot."""
|
|
302
|
+
if pd.api.types.is_object_dtype(dtype):
|
|
303
|
+
return True
|
|
304
|
+
return 'uint64' in str(dtype).lower()
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _decimal_dtype(series: pd.Series, arrow_type: Any | None = None) -> str:
|
|
308
|
+
r"""``int64`` for a whole-number ``Decimal`` column, ``float64`` otherwise.
|
|
309
|
+
|
|
310
|
+
``DECIMAL`` and ``NUMBER`` are how a warehouse spells an exact number, and
|
|
311
|
+
a driver hands them back as ``Decimal`` objects. Calling the column
|
|
312
|
+
``float64`` because that is what floats are made of loses every value past
|
|
313
|
+
2**53, which includes most warehouse ids. A column whose values are whole
|
|
314
|
+
is reported as the integer it is, and travels under the rule that keeps
|
|
315
|
+
those exact.
|
|
316
|
+
|
|
317
|
+
A whole number too wide even for ``int64`` travels as ``string``, which
|
|
318
|
+
keeps the digits exactly and is what the relational encoder does with the
|
|
319
|
+
same input. Anything else is a ``float64``, and :func:`serialize_cell`
|
|
320
|
+
refuses the values that cannot survive one.
|
|
321
|
+
"""
|
|
322
|
+
if arrow_type is not None:
|
|
323
|
+
# The declared type answers both questions without reading a cell: a
|
|
324
|
+
# scale above zero cannot be whole, and 18 digits always fit int64.
|
|
325
|
+
# Only an ambiguous precision has to look at the values.
|
|
326
|
+
if arrow_type.scale > 0:
|
|
327
|
+
return 'float64'
|
|
328
|
+
if arrow_type.precision <= _INT64_SAFE_PRECISION:
|
|
329
|
+
return 'int64'
|
|
330
|
+
|
|
331
|
+
integral = True
|
|
332
|
+
for value in series:
|
|
333
|
+
if _is_missing(value):
|
|
334
|
+
continue
|
|
335
|
+
if not isinstance(value, Decimal):
|
|
336
|
+
return 'float64'
|
|
337
|
+
if not value.is_finite():
|
|
338
|
+
return 'float64'
|
|
339
|
+
# `as_tuple().exponent` and `adjusted()` answer "is it whole" and "how
|
|
340
|
+
# many digits" off the representation, so a 400-digit value never has
|
|
341
|
+
# to be materialised as an int just to be measured.
|
|
342
|
+
if value.as_tuple().exponent < 0 and value != value.to_integral_value():
|
|
343
|
+
return 'float64'
|
|
344
|
+
if value.adjusted() >= _INT64_SAFE_PRECISION and not (
|
|
345
|
+
INT64_MIN <= value <= INT64_MAX
|
|
346
|
+
):
|
|
347
|
+
integral = False
|
|
348
|
+
return 'int64' if integral else 'string'
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def widen_tfm_dtype(dtypes: Iterable[str]) -> str:
|
|
352
|
+
r"""The narrowest wire dtype that can hold every one of ``dtypes``.
|
|
353
|
+
|
|
354
|
+
A column typed differently in two frames must travel under a dtype that
|
|
355
|
+
represents both, otherwise the values of one frame are coerced to the other
|
|
356
|
+
frame's dtype and silently corrupted.
|
|
357
|
+
|
|
358
|
+
:data:`UNTYPED` candidates are dropped: a column that carries no type
|
|
359
|
+
information cannot make the union any wider, and letting it widen the union
|
|
360
|
+
to ``'string'`` would re-type the other frame's real values.
|
|
361
|
+
"""
|
|
362
|
+
candidates = [dtype for dtype in dict.fromkeys(dtypes) if dtype]
|
|
363
|
+
if not candidates:
|
|
364
|
+
return 'string'
|
|
365
|
+
if len(candidates) == 1:
|
|
366
|
+
return candidates[0]
|
|
367
|
+
if any(dtype not in _NUMERIC_WIDTH for dtype in candidates):
|
|
368
|
+
return 'string'
|
|
369
|
+
widest = max(candidates, key=_NUMERIC_WIDTH.__getitem__)
|
|
370
|
+
if widest == 'float32' and 'int64' in candidates:
|
|
371
|
+
return 'float64'
|
|
372
|
+
return widest
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
#: Deep enough for any real inference mapping, shallow enough that the walk
|
|
376
|
+
#: below cannot exhaust the interpreter stack before reporting.
|
|
377
|
+
_MAX_JSON_DEPTH = 100
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _refuse_inexact_integer(value: Any) -> None:
|
|
381
|
+
r"""Refuse an integer a float64 JSON reader would not read back exactly.
|
|
382
|
+
|
|
383
|
+
Asked by round trip rather than against the safe-integer bound. Every
|
|
384
|
+
integer up to that bound is representable, but so are plenty above it:
|
|
385
|
+
``2 ** 53`` is exact and only the odd numbers past it are not. Refusing on
|
|
386
|
+
the bound alone would reject values that travel perfectly well.
|
|
387
|
+
"""
|
|
388
|
+
if not isinstance(value, Integral) or isinstance(value, bool):
|
|
389
|
+
return
|
|
390
|
+
number = int(value)
|
|
391
|
+
try:
|
|
392
|
+
if int(float(number)) == number:
|
|
393
|
+
return
|
|
394
|
+
except (OverflowError, ValueError):
|
|
395
|
+
pass
|
|
396
|
+
raise StructuredDataError(
|
|
397
|
+
f'value {number} cannot be sent as a JSON number without losing '
|
|
398
|
+
f'precision, since a float64 does not hold it exactly; pass it as a '
|
|
399
|
+
f'string if the model accepts one',
|
|
400
|
+
code='INVALID_REQUEST',
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def json_ready(
|
|
405
|
+
value: Any, _depth: int = 0, _seen: frozenset[int] = frozenset()
|
|
406
|
+
) -> Any:
|
|
407
|
+
r"""``value`` with numpy scalars replaced by their Python equivalents.
|
|
408
|
+
|
|
409
|
+
Anything a caller reads out of their own frame is a numpy scalar, and the
|
|
410
|
+
JSON encoder has no rule for one. Thus,
|
|
411
|
+
``inference={'seed': df['seed'][0]}``
|
|
412
|
+
used to die inside ``requests`` as a ``TypeError`` naming neither the key
|
|
413
|
+
nor this client. Applied to the small hand-written parts of a payload;
|
|
414
|
+
table cells go through :func:`serialize_cell`, which types them properly.
|
|
415
|
+
|
|
416
|
+
An integer too large for a float64 to hold exactly is refused rather than
|
|
417
|
+
sent. A table column has a declared type and can travel as a string, but
|
|
418
|
+
these keys have whatever meaning the model gives them, so a string may not
|
|
419
|
+
be what the far side reads. Refusing names the value at the call site;
|
|
420
|
+
sending it means a reader on a float64 JSON stack gets a different number
|
|
421
|
+
than the caller passed.
|
|
422
|
+
"""
|
|
423
|
+
if isinstance(value, np.generic):
|
|
424
|
+
value = value.item()
|
|
425
|
+
_refuse_inexact_integer(value)
|
|
426
|
+
if id(value) in _seen:
|
|
427
|
+
raise StructuredDataError(
|
|
428
|
+
'this value refers back to itself, so it cannot be sent as JSON',
|
|
429
|
+
code='INVALID_REQUEST',
|
|
430
|
+
)
|
|
431
|
+
if _depth > _MAX_JSON_DEPTH:
|
|
432
|
+
# Reached before the interpreter's own limit, so a nested or cyclic
|
|
433
|
+
# mapping is named here rather than escaping the payload build as a
|
|
434
|
+
# bare RecursionError, which the transport's guard never sees.
|
|
435
|
+
raise StructuredDataError(
|
|
436
|
+
f'this value nests more than {_MAX_JSON_DEPTH} levels deep, or '
|
|
437
|
+
f'refers back to itself, so it cannot be sent as JSON',
|
|
438
|
+
code='INVALID_REQUEST',
|
|
439
|
+
)
|
|
440
|
+
if isinstance(value, Mapping):
|
|
441
|
+
nested = _seen | {id(value)}
|
|
442
|
+
return {
|
|
443
|
+
key: json_ready(item, _depth + 1, nested)
|
|
444
|
+
for key, item in value.items()
|
|
445
|
+
}
|
|
446
|
+
if isinstance(value, (list, tuple)):
|
|
447
|
+
nested = _seen | {id(value)}
|
|
448
|
+
return [json_ready(item, _depth + 1, nested) for item in value]
|
|
449
|
+
return value
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _is_missing(value: Any) -> bool:
|
|
453
|
+
if value is None:
|
|
454
|
+
return True
|
|
455
|
+
try:
|
|
456
|
+
result = pd.isna(value)
|
|
457
|
+
except (TypeError, ValueError):
|
|
458
|
+
return False
|
|
459
|
+
if isinstance(result, (bool, np.bool_)):
|
|
460
|
+
return bool(result)
|
|
461
|
+
return False # Array-likes (lists, ndarrays) are values, not nulls.
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _require_finite(value: Any) -> None:
|
|
465
|
+
r"""``inf`` has no JSON literal; emitting it produces invalid JSON.
|
|
466
|
+
|
|
467
|
+
Asked of the value rather than of its type: a ``Decimal('Infinity')`` from
|
|
468
|
+
a warehouse is as unsendable as a ``float('inf')``, and testing only for
|
|
469
|
+
``float`` let it through to fail later inside the JSON encoder, as a
|
|
470
|
+
transport error that named neither the column nor the value.
|
|
471
|
+
"""
|
|
472
|
+
if isinstance(value, Decimal):
|
|
473
|
+
if not value.is_finite():
|
|
474
|
+
raise StructuredDataError(
|
|
475
|
+
f'value {value!r} is not JSON-representable; replace '
|
|
476
|
+
'non-finite numbers with a finite value or a null',
|
|
477
|
+
code='INVALID_REQUEST',
|
|
478
|
+
)
|
|
479
|
+
return
|
|
480
|
+
if isinstance(value, (float, np.floating)) and not math.isfinite(value):
|
|
481
|
+
raise StructuredDataError(
|
|
482
|
+
f'value {value!r} is not JSON-representable; replace non-finite '
|
|
483
|
+
'numbers with a finite value or a null',
|
|
484
|
+
code='INVALID_REQUEST',
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _exceeds_float64(value: Any) -> bool:
|
|
489
|
+
r"""Whether ``value`` carries precision a ``float64`` would drop.
|
|
490
|
+
|
|
491
|
+
Two shapes reach the float branch already carrying more than a float64
|
|
492
|
+
holds. A numeric *string* is exact text, and `float()` rounds it without
|
|
493
|
+
complaint. A `longdouble` is wider than a `float64` on the platforms that
|
|
494
|
+
have one, so the same digits are lost on the way down. Both are silent,
|
|
495
|
+
which is the one thing the wire rules exist to prevent.
|
|
496
|
+
"""
|
|
497
|
+
if isinstance(value, str):
|
|
498
|
+
try:
|
|
499
|
+
exact = Decimal(value)
|
|
500
|
+
except (ArithmeticError, ValueError):
|
|
501
|
+
return False
|
|
502
|
+
if not exact.is_finite():
|
|
503
|
+
return False
|
|
504
|
+
return Decimal(repr(float(exact))) != exact
|
|
505
|
+
if isinstance(value, np.floating) and value.dtype.itemsize > 8:
|
|
506
|
+
if not np.isfinite(value):
|
|
507
|
+
return False
|
|
508
|
+
return np.longdouble(float(value)) != value
|
|
509
|
+
return False
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _integral_value(value: Any) -> int | None:
|
|
513
|
+
r"""``value`` as an ``int`` when it is a whole number, else ``None``.
|
|
514
|
+
|
|
515
|
+
Asked of the value, not of its type. ``numbers.Integral`` covers ``int``
|
|
516
|
+
and ``np.integer`` but not ``Decimal``, and a numpy integer compares equal
|
|
517
|
+
to the ``float`` it rounds to. A type test and an ``!=`` each let a
|
|
518
|
+
different unrepresentable value through the guard below.
|
|
519
|
+
"""
|
|
520
|
+
if isinstance(value, bool):
|
|
521
|
+
return None
|
|
522
|
+
if isinstance(value, Integral):
|
|
523
|
+
return int(value)
|
|
524
|
+
if (
|
|
525
|
+
isinstance(value, Decimal)
|
|
526
|
+
and value.is_finite()
|
|
527
|
+
and value == value.to_integral_value()
|
|
528
|
+
):
|
|
529
|
+
return int(value)
|
|
530
|
+
return None
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def serialize_cell(value: Any, dtype: str) -> Any:
|
|
534
|
+
if _is_missing(value):
|
|
535
|
+
return None
|
|
536
|
+
if dtype == 'timestamp[us]':
|
|
537
|
+
timestamp = pd.Timestamp(value)
|
|
538
|
+
if timestamp.tzinfo is None:
|
|
539
|
+
timestamp = timestamp.tz_localize('UTC')
|
|
540
|
+
else:
|
|
541
|
+
timestamp = timestamp.tz_convert('UTC')
|
|
542
|
+
return timestamp.isoformat(timespec='microseconds').replace(
|
|
543
|
+
'+00:00', 'Z'
|
|
544
|
+
)
|
|
545
|
+
if dtype in ('int64', 'int32'):
|
|
546
|
+
_require_finite(value)
|
|
547
|
+
integer_value = int(value)
|
|
548
|
+
if not isinstance(value, Integral) and integer_value != value:
|
|
549
|
+
raise StructuredDataError(
|
|
550
|
+
f'value {value!r} cannot be sent as {dtype} without losing its '
|
|
551
|
+
'fractional part',
|
|
552
|
+
code='INVALID_REQUEST',
|
|
553
|
+
)
|
|
554
|
+
if not INT64_MIN <= integer_value <= INT64_MAX:
|
|
555
|
+
raise StructuredDataError(
|
|
556
|
+
f'value {value!r} is outside the range {dtype} represents '
|
|
557
|
+
f'({INT64_MIN} to {INT64_MAX}); send the column as strings',
|
|
558
|
+
code='INVALID_REQUEST',
|
|
559
|
+
)
|
|
560
|
+
if (
|
|
561
|
+
integer_value > JSON_SAFE_INT_MAX
|
|
562
|
+
or integer_value < JSON_SAFE_INT_MIN
|
|
563
|
+
):
|
|
564
|
+
return str(integer_value)
|
|
565
|
+
return integer_value
|
|
566
|
+
if dtype in ('float64', 'float32'):
|
|
567
|
+
_require_finite(value)
|
|
568
|
+
try:
|
|
569
|
+
number = float(value)
|
|
570
|
+
except OverflowError:
|
|
571
|
+
# A Python int has no width, so `float()` refuses outright rather
|
|
572
|
+
# than returning `inf` the way a wide Decimal does. Same fault,
|
|
573
|
+
# different exception, and it must not escape raw.
|
|
574
|
+
raise StructuredDataError(
|
|
575
|
+
f'value {value!r} is outside the range {dtype} represents; '
|
|
576
|
+
'send the column as strings to keep it exact',
|
|
577
|
+
code='INVALID_REQUEST',
|
|
578
|
+
) from None
|
|
579
|
+
if not math.isfinite(number):
|
|
580
|
+
# A finite value whose float conversion is not: too large for
|
|
581
|
+
# the dtype to hold at all, rather than merely rounded by it.
|
|
582
|
+
raise StructuredDataError(
|
|
583
|
+
f'value {value!r} is outside the range {dtype} represents; '
|
|
584
|
+
'send the column as strings to keep it exact',
|
|
585
|
+
code='INVALID_REQUEST',
|
|
586
|
+
)
|
|
587
|
+
if _exceeds_float64(value):
|
|
588
|
+
raise StructuredDataError(
|
|
589
|
+
f'value {value!r} cannot be sent as {dtype} without losing '
|
|
590
|
+
f'precision; send the column as strings to keep it exact',
|
|
591
|
+
code='INVALID_REQUEST',
|
|
592
|
+
)
|
|
593
|
+
whole = _integral_value(value)
|
|
594
|
+
if (
|
|
595
|
+
whole is None
|
|
596
|
+
and isinstance(value, Decimal)
|
|
597
|
+
and abs(value) > JSON_SAFE_INT_MAX
|
|
598
|
+
):
|
|
599
|
+
# Past 2**53 a float64 cannot even hold the integer part, so the
|
|
600
|
+
# fractional value it becomes is not a rounding of the original in
|
|
601
|
+
# any useful sense. Below that the loss is ordinary float
|
|
602
|
+
# behaviour, which any float column already has.
|
|
603
|
+
raise StructuredDataError(
|
|
604
|
+
f'value {value!r} is too large for {dtype} to represent; '
|
|
605
|
+
'send the column as strings to keep it exact',
|
|
606
|
+
code='INVALID_REQUEST',
|
|
607
|
+
)
|
|
608
|
+
if whole is not None and int(number) != whole:
|
|
609
|
+
raise StructuredDataError(
|
|
610
|
+
f'value {value!r} cannot be sent as {dtype} without losing '
|
|
611
|
+
f'precision, because it is outside the range a {dtype} '
|
|
612
|
+
'represents exactly; give the column the same integer dtype '
|
|
613
|
+
'in every frame, or convert it to strings',
|
|
614
|
+
code='INVALID_REQUEST',
|
|
615
|
+
)
|
|
616
|
+
return number
|
|
617
|
+
if dtype == 'bool':
|
|
618
|
+
return bool(value)
|
|
619
|
+
if isinstance(value, Decimal):
|
|
620
|
+
# `str` switches to scientific notation once the exponent is positive,
|
|
621
|
+
# so a whole number too wide for int64 (the reason this column is a
|
|
622
|
+
# string at all) would travel as '1E+400' rather than its digits.
|
|
623
|
+
return format(value, 'f')
|
|
624
|
+
return str(value)
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def serialize_column(series: pd.Series, dtype: str) -> list[Any]:
|
|
628
|
+
# Every table reaches the wire through here, including the rows being
|
|
629
|
+
# scored, which no schema describes. A list cell would otherwise be written
|
|
630
|
+
# by `serialize_cell`'s `str(value)` fallback as the repr of a Python list.
|
|
631
|
+
require_scalar_cells(series, f'column {series.name!r}')
|
|
632
|
+
return [serialize_cell(value, dtype) for value in series.tolist()]
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
_TRUE = frozenset({'true', 't', '1'})
|
|
636
|
+
_FALSE = frozenset({'false', 'f', '0'})
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _as_bool(value: Any) -> Any:
|
|
640
|
+
"""A declared bool read back from what the wire spells it as.
|
|
641
|
+
|
|
642
|
+
`bool('false')` is True, so a string is matched against the spellings
|
|
643
|
+
rather than cast. Anything else is returned untouched, as everywhere else
|
|
644
|
+
here.
|
|
645
|
+
"""
|
|
646
|
+
if isinstance(value, bool):
|
|
647
|
+
return value
|
|
648
|
+
if isinstance(value, str):
|
|
649
|
+
text = value.strip().lower()
|
|
650
|
+
if text in _TRUE:
|
|
651
|
+
return True
|
|
652
|
+
if text in _FALSE:
|
|
653
|
+
return False
|
|
654
|
+
return value
|
|
655
|
+
if isinstance(value, Integral) and value in (0, 1):
|
|
656
|
+
return bool(value)
|
|
657
|
+
return value
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def as_declared(value: Any, dtype: str | None) -> Any:
|
|
661
|
+
r"""``value`` read back as ``dtype``, or unchanged if it cannot be.
|
|
662
|
+
|
|
663
|
+
The wire is text in places where the caller had a number: an integer wider
|
|
664
|
+
than a double travels as a base-10 string by contract, and one endpoint
|
|
665
|
+
renders a class label as a string where the other sends the number. Both
|
|
666
|
+
are legal, and both leave the caller holding a different type depending on
|
|
667
|
+
which model answered. Thus, `result.prediction == 0` counted nothing on one
|
|
668
|
+
of them, silently and in the direction of "this class never occurs".
|
|
669
|
+
|
|
670
|
+
Reading it back is a cast, not a parse: anything that does not convert
|
|
671
|
+
exactly is returned untouched rather than guessed at.
|
|
672
|
+
|
|
673
|
+
A bool is an int in Python and is not a number the caller wrote, so it is
|
|
674
|
+
excluded from the numeric branches. From text, `int` accepts only an exact
|
|
675
|
+
integer literal, so reaching the conversion means it was one. From a
|
|
676
|
+
number,
|
|
677
|
+
`int(1.9)` does not raise and a value that is not whole is not this integer
|
|
678
|
+
under another name, which is true of a float, a numpy float and a `Decimal`
|
|
679
|
+
alike and so is asked of the value rather than of its type.
|
|
680
|
+
|
|
681
|
+
A declared `bool` is read from the spellings the wire actually uses,
|
|
682
|
+
because
|
|
683
|
+
`bool('false')` is True and would invert the answer.
|
|
684
|
+
"""
|
|
685
|
+
if value is None or dtype is None:
|
|
686
|
+
return value
|
|
687
|
+
if dtype.startswith('bool'):
|
|
688
|
+
return _as_bool(value)
|
|
689
|
+
if isinstance(value, bool):
|
|
690
|
+
return value
|
|
691
|
+
try:
|
|
692
|
+
if dtype.startswith('int'):
|
|
693
|
+
read = int(value)
|
|
694
|
+
if isinstance(value, str):
|
|
695
|
+
return read
|
|
696
|
+
return read if read == value else value
|
|
697
|
+
if dtype.startswith('float'):
|
|
698
|
+
return float(value)
|
|
699
|
+
except (TypeError, ValueError, OverflowError):
|
|
700
|
+
return value
|
|
701
|
+
return value
|