structured-data-models-client 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm_client/__init__.py +70 -0
- sdm_client/_version.py +5 -0
- sdm_client/adapters/__init__.py +9 -0
- sdm_client/adapters/relational.py +870 -0
- sdm_client/adapters/tabular.py +580 -0
- sdm_client/base.py +300 -0
- sdm_client/client.py +1416 -0
- sdm_client/core/__init__.py +5 -0
- sdm_client/core/databricks_serving.py +611 -0
- sdm_client/core/databricks_user_agent.py +76 -0
- sdm_client/core/dates.py +278 -0
- sdm_client/core/transport.py +898 -0
- sdm_client/core/warnings.py +51 -0
- sdm_client/decompose.py +421 -0
- sdm_client/errors.py +177 -0
- sdm_client/materializers/__init__.py +10 -0
- sdm_client/materializers/relational.py +1786 -0
- sdm_client/materializers/sources.py +331 -0
- sdm_client/materializers/tabular.py +339 -0
- sdm_client/models.py +251 -0
- sdm_client/py.typed +0 -0
- sdm_client/query.py +647 -0
- sdm_client/request.py +1127 -0
- sdm_client/result.py +328 -0
- sdm_client/schema.py +3187 -0
- sdm_client/wire/__init__.py +46 -0
- sdm_client/wire/answers.py +80 -0
- sdm_client/wire/dtypes.py +701 -0
- sdm_client/wire/keys.py +363 -0
- sdm_client/wire/tables.py +50 -0
- structured_data_models_client-1.0.0.dist-info/METADATA +99 -0
- structured_data_models_client-1.0.0.dist-info/RECORD +36 -0
- structured_data_models_client-1.0.0.dist-info/WHEEL +5 -0
- structured_data_models_client-1.0.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models_client-1.0.0.dist-info/licenses/NOTICE +10 -0
- structured_data_models_client-1.0.0.dist-info/top_level.txt +1 -0
sdm_client/request.py
ADDED
|
@@ -0,0 +1,1127 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
|
|
2
|
+
# All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
r"""One request type, for every model.
|
|
6
|
+
|
|
7
|
+
A :class:`Request` is the wire envelope minus the two things that belong to the
|
|
8
|
+
call rather than the question: which model answers it, and which output fields
|
|
9
|
+
you want back. Both are arguments to
|
|
10
|
+
:meth:`~sdm_client.SDMClient.predict`.
|
|
11
|
+
|
|
12
|
+
It is not a new abstraction. The Universal TFM contract already unified the
|
|
13
|
+
single-table and relational cases under one envelope, and a tabular request is
|
|
14
|
+
simply the one where the schema has one table and no relationships.
|
|
15
|
+
|
|
16
|
+
Two builders produce it. This module holds the direct one:
|
|
17
|
+
|
|
18
|
+
>>> request = Request(
|
|
19
|
+
... context=labelled_rows, predict=rows_to_score,
|
|
20
|
+
... target='churn', task='classification',
|
|
21
|
+
... )
|
|
22
|
+
|
|
23
|
+
The other is a predictive query, which arrives in a later phase.
|
|
24
|
+
|
|
25
|
+
``Request`` is **singular**. A run that exceeds the endpoint's per-call cap is
|
|
26
|
+
split into several round trips by the adapter, because batching is transport
|
|
27
|
+
and does not belong in a type you read.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import json
|
|
33
|
+
import math
|
|
34
|
+
import re
|
|
35
|
+
import warnings
|
|
36
|
+
from collections.abc import Mapping, Sequence
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
import pandas as pd
|
|
41
|
+
|
|
42
|
+
from sdm_client.core.warnings import SDMWarning, caller_stacklevel
|
|
43
|
+
from sdm_client.decompose import Decomposition
|
|
44
|
+
from sdm_client.errors import StructuredDataError, UnsupportedRequestError
|
|
45
|
+
from sdm_client.schema import Schema, supports_dtype
|
|
46
|
+
from sdm_client.wire import UNTYPED, infer_tfm_dtype, is_container
|
|
47
|
+
from sdm_client.wire.keys import key_dtypes, keys_from_a_frame
|
|
48
|
+
|
|
49
|
+
# Task kinds the contract carries. A model narrows this further: a tabular
|
|
50
|
+
# model takes only the coarse 'classification', and its adapter says so.
|
|
51
|
+
TASK_KINDS = (
|
|
52
|
+
'classification',
|
|
53
|
+
'binary_classification',
|
|
54
|
+
'multiclass_classification',
|
|
55
|
+
'regression',
|
|
56
|
+
'forecasting',
|
|
57
|
+
'temporal_link_prediction',
|
|
58
|
+
'ranking',
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
_CLASSIFICATION_KINDS = frozenset(
|
|
62
|
+
{'classification', 'binary_classification', 'multiclass_classification'}
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Target:
|
|
68
|
+
r"""The column being predicted, and the dtype it is declared as.
|
|
69
|
+
|
|
70
|
+
The dtype has to match the schema's dtype for the same column exactly: the
|
|
71
|
+
endpoint compares them and answers 422 when they differ, so this is taken
|
|
72
|
+
from the schema rather than guessed at separately.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
column_name: str
|
|
76
|
+
dtype: str
|
|
77
|
+
table_name: str | None = None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class Task:
|
|
82
|
+
r"""What is being asked.
|
|
83
|
+
|
|
84
|
+
Attributes:
|
|
85
|
+
kind: One of :data:`TASK_KINDS`.
|
|
86
|
+
target: The column being predicted.
|
|
87
|
+
entity_table: The table whose rows are the entities, when the question
|
|
88
|
+
is relational. ``None`` for a single-table question.
|
|
89
|
+
options: Task options the contract carries but this client does not
|
|
90
|
+
model individually, such as ``anchor_time_column``.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
kind: str
|
|
94
|
+
target: Target
|
|
95
|
+
entity_table: str | None = None
|
|
96
|
+
options: dict[str, Any] = field(default_factory=dict)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def is_classification(self) -> bool:
|
|
100
|
+
return self.kind in _CLASSIFICATION_KINDS
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_A_QUOTED_LITERAL = re.compile(r""""(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'""")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _one_spelling_of_each_literal(text: str) -> str:
|
|
107
|
+
r"""Rewrite a query's string literals to one spelling of the quote.
|
|
108
|
+
|
|
109
|
+
``channel="web"`` and ``channel='web'`` are one question.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
def canonical(match: re.Match[str]) -> str:
|
|
113
|
+
literal = match.group(0)
|
|
114
|
+
inner = literal[1:-1]
|
|
115
|
+
if literal[0] == "'":
|
|
116
|
+
inner = inner.replace("\\'", "'").replace('"', '\\"')
|
|
117
|
+
try:
|
|
118
|
+
return json.dumps(json.loads(f'"{inner}"'))
|
|
119
|
+
except json.JSONDecodeError:
|
|
120
|
+
return literal
|
|
121
|
+
|
|
122
|
+
return _A_QUOTED_LITERAL.sub(canonical, text)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _what_the_context_holds(context: Split | None) -> int | None:
|
|
126
|
+
r"""A value standing for the labelled rows, or ``None`` for gathered ones.
|
|
127
|
+
|
|
128
|
+
Frames carry their own labels, so two requests over different ones ask
|
|
129
|
+
different questions. A gathered context is sampled from the schema, so two
|
|
130
|
+
materializations of one query differ in rows and not in question.
|
|
131
|
+
"""
|
|
132
|
+
if context is None:
|
|
133
|
+
return None
|
|
134
|
+
frames = [context.instance_table, *context.related_tables.values()]
|
|
135
|
+
total = hash(tuple(tuple(map(str, frame.columns)) for frame in frames))
|
|
136
|
+
for frame in frames:
|
|
137
|
+
try:
|
|
138
|
+
total ^= int(pd.util.hash_pandas_object(frame, index=False).sum())
|
|
139
|
+
except TypeError:
|
|
140
|
+
total ^= hash((len(frame), len(frame.columns)))
|
|
141
|
+
return total
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class Question:
|
|
146
|
+
r"""What a request asks, apart from the rows it asks about.
|
|
147
|
+
|
|
148
|
+
Assembled from the request rather than recorded by one constructor, so a
|
|
149
|
+
request built from frames carries one too.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
query: str | None
|
|
153
|
+
kind: str
|
|
154
|
+
target: str
|
|
155
|
+
dtype: str
|
|
156
|
+
entity_table: str | None
|
|
157
|
+
context: int | None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _judge_a_context_that_teaches_nothing(
|
|
161
|
+
column: str, target: pd.Series, kind: str
|
|
162
|
+
) -> None:
|
|
163
|
+
r"""Remark on a context whose every row says the same thing.
|
|
164
|
+
|
|
165
|
+
Said rather than refused: this endpoint answers a single-class
|
|
166
|
+
classification, returning that class at probability 1.0, so refusing it
|
|
167
|
+
here turned a route that gathers into one that rejects what the route that
|
|
168
|
+
does not gather is answered for. A context with no answered rows is left to
|
|
169
|
+
``Request``.
|
|
170
|
+
|
|
171
|
+
Only of the two kinds one value in a column means something for. A
|
|
172
|
+
forecast's target is a horizon and a ranking's is a candidate list, and one
|
|
173
|
+
value there does not mean what it means here.
|
|
174
|
+
"""
|
|
175
|
+
answered = target.dropna()
|
|
176
|
+
if len(answered) == 0 or answered.nunique() >= 2:
|
|
177
|
+
return
|
|
178
|
+
if kind not in _CLASSIFICATION_KINDS and kind != 'regression':
|
|
179
|
+
return
|
|
180
|
+
|
|
181
|
+
said = (
|
|
182
|
+
'a classification needs at least two classes to learn from, and the '
|
|
183
|
+
'answer will be that one class at probability 1.0'
|
|
184
|
+
if kind in _CLASSIFICATION_KINDS
|
|
185
|
+
else 'nothing in the context distinguishes one answer from another, '
|
|
186
|
+
'so the answer cannot say much'
|
|
187
|
+
)
|
|
188
|
+
warnings.warn(
|
|
189
|
+
f'every context row has the same {column}, so {said}. If the rows '
|
|
190
|
+
f'being scored are the ones that differ, score fewer of them; if '
|
|
191
|
+
f'{column} is constant in the whole table, there is no context that '
|
|
192
|
+
f'answers this question.',
|
|
193
|
+
category=SDMWarning,
|
|
194
|
+
stacklevel=caller_stacklevel(),
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def warn_of_rows_scored_from_their_own_answers(request: Request) -> None:
|
|
199
|
+
r"""Say when the rows being scored sit in the context with their answers.
|
|
200
|
+
|
|
201
|
+
Both gathered routes take such rows out of the context they send, and a
|
|
202
|
+
session says so when it cannot. A request built from frames is handed both
|
|
203
|
+
halves and did neither, so the model was shown the answer to the question
|
|
204
|
+
it was being asked, which every other route here prevents.
|
|
205
|
+
|
|
206
|
+
Asked where both halves travel together rather than where the request is
|
|
207
|
+
built, because that is the only call at which the pairing is a leak. A
|
|
208
|
+
request built to score against a *session's* context sends only its rows,
|
|
209
|
+
and warning about a context that never travels made a correct call fail
|
|
210
|
+
under warnings-as-errors.
|
|
211
|
+
"""
|
|
212
|
+
if request.payloads:
|
|
213
|
+
return
|
|
214
|
+
schema, entity_table = request.schema, request.entity_table
|
|
215
|
+
target = request.task.target.column_name
|
|
216
|
+
if schema is None or entity_table is None or not target:
|
|
217
|
+
return
|
|
218
|
+
try:
|
|
219
|
+
context = request.context.instance_table
|
|
220
|
+
predict = request.predict.instance_table
|
|
221
|
+
except Exception: # noqa: BLE001 - a warning is never worth a failure
|
|
222
|
+
return
|
|
223
|
+
if target not in context.columns:
|
|
224
|
+
return
|
|
225
|
+
_warn_of_rows_scored_from_their_own_answers(
|
|
226
|
+
context, predict, schema, entity_table, target
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _warn_of_rows_scored_from_their_own_answers(
|
|
231
|
+
context: pd.DataFrame,
|
|
232
|
+
predict: pd.DataFrame,
|
|
233
|
+
schema: Schema,
|
|
234
|
+
entity_table: str,
|
|
235
|
+
target: str,
|
|
236
|
+
) -> None:
|
|
237
|
+
r"""The comparison itself, given both halves as frames."""
|
|
238
|
+
key = _as_key_tuple(schema[entity_table].primary_key)
|
|
239
|
+
# Every component or none. Reducing a composite key to the columns both
|
|
240
|
+
# frames happen to carry compares a part of an identity, and two rows that
|
|
241
|
+
# share a part are not the same row: one tenant's entity 1 was reported as
|
|
242
|
+
# another tenant's, scored from an answer that is not its own.
|
|
243
|
+
if not key or any(
|
|
244
|
+
column not in context.columns or column not in predict.columns
|
|
245
|
+
for column in key
|
|
246
|
+
):
|
|
247
|
+
return
|
|
248
|
+
key = list(key)
|
|
249
|
+
answered = context[context[target].notna()]
|
|
250
|
+
if answered.empty:
|
|
251
|
+
return
|
|
252
|
+
# Compared as the wire spells them, through the encoder that writes them:
|
|
253
|
+
# a key column whose two halves hold different python types is one column
|
|
254
|
+
# written one way, so values that differ here are the same value by the
|
|
255
|
+
# time the model reads them. One dtype for the whole request, not one per
|
|
256
|
+
# half. Settled per half, the same entity came out `1.0` on the side
|
|
257
|
+
# read from the context and `1` on the side being scored, and the
|
|
258
|
+
# comparison reported overlaps the encoded request does not have.
|
|
259
|
+
table = schema[entity_table]
|
|
260
|
+
dtypes = key_dtypes(table, key, predict)
|
|
261
|
+
answers = set(keys_from_a_frame(answered, key, dtypes))
|
|
262
|
+
if not answers:
|
|
263
|
+
return
|
|
264
|
+
# Rows, not keys: the sentence counts rows being scored, and two rows
|
|
265
|
+
# asking about one entity are two rows whose answer is being shown.
|
|
266
|
+
shown = sum(
|
|
267
|
+
1
|
|
268
|
+
for named in keys_from_a_frame(predict, key, dtypes)
|
|
269
|
+
if named in answers
|
|
270
|
+
)
|
|
271
|
+
if not shown:
|
|
272
|
+
return
|
|
273
|
+
warnings.warn(
|
|
274
|
+
f'{shown} of the {len(predict)} rows being scored are also in '
|
|
275
|
+
f'context, carrying the {target!r} being asked for, so the model is '
|
|
276
|
+
f'being shown the answers to rows it is being asked about. Drop them '
|
|
277
|
+
f'from context, or gather the request with materialize(), which '
|
|
278
|
+
f'leaves the rows it scores out of the context it sends.',
|
|
279
|
+
category=SDMWarning,
|
|
280
|
+
stacklevel=caller_stacklevel(),
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@dataclass(frozen=True)
|
|
285
|
+
class Split:
|
|
286
|
+
r"""One half of a request: the instance rows, plus any related tables.
|
|
287
|
+
|
|
288
|
+
``context`` is the half where the answer is known; ``predict`` is the half
|
|
289
|
+
being asked about. Both carry the same shape, which is what lets one type
|
|
290
|
+
describe a single-table and a relational request alike.
|
|
291
|
+
"""
|
|
292
|
+
|
|
293
|
+
instance_table: pd.DataFrame
|
|
294
|
+
related_tables: dict[str, pd.DataFrame] = field(default_factory=dict)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
class Request:
|
|
298
|
+
r"""What to ask, independent of which model answers it.
|
|
299
|
+
|
|
300
|
+
Attributes:
|
|
301
|
+
task: The question.
|
|
302
|
+
schema: The tables and how they relate.
|
|
303
|
+
context: Rows where the answer is known.
|
|
304
|
+
predict: Rows being asked about.
|
|
305
|
+
inference: Gathering knobs, e.g. ``num_context_samples``. The adapter
|
|
306
|
+
turns these into whatever the endpoint expects; ``run_mode`` is
|
|
307
|
+
derived there rather than being something you set.
|
|
308
|
+
query: The predictive query this was built from, when it was built
|
|
309
|
+
from one, and ``None`` when it was built from frames.
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
query: str | None = None
|
|
313
|
+
|
|
314
|
+
# How this question was rewritten, when it had to be: the kind of question
|
|
315
|
+
# that was asked, what each part asks after (a candidate or a step) as the
|
|
316
|
+
# caller gave it, and how many candidates a ranking keeps. Re-deriving any
|
|
317
|
+
# of this from the query text would read a key as a string, and would call
|
|
318
|
+
# a forecast over a table named `rank_events` a ranking. Internal, because
|
|
319
|
+
# it is how the client puts the parts' answers together, not something a
|
|
320
|
+
# caller reads: `parts` and `asked` say what a caller needs to see.
|
|
321
|
+
_decomposition: Decomposition | None = None
|
|
322
|
+
|
|
323
|
+
def __init__(
|
|
324
|
+
self,
|
|
325
|
+
context: pd.DataFrame,
|
|
326
|
+
predict: pd.DataFrame,
|
|
327
|
+
*,
|
|
328
|
+
target: str,
|
|
329
|
+
task: str,
|
|
330
|
+
related_tables: Mapping[str, pd.DataFrame] | None = None,
|
|
331
|
+
schema: Schema | None = None,
|
|
332
|
+
entity_table: str = 'instance',
|
|
333
|
+
inference: Mapping[str, Any] | None = None,
|
|
334
|
+
task_options: Mapping[str, Any] | None = None,
|
|
335
|
+
) -> None:
|
|
336
|
+
r"""Build a request from frames you already have.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
context: Rows where the answer is known. Must contain ``target``.
|
|
340
|
+
predict: Rows to score. Must not contain ``target``; these are the
|
|
341
|
+
rows whose answer you do not have, including rows the model
|
|
342
|
+
has never seen.
|
|
343
|
+
target: The column being predicted.
|
|
344
|
+
task: One of :data:`TASK_KINDS`. Stated, never inferred from the
|
|
345
|
+
target's cardinality. Guessing there is what turns a
|
|
346
|
+
two-class target into a multiclass one.
|
|
347
|
+
related_tables: Refused. A relational request is gathered by
|
|
348
|
+
``PredictiveQuery(...).materialize(...)``; no model can be
|
|
349
|
+
sent one built from frames here.
|
|
350
|
+
schema: Override the inferred schema. Rarely needed.
|
|
351
|
+
entity_table: The name the instance table is known by. Only visible
|
|
352
|
+
in errors and in a query, so the default is fine.
|
|
353
|
+
inference: Gathering knobs passed to the adapter.
|
|
354
|
+
task_options: Task fields the contract carries that this client
|
|
355
|
+
does not model individually.
|
|
356
|
+
|
|
357
|
+
Raises:
|
|
358
|
+
StructuredDataError: If a frame is not a DataFrame, ``task`` is not
|
|
359
|
+
a known kind, ``context`` has no rows, ``target`` is missing
|
|
360
|
+
from ``context``, or ``predict`` carries the target column.
|
|
361
|
+
UnsupportedRequestError: If ``related_tables`` is given.
|
|
362
|
+
"""
|
|
363
|
+
context = _require_frame(context, 'context')
|
|
364
|
+
predict = _require_frame(predict, 'predict')
|
|
365
|
+
related = {
|
|
366
|
+
name: _require_frame(frame, f'related_tables[{name!r}]')
|
|
367
|
+
for name, frame in (related_tables or {}).items()
|
|
368
|
+
}
|
|
369
|
+
_refuse_tables_no_model_can_be_given(related)
|
|
370
|
+
|
|
371
|
+
if task not in TASK_KINDS:
|
|
372
|
+
raise StructuredDataError(
|
|
373
|
+
f'task must be one of {list(TASK_KINDS)}, got {task!r}',
|
|
374
|
+
code='INVALID_REQUEST',
|
|
375
|
+
)
|
|
376
|
+
if target in context.columns and _has_container_cells(context[target]):
|
|
377
|
+
# Said here because this is the only place that knows which column
|
|
378
|
+
# is the target. SDM's own `Task.classification` covers "binary,
|
|
379
|
+
# multi-class, or multi-label", but the wire does not: `CellValue`
|
|
380
|
+
# is documented as excluding array-valued prediction scalars on
|
|
381
|
+
# purpose, so a multi-label answer has no shape to come back in.
|
|
382
|
+
raise StructuredDataError(
|
|
383
|
+
f'{target!r} holds a list per row, which is a multi-label '
|
|
384
|
+
f'target. These models predict one value per row, and the '
|
|
385
|
+
f'contract has no form for a set of labels, so this cannot be '
|
|
386
|
+
f'asked or answered',
|
|
387
|
+
code='UNSUPPORTED_REQUEST',
|
|
388
|
+
)
|
|
389
|
+
if len(context) == 0:
|
|
390
|
+
# A zero-row `predict` is legal. The contract sets no minimum on
|
|
391
|
+
# rows, and an empty batch inside a loop is a reasonable no-op.
|
|
392
|
+
# A zero-row context is not: a model that predicts from labelled
|
|
393
|
+
# examples cannot be given none, and the endpoint answers a 500.
|
|
394
|
+
raise StructuredDataError(
|
|
395
|
+
'context has no rows, so there is nothing for the model to '
|
|
396
|
+
'learn the answer from',
|
|
397
|
+
code='INVALID_REQUEST',
|
|
398
|
+
)
|
|
399
|
+
if target not in context.columns:
|
|
400
|
+
raise StructuredDataError(
|
|
401
|
+
f'context has no target column {target!r}; it has '
|
|
402
|
+
f'{sorted(map(str, context.columns))}',
|
|
403
|
+
code='INVALID_REQUEST',
|
|
404
|
+
)
|
|
405
|
+
if target in predict.columns:
|
|
406
|
+
raise StructuredDataError(
|
|
407
|
+
f'predict carries the target column {target!r}. Those are the '
|
|
408
|
+
f'rows whose answer is being asked for, so the column has to '
|
|
409
|
+
f'be absent; drop it with predict.drop(columns=[{target!r}])',
|
|
410
|
+
code='INVALID_REQUEST',
|
|
411
|
+
)
|
|
412
|
+
if context[target].notna().sum() == 0:
|
|
413
|
+
raise StructuredDataError(
|
|
414
|
+
'context must contain at least one row with a known target',
|
|
415
|
+
code='INVALID_REQUEST',
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
missing = [
|
|
419
|
+
column
|
|
420
|
+
for column in predict.columns
|
|
421
|
+
if column not in context.columns
|
|
422
|
+
]
|
|
423
|
+
if missing:
|
|
424
|
+
raise StructuredDataError(
|
|
425
|
+
f'predict has column(s) {missing} that context does not; the '
|
|
426
|
+
f'model is shown context first, so it cannot score a feature '
|
|
427
|
+
f'it has never been given',
|
|
428
|
+
code='INVALID_REQUEST',
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
if schema is None:
|
|
432
|
+
schema = Schema.from_data(
|
|
433
|
+
{entity_table: context, **related},
|
|
434
|
+
relationships=None if related else (),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
_check_scored_values_mean_what_the_schema_says(
|
|
438
|
+
predict, schema, entity_table
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
# And the other direction: a feature the model was shown has to be
|
|
442
|
+
# supplied for the rows being scored. The endpoint rejects a predict
|
|
443
|
+
# half that is missing one, so name it here instead. A primary key is
|
|
444
|
+
# identity rather than a feature, so scoring rows may omit it.
|
|
445
|
+
identity = set(_as_key_tuple(schema[entity_table].primary_key))
|
|
446
|
+
absent = [
|
|
447
|
+
column
|
|
448
|
+
for column in context.columns
|
|
449
|
+
if column != target
|
|
450
|
+
and column not in identity
|
|
451
|
+
and column not in predict.columns
|
|
452
|
+
]
|
|
453
|
+
if absent:
|
|
454
|
+
raise StructuredDataError(
|
|
455
|
+
f'predict is missing column(s) {absent} that context carries. '
|
|
456
|
+
f'Every feature the model is shown has to be supplied for the '
|
|
457
|
+
f'rows being scored; drop them from context, or supply them',
|
|
458
|
+
code='INVALID_REQUEST',
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
_judge_a_context_that_teaches_nothing(target, context[target], task)
|
|
462
|
+
|
|
463
|
+
self.schema = schema
|
|
464
|
+
self.task = Task(
|
|
465
|
+
kind=task,
|
|
466
|
+
target=Target(
|
|
467
|
+
column_name=target,
|
|
468
|
+
dtype=schema[entity_table][target].dtype,
|
|
469
|
+
table_name=entity_table,
|
|
470
|
+
),
|
|
471
|
+
entity_table=entity_table if related else None,
|
|
472
|
+
options=dict(task_options or {}),
|
|
473
|
+
)
|
|
474
|
+
self._context = Split(instance_table=context, related_tables=related)
|
|
475
|
+
self._predict = Split(instance_table=predict, related_tables=related)
|
|
476
|
+
self.inference = dict(inference or {})
|
|
477
|
+
self.requested_inference = dict(inference or {})
|
|
478
|
+
self.entity_table = entity_table
|
|
479
|
+
|
|
480
|
+
@classmethod
|
|
481
|
+
def materialized(
|
|
482
|
+
cls,
|
|
483
|
+
*,
|
|
484
|
+
payloads: tuple[dict[str, Any], ...],
|
|
485
|
+
schema: Schema,
|
|
486
|
+
query: str,
|
|
487
|
+
indices: list[Any],
|
|
488
|
+
entity_table: str | None = None,
|
|
489
|
+
windowed: bool = False,
|
|
490
|
+
requested_inference: Mapping[str, Any] | None = None,
|
|
491
|
+
) -> Request:
|
|
492
|
+
r"""A request whose data has already been gathered.
|
|
493
|
+
|
|
494
|
+
The predictive-query path does its gathering in ``materialize()``, so
|
|
495
|
+
what arrives here is one or more finished wire envelopes rather than
|
|
496
|
+
frames to encode. More than one when the run exceeds the endpoint's
|
|
497
|
+
per-call cap; the adapter sends each and the results are merged.
|
|
498
|
+
|
|
499
|
+
This is still one request. Batching is transport, and a caller reads
|
|
500
|
+
this object to see one question.
|
|
501
|
+
|
|
502
|
+
`entity_table` is named by the query rather than taken from whichever
|
|
503
|
+
table the caller's dict happened to hold first. Every pre-send
|
|
504
|
+
description of a relational request read it from insertion order, so
|
|
505
|
+
it answered correctly or incorrectly depending on how the schema was
|
|
506
|
+
built, and a programmatic caller could not use it as a guard.
|
|
507
|
+
|
|
508
|
+
It is optional because it was not there to pass before, and a caller
|
|
509
|
+
who does not name it is not asking for insertion order back: the
|
|
510
|
+
envelopes carry the query's own answer, which is read instead.
|
|
511
|
+
"""
|
|
512
|
+
request = cls.__new__(cls)
|
|
513
|
+
request._windowed = windowed
|
|
514
|
+
if entity_table is None:
|
|
515
|
+
entity_table = _entity_table_in(payloads)
|
|
516
|
+
request.schema = schema
|
|
517
|
+
first = payloads[0]
|
|
518
|
+
target = first.get('task', {}).get('target', {})
|
|
519
|
+
request.task = Task(
|
|
520
|
+
kind=first.get('task', {}).get('kind', 'classification'),
|
|
521
|
+
target=Target(
|
|
522
|
+
column_name=target.get('column_name', ''),
|
|
523
|
+
dtype=target.get('dtype', ''),
|
|
524
|
+
),
|
|
525
|
+
entity_table=entity_table,
|
|
526
|
+
options=dict(first.get('task', {}).get('options', {})),
|
|
527
|
+
)
|
|
528
|
+
request._context = None
|
|
529
|
+
request._predict = None
|
|
530
|
+
request.inference = dict(first.get('inference', {}))
|
|
531
|
+
request.entity_table = entity_table
|
|
532
|
+
request.query = query
|
|
533
|
+
request.indices = indices
|
|
534
|
+
request._payloads = payloads
|
|
535
|
+
# What the caller asked for, as distinct from what the engine filled
|
|
536
|
+
# in. An adapter may drop the engine's defaults silently; it must not
|
|
537
|
+
# drop something that was asked for.
|
|
538
|
+
request.requested_inference = dict(requested_inference or {})
|
|
539
|
+
return request
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def _question(self) -> Question:
|
|
543
|
+
r"""What this request asks. See :class:`Question`."""
|
|
544
|
+
target = self.task.target
|
|
545
|
+
query = self.query
|
|
546
|
+
return Question(
|
|
547
|
+
query=None
|
|
548
|
+
if query is None
|
|
549
|
+
else _one_spelling_of_each_literal(query),
|
|
550
|
+
kind=self.task.kind,
|
|
551
|
+
target=target.column_name,
|
|
552
|
+
dtype=target.dtype,
|
|
553
|
+
entity_table=self.entity_table,
|
|
554
|
+
context=_what_the_context_holds(self._context),
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
@property
|
|
558
|
+
def context(self) -> Split:
|
|
559
|
+
r"""The rows where the answer is known.
|
|
560
|
+
|
|
561
|
+
Read from the wire envelopes on the route that keeps its data as
|
|
562
|
+
envelopes, and on first access rather than while materializing: the
|
|
563
|
+
adapter sends those envelopes as they are, so building the frames
|
|
564
|
+
eagerly would copy every context row of every request on a path where
|
|
565
|
+
nothing reads them.
|
|
566
|
+
"""
|
|
567
|
+
return self._half('context')
|
|
568
|
+
|
|
569
|
+
@context.setter
|
|
570
|
+
def context(self, value: Split) -> None:
|
|
571
|
+
self._refuse_if_gathered('context')
|
|
572
|
+
self._context = value
|
|
573
|
+
|
|
574
|
+
@property
|
|
575
|
+
def predict(self) -> Split:
|
|
576
|
+
r"""The rows being asked about. Built the way `context` is."""
|
|
577
|
+
return self._half('predict')
|
|
578
|
+
|
|
579
|
+
@predict.setter
|
|
580
|
+
def predict(self, value: Split) -> None:
|
|
581
|
+
self._refuse_if_gathered('predict')
|
|
582
|
+
self._predict = value
|
|
583
|
+
|
|
584
|
+
def _half(self, half: str) -> Split:
|
|
585
|
+
r"""One half, read from the envelopes that carry it.
|
|
586
|
+
|
|
587
|
+
Read on access rather than while materializing, because the adapter
|
|
588
|
+
sends those envelopes as they are and building the frames eagerly
|
|
589
|
+
would copy every row of every request on the path where nothing looks
|
|
590
|
+
at them.
|
|
591
|
+
|
|
592
|
+
Not held between reads. The envelopes are edited in place when leaked
|
|
593
|
+
context rows are taken out of them, so a kept frame would go on
|
|
594
|
+
describing rows the request no longer sends, and a cache correct only
|
|
595
|
+
for the edits made today is a trap for the ones made later. Reading is
|
|
596
|
+
a few milliseconds per thousand rows, against a call that spends
|
|
597
|
+
hundreds on the wire.
|
|
598
|
+
|
|
599
|
+
Where there are envelopes they are the request, and they are what is
|
|
600
|
+
read, in step with the counts beside them and with what the adapter
|
|
601
|
+
sends. A half stored alongside them cannot have been put there through
|
|
602
|
+
the setter, which refuses exactly that, but an object restored from an
|
|
603
|
+
older pickle carries one; reading it would describe rows that are
|
|
604
|
+
neither counted nor sent.
|
|
605
|
+
|
|
606
|
+
Which makes these a description of the request rather than a handle
|
|
607
|
+
into it. Editing a frame read from here changes what is sent only on
|
|
608
|
+
the route that happens to keep its data that way; on the other it is
|
|
609
|
+
read from the envelopes again and the edit is not there. Assigning the
|
|
610
|
+
half is what changes the request, and does so on both.
|
|
611
|
+
"""
|
|
612
|
+
payloads = self.payloads
|
|
613
|
+
if payloads:
|
|
614
|
+
return _split_from(payloads, half)
|
|
615
|
+
held = getattr(self, f'_{half}', None)
|
|
616
|
+
return held if held is not None else Split(pd.DataFrame())
|
|
617
|
+
|
|
618
|
+
def _refuse_if_gathered(self, half: str) -> None:
|
|
619
|
+
r"""Refuse to replace a half of a request that is already gathered.
|
|
620
|
+
|
|
621
|
+
A gathered request holds finished wire envelopes, and the adapter
|
|
622
|
+
sends those. An envelope carries what a frame cannot: the entity
|
|
623
|
+
references the rows are addressed by, and the anchor each is asked
|
|
624
|
+
at. A `Split` put here could not be turned back into one. It
|
|
625
|
+
would change what the request reports and not what it sends, which is
|
|
626
|
+
worse than not being allowed: the object would describe rows the model
|
|
627
|
+
never sees.
|
|
628
|
+
|
|
629
|
+
Gathered is read the same way here as everywhere else on this object,
|
|
630
|
+
so that the route a request is on is one answer rather than four.
|
|
631
|
+
"""
|
|
632
|
+
if not self.payloads:
|
|
633
|
+
return
|
|
634
|
+
raise StructuredDataError(
|
|
635
|
+
f'{half} cannot be replaced on a request gathered by a predictive '
|
|
636
|
+
f'query: its rows are already encoded, and the encoding carries '
|
|
637
|
+
f'what a frame does not. Materialize the query again for the rows '
|
|
638
|
+
f'you want instead.',
|
|
639
|
+
code='INVALID_REQUEST',
|
|
640
|
+
)
|
|
641
|
+
|
|
642
|
+
@property
|
|
643
|
+
def parts(self) -> tuple[Request, ...] | None:
|
|
644
|
+
r"""The requests this one is asked as, when it is asked as several.
|
|
645
|
+
|
|
646
|
+
A ranking and a forecast have no task on the endpoint, so each is
|
|
647
|
+
asked as a set of questions it does serve, one per candidate and one
|
|
648
|
+
per step. They are kept together here because the answer that was
|
|
649
|
+
asked for is the one they make between them, not any one of them.
|
|
650
|
+
"""
|
|
651
|
+
return getattr(self, '_parts', None)
|
|
652
|
+
|
|
653
|
+
@property
|
|
654
|
+
def asked(self) -> str | None:
|
|
655
|
+
r"""The query a caller wrote, when this was gathered as several."""
|
|
656
|
+
return getattr(self, '_asked', None)
|
|
657
|
+
|
|
658
|
+
@property
|
|
659
|
+
def target_moves_with_time(self) -> bool:
|
|
660
|
+
r"""Whether the answer is a window rather than a standing value.
|
|
661
|
+
|
|
662
|
+
An aggregation is a window: the same entity asked about at a later
|
|
663
|
+
moment is a different question, and the answer to one is not the
|
|
664
|
+
answer to the other. A column is not: it has one value, and whoever
|
|
665
|
+
has seen it has the answer whenever they are asked.
|
|
666
|
+
|
|
667
|
+
Which is the difference between a context that has answered the
|
|
668
|
+
question and one that merely mentions the same entity, so anything
|
|
669
|
+
deciding whether a row was already given away has to know it.
|
|
670
|
+
"""
|
|
671
|
+
return getattr(self, '_windowed', False)
|
|
672
|
+
|
|
673
|
+
@property
|
|
674
|
+
def payloads(self) -> tuple[dict[str, Any], ...] | None:
|
|
675
|
+
r"""The gathered wire envelopes, when this came from a query.
|
|
676
|
+
|
|
677
|
+
No envelopes and never having been gathered are the same request, and
|
|
678
|
+
are reported the same way. They were told apart once, so an empty
|
|
679
|
+
tuple read as gathered here and as not gathered where the rows are,
|
|
680
|
+
and an object could belong to neither route: the halves it reported
|
|
681
|
+
came from its frames while the adapters refused it for carrying
|
|
682
|
+
envelopes.
|
|
683
|
+
"""
|
|
684
|
+
mine = getattr(self, '_payloads', None) or None
|
|
685
|
+
parts = self.parts
|
|
686
|
+
if parts is None:
|
|
687
|
+
return mine
|
|
688
|
+
# A question asked as several carries all of their envelopes: this is
|
|
689
|
+
# the object the caller holds, and its size is the size of what will
|
|
690
|
+
# be sent. Reporting one part's would undercount by as many parts as
|
|
691
|
+
# there are, which for a ranking is one per candidate.
|
|
692
|
+
gathered = tuple(
|
|
693
|
+
envelope for part in parts for envelope in part.payloads or ()
|
|
694
|
+
)
|
|
695
|
+
return gathered or mine
|
|
696
|
+
|
|
697
|
+
@property
|
|
698
|
+
def is_relational(self) -> bool:
|
|
699
|
+
r"""Whether this request needs a model that can read related tables."""
|
|
700
|
+
return bool(
|
|
701
|
+
self.context.related_tables
|
|
702
|
+
or self.schema.relationships
|
|
703
|
+
or len(self.schema) > 1
|
|
704
|
+
)
|
|
705
|
+
|
|
706
|
+
@property
|
|
707
|
+
def context_rows(self) -> int:
|
|
708
|
+
r"""How many rows the model will learn the answer from, in total.
|
|
709
|
+
|
|
710
|
+
A relational request keeps its rows in the gathered envelopes rather
|
|
711
|
+
than in ``context``, which stays an empty frame. Reading the frame
|
|
712
|
+
reported nought for every graph request. These two counts are the
|
|
713
|
+
only description of a request there is before it is sent, so the one
|
|
714
|
+
shape whose size is worth checking was the one that would not say.
|
|
715
|
+
|
|
716
|
+
Summed across envelopes, because a request gathered in several parts
|
|
717
|
+
gives each part its own context: that is why a session refuses a split
|
|
718
|
+
request rather than pinning the first one's.
|
|
719
|
+
|
|
720
|
+
A sum is not what any one call sends, and the endpoint's cap is per
|
|
721
|
+
call. A request batched for size gathers a context for each batch,
|
|
722
|
+
around that batch's own rows, so this counts every one of them;
|
|
723
|
+
`_context_rows_per_call` is the largest, which is the figure the cap
|
|
724
|
+
is held against, and `__repr__` says both.
|
|
725
|
+
"""
|
|
726
|
+
return self._rows('context')
|
|
727
|
+
|
|
728
|
+
@property
|
|
729
|
+
def predict_rows(self) -> int:
|
|
730
|
+
r"""How many rows are being asked about, across every call."""
|
|
731
|
+
return self._rows('predict')
|
|
732
|
+
|
|
733
|
+
def _rows(self, half: str) -> int:
|
|
734
|
+
if self.payloads:
|
|
735
|
+
return sum(
|
|
736
|
+
_envelope_rows(payload, half) or 0 for payload in self.payloads
|
|
737
|
+
)
|
|
738
|
+
return len(getattr(self, half).instance_table)
|
|
739
|
+
|
|
740
|
+
def _context_rows_per_call(self) -> int:
|
|
741
|
+
r"""The most context rows any one call of this request sends.
|
|
742
|
+
|
|
743
|
+
The endpoint caps the context of each call, and a request gathered in
|
|
744
|
+
several calls sends a context with each, so ``context_rows``, their
|
|
745
|
+
sum, is not what any call carries.
|
|
746
|
+
"""
|
|
747
|
+
if self.payloads:
|
|
748
|
+
return max(
|
|
749
|
+
_envelope_rows(payload, 'context') or 0
|
|
750
|
+
for payload in self.payloads
|
|
751
|
+
)
|
|
752
|
+
return len(self.context.instance_table)
|
|
753
|
+
|
|
754
|
+
@property
|
|
755
|
+
def nothing_to_score(self) -> bool:
|
|
756
|
+
r"""Whether this request is known to ask about no rows at all.
|
|
757
|
+
|
|
758
|
+
Distinct from ``predict_rows == 0``, and the distinction matters:
|
|
759
|
+
`predict()` answers a request with nothing in it without sending, so a
|
|
760
|
+
payload whose rows simply cannot be read must not be mistaken for one
|
|
761
|
+
that is empty. An unreadable envelope is a request that should reach
|
|
762
|
+
the endpoint and be refused, not one that quietly succeeds with an
|
|
763
|
+
empty answer.
|
|
764
|
+
|
|
765
|
+
"""
|
|
766
|
+
if self.payloads:
|
|
767
|
+
counts = [
|
|
768
|
+
_envelope_rows(payload, 'predict') for payload in self.payloads
|
|
769
|
+
]
|
|
770
|
+
return all(count is not None for count in counts) and not any(
|
|
771
|
+
counts
|
|
772
|
+
)
|
|
773
|
+
return len(self.predict.instance_table) == 0
|
|
774
|
+
|
|
775
|
+
def __repr__(self) -> str:
|
|
776
|
+
r"""What this request asks, and how much of it travels per call.
|
|
777
|
+
|
|
778
|
+
A request gathered in several calls carries a context in each, so the
|
|
779
|
+
total alone reads as a context far wider than any call carries. The
|
|
780
|
+
number of calls is said where there is more than one, and the context
|
|
781
|
+
is the widest one call's, since that is what the cap is held to.
|
|
782
|
+
"""
|
|
783
|
+
shape = 'relational' if self.is_relational else 'single-table'
|
|
784
|
+
calls = len(self.payloads or ())
|
|
785
|
+
context = (
|
|
786
|
+
f'up to {self._context_rows_per_call()} context rows per call '
|
|
787
|
+
f'across {calls} calls'
|
|
788
|
+
if calls > 1
|
|
789
|
+
else f'{self.context_rows} context rows'
|
|
790
|
+
)
|
|
791
|
+
return (
|
|
792
|
+
f'Request({self.task.kind}, target={self.task.target.column_name!r}, '
|
|
793
|
+
f'{context}, {self.predict_rows} to score, '
|
|
794
|
+
f'{shape})'
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _as_key_tuple(key: Any) -> tuple[str, ...]:
|
|
799
|
+
if key is None:
|
|
800
|
+
return ()
|
|
801
|
+
if isinstance(key, str):
|
|
802
|
+
return (key,)
|
|
803
|
+
return tuple(key)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _has_container_cells(series: pd.Series) -> bool:
|
|
807
|
+
r"""The same question the wire asks, so the two cannot disagree."""
|
|
808
|
+
return any(is_container(value) for value in series)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _require_frame(value: Any, name: str) -> pd.DataFrame:
|
|
812
|
+
r"""Reject a table argument that is not a frame, naming the parameter.
|
|
813
|
+
|
|
814
|
+
Every table here is used as a frame immediately, so a dict, a list of
|
|
815
|
+
records or a Series, the shapes a pandas user reaches for before building
|
|
816
|
+
a frame, would otherwise surface as an ``AttributeError`` from inside an
|
|
817
|
+
adapter, which ``except StructuredDataError`` does not catch and which
|
|
818
|
+
names neither the argument nor the type it should have been.
|
|
819
|
+
"""
|
|
820
|
+
if not isinstance(value, pd.DataFrame):
|
|
821
|
+
hint = (
|
|
822
|
+
'; wrap a single row with pd.DataFrame([row])'
|
|
823
|
+
if isinstance(value, (dict, pd.Series))
|
|
824
|
+
else '; build one with pd.DataFrame(...)'
|
|
825
|
+
)
|
|
826
|
+
raise StructuredDataError(
|
|
827
|
+
f'{name} must be a pandas DataFrame, got '
|
|
828
|
+
f'{type(value).__name__}{hint}',
|
|
829
|
+
code='INVALID_REQUEST',
|
|
830
|
+
)
|
|
831
|
+
return value
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def _refuse_tables_no_model_can_be_given(
|
|
835
|
+
related: Mapping[str, pd.DataFrame],
|
|
836
|
+
) -> None:
|
|
837
|
+
r"""Refuse related tables handed to the builder that cannot carry them.
|
|
838
|
+
|
|
839
|
+
Neither model can be sent a request built this way. The relational one is
|
|
840
|
+
sent gathered envelopes, which contain neighbourhoods sampled around each
|
|
841
|
+
entity with columns folded onto the instance table, and refuses frames. The
|
|
842
|
+
tabular one reads one table and would drop these without saying so. So the
|
|
843
|
+
argument took frames, built an object, and every path that could have sent
|
|
844
|
+
it said no, one call later than the mistake.
|
|
845
|
+
|
|
846
|
+
Said here, where the tables are offered, and naming the builder that does
|
|
847
|
+
gather them.
|
|
848
|
+
"""
|
|
849
|
+
if not related:
|
|
850
|
+
return
|
|
851
|
+
raise UnsupportedRequestError(
|
|
852
|
+
f'related_tables {sorted(related)} makes this a relational request, '
|
|
853
|
+
f'and no model can be sent one built from frames: the relational '
|
|
854
|
+
f'model is sent tables gathered around each entity, and the tabular '
|
|
855
|
+
f'model reads a single table. Gather it with '
|
|
856
|
+
f'PredictiveQuery(...).materialize(indices=[...], '
|
|
857
|
+
f"model='kumo-relational'), whose schema holds the related tables and "
|
|
858
|
+
f'the relationships that reach them.',
|
|
859
|
+
details={'related_tables': sorted(related)},
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def _entity_table_in(payloads: Sequence[Mapping[str, Any]]) -> str:
|
|
864
|
+
r"""The table the query asks about, as the envelopes record it.
|
|
865
|
+
|
|
866
|
+
`task.entity_table_names` is the query's own answer and is what the
|
|
867
|
+
caller would otherwise have passed. Read here so that a caller who
|
|
868
|
+
predates the argument gets that rather than the insertion order this
|
|
869
|
+
replaced.
|
|
870
|
+
"""
|
|
871
|
+
for payload in payloads:
|
|
872
|
+
task = payload.get('task') if isinstance(payload, Mapping) else None
|
|
873
|
+
named = (task or {}).get('entity_table_names')
|
|
874
|
+
if isinstance(named, (list, tuple)) and named:
|
|
875
|
+
return str(named[0])
|
|
876
|
+
return ''
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _wire_frame(table: Any) -> pd.DataFrame | None:
|
|
880
|
+
r"""One wire table as a frame, or nothing if it cannot be read as one."""
|
|
881
|
+
if not isinstance(table, Mapping):
|
|
882
|
+
return None
|
|
883
|
+
columns = table.get('columns')
|
|
884
|
+
rows = table.get('rows')
|
|
885
|
+
if not isinstance(columns, list) or not isinstance(rows, list):
|
|
886
|
+
return None
|
|
887
|
+
try:
|
|
888
|
+
return pd.DataFrame(rows, columns=columns)
|
|
889
|
+
except (TypeError, ValueError, OverflowError):
|
|
890
|
+
# A row wider than its header raises rather than padding, and a
|
|
891
|
+
# number too large for pandas to hold raises as well. Reading a
|
|
892
|
+
# request is a description of it, and a description that cannot be
|
|
893
|
+
# made is not worth failing the call for.
|
|
894
|
+
return None
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
#: What ties a related row to the instance it hangs off.
|
|
898
|
+
INSTANCE_ID = 'instance_id'
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
def _is_a_number(value: Any) -> bool:
|
|
902
|
+
r"""Whether ``value`` is a number that can be moved along."""
|
|
903
|
+
return (
|
|
904
|
+
isinstance(value, (int, float))
|
|
905
|
+
and not isinstance(value, bool)
|
|
906
|
+
and not pd.isna(value)
|
|
907
|
+
# An infinity is a number that no other can be moved past, and
|
|
908
|
+
# rounding one has no answer at all. An integer too large to ask the
|
|
909
|
+
# question of is the same case, and asking raises.
|
|
910
|
+
and _is_finite(value)
|
|
911
|
+
)
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _is_finite(value: int | float) -> bool:
|
|
915
|
+
try:
|
|
916
|
+
return math.isfinite(value)
|
|
917
|
+
except (OverflowError, ValueError):
|
|
918
|
+
return False
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _instance_bounds(
|
|
922
|
+
frame: pd.DataFrame | None,
|
|
923
|
+
) -> tuple[int, int | None]:
|
|
924
|
+
r"""How far an ``instance_id`` column reaches, rounded outwards.
|
|
925
|
+
|
|
926
|
+
Outwards because these bound a shift: truncating toward zero would round
|
|
927
|
+
a floor of `-0.5` up to `0` and leave the shift half a step short, which
|
|
928
|
+
is enough for two envelopes to land on the same id.
|
|
929
|
+
|
|
930
|
+
Only the values a shift can move are measured. Reading `'7'` as seven
|
|
931
|
+
would let it decide how far the next envelope moves while staying where
|
|
932
|
+
it is itself, so it would set the distance for a journey it does not
|
|
933
|
+
make.
|
|
934
|
+
"""
|
|
935
|
+
if frame is None or INSTANCE_ID not in frame or frame.empty:
|
|
936
|
+
return -1, None
|
|
937
|
+
movable = [value for value in frame[INSTANCE_ID] if _is_a_number(value)]
|
|
938
|
+
if not movable:
|
|
939
|
+
return -1, None
|
|
940
|
+
return math.ceil(max(movable)), math.floor(min(movable))
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _past(frame: pd.DataFrame | None, shift: int) -> pd.DataFrame | None:
|
|
944
|
+
r"""``frame`` with its numeric ``instance_id`` moved along by ``shift``.
|
|
945
|
+
|
|
946
|
+
Only the numbers move. An id that is not one cannot be moved past
|
|
947
|
+
anything, and replacing it with the nothing it coerces to would lose the
|
|
948
|
+
value rather than relocate it. It is left where it is, still tying
|
|
949
|
+
its related rows to the instance it always named.
|
|
950
|
+
"""
|
|
951
|
+
if frame is None or not shift or INSTANCE_ID not in frame:
|
|
952
|
+
return frame
|
|
953
|
+
moved = frame.copy()
|
|
954
|
+
moved[INSTANCE_ID] = moved[INSTANCE_ID].map(
|
|
955
|
+
lambda value: value + shift if _is_a_number(value) else value
|
|
956
|
+
)
|
|
957
|
+
return moved
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _split_from(payloads: Sequence[Mapping[str, Any]], half: str) -> Split:
|
|
961
|
+
r"""The rows of one half, gathered from the envelopes that carry them.
|
|
962
|
+
|
|
963
|
+
A relational request keeps its data as finished wire envelopes, and these
|
|
964
|
+
two attributes are how a caller reads what it holds. Leaving them empty
|
|
965
|
+
described the shape of the storage rather than the request: a relational
|
|
966
|
+
`context` reported no rows while `context_rows` on the same object
|
|
967
|
+
reported several hundred, so the object answered its own question twice
|
|
968
|
+
and differently.
|
|
969
|
+
|
|
970
|
+
Concatenated across envelopes, because more than one is batching and
|
|
971
|
+
batching is transport. A caller reads this object to see one question, so
|
|
972
|
+
the halves it exposes are the whole of each half.
|
|
973
|
+
|
|
974
|
+
An `instance_id` that is not a number is left exactly as it is. There is
|
|
975
|
+
no arithmetic that moves it past another, so two envelopes using the same
|
|
976
|
+
one stay indistinguishable here, as they already were. Two envelopes
|
|
977
|
+
sharing an `instance_id` are a malformed envelope, not something this
|
|
978
|
+
can repair.
|
|
979
|
+
"""
|
|
980
|
+
instances: list[pd.DataFrame] = []
|
|
981
|
+
related: dict[str, list[pd.DataFrame]] = {}
|
|
982
|
+
already = 0
|
|
983
|
+
first = True
|
|
984
|
+
for payload in payloads:
|
|
985
|
+
half_of = payload.get(half) if isinstance(payload, Mapping) else None
|
|
986
|
+
if not isinstance(half_of, Mapping):
|
|
987
|
+
continue
|
|
988
|
+
frame = _wire_frame(half_of.get('instance_table'))
|
|
989
|
+
tables = half_of.get('related_tables')
|
|
990
|
+
parts = [
|
|
991
|
+
(name, _wire_frame(table))
|
|
992
|
+
for name, table in (
|
|
993
|
+
tables.items() if isinstance(tables, Mapping) else ()
|
|
994
|
+
)
|
|
995
|
+
]
|
|
996
|
+
# Each envelope numbers its rows from zero, and `instance_id` is what
|
|
997
|
+
# ties a related row to the instance it hangs off. Stacking them
|
|
998
|
+
# unchanged puts the second envelope's row 0 beside the first's, so a
|
|
999
|
+
# join that was exact within one envelope silently crosses between
|
|
1000
|
+
# them. The first envelope is left as it arrived; each one after has
|
|
1001
|
+
# its lowest id put where the numbers so far ran out, which an offset
|
|
1002
|
+
# alone did not do: an envelope numbering from below zero landed
|
|
1003
|
+
# back among ids already taken however far the offset had advanced.
|
|
1004
|
+
numbered = [frame, *(table for _, table in parts)]
|
|
1005
|
+
bounds = [_instance_bounds(t) for t in numbered]
|
|
1006
|
+
highest = max((top for top, _ in bounds), default=-1)
|
|
1007
|
+
lowest = min(
|
|
1008
|
+
(floor for _, floor in bounds if floor is not None), default=None
|
|
1009
|
+
)
|
|
1010
|
+
shift = 0 if first or lowest is None else already - lowest
|
|
1011
|
+
if (moved := _past(frame, shift)) is not None:
|
|
1012
|
+
instances.append(moved)
|
|
1013
|
+
for name, table in parts:
|
|
1014
|
+
if (shifted := _past(table, shift)) is not None:
|
|
1015
|
+
related.setdefault(name, []).append(shifted)
|
|
1016
|
+
# An envelope that carried no numbered rows has not taken a place in
|
|
1017
|
+
# the sequence: it reserves none of it, and the next one that does is
|
|
1018
|
+
# still the first. Counting it moved the first readable envelope off
|
|
1019
|
+
# the numbers it arrived with, which is the one thing this promises
|
|
1020
|
+
# not to do.
|
|
1021
|
+
if lowest is not None:
|
|
1022
|
+
seen = 0 if frame is None else len(frame)
|
|
1023
|
+
already = max(already + seen, highest + shift + 1)
|
|
1024
|
+
first = False
|
|
1025
|
+
|
|
1026
|
+
def joined(frames: list[pd.DataFrame]) -> pd.DataFrame:
|
|
1027
|
+
if not frames:
|
|
1028
|
+
return pd.DataFrame()
|
|
1029
|
+
if len(frames) == 1:
|
|
1030
|
+
return frames[0]
|
|
1031
|
+
return pd.concat(frames, ignore_index=True)
|
|
1032
|
+
|
|
1033
|
+
return Split(
|
|
1034
|
+
instance_table=joined(instances),
|
|
1035
|
+
related_tables={name: joined(f) for name, f in related.items()},
|
|
1036
|
+
)
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def _envelope_rows(payload: Mapping[str, Any], half: str) -> int | None:
|
|
1040
|
+
r"""How many instance rows one wire envelope carries, or `None`.
|
|
1041
|
+
|
|
1042
|
+
Read defensively, and report not knowing as not knowing. These counts are
|
|
1043
|
+
what `repr` shows, so a request that cannot describe itself is worse than
|
|
1044
|
+
one that describes itself roughly. However, `nothing_to_score` decides
|
|
1045
|
+
whether
|
|
1046
|
+
to answer without sending, and there `None` and `0` are opposite answers.
|
|
1047
|
+
|
|
1048
|
+
A list, specifically: `Sized` admits `{}` and `''`, which would read as an
|
|
1049
|
+
envelope carrying no rows rather than as one that cannot be read, and
|
|
1050
|
+
those are the two answers this has to keep apart.
|
|
1051
|
+
"""
|
|
1052
|
+
if not isinstance(payload, Mapping):
|
|
1053
|
+
return None
|
|
1054
|
+
half_of = payload.get(half)
|
|
1055
|
+
if not isinstance(half_of, Mapping):
|
|
1056
|
+
return None
|
|
1057
|
+
instance_table = half_of.get('instance_table')
|
|
1058
|
+
if not isinstance(instance_table, Mapping):
|
|
1059
|
+
return None
|
|
1060
|
+
rows = instance_table.get('rows')
|
|
1061
|
+
return len(rows) if isinstance(rows, (list, tuple)) else None
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _check_scored_values_mean_what_the_schema_says(
|
|
1065
|
+
predict: pd.DataFrame, schema: Schema, entity_table: str | None
|
|
1066
|
+
) -> None:
|
|
1067
|
+
r"""Refuse rows whose values contradict what their column means.
|
|
1068
|
+
|
|
1069
|
+
The columns of the two halves are already checked against each other, by
|
|
1070
|
+
name, each with a message naming the fix. The values were not, and the
|
|
1071
|
+
adapter rightly widens a column's dtype to cover both halves for the
|
|
1072
|
+
case it was written for: an all-null context column carries no type, and
|
|
1073
|
+
taking the context's answer alone would send the scored half's integers as
|
|
1074
|
+
text.
|
|
1075
|
+
|
|
1076
|
+
Nothing bounded it. A column the schema declares `float64` meaning
|
|
1077
|
+
`numerical` could be widened to `string` by scored rows holding text, and
|
|
1078
|
+
`stype` is read from the schema on another path and is not widened with it,
|
|
1079
|
+
so the request went out declaring a column that is `string` and
|
|
1080
|
+
`numerical` at once, which is not a thing a column can be. It was
|
|
1081
|
+
answered rather than refused: measured against the live endpoint on a
|
|
1082
|
+
target that is exactly three times a feature, the scored rows came back
|
|
1083
|
+
300.0, 329.9 and 299.7 where the truth is 300, 330 and 360. Plausible
|
|
1084
|
+
numbers, one wrong by a sixth, and nothing in the answer to say so.
|
|
1085
|
+
|
|
1086
|
+
Only a contradiction is refused. Integers in a `float64` column and unseen
|
|
1087
|
+
labels in a `categorical` one still mean what the column means, and pass.
|
|
1088
|
+
|
|
1089
|
+
Runs after the schema is settled rather than beside the column checks it
|
|
1090
|
+
belongs with: a request built by hand may carry no schema at all, and one
|
|
1091
|
+
is inferred from the context a few lines below. Checking before that ran
|
|
1092
|
+
left the hand-built request, which has no query to validate it, as the only
|
|
1093
|
+
one not covered.
|
|
1094
|
+
|
|
1095
|
+
A column is not skipped merely for holding nothing. `pd.Series([],
|
|
1096
|
+
dtype='int64')` still says int64, and a zero-row batch declaring a type
|
|
1097
|
+
the column cannot mean builds the same contradictory envelope as a full
|
|
1098
|
+
one. A column that genuinely carries no type reports UNTYPED and is
|
|
1099
|
+
skipped for that reason instead. An all-null context column, by contrast,
|
|
1100
|
+
has no claim to contradict: the `categorical` it was given is a
|
|
1101
|
+
placeholder rather than a finding, so the scored rows decide both its type
|
|
1102
|
+
and its meaning. Once a caller has said what the column means, that is a
|
|
1103
|
+
claim like any other.
|
|
1104
|
+
"""
|
|
1105
|
+
if predict is None or entity_table is None or entity_table not in schema:
|
|
1106
|
+
return
|
|
1107
|
+
table = schema[entity_table]
|
|
1108
|
+
for name in predict.columns:
|
|
1109
|
+
column = table.columns.get(name)
|
|
1110
|
+
if column is None:
|
|
1111
|
+
continue
|
|
1112
|
+
if column.untyped and not column.stype_declared:
|
|
1113
|
+
continue
|
|
1114
|
+
arriving = infer_tfm_dtype(predict[name])
|
|
1115
|
+
if arriving == UNTYPED or supports_dtype(column.stype, arriving):
|
|
1116
|
+
continue
|
|
1117
|
+
raise StructuredDataError(
|
|
1118
|
+
f'the rows being scored hold {arriving!r} in column {name!r}, and '
|
|
1119
|
+
f'the schema declares it {column.dtype!r} meaning '
|
|
1120
|
+
f'{column.stype!r}, which {arriving!r} cannot mean. Sent, the '
|
|
1121
|
+
f'column would be described as both at once and answered with '
|
|
1122
|
+
f'numbers that are wrong without saying so. Convert it before '
|
|
1123
|
+
f'scoring: predict[{name!r}] = pd.to_numeric(predict[{name!r}]) '
|
|
1124
|
+
f'or say what the column really is with '
|
|
1125
|
+
f'schema[{entity_table!r}][{name!r}].stype = ...',
|
|
1126
|
+
code='INVALID_REQUEST',
|
|
1127
|
+
)
|