structured-data-models-client 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdm_client/request.py ADDED
@@ -0,0 +1,1127 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
2
+ # All rights reserved.
3
+ # SPDX-License-Identifier: Apache-2.0
4
+
5
+ r"""One request type, for every model.
6
+
7
+ A :class:`Request` is the wire envelope minus the two things that belong to the
8
+ call rather than the question: which model answers it, and which output fields
9
+ you want back. Both are arguments to
10
+ :meth:`~sdm_client.SDMClient.predict`.
11
+
12
+ It is not a new abstraction. The Universal TFM contract already unified the
13
+ single-table and relational cases under one envelope, and a tabular request is
14
+ simply the one where the schema has one table and no relationships.
15
+
16
+ Two builders produce it. This module holds the direct one:
17
+
18
+ >>> request = Request(
19
+ ... context=labelled_rows, predict=rows_to_score,
20
+ ... target='churn', task='classification',
21
+ ... )
22
+
23
+ The other is a predictive query, which arrives in a later phase.
24
+
25
+ ``Request`` is **singular**. A run that exceeds the endpoint's per-call cap is
26
+ split into several round trips by the adapter, because batching is transport
27
+ and does not belong in a type you read.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json
33
+ import math
34
+ import re
35
+ import warnings
36
+ from collections.abc import Mapping, Sequence
37
+ from dataclasses import dataclass, field
38
+ from typing import Any
39
+
40
+ import pandas as pd
41
+
42
+ from sdm_client.core.warnings import SDMWarning, caller_stacklevel
43
+ from sdm_client.decompose import Decomposition
44
+ from sdm_client.errors import StructuredDataError, UnsupportedRequestError
45
+ from sdm_client.schema import Schema, supports_dtype
46
+ from sdm_client.wire import UNTYPED, infer_tfm_dtype, is_container
47
+ from sdm_client.wire.keys import key_dtypes, keys_from_a_frame
48
+
49
+ # Task kinds the contract carries. A model narrows this further: a tabular
50
+ # model takes only the coarse 'classification', and its adapter says so.
51
+ TASK_KINDS = (
52
+ 'classification',
53
+ 'binary_classification',
54
+ 'multiclass_classification',
55
+ 'regression',
56
+ 'forecasting',
57
+ 'temporal_link_prediction',
58
+ 'ranking',
59
+ )
60
+
61
+ _CLASSIFICATION_KINDS = frozenset(
62
+ {'classification', 'binary_classification', 'multiclass_classification'}
63
+ )
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Target:
68
+ r"""The column being predicted, and the dtype it is declared as.
69
+
70
+ The dtype has to match the schema's dtype for the same column exactly: the
71
+ endpoint compares them and answers 422 when they differ, so this is taken
72
+ from the schema rather than guessed at separately.
73
+ """
74
+
75
+ column_name: str
76
+ dtype: str
77
+ table_name: str | None = None
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class Task:
82
+ r"""What is being asked.
83
+
84
+ Attributes:
85
+ kind: One of :data:`TASK_KINDS`.
86
+ target: The column being predicted.
87
+ entity_table: The table whose rows are the entities, when the question
88
+ is relational. ``None`` for a single-table question.
89
+ options: Task options the contract carries but this client does not
90
+ model individually, such as ``anchor_time_column``.
91
+ """
92
+
93
+ kind: str
94
+ target: Target
95
+ entity_table: str | None = None
96
+ options: dict[str, Any] = field(default_factory=dict)
97
+
98
+ @property
99
+ def is_classification(self) -> bool:
100
+ return self.kind in _CLASSIFICATION_KINDS
101
+
102
+
103
+ _A_QUOTED_LITERAL = re.compile(r""""(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'""")
104
+
105
+
106
+ def _one_spelling_of_each_literal(text: str) -> str:
107
+ r"""Rewrite a query's string literals to one spelling of the quote.
108
+
109
+ ``channel="web"`` and ``channel='web'`` are one question.
110
+ """
111
+
112
+ def canonical(match: re.Match[str]) -> str:
113
+ literal = match.group(0)
114
+ inner = literal[1:-1]
115
+ if literal[0] == "'":
116
+ inner = inner.replace("\\'", "'").replace('"', '\\"')
117
+ try:
118
+ return json.dumps(json.loads(f'"{inner}"'))
119
+ except json.JSONDecodeError:
120
+ return literal
121
+
122
+ return _A_QUOTED_LITERAL.sub(canonical, text)
123
+
124
+
125
+ def _what_the_context_holds(context: Split | None) -> int | None:
126
+ r"""A value standing for the labelled rows, or ``None`` for gathered ones.
127
+
128
+ Frames carry their own labels, so two requests over different ones ask
129
+ different questions. A gathered context is sampled from the schema, so two
130
+ materializations of one query differ in rows and not in question.
131
+ """
132
+ if context is None:
133
+ return None
134
+ frames = [context.instance_table, *context.related_tables.values()]
135
+ total = hash(tuple(tuple(map(str, frame.columns)) for frame in frames))
136
+ for frame in frames:
137
+ try:
138
+ total ^= int(pd.util.hash_pandas_object(frame, index=False).sum())
139
+ except TypeError:
140
+ total ^= hash((len(frame), len(frame.columns)))
141
+ return total
142
+
143
+
144
+ @dataclass(frozen=True)
145
+ class Question:
146
+ r"""What a request asks, apart from the rows it asks about.
147
+
148
+ Assembled from the request rather than recorded by one constructor, so a
149
+ request built from frames carries one too.
150
+ """
151
+
152
+ query: str | None
153
+ kind: str
154
+ target: str
155
+ dtype: str
156
+ entity_table: str | None
157
+ context: int | None
158
+
159
+
160
+ def _judge_a_context_that_teaches_nothing(
161
+ column: str, target: pd.Series, kind: str
162
+ ) -> None:
163
+ r"""Remark on a context whose every row says the same thing.
164
+
165
+ Said rather than refused: this endpoint answers a single-class
166
+ classification, returning that class at probability 1.0, so refusing it
167
+ here turned a route that gathers into one that rejects what the route that
168
+ does not gather is answered for. A context with no answered rows is left to
169
+ ``Request``.
170
+
171
+ Only of the two kinds one value in a column means something for. A
172
+ forecast's target is a horizon and a ranking's is a candidate list, and one
173
+ value there does not mean what it means here.
174
+ """
175
+ answered = target.dropna()
176
+ if len(answered) == 0 or answered.nunique() >= 2:
177
+ return
178
+ if kind not in _CLASSIFICATION_KINDS and kind != 'regression':
179
+ return
180
+
181
+ said = (
182
+ 'a classification needs at least two classes to learn from, and the '
183
+ 'answer will be that one class at probability 1.0'
184
+ if kind in _CLASSIFICATION_KINDS
185
+ else 'nothing in the context distinguishes one answer from another, '
186
+ 'so the answer cannot say much'
187
+ )
188
+ warnings.warn(
189
+ f'every context row has the same {column}, so {said}. If the rows '
190
+ f'being scored are the ones that differ, score fewer of them; if '
191
+ f'{column} is constant in the whole table, there is no context that '
192
+ f'answers this question.',
193
+ category=SDMWarning,
194
+ stacklevel=caller_stacklevel(),
195
+ )
196
+
197
+
198
+ def warn_of_rows_scored_from_their_own_answers(request: Request) -> None:
199
+ r"""Say when the rows being scored sit in the context with their answers.
200
+
201
+ Both gathered routes take such rows out of the context they send, and a
202
+ session says so when it cannot. A request built from frames is handed both
203
+ halves and did neither, so the model was shown the answer to the question
204
+ it was being asked, which every other route here prevents.
205
+
206
+ Asked where both halves travel together rather than where the request is
207
+ built, because that is the only call at which the pairing is a leak. A
208
+ request built to score against a *session's* context sends only its rows,
209
+ and warning about a context that never travels made a correct call fail
210
+ under warnings-as-errors.
211
+ """
212
+ if request.payloads:
213
+ return
214
+ schema, entity_table = request.schema, request.entity_table
215
+ target = request.task.target.column_name
216
+ if schema is None or entity_table is None or not target:
217
+ return
218
+ try:
219
+ context = request.context.instance_table
220
+ predict = request.predict.instance_table
221
+ except Exception: # noqa: BLE001 - a warning is never worth a failure
222
+ return
223
+ if target not in context.columns:
224
+ return
225
+ _warn_of_rows_scored_from_their_own_answers(
226
+ context, predict, schema, entity_table, target
227
+ )
228
+
229
+
230
+ def _warn_of_rows_scored_from_their_own_answers(
231
+ context: pd.DataFrame,
232
+ predict: pd.DataFrame,
233
+ schema: Schema,
234
+ entity_table: str,
235
+ target: str,
236
+ ) -> None:
237
+ r"""The comparison itself, given both halves as frames."""
238
+ key = _as_key_tuple(schema[entity_table].primary_key)
239
+ # Every component or none. Reducing a composite key to the columns both
240
+ # frames happen to carry compares a part of an identity, and two rows that
241
+ # share a part are not the same row: one tenant's entity 1 was reported as
242
+ # another tenant's, scored from an answer that is not its own.
243
+ if not key or any(
244
+ column not in context.columns or column not in predict.columns
245
+ for column in key
246
+ ):
247
+ return
248
+ key = list(key)
249
+ answered = context[context[target].notna()]
250
+ if answered.empty:
251
+ return
252
+ # Compared as the wire spells them, through the encoder that writes them:
253
+ # a key column whose two halves hold different python types is one column
254
+ # written one way, so values that differ here are the same value by the
255
+ # time the model reads them. One dtype for the whole request, not one per
256
+ # half. Settled per half, the same entity came out `1.0` on the side
257
+ # read from the context and `1` on the side being scored, and the
258
+ # comparison reported overlaps the encoded request does not have.
259
+ table = schema[entity_table]
260
+ dtypes = key_dtypes(table, key, predict)
261
+ answers = set(keys_from_a_frame(answered, key, dtypes))
262
+ if not answers:
263
+ return
264
+ # Rows, not keys: the sentence counts rows being scored, and two rows
265
+ # asking about one entity are two rows whose answer is being shown.
266
+ shown = sum(
267
+ 1
268
+ for named in keys_from_a_frame(predict, key, dtypes)
269
+ if named in answers
270
+ )
271
+ if not shown:
272
+ return
273
+ warnings.warn(
274
+ f'{shown} of the {len(predict)} rows being scored are also in '
275
+ f'context, carrying the {target!r} being asked for, so the model is '
276
+ f'being shown the answers to rows it is being asked about. Drop them '
277
+ f'from context, or gather the request with materialize(), which '
278
+ f'leaves the rows it scores out of the context it sends.',
279
+ category=SDMWarning,
280
+ stacklevel=caller_stacklevel(),
281
+ )
282
+
283
+
284
+ @dataclass(frozen=True)
285
+ class Split:
286
+ r"""One half of a request: the instance rows, plus any related tables.
287
+
288
+ ``context`` is the half where the answer is known; ``predict`` is the half
289
+ being asked about. Both carry the same shape, which is what lets one type
290
+ describe a single-table and a relational request alike.
291
+ """
292
+
293
+ instance_table: pd.DataFrame
294
+ related_tables: dict[str, pd.DataFrame] = field(default_factory=dict)
295
+
296
+
297
+ class Request:
298
+ r"""What to ask, independent of which model answers it.
299
+
300
+ Attributes:
301
+ task: The question.
302
+ schema: The tables and how they relate.
303
+ context: Rows where the answer is known.
304
+ predict: Rows being asked about.
305
+ inference: Gathering knobs, e.g. ``num_context_samples``. The adapter
306
+ turns these into whatever the endpoint expects; ``run_mode`` is
307
+ derived there rather than being something you set.
308
+ query: The predictive query this was built from, when it was built
309
+ from one, and ``None`` when it was built from frames.
310
+ """
311
+
312
+ query: str | None = None
313
+
314
+ # How this question was rewritten, when it had to be: the kind of question
315
+ # that was asked, what each part asks after (a candidate or a step) as the
316
+ # caller gave it, and how many candidates a ranking keeps. Re-deriving any
317
+ # of this from the query text would read a key as a string, and would call
318
+ # a forecast over a table named `rank_events` a ranking. Internal, because
319
+ # it is how the client puts the parts' answers together, not something a
320
+ # caller reads: `parts` and `asked` say what a caller needs to see.
321
+ _decomposition: Decomposition | None = None
322
+
323
+ def __init__(
324
+ self,
325
+ context: pd.DataFrame,
326
+ predict: pd.DataFrame,
327
+ *,
328
+ target: str,
329
+ task: str,
330
+ related_tables: Mapping[str, pd.DataFrame] | None = None,
331
+ schema: Schema | None = None,
332
+ entity_table: str = 'instance',
333
+ inference: Mapping[str, Any] | None = None,
334
+ task_options: Mapping[str, Any] | None = None,
335
+ ) -> None:
336
+ r"""Build a request from frames you already have.
337
+
338
+ Args:
339
+ context: Rows where the answer is known. Must contain ``target``.
340
+ predict: Rows to score. Must not contain ``target``; these are the
341
+ rows whose answer you do not have, including rows the model
342
+ has never seen.
343
+ target: The column being predicted.
344
+ task: One of :data:`TASK_KINDS`. Stated, never inferred from the
345
+ target's cardinality. Guessing there is what turns a
346
+ two-class target into a multiclass one.
347
+ related_tables: Refused. A relational request is gathered by
348
+ ``PredictiveQuery(...).materialize(...)``; no model can be
349
+ sent one built from frames here.
350
+ schema: Override the inferred schema. Rarely needed.
351
+ entity_table: The name the instance table is known by. Only visible
352
+ in errors and in a query, so the default is fine.
353
+ inference: Gathering knobs passed to the adapter.
354
+ task_options: Task fields the contract carries that this client
355
+ does not model individually.
356
+
357
+ Raises:
358
+ StructuredDataError: If a frame is not a DataFrame, ``task`` is not
359
+ a known kind, ``context`` has no rows, ``target`` is missing
360
+ from ``context``, or ``predict`` carries the target column.
361
+ UnsupportedRequestError: If ``related_tables`` is given.
362
+ """
363
+ context = _require_frame(context, 'context')
364
+ predict = _require_frame(predict, 'predict')
365
+ related = {
366
+ name: _require_frame(frame, f'related_tables[{name!r}]')
367
+ for name, frame in (related_tables or {}).items()
368
+ }
369
+ _refuse_tables_no_model_can_be_given(related)
370
+
371
+ if task not in TASK_KINDS:
372
+ raise StructuredDataError(
373
+ f'task must be one of {list(TASK_KINDS)}, got {task!r}',
374
+ code='INVALID_REQUEST',
375
+ )
376
+ if target in context.columns and _has_container_cells(context[target]):
377
+ # Said here because this is the only place that knows which column
378
+ # is the target. SDM's own `Task.classification` covers "binary,
379
+ # multi-class, or multi-label", but the wire does not: `CellValue`
380
+ # is documented as excluding array-valued prediction scalars on
381
+ # purpose, so a multi-label answer has no shape to come back in.
382
+ raise StructuredDataError(
383
+ f'{target!r} holds a list per row, which is a multi-label '
384
+ f'target. These models predict one value per row, and the '
385
+ f'contract has no form for a set of labels, so this cannot be '
386
+ f'asked or answered',
387
+ code='UNSUPPORTED_REQUEST',
388
+ )
389
+ if len(context) == 0:
390
+ # A zero-row `predict` is legal. The contract sets no minimum on
391
+ # rows, and an empty batch inside a loop is a reasonable no-op.
392
+ # A zero-row context is not: a model that predicts from labelled
393
+ # examples cannot be given none, and the endpoint answers a 500.
394
+ raise StructuredDataError(
395
+ 'context has no rows, so there is nothing for the model to '
396
+ 'learn the answer from',
397
+ code='INVALID_REQUEST',
398
+ )
399
+ if target not in context.columns:
400
+ raise StructuredDataError(
401
+ f'context has no target column {target!r}; it has '
402
+ f'{sorted(map(str, context.columns))}',
403
+ code='INVALID_REQUEST',
404
+ )
405
+ if target in predict.columns:
406
+ raise StructuredDataError(
407
+ f'predict carries the target column {target!r}. Those are the '
408
+ f'rows whose answer is being asked for, so the column has to '
409
+ f'be absent; drop it with predict.drop(columns=[{target!r}])',
410
+ code='INVALID_REQUEST',
411
+ )
412
+ if context[target].notna().sum() == 0:
413
+ raise StructuredDataError(
414
+ 'context must contain at least one row with a known target',
415
+ code='INVALID_REQUEST',
416
+ )
417
+
418
+ missing = [
419
+ column
420
+ for column in predict.columns
421
+ if column not in context.columns
422
+ ]
423
+ if missing:
424
+ raise StructuredDataError(
425
+ f'predict has column(s) {missing} that context does not; the '
426
+ f'model is shown context first, so it cannot score a feature '
427
+ f'it has never been given',
428
+ code='INVALID_REQUEST',
429
+ )
430
+
431
+ if schema is None:
432
+ schema = Schema.from_data(
433
+ {entity_table: context, **related},
434
+ relationships=None if related else (),
435
+ )
436
+
437
+ _check_scored_values_mean_what_the_schema_says(
438
+ predict, schema, entity_table
439
+ )
440
+
441
+ # And the other direction: a feature the model was shown has to be
442
+ # supplied for the rows being scored. The endpoint rejects a predict
443
+ # half that is missing one, so name it here instead. A primary key is
444
+ # identity rather than a feature, so scoring rows may omit it.
445
+ identity = set(_as_key_tuple(schema[entity_table].primary_key))
446
+ absent = [
447
+ column
448
+ for column in context.columns
449
+ if column != target
450
+ and column not in identity
451
+ and column not in predict.columns
452
+ ]
453
+ if absent:
454
+ raise StructuredDataError(
455
+ f'predict is missing column(s) {absent} that context carries. '
456
+ f'Every feature the model is shown has to be supplied for the '
457
+ f'rows being scored; drop them from context, or supply them',
458
+ code='INVALID_REQUEST',
459
+ )
460
+
461
+ _judge_a_context_that_teaches_nothing(target, context[target], task)
462
+
463
+ self.schema = schema
464
+ self.task = Task(
465
+ kind=task,
466
+ target=Target(
467
+ column_name=target,
468
+ dtype=schema[entity_table][target].dtype,
469
+ table_name=entity_table,
470
+ ),
471
+ entity_table=entity_table if related else None,
472
+ options=dict(task_options or {}),
473
+ )
474
+ self._context = Split(instance_table=context, related_tables=related)
475
+ self._predict = Split(instance_table=predict, related_tables=related)
476
+ self.inference = dict(inference or {})
477
+ self.requested_inference = dict(inference or {})
478
+ self.entity_table = entity_table
479
+
480
+ @classmethod
481
+ def materialized(
482
+ cls,
483
+ *,
484
+ payloads: tuple[dict[str, Any], ...],
485
+ schema: Schema,
486
+ query: str,
487
+ indices: list[Any],
488
+ entity_table: str | None = None,
489
+ windowed: bool = False,
490
+ requested_inference: Mapping[str, Any] | None = None,
491
+ ) -> Request:
492
+ r"""A request whose data has already been gathered.
493
+
494
+ The predictive-query path does its gathering in ``materialize()``, so
495
+ what arrives here is one or more finished wire envelopes rather than
496
+ frames to encode. More than one when the run exceeds the endpoint's
497
+ per-call cap; the adapter sends each and the results are merged.
498
+
499
+ This is still one request. Batching is transport, and a caller reads
500
+ this object to see one question.
501
+
502
+ `entity_table` is named by the query rather than taken from whichever
503
+ table the caller's dict happened to hold first. Every pre-send
504
+ description of a relational request read it from insertion order, so
505
+ it answered correctly or incorrectly depending on how the schema was
506
+ built, and a programmatic caller could not use it as a guard.
507
+
508
+ It is optional because it was not there to pass before, and a caller
509
+ who does not name it is not asking for insertion order back: the
510
+ envelopes carry the query's own answer, which is read instead.
511
+ """
512
+ request = cls.__new__(cls)
513
+ request._windowed = windowed
514
+ if entity_table is None:
515
+ entity_table = _entity_table_in(payloads)
516
+ request.schema = schema
517
+ first = payloads[0]
518
+ target = first.get('task', {}).get('target', {})
519
+ request.task = Task(
520
+ kind=first.get('task', {}).get('kind', 'classification'),
521
+ target=Target(
522
+ column_name=target.get('column_name', ''),
523
+ dtype=target.get('dtype', ''),
524
+ ),
525
+ entity_table=entity_table,
526
+ options=dict(first.get('task', {}).get('options', {})),
527
+ )
528
+ request._context = None
529
+ request._predict = None
530
+ request.inference = dict(first.get('inference', {}))
531
+ request.entity_table = entity_table
532
+ request.query = query
533
+ request.indices = indices
534
+ request._payloads = payloads
535
+ # What the caller asked for, as distinct from what the engine filled
536
+ # in. An adapter may drop the engine's defaults silently; it must not
537
+ # drop something that was asked for.
538
+ request.requested_inference = dict(requested_inference or {})
539
+ return request
540
+
541
+ @property
542
+ def _question(self) -> Question:
543
+ r"""What this request asks. See :class:`Question`."""
544
+ target = self.task.target
545
+ query = self.query
546
+ return Question(
547
+ query=None
548
+ if query is None
549
+ else _one_spelling_of_each_literal(query),
550
+ kind=self.task.kind,
551
+ target=target.column_name,
552
+ dtype=target.dtype,
553
+ entity_table=self.entity_table,
554
+ context=_what_the_context_holds(self._context),
555
+ )
556
+
557
+ @property
558
+ def context(self) -> Split:
559
+ r"""The rows where the answer is known.
560
+
561
+ Read from the wire envelopes on the route that keeps its data as
562
+ envelopes, and on first access rather than while materializing: the
563
+ adapter sends those envelopes as they are, so building the frames
564
+ eagerly would copy every context row of every request on a path where
565
+ nothing reads them.
566
+ """
567
+ return self._half('context')
568
+
569
+ @context.setter
570
+ def context(self, value: Split) -> None:
571
+ self._refuse_if_gathered('context')
572
+ self._context = value
573
+
574
+ @property
575
+ def predict(self) -> Split:
576
+ r"""The rows being asked about. Built the way `context` is."""
577
+ return self._half('predict')
578
+
579
+ @predict.setter
580
+ def predict(self, value: Split) -> None:
581
+ self._refuse_if_gathered('predict')
582
+ self._predict = value
583
+
584
+ def _half(self, half: str) -> Split:
585
+ r"""One half, read from the envelopes that carry it.
586
+
587
+ Read on access rather than while materializing, because the adapter
588
+ sends those envelopes as they are and building the frames eagerly
589
+ would copy every row of every request on the path where nothing looks
590
+ at them.
591
+
592
+ Not held between reads. The envelopes are edited in place when leaked
593
+ context rows are taken out of them, so a kept frame would go on
594
+ describing rows the request no longer sends, and a cache correct only
595
+ for the edits made today is a trap for the ones made later. Reading is
596
+ a few milliseconds per thousand rows, against a call that spends
597
+ hundreds on the wire.
598
+
599
+ Where there are envelopes they are the request, and they are what is
600
+ read, in step with the counts beside them and with what the adapter
601
+ sends. A half stored alongside them cannot have been put there through
602
+ the setter, which refuses exactly that, but an object restored from an
603
+ older pickle carries one; reading it would describe rows that are
604
+ neither counted nor sent.
605
+
606
+ Which makes these a description of the request rather than a handle
607
+ into it. Editing a frame read from here changes what is sent only on
608
+ the route that happens to keep its data that way; on the other it is
609
+ read from the envelopes again and the edit is not there. Assigning the
610
+ half is what changes the request, and does so on both.
611
+ """
612
+ payloads = self.payloads
613
+ if payloads:
614
+ return _split_from(payloads, half)
615
+ held = getattr(self, f'_{half}', None)
616
+ return held if held is not None else Split(pd.DataFrame())
617
+
618
+ def _refuse_if_gathered(self, half: str) -> None:
619
+ r"""Refuse to replace a half of a request that is already gathered.
620
+
621
+ A gathered request holds finished wire envelopes, and the adapter
622
+ sends those. An envelope carries what a frame cannot: the entity
623
+ references the rows are addressed by, and the anchor each is asked
624
+ at. A `Split` put here could not be turned back into one. It
625
+ would change what the request reports and not what it sends, which is
626
+ worse than not being allowed: the object would describe rows the model
627
+ never sees.
628
+
629
+ Gathered is read the same way here as everywhere else on this object,
630
+ so that the route a request is on is one answer rather than four.
631
+ """
632
+ if not self.payloads:
633
+ return
634
+ raise StructuredDataError(
635
+ f'{half} cannot be replaced on a request gathered by a predictive '
636
+ f'query: its rows are already encoded, and the encoding carries '
637
+ f'what a frame does not. Materialize the query again for the rows '
638
+ f'you want instead.',
639
+ code='INVALID_REQUEST',
640
+ )
641
+
642
+ @property
643
+ def parts(self) -> tuple[Request, ...] | None:
644
+ r"""The requests this one is asked as, when it is asked as several.
645
+
646
+ A ranking and a forecast have no task on the endpoint, so each is
647
+ asked as a set of questions it does serve, one per candidate and one
648
+ per step. They are kept together here because the answer that was
649
+ asked for is the one they make between them, not any one of them.
650
+ """
651
+ return getattr(self, '_parts', None)
652
+
653
+ @property
654
+ def asked(self) -> str | None:
655
+ r"""The query a caller wrote, when this was gathered as several."""
656
+ return getattr(self, '_asked', None)
657
+
658
+ @property
659
+ def target_moves_with_time(self) -> bool:
660
+ r"""Whether the answer is a window rather than a standing value.
661
+
662
+ An aggregation is a window: the same entity asked about at a later
663
+ moment is a different question, and the answer to one is not the
664
+ answer to the other. A column is not: it has one value, and whoever
665
+ has seen it has the answer whenever they are asked.
666
+
667
+ Which is the difference between a context that has answered the
668
+ question and one that merely mentions the same entity, so anything
669
+ deciding whether a row was already given away has to know it.
670
+ """
671
+ return getattr(self, '_windowed', False)
672
+
673
+ @property
674
+ def payloads(self) -> tuple[dict[str, Any], ...] | None:
675
+ r"""The gathered wire envelopes, when this came from a query.
676
+
677
+ No envelopes and never having been gathered are the same request, and
678
+ are reported the same way. They were told apart once, so an empty
679
+ tuple read as gathered here and as not gathered where the rows are,
680
+ and an object could belong to neither route: the halves it reported
681
+ came from its frames while the adapters refused it for carrying
682
+ envelopes.
683
+ """
684
+ mine = getattr(self, '_payloads', None) or None
685
+ parts = self.parts
686
+ if parts is None:
687
+ return mine
688
+ # A question asked as several carries all of their envelopes: this is
689
+ # the object the caller holds, and its size is the size of what will
690
+ # be sent. Reporting one part's would undercount by as many parts as
691
+ # there are, which for a ranking is one per candidate.
692
+ gathered = tuple(
693
+ envelope for part in parts for envelope in part.payloads or ()
694
+ )
695
+ return gathered or mine
696
+
697
+ @property
698
+ def is_relational(self) -> bool:
699
+ r"""Whether this request needs a model that can read related tables."""
700
+ return bool(
701
+ self.context.related_tables
702
+ or self.schema.relationships
703
+ or len(self.schema) > 1
704
+ )
705
+
706
+ @property
707
+ def context_rows(self) -> int:
708
+ r"""How many rows the model will learn the answer from, in total.
709
+
710
+ A relational request keeps its rows in the gathered envelopes rather
711
+ than in ``context``, which stays an empty frame. Reading the frame
712
+ reported nought for every graph request. These two counts are the
713
+ only description of a request there is before it is sent, so the one
714
+ shape whose size is worth checking was the one that would not say.
715
+
716
+ Summed across envelopes, because a request gathered in several parts
717
+ gives each part its own context: that is why a session refuses a split
718
+ request rather than pinning the first one's.
719
+
720
+ A sum is not what any one call sends, and the endpoint's cap is per
721
+ call. A request batched for size gathers a context for each batch,
722
+ around that batch's own rows, so this counts every one of them;
723
+ `_context_rows_per_call` is the largest, which is the figure the cap
724
+ is held against, and `__repr__` says both.
725
+ """
726
+ return self._rows('context')
727
+
728
+ @property
729
+ def predict_rows(self) -> int:
730
+ r"""How many rows are being asked about, across every call."""
731
+ return self._rows('predict')
732
+
733
+ def _rows(self, half: str) -> int:
734
+ if self.payloads:
735
+ return sum(
736
+ _envelope_rows(payload, half) or 0 for payload in self.payloads
737
+ )
738
+ return len(getattr(self, half).instance_table)
739
+
740
+ def _context_rows_per_call(self) -> int:
741
+ r"""The most context rows any one call of this request sends.
742
+
743
+ The endpoint caps the context of each call, and a request gathered in
744
+ several calls sends a context with each, so ``context_rows``, their
745
+ sum, is not what any call carries.
746
+ """
747
+ if self.payloads:
748
+ return max(
749
+ _envelope_rows(payload, 'context') or 0
750
+ for payload in self.payloads
751
+ )
752
+ return len(self.context.instance_table)
753
+
754
+ @property
755
+ def nothing_to_score(self) -> bool:
756
+ r"""Whether this request is known to ask about no rows at all.
757
+
758
+ Distinct from ``predict_rows == 0``, and the distinction matters:
759
+ `predict()` answers a request with nothing in it without sending, so a
760
+ payload whose rows simply cannot be read must not be mistaken for one
761
+ that is empty. An unreadable envelope is a request that should reach
762
+ the endpoint and be refused, not one that quietly succeeds with an
763
+ empty answer.
764
+
765
+ """
766
+ if self.payloads:
767
+ counts = [
768
+ _envelope_rows(payload, 'predict') for payload in self.payloads
769
+ ]
770
+ return all(count is not None for count in counts) and not any(
771
+ counts
772
+ )
773
+ return len(self.predict.instance_table) == 0
774
+
775
+ def __repr__(self) -> str:
776
+ r"""What this request asks, and how much of it travels per call.
777
+
778
+ A request gathered in several calls carries a context in each, so the
779
+ total alone reads as a context far wider than any call carries. The
780
+ number of calls is said where there is more than one, and the context
781
+ is the widest one call's, since that is what the cap is held to.
782
+ """
783
+ shape = 'relational' if self.is_relational else 'single-table'
784
+ calls = len(self.payloads or ())
785
+ context = (
786
+ f'up to {self._context_rows_per_call()} context rows per call '
787
+ f'across {calls} calls'
788
+ if calls > 1
789
+ else f'{self.context_rows} context rows'
790
+ )
791
+ return (
792
+ f'Request({self.task.kind}, target={self.task.target.column_name!r}, '
793
+ f'{context}, {self.predict_rows} to score, '
794
+ f'{shape})'
795
+ )
796
+
797
+
798
+ def _as_key_tuple(key: Any) -> tuple[str, ...]:
799
+ if key is None:
800
+ return ()
801
+ if isinstance(key, str):
802
+ return (key,)
803
+ return tuple(key)
804
+
805
+
806
+ def _has_container_cells(series: pd.Series) -> bool:
807
+ r"""The same question the wire asks, so the two cannot disagree."""
808
+ return any(is_container(value) for value in series)
809
+
810
+
811
+ def _require_frame(value: Any, name: str) -> pd.DataFrame:
812
+ r"""Reject a table argument that is not a frame, naming the parameter.
813
+
814
+ Every table here is used as a frame immediately, so a dict, a list of
815
+ records or a Series, the shapes a pandas user reaches for before building
816
+ a frame, would otherwise surface as an ``AttributeError`` from inside an
817
+ adapter, which ``except StructuredDataError`` does not catch and which
818
+ names neither the argument nor the type it should have been.
819
+ """
820
+ if not isinstance(value, pd.DataFrame):
821
+ hint = (
822
+ '; wrap a single row with pd.DataFrame([row])'
823
+ if isinstance(value, (dict, pd.Series))
824
+ else '; build one with pd.DataFrame(...)'
825
+ )
826
+ raise StructuredDataError(
827
+ f'{name} must be a pandas DataFrame, got '
828
+ f'{type(value).__name__}{hint}',
829
+ code='INVALID_REQUEST',
830
+ )
831
+ return value
832
+
833
+
834
+ def _refuse_tables_no_model_can_be_given(
835
+ related: Mapping[str, pd.DataFrame],
836
+ ) -> None:
837
+ r"""Refuse related tables handed to the builder that cannot carry them.
838
+
839
+ Neither model can be sent a request built this way. The relational one is
840
+ sent gathered envelopes, which contain neighbourhoods sampled around each
841
+ entity with columns folded onto the instance table, and refuses frames. The
842
+ tabular one reads one table and would drop these without saying so. So the
843
+ argument took frames, built an object, and every path that could have sent
844
+ it said no, one call later than the mistake.
845
+
846
+ Said here, where the tables are offered, and naming the builder that does
847
+ gather them.
848
+ """
849
+ if not related:
850
+ return
851
+ raise UnsupportedRequestError(
852
+ f'related_tables {sorted(related)} makes this a relational request, '
853
+ f'and no model can be sent one built from frames: the relational '
854
+ f'model is sent tables gathered around each entity, and the tabular '
855
+ f'model reads a single table. Gather it with '
856
+ f'PredictiveQuery(...).materialize(indices=[...], '
857
+ f"model='kumo-relational'), whose schema holds the related tables and "
858
+ f'the relationships that reach them.',
859
+ details={'related_tables': sorted(related)},
860
+ )
861
+
862
+
863
+ def _entity_table_in(payloads: Sequence[Mapping[str, Any]]) -> str:
864
+ r"""The table the query asks about, as the envelopes record it.
865
+
866
+ `task.entity_table_names` is the query's own answer and is what the
867
+ caller would otherwise have passed. Read here so that a caller who
868
+ predates the argument gets that rather than the insertion order this
869
+ replaced.
870
+ """
871
+ for payload in payloads:
872
+ task = payload.get('task') if isinstance(payload, Mapping) else None
873
+ named = (task or {}).get('entity_table_names')
874
+ if isinstance(named, (list, tuple)) and named:
875
+ return str(named[0])
876
+ return ''
877
+
878
+
879
+ def _wire_frame(table: Any) -> pd.DataFrame | None:
880
+ r"""One wire table as a frame, or nothing if it cannot be read as one."""
881
+ if not isinstance(table, Mapping):
882
+ return None
883
+ columns = table.get('columns')
884
+ rows = table.get('rows')
885
+ if not isinstance(columns, list) or not isinstance(rows, list):
886
+ return None
887
+ try:
888
+ return pd.DataFrame(rows, columns=columns)
889
+ except (TypeError, ValueError, OverflowError):
890
+ # A row wider than its header raises rather than padding, and a
891
+ # number too large for pandas to hold raises as well. Reading a
892
+ # request is a description of it, and a description that cannot be
893
+ # made is not worth failing the call for.
894
+ return None
895
+
896
+
897
+ #: What ties a related row to the instance it hangs off.
898
+ INSTANCE_ID = 'instance_id'
899
+
900
+
901
+ def _is_a_number(value: Any) -> bool:
902
+ r"""Whether ``value`` is a number that can be moved along."""
903
+ return (
904
+ isinstance(value, (int, float))
905
+ and not isinstance(value, bool)
906
+ and not pd.isna(value)
907
+ # An infinity is a number that no other can be moved past, and
908
+ # rounding one has no answer at all. An integer too large to ask the
909
+ # question of is the same case, and asking raises.
910
+ and _is_finite(value)
911
+ )
912
+
913
+
914
+ def _is_finite(value: int | float) -> bool:
915
+ try:
916
+ return math.isfinite(value)
917
+ except (OverflowError, ValueError):
918
+ return False
919
+
920
+
921
+ def _instance_bounds(
922
+ frame: pd.DataFrame | None,
923
+ ) -> tuple[int, int | None]:
924
+ r"""How far an ``instance_id`` column reaches, rounded outwards.
925
+
926
+ Outwards because these bound a shift: truncating toward zero would round
927
+ a floor of `-0.5` up to `0` and leave the shift half a step short, which
928
+ is enough for two envelopes to land on the same id.
929
+
930
+ Only the values a shift can move are measured. Reading `'7'` as seven
931
+ would let it decide how far the next envelope moves while staying where
932
+ it is itself, so it would set the distance for a journey it does not
933
+ make.
934
+ """
935
+ if frame is None or INSTANCE_ID not in frame or frame.empty:
936
+ return -1, None
937
+ movable = [value for value in frame[INSTANCE_ID] if _is_a_number(value)]
938
+ if not movable:
939
+ return -1, None
940
+ return math.ceil(max(movable)), math.floor(min(movable))
941
+
942
+
943
+ def _past(frame: pd.DataFrame | None, shift: int) -> pd.DataFrame | None:
944
+ r"""``frame`` with its numeric ``instance_id`` moved along by ``shift``.
945
+
946
+ Only the numbers move. An id that is not one cannot be moved past
947
+ anything, and replacing it with the nothing it coerces to would lose the
948
+ value rather than relocate it. It is left where it is, still tying
949
+ its related rows to the instance it always named.
950
+ """
951
+ if frame is None or not shift or INSTANCE_ID not in frame:
952
+ return frame
953
+ moved = frame.copy()
954
+ moved[INSTANCE_ID] = moved[INSTANCE_ID].map(
955
+ lambda value: value + shift if _is_a_number(value) else value
956
+ )
957
+ return moved
958
+
959
+
960
+ def _split_from(payloads: Sequence[Mapping[str, Any]], half: str) -> Split:
961
+ r"""The rows of one half, gathered from the envelopes that carry them.
962
+
963
+ A relational request keeps its data as finished wire envelopes, and these
964
+ two attributes are how a caller reads what it holds. Leaving them empty
965
+ described the shape of the storage rather than the request: a relational
966
+ `context` reported no rows while `context_rows` on the same object
967
+ reported several hundred, so the object answered its own question twice
968
+ and differently.
969
+
970
+ Concatenated across envelopes, because more than one is batching and
971
+ batching is transport. A caller reads this object to see one question, so
972
+ the halves it exposes are the whole of each half.
973
+
974
+ An `instance_id` that is not a number is left exactly as it is. There is
975
+ no arithmetic that moves it past another, so two envelopes using the same
976
+ one stay indistinguishable here, as they already were. Two envelopes
977
+ sharing an `instance_id` are a malformed envelope, not something this
978
+ can repair.
979
+ """
980
+ instances: list[pd.DataFrame] = []
981
+ related: dict[str, list[pd.DataFrame]] = {}
982
+ already = 0
983
+ first = True
984
+ for payload in payloads:
985
+ half_of = payload.get(half) if isinstance(payload, Mapping) else None
986
+ if not isinstance(half_of, Mapping):
987
+ continue
988
+ frame = _wire_frame(half_of.get('instance_table'))
989
+ tables = half_of.get('related_tables')
990
+ parts = [
991
+ (name, _wire_frame(table))
992
+ for name, table in (
993
+ tables.items() if isinstance(tables, Mapping) else ()
994
+ )
995
+ ]
996
+ # Each envelope numbers its rows from zero, and `instance_id` is what
997
+ # ties a related row to the instance it hangs off. Stacking them
998
+ # unchanged puts the second envelope's row 0 beside the first's, so a
999
+ # join that was exact within one envelope silently crosses between
1000
+ # them. The first envelope is left as it arrived; each one after has
1001
+ # its lowest id put where the numbers so far ran out, which an offset
1002
+ # alone did not do: an envelope numbering from below zero landed
1003
+ # back among ids already taken however far the offset had advanced.
1004
+ numbered = [frame, *(table for _, table in parts)]
1005
+ bounds = [_instance_bounds(t) for t in numbered]
1006
+ highest = max((top for top, _ in bounds), default=-1)
1007
+ lowest = min(
1008
+ (floor for _, floor in bounds if floor is not None), default=None
1009
+ )
1010
+ shift = 0 if first or lowest is None else already - lowest
1011
+ if (moved := _past(frame, shift)) is not None:
1012
+ instances.append(moved)
1013
+ for name, table in parts:
1014
+ if (shifted := _past(table, shift)) is not None:
1015
+ related.setdefault(name, []).append(shifted)
1016
+ # An envelope that carried no numbered rows has not taken a place in
1017
+ # the sequence: it reserves none of it, and the next one that does is
1018
+ # still the first. Counting it moved the first readable envelope off
1019
+ # the numbers it arrived with, which is the one thing this promises
1020
+ # not to do.
1021
+ if lowest is not None:
1022
+ seen = 0 if frame is None else len(frame)
1023
+ already = max(already + seen, highest + shift + 1)
1024
+ first = False
1025
+
1026
+ def joined(frames: list[pd.DataFrame]) -> pd.DataFrame:
1027
+ if not frames:
1028
+ return pd.DataFrame()
1029
+ if len(frames) == 1:
1030
+ return frames[0]
1031
+ return pd.concat(frames, ignore_index=True)
1032
+
1033
+ return Split(
1034
+ instance_table=joined(instances),
1035
+ related_tables={name: joined(f) for name, f in related.items()},
1036
+ )
1037
+
1038
+
1039
+ def _envelope_rows(payload: Mapping[str, Any], half: str) -> int | None:
1040
+ r"""How many instance rows one wire envelope carries, or `None`.
1041
+
1042
+ Read defensively, and report not knowing as not knowing. These counts are
1043
+ what `repr` shows, so a request that cannot describe itself is worse than
1044
+ one that describes itself roughly. However, `nothing_to_score` decides
1045
+ whether
1046
+ to answer without sending, and there `None` and `0` are opposite answers.
1047
+
1048
+ A list, specifically: `Sized` admits `{}` and `''`, which would read as an
1049
+ envelope carrying no rows rather than as one that cannot be read, and
1050
+ those are the two answers this has to keep apart.
1051
+ """
1052
+ if not isinstance(payload, Mapping):
1053
+ return None
1054
+ half_of = payload.get(half)
1055
+ if not isinstance(half_of, Mapping):
1056
+ return None
1057
+ instance_table = half_of.get('instance_table')
1058
+ if not isinstance(instance_table, Mapping):
1059
+ return None
1060
+ rows = instance_table.get('rows')
1061
+ return len(rows) if isinstance(rows, (list, tuple)) else None
1062
+
1063
+
1064
+ def _check_scored_values_mean_what_the_schema_says(
1065
+ predict: pd.DataFrame, schema: Schema, entity_table: str | None
1066
+ ) -> None:
1067
+ r"""Refuse rows whose values contradict what their column means.
1068
+
1069
+ The columns of the two halves are already checked against each other, by
1070
+ name, each with a message naming the fix. The values were not, and the
1071
+ adapter rightly widens a column's dtype to cover both halves for the
1072
+ case it was written for: an all-null context column carries no type, and
1073
+ taking the context's answer alone would send the scored half's integers as
1074
+ text.
1075
+
1076
+ Nothing bounded it. A column the schema declares `float64` meaning
1077
+ `numerical` could be widened to `string` by scored rows holding text, and
1078
+ `stype` is read from the schema on another path and is not widened with it,
1079
+ so the request went out declaring a column that is `string` and
1080
+ `numerical` at once, which is not a thing a column can be. It was
1081
+ answered rather than refused: measured against the live endpoint on a
1082
+ target that is exactly three times a feature, the scored rows came back
1083
+ 300.0, 329.9 and 299.7 where the truth is 300, 330 and 360. Plausible
1084
+ numbers, one wrong by a sixth, and nothing in the answer to say so.
1085
+
1086
+ Only a contradiction is refused. Integers in a `float64` column and unseen
1087
+ labels in a `categorical` one still mean what the column means, and pass.
1088
+
1089
+ Runs after the schema is settled rather than beside the column checks it
1090
+ belongs with: a request built by hand may carry no schema at all, and one
1091
+ is inferred from the context a few lines below. Checking before that ran
1092
+ left the hand-built request, which has no query to validate it, as the only
1093
+ one not covered.
1094
+
1095
+ A column is not skipped merely for holding nothing. `pd.Series([],
1096
+ dtype='int64')` still says int64, and a zero-row batch declaring a type
1097
+ the column cannot mean builds the same contradictory envelope as a full
1098
+ one. A column that genuinely carries no type reports UNTYPED and is
1099
+ skipped for that reason instead. An all-null context column, by contrast,
1100
+ has no claim to contradict: the `categorical` it was given is a
1101
+ placeholder rather than a finding, so the scored rows decide both its type
1102
+ and its meaning. Once a caller has said what the column means, that is a
1103
+ claim like any other.
1104
+ """
1105
+ if predict is None or entity_table is None or entity_table not in schema:
1106
+ return
1107
+ table = schema[entity_table]
1108
+ for name in predict.columns:
1109
+ column = table.columns.get(name)
1110
+ if column is None:
1111
+ continue
1112
+ if column.untyped and not column.stype_declared:
1113
+ continue
1114
+ arriving = infer_tfm_dtype(predict[name])
1115
+ if arriving == UNTYPED or supports_dtype(column.stype, arriving):
1116
+ continue
1117
+ raise StructuredDataError(
1118
+ f'the rows being scored hold {arriving!r} in column {name!r}, and '
1119
+ f'the schema declares it {column.dtype!r} meaning '
1120
+ f'{column.stype!r}, which {arriving!r} cannot mean. Sent, the '
1121
+ f'column would be described as both at once and answered with '
1122
+ f'numbers that are wrong without saying so. Convert it before '
1123
+ f'scoring: predict[{name!r}] = pd.to_numeric(predict[{name!r}]) '
1124
+ f'or say what the column really is with '
1125
+ f'schema[{entity_table!r}][{name!r}].stype = ...',
1126
+ code='INVALID_REQUEST',
1127
+ )