structured-data-models-client 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdm_client/schema.py ADDED
@@ -0,0 +1,3187 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
2
+ # All rights reserved.
3
+ # SPDX-License-Identifier: Apache-2.0
4
+
5
+ r"""The tables a request is about, and what the model should make of them.
6
+
7
+ A :class:`Schema` holds your tables, what each column means, and how the tables
8
+ relate. One table or many is the same object. A single-table model is the
9
+ degenerate case of the same shape, not a second system, which is why this is
10
+ not called a graph.
11
+
12
+ Everything is inferred. You override only where inference is wrong:
13
+
14
+ >>> schema = Schema.from_data({'users': users_df})
15
+ >>> schema['users'].primary_key = 'user_id'
16
+ >>> schema['users']['churn'].stype = 'categorical'
17
+
18
+ **Inference reads values, so a code stored as a number reads as a number.** A
19
+ postcode, a store number or a status code is ``numerical`` here, and the model
20
+ will treat its distances as meaningful. No amount of inference settles that.
21
+ Only you know the column is a label. Correct it:
22
+
23
+ >>> schema['users']['zip_code'].stype = 'categorical'
24
+
25
+ Relationship inference is deliberately conservative in the same spirit: it
26
+ links a column only to a primary key of exactly the same name, so
27
+ ``orders.user_id`` finds ``users.user_id`` but not ``users.id``. A wrong link
28
+ changes what the model is shown and is harder to notice than a missing one, so
29
+ pass ``relationships=`` when the names do not line up.
30
+
31
+ Nothing here contacts a model. Inference reads your frames locally.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import warnings
37
+ from collections.abc import Iterable, Iterator, Mapping, Sequence
38
+ from collections.abc import Set as AbstractSet
39
+ from dataclasses import dataclass, field, replace
40
+ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
41
+
42
+ import pandas as pd
43
+ from predictive_query.time_columns import (
44
+ TimeColumnChoice,
45
+ choose_time_column,
46
+ taken_unchecked,
47
+ )
48
+
49
+ from sdm_client.core import dates
50
+ from sdm_client.core.warnings import SDMWarning, caller_stacklevel
51
+ from sdm_client.errors import StructuredDataError
52
+ from sdm_client.wire import (
53
+ UNTYPED,
54
+ infer_tfm_dtype,
55
+ require_scalar_cells,
56
+ )
57
+
58
+ if TYPE_CHECKING:
59
+ from sdm_connectors import Source
60
+
61
+ # Semantic types this client can declare. These are the contract's, and match
62
+ # ``predictive_query.Stype`` one for one.
63
+ SEMANTIC_TYPES = frozenset(
64
+ {
65
+ 'ID',
66
+ 'categorical',
67
+ 'multicategorical',
68
+ 'numerical',
69
+ 'sequence',
70
+ 'text',
71
+ 'timestamp',
72
+ }
73
+ )
74
+
75
+ # Above this many distinct values, a string column is prose rather than a set
76
+ # of labels. The exact number matters less than having one: a model encodes
77
+ # the two differently, and 'text' on a 3-value column wastes the signal.
78
+ _MAX_CATEGORICAL_CARDINALITY = 64
79
+
80
+ # A column is only a candidate identifier if nearly every value is distinct.
81
+ # Exactly 1.0 would reject a key with a single duplicated row, which is common
82
+ # in an extract and still the column the user means.
83
+ _KEY_UNIQUENESS = 0.99
84
+
85
+ # And below this, the repeats are the ordinary many-to-one shape rather than a
86
+ # fault in a key: a fact table's foreign key repeats on the order of its row
87
+ # count, while a key with a bad extract behind it repeats a handful of times.
88
+ # Between the two is where a column was meant to be the key and missed.
89
+ _NEARLY_A_KEY = 0.9
90
+
91
+ _KEY_SUFFIXES = ('_id', '_key', 'id', 'key')
92
+
93
+ _T = TypeVar('_T', bound='_TableShape')
94
+
95
+
96
+ @dataclass
97
+ class Column:
98
+ r"""Describe one column and how the model should treat it.
99
+
100
+ Attributes:
101
+ name: The column name, as it appears in the frame.
102
+ dtype: The wire dtype, e.g. ``'int64'`` or ``'timestamp[us]'``.
103
+ Inferred from the values; correcting it is rarely necessary.
104
+ stype: What the column *means*: ``'numerical'``, ``'categorical'``,
105
+ ``'ID'``, ``'text'``, ``'timestamp'``. This is the one worth
106
+ correcting: it decides how the model encodes the column.
107
+ stype_declared: Whether the meaning was assigned rather than inferred.
108
+ An all-null column is typed ``categorical`` because a schema has to
109
+ say something, and that placeholder is not a claim about the
110
+ values; a meaning the caller assigned is. Recorded so the two can
111
+ be told apart, since only the second is worth refusing a
112
+ contradiction against.
113
+ untyped: Whether the values carried no type at all, as in an all-null
114
+ object column. ``dtype`` is ``'string'`` for those, because a
115
+ schema has to name something and a string round-trips whatever
116
+ arrives, but the frame being scored may hold real values and
117
+ decide the dtype instead. Recorded so the adapter can read it
118
+ here rather than re-deriving it from the frame, which is how the
119
+ schema and the request came to disagree.
120
+ """
121
+
122
+ name: str
123
+ dtype: str
124
+ stype: str
125
+ untyped: bool = field(init=False, default=False)
126
+ stype_declared: bool = field(init=False, default=False)
127
+ _stype_before_key: str | None = field(init=False, default=None, repr=False)
128
+
129
+ def __post_init__(self) -> None:
130
+ self._check_stype(self.stype)
131
+
132
+ def _mark_as_key(self) -> None:
133
+ r"""Type this column as an identifier because the schema joins on it.
134
+
135
+ A key typed from its dtype alone becomes ``numerical``, the model
136
+ stores it as a float, and the join against an integer key fails
137
+ downstream. That is true of a column the caller called ``numerical``
138
+ too: what a key means is settled by its being a key, not by a
139
+ preference, so a declared meaning is overridden here rather than
140
+ honoured. Like an inferred meaning, it is recorded so it returns if
141
+ the column stops being a key.
142
+
143
+ This goes around ``__setattr__``: routed through it the schema's own
144
+ marking set ``stype_declared``, becoming indistinguishable from a
145
+ meaning the caller chose, and what had been decided was gone with
146
+ nothing to restore.
147
+ """
148
+ if self.stype == 'ID':
149
+ return
150
+ object.__setattr__(self, '_stype_before_key', self.stype)
151
+ object.__setattr__(self, 'stype', 'ID')
152
+
153
+ def _release_as_key(self) -> None:
154
+ r"""Restore a column's meaning after the schema stops joining on it.
155
+
156
+ Through the same adoption an assignment goes through: a column that
157
+ means `timestamp` orders the table it belongs to, and restoring the
158
+ meaning around that left a table saying it held a timestamp while
159
+ naming no time column. `check_consistent()` accepted that state, and
160
+ every temporal query then failed on.
161
+ """
162
+ if self._stype_before_key is None:
163
+ return
164
+ restored = self._stype_before_key
165
+ object.__setattr__(self, 'stype', restored)
166
+ object.__setattr__(self, '_stype_before_key', None)
167
+ if restored == 'timestamp':
168
+ self._adopt_as_time_column()
169
+
170
+ def __setattr__(self, key: str, value: Any) -> None:
171
+ r"""Assign a field, keeping the three that describe each other true.
172
+
173
+ `untyped` is derived rather than declared. It records that inference
174
+ found no type, and the adapter reads it as authority over `dtype`, so
175
+ letting it be set is letting a column claim a dtype and hold none.
176
+ Naming the dtype is the information it was missing, which is why doing
177
+ so clears it and the adapter stops substituting.
178
+
179
+ `stype_declared` separates a meaning that was chosen from one that was
180
+ inferred, and a second assignment is what distinguishes them.
181
+ """
182
+ if key == 'untyped':
183
+ raise StructuredDataError(
184
+ "'untyped' is inferred from the values and cannot be set; "
185
+ 'assign the dtype you want instead',
186
+ code='INVALID_REQUEST',
187
+ )
188
+ if key == 'stype':
189
+ self._check_stype(value)
190
+ self._check_supports_dtype(value, getattr(self, 'dtype', None))
191
+ if hasattr(self, 'stype'):
192
+ object.__setattr__(self, 'stype_declared', True)
193
+ if key == 'dtype' and getattr(self, 'untyped', False):
194
+ object.__setattr__(self, 'untyped', False)
195
+ object.__setattr__(self, key, value)
196
+ if key == 'stype' and value == 'timestamp':
197
+ self._adopt_as_time_column()
198
+ if key == 'stype':
199
+ self._stay_a_key_if_the_schema_joins_on_this(value)
200
+
201
+ def _stay_a_key_if_the_schema_joins_on_this(self, declared: str) -> None:
202
+ r"""Put ``ID`` back if this column is one the schema joins on.
203
+
204
+ A key's meaning is settled by its being a key, and the schema says so
205
+ wherever it is asked. The answer was only re-derived when the
206
+ keys changed, so between declaring a meaning on a live key and the
207
+ next such moment the column read as whatever was declared. Nothing was
208
+ sent that way, because a request re-derives as it takes its copy; the
209
+ object simply described itself wrongly in the meantime.
210
+
211
+ The declaration replaces what there was to give back, rather than
212
+ being discarded beside it: declaring ``ID`` on a column that is
213
+ already a key left the meaning inference had found still recorded, and
214
+ moving the key elsewhere handed that back over the caller's own word.
215
+ """
216
+ if declared == 'ID':
217
+ object.__setattr__(self, '_stype_before_key', None)
218
+ return
219
+ table = getattr(self, '_table', None)
220
+ schema = getattr(table, '_schema', None) if table is not None else None
221
+ if schema is None:
222
+ if table is not None and self.name in _as_key_tuple(
223
+ getattr(table, 'primary_key', None)
224
+ ):
225
+ self._mark_as_key()
226
+ return
227
+ schema._mark_key_columns()
228
+
229
+ def _adopt_as_time_column(self) -> None:
230
+ r"""Let a column named as a timestamp order the table it belongs to.
231
+
232
+ `Table._check_time_column` already sets a column's stype when the
233
+ column is named as the time column. Only that direction was wired, so
234
+ a caller correcting an inferred schema the way the SDK's own error
235
+ messages teach, ``schema[table][column].stype = 'timestamp'``, got
236
+ a table reporting `stype='timestamp'` and `time_column=None` at the
237
+ same time, and every temporal query on it was refused for want of a
238
+ time column the schema said was there.
239
+
240
+ Only an absent time column is filled: naming a second timestamp does
241
+ not displace the first. Nothing is adopted when inference already
242
+ chose, so this resolves the contradiction rather than overriding a
243
+ decision.
244
+
245
+ A column that is also part of the key is left alone rather than
246
+ refused: the two meanings contradict, and raising from an stype
247
+ assignment would report it from the wrong place. `check_consistent`
248
+ is where that is said.
249
+
250
+ The time column is set directly because the checked route assigns this
251
+ column's stype, which would re-enter here before the attribute it
252
+ guards on exists.
253
+ """
254
+ table = getattr(self, '_table', None)
255
+ if table is None or getattr(table, 'time_column', None) is not None:
256
+ return
257
+ if self.name in _as_key_tuple(getattr(table, 'primary_key', None)):
258
+ return
259
+ object.__setattr__(table, 'time_column', self.name)
260
+
261
+ def check_consistent(self) -> None:
262
+ r"""Refuse a column whose storage and meaning contradict each other.
263
+
264
+ Assigning a meaning is checked against the storage as it happens, but
265
+ the reverse is not, because a caller re-declaring storage often has to
266
+ set the two one at a time and either order passes through a pair that
267
+ does not hold. The pair is asked about here instead, once the caller
268
+ has finished and before anything is sent.
269
+ """
270
+ if self.dtype is None or self.stype is None:
271
+ return
272
+ self._check_supports_dtype(self.stype, self.dtype)
273
+
274
+ @staticmethod
275
+ def _check_stype(value: Any) -> None:
276
+ if value not in SEMANTIC_TYPES:
277
+ raise StructuredDataError(
278
+ f'{value!r} is not a semantic type; use one of '
279
+ f'{sorted(SEMANTIC_TYPES)}',
280
+ code='INVALID_REQUEST',
281
+ )
282
+
283
+ def _check_supports_dtype(self, stype: str, dtype: str | None) -> None:
284
+ r"""Refuse a meaning the values cannot carry.
285
+
286
+ The language already decides this, and used to decide it at
287
+ ``materialize()``, several cells after the assignment that caused it,
288
+ as a bare ``ValueError``. Asking the same question here changes only
289
+ when the answer arrives. Nothing is refused that was going to be
290
+ accepted.
291
+ """
292
+ if dtype is None:
293
+ return
294
+ if not supports_dtype(stype, dtype):
295
+ raise StructuredDataError(
296
+ f'column {self.name!r} holds {dtype!r}, which cannot mean '
297
+ f'{stype!r}',
298
+ code='INVALID_REQUEST',
299
+ )
300
+
301
+ def __repr__(self) -> str:
302
+ untyped = ', untyped=True' if self.untyped else ''
303
+ return (
304
+ f'Column({self.name!r}, dtype={self.dtype!r}, '
305
+ f'stype={self.stype!r}{untyped})'
306
+ )
307
+
308
+
309
+ class _TableShape:
310
+ r"""What a table says about itself, whether or not it holds the rows.
311
+
312
+ Keys, the time column and what each column means are declared the same
313
+ way on a table of rows and on one that stays in its source; only the
314
+ checks that read rows differ, and those ask for the rows rather than
315
+ assume them.
316
+ """
317
+
318
+ name: str
319
+ columns: dict[str, Column]
320
+ primary_key: str | Sequence[str] | None
321
+ time_column: str | None
322
+ duplicate_keys: int | None
323
+
324
+ def _own_columns(self) -> None:
325
+ r"""Point every column at this table, and take a copy of any that is
326
+ already pointing somewhere else.
327
+
328
+ The back-reference is what lets a column make itself the time column
329
+ when it is told it is a timestamp. One `Column` handed to two tables
330
+ could only point at one of them, so setting the stype through the
331
+ first would have ordered the second. A column belongs to a table;
332
+ handing the same one to two is asking for two columns.
333
+
334
+ A borrowed column is copied into a new dict, so that a caller holding
335
+ the one they passed does not find entries in it they did not put
336
+ there. The back-reference itself is set directly: it is not a field,
337
+ and must stay out of equality and `repr`.
338
+ """
339
+ # By identity. A `Table` is a dataclass, so `in (None, self)` asks
340
+ # `==`, which compares the frames inside and raises on any table that
341
+ # holds one.
342
+ borrowed = [
343
+ name
344
+ for name, column in self.columns.items()
345
+ if (owner := getattr(column, '_table', None)) is not None
346
+ and owner is not self
347
+ ]
348
+ if borrowed:
349
+ object.__setattr__(self, 'columns', dict(self.columns))
350
+ for name in borrowed:
351
+ self.columns[name] = _copy_column(self.columns[name])
352
+ for column in self.columns.values():
353
+ object.__setattr__(column, '_table', self)
354
+
355
+ def __eq__(self, other: object) -> bool:
356
+ r"""Whether two tables describe the same thing.
357
+
358
+ Written out because the generated one compares the frames with `==`,
359
+ which for a DataFrame is a frame of booleans. Asking whether two
360
+ tables are equal raised `ValueError: The truth value of a DataFrame is
361
+ ambiguous` from inside pandas, naming nothing the caller wrote. The
362
+ rows are compared as rows, and everything else by value, which is what
363
+ the question means.
364
+ """
365
+ if not isinstance(other, _TableShape) or type(other) is not type(self):
366
+ return NotImplemented
367
+ described = (
368
+ self.name == other.name
369
+ and self.columns == other.columns
370
+ and _as_key_tuple(self.primary_key)
371
+ == _as_key_tuple(other.primary_key)
372
+ and self.time_column == other.time_column
373
+ and getattr(self, 'source_name', None)
374
+ == getattr(other, 'source_name', None)
375
+ )
376
+ if not described:
377
+ return False
378
+ mine = getattr(self, 'data', None)
379
+ theirs = getattr(other, 'data', None)
380
+ if mine is None or theirs is None:
381
+ return mine is theirs
382
+ return bool(mine.equals(theirs))
383
+
384
+ __hash__ = None # type: ignore[assignment]
385
+
386
+ def __copy__(self: _T) -> _T:
387
+ r"""A shallow copy is still a table of its own columns.
388
+
389
+ Sharing them left the copy unable to answer for itself. Its columns
390
+ pointed at the table they came from, so changing the copy's key typed
391
+ nothing. It could also retype the original once the copy was put in a
392
+ schema.
393
+ """
394
+ return _copy_table(self)
395
+
396
+ def __setattr__(self, key: str, value: Any) -> None:
397
+ if key in ('primary_key', 'time_column') and 'columns' in self.__dict__:
398
+ # Before anything is marked, not after. Naming a key or a time
399
+ # column types the columns it names, and ownership was taken in
400
+ # `__post_init__` (which runs after the fields are assigned), so
401
+ # a table built on another's columns typed *those* first and left
402
+ # the table it borrowed them from changed.
403
+ self._own_columns()
404
+ if key == 'primary_key':
405
+ # Read once, here, and what was read is what is kept. A key is
406
+ # consulted many times afterwards: to check it, to type its
407
+ # columns, and to copy the table. An iterable that empties on
408
+ # the first read arrived as "was given an empty primary key"
409
+ # about a key that named two columns. Reading it before anything
410
+ # is assigned also stops a key this cannot read from being taken
411
+ # and leaving a table that can be neither copied nor sent.
412
+ names = _as_key_tuple(value)
413
+ if value is not None and not isinstance(value, str):
414
+ value = names
415
+ if value is None:
416
+ object.__setattr__(self, 'duplicate_keys', 0)
417
+ object.__setattr__(self, key, None)
418
+ self._re_derive_key_columns()
419
+ return
420
+ self._check_key(value)
421
+ object.__setattr__(self, key, value)
422
+ self._adopt_key(value)
423
+ self._re_derive_key_columns()
424
+ return
425
+ if key == 'time_column' and value is not None:
426
+ self._check_time_column(value)
427
+ object.__setattr__(self, key, value)
428
+
429
+ def _re_derive_key_columns(self) -> None:
430
+ r"""Ask this table's schema which of its columns are still keys.
431
+
432
+ A key that has just been replaced is no longer a key, and nothing said
433
+ so: the column it was kept the ``ID`` the old key gave it. Only the
434
+ schema knows whether some relationship still joins on that column, so
435
+ the question goes there; a table held on its own answers it for its own
436
+ key alone.
437
+ """
438
+ schema = getattr(self, '_schema', None)
439
+ if schema is not None:
440
+ schema._mark_key_columns()
441
+ return
442
+ named = set(_as_key_tuple(getattr(self, 'primary_key', None)))
443
+ for name, column in getattr(self, 'columns', {}).items():
444
+ if name not in named:
445
+ column._release_as_key()
446
+
447
+ def _adopt_key(self, value: str | Sequence[str]) -> None:
448
+ r"""Make the rest of the table agree with the key it was just given.
449
+
450
+ A key identifies a row, so its columns mean ``ID``. The engine's own
451
+ setter does the same thing. Any imperfection in the uniqueness
452
+ check tolerated is recorded and said out loud. Inference does all three
453
+ after nominating a key; assignment has to as well, or the two routes
454
+ describe the same table differently.
455
+
456
+ A repeat is the one visible sign that the column identifies something
457
+ coarser than a row, most often a parent on a table whose real key is
458
+ that column together with another. Naming that possibility is what
459
+ makes the warning actionable; saying only that the rows are
460
+ indistinguishable describes the symptom. A column that cannot be a key
461
+ at all has already been refused by `_check_key`.
462
+
463
+ Which is why only a column `_check_key` would take is offered. The
464
+ search filtered nothing, and the column that distinguishes two rows of
465
+ the same entity is very often the time column, as in the ordinary "two
466
+ snapshots of one account" shape. The composite key the warning
467
+ recommended was the one the next line refuses. Where the only
468
+ disambiguating column is one a key cannot hold, that is said instead
469
+ of recommending it.
470
+ """
471
+ names = _as_key_tuple(value)
472
+ for name in names:
473
+ self.columns[name]._mark_as_key()
474
+ frame = getattr(self, 'data', None)
475
+ if frame is None:
476
+ # A table in its source is counted by `Schema.from_source`, and a
477
+ # key named afterwards has not been.
478
+ object.__setattr__(self, 'duplicate_keys', None)
479
+ return
480
+ if frame.empty:
481
+ return
482
+ quality = _key_quality(frame, names)
483
+ object.__setattr__(self, 'duplicate_keys', quality.duplicates)
484
+ if quality.duplicates:
485
+ distinguishing = [
486
+ other
487
+ for other in self.columns
488
+ if other not in names
489
+ and _key_quality(frame, (*names, other)).duplicates == 0
490
+ ]
491
+ usable = [
492
+ other
493
+ for other in distinguishing
494
+ if self._could_hold_a_key(other)
495
+ ]
496
+ if usable:
497
+ remedy = (
498
+ f' {".".join(names)!r} is unique together with '
499
+ f'{", ".join(repr(other) for other in usable)}, which is '
500
+ f'what a composite key looks like; pass that as the '
501
+ f'primary key if so.'
502
+ )
503
+ elif distinguishing:
504
+ remedy = (
505
+ f' They differ only by '
506
+ f'{_and_list(distinguishing)}, which a primary key '
507
+ f'cannot be made of, so there is no composite key to name '
508
+ f'here: keep one row per {".".join(names)!r} if the model '
509
+ f'should see one, or name a different primary key.'
510
+ )
511
+ else:
512
+ remedy = _NAME_ANOTHER_KEY
513
+ _warn_a_key_repeats(
514
+ self.name, names, quality.duplicates, quality.example, remedy
515
+ )
516
+
517
+ def _could_hold_a_key(self, name: str) -> bool:
518
+ r"""Whether `_check_key` would take this column as part of a key.
519
+
520
+ The conditions it refuses on that a column carries on its own: a dtype
521
+ no identifier can be made of, being the table's time column, and
522
+ holding a null. Asked here so that what is recommended is what is
523
+ accepted. A column with one null was offered as half of a composite
524
+ key and refused by `_check_key` on the very next call.
525
+
526
+ Uniqueness is not asked here. It is a property of the combination, and
527
+ the caller of this is choosing what to combine.
528
+ """
529
+ column = self.columns.get(name)
530
+ if column is None or name == getattr(self, 'time_column', None):
531
+ return False
532
+ if not supports_dtype('ID', column.dtype):
533
+ return False
534
+ frame = getattr(self, 'data', None)
535
+ if frame is None or name not in frame.columns:
536
+ return True
537
+ return not bool(frame[name].isna().any())
538
+
539
+ def _check_name(self, name: str) -> None:
540
+ if not isinstance(name, str) or not name:
541
+ raise StructuredDataError(
542
+ f'The name {name!r} must be a non-empty string.',
543
+ code='INVALID_REQUEST',
544
+ )
545
+ name_invalid_chars = _invalid_name_chars(name)
546
+ if len(name_invalid_chars) > 0:
547
+ raise StructuredDataError(
548
+ f'The name {name!r} contains '
549
+ f'one or more invalid characters: {name_invalid_chars}.',
550
+ code='INVALID_REQUEST',
551
+ )
552
+
553
+ def _check_column(self, name: str) -> None:
554
+ if name not in getattr(self, 'columns', {}):
555
+ raise StructuredDataError(
556
+ f'table {self.name!r} has no column {name!r}; it has '
557
+ f'{sorted(getattr(self, "columns", {}))}',
558
+ code='INVALID_REQUEST',
559
+ )
560
+
561
+ def _check_key(self, value: str | Sequence[str]) -> None:
562
+ r"""A key has to exist, and has to identify a row.
563
+
564
+ Inference deliberately tolerates a nearly-unique column, because a
565
+ single duplicated row is common in an extract and still the column the
566
+ caller means. An explicit key is an assertion rather than a guess, so
567
+ it is held to the same standard the inference applies. A caller
568
+ who states something false should hear so.
569
+
570
+ A null is refused however few there are. It is not a near miss on
571
+ uniqueness: the engine drops those rows rather than scoring them, and
572
+ a single one passed silently because `duplicated` does not flag one
573
+ null.
574
+ """
575
+ names = (value,) if isinstance(value, str) else tuple(value)
576
+ if not names:
577
+ raise StructuredDataError(
578
+ f'table {self.name!r} was given an empty primary key',
579
+ code='INVALID_REQUEST',
580
+ )
581
+ for name in names:
582
+ self._check_column(name)
583
+ column = self.columns[name]
584
+ if not supports_dtype('ID', column.dtype):
585
+ raise StructuredDataError(
586
+ f'column {name!r} holds {column.dtype!r}, which cannot '
587
+ f'identify a row, so it cannot be a primary key',
588
+ code='INVALID_REQUEST',
589
+ )
590
+ if name == getattr(self, 'time_column', None):
591
+ raise StructuredDataError(
592
+ f'column {name!r} is already the time column of table '
593
+ f'{self.name!r}, so it cannot also be its primary key',
594
+ code='INVALID_REQUEST',
595
+ )
596
+ frame = getattr(self, 'data', None)
597
+ if frame is None or frame.empty:
598
+ return
599
+ quality = _key_quality(frame, names)
600
+ if quality.nulls:
601
+ empty = [
602
+ name
603
+ for name in names
604
+ if name in frame.columns and frame[name].isna().any()
605
+ ]
606
+ raise StructuredDataError(
607
+ _a_key_missing_values(self.name, names, quality.nulls, empty),
608
+ code='INVALID_REQUEST',
609
+ )
610
+ if _too_many_repeats(quality.duplicates, len(frame)):
611
+ raise StructuredDataError(
612
+ _not_unique_enough(
613
+ self.name,
614
+ names,
615
+ quality.duplicates,
616
+ len(frame),
617
+ quality.example,
618
+ ),
619
+ code='INVALID_REQUEST',
620
+ )
621
+
622
+ def _check_time_column(self, name: str) -> None:
623
+ self._check_column(name)
624
+ if name in _as_key_tuple(getattr(self, 'primary_key', None)):
625
+ raise StructuredDataError(
626
+ f'column {name!r} is already the primary key of table '
627
+ f'{self.name!r}, so it cannot also be its time column',
628
+ code='INVALID_REQUEST',
629
+ )
630
+ if name in getattr(self, 'columns', {}):
631
+ self.columns[name].stype = 'timestamp'
632
+
633
+ def __getitem__(self, column: str) -> Column:
634
+ try:
635
+ return self.columns[column]
636
+ except KeyError:
637
+ raise StructuredDataError(
638
+ f'table {self.name!r} has no column {column!r}; it has '
639
+ f'{sorted(self.columns)}',
640
+ code='INVALID_REQUEST',
641
+ ) from None
642
+
643
+ def __contains__(self, column: object) -> bool:
644
+ return column in self.columns
645
+
646
+ def __iter__(self) -> Iterator[str]:
647
+ return iter(self.columns)
648
+
649
+ def __len__(self) -> int:
650
+ return len(self.columns)
651
+
652
+
653
+ # `eq=False` on both, so the comparison above is the one they use: a
654
+ # generated one would compare the frames with `==`.
655
+ @dataclass(eq=False)
656
+ class Table(_TableShape):
657
+ r"""One table: its rows, its columns, and which column identifies a row.
658
+
659
+ Attributes:
660
+ name: The name the schema knows this table by. A query names columns
661
+ through it, so it is part of the interface rather than a label.
662
+ data: The rows.
663
+ columns: Column metadata, keyed by column name.
664
+ primary_key: The column identifying a row, or ``None``. A single-table
665
+ request needs none; a relational one needs one per table, and it is
666
+ inferred, so you rarely set it.
667
+ time_column: The column ordering rows in time, or ``None``. Only a
668
+ temporal question needs one.
669
+ """
670
+
671
+ name: str
672
+ data: pd.DataFrame
673
+ columns: dict[str, Column]
674
+ primary_key: str | Sequence[str] | None = None
675
+ time_column: str | None = None
676
+ #: How many rows repeat a primary key value another row already used.
677
+ #: Inference tolerates a few, deliberately; this is what it tolerated.
678
+ #: Not an init argument: the generated ``__init__`` assigns fields in
679
+ #: order, so a settable one would overwrite what assigning the key just
680
+ #: computed, and every explicitly-built table would report zero.
681
+ duplicate_keys: int = field(init=False, default=0)
682
+
683
+ def __post_init__(self) -> None:
684
+ r"""Check the names and the cells, and take ownership of the columns.
685
+
686
+ `Schema.from_data` checks the cells while inferring, but a hand-built
687
+ `Table` reached the adapters unchecked, where a list cell surfaced
688
+ much later as `TypeError: unhashable type: 'list'` raised from label
689
+ collection, naming nothing the caller had written.
690
+ """
691
+ self._check_name(self.name)
692
+ frame = getattr(self, 'data', None)
693
+ if frame is not None:
694
+ for column in frame.columns:
695
+ self._check_name(column)
696
+ require_scalar_cells(frame[column], f'{self.name}.{column}')
697
+ self._own_columns()
698
+
699
+ def __repr__(self) -> str:
700
+ key = f', primary_key={self.primary_key!r}' if self.primary_key else ''
701
+ time = f', time_column={self.time_column!r}' if self.time_column else ''
702
+ repeats = (
703
+ f', {self.duplicate_keys} repeated keys'
704
+ if self.duplicate_keys
705
+ else ''
706
+ )
707
+ return (
708
+ f'Table({self.name!r}, {len(self.data)} rows, '
709
+ f'{len(self.columns)} columns{key}{time}{repeats})'
710
+ )
711
+
712
+
713
+ @dataclass(eq=False)
714
+ class SourceTable(_TableShape):
715
+ r"""One table that stays in its source: its columns, and its keys.
716
+
717
+ Built by :meth:`Schema.from_source`. It holds no rows: sampling reads them
718
+ from the source, a hop at a time, when a query is materialized. Keys, time
719
+ columns and semantic types are corrected exactly as on a :class:`Table`.
720
+
721
+ Attributes:
722
+ name: The name the schema knows this table by. A query names columns
723
+ through it.
724
+ source_name: The table's name in the source.
725
+ columns: Column metadata, keyed by column name.
726
+ primary_key: The column identifying a row, or ``None``.
727
+ time_column: The column ordering rows in time, or ``None``.
728
+ """
729
+
730
+ name: str
731
+ source_name: str
732
+ columns: dict[str, Column]
733
+ primary_key: str | Sequence[str] | None = None
734
+ time_column: str | None = None
735
+ #: How many rows repeat a primary key value another row already used, as
736
+ #: :meth:`Schema.from_source` counted them in the source. Zero for a key
737
+ #: the source enforces, and for a table with no key. ``None`` where
738
+ #: nothing was counted: ``check_keys=False``, a semantic model's table,
739
+ #: or a key named after the schema was built.
740
+ duplicate_keys: int | None = field(init=False, default=None)
741
+
742
+ def __post_init__(self) -> None:
743
+ self._check_name(self.name)
744
+ if not isinstance(self.source_name, str) or not self.source_name:
745
+ raise StructuredDataError(
746
+ f'table {self.name!r} needs the name it has in its source, '
747
+ f'got {self.source_name!r}',
748
+ code='INVALID_REQUEST',
749
+ )
750
+ for column in self.columns:
751
+ self._check_name(column)
752
+ self._own_columns()
753
+
754
+ def __delitem__(self, column: str) -> None:
755
+ r"""Leave a column out: it is neither read from the source nor sent.
756
+
757
+ >>> del schema['orders']['rowguid']
758
+
759
+ A table in its source cannot be edited to suit a request, and some
760
+ columns only mislead a model: a per-row UUID identifies a row and
761
+ says nothing about it. A table of frames drops a column from its frame
762
+ before :meth:`Schema.from_data` instead.
763
+ """
764
+ self._check_column(column)
765
+ role = (
766
+ 'primary key'
767
+ if column in _as_key_tuple(self.primary_key)
768
+ else 'time column'
769
+ if column == self.time_column
770
+ else None
771
+ )
772
+ if role is not None:
773
+ raise StructuredDataError(
774
+ f"{self.name}.{column} is the table's {role}; set that to "
775
+ f'something else, or None, before leaving the column out',
776
+ code='INVALID_REQUEST',
777
+ )
778
+ del self.columns[column]
779
+
780
+ def __repr__(self) -> str:
781
+ key = f', primary_key={self.primary_key!r}' if self.primary_key else ''
782
+ time = f', time_column={self.time_column!r}' if self.time_column else ''
783
+ return (
784
+ f'SourceTable({self.name!r}, from {self.source_name!r}, '
785
+ f'{len(self.columns)} columns{key}{time})'
786
+ )
787
+
788
+
789
+ @dataclass(frozen=True)
790
+ class Relationship:
791
+ r"""A foreign key: ``source_table.source_column`` points at a table's key.
792
+
793
+ Directed from the many side to the one side, which is the direction a join
794
+ reads and the direction the wire envelope carries.
795
+ """
796
+
797
+ source_table: str
798
+ source_column: str
799
+ target_table: str
800
+ target_column: str
801
+
802
+ def __repr__(self) -> str:
803
+ return (
804
+ f'{self.source_table}.{self.source_column} -> '
805
+ f'{self.target_table}.{self.target_column}'
806
+ )
807
+
808
+
809
+ class Schema:
810
+ r"""The tables a request is about.
811
+
812
+ Build one with :meth:`from_data`. Index it by table name, and index a table
813
+ by column name:
814
+
815
+ >>> schema = Schema.from_data({'users': users, 'orders': orders})
816
+ >>> schema['orders']['price'].stype
817
+ 'numerical'
818
+
819
+ One table and many tables are the same object. A single-table model simply
820
+ rejects a schema that has relationships, and says so before anything is
821
+ sent.
822
+ """
823
+
824
+ def check_consistent(self) -> None:
825
+ r"""Refuse any column whose storage and meaning contradict, and any
826
+ relationship that cannot join.
827
+
828
+ Asked once, as the schema becomes a request, because a caller
829
+ re-declaring a column sets its storage and its meaning one at a time
830
+ and either order passes through a pair that does not hold.
831
+
832
+ The relationships are asked about here as well as on assignment.
833
+ `relationships` is an ordinary list, so `schema.relationships.append(
834
+ link)` reaches it without passing the setter, and this is the question
835
+ a caller runs to find out whether their schema holds together. It
836
+ should not be the one place that does not look.
837
+
838
+ Which columns are keys is re-derived first. `tables` and a table's
839
+ `columns` are ordinary dictionaries, so a table or a column replaced
840
+ through one arrives never having been asked, and a relationship key
841
+ left typed ``numerical`` would otherwise pass this and be sent as a
842
+ float.
843
+ """
844
+ self._mark_key_columns()
845
+ for table in self.tables.values():
846
+ for column in table.columns.values():
847
+ column.check_consistent()
848
+ self._check_relationships(self.relationships)
849
+
850
+ def __init__(
851
+ self,
852
+ tables: Mapping[str, Table | SourceTable],
853
+ relationships: Sequence[Relationship] = (),
854
+ *,
855
+ source: Source | None = None,
856
+ ) -> None:
857
+ _require_one_kind_of_table(tables, source)
858
+ self._source = source
859
+ # Copied, because a schema types the key columns of its own tables
860
+ # and two schemas handed the same ones fought over them: building an
861
+ # unlinked schema retyped a linked schema's join key, and checking the
862
+ # linked one retyped the unlinked one's back.
863
+ self._tables = {
864
+ name: _copy_table(table) for name, table in tables.items()
865
+ }
866
+ self.relationships = relationships
867
+ # The engine's own reading of the source's tables, kept so that each
868
+ # question does not read them again. Set by the first read.
869
+ self._engine_tables: Any = None
870
+ self._conversion_notes: tuple[str, ...] = ()
871
+
872
+ @property
873
+ def source(self) -> Source | None:
874
+ r"""Where the tables are, for a schema built with :meth:`from_source`.
875
+
876
+ ``None`` for a schema of frames. A schema has one source or none,
877
+ because sampling runs against one: see :meth:`from_source`.
878
+ """
879
+ return self._source
880
+
881
+ @property
882
+ def conversion_notes(self) -> tuple[str, ...]:
883
+ r"""How :meth:`from_source` read a semantic model where nothing was
884
+ lost, for a person to read.
885
+
886
+ For a Databricks metric view, one note each: the predictive query
887
+ aggregation that matches a simple measure, such as
888
+ ``'revenue' is SUM(order_lines.LineTotal, 0, 30, days)``, the time
889
+ column taken in place of a date the view truncates, unless
890
+ ``time_columns`` named another, and a parameter read as its default.
891
+ Tables are named as the schema names them. What a model loses is
892
+ said in an :class:`SDMWarning` instead, and a time column taken in
893
+ place of a truncated date is said in the warning on how time columns
894
+ were inferred as well, with the override that names another. Empty
895
+ for every other schema.
896
+
897
+ The wording of a note is not part of the contract, so do not parse
898
+ it.
899
+ """
900
+ return self._conversion_notes
901
+
902
+ def __setattr__(self, key: str, value: Any) -> None:
903
+ r"""Hold an assigned relationship to the same standard as a passed one.
904
+
905
+ `Schema.from_data(relationships=[link])` checks that both columns exist
906
+ and that their dtypes can join; `schema.relationships = [link]` is the
907
+ same statement and checked nothing, so a link that one route refuses
908
+ the other accepted. `check_consistent()`, which a caller runs to
909
+ ask exactly this, passed it.
910
+ """
911
+ if key == 'relationships' and hasattr(self, '_tables'):
912
+ value = _Links(self._settle(value), self)
913
+ object.__setattr__(self, key, value)
914
+ self._mark_key_columns()
915
+ return
916
+ object.__setattr__(self, key, value)
917
+
918
+ def __copy__(self) -> Schema:
919
+ r"""A shallow copy is still a schema of its own tables.
920
+
921
+ `copy.copy` shared the tables and their columns, so editing the copy
922
+ edited the original. This was always a trap, and became a
923
+ destructive one once key columns were re-derived: clearing the copy's
924
+ relationships retyped a column of a schema that still had them.
925
+
926
+ The rows are still shared, as they are in `snapshot()`: they are the
927
+ data, and copying a warehouse extract is not what a caller asking for
928
+ a copy of the schema is asking for.
929
+ """
930
+ return self.snapshot()
931
+
932
+ def __setstate__(self, state: dict[str, Any]) -> None:
933
+ r"""Restore a copied or unpickled schema with its links still checked.
934
+
935
+ Pickle and `deepcopy` put the attributes back without going through
936
+ `__setattr__`, so the copy would hold a plain list and quietly stop
937
+ checking what is added to it. Said nothing while it does: a copy is
938
+ not a new declaration, and the notices were given when the schema it
939
+ was copied from was built.
940
+
941
+ A schema pickled before it could be bound to a source has neither
942
+ that source nor the engine's reading of it, which is what it is: a
943
+ schema of frames.
944
+ """
945
+ object.__setattr__(
946
+ self,
947
+ '__dict__',
948
+ {
949
+ '_source': None,
950
+ '_engine_tables': None,
951
+ '_conversion_notes': (),
952
+ **state,
953
+ },
954
+ )
955
+ with warnings.catch_warnings():
956
+ warnings.simplefilter('ignore', SDMWarning)
957
+ self.relationships = list(state.get('relationships') or ())
958
+
959
+ def _settle(self, links: Sequence[Relationship]) -> list[Relationship]:
960
+ r"""The relationships as this schema will hold them, having said so.
961
+
962
+ One place, so that what is checked, de-duplicated and warned about does
963
+ not depend on how the link was added.
964
+ """
965
+ given = list(links)
966
+ self._check_relationships(given)
967
+ settled = _each_link_once(given)
968
+ _warn_two_ways_to_reach_one_table(self._tables, settled)
969
+ return settled
970
+
971
+ def _check_relationships(self, links: Sequence[Relationship]) -> None:
972
+ for link in links:
973
+ if not isinstance(link, Relationship):
974
+ raise StructuredDataError(
975
+ f'a relationship must be a Relationship, got '
976
+ f'{type(link).__name__}: {link!r}. Build one with '
977
+ f'Relationship(source_table, source_column, '
978
+ f'target_table, target_column)',
979
+ code='INVALID_REQUEST',
980
+ )
981
+ _require_column(self._tables, link.source_table, link.source_column)
982
+ _require_column(self._tables, link.target_table, link.target_column)
983
+ _require_joinable(self._tables, link)
984
+
985
+ def snapshot(self) -> Schema:
986
+ r"""This schema as it stands, so a later edit cannot reach back.
987
+
988
+ `materialize()` is the gathering step and what it returns is read as
989
+ finished, but the request kept the caller's schema by reference and the
990
+ adapter reads it again at send time. Editing a column's meaning
991
+ after materializing changed what was sent, silently, and an agent
992
+ reusing one schema across several questions had no way to see it.
993
+
994
+ The rows are shared rather than copied: they are the data, they are
995
+ what the request is about, and duplicating a warehouse extract per
996
+ question to guard against an edit nobody makes to a frame would cost
997
+ more than the defect. What is copied is everything a schema *says*.
998
+ """
999
+ copied = Schema(
1000
+ {name: _copy_table(table) for name, table in self._tables.items()},
1001
+ list(self.relationships),
1002
+ source=self._source,
1003
+ )
1004
+ # What the engine read of the source is shared, not copied: it is what
1005
+ # the tables are, and every gather is given its own copy of it.
1006
+ copied._engine_tables = self._engine_tables
1007
+ copied._conversion_notes = self._conversion_notes
1008
+ return copied
1009
+
1010
+ @classmethod
1011
+ def from_data(
1012
+ cls,
1013
+ tables: Mapping[str, pd.DataFrame],
1014
+ *,
1015
+ primary_keys: Mapping[str, str | Sequence[str]] | None = None,
1016
+ time_columns: Mapping[str, str | None] | None = None,
1017
+ relationships: Sequence[Relationship] | None = None,
1018
+ ) -> Schema:
1019
+ r"""Infer a schema from frames.
1020
+
1021
+ Args:
1022
+ tables: Frames by table name. The name matters: a predictive query
1023
+ names columns through it.
1024
+ primary_keys: Overrides for the inferred key of a table. A
1025
+ sequence of column names declares a composite key, which the
1026
+ setter has always taken; the annotation said otherwise and so
1027
+ pushed anyone with one onto the assignment route, which until
1028
+ recently validated nothing.
1029
+ time_columns: Overrides for the inferred time column of a table.
1030
+ ``None`` says the table has none.
1031
+ relationships: Overrides for the inferred relationships. Pass an
1032
+ empty sequence to assert there are none.
1033
+
1034
+ Returns:
1035
+ A schema with every dtype, semantic type, key, time column and
1036
+ relationship filled in.
1037
+
1038
+ Raises:
1039
+ StructuredDataError: If ``tables`` is empty, a value is not a
1040
+ DataFrame, a name is not a string, or an override names
1041
+ something that does not exist.
1042
+
1043
+ An inferred relationship is held to the same contract as an explicit
1044
+ one, and dropped rather than refused when it fails. Inference matches
1045
+ on name alone, so it can pair columns whose dtypes cannot join, and
1046
+ `query.py` would send that edge unchanged; a caller who wrote the
1047
+ relationship is told it cannot join, while a caller who never asked
1048
+ for it is not refused for a guess this made on their behalf.
1049
+ """
1050
+ if not tables:
1051
+ raise StructuredDataError(
1052
+ 'a schema needs at least one table', code='INVALID_REQUEST'
1053
+ )
1054
+
1055
+ built: dict[str, Table] = {}
1056
+ chose: dict[str, TimeColumnChoice | None] = {}
1057
+ for name, frame in tables.items():
1058
+ if not isinstance(name, str) or not name:
1059
+ raise StructuredDataError(
1060
+ f'table names must be non-empty strings; got {name!r}',
1061
+ code='INVALID_REQUEST',
1062
+ )
1063
+ if not isinstance(frame, pd.DataFrame):
1064
+ raise StructuredDataError(
1065
+ f'table {name!r} must be a pandas DataFrame, got '
1066
+ f'{type(frame).__name__}',
1067
+ code='INVALID_REQUEST',
1068
+ )
1069
+ built[name], chose[name] = _infer_table_and_time_column(name, frame)
1070
+
1071
+ _apply_keys_and_times(built, primary_keys, time_columns)
1072
+ if relationships is None:
1073
+ inferred = _joinable_inferred_links(
1074
+ built, _infer_relationships(built)
1075
+ )
1076
+ else:
1077
+ inferred = _checked_links(built, relationships)
1078
+
1079
+ _settle_edit_stamps(built, chose, time_columns, inferred)
1080
+ with warnings.catch_warnings():
1081
+ warnings.simplefilter('ignore', SDMWarning)
1082
+ schema = cls(built, inferred)
1083
+ schema._mark_key_columns()
1084
+ dates = {
1085
+ name: named
1086
+ for name, table in built.items()
1087
+ if (named := _warn_dates_stored_as_text(table)) is not None
1088
+ }
1089
+ if relationships is None:
1090
+ spoken_for = _warn_repeats_cost_the_table_its_key(built)
1091
+ _warn_table_cannot_be_linked_to(built, dates, spoken_for)
1092
+ _warn_one_key_name_keys_several_tables(built)
1093
+ _warn_stype_was_decided_by_the_rows_read(built)
1094
+ _warn_how_time_columns_were_inferred(schema, chose, time_columns)
1095
+ _warn_two_ways_to_reach_one_table(built, schema.relationships)
1096
+ return schema
1097
+
1098
+ @classmethod
1099
+ def from_source(
1100
+ cls,
1101
+ source: Source,
1102
+ tables: Mapping[str, str] | None = None,
1103
+ *,
1104
+ primary_keys: Mapping[str, str | Sequence[str]] | None = None,
1105
+ time_columns: Mapping[str, str | None] | None = None,
1106
+ relationships: Sequence[Relationship] | None = None,
1107
+ check_keys: bool = True,
1108
+ ) -> Schema:
1109
+ r"""Describe tables that stay in their source.
1110
+
1111
+ >>> source = DatabricksSource(spark, catalog='main', schema='sales')
1112
+ >>> schema = Schema.from_source(
1113
+ ... source,
1114
+ ... {'customers': 'adventureworks_customers',
1115
+ ... 'orders': 'adventureworks_orders'},
1116
+ ... time_columns={'orders': 'order_date'},
1117
+ ... )
1118
+
1119
+ Nothing is read into memory. Column types come from the source's
1120
+ catalog and semantic types from a preview of at most 1,000 rows. Keys
1121
+ and relationships come from the constraints the catalog declares, and
1122
+ where it declares none, from names, by the same rule as
1123
+ :meth:`from_data`. When a query is materialized, the engine samples
1124
+ the tables with SQL, a hop at a time, and only the rows it samples
1125
+ leave the source.
1126
+
1127
+ Each key is then checked against every row, by the rules
1128
+ :meth:`from_data` applies to a frame. Databricks and Snowflake do not
1129
+ enforce a declared key, and a key inferred from a preview has only
1130
+ been seen to hold there, so a repeated value would otherwise surface
1131
+ as rows that change between seeds and context examples that
1132
+ contradict each other. A declared key, or one named in
1133
+ ``primary_keys``, warns when up to 1% of rows repeat a value and is
1134
+ refused beyond that; an inferred key is declined beyond it. The count
1135
+ is one statement for all the tables, a full scan of each, and it is
1136
+ skipped for a key the source enforces, such as a PostgreSQL or DuckDB
1137
+ primary key. :attr:`SourceTable.duplicate_keys` holds what it found.
1138
+
1139
+ A semantic model, such as a Databricks metric view or a Snowflake
1140
+ semantic view, is a source of its own, and already says how its
1141
+ tables join:
1142
+
1143
+ >>> schema = Schema.from_source(source.semantic_model('sales_metrics'))
1144
+
1145
+ Its keys and relationships come from the model, and its tables are
1146
+ named as the model names them. No relationship is inferred from
1147
+ column names beside the model's own joins, which would give a table
1148
+ a second path the model never declared. ``relationships=`` replaces
1149
+ the model's own, so to add one, pass them with it:
1150
+ ``relationships=[*Schema.from_source(model).relationships, extra]``.
1151
+ What the model has that a schema cannot hold is said in one
1152
+ :class:`SDMWarning`, and how it was read where nothing was lost, such
1153
+ as the aggregation each measure is, is in :attr:`conversion_notes`.
1154
+
1155
+ Everything that differs between warehouses belongs to the source, so
1156
+ this reads the same whichever one the tables are in. A schema has
1157
+ exactly one source: sampling sends the ids each hop found to the next
1158
+ query, so it runs against one engine.
1159
+
1160
+ Args:
1161
+ source: Where the tables are: an ``sdm_connectors.Source``, or a
1162
+ ``SemanticModel`` got from one.
1163
+ tables: For each name a query will use, the table's name in the
1164
+ source. Left out, every table the source describes is taken,
1165
+ under its own name: every table in the namespace, or every
1166
+ table of the semantic model.
1167
+ primary_keys: Overrides for a table's key.
1168
+ time_columns: Overrides for a table's time column. ``None``
1169
+ says the table has none.
1170
+ relationships: Replaces the relationships, a semantic model's
1171
+ own included. Pass an empty sequence to assert there are
1172
+ none.
1173
+ check_keys: Count each key's repeats and missing values in the
1174
+ source. ``False`` skips the scan, and the checks with it, and
1175
+ leaves :attr:`SourceTable.duplicate_keys` ``None``. The tables
1176
+ of a semantic model are never counted: the model derives them.
1177
+
1178
+ Returns:
1179
+ A schema of :class:`SourceTable`, bound to ``source``.
1180
+
1181
+ Raises:
1182
+ StructuredDataError: If ``tables`` is empty, a name is not a
1183
+ string, a table is not in the source, the source describes no
1184
+ table, an override names something that does not exist, or a
1185
+ declared or named key repeats a value in more than 1% of rows
1186
+ or a named key has no value in some row.
1187
+ MissingExtraError: If the relational engine, which reads the
1188
+ source, is not installed.
1189
+ GraphConstructionError: The relational engine's, if the
1190
+ warehouse fails while a semantic model is read, with the
1191
+ driver's error as its ``__cause__``.
1192
+ EmptyTableError: The relational engine's, a ``RuntimeError``, if
1193
+ a semantic model's source table holds no rows.
1194
+ """
1195
+ from sdm_client.materializers.sources import (
1196
+ EngineTables,
1197
+ engine_graph,
1198
+ is_semantic_model,
1199
+ semantic_model_graph,
1200
+ )
1201
+
1202
+ _require_a_source(source)
1203
+ if tables is not None:
1204
+ _require_table_names(tables)
1205
+
1206
+ read = EngineTables()
1207
+ engine_tables: dict[str, Any] | None = None
1208
+ chose: dict[str, TimeColumnChoice | None] = {}
1209
+ notes: tuple[str, ...] = ()
1210
+ if is_semantic_model(source):
1211
+ graph = semantic_model_graph(source)
1212
+ read.keep({name: graph[name] for name in graph.tables})
1213
+ offered = list(graph.tables)
1214
+ chosen = (
1215
+ dict(tables)
1216
+ if tables is not None
1217
+ else {name: name for name in offered}
1218
+ )
1219
+ absent = sorted(set(chosen.values()) - set(offered))
1220
+ if absent:
1221
+ raise StructuredDataError(
1222
+ f'{absent} are not tables of {source!r}; it has '
1223
+ f'{sorted(offered)}',
1224
+ code='INVALID_REQUEST',
1225
+ )
1226
+ as_named = {model: name for name, model in chosen.items()}
1227
+ built = {
1228
+ name: _describe_source_table(graph[model], model, name=name)
1229
+ for name, model in chosen.items()
1230
+ }
1231
+ chose = {
1232
+ name: graph[model].time_column_choice
1233
+ for name, model in chosen.items()
1234
+ }
1235
+ declared = _model_links(graph, built, as_named)
1236
+ else:
1237
+ graph = engine_graph(source, tables, infer_metadata=True)
1238
+ chosen = (
1239
+ dict(tables)
1240
+ if tables is not None
1241
+ else {name: name for name in graph.tables}
1242
+ )
1243
+ read.keep({named: graph[name] for name, named in chosen.items()})
1244
+ built = {
1245
+ name: _describe_source_table(graph[name], named)
1246
+ for name, named in chosen.items()
1247
+ }
1248
+ chose = {name: graph[name].time_column_choice for name in chosen}
1249
+ declared = _declared_links(graph, chosen, built)
1250
+ if relationships is None:
1251
+ _key_by_the_referenced_column(built, declared)
1252
+ engine_tables = {name: graph[name] for name in chosen}
1253
+
1254
+ found = {
1255
+ name: _as_key_tuple(table.primary_key)
1256
+ for name, table in built.items()
1257
+ }
1258
+ _apply_keys_and_times(built, primary_keys, time_columns)
1259
+ if is_semantic_model(source):
1260
+ notes = _schema_notes(graph.conversion_notes, built, as_named)
1261
+ if check_keys and engine_tables is not None:
1262
+ # Before relationships are inferred: a declined key is one no
1263
+ # relationship can point at.
1264
+ _check_source_keys(
1265
+ source,
1266
+ built,
1267
+ engine_tables,
1268
+ found,
1269
+ set(primary_keys or ()),
1270
+ infer_links=relationships is None,
1271
+ )
1272
+ else:
1273
+ for table in built.values():
1274
+ object.__setattr__(table, 'duplicate_keys', None)
1275
+ if relationships is None:
1276
+ covered = {
1277
+ (link.source_table, link.source_column) for link in declared
1278
+ }
1279
+ # A semantic model's joins are all the links it means.
1280
+ by_name = (
1281
+ []
1282
+ if is_semantic_model(source)
1283
+ else [
1284
+ link
1285
+ for link in _infer_relationships(built)
1286
+ if (link.source_table, link.source_column) not in covered
1287
+ ]
1288
+ )
1289
+ links = _joinable_inferred_links(built, declared + by_name)
1290
+ else:
1291
+ links = _checked_links(built, relationships)
1292
+
1293
+ _settle_edit_stamps(built, chose, time_columns, links)
1294
+ with warnings.catch_warnings():
1295
+ warnings.simplefilter('ignore', SDMWarning)
1296
+ schema = cls(built, links, source=source)
1297
+ schema._engine_tables = read
1298
+ schema._conversion_notes = notes
1299
+ schema._mark_key_columns()
1300
+ _warn_how_time_columns_were_inferred(schema, chose, time_columns)
1301
+ _warn_two_ways_to_reach_one_table(built, schema.relationships)
1302
+ return schema
1303
+
1304
+ @property
1305
+ def tables(self) -> dict[str, Table | SourceTable]:
1306
+ r"""The tables, by name."""
1307
+ return self._tables
1308
+
1309
+ def __getitem__(self, table: str) -> Table | SourceTable:
1310
+ try:
1311
+ return self._tables[table]
1312
+ except KeyError:
1313
+ raise StructuredDataError(
1314
+ f'no table {table!r} in this schema; it has '
1315
+ f'{sorted(self._tables)}',
1316
+ code='INVALID_REQUEST',
1317
+ ) from None
1318
+
1319
+ def __contains__(self, table: object) -> bool:
1320
+ return table in self._tables
1321
+
1322
+ def __iter__(self) -> Iterator[str]:
1323
+ return iter(self._tables)
1324
+
1325
+ def __len__(self) -> int:
1326
+ return len(self._tables)
1327
+
1328
+ def _own_tables(self) -> None:
1329
+ r"""Take a copy of any table that still belongs to another schema.
1330
+
1331
+ `tables` hands back the live dictionary, so a table can arrive from a
1332
+ schema that is still using it. Typing its key columns for this
1333
+ schema would retype them for that one. The same rule a table applies
1334
+ to the columns it is handed.
1335
+ """
1336
+ borrowed = [
1337
+ name
1338
+ for name, table in self._tables.items()
1339
+ if (owner := getattr(table, '_schema', None)) is not None
1340
+ and owner is not self
1341
+ ]
1342
+ for name in borrowed:
1343
+ self._tables[name] = _copy_table(self._tables[name])
1344
+ for table in self._tables.values():
1345
+ object.__setattr__(table, '_schema', self)
1346
+ # And the columns inside them, for the same reason: a column
1347
+ # inserted through `table.columns` arrives still belonging to the
1348
+ # table it came from, and typing it here typed it there.
1349
+ table._own_columns()
1350
+
1351
+ def _mark_key_columns(self) -> None:
1352
+ r"""Mark columns this schema joins on or no longer joins on.
1353
+
1354
+ Derived from the schema as it stands rather than applied once when it
1355
+ was built. Marking only on the way in meant a relationship dropped
1356
+ afterwards left its source column typed ``ID``, so two schemas with
1357
+ the same tables and no relationships asked different questions: one
1358
+ a regression and the other a multiclass classification, according to a
1359
+ link only one of them had ever had.
1360
+ """
1361
+ self._own_tables()
1362
+ keys: dict[str, set[str]] = {}
1363
+ for name, table in self._tables.items():
1364
+ keys[name] = {
1365
+ column
1366
+ for column in _as_key_tuple(table.primary_key)
1367
+ if column in table.columns
1368
+ }
1369
+ for link in self.relationships:
1370
+ source = self._tables.get(link.source_table)
1371
+ if source is not None and link.source_column in source.columns:
1372
+ keys[link.source_table].add(link.source_column)
1373
+
1374
+ for name, table in self._tables.items():
1375
+ for column_name, column in table.columns.items():
1376
+ # A column replaced through the ordinary `columns` dictionary
1377
+ # arrives with no idea which table it is in, and could not be
1378
+ # asked about again.
1379
+ object.__setattr__(column, '_table', table)
1380
+ if column_name in keys[name]:
1381
+ column._mark_as_key()
1382
+ else:
1383
+ column._release_as_key()
1384
+
1385
+ def __repr__(self) -> str:
1386
+ return (
1387
+ f'Schema({len(self._tables)} tables: {sorted(self._tables)}, '
1388
+ f'{len(self.relationships)} relationships)'
1389
+ )
1390
+
1391
+
1392
+ def _require_table(tables: Mapping[str, _T], name: str) -> _T:
1393
+ if name not in tables:
1394
+ raise StructuredDataError(
1395
+ f'no table {name!r} in this schema; it has {sorted(tables)}',
1396
+ code='INVALID_REQUEST',
1397
+ )
1398
+ return tables[name]
1399
+
1400
+
1401
+ def _schema_notes(
1402
+ notes: Sequence[Any],
1403
+ built: Mapping[str, _TableShape],
1404
+ as_named: Mapping[str, str],
1405
+ ) -> tuple[str, ...]:
1406
+ r"""A semantic model's conversion notes as text, naming its tables as
1407
+ the schema names them.
1408
+
1409
+ A note about a table the schema leaves out, or about a time column its
1410
+ table no longer has, says nothing the schema holds, so it is left out.
1411
+
1412
+ Args:
1413
+ notes: The engine's notes on how it read the model.
1414
+ built: The schema's tables, by name.
1415
+ as_named: The schema's name for each of the model's tables it holds.
1416
+ """
1417
+ from sdm_relational_engine.rfm.view_conversion import (
1418
+ MeasureNote,
1419
+ TimeColumnNote,
1420
+ )
1421
+
1422
+ said: list[str] = []
1423
+ for note in notes:
1424
+ if isinstance(note, MeasureNote | TimeColumnNote):
1425
+ name = as_named.get(note.table)
1426
+ if name is None or (
1427
+ isinstance(note, TimeColumnNote)
1428
+ and built[name].time_column != note.column
1429
+ ):
1430
+ continue
1431
+ note = replace(note, table=name)
1432
+ said.append(str(note))
1433
+ return tuple(said)
1434
+
1435
+
1436
+ def _apply_keys_and_times(
1437
+ built: Mapping[str, _TableShape],
1438
+ primary_keys: Mapping[str, str | Sequence[str]] | None,
1439
+ time_columns: Mapping[str, str | None] | None,
1440
+ ) -> None:
1441
+ for name, key in (primary_keys or {}).items():
1442
+ table = _require_table(built, name)
1443
+ # Read once, and the names are what is passed on. Reading it here
1444
+ # to check the columns exist and then handing the original along
1445
+ # emptied a key that can only be read once, exactly as the setter
1446
+ # used to before it read once itself.
1447
+ names = _as_key_tuple(key)
1448
+ for column in names:
1449
+ _require_column(built, name, column)
1450
+ table.primary_key = key if isinstance(key, str) else names
1451
+ for name, named in (time_columns or {}).items():
1452
+ _require_table(built, name).time_column = (
1453
+ None if named is None else _require_column(built, name, named)
1454
+ )
1455
+
1456
+
1457
+ def _joinable_inferred_links(
1458
+ built: Mapping[str, _TableShape], inferred: Sequence[Relationship]
1459
+ ) -> list[Relationship]:
1460
+ r"""The inferred links that can join, having said which could not.
1461
+
1462
+ An inferred relationship is held to the same contract as an explicit one,
1463
+ and dropped rather than refused when it fails: a caller who never asked
1464
+ for it is not refused for a guess made on their behalf.
1465
+ """
1466
+ unjoinable = [link for link in inferred if not _is_joinable(built, link)]
1467
+ _warn_of_a_link_that_cannot_join(built, unjoinable)
1468
+ return [link for link in inferred if _is_joinable(built, link)]
1469
+
1470
+
1471
+ def _key_by_the_referenced_column(
1472
+ built: Mapping[str, SourceTable], declared: Sequence[Relationship]
1473
+ ) -> None:
1474
+ r"""Key a keyless table by the column declared foreign keys refer to.
1475
+
1476
+ A catalog lets a foreign key refer only to a primary key or a unique
1477
+ column, so that column identifies a row, as a lone unique column does when
1478
+ the engine infers a key. Before the keys are checked, so this one is too.
1479
+ Foreign keys naming different columns of one table leave it keyless.
1480
+ """
1481
+ referenced: dict[str, set[str]] = {}
1482
+ for link in declared:
1483
+ referenced.setdefault(link.target_table, set()).add(link.target_column)
1484
+ for name, columns in referenced.items():
1485
+ if built[name].primary_key is None and len(columns) == 1:
1486
+ built[name].primary_key = next(iter(columns))
1487
+
1488
+
1489
+ def _checked_links(
1490
+ built: Mapping[str, _TableShape], relationships: Sequence[Relationship]
1491
+ ) -> list[Relationship]:
1492
+ links = list(relationships)
1493
+ for link in links:
1494
+ if not isinstance(link, Relationship):
1495
+ raise StructuredDataError(
1496
+ f'a relationship must be a Relationship, got '
1497
+ f'{type(link).__name__}: {link!r}. Build one with '
1498
+ f'Relationship(source_table, source_column, '
1499
+ f'target_table, target_column)',
1500
+ code='INVALID_REQUEST',
1501
+ )
1502
+ _require_column(built, link.source_table, link.source_column)
1503
+ _require_column(built, link.target_table, link.target_column)
1504
+ _require_joinable(built, link)
1505
+ return links
1506
+
1507
+
1508
+ #: How a message names each source's engine.
1509
+ _BACKEND_NAMES = {
1510
+ 'databricks': 'Databricks',
1511
+ 'snowflake': 'Snowflake',
1512
+ 'postgres': 'PostgreSQL',
1513
+ 'duckdb': 'DuckDB',
1514
+ 'sqlite': 'SQLite',
1515
+ }
1516
+
1517
+ _SKIP_THE_CHECK = (
1518
+ 'Name another key with primary_keys=, or pass check_keys=False to skip '
1519
+ '(one full scan per table).'
1520
+ )
1521
+
1522
+
1523
+ def _check_source_keys(
1524
+ source: Any,
1525
+ built: Mapping[str, SourceTable],
1526
+ engine_tables: Mapping[str, Any],
1527
+ found: Mapping[str, tuple[str, ...]],
1528
+ stated: AbstractSet[str],
1529
+ *,
1530
+ infer_links: bool,
1531
+ ) -> None:
1532
+ r"""Hold the keys of tables in a source to the rules a key of frames
1533
+ meets, counting the rows where they are.
1534
+
1535
+ A stated key, from ``primary_keys=`` or the catalog, warns at up to 1% of
1536
+ rows repeating a value and is refused above that. A key the engine
1537
+ inferred is kept up to the same 1% and declined beyond it, as inference
1538
+ over frames declines it. A key the source enforces is not counted.
1539
+
1540
+ Args:
1541
+ source: The source the tables are in.
1542
+ built: The tables, keyed as they will be.
1543
+ engine_tables: The engine's reading of each table, by schema name.
1544
+ found: The key each table had before ``primary_keys=`` was applied.
1545
+ stated: The tables whose key ``primary_keys=`` named.
1546
+ infer_links: Whether relationships will be inferred, which is when a
1547
+ declined key is worth a word.
1548
+ """
1549
+ from sdm_relational_engine.rfm.base import count_keys
1550
+
1551
+ for table in built.values():
1552
+ object.__setattr__(table, 'duplicate_keys', 0)
1553
+ keyed = {
1554
+ name: _as_key_tuple(table.primary_key)
1555
+ for name, table in built.items()
1556
+ if table.primary_key is not None
1557
+ }
1558
+ if not keyed:
1559
+ return
1560
+ counts = dict(
1561
+ zip(
1562
+ keyed,
1563
+ count_keys([(engine_tables[name], keyed[name]) for name in keyed]),
1564
+ strict=True,
1565
+ )
1566
+ )
1567
+ backend = _BACKEND_NAMES.get(source.backend, source.backend)
1568
+ declined: dict[str, Any] = {}
1569
+ for name, names in keyed.items():
1570
+ count = counts[name]
1571
+ rows, distinct = count.rows, count.distinct
1572
+ if rows is None or distinct is None:
1573
+ continue
1574
+ table = built[name]
1575
+ declared = (
1576
+ name not in stated
1577
+ and names == found[name]
1578
+ and engine_tables[name]._declares_key(names)
1579
+ )
1580
+ repeats = count.repeats
1581
+ if name not in stated and not declared:
1582
+ # Inferred, so declined as inference over a frame declines it.
1583
+ if distinct < rows * _KEY_UNIQUENESS:
1584
+ declined[name] = count
1585
+ table.primary_key = None
1586
+ continue
1587
+ elif count.nulls and name in stated:
1588
+ empty = names if len(names) == 1 else None
1589
+ missing = _a_key_missing_values(name, names, count.nulls, empty)
1590
+ raise StructuredDataError(
1591
+ f'{missing}. {_SKIP_THE_CHECK}', code='INVALID_REQUEST'
1592
+ )
1593
+ elif _too_many_repeats(repeats, rows):
1594
+ found_out = _not_unique_enough(
1595
+ name, names, repeats, rows, count.example
1596
+ )
1597
+ unenforced = (
1598
+ f'{backend} does not enforce declared keys. '
1599
+ if declared and not count.enforced
1600
+ else ''
1601
+ )
1602
+ raise StructuredDataError(
1603
+ f'{found_out}. {unenforced}{_SKIP_THE_CHECK}',
1604
+ code='INVALID_REQUEST',
1605
+ )
1606
+ if count.nulls:
1607
+ _warn_a_key_has_no_value(name, '.'.join(names), count.nulls)
1608
+ if repeats:
1609
+ _warn_a_key_repeats(
1610
+ name, names, repeats, count.example, _NAME_ANOTHER_KEY
1611
+ )
1612
+ object.__setattr__(table, 'duplicate_keys', repeats)
1613
+
1614
+ if not infer_links or not declined:
1615
+ return
1616
+ named_elsewhere = _tables_naming_each_column(built)
1617
+ keyed_by = {
1618
+ table.primary_key
1619
+ for table in built.values()
1620
+ if isinstance(table.primary_key, str)
1621
+ }
1622
+ for name, count in declined.items():
1623
+ column = keyed[name][0]
1624
+ rows, distinct = count.rows or 0, count.distinct or 0
1625
+ if (
1626
+ len(keyed[name]) > 1
1627
+ or column in keyed_by
1628
+ or not named_elsewhere.get(column, set()) - {name}
1629
+ or distinct < rows * _NEARLY_A_KEY
1630
+ or not count.repeats
1631
+ ):
1632
+ continue
1633
+ _warn_a_key_was_declined(
1634
+ name,
1635
+ column,
1636
+ count.repeats,
1637
+ rows,
1638
+ count.example,
1639
+ f'Keep one row per {column!r} in the source, or name a column '
1640
+ f'that does identify a row with primary_keys={{{name!r}: ...}}.',
1641
+ )
1642
+
1643
+
1644
+ def _require_table_names(tables: Mapping[str, str]) -> None:
1645
+ if not tables:
1646
+ raise StructuredDataError(
1647
+ 'a schema needs at least one table; leave tables out to take every '
1648
+ 'table the source describes',
1649
+ code='INVALID_REQUEST',
1650
+ )
1651
+ for name, source_name in tables.items():
1652
+ if not isinstance(name, str) or not name:
1653
+ raise StructuredDataError(
1654
+ f'table names must be non-empty strings; got {name!r}',
1655
+ code='INVALID_REQUEST',
1656
+ )
1657
+ if not isinstance(source_name, str) or not source_name:
1658
+ raise StructuredDataError(
1659
+ f'table {name!r} must map to its name in the source, a '
1660
+ f'non-empty string; got {source_name!r}',
1661
+ code='INVALID_REQUEST',
1662
+ )
1663
+
1664
+
1665
+ def _require_a_source(source: object) -> None:
1666
+ try:
1667
+ from sdm_connectors import Source
1668
+ except ImportError: # pragma: no cover - nothing could have built one
1669
+ Source = None # type: ignore[assignment,misc]
1670
+ if Source is None or not isinstance(source, Source):
1671
+ raise StructuredDataError(
1672
+ f'a schema over tables in place needs an sdm_connectors.Source, '
1673
+ f'got {type(source).__name__}. Build one from a connection: '
1674
+ f'DatabricksSource(databricks.sql.connect(...), catalog=..., '
1675
+ f'schema=...), or DatabricksSource(spark, ...) in a notebook',
1676
+ code='INVALID_REQUEST',
1677
+ )
1678
+
1679
+
1680
+ def _require_one_kind_of_table(
1681
+ tables: Mapping[str, Table | SourceTable], source: object
1682
+ ) -> None:
1683
+ r"""Refuse a schema that mixes frames with tables in place, or tables in
1684
+ place without the source they are in.
1685
+
1686
+ The engine samples one graph with one sampler: an in-memory one over
1687
+ frames, or SQL against one source. A mixture has no sampler to run it.
1688
+ """
1689
+ in_place = sorted(
1690
+ name for name, table in tables.items() if isinstance(table, SourceTable)
1691
+ )
1692
+ in_memory = sorted(
1693
+ name
1694
+ for name, table in tables.items()
1695
+ if not isinstance(table, SourceTable)
1696
+ )
1697
+ if in_place and in_memory:
1698
+ raise StructuredDataError(
1699
+ f'{in_place} stay in their source and {in_memory} are frames; a '
1700
+ f'schema samples one or the other, so read the first into frames '
1701
+ f'or put the second in the source',
1702
+ code='INVALID_REQUEST',
1703
+ )
1704
+ if in_place and source is None:
1705
+ raise StructuredDataError(
1706
+ f'{in_place} stay in their source, so the schema needs that '
1707
+ f'source; build it with Schema.from_source',
1708
+ code='INVALID_REQUEST',
1709
+ )
1710
+ if in_memory and source is not None:
1711
+ raise StructuredDataError(
1712
+ f'{in_memory} are frames, which a source does not hold; drop '
1713
+ f'source= or build the schema with Schema.from_source',
1714
+ code='INVALID_REQUEST',
1715
+ )
1716
+ if source is not None:
1717
+ _require_a_source(source)
1718
+
1719
+
1720
+ # What a name in a predictive query cannot hold: `table.column` splits at the
1721
+ # dot, and a backtick or line break ends the name.
1722
+ _INVALID_NAME_CHARS = ('`', '\n', '\r', '.')
1723
+
1724
+
1725
+ def _invalid_name_chars(name: str) -> list[str]:
1726
+ return [char for char in _INVALID_NAME_CHARS if char in name]
1727
+
1728
+
1729
+ def _describe_source_table(
1730
+ engine_table: Any, source_name: str, *, name: str | None = None
1731
+ ) -> SourceTable:
1732
+ r"""A table in place, described from what the engine read of it.
1733
+
1734
+ A column whose values have no scalar form on the wire, such as a list,
1735
+ binary or a duration, is left out and named, rather than refusing the
1736
+ whole table:
1737
+ the caller cannot change a warehouse table's columns to suit a request, and
1738
+ the engine then leaves it unread as well. So is a column whose name a
1739
+ predictive query cannot say, such as a metric view's dimension `a.b`.
1740
+ """
1741
+ from sdm_client.materializers.sources import wire_dtype
1742
+
1743
+ columns: dict[str, Column] = {}
1744
+ unreadable: list[str] = []
1745
+ unnamable: list[str] = []
1746
+ for engine_column in engine_table.columns:
1747
+ if _invalid_name_chars(engine_column.name):
1748
+ unnamable.append(repr(engine_column.name))
1749
+ continue
1750
+ dtype = wire_dtype(engine_column.dtype)
1751
+ stype = str(getattr(engine_column.stype, 'value', engine_column.stype))
1752
+ if dtype is None or stype not in SEMANTIC_TYPES:
1753
+ unreadable.append(f'{engine_column.name} ({engine_column.dtype})')
1754
+ continue
1755
+ columns[engine_column.name] = Column(engine_column.name, dtype, stype)
1756
+ if unreadable:
1757
+ warnings.warn(
1758
+ f'{name or engine_table.name} leaves out {", ".join(unreadable)}: a '
1759
+ f'request carries one scalar value per cell, and these hold none',
1760
+ category=SDMWarning,
1761
+ stacklevel=caller_stacklevel(),
1762
+ )
1763
+ if unnamable:
1764
+ warnings.warn(
1765
+ f'{name or engine_table.name} leaves out {", ".join(unnamable)}: '
1766
+ f'a predictive query cannot name a column holding a dot, a '
1767
+ f'backtick or a line break',
1768
+ category=SDMWarning,
1769
+ stacklevel=caller_stacklevel(),
1770
+ )
1771
+ table = SourceTable(
1772
+ name=name or engine_table.name,
1773
+ source_name=source_name,
1774
+ columns=columns,
1775
+ )
1776
+ keys = tuple(engine_table.primary_key_columns or ())
1777
+ if keys and all(key in columns for key in keys):
1778
+ table.primary_key = keys[0] if len(keys) == 1 else keys
1779
+ time = engine_table.time_column
1780
+ if time is not None and time.name in columns and time.name not in keys:
1781
+ table.time_column = time.name
1782
+ return table
1783
+
1784
+
1785
+ def _model_links(
1786
+ graph: Any, built: Mapping[str, SourceTable], as_named: Mapping[str, str]
1787
+ ) -> list[Relationship]:
1788
+ r"""The relationships a semantic model's joins make, as the schema says
1789
+ them.
1790
+
1791
+ Each join links the many side's key to the one side's, which the model
1792
+ makes that table's primary key. A join to a table the schema did not take
1793
+ links nothing.
1794
+ """
1795
+ links = []
1796
+ for edge in graph.edges:
1797
+ source_name = as_named.get(edge.src_table)
1798
+ target_name = as_named.get(edge.dst_table)
1799
+ if source_name is None or target_name is None:
1800
+ continue
1801
+ target = built[target_name]
1802
+ source = built[source_name]
1803
+ key = target.primary_key
1804
+ if not isinstance(key, str) or edge.fkey not in source.columns:
1805
+ continue
1806
+ links.append(
1807
+ Relationship(
1808
+ source_table=source_name,
1809
+ source_column=edge.fkey,
1810
+ target_table=target_name,
1811
+ target_column=key,
1812
+ )
1813
+ )
1814
+ return links
1815
+
1816
+
1817
+ def _declared_links(
1818
+ graph: Any, tables: Mapping[str, str], built: Mapping[str, SourceTable]
1819
+ ) -> list[Relationship]:
1820
+ r"""The foreign keys the source's catalog declares between these tables.
1821
+
1822
+ A constraint naming a table outside the schema links nothing here, and
1823
+ one naming a column the schema left out is skipped with it.
1824
+ """
1825
+ # `dst_table` is qualified the way `source_name` is, not the way the caller
1826
+ # named the table.
1827
+ by_source_name = {graph[name].source_name: name for name in tables}
1828
+ links = []
1829
+ for name in tables:
1830
+ for key in graph[name].source_foreign_keys:
1831
+ target = _table_named(by_source_name, key.dst_table)
1832
+ if target is None:
1833
+ continue
1834
+ if key.name not in built[name].columns:
1835
+ continue
1836
+ if key.primary_key not in built[target].columns:
1837
+ continue
1838
+ links.append(
1839
+ Relationship(
1840
+ source_table=name,
1841
+ source_column=key.name,
1842
+ target_table=target,
1843
+ target_column=key.primary_key,
1844
+ )
1845
+ )
1846
+ return links
1847
+
1848
+
1849
+ def _table_named(
1850
+ by_source_name: Mapping[str, str], dst_table: str
1851
+ ) -> str | None:
1852
+ r"""The table a foreign key's target names, matched exactly if it can be.
1853
+
1854
+ SQLite and Databricks fold names to lower case, so a case-insensitive match
1855
+ still counts, but only when one table answers to it. PostgreSQL and
1856
+ Snowflake can hold both `"Parent"` and `parent`, and guessing between them
1857
+ would attach the key to the wrong one.
1858
+ """
1859
+ if dst_table in by_source_name:
1860
+ return by_source_name[dst_table]
1861
+ folded = [
1862
+ name
1863
+ for source_name, name in by_source_name.items()
1864
+ if source_name.lower() == dst_table.lower()
1865
+ ]
1866
+ return folded[0] if len(folded) == 1 else None
1867
+
1868
+
1869
+ def _require_column(
1870
+ tables: Mapping[str, _TableShape], table: str, column: str
1871
+ ) -> str:
1872
+ if column not in _require_table(tables, table).columns:
1873
+ raise StructuredDataError(
1874
+ f'table {table!r} has no column {column!r}; it has '
1875
+ f'{sorted(tables[table].columns)}',
1876
+ code='INVALID_REQUEST',
1877
+ )
1878
+ return column
1879
+
1880
+
1881
+ def _as_key_tuple(key: Any) -> tuple[str, ...]:
1882
+ r"""A primary key as the tuple of column names it is.
1883
+
1884
+ A single name, or any ordered iterable of them, including a `pd.Index`,
1885
+ which is what `frame.columns` is and so the most natural way to name a
1886
+ composite key from the data it came from. `pd.Index` is not a
1887
+ `collections.abc.Sequence`, so asking that question rejected it.
1888
+
1889
+ A set is not one of them. A composite key's columns are in an order, and
1890
+ a set has none, so the key it named would differ between runs.
1891
+ """
1892
+ if key is None:
1893
+ return ()
1894
+ if isinstance(key, str):
1895
+ return (key,)
1896
+ if isinstance(key, (bytes, Mapping, AbstractSet)) or not isinstance(
1897
+ key, Iterable
1898
+ ):
1899
+ raise StructuredDataError(
1900
+ f'a primary key names a column or a sequence of them, got {key!r}',
1901
+ code='INVALID_REQUEST',
1902
+ )
1903
+ names = list(key)
1904
+ if not all(isinstance(name, str) for name in names):
1905
+ raise StructuredDataError(
1906
+ f'a primary key names a column or a sequence of them, got {key!r}',
1907
+ code='INVALID_REQUEST',
1908
+ )
1909
+ return tuple(names)
1910
+
1911
+
1912
+ def _infer_table(name: str, frame: pd.DataFrame) -> Table:
1913
+ r"""Describe a frame as a table; see
1914
+ :func:`_infer_table_and_time_column`.
1915
+ """
1916
+ return _infer_table_and_time_column(name, frame)[0]
1917
+
1918
+
1919
+ def _infer_table_and_time_column(
1920
+ name: str, frame: pd.DataFrame
1921
+ ) -> tuple[Table, TimeColumnChoice]:
1922
+ r"""Describe a frame as a table, inferring the types, key and time column,
1923
+ and return with it how the time column was chosen.
1924
+
1925
+ The cells are checked before any column is described. Typing a column asks
1926
+ for its distinct values, and a list cell makes that raise `unhashable
1927
+ type: 'list'` from inside pandas, naming nothing the caller wrote.
1928
+ `Table.__post_init__` asks the same question again for a table built by
1929
+ hand, which is the route that has not passed through here.
1930
+ """
1931
+ if frame.columns.has_duplicates:
1932
+ duplicates = sorted(
1933
+ {str(c) for c in frame.columns[frame.columns.duplicated()]}
1934
+ )
1935
+ raise StructuredDataError(
1936
+ f'table {name!r} has duplicate column name(s) {duplicates}',
1937
+ code='INVALID_REQUEST',
1938
+ )
1939
+ non_strings = [c for c in frame.columns if not isinstance(c, str)]
1940
+ if non_strings:
1941
+ raise StructuredDataError(
1942
+ f'table {name!r} column names must be strings; got {non_strings}',
1943
+ code='INVALID_REQUEST',
1944
+ )
1945
+ for column in frame.columns:
1946
+ require_scalar_cells(frame[column], f'{name}.{column}')
1947
+
1948
+ columns = {
1949
+ column: _column(frame[column], column) for column in frame.columns
1950
+ }
1951
+ table = Table(name=name, data=frame, columns=columns)
1952
+ inferred = _infer_primary_key(frame, columns)
1953
+ if inferred is not None and frame[inferred].isna().any():
1954
+ # Tolerated here and refused on assignment, the same way a nearly
1955
+ # unique column is: inference is a guess and a caller's key is an
1956
+ # assertion. Refusing would break `Schema.from_data` on frames that
1957
+ # worked before, including any relationship pointing at this table,
1958
+ # but the engine drops these rows without a word, so the guess is
1959
+ # said out loud.
1960
+ _warn_a_key_has_no_value(
1961
+ name, inferred, int(frame[inferred].isna().sum())
1962
+ )
1963
+ object.__setattr__(table, 'primary_key', inferred)
1964
+ table._adopt_key(inferred)
1965
+ return table, _infer_time_column(table, frame)
1966
+ # Assigning the key is what marks its column, counts what the uniqueness
1967
+ # threshold tolerated and says so; doing any of it a second time here would
1968
+ # report the same duplicate twice, from inside this module.
1969
+ table.primary_key = inferred
1970
+ return table, _infer_time_column(table, frame)
1971
+
1972
+
1973
+ def _copy_table(table: _T) -> _T:
1974
+ r"""The same table again, field for field, judging nothing.
1975
+
1976
+ Built without the constructor and without the checked setters, for two
1977
+ reasons. Inference deliberately accepts a key with a null in it. It warns
1978
+ and continues because refusing would break frames that worked. However,
1979
+ assignment refuses one, so copying through the setter turned a schema
1980
+ `from_data` had accepted into a request that could not be materialized.
1981
+ And `Table.__post_init__` walks every cell of every column, which on a
1982
+ warehouse extract is a second full scan added to every single request.
1983
+ """
1984
+ copied = type(table).__new__(type(table))
1985
+ object.__setattr__(copied, 'name', table.name)
1986
+ for held in ('data', 'source_name'):
1987
+ if hasattr(table, held):
1988
+ object.__setattr__(copied, held, getattr(table, held))
1989
+ object.__setattr__(
1990
+ copied,
1991
+ 'columns',
1992
+ {name: _copy_column(column) for name, column in table.columns.items()},
1993
+ )
1994
+ for carried in ('primary_key', 'time_column', 'duplicate_keys'):
1995
+ object.__setattr__(copied, carried, getattr(table, carried))
1996
+ for column in copied.columns.values():
1997
+ object.__setattr__(column, '_table', copied)
1998
+ return copied
1999
+
2000
+
2001
+ def _copy_column(column: Column) -> Column:
2002
+ r"""The same column again, without re-judging it.
2003
+
2004
+ Built field by field rather than through the constructor: a schema is
2005
+ allowed to be inconsistent between two assignments. That is what
2006
+ `check_consistent` checks later, so copying one must not be
2007
+ the moment its storage and meaning are held against each other.
2008
+ """
2009
+ made = Column.__new__(Column)
2010
+ for field_name in (
2011
+ 'name',
2012
+ 'dtype',
2013
+ 'stype',
2014
+ 'untyped',
2015
+ 'stype_declared',
2016
+ '_stype_before_key',
2017
+ ):
2018
+ object.__setattr__(made, field_name, getattr(column, field_name))
2019
+ return made
2020
+
2021
+
2022
+ def _column(series: pd.Series, name: str) -> Column:
2023
+ r"""One column, with the record of whether inference had anything to go on.
2024
+
2025
+ `untyped` is set after construction because it is not an init field: a
2026
+ caller must not be able to build a column that names a dtype and claims to
2027
+ have none, which is the state the adapter resolves by discarding the dtype.
2028
+
2029
+ The values are read once and the answer passed down. `infer_tfm_dtype` on
2030
+ an object column walks every cell. Asking it three times for the dtype,
2031
+ again inside the stype, and again for `untyped` is what made
2032
+ describing a million rows of warehouse `Decimal` take five seconds before
2033
+ anything was sent. The same data already typed takes three hundredths.
2034
+ """
2035
+ inferred = infer_tfm_dtype(series)
2036
+ built = Column(
2037
+ name=name,
2038
+ dtype=_dtype_of(inferred),
2039
+ stype=_stype_of(series, inferred),
2040
+ )
2041
+ if inferred == UNTYPED:
2042
+ object.__setattr__(built, 'untyped', True)
2043
+ return built
2044
+
2045
+
2046
+ def _dtype_of(inferred: str) -> str:
2047
+ # An all-null object column carries no type. Calling it a string is the
2048
+ # safe fallback: it round-trips whatever arrives without claiming a width.
2049
+ return 'string' if inferred == UNTYPED else inferred
2050
+
2051
+
2052
+ def _stype_of(series: pd.Series, inferred: str | None = None) -> str:
2053
+ dtype = _dtype_of(infer_tfm_dtype(series) if inferred is None else inferred)
2054
+ if dtype.startswith('timestamp'):
2055
+ return 'timestamp'
2056
+ if dtype in {'int32', 'int64', 'float32', 'float64'}:
2057
+ return 'numerical'
2058
+ if dtype == 'bool':
2059
+ return 'categorical'
2060
+ distinct = series.nunique(dropna=True)
2061
+ return 'categorical' if distinct <= _MAX_CATEGORICAL_CARDINALITY else 'text'
2062
+
2063
+
2064
+ def _infer_primary_key(
2065
+ frame: pd.DataFrame, columns: Mapping[str, Column]
2066
+ ) -> str | None:
2067
+ r"""The column that identifies a row, if one is obvious.
2068
+
2069
+ Requires a key-shaped name, values that can identify a row, and near-total
2070
+ uniqueness. Uniqueness alone would nominate any high-cardinality
2071
+ measurement, and a name alone would nominate a foreign key.
2072
+
2073
+ **Uniqueness here is a property of the frame, not of the table it came
2074
+ from.** A foreign key drawn sparsely enough looks exactly like a key: take
2075
+ 200 rows of a fact table whose real key is `(order, line)` and `order`
2076
+ will very likely hold 200 distinct values, because two rows of the same
2077
+ order are unlikely to be drawn together. Nothing in the frame distinguishes
2078
+ that from a genuine key, so a frame that is a sample can nominate a foreign
2079
+ key and no amount of checking here will notice. Declare the key when the
2080
+ frame is a sample; inference is for the case where the frame is the table.
2081
+ """
2082
+ if len(frame) == 0:
2083
+ return None
2084
+ for name, column in columns.items():
2085
+ lowered = name.lower()
2086
+ if not lowered.endswith(_KEY_SUFFIXES):
2087
+ continue
2088
+ if not supports_dtype('ID', column.dtype):
2089
+ # A key-shaped name on values that cannot identify a row, such as a
2090
+ # bool
2091
+ # column called `id`, say. Nominating it would refuse the frame at
2092
+ # construction for a key nobody asked for.
2093
+ continue
2094
+ distinct = frame[name].nunique(dropna=True)
2095
+ if distinct >= len(frame) * _KEY_UNIQUENESS:
2096
+ return name
2097
+ return None
2098
+
2099
+
2100
+ #: Above this share of rows being distinct, a string column's value count is
2101
+ #: bounded by the frame rather than by the column, so it says nothing about
2102
+ #: how many values the column really holds.
2103
+ _UNSATURATED = 0.5
2104
+
2105
+ #: And below this share of the categorical limit, growth has too far to go for
2106
+ #: the frame to be what decides: two values in four rows are unsaturated by
2107
+ #: the measure above and are still nowhere near becoming sixty-five.
2108
+ _WITHIN_REACH = 0.5
2109
+
2110
+
2111
+ def _warn_stype_was_decided_by_the_rows_read(
2112
+ tables: Mapping[str, Table],
2113
+ ) -> None:
2114
+ r"""Say when `categorical` was decided by the size of the frame.
2115
+
2116
+ A string column is called `categorical` when it holds few enough distinct
2117
+ values and `text` otherwise, and the count is taken from the frame handed
2118
+ in. For a column whose values repeat that is the same answer at any size:
2119
+ `state` holds 27 values in ten thousand rows and in a hundred thousand.
2120
+ For one that does not, the count is not a property of the column at all.
2121
+ It is a property of how many rows were read, and reading more rows moves
2122
+ it. So a request written against `df.head(100)` and then pointed at the
2123
+ table describes a column two different ways, with nothing in between to
2124
+ say the description changed.
2125
+
2126
+ Which is worth saying only where it can happen, and that takes both of two
2127
+ measures. The first is whether the values have run out: a column whose
2128
+ distinct count tracks its row count has not shown how many values it has,
2129
+ only how many rows were looked at. The second is whether there is anything
2130
+ within reach. Two values in four rows have not saturated either, and are
2131
+ nowhere near becoming sixty-five, so saying so is noise on every small
2132
+ frame. Nearness to the threshold alone is not the signal, because 53
2133
+ distinct in 100 rows is not near 64 by any measure and still flips.
2134
+
2135
+ This is the same property the primary-key inference documents about
2136
+ uniqueness, and it has the same remedy, which is to say what the column is
2137
+ when the frame is a sample.
2138
+
2139
+ The count decides the meaning for strings only, which is what this is
2140
+ about: a bool is categorical by its type rather than by how many values it
2141
+ holds. That makes the dtype check a precondition rather than a filter
2142
+ today, since the only other categorical dtype is bool and two values never
2143
+ come within reach of the limit. It is stated so that a dtype added to the
2144
+ categorical side later does not inherit a message about a count that did
2145
+ not decide it.
2146
+
2147
+ The share is measured against the rows that hold a value, because that is
2148
+ what the distinct count was taken from. A column of forty values among a
2149
+ hundred and sixty nulls has saturated nothing, and dividing by the height
2150
+ of the frame said it had.
2151
+ """
2152
+ for name, table in tables.items():
2153
+ frame = table.data
2154
+ if frame is None or len(frame) == 0:
2155
+ continue
2156
+ for column, meta in table.columns.items():
2157
+ if meta.stype_declared or meta.stype != 'categorical':
2158
+ continue
2159
+ if meta.dtype != 'string' or column not in frame:
2160
+ continue
2161
+ distinct = frame[column].nunique(dropna=True)
2162
+ present = int(frame[column].count())
2163
+ if distinct < present * _UNSATURATED:
2164
+ continue
2165
+ if distinct < _MAX_CATEGORICAL_CARDINALITY * _WITHIN_REACH:
2166
+ continue
2167
+ warnings.warn(
2168
+ f'{name}.{column} holds {distinct} distinct values in the '
2169
+ f'{present} rows that have one, so the rows ran out before '
2170
+ f'the values did and the count is a measure of the frame '
2171
+ f'rather than of the column. It is read as categorical here '
2172
+ f'because {distinct} is at most '
2173
+ f'{_MAX_CATEGORICAL_CARDINALITY}; on more rows the same '
2174
+ f'column could pass that and be read as text, which the '
2175
+ f'models treat differently. If this frame is a sample, say '
2176
+ f'what the column is with '
2177
+ f"schema[{name!r}][{column!r}].stype = 'categorical' (or "
2178
+ f"'text'), so that it does not change under a larger one.",
2179
+ category=SDMWarning,
2180
+ stacklevel=caller_stacklevel(),
2181
+ )
2182
+
2183
+
2184
+ def _warn_dates_stored_as_text(table: Table) -> str | None:
2185
+ r"""Say when the only candidate timestamp is a string, and was not used.
2186
+
2187
+ Inference types a string by how many distinct values it holds and never
2188
+ reads it as a date, which is deliberate: a parser loose enough to read a
2189
+ warehouse `DATE` column also reads part codes as dates and dies later, on
2190
+ a value it should not have touched. The cost is that a wholly temporal
2191
+ dataset, such as a warehouse that ordinarily returns `DATE` as `STRING`,
2192
+ infers to a schema with no time in it. The caller learns
2193
+ only when a temporal query is refused, or never, if the query they wrote
2194
+ happened to be static.
2195
+
2196
+ So the column is still not parsed, and the declining is no longer silent.
2197
+ Only a table that ended with no time column is worth a word; one that
2198
+ found its timestamp does not need to hear about a date-shaped string.
2199
+
2200
+ The shape test is not a correctness check: the conversion below rejects
2201
+ everything it would, and more. It is there so that the ordinary case, a
2202
+ table whose string columns are names and descriptions, does not pay to
2203
+ have every one of them parsed as a date.
2204
+
2205
+ Every remaining question is answered by running that conversion over the
2206
+ whole column rather than sampling it or reasoning about it: whether the
2207
+ values past the sample hold their shape, whether they are real dates, and
2208
+ which form of the call the caller should be told to make.
2209
+ """
2210
+ frame = table.data
2211
+ if table.time_column is not None or frame is None or frame.empty:
2212
+ return None
2213
+ for name, column in table.columns.items():
2214
+ if column.dtype != 'string' or column.stype == 'ID':
2215
+ continue
2216
+ present = frame[name].dropna()
2217
+ if not dates.looks_unambiguous(present):
2218
+ continue
2219
+ how = dates.conversion_that_works(present)
2220
+ if how is None:
2221
+ continue
2222
+ warnings.warn(
2223
+ f'{table.name}.{name} holds dates but is stored as text, so it is '
2224
+ f'not the time column and no temporal query can use it. Convert '
2225
+ f'it before inferring: '
2226
+ f'frame[{name!r}] = pd.to_datetime(frame[{name!r}]{how}), or '
2227
+ f'set schema[{table.name!r}][{name!r}].stype = {"timestamp"!r} '
2228
+ f'afterwards.',
2229
+ category=SDMWarning,
2230
+ stacklevel=caller_stacklevel(),
2231
+ )
2232
+ return name
2233
+ return None
2234
+
2235
+
2236
+ def _infer_time_column(table: Table, frame: pd.DataFrame) -> TimeColumnChoice:
2237
+ r"""Set the time column, and return how it was chosen.
2238
+
2239
+ The rule is the one the engine applies to a source, so a frame and the
2240
+ same rows in a warehouse infer the same column.
2241
+ """
2242
+ stamps = [
2243
+ name
2244
+ for name, column in table.columns.items()
2245
+ if column.stype == 'timestamp' and name != table.primary_key
2246
+ ]
2247
+ choice = choose_time_column(frame, stamps)
2248
+ table.time_column = choice.column
2249
+ return choice
2250
+
2251
+
2252
+ def _settle_edit_stamps(
2253
+ built: Mapping[str, _TableShape],
2254
+ chose: dict[str, TimeColumnChoice | None],
2255
+ time_columns: Mapping[str, str | None] | None,
2256
+ links: Sequence[Relationship],
2257
+ ) -> None:
2258
+ r"""Take or refuse a lone edit stamp by whether another table links to
2259
+ the table it is in.
2260
+
2261
+ A table something links to is an entity and refuses the stamp of its
2262
+ last edit; one nothing links to is a log and takes it. Decided on the
2263
+ schema's final links, the one fact both paths share by now: the two
2264
+ guess keys differently, so a log's `update_id` is a key from a frame
2265
+ and none from a source, and a rule on the key made them disagree.
2266
+ """
2267
+ linked = {link.target_table for link in links}
2268
+ for name, choice in chose.items():
2269
+ if choice is None or name in (time_columns or {}):
2270
+ continue
2271
+ table = built[name]
2272
+ settled = choice.for_links(name in linked)
2273
+ if settled != choice:
2274
+ chose[name] = settled
2275
+ table.time_column = settled.column
2276
+
2277
+
2278
+ def _warn_how_time_columns_were_inferred(
2279
+ schema: Schema,
2280
+ chose: Mapping[str, TimeColumnChoice | None],
2281
+ given: Mapping[str, str | None] | None,
2282
+ ) -> None:
2283
+ """Say, in one warning, which time columns inference took or refused on
2284
+ a guess, and the override that settles each.
2285
+
2286
+ A guess about a column's name or values decides what the model may know
2287
+ at the moment it answers, so it is said out loud: a column taken as the
2288
+ earliest of several, a stamp of the last edit taken as the only timestamp
2289
+ left, one taken although the rows read hold one value or mostly none,
2290
+ because the query over the whole table failed, and, for a table left
2291
+ without a time column, each timestamp refused and why. That table fails its first windowed query, and the refusal
2292
+ cannot say which of its columns inference passed over.
2293
+
2294
+ Only where something could be windowed: a lone table is asked
2295
+ single-table questions, which have no window. A table whose time column
2296
+ the caller named is not mentioned.
2297
+ """
2298
+ tables = schema.tables
2299
+ if len(tables) < 2 and not schema.relationships:
2300
+ return
2301
+ lines: list[str] = []
2302
+ for name, table in sorted(tables.items()):
2303
+ choice = chose.get(name)
2304
+ if choice is None or name in (given or {}):
2305
+ continue
2306
+ column = table.time_column
2307
+ if column is not None and column == choice.column:
2308
+ if choice.instead_of:
2309
+ one = len(choice.instead_of) == 1
2310
+ lines.append(
2311
+ f'{name}.{column} was taken as the time column rather '
2312
+ f'than {_and_list(choice.instead_of)}, which '
2313
+ f'{"truncates or casts" if one else "truncate or cast"} '
2314
+ f'it, since it says when a row happened more precisely. '
2315
+ f'If a question should count by '
2316
+ f'{"it" if one else "one of those"}, name it: '
2317
+ f'time_columns={{{name!r}: {choice.instead_of[0]!r}}}.'
2318
+ )
2319
+ if choice.how == 'earliest':
2320
+ others = [other for other in choice.among if other != column]
2321
+ lines.append(
2322
+ f'{name}.{column} was taken as the time column because '
2323
+ f'its values start earliest of {_and_list(choice.among)}. '
2324
+ f'If another says when a row became true, name it: '
2325
+ f'time_columns={{{name!r}: {others[0]!r}}}. If none does, '
2326
+ f'clear it: time_columns={{{name!r}: None}}.'
2327
+ )
2328
+ elif choice.how == 'edit_stamp':
2329
+ lines.append(
2330
+ f'{name}.{column} was taken as the time column although '
2331
+ f'it looks like when a row was last changed, because the '
2332
+ f'table has no other timestamp and no table links to it, '
2333
+ f'so it reads as a log of changes. If it does not say when a row '
2334
+ f'became true, clear it: time_columns={{{name!r}: None}}.'
2335
+ )
2336
+ lines.extend(
2337
+ taken_unchecked(name, doubt, f'time_columns={{{name!r}: None}}')
2338
+ for doubt in choice.unconfirmed
2339
+ if doubt.column == column
2340
+ )
2341
+ elif column is None:
2342
+ lines.extend(
2343
+ f'{name}.{rejected.column} was not taken as the time column '
2344
+ f'because {rejected.why}. If it does say when a row became '
2345
+ f'true, name it: '
2346
+ f'time_columns={{{name!r}: {rejected.column!r}}}.'
2347
+ for rejected in choice.rejected
2348
+ if rejected.column in table.columns
2349
+ )
2350
+ if lines:
2351
+ warnings.warn(
2352
+ 'Some time columns were inferred, or refused, on a guess about '
2353
+ 'what a column holds:\n' + '\n'.join(f'- {line}' for line in lines),
2354
+ category=SDMWarning,
2355
+ stacklevel=caller_stacklevel(),
2356
+ )
2357
+
2358
+
2359
+ def _warn_table_cannot_be_linked_to(
2360
+ tables: Mapping[str, Table],
2361
+ dates: Mapping[str, str],
2362
+ already_said: AbstractSet[str] = frozenset(),
2363
+ ) -> None:
2364
+ r"""Say when a table has no key and something wanted to point at it.
2365
+
2366
+ `_infer_primary_key` requires a key-shaped name, and rightly: uniqueness
2367
+ alone would nominate any high-cardinality measurement. But a great many
2368
+ real keys are domain codes such as `IATA_CODE`, `SKU`, `ISBN`, and `VIN`.
2369
+ No suffix rule will cover them.
2370
+
2371
+ That is not merely a missing key. `_infer_relationships` can only point an
2372
+ edge at a *declared* key, so a table that has none is not a candidate
2373
+ target for anything: every relationship into it is lost, the tables are
2374
+ dropped from the request, and the model answers from less data than it was
2375
+ given with nothing on screen to say so.
2376
+
2377
+ Nominating a key here was tried and withdrawn. Uniqueness within one frame
2378
+ does not distinguish a key from a foreign key that happens not to repeat in
2379
+ the rows to hand. A two-row extract of `line_items` has a unique
2380
+ `ORDER_NO`, so nominating turns a child into a parent and invents an edge
2381
+ between two facts. Worse, a second table keyed on a name makes that name
2382
+ ambiguous, and `_infer_relationships` then declines *every* edge using it:
2383
+ the guess does not merely risk a wrong link, it can delete the right ones
2384
+ from a schema whose keys the caller declared correctly.
2385
+
2386
+ So the caller is asked instead, and only where the question is real. A
2387
+ keyless table no other table names loses no edges. Nor does one whose
2388
+ candidate column is already another table's key. That column is a
2389
+ foreign key, naming it here would make the name ambiguous, and ambiguity
2390
+ deletes the edges that were working. A warning that talks a caller into
2391
+ the failure it was written to prevent is worse than silence.
2392
+
2393
+ `already_said` names the tables the repeated-key warning has spoken about.
2394
+ Said of one table the two contradict, with remedies that cannot both be
2395
+ followed.
2396
+
2397
+ `dates` names, per table, a column already reported as a date stored as
2398
+ text. A column cannot be both the time column and the primary key, so
2399
+ asking for it here would contradict the other warning, and a caller who
2400
+ followed both in order would end with neither.
2401
+
2402
+ How many columns could be the key and how many tables they are shared with
2403
+ are two counts, and they do not move together: several columns can be
2404
+ shared with one table, and one column with several. Agreeing either by the
2405
+ other made the sentence disagree with the graph it describes.
2406
+ """
2407
+ named_elsewhere = _tables_naming_each_column(tables)
2408
+ keyed_by = {
2409
+ table.primary_key
2410
+ for table in tables.values()
2411
+ if isinstance(table.primary_key, str)
2412
+ }
2413
+
2414
+ for name, table in tables.items():
2415
+ if table.primary_key is not None or name in already_said:
2416
+ continue
2417
+ frame = table.data
2418
+ if frame is None or frame.empty:
2419
+ continue
2420
+ candidates = [
2421
+ column
2422
+ for column, meta in table.columns.items()
2423
+ if _could_identify_a_row(table, frame, column, meta)
2424
+ and len(named_elsewhere.get(column, ())) > 1
2425
+ and column not in keyed_by
2426
+ and column != dates.get(name)
2427
+ ]
2428
+ if not candidates:
2429
+ continue
2430
+ elsewhere = {
2431
+ other
2432
+ for column in candidates
2433
+ for other in named_elsewhere.get(column, ())
2434
+ if other != name
2435
+ }
2436
+ warnings.warn(
2437
+ _unlinkable_table_reads(name, candidates, len(elsewhere)),
2438
+ category=SDMWarning,
2439
+ stacklevel=caller_stacklevel(),
2440
+ )
2441
+
2442
+
2443
+ def _tables_naming_each_column(
2444
+ tables: Mapping[str, _TableShape],
2445
+ ) -> dict[str, set[str]]:
2446
+ r"""Which tables hold a column of each name.
2447
+
2448
+ A name shared by two tables is what an inferred edge is made of, so it is
2449
+ also what every question about a missing edge is asked against.
2450
+ """
2451
+ named: dict[str, set[str]] = {}
2452
+ for name, table in tables.items():
2453
+ for column in table.columns:
2454
+ named.setdefault(column, set()).add(name)
2455
+ return named
2456
+
2457
+
2458
+ def _warn_repeats_cost_the_table_its_key(
2459
+ tables: Mapping[str, Table],
2460
+ ) -> set[str]:
2461
+ r"""Say when a key-shaped column was declined for its repeats.
2462
+
2463
+ `_infer_primary_key` wants `_KEY_UNIQUENESS` of the values distinct, and
2464
+ tolerating a little short of total uniqueness is right: one duplicated row
2465
+ is common in an extract and still the column the caller means. But the
2466
+ tolerance has an edge, and the behaviour either side of it is the wrong way
2467
+ round. One duplicated row nominates the key and warns; two nominate
2468
+ nothing, and a table with no key is not a candidate target for any edge, so
2469
+ every relationship pointing at it disappears, with nothing said at
2470
+ inference or at materialization. The louder outcome is the smaller
2471
+ problem.
2472
+
2473
+ `_warn_table_cannot_be_linked_to` is the warning for a table nothing can
2474
+ point at, and it cannot see this one: it asks for columns that identify a
2475
+ row perfectly, and the column that caused the loss is by definition not
2476
+ one of them. So it is asked about here instead, where the repeats are the
2477
+ subject rather than the disqualification.
2478
+
2479
+ Only a column another table names is worth a word, on the same reasoning:
2480
+ a key nothing points at loses no edge by being absent.
2481
+
2482
+ `_NEARLY_A_KEY` is what separates a key with a fault in it from an
2483
+ ordinary foreign key, which repeats on the order of the row count rather
2484
+ than a handful of times. A sampled extract makes an ordinary foreign key
2485
+ nearly unique too, so a column another table is already keyed by is left
2486
+ alone: the edge through it works, and the remedy printed here deletes it.
2487
+
2488
+ Returns the tables it spoke about, so the keyless-table warning does not
2489
+ follow it with a contradicting account of the same one.
2490
+ """
2491
+ named_elsewhere = _tables_naming_each_column(tables)
2492
+ keyed_by = {
2493
+ table.primary_key
2494
+ for table in tables.values()
2495
+ if isinstance(table.primary_key, str)
2496
+ }
2497
+ spoken_for = set()
2498
+ for name, table in tables.items():
2499
+ if table.primary_key is not None:
2500
+ continue
2501
+ frame = table.data
2502
+ if frame is None or frame.empty:
2503
+ continue
2504
+ for column, meta in table.columns.items():
2505
+ if not column.lower().endswith(_KEY_SUFFIXES):
2506
+ continue
2507
+ if not supports_dtype('ID', meta.dtype):
2508
+ continue
2509
+ if column in keyed_by:
2510
+ continue
2511
+ if not named_elsewhere.get(column, set()) - {name}:
2512
+ continue
2513
+ if frame[column].nunique(dropna=True) < len(frame) * _NEARLY_A_KEY:
2514
+ continue
2515
+ quality = _key_quality(frame, (column,))
2516
+ if not quality.duplicates:
2517
+ continue
2518
+ _warn_a_key_was_declined(
2519
+ name,
2520
+ column,
2521
+ quality.duplicates,
2522
+ len(frame),
2523
+ quality.example,
2524
+ f'Keep one row per {column!r}: '
2525
+ f'frame = frame.drop_duplicates({column!r}), and infer '
2526
+ f'again, or name a column that does identify a row with '
2527
+ f'primary_keys={{{name!r}: ...}}.',
2528
+ )
2529
+ spoken_for.add(name)
2530
+ break
2531
+ return spoken_for
2532
+
2533
+
2534
+ def _warn_one_key_name_keys_several_tables(
2535
+ tables: Mapping[str, Table],
2536
+ ) -> None:
2537
+ r"""Say when a key name is shared, so no edge can be inferred through it.
2538
+
2539
+ `_infer_relationships` declines every edge naming a key that more than one
2540
+ table is keyed by, and rightly: `id` on four tables identifies a row in
2541
+ each of them and names a parent in none, so choosing one would be a guess.
2542
+ What was missing is that the declining was silent. Four frames infer to
2543
+ four tables and no relationships at all, and the only thing on screen is
2544
+ whatever unrelated notice some column happened to earn, which then reads
2545
+ as the whole account of what inference did.
2546
+
2547
+ Said per shared name rather than per lost edge. A table keyed by an
2548
+ ambiguous name can never be the target of an inferred relationship, so the
2549
+ loss is the name's, and listing candidate edges that were never built
2550
+ would be inventing a graph to describe.
2551
+ """
2552
+ keyed_by: dict[str, list[str]] = {}
2553
+ for name, table in tables.items():
2554
+ if isinstance(table.primary_key, str):
2555
+ keyed_by.setdefault(table.primary_key, []).append(name)
2556
+
2557
+ for key, names in sorted(keyed_by.items()):
2558
+ if len(names) < 2:
2559
+ continue
2560
+ shared = sorted(names)
2561
+ warnings.warn(
2562
+ f'{key!r} is the primary key of {_and_list(shared)}, so a column '
2563
+ f'named {key!r} identifies a row in each of them and names a '
2564
+ f'parent in none. No relationship is inferred through an '
2565
+ f'ambiguous name, so nothing can point at any of those tables: '
2566
+ f"none of them is gathered as another table's related rows, and "
2567
+ f'a question about one of them is answered from that table alone. '
2568
+ f'Say what the graph is: '
2569
+ f'relationships=[Relationship(child, column, {shared[0]!r}, '
2570
+ f'{key!r}), ...], or give each table a key of its own name, so '
2571
+ f'that the column naming it can be matched to it.',
2572
+ category=SDMWarning,
2573
+ stacklevel=caller_stacklevel(),
2574
+ )
2575
+
2576
+
2577
+ def _unlinkable_table_reads(
2578
+ name: str, candidates: Sequence[str], tables_sharing: int
2579
+ ) -> str:
2580
+ r"""What to say about a table nothing can link to.
2581
+
2582
+ Kept apart from the search that found it, because the two are different
2583
+ kinds of difficult. Deciding which columns could be the key is about the
2584
+ data; saying so is about English, and the sentence has to agree with the
2585
+ graph on two counts that move independently: how many columns could be
2586
+ the key, and how many tables they are shared with. Several columns can be
2587
+ shared with one table, and one column with several.
2588
+ """
2589
+ many, shared_by_many = len(candidates) > 1, tables_sharing > 1
2590
+ subject, verb = ('they', 'are') if many else ('it', 'is')
2591
+ tables = 'tables' if shared_by_many else 'table'
2592
+ return (
2593
+ f'{name!r} has no primary key, and nothing can link to a table '
2594
+ f'without one: every relationship pointing at {name!r} is dropped, so '
2595
+ f'nothing is gathered through it, and a question about another table '
2596
+ f'reaches it only where {name!r} names that table itself. '
2597
+ f'{_and_list(candidates)} {"identify" if many else "identifies"} '
2598
+ f'every row of {name!r} and {verb} named by '
2599
+ f'{"other tables" if shared_by_many else "another table"} too, which '
2600
+ f'makes {"each of them" if many else "it"} either a key or a foreign '
2601
+ f'key that happens not to repeat in the rows read. '
2602
+ f'If {"one of them identifies" if many else "it identifies"} '
2603
+ f'{name!r}, say so with primary_keys={{{name!r}: '
2604
+ f'{repr(candidates[0]) if not many else "..."}}}; '
2605
+ f'if {subject} instead '
2606
+ f'{"name" if many else "names"} the rows of the {tables} {subject} '
2607
+ f'{verb} shared with, '
2608
+ f'{"those tables are the ones that need" if shared_by_many else "that table is the one that needs"} '
2609
+ f'the key.'
2610
+ )
2611
+
2612
+
2613
+ def _each_link_once(
2614
+ links: Sequence[Relationship],
2615
+ ) -> list[Relationship]:
2616
+ """Deduplicate relationships while preserving their order.
2617
+
2618
+ The materializer already sends one edge for it, so keeping two here made
2619
+ the two halves of the client disagree about what the graph is, and the
2620
+ ambiguity warning read a repeat as a second way across the pair.
2621
+ """
2622
+ seen: set[tuple[str, str, str, str]] = set()
2623
+ once = []
2624
+ for link in links:
2625
+ named = (
2626
+ link.source_table,
2627
+ link.source_column,
2628
+ link.target_table,
2629
+ link.target_column,
2630
+ )
2631
+ if named in seen:
2632
+ continue
2633
+ seen.add(named)
2634
+ once.append(link)
2635
+ return once
2636
+
2637
+
2638
+ def _warn_of_a_pair_joined_both_ways(links: Sequence[Relationship]) -> None:
2639
+ """Say when two tables name each other, which reads as an ambiguous pair.
2640
+
2641
+ A parent naming one representative child is an ordinary shape and the
2642
+ schema takes it. The sampler walks an edge in both directions, so a
2643
+ declared reverse edge is a second way across the same pair, and every
2644
+ aggregation between them is then refused for a foreign key it says it
2645
+ cannot find uniquely when there is exactly one. The reverse edge is
2646
+ what made it two.
2647
+
2648
+ A table pointing at itself is one link, not two: the test for a reverse
2649
+ edge is otherwise satisfied by the edge itself.
2650
+ """
2651
+ pairs = {(link.source_table, link.target_table) for link in links}
2652
+ said = set()
2653
+ for source, target in sorted(pairs):
2654
+ if source == target:
2655
+ continue
2656
+ if (target, source) not in pairs or (target, source) in said:
2657
+ continue
2658
+ said.add((source, target))
2659
+ warnings.warn(
2660
+ f'{source!r} and {target!r} name each other, and a relationship is '
2661
+ f'crossed in both directions, so this is two ways over one pair and '
2662
+ f'an aggregation between them is refused for not having a single '
2663
+ f'one. Declare whichever of the two a question is about and leave '
2664
+ f'the other out.',
2665
+ category=SDMWarning,
2666
+ stacklevel=caller_stacklevel(),
2667
+ )
2668
+
2669
+
2670
+ class _Links(list):
2671
+ _schema: Schema
2672
+
2673
+ r"""The relationships of one schema, checked however they are added.
2674
+
2675
+ `append` is the route this client's own advice prints, and a check on
2676
+ assignment alone left it unchecked.
2677
+ """
2678
+
2679
+ def __init__(self, links: Sequence[Relationship], schema: Schema) -> None:
2680
+ super().__init__(links)
2681
+ object.__setattr__(self, '_schema', schema)
2682
+
2683
+ def _resettle(self, links: list[Relationship]) -> None:
2684
+ settled = self._schema._settle(links)
2685
+ super().clear()
2686
+ super().extend(settled)
2687
+ self._schema._mark_key_columns()
2688
+
2689
+ def remove(self, link: Any) -> None:
2690
+ kept = list(self)
2691
+ kept.remove(link)
2692
+ self._resettle(kept)
2693
+
2694
+ def pop(self, index: Any = -1) -> Relationship:
2695
+ kept = list(self)
2696
+ gone = kept.pop(index)
2697
+ self._resettle(kept)
2698
+ return gone
2699
+
2700
+ def clear(self) -> None:
2701
+ self._resettle([])
2702
+
2703
+ def __delitem__(self, index: Any) -> None:
2704
+ kept = list(self)
2705
+ del kept[index]
2706
+ self._resettle(kept)
2707
+
2708
+ def append(self, link: Relationship) -> None:
2709
+ self._resettle([*self, link])
2710
+
2711
+ def extend(self, links: Any) -> None:
2712
+ self._resettle([*self, *links])
2713
+
2714
+ def insert(self, index: Any, link: Any) -> None:
2715
+ kept = list(self)
2716
+ kept.insert(index, link)
2717
+ self._resettle(kept)
2718
+
2719
+ def __iadd__(self, links: Any) -> _Links: # type: ignore[misc]
2720
+ self._resettle([*self, *links])
2721
+ return self
2722
+
2723
+ def __imul__(self, count: Any) -> _Links: # type: ignore[misc]
2724
+ self._resettle(list(self) * count)
2725
+ return self
2726
+
2727
+ def __setitem__(self, index: Any, link: Any) -> None:
2728
+ kept = list(self)
2729
+ kept[index] = link
2730
+ self._resettle(kept)
2731
+
2732
+ def __reduce__(self) -> tuple[Any, ...]:
2733
+ r"""Copied and pickled as a plain list, and re-checked on arrival.
2734
+
2735
+ The schema this belongs to is held here, so copying the list the
2736
+ ordinary way copies the schema, whose state is restored after its
2737
+ own `__setattr__` would have run. `Schema.__setstate__` puts the list
2738
+ back under this type once the copy has its tables.
2739
+ """
2740
+ return (list, (list(self),))
2741
+
2742
+
2743
+ def _warn_two_ways_to_reach_one_table(
2744
+ tables: Mapping[str, _TableShape], links: Sequence[Relationship]
2745
+ ) -> None:
2746
+ r"""Say when one table reaches another by more than one column.
2747
+
2748
+ Both are real relationships and keeping both is right: a sales line names
2749
+ the customer it was sold to and the one it ships to, and they are
2750
+ different facts. But an aggregation across a pair of tables has to be
2751
+ resolved to one column, and a query that crosses an ambiguous pair is
2752
+ refused when it is asked, which reads as a problem with the
2753
+ query rather than with the graph it was written against.
2754
+
2755
+ Said here so that it is read where the cause is. Nothing is dropped: which
2756
+ of them a question means is the caller's to decide, and choosing for them
2757
+ is choosing what their model sees.
2758
+ """
2759
+ ways: dict[tuple[str, str], list[str]] = {}
2760
+ for link in links:
2761
+ pair = (link.source_table, link.target_table)
2762
+ columns = ways.setdefault(pair, [])
2763
+ if link.source_column not in columns:
2764
+ columns.append(link.source_column)
2765
+
2766
+ _warn_of_a_pair_joined_both_ways(links)
2767
+ for (source, target), columns in sorted(ways.items()):
2768
+ if len(columns) < 2:
2769
+ continue
2770
+ kept, *rest = sorted(columns)
2771
+ warnings.warn(
2772
+ f'{source!r} reaches {target!r} through {_and_list(sorted(columns))}'
2773
+ f', and an aggregation across the two tables has to be resolved to '
2774
+ f'one of them, so a query that crosses this pair is refused until '
2775
+ f'it is. They are all kept because they are all real, and which '
2776
+ f'one a question means is not something this can decide. To keep '
2777
+ f'{kept!r}, drop the others: schema.relationships = [link for '
2778
+ f'link in schema.relationships if not (link.source_table == '
2779
+ f'{source!r} and link.target_table == {target!r} and '
2780
+ f'link.source_column in {sorted(rest)!r})].',
2781
+ category=SDMWarning,
2782
+ stacklevel=caller_stacklevel(),
2783
+ )
2784
+
2785
+
2786
+ def _could_identify_a_row(
2787
+ table: Table, frame: pd.DataFrame, column: str, meta: Column
2788
+ ) -> bool:
2789
+ if column == table.time_column:
2790
+ return False
2791
+ if meta.dtype != 'string' and not meta.dtype.startswith('int'):
2792
+ return False
2793
+ if frame[column].isna().any():
2794
+ return False
2795
+ return _key_quality(frame, (column,)).duplicates == 0
2796
+
2797
+
2798
+ def _and_list(names: Sequence[str]) -> str:
2799
+ quoted = [repr(name) for name in names]
2800
+ if len(quoted) < 3:
2801
+ return ' and '.join(quoted)
2802
+ return f'{", ".join(quoted[:-1])} and {quoted[-1]}'
2803
+
2804
+
2805
+ def _infer_relationships(
2806
+ tables: Mapping[str, _TableShape],
2807
+ ) -> list[Relationship]:
2808
+ r"""Link a column to another table's primary key of the same name.
2809
+
2810
+ Deliberately conservative: only an exact name match against a declared
2811
+ primary key counts. A wrong link changes what the model is shown, and is
2812
+ harder to notice than a missing one, which simply narrows the question.
2813
+
2814
+ Which is why a name more than one table is keyed by links nothing. `ID` on
2815
+ four tables identifies a row in each of them and names a parent in none, so
2816
+ choosing one would be a guess in the direction this is supposed to avoid.
2817
+ Value overlap cannot break the tie either: a
2818
+ fabricated edge has been measured joining 97% of rows and a real one 91%,
2819
+ so no threshold separates them.
2820
+ """
2821
+ ambiguous: set[str] = set()
2822
+ keys: dict[str, str] = {}
2823
+ for name, table in tables.items():
2824
+ key = table.primary_key
2825
+ if not isinstance(key, str):
2826
+ continue
2827
+ if key in keys:
2828
+ ambiguous.add(key)
2829
+ keys[key] = name
2830
+
2831
+ links = []
2832
+ for name, table in tables.items():
2833
+ for column in table.columns:
2834
+ target = keys.get(column)
2835
+ if target is None or target == name or column in ambiguous:
2836
+ continue
2837
+ links.append(
2838
+ Relationship(
2839
+ source_table=name,
2840
+ source_column=column,
2841
+ target_table=target,
2842
+ target_column=column,
2843
+ )
2844
+ )
2845
+ return links
2846
+
2847
+
2848
+ def _warn_of_a_link_that_cannot_join(
2849
+ tables: Mapping[str, _TableShape], dropped: Sequence[Relationship]
2850
+ ) -> None:
2851
+ """Say why an inferred or declared link could not join.
2852
+
2853
+ Dropping it is right: joining `int64` to `string` matches nothing, and an
2854
+ edge that matches nothing is worse than no edge. But it was dropped in
2855
+ silence, and the table then either goes unsent or the query is refused for
2856
+ a foreign key that looks as though it should exist.
2857
+ """
2858
+ for link in dropped:
2859
+ source = tables[link.source_table].columns[link.source_column]
2860
+ target_table = tables[link.target_table]
2861
+ target = target_table.columns[link.target_column]
2862
+ key = _as_key_tuple(target_table.primary_key)
2863
+ if key != (link.target_column,):
2864
+ # A declared foreign key may name a unique column that is not the
2865
+ # key, which is no fault of the dtypes.
2866
+ has = (
2867
+ f'the primary key of {link.target_table!r} is '
2868
+ f'{_and_list(list(key))}'
2869
+ if key
2870
+ else f'{link.target_table!r} has no primary key'
2871
+ )
2872
+ warnings.warn(
2873
+ f'{link.source_table}.{link.source_column} refers to '
2874
+ f'{link.target_table}.{link.target_column}, but {has}. A '
2875
+ f'relationship joins to a primary key, so none was made from '
2876
+ f'it. Pass '
2877
+ f'primary_keys={{{link.target_table!r}: '
2878
+ f'{link.target_column!r}}} if {link.target_column!r} '
2879
+ f'identifies a row of {link.target_table!r}.',
2880
+ category=SDMWarning,
2881
+ stacklevel=caller_stacklevel(),
2882
+ )
2883
+ continue
2884
+ warnings.warn(
2885
+ f'{link.source_table}.{link.source_column} names the key of '
2886
+ f'{link.target_table!r} and holds {source.dtype!r} where that key '
2887
+ f'holds {target.dtype!r}, which cannot match, so no relationship '
2888
+ f'was made from it. Store the two as one type if they are the same '
2889
+ f'identifier.',
2890
+ category=SDMWarning,
2891
+ stacklevel=caller_stacklevel(),
2892
+ )
2893
+
2894
+
2895
+ def _require_joinable(
2896
+ tables: Mapping[str, _TableShape], link: Relationship
2897
+ ) -> None:
2898
+ r"""A relationship has to be one the engine can actually build.
2899
+
2900
+ An edge joins a foreign key to the *primary key* of the table it points at.
2901
+ That is the only shape the graph has, so a link naming some other
2902
+ column is not narrower, it is different: the language would resolve one
2903
+ graph while the engine built another. And the two columns have to hold
2904
+ comparable values, or the join matches nothing.
2905
+ """
2906
+ target = tables[link.target_table]
2907
+ key = target.primary_key
2908
+ keys = _as_key_tuple(key)
2909
+ if len(keys) > 1:
2910
+ raise StructuredDataError(
2911
+ f'{link} points at {link.target_table!r}, whose primary key is '
2912
+ f'{list(keys)}. An edge joins one column to one key, so a table '
2913
+ f'with a composite key cannot be the target of a relationship',
2914
+ code='INVALID_REQUEST',
2915
+ )
2916
+ if link.target_column not in keys:
2917
+ named = f'{key!r}' if key is not None else 'no primary key'
2918
+ raise StructuredDataError(
2919
+ f'{link} does not join to a primary key: table '
2920
+ f'{link.target_table!r} has {named}. A relationship points at the '
2921
+ f'key that identifies a row, so set that key first if it is not '
2922
+ f'the one inference found',
2923
+ code='INVALID_REQUEST',
2924
+ )
2925
+ source_dtype = tables[link.source_table][link.source_column].dtype
2926
+ target_dtype = target[link.target_column].dtype
2927
+ if _join_family(source_dtype) != _join_family(target_dtype):
2928
+ raise StructuredDataError(
2929
+ f'{link} joins {source_dtype!r} to {target_dtype!r}, which cannot '
2930
+ f'match. Give both columns the same kind of value',
2931
+ code='INVALID_REQUEST',
2932
+ )
2933
+
2934
+
2935
+ def _join_family(dtype: str) -> str:
2936
+ r"""What has to agree across a join: a number matches a number."""
2937
+ if dtype.startswith(('int', 'float')):
2938
+ return 'number'
2939
+ if dtype.startswith('timestamp'):
2940
+ return 'timestamp'
2941
+ return dtype
2942
+
2943
+
2944
+ def supports_dtype(stype: str, dtype: str) -> bool:
2945
+ r"""Whether a column of ``dtype`` can mean ``stype``.
2946
+
2947
+ Answered by the language rather than by a second table kept here, so the
2948
+ two cannot drift. An unrecognised pair is allowed through: this is a guard
2949
+ against a known-wrong combination, not a whitelist.
2950
+ """
2951
+ from predictive_query.schema.typing import Dtype, Stype
2952
+
2953
+ try:
2954
+ return Stype(stype).supports_dtype(
2955
+ Dtype(_WIRE_TO_PQL_DTYPE.get(dtype, dtype))
2956
+ )
2957
+ except ValueError:
2958
+ return True
2959
+
2960
+
2961
+ class _KeyQuality(NamedTuple):
2962
+ r"""What one pass over a key column found.
2963
+
2964
+ Computed once and handed on. The check, the count that gets recorded and
2965
+ the example in the message all used to derive this separately, which meant
2966
+ three full passes over the same column for every key assignment.
2967
+ """
2968
+
2969
+ duplicates: int
2970
+ nulls: int
2971
+ example: Any
2972
+
2973
+
2974
+ def _is_joinable(tables: Mapping[str, _TableShape], link: Relationship) -> bool:
2975
+ try:
2976
+ _require_joinable(tables, link)
2977
+ except StructuredDataError:
2978
+ return False
2979
+ return True
2980
+
2981
+
2982
+ def _the_key_of(table: str, names: Sequence[str]) -> str:
2983
+ """How to name a key in a message, for one column or several.
2984
+
2985
+ Joining the parts with a dot rendered `['a', 'b']` of table `t` as
2986
+ `t.a.b`, which reads as one column named `a.b`.
2987
+ """
2988
+ if len(names) == 1:
2989
+ return f'{table}.{names[0]}'
2990
+ return f'the primary key of {table!r}, {_and_list(list(names))},'
2991
+
2992
+
2993
+ _NAME_ANOTHER_KEY = (
2994
+ ' Name a different primary key if this column is a foreign key rather '
2995
+ 'than an identifier.'
2996
+ )
2997
+
2998
+
2999
+ def _too_many_repeats(duplicates: int, rows: int) -> bool:
3000
+ r"""Whether a stated key repeats more than inference would tolerate."""
3001
+ return duplicates > rows * (1 - _KEY_UNIQUENESS)
3002
+
3003
+
3004
+ def _not_unique_enough(
3005
+ table: str, names: Sequence[str], duplicates: int, rows: int, example: Any
3006
+ ) -> str:
3007
+ return (
3008
+ f'{_the_key_of(table, names)} is not unique enough to identify a '
3009
+ f'row: {duplicates:,} of {rows:,} rows repeat a value already used, '
3010
+ f'e.g. {example!r}'
3011
+ )
3012
+
3013
+
3014
+ def _a_key_missing_values(
3015
+ table: str, names: Sequence[str], nulls: int, empty: Sequence[str] | None
3016
+ ) -> str:
3017
+ r"""Why a stated key with a null in it is refused.
3018
+
3019
+ ``empty`` names the columns holding the nulls, or is ``None`` where only
3020
+ the rows missing some part of a composite key were counted.
3021
+ """
3022
+ where = f', in {_and_list(empty)},' if empty else ' in part of it,'
3023
+ several = empty is None or len(empty) > 1
3024
+ return (
3025
+ f'{_the_key_of(table, names)} has {nulls:,} row(s) with no value'
3026
+ f'{where} so those rows cannot be identified; fill or drop them '
3027
+ f'before using {"them" if several else "it"} as a primary key'
3028
+ )
3029
+
3030
+
3031
+ def _warn_a_key_repeats(
3032
+ table: str, names: Sequence[str], duplicates: int, example: Any, remedy: str
3033
+ ) -> None:
3034
+ warnings.warn(
3035
+ f'{table}.{".".join(names)} identifies a row {duplicates:,} time(s) '
3036
+ f'it has already been used, e.g. {example!r}. Those rows are '
3037
+ f'indistinguishable to the model.{remedy}',
3038
+ category=SDMWarning,
3039
+ stacklevel=caller_stacklevel(),
3040
+ )
3041
+
3042
+
3043
+ def _warn_a_key_has_no_value(table: str, column: str, missing: int) -> None:
3044
+ r"""Said of a key that was inferred rather than stated, which a null in
3045
+ it does not cost: inference is a guess, and refusing would break tables
3046
+ that worked before.
3047
+ """
3048
+ warnings.warn(
3049
+ f'{table}.{column} identifies a row, but {missing:,} row(s) have no '
3050
+ f'value for it. Those rows cannot be scored and the model will not '
3051
+ f'see them; fill or drop them, or name a different primary key.',
3052
+ category=SDMWarning,
3053
+ stacklevel=caller_stacklevel(),
3054
+ )
3055
+
3056
+
3057
+ def _warn_a_key_was_declined(
3058
+ table: str,
3059
+ column: str,
3060
+ duplicates: int,
3061
+ rows: int,
3062
+ example: Any,
3063
+ remedy: str,
3064
+ ) -> None:
3065
+ warnings.warn(
3066
+ f'{table}.{column} is named like a key and another table names it '
3067
+ f'too, but {duplicates:,} of {rows:,} rows repeat a value already '
3068
+ f'used, e.g. {example!r}, so it does not identify a row and '
3069
+ f'{table!r} gets no primary key. Nothing can link to a table without '
3070
+ f'one: every relationship pointing at {table!r} is dropped, so '
3071
+ f'nothing is gathered through it, and a question about another table '
3072
+ f'reaches it only where {table!r} names that table itself. {remedy}',
3073
+ category=SDMWarning,
3074
+ stacklevel=caller_stacklevel(),
3075
+ )
3076
+
3077
+
3078
+ def _key_quality(frame: pd.DataFrame, names: Sequence[str]) -> _KeyQuality:
3079
+ columns = list(names)
3080
+ if any(name not in frame.columns for name in columns):
3081
+ return _KeyQuality(0, 0, None)
3082
+ key = frame[columns[0]] if len(columns) == 1 else frame[columns]
3083
+ nulls = int(
3084
+ key.isna().any(axis=1).sum() if len(columns) > 1 else key.isna().sum()
3085
+ )
3086
+ # `duplicated` treats null as equal to null, so nulls would otherwise be
3087
+ # counted as repeats and reported as a duplicate problem, which has a
3088
+ # different fix. Count them separately and leave them out of this.
3089
+ present = (
3090
+ key[~key.isna().any(axis=1)] if len(columns) > 1 else key[key.notna()]
3091
+ )
3092
+ repeated_mask = present.duplicated()
3093
+ duplicates = int(repeated_mask.sum())
3094
+ example = None
3095
+ if duplicates:
3096
+ first = present[repeated_mask].iloc[0]
3097
+ example = first if len(columns) == 1 else tuple(first)
3098
+ return _KeyQuality(duplicates, nulls, example)
3099
+
3100
+
3101
+ def to_schema_definition(schema: Schema) -> Any:
3102
+ r"""This schema as the language's own schema definition.
3103
+
3104
+ The language resolves a query against names and types with no rows
3105
+ attached, which is what this hands it. Built here rather than in the query
3106
+ module so that a schema has exactly one translation into what PQL reads.
3107
+
3108
+ `TableDefinition.pkey` is a single column name, so the language has no
3109
+ form for a composite key and cannot be given one. It is dropped rather
3110
+ than refused, because the engine does support one and the request that
3111
+ follows carries it. What the drop costs is that the language resolves the
3112
+ query against a table it believes has no identity, so a question that
3113
+ needs one is answered as though the table had none, which is why it is
3114
+ said rather than done quietly.
3115
+ """
3116
+ from predictive_query.schema.definition import (
3117
+ ColumnKey,
3118
+ ColumnKeyGroup,
3119
+ SchemaDefinition,
3120
+ )
3121
+ from predictive_query.schema.table import Column as PQLColumn
3122
+ from predictive_query.schema.table import TableDefinition
3123
+ from predictive_query.schema.typing import Dtype, Stype
3124
+
3125
+ tables = {}
3126
+ for name, table in schema.tables.items():
3127
+ columns = []
3128
+ for column in table.columns.values():
3129
+ try:
3130
+ dtype = Dtype(
3131
+ _WIRE_TO_PQL_DTYPE.get(column.dtype, column.dtype)
3132
+ )
3133
+ stype = Stype(column.stype)
3134
+ except ValueError:
3135
+ continue
3136
+ columns.append(
3137
+ PQLColumn(name=column.name, stype=stype, dtype=dtype)
3138
+ )
3139
+ key = table.primary_key
3140
+ if key is not None and not isinstance(key, str):
3141
+ warnings.warn(
3142
+ f'table {name!r} has the composite primary key '
3143
+ f'{list(key)}, and a predictive query can only name a single '
3144
+ f'key column, so this query is resolved against {name!r} as '
3145
+ f'though it had no key. The request still carries the '
3146
+ f'composite key; only the query does not see it. Name a '
3147
+ f'single column as the primary key if the query needs to '
3148
+ f'identify a row of {name!r}.',
3149
+ category=SDMWarning,
3150
+ stacklevel=caller_stacklevel(),
3151
+ )
3152
+ tables[name] = TableDefinition(
3153
+ cols=columns,
3154
+ pkey=key if isinstance(key, str) else None,
3155
+ time_col=table.time_column,
3156
+ )
3157
+
3158
+ groups = [
3159
+ ColumnKeyGroup(
3160
+ # The group is documented as sorted and deduped, and a ColumnKey
3161
+ # has no ordering of its own, so sort on what it holds.
3162
+ columns=tuple(
3163
+ sorted(
3164
+ (
3165
+ ColumnKey(link.source_table, link.source_column),
3166
+ ColumnKey(link.target_table, link.target_column),
3167
+ ),
3168
+ key=lambda key: (key.table_name, key.col_name),
3169
+ )
3170
+ )
3171
+ )
3172
+ for link in schema.relationships
3173
+ ]
3174
+ return SchemaDefinition(tables=tables, col_groups=groups)
3175
+
3176
+
3177
+ # The wire spells a few types differently from the language.
3178
+ _WIRE_TO_PQL_DTYPE = {
3179
+ 'int64': 'int',
3180
+ 'int32': 'int',
3181
+ 'float64': 'float',
3182
+ 'float32': 'float',
3183
+ 'timestamp[us]': 'time',
3184
+ 'timestamp[ns]': 'time',
3185
+ 'timestamp[ms]': 'time',
3186
+ 'timestamp[s]': 'time',
3187
+ }