structured-data-models-client 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm_client/__init__.py +70 -0
- sdm_client/_version.py +5 -0
- sdm_client/adapters/__init__.py +9 -0
- sdm_client/adapters/relational.py +870 -0
- sdm_client/adapters/tabular.py +580 -0
- sdm_client/base.py +300 -0
- sdm_client/client.py +1416 -0
- sdm_client/core/__init__.py +5 -0
- sdm_client/core/databricks_serving.py +611 -0
- sdm_client/core/databricks_user_agent.py +76 -0
- sdm_client/core/dates.py +278 -0
- sdm_client/core/transport.py +898 -0
- sdm_client/core/warnings.py +51 -0
- sdm_client/decompose.py +421 -0
- sdm_client/errors.py +177 -0
- sdm_client/materializers/__init__.py +10 -0
- sdm_client/materializers/relational.py +1786 -0
- sdm_client/materializers/sources.py +331 -0
- sdm_client/materializers/tabular.py +339 -0
- sdm_client/models.py +251 -0
- sdm_client/py.typed +0 -0
- sdm_client/query.py +647 -0
- sdm_client/request.py +1127 -0
- sdm_client/result.py +328 -0
- sdm_client/schema.py +3187 -0
- sdm_client/wire/__init__.py +46 -0
- sdm_client/wire/answers.py +80 -0
- sdm_client/wire/dtypes.py +701 -0
- sdm_client/wire/keys.py +363 -0
- sdm_client/wire/tables.py +50 -0
- structured_data_models_client-1.0.0.dist-info/METADATA +99 -0
- structured_data_models_client-1.0.0.dist-info/RECORD +36 -0
- structured_data_models_client-1.0.0.dist-info/WHEEL +5 -0
- structured_data_models_client-1.0.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models_client-1.0.0.dist-info/licenses/NOTICE +10 -0
- structured_data_models_client-1.0.0.dist-info/top_level.txt +1 -0
sdm_client/schema.py
ADDED
|
@@ -0,0 +1,3187 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
|
|
2
|
+
# All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
r"""The tables a request is about, and what the model should make of them.
|
|
6
|
+
|
|
7
|
+
A :class:`Schema` holds your tables, what each column means, and how the tables
|
|
8
|
+
relate. One table or many is the same object. A single-table model is the
|
|
9
|
+
degenerate case of the same shape, not a second system, which is why this is
|
|
10
|
+
not called a graph.
|
|
11
|
+
|
|
12
|
+
Everything is inferred. You override only where inference is wrong:
|
|
13
|
+
|
|
14
|
+
>>> schema = Schema.from_data({'users': users_df})
|
|
15
|
+
>>> schema['users'].primary_key = 'user_id'
|
|
16
|
+
>>> schema['users']['churn'].stype = 'categorical'
|
|
17
|
+
|
|
18
|
+
**Inference reads values, so a code stored as a number reads as a number.** A
|
|
19
|
+
postcode, a store number or a status code is ``numerical`` here, and the model
|
|
20
|
+
will treat its distances as meaningful. No amount of inference settles that.
|
|
21
|
+
Only you know the column is a label. Correct it:
|
|
22
|
+
|
|
23
|
+
>>> schema['users']['zip_code'].stype = 'categorical'
|
|
24
|
+
|
|
25
|
+
Relationship inference is deliberately conservative in the same spirit: it
|
|
26
|
+
links a column only to a primary key of exactly the same name, so
|
|
27
|
+
``orders.user_id`` finds ``users.user_id`` but not ``users.id``. A wrong link
|
|
28
|
+
changes what the model is shown and is harder to notice than a missing one, so
|
|
29
|
+
pass ``relationships=`` when the names do not line up.
|
|
30
|
+
|
|
31
|
+
Nothing here contacts a model. Inference reads your frames locally.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import warnings
|
|
37
|
+
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
38
|
+
from collections.abc import Set as AbstractSet
|
|
39
|
+
from dataclasses import dataclass, field, replace
|
|
40
|
+
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
|
|
41
|
+
|
|
42
|
+
import pandas as pd
|
|
43
|
+
from predictive_query.time_columns import (
|
|
44
|
+
TimeColumnChoice,
|
|
45
|
+
choose_time_column,
|
|
46
|
+
taken_unchecked,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
from sdm_client.core import dates
|
|
50
|
+
from sdm_client.core.warnings import SDMWarning, caller_stacklevel
|
|
51
|
+
from sdm_client.errors import StructuredDataError
|
|
52
|
+
from sdm_client.wire import (
|
|
53
|
+
UNTYPED,
|
|
54
|
+
infer_tfm_dtype,
|
|
55
|
+
require_scalar_cells,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if TYPE_CHECKING:
|
|
59
|
+
from sdm_connectors import Source
|
|
60
|
+
|
|
61
|
+
# Semantic types this client can declare. These are the contract's, and match
|
|
62
|
+
# ``predictive_query.Stype`` one for one.
|
|
63
|
+
SEMANTIC_TYPES = frozenset(
|
|
64
|
+
{
|
|
65
|
+
'ID',
|
|
66
|
+
'categorical',
|
|
67
|
+
'multicategorical',
|
|
68
|
+
'numerical',
|
|
69
|
+
'sequence',
|
|
70
|
+
'text',
|
|
71
|
+
'timestamp',
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# Above this many distinct values, a string column is prose rather than a set
|
|
76
|
+
# of labels. The exact number matters less than having one: a model encodes
|
|
77
|
+
# the two differently, and 'text' on a 3-value column wastes the signal.
|
|
78
|
+
_MAX_CATEGORICAL_CARDINALITY = 64
|
|
79
|
+
|
|
80
|
+
# A column is only a candidate identifier if nearly every value is distinct.
|
|
81
|
+
# Exactly 1.0 would reject a key with a single duplicated row, which is common
|
|
82
|
+
# in an extract and still the column the user means.
|
|
83
|
+
_KEY_UNIQUENESS = 0.99
|
|
84
|
+
|
|
85
|
+
# And below this, the repeats are the ordinary many-to-one shape rather than a
|
|
86
|
+
# fault in a key: a fact table's foreign key repeats on the order of its row
|
|
87
|
+
# count, while a key with a bad extract behind it repeats a handful of times.
|
|
88
|
+
# Between the two is where a column was meant to be the key and missed.
|
|
89
|
+
_NEARLY_A_KEY = 0.9
|
|
90
|
+
|
|
91
|
+
_KEY_SUFFIXES = ('_id', '_key', 'id', 'key')
|
|
92
|
+
|
|
93
|
+
_T = TypeVar('_T', bound='_TableShape')
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class Column:
|
|
98
|
+
r"""Describe one column and how the model should treat it.
|
|
99
|
+
|
|
100
|
+
Attributes:
|
|
101
|
+
name: The column name, as it appears in the frame.
|
|
102
|
+
dtype: The wire dtype, e.g. ``'int64'`` or ``'timestamp[us]'``.
|
|
103
|
+
Inferred from the values; correcting it is rarely necessary.
|
|
104
|
+
stype: What the column *means*: ``'numerical'``, ``'categorical'``,
|
|
105
|
+
``'ID'``, ``'text'``, ``'timestamp'``. This is the one worth
|
|
106
|
+
correcting: it decides how the model encodes the column.
|
|
107
|
+
stype_declared: Whether the meaning was assigned rather than inferred.
|
|
108
|
+
An all-null column is typed ``categorical`` because a schema has to
|
|
109
|
+
say something, and that placeholder is not a claim about the
|
|
110
|
+
values; a meaning the caller assigned is. Recorded so the two can
|
|
111
|
+
be told apart, since only the second is worth refusing a
|
|
112
|
+
contradiction against.
|
|
113
|
+
untyped: Whether the values carried no type at all, as in an all-null
|
|
114
|
+
object column. ``dtype`` is ``'string'`` for those, because a
|
|
115
|
+
schema has to name something and a string round-trips whatever
|
|
116
|
+
arrives, but the frame being scored may hold real values and
|
|
117
|
+
decide the dtype instead. Recorded so the adapter can read it
|
|
118
|
+
here rather than re-deriving it from the frame, which is how the
|
|
119
|
+
schema and the request came to disagree.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
name: str
|
|
123
|
+
dtype: str
|
|
124
|
+
stype: str
|
|
125
|
+
untyped: bool = field(init=False, default=False)
|
|
126
|
+
stype_declared: bool = field(init=False, default=False)
|
|
127
|
+
_stype_before_key: str | None = field(init=False, default=None, repr=False)
|
|
128
|
+
|
|
129
|
+
def __post_init__(self) -> None:
|
|
130
|
+
self._check_stype(self.stype)
|
|
131
|
+
|
|
132
|
+
def _mark_as_key(self) -> None:
|
|
133
|
+
r"""Type this column as an identifier because the schema joins on it.
|
|
134
|
+
|
|
135
|
+
A key typed from its dtype alone becomes ``numerical``, the model
|
|
136
|
+
stores it as a float, and the join against an integer key fails
|
|
137
|
+
downstream. That is true of a column the caller called ``numerical``
|
|
138
|
+
too: what a key means is settled by its being a key, not by a
|
|
139
|
+
preference, so a declared meaning is overridden here rather than
|
|
140
|
+
honoured. Like an inferred meaning, it is recorded so it returns if
|
|
141
|
+
the column stops being a key.
|
|
142
|
+
|
|
143
|
+
This goes around ``__setattr__``: routed through it the schema's own
|
|
144
|
+
marking set ``stype_declared``, becoming indistinguishable from a
|
|
145
|
+
meaning the caller chose, and what had been decided was gone with
|
|
146
|
+
nothing to restore.
|
|
147
|
+
"""
|
|
148
|
+
if self.stype == 'ID':
|
|
149
|
+
return
|
|
150
|
+
object.__setattr__(self, '_stype_before_key', self.stype)
|
|
151
|
+
object.__setattr__(self, 'stype', 'ID')
|
|
152
|
+
|
|
153
|
+
def _release_as_key(self) -> None:
|
|
154
|
+
r"""Restore a column's meaning after the schema stops joining on it.
|
|
155
|
+
|
|
156
|
+
Through the same adoption an assignment goes through: a column that
|
|
157
|
+
means `timestamp` orders the table it belongs to, and restoring the
|
|
158
|
+
meaning around that left a table saying it held a timestamp while
|
|
159
|
+
naming no time column. `check_consistent()` accepted that state, and
|
|
160
|
+
every temporal query then failed on.
|
|
161
|
+
"""
|
|
162
|
+
if self._stype_before_key is None:
|
|
163
|
+
return
|
|
164
|
+
restored = self._stype_before_key
|
|
165
|
+
object.__setattr__(self, 'stype', restored)
|
|
166
|
+
object.__setattr__(self, '_stype_before_key', None)
|
|
167
|
+
if restored == 'timestamp':
|
|
168
|
+
self._adopt_as_time_column()
|
|
169
|
+
|
|
170
|
+
def __setattr__(self, key: str, value: Any) -> None:
|
|
171
|
+
r"""Assign a field, keeping the three that describe each other true.
|
|
172
|
+
|
|
173
|
+
`untyped` is derived rather than declared. It records that inference
|
|
174
|
+
found no type, and the adapter reads it as authority over `dtype`, so
|
|
175
|
+
letting it be set is letting a column claim a dtype and hold none.
|
|
176
|
+
Naming the dtype is the information it was missing, which is why doing
|
|
177
|
+
so clears it and the adapter stops substituting.
|
|
178
|
+
|
|
179
|
+
`stype_declared` separates a meaning that was chosen from one that was
|
|
180
|
+
inferred, and a second assignment is what distinguishes them.
|
|
181
|
+
"""
|
|
182
|
+
if key == 'untyped':
|
|
183
|
+
raise StructuredDataError(
|
|
184
|
+
"'untyped' is inferred from the values and cannot be set; "
|
|
185
|
+
'assign the dtype you want instead',
|
|
186
|
+
code='INVALID_REQUEST',
|
|
187
|
+
)
|
|
188
|
+
if key == 'stype':
|
|
189
|
+
self._check_stype(value)
|
|
190
|
+
self._check_supports_dtype(value, getattr(self, 'dtype', None))
|
|
191
|
+
if hasattr(self, 'stype'):
|
|
192
|
+
object.__setattr__(self, 'stype_declared', True)
|
|
193
|
+
if key == 'dtype' and getattr(self, 'untyped', False):
|
|
194
|
+
object.__setattr__(self, 'untyped', False)
|
|
195
|
+
object.__setattr__(self, key, value)
|
|
196
|
+
if key == 'stype' and value == 'timestamp':
|
|
197
|
+
self._adopt_as_time_column()
|
|
198
|
+
if key == 'stype':
|
|
199
|
+
self._stay_a_key_if_the_schema_joins_on_this(value)
|
|
200
|
+
|
|
201
|
+
def _stay_a_key_if_the_schema_joins_on_this(self, declared: str) -> None:
|
|
202
|
+
r"""Put ``ID`` back if this column is one the schema joins on.
|
|
203
|
+
|
|
204
|
+
A key's meaning is settled by its being a key, and the schema says so
|
|
205
|
+
wherever it is asked. The answer was only re-derived when the
|
|
206
|
+
keys changed, so between declaring a meaning on a live key and the
|
|
207
|
+
next such moment the column read as whatever was declared. Nothing was
|
|
208
|
+
sent that way, because a request re-derives as it takes its copy; the
|
|
209
|
+
object simply described itself wrongly in the meantime.
|
|
210
|
+
|
|
211
|
+
The declaration replaces what there was to give back, rather than
|
|
212
|
+
being discarded beside it: declaring ``ID`` on a column that is
|
|
213
|
+
already a key left the meaning inference had found still recorded, and
|
|
214
|
+
moving the key elsewhere handed that back over the caller's own word.
|
|
215
|
+
"""
|
|
216
|
+
if declared == 'ID':
|
|
217
|
+
object.__setattr__(self, '_stype_before_key', None)
|
|
218
|
+
return
|
|
219
|
+
table = getattr(self, '_table', None)
|
|
220
|
+
schema = getattr(table, '_schema', None) if table is not None else None
|
|
221
|
+
if schema is None:
|
|
222
|
+
if table is not None and self.name in _as_key_tuple(
|
|
223
|
+
getattr(table, 'primary_key', None)
|
|
224
|
+
):
|
|
225
|
+
self._mark_as_key()
|
|
226
|
+
return
|
|
227
|
+
schema._mark_key_columns()
|
|
228
|
+
|
|
229
|
+
def _adopt_as_time_column(self) -> None:
|
|
230
|
+
r"""Let a column named as a timestamp order the table it belongs to.
|
|
231
|
+
|
|
232
|
+
`Table._check_time_column` already sets a column's stype when the
|
|
233
|
+
column is named as the time column. Only that direction was wired, so
|
|
234
|
+
a caller correcting an inferred schema the way the SDK's own error
|
|
235
|
+
messages teach, ``schema[table][column].stype = 'timestamp'``, got
|
|
236
|
+
a table reporting `stype='timestamp'` and `time_column=None` at the
|
|
237
|
+
same time, and every temporal query on it was refused for want of a
|
|
238
|
+
time column the schema said was there.
|
|
239
|
+
|
|
240
|
+
Only an absent time column is filled: naming a second timestamp does
|
|
241
|
+
not displace the first. Nothing is adopted when inference already
|
|
242
|
+
chose, so this resolves the contradiction rather than overriding a
|
|
243
|
+
decision.
|
|
244
|
+
|
|
245
|
+
A column that is also part of the key is left alone rather than
|
|
246
|
+
refused: the two meanings contradict, and raising from an stype
|
|
247
|
+
assignment would report it from the wrong place. `check_consistent`
|
|
248
|
+
is where that is said.
|
|
249
|
+
|
|
250
|
+
The time column is set directly because the checked route assigns this
|
|
251
|
+
column's stype, which would re-enter here before the attribute it
|
|
252
|
+
guards on exists.
|
|
253
|
+
"""
|
|
254
|
+
table = getattr(self, '_table', None)
|
|
255
|
+
if table is None or getattr(table, 'time_column', None) is not None:
|
|
256
|
+
return
|
|
257
|
+
if self.name in _as_key_tuple(getattr(table, 'primary_key', None)):
|
|
258
|
+
return
|
|
259
|
+
object.__setattr__(table, 'time_column', self.name)
|
|
260
|
+
|
|
261
|
+
def check_consistent(self) -> None:
|
|
262
|
+
r"""Refuse a column whose storage and meaning contradict each other.
|
|
263
|
+
|
|
264
|
+
Assigning a meaning is checked against the storage as it happens, but
|
|
265
|
+
the reverse is not, because a caller re-declaring storage often has to
|
|
266
|
+
set the two one at a time and either order passes through a pair that
|
|
267
|
+
does not hold. The pair is asked about here instead, once the caller
|
|
268
|
+
has finished and before anything is sent.
|
|
269
|
+
"""
|
|
270
|
+
if self.dtype is None or self.stype is None:
|
|
271
|
+
return
|
|
272
|
+
self._check_supports_dtype(self.stype, self.dtype)
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def _check_stype(value: Any) -> None:
|
|
276
|
+
if value not in SEMANTIC_TYPES:
|
|
277
|
+
raise StructuredDataError(
|
|
278
|
+
f'{value!r} is not a semantic type; use one of '
|
|
279
|
+
f'{sorted(SEMANTIC_TYPES)}',
|
|
280
|
+
code='INVALID_REQUEST',
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
def _check_supports_dtype(self, stype: str, dtype: str | None) -> None:
|
|
284
|
+
r"""Refuse a meaning the values cannot carry.
|
|
285
|
+
|
|
286
|
+
The language already decides this, and used to decide it at
|
|
287
|
+
``materialize()``, several cells after the assignment that caused it,
|
|
288
|
+
as a bare ``ValueError``. Asking the same question here changes only
|
|
289
|
+
when the answer arrives. Nothing is refused that was going to be
|
|
290
|
+
accepted.
|
|
291
|
+
"""
|
|
292
|
+
if dtype is None:
|
|
293
|
+
return
|
|
294
|
+
if not supports_dtype(stype, dtype):
|
|
295
|
+
raise StructuredDataError(
|
|
296
|
+
f'column {self.name!r} holds {dtype!r}, which cannot mean '
|
|
297
|
+
f'{stype!r}',
|
|
298
|
+
code='INVALID_REQUEST',
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
def __repr__(self) -> str:
|
|
302
|
+
untyped = ', untyped=True' if self.untyped else ''
|
|
303
|
+
return (
|
|
304
|
+
f'Column({self.name!r}, dtype={self.dtype!r}, '
|
|
305
|
+
f'stype={self.stype!r}{untyped})'
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class _TableShape:
|
|
310
|
+
r"""What a table says about itself, whether or not it holds the rows.
|
|
311
|
+
|
|
312
|
+
Keys, the time column and what each column means are declared the same
|
|
313
|
+
way on a table of rows and on one that stays in its source; only the
|
|
314
|
+
checks that read rows differ, and those ask for the rows rather than
|
|
315
|
+
assume them.
|
|
316
|
+
"""
|
|
317
|
+
|
|
318
|
+
name: str
|
|
319
|
+
columns: dict[str, Column]
|
|
320
|
+
primary_key: str | Sequence[str] | None
|
|
321
|
+
time_column: str | None
|
|
322
|
+
duplicate_keys: int | None
|
|
323
|
+
|
|
324
|
+
def _own_columns(self) -> None:
|
|
325
|
+
r"""Point every column at this table, and take a copy of any that is
|
|
326
|
+
already pointing somewhere else.
|
|
327
|
+
|
|
328
|
+
The back-reference is what lets a column make itself the time column
|
|
329
|
+
when it is told it is a timestamp. One `Column` handed to two tables
|
|
330
|
+
could only point at one of them, so setting the stype through the
|
|
331
|
+
first would have ordered the second. A column belongs to a table;
|
|
332
|
+
handing the same one to two is asking for two columns.
|
|
333
|
+
|
|
334
|
+
A borrowed column is copied into a new dict, so that a caller holding
|
|
335
|
+
the one they passed does not find entries in it they did not put
|
|
336
|
+
there. The back-reference itself is set directly: it is not a field,
|
|
337
|
+
and must stay out of equality and `repr`.
|
|
338
|
+
"""
|
|
339
|
+
# By identity. A `Table` is a dataclass, so `in (None, self)` asks
|
|
340
|
+
# `==`, which compares the frames inside and raises on any table that
|
|
341
|
+
# holds one.
|
|
342
|
+
borrowed = [
|
|
343
|
+
name
|
|
344
|
+
for name, column in self.columns.items()
|
|
345
|
+
if (owner := getattr(column, '_table', None)) is not None
|
|
346
|
+
and owner is not self
|
|
347
|
+
]
|
|
348
|
+
if borrowed:
|
|
349
|
+
object.__setattr__(self, 'columns', dict(self.columns))
|
|
350
|
+
for name in borrowed:
|
|
351
|
+
self.columns[name] = _copy_column(self.columns[name])
|
|
352
|
+
for column in self.columns.values():
|
|
353
|
+
object.__setattr__(column, '_table', self)
|
|
354
|
+
|
|
355
|
+
def __eq__(self, other: object) -> bool:
|
|
356
|
+
r"""Whether two tables describe the same thing.
|
|
357
|
+
|
|
358
|
+
Written out because the generated one compares the frames with `==`,
|
|
359
|
+
which for a DataFrame is a frame of booleans. Asking whether two
|
|
360
|
+
tables are equal raised `ValueError: The truth value of a DataFrame is
|
|
361
|
+
ambiguous` from inside pandas, naming nothing the caller wrote. The
|
|
362
|
+
rows are compared as rows, and everything else by value, which is what
|
|
363
|
+
the question means.
|
|
364
|
+
"""
|
|
365
|
+
if not isinstance(other, _TableShape) or type(other) is not type(self):
|
|
366
|
+
return NotImplemented
|
|
367
|
+
described = (
|
|
368
|
+
self.name == other.name
|
|
369
|
+
and self.columns == other.columns
|
|
370
|
+
and _as_key_tuple(self.primary_key)
|
|
371
|
+
== _as_key_tuple(other.primary_key)
|
|
372
|
+
and self.time_column == other.time_column
|
|
373
|
+
and getattr(self, 'source_name', None)
|
|
374
|
+
== getattr(other, 'source_name', None)
|
|
375
|
+
)
|
|
376
|
+
if not described:
|
|
377
|
+
return False
|
|
378
|
+
mine = getattr(self, 'data', None)
|
|
379
|
+
theirs = getattr(other, 'data', None)
|
|
380
|
+
if mine is None or theirs is None:
|
|
381
|
+
return mine is theirs
|
|
382
|
+
return bool(mine.equals(theirs))
|
|
383
|
+
|
|
384
|
+
__hash__ = None # type: ignore[assignment]
|
|
385
|
+
|
|
386
|
+
def __copy__(self: _T) -> _T:
|
|
387
|
+
r"""A shallow copy is still a table of its own columns.
|
|
388
|
+
|
|
389
|
+
Sharing them left the copy unable to answer for itself. Its columns
|
|
390
|
+
pointed at the table they came from, so changing the copy's key typed
|
|
391
|
+
nothing. It could also retype the original once the copy was put in a
|
|
392
|
+
schema.
|
|
393
|
+
"""
|
|
394
|
+
return _copy_table(self)
|
|
395
|
+
|
|
396
|
+
def __setattr__(self, key: str, value: Any) -> None:
|
|
397
|
+
if key in ('primary_key', 'time_column') and 'columns' in self.__dict__:
|
|
398
|
+
# Before anything is marked, not after. Naming a key or a time
|
|
399
|
+
# column types the columns it names, and ownership was taken in
|
|
400
|
+
# `__post_init__` (which runs after the fields are assigned), so
|
|
401
|
+
# a table built on another's columns typed *those* first and left
|
|
402
|
+
# the table it borrowed them from changed.
|
|
403
|
+
self._own_columns()
|
|
404
|
+
if key == 'primary_key':
|
|
405
|
+
# Read once, here, and what was read is what is kept. A key is
|
|
406
|
+
# consulted many times afterwards: to check it, to type its
|
|
407
|
+
# columns, and to copy the table. An iterable that empties on
|
|
408
|
+
# the first read arrived as "was given an empty primary key"
|
|
409
|
+
# about a key that named two columns. Reading it before anything
|
|
410
|
+
# is assigned also stops a key this cannot read from being taken
|
|
411
|
+
# and leaving a table that can be neither copied nor sent.
|
|
412
|
+
names = _as_key_tuple(value)
|
|
413
|
+
if value is not None and not isinstance(value, str):
|
|
414
|
+
value = names
|
|
415
|
+
if value is None:
|
|
416
|
+
object.__setattr__(self, 'duplicate_keys', 0)
|
|
417
|
+
object.__setattr__(self, key, None)
|
|
418
|
+
self._re_derive_key_columns()
|
|
419
|
+
return
|
|
420
|
+
self._check_key(value)
|
|
421
|
+
object.__setattr__(self, key, value)
|
|
422
|
+
self._adopt_key(value)
|
|
423
|
+
self._re_derive_key_columns()
|
|
424
|
+
return
|
|
425
|
+
if key == 'time_column' and value is not None:
|
|
426
|
+
self._check_time_column(value)
|
|
427
|
+
object.__setattr__(self, key, value)
|
|
428
|
+
|
|
429
|
+
def _re_derive_key_columns(self) -> None:
|
|
430
|
+
r"""Ask this table's schema which of its columns are still keys.
|
|
431
|
+
|
|
432
|
+
A key that has just been replaced is no longer a key, and nothing said
|
|
433
|
+
so: the column it was kept the ``ID`` the old key gave it. Only the
|
|
434
|
+
schema knows whether some relationship still joins on that column, so
|
|
435
|
+
the question goes there; a table held on its own answers it for its own
|
|
436
|
+
key alone.
|
|
437
|
+
"""
|
|
438
|
+
schema = getattr(self, '_schema', None)
|
|
439
|
+
if schema is not None:
|
|
440
|
+
schema._mark_key_columns()
|
|
441
|
+
return
|
|
442
|
+
named = set(_as_key_tuple(getattr(self, 'primary_key', None)))
|
|
443
|
+
for name, column in getattr(self, 'columns', {}).items():
|
|
444
|
+
if name not in named:
|
|
445
|
+
column._release_as_key()
|
|
446
|
+
|
|
447
|
+
def _adopt_key(self, value: str | Sequence[str]) -> None:
|
|
448
|
+
r"""Make the rest of the table agree with the key it was just given.
|
|
449
|
+
|
|
450
|
+
A key identifies a row, so its columns mean ``ID``. The engine's own
|
|
451
|
+
setter does the same thing. Any imperfection in the uniqueness
|
|
452
|
+
check tolerated is recorded and said out loud. Inference does all three
|
|
453
|
+
after nominating a key; assignment has to as well, or the two routes
|
|
454
|
+
describe the same table differently.
|
|
455
|
+
|
|
456
|
+
A repeat is the one visible sign that the column identifies something
|
|
457
|
+
coarser than a row, most often a parent on a table whose real key is
|
|
458
|
+
that column together with another. Naming that possibility is what
|
|
459
|
+
makes the warning actionable; saying only that the rows are
|
|
460
|
+
indistinguishable describes the symptom. A column that cannot be a key
|
|
461
|
+
at all has already been refused by `_check_key`.
|
|
462
|
+
|
|
463
|
+
Which is why only a column `_check_key` would take is offered. The
|
|
464
|
+
search filtered nothing, and the column that distinguishes two rows of
|
|
465
|
+
the same entity is very often the time column, as in the ordinary "two
|
|
466
|
+
snapshots of one account" shape. The composite key the warning
|
|
467
|
+
recommended was the one the next line refuses. Where the only
|
|
468
|
+
disambiguating column is one a key cannot hold, that is said instead
|
|
469
|
+
of recommending it.
|
|
470
|
+
"""
|
|
471
|
+
names = _as_key_tuple(value)
|
|
472
|
+
for name in names:
|
|
473
|
+
self.columns[name]._mark_as_key()
|
|
474
|
+
frame = getattr(self, 'data', None)
|
|
475
|
+
if frame is None:
|
|
476
|
+
# A table in its source is counted by `Schema.from_source`, and a
|
|
477
|
+
# key named afterwards has not been.
|
|
478
|
+
object.__setattr__(self, 'duplicate_keys', None)
|
|
479
|
+
return
|
|
480
|
+
if frame.empty:
|
|
481
|
+
return
|
|
482
|
+
quality = _key_quality(frame, names)
|
|
483
|
+
object.__setattr__(self, 'duplicate_keys', quality.duplicates)
|
|
484
|
+
if quality.duplicates:
|
|
485
|
+
distinguishing = [
|
|
486
|
+
other
|
|
487
|
+
for other in self.columns
|
|
488
|
+
if other not in names
|
|
489
|
+
and _key_quality(frame, (*names, other)).duplicates == 0
|
|
490
|
+
]
|
|
491
|
+
usable = [
|
|
492
|
+
other
|
|
493
|
+
for other in distinguishing
|
|
494
|
+
if self._could_hold_a_key(other)
|
|
495
|
+
]
|
|
496
|
+
if usable:
|
|
497
|
+
remedy = (
|
|
498
|
+
f' {".".join(names)!r} is unique together with '
|
|
499
|
+
f'{", ".join(repr(other) for other in usable)}, which is '
|
|
500
|
+
f'what a composite key looks like; pass that as the '
|
|
501
|
+
f'primary key if so.'
|
|
502
|
+
)
|
|
503
|
+
elif distinguishing:
|
|
504
|
+
remedy = (
|
|
505
|
+
f' They differ only by '
|
|
506
|
+
f'{_and_list(distinguishing)}, which a primary key '
|
|
507
|
+
f'cannot be made of, so there is no composite key to name '
|
|
508
|
+
f'here: keep one row per {".".join(names)!r} if the model '
|
|
509
|
+
f'should see one, or name a different primary key.'
|
|
510
|
+
)
|
|
511
|
+
else:
|
|
512
|
+
remedy = _NAME_ANOTHER_KEY
|
|
513
|
+
_warn_a_key_repeats(
|
|
514
|
+
self.name, names, quality.duplicates, quality.example, remedy
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
def _could_hold_a_key(self, name: str) -> bool:
|
|
518
|
+
r"""Whether `_check_key` would take this column as part of a key.
|
|
519
|
+
|
|
520
|
+
The conditions it refuses on that a column carries on its own: a dtype
|
|
521
|
+
no identifier can be made of, being the table's time column, and
|
|
522
|
+
holding a null. Asked here so that what is recommended is what is
|
|
523
|
+
accepted. A column with one null was offered as half of a composite
|
|
524
|
+
key and refused by `_check_key` on the very next call.
|
|
525
|
+
|
|
526
|
+
Uniqueness is not asked here. It is a property of the combination, and
|
|
527
|
+
the caller of this is choosing what to combine.
|
|
528
|
+
"""
|
|
529
|
+
column = self.columns.get(name)
|
|
530
|
+
if column is None or name == getattr(self, 'time_column', None):
|
|
531
|
+
return False
|
|
532
|
+
if not supports_dtype('ID', column.dtype):
|
|
533
|
+
return False
|
|
534
|
+
frame = getattr(self, 'data', None)
|
|
535
|
+
if frame is None or name not in frame.columns:
|
|
536
|
+
return True
|
|
537
|
+
return not bool(frame[name].isna().any())
|
|
538
|
+
|
|
539
|
+
def _check_name(self, name: str) -> None:
|
|
540
|
+
if not isinstance(name, str) or not name:
|
|
541
|
+
raise StructuredDataError(
|
|
542
|
+
f'The name {name!r} must be a non-empty string.',
|
|
543
|
+
code='INVALID_REQUEST',
|
|
544
|
+
)
|
|
545
|
+
name_invalid_chars = _invalid_name_chars(name)
|
|
546
|
+
if len(name_invalid_chars) > 0:
|
|
547
|
+
raise StructuredDataError(
|
|
548
|
+
f'The name {name!r} contains '
|
|
549
|
+
f'one or more invalid characters: {name_invalid_chars}.',
|
|
550
|
+
code='INVALID_REQUEST',
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
def _check_column(self, name: str) -> None:
|
|
554
|
+
if name not in getattr(self, 'columns', {}):
|
|
555
|
+
raise StructuredDataError(
|
|
556
|
+
f'table {self.name!r} has no column {name!r}; it has '
|
|
557
|
+
f'{sorted(getattr(self, "columns", {}))}',
|
|
558
|
+
code='INVALID_REQUEST',
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
def _check_key(self, value: str | Sequence[str]) -> None:
|
|
562
|
+
r"""A key has to exist, and has to identify a row.
|
|
563
|
+
|
|
564
|
+
Inference deliberately tolerates a nearly-unique column, because a
|
|
565
|
+
single duplicated row is common in an extract and still the column the
|
|
566
|
+
caller means. An explicit key is an assertion rather than a guess, so
|
|
567
|
+
it is held to the same standard the inference applies. A caller
|
|
568
|
+
who states something false should hear so.
|
|
569
|
+
|
|
570
|
+
A null is refused however few there are. It is not a near miss on
|
|
571
|
+
uniqueness: the engine drops those rows rather than scoring them, and
|
|
572
|
+
a single one passed silently because `duplicated` does not flag one
|
|
573
|
+
null.
|
|
574
|
+
"""
|
|
575
|
+
names = (value,) if isinstance(value, str) else tuple(value)
|
|
576
|
+
if not names:
|
|
577
|
+
raise StructuredDataError(
|
|
578
|
+
f'table {self.name!r} was given an empty primary key',
|
|
579
|
+
code='INVALID_REQUEST',
|
|
580
|
+
)
|
|
581
|
+
for name in names:
|
|
582
|
+
self._check_column(name)
|
|
583
|
+
column = self.columns[name]
|
|
584
|
+
if not supports_dtype('ID', column.dtype):
|
|
585
|
+
raise StructuredDataError(
|
|
586
|
+
f'column {name!r} holds {column.dtype!r}, which cannot '
|
|
587
|
+
f'identify a row, so it cannot be a primary key',
|
|
588
|
+
code='INVALID_REQUEST',
|
|
589
|
+
)
|
|
590
|
+
if name == getattr(self, 'time_column', None):
|
|
591
|
+
raise StructuredDataError(
|
|
592
|
+
f'column {name!r} is already the time column of table '
|
|
593
|
+
f'{self.name!r}, so it cannot also be its primary key',
|
|
594
|
+
code='INVALID_REQUEST',
|
|
595
|
+
)
|
|
596
|
+
frame = getattr(self, 'data', None)
|
|
597
|
+
if frame is None or frame.empty:
|
|
598
|
+
return
|
|
599
|
+
quality = _key_quality(frame, names)
|
|
600
|
+
if quality.nulls:
|
|
601
|
+
empty = [
|
|
602
|
+
name
|
|
603
|
+
for name in names
|
|
604
|
+
if name in frame.columns and frame[name].isna().any()
|
|
605
|
+
]
|
|
606
|
+
raise StructuredDataError(
|
|
607
|
+
_a_key_missing_values(self.name, names, quality.nulls, empty),
|
|
608
|
+
code='INVALID_REQUEST',
|
|
609
|
+
)
|
|
610
|
+
if _too_many_repeats(quality.duplicates, len(frame)):
|
|
611
|
+
raise StructuredDataError(
|
|
612
|
+
_not_unique_enough(
|
|
613
|
+
self.name,
|
|
614
|
+
names,
|
|
615
|
+
quality.duplicates,
|
|
616
|
+
len(frame),
|
|
617
|
+
quality.example,
|
|
618
|
+
),
|
|
619
|
+
code='INVALID_REQUEST',
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
def _check_time_column(self, name: str) -> None:
|
|
623
|
+
self._check_column(name)
|
|
624
|
+
if name in _as_key_tuple(getattr(self, 'primary_key', None)):
|
|
625
|
+
raise StructuredDataError(
|
|
626
|
+
f'column {name!r} is already the primary key of table '
|
|
627
|
+
f'{self.name!r}, so it cannot also be its time column',
|
|
628
|
+
code='INVALID_REQUEST',
|
|
629
|
+
)
|
|
630
|
+
if name in getattr(self, 'columns', {}):
|
|
631
|
+
self.columns[name].stype = 'timestamp'
|
|
632
|
+
|
|
633
|
+
def __getitem__(self, column: str) -> Column:
|
|
634
|
+
try:
|
|
635
|
+
return self.columns[column]
|
|
636
|
+
except KeyError:
|
|
637
|
+
raise StructuredDataError(
|
|
638
|
+
f'table {self.name!r} has no column {column!r}; it has '
|
|
639
|
+
f'{sorted(self.columns)}',
|
|
640
|
+
code='INVALID_REQUEST',
|
|
641
|
+
) from None
|
|
642
|
+
|
|
643
|
+
def __contains__(self, column: object) -> bool:
|
|
644
|
+
return column in self.columns
|
|
645
|
+
|
|
646
|
+
def __iter__(self) -> Iterator[str]:
|
|
647
|
+
return iter(self.columns)
|
|
648
|
+
|
|
649
|
+
def __len__(self) -> int:
|
|
650
|
+
return len(self.columns)
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
# `eq=False` on both, so the comparison above is the one they use: a
|
|
654
|
+
# generated one would compare the frames with `==`.
|
|
655
|
+
@dataclass(eq=False)
|
|
656
|
+
class Table(_TableShape):
|
|
657
|
+
r"""One table: its rows, its columns, and which column identifies a row.
|
|
658
|
+
|
|
659
|
+
Attributes:
|
|
660
|
+
name: The name the schema knows this table by. A query names columns
|
|
661
|
+
through it, so it is part of the interface rather than a label.
|
|
662
|
+
data: The rows.
|
|
663
|
+
columns: Column metadata, keyed by column name.
|
|
664
|
+
primary_key: The column identifying a row, or ``None``. A single-table
|
|
665
|
+
request needs none; a relational one needs one per table, and it is
|
|
666
|
+
inferred, so you rarely set it.
|
|
667
|
+
time_column: The column ordering rows in time, or ``None``. Only a
|
|
668
|
+
temporal question needs one.
|
|
669
|
+
"""
|
|
670
|
+
|
|
671
|
+
name: str
|
|
672
|
+
data: pd.DataFrame
|
|
673
|
+
columns: dict[str, Column]
|
|
674
|
+
primary_key: str | Sequence[str] | None = None
|
|
675
|
+
time_column: str | None = None
|
|
676
|
+
#: How many rows repeat a primary key value another row already used.
|
|
677
|
+
#: Inference tolerates a few, deliberately; this is what it tolerated.
|
|
678
|
+
#: Not an init argument: the generated ``__init__`` assigns fields in
|
|
679
|
+
#: order, so a settable one would overwrite what assigning the key just
|
|
680
|
+
#: computed, and every explicitly-built table would report zero.
|
|
681
|
+
duplicate_keys: int = field(init=False, default=0)
|
|
682
|
+
|
|
683
|
+
def __post_init__(self) -> None:
|
|
684
|
+
r"""Check the names and the cells, and take ownership of the columns.
|
|
685
|
+
|
|
686
|
+
`Schema.from_data` checks the cells while inferring, but a hand-built
|
|
687
|
+
`Table` reached the adapters unchecked, where a list cell surfaced
|
|
688
|
+
much later as `TypeError: unhashable type: 'list'` raised from label
|
|
689
|
+
collection, naming nothing the caller had written.
|
|
690
|
+
"""
|
|
691
|
+
self._check_name(self.name)
|
|
692
|
+
frame = getattr(self, 'data', None)
|
|
693
|
+
if frame is not None:
|
|
694
|
+
for column in frame.columns:
|
|
695
|
+
self._check_name(column)
|
|
696
|
+
require_scalar_cells(frame[column], f'{self.name}.{column}')
|
|
697
|
+
self._own_columns()
|
|
698
|
+
|
|
699
|
+
def __repr__(self) -> str:
|
|
700
|
+
key = f', primary_key={self.primary_key!r}' if self.primary_key else ''
|
|
701
|
+
time = f', time_column={self.time_column!r}' if self.time_column else ''
|
|
702
|
+
repeats = (
|
|
703
|
+
f', {self.duplicate_keys} repeated keys'
|
|
704
|
+
if self.duplicate_keys
|
|
705
|
+
else ''
|
|
706
|
+
)
|
|
707
|
+
return (
|
|
708
|
+
f'Table({self.name!r}, {len(self.data)} rows, '
|
|
709
|
+
f'{len(self.columns)} columns{key}{time}{repeats})'
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
@dataclass(eq=False)
|
|
714
|
+
class SourceTable(_TableShape):
|
|
715
|
+
r"""One table that stays in its source: its columns, and its keys.
|
|
716
|
+
|
|
717
|
+
Built by :meth:`Schema.from_source`. It holds no rows: sampling reads them
|
|
718
|
+
from the source, a hop at a time, when a query is materialized. Keys, time
|
|
719
|
+
columns and semantic types are corrected exactly as on a :class:`Table`.
|
|
720
|
+
|
|
721
|
+
Attributes:
|
|
722
|
+
name: The name the schema knows this table by. A query names columns
|
|
723
|
+
through it.
|
|
724
|
+
source_name: The table's name in the source.
|
|
725
|
+
columns: Column metadata, keyed by column name.
|
|
726
|
+
primary_key: The column identifying a row, or ``None``.
|
|
727
|
+
time_column: The column ordering rows in time, or ``None``.
|
|
728
|
+
"""
|
|
729
|
+
|
|
730
|
+
name: str
|
|
731
|
+
source_name: str
|
|
732
|
+
columns: dict[str, Column]
|
|
733
|
+
primary_key: str | Sequence[str] | None = None
|
|
734
|
+
time_column: str | None = None
|
|
735
|
+
#: How many rows repeat a primary key value another row already used, as
|
|
736
|
+
#: :meth:`Schema.from_source` counted them in the source. Zero for a key
|
|
737
|
+
#: the source enforces, and for a table with no key. ``None`` where
|
|
738
|
+
#: nothing was counted: ``check_keys=False``, a semantic model's table,
|
|
739
|
+
#: or a key named after the schema was built.
|
|
740
|
+
duplicate_keys: int | None = field(init=False, default=None)
|
|
741
|
+
|
|
742
|
+
def __post_init__(self) -> None:
|
|
743
|
+
self._check_name(self.name)
|
|
744
|
+
if not isinstance(self.source_name, str) or not self.source_name:
|
|
745
|
+
raise StructuredDataError(
|
|
746
|
+
f'table {self.name!r} needs the name it has in its source, '
|
|
747
|
+
f'got {self.source_name!r}',
|
|
748
|
+
code='INVALID_REQUEST',
|
|
749
|
+
)
|
|
750
|
+
for column in self.columns:
|
|
751
|
+
self._check_name(column)
|
|
752
|
+
self._own_columns()
|
|
753
|
+
|
|
754
|
+
def __delitem__(self, column: str) -> None:
|
|
755
|
+
r"""Leave a column out: it is neither read from the source nor sent.
|
|
756
|
+
|
|
757
|
+
>>> del schema['orders']['rowguid']
|
|
758
|
+
|
|
759
|
+
A table in its source cannot be edited to suit a request, and some
|
|
760
|
+
columns only mislead a model: a per-row UUID identifies a row and
|
|
761
|
+
says nothing about it. A table of frames drops a column from its frame
|
|
762
|
+
before :meth:`Schema.from_data` instead.
|
|
763
|
+
"""
|
|
764
|
+
self._check_column(column)
|
|
765
|
+
role = (
|
|
766
|
+
'primary key'
|
|
767
|
+
if column in _as_key_tuple(self.primary_key)
|
|
768
|
+
else 'time column'
|
|
769
|
+
if column == self.time_column
|
|
770
|
+
else None
|
|
771
|
+
)
|
|
772
|
+
if role is not None:
|
|
773
|
+
raise StructuredDataError(
|
|
774
|
+
f"{self.name}.{column} is the table's {role}; set that to "
|
|
775
|
+
f'something else, or None, before leaving the column out',
|
|
776
|
+
code='INVALID_REQUEST',
|
|
777
|
+
)
|
|
778
|
+
del self.columns[column]
|
|
779
|
+
|
|
780
|
+
def __repr__(self) -> str:
|
|
781
|
+
key = f', primary_key={self.primary_key!r}' if self.primary_key else ''
|
|
782
|
+
time = f', time_column={self.time_column!r}' if self.time_column else ''
|
|
783
|
+
return (
|
|
784
|
+
f'SourceTable({self.name!r}, from {self.source_name!r}, '
|
|
785
|
+
f'{len(self.columns)} columns{key}{time})'
|
|
786
|
+
)
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
@dataclass(frozen=True)
|
|
790
|
+
class Relationship:
|
|
791
|
+
r"""A foreign key: ``source_table.source_column`` points at a table's key.
|
|
792
|
+
|
|
793
|
+
Directed from the many side to the one side, which is the direction a join
|
|
794
|
+
reads and the direction the wire envelope carries.
|
|
795
|
+
"""
|
|
796
|
+
|
|
797
|
+
source_table: str
|
|
798
|
+
source_column: str
|
|
799
|
+
target_table: str
|
|
800
|
+
target_column: str
|
|
801
|
+
|
|
802
|
+
def __repr__(self) -> str:
|
|
803
|
+
return (
|
|
804
|
+
f'{self.source_table}.{self.source_column} -> '
|
|
805
|
+
f'{self.target_table}.{self.target_column}'
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
class Schema:
|
|
810
|
+
r"""The tables a request is about.
|
|
811
|
+
|
|
812
|
+
Build one with :meth:`from_data`. Index it by table name, and index a table
|
|
813
|
+
by column name:
|
|
814
|
+
|
|
815
|
+
>>> schema = Schema.from_data({'users': users, 'orders': orders})
|
|
816
|
+
>>> schema['orders']['price'].stype
|
|
817
|
+
'numerical'
|
|
818
|
+
|
|
819
|
+
One table and many tables are the same object. A single-table model simply
|
|
820
|
+
rejects a schema that has relationships, and says so before anything is
|
|
821
|
+
sent.
|
|
822
|
+
"""
|
|
823
|
+
|
|
824
|
+
def check_consistent(self) -> None:
|
|
825
|
+
r"""Refuse any column whose storage and meaning contradict, and any
|
|
826
|
+
relationship that cannot join.
|
|
827
|
+
|
|
828
|
+
Asked once, as the schema becomes a request, because a caller
|
|
829
|
+
re-declaring a column sets its storage and its meaning one at a time
|
|
830
|
+
and either order passes through a pair that does not hold.
|
|
831
|
+
|
|
832
|
+
The relationships are asked about here as well as on assignment.
|
|
833
|
+
`relationships` is an ordinary list, so `schema.relationships.append(
|
|
834
|
+
link)` reaches it without passing the setter, and this is the question
|
|
835
|
+
a caller runs to find out whether their schema holds together. It
|
|
836
|
+
should not be the one place that does not look.
|
|
837
|
+
|
|
838
|
+
Which columns are keys is re-derived first. `tables` and a table's
|
|
839
|
+
`columns` are ordinary dictionaries, so a table or a column replaced
|
|
840
|
+
through one arrives never having been asked, and a relationship key
|
|
841
|
+
left typed ``numerical`` would otherwise pass this and be sent as a
|
|
842
|
+
float.
|
|
843
|
+
"""
|
|
844
|
+
self._mark_key_columns()
|
|
845
|
+
for table in self.tables.values():
|
|
846
|
+
for column in table.columns.values():
|
|
847
|
+
column.check_consistent()
|
|
848
|
+
self._check_relationships(self.relationships)
|
|
849
|
+
|
|
850
|
+
def __init__(
|
|
851
|
+
self,
|
|
852
|
+
tables: Mapping[str, Table | SourceTable],
|
|
853
|
+
relationships: Sequence[Relationship] = (),
|
|
854
|
+
*,
|
|
855
|
+
source: Source | None = None,
|
|
856
|
+
) -> None:
|
|
857
|
+
_require_one_kind_of_table(tables, source)
|
|
858
|
+
self._source = source
|
|
859
|
+
# Copied, because a schema types the key columns of its own tables
|
|
860
|
+
# and two schemas handed the same ones fought over them: building an
|
|
861
|
+
# unlinked schema retyped a linked schema's join key, and checking the
|
|
862
|
+
# linked one retyped the unlinked one's back.
|
|
863
|
+
self._tables = {
|
|
864
|
+
name: _copy_table(table) for name, table in tables.items()
|
|
865
|
+
}
|
|
866
|
+
self.relationships = relationships
|
|
867
|
+
# The engine's own reading of the source's tables, kept so that each
|
|
868
|
+
# question does not read them again. Set by the first read.
|
|
869
|
+
self._engine_tables: Any = None
|
|
870
|
+
self._conversion_notes: tuple[str, ...] = ()
|
|
871
|
+
|
|
872
|
+
@property
|
|
873
|
+
def source(self) -> Source | None:
|
|
874
|
+
r"""Where the tables are, for a schema built with :meth:`from_source`.
|
|
875
|
+
|
|
876
|
+
``None`` for a schema of frames. A schema has one source or none,
|
|
877
|
+
because sampling runs against one: see :meth:`from_source`.
|
|
878
|
+
"""
|
|
879
|
+
return self._source
|
|
880
|
+
|
|
881
|
+
@property
|
|
882
|
+
def conversion_notes(self) -> tuple[str, ...]:
|
|
883
|
+
r"""How :meth:`from_source` read a semantic model where nothing was
|
|
884
|
+
lost, for a person to read.
|
|
885
|
+
|
|
886
|
+
For a Databricks metric view, one note each: the predictive query
|
|
887
|
+
aggregation that matches a simple measure, such as
|
|
888
|
+
``'revenue' is SUM(order_lines.LineTotal, 0, 30, days)``, the time
|
|
889
|
+
column taken in place of a date the view truncates, unless
|
|
890
|
+
``time_columns`` named another, and a parameter read as its default.
|
|
891
|
+
Tables are named as the schema names them. What a model loses is
|
|
892
|
+
said in an :class:`SDMWarning` instead, and a time column taken in
|
|
893
|
+
place of a truncated date is said in the warning on how time columns
|
|
894
|
+
were inferred as well, with the override that names another. Empty
|
|
895
|
+
for every other schema.
|
|
896
|
+
|
|
897
|
+
The wording of a note is not part of the contract, so do not parse
|
|
898
|
+
it.
|
|
899
|
+
"""
|
|
900
|
+
return self._conversion_notes
|
|
901
|
+
|
|
902
|
+
def __setattr__(self, key: str, value: Any) -> None:
|
|
903
|
+
r"""Hold an assigned relationship to the same standard as a passed one.
|
|
904
|
+
|
|
905
|
+
`Schema.from_data(relationships=[link])` checks that both columns exist
|
|
906
|
+
and that their dtypes can join; `schema.relationships = [link]` is the
|
|
907
|
+
same statement and checked nothing, so a link that one route refuses
|
|
908
|
+
the other accepted. `check_consistent()`, which a caller runs to
|
|
909
|
+
ask exactly this, passed it.
|
|
910
|
+
"""
|
|
911
|
+
if key == 'relationships' and hasattr(self, '_tables'):
|
|
912
|
+
value = _Links(self._settle(value), self)
|
|
913
|
+
object.__setattr__(self, key, value)
|
|
914
|
+
self._mark_key_columns()
|
|
915
|
+
return
|
|
916
|
+
object.__setattr__(self, key, value)
|
|
917
|
+
|
|
918
|
+
def __copy__(self) -> Schema:
|
|
919
|
+
r"""A shallow copy is still a schema of its own tables.
|
|
920
|
+
|
|
921
|
+
`copy.copy` shared the tables and their columns, so editing the copy
|
|
922
|
+
edited the original. This was always a trap, and became a
|
|
923
|
+
destructive one once key columns were re-derived: clearing the copy's
|
|
924
|
+
relationships retyped a column of a schema that still had them.
|
|
925
|
+
|
|
926
|
+
The rows are still shared, as they are in `snapshot()`: they are the
|
|
927
|
+
data, and copying a warehouse extract is not what a caller asking for
|
|
928
|
+
a copy of the schema is asking for.
|
|
929
|
+
"""
|
|
930
|
+
return self.snapshot()
|
|
931
|
+
|
|
932
|
+
def __setstate__(self, state: dict[str, Any]) -> None:
|
|
933
|
+
r"""Restore a copied or unpickled schema with its links still checked.
|
|
934
|
+
|
|
935
|
+
Pickle and `deepcopy` put the attributes back without going through
|
|
936
|
+
`__setattr__`, so the copy would hold a plain list and quietly stop
|
|
937
|
+
checking what is added to it. Said nothing while it does: a copy is
|
|
938
|
+
not a new declaration, and the notices were given when the schema it
|
|
939
|
+
was copied from was built.
|
|
940
|
+
|
|
941
|
+
A schema pickled before it could be bound to a source has neither
|
|
942
|
+
that source nor the engine's reading of it, which is what it is: a
|
|
943
|
+
schema of frames.
|
|
944
|
+
"""
|
|
945
|
+
object.__setattr__(
|
|
946
|
+
self,
|
|
947
|
+
'__dict__',
|
|
948
|
+
{
|
|
949
|
+
'_source': None,
|
|
950
|
+
'_engine_tables': None,
|
|
951
|
+
'_conversion_notes': (),
|
|
952
|
+
**state,
|
|
953
|
+
},
|
|
954
|
+
)
|
|
955
|
+
with warnings.catch_warnings():
|
|
956
|
+
warnings.simplefilter('ignore', SDMWarning)
|
|
957
|
+
self.relationships = list(state.get('relationships') or ())
|
|
958
|
+
|
|
959
|
+
def _settle(self, links: Sequence[Relationship]) -> list[Relationship]:
|
|
960
|
+
r"""The relationships as this schema will hold them, having said so.
|
|
961
|
+
|
|
962
|
+
One place, so that what is checked, de-duplicated and warned about does
|
|
963
|
+
not depend on how the link was added.
|
|
964
|
+
"""
|
|
965
|
+
given = list(links)
|
|
966
|
+
self._check_relationships(given)
|
|
967
|
+
settled = _each_link_once(given)
|
|
968
|
+
_warn_two_ways_to_reach_one_table(self._tables, settled)
|
|
969
|
+
return settled
|
|
970
|
+
|
|
971
|
+
def _check_relationships(self, links: Sequence[Relationship]) -> None:
|
|
972
|
+
for link in links:
|
|
973
|
+
if not isinstance(link, Relationship):
|
|
974
|
+
raise StructuredDataError(
|
|
975
|
+
f'a relationship must be a Relationship, got '
|
|
976
|
+
f'{type(link).__name__}: {link!r}. Build one with '
|
|
977
|
+
f'Relationship(source_table, source_column, '
|
|
978
|
+
f'target_table, target_column)',
|
|
979
|
+
code='INVALID_REQUEST',
|
|
980
|
+
)
|
|
981
|
+
_require_column(self._tables, link.source_table, link.source_column)
|
|
982
|
+
_require_column(self._tables, link.target_table, link.target_column)
|
|
983
|
+
_require_joinable(self._tables, link)
|
|
984
|
+
|
|
985
|
+
def snapshot(self) -> Schema:
|
|
986
|
+
r"""This schema as it stands, so a later edit cannot reach back.
|
|
987
|
+
|
|
988
|
+
`materialize()` is the gathering step and what it returns is read as
|
|
989
|
+
finished, but the request kept the caller's schema by reference and the
|
|
990
|
+
adapter reads it again at send time. Editing a column's meaning
|
|
991
|
+
after materializing changed what was sent, silently, and an agent
|
|
992
|
+
reusing one schema across several questions had no way to see it.
|
|
993
|
+
|
|
994
|
+
The rows are shared rather than copied: they are the data, they are
|
|
995
|
+
what the request is about, and duplicating a warehouse extract per
|
|
996
|
+
question to guard against an edit nobody makes to a frame would cost
|
|
997
|
+
more than the defect. What is copied is everything a schema *says*.
|
|
998
|
+
"""
|
|
999
|
+
copied = Schema(
|
|
1000
|
+
{name: _copy_table(table) for name, table in self._tables.items()},
|
|
1001
|
+
list(self.relationships),
|
|
1002
|
+
source=self._source,
|
|
1003
|
+
)
|
|
1004
|
+
# What the engine read of the source is shared, not copied: it is what
|
|
1005
|
+
# the tables are, and every gather is given its own copy of it.
|
|
1006
|
+
copied._engine_tables = self._engine_tables
|
|
1007
|
+
copied._conversion_notes = self._conversion_notes
|
|
1008
|
+
return copied
|
|
1009
|
+
|
|
1010
|
+
@classmethod
|
|
1011
|
+
def from_data(
|
|
1012
|
+
cls,
|
|
1013
|
+
tables: Mapping[str, pd.DataFrame],
|
|
1014
|
+
*,
|
|
1015
|
+
primary_keys: Mapping[str, str | Sequence[str]] | None = None,
|
|
1016
|
+
time_columns: Mapping[str, str | None] | None = None,
|
|
1017
|
+
relationships: Sequence[Relationship] | None = None,
|
|
1018
|
+
) -> Schema:
|
|
1019
|
+
r"""Infer a schema from frames.
|
|
1020
|
+
|
|
1021
|
+
Args:
|
|
1022
|
+
tables: Frames by table name. The name matters: a predictive query
|
|
1023
|
+
names columns through it.
|
|
1024
|
+
primary_keys: Overrides for the inferred key of a table. A
|
|
1025
|
+
sequence of column names declares a composite key, which the
|
|
1026
|
+
setter has always taken; the annotation said otherwise and so
|
|
1027
|
+
pushed anyone with one onto the assignment route, which until
|
|
1028
|
+
recently validated nothing.
|
|
1029
|
+
time_columns: Overrides for the inferred time column of a table.
|
|
1030
|
+
``None`` says the table has none.
|
|
1031
|
+
relationships: Overrides for the inferred relationships. Pass an
|
|
1032
|
+
empty sequence to assert there are none.
|
|
1033
|
+
|
|
1034
|
+
Returns:
|
|
1035
|
+
A schema with every dtype, semantic type, key, time column and
|
|
1036
|
+
relationship filled in.
|
|
1037
|
+
|
|
1038
|
+
Raises:
|
|
1039
|
+
StructuredDataError: If ``tables`` is empty, a value is not a
|
|
1040
|
+
DataFrame, a name is not a string, or an override names
|
|
1041
|
+
something that does not exist.
|
|
1042
|
+
|
|
1043
|
+
An inferred relationship is held to the same contract as an explicit
|
|
1044
|
+
one, and dropped rather than refused when it fails. Inference matches
|
|
1045
|
+
on name alone, so it can pair columns whose dtypes cannot join, and
|
|
1046
|
+
`query.py` would send that edge unchanged; a caller who wrote the
|
|
1047
|
+
relationship is told it cannot join, while a caller who never asked
|
|
1048
|
+
for it is not refused for a guess this made on their behalf.
|
|
1049
|
+
"""
|
|
1050
|
+
if not tables:
|
|
1051
|
+
raise StructuredDataError(
|
|
1052
|
+
'a schema needs at least one table', code='INVALID_REQUEST'
|
|
1053
|
+
)
|
|
1054
|
+
|
|
1055
|
+
built: dict[str, Table] = {}
|
|
1056
|
+
chose: dict[str, TimeColumnChoice | None] = {}
|
|
1057
|
+
for name, frame in tables.items():
|
|
1058
|
+
if not isinstance(name, str) or not name:
|
|
1059
|
+
raise StructuredDataError(
|
|
1060
|
+
f'table names must be non-empty strings; got {name!r}',
|
|
1061
|
+
code='INVALID_REQUEST',
|
|
1062
|
+
)
|
|
1063
|
+
if not isinstance(frame, pd.DataFrame):
|
|
1064
|
+
raise StructuredDataError(
|
|
1065
|
+
f'table {name!r} must be a pandas DataFrame, got '
|
|
1066
|
+
f'{type(frame).__name__}',
|
|
1067
|
+
code='INVALID_REQUEST',
|
|
1068
|
+
)
|
|
1069
|
+
built[name], chose[name] = _infer_table_and_time_column(name, frame)
|
|
1070
|
+
|
|
1071
|
+
_apply_keys_and_times(built, primary_keys, time_columns)
|
|
1072
|
+
if relationships is None:
|
|
1073
|
+
inferred = _joinable_inferred_links(
|
|
1074
|
+
built, _infer_relationships(built)
|
|
1075
|
+
)
|
|
1076
|
+
else:
|
|
1077
|
+
inferred = _checked_links(built, relationships)
|
|
1078
|
+
|
|
1079
|
+
_settle_edit_stamps(built, chose, time_columns, inferred)
|
|
1080
|
+
with warnings.catch_warnings():
|
|
1081
|
+
warnings.simplefilter('ignore', SDMWarning)
|
|
1082
|
+
schema = cls(built, inferred)
|
|
1083
|
+
schema._mark_key_columns()
|
|
1084
|
+
dates = {
|
|
1085
|
+
name: named
|
|
1086
|
+
for name, table in built.items()
|
|
1087
|
+
if (named := _warn_dates_stored_as_text(table)) is not None
|
|
1088
|
+
}
|
|
1089
|
+
if relationships is None:
|
|
1090
|
+
spoken_for = _warn_repeats_cost_the_table_its_key(built)
|
|
1091
|
+
_warn_table_cannot_be_linked_to(built, dates, spoken_for)
|
|
1092
|
+
_warn_one_key_name_keys_several_tables(built)
|
|
1093
|
+
_warn_stype_was_decided_by_the_rows_read(built)
|
|
1094
|
+
_warn_how_time_columns_were_inferred(schema, chose, time_columns)
|
|
1095
|
+
_warn_two_ways_to_reach_one_table(built, schema.relationships)
|
|
1096
|
+
return schema
|
|
1097
|
+
|
|
1098
|
+
@classmethod
|
|
1099
|
+
def from_source(
|
|
1100
|
+
cls,
|
|
1101
|
+
source: Source,
|
|
1102
|
+
tables: Mapping[str, str] | None = None,
|
|
1103
|
+
*,
|
|
1104
|
+
primary_keys: Mapping[str, str | Sequence[str]] | None = None,
|
|
1105
|
+
time_columns: Mapping[str, str | None] | None = None,
|
|
1106
|
+
relationships: Sequence[Relationship] | None = None,
|
|
1107
|
+
check_keys: bool = True,
|
|
1108
|
+
) -> Schema:
|
|
1109
|
+
r"""Describe tables that stay in their source.
|
|
1110
|
+
|
|
1111
|
+
>>> source = DatabricksSource(spark, catalog='main', schema='sales')
|
|
1112
|
+
>>> schema = Schema.from_source(
|
|
1113
|
+
... source,
|
|
1114
|
+
... {'customers': 'adventureworks_customers',
|
|
1115
|
+
... 'orders': 'adventureworks_orders'},
|
|
1116
|
+
... time_columns={'orders': 'order_date'},
|
|
1117
|
+
... )
|
|
1118
|
+
|
|
1119
|
+
Nothing is read into memory. Column types come from the source's
|
|
1120
|
+
catalog and semantic types from a preview of at most 1,000 rows. Keys
|
|
1121
|
+
and relationships come from the constraints the catalog declares, and
|
|
1122
|
+
where it declares none, from names, by the same rule as
|
|
1123
|
+
:meth:`from_data`. When a query is materialized, the engine samples
|
|
1124
|
+
the tables with SQL, a hop at a time, and only the rows it samples
|
|
1125
|
+
leave the source.
|
|
1126
|
+
|
|
1127
|
+
Each key is then checked against every row, by the rules
|
|
1128
|
+
:meth:`from_data` applies to a frame. Databricks and Snowflake do not
|
|
1129
|
+
enforce a declared key, and a key inferred from a preview has only
|
|
1130
|
+
been seen to hold there, so a repeated value would otherwise surface
|
|
1131
|
+
as rows that change between seeds and context examples that
|
|
1132
|
+
contradict each other. A declared key, or one named in
|
|
1133
|
+
``primary_keys``, warns when up to 1% of rows repeat a value and is
|
|
1134
|
+
refused beyond that; an inferred key is declined beyond it. The count
|
|
1135
|
+
is one statement for all the tables, a full scan of each, and it is
|
|
1136
|
+
skipped for a key the source enforces, such as a PostgreSQL or DuckDB
|
|
1137
|
+
primary key. :attr:`SourceTable.duplicate_keys` holds what it found.
|
|
1138
|
+
|
|
1139
|
+
A semantic model, such as a Databricks metric view or a Snowflake
|
|
1140
|
+
semantic view, is a source of its own, and already says how its
|
|
1141
|
+
tables join:
|
|
1142
|
+
|
|
1143
|
+
>>> schema = Schema.from_source(source.semantic_model('sales_metrics'))
|
|
1144
|
+
|
|
1145
|
+
Its keys and relationships come from the model, and its tables are
|
|
1146
|
+
named as the model names them. No relationship is inferred from
|
|
1147
|
+
column names beside the model's own joins, which would give a table
|
|
1148
|
+
a second path the model never declared. ``relationships=`` replaces
|
|
1149
|
+
the model's own, so to add one, pass them with it:
|
|
1150
|
+
``relationships=[*Schema.from_source(model).relationships, extra]``.
|
|
1151
|
+
What the model has that a schema cannot hold is said in one
|
|
1152
|
+
:class:`SDMWarning`, and how it was read where nothing was lost, such
|
|
1153
|
+
as the aggregation each measure is, is in :attr:`conversion_notes`.
|
|
1154
|
+
|
|
1155
|
+
Everything that differs between warehouses belongs to the source, so
|
|
1156
|
+
this reads the same whichever one the tables are in. A schema has
|
|
1157
|
+
exactly one source: sampling sends the ids each hop found to the next
|
|
1158
|
+
query, so it runs against one engine.
|
|
1159
|
+
|
|
1160
|
+
Args:
|
|
1161
|
+
source: Where the tables are: an ``sdm_connectors.Source``, or a
|
|
1162
|
+
``SemanticModel`` got from one.
|
|
1163
|
+
tables: For each name a query will use, the table's name in the
|
|
1164
|
+
source. Left out, every table the source describes is taken,
|
|
1165
|
+
under its own name: every table in the namespace, or every
|
|
1166
|
+
table of the semantic model.
|
|
1167
|
+
primary_keys: Overrides for a table's key.
|
|
1168
|
+
time_columns: Overrides for a table's time column. ``None``
|
|
1169
|
+
says the table has none.
|
|
1170
|
+
relationships: Replaces the relationships, a semantic model's
|
|
1171
|
+
own included. Pass an empty sequence to assert there are
|
|
1172
|
+
none.
|
|
1173
|
+
check_keys: Count each key's repeats and missing values in the
|
|
1174
|
+
source. ``False`` skips the scan, and the checks with it, and
|
|
1175
|
+
leaves :attr:`SourceTable.duplicate_keys` ``None``. The tables
|
|
1176
|
+
of a semantic model are never counted: the model derives them.
|
|
1177
|
+
|
|
1178
|
+
Returns:
|
|
1179
|
+
A schema of :class:`SourceTable`, bound to ``source``.
|
|
1180
|
+
|
|
1181
|
+
Raises:
|
|
1182
|
+
StructuredDataError: If ``tables`` is empty, a name is not a
|
|
1183
|
+
string, a table is not in the source, the source describes no
|
|
1184
|
+
table, an override names something that does not exist, or a
|
|
1185
|
+
declared or named key repeats a value in more than 1% of rows
|
|
1186
|
+
or a named key has no value in some row.
|
|
1187
|
+
MissingExtraError: If the relational engine, which reads the
|
|
1188
|
+
source, is not installed.
|
|
1189
|
+
GraphConstructionError: The relational engine's, if the
|
|
1190
|
+
warehouse fails while a semantic model is read, with the
|
|
1191
|
+
driver's error as its ``__cause__``.
|
|
1192
|
+
EmptyTableError: The relational engine's, a ``RuntimeError``, if
|
|
1193
|
+
a semantic model's source table holds no rows.
|
|
1194
|
+
"""
|
|
1195
|
+
from sdm_client.materializers.sources import (
|
|
1196
|
+
EngineTables,
|
|
1197
|
+
engine_graph,
|
|
1198
|
+
is_semantic_model,
|
|
1199
|
+
semantic_model_graph,
|
|
1200
|
+
)
|
|
1201
|
+
|
|
1202
|
+
_require_a_source(source)
|
|
1203
|
+
if tables is not None:
|
|
1204
|
+
_require_table_names(tables)
|
|
1205
|
+
|
|
1206
|
+
read = EngineTables()
|
|
1207
|
+
engine_tables: dict[str, Any] | None = None
|
|
1208
|
+
chose: dict[str, TimeColumnChoice | None] = {}
|
|
1209
|
+
notes: tuple[str, ...] = ()
|
|
1210
|
+
if is_semantic_model(source):
|
|
1211
|
+
graph = semantic_model_graph(source)
|
|
1212
|
+
read.keep({name: graph[name] for name in graph.tables})
|
|
1213
|
+
offered = list(graph.tables)
|
|
1214
|
+
chosen = (
|
|
1215
|
+
dict(tables)
|
|
1216
|
+
if tables is not None
|
|
1217
|
+
else {name: name for name in offered}
|
|
1218
|
+
)
|
|
1219
|
+
absent = sorted(set(chosen.values()) - set(offered))
|
|
1220
|
+
if absent:
|
|
1221
|
+
raise StructuredDataError(
|
|
1222
|
+
f'{absent} are not tables of {source!r}; it has '
|
|
1223
|
+
f'{sorted(offered)}',
|
|
1224
|
+
code='INVALID_REQUEST',
|
|
1225
|
+
)
|
|
1226
|
+
as_named = {model: name for name, model in chosen.items()}
|
|
1227
|
+
built = {
|
|
1228
|
+
name: _describe_source_table(graph[model], model, name=name)
|
|
1229
|
+
for name, model in chosen.items()
|
|
1230
|
+
}
|
|
1231
|
+
chose = {
|
|
1232
|
+
name: graph[model].time_column_choice
|
|
1233
|
+
for name, model in chosen.items()
|
|
1234
|
+
}
|
|
1235
|
+
declared = _model_links(graph, built, as_named)
|
|
1236
|
+
else:
|
|
1237
|
+
graph = engine_graph(source, tables, infer_metadata=True)
|
|
1238
|
+
chosen = (
|
|
1239
|
+
dict(tables)
|
|
1240
|
+
if tables is not None
|
|
1241
|
+
else {name: name for name in graph.tables}
|
|
1242
|
+
)
|
|
1243
|
+
read.keep({named: graph[name] for name, named in chosen.items()})
|
|
1244
|
+
built = {
|
|
1245
|
+
name: _describe_source_table(graph[name], named)
|
|
1246
|
+
for name, named in chosen.items()
|
|
1247
|
+
}
|
|
1248
|
+
chose = {name: graph[name].time_column_choice for name in chosen}
|
|
1249
|
+
declared = _declared_links(graph, chosen, built)
|
|
1250
|
+
if relationships is None:
|
|
1251
|
+
_key_by_the_referenced_column(built, declared)
|
|
1252
|
+
engine_tables = {name: graph[name] for name in chosen}
|
|
1253
|
+
|
|
1254
|
+
found = {
|
|
1255
|
+
name: _as_key_tuple(table.primary_key)
|
|
1256
|
+
for name, table in built.items()
|
|
1257
|
+
}
|
|
1258
|
+
_apply_keys_and_times(built, primary_keys, time_columns)
|
|
1259
|
+
if is_semantic_model(source):
|
|
1260
|
+
notes = _schema_notes(graph.conversion_notes, built, as_named)
|
|
1261
|
+
if check_keys and engine_tables is not None:
|
|
1262
|
+
# Before relationships are inferred: a declined key is one no
|
|
1263
|
+
# relationship can point at.
|
|
1264
|
+
_check_source_keys(
|
|
1265
|
+
source,
|
|
1266
|
+
built,
|
|
1267
|
+
engine_tables,
|
|
1268
|
+
found,
|
|
1269
|
+
set(primary_keys or ()),
|
|
1270
|
+
infer_links=relationships is None,
|
|
1271
|
+
)
|
|
1272
|
+
else:
|
|
1273
|
+
for table in built.values():
|
|
1274
|
+
object.__setattr__(table, 'duplicate_keys', None)
|
|
1275
|
+
if relationships is None:
|
|
1276
|
+
covered = {
|
|
1277
|
+
(link.source_table, link.source_column) for link in declared
|
|
1278
|
+
}
|
|
1279
|
+
# A semantic model's joins are all the links it means.
|
|
1280
|
+
by_name = (
|
|
1281
|
+
[]
|
|
1282
|
+
if is_semantic_model(source)
|
|
1283
|
+
else [
|
|
1284
|
+
link
|
|
1285
|
+
for link in _infer_relationships(built)
|
|
1286
|
+
if (link.source_table, link.source_column) not in covered
|
|
1287
|
+
]
|
|
1288
|
+
)
|
|
1289
|
+
links = _joinable_inferred_links(built, declared + by_name)
|
|
1290
|
+
else:
|
|
1291
|
+
links = _checked_links(built, relationships)
|
|
1292
|
+
|
|
1293
|
+
_settle_edit_stamps(built, chose, time_columns, links)
|
|
1294
|
+
with warnings.catch_warnings():
|
|
1295
|
+
warnings.simplefilter('ignore', SDMWarning)
|
|
1296
|
+
schema = cls(built, links, source=source)
|
|
1297
|
+
schema._engine_tables = read
|
|
1298
|
+
schema._conversion_notes = notes
|
|
1299
|
+
schema._mark_key_columns()
|
|
1300
|
+
_warn_how_time_columns_were_inferred(schema, chose, time_columns)
|
|
1301
|
+
_warn_two_ways_to_reach_one_table(built, schema.relationships)
|
|
1302
|
+
return schema
|
|
1303
|
+
|
|
1304
|
+
@property
|
|
1305
|
+
def tables(self) -> dict[str, Table | SourceTable]:
|
|
1306
|
+
r"""The tables, by name."""
|
|
1307
|
+
return self._tables
|
|
1308
|
+
|
|
1309
|
+
def __getitem__(self, table: str) -> Table | SourceTable:
|
|
1310
|
+
try:
|
|
1311
|
+
return self._tables[table]
|
|
1312
|
+
except KeyError:
|
|
1313
|
+
raise StructuredDataError(
|
|
1314
|
+
f'no table {table!r} in this schema; it has '
|
|
1315
|
+
f'{sorted(self._tables)}',
|
|
1316
|
+
code='INVALID_REQUEST',
|
|
1317
|
+
) from None
|
|
1318
|
+
|
|
1319
|
+
def __contains__(self, table: object) -> bool:
|
|
1320
|
+
return table in self._tables
|
|
1321
|
+
|
|
1322
|
+
def __iter__(self) -> Iterator[str]:
|
|
1323
|
+
return iter(self._tables)
|
|
1324
|
+
|
|
1325
|
+
def __len__(self) -> int:
|
|
1326
|
+
return len(self._tables)
|
|
1327
|
+
|
|
1328
|
+
def _own_tables(self) -> None:
|
|
1329
|
+
r"""Take a copy of any table that still belongs to another schema.
|
|
1330
|
+
|
|
1331
|
+
`tables` hands back the live dictionary, so a table can arrive from a
|
|
1332
|
+
schema that is still using it. Typing its key columns for this
|
|
1333
|
+
schema would retype them for that one. The same rule a table applies
|
|
1334
|
+
to the columns it is handed.
|
|
1335
|
+
"""
|
|
1336
|
+
borrowed = [
|
|
1337
|
+
name
|
|
1338
|
+
for name, table in self._tables.items()
|
|
1339
|
+
if (owner := getattr(table, '_schema', None)) is not None
|
|
1340
|
+
and owner is not self
|
|
1341
|
+
]
|
|
1342
|
+
for name in borrowed:
|
|
1343
|
+
self._tables[name] = _copy_table(self._tables[name])
|
|
1344
|
+
for table in self._tables.values():
|
|
1345
|
+
object.__setattr__(table, '_schema', self)
|
|
1346
|
+
# And the columns inside them, for the same reason: a column
|
|
1347
|
+
# inserted through `table.columns` arrives still belonging to the
|
|
1348
|
+
# table it came from, and typing it here typed it there.
|
|
1349
|
+
table._own_columns()
|
|
1350
|
+
|
|
1351
|
+
def _mark_key_columns(self) -> None:
|
|
1352
|
+
r"""Mark columns this schema joins on or no longer joins on.
|
|
1353
|
+
|
|
1354
|
+
Derived from the schema as it stands rather than applied once when it
|
|
1355
|
+
was built. Marking only on the way in meant a relationship dropped
|
|
1356
|
+
afterwards left its source column typed ``ID``, so two schemas with
|
|
1357
|
+
the same tables and no relationships asked different questions: one
|
|
1358
|
+
a regression and the other a multiclass classification, according to a
|
|
1359
|
+
link only one of them had ever had.
|
|
1360
|
+
"""
|
|
1361
|
+
self._own_tables()
|
|
1362
|
+
keys: dict[str, set[str]] = {}
|
|
1363
|
+
for name, table in self._tables.items():
|
|
1364
|
+
keys[name] = {
|
|
1365
|
+
column
|
|
1366
|
+
for column in _as_key_tuple(table.primary_key)
|
|
1367
|
+
if column in table.columns
|
|
1368
|
+
}
|
|
1369
|
+
for link in self.relationships:
|
|
1370
|
+
source = self._tables.get(link.source_table)
|
|
1371
|
+
if source is not None and link.source_column in source.columns:
|
|
1372
|
+
keys[link.source_table].add(link.source_column)
|
|
1373
|
+
|
|
1374
|
+
for name, table in self._tables.items():
|
|
1375
|
+
for column_name, column in table.columns.items():
|
|
1376
|
+
# A column replaced through the ordinary `columns` dictionary
|
|
1377
|
+
# arrives with no idea which table it is in, and could not be
|
|
1378
|
+
# asked about again.
|
|
1379
|
+
object.__setattr__(column, '_table', table)
|
|
1380
|
+
if column_name in keys[name]:
|
|
1381
|
+
column._mark_as_key()
|
|
1382
|
+
else:
|
|
1383
|
+
column._release_as_key()
|
|
1384
|
+
|
|
1385
|
+
def __repr__(self) -> str:
|
|
1386
|
+
return (
|
|
1387
|
+
f'Schema({len(self._tables)} tables: {sorted(self._tables)}, '
|
|
1388
|
+
f'{len(self.relationships)} relationships)'
|
|
1389
|
+
)
|
|
1390
|
+
|
|
1391
|
+
|
|
1392
|
+
def _require_table(tables: Mapping[str, _T], name: str) -> _T:
|
|
1393
|
+
if name not in tables:
|
|
1394
|
+
raise StructuredDataError(
|
|
1395
|
+
f'no table {name!r} in this schema; it has {sorted(tables)}',
|
|
1396
|
+
code='INVALID_REQUEST',
|
|
1397
|
+
)
|
|
1398
|
+
return tables[name]
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def _schema_notes(
|
|
1402
|
+
notes: Sequence[Any],
|
|
1403
|
+
built: Mapping[str, _TableShape],
|
|
1404
|
+
as_named: Mapping[str, str],
|
|
1405
|
+
) -> tuple[str, ...]:
|
|
1406
|
+
r"""A semantic model's conversion notes as text, naming its tables as
|
|
1407
|
+
the schema names them.
|
|
1408
|
+
|
|
1409
|
+
A note about a table the schema leaves out, or about a time column its
|
|
1410
|
+
table no longer has, says nothing the schema holds, so it is left out.
|
|
1411
|
+
|
|
1412
|
+
Args:
|
|
1413
|
+
notes: The engine's notes on how it read the model.
|
|
1414
|
+
built: The schema's tables, by name.
|
|
1415
|
+
as_named: The schema's name for each of the model's tables it holds.
|
|
1416
|
+
"""
|
|
1417
|
+
from sdm_relational_engine.rfm.view_conversion import (
|
|
1418
|
+
MeasureNote,
|
|
1419
|
+
TimeColumnNote,
|
|
1420
|
+
)
|
|
1421
|
+
|
|
1422
|
+
said: list[str] = []
|
|
1423
|
+
for note in notes:
|
|
1424
|
+
if isinstance(note, MeasureNote | TimeColumnNote):
|
|
1425
|
+
name = as_named.get(note.table)
|
|
1426
|
+
if name is None or (
|
|
1427
|
+
isinstance(note, TimeColumnNote)
|
|
1428
|
+
and built[name].time_column != note.column
|
|
1429
|
+
):
|
|
1430
|
+
continue
|
|
1431
|
+
note = replace(note, table=name)
|
|
1432
|
+
said.append(str(note))
|
|
1433
|
+
return tuple(said)
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def _apply_keys_and_times(
|
|
1437
|
+
built: Mapping[str, _TableShape],
|
|
1438
|
+
primary_keys: Mapping[str, str | Sequence[str]] | None,
|
|
1439
|
+
time_columns: Mapping[str, str | None] | None,
|
|
1440
|
+
) -> None:
|
|
1441
|
+
for name, key in (primary_keys or {}).items():
|
|
1442
|
+
table = _require_table(built, name)
|
|
1443
|
+
# Read once, and the names are what is passed on. Reading it here
|
|
1444
|
+
# to check the columns exist and then handing the original along
|
|
1445
|
+
# emptied a key that can only be read once, exactly as the setter
|
|
1446
|
+
# used to before it read once itself.
|
|
1447
|
+
names = _as_key_tuple(key)
|
|
1448
|
+
for column in names:
|
|
1449
|
+
_require_column(built, name, column)
|
|
1450
|
+
table.primary_key = key if isinstance(key, str) else names
|
|
1451
|
+
for name, named in (time_columns or {}).items():
|
|
1452
|
+
_require_table(built, name).time_column = (
|
|
1453
|
+
None if named is None else _require_column(built, name, named)
|
|
1454
|
+
)
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def _joinable_inferred_links(
|
|
1458
|
+
built: Mapping[str, _TableShape], inferred: Sequence[Relationship]
|
|
1459
|
+
) -> list[Relationship]:
|
|
1460
|
+
r"""The inferred links that can join, having said which could not.
|
|
1461
|
+
|
|
1462
|
+
An inferred relationship is held to the same contract as an explicit one,
|
|
1463
|
+
and dropped rather than refused when it fails: a caller who never asked
|
|
1464
|
+
for it is not refused for a guess made on their behalf.
|
|
1465
|
+
"""
|
|
1466
|
+
unjoinable = [link for link in inferred if not _is_joinable(built, link)]
|
|
1467
|
+
_warn_of_a_link_that_cannot_join(built, unjoinable)
|
|
1468
|
+
return [link for link in inferred if _is_joinable(built, link)]
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def _key_by_the_referenced_column(
|
|
1472
|
+
built: Mapping[str, SourceTable], declared: Sequence[Relationship]
|
|
1473
|
+
) -> None:
|
|
1474
|
+
r"""Key a keyless table by the column declared foreign keys refer to.
|
|
1475
|
+
|
|
1476
|
+
A catalog lets a foreign key refer only to a primary key or a unique
|
|
1477
|
+
column, so that column identifies a row, as a lone unique column does when
|
|
1478
|
+
the engine infers a key. Before the keys are checked, so this one is too.
|
|
1479
|
+
Foreign keys naming different columns of one table leave it keyless.
|
|
1480
|
+
"""
|
|
1481
|
+
referenced: dict[str, set[str]] = {}
|
|
1482
|
+
for link in declared:
|
|
1483
|
+
referenced.setdefault(link.target_table, set()).add(link.target_column)
|
|
1484
|
+
for name, columns in referenced.items():
|
|
1485
|
+
if built[name].primary_key is None and len(columns) == 1:
|
|
1486
|
+
built[name].primary_key = next(iter(columns))
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _checked_links(
|
|
1490
|
+
built: Mapping[str, _TableShape], relationships: Sequence[Relationship]
|
|
1491
|
+
) -> list[Relationship]:
|
|
1492
|
+
links = list(relationships)
|
|
1493
|
+
for link in links:
|
|
1494
|
+
if not isinstance(link, Relationship):
|
|
1495
|
+
raise StructuredDataError(
|
|
1496
|
+
f'a relationship must be a Relationship, got '
|
|
1497
|
+
f'{type(link).__name__}: {link!r}. Build one with '
|
|
1498
|
+
f'Relationship(source_table, source_column, '
|
|
1499
|
+
f'target_table, target_column)',
|
|
1500
|
+
code='INVALID_REQUEST',
|
|
1501
|
+
)
|
|
1502
|
+
_require_column(built, link.source_table, link.source_column)
|
|
1503
|
+
_require_column(built, link.target_table, link.target_column)
|
|
1504
|
+
_require_joinable(built, link)
|
|
1505
|
+
return links
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
#: How a message names each source's engine.
|
|
1509
|
+
_BACKEND_NAMES = {
|
|
1510
|
+
'databricks': 'Databricks',
|
|
1511
|
+
'snowflake': 'Snowflake',
|
|
1512
|
+
'postgres': 'PostgreSQL',
|
|
1513
|
+
'duckdb': 'DuckDB',
|
|
1514
|
+
'sqlite': 'SQLite',
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
_SKIP_THE_CHECK = (
|
|
1518
|
+
'Name another key with primary_keys=, or pass check_keys=False to skip '
|
|
1519
|
+
'(one full scan per table).'
|
|
1520
|
+
)
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
def _check_source_keys(
|
|
1524
|
+
source: Any,
|
|
1525
|
+
built: Mapping[str, SourceTable],
|
|
1526
|
+
engine_tables: Mapping[str, Any],
|
|
1527
|
+
found: Mapping[str, tuple[str, ...]],
|
|
1528
|
+
stated: AbstractSet[str],
|
|
1529
|
+
*,
|
|
1530
|
+
infer_links: bool,
|
|
1531
|
+
) -> None:
|
|
1532
|
+
r"""Hold the keys of tables in a source to the rules a key of frames
|
|
1533
|
+
meets, counting the rows where they are.
|
|
1534
|
+
|
|
1535
|
+
A stated key, from ``primary_keys=`` or the catalog, warns at up to 1% of
|
|
1536
|
+
rows repeating a value and is refused above that. A key the engine
|
|
1537
|
+
inferred is kept up to the same 1% and declined beyond it, as inference
|
|
1538
|
+
over frames declines it. A key the source enforces is not counted.
|
|
1539
|
+
|
|
1540
|
+
Args:
|
|
1541
|
+
source: The source the tables are in.
|
|
1542
|
+
built: The tables, keyed as they will be.
|
|
1543
|
+
engine_tables: The engine's reading of each table, by schema name.
|
|
1544
|
+
found: The key each table had before ``primary_keys=`` was applied.
|
|
1545
|
+
stated: The tables whose key ``primary_keys=`` named.
|
|
1546
|
+
infer_links: Whether relationships will be inferred, which is when a
|
|
1547
|
+
declined key is worth a word.
|
|
1548
|
+
"""
|
|
1549
|
+
from sdm_relational_engine.rfm.base import count_keys
|
|
1550
|
+
|
|
1551
|
+
for table in built.values():
|
|
1552
|
+
object.__setattr__(table, 'duplicate_keys', 0)
|
|
1553
|
+
keyed = {
|
|
1554
|
+
name: _as_key_tuple(table.primary_key)
|
|
1555
|
+
for name, table in built.items()
|
|
1556
|
+
if table.primary_key is not None
|
|
1557
|
+
}
|
|
1558
|
+
if not keyed:
|
|
1559
|
+
return
|
|
1560
|
+
counts = dict(
|
|
1561
|
+
zip(
|
|
1562
|
+
keyed,
|
|
1563
|
+
count_keys([(engine_tables[name], keyed[name]) for name in keyed]),
|
|
1564
|
+
strict=True,
|
|
1565
|
+
)
|
|
1566
|
+
)
|
|
1567
|
+
backend = _BACKEND_NAMES.get(source.backend, source.backend)
|
|
1568
|
+
declined: dict[str, Any] = {}
|
|
1569
|
+
for name, names in keyed.items():
|
|
1570
|
+
count = counts[name]
|
|
1571
|
+
rows, distinct = count.rows, count.distinct
|
|
1572
|
+
if rows is None or distinct is None:
|
|
1573
|
+
continue
|
|
1574
|
+
table = built[name]
|
|
1575
|
+
declared = (
|
|
1576
|
+
name not in stated
|
|
1577
|
+
and names == found[name]
|
|
1578
|
+
and engine_tables[name]._declares_key(names)
|
|
1579
|
+
)
|
|
1580
|
+
repeats = count.repeats
|
|
1581
|
+
if name not in stated and not declared:
|
|
1582
|
+
# Inferred, so declined as inference over a frame declines it.
|
|
1583
|
+
if distinct < rows * _KEY_UNIQUENESS:
|
|
1584
|
+
declined[name] = count
|
|
1585
|
+
table.primary_key = None
|
|
1586
|
+
continue
|
|
1587
|
+
elif count.nulls and name in stated:
|
|
1588
|
+
empty = names if len(names) == 1 else None
|
|
1589
|
+
missing = _a_key_missing_values(name, names, count.nulls, empty)
|
|
1590
|
+
raise StructuredDataError(
|
|
1591
|
+
f'{missing}. {_SKIP_THE_CHECK}', code='INVALID_REQUEST'
|
|
1592
|
+
)
|
|
1593
|
+
elif _too_many_repeats(repeats, rows):
|
|
1594
|
+
found_out = _not_unique_enough(
|
|
1595
|
+
name, names, repeats, rows, count.example
|
|
1596
|
+
)
|
|
1597
|
+
unenforced = (
|
|
1598
|
+
f'{backend} does not enforce declared keys. '
|
|
1599
|
+
if declared and not count.enforced
|
|
1600
|
+
else ''
|
|
1601
|
+
)
|
|
1602
|
+
raise StructuredDataError(
|
|
1603
|
+
f'{found_out}. {unenforced}{_SKIP_THE_CHECK}',
|
|
1604
|
+
code='INVALID_REQUEST',
|
|
1605
|
+
)
|
|
1606
|
+
if count.nulls:
|
|
1607
|
+
_warn_a_key_has_no_value(name, '.'.join(names), count.nulls)
|
|
1608
|
+
if repeats:
|
|
1609
|
+
_warn_a_key_repeats(
|
|
1610
|
+
name, names, repeats, count.example, _NAME_ANOTHER_KEY
|
|
1611
|
+
)
|
|
1612
|
+
object.__setattr__(table, 'duplicate_keys', repeats)
|
|
1613
|
+
|
|
1614
|
+
if not infer_links or not declined:
|
|
1615
|
+
return
|
|
1616
|
+
named_elsewhere = _tables_naming_each_column(built)
|
|
1617
|
+
keyed_by = {
|
|
1618
|
+
table.primary_key
|
|
1619
|
+
for table in built.values()
|
|
1620
|
+
if isinstance(table.primary_key, str)
|
|
1621
|
+
}
|
|
1622
|
+
for name, count in declined.items():
|
|
1623
|
+
column = keyed[name][0]
|
|
1624
|
+
rows, distinct = count.rows or 0, count.distinct or 0
|
|
1625
|
+
if (
|
|
1626
|
+
len(keyed[name]) > 1
|
|
1627
|
+
or column in keyed_by
|
|
1628
|
+
or not named_elsewhere.get(column, set()) - {name}
|
|
1629
|
+
or distinct < rows * _NEARLY_A_KEY
|
|
1630
|
+
or not count.repeats
|
|
1631
|
+
):
|
|
1632
|
+
continue
|
|
1633
|
+
_warn_a_key_was_declined(
|
|
1634
|
+
name,
|
|
1635
|
+
column,
|
|
1636
|
+
count.repeats,
|
|
1637
|
+
rows,
|
|
1638
|
+
count.example,
|
|
1639
|
+
f'Keep one row per {column!r} in the source, or name a column '
|
|
1640
|
+
f'that does identify a row with primary_keys={{{name!r}: ...}}.',
|
|
1641
|
+
)
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
def _require_table_names(tables: Mapping[str, str]) -> None:
|
|
1645
|
+
if not tables:
|
|
1646
|
+
raise StructuredDataError(
|
|
1647
|
+
'a schema needs at least one table; leave tables out to take every '
|
|
1648
|
+
'table the source describes',
|
|
1649
|
+
code='INVALID_REQUEST',
|
|
1650
|
+
)
|
|
1651
|
+
for name, source_name in tables.items():
|
|
1652
|
+
if not isinstance(name, str) or not name:
|
|
1653
|
+
raise StructuredDataError(
|
|
1654
|
+
f'table names must be non-empty strings; got {name!r}',
|
|
1655
|
+
code='INVALID_REQUEST',
|
|
1656
|
+
)
|
|
1657
|
+
if not isinstance(source_name, str) or not source_name:
|
|
1658
|
+
raise StructuredDataError(
|
|
1659
|
+
f'table {name!r} must map to its name in the source, a '
|
|
1660
|
+
f'non-empty string; got {source_name!r}',
|
|
1661
|
+
code='INVALID_REQUEST',
|
|
1662
|
+
)
|
|
1663
|
+
|
|
1664
|
+
|
|
1665
|
+
def _require_a_source(source: object) -> None:
|
|
1666
|
+
try:
|
|
1667
|
+
from sdm_connectors import Source
|
|
1668
|
+
except ImportError: # pragma: no cover - nothing could have built one
|
|
1669
|
+
Source = None # type: ignore[assignment,misc]
|
|
1670
|
+
if Source is None or not isinstance(source, Source):
|
|
1671
|
+
raise StructuredDataError(
|
|
1672
|
+
f'a schema over tables in place needs an sdm_connectors.Source, '
|
|
1673
|
+
f'got {type(source).__name__}. Build one from a connection: '
|
|
1674
|
+
f'DatabricksSource(databricks.sql.connect(...), catalog=..., '
|
|
1675
|
+
f'schema=...), or DatabricksSource(spark, ...) in a notebook',
|
|
1676
|
+
code='INVALID_REQUEST',
|
|
1677
|
+
)
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
def _require_one_kind_of_table(
|
|
1681
|
+
tables: Mapping[str, Table | SourceTable], source: object
|
|
1682
|
+
) -> None:
|
|
1683
|
+
r"""Refuse a schema that mixes frames with tables in place, or tables in
|
|
1684
|
+
place without the source they are in.
|
|
1685
|
+
|
|
1686
|
+
The engine samples one graph with one sampler: an in-memory one over
|
|
1687
|
+
frames, or SQL against one source. A mixture has no sampler to run it.
|
|
1688
|
+
"""
|
|
1689
|
+
in_place = sorted(
|
|
1690
|
+
name for name, table in tables.items() if isinstance(table, SourceTable)
|
|
1691
|
+
)
|
|
1692
|
+
in_memory = sorted(
|
|
1693
|
+
name
|
|
1694
|
+
for name, table in tables.items()
|
|
1695
|
+
if not isinstance(table, SourceTable)
|
|
1696
|
+
)
|
|
1697
|
+
if in_place and in_memory:
|
|
1698
|
+
raise StructuredDataError(
|
|
1699
|
+
f'{in_place} stay in their source and {in_memory} are frames; a '
|
|
1700
|
+
f'schema samples one or the other, so read the first into frames '
|
|
1701
|
+
f'or put the second in the source',
|
|
1702
|
+
code='INVALID_REQUEST',
|
|
1703
|
+
)
|
|
1704
|
+
if in_place and source is None:
|
|
1705
|
+
raise StructuredDataError(
|
|
1706
|
+
f'{in_place} stay in their source, so the schema needs that '
|
|
1707
|
+
f'source; build it with Schema.from_source',
|
|
1708
|
+
code='INVALID_REQUEST',
|
|
1709
|
+
)
|
|
1710
|
+
if in_memory and source is not None:
|
|
1711
|
+
raise StructuredDataError(
|
|
1712
|
+
f'{in_memory} are frames, which a source does not hold; drop '
|
|
1713
|
+
f'source= or build the schema with Schema.from_source',
|
|
1714
|
+
code='INVALID_REQUEST',
|
|
1715
|
+
)
|
|
1716
|
+
if source is not None:
|
|
1717
|
+
_require_a_source(source)
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
# What a name in a predictive query cannot hold: `table.column` splits at the
|
|
1721
|
+
# dot, and a backtick or line break ends the name.
|
|
1722
|
+
_INVALID_NAME_CHARS = ('`', '\n', '\r', '.')
|
|
1723
|
+
|
|
1724
|
+
|
|
1725
|
+
def _invalid_name_chars(name: str) -> list[str]:
|
|
1726
|
+
return [char for char in _INVALID_NAME_CHARS if char in name]
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
def _describe_source_table(
|
|
1730
|
+
engine_table: Any, source_name: str, *, name: str | None = None
|
|
1731
|
+
) -> SourceTable:
|
|
1732
|
+
r"""A table in place, described from what the engine read of it.
|
|
1733
|
+
|
|
1734
|
+
A column whose values have no scalar form on the wire, such as a list,
|
|
1735
|
+
binary or a duration, is left out and named, rather than refusing the
|
|
1736
|
+
whole table:
|
|
1737
|
+
the caller cannot change a warehouse table's columns to suit a request, and
|
|
1738
|
+
the engine then leaves it unread as well. So is a column whose name a
|
|
1739
|
+
predictive query cannot say, such as a metric view's dimension `a.b`.
|
|
1740
|
+
"""
|
|
1741
|
+
from sdm_client.materializers.sources import wire_dtype
|
|
1742
|
+
|
|
1743
|
+
columns: dict[str, Column] = {}
|
|
1744
|
+
unreadable: list[str] = []
|
|
1745
|
+
unnamable: list[str] = []
|
|
1746
|
+
for engine_column in engine_table.columns:
|
|
1747
|
+
if _invalid_name_chars(engine_column.name):
|
|
1748
|
+
unnamable.append(repr(engine_column.name))
|
|
1749
|
+
continue
|
|
1750
|
+
dtype = wire_dtype(engine_column.dtype)
|
|
1751
|
+
stype = str(getattr(engine_column.stype, 'value', engine_column.stype))
|
|
1752
|
+
if dtype is None or stype not in SEMANTIC_TYPES:
|
|
1753
|
+
unreadable.append(f'{engine_column.name} ({engine_column.dtype})')
|
|
1754
|
+
continue
|
|
1755
|
+
columns[engine_column.name] = Column(engine_column.name, dtype, stype)
|
|
1756
|
+
if unreadable:
|
|
1757
|
+
warnings.warn(
|
|
1758
|
+
f'{name or engine_table.name} leaves out {", ".join(unreadable)}: a '
|
|
1759
|
+
f'request carries one scalar value per cell, and these hold none',
|
|
1760
|
+
category=SDMWarning,
|
|
1761
|
+
stacklevel=caller_stacklevel(),
|
|
1762
|
+
)
|
|
1763
|
+
if unnamable:
|
|
1764
|
+
warnings.warn(
|
|
1765
|
+
f'{name or engine_table.name} leaves out {", ".join(unnamable)}: '
|
|
1766
|
+
f'a predictive query cannot name a column holding a dot, a '
|
|
1767
|
+
f'backtick or a line break',
|
|
1768
|
+
category=SDMWarning,
|
|
1769
|
+
stacklevel=caller_stacklevel(),
|
|
1770
|
+
)
|
|
1771
|
+
table = SourceTable(
|
|
1772
|
+
name=name or engine_table.name,
|
|
1773
|
+
source_name=source_name,
|
|
1774
|
+
columns=columns,
|
|
1775
|
+
)
|
|
1776
|
+
keys = tuple(engine_table.primary_key_columns or ())
|
|
1777
|
+
if keys and all(key in columns for key in keys):
|
|
1778
|
+
table.primary_key = keys[0] if len(keys) == 1 else keys
|
|
1779
|
+
time = engine_table.time_column
|
|
1780
|
+
if time is not None and time.name in columns and time.name not in keys:
|
|
1781
|
+
table.time_column = time.name
|
|
1782
|
+
return table
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
def _model_links(
|
|
1786
|
+
graph: Any, built: Mapping[str, SourceTable], as_named: Mapping[str, str]
|
|
1787
|
+
) -> list[Relationship]:
|
|
1788
|
+
r"""The relationships a semantic model's joins make, as the schema says
|
|
1789
|
+
them.
|
|
1790
|
+
|
|
1791
|
+
Each join links the many side's key to the one side's, which the model
|
|
1792
|
+
makes that table's primary key. A join to a table the schema did not take
|
|
1793
|
+
links nothing.
|
|
1794
|
+
"""
|
|
1795
|
+
links = []
|
|
1796
|
+
for edge in graph.edges:
|
|
1797
|
+
source_name = as_named.get(edge.src_table)
|
|
1798
|
+
target_name = as_named.get(edge.dst_table)
|
|
1799
|
+
if source_name is None or target_name is None:
|
|
1800
|
+
continue
|
|
1801
|
+
target = built[target_name]
|
|
1802
|
+
source = built[source_name]
|
|
1803
|
+
key = target.primary_key
|
|
1804
|
+
if not isinstance(key, str) or edge.fkey not in source.columns:
|
|
1805
|
+
continue
|
|
1806
|
+
links.append(
|
|
1807
|
+
Relationship(
|
|
1808
|
+
source_table=source_name,
|
|
1809
|
+
source_column=edge.fkey,
|
|
1810
|
+
target_table=target_name,
|
|
1811
|
+
target_column=key,
|
|
1812
|
+
)
|
|
1813
|
+
)
|
|
1814
|
+
return links
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def _declared_links(
|
|
1818
|
+
graph: Any, tables: Mapping[str, str], built: Mapping[str, SourceTable]
|
|
1819
|
+
) -> list[Relationship]:
|
|
1820
|
+
r"""The foreign keys the source's catalog declares between these tables.
|
|
1821
|
+
|
|
1822
|
+
A constraint naming a table outside the schema links nothing here, and
|
|
1823
|
+
one naming a column the schema left out is skipped with it.
|
|
1824
|
+
"""
|
|
1825
|
+
# `dst_table` is qualified the way `source_name` is, not the way the caller
|
|
1826
|
+
# named the table.
|
|
1827
|
+
by_source_name = {graph[name].source_name: name for name in tables}
|
|
1828
|
+
links = []
|
|
1829
|
+
for name in tables:
|
|
1830
|
+
for key in graph[name].source_foreign_keys:
|
|
1831
|
+
target = _table_named(by_source_name, key.dst_table)
|
|
1832
|
+
if target is None:
|
|
1833
|
+
continue
|
|
1834
|
+
if key.name not in built[name].columns:
|
|
1835
|
+
continue
|
|
1836
|
+
if key.primary_key not in built[target].columns:
|
|
1837
|
+
continue
|
|
1838
|
+
links.append(
|
|
1839
|
+
Relationship(
|
|
1840
|
+
source_table=name,
|
|
1841
|
+
source_column=key.name,
|
|
1842
|
+
target_table=target,
|
|
1843
|
+
target_column=key.primary_key,
|
|
1844
|
+
)
|
|
1845
|
+
)
|
|
1846
|
+
return links
|
|
1847
|
+
|
|
1848
|
+
|
|
1849
|
+
def _table_named(
|
|
1850
|
+
by_source_name: Mapping[str, str], dst_table: str
|
|
1851
|
+
) -> str | None:
|
|
1852
|
+
r"""The table a foreign key's target names, matched exactly if it can be.
|
|
1853
|
+
|
|
1854
|
+
SQLite and Databricks fold names to lower case, so a case-insensitive match
|
|
1855
|
+
still counts, but only when one table answers to it. PostgreSQL and
|
|
1856
|
+
Snowflake can hold both `"Parent"` and `parent`, and guessing between them
|
|
1857
|
+
would attach the key to the wrong one.
|
|
1858
|
+
"""
|
|
1859
|
+
if dst_table in by_source_name:
|
|
1860
|
+
return by_source_name[dst_table]
|
|
1861
|
+
folded = [
|
|
1862
|
+
name
|
|
1863
|
+
for source_name, name in by_source_name.items()
|
|
1864
|
+
if source_name.lower() == dst_table.lower()
|
|
1865
|
+
]
|
|
1866
|
+
return folded[0] if len(folded) == 1 else None
|
|
1867
|
+
|
|
1868
|
+
|
|
1869
|
+
def _require_column(
|
|
1870
|
+
tables: Mapping[str, _TableShape], table: str, column: str
|
|
1871
|
+
) -> str:
|
|
1872
|
+
if column not in _require_table(tables, table).columns:
|
|
1873
|
+
raise StructuredDataError(
|
|
1874
|
+
f'table {table!r} has no column {column!r}; it has '
|
|
1875
|
+
f'{sorted(tables[table].columns)}',
|
|
1876
|
+
code='INVALID_REQUEST',
|
|
1877
|
+
)
|
|
1878
|
+
return column
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
def _as_key_tuple(key: Any) -> tuple[str, ...]:
|
|
1882
|
+
r"""A primary key as the tuple of column names it is.
|
|
1883
|
+
|
|
1884
|
+
A single name, or any ordered iterable of them, including a `pd.Index`,
|
|
1885
|
+
which is what `frame.columns` is and so the most natural way to name a
|
|
1886
|
+
composite key from the data it came from. `pd.Index` is not a
|
|
1887
|
+
`collections.abc.Sequence`, so asking that question rejected it.
|
|
1888
|
+
|
|
1889
|
+
A set is not one of them. A composite key's columns are in an order, and
|
|
1890
|
+
a set has none, so the key it named would differ between runs.
|
|
1891
|
+
"""
|
|
1892
|
+
if key is None:
|
|
1893
|
+
return ()
|
|
1894
|
+
if isinstance(key, str):
|
|
1895
|
+
return (key,)
|
|
1896
|
+
if isinstance(key, (bytes, Mapping, AbstractSet)) or not isinstance(
|
|
1897
|
+
key, Iterable
|
|
1898
|
+
):
|
|
1899
|
+
raise StructuredDataError(
|
|
1900
|
+
f'a primary key names a column or a sequence of them, got {key!r}',
|
|
1901
|
+
code='INVALID_REQUEST',
|
|
1902
|
+
)
|
|
1903
|
+
names = list(key)
|
|
1904
|
+
if not all(isinstance(name, str) for name in names):
|
|
1905
|
+
raise StructuredDataError(
|
|
1906
|
+
f'a primary key names a column or a sequence of them, got {key!r}',
|
|
1907
|
+
code='INVALID_REQUEST',
|
|
1908
|
+
)
|
|
1909
|
+
return tuple(names)
|
|
1910
|
+
|
|
1911
|
+
|
|
1912
|
+
def _infer_table(name: str, frame: pd.DataFrame) -> Table:
|
|
1913
|
+
r"""Describe a frame as a table; see
|
|
1914
|
+
:func:`_infer_table_and_time_column`.
|
|
1915
|
+
"""
|
|
1916
|
+
return _infer_table_and_time_column(name, frame)[0]
|
|
1917
|
+
|
|
1918
|
+
|
|
1919
|
+
def _infer_table_and_time_column(
|
|
1920
|
+
name: str, frame: pd.DataFrame
|
|
1921
|
+
) -> tuple[Table, TimeColumnChoice]:
|
|
1922
|
+
r"""Describe a frame as a table, inferring the types, key and time column,
|
|
1923
|
+
and return with it how the time column was chosen.
|
|
1924
|
+
|
|
1925
|
+
The cells are checked before any column is described. Typing a column asks
|
|
1926
|
+
for its distinct values, and a list cell makes that raise `unhashable
|
|
1927
|
+
type: 'list'` from inside pandas, naming nothing the caller wrote.
|
|
1928
|
+
`Table.__post_init__` asks the same question again for a table built by
|
|
1929
|
+
hand, which is the route that has not passed through here.
|
|
1930
|
+
"""
|
|
1931
|
+
if frame.columns.has_duplicates:
|
|
1932
|
+
duplicates = sorted(
|
|
1933
|
+
{str(c) for c in frame.columns[frame.columns.duplicated()]}
|
|
1934
|
+
)
|
|
1935
|
+
raise StructuredDataError(
|
|
1936
|
+
f'table {name!r} has duplicate column name(s) {duplicates}',
|
|
1937
|
+
code='INVALID_REQUEST',
|
|
1938
|
+
)
|
|
1939
|
+
non_strings = [c for c in frame.columns if not isinstance(c, str)]
|
|
1940
|
+
if non_strings:
|
|
1941
|
+
raise StructuredDataError(
|
|
1942
|
+
f'table {name!r} column names must be strings; got {non_strings}',
|
|
1943
|
+
code='INVALID_REQUEST',
|
|
1944
|
+
)
|
|
1945
|
+
for column in frame.columns:
|
|
1946
|
+
require_scalar_cells(frame[column], f'{name}.{column}')
|
|
1947
|
+
|
|
1948
|
+
columns = {
|
|
1949
|
+
column: _column(frame[column], column) for column in frame.columns
|
|
1950
|
+
}
|
|
1951
|
+
table = Table(name=name, data=frame, columns=columns)
|
|
1952
|
+
inferred = _infer_primary_key(frame, columns)
|
|
1953
|
+
if inferred is not None and frame[inferred].isna().any():
|
|
1954
|
+
# Tolerated here and refused on assignment, the same way a nearly
|
|
1955
|
+
# unique column is: inference is a guess and a caller's key is an
|
|
1956
|
+
# assertion. Refusing would break `Schema.from_data` on frames that
|
|
1957
|
+
# worked before, including any relationship pointing at this table,
|
|
1958
|
+
# but the engine drops these rows without a word, so the guess is
|
|
1959
|
+
# said out loud.
|
|
1960
|
+
_warn_a_key_has_no_value(
|
|
1961
|
+
name, inferred, int(frame[inferred].isna().sum())
|
|
1962
|
+
)
|
|
1963
|
+
object.__setattr__(table, 'primary_key', inferred)
|
|
1964
|
+
table._adopt_key(inferred)
|
|
1965
|
+
return table, _infer_time_column(table, frame)
|
|
1966
|
+
# Assigning the key is what marks its column, counts what the uniqueness
|
|
1967
|
+
# threshold tolerated and says so; doing any of it a second time here would
|
|
1968
|
+
# report the same duplicate twice, from inside this module.
|
|
1969
|
+
table.primary_key = inferred
|
|
1970
|
+
return table, _infer_time_column(table, frame)
|
|
1971
|
+
|
|
1972
|
+
|
|
1973
|
+
def _copy_table(table: _T) -> _T:
|
|
1974
|
+
r"""The same table again, field for field, judging nothing.
|
|
1975
|
+
|
|
1976
|
+
Built without the constructor and without the checked setters, for two
|
|
1977
|
+
reasons. Inference deliberately accepts a key with a null in it. It warns
|
|
1978
|
+
and continues because refusing would break frames that worked. However,
|
|
1979
|
+
assignment refuses one, so copying through the setter turned a schema
|
|
1980
|
+
`from_data` had accepted into a request that could not be materialized.
|
|
1981
|
+
And `Table.__post_init__` walks every cell of every column, which on a
|
|
1982
|
+
warehouse extract is a second full scan added to every single request.
|
|
1983
|
+
"""
|
|
1984
|
+
copied = type(table).__new__(type(table))
|
|
1985
|
+
object.__setattr__(copied, 'name', table.name)
|
|
1986
|
+
for held in ('data', 'source_name'):
|
|
1987
|
+
if hasattr(table, held):
|
|
1988
|
+
object.__setattr__(copied, held, getattr(table, held))
|
|
1989
|
+
object.__setattr__(
|
|
1990
|
+
copied,
|
|
1991
|
+
'columns',
|
|
1992
|
+
{name: _copy_column(column) for name, column in table.columns.items()},
|
|
1993
|
+
)
|
|
1994
|
+
for carried in ('primary_key', 'time_column', 'duplicate_keys'):
|
|
1995
|
+
object.__setattr__(copied, carried, getattr(table, carried))
|
|
1996
|
+
for column in copied.columns.values():
|
|
1997
|
+
object.__setattr__(column, '_table', copied)
|
|
1998
|
+
return copied
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
def _copy_column(column: Column) -> Column:
|
|
2002
|
+
r"""The same column again, without re-judging it.
|
|
2003
|
+
|
|
2004
|
+
Built field by field rather than through the constructor: a schema is
|
|
2005
|
+
allowed to be inconsistent between two assignments. That is what
|
|
2006
|
+
`check_consistent` checks later, so copying one must not be
|
|
2007
|
+
the moment its storage and meaning are held against each other.
|
|
2008
|
+
"""
|
|
2009
|
+
made = Column.__new__(Column)
|
|
2010
|
+
for field_name in (
|
|
2011
|
+
'name',
|
|
2012
|
+
'dtype',
|
|
2013
|
+
'stype',
|
|
2014
|
+
'untyped',
|
|
2015
|
+
'stype_declared',
|
|
2016
|
+
'_stype_before_key',
|
|
2017
|
+
):
|
|
2018
|
+
object.__setattr__(made, field_name, getattr(column, field_name))
|
|
2019
|
+
return made
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
def _column(series: pd.Series, name: str) -> Column:
|
|
2023
|
+
r"""One column, with the record of whether inference had anything to go on.
|
|
2024
|
+
|
|
2025
|
+
`untyped` is set after construction because it is not an init field: a
|
|
2026
|
+
caller must not be able to build a column that names a dtype and claims to
|
|
2027
|
+
have none, which is the state the adapter resolves by discarding the dtype.
|
|
2028
|
+
|
|
2029
|
+
The values are read once and the answer passed down. `infer_tfm_dtype` on
|
|
2030
|
+
an object column walks every cell. Asking it three times for the dtype,
|
|
2031
|
+
again inside the stype, and again for `untyped` is what made
|
|
2032
|
+
describing a million rows of warehouse `Decimal` take five seconds before
|
|
2033
|
+
anything was sent. The same data already typed takes three hundredths.
|
|
2034
|
+
"""
|
|
2035
|
+
inferred = infer_tfm_dtype(series)
|
|
2036
|
+
built = Column(
|
|
2037
|
+
name=name,
|
|
2038
|
+
dtype=_dtype_of(inferred),
|
|
2039
|
+
stype=_stype_of(series, inferred),
|
|
2040
|
+
)
|
|
2041
|
+
if inferred == UNTYPED:
|
|
2042
|
+
object.__setattr__(built, 'untyped', True)
|
|
2043
|
+
return built
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def _dtype_of(inferred: str) -> str:
|
|
2047
|
+
# An all-null object column carries no type. Calling it a string is the
|
|
2048
|
+
# safe fallback: it round-trips whatever arrives without claiming a width.
|
|
2049
|
+
return 'string' if inferred == UNTYPED else inferred
|
|
2050
|
+
|
|
2051
|
+
|
|
2052
|
+
def _stype_of(series: pd.Series, inferred: str | None = None) -> str:
|
|
2053
|
+
dtype = _dtype_of(infer_tfm_dtype(series) if inferred is None else inferred)
|
|
2054
|
+
if dtype.startswith('timestamp'):
|
|
2055
|
+
return 'timestamp'
|
|
2056
|
+
if dtype in {'int32', 'int64', 'float32', 'float64'}:
|
|
2057
|
+
return 'numerical'
|
|
2058
|
+
if dtype == 'bool':
|
|
2059
|
+
return 'categorical'
|
|
2060
|
+
distinct = series.nunique(dropna=True)
|
|
2061
|
+
return 'categorical' if distinct <= _MAX_CATEGORICAL_CARDINALITY else 'text'
|
|
2062
|
+
|
|
2063
|
+
|
|
2064
|
+
def _infer_primary_key(
|
|
2065
|
+
frame: pd.DataFrame, columns: Mapping[str, Column]
|
|
2066
|
+
) -> str | None:
|
|
2067
|
+
r"""The column that identifies a row, if one is obvious.
|
|
2068
|
+
|
|
2069
|
+
Requires a key-shaped name, values that can identify a row, and near-total
|
|
2070
|
+
uniqueness. Uniqueness alone would nominate any high-cardinality
|
|
2071
|
+
measurement, and a name alone would nominate a foreign key.
|
|
2072
|
+
|
|
2073
|
+
**Uniqueness here is a property of the frame, not of the table it came
|
|
2074
|
+
from.** A foreign key drawn sparsely enough looks exactly like a key: take
|
|
2075
|
+
200 rows of a fact table whose real key is `(order, line)` and `order`
|
|
2076
|
+
will very likely hold 200 distinct values, because two rows of the same
|
|
2077
|
+
order are unlikely to be drawn together. Nothing in the frame distinguishes
|
|
2078
|
+
that from a genuine key, so a frame that is a sample can nominate a foreign
|
|
2079
|
+
key and no amount of checking here will notice. Declare the key when the
|
|
2080
|
+
frame is a sample; inference is for the case where the frame is the table.
|
|
2081
|
+
"""
|
|
2082
|
+
if len(frame) == 0:
|
|
2083
|
+
return None
|
|
2084
|
+
for name, column in columns.items():
|
|
2085
|
+
lowered = name.lower()
|
|
2086
|
+
if not lowered.endswith(_KEY_SUFFIXES):
|
|
2087
|
+
continue
|
|
2088
|
+
if not supports_dtype('ID', column.dtype):
|
|
2089
|
+
# A key-shaped name on values that cannot identify a row, such as a
|
|
2090
|
+
# bool
|
|
2091
|
+
# column called `id`, say. Nominating it would refuse the frame at
|
|
2092
|
+
# construction for a key nobody asked for.
|
|
2093
|
+
continue
|
|
2094
|
+
distinct = frame[name].nunique(dropna=True)
|
|
2095
|
+
if distinct >= len(frame) * _KEY_UNIQUENESS:
|
|
2096
|
+
return name
|
|
2097
|
+
return None
|
|
2098
|
+
|
|
2099
|
+
|
|
2100
|
+
#: Above this share of rows being distinct, a string column's value count is
|
|
2101
|
+
#: bounded by the frame rather than by the column, so it says nothing about
|
|
2102
|
+
#: how many values the column really holds.
|
|
2103
|
+
_UNSATURATED = 0.5
|
|
2104
|
+
|
|
2105
|
+
#: And below this share of the categorical limit, growth has too far to go for
|
|
2106
|
+
#: the frame to be what decides: two values in four rows are unsaturated by
|
|
2107
|
+
#: the measure above and are still nowhere near becoming sixty-five.
|
|
2108
|
+
_WITHIN_REACH = 0.5
|
|
2109
|
+
|
|
2110
|
+
|
|
2111
|
+
def _warn_stype_was_decided_by_the_rows_read(
|
|
2112
|
+
tables: Mapping[str, Table],
|
|
2113
|
+
) -> None:
|
|
2114
|
+
r"""Say when `categorical` was decided by the size of the frame.
|
|
2115
|
+
|
|
2116
|
+
A string column is called `categorical` when it holds few enough distinct
|
|
2117
|
+
values and `text` otherwise, and the count is taken from the frame handed
|
|
2118
|
+
in. For a column whose values repeat that is the same answer at any size:
|
|
2119
|
+
`state` holds 27 values in ten thousand rows and in a hundred thousand.
|
|
2120
|
+
For one that does not, the count is not a property of the column at all.
|
|
2121
|
+
It is a property of how many rows were read, and reading more rows moves
|
|
2122
|
+
it. So a request written against `df.head(100)` and then pointed at the
|
|
2123
|
+
table describes a column two different ways, with nothing in between to
|
|
2124
|
+
say the description changed.
|
|
2125
|
+
|
|
2126
|
+
Which is worth saying only where it can happen, and that takes both of two
|
|
2127
|
+
measures. The first is whether the values have run out: a column whose
|
|
2128
|
+
distinct count tracks its row count has not shown how many values it has,
|
|
2129
|
+
only how many rows were looked at. The second is whether there is anything
|
|
2130
|
+
within reach. Two values in four rows have not saturated either, and are
|
|
2131
|
+
nowhere near becoming sixty-five, so saying so is noise on every small
|
|
2132
|
+
frame. Nearness to the threshold alone is not the signal, because 53
|
|
2133
|
+
distinct in 100 rows is not near 64 by any measure and still flips.
|
|
2134
|
+
|
|
2135
|
+
This is the same property the primary-key inference documents about
|
|
2136
|
+
uniqueness, and it has the same remedy, which is to say what the column is
|
|
2137
|
+
when the frame is a sample.
|
|
2138
|
+
|
|
2139
|
+
The count decides the meaning for strings only, which is what this is
|
|
2140
|
+
about: a bool is categorical by its type rather than by how many values it
|
|
2141
|
+
holds. That makes the dtype check a precondition rather than a filter
|
|
2142
|
+
today, since the only other categorical dtype is bool and two values never
|
|
2143
|
+
come within reach of the limit. It is stated so that a dtype added to the
|
|
2144
|
+
categorical side later does not inherit a message about a count that did
|
|
2145
|
+
not decide it.
|
|
2146
|
+
|
|
2147
|
+
The share is measured against the rows that hold a value, because that is
|
|
2148
|
+
what the distinct count was taken from. A column of forty values among a
|
|
2149
|
+
hundred and sixty nulls has saturated nothing, and dividing by the height
|
|
2150
|
+
of the frame said it had.
|
|
2151
|
+
"""
|
|
2152
|
+
for name, table in tables.items():
|
|
2153
|
+
frame = table.data
|
|
2154
|
+
if frame is None or len(frame) == 0:
|
|
2155
|
+
continue
|
|
2156
|
+
for column, meta in table.columns.items():
|
|
2157
|
+
if meta.stype_declared or meta.stype != 'categorical':
|
|
2158
|
+
continue
|
|
2159
|
+
if meta.dtype != 'string' or column not in frame:
|
|
2160
|
+
continue
|
|
2161
|
+
distinct = frame[column].nunique(dropna=True)
|
|
2162
|
+
present = int(frame[column].count())
|
|
2163
|
+
if distinct < present * _UNSATURATED:
|
|
2164
|
+
continue
|
|
2165
|
+
if distinct < _MAX_CATEGORICAL_CARDINALITY * _WITHIN_REACH:
|
|
2166
|
+
continue
|
|
2167
|
+
warnings.warn(
|
|
2168
|
+
f'{name}.{column} holds {distinct} distinct values in the '
|
|
2169
|
+
f'{present} rows that have one, so the rows ran out before '
|
|
2170
|
+
f'the values did and the count is a measure of the frame '
|
|
2171
|
+
f'rather than of the column. It is read as categorical here '
|
|
2172
|
+
f'because {distinct} is at most '
|
|
2173
|
+
f'{_MAX_CATEGORICAL_CARDINALITY}; on more rows the same '
|
|
2174
|
+
f'column could pass that and be read as text, which the '
|
|
2175
|
+
f'models treat differently. If this frame is a sample, say '
|
|
2176
|
+
f'what the column is with '
|
|
2177
|
+
f"schema[{name!r}][{column!r}].stype = 'categorical' (or "
|
|
2178
|
+
f"'text'), so that it does not change under a larger one.",
|
|
2179
|
+
category=SDMWarning,
|
|
2180
|
+
stacklevel=caller_stacklevel(),
|
|
2181
|
+
)
|
|
2182
|
+
|
|
2183
|
+
|
|
2184
|
+
def _warn_dates_stored_as_text(table: Table) -> str | None:
|
|
2185
|
+
r"""Say when the only candidate timestamp is a string, and was not used.
|
|
2186
|
+
|
|
2187
|
+
Inference types a string by how many distinct values it holds and never
|
|
2188
|
+
reads it as a date, which is deliberate: a parser loose enough to read a
|
|
2189
|
+
warehouse `DATE` column also reads part codes as dates and dies later, on
|
|
2190
|
+
a value it should not have touched. The cost is that a wholly temporal
|
|
2191
|
+
dataset, such as a warehouse that ordinarily returns `DATE` as `STRING`,
|
|
2192
|
+
infers to a schema with no time in it. The caller learns
|
|
2193
|
+
only when a temporal query is refused, or never, if the query they wrote
|
|
2194
|
+
happened to be static.
|
|
2195
|
+
|
|
2196
|
+
So the column is still not parsed, and the declining is no longer silent.
|
|
2197
|
+
Only a table that ended with no time column is worth a word; one that
|
|
2198
|
+
found its timestamp does not need to hear about a date-shaped string.
|
|
2199
|
+
|
|
2200
|
+
The shape test is not a correctness check: the conversion below rejects
|
|
2201
|
+
everything it would, and more. It is there so that the ordinary case, a
|
|
2202
|
+
table whose string columns are names and descriptions, does not pay to
|
|
2203
|
+
have every one of them parsed as a date.
|
|
2204
|
+
|
|
2205
|
+
Every remaining question is answered by running that conversion over the
|
|
2206
|
+
whole column rather than sampling it or reasoning about it: whether the
|
|
2207
|
+
values past the sample hold their shape, whether they are real dates, and
|
|
2208
|
+
which form of the call the caller should be told to make.
|
|
2209
|
+
"""
|
|
2210
|
+
frame = table.data
|
|
2211
|
+
if table.time_column is not None or frame is None or frame.empty:
|
|
2212
|
+
return None
|
|
2213
|
+
for name, column in table.columns.items():
|
|
2214
|
+
if column.dtype != 'string' or column.stype == 'ID':
|
|
2215
|
+
continue
|
|
2216
|
+
present = frame[name].dropna()
|
|
2217
|
+
if not dates.looks_unambiguous(present):
|
|
2218
|
+
continue
|
|
2219
|
+
how = dates.conversion_that_works(present)
|
|
2220
|
+
if how is None:
|
|
2221
|
+
continue
|
|
2222
|
+
warnings.warn(
|
|
2223
|
+
f'{table.name}.{name} holds dates but is stored as text, so it is '
|
|
2224
|
+
f'not the time column and no temporal query can use it. Convert '
|
|
2225
|
+
f'it before inferring: '
|
|
2226
|
+
f'frame[{name!r}] = pd.to_datetime(frame[{name!r}]{how}), or '
|
|
2227
|
+
f'set schema[{table.name!r}][{name!r}].stype = {"timestamp"!r} '
|
|
2228
|
+
f'afterwards.',
|
|
2229
|
+
category=SDMWarning,
|
|
2230
|
+
stacklevel=caller_stacklevel(),
|
|
2231
|
+
)
|
|
2232
|
+
return name
|
|
2233
|
+
return None
|
|
2234
|
+
|
|
2235
|
+
|
|
2236
|
+
def _infer_time_column(table: Table, frame: pd.DataFrame) -> TimeColumnChoice:
|
|
2237
|
+
r"""Set the time column, and return how it was chosen.
|
|
2238
|
+
|
|
2239
|
+
The rule is the one the engine applies to a source, so a frame and the
|
|
2240
|
+
same rows in a warehouse infer the same column.
|
|
2241
|
+
"""
|
|
2242
|
+
stamps = [
|
|
2243
|
+
name
|
|
2244
|
+
for name, column in table.columns.items()
|
|
2245
|
+
if column.stype == 'timestamp' and name != table.primary_key
|
|
2246
|
+
]
|
|
2247
|
+
choice = choose_time_column(frame, stamps)
|
|
2248
|
+
table.time_column = choice.column
|
|
2249
|
+
return choice
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
def _settle_edit_stamps(
|
|
2253
|
+
built: Mapping[str, _TableShape],
|
|
2254
|
+
chose: dict[str, TimeColumnChoice | None],
|
|
2255
|
+
time_columns: Mapping[str, str | None] | None,
|
|
2256
|
+
links: Sequence[Relationship],
|
|
2257
|
+
) -> None:
|
|
2258
|
+
r"""Take or refuse a lone edit stamp by whether another table links to
|
|
2259
|
+
the table it is in.
|
|
2260
|
+
|
|
2261
|
+
A table something links to is an entity and refuses the stamp of its
|
|
2262
|
+
last edit; one nothing links to is a log and takes it. Decided on the
|
|
2263
|
+
schema's final links, the one fact both paths share by now: the two
|
|
2264
|
+
guess keys differently, so a log's `update_id` is a key from a frame
|
|
2265
|
+
and none from a source, and a rule on the key made them disagree.
|
|
2266
|
+
"""
|
|
2267
|
+
linked = {link.target_table for link in links}
|
|
2268
|
+
for name, choice in chose.items():
|
|
2269
|
+
if choice is None or name in (time_columns or {}):
|
|
2270
|
+
continue
|
|
2271
|
+
table = built[name]
|
|
2272
|
+
settled = choice.for_links(name in linked)
|
|
2273
|
+
if settled != choice:
|
|
2274
|
+
chose[name] = settled
|
|
2275
|
+
table.time_column = settled.column
|
|
2276
|
+
|
|
2277
|
+
|
|
2278
|
+
def _warn_how_time_columns_were_inferred(
|
|
2279
|
+
schema: Schema,
|
|
2280
|
+
chose: Mapping[str, TimeColumnChoice | None],
|
|
2281
|
+
given: Mapping[str, str | None] | None,
|
|
2282
|
+
) -> None:
|
|
2283
|
+
"""Say, in one warning, which time columns inference took or refused on
|
|
2284
|
+
a guess, and the override that settles each.
|
|
2285
|
+
|
|
2286
|
+
A guess about a column's name or values decides what the model may know
|
|
2287
|
+
at the moment it answers, so it is said out loud: a column taken as the
|
|
2288
|
+
earliest of several, a stamp of the last edit taken as the only timestamp
|
|
2289
|
+
left, one taken although the rows read hold one value or mostly none,
|
|
2290
|
+
because the query over the whole table failed, and, for a table left
|
|
2291
|
+
without a time column, each timestamp refused and why. That table fails its first windowed query, and the refusal
|
|
2292
|
+
cannot say which of its columns inference passed over.
|
|
2293
|
+
|
|
2294
|
+
Only where something could be windowed: a lone table is asked
|
|
2295
|
+
single-table questions, which have no window. A table whose time column
|
|
2296
|
+
the caller named is not mentioned.
|
|
2297
|
+
"""
|
|
2298
|
+
tables = schema.tables
|
|
2299
|
+
if len(tables) < 2 and not schema.relationships:
|
|
2300
|
+
return
|
|
2301
|
+
lines: list[str] = []
|
|
2302
|
+
for name, table in sorted(tables.items()):
|
|
2303
|
+
choice = chose.get(name)
|
|
2304
|
+
if choice is None or name in (given or {}):
|
|
2305
|
+
continue
|
|
2306
|
+
column = table.time_column
|
|
2307
|
+
if column is not None and column == choice.column:
|
|
2308
|
+
if choice.instead_of:
|
|
2309
|
+
one = len(choice.instead_of) == 1
|
|
2310
|
+
lines.append(
|
|
2311
|
+
f'{name}.{column} was taken as the time column rather '
|
|
2312
|
+
f'than {_and_list(choice.instead_of)}, which '
|
|
2313
|
+
f'{"truncates or casts" if one else "truncate or cast"} '
|
|
2314
|
+
f'it, since it says when a row happened more precisely. '
|
|
2315
|
+
f'If a question should count by '
|
|
2316
|
+
f'{"it" if one else "one of those"}, name it: '
|
|
2317
|
+
f'time_columns={{{name!r}: {choice.instead_of[0]!r}}}.'
|
|
2318
|
+
)
|
|
2319
|
+
if choice.how == 'earliest':
|
|
2320
|
+
others = [other for other in choice.among if other != column]
|
|
2321
|
+
lines.append(
|
|
2322
|
+
f'{name}.{column} was taken as the time column because '
|
|
2323
|
+
f'its values start earliest of {_and_list(choice.among)}. '
|
|
2324
|
+
f'If another says when a row became true, name it: '
|
|
2325
|
+
f'time_columns={{{name!r}: {others[0]!r}}}. If none does, '
|
|
2326
|
+
f'clear it: time_columns={{{name!r}: None}}.'
|
|
2327
|
+
)
|
|
2328
|
+
elif choice.how == 'edit_stamp':
|
|
2329
|
+
lines.append(
|
|
2330
|
+
f'{name}.{column} was taken as the time column although '
|
|
2331
|
+
f'it looks like when a row was last changed, because the '
|
|
2332
|
+
f'table has no other timestamp and no table links to it, '
|
|
2333
|
+
f'so it reads as a log of changes. If it does not say when a row '
|
|
2334
|
+
f'became true, clear it: time_columns={{{name!r}: None}}.'
|
|
2335
|
+
)
|
|
2336
|
+
lines.extend(
|
|
2337
|
+
taken_unchecked(name, doubt, f'time_columns={{{name!r}: None}}')
|
|
2338
|
+
for doubt in choice.unconfirmed
|
|
2339
|
+
if doubt.column == column
|
|
2340
|
+
)
|
|
2341
|
+
elif column is None:
|
|
2342
|
+
lines.extend(
|
|
2343
|
+
f'{name}.{rejected.column} was not taken as the time column '
|
|
2344
|
+
f'because {rejected.why}. If it does say when a row became '
|
|
2345
|
+
f'true, name it: '
|
|
2346
|
+
f'time_columns={{{name!r}: {rejected.column!r}}}.'
|
|
2347
|
+
for rejected in choice.rejected
|
|
2348
|
+
if rejected.column in table.columns
|
|
2349
|
+
)
|
|
2350
|
+
if lines:
|
|
2351
|
+
warnings.warn(
|
|
2352
|
+
'Some time columns were inferred, or refused, on a guess about '
|
|
2353
|
+
'what a column holds:\n' + '\n'.join(f'- {line}' for line in lines),
|
|
2354
|
+
category=SDMWarning,
|
|
2355
|
+
stacklevel=caller_stacklevel(),
|
|
2356
|
+
)
|
|
2357
|
+
|
|
2358
|
+
|
|
2359
|
+
def _warn_table_cannot_be_linked_to(
|
|
2360
|
+
tables: Mapping[str, Table],
|
|
2361
|
+
dates: Mapping[str, str],
|
|
2362
|
+
already_said: AbstractSet[str] = frozenset(),
|
|
2363
|
+
) -> None:
|
|
2364
|
+
r"""Say when a table has no key and something wanted to point at it.
|
|
2365
|
+
|
|
2366
|
+
`_infer_primary_key` requires a key-shaped name, and rightly: uniqueness
|
|
2367
|
+
alone would nominate any high-cardinality measurement. But a great many
|
|
2368
|
+
real keys are domain codes such as `IATA_CODE`, `SKU`, `ISBN`, and `VIN`.
|
|
2369
|
+
No suffix rule will cover them.
|
|
2370
|
+
|
|
2371
|
+
That is not merely a missing key. `_infer_relationships` can only point an
|
|
2372
|
+
edge at a *declared* key, so a table that has none is not a candidate
|
|
2373
|
+
target for anything: every relationship into it is lost, the tables are
|
|
2374
|
+
dropped from the request, and the model answers from less data than it was
|
|
2375
|
+
given with nothing on screen to say so.
|
|
2376
|
+
|
|
2377
|
+
Nominating a key here was tried and withdrawn. Uniqueness within one frame
|
|
2378
|
+
does not distinguish a key from a foreign key that happens not to repeat in
|
|
2379
|
+
the rows to hand. A two-row extract of `line_items` has a unique
|
|
2380
|
+
`ORDER_NO`, so nominating turns a child into a parent and invents an edge
|
|
2381
|
+
between two facts. Worse, a second table keyed on a name makes that name
|
|
2382
|
+
ambiguous, and `_infer_relationships` then declines *every* edge using it:
|
|
2383
|
+
the guess does not merely risk a wrong link, it can delete the right ones
|
|
2384
|
+
from a schema whose keys the caller declared correctly.
|
|
2385
|
+
|
|
2386
|
+
So the caller is asked instead, and only where the question is real. A
|
|
2387
|
+
keyless table no other table names loses no edges. Nor does one whose
|
|
2388
|
+
candidate column is already another table's key. That column is a
|
|
2389
|
+
foreign key, naming it here would make the name ambiguous, and ambiguity
|
|
2390
|
+
deletes the edges that were working. A warning that talks a caller into
|
|
2391
|
+
the failure it was written to prevent is worse than silence.
|
|
2392
|
+
|
|
2393
|
+
`already_said` names the tables the repeated-key warning has spoken about.
|
|
2394
|
+
Said of one table the two contradict, with remedies that cannot both be
|
|
2395
|
+
followed.
|
|
2396
|
+
|
|
2397
|
+
`dates` names, per table, a column already reported as a date stored as
|
|
2398
|
+
text. A column cannot be both the time column and the primary key, so
|
|
2399
|
+
asking for it here would contradict the other warning, and a caller who
|
|
2400
|
+
followed both in order would end with neither.
|
|
2401
|
+
|
|
2402
|
+
How many columns could be the key and how many tables they are shared with
|
|
2403
|
+
are two counts, and they do not move together: several columns can be
|
|
2404
|
+
shared with one table, and one column with several. Agreeing either by the
|
|
2405
|
+
other made the sentence disagree with the graph it describes.
|
|
2406
|
+
"""
|
|
2407
|
+
named_elsewhere = _tables_naming_each_column(tables)
|
|
2408
|
+
keyed_by = {
|
|
2409
|
+
table.primary_key
|
|
2410
|
+
for table in tables.values()
|
|
2411
|
+
if isinstance(table.primary_key, str)
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
for name, table in tables.items():
|
|
2415
|
+
if table.primary_key is not None or name in already_said:
|
|
2416
|
+
continue
|
|
2417
|
+
frame = table.data
|
|
2418
|
+
if frame is None or frame.empty:
|
|
2419
|
+
continue
|
|
2420
|
+
candidates = [
|
|
2421
|
+
column
|
|
2422
|
+
for column, meta in table.columns.items()
|
|
2423
|
+
if _could_identify_a_row(table, frame, column, meta)
|
|
2424
|
+
and len(named_elsewhere.get(column, ())) > 1
|
|
2425
|
+
and column not in keyed_by
|
|
2426
|
+
and column != dates.get(name)
|
|
2427
|
+
]
|
|
2428
|
+
if not candidates:
|
|
2429
|
+
continue
|
|
2430
|
+
elsewhere = {
|
|
2431
|
+
other
|
|
2432
|
+
for column in candidates
|
|
2433
|
+
for other in named_elsewhere.get(column, ())
|
|
2434
|
+
if other != name
|
|
2435
|
+
}
|
|
2436
|
+
warnings.warn(
|
|
2437
|
+
_unlinkable_table_reads(name, candidates, len(elsewhere)),
|
|
2438
|
+
category=SDMWarning,
|
|
2439
|
+
stacklevel=caller_stacklevel(),
|
|
2440
|
+
)
|
|
2441
|
+
|
|
2442
|
+
|
|
2443
|
+
def _tables_naming_each_column(
|
|
2444
|
+
tables: Mapping[str, _TableShape],
|
|
2445
|
+
) -> dict[str, set[str]]:
|
|
2446
|
+
r"""Which tables hold a column of each name.
|
|
2447
|
+
|
|
2448
|
+
A name shared by two tables is what an inferred edge is made of, so it is
|
|
2449
|
+
also what every question about a missing edge is asked against.
|
|
2450
|
+
"""
|
|
2451
|
+
named: dict[str, set[str]] = {}
|
|
2452
|
+
for name, table in tables.items():
|
|
2453
|
+
for column in table.columns:
|
|
2454
|
+
named.setdefault(column, set()).add(name)
|
|
2455
|
+
return named
|
|
2456
|
+
|
|
2457
|
+
|
|
2458
|
+
def _warn_repeats_cost_the_table_its_key(
|
|
2459
|
+
tables: Mapping[str, Table],
|
|
2460
|
+
) -> set[str]:
|
|
2461
|
+
r"""Say when a key-shaped column was declined for its repeats.
|
|
2462
|
+
|
|
2463
|
+
`_infer_primary_key` wants `_KEY_UNIQUENESS` of the values distinct, and
|
|
2464
|
+
tolerating a little short of total uniqueness is right: one duplicated row
|
|
2465
|
+
is common in an extract and still the column the caller means. But the
|
|
2466
|
+
tolerance has an edge, and the behaviour either side of it is the wrong way
|
|
2467
|
+
round. One duplicated row nominates the key and warns; two nominate
|
|
2468
|
+
nothing, and a table with no key is not a candidate target for any edge, so
|
|
2469
|
+
every relationship pointing at it disappears, with nothing said at
|
|
2470
|
+
inference or at materialization. The louder outcome is the smaller
|
|
2471
|
+
problem.
|
|
2472
|
+
|
|
2473
|
+
`_warn_table_cannot_be_linked_to` is the warning for a table nothing can
|
|
2474
|
+
point at, and it cannot see this one: it asks for columns that identify a
|
|
2475
|
+
row perfectly, and the column that caused the loss is by definition not
|
|
2476
|
+
one of them. So it is asked about here instead, where the repeats are the
|
|
2477
|
+
subject rather than the disqualification.
|
|
2478
|
+
|
|
2479
|
+
Only a column another table names is worth a word, on the same reasoning:
|
|
2480
|
+
a key nothing points at loses no edge by being absent.
|
|
2481
|
+
|
|
2482
|
+
`_NEARLY_A_KEY` is what separates a key with a fault in it from an
|
|
2483
|
+
ordinary foreign key, which repeats on the order of the row count rather
|
|
2484
|
+
than a handful of times. A sampled extract makes an ordinary foreign key
|
|
2485
|
+
nearly unique too, so a column another table is already keyed by is left
|
|
2486
|
+
alone: the edge through it works, and the remedy printed here deletes it.
|
|
2487
|
+
|
|
2488
|
+
Returns the tables it spoke about, so the keyless-table warning does not
|
|
2489
|
+
follow it with a contradicting account of the same one.
|
|
2490
|
+
"""
|
|
2491
|
+
named_elsewhere = _tables_naming_each_column(tables)
|
|
2492
|
+
keyed_by = {
|
|
2493
|
+
table.primary_key
|
|
2494
|
+
for table in tables.values()
|
|
2495
|
+
if isinstance(table.primary_key, str)
|
|
2496
|
+
}
|
|
2497
|
+
spoken_for = set()
|
|
2498
|
+
for name, table in tables.items():
|
|
2499
|
+
if table.primary_key is not None:
|
|
2500
|
+
continue
|
|
2501
|
+
frame = table.data
|
|
2502
|
+
if frame is None or frame.empty:
|
|
2503
|
+
continue
|
|
2504
|
+
for column, meta in table.columns.items():
|
|
2505
|
+
if not column.lower().endswith(_KEY_SUFFIXES):
|
|
2506
|
+
continue
|
|
2507
|
+
if not supports_dtype('ID', meta.dtype):
|
|
2508
|
+
continue
|
|
2509
|
+
if column in keyed_by:
|
|
2510
|
+
continue
|
|
2511
|
+
if not named_elsewhere.get(column, set()) - {name}:
|
|
2512
|
+
continue
|
|
2513
|
+
if frame[column].nunique(dropna=True) < len(frame) * _NEARLY_A_KEY:
|
|
2514
|
+
continue
|
|
2515
|
+
quality = _key_quality(frame, (column,))
|
|
2516
|
+
if not quality.duplicates:
|
|
2517
|
+
continue
|
|
2518
|
+
_warn_a_key_was_declined(
|
|
2519
|
+
name,
|
|
2520
|
+
column,
|
|
2521
|
+
quality.duplicates,
|
|
2522
|
+
len(frame),
|
|
2523
|
+
quality.example,
|
|
2524
|
+
f'Keep one row per {column!r}: '
|
|
2525
|
+
f'frame = frame.drop_duplicates({column!r}), and infer '
|
|
2526
|
+
f'again, or name a column that does identify a row with '
|
|
2527
|
+
f'primary_keys={{{name!r}: ...}}.',
|
|
2528
|
+
)
|
|
2529
|
+
spoken_for.add(name)
|
|
2530
|
+
break
|
|
2531
|
+
return spoken_for
|
|
2532
|
+
|
|
2533
|
+
|
|
2534
|
+
def _warn_one_key_name_keys_several_tables(
|
|
2535
|
+
tables: Mapping[str, Table],
|
|
2536
|
+
) -> None:
|
|
2537
|
+
r"""Say when a key name is shared, so no edge can be inferred through it.
|
|
2538
|
+
|
|
2539
|
+
`_infer_relationships` declines every edge naming a key that more than one
|
|
2540
|
+
table is keyed by, and rightly: `id` on four tables identifies a row in
|
|
2541
|
+
each of them and names a parent in none, so choosing one would be a guess.
|
|
2542
|
+
What was missing is that the declining was silent. Four frames infer to
|
|
2543
|
+
four tables and no relationships at all, and the only thing on screen is
|
|
2544
|
+
whatever unrelated notice some column happened to earn, which then reads
|
|
2545
|
+
as the whole account of what inference did.
|
|
2546
|
+
|
|
2547
|
+
Said per shared name rather than per lost edge. A table keyed by an
|
|
2548
|
+
ambiguous name can never be the target of an inferred relationship, so the
|
|
2549
|
+
loss is the name's, and listing candidate edges that were never built
|
|
2550
|
+
would be inventing a graph to describe.
|
|
2551
|
+
"""
|
|
2552
|
+
keyed_by: dict[str, list[str]] = {}
|
|
2553
|
+
for name, table in tables.items():
|
|
2554
|
+
if isinstance(table.primary_key, str):
|
|
2555
|
+
keyed_by.setdefault(table.primary_key, []).append(name)
|
|
2556
|
+
|
|
2557
|
+
for key, names in sorted(keyed_by.items()):
|
|
2558
|
+
if len(names) < 2:
|
|
2559
|
+
continue
|
|
2560
|
+
shared = sorted(names)
|
|
2561
|
+
warnings.warn(
|
|
2562
|
+
f'{key!r} is the primary key of {_and_list(shared)}, so a column '
|
|
2563
|
+
f'named {key!r} identifies a row in each of them and names a '
|
|
2564
|
+
f'parent in none. No relationship is inferred through an '
|
|
2565
|
+
f'ambiguous name, so nothing can point at any of those tables: '
|
|
2566
|
+
f"none of them is gathered as another table's related rows, and "
|
|
2567
|
+
f'a question about one of them is answered from that table alone. '
|
|
2568
|
+
f'Say what the graph is: '
|
|
2569
|
+
f'relationships=[Relationship(child, column, {shared[0]!r}, '
|
|
2570
|
+
f'{key!r}), ...], or give each table a key of its own name, so '
|
|
2571
|
+
f'that the column naming it can be matched to it.',
|
|
2572
|
+
category=SDMWarning,
|
|
2573
|
+
stacklevel=caller_stacklevel(),
|
|
2574
|
+
)
|
|
2575
|
+
|
|
2576
|
+
|
|
2577
|
+
def _unlinkable_table_reads(
|
|
2578
|
+
name: str, candidates: Sequence[str], tables_sharing: int
|
|
2579
|
+
) -> str:
|
|
2580
|
+
r"""What to say about a table nothing can link to.
|
|
2581
|
+
|
|
2582
|
+
Kept apart from the search that found it, because the two are different
|
|
2583
|
+
kinds of difficult. Deciding which columns could be the key is about the
|
|
2584
|
+
data; saying so is about English, and the sentence has to agree with the
|
|
2585
|
+
graph on two counts that move independently: how many columns could be
|
|
2586
|
+
the key, and how many tables they are shared with. Several columns can be
|
|
2587
|
+
shared with one table, and one column with several.
|
|
2588
|
+
"""
|
|
2589
|
+
many, shared_by_many = len(candidates) > 1, tables_sharing > 1
|
|
2590
|
+
subject, verb = ('they', 'are') if many else ('it', 'is')
|
|
2591
|
+
tables = 'tables' if shared_by_many else 'table'
|
|
2592
|
+
return (
|
|
2593
|
+
f'{name!r} has no primary key, and nothing can link to a table '
|
|
2594
|
+
f'without one: every relationship pointing at {name!r} is dropped, so '
|
|
2595
|
+
f'nothing is gathered through it, and a question about another table '
|
|
2596
|
+
f'reaches it only where {name!r} names that table itself. '
|
|
2597
|
+
f'{_and_list(candidates)} {"identify" if many else "identifies"} '
|
|
2598
|
+
f'every row of {name!r} and {verb} named by '
|
|
2599
|
+
f'{"other tables" if shared_by_many else "another table"} too, which '
|
|
2600
|
+
f'makes {"each of them" if many else "it"} either a key or a foreign '
|
|
2601
|
+
f'key that happens not to repeat in the rows read. '
|
|
2602
|
+
f'If {"one of them identifies" if many else "it identifies"} '
|
|
2603
|
+
f'{name!r}, say so with primary_keys={{{name!r}: '
|
|
2604
|
+
f'{repr(candidates[0]) if not many else "..."}}}; '
|
|
2605
|
+
f'if {subject} instead '
|
|
2606
|
+
f'{"name" if many else "names"} the rows of the {tables} {subject} '
|
|
2607
|
+
f'{verb} shared with, '
|
|
2608
|
+
f'{"those tables are the ones that need" if shared_by_many else "that table is the one that needs"} '
|
|
2609
|
+
f'the key.'
|
|
2610
|
+
)
|
|
2611
|
+
|
|
2612
|
+
|
|
2613
|
+
def _each_link_once(
|
|
2614
|
+
links: Sequence[Relationship],
|
|
2615
|
+
) -> list[Relationship]:
|
|
2616
|
+
"""Deduplicate relationships while preserving their order.
|
|
2617
|
+
|
|
2618
|
+
The materializer already sends one edge for it, so keeping two here made
|
|
2619
|
+
the two halves of the client disagree about what the graph is, and the
|
|
2620
|
+
ambiguity warning read a repeat as a second way across the pair.
|
|
2621
|
+
"""
|
|
2622
|
+
seen: set[tuple[str, str, str, str]] = set()
|
|
2623
|
+
once = []
|
|
2624
|
+
for link in links:
|
|
2625
|
+
named = (
|
|
2626
|
+
link.source_table,
|
|
2627
|
+
link.source_column,
|
|
2628
|
+
link.target_table,
|
|
2629
|
+
link.target_column,
|
|
2630
|
+
)
|
|
2631
|
+
if named in seen:
|
|
2632
|
+
continue
|
|
2633
|
+
seen.add(named)
|
|
2634
|
+
once.append(link)
|
|
2635
|
+
return once
|
|
2636
|
+
|
|
2637
|
+
|
|
2638
|
+
def _warn_of_a_pair_joined_both_ways(links: Sequence[Relationship]) -> None:
|
|
2639
|
+
"""Say when two tables name each other, which reads as an ambiguous pair.
|
|
2640
|
+
|
|
2641
|
+
A parent naming one representative child is an ordinary shape and the
|
|
2642
|
+
schema takes it. The sampler walks an edge in both directions, so a
|
|
2643
|
+
declared reverse edge is a second way across the same pair, and every
|
|
2644
|
+
aggregation between them is then refused for a foreign key it says it
|
|
2645
|
+
cannot find uniquely when there is exactly one. The reverse edge is
|
|
2646
|
+
what made it two.
|
|
2647
|
+
|
|
2648
|
+
A table pointing at itself is one link, not two: the test for a reverse
|
|
2649
|
+
edge is otherwise satisfied by the edge itself.
|
|
2650
|
+
"""
|
|
2651
|
+
pairs = {(link.source_table, link.target_table) for link in links}
|
|
2652
|
+
said = set()
|
|
2653
|
+
for source, target in sorted(pairs):
|
|
2654
|
+
if source == target:
|
|
2655
|
+
continue
|
|
2656
|
+
if (target, source) not in pairs or (target, source) in said:
|
|
2657
|
+
continue
|
|
2658
|
+
said.add((source, target))
|
|
2659
|
+
warnings.warn(
|
|
2660
|
+
f'{source!r} and {target!r} name each other, and a relationship is '
|
|
2661
|
+
f'crossed in both directions, so this is two ways over one pair and '
|
|
2662
|
+
f'an aggregation between them is refused for not having a single '
|
|
2663
|
+
f'one. Declare whichever of the two a question is about and leave '
|
|
2664
|
+
f'the other out.',
|
|
2665
|
+
category=SDMWarning,
|
|
2666
|
+
stacklevel=caller_stacklevel(),
|
|
2667
|
+
)
|
|
2668
|
+
|
|
2669
|
+
|
|
2670
|
+
class _Links(list):
|
|
2671
|
+
_schema: Schema
|
|
2672
|
+
|
|
2673
|
+
r"""The relationships of one schema, checked however they are added.
|
|
2674
|
+
|
|
2675
|
+
`append` is the route this client's own advice prints, and a check on
|
|
2676
|
+
assignment alone left it unchecked.
|
|
2677
|
+
"""
|
|
2678
|
+
|
|
2679
|
+
def __init__(self, links: Sequence[Relationship], schema: Schema) -> None:
|
|
2680
|
+
super().__init__(links)
|
|
2681
|
+
object.__setattr__(self, '_schema', schema)
|
|
2682
|
+
|
|
2683
|
+
def _resettle(self, links: list[Relationship]) -> None:
|
|
2684
|
+
settled = self._schema._settle(links)
|
|
2685
|
+
super().clear()
|
|
2686
|
+
super().extend(settled)
|
|
2687
|
+
self._schema._mark_key_columns()
|
|
2688
|
+
|
|
2689
|
+
def remove(self, link: Any) -> None:
|
|
2690
|
+
kept = list(self)
|
|
2691
|
+
kept.remove(link)
|
|
2692
|
+
self._resettle(kept)
|
|
2693
|
+
|
|
2694
|
+
def pop(self, index: Any = -1) -> Relationship:
|
|
2695
|
+
kept = list(self)
|
|
2696
|
+
gone = kept.pop(index)
|
|
2697
|
+
self._resettle(kept)
|
|
2698
|
+
return gone
|
|
2699
|
+
|
|
2700
|
+
def clear(self) -> None:
|
|
2701
|
+
self._resettle([])
|
|
2702
|
+
|
|
2703
|
+
def __delitem__(self, index: Any) -> None:
|
|
2704
|
+
kept = list(self)
|
|
2705
|
+
del kept[index]
|
|
2706
|
+
self._resettle(kept)
|
|
2707
|
+
|
|
2708
|
+
def append(self, link: Relationship) -> None:
|
|
2709
|
+
self._resettle([*self, link])
|
|
2710
|
+
|
|
2711
|
+
def extend(self, links: Any) -> None:
|
|
2712
|
+
self._resettle([*self, *links])
|
|
2713
|
+
|
|
2714
|
+
def insert(self, index: Any, link: Any) -> None:
|
|
2715
|
+
kept = list(self)
|
|
2716
|
+
kept.insert(index, link)
|
|
2717
|
+
self._resettle(kept)
|
|
2718
|
+
|
|
2719
|
+
def __iadd__(self, links: Any) -> _Links: # type: ignore[misc]
|
|
2720
|
+
self._resettle([*self, *links])
|
|
2721
|
+
return self
|
|
2722
|
+
|
|
2723
|
+
def __imul__(self, count: Any) -> _Links: # type: ignore[misc]
|
|
2724
|
+
self._resettle(list(self) * count)
|
|
2725
|
+
return self
|
|
2726
|
+
|
|
2727
|
+
def __setitem__(self, index: Any, link: Any) -> None:
|
|
2728
|
+
kept = list(self)
|
|
2729
|
+
kept[index] = link
|
|
2730
|
+
self._resettle(kept)
|
|
2731
|
+
|
|
2732
|
+
def __reduce__(self) -> tuple[Any, ...]:
|
|
2733
|
+
r"""Copied and pickled as a plain list, and re-checked on arrival.
|
|
2734
|
+
|
|
2735
|
+
The schema this belongs to is held here, so copying the list the
|
|
2736
|
+
ordinary way copies the schema, whose state is restored after its
|
|
2737
|
+
own `__setattr__` would have run. `Schema.__setstate__` puts the list
|
|
2738
|
+
back under this type once the copy has its tables.
|
|
2739
|
+
"""
|
|
2740
|
+
return (list, (list(self),))
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def _warn_two_ways_to_reach_one_table(
|
|
2744
|
+
tables: Mapping[str, _TableShape], links: Sequence[Relationship]
|
|
2745
|
+
) -> None:
|
|
2746
|
+
r"""Say when one table reaches another by more than one column.
|
|
2747
|
+
|
|
2748
|
+
Both are real relationships and keeping both is right: a sales line names
|
|
2749
|
+
the customer it was sold to and the one it ships to, and they are
|
|
2750
|
+
different facts. But an aggregation across a pair of tables has to be
|
|
2751
|
+
resolved to one column, and a query that crosses an ambiguous pair is
|
|
2752
|
+
refused when it is asked, which reads as a problem with the
|
|
2753
|
+
query rather than with the graph it was written against.
|
|
2754
|
+
|
|
2755
|
+
Said here so that it is read where the cause is. Nothing is dropped: which
|
|
2756
|
+
of them a question means is the caller's to decide, and choosing for them
|
|
2757
|
+
is choosing what their model sees.
|
|
2758
|
+
"""
|
|
2759
|
+
ways: dict[tuple[str, str], list[str]] = {}
|
|
2760
|
+
for link in links:
|
|
2761
|
+
pair = (link.source_table, link.target_table)
|
|
2762
|
+
columns = ways.setdefault(pair, [])
|
|
2763
|
+
if link.source_column not in columns:
|
|
2764
|
+
columns.append(link.source_column)
|
|
2765
|
+
|
|
2766
|
+
_warn_of_a_pair_joined_both_ways(links)
|
|
2767
|
+
for (source, target), columns in sorted(ways.items()):
|
|
2768
|
+
if len(columns) < 2:
|
|
2769
|
+
continue
|
|
2770
|
+
kept, *rest = sorted(columns)
|
|
2771
|
+
warnings.warn(
|
|
2772
|
+
f'{source!r} reaches {target!r} through {_and_list(sorted(columns))}'
|
|
2773
|
+
f', and an aggregation across the two tables has to be resolved to '
|
|
2774
|
+
f'one of them, so a query that crosses this pair is refused until '
|
|
2775
|
+
f'it is. They are all kept because they are all real, and which '
|
|
2776
|
+
f'one a question means is not something this can decide. To keep '
|
|
2777
|
+
f'{kept!r}, drop the others: schema.relationships = [link for '
|
|
2778
|
+
f'link in schema.relationships if not (link.source_table == '
|
|
2779
|
+
f'{source!r} and link.target_table == {target!r} and '
|
|
2780
|
+
f'link.source_column in {sorted(rest)!r})].',
|
|
2781
|
+
category=SDMWarning,
|
|
2782
|
+
stacklevel=caller_stacklevel(),
|
|
2783
|
+
)
|
|
2784
|
+
|
|
2785
|
+
|
|
2786
|
+
def _could_identify_a_row(
|
|
2787
|
+
table: Table, frame: pd.DataFrame, column: str, meta: Column
|
|
2788
|
+
) -> bool:
|
|
2789
|
+
if column == table.time_column:
|
|
2790
|
+
return False
|
|
2791
|
+
if meta.dtype != 'string' and not meta.dtype.startswith('int'):
|
|
2792
|
+
return False
|
|
2793
|
+
if frame[column].isna().any():
|
|
2794
|
+
return False
|
|
2795
|
+
return _key_quality(frame, (column,)).duplicates == 0
|
|
2796
|
+
|
|
2797
|
+
|
|
2798
|
+
def _and_list(names: Sequence[str]) -> str:
|
|
2799
|
+
quoted = [repr(name) for name in names]
|
|
2800
|
+
if len(quoted) < 3:
|
|
2801
|
+
return ' and '.join(quoted)
|
|
2802
|
+
return f'{", ".join(quoted[:-1])} and {quoted[-1]}'
|
|
2803
|
+
|
|
2804
|
+
|
|
2805
|
+
def _infer_relationships(
|
|
2806
|
+
tables: Mapping[str, _TableShape],
|
|
2807
|
+
) -> list[Relationship]:
|
|
2808
|
+
r"""Link a column to another table's primary key of the same name.
|
|
2809
|
+
|
|
2810
|
+
Deliberately conservative: only an exact name match against a declared
|
|
2811
|
+
primary key counts. A wrong link changes what the model is shown, and is
|
|
2812
|
+
harder to notice than a missing one, which simply narrows the question.
|
|
2813
|
+
|
|
2814
|
+
Which is why a name more than one table is keyed by links nothing. `ID` on
|
|
2815
|
+
four tables identifies a row in each of them and names a parent in none, so
|
|
2816
|
+
choosing one would be a guess in the direction this is supposed to avoid.
|
|
2817
|
+
Value overlap cannot break the tie either: a
|
|
2818
|
+
fabricated edge has been measured joining 97% of rows and a real one 91%,
|
|
2819
|
+
so no threshold separates them.
|
|
2820
|
+
"""
|
|
2821
|
+
ambiguous: set[str] = set()
|
|
2822
|
+
keys: dict[str, str] = {}
|
|
2823
|
+
for name, table in tables.items():
|
|
2824
|
+
key = table.primary_key
|
|
2825
|
+
if not isinstance(key, str):
|
|
2826
|
+
continue
|
|
2827
|
+
if key in keys:
|
|
2828
|
+
ambiguous.add(key)
|
|
2829
|
+
keys[key] = name
|
|
2830
|
+
|
|
2831
|
+
links = []
|
|
2832
|
+
for name, table in tables.items():
|
|
2833
|
+
for column in table.columns:
|
|
2834
|
+
target = keys.get(column)
|
|
2835
|
+
if target is None or target == name or column in ambiguous:
|
|
2836
|
+
continue
|
|
2837
|
+
links.append(
|
|
2838
|
+
Relationship(
|
|
2839
|
+
source_table=name,
|
|
2840
|
+
source_column=column,
|
|
2841
|
+
target_table=target,
|
|
2842
|
+
target_column=column,
|
|
2843
|
+
)
|
|
2844
|
+
)
|
|
2845
|
+
return links
|
|
2846
|
+
|
|
2847
|
+
|
|
2848
|
+
def _warn_of_a_link_that_cannot_join(
|
|
2849
|
+
tables: Mapping[str, _TableShape], dropped: Sequence[Relationship]
|
|
2850
|
+
) -> None:
|
|
2851
|
+
"""Say why an inferred or declared link could not join.
|
|
2852
|
+
|
|
2853
|
+
Dropping it is right: joining `int64` to `string` matches nothing, and an
|
|
2854
|
+
edge that matches nothing is worse than no edge. But it was dropped in
|
|
2855
|
+
silence, and the table then either goes unsent or the query is refused for
|
|
2856
|
+
a foreign key that looks as though it should exist.
|
|
2857
|
+
"""
|
|
2858
|
+
for link in dropped:
|
|
2859
|
+
source = tables[link.source_table].columns[link.source_column]
|
|
2860
|
+
target_table = tables[link.target_table]
|
|
2861
|
+
target = target_table.columns[link.target_column]
|
|
2862
|
+
key = _as_key_tuple(target_table.primary_key)
|
|
2863
|
+
if key != (link.target_column,):
|
|
2864
|
+
# A declared foreign key may name a unique column that is not the
|
|
2865
|
+
# key, which is no fault of the dtypes.
|
|
2866
|
+
has = (
|
|
2867
|
+
f'the primary key of {link.target_table!r} is '
|
|
2868
|
+
f'{_and_list(list(key))}'
|
|
2869
|
+
if key
|
|
2870
|
+
else f'{link.target_table!r} has no primary key'
|
|
2871
|
+
)
|
|
2872
|
+
warnings.warn(
|
|
2873
|
+
f'{link.source_table}.{link.source_column} refers to '
|
|
2874
|
+
f'{link.target_table}.{link.target_column}, but {has}. A '
|
|
2875
|
+
f'relationship joins to a primary key, so none was made from '
|
|
2876
|
+
f'it. Pass '
|
|
2877
|
+
f'primary_keys={{{link.target_table!r}: '
|
|
2878
|
+
f'{link.target_column!r}}} if {link.target_column!r} '
|
|
2879
|
+
f'identifies a row of {link.target_table!r}.',
|
|
2880
|
+
category=SDMWarning,
|
|
2881
|
+
stacklevel=caller_stacklevel(),
|
|
2882
|
+
)
|
|
2883
|
+
continue
|
|
2884
|
+
warnings.warn(
|
|
2885
|
+
f'{link.source_table}.{link.source_column} names the key of '
|
|
2886
|
+
f'{link.target_table!r} and holds {source.dtype!r} where that key '
|
|
2887
|
+
f'holds {target.dtype!r}, which cannot match, so no relationship '
|
|
2888
|
+
f'was made from it. Store the two as one type if they are the same '
|
|
2889
|
+
f'identifier.',
|
|
2890
|
+
category=SDMWarning,
|
|
2891
|
+
stacklevel=caller_stacklevel(),
|
|
2892
|
+
)
|
|
2893
|
+
|
|
2894
|
+
|
|
2895
|
+
def _require_joinable(
|
|
2896
|
+
tables: Mapping[str, _TableShape], link: Relationship
|
|
2897
|
+
) -> None:
|
|
2898
|
+
r"""A relationship has to be one the engine can actually build.
|
|
2899
|
+
|
|
2900
|
+
An edge joins a foreign key to the *primary key* of the table it points at.
|
|
2901
|
+
That is the only shape the graph has, so a link naming some other
|
|
2902
|
+
column is not narrower, it is different: the language would resolve one
|
|
2903
|
+
graph while the engine built another. And the two columns have to hold
|
|
2904
|
+
comparable values, or the join matches nothing.
|
|
2905
|
+
"""
|
|
2906
|
+
target = tables[link.target_table]
|
|
2907
|
+
key = target.primary_key
|
|
2908
|
+
keys = _as_key_tuple(key)
|
|
2909
|
+
if len(keys) > 1:
|
|
2910
|
+
raise StructuredDataError(
|
|
2911
|
+
f'{link} points at {link.target_table!r}, whose primary key is '
|
|
2912
|
+
f'{list(keys)}. An edge joins one column to one key, so a table '
|
|
2913
|
+
f'with a composite key cannot be the target of a relationship',
|
|
2914
|
+
code='INVALID_REQUEST',
|
|
2915
|
+
)
|
|
2916
|
+
if link.target_column not in keys:
|
|
2917
|
+
named = f'{key!r}' if key is not None else 'no primary key'
|
|
2918
|
+
raise StructuredDataError(
|
|
2919
|
+
f'{link} does not join to a primary key: table '
|
|
2920
|
+
f'{link.target_table!r} has {named}. A relationship points at the '
|
|
2921
|
+
f'key that identifies a row, so set that key first if it is not '
|
|
2922
|
+
f'the one inference found',
|
|
2923
|
+
code='INVALID_REQUEST',
|
|
2924
|
+
)
|
|
2925
|
+
source_dtype = tables[link.source_table][link.source_column].dtype
|
|
2926
|
+
target_dtype = target[link.target_column].dtype
|
|
2927
|
+
if _join_family(source_dtype) != _join_family(target_dtype):
|
|
2928
|
+
raise StructuredDataError(
|
|
2929
|
+
f'{link} joins {source_dtype!r} to {target_dtype!r}, which cannot '
|
|
2930
|
+
f'match. Give both columns the same kind of value',
|
|
2931
|
+
code='INVALID_REQUEST',
|
|
2932
|
+
)
|
|
2933
|
+
|
|
2934
|
+
|
|
2935
|
+
def _join_family(dtype: str) -> str:
|
|
2936
|
+
r"""What has to agree across a join: a number matches a number."""
|
|
2937
|
+
if dtype.startswith(('int', 'float')):
|
|
2938
|
+
return 'number'
|
|
2939
|
+
if dtype.startswith('timestamp'):
|
|
2940
|
+
return 'timestamp'
|
|
2941
|
+
return dtype
|
|
2942
|
+
|
|
2943
|
+
|
|
2944
|
+
def supports_dtype(stype: str, dtype: str) -> bool:
|
|
2945
|
+
r"""Whether a column of ``dtype`` can mean ``stype``.
|
|
2946
|
+
|
|
2947
|
+
Answered by the language rather than by a second table kept here, so the
|
|
2948
|
+
two cannot drift. An unrecognised pair is allowed through: this is a guard
|
|
2949
|
+
against a known-wrong combination, not a whitelist.
|
|
2950
|
+
"""
|
|
2951
|
+
from predictive_query.schema.typing import Dtype, Stype
|
|
2952
|
+
|
|
2953
|
+
try:
|
|
2954
|
+
return Stype(stype).supports_dtype(
|
|
2955
|
+
Dtype(_WIRE_TO_PQL_DTYPE.get(dtype, dtype))
|
|
2956
|
+
)
|
|
2957
|
+
except ValueError:
|
|
2958
|
+
return True
|
|
2959
|
+
|
|
2960
|
+
|
|
2961
|
+
class _KeyQuality(NamedTuple):
|
|
2962
|
+
r"""What one pass over a key column found.
|
|
2963
|
+
|
|
2964
|
+
Computed once and handed on. The check, the count that gets recorded and
|
|
2965
|
+
the example in the message all used to derive this separately, which meant
|
|
2966
|
+
three full passes over the same column for every key assignment.
|
|
2967
|
+
"""
|
|
2968
|
+
|
|
2969
|
+
duplicates: int
|
|
2970
|
+
nulls: int
|
|
2971
|
+
example: Any
|
|
2972
|
+
|
|
2973
|
+
|
|
2974
|
+
def _is_joinable(tables: Mapping[str, _TableShape], link: Relationship) -> bool:
|
|
2975
|
+
try:
|
|
2976
|
+
_require_joinable(tables, link)
|
|
2977
|
+
except StructuredDataError:
|
|
2978
|
+
return False
|
|
2979
|
+
return True
|
|
2980
|
+
|
|
2981
|
+
|
|
2982
|
+
def _the_key_of(table: str, names: Sequence[str]) -> str:
|
|
2983
|
+
"""How to name a key in a message, for one column or several.
|
|
2984
|
+
|
|
2985
|
+
Joining the parts with a dot rendered `['a', 'b']` of table `t` as
|
|
2986
|
+
`t.a.b`, which reads as one column named `a.b`.
|
|
2987
|
+
"""
|
|
2988
|
+
if len(names) == 1:
|
|
2989
|
+
return f'{table}.{names[0]}'
|
|
2990
|
+
return f'the primary key of {table!r}, {_and_list(list(names))},'
|
|
2991
|
+
|
|
2992
|
+
|
|
2993
|
+
_NAME_ANOTHER_KEY = (
|
|
2994
|
+
' Name a different primary key if this column is a foreign key rather '
|
|
2995
|
+
'than an identifier.'
|
|
2996
|
+
)
|
|
2997
|
+
|
|
2998
|
+
|
|
2999
|
+
def _too_many_repeats(duplicates: int, rows: int) -> bool:
|
|
3000
|
+
r"""Whether a stated key repeats more than inference would tolerate."""
|
|
3001
|
+
return duplicates > rows * (1 - _KEY_UNIQUENESS)
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
def _not_unique_enough(
|
|
3005
|
+
table: str, names: Sequence[str], duplicates: int, rows: int, example: Any
|
|
3006
|
+
) -> str:
|
|
3007
|
+
return (
|
|
3008
|
+
f'{_the_key_of(table, names)} is not unique enough to identify a '
|
|
3009
|
+
f'row: {duplicates:,} of {rows:,} rows repeat a value already used, '
|
|
3010
|
+
f'e.g. {example!r}'
|
|
3011
|
+
)
|
|
3012
|
+
|
|
3013
|
+
|
|
3014
|
+
def _a_key_missing_values(
|
|
3015
|
+
table: str, names: Sequence[str], nulls: int, empty: Sequence[str] | None
|
|
3016
|
+
) -> str:
|
|
3017
|
+
r"""Why a stated key with a null in it is refused.
|
|
3018
|
+
|
|
3019
|
+
``empty`` names the columns holding the nulls, or is ``None`` where only
|
|
3020
|
+
the rows missing some part of a composite key were counted.
|
|
3021
|
+
"""
|
|
3022
|
+
where = f', in {_and_list(empty)},' if empty else ' in part of it,'
|
|
3023
|
+
several = empty is None or len(empty) > 1
|
|
3024
|
+
return (
|
|
3025
|
+
f'{_the_key_of(table, names)} has {nulls:,} row(s) with no value'
|
|
3026
|
+
f'{where} so those rows cannot be identified; fill or drop them '
|
|
3027
|
+
f'before using {"them" if several else "it"} as a primary key'
|
|
3028
|
+
)
|
|
3029
|
+
|
|
3030
|
+
|
|
3031
|
+
def _warn_a_key_repeats(
|
|
3032
|
+
table: str, names: Sequence[str], duplicates: int, example: Any, remedy: str
|
|
3033
|
+
) -> None:
|
|
3034
|
+
warnings.warn(
|
|
3035
|
+
f'{table}.{".".join(names)} identifies a row {duplicates:,} time(s) '
|
|
3036
|
+
f'it has already been used, e.g. {example!r}. Those rows are '
|
|
3037
|
+
f'indistinguishable to the model.{remedy}',
|
|
3038
|
+
category=SDMWarning,
|
|
3039
|
+
stacklevel=caller_stacklevel(),
|
|
3040
|
+
)
|
|
3041
|
+
|
|
3042
|
+
|
|
3043
|
+
def _warn_a_key_has_no_value(table: str, column: str, missing: int) -> None:
|
|
3044
|
+
r"""Said of a key that was inferred rather than stated, which a null in
|
|
3045
|
+
it does not cost: inference is a guess, and refusing would break tables
|
|
3046
|
+
that worked before.
|
|
3047
|
+
"""
|
|
3048
|
+
warnings.warn(
|
|
3049
|
+
f'{table}.{column} identifies a row, but {missing:,} row(s) have no '
|
|
3050
|
+
f'value for it. Those rows cannot be scored and the model will not '
|
|
3051
|
+
f'see them; fill or drop them, or name a different primary key.',
|
|
3052
|
+
category=SDMWarning,
|
|
3053
|
+
stacklevel=caller_stacklevel(),
|
|
3054
|
+
)
|
|
3055
|
+
|
|
3056
|
+
|
|
3057
|
+
def _warn_a_key_was_declined(
|
|
3058
|
+
table: str,
|
|
3059
|
+
column: str,
|
|
3060
|
+
duplicates: int,
|
|
3061
|
+
rows: int,
|
|
3062
|
+
example: Any,
|
|
3063
|
+
remedy: str,
|
|
3064
|
+
) -> None:
|
|
3065
|
+
warnings.warn(
|
|
3066
|
+
f'{table}.{column} is named like a key and another table names it '
|
|
3067
|
+
f'too, but {duplicates:,} of {rows:,} rows repeat a value already '
|
|
3068
|
+
f'used, e.g. {example!r}, so it does not identify a row and '
|
|
3069
|
+
f'{table!r} gets no primary key. Nothing can link to a table without '
|
|
3070
|
+
f'one: every relationship pointing at {table!r} is dropped, so '
|
|
3071
|
+
f'nothing is gathered through it, and a question about another table '
|
|
3072
|
+
f'reaches it only where {table!r} names that table itself. {remedy}',
|
|
3073
|
+
category=SDMWarning,
|
|
3074
|
+
stacklevel=caller_stacklevel(),
|
|
3075
|
+
)
|
|
3076
|
+
|
|
3077
|
+
|
|
3078
|
+
def _key_quality(frame: pd.DataFrame, names: Sequence[str]) -> _KeyQuality:
|
|
3079
|
+
columns = list(names)
|
|
3080
|
+
if any(name not in frame.columns for name in columns):
|
|
3081
|
+
return _KeyQuality(0, 0, None)
|
|
3082
|
+
key = frame[columns[0]] if len(columns) == 1 else frame[columns]
|
|
3083
|
+
nulls = int(
|
|
3084
|
+
key.isna().any(axis=1).sum() if len(columns) > 1 else key.isna().sum()
|
|
3085
|
+
)
|
|
3086
|
+
# `duplicated` treats null as equal to null, so nulls would otherwise be
|
|
3087
|
+
# counted as repeats and reported as a duplicate problem, which has a
|
|
3088
|
+
# different fix. Count them separately and leave them out of this.
|
|
3089
|
+
present = (
|
|
3090
|
+
key[~key.isna().any(axis=1)] if len(columns) > 1 else key[key.notna()]
|
|
3091
|
+
)
|
|
3092
|
+
repeated_mask = present.duplicated()
|
|
3093
|
+
duplicates = int(repeated_mask.sum())
|
|
3094
|
+
example = None
|
|
3095
|
+
if duplicates:
|
|
3096
|
+
first = present[repeated_mask].iloc[0]
|
|
3097
|
+
example = first if len(columns) == 1 else tuple(first)
|
|
3098
|
+
return _KeyQuality(duplicates, nulls, example)
|
|
3099
|
+
|
|
3100
|
+
|
|
3101
|
+
def to_schema_definition(schema: Schema) -> Any:
|
|
3102
|
+
r"""This schema as the language's own schema definition.
|
|
3103
|
+
|
|
3104
|
+
The language resolves a query against names and types with no rows
|
|
3105
|
+
attached, which is what this hands it. Built here rather than in the query
|
|
3106
|
+
module so that a schema has exactly one translation into what PQL reads.
|
|
3107
|
+
|
|
3108
|
+
`TableDefinition.pkey` is a single column name, so the language has no
|
|
3109
|
+
form for a composite key and cannot be given one. It is dropped rather
|
|
3110
|
+
than refused, because the engine does support one and the request that
|
|
3111
|
+
follows carries it. What the drop costs is that the language resolves the
|
|
3112
|
+
query against a table it believes has no identity, so a question that
|
|
3113
|
+
needs one is answered as though the table had none, which is why it is
|
|
3114
|
+
said rather than done quietly.
|
|
3115
|
+
"""
|
|
3116
|
+
from predictive_query.schema.definition import (
|
|
3117
|
+
ColumnKey,
|
|
3118
|
+
ColumnKeyGroup,
|
|
3119
|
+
SchemaDefinition,
|
|
3120
|
+
)
|
|
3121
|
+
from predictive_query.schema.table import Column as PQLColumn
|
|
3122
|
+
from predictive_query.schema.table import TableDefinition
|
|
3123
|
+
from predictive_query.schema.typing import Dtype, Stype
|
|
3124
|
+
|
|
3125
|
+
tables = {}
|
|
3126
|
+
for name, table in schema.tables.items():
|
|
3127
|
+
columns = []
|
|
3128
|
+
for column in table.columns.values():
|
|
3129
|
+
try:
|
|
3130
|
+
dtype = Dtype(
|
|
3131
|
+
_WIRE_TO_PQL_DTYPE.get(column.dtype, column.dtype)
|
|
3132
|
+
)
|
|
3133
|
+
stype = Stype(column.stype)
|
|
3134
|
+
except ValueError:
|
|
3135
|
+
continue
|
|
3136
|
+
columns.append(
|
|
3137
|
+
PQLColumn(name=column.name, stype=stype, dtype=dtype)
|
|
3138
|
+
)
|
|
3139
|
+
key = table.primary_key
|
|
3140
|
+
if key is not None and not isinstance(key, str):
|
|
3141
|
+
warnings.warn(
|
|
3142
|
+
f'table {name!r} has the composite primary key '
|
|
3143
|
+
f'{list(key)}, and a predictive query can only name a single '
|
|
3144
|
+
f'key column, so this query is resolved against {name!r} as '
|
|
3145
|
+
f'though it had no key. The request still carries the '
|
|
3146
|
+
f'composite key; only the query does not see it. Name a '
|
|
3147
|
+
f'single column as the primary key if the query needs to '
|
|
3148
|
+
f'identify a row of {name!r}.',
|
|
3149
|
+
category=SDMWarning,
|
|
3150
|
+
stacklevel=caller_stacklevel(),
|
|
3151
|
+
)
|
|
3152
|
+
tables[name] = TableDefinition(
|
|
3153
|
+
cols=columns,
|
|
3154
|
+
pkey=key if isinstance(key, str) else None,
|
|
3155
|
+
time_col=table.time_column,
|
|
3156
|
+
)
|
|
3157
|
+
|
|
3158
|
+
groups = [
|
|
3159
|
+
ColumnKeyGroup(
|
|
3160
|
+
# The group is documented as sorted and deduped, and a ColumnKey
|
|
3161
|
+
# has no ordering of its own, so sort on what it holds.
|
|
3162
|
+
columns=tuple(
|
|
3163
|
+
sorted(
|
|
3164
|
+
(
|
|
3165
|
+
ColumnKey(link.source_table, link.source_column),
|
|
3166
|
+
ColumnKey(link.target_table, link.target_column),
|
|
3167
|
+
),
|
|
3168
|
+
key=lambda key: (key.table_name, key.col_name),
|
|
3169
|
+
)
|
|
3170
|
+
)
|
|
3171
|
+
)
|
|
3172
|
+
for link in schema.relationships
|
|
3173
|
+
]
|
|
3174
|
+
return SchemaDefinition(tables=tables, col_groups=groups)
|
|
3175
|
+
|
|
3176
|
+
|
|
3177
|
+
# The wire spells a few types differently from the language.
|
|
3178
|
+
_WIRE_TO_PQL_DTYPE = {
|
|
3179
|
+
'int64': 'int',
|
|
3180
|
+
'int32': 'int',
|
|
3181
|
+
'float64': 'float',
|
|
3182
|
+
'float32': 'float',
|
|
3183
|
+
'timestamp[us]': 'time',
|
|
3184
|
+
'timestamp[ns]': 'time',
|
|
3185
|
+
'timestamp[ms]': 'time',
|
|
3186
|
+
'timestamp[s]': 'time',
|
|
3187
|
+
}
|