lorebase-core 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_core/__init__.py +1 -0
- lorebase_core/canonical/__init__.py +35 -0
- lorebase_core/canonical/hashing.py +128 -0
- lorebase_core/canonical/jcs.py +162 -0
- lorebase_core/expr/__init__.py +118 -0
- lorebase_core/expr/ast.py +443 -0
- lorebase_core/expr/checker.py +428 -0
- lorebase_core/expr/compiler.py +87 -0
- lorebase_core/expr/context.py +82 -0
- lorebase_core/expr/derivations.py +306 -0
- lorebase_core/expr/environment.py +176 -0
- lorebase_core/expr/errors.py +98 -0
- lorebase_core/expr/evaluator.py +414 -0
- lorebase_core/expr/functions.py +898 -0
- lorebase_core/expr/limits.py +52 -0
- lorebase_core/expr/operations.py +479 -0
- lorebase_core/expr/ops.py +243 -0
- lorebase_core/expr/resolver.py +96 -0
- lorebase_core/expr/rows.py +148 -0
- lorebase_core/expr/sensitivity.py +337 -0
- lorebase_core/expr/types.py +227 -0
- lorebase_core/expr/validation.py +51 -0
- lorebase_core/expr/values.py +169 -0
- lorebase_core/expr/visibility.py +363 -0
- lorebase_core/infer/__init__.py +36 -0
- lorebase_core/infer/columns.py +621 -0
- lorebase_core/infer/engine.py +546 -0
- lorebase_core/infer/keys.py +164 -0
- lorebase_core/infer/names.py +145 -0
- lorebase_core/infer/options.py +290 -0
- lorebase_core/infer/profile.py +54 -0
- lorebase_core/infer/proposal.py +122 -0
- lorebase_core/infer/references.py +314 -0
- lorebase_core/infer/suggestions.py +731 -0
- lorebase_core/infer/tables.py +81 -0
- lorebase_core/merge/__init__.py +89 -0
- lorebase_core/merge/conflicts.py +262 -0
- lorebase_core/merge/consistency.py +440 -0
- lorebase_core/merge/data_merge.py +522 -0
- lorebase_core/merge/engine.py +476 -0
- lorebase_core/merge/index.py +100 -0
- lorebase_core/merge/objects.py +204 -0
- lorebase_core/merge/policies.py +322 -0
- lorebase_core/merge/resolve.py +321 -0
- lorebase_core/merge/schema_merge.py +429 -0
- lorebase_core/merge/state.py +443 -0
- lorebase_core/model/__init__.py +245 -0
- lorebase_core/model/attribute.py +72 -0
- lorebase_core/model/base.py +61 -0
- lorebase_core/model/derivation.py +95 -0
- lorebase_core/model/domain.py +71 -0
- lorebase_core/model/embedding.py +193 -0
- lorebase_core/model/ids.py +78 -0
- lorebase_core/model/kind.py +85 -0
- lorebase_core/model/lineage.py +114 -0
- lorebase_core/model/operation.py +161 -0
- lorebase_core/model/paths.py +141 -0
- lorebase_core/model/record.py +127 -0
- lorebase_core/model/relation.py +83 -0
- lorebase_core/model/schema.py +219 -0
- lorebase_core/model/source.py +204 -0
- lorebase_core/model/trait.py +37 -0
- lorebase_core/model/vault.py +43 -0
- lorebase_core/model/yaml_schema.py +1058 -0
- lorebase_core/query/__init__.py +97 -0
- lorebase_core/query/dsl.py +227 -0
- lorebase_core/query/resolve.py +794 -0
- lorebase_core/revision/__init__.py +59 -0
- lorebase_core/revision/ancestry.py +272 -0
- lorebase_core/revision/changeset.py +62 -0
- lorebase_core/revision/revision.py +121 -0
- lorebase_core/tree/__init__.py +76 -0
- lorebase_core/tree/diff.py +119 -0
- lorebase_core/tree/hamt.py +333 -0
- lorebase_core/tree/node.py +280 -0
- lorebase_core/tree/params.py +33 -0
- lorebase_core/tree/root.py +70 -0
- lorebase_core/tree/store.py +98 -0
- lorebase_core/types/__init__.py +71 -0
- lorebase_core/types/casts.py +271 -0
- lorebase_core/types/constraints.py +299 -0
- lorebase_core/types/encoding.py +538 -0
- lorebase_core/types/errors.py +41 -0
- lorebase_core/types/matching.py +40 -0
- lorebase_core/types/types.py +187 -0
- lorebase_core/validate/__init__.py +49 -0
- lorebase_core/validate/changes.py +70 -0
- lorebase_core/validate/changeset.py +551 -0
- lorebase_core/validate/data_rules.py +291 -0
- lorebase_core/validate/issues.py +123 -0
- lorebase_core/validate/schema_rules.py +1304 -0
- lorebase_core-0.2.3.dist-info/METADATA +9 -0
- lorebase_core-0.2.3.dist-info/RECORD +94 -0
- lorebase_core-0.2.3.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Primary key choice.
|
|
2
|
+
|
|
3
|
+
Candidates are required, unique columns of a keyable type (string, integer, date, uri). They are
|
|
4
|
+
ranked by ``key_name_patterns`` (``id``, ``key``, ``code``, ``{table}_id``... then ``*_id``...),
|
|
5
|
+
then by column order. Without a named candidate the first column may be the key
|
|
6
|
+
(``allow_unnamed_primary_key``); without any single candidate, pairs (and triples) of key-named
|
|
7
|
+
or reference-named columns are tried. A declared primary key always wins.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from itertools import combinations
|
|
15
|
+
|
|
16
|
+
from lorebase_core.infer.columns import ColumnAnalysis, support
|
|
17
|
+
from lorebase_core.infer.names import rank
|
|
18
|
+
from lorebase_core.infer.options import InferenceOptions
|
|
19
|
+
from lorebase_core.types import AttributeType
|
|
20
|
+
|
|
21
|
+
#: Types a primary key candidate may have.
|
|
22
|
+
KEY_CANDIDATE_TYPES = frozenset(
|
|
23
|
+
{AttributeType.STRING, AttributeType.INTEGER, AttributeType.DATE, AttributeType.URI}
|
|
24
|
+
)
|
|
25
|
+
#: Confidence of a key chosen without a name hint, before support and sampling factors.
|
|
26
|
+
UNNAMED_KEY_CONFIDENCE = 0.6
|
|
27
|
+
#: Confidence of a composite key, before support and sampling factors.
|
|
28
|
+
COMPOSITE_KEY_CONFIDENCE = 0.85
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class KeyChoice:
|
|
33
|
+
"""The chosen key: column indexes in key order."""
|
|
34
|
+
|
|
35
|
+
columns: tuple[int, ...]
|
|
36
|
+
confidence: float
|
|
37
|
+
reasons: tuple[str, ...]
|
|
38
|
+
#: Named single-column candidates, best first (more than one is an ambiguity).
|
|
39
|
+
candidates: tuple[int, ...] = ()
|
|
40
|
+
declared: bool = False
|
|
41
|
+
#: The key column carries generated Record keys: the Kind uses minted keys.
|
|
42
|
+
minted: bool = False
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def single(self) -> int | None:
|
|
46
|
+
return self.columns[0] if len(self.columns) == 1 else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def key_patterns(table_names: Sequence[str], options: InferenceOptions) -> list[str]:
|
|
50
|
+
"""``key_name_patterns`` with ``{table}`` expanded to each table name form, in rank order."""
|
|
51
|
+
expanded: list[str] = []
|
|
52
|
+
for pattern in options.key_name_patterns:
|
|
53
|
+
forms = [pattern.replace("{table}", name) for name in table_names]
|
|
54
|
+
for form in forms if "{table}" in pattern else [pattern]:
|
|
55
|
+
if form not in expanded:
|
|
56
|
+
expanded.append(form)
|
|
57
|
+
return expanded
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _factor(columns: Sequence[ColumnAnalysis], sampled: bool, options: InferenceOptions) -> float:
|
|
61
|
+
non_null = min((c.non_null for c in columns), default=0)
|
|
62
|
+
sampling = options.sampled_uniqueness_factor if sampled else 1.0
|
|
63
|
+
return support(non_null, options) * sampling
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _unique_rows(columns: Sequence[ColumnAnalysis]) -> bool:
|
|
67
|
+
rows = list(zip(*(c.row_keys for c in columns), strict=False))
|
|
68
|
+
if not rows or any(None in row for row in rows):
|
|
69
|
+
return False
|
|
70
|
+
return len(set(rows)) == len(rows)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def choose_key(
|
|
74
|
+
table_names: Sequence[str],
|
|
75
|
+
columns: Sequence[ColumnAnalysis],
|
|
76
|
+
declared: tuple[str, ...] | None,
|
|
77
|
+
options: InferenceOptions,
|
|
78
|
+
*,
|
|
79
|
+
sampled: bool,
|
|
80
|
+
reference_like: frozenset[int] = frozenset(),
|
|
81
|
+
) -> KeyChoice | None:
|
|
82
|
+
"""The primary key of a table, or ``None`` when it has none (Records get minted keys).
|
|
83
|
+
|
|
84
|
+
``reference_like`` lists columns named like references (``*_id``, ``from_key``): they may
|
|
85
|
+
form composite keys even when ``key_name_patterns`` does not list them.
|
|
86
|
+
"""
|
|
87
|
+
by_name = {c.name: c for c in columns}
|
|
88
|
+
if declared:
|
|
89
|
+
found = [by_name[name] for name in declared if name in by_name]
|
|
90
|
+
if len(found) == len(declared):
|
|
91
|
+
return KeyChoice(
|
|
92
|
+
columns=tuple(c.index for c in found),
|
|
93
|
+
confidence=1.0,
|
|
94
|
+
reasons=(f"declared primary key ({', '.join(declared)})",),
|
|
95
|
+
declared=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
patterns = key_patterns(table_names, options)
|
|
99
|
+
candidates = [
|
|
100
|
+
c for c in columns if c.required and c.unique and c.type.name in KEY_CANDIDATE_TYPES
|
|
101
|
+
]
|
|
102
|
+
ranked: list[tuple[int, int, ColumnAnalysis]] = []
|
|
103
|
+
for column in candidates:
|
|
104
|
+
position = rank(column.snake, patterns)
|
|
105
|
+
if position is not None:
|
|
106
|
+
ranked.append((position, column.index, column))
|
|
107
|
+
ranked.sort(key=lambda item: (item[0], item[1]))
|
|
108
|
+
named = tuple(c.index for _, _, c in ranked)
|
|
109
|
+
if ranked:
|
|
110
|
+
best_rank, _, best = ranked[0]
|
|
111
|
+
reasons = [
|
|
112
|
+
f"{best.name!r} is unique and never null",
|
|
113
|
+
f"named like a key ({patterns[best_rank]})",
|
|
114
|
+
]
|
|
115
|
+
if len(ranked) > 1:
|
|
116
|
+
others = ", ".join(repr(c.name) for _, _, c in ranked[1:])
|
|
117
|
+
reasons.append(f"other key candidates: {others}")
|
|
118
|
+
return KeyChoice(
|
|
119
|
+
columns=(best.index,),
|
|
120
|
+
confidence=round(_factor([best], sampled, options), 4),
|
|
121
|
+
reasons=tuple(reasons),
|
|
122
|
+
candidates=named,
|
|
123
|
+
)
|
|
124
|
+
if options.allow_unnamed_primary_key and candidates and candidates[0].index == 0:
|
|
125
|
+
first = candidates[0]
|
|
126
|
+
return KeyChoice(
|
|
127
|
+
columns=(first.index,),
|
|
128
|
+
confidence=round(UNNAMED_KEY_CONFIDENCE * _factor([first], sampled, options), 4),
|
|
129
|
+
reasons=(
|
|
130
|
+
f"first column {first.name!r} is unique and never null",
|
|
131
|
+
"no key-like column name",
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
keyish = [
|
|
136
|
+
c
|
|
137
|
+
for c in columns
|
|
138
|
+
if c.required
|
|
139
|
+
and c.type.name in KEY_CANDIDATE_TYPES
|
|
140
|
+
and (rank(c.snake, patterns) is not None or c.index in reference_like)
|
|
141
|
+
]
|
|
142
|
+
for size in range(2, options.composite_key_max_columns + 1):
|
|
143
|
+
for combo in combinations(keyish, size):
|
|
144
|
+
if _unique_rows(combo):
|
|
145
|
+
names = ", ".join(c.name for c in combo)
|
|
146
|
+
return KeyChoice(
|
|
147
|
+
columns=tuple(c.index for c in combo),
|
|
148
|
+
confidence=round(
|
|
149
|
+
COMPOSITE_KEY_CONFIDENCE * _factor(combo, sampled, options), 4
|
|
150
|
+
),
|
|
151
|
+
reasons=(f"({names}) is unique and never null: composite key",),
|
|
152
|
+
)
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def is_minted(
|
|
157
|
+
choice: KeyChoice, columns: Sequence[ColumnAnalysis], options: InferenceOptions
|
|
158
|
+
) -> bool:
|
|
159
|
+
"""Whether the single key column carries generated Record keys (see ``minted_key_columns``)."""
|
|
160
|
+
single = choice.single
|
|
161
|
+
if single is None:
|
|
162
|
+
return False
|
|
163
|
+
column = columns[single]
|
|
164
|
+
return column.snake in options.minted_key_columns and column.generated_ids
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Stable name normalization: table and column names to api names.
|
|
2
|
+
|
|
3
|
+
- ``normalize``: any source name to ``snake_case`` (``PurchaseOrders`` -> ``purchase_orders``,
|
|
4
|
+
``Unit Price (EUR)`` -> ``unit_price_eur``, ``2nd_line`` -> ``c_2nd_line``).
|
|
5
|
+
- ``singular`` / ``plural``: small English rules on the last word only.
|
|
6
|
+
- ``pascal``: ``purchase_order`` -> ``PurchaseOrder``; ``humanize``: ``Purchase order``.
|
|
7
|
+
- ``matches``: ``fnmatch`` of a normalized name against glob patterns.
|
|
8
|
+
|
|
9
|
+
The functions are pure and deterministic, so the same source names always give the same api
|
|
10
|
+
names.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import unicodedata
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
from fnmatch import fnmatchcase
|
|
19
|
+
|
|
20
|
+
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
|
|
21
|
+
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
|
|
22
|
+
_VOWELS = frozenset("aeiou")
|
|
23
|
+
#: Words whose singular and plural are the same, or that end in ``s`` in the singular.
|
|
24
|
+
_INVARIANT = frozenset(
|
|
25
|
+
{"data", "series", "species", "news", "status", "address", "analysis", "basis", "bus"}
|
|
26
|
+
)
|
|
27
|
+
#: Fallback prefix of names that would start with a digit or be empty.
|
|
28
|
+
NAME_PREFIX = "c"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def normalize(name: str) -> str:
|
|
32
|
+
"""``snake_case`` form of a source name: ASCII letters, digits and single underscores."""
|
|
33
|
+
ascii_name = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode("ascii")
|
|
34
|
+
split = _CAMEL_BOUNDARY.sub("_", ascii_name.strip())
|
|
35
|
+
snake = _NON_ALNUM.sub("_", split.lower()).strip("_")
|
|
36
|
+
if not snake:
|
|
37
|
+
return NAME_PREFIX
|
|
38
|
+
if snake[0].isdigit():
|
|
39
|
+
return f"{NAME_PREFIX}_{snake}"
|
|
40
|
+
return snake
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def strip_prefixes(name: str, prefixes: Iterable[str]) -> str:
|
|
44
|
+
"""``name`` without the first matching prefix, unless nothing would remain."""
|
|
45
|
+
for prefix in prefixes:
|
|
46
|
+
if prefix and name.startswith(prefix) and len(name) > len(prefix):
|
|
47
|
+
return name[len(prefix) :]
|
|
48
|
+
return name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _last_word(snake: str) -> tuple[str, str]:
|
|
52
|
+
head, sep, last = snake.rpartition("_")
|
|
53
|
+
return (head + sep, last)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def singular(snake: str) -> str:
|
|
57
|
+
"""Singular of the last word: ``suppliers`` -> ``supplier``, ``categories`` -> ``category``."""
|
|
58
|
+
head, word = _last_word(snake)
|
|
59
|
+
if word in _INVARIANT or len(word) < 3:
|
|
60
|
+
return snake
|
|
61
|
+
if word.endswith("ies") and len(word) > 4:
|
|
62
|
+
word = word[:-3] + "y"
|
|
63
|
+
elif word.endswith(("sses", "shes", "ches", "xes", "zes")):
|
|
64
|
+
word = word[:-2]
|
|
65
|
+
elif word.endswith("s") and not word.endswith(("ss", "us", "is")):
|
|
66
|
+
word = word[:-1]
|
|
67
|
+
return head + word
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def plural(snake: str) -> str:
|
|
71
|
+
"""Plural of the last word: ``order`` -> ``orders``, ``category`` -> ``categories``."""
|
|
72
|
+
head, word = _last_word(snake)
|
|
73
|
+
if word in _INVARIANT:
|
|
74
|
+
return snake
|
|
75
|
+
if word.endswith("y") and len(word) > 1 and word[-2] not in _VOWELS:
|
|
76
|
+
word = word[:-1] + "ies"
|
|
77
|
+
elif word.endswith(("s", "x", "z", "ch", "sh")):
|
|
78
|
+
word += "es"
|
|
79
|
+
else:
|
|
80
|
+
word += "s"
|
|
81
|
+
return head + word
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def pascal(snake: str) -> str:
|
|
85
|
+
"""``purchase_order`` -> ``PurchaseOrder``; a leading digit gets the fallback prefix."""
|
|
86
|
+
text = "".join(part[:1].upper() + part[1:] for part in snake.split("_") if part)
|
|
87
|
+
if not text or not text[0].isalpha():
|
|
88
|
+
return NAME_PREFIX.upper() + text
|
|
89
|
+
return text
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def humanize(snake: str) -> str:
|
|
93
|
+
"""``purchase_order`` -> ``Purchase order``."""
|
|
94
|
+
words = snake.replace("_", " ").strip()
|
|
95
|
+
return words[:1].upper() + words[1:]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def matches(name: str, patterns: Iterable[str]) -> bool:
|
|
99
|
+
"""Whether the normalized ``name`` matches one of the glob ``patterns``."""
|
|
100
|
+
return any(fnmatchcase(name, pattern) for pattern in patterns)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def rank(name: str, patterns: Iterable[str]) -> int | None:
|
|
104
|
+
"""Index of the first pattern ``name`` matches, or ``None``."""
|
|
105
|
+
for index, pattern in enumerate(patterns):
|
|
106
|
+
if fnmatchcase(name, pattern):
|
|
107
|
+
return index
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def strip_suffix(name: str, suffixes: Iterable[str]) -> str | None:
|
|
112
|
+
"""``name`` without the first matching suffix (``supplier_id`` -> ``supplier``), or
|
|
113
|
+
``None`` when no suffix matches or nothing would remain."""
|
|
114
|
+
for suffix in suffixes:
|
|
115
|
+
if suffix and name.endswith(suffix) and len(name) > len(suffix):
|
|
116
|
+
return name[: -len(suffix)].rstrip("_") or None
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def unique_name(candidate: str, taken: set[str], alternatives: Iterable[str] = ()) -> str:
|
|
121
|
+
"""The first of ``candidate``, ``alternatives`` and ``candidate_2``, ``candidate_3``... not
|
|
122
|
+
in ``taken``; the result is added to ``taken``."""
|
|
123
|
+
for name in (candidate, *alternatives):
|
|
124
|
+
if name not in taken:
|
|
125
|
+
taken.add(name)
|
|
126
|
+
return name
|
|
127
|
+
index = 2
|
|
128
|
+
while f"{candidate}_{index}" in taken:
|
|
129
|
+
index += 1
|
|
130
|
+
result = f"{candidate}_{index}"
|
|
131
|
+
taken.add(result)
|
|
132
|
+
return result
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def unique_pascal(candidate: str, taken: set[str]) -> str:
|
|
136
|
+
"""PascalCase variant of ``unique_name`` (``Order``, ``Order2``)."""
|
|
137
|
+
if candidate not in taken:
|
|
138
|
+
taken.add(candidate)
|
|
139
|
+
return candidate
|
|
140
|
+
index = 2
|
|
141
|
+
while f"{candidate}{index}" in taken:
|
|
142
|
+
index += 1
|
|
143
|
+
result = f"{candidate}{index}"
|
|
144
|
+
taken.add(result)
|
|
145
|
+
return result
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Thresholds and naming rules of schema inference, with documented defaults.
|
|
2
|
+
|
|
3
|
+
Every heuristic of ``lorebase_core.infer`` reads its threshold from ``InferenceOptions``; nothing
|
|
4
|
+
is hard-coded in the rules. Services expose these options as settings (see
|
|
5
|
+
``config/lorebase.example.yaml``, ``keeper.schema_inference.options``).
|
|
6
|
+
|
|
7
|
+
Name patterns are ``fnmatch`` globs matched against normalized ``snake_case`` names. In key
|
|
8
|
+
patterns ``{table}`` stands for the normalized table name and for its singular form.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Self
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
16
|
+
|
|
17
|
+
from lorebase_core.model.base import ElementStatus
|
|
18
|
+
from lorebase_core.validate.schema_rules import DEFAULT_RESERVED_ATTRIBUTE_NAMES
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class InferenceOptions(BaseModel):
|
|
22
|
+
"""Options of ``infer_schema``. Defaults suit small to medium operational tables."""
|
|
23
|
+
|
|
24
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
25
|
+
|
|
26
|
+
# Sampling and value reading ----------------------------------------------------------
|
|
27
|
+
sample_size: int = Field(
|
|
28
|
+
default=10000,
|
|
29
|
+
ge=1,
|
|
30
|
+
description="Values read per column; longer value iterables are cut (head sample).",
|
|
31
|
+
)
|
|
32
|
+
null_tokens: tuple[str, ...] = Field(
|
|
33
|
+
default=("", "NULL", "null"),
|
|
34
|
+
description="Text values read as null (after trimming). Exact, case-sensitive match.",
|
|
35
|
+
)
|
|
36
|
+
trim_text: bool = Field(default=True, description="Trim whitespace around text values.")
|
|
37
|
+
max_evidence_samples: int = Field(
|
|
38
|
+
default=5, ge=0, description="Sample values kept in each Evidence entry."
|
|
39
|
+
)
|
|
40
|
+
min_support_values: int = Field(
|
|
41
|
+
default=20,
|
|
42
|
+
ge=1,
|
|
43
|
+
description="Non-null values needed for full confidence; fewer scale confidence down "
|
|
44
|
+
"linearly (to half at zero values).",
|
|
45
|
+
)
|
|
46
|
+
sampled_uniqueness_factor: float = Field(
|
|
47
|
+
default=0.8,
|
|
48
|
+
ge=0,
|
|
49
|
+
le=1,
|
|
50
|
+
description="Confidence factor of uniqueness and inclusion seen on a sample only "
|
|
51
|
+
"(row_count above the values read).",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
# Types ---------------------------------------------------------------------------------
|
|
55
|
+
true_tokens: tuple[str, ...] = Field(
|
|
56
|
+
default=("true", "t", "yes", "y"), description="Boolean true, compared case-insensitively."
|
|
57
|
+
)
|
|
58
|
+
false_tokens: tuple[str, ...] = Field(
|
|
59
|
+
default=("false", "f", "no", "n"),
|
|
60
|
+
description="Boolean false, compared case-insensitively.",
|
|
61
|
+
)
|
|
62
|
+
type_tolerance: float = Field(
|
|
63
|
+
default=0.0,
|
|
64
|
+
ge=0,
|
|
65
|
+
lt=1,
|
|
66
|
+
description="Share of non-null values allowed not to parse as the detected type. "
|
|
67
|
+
"Outliers are reported as a mixed_types issue.",
|
|
68
|
+
)
|
|
69
|
+
mixed_type_report_ratio: float = Field(
|
|
70
|
+
default=0.5,
|
|
71
|
+
gt=0,
|
|
72
|
+
le=1,
|
|
73
|
+
description="A text column where at least this share parses as one non-text type is "
|
|
74
|
+
"reported as mixed_types.",
|
|
75
|
+
)
|
|
76
|
+
leading_zero_integers_as_text: bool = Field(
|
|
77
|
+
default=True, description="Integers written with leading zeros (007) stay text codes."
|
|
78
|
+
)
|
|
79
|
+
decimal_max_scale: int = Field(
|
|
80
|
+
default=4,
|
|
81
|
+
ge=0,
|
|
82
|
+
description="Numbers with at most this many fractional digits are decimal; more: float.",
|
|
83
|
+
)
|
|
84
|
+
decimal_precision_headroom: int = Field(
|
|
85
|
+
default=2,
|
|
86
|
+
ge=0,
|
|
87
|
+
description="Integer digits added to the largest seen when proposing decimal precision.",
|
|
88
|
+
)
|
|
89
|
+
decimal_max_precision: int = Field(default=38, ge=1, description="Cap of decimal precision.")
|
|
90
|
+
date_formats: tuple[str, ...] = Field(
|
|
91
|
+
default=("%Y-%m-%d",),
|
|
92
|
+
description="strptime formats of date values. Day-month ambiguity is left to the user.",
|
|
93
|
+
)
|
|
94
|
+
timestamp_formats: tuple[str, ...] = Field(
|
|
95
|
+
default=("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S"),
|
|
96
|
+
description="strptime formats of timestamps without offset, read as UTC. RFC 3339 "
|
|
97
|
+
"values with an offset are always recognized.",
|
|
98
|
+
)
|
|
99
|
+
enum_max_cardinality: int = Field(
|
|
100
|
+
default=12, ge=1, description="Text columns with at most this many distinct values ..."
|
|
101
|
+
)
|
|
102
|
+
enum_max_distinct_ratio: float = Field(
|
|
103
|
+
default=0.5, gt=0, le=1, description="... and at most this distinct/non-null ratio ..."
|
|
104
|
+
)
|
|
105
|
+
enum_min_values: int = Field(
|
|
106
|
+
default=10, ge=1, description="... and at least this many non-null values are enum."
|
|
107
|
+
)
|
|
108
|
+
enum_max_value_length: int = Field(
|
|
109
|
+
default=40, ge=1, description="Longer values never form an enum."
|
|
110
|
+
)
|
|
111
|
+
text_min_length: int = Field(
|
|
112
|
+
default=256, ge=1, description="Text values longer than this make the column `text`."
|
|
113
|
+
)
|
|
114
|
+
text_name_hints: tuple[str, ...] = Field(
|
|
115
|
+
default=(
|
|
116
|
+
"description",
|
|
117
|
+
"*_description",
|
|
118
|
+
"summary",
|
|
119
|
+
"reason",
|
|
120
|
+
"notes",
|
|
121
|
+
"note",
|
|
122
|
+
"comment",
|
|
123
|
+
"comments",
|
|
124
|
+
"body",
|
|
125
|
+
"content",
|
|
126
|
+
"details",
|
|
127
|
+
),
|
|
128
|
+
description="Column names whose prose values (see text_hint_min_words) are `text`.",
|
|
129
|
+
)
|
|
130
|
+
text_hint_min_words: float = Field(
|
|
131
|
+
default=3.0, ge=0, description="Average words per value needed by text_name_hints."
|
|
132
|
+
)
|
|
133
|
+
string_length_buckets: tuple[int, ...] = Field(
|
|
134
|
+
default=(16, 32, 64, 128, 256),
|
|
135
|
+
description="max_length of strings is the longest value rounded up to a bucket.",
|
|
136
|
+
)
|
|
137
|
+
propose_max_length: bool = Field(
|
|
138
|
+
default=True, description="Propose a max_length constraint on strings."
|
|
139
|
+
)
|
|
140
|
+
propose_patterns: bool = Field(
|
|
141
|
+
default=True,
|
|
142
|
+
description="Propose a pattern for code-like strings sharing one shape (S-1001, PL).",
|
|
143
|
+
)
|
|
144
|
+
pattern_min_values: int = Field(
|
|
145
|
+
default=5, ge=1, description="Distinct values needed before a pattern is proposed."
|
|
146
|
+
)
|
|
147
|
+
propose_non_negative_min: bool = Field(
|
|
148
|
+
default=True, description="Propose min 0 on numbers that are never negative."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Nullability and uniqueness -----------------------------------------------------------
|
|
152
|
+
required_max_null_ratio: float = Field(
|
|
153
|
+
default=0.0,
|
|
154
|
+
ge=0,
|
|
155
|
+
lt=1,
|
|
156
|
+
description="Columns with at most this share of nulls are required.",
|
|
157
|
+
)
|
|
158
|
+
unique_min_values: int = Field(
|
|
159
|
+
default=2, ge=1, description="Non-null values needed before uniqueness counts."
|
|
160
|
+
)
|
|
161
|
+
unique_name_patterns: tuple[str, ...] = Field(
|
|
162
|
+
default=("*_number", "*_code", "*_id", "*_key", "email", "*_email", "vat*", "iban"),
|
|
163
|
+
description="Unique non-key columns flagged `unique` only when named like this.",
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# Keys ----------------------------------------------------------------------------------
|
|
167
|
+
key_name_patterns: tuple[str, ...] = Field(
|
|
168
|
+
default=(
|
|
169
|
+
"id",
|
|
170
|
+
"key",
|
|
171
|
+
"code",
|
|
172
|
+
"{table}_id",
|
|
173
|
+
"{table}_key",
|
|
174
|
+
"{table}_code",
|
|
175
|
+
"{table}_number",
|
|
176
|
+
"{table}_no",
|
|
177
|
+
"*_id",
|
|
178
|
+
"*_key",
|
|
179
|
+
"*_code",
|
|
180
|
+
"*_number",
|
|
181
|
+
"*_no",
|
|
182
|
+
),
|
|
183
|
+
description="Ranked names of primary key candidates (unique, non-null columns).",
|
|
184
|
+
)
|
|
185
|
+
allow_unnamed_primary_key: bool = Field(
|
|
186
|
+
default=True,
|
|
187
|
+
description="When no candidate matches key_name_patterns, the first column may still "
|
|
188
|
+
"be the key if it is unique and non-null.",
|
|
189
|
+
)
|
|
190
|
+
composite_key_max_columns: int = Field(
|
|
191
|
+
default=2,
|
|
192
|
+
ge=1,
|
|
193
|
+
le=3,
|
|
194
|
+
description="Largest composite key tried (key-named and foreign key-like columns only).",
|
|
195
|
+
)
|
|
196
|
+
minted_key_columns: tuple[str, ...] = Field(
|
|
197
|
+
default=("key",),
|
|
198
|
+
description="A key column with one of these names holding generated ids (ULID, UUID) "
|
|
199
|
+
"carries Record keys: the Kind gets minted keys and no key Attribute.",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# Foreign keys and Relations ------------------------------------------------------------
|
|
203
|
+
min_inclusion_ratio: float = Field(
|
|
204
|
+
default=0.95,
|
|
205
|
+
gt=0,
|
|
206
|
+
le=1,
|
|
207
|
+
description="Share of a column's distinct values found in another table's key for a "
|
|
208
|
+
"foreign key (and a Relation) to be proposed.",
|
|
209
|
+
)
|
|
210
|
+
foreign_key_suffixes: tuple[str, ...] = Field(
|
|
211
|
+
default=("_id", "_key", "_code", "_number", "_no", "_ref", "_fk"),
|
|
212
|
+
description="Suffixes marking reference columns; stripped to name the Relation.",
|
|
213
|
+
)
|
|
214
|
+
integer_reference_needs_name: bool = Field(
|
|
215
|
+
default=True,
|
|
216
|
+
description="Integer columns reference another table only when their name points at "
|
|
217
|
+
"it (small integers are included in most integer keys).",
|
|
218
|
+
)
|
|
219
|
+
weak_name_factor: float = Field(
|
|
220
|
+
default=0.9,
|
|
221
|
+
ge=0,
|
|
222
|
+
le=1,
|
|
223
|
+
description="Confidence factor of a reference whose name has a reference suffix but "
|
|
224
|
+
"does not name the target.",
|
|
225
|
+
)
|
|
226
|
+
no_name_factor: float = Field(
|
|
227
|
+
default=0.75,
|
|
228
|
+
ge=0,
|
|
229
|
+
le=1,
|
|
230
|
+
description="Confidence factor of a reference whose name gives no hint at all.",
|
|
231
|
+
)
|
|
232
|
+
detect_junction_tables: bool = Field(
|
|
233
|
+
default=True,
|
|
234
|
+
description="Tables made of two foreign keys (plus link attributes) become Relations.",
|
|
235
|
+
)
|
|
236
|
+
junction_max_attributes: int = Field(
|
|
237
|
+
default=8, ge=0, description="Link attributes a junction table may carry."
|
|
238
|
+
)
|
|
239
|
+
one_to_one_needs_declared_unique: bool = Field(
|
|
240
|
+
default=True,
|
|
241
|
+
description="Uniqueness seen in data never makes a Relation one_to_one; a declared "
|
|
242
|
+
"unique constraint (or a key reference) does.",
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
# Names ---------------------------------------------------------------------------------
|
|
246
|
+
singular_kind_names: bool = Field(
|
|
247
|
+
default=True, description="Kind names are the singular of table names (suppliers)."
|
|
248
|
+
)
|
|
249
|
+
strip_table_prefixes: tuple[str, ...] = Field(
|
|
250
|
+
default=(), description="Prefixes removed from table names, e.g. tbl_."
|
|
251
|
+
)
|
|
252
|
+
title_names: tuple[str, ...] = Field(
|
|
253
|
+
default=("name", "title", "label", "display_name", "*_name", "*_title", "*_label"),
|
|
254
|
+
description="Ranked names of the title Attribute.",
|
|
255
|
+
)
|
|
256
|
+
propose_search: bool = Field(
|
|
257
|
+
default=True,
|
|
258
|
+
description="Make the title Attribute searchable (weight A) and text Attributes "
|
|
259
|
+
"searchable (weight B).",
|
|
260
|
+
)
|
|
261
|
+
reserved_attribute_names: tuple[str, ...] = Field(
|
|
262
|
+
default=tuple(sorted(DEFAULT_RESERVED_ATTRIBUTE_NAMES)),
|
|
263
|
+
description="Attribute names reserved by Lorebase; colliding columns are renamed "
|
|
264
|
+
"<kind>_<column>.",
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
# Ids and status ------------------------------------------------------------------------
|
|
268
|
+
element_status: ElementStatus = Field(
|
|
269
|
+
default=ElementStatus.DRAFT,
|
|
270
|
+
description="Status of proposed Kinds and Relations; draft elements need no "
|
|
271
|
+
"descriptions until a reviewer activates them.",
|
|
272
|
+
)
|
|
273
|
+
id_seed: str = Field(
|
|
274
|
+
default="lorebase.infer",
|
|
275
|
+
description="Seed of the deterministic ids: the same input and seed give the same ids.",
|
|
276
|
+
)
|
|
277
|
+
id_timestamp_ms: int = Field(
|
|
278
|
+
default=0, ge=0, lt=2**48, description="Timestamp part of the deterministic ULIDs."
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
@model_validator(mode="after")
|
|
282
|
+
def _check(self) -> Self:
|
|
283
|
+
if not self.string_length_buckets or list(self.string_length_buckets) != sorted(
|
|
284
|
+
set(self.string_length_buckets)
|
|
285
|
+
):
|
|
286
|
+
raise ValueError("string_length_buckets must be strictly increasing and non-empty")
|
|
287
|
+
overlap = {t.lower() for t in self.true_tokens} & {t.lower() for t in self.false_tokens}
|
|
288
|
+
if overlap:
|
|
289
|
+
raise ValueError(f"tokens both true and false: {sorted(overlap)}")
|
|
290
|
+
return self
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Input of schema inference: what a profiler saw of each table.
|
|
2
|
+
|
|
3
|
+
A profiler (CSV reader, database catalog reader) turns a structured source into
|
|
4
|
+
``TableProfile`` objects; inference never reads files or connects to anything. Values are
|
|
5
|
+
Python natives (``str`` from CSV; ``int``, ``Decimal``, ``date``, ``bool``... from databases);
|
|
6
|
+
``None`` is null. A profile may also carry what a catalog declares (types, nullability, keys,
|
|
7
|
+
foreign keys, comments): declared facts win over facts seen in values.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from lorebase_core.types import TypeSpec
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class DeclaredForeignKey:
|
|
20
|
+
"""A foreign key constraint of the source catalog."""
|
|
21
|
+
|
|
22
|
+
columns: tuple[str, ...]
|
|
23
|
+
target_table: str
|
|
24
|
+
target_columns: tuple[str, ...]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class ColumnProfile:
|
|
29
|
+
"""One column: its values (sampled or complete) and what a catalog declares about it.
|
|
30
|
+
|
|
31
|
+
``values`` may be any iterable, including a generator over a complete column; inference
|
|
32
|
+
reads at most ``InferenceOptions.sample_size`` of them.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
name: str
|
|
36
|
+
values: Iterable[object] = ()
|
|
37
|
+
declared_type: TypeSpec | None = None
|
|
38
|
+
declared_max_length: int | None = None
|
|
39
|
+
nullable: bool | None = None
|
|
40
|
+
unique: bool | None = None
|
|
41
|
+
comment: str | None = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class TableProfile:
|
|
46
|
+
"""One table. ``row_count`` is the size of the whole table when the values are a sample;
|
|
47
|
+
``None`` means the values are the whole table."""
|
|
48
|
+
|
|
49
|
+
name: str
|
|
50
|
+
columns: tuple[ColumnProfile, ...]
|
|
51
|
+
row_count: int | None = None
|
|
52
|
+
primary_key: tuple[str, ...] | None = None
|
|
53
|
+
foreign_keys: tuple[DeclaredForeignKey, ...] = field(default=())
|
|
54
|
+
comment: str | None = None
|