etlplus 0.6.1__py3-none-any.whl → 0.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/database/__init__.py +20 -1
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +146 -0
- etlplus/database/orm.py +347 -0
- etlplus/database/schema.py +273 -0
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/METADATA +4 -1
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/RECORD +11 -8
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/WHEEL +0 -0
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.6.1.dist-info → etlplus-0.7.1.dist-info}/top_level.txt +0 -0
etlplus/database/__init__.py
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
"""
|
|
2
2
|
:mod:`etlplus.database` package.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Database utilities for:
|
|
5
5
|
- DDL rendering and schema management.
|
|
6
|
+
- Schema parsing from configuration files.
|
|
7
|
+
- Dynamic ORM generation.
|
|
8
|
+
- Database engine/session management.
|
|
6
9
|
"""
|
|
7
10
|
|
|
8
11
|
from __future__ import annotations
|
|
@@ -11,13 +14,29 @@ from .ddl import load_table_spec
|
|
|
11
14
|
from .ddl import render_table_sql
|
|
12
15
|
from .ddl import render_tables
|
|
13
16
|
from .ddl import render_tables_to_string
|
|
17
|
+
from .engine import engine
|
|
18
|
+
from .engine import load_database_url_from_config
|
|
19
|
+
from .engine import make_engine
|
|
20
|
+
from .engine import session
|
|
21
|
+
from .orm import build_models
|
|
22
|
+
from .orm import load_and_build_models
|
|
23
|
+
from .schema import load_table_specs
|
|
14
24
|
|
|
15
25
|
# SECTION: EXPORTS ========================================================== #
|
|
16
26
|
|
|
17
27
|
|
|
18
28
|
__all__ = [
|
|
29
|
+
# Functions
|
|
30
|
+
'build_models',
|
|
31
|
+
'load_and_build_models',
|
|
32
|
+
'load_database_url_from_config',
|
|
19
33
|
'load_table_spec',
|
|
34
|
+
'load_table_specs',
|
|
35
|
+
'make_engine',
|
|
20
36
|
'render_table_sql',
|
|
21
37
|
'render_tables',
|
|
22
38
|
'render_tables_to_string',
|
|
39
|
+
# Singletons
|
|
40
|
+
'engine',
|
|
41
|
+
'session',
|
|
23
42
|
]
|
etlplus/database/ddl.py
CHANGED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database.engine` module.
|
|
3
|
+
|
|
4
|
+
Lightweight engine/session factory with optional config-driven URL loading.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from sqlalchemy import create_engine
|
|
15
|
+
from sqlalchemy.engine import Engine
|
|
16
|
+
from sqlalchemy.orm import sessionmaker
|
|
17
|
+
|
|
18
|
+
from ..file import File
|
|
19
|
+
|
|
20
|
+
# SECTION: EXPORTS ========================================================== #
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
# Functions
|
|
25
|
+
'load_database_url_from_config',
|
|
26
|
+
'make_engine',
|
|
27
|
+
# Singletons
|
|
28
|
+
'engine',
|
|
29
|
+
'session',
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
DATABASE_URL: str = (
|
|
37
|
+
os.getenv('DATABASE_URL')
|
|
38
|
+
or os.getenv('DATABASE_DSN')
|
|
39
|
+
or 'sqlite+pysqlite:///:memory:'
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _resolve_url_from_mapping(cfg: Mapping[str, Any]) -> str | None:
|
|
47
|
+
"""
|
|
48
|
+
Return a URL/DSN from a mapping if present.
|
|
49
|
+
|
|
50
|
+
Parameters
|
|
51
|
+
----------
|
|
52
|
+
cfg : Mapping[str, Any]
|
|
53
|
+
Configuration mapping potentially containing connection fields.
|
|
54
|
+
|
|
55
|
+
Returns
|
|
56
|
+
-------
|
|
57
|
+
str | None
|
|
58
|
+
Resolved URL/DSN string, if present.
|
|
59
|
+
"""
|
|
60
|
+
conn = cfg.get('connection_string') or cfg.get('url') or cfg.get('dsn')
|
|
61
|
+
if isinstance(conn, str) and conn.strip():
|
|
62
|
+
return conn.strip()
|
|
63
|
+
|
|
64
|
+
# Some configs nest defaults.
|
|
65
|
+
# E.g., databases: { mssql: { default: {...} } }
|
|
66
|
+
default_cfg = cfg.get('default')
|
|
67
|
+
if isinstance(default_cfg, Mapping):
|
|
68
|
+
return _resolve_url_from_mapping(default_cfg)
|
|
69
|
+
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def load_database_url_from_config(
|
|
77
|
+
path: str | Path,
|
|
78
|
+
*,
|
|
79
|
+
name: str | None = None,
|
|
80
|
+
) -> str:
|
|
81
|
+
"""
|
|
82
|
+
Extract a database URL/DSN from a YAML/JSON config file.
|
|
83
|
+
|
|
84
|
+
The loader is schema-tolerant: it looks for a top-level "databases" map
|
|
85
|
+
and then for a named entry (``name``). Each entry may contain either a
|
|
86
|
+
``connection_string``/``url``/``dsn`` or a nested ``default`` block with
|
|
87
|
+
those fields.
|
|
88
|
+
|
|
89
|
+
Parameters
|
|
90
|
+
----------
|
|
91
|
+
path : str | Path
|
|
92
|
+
Location of the configuration file.
|
|
93
|
+
name : str | None, optional
|
|
94
|
+
Named database entry under the ``databases`` map (default:
|
|
95
|
+
``default``).
|
|
96
|
+
|
|
97
|
+
Returns
|
|
98
|
+
-------
|
|
99
|
+
str
|
|
100
|
+
Resolved database URL/DSN string.
|
|
101
|
+
|
|
102
|
+
Raises
|
|
103
|
+
------
|
|
104
|
+
KeyError
|
|
105
|
+
If the specified database entry is not found.
|
|
106
|
+
TypeError
|
|
107
|
+
If the config structure is invalid.
|
|
108
|
+
ValueError
|
|
109
|
+
If no connection string/URL/DSN is found for the specified entry.
|
|
110
|
+
"""
|
|
111
|
+
cfg = File.read_file(Path(path))
|
|
112
|
+
if not isinstance(cfg, Mapping):
|
|
113
|
+
raise TypeError('Database config must be a mapping')
|
|
114
|
+
|
|
115
|
+
databases = cfg.get('databases') if isinstance(cfg, Mapping) else None
|
|
116
|
+
if not isinstance(databases, Mapping):
|
|
117
|
+
raise KeyError('Config missing top-level "databases" mapping')
|
|
118
|
+
|
|
119
|
+
target = name or 'default'
|
|
120
|
+
entry = databases.get(target)
|
|
121
|
+
if entry is None:
|
|
122
|
+
raise KeyError(f'Database entry "{target}" not found in config')
|
|
123
|
+
if not isinstance(entry, Mapping):
|
|
124
|
+
raise TypeError(f'Database entry "{target}" must be a mapping')
|
|
125
|
+
|
|
126
|
+
url = _resolve_url_from_mapping(entry)
|
|
127
|
+
if not url:
|
|
128
|
+
raise ValueError(
|
|
129
|
+
f'Database entry "{target}" lacks connection_string/url/dsn',
|
|
130
|
+
)
|
|
131
|
+
return url
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
|
|
135
|
+
"""Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
|
|
136
|
+
|
|
137
|
+
resolved_url = url or DATABASE_URL
|
|
138
|
+
return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# SECTION: SINGLETONS ======================================================= #
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# Default engine/session for callers that rely on module-level singletons.
|
|
145
|
+
engine = make_engine()
|
|
146
|
+
session = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
etlplus/database/orm.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database.orm` module.
|
|
3
|
+
|
|
4
|
+
Dynamic SQLAlchemy model generation from YAML table specs.
|
|
5
|
+
|
|
6
|
+
Usage
|
|
7
|
+
-----
|
|
8
|
+
>>> from etlplus.database.orm import load_and_build_models
|
|
9
|
+
>>> registry = load_and_build_models('examples/configs/ddl_spec.yml')
|
|
10
|
+
>>> Player = registry['dbo.Customers']
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from sqlalchemy import Boolean
|
|
21
|
+
from sqlalchemy import CheckConstraint
|
|
22
|
+
from sqlalchemy import Date
|
|
23
|
+
from sqlalchemy import DateTime
|
|
24
|
+
from sqlalchemy import Enum
|
|
25
|
+
from sqlalchemy import Float
|
|
26
|
+
from sqlalchemy import ForeignKey
|
|
27
|
+
from sqlalchemy import ForeignKeyConstraint
|
|
28
|
+
from sqlalchemy import Index
|
|
29
|
+
from sqlalchemy import Integer
|
|
30
|
+
from sqlalchemy import LargeBinary
|
|
31
|
+
from sqlalchemy import Numeric
|
|
32
|
+
from sqlalchemy import PrimaryKeyConstraint
|
|
33
|
+
from sqlalchemy import String
|
|
34
|
+
from sqlalchemy import Text
|
|
35
|
+
from sqlalchemy import Time
|
|
36
|
+
from sqlalchemy import UniqueConstraint
|
|
37
|
+
from sqlalchemy import text
|
|
38
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
|
39
|
+
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
|
|
40
|
+
from sqlalchemy.orm import DeclarativeBase
|
|
41
|
+
from sqlalchemy.orm import mapped_column
|
|
42
|
+
from sqlalchemy.types import TypeEngine
|
|
43
|
+
|
|
44
|
+
from .schema import ForeignKeySpec
|
|
45
|
+
from .schema import TableSpec
|
|
46
|
+
from .schema import load_table_specs
|
|
47
|
+
|
|
48
|
+
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
# Classes
|
|
52
|
+
'Base',
|
|
53
|
+
# Functions
|
|
54
|
+
'build_models',
|
|
55
|
+
'load_and_build_models',
|
|
56
|
+
'resolve_type',
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_TYPE_MAPPING: dict[str, Callable[[list[int]], TypeEngine]] = {
|
|
61
|
+
'int': lambda _: Integer(),
|
|
62
|
+
'integer': lambda _: Integer(),
|
|
63
|
+
'bigint': lambda _: Integer(),
|
|
64
|
+
'smallint': lambda _: Integer(),
|
|
65
|
+
'bool': lambda _: Boolean(),
|
|
66
|
+
'boolean': lambda _: Boolean(),
|
|
67
|
+
'uuid': lambda _: PG_UUID(as_uuid=True),
|
|
68
|
+
'uniqueidentifier': lambda _: PG_UUID(as_uuid=True),
|
|
69
|
+
'rowversion': lambda _: LargeBinary(),
|
|
70
|
+
'varbinary': lambda _: LargeBinary(),
|
|
71
|
+
'blob': lambda _: LargeBinary(),
|
|
72
|
+
'text': lambda _: Text(),
|
|
73
|
+
'string': lambda _: Text(),
|
|
74
|
+
'varchar': lambda p: String(length=p[0]) if p else String(),
|
|
75
|
+
'nvarchar': lambda p: String(length=p[0]) if p else String(),
|
|
76
|
+
'char': lambda p: String(length=p[0] if p else 1),
|
|
77
|
+
'nchar': lambda p: String(length=p[0] if p else 1),
|
|
78
|
+
'numeric': lambda p: Numeric(
|
|
79
|
+
precision=p[0] if p else None,
|
|
80
|
+
scale=p[1] if len(p) > 1 else None,
|
|
81
|
+
),
|
|
82
|
+
'decimal': lambda p: Numeric(
|
|
83
|
+
precision=p[0] if p else None,
|
|
84
|
+
scale=p[1] if len(p) > 1 else None,
|
|
85
|
+
),
|
|
86
|
+
'float': lambda _: Float(),
|
|
87
|
+
'real': lambda _: Float(),
|
|
88
|
+
'double': lambda _: Float(),
|
|
89
|
+
'datetime': lambda _: DateTime(timezone=True),
|
|
90
|
+
'datetime2': lambda _: DateTime(timezone=True),
|
|
91
|
+
'timestamp': lambda _: DateTime(timezone=True),
|
|
92
|
+
'date': lambda _: Date(),
|
|
93
|
+
'time': lambda _: Time(),
|
|
94
|
+
'json': lambda _: JSONB(),
|
|
95
|
+
'jsonb': lambda _: JSONB(),
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# SECTION: CLASSES ========================================================== #
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class Base(DeclarativeBase):
|
|
103
|
+
"""Base class for all ORM models."""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _class_name(
|
|
110
|
+
table: str,
|
|
111
|
+
) -> str:
|
|
112
|
+
"""
|
|
113
|
+
Convert table name to PascalCase class name.
|
|
114
|
+
|
|
115
|
+
Parameters
|
|
116
|
+
----------
|
|
117
|
+
table : str
|
|
118
|
+
Table name.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
str
|
|
123
|
+
PascalCase class name.
|
|
124
|
+
"""
|
|
125
|
+
parts = re.split(r'[^A-Za-z0-9]+', table)
|
|
126
|
+
return ''.join(p.capitalize() for p in parts if p)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _parse_type_decl(
|
|
130
|
+
type_str: str,
|
|
131
|
+
) -> tuple[str, list[int]]:
|
|
132
|
+
"""
|
|
133
|
+
Parse a type declaration string into its name and parameters.
|
|
134
|
+
|
|
135
|
+
Parameters
|
|
136
|
+
----------
|
|
137
|
+
type_str : str
|
|
138
|
+
Type declaration string, e.g., "varchar(255)".
|
|
139
|
+
|
|
140
|
+
Returns
|
|
141
|
+
-------
|
|
142
|
+
tuple[str, list[int]]
|
|
143
|
+
A tuple containing the type name and a list of integer parameters.
|
|
144
|
+
"""
|
|
145
|
+
m = re.match(
|
|
146
|
+
r'^(?P<name>[A-Za-z0-9_]+)(?:\((?P<params>[^)]*)\))?$',
|
|
147
|
+
type_str.strip(),
|
|
148
|
+
)
|
|
149
|
+
if not m:
|
|
150
|
+
return type_str.lower(), []
|
|
151
|
+
name = m.group('name').lower()
|
|
152
|
+
params_raw = m.group('params')
|
|
153
|
+
if not params_raw:
|
|
154
|
+
return name, []
|
|
155
|
+
params = [p.strip() for p in params_raw.split(',') if p.strip()]
|
|
156
|
+
parsed: list[int] = []
|
|
157
|
+
for p in params:
|
|
158
|
+
try:
|
|
159
|
+
parsed.append(int(p))
|
|
160
|
+
except ValueError:
|
|
161
|
+
continue
|
|
162
|
+
return name, parsed
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _table_kwargs(
|
|
166
|
+
spec: TableSpec,
|
|
167
|
+
) -> dict[str, str]:
|
|
168
|
+
"""
|
|
169
|
+
Generate table keyword arguments based on the table specification.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
spec : TableSpec
|
|
174
|
+
Table specification.
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
dict[str, str]
|
|
179
|
+
Dictionary of table keyword arguments.
|
|
180
|
+
"""
|
|
181
|
+
kwargs: dict[str, str] = {}
|
|
182
|
+
if spec.schema_name:
|
|
183
|
+
kwargs['schema'] = spec.schema_name
|
|
184
|
+
return kwargs
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def build_models(
|
|
191
|
+
specs: list[TableSpec],
|
|
192
|
+
*,
|
|
193
|
+
base: type[DeclarativeBase] = Base,
|
|
194
|
+
) -> dict[str, type[DeclarativeBase]]:
|
|
195
|
+
"""
|
|
196
|
+
Build SQLAlchemy ORM models from table specifications.
|
|
197
|
+
Parameters
|
|
198
|
+
----------
|
|
199
|
+
specs : list[TableSpec]
|
|
200
|
+
List of table specifications.
|
|
201
|
+
base : type[DeclarativeBase], optional
|
|
202
|
+
Base class for the ORM models (default: :class:`Base`).
|
|
203
|
+
Returns
|
|
204
|
+
-------
|
|
205
|
+
dict[str, type[DeclarativeBase]]
|
|
206
|
+
Registry mapping fully qualified table names to ORM model classes.
|
|
207
|
+
"""
|
|
208
|
+
registry: dict[str, type[DeclarativeBase]] = {}
|
|
209
|
+
|
|
210
|
+
for spec in specs:
|
|
211
|
+
table_args: list[object] = []
|
|
212
|
+
table_kwargs = _table_kwargs(spec)
|
|
213
|
+
pk_cols = set(spec.primary_key.columns) if spec.primary_key else set()
|
|
214
|
+
|
|
215
|
+
# Pre-handle multi-column constraints.
|
|
216
|
+
if spec.primary_key and len(spec.primary_key.columns) > 1:
|
|
217
|
+
table_args.append(
|
|
218
|
+
PrimaryKeyConstraint(
|
|
219
|
+
*spec.primary_key.columns,
|
|
220
|
+
name=spec.primary_key.name,
|
|
221
|
+
),
|
|
222
|
+
)
|
|
223
|
+
for uc in spec.unique_constraints:
|
|
224
|
+
table_args.append(UniqueConstraint(*uc.columns, name=uc.name))
|
|
225
|
+
for idx in spec.indexes:
|
|
226
|
+
table_args.append(
|
|
227
|
+
Index(
|
|
228
|
+
idx.name,
|
|
229
|
+
*idx.columns,
|
|
230
|
+
unique=idx.unique,
|
|
231
|
+
postgresql_where=text(idx.where) if idx.where else None,
|
|
232
|
+
),
|
|
233
|
+
)
|
|
234
|
+
composite_fks = [fk for fk in spec.foreign_keys if len(fk.columns) > 1]
|
|
235
|
+
for fk in composite_fks:
|
|
236
|
+
table_args.append(
|
|
237
|
+
ForeignKeyConstraint(
|
|
238
|
+
fk.columns,
|
|
239
|
+
[f'{fk.ref_table}.{c}' for c in fk.ref_columns],
|
|
240
|
+
ondelete=fk.ondelete,
|
|
241
|
+
),
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
fk_by_column = {
|
|
245
|
+
fk.columns[0]: fk
|
|
246
|
+
for fk in spec.foreign_keys
|
|
247
|
+
if len(fk.columns) == 1 and len(fk.ref_columns) == 1
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
attrs: dict[str, object] = {'__tablename__': spec.table}
|
|
251
|
+
|
|
252
|
+
for col in spec.columns:
|
|
253
|
+
col_fk: ForeignKeySpec | None = fk_by_column.get(col.name)
|
|
254
|
+
fk_arg = (
|
|
255
|
+
ForeignKey(
|
|
256
|
+
f'{col_fk.ref_table}.{col_fk.ref_columns[0]}',
|
|
257
|
+
ondelete=col_fk.ondelete,
|
|
258
|
+
)
|
|
259
|
+
if col_fk
|
|
260
|
+
else None
|
|
261
|
+
)
|
|
262
|
+
col_type: TypeEngine = (
|
|
263
|
+
Enum(*col.enum, name=f'{spec.table}_{col.name}_enum')
|
|
264
|
+
if col.enum
|
|
265
|
+
else resolve_type(col.type)
|
|
266
|
+
)
|
|
267
|
+
fk_args: list[ForeignKey] = []
|
|
268
|
+
if fk_arg:
|
|
269
|
+
fk_args.append(fk_arg)
|
|
270
|
+
|
|
271
|
+
kwargs: dict[str, Any] = {
|
|
272
|
+
'nullable': col.nullable,
|
|
273
|
+
'primary_key': col.name in pk_cols and len(pk_cols) == 1,
|
|
274
|
+
'unique': col.unique,
|
|
275
|
+
}
|
|
276
|
+
if col.default:
|
|
277
|
+
kwargs['server_default'] = text(col.default)
|
|
278
|
+
if col.identity:
|
|
279
|
+
kwargs['autoincrement'] = True
|
|
280
|
+
|
|
281
|
+
attrs[col.name] = mapped_column(*fk_args, type_=col_type, **kwargs)
|
|
282
|
+
|
|
283
|
+
if col.check:
|
|
284
|
+
table_args.append(
|
|
285
|
+
CheckConstraint(
|
|
286
|
+
col.check,
|
|
287
|
+
name=f'ck_{spec.table}_{col.name}',
|
|
288
|
+
),
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
if table_args or table_kwargs:
|
|
292
|
+
args_tuple = tuple(table_args)
|
|
293
|
+
attrs['__table_args__'] = (
|
|
294
|
+
(*args_tuple, table_kwargs) if table_kwargs else args_tuple
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
cls_name = _class_name(spec.table)
|
|
298
|
+
model_cls = type(cls_name, (base,), attrs)
|
|
299
|
+
registry[spec.fq_name] = model_cls
|
|
300
|
+
|
|
301
|
+
return registry
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def load_and_build_models(
|
|
305
|
+
path: str | Path,
|
|
306
|
+
*,
|
|
307
|
+
base: type[DeclarativeBase] = Base,
|
|
308
|
+
) -> dict[str, type[DeclarativeBase]]:
|
|
309
|
+
"""
|
|
310
|
+
Load table specifications from a file and build SQLAlchemy models.
|
|
311
|
+
|
|
312
|
+
Parameters
|
|
313
|
+
----------
|
|
314
|
+
path : str | Path
|
|
315
|
+
Path to the YAML file containing table specifications.
|
|
316
|
+
base : type[DeclarativeBase], optional
|
|
317
|
+
Base class for the ORM models (default: :class:`Base`).
|
|
318
|
+
|
|
319
|
+
Returns
|
|
320
|
+
-------
|
|
321
|
+
dict[str, type[DeclarativeBase]]
|
|
322
|
+
Registry mapping fully qualified table names to ORM model classes.
|
|
323
|
+
"""
|
|
324
|
+
return build_models(load_table_specs(path), base=base)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def resolve_type(
|
|
328
|
+
type_str: str,
|
|
329
|
+
) -> TypeEngine:
|
|
330
|
+
"""
|
|
331
|
+
Resolve a string type declaration to a SQLAlchemy :class:`TypeEngine`.
|
|
332
|
+
|
|
333
|
+
Parameters
|
|
334
|
+
----------
|
|
335
|
+
type_str : str
|
|
336
|
+
String representation of the type declaration.
|
|
337
|
+
|
|
338
|
+
Returns
|
|
339
|
+
-------
|
|
340
|
+
TypeEngine
|
|
341
|
+
SQLAlchemy type engine instance corresponding to the type declaration.
|
|
342
|
+
"""
|
|
343
|
+
name, params = _parse_type_decl(type_str)
|
|
344
|
+
factory = _TYPE_MAPPING.get(name)
|
|
345
|
+
if factory:
|
|
346
|
+
return factory(params)
|
|
347
|
+
return Text()
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database.schema` module.
|
|
3
|
+
|
|
4
|
+
Helpers for loading and translating YAML definitions of database table schema
|
|
5
|
+
specifications into Pydantic models for dynamic SQLAlchemy generation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
from typing import ClassVar
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel
|
|
15
|
+
from pydantic import ConfigDict
|
|
16
|
+
from pydantic import Field
|
|
17
|
+
|
|
18
|
+
from ..file import File
|
|
19
|
+
|
|
20
|
+
# SECTION: EXPORTS ========================================================== #
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
# Classes
|
|
25
|
+
'ColumnSpec',
|
|
26
|
+
'ForeignKeySpec',
|
|
27
|
+
'IdentitySpec',
|
|
28
|
+
'IndexSpec',
|
|
29
|
+
'PrimaryKeySpec',
|
|
30
|
+
'UniqueConstraintSpec',
|
|
31
|
+
'TableSpec',
|
|
32
|
+
# Functions
|
|
33
|
+
'load_table_specs',
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# SECTION: CLASSES ========================================================== #
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ColumnSpec(BaseModel):
|
|
41
|
+
"""
|
|
42
|
+
Column specification suitable for ODBC / SQLite DDL.
|
|
43
|
+
|
|
44
|
+
Attributes
|
|
45
|
+
----------
|
|
46
|
+
model_config : ClassVar[ConfigDict]
|
|
47
|
+
Pydantic model configuration.
|
|
48
|
+
name : str
|
|
49
|
+
Unquoted column name.
|
|
50
|
+
type : str
|
|
51
|
+
SQL type string, e.g., INT, NVARCHAR(100).
|
|
52
|
+
nullable : bool
|
|
53
|
+
True if NULL values are allowed.
|
|
54
|
+
default : str | None
|
|
55
|
+
Default value expression, or None if no default.
|
|
56
|
+
identity : IdentitySpec | None
|
|
57
|
+
Identity specification, or None if not an identity column.
|
|
58
|
+
check : str | None
|
|
59
|
+
Check constraint expression, or None if no check constraint.
|
|
60
|
+
enum : list[str] | None
|
|
61
|
+
List of allowed string values for enum-like columns, or None.
|
|
62
|
+
unique : bool
|
|
63
|
+
True if the column has a UNIQUE constraint.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
67
|
+
|
|
68
|
+
name: str
|
|
69
|
+
type: str = Field(description='SQL type string, e.g., INT, NVARCHAR(100)')
|
|
70
|
+
nullable: bool = True
|
|
71
|
+
default: str | None = None
|
|
72
|
+
identity: IdentitySpec | None = None
|
|
73
|
+
check: str | None = None
|
|
74
|
+
enum: list[str] | None = None
|
|
75
|
+
unique: bool = False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class ForeignKeySpec(BaseModel):
|
|
79
|
+
"""
|
|
80
|
+
Foreign key specification.
|
|
81
|
+
|
|
82
|
+
Attributes
|
|
83
|
+
----------
|
|
84
|
+
model_config : ClassVar[ConfigDict]
|
|
85
|
+
Pydantic model configuration.
|
|
86
|
+
columns : list[str]
|
|
87
|
+
List of local column names.
|
|
88
|
+
ref_table : str
|
|
89
|
+
Referenced table name.
|
|
90
|
+
ref_columns : list[str]
|
|
91
|
+
List of referenced column names.
|
|
92
|
+
ondelete : str | None
|
|
93
|
+
ON DELETE action, or None.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
97
|
+
|
|
98
|
+
columns: list[str]
|
|
99
|
+
ref_table: str
|
|
100
|
+
ref_columns: list[str]
|
|
101
|
+
ondelete: str | None = None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class IdentitySpec(BaseModel):
|
|
105
|
+
"""
|
|
106
|
+
Identity specification.
|
|
107
|
+
|
|
108
|
+
Attributes
|
|
109
|
+
----------
|
|
110
|
+
model_config : ClassVar[ConfigDict]
|
|
111
|
+
Pydantic model configuration.
|
|
112
|
+
seed : int | None
|
|
113
|
+
Identity seed value (default: 1).
|
|
114
|
+
increment : int | None
|
|
115
|
+
Identity increment value (default: 1).
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
119
|
+
|
|
120
|
+
seed: int | None = Field(default=None, ge=1)
|
|
121
|
+
increment: int | None = Field(default=None, ge=1)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class IndexSpec(BaseModel):
|
|
125
|
+
"""
|
|
126
|
+
Index specification.
|
|
127
|
+
|
|
128
|
+
Attributes
|
|
129
|
+
----------
|
|
130
|
+
model_config : ClassVar[ConfigDict]
|
|
131
|
+
Pydantic model configuration.
|
|
132
|
+
name : str
|
|
133
|
+
Index name.
|
|
134
|
+
columns : list[str]
|
|
135
|
+
List of column names included in the index.
|
|
136
|
+
unique : bool
|
|
137
|
+
True if the index is unique.
|
|
138
|
+
where : str | None
|
|
139
|
+
Optional WHERE clause for filtered indexes.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
143
|
+
|
|
144
|
+
name: str
|
|
145
|
+
columns: list[str]
|
|
146
|
+
unique: bool = False
|
|
147
|
+
where: str | None = None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class PrimaryKeySpec(BaseModel):
|
|
151
|
+
"""
|
|
152
|
+
Primary key specification.
|
|
153
|
+
|
|
154
|
+
Attributes
|
|
155
|
+
----------
|
|
156
|
+
model_config : ClassVar[ConfigDict]
|
|
157
|
+
Pydantic model configuration.
|
|
158
|
+
name : str | None
|
|
159
|
+
Primary key constraint name, or None if unnamed.
|
|
160
|
+
columns : list[str]
|
|
161
|
+
List of column names included in the primary key.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
165
|
+
|
|
166
|
+
name: str | None = None
|
|
167
|
+
columns: list[str]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class UniqueConstraintSpec(BaseModel):
|
|
171
|
+
"""
|
|
172
|
+
Unique constraint specification.
|
|
173
|
+
|
|
174
|
+
Attributes
|
|
175
|
+
----------
|
|
176
|
+
model_config : ClassVar[ConfigDict]
|
|
177
|
+
Pydantic model configuration.
|
|
178
|
+
name : str | None
|
|
179
|
+
Unique constraint name, or None if unnamed.
|
|
180
|
+
columns : list[str]
|
|
181
|
+
List of column names included in the unique constraint.
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
185
|
+
|
|
186
|
+
name: str | None = None
|
|
187
|
+
columns: list[str]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class TableSpec(BaseModel):
|
|
191
|
+
"""
|
|
192
|
+
Table specification.
|
|
193
|
+
|
|
194
|
+
Attributes
|
|
195
|
+
----------
|
|
196
|
+
model_config : ClassVar[ConfigDict]
|
|
197
|
+
Pydantic model configuration.
|
|
198
|
+
table : str
|
|
199
|
+
Table name.
|
|
200
|
+
schema_name : str | None
|
|
201
|
+
Schema name, or None if not specified.
|
|
202
|
+
create_schema : bool
|
|
203
|
+
Whether to create the schema if it does not exist.
|
|
204
|
+
columns : list[ColumnSpec]
|
|
205
|
+
List of column specifications.
|
|
206
|
+
primary_key : PrimaryKeySpec | None
|
|
207
|
+
Primary key specification, or None if no primary key.
|
|
208
|
+
unique_constraints : list[UniqueConstraintSpec]
|
|
209
|
+
List of unique constraint specifications.
|
|
210
|
+
indexes : list[IndexSpec]
|
|
211
|
+
List of index specifications.
|
|
212
|
+
foreign_keys : list[ForeignKeySpec]
|
|
213
|
+
List of foreign key specifications.
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
|
|
217
|
+
|
|
218
|
+
table: str = Field(alias='name')
|
|
219
|
+
schema_name: str | None = Field(default=None, alias='schema')
|
|
220
|
+
create_schema: bool = False
|
|
221
|
+
columns: list[ColumnSpec]
|
|
222
|
+
primary_key: PrimaryKeySpec | None = None
|
|
223
|
+
unique_constraints: list[UniqueConstraintSpec] = Field(
|
|
224
|
+
default_factory=list,
|
|
225
|
+
)
|
|
226
|
+
indexes: list[IndexSpec] = Field(default_factory=list)
|
|
227
|
+
foreign_keys: list[ForeignKeySpec] = Field(default_factory=list)
|
|
228
|
+
|
|
229
|
+
# -- Properties -- #
|
|
230
|
+
|
|
231
|
+
@property
|
|
232
|
+
def fq_name(self) -> str:
|
|
233
|
+
"""
|
|
234
|
+
Fully qualified table name, including schema if specified.
|
|
235
|
+
"""
|
|
236
|
+
return (
|
|
237
|
+
f'{self.schema_name}.{self.table}'
|
|
238
|
+
if self.schema_name
|
|
239
|
+
else self.table
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def load_table_specs(
|
|
247
|
+
path: str | Path,
|
|
248
|
+
) -> list[TableSpec]:
|
|
249
|
+
"""
|
|
250
|
+
Load table specifications from a YAML file.
|
|
251
|
+
|
|
252
|
+
Parameters
|
|
253
|
+
----------
|
|
254
|
+
path : str | Path
|
|
255
|
+
Path to the YAML file containing table specifications.
|
|
256
|
+
|
|
257
|
+
Returns
|
|
258
|
+
-------
|
|
259
|
+
list[TableSpec]
|
|
260
|
+
A list of TableSpec instances parsed from the YAML file.
|
|
261
|
+
"""
|
|
262
|
+
data = File.read_file(Path(path))
|
|
263
|
+
if not data:
|
|
264
|
+
return []
|
|
265
|
+
|
|
266
|
+
if isinstance(data, dict) and 'table_schemas' in data:
|
|
267
|
+
items: list[Any] = data['table_schemas'] or []
|
|
268
|
+
elif isinstance(data, list):
|
|
269
|
+
items = data
|
|
270
|
+
else:
|
|
271
|
+
items = [data]
|
|
272
|
+
|
|
273
|
+
return [TableSpec.model_validate(item) for item in items]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -21,7 +21,10 @@ Requires-Dist: jinja2>=3.1.6
|
|
|
21
21
|
Requires-Dist: pyodbc>=5.3.0
|
|
22
22
|
Requires-Dist: python-dotenv>=1.2.1
|
|
23
23
|
Requires-Dist: pandas>=2.3.3
|
|
24
|
+
Requires-Dist: pydantic>=2.12.5
|
|
25
|
+
Requires-Dist: PyYAML>=6.0.3
|
|
24
26
|
Requires-Dist: requests>=2.32.5
|
|
27
|
+
Requires-Dist: SQLAlchemy>=2.0.45
|
|
25
28
|
Requires-Dist: typer>=0.21.0
|
|
26
29
|
Provides-Extra: dev
|
|
27
30
|
Requires-Dist: black>=25.9.0; extra == "dev"
|
|
@@ -41,16 +41,19 @@ etlplus/config/pipeline.py,sha256=Va4MQY6KEyKqHGMKPmh09ZcGpx95br-iNUjpkqtzVbw,95
|
|
|
41
41
|
etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
|
|
42
42
|
etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
43
43
|
etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
|
|
44
|
-
etlplus/database/__init__.py,sha256=
|
|
45
|
-
etlplus/database/ddl.py,sha256=
|
|
44
|
+
etlplus/database/__init__.py,sha256=0gWnMlQiVHS6SVUxIT9zklQUHU36y-2RF_gN1cx7icg,1018
|
|
45
|
+
etlplus/database/ddl.py,sha256=lIar9KIOoBRslp_P0DnpoMDXzkjt64J5-iVV7CeSV_M,7747
|
|
46
|
+
etlplus/database/engine.py,sha256=54f-XtNKIuogJhsLV9cX_xPoBwcl_HNJTL5HqMCi8kw,3986
|
|
47
|
+
etlplus/database/orm.py,sha256=StjeguokM70oNKq7mNXLyc4_mYUZR-EKW3oGRlsd8QE,9962
|
|
48
|
+
etlplus/database/schema.py,sha256=BmRP2wwX2xex1phLm0tnHrP6A2AQgguA-hSLnK0xwwc,7003
|
|
46
49
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
47
50
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
48
51
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
49
52
|
etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
|
|
50
53
|
etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
|
|
51
|
-
etlplus-0.
|
|
52
|
-
etlplus-0.
|
|
53
|
-
etlplus-0.
|
|
54
|
-
etlplus-0.
|
|
55
|
-
etlplus-0.
|
|
56
|
-
etlplus-0.
|
|
54
|
+
etlplus-0.7.1.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
55
|
+
etlplus-0.7.1.dist-info/METADATA,sha256=Y2MYm3-8rGosgvY29er_Pt_j9qxgBVndnRXK3k0xUBM,19383
|
|
56
|
+
etlplus-0.7.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
57
|
+
etlplus-0.7.1.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
58
|
+
etlplus-0.7.1.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
59
|
+
etlplus-0.7.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|