etlplus 0.7.1__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.7.1/etlplus.egg-info → etlplus-0.7.2}/PKG-INFO +1 -1
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/database/__init__.py +2 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/database/ddl.py +37 -29
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/database/engine.py +10 -5
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/database/orm.py +18 -11
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/database/schema.py +3 -2
- etlplus-0.7.2/etlplus/database/types.py +38 -0
- {etlplus-0.7.1 → etlplus-0.7.2/etlplus.egg-info}/PKG-INFO +1 -1
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus.egg-info/SOURCES.txt +1 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.coveragerc +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.editorconfig +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.gitattributes +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.github/workflows/ci.yml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.gitignore +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.pre-commit-config.yaml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/.ruff.toml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/CONTRIBUTING.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/DEMO.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/LICENSE +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/MANIFEST.in +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/Makefile +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/README.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/REFERENCES.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/docs/pipeline-guide.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/__main__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/__version__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/README.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/auth.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/endpoint_client.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/errors.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/pagination/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/pagination/client.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/pagination/config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/rate_limiting/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/rate_limiting/config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/rate_limiting/rate_limiter.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/request_manager.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/retry_manager.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/transport.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/api/types.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/cli/app.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/cli/handlers.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/cli/main.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/connector.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/jobs.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/pipeline.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/profile.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/types.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/config/utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/enums.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/extract.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/file.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/load.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/mixins.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/py.typed +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/run.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/run_helpers.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/templates/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/templates/ddl.sql.j2 +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/templates/view.sql.j2 +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/transform.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/types.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/validate.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/validation/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus/validation/utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/README.md +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/configs/ddl_spec.yml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/configs/pipeline.yml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/data/sample.csv +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/data/sample.json +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/data/sample.xml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/data/sample.xsd +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/data/sample.yaml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/examples/quickstart_python.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/pyproject.toml +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/pytest.ini +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/setup.cfg +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/setup.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/__init__.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/conftest.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/conftest.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_cli.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_examples_data_parity.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_pagination_strategy.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_pipeline_smoke.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_pipeline_yaml_load.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_run.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_run_profile_pagination_defaults.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/integration/test_i_run_profile_rate_limit_defaults.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/conftest.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_auth.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_endpoint_client.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_mocks.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_pagination_client.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_pagination_config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_paginator.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_rate_limit_config.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_rate_limiter.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_request_manager.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_retry_manager.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_transport.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/api/test_u_types.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/cli/conftest.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/cli/test_u_cli_app.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/cli/test_u_cli_handlers.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/cli/test_u_cli_main.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/config/test_u_config_utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/config/test_u_connector.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/config/test_u_jobs.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/config/test_u_pipeline.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/conftest.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/database/test_u_database_ddl.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/database/test_u_database_engine.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/database/test_u_database_orm.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/database/test_u_database_schema.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_enums.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_extract.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_file.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_load.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_main.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_mixins.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_run.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_run_helpers.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_transform.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_validate.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/test_u_version.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tests/unit/validation/test_u_validation_utils.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tools/run_pipeline.py +0 -0
- {etlplus-0.7.1 → etlplus-0.7.2}/tools/update_demo_snippets.py +0 -0
|
@@ -18,6 +18,7 @@ from .engine import engine
|
|
|
18
18
|
from .engine import load_database_url_from_config
|
|
19
19
|
from .engine import make_engine
|
|
20
20
|
from .engine import session
|
|
21
|
+
from .orm import Base
|
|
21
22
|
from .orm import build_models
|
|
22
23
|
from .orm import load_and_build_models
|
|
23
24
|
from .schema import load_table_specs
|
|
@@ -36,6 +37,7 @@ __all__ = [
|
|
|
36
37
|
'render_table_sql',
|
|
37
38
|
'render_tables',
|
|
38
39
|
'render_tables_to_string',
|
|
40
|
+
'Base',
|
|
39
41
|
# Singletons
|
|
40
42
|
'engine',
|
|
41
43
|
'session',
|
|
@@ -15,7 +15,6 @@ import os
|
|
|
15
15
|
from collections.abc import Iterable
|
|
16
16
|
from collections.abc import Mapping
|
|
17
17
|
from pathlib import Path
|
|
18
|
-
from typing import Any
|
|
19
18
|
from typing import Final
|
|
20
19
|
|
|
21
20
|
from jinja2 import DictLoader
|
|
@@ -24,6 +23,9 @@ from jinja2 import FileSystemLoader
|
|
|
24
23
|
from jinja2 import StrictUndefined
|
|
25
24
|
|
|
26
25
|
from ..file import File
|
|
26
|
+
from ..types import StrAnyMap
|
|
27
|
+
from ..types import StrPath
|
|
28
|
+
from .types import TemplateKey
|
|
27
29
|
|
|
28
30
|
# SECTION: EXPORTS ========================================================== #
|
|
29
31
|
|
|
@@ -52,7 +54,7 @@ _SUPPORTED_SPEC_SUFFIXES: Final[frozenset[str]] = frozenset(
|
|
|
52
54
|
# SECTION: CONSTANTS ======================================================== #
|
|
53
55
|
|
|
54
56
|
|
|
55
|
-
TEMPLATES: Final[dict[
|
|
57
|
+
TEMPLATES: Final[dict[TemplateKey, str]] = {
|
|
56
58
|
'ddl': 'ddl.sql.j2',
|
|
57
59
|
'view': 'view.sql.j2',
|
|
58
60
|
}
|
|
@@ -64,7 +66,8 @@ TEMPLATES: Final[dict[str, str]] = {
|
|
|
64
66
|
def _load_template_text(
|
|
65
67
|
filename: str,
|
|
66
68
|
) -> str:
|
|
67
|
-
"""
|
|
69
|
+
"""
|
|
70
|
+
Return the bundled template text.
|
|
68
71
|
|
|
69
72
|
Parameters
|
|
70
73
|
----------
|
|
@@ -99,16 +102,17 @@ def _load_template_text(
|
|
|
99
102
|
|
|
100
103
|
def _resolve_template(
|
|
101
104
|
*,
|
|
102
|
-
template_key:
|
|
103
|
-
template_path:
|
|
105
|
+
template_key: TemplateKey | None,
|
|
106
|
+
template_path: StrPath | None,
|
|
104
107
|
) -> tuple[Environment, str]:
|
|
105
|
-
"""
|
|
108
|
+
"""
|
|
109
|
+
Return environment and template name for rendering.
|
|
106
110
|
|
|
107
111
|
Parameters
|
|
108
112
|
----------
|
|
109
|
-
template_key :
|
|
113
|
+
template_key : TemplateKey | None
|
|
110
114
|
Named template key bundled with the package.
|
|
111
|
-
template_path :
|
|
115
|
+
template_path : StrPath | None
|
|
112
116
|
Explicit template file override.
|
|
113
117
|
|
|
114
118
|
Returns
|
|
@@ -123,7 +127,11 @@ def _resolve_template(
|
|
|
123
127
|
ValueError
|
|
124
128
|
If the template key is unknown.
|
|
125
129
|
"""
|
|
126
|
-
file_override =
|
|
130
|
+
file_override = (
|
|
131
|
+
str(template_path)
|
|
132
|
+
if template_path is not None
|
|
133
|
+
else os.environ.get('TEMPLATE_NAME')
|
|
134
|
+
)
|
|
127
135
|
if file_override:
|
|
128
136
|
path = Path(file_override)
|
|
129
137
|
if not path.exists():
|
|
@@ -137,14 +145,14 @@ def _resolve_template(
|
|
|
137
145
|
)
|
|
138
146
|
return env, path.name
|
|
139
147
|
|
|
140
|
-
key =
|
|
148
|
+
key: TemplateKey = template_key or 'ddl'
|
|
141
149
|
if key not in TEMPLATES:
|
|
142
150
|
choices = ', '.join(sorted(TEMPLATES))
|
|
143
151
|
raise ValueError(
|
|
144
152
|
f'Unknown template key "{key}". Choose from: {choices}',
|
|
145
153
|
)
|
|
146
154
|
|
|
147
|
-
# Load template from package data
|
|
155
|
+
# Load template from package data.
|
|
148
156
|
template_filename = TEMPLATES[key]
|
|
149
157
|
template_source = _load_template_text(template_filename)
|
|
150
158
|
|
|
@@ -161,19 +169,19 @@ def _resolve_template(
|
|
|
161
169
|
|
|
162
170
|
|
|
163
171
|
def load_table_spec(
|
|
164
|
-
path:
|
|
165
|
-
) ->
|
|
172
|
+
path: StrPath,
|
|
173
|
+
) -> StrAnyMap:
|
|
166
174
|
"""
|
|
167
175
|
Load a table specification from disk.
|
|
168
176
|
|
|
169
177
|
Parameters
|
|
170
178
|
----------
|
|
171
|
-
path :
|
|
179
|
+
path : StrPath
|
|
172
180
|
Path to the JSON or YAML specification file.
|
|
173
181
|
|
|
174
182
|
Returns
|
|
175
183
|
-------
|
|
176
|
-
|
|
184
|
+
StrAnyMap
|
|
177
185
|
Parsed table specification mapping.
|
|
178
186
|
|
|
179
187
|
Raises
|
|
@@ -210,9 +218,9 @@ def load_table_spec(
|
|
|
210
218
|
|
|
211
219
|
|
|
212
220
|
def render_table_sql(
|
|
213
|
-
spec:
|
|
221
|
+
spec: StrAnyMap,
|
|
214
222
|
*,
|
|
215
|
-
template:
|
|
223
|
+
template: TemplateKey | None = 'ddl',
|
|
216
224
|
template_path: str | None = None,
|
|
217
225
|
) -> str:
|
|
218
226
|
"""
|
|
@@ -220,9 +228,9 @@ def render_table_sql(
|
|
|
220
228
|
|
|
221
229
|
Parameters
|
|
222
230
|
----------
|
|
223
|
-
spec :
|
|
231
|
+
spec : StrAnyMap
|
|
224
232
|
Table specification mapping.
|
|
225
|
-
template :
|
|
233
|
+
template : TemplateKey | None, optional
|
|
226
234
|
Template key to use (default: 'ddl').
|
|
227
235
|
template_path : str | None, optional
|
|
228
236
|
Path to a custom template file (overrides ``template``).
|
|
@@ -241,9 +249,9 @@ def render_table_sql(
|
|
|
241
249
|
|
|
242
250
|
|
|
243
251
|
def render_tables(
|
|
244
|
-
specs: Iterable[
|
|
252
|
+
specs: Iterable[StrAnyMap],
|
|
245
253
|
*,
|
|
246
|
-
template:
|
|
254
|
+
template: TemplateKey | None = 'ddl',
|
|
247
255
|
template_path: str | None = None,
|
|
248
256
|
) -> list[str]:
|
|
249
257
|
"""
|
|
@@ -251,9 +259,9 @@ def render_tables(
|
|
|
251
259
|
|
|
252
260
|
Parameters
|
|
253
261
|
----------
|
|
254
|
-
specs : Iterable[
|
|
262
|
+
specs : Iterable[StrAnyMap]
|
|
255
263
|
Table specification mappings.
|
|
256
|
-
template :
|
|
264
|
+
template : TemplateKey | None, optional
|
|
257
265
|
Template key to use (default: 'ddl').
|
|
258
266
|
template_path : str | None, optional
|
|
259
267
|
Path to a custom template file (overrides ``template``).
|
|
@@ -271,21 +279,21 @@ def render_tables(
|
|
|
271
279
|
|
|
272
280
|
|
|
273
281
|
def render_tables_to_string(
|
|
274
|
-
spec_paths: Iterable[
|
|
282
|
+
spec_paths: Iterable[StrPath],
|
|
275
283
|
*,
|
|
276
|
-
template:
|
|
277
|
-
template_path:
|
|
284
|
+
template: TemplateKey | None = 'ddl',
|
|
285
|
+
template_path: StrPath | None = None,
|
|
278
286
|
) -> str:
|
|
279
287
|
"""
|
|
280
288
|
Render one or more specs and concatenate the SQL payloads.
|
|
281
289
|
|
|
282
290
|
Parameters
|
|
283
291
|
----------
|
|
284
|
-
spec_paths : Iterable[
|
|
292
|
+
spec_paths : Iterable[StrPath]
|
|
285
293
|
Paths to table specification files.
|
|
286
|
-
template :
|
|
294
|
+
template : TemplateKey | None, optional
|
|
287
295
|
Template key bundled with ETLPlus. Defaults to ``'ddl'``.
|
|
288
|
-
template_path :
|
|
296
|
+
template_path : StrPath | None, optional
|
|
289
297
|
Custom Jinja template to override the bundled templates.
|
|
290
298
|
|
|
291
299
|
Returns
|
|
@@ -10,12 +10,15 @@ import os
|
|
|
10
10
|
from collections.abc import Mapping
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
|
+
from typing import Final
|
|
13
14
|
|
|
14
15
|
from sqlalchemy import create_engine
|
|
15
16
|
from sqlalchemy.engine import Engine
|
|
16
17
|
from sqlalchemy.orm import sessionmaker
|
|
17
18
|
|
|
18
19
|
from ..file import File
|
|
20
|
+
from ..types import StrAnyMap
|
|
21
|
+
from ..types import StrPath
|
|
19
22
|
|
|
20
23
|
# SECTION: EXPORTS ========================================================== #
|
|
21
24
|
|
|
@@ -33,7 +36,7 @@ __all__ = [
|
|
|
33
36
|
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
34
37
|
|
|
35
38
|
|
|
36
|
-
DATABASE_URL: str = (
|
|
39
|
+
DATABASE_URL: Final[str] = (
|
|
37
40
|
os.getenv('DATABASE_URL')
|
|
38
41
|
or os.getenv('DATABASE_DSN')
|
|
39
42
|
or 'sqlite+pysqlite:///:memory:'
|
|
@@ -43,13 +46,15 @@ DATABASE_URL: str = (
|
|
|
43
46
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
44
47
|
|
|
45
48
|
|
|
46
|
-
def _resolve_url_from_mapping(
|
|
49
|
+
def _resolve_url_from_mapping(
|
|
50
|
+
cfg: StrAnyMap,
|
|
51
|
+
) -> str | None:
|
|
47
52
|
"""
|
|
48
53
|
Return a URL/DSN from a mapping if present.
|
|
49
54
|
|
|
50
55
|
Parameters
|
|
51
56
|
----------
|
|
52
|
-
cfg :
|
|
57
|
+
cfg : StrAnyMap
|
|
53
58
|
Configuration mapping potentially containing connection fields.
|
|
54
59
|
|
|
55
60
|
Returns
|
|
@@ -74,7 +79,7 @@ def _resolve_url_from_mapping(cfg: Mapping[str, Any]) -> str | None:
|
|
|
74
79
|
|
|
75
80
|
|
|
76
81
|
def load_database_url_from_config(
|
|
77
|
-
path:
|
|
82
|
+
path: StrPath,
|
|
78
83
|
*,
|
|
79
84
|
name: str | None = None,
|
|
80
85
|
) -> str:
|
|
@@ -88,7 +93,7 @@ def load_database_url_from_config(
|
|
|
88
93
|
|
|
89
94
|
Parameters
|
|
90
95
|
----------
|
|
91
|
-
path :
|
|
96
|
+
path : StrPath
|
|
92
97
|
Location of the configuration file.
|
|
93
98
|
name : str | None, optional
|
|
94
99
|
Named database entry under the ``databases`` map (default:
|
|
@@ -13,9 +13,8 @@ Usage
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
import re
|
|
16
|
-
from collections.abc import Callable
|
|
17
|
-
from pathlib import Path
|
|
18
16
|
from typing import Any
|
|
17
|
+
from typing import Final
|
|
19
18
|
|
|
20
19
|
from sqlalchemy import Boolean
|
|
21
20
|
from sqlalchemy import CheckConstraint
|
|
@@ -41,11 +40,15 @@ from sqlalchemy.orm import DeclarativeBase
|
|
|
41
40
|
from sqlalchemy.orm import mapped_column
|
|
42
41
|
from sqlalchemy.types import TypeEngine
|
|
43
42
|
|
|
43
|
+
from ..types import StrPath
|
|
44
44
|
from .schema import ForeignKeySpec
|
|
45
45
|
from .schema import TableSpec
|
|
46
46
|
from .schema import load_table_specs
|
|
47
|
+
from .types import ModelRegistry
|
|
48
|
+
from .types import TypeFactory
|
|
49
|
+
|
|
50
|
+
# SECTION: EXPORTS ========================================================== #
|
|
47
51
|
|
|
48
|
-
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
49
52
|
|
|
50
53
|
__all__ = [
|
|
51
54
|
# Classes
|
|
@@ -57,7 +60,9 @@ __all__ = [
|
|
|
57
60
|
]
|
|
58
61
|
|
|
59
62
|
|
|
60
|
-
|
|
63
|
+
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
64
|
+
|
|
65
|
+
_TYPE_MAPPING: Final[dict[str, TypeFactory]] = {
|
|
61
66
|
'int': lambda _: Integer(),
|
|
62
67
|
'integer': lambda _: Integer(),
|
|
63
68
|
'bigint': lambda _: Integer(),
|
|
@@ -102,6 +107,8 @@ _TYPE_MAPPING: dict[str, Callable[[list[int]], TypeEngine]] = {
|
|
|
102
107
|
class Base(DeclarativeBase):
|
|
103
108
|
"""Base class for all ORM models."""
|
|
104
109
|
|
|
110
|
+
__abstract__ = True
|
|
111
|
+
|
|
105
112
|
|
|
106
113
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
107
114
|
|
|
@@ -191,7 +198,7 @@ def build_models(
|
|
|
191
198
|
specs: list[TableSpec],
|
|
192
199
|
*,
|
|
193
200
|
base: type[DeclarativeBase] = Base,
|
|
194
|
-
) ->
|
|
201
|
+
) -> ModelRegistry:
|
|
195
202
|
"""
|
|
196
203
|
Build SQLAlchemy ORM models from table specifications.
|
|
197
204
|
Parameters
|
|
@@ -202,10 +209,10 @@ def build_models(
|
|
|
202
209
|
Base class for the ORM models (default: :class:`Base`).
|
|
203
210
|
Returns
|
|
204
211
|
-------
|
|
205
|
-
|
|
212
|
+
ModelRegistry
|
|
206
213
|
Registry mapping fully qualified table names to ORM model classes.
|
|
207
214
|
"""
|
|
208
|
-
registry:
|
|
215
|
+
registry: ModelRegistry = {}
|
|
209
216
|
|
|
210
217
|
for spec in specs:
|
|
211
218
|
table_args: list[object] = []
|
|
@@ -302,23 +309,23 @@ def build_models(
|
|
|
302
309
|
|
|
303
310
|
|
|
304
311
|
def load_and_build_models(
|
|
305
|
-
path:
|
|
312
|
+
path: StrPath,
|
|
306
313
|
*,
|
|
307
314
|
base: type[DeclarativeBase] = Base,
|
|
308
|
-
) ->
|
|
315
|
+
) -> ModelRegistry:
|
|
309
316
|
"""
|
|
310
317
|
Load table specifications from a file and build SQLAlchemy models.
|
|
311
318
|
|
|
312
319
|
Parameters
|
|
313
320
|
----------
|
|
314
|
-
path :
|
|
321
|
+
path : StrPath
|
|
315
322
|
Path to the YAML file containing table specifications.
|
|
316
323
|
base : type[DeclarativeBase], optional
|
|
317
324
|
Base class for the ORM models (default: :class:`Base`).
|
|
318
325
|
|
|
319
326
|
Returns
|
|
320
327
|
-------
|
|
321
|
-
|
|
328
|
+
ModelRegistry
|
|
322
329
|
Registry mapping fully qualified table names to ORM model classes.
|
|
323
330
|
"""
|
|
324
331
|
return build_models(load_table_specs(path), base=base)
|
|
@@ -16,6 +16,7 @@ from pydantic import ConfigDict
|
|
|
16
16
|
from pydantic import Field
|
|
17
17
|
|
|
18
18
|
from ..file import File
|
|
19
|
+
from ..types import StrPath
|
|
19
20
|
|
|
20
21
|
# SECTION: EXPORTS ========================================================== #
|
|
21
22
|
|
|
@@ -244,14 +245,14 @@ class TableSpec(BaseModel):
|
|
|
244
245
|
|
|
245
246
|
|
|
246
247
|
def load_table_specs(
|
|
247
|
-
path:
|
|
248
|
+
path: StrPath,
|
|
248
249
|
) -> list[TableSpec]:
|
|
249
250
|
"""
|
|
250
251
|
Load table specifications from a YAML file.
|
|
251
252
|
|
|
252
253
|
Parameters
|
|
253
254
|
----------
|
|
254
|
-
path :
|
|
255
|
+
path : StrPath
|
|
255
256
|
Path to the YAML file containing table specifications.
|
|
256
257
|
|
|
257
258
|
Returns
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database.types` module.
|
|
3
|
+
|
|
4
|
+
Shared type aliases leveraged across :mod:`etlplus.database` modules.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from typing import Literal
|
|
11
|
+
|
|
12
|
+
from sqlalchemy.orm import DeclarativeBase
|
|
13
|
+
from sqlalchemy.types import TypeEngine
|
|
14
|
+
|
|
15
|
+
# SECTION: EXPORTS ========================================================== #
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
# Type Aliases
|
|
20
|
+
'ModelRegistry',
|
|
21
|
+
'TemplateKey',
|
|
22
|
+
'TypeFactory',
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# SECTION: TYPE ALIASES ===================================================== #
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# pylint: disable=invalid-name
|
|
30
|
+
|
|
31
|
+
# Registry mapping fully qualified table names to declarative classes.
|
|
32
|
+
type ModelRegistry = dict[str, type[DeclarativeBase]]
|
|
33
|
+
|
|
34
|
+
# Allowed template keys for bundled DDL rendering.
|
|
35
|
+
type TemplateKey = Literal['ddl', 'view']
|
|
36
|
+
|
|
37
|
+
# Callable producing a SQLAlchemy TypeEngine from parsed parameters.
|
|
38
|
+
type TypeFactory = Callable[[list[int]], TypeEngine]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|