etlplus 0.4.7__py3-none-any.whl → 0.8.3__py3-none-any.whl

etlplus/templates/ddl.sql.j2

@@ -0,0 +1,128 @@
+{# ---------- helpers ---------- #}
+{% macro qid(name) -%}[{{ name | replace(']', ']]') }}]{%- endmacro %}
+{% macro qname(schema, name) -%}{{ qid(schema) }}.{{ qid(name) }}{%- endmacro %}
+{% macro df_name(table, col) -%}DF_{{ table }}_{{ col }}{%- endmacro %}
+{% macro ck_name(table, col) -%}CK_{{ table }}_{{ col }}{%- endmacro %}
+
+{# ---------- create schema (optional) ---------- #}
+{% if spec.create_schema | default(true) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.schemas
+        WHERE name = N'{{ spec.schema }}'
+    )
+    EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
+GO
+{% endif %}
+
+{# ---------- create table guarded ---------- #}
+{% set fqtn = qname(spec.schema, spec.table) %}
+IF OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}', N'U') IS NULL
+    BEGIN
+        CREATE TABLE {{ fqtn }} (
+        {% set ns = namespace(maxlen=0) %}
+        {% for c in spec.columns %}
+            {% set nlen = (c.name|string)|length %}
+            {% if nlen > ns.maxlen %}{% set ns.maxlen = nlen %}{% endif %}
+        {% endfor %}
+        {% set indent_cols = '            ' %}
+        {% for c in spec.columns %}
+            {# Map-safe lookups so StrictUndefined never bites on missing keys #}
+            {%- set ctype = c.get('type') -%}
+            {%- set ident = c.get('identity') -%}
+            {%- set computed = c.get('computed') -%}
+            {%- set nullable = c.get('nullable', true) -%}
+            {%- set defv = c.get('default') -%}
+            {%- set checkv = c.get('check') -%}
+            {{- indent_cols -}}{{ qid(c.name) }} {{ ctype }}
+            {%- if computed %} AS ({{ computed }}) PERSISTED{%- endif -%}
+            {%- if nullable %} NULL{%- else %} NOT NULL{%- endif -%}
+            {%- if ident %} IDENTITY({{ ident.get('seed', 1) }}, {{ ident.get('increment', 1) }}){%- endif -%}
+            {%- if defv %} CONSTRAINT {{ qid(df_name(spec.table, c.name)) }} DEFAULT ({{ defv }}){%- endif -%}
+            {%- if checkv %} CONSTRAINT {{ qid(ck_name(spec.table, c.name)) }} CHECK ({{ checkv }}){%- endif -%}
+            {{ "," if not loop.last }}
+        {% endfor %}
+
+        {%- if spec.primary_key is defined and spec.primary_key %}
+            , CONSTRAINT {{ qid(spec.primary_key.name
+                | default('PK_' ~ spec.table)) }}
+            PRIMARY KEY CLUSTERED (
+                {%- for col in spec.primary_key.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+        {%- endif %}
+
+        {%- for uq in spec.unique_constraints | default([]) %}
+            , CONSTRAINT {{ qid(uq.name) }} UNIQUE (
+                {%- for col in uq.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+        {%- endfor %}
+        );
+    END
+GO
+
+{# ---------- indexes ---------- #}
+{% for ix in spec.indexes | default([]) %}
+{%- set ix_unique = ix.get('unique', false) -%}
+{%- set ix_include = ix.get('include') -%}
+{%- set ix_where = ix.get('where') -%}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.indexes
+        WHERE name = N'{{ ix.name }}'
+        AND object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}')
+    )
+    BEGIN
+        CREATE{% if ix_unique %} UNIQUE{% endif %} INDEX {{ qid(ix.name) }}
+            ON {{ fqtn }} (
+                {%- for col in ix.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+            {%- if ix_include %}
+            INCLUDE (
+                {%- for col in ix_include -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+            {%- endif -%}
+            {%- if ix_where %} WHERE {{ ix_where }}{% endif %};
+    END
+GO
+{% endfor %}
+
+{# ---------- foreign keys ---------- #}
+{% for fk in spec.foreign_keys | default([]) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.foreign_keys
+        WHERE
+            name = N'{{ fk.name }}'
+            AND parent_object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}'
+    )
+)
+BEGIN
+    ALTER TABLE {{ fqtn }} WITH CHECK
+    ADD CONSTRAINT {{ qid(fk.name) }} FOREIGN KEY (
+        {%- for col in fk.columns -%}
+            {{ qid(col) }}{{ ", " if not loop.last }}
+        {%- endfor -%}
+    ) REFERENCES
+    {{ qname(fk.ref_schema | default('dbo'), fk.ref_table) }} (
+        {%- for col in fk.ref_columns -%}
+            {{ qid(col) }}{{ ", " if not loop.last }}
+        {%- endfor -%}
+    )
+    {%- set on_upd = fk.get('on_update') -%}
+    {%- set on_del = fk.get('on_delete') -%}
+    {%- if on_upd %} ON UPDATE {{ on_upd }}{% endif -%}
+    {%- if on_del %} ON DELETE {{ on_del }}{% endif %};
+END
+GO
+{% endfor %}

etlplus/templates/view.sql.j2

@@ -0,0 +1,69 @@
+{# -- Helpers -- #}
+{% macro qid(name) -%}
+[{{ name | replace(']', ']]') }}]
+{%- endmacro %}
+
+{% macro qname(schema, name) -%}
+{{ qid(schema) }}.{{ qid(name) }}
+{%- endmacro %}
+
+{# Convert any expression to nvarchar(MAX) with N'' for NULLs #}
+{% macro to_str(expr) -%}
+ISNULL(CONVERT(NVARCHAR(MAX), {{ expr }}), N'')
+{%- endmacro %}
+
+{# Normalize truthy values to 'TRUE' / 'FALSE' #}
+{% macro truthy(expr) -%}
+IIF(CONVERT(VARCHAR(5), {{ expr }}) IN ('TRUE', '1', 'YES'), 'TRUE', 'FALSE')
+{%- endmacro %}
+
+{# -- Create schema (optional) -- #}
+{% if spec.create_schema | default(true) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.schemas
+        WHERE name = N'{{ spec.schema }}'
+    )
+    EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
+GO
+{% endif %}
+
+{# -- SET options (match your source view) -- #}
+SET QUOTED_IDENTIFIER ON
+SET ANSI_NULLS ON
+GO
+
+{# -- Derive names with fallbacks so table-style specs also work -- #}
+{% set view_name = spec.view if spec.view is defined else ('vw_' ~ spec.table) %}
+{% set src_schema = (spec.from_schema if spec.from_schema is defined else (spec.schema if spec.schema is defined else 'dbo')) %}
+{% set src_table = (
+    spec.from_table if spec.from_table is defined else
+    (spec.source_table if spec.source_table is defined else spec.table)
+) %}
+{% set indent_cols = '    ' %}
+
+{# -- Create view -- #}
+CREATE VIEW {{ qname(spec.schema | default('dbo'), view_name) }} AS
+SELECT
+{% if spec.projections is defined and spec.projections %}
+    {% for p in spec.projections %}
+        {%- if p.type is defined and (p.type | lower)[:8] == 'nvarchar' -%}
+    {{- indent_cols -}}{{ to_str(p.expr) }} AS {{ qid(p.alias) }}
+        {%- else -%}
+    {{- indent_cols -}}{{ p.expr }}
+        {%- endif -%}
+    {{ "," if not loop.last }}
+    {% endfor %}
+{% else %}
+    {% for c in spec.columns | default([]) %}
+        {%- if c.type is defined and (c.type | lower)[:8] == 'nvarchar' -%}
+    {{- indent_cols -}}{{ to_str(qid(c.name)) }} AS {{ qid(c.name) }}
+        {%- else -%}
+    {{- indent_cols -}}{{ qid(c.name) }}
+        {%- endif -%}
+    {{ "," if not loop.last }}
+    {% endfor %}
+{%- endif %}
+FROM {{ qname(src_schema, src_table) }};
+GO

etlplus/types.py

@@ -225,3 +225,8 @@ type Sleeper = Callable[[float], None]
 
 # Numeric timeout in seconds or ``None`` for no timeout.
 type Timeout = float | None
+
+# -- Templates -- #
+
+# Allowed template keys for bundled DDL rendering.
+type TemplateKey = Literal['ddl', 'view']

etlplus/utils.py

@@ -6,7 +6,6 @@ Small shared helpers used across modules.
 
 from __future__ import annotations
 
-import argparse
 import json
 from collections.abc import Callable
 from collections.abc import Mapping
@@ -22,7 +21,6 @@ from .types import StrAnyMap
 __all__ = [
     # Data utilities
     'count_records',
-    'json_type',
     'print_json',
     # Mapping utilities
     'cast_str_dict',
@@ -119,35 +117,6 @@ def count_records(
     return len(data) if isinstance(data, list) else 1
 
 
-def json_type(
-    option: str,
-) -> Any:
-    """
-    Argparse ``type=`` hook that parses a JSON string.
-
-    Parameters
-    ----------
-    option : str
-        Raw CLI string to parse as JSON.
-
-    Returns
-    -------
-    Any
-        Parsed JSON value.
-
-    Raises
-    ------
-    argparse.ArgumentTypeError
-        If the input cannot be parsed as JSON.
-    """
-    try:
-        return json.loads(option)
-    except json.JSONDecodeError as e:  # pragma: no cover - argparse path
-        raise argparse.ArgumentTypeError(
-            f'Invalid JSON: {e.msg} (pos {e.pos})',
-        ) from e
-
-
 def maybe_mapping(
     value: Any,
 ) -> StrAnyMap | None:

{etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.4.7
+Version: 0.8.3
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -21,7 +21,10 @@ Requires-Dist: jinja2>=3.1.6
 Requires-Dist: pyodbc>=5.3.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: PyYAML>=6.0.3
 Requires-Dist: requests>=2.32.5
+Requires-Dist: SQLAlchemy>=2.0.45
 Requires-Dist: typer>=0.21.0
 Provides-Extra: dev
 Requires-Dist: black>=25.9.0; extra == "dev"
@@ -61,6 +64,8 @@ package and command-line interface for data extraction, validation, transformati
   - [Quickstart](#quickstart)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
+      - [Check Pipelines](#check-pipelines)
+      - [Render SQL DDL](#render-sql-ddl)
       - [Extract Data](#extract-data)
       - [Validate Data](#validate-data)
       - [Transform Data](#transform-data)
@@ -88,6 +93,14 @@ package and command-line interface for data extraction, validation, transformati
 
 ## Features
 
+- **Check** data pipeline definitions before running them:
+  - Summarize jobs, sources, targets, and transforms
+  - Confirm configuration changes by printing focused sections on demand
+
+- **Render** SQL DDL from shared table specs:
+  - Generate CREATE TABLE or view statements
+  - Swap templates or direct output to files for database migrations
+
 - **Extract** data from multiple sources:
   - Files (CSV, JSON, XML, YAML)
   - Databases (connection string support)
@@ -169,6 +182,47 @@ etlplus --help
 etlplus --version
 ```
 
+The CLI is implemented with Typer (Click-based). There is no argparse compatibility layer, so rely
+on the documented commands/flags and run `etlplus <command> --help` for current options.
+
+#### Check Pipelines
+
+Use `etlplus check` to explore pipeline YAML definitions without running them. The command can print
+job names, summarize configured sources and targets, or drill into specific sections.
+
+List jobs and show a pipeline summary:
+```bash
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+```
+
+Show sources or transforms for troubleshooting:
+```bash
+etlplus check --config examples/configs/pipeline.yml --sources
+etlplus check --config examples/configs/pipeline.yml --transforms
+```
+
+#### Render SQL DDL
+
+Use `etlplus render` to turn table schema specs into ready-to-run SQL. Render from a pipeline config
+or from a standalone schema file, and choose the built-in `ddl` or `view` templates (or provide your
+own).
+
+Render all tables defined in a pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --template ddl
+```
+
+Render a single table in that pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --table customers --template view
+```
+
+Render from a standalone table spec to a file:
+```bash
+etlplus render --spec schemas/customer.yml --template view -o temp/customer_view.sql
+```
+
 #### Extract Data
 
 Note: For file sources, the format is normally inferred from the filename extension. Use
@@ -306,6 +360,17 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
 - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
 - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
 
+CLI quick reference for pipelines:
+
+```bash
+# List jobs or show a pipeline summary
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+
+# Run a job
+etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
+```
+
 ### Complete ETL Pipeline Example
 
 ```bash

{etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/RECORD

@@ -7,13 +7,13 @@ etlplus/file.py,sha256=RxIAsGDN4f_vNA2B5-ct88JNd_ISAyYbooIRE5DstS8,17972
 etlplus/load.py,sha256=BwF3gT4gIr-5CvNMz_aLTCl-w2ihWSTxNVd4X92XFwI,8737
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/run.py,sha256=zl_Yx35spcgaa9Xx7-kcJEb1CAYyMIiqtPlsSrYxRfs,12448
+etlplus/run.py,sha256=X4kp5FQlIWVf1_d9oSrchKau7BFDCE1Zkscvu7WPaWw,12340
 etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
 etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
-etlplus/types.py,sha256=SJiZ7wJiSnV4CEvF-9E5nSFLBo4DT9OqHQqj1GSHkv8,6042
-etlplus/utils.py,sha256=_fn8b-SAdxiw28VX-Ugr8sZUPZI9mEkWKAGExlgxhJA,13993
+etlplus/types.py,sha256=1hsDlnF6r76zAwaUYay-i6pCM-Y0IU5nP7Crj8PLCQ4,6157
+etlplus/utils.py,sha256=GrRH6N9u_U-4gs6PYpO0_Y2RwsyD3Ju9cup0b0byUbk,13367
 etlplus/validate.py,sha256=7rJoEI_SIILdPpoBqqh2UJqg9oeReDz34mYSlc3t7Qg,12989
-etlplus/api/README.md,sha256=UkK5PiZWXbbnMNP0MaPa56S88PjSqOwhMNCyswOhvKc,7329
+etlplus/api/README.md,sha256=ZiyjxLz0LfFCzeYKXwtH8yY1OJ4hXCju7t2ICroFoU8,7215
 etlplus/api/__init__.py,sha256=P2JUYFy6Ep4t6xnsBiCBfQCkQLHYYhA-yXPXCobS8Y0,4295
 etlplus/api/auth.py,sha256=GOO5on-LoMS1GXTAhtK9rFcfpjbBcNeA6NE5UZwIq0g,12158
 etlplus/api/config.py,sha256=wRpOaZ31sPReVzEMme0jKl_37nqgraESwuYSNxP_xDo,17397
@@ -31,21 +31,35 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
 etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
 etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
 etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
-etlplus/cli/app.py,sha256=pc9VDUb3Qc8u5-XyDrHJkrSR9D3bq4e9zLbaD8KzyfY,32618
-etlplus/cli/handlers.py,sha256=aI_ZlnJCGGkVnVJJPhmPRCXc31MxtLaOeqqJoo3ci48,15816
-etlplus/cli/main.py,sha256=LaE3q4jbQ8KTTvs4D_khpOdlTKrdaPmxB28WML6hnLg,15059
+etlplus/cli/commands.py,sha256=-lO8XX2IeTvdfZLeI_UpT2kH4qZuNpfujArw3IKS2vY,22585
+etlplus/cli/constants.py,sha256=NJ6IvNyYEI8IdB7eMcc-vteQiiIwqid5YvmUk-5DRHY,1839
+etlplus/cli/handlers.py,sha256=WBzJZz7ESU4Hljog_ON4g9PLIW6U6io_F9op7daksKY,17781
+etlplus/cli/io.py,sha256=hkmbDh0HFrZhDjfZbma0Lp4tc0wrTwdD9Snqa-wNAd4,7308
+etlplus/cli/main.py,sha256=_ipwLgIzkf8ks8cbu2KkVBbY3gcdaqDR98WUBdwviiw,5191
+etlplus/cli/options.py,sha256=dswqAdnQ4fRJGI3xEFUyMifp0ry3rm2cCaPyEqI_99c,1199
+etlplus/cli/state.py,sha256=20uKdYddYvAlR-HUQJAV-ThyXMZaPiSk9cN-tevRIpw,7991
+etlplus/cli/types.py,sha256=tclhKVJXDqHzlTQBYKARfqMgDOcuBJ-Zej2pvFy96WM,652
 etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
 etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
 etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
-etlplus/config/pipeline.py,sha256=7w0WWe21vvPJvTlJoRTZh2j92I1_GOcdwan6xyKfQII,8973
+etlplus/config/pipeline.py,sha256=Va4MQY6KEyKqHGMKPmh09ZcGpx95br-iNUjpkqtzVbw,9500
 etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
 etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
 etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
+etlplus/database/__init__.py,sha256=AKJsDl2RHuRGPS-eXgNJeh4aSncJP5Y0yLApBF6i7i8,1052
+etlplus/database/ddl.py,sha256=z9KvHi1MPhPBLHxMDdqJgLTp3A2-lcz0gqhZ7HIE6kU,7916
+etlplus/database/engine.py,sha256=7rr7YndA8LwyWJL8k1YhQbqxxmW4gWEUQjp0NwQcYtc,4061
+etlplus/database/orm.py,sha256=gCSqH-CjQz6tV9133-VqgiwokK5ylun0BwXaIWfImAo,10008
+etlplus/database/schema.py,sha256=HNTgglI8qvQLInr7gq--2lLmLKHzAZTL2MJUOIw9DlY,7025
+etlplus/database/types.py,sha256=_pkQyC14TzAlgyeIqZG4F5LWYknZbHw3TW68Auk7Ya0,795
+etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
+etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
+etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
 etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
 etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
-etlplus-0.4.7.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.4.7.dist-info/METADATA,sha256=zD88Gnu6IAFvbIR_L--yZE5EN2VpEK-Q3B1CZypmF9I,17278
-etlplus-0.4.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-etlplus-0.4.7.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.4.7.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.4.7.dist-info/RECORD,,
+etlplus-0.8.3.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.8.3.dist-info/METADATA,sha256=fCmfB5aa9jgUEb2s_l-5j5b3vooe-l3M0pNYrrjhYAg,19513
+etlplus-0.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+etlplus-0.8.3.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.8.3.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.8.3.dist-info/RECORD,,