etlplus 0.4.6__py3-none-any.whl → 0.7.0__py3-none-any.whl

etlplus/templates/ddl.sql.j2

@@ -0,0 +1,128 @@
+{# ---------- helpers ---------- #}
+{% macro qid(name) -%}[{{ name | replace(']', ']]') }}]{%- endmacro %}
+{% macro qname(schema, name) -%}{{ qid(schema) }}.{{ qid(name) }}{%- endmacro %}
+{% macro df_name(table, col) -%}DF_{{ table }}_{{ col }}{%- endmacro %}
+{% macro ck_name(table, col) -%}CK_{{ table }}_{{ col }}{%- endmacro %}
+
+{# ---------- create schema (optional) ---------- #}
+{% if spec.create_schema | default(true) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.schemas
+        WHERE name = N'{{ spec.schema }}'
+    )
+    EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
+GO
+{% endif %}
+
+{# ---------- create table guarded ---------- #}
+{% set fqtn = qname(spec.schema, spec.table) %}
+IF OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}', N'U') IS NULL
+    BEGIN
+        CREATE TABLE {{ fqtn }} (
+        {% set ns = namespace(maxlen=0) %}
+        {% for c in spec.columns %}
+            {% set nlen = (c.name|string)|length %}
+            {% if nlen > ns.maxlen %}{% set ns.maxlen = nlen %}{% endif %}
+        {% endfor %}
+        {% set indent_cols = '            ' %}
+        {% for c in spec.columns %}
+            {# Map-safe lookups so StrictUndefined never bites on missing keys #}
+            {%- set ctype = c.get('type') -%}
+            {%- set ident = c.get('identity') -%}
+            {%- set computed = c.get('computed') -%}
+            {%- set nullable = c.get('nullable', true) -%}
+            {%- set defv = c.get('default') -%}
+            {%- set checkv = c.get('check') -%}
+            {{- indent_cols -}}{{ qid(c.name) }} {{ ctype }}
+            {%- if computed %} AS ({{ computed }}) PERSISTED{%- endif -%}
+            {%- if nullable %} NULL{%- else %} NOT NULL{%- endif -%}
+            {%- if ident %} IDENTITY({{ ident.get('seed', 1) }}, {{ ident.get('increment', 1) }}){%- endif -%}
+            {%- if defv %} CONSTRAINT {{ qid(df_name(spec.table, c.name)) }} DEFAULT ({{ defv }}){%- endif -%}
+            {%- if checkv %} CONSTRAINT {{ qid(ck_name(spec.table, c.name)) }} CHECK ({{ checkv }}){%- endif -%}
+            {{ "," if not loop.last }}
+        {% endfor %}
+
+        {%- if spec.primary_key is defined and spec.primary_key %}
+            , CONSTRAINT {{ qid(spec.primary_key.name
+                | default('PK_' ~ spec.table)) }}
+            PRIMARY KEY CLUSTERED (
+                {%- for col in spec.primary_key.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+        {%- endif %}
+
+        {%- for uq in spec.unique_constraints | default([]) %}
+            , CONSTRAINT {{ qid(uq.name) }} UNIQUE (
+                {%- for col in uq.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+        {%- endfor %}
+        );
+    END
+GO
+
+{# ---------- indexes ---------- #}
+{% for ix in spec.indexes | default([]) %}
+{%- set ix_unique = ix.get('unique', false) -%}
+{%- set ix_include = ix.get('include') -%}
+{%- set ix_where = ix.get('where') -%}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.indexes
+        WHERE name = N'{{ ix.name }}'
+        AND object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}')
+    )
+    BEGIN
+        CREATE{% if ix_unique %} UNIQUE{% endif %} INDEX {{ qid(ix.name) }}
+            ON {{ fqtn }} (
+                {%- for col in ix.columns -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+            {%- if ix_include %}
+            INCLUDE (
+                {%- for col in ix_include -%}
+                    {{ qid(col) }}{{ ", " if not loop.last }}
+                {%- endfor -%}
+            )
+            {%- endif -%}
+            {%- if ix_where %} WHERE {{ ix_where }}{% endif %};
+    END
+GO
+{% endfor %}
+
+{# ---------- foreign keys ---------- #}
+{% for fk in spec.foreign_keys | default([]) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.foreign_keys
+        WHERE
+            name = N'{{ fk.name }}'
+            AND parent_object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}'
+    )
+)
+BEGIN
+    ALTER TABLE {{ fqtn }} WITH CHECK
+    ADD CONSTRAINT {{ qid(fk.name) }} FOREIGN KEY (
+        {%- for col in fk.columns -%}
+            {{ qid(col) }}{{ ", " if not loop.last }}
+        {%- endfor -%}
+    ) REFERENCES
+    {{ qname(fk.ref_schema | default('dbo'), fk.ref_table) }} (
+        {%- for col in fk.ref_columns -%}
+            {{ qid(col) }}{{ ", " if not loop.last }}
+        {%- endfor -%}
+    )
+    {%- set on_upd = fk.get('on_update') -%}
+    {%- set on_del = fk.get('on_delete') -%}
+    {%- if on_upd %} ON UPDATE {{ on_upd }}{% endif -%}
+    {%- if on_del %} ON DELETE {{ on_del }}{% endif %};
+END
+GO
+{% endfor %}

etlplus/templates/view.sql.j2

@@ -0,0 +1,69 @@
+{# -- Helpers -- #}
+{% macro qid(name) -%}
+[{{ name | replace(']', ']]') }}]
+{%- endmacro %}
+
+{% macro qname(schema, name) -%}
+{{ qid(schema) }}.{{ qid(name) }}
+{%- endmacro %}
+
+{# Convert any expression to nvarchar(MAX) with N'' for NULLs #}
+{% macro to_str(expr) -%}
+ISNULL(CONVERT(NVARCHAR(MAX), {{ expr }}), N'')
+{%- endmacro %}
+
+{# Normalize truthy values to 'TRUE' / 'FALSE' #}
+{% macro truthy(expr) -%}
+IIF(CONVERT(VARCHAR(5), {{ expr }}) IN ('TRUE', '1', 'YES'), 'TRUE', 'FALSE')
+{%- endmacro %}
+
+{# -- Create schema (optional) -- #}
+{% if spec.create_schema | default(true) %}
+IF
+    NOT EXISTS (
+        SELECT 1
+        FROM sys.schemas
+        WHERE name = N'{{ spec.schema }}'
+    )
+    EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
+GO
+{% endif %}
+
+{# -- SET options (match your source view) -- #}
+SET QUOTED_IDENTIFIER ON
+SET ANSI_NULLS ON
+GO
+
+{# -- Derive names with fallbacks so table-style specs also work -- #}
+{% set view_name = spec.view if spec.view is defined else ('vw_' ~ spec.table) %}
+{% set src_schema = (spec.from_schema if spec.from_schema is defined else (spec.schema if spec.schema is defined else 'dbo')) %}
+{% set src_table = (
+    spec.from_table if spec.from_table is defined else
+    (spec.source_table if spec.source_table is defined else spec.table)
+) %}
+{% set indent_cols = '    ' %}
+
+{# -- Create view -- #}
+CREATE VIEW {{ qname(spec.schema | default('dbo'), view_name) }} AS
+SELECT
+{% if spec.projections is defined and spec.projections %}
+    {% for p in spec.projections %}
+        {%- if p.type is defined and (p.type | lower)[:8] == 'nvarchar' -%}
+    {{- indent_cols -}}{{ to_str(p.expr) }} AS {{ qid(p.alias) }}
+        {%- else -%}
+    {{- indent_cols -}}{{ p.expr }}
+        {%- endif -%}
+    {{ "," if not loop.last }}
+    {% endfor %}
+{% else %}
+    {% for c in spec.columns | default([]) %}
+        {%- if c.type is defined and (c.type | lower)[:8] == 'nvarchar' -%}
+    {{- indent_cols -}}{{ to_str(qid(c.name)) }} AS {{ qid(c.name) }}
+        {%- else -%}
+    {{- indent_cols -}}{{ qid(c.name) }}
+        {%- endif -%}
+    {{ "," if not loop.last }}
+    {% endfor %}
+{%- endif %}
+FROM {{ qname(src_schema, src_table) }};
+GO

{etlplus-0.4.6.dist-info → etlplus-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.4.6
+Version: 0.7.0
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -21,7 +21,10 @@ Requires-Dist: jinja2>=3.1.6
 Requires-Dist: pyodbc>=5.3.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: PyYAML>=6.0.3
 Requires-Dist: requests>=2.32.5
+Requires-Dist: SQLAlchemy>=2.0.45
 Requires-Dist: typer>=0.21.0
 Provides-Extra: dev
 Requires-Dist: black>=25.9.0; extra == "dev"
@@ -61,6 +64,8 @@ package and command-line interface for data extraction, validation, transformati
   - [Quickstart](#quickstart)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
+      - [Inspect Pipelines](#inspect-pipelines)
+      - [Render SQL DDL](#render-sql-ddl)
       - [Extract Data](#extract-data)
       - [Validate Data](#validate-data)
       - [Transform Data](#transform-data)
@@ -88,6 +93,14 @@ package and command-line interface for data extraction, validation, transformati
 
 ## Features
 
+- **Check** data pipeline definitions before running them:
+  - Summarize jobs, sources, targets, and transforms
+  - Confirm configuration changes by printing focused sections on demand
+
+- **Render** SQL DDL from shared table specs:
+  - Generate CREATE TABLE or view statements
+  - Swap templates or direct output to files for database migrations
+
 - **Extract** data from multiple sources:
   - Files (CSV, JSON, XML, YAML)
   - Databases (connection string support)
@@ -169,6 +182,44 @@ etlplus --help
 etlplus --version
 ```
 
+#### Check Pipelines
+
+Use `etlplus check` to explore pipeline YAML definitions without running them. The command can print
+job names, summarize configured sources and targets, or drill into specific sections.
+
+List jobs and show a pipeline summary:
+```bash
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+```
+
+Show sources or transforms for troubleshooting:
+```bash
+etlplus check --config examples/configs/pipeline.yml --sources
+etlplus check --config examples/configs/pipeline.yml --transforms
+```
+
+#### Render SQL DDL
+
+Use `etlplus render` to turn table schema specs into ready-to-run SQL. Render from a pipeline config
+or from a standalone schema file, and choose the built-in `ddl` or `view` templates (or provide your
+own).
+
+Render all tables defined in a pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --template ddl
+```
+
+Render a single table in that pipeline:
+```bash
+etlplus render --config examples/configs/pipeline.yml --table customers --template view
+```
+
+Render from a standalone table spec to a file:
+```bash
+etlplus render --spec schemas/customer.yml --template view -o temp/customer_view.sql
+```
+
 #### Extract Data
 
 Note: For file sources, the format is normally inferred from the filename extension. Use
@@ -306,6 +357,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
 - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
 - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
 
+CLI quick reference for pipelines:
+
+```bash
+# List jobs or show a pipeline summary
+etlplus check --config examples/configs/pipeline.yml --jobs
+etlplus check --config examples/configs/pipeline.yml --summary
+
+# Run a job
+etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
+
+# Deprecated shim (will be removed): etlplus pipeline
+```
+
 ### Complete ETL Pipeline Example
 
 ```bash

{etlplus-0.4.6.dist-info → etlplus-0.7.0.dist-info}/RECORD

@@ -7,7 +7,7 @@ etlplus/file.py,sha256=RxIAsGDN4f_vNA2B5-ct88JNd_ISAyYbooIRE5DstS8,17972
 etlplus/load.py,sha256=BwF3gT4gIr-5CvNMz_aLTCl-w2ihWSTxNVd4X92XFwI,8737
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/run.py,sha256=zl_Yx35spcgaa9Xx7-kcJEb1CAYyMIiqtPlsSrYxRfs,12448
+etlplus/run.py,sha256=X4kp5FQlIWVf1_d9oSrchKau7BFDCE1Zkscvu7WPaWw,12340
 etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
 etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
 etlplus/types.py,sha256=SJiZ7wJiSnV4CEvF-9E5nSFLBo4DT9OqHQqj1GSHkv8,6042
@@ -31,21 +31,29 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
 etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
 etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
 etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
-etlplus/cli/app.py,sha256=pc9VDUb3Qc8u5-XyDrHJkrSR9D3bq4e9zLbaD8KzyfY,32618
-etlplus/cli/handlers.py,sha256=aI_ZlnJCGGkVnVJJPhmPRCXc31MxtLaOeqqJoo3ci48,15816
-etlplus/cli/main.py,sha256=9hoitdc9FisrXzwZniTglPWwKsODFAW-A-2QQV4NkBs,12565
+etlplus/cli/app.py,sha256=SYPO-NDwXgymJrACw39jZ_NJrSKAs0O8anuWR5o42WM,35893
+etlplus/cli/handlers.py,sha256=nFMvqHQhJ8kJZPisDCiUHeOhjlqAO6hJvRjXiJTcU74,18951
+etlplus/cli/main.py,sha256=ijYOy72SEsxrTEBan5yADW8CZyr0yddVF8HeMgFw6Zg,16576
 etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
 etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
 etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
-etlplus/config/pipeline.py,sha256=7w0WWe21vvPJvTlJoRTZh2j92I1_GOcdwan6xyKfQII,8973
+etlplus/config/pipeline.py,sha256=Va4MQY6KEyKqHGMKPmh09ZcGpx95br-iNUjpkqtzVbw,9500
 etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
 etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
 etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
+etlplus/database/__init__.py,sha256=0gWnMlQiVHS6SVUxIT9zklQUHU36y-2RF_gN1cx7icg,1018
+etlplus/database/ddl.py,sha256=lIar9KIOoBRslp_P0DnpoMDXzkjt64J5-iVV7CeSV_M,7747
+etlplus/database/engine.py,sha256=54f-XtNKIuogJhsLV9cX_xPoBwcl_HNJTL5HqMCi8kw,3986
+etlplus/database/orm.py,sha256=StjeguokM70oNKq7mNXLyc4_mYUZR-EKW3oGRlsd8QE,9962
+etlplus/database/schema.py,sha256=BmRP2wwX2xex1phLm0tnHrP6A2AQgguA-hSLnK0xwwc,7003
+etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
+etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
+etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
 etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
 etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
-etlplus-0.4.6.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.4.6.dist-info/METADATA,sha256=oq0-zYAVKNkZFCBPecObqjJ2LuYcbSX5kVjCErpiDO4,17278
-etlplus-0.4.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-etlplus-0.4.6.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.4.6.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.4.6.dist-info/RECORD,,
+etlplus-0.7.0.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.7.0.dist-info/METADATA,sha256=ulMPDyXMX6p-NcxMBSZfegGrv0LNwAu_686_TVrkJPM,19383
+etlplus-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+etlplus-0.7.0.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.7.0.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.7.0.dist-info/RECORD,,