etlplus 0.5.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/__init__.py +43 -0
- etlplus/__main__.py +22 -0
- etlplus/__version__.py +14 -0
- etlplus/api/README.md +237 -0
- etlplus/api/__init__.py +136 -0
- etlplus/api/auth.py +432 -0
- etlplus/api/config.py +633 -0
- etlplus/api/endpoint_client.py +885 -0
- etlplus/api/errors.py +170 -0
- etlplus/api/pagination/__init__.py +47 -0
- etlplus/api/pagination/client.py +188 -0
- etlplus/api/pagination/config.py +440 -0
- etlplus/api/pagination/paginator.py +775 -0
- etlplus/api/rate_limiting/__init__.py +38 -0
- etlplus/api/rate_limiting/config.py +343 -0
- etlplus/api/rate_limiting/rate_limiter.py +266 -0
- etlplus/api/request_manager.py +589 -0
- etlplus/api/retry_manager.py +430 -0
- etlplus/api/transport.py +325 -0
- etlplus/api/types.py +172 -0
- etlplus/cli/__init__.py +15 -0
- etlplus/cli/app.py +1367 -0
- etlplus/cli/handlers.py +775 -0
- etlplus/cli/main.py +616 -0
- etlplus/config/__init__.py +56 -0
- etlplus/config/connector.py +372 -0
- etlplus/config/jobs.py +311 -0
- etlplus/config/pipeline.py +339 -0
- etlplus/config/profile.py +78 -0
- etlplus/config/types.py +204 -0
- etlplus/config/utils.py +120 -0
- etlplus/ddl.py +197 -0
- etlplus/enums.py +414 -0
- etlplus/extract.py +218 -0
- etlplus/file.py +657 -0
- etlplus/load.py +336 -0
- etlplus/mixins.py +62 -0
- etlplus/py.typed +0 -0
- etlplus/run.py +368 -0
- etlplus/run_helpers.py +843 -0
- etlplus/templates/__init__.py +5 -0
- etlplus/templates/ddl.sql.j2 +128 -0
- etlplus/templates/view.sql.j2 +69 -0
- etlplus/transform.py +1049 -0
- etlplus/types.py +227 -0
- etlplus/utils.py +638 -0
- etlplus/validate.py +493 -0
- etlplus/validation/__init__.py +44 -0
- etlplus/validation/utils.py +389 -0
- etlplus-0.5.4.dist-info/METADATA +616 -0
- etlplus-0.5.4.dist-info/RECORD +55 -0
- etlplus-0.5.4.dist-info/WHEEL +5 -0
- etlplus-0.5.4.dist-info/entry_points.txt +2 -0
- etlplus-0.5.4.dist-info/licenses/LICENSE +21 -0
- etlplus-0.5.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
{# ---------- helpers ---------- #}
|
|
2
|
+
{% macro qid(name) -%}[{{ name | replace(']', ']]') }}]{%- endmacro %}
|
|
3
|
+
{% macro qname(schema, name) -%}{{ qid(schema) }}.{{ qid(name) }}{%- endmacro %}
|
|
4
|
+
{% macro df_name(table, col) -%}DF_{{ table }}_{{ col }}{%- endmacro %}
|
|
5
|
+
{% macro ck_name(table, col) -%}CK_{{ table }}_{{ col }}{%- endmacro %}
|
|
6
|
+
|
|
7
|
+
{# ---------- create schema (optional) ---------- #}
|
|
8
|
+
{% if spec.create_schema | default(true) %}
|
|
9
|
+
IF
|
|
10
|
+
NOT EXISTS (
|
|
11
|
+
SELECT 1
|
|
12
|
+
FROM sys.schemas
|
|
13
|
+
WHERE name = N'{{ spec.schema }}'
|
|
14
|
+
)
|
|
15
|
+
EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
|
|
16
|
+
GO
|
|
17
|
+
{% endif %}
|
|
18
|
+
|
|
19
|
+
{# ---------- create table guarded ---------- #}
|
|
20
|
+
{% set fqtn = qname(spec.schema, spec.table) %}
|
|
21
|
+
IF OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}', N'U') IS NULL
|
|
22
|
+
BEGIN
|
|
23
|
+
CREATE TABLE {{ fqtn }} (
|
|
24
|
+
{% set ns = namespace(maxlen=0) %}
|
|
25
|
+
{% for c in spec.columns %}
|
|
26
|
+
{% set nlen = (c.name|string)|length %}
|
|
27
|
+
{% if nlen > ns.maxlen %}{% set ns.maxlen = nlen %}{% endif %}
|
|
28
|
+
{% endfor %}
|
|
29
|
+
{% set indent_cols = ' ' %}
|
|
30
|
+
{% for c in spec.columns %}
|
|
31
|
+
{# Map-safe lookups so StrictUndefined never bites on missing keys #}
|
|
32
|
+
{%- set ctype = c.get('type') -%}
|
|
33
|
+
{%- set ident = c.get('identity') -%}
|
|
34
|
+
{%- set computed = c.get('computed') -%}
|
|
35
|
+
{%- set nullable = c.get('nullable', true) -%}
|
|
36
|
+
{%- set defv = c.get('default') -%}
|
|
37
|
+
{%- set checkv = c.get('check') -%}
|
|
38
|
+
{{- indent_cols -}}{{ qid(c.name) }} {{ ctype }}
|
|
39
|
+
{%- if computed %} AS ({{ computed }}) PERSISTED{%- endif -%}
|
|
40
|
+
{%- if nullable %} NULL{%- else %} NOT NULL{%- endif -%}
|
|
41
|
+
{%- if ident %} IDENTITY({{ ident.get('seed', 1) }}, {{ ident.get('increment', 1) }}){%- endif -%}
|
|
42
|
+
{%- if defv %} CONSTRAINT {{ qid(df_name(spec.table, c.name)) }} DEFAULT ({{ defv }}){%- endif -%}
|
|
43
|
+
{%- if checkv %} CONSTRAINT {{ qid(ck_name(spec.table, c.name)) }} CHECK ({{ checkv }}){%- endif -%}
|
|
44
|
+
{{ "," if not loop.last }}
|
|
45
|
+
{% endfor %}
|
|
46
|
+
|
|
47
|
+
{%- if spec.primary_key is defined and spec.primary_key %}
|
|
48
|
+
, CONSTRAINT {{ qid(spec.primary_key.name
|
|
49
|
+
| default('PK_' ~ spec.table)) }}
|
|
50
|
+
PRIMARY KEY CLUSTERED (
|
|
51
|
+
{%- for col in spec.primary_key.columns -%}
|
|
52
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
53
|
+
{%- endfor -%}
|
|
54
|
+
)
|
|
55
|
+
{%- endif %}
|
|
56
|
+
|
|
57
|
+
{%- for uq in spec.unique_constraints | default([]) %}
|
|
58
|
+
, CONSTRAINT {{ qid(uq.name) }} UNIQUE (
|
|
59
|
+
{%- for col in uq.columns -%}
|
|
60
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
61
|
+
{%- endfor -%}
|
|
62
|
+
)
|
|
63
|
+
{%- endfor %}
|
|
64
|
+
);
|
|
65
|
+
END
|
|
66
|
+
GO
|
|
67
|
+
|
|
68
|
+
{# ---------- indexes ---------- #}
|
|
69
|
+
{% for ix in spec.indexes | default([]) %}
|
|
70
|
+
{%- set ix_unique = ix.get('unique', false) -%}
|
|
71
|
+
{%- set ix_include = ix.get('include') -%}
|
|
72
|
+
{%- set ix_where = ix.get('where') -%}
|
|
73
|
+
IF
|
|
74
|
+
NOT EXISTS (
|
|
75
|
+
SELECT 1
|
|
76
|
+
FROM sys.indexes
|
|
77
|
+
WHERE name = N'{{ ix.name }}'
|
|
78
|
+
AND object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}')
|
|
79
|
+
)
|
|
80
|
+
BEGIN
|
|
81
|
+
CREATE{% if ix_unique %} UNIQUE{% endif %} INDEX {{ qid(ix.name) }}
|
|
82
|
+
ON {{ fqtn }} (
|
|
83
|
+
{%- for col in ix.columns -%}
|
|
84
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
85
|
+
{%- endfor -%}
|
|
86
|
+
)
|
|
87
|
+
{%- if ix_include %}
|
|
88
|
+
INCLUDE (
|
|
89
|
+
{%- for col in ix_include -%}
|
|
90
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
91
|
+
{%- endfor -%}
|
|
92
|
+
)
|
|
93
|
+
{%- endif -%}
|
|
94
|
+
{%- if ix_where %} WHERE {{ ix_where }}{% endif %};
|
|
95
|
+
END
|
|
96
|
+
GO
|
|
97
|
+
{% endfor %}
|
|
98
|
+
|
|
99
|
+
{# ---------- foreign keys ---------- #}
|
|
100
|
+
{% for fk in spec.foreign_keys | default([]) %}
|
|
101
|
+
IF
|
|
102
|
+
NOT EXISTS (
|
|
103
|
+
SELECT 1
|
|
104
|
+
FROM sys.foreign_keys
|
|
105
|
+
WHERE
|
|
106
|
+
name = N'{{ fk.name }}'
|
|
107
|
+
AND parent_object_id = OBJECT_ID(N'{{ spec.schema }}.{{ spec.table }}'
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
BEGIN
|
|
111
|
+
ALTER TABLE {{ fqtn }} WITH CHECK
|
|
112
|
+
ADD CONSTRAINT {{ qid(fk.name) }} FOREIGN KEY (
|
|
113
|
+
{%- for col in fk.columns -%}
|
|
114
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
115
|
+
{%- endfor -%}
|
|
116
|
+
) REFERENCES
|
|
117
|
+
{{ qname(fk.ref_schema | default('dbo'), fk.ref_table) }} (
|
|
118
|
+
{%- for col in fk.ref_columns -%}
|
|
119
|
+
{{ qid(col) }}{{ ", " if not loop.last }}
|
|
120
|
+
{%- endfor -%}
|
|
121
|
+
)
|
|
122
|
+
{%- set on_upd = fk.get('on_update') -%}
|
|
123
|
+
{%- set on_del = fk.get('on_delete') -%}
|
|
124
|
+
{%- if on_upd %} ON UPDATE {{ on_upd }}{% endif -%}
|
|
125
|
+
{%- if on_del %} ON DELETE {{ on_del }}{% endif %};
|
|
126
|
+
END
|
|
127
|
+
GO
|
|
128
|
+
{% endfor %}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{# -- Helpers -- #}
|
|
2
|
+
{% macro qid(name) -%}
|
|
3
|
+
[{{ name | replace(']', ']]') }}]
|
|
4
|
+
{%- endmacro %}
|
|
5
|
+
|
|
6
|
+
{% macro qname(schema, name) -%}
|
|
7
|
+
{{ qid(schema) }}.{{ qid(name) }}
|
|
8
|
+
{%- endmacro %}
|
|
9
|
+
|
|
10
|
+
{# Convert any expression to nvarchar(MAX) with N'' for NULLs #}
|
|
11
|
+
{% macro to_str(expr) -%}
|
|
12
|
+
ISNULL(CONVERT(NVARCHAR(MAX), {{ expr }}), N'')
|
|
13
|
+
{%- endmacro %}
|
|
14
|
+
|
|
15
|
+
{# Normalize truthy values to 'TRUE' / 'FALSE' #}
|
|
16
|
+
{% macro truthy(expr) -%}
|
|
17
|
+
IIF(CONVERT(VARCHAR(5), {{ expr }}) IN ('TRUE', '1', 'YES'), 'TRUE', 'FALSE')
|
|
18
|
+
{%- endmacro %}
|
|
19
|
+
|
|
20
|
+
{# -- Create schema (optional) -- #}
|
|
21
|
+
{% if spec.create_schema | default(true) %}
|
|
22
|
+
IF
|
|
23
|
+
NOT EXISTS (
|
|
24
|
+
SELECT 1
|
|
25
|
+
FROM sys.schemas
|
|
26
|
+
WHERE name = N'{{ spec.schema }}'
|
|
27
|
+
)
|
|
28
|
+
EXEC ('CREATE SCHEMA {{ qid(spec.schema) }}');
|
|
29
|
+
GO
|
|
30
|
+
{% endif %}
|
|
31
|
+
|
|
32
|
+
{# -- SET options (match your source view) -- #}
|
|
33
|
+
SET QUOTED_IDENTIFIER ON
|
|
34
|
+
SET ANSI_NULLS ON
|
|
35
|
+
GO
|
|
36
|
+
|
|
37
|
+
{# -- Derive names with fallbacks so table-style specs also work -- #}
|
|
38
|
+
{% set view_name = spec.view if spec.view is defined else ('vw_' ~ spec.table) %}
|
|
39
|
+
{% set src_schema = (spec.from_schema if spec.from_schema is defined else (spec.schema if spec.schema is defined else 'dbo')) %}
|
|
40
|
+
{% set src_table = (
|
|
41
|
+
spec.from_table if spec.from_table is defined else
|
|
42
|
+
(spec.source_table if spec.source_table is defined else spec.table)
|
|
43
|
+
) %}
|
|
44
|
+
{% set indent_cols = ' ' %}
|
|
45
|
+
|
|
46
|
+
{# -- Create view -- #}
|
|
47
|
+
CREATE VIEW {{ qname(spec.schema | default('dbo'), view_name) }} AS
|
|
48
|
+
SELECT
|
|
49
|
+
{% if spec.projections is defined and spec.projections %}
|
|
50
|
+
{% for p in spec.projections %}
|
|
51
|
+
{%- if p.type is defined and (p.type | lower)[:8] == 'nvarchar' -%}
|
|
52
|
+
{{- indent_cols -}}{{ to_str(p.expr) }} AS {{ qid(p.alias) }}
|
|
53
|
+
{%- else -%}
|
|
54
|
+
{{- indent_cols -}}{{ p.expr }}
|
|
55
|
+
{%- endif -%}
|
|
56
|
+
{{ "," if not loop.last }}
|
|
57
|
+
{% endfor %}
|
|
58
|
+
{% else %}
|
|
59
|
+
{% for c in spec.columns | default([]) %}
|
|
60
|
+
{%- if c.type is defined and (c.type | lower)[:8] == 'nvarchar' -%}
|
|
61
|
+
{{- indent_cols -}}{{ to_str(qid(c.name)) }} AS {{ qid(c.name) }}
|
|
62
|
+
{%- else -%}
|
|
63
|
+
{{- indent_cols -}}{{ qid(c.name) }}
|
|
64
|
+
{%- endif -%}
|
|
65
|
+
{{ "," if not loop.last }}
|
|
66
|
+
{% endfor %}
|
|
67
|
+
{%- endif %}
|
|
68
|
+
FROM {{ qname(src_schema, src_table) }};
|
|
69
|
+
GO
|