sandwich 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. {sandwich-0.1.0 → sandwich-0.2.0}/.gitignore +13 -13
  2. sandwich-0.2.0/PKG-INFO +144 -0
  3. sandwich-0.2.0/README.md +124 -0
  4. {sandwich-0.1.0 → sandwich-0.2.0}/pyproject.toml +40 -38
  5. sandwich-0.1.0/PKG-INFO +0 -16
  6. sandwich-0.1.0/README.md +0 -0
  7. {sandwich-0.1.0 → sandwich-0.2.0}/LICENSE +0 -0
  8. {sandwich-0.1.0 → sandwich-0.2.0}/sandwich/py.typed +0 -0
{sandwich-0.1.0 → sandwich-0.2.0}/.gitignore RENAMED
@@ -1,13 +1,13 @@
1
- # Python-generated files
2
- __pycache__/
3
- *.py[oc]
4
- build/
5
- dist/
6
- wheels/
7
- *.egg-info
8
-
9
- # Virtual environments
10
- .venv
11
-
12
- # config
13
- .env
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # config
13
+ .env
sandwich-0.2.0/PKG-INFO ADDED
@@ -0,0 +1,144 @@
1
+ Metadata-Version: 2.4
2
+ Name: sandwich
3
+ Version: 0.2.0
4
+ Summary: DataVault 2.0 code gen
5
+ Author-email: Andrey Morozov <andrey@morozov.lv>
6
+ License-File: LICENSE
7
+ Keywords: DWH,Data Vault 2.0
8
+ Classifier: Development Status :: 1 - Planning
9
+ Classifier: Environment :: Console
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Typing :: Typed
15
+ Requires-Python: >=3.12
16
+ Requires-Dist: dotenv>=0.9.9
17
+ Requires-Dist: pyodbc>=5.3.0
18
+ Requires-Dist: sqlalchemy>=2.0.44
19
+ Description-Content-Type: text/markdown
20
+
21
+ ## Data Vault 2.0 scaffolding tool
22
+ This tool is designed to streamline the process of creating Data Vault 2.0 entities,
23
+ such as hubs, links, and satellites.
24
+ As well as building information layer objects such as dim and fact tables
25
+ from a multidimensional paradigm.
26
+
27
+ ### How it works:
28
+ User: provides a staging view `stg.[entity_name]` (or a table if the staging layer persisted)
29
+ with all requirements for the `[entity_name]` defined in the schema (how to define see below).
30
+ Tool:
31
+ 1. Validates metadata of the provided staging view or table.
32
+ 2. Generates the necessary DDL statements to create the Data Vault 2.0 entities.
33
+ 3. Generates ELT procedures to load data to the generated entities.
34
+ 4. Generates support procedures such as `meta.Drop_all_related_to_[entity_name]` and `elt.Run_all_related_to_[entity_name]`
35
+
36
+ ```text
37
+ +----------------------+
38
+ | hub.[entity_name] |
39
+ +----------------------+
40
+ ^
41
+ o 1.define +-------------------+ | 3.create
42
+ /|\ -------> | stg.[entity_name] | # +----------------------+
43
+ / \ +-------------------+ /|\ ---------> | sat.[entity_name] |
44
+ User ---------------------------------------> / \ 3.create +----------------------+
45
+ 2.use Tool
46
+ | 3.create
47
+ v
48
+ +----------------------+
49
+ | dim.[entity_name] |
50
+ +----------------------+
51
+
52
+ ```
53
+
54
+ ### How to define a staging view or table:
55
+ * `bk_` (BusinessKey) - at least one `bk_` column
56
+ * `hk_[entity_name]` (HashKey) - exactly one `hk_[entity_name]` column if you want a `hub` table created
57
+ * `LoadDate` - required by dv2 standard for an auditability
58
+ * `RecordSource` - required by dv2 standard for an auditability
59
+ * `HashDiff` - optional, required if you want to have a scd2 type `dim` table created
60
+ * `IsAvailable` - optional, required if you want to track missing/deleted records
61
+ * all other columns will be considered as business columns and will be included to the `sat` table definition
62
+
63
+
64
+ | staging fields | scd2dim profile |
65
+ |--------------------|-----------------|
66
+ | bk_ | ✅ |
67
+ | hk_`[entity_name]` | ✅ |
68
+ | LoadDate | ✅ |
69
+ | RecordSource | ✅ |
70
+ | HashDiff | ✅ |
71
+ | IsAvailable | ✅ |
72
+
73
+ ```sql
74
+ -- staging view example for the scd2dim profile (mssql)
75
+ create view [stg].[UR_officers] as
76
+ select cast(31 as bigint) [bk_id]
77
+ , core.StringToHash1(cast(31 as bigint)) [hk_UR_officers]
78
+ , sysdatetime() [LoadDate]
79
+ , cast('LobSystem.dbo.officers_daily' as varchar(200)) [RecordSource]
80
+ , core.StringToHash8(
81
+ cast('uri' as nvarchar(100))
82
+ , cast('00000000000000' as varchar(20))
83
+ , cast('NATURAL_PERSON' as varchar(50))
84
+ , cast(null as varchar(20))
85
+ , cast('INDIVIDUALLY' as varchar(50))
86
+ , cast(0 as int)
87
+ , cast('2008-04-07' as date)
88
+ , cast('2008-04-07 18:00:54.000' as datetime)
89
+ ) [HashDiff]
90
+ , cast('uri' as nvarchar(100)) [uri]
91
+ , cast('00000000000000' as varchar(20)) [at_legal_entity_registration_number]
92
+ , cast('NATURAL_PERSON' as varchar(50)) [entity_type]
93
+ , cast(null as varchar(20)) [legal_entity_registration_number]
94
+ , cast('INDIVIDUALLY' as varchar(50)) [rights_of_representation_type]
95
+ , cast(0 as int) [representation_with_at_least]
96
+ , cast('2008-04-07' as date) [registered_on]
97
+ , cast('2008-04-07 18:00:54.000' as datetime) [last_modified_at]
98
+ , cast(1 as bit) [IsAvailable]
99
+ ```
100
+ ### scd2dim profile example:
101
+ | stg | hub | sat | dim |
102
+ |--------------------|------------------------|----------------------------|--------------------|
103
+ | | | | hk_`[entity_name]` |
104
+ | BKs... | (uk)BKs... | BKs... | (pk)BKs... |
105
+ | hk_`[entity_name]` | (pk)hk_`[entity_name]` | (pk)(fk)hk_`[entity_name]` | |
106
+ | LoadDate | LoadDate | (pk)LoadDate | |
107
+ | RecordSource | RecordSource | RecordSource | |
108
+ | HashDiff | | HashDiff | |
109
+ | FLDs... | | FLDs... | FLDs... |
110
+ | IsAvailable | | IsAvailable | IsAvailable |
111
+ | | | | IsCurrent |
112
+ | | | | (pk)DateFrom |
113
+ | | | | DateTo |
114
+
115
+ ### link2fact profile example:
116
+ | stg | link | sat | fact |
117
+ |--------------------|------------------------|----------------------------|------|
118
+ | HKs... | (uk)(fk)HKs... | | |
119
+ | hk_`[entity_name]` | (pk)hk_`[entity_name]` | (pk)(fk)hk_`[entity_name]` | |
120
+ | degenerate_field | (uk)degenerate_field | degenerate_field | |
121
+ | LoadDate | LoadDate | LoadDate | |
122
+ | RecordSource | RecordSource | RecordSource | |
123
+ | FLDs... | | FLDs... | |
124
+
125
+
126
+ ### Schemas:
127
+ * `core` - framework-related code
128
+ * `stg` - staging layer for both virtual (views) and materialized (tables)
129
+ * `hub` - hub layer
130
+ * `sat` - satellite layer
131
+ * `dim` - dimension layer (information vault)
132
+ * `fact` - fact layer (information vault)
133
+ * `elt` - ELT procedures
134
+ * `job` - top level ELT procedures
135
+ * `meta` - metadata vault
136
+ * `proxy` - source data for materialized staging area (meant for wrapping external data sources as SQL views)
137
+
138
+ ### DV2-related schemas layering
139
+ | LoB* | staging | raw vault | business vault | information vault |
140
+ |-------|---------|-----------|----------------|-------------------|
141
+ | proxy | stg | hub | sal | dim |
142
+ | | | sat | | fact |
143
+ | | | link | | |
144
+ _* Line of Business applications_
sandwich-0.2.0/README.md ADDED
@@ -0,0 +1,124 @@
1
+ ## Data Vault 2.0 scaffolding tool
2
+ This tool is designed to streamline the process of creating Data Vault 2.0 entities,
3
+ such as hubs, links, and satellites.
4
+ As well as building information layer objects such as dim and fact tables
5
+ from a multidimensional paradigm.
6
+
7
+ ### How it works:
8
+ User: provides a staging view `stg.[entity_name]` (or a table if the staging layer persisted)
9
+ with all requirements for the `[entity_name]` defined in the schema (how to define see below).
10
+ Tool:
11
+ 1. Validates metadata of the provided staging view or table.
12
+ 2. Generates the necessary DDL statements to create the Data Vault 2.0 entities.
13
+ 3. Generates ELT procedures to load data to the generated entities.
14
+ 4. Generates support procedures such as `meta.Drop_all_related_to_[entity_name]` and `elt.Run_all_related_to_[entity_name]`
15
+
16
+ ```text
17
+ +----------------------+
18
+ | hub.[entity_name] |
19
+ +----------------------+
20
+ ^
21
+ o 1.define +-------------------+ | 3.create
22
+ /|\ -------> | stg.[entity_name] | # +----------------------+
23
+ / \ +-------------------+ /|\ ---------> | sat.[entity_name] |
24
+ User ---------------------------------------> / \ 3.create +----------------------+
25
+ 2.use Tool
26
+ | 3.create
27
+ v
28
+ +----------------------+
29
+ | dim.[entity_name] |
30
+ +----------------------+
31
+
32
+ ```
33
+
34
+ ### How to define a staging view or table:
35
+ * `bk_` (BusinessKey) - at least one `bk_` column
36
+ * `hk_[entity_name]` (HashKey) - exactly one `hk_[entity_name]` column if you want a `hub` table created
37
+ * `LoadDate` - required by dv2 standard for an auditability
38
+ * `RecordSource` - required by dv2 standard for an auditability
39
+ * `HashDiff` - optional, required if you want to have a scd2 type `dim` table created
40
+ * `IsAvailable` - optional, required if you want to track missing/deleted records
41
+ * all other columns will be considered as business columns and will be included to the `sat` table definition
42
+
43
+
44
+ | staging fields | scd2dim profile |
45
+ |--------------------|-----------------|
46
+ | bk_ | ✅ |
47
+ | hk_`[entity_name]` | ✅ |
48
+ | LoadDate | ✅ |
49
+ | RecordSource | ✅ |
50
+ | HashDiff | ✅ |
51
+ | IsAvailable | ✅ |
52
+
53
+ ```sql
54
+ -- staging view example for the scd2dim profile (mssql)
55
+ create view [stg].[UR_officers] as
56
+ select cast(31 as bigint) [bk_id]
57
+ , core.StringToHash1(cast(31 as bigint)) [hk_UR_officers]
58
+ , sysdatetime() [LoadDate]
59
+ , cast('LobSystem.dbo.officers_daily' as varchar(200)) [RecordSource]
60
+ , core.StringToHash8(
61
+ cast('uri' as nvarchar(100))
62
+ , cast('00000000000000' as varchar(20))
63
+ , cast('NATURAL_PERSON' as varchar(50))
64
+ , cast(null as varchar(20))
65
+ , cast('INDIVIDUALLY' as varchar(50))
66
+ , cast(0 as int)
67
+ , cast('2008-04-07' as date)
68
+ , cast('2008-04-07 18:00:54.000' as datetime)
69
+ ) [HashDiff]
70
+ , cast('uri' as nvarchar(100)) [uri]
71
+ , cast('00000000000000' as varchar(20)) [at_legal_entity_registration_number]
72
+ , cast('NATURAL_PERSON' as varchar(50)) [entity_type]
73
+ , cast(null as varchar(20)) [legal_entity_registration_number]
74
+ , cast('INDIVIDUALLY' as varchar(50)) [rights_of_representation_type]
75
+ , cast(0 as int) [representation_with_at_least]
76
+ , cast('2008-04-07' as date) [registered_on]
77
+ , cast('2008-04-07 18:00:54.000' as datetime) [last_modified_at]
78
+ , cast(1 as bit) [IsAvailable]
79
+ ```
80
+ ### scd2dim profile example:
81
+ | stg | hub | sat | dim |
82
+ |--------------------|------------------------|----------------------------|--------------------|
83
+ | | | | hk_`[entity_name]` |
84
+ | BKs... | (uk)BKs... | BKs... | (pk)BKs... |
85
+ | hk_`[entity_name]` | (pk)hk_`[entity_name]` | (pk)(fk)hk_`[entity_name]` | |
86
+ | LoadDate | LoadDate | (pk)LoadDate | |
87
+ | RecordSource | RecordSource | RecordSource | |
88
+ | HashDiff | | HashDiff | |
89
+ | FLDs... | | FLDs... | FLDs... |
90
+ | IsAvailable | | IsAvailable | IsAvailable |
91
+ | | | | IsCurrent |
92
+ | | | | (pk)DateFrom |
93
+ | | | | DateTo |
94
+
95
+ ### link2fact profile example:
96
+ | stg | link | sat | fact |
97
+ |--------------------|------------------------|----------------------------|------|
98
+ | HKs... | (uk)(fk)HKs... | | |
99
+ | hk_`[entity_name]` | (pk)hk_`[entity_name]` | (pk)(fk)hk_`[entity_name]` | |
100
+ | degenerate_field | (uk)degenerate_field | degenerate_field | |
101
+ | LoadDate | LoadDate | LoadDate | |
102
+ | RecordSource | RecordSource | RecordSource | |
103
+ | FLDs... | | FLDs... | |
104
+
105
+
106
+ ### Schemas:
107
+ * `core` - framework-related code
108
+ * `stg` - staging layer for both virtual (views) and materialized (tables)
109
+ * `hub` - hub layer
110
+ * `sat` - satellite layer
111
+ * `dim` - dimension layer (information vault)
112
+ * `fact` - fact layer (information vault)
113
+ * `elt` - ELT procedures
114
+ * `job` - top level ELT procedures
115
+ * `meta` - metadata vault
116
+ * `proxy` - source data for materialized staging area (meant for wrapping external data sources as SQL views)
117
+
118
+ ### DV2-related schemas layering
119
+ | LoB* | staging | raw vault | business vault | information vault |
120
+ |-------|---------|-----------|----------------|-------------------|
121
+ | proxy | stg | hub | sal | dim |
122
+ | | | sat | | fact |
123
+ | | | link | | |
124
+ _* Line of Business applications_
{sandwich-0.1.0 → sandwich-0.2.0}/pyproject.toml RENAMED
@@ -1,38 +1,40 @@
1
- [project]
2
- name = "sandwich"
3
- version = "0.1.0"
4
- description = "DataVault 2.0 code gen"
5
- readme = "README.md"
6
- requires-python = ">=3.12"
7
- authors = [
8
- { name = "Andrey Morozov", email = "andrey@morozov.lv" }
9
- ]
10
- keywords = ["DWH", "Data Vault 2.0"]
11
- classifiers = [
12
- "Programming Language :: Python :: 3",
13
- "License :: OSI Approved :: MIT License",
14
- "Operating System :: OS Independent",
15
- "Environment :: Console",
16
- "Development Status :: 1 - Planning",
17
- "Intended Audience :: Developers",
18
- ]
19
- dependencies = [
20
- "pyodbc>=5.3.0",
21
- "sqlalchemy>=2.0.44",
22
- ]
23
-
24
- [dependency-groups]
25
- dev = [
26
- "dotenv>=0.9.9",
27
- "mypy>=1.19.0",
28
- "pytest>=9.0.1",
29
- ]
30
-
31
- [build-system]
32
- requires = ["hatchling"]
33
- build-backend = "hatchling.build"
34
-
35
- [tool.hatch.build]
36
- include = [
37
- "sandwich/py.typed",
38
- ]
1
+ [project]
2
+ name = "sandwich"
3
+ version = "0.2.0"
4
+ description = "DataVault 2.0 code gen"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ authors = [
8
+ { name = "Andrey Morozov", email = "andrey@morozov.lv" }
9
+ ]
10
+ keywords = ["DWH", "Data Vault 2.0"]
11
+ classifiers = [
12
+ "Programming Language :: Python :: 3",
13
+ "License :: OSI Approved :: MIT License",
14
+ "Operating System :: OS Independent",
15
+ "Environment :: Console",
16
+ "Development Status :: 1 - Planning",
17
+ "Intended Audience :: Developers",
18
+ "Typing :: Typed",
19
+ ]
20
+ dependencies = [
21
+ "pyodbc>=5.3.0",
22
+ "sqlalchemy>=2.0.44",
23
+ "dotenv>=0.9.9",
24
+ ]
25
+
26
+ [dependency-groups]
27
+ dev = [
28
+ "mypy>=1.19.0",
29
+ "psycopg2>=2.9.11",
30
+ "pytest>=9.0.1",
31
+ ]
32
+
33
+ [build-system]
34
+ requires = ["hatchling"]
35
+ build-backend = "hatchling.build"
36
+
37
+ [tool.hatch.build]
38
+ include = [
39
+ "sandwich/py.typed",
40
+ ]
sandwich-0.1.0/PKG-INFO DELETED
@@ -1,16 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: sandwich
3
- Version: 0.1.0
4
- Summary: DataVault 2.0 code gen
5
- Author-email: Andrey Morozov <andrey@morozov.lv>
6
- License-File: LICENSE
7
- Keywords: DWH,Data Vault 2.0
8
- Classifier: Development Status :: 1 - Planning
9
- Classifier: Environment :: Console
10
- Classifier: Intended Audience :: Developers
11
- Classifier: License :: OSI Approved :: MIT License
12
- Classifier: Operating System :: OS Independent
13
- Classifier: Programming Language :: Python :: 3
14
- Requires-Python: >=3.12
15
- Requires-Dist: pyodbc>=5.3.0
16
- Requires-Dist: sqlalchemy>=2.0.44
sandwich-0.1.0/README.md DELETED
File without changes
{sandwich-0.1.0 → sandwich-0.2.0}/LICENSE RENAMED
File without changes
{sandwich-0.1.0 → sandwich-0.2.0}/sandwich/py.typed RENAMED
File without changes