nesso-cli 0.10.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nesso_cli-0.10.10/PKG-INFO +106 -0
- nesso_cli-0.10.10/README.md +37 -0
- nesso_cli-0.10.10/pyproject.toml +117 -0
- nesso_cli-0.10.10/src/nesso_cli/__init__.py +14 -0
- nesso_cli-0.10.10/src/nesso_cli/jobs/.gitkeep +0 -0
- nesso_cli-0.10.10/src/nesso_cli/jobs/main.py +3 -0
- nesso_cli-0.10.10/src/nesso_cli/models/__init__.py +0 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/__init__.py +0 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/__init__.py +2 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/LICENSE.md +26 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/__init__.py +25 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/templater.py +234 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/project.py +6294 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/sqlfluff_util.py +142 -0
- nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/state.py +1 -0
- nesso_cli-0.10.10/src/nesso_cli/models/base_model.py +347 -0
- nesso_cli-0.10.10/src/nesso_cli/models/common.py +529 -0
- nesso_cli-0.10.10/src/nesso_cli/models/config.py +49 -0
- nesso_cli-0.10.10/src/nesso_cli/models/context.py +20 -0
- nesso_cli-0.10.10/src/nesso_cli/models/init.py +594 -0
- nesso_cli-0.10.10/src/nesso_cli/models/main.py +121 -0
- nesso_cli-0.10.10/src/nesso_cli/models/metadata.py +24 -0
- nesso_cli-0.10.10/src/nesso_cli/models/model.py +129 -0
- nesso_cli-0.10.10/src/nesso_cli/models/models.py +947 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/dbt_project.yml +3 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/codegen_helpers.sql +285 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/create_description_markdown.sql +21 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/dbt_profiler.sql +368 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_base_model.sql +45 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_model_yaml_boilerplate.sql +362 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_seed_yaml_boilerplate.sql +72 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_source_yaml_boilerplate.sql +149 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_custom_schema.sql +8 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_source_pii_columns.sql +19 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_table_columns.sql +23 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/hash_column.sql +15 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/hash_source_pii_columns.sql +13 -0
- nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/redshift_external_tables_fix.sql +36 -0
- nesso_cli-0.10.10/src/nesso_cli/models/seed.py +264 -0
- nesso_cli-0.10.10/src/nesso_cli/models/source.py +489 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/.gitignore +158 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/extensions.list +40 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/install_extensions.sh +1 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/settings.json +27 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/CONTRIBUTING.md +9 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/README.md +45 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/config.yml +10 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/dbt_project.yml +51 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/packages.yml +20 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/prepare.sh +50 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/requirements.txt +1 -0
- nesso_cli-0.10.10/src/nesso_cli/models/templates/{{ bronze_schema }}.yml +6 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/.env.example +1 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/compose.yml +17 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/conftest.py +612 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/.nesso/config.yml +10 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/dbt_project.yml +33 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/packages.yml +8 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/profiles.yml +23 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/average salary test.xlsx +0 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/countries_example.csv +9 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/dbt_project.yml +33 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/packages.yml +7 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/profiles.yml +12 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/functional/test_da_workflow.py +239 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/pytest.ini +6 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_base_model.py +517 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_common.py +263 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_dbt_macros.py +65 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_init.py +426 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_main.py +49 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_metadata.py +13 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_model.py +157 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_models.py +256 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_seed.py +324 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_source.py +553 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_update.py +626 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/test_yaml_generators.py +24 -0
- nesso_cli-0.10.10/src/nesso_cli/models/tests/validate_tables.py +337 -0
- nesso_cli-0.10.10/src/nesso_cli/models/update.py +128 -0
- nesso_cli-0.10.10/src/nesso_cli/models/yaml_generators.py +45 -0
- nesso_cli-0.10.10/src/nesso_cli/nesso_cli.py +39 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: nesso_cli
|
|
3
|
+
Version: 0.10.10
|
|
4
|
+
Summary: A CLI tool for managing data models.
|
|
5
|
+
Keywords: cli,dbt,dyvenia,data,nesso,models
|
|
6
|
+
Author-email: Michał Zawadzki <mzawadzki@dyvenia.com>
|
|
7
|
+
Requires-Python: >=3.10, <3.13
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: loguru == 0.7.2
|
|
10
|
+
Requires-Dist: python-dotenv == 1.0.0
|
|
11
|
+
Requires-Dist: ruamel.yaml == 0.18.5
|
|
12
|
+
Requires-Dist: pydantic >=1.10.11,<2
|
|
13
|
+
Requires-Dist: typer>=0.9.0
|
|
14
|
+
Requires-Dist: nesso_cli[trino] ; extra == "all"
|
|
15
|
+
Requires-Dist: nesso_cli[redshift] ; extra == "all"
|
|
16
|
+
Requires-Dist: nesso_cli[databricks] ; extra == "all"
|
|
17
|
+
Requires-Dist: nesso_cli[sqlserver] ; extra == "all"
|
|
18
|
+
Requires-Dist: nesso_cli[postgres] ; extra == "all"
|
|
19
|
+
Requires-Dist: nesso_cli[duckdb] ; extra == "all"
|
|
20
|
+
Requires-Dist: dbt-databricks == 1.3.2 ; extra == "databricks"
|
|
21
|
+
Requires-Dist: nesso_cli[test] ; extra == "dev"
|
|
22
|
+
Requires-Dist: nesso_cli[docs] ; extra == "dev"
|
|
23
|
+
Requires-Dist: dbt-osmosis>=0.12.4 ; extra == "dev"
|
|
24
|
+
Requires-Dist: sqlfluff>=2.3.5 ; extra == "dev"
|
|
25
|
+
Requires-Dist: mkdocstrings[python] == 0.24.0 ; extra == "docs"
|
|
26
|
+
Requires-Dist: mkdocs-material[imaging] == 9.4.11 ; extra == "docs"
|
|
27
|
+
Requires-Dist: mkdocs-include-dir-to-nav == 1.2.0 ; extra == "docs"
|
|
28
|
+
Requires-Dist: neoteroi-mkdocs == 1.0.4 ; extra == "docs"
|
|
29
|
+
Requires-Dist: mkdocs-jupyter == 0.24.6 ; extra == "docs"
|
|
30
|
+
Requires-Dist: mike == 2.0.0 ; extra == "docs"
|
|
31
|
+
Requires-Dist: mkdocs-table-reader-plugin == 2.0.3 ; extra == "docs"
|
|
32
|
+
Requires-Dist: mkdocs-include-markdown-plugin == 6.0.4 ; extra == "docs"
|
|
33
|
+
Requires-Dist: mkdocs-git-revision-date-plugin == 0.3.2 ; extra == "docs"
|
|
34
|
+
Requires-Dist: mkdocs-glightbox>=0.3.5 ; extra == "docs"
|
|
35
|
+
Requires-Dist: mkdocs-material-extensions==1.3.1 ; extra == "docs"
|
|
36
|
+
Requires-Dist: dbt-duckdb == 1.7.4 ; extra == "duckdb"
|
|
37
|
+
Requires-Dist: duckdb == 0.10.1 ; extra == "duckdb"
|
|
38
|
+
Requires-Dist: dbt-postgres == 1.7.12 ; extra == "postgres"
|
|
39
|
+
Requires-Dist: dbt-redshift == 1.3.0 ; extra == "redshift"
|
|
40
|
+
Requires-Dist: dyvenia-dbt-sqlserver == 1.3.2 ; extra == "sqlserver"
|
|
41
|
+
Requires-Dist: pytest == 7.4.3 ; extra == "test"
|
|
42
|
+
Requires-Dist: pytest-cov == 4.1.0 ; extra == "test"
|
|
43
|
+
Requires-Dist: pytest-env == 1.1.1 ; extra == "test"
|
|
44
|
+
Requires-Dist: coverage == 7.3.2 ; extra == "test"
|
|
45
|
+
Requires-Dist: black == 22.12.0 ; extra == "test"
|
|
46
|
+
Requires-Dist: mypy == 1.7.0 ; extra == "test"
|
|
47
|
+
Requires-Dist: sqlalchemy == 1.4.50 ; extra == "test"
|
|
48
|
+
Requires-Dist: faker == 20.1.0 ; extra == "test"
|
|
49
|
+
Requires-Dist: getkey == 0.6.5 ; extra == "test"
|
|
50
|
+
Requires-Dist: pandas == 1.5.3 ; extra == "test"
|
|
51
|
+
Requires-Dist: mock == 5.1.0 ; extra == "test"
|
|
52
|
+
Requires-Dist: genbadge[coverage] == 1.1.0 ; extra == "test"
|
|
53
|
+
Requires-Dist: flake8 == 6.1.0 ; extra == "test"
|
|
54
|
+
Requires-Dist: Flake8-pyproject == 1.2.3 ; extra == "test"
|
|
55
|
+
Requires-Dist: tomlkit == 0.12.3 ; extra == "test"
|
|
56
|
+
Requires-Dist: interrogate[png] == 1.5.0 ; extra == "test"
|
|
57
|
+
Requires-Dist: dbt-postgres == 1.7.12 ; extra == "test"
|
|
58
|
+
Requires-Dist: dbt-trino == 1.3.2 ; extra == "trino"
|
|
59
|
+
Provides-Extra: all
|
|
60
|
+
Provides-Extra: databricks
|
|
61
|
+
Provides-Extra: dev
|
|
62
|
+
Provides-Extra: docs
|
|
63
|
+
Provides-Extra: duckdb
|
|
64
|
+
Provides-Extra: postgres
|
|
65
|
+
Provides-Extra: redshift
|
|
66
|
+
Provides-Extra: sqlserver
|
|
67
|
+
Provides-Extra: test
|
|
68
|
+
Provides-Extra: trino
|
|
69
|
+
|
|
70
|
+
# nesso-cli
|
|
71
|
+
|
|
72
|
+

|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
**Documentation**: 📚 [dyvenia docs (internal)][mkdocs page]
|
|
76
|
+
|
|
77
|
+
**Source Code**: 💾 [dyvenia/nesso-cli][github page]
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
<!-- body-begin -->
|
|
82
|
+
|
|
83
|
+
The [CLI](https://www.w3schools.com/whatis/whatis_cli.asp) interface of the [nesso data platform].
|
|
84
|
+
|
|
85
|
+
## Features
|
|
86
|
+
|
|
87
|
+
- [x] simplify and automate data modelling
|
|
88
|
+
- [x] simplify and automate metadata generation
|
|
89
|
+
- [x] manage nesso project configuration
|
|
90
|
+
- [ ] simplify and automate job scheduling (coming soon!)
|
|
91
|
+
|
|
92
|
+
## Where does nesso-cli fit in?
|
|
93
|
+
|
|
94
|
+
Currently, nesso-cli contains a single module, `models` (`nesso models`), which is used for the T in ELTC (Extract, Load, Transform, Catalog), sitting between data ingestion (`viadot`) and metadata ingestion (`luma-cli`):
|
|
95
|
+
|
|
96
|
+

|
|
97
|
+
|
|
98
|
+
In the future, nesso-cli will include additional modules to allow interacting with different components of the nesso data platform through a unified interface.
|
|
99
|
+
|
|
100
|
+
The next planned module is `jobs`, which will allow creating and scheduling EL and ELTC jobs via a simple CLI interface. Currently, this is done by creating jobs manually in Python and then manually scheduling them in Prefect. We hope to replace this tedious and error-prone (though repeatable) process with simple commands, such as `nesso jobs deployment create --job my_job --schedule "0 0 * * *"`, as well as interactive commands which will guide user through a set of limited choices, such as `nesso jobs job create`.
|
|
101
|
+
|
|
102
|
+
[github page]: https://github.com/dyvenia/nesso-cli
|
|
103
|
+
[mkdocs page]: https://nesso-cli.docs.dyvenia.com/
|
|
104
|
+
[nesso data platform]: https://nesso.docs.dyvenia.com/
|
|
105
|
+
|
|
106
|
+
<!-- body-end -->
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# nesso-cli
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
**Documentation**: 📚 [dyvenia docs (internal)][mkdocs page]
|
|
7
|
+
|
|
8
|
+
**Source Code**: 💾 [dyvenia/nesso-cli][github page]
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
<!-- body-begin -->
|
|
13
|
+
|
|
14
|
+
The [CLI](https://www.w3schools.com/whatis/whatis_cli.asp) interface of the [nesso data platform].
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
- [x] simplify and automate data modelling
|
|
19
|
+
- [x] simplify and automate metadata generation
|
|
20
|
+
- [x] manage nesso project configuration
|
|
21
|
+
- [ ] simplify and automate job scheduling (coming soon!)
|
|
22
|
+
|
|
23
|
+
## Where does nesso-cli fit in?
|
|
24
|
+
|
|
25
|
+
Currently, nesso-cli contains a single module, `models` (`nesso models`), which is used for the T in ELTC (Extract, Load, Transform, Catalog), sitting between data ingestion (`viadot`) and metadata ingestion (`luma-cli`):
|
|
26
|
+
|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
In the future, nesso-cli will include additional modules to allow interacting with different components of the nesso data platform through a unified interface.
|
|
30
|
+
|
|
31
|
+
The next planned module is `jobs`, which will allow creating and scheduling EL and ELTC jobs via a simple CLI interface. Currently, this is done by creating jobs manually in Python and then manually scheduling them in Prefect. We hope to replace this tedious and error-prone (though repeatable) process with simple commands, such as `nesso jobs deployment create --job my_job --schedule "0 0 * * *"`, as well as interactive commands which will guide user through a set of limited choices, such as `nesso jobs job create`.
|
|
32
|
+
|
|
33
|
+
[github page]: https://github.com/dyvenia/nesso-cli
|
|
34
|
+
[mkdocs page]: https://nesso-cli.docs.dyvenia.com/
|
|
35
|
+
[nesso data platform]: https://nesso.docs.dyvenia.com/
|
|
36
|
+
|
|
37
|
+
<!-- body-end -->
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "nesso_cli"
|
|
3
|
+
description = "A CLI tool for managing data models."
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
version = "0.10.10"
|
|
6
|
+
authors = [{ name = "Michał Zawadzki", email = "mzawadzki@dyvenia.com" }]
|
|
7
|
+
classifiers = []
|
|
8
|
+
keywords = ["cli", "dbt", "dyvenia", "data", "nesso", "models"]
|
|
9
|
+
requires-python = ">=3.10, <3.13"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"loguru == 0.7.2",
|
|
12
|
+
"python-dotenv == 1.0.0",
|
|
13
|
+
"ruamel.yaml == 0.18.5",
|
|
14
|
+
"pydantic >=1.10.11,<2",
|
|
15
|
+
"typer>=0.9.0",
|
|
16
|
+
# Below fork of the dbt-core-interface is vendored till https://github.com/z3z1ma/dbt-core-interface/pull/117 is merged.
|
|
17
|
+
# "dbt-core-interface @ git+https://github.com/trymzet/dbt-core-interface",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.optional-dependencies]
|
|
21
|
+
test = [
|
|
22
|
+
"pytest == 7.4.3",
|
|
23
|
+
"pytest-cov == 4.1.0",
|
|
24
|
+
"pytest-env == 1.1.1",
|
|
25
|
+
"coverage == 7.3.2",
|
|
26
|
+
"black == 22.12.0",
|
|
27
|
+
"mypy == 1.7.0",
|
|
28
|
+
"sqlalchemy == 1.4.50",
|
|
29
|
+
"faker == 20.1.0",
|
|
30
|
+
"getkey == 0.6.5",
|
|
31
|
+
"pandas == 1.5.3",
|
|
32
|
+
"mock == 5.1.0",
|
|
33
|
+
"genbadge[coverage] == 1.1.0",
|
|
34
|
+
"flake8 == 6.1.0",
|
|
35
|
+
"Flake8-pyproject == 1.2.3",
|
|
36
|
+
"tomlkit == 0.12.3",
|
|
37
|
+
"interrogate[png] == 1.5.0",
|
|
38
|
+
"dbt-postgres == 1.7.12",
|
|
39
|
+
]
|
|
40
|
+
docs = [
|
|
41
|
+
"mkdocstrings[python] == 0.24.0",
|
|
42
|
+
"mkdocs-material[imaging] == 9.4.11",
|
|
43
|
+
"mkdocs-include-dir-to-nav == 1.2.0",
|
|
44
|
+
"neoteroi-mkdocs == 1.0.4",
|
|
45
|
+
"mkdocs-jupyter == 0.24.6",
|
|
46
|
+
"mike == 2.0.0",
|
|
47
|
+
"mkdocs-table-reader-plugin == 2.0.3",
|
|
48
|
+
"mkdocs-include-markdown-plugin == 6.0.4",
|
|
49
|
+
"mkdocs-git-revision-date-plugin == 0.3.2",
|
|
50
|
+
"mkdocs-glightbox>=0.3.5",
|
|
51
|
+
"mkdocs-material-extensions==1.3.1",
|
|
52
|
+
]
|
|
53
|
+
dev = [
|
|
54
|
+
"nesso_cli[test]",
|
|
55
|
+
"nesso_cli[docs]",
|
|
56
|
+
"dbt-osmosis>=0.12.4",
|
|
57
|
+
"sqlfluff>=2.3.5",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
trino = ["dbt-trino == 1.3.2"]
|
|
61
|
+
redshift = ["dbt-redshift == 1.3.0"]
|
|
62
|
+
databricks = ["dbt-databricks == 1.3.2"]
|
|
63
|
+
# Workaround till https://github.com/dbt-msft/dbt-sqlserver/pull/460 is merged.
|
|
64
|
+
sqlserver = ["dyvenia-dbt-sqlserver == 1.3.2"]
|
|
65
|
+
postgres = ["dbt-postgres == 1.7.12"]
|
|
66
|
+
duckdb = ["dbt-duckdb == 1.7.4", "duckdb == 0.10.1"]
|
|
67
|
+
|
|
68
|
+
all = [
|
|
69
|
+
"nesso_cli[trino]",
|
|
70
|
+
"nesso_cli[redshift]",
|
|
71
|
+
"nesso_cli[databricks]",
|
|
72
|
+
"nesso_cli[sqlserver]",
|
|
73
|
+
"nesso_cli[postgres]",
|
|
74
|
+
"nesso_cli[duckdb]",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[project.scripts]
|
|
78
|
+
nesso = "nesso_cli.nesso_cli:cli"
|
|
79
|
+
|
|
80
|
+
[tool.setuptools]
|
|
81
|
+
packages = ["cli"]
|
|
82
|
+
|
|
83
|
+
[tool.black]
|
|
84
|
+
line-length = 88
|
|
85
|
+
exclude = '''
|
|
86
|
+
/(
|
|
87
|
+
\.git
|
|
88
|
+
| \.mypy_cache
|
|
89
|
+
| \.tox
|
|
90
|
+
| venv
|
|
91
|
+
| \.venv
|
|
92
|
+
| _build
|
|
93
|
+
| buck-out
|
|
94
|
+
| build
|
|
95
|
+
| dist
|
|
96
|
+
)/
|
|
97
|
+
'''
|
|
98
|
+
|
|
99
|
+
[tool.isort]
|
|
100
|
+
# make it compatible with Black
|
|
101
|
+
profile = "black"
|
|
102
|
+
|
|
103
|
+
[tool.flake8]
|
|
104
|
+
extend-ignore = "F722"
|
|
105
|
+
exclude = "_vendored"
|
|
106
|
+
|
|
107
|
+
max-line-length = 88
|
|
108
|
+
[tool.interrogate]
|
|
109
|
+
fail-under = 40
|
|
110
|
+
quiet = false
|
|
111
|
+
exclude = ["src/nesso_cli/models/tests/"]
|
|
112
|
+
generate-badge = "coverage/docstring_coverage.svg"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
[build-system]
|
|
116
|
+
build-backend = "flit_core.buildapi"
|
|
117
|
+
requires = ["flit_core >=3.8,<4"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Description placeholder (required by Flit)."""
|
|
2
|
+
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
|
|
5
|
+
from .jobs import main # noqa
|
|
6
|
+
from .models import base_model, common, main, model, seed, source # noqa
|
|
7
|
+
|
|
8
|
+
# This allows specifying the version in a single place (`pyproject.toml`).
|
|
9
|
+
# See https://gist.github.com/benkehoe/066a73903e84576a8d6d911cfedc2df6.
|
|
10
|
+
try:
|
|
11
|
+
# __package__ allows for the case where __name__ is "__main__"
|
|
12
|
+
__version__ = importlib.metadata.version(__package__ or __name__)
|
|
13
|
+
except importlib.metadata.PackageNotFoundError:
|
|
14
|
+
__version__ = "0.0.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Portions of this software are adapted from the sqlfluff-templater-dbt
|
|
2
|
+
package. The original license and copyright are reproduced below.
|
|
3
|
+
|
|
4
|
+
==
|
|
5
|
+
|
|
6
|
+
MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2018 Alan Cruickshank
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Defines the hook endpoints for the dbt templater plugin."""
|
|
2
|
+
import logging
|
|
3
|
+
|
|
4
|
+
from sqlfluff.core.plugin import hookimpl
|
|
5
|
+
|
|
6
|
+
from nesso_cli.models._vendored.dbt_core_interface.dbt_templater.templater import (
|
|
7
|
+
DCIDbtTemplater,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
LOGGER = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@hookimpl
|
|
14
|
+
def get_templaters():
|
|
15
|
+
"""Get templaters."""
|
|
16
|
+
def create_templater(**kwargs):
|
|
17
|
+
import nesso_cli.models.dbt_core_interface.state
|
|
18
|
+
assert nesso_cli.models.dbt_core_interface.state.dbt_project_container is not None, "dbt_project_container is None"
|
|
19
|
+
return DCIDbtTemplater(
|
|
20
|
+
dbt_project_container=nesso_cli.models.dbt_core_interface.state.dbt_project_container,
|
|
21
|
+
**kwargs
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
create_templater.name = DCIDbtTemplater.name
|
|
25
|
+
return [create_templater]
|
nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/templater.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Specialized dbt templater for embedding in the dbt-core-interface server."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os.path
|
|
5
|
+
import threading
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from dbt.version import get_installed_version
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
from dbt.exceptions import (
|
|
13
|
+
CompilationException as DbtCompilationException,
|
|
14
|
+
)
|
|
15
|
+
except ImportError:
|
|
16
|
+
from dbt.exceptions import (
|
|
17
|
+
CompilationError as DbtCompilationException,
|
|
18
|
+
)
|
|
19
|
+
from jinja2 import Environment
|
|
20
|
+
from jinja2_simple_tags import StandaloneTag
|
|
21
|
+
from sqlfluff.core.errors import SQLFluffSkipFile, SQLTemplaterError
|
|
22
|
+
from sqlfluff.core.templaters.base import TemplatedFile, large_file_check
|
|
23
|
+
from sqlfluff.core.templaters.jinja import JinjaTemplater
|
|
24
|
+
|
|
25
|
+
from nesso_cli.models._vendored.dbt_core_interface.project import DbtProject
|
|
26
|
+
|
|
27
|
+
templater_logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
DBT_VERSION = get_installed_version()
|
|
30
|
+
DBT_VERSION_TUPLE = (int(DBT_VERSION.major), int(DBT_VERSION.minor))
|
|
31
|
+
|
|
32
|
+
if DBT_VERSION_TUPLE >= (1, 3):
|
|
33
|
+
COMPILED_SQL_ATTRIBUTE = "compiled_code"
|
|
34
|
+
RAW_SQL_ATTRIBUTE = "raw_code"
|
|
35
|
+
else:
|
|
36
|
+
COMPILED_SQL_ATTRIBUTE = "compiled_sql"
|
|
37
|
+
RAW_SQL_ATTRIBUTE = "raw_sql"
|
|
38
|
+
|
|
39
|
+
local = threading.local()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class DCIDbtTemplater(JinjaTemplater):
|
|
43
|
+
"""dbt templater for dbt-core-interface, based on sqlfluff-templater-dbt."""
|
|
44
|
+
|
|
45
|
+
# Same templater name as sqlfluff-templater-dbt. It is functionally
|
|
46
|
+
# equivalent to that templater, but optimized for dbt-core-interface.
|
|
47
|
+
# The two templaters cannot be installed in the same virtualenv.
|
|
48
|
+
name = "dbt"
|
|
49
|
+
|
|
50
|
+
def __init__(self, **kwargs):
|
|
51
|
+
self.dbt_project_container = kwargs.pop("dbt_project_container")
|
|
52
|
+
super().__init__(**kwargs)
|
|
53
|
+
|
|
54
|
+
def config_pairs(self): # pragma: no cover
|
|
55
|
+
"""Returns info about the given templater for output by the cli."""
|
|
56
|
+
return [("templater", self.name), ("dbt", DBT_VERSION.to_version_string())]
|
|
57
|
+
|
|
58
|
+
@large_file_check
|
|
59
|
+
def process(self, *, fname: str, in_str=None, config, **kwargs):
|
|
60
|
+
"""Compile a dbt model and return the compiled SQL."""
|
|
61
|
+
try:
|
|
62
|
+
return self._unsafe_process(
|
|
63
|
+
os.path.abspath(fname) if in_str is None else None, in_str, config
|
|
64
|
+
)
|
|
65
|
+
except DbtCompilationException as e:
|
|
66
|
+
if e.node:
|
|
67
|
+
return None, [
|
|
68
|
+
SQLTemplaterError(
|
|
69
|
+
f"dbt compilation error on file '{e.node.original_file_path}', " f"{e.msg}"
|
|
70
|
+
)
|
|
71
|
+
]
|
|
72
|
+
else:
|
|
73
|
+
raise # pragma: no cover
|
|
74
|
+
# If a SQLFluff error is raised, just pass it through
|
|
75
|
+
except SQLTemplaterError as e: # pragma: no cover
|
|
76
|
+
return None, [e]
|
|
77
|
+
|
|
78
|
+
def _find_node(self, project: DbtProject, fname: str):
|
|
79
|
+
expected_node_path = os.path.relpath(fname, start=os.path.abspath(project.config.project_root))
|
|
80
|
+
node = project.get_node_by_path(expected_node_path)
|
|
81
|
+
if node:
|
|
82
|
+
return node
|
|
83
|
+
skip_reason = self._find_skip_reason(project, expected_node_path)
|
|
84
|
+
if skip_reason:
|
|
85
|
+
raise SQLFluffSkipFile(f"Skipped file {fname} because it is {skip_reason}")
|
|
86
|
+
raise SQLFluffSkipFile(f"File {fname} was not found in dbt project") # pragma: no cover
|
|
87
|
+
|
|
88
|
+
@staticmethod
|
|
89
|
+
def _find_skip_reason(project: DbtProject, expected_node_path: str) -> Optional[str]:
|
|
90
|
+
"""Return string reason if model okay to skip, otherwise None."""
|
|
91
|
+
# Scan macros.
|
|
92
|
+
for macro in project.dbt.macros.values():
|
|
93
|
+
if macro.original_file_path == expected_node_path:
|
|
94
|
+
return "a macro"
|
|
95
|
+
|
|
96
|
+
# Scan disabled nodes.
|
|
97
|
+
for nodes in project.dbt.disabled.values():
|
|
98
|
+
for node in nodes:
|
|
99
|
+
if node.original_file_path == expected_node_path:
|
|
100
|
+
return "disabled"
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def from_string(*args, **kwargs):
|
|
105
|
+
"""Replaces (via monkeypatch) the jinja2.Environment function."""
|
|
106
|
+
globals = kwargs.get("globals")
|
|
107
|
+
if globals and hasattr(local, "target_sql"):
|
|
108
|
+
model = globals.get("model")
|
|
109
|
+
if model:
|
|
110
|
+
# Is it processing the node we're interested in?
|
|
111
|
+
if isinstance(local.target_sql, Path):
|
|
112
|
+
the_one = str(local.target_sql) == model.get("original_file_path")
|
|
113
|
+
else:
|
|
114
|
+
the_one = local.target_sql == args[1]
|
|
115
|
+
if the_one:
|
|
116
|
+
# Yes. Capture the important arguments and create
|
|
117
|
+
# a make_template() function.
|
|
118
|
+
env = args[0]
|
|
119
|
+
globals = args[2] if len(args) >= 3 else kwargs["globals"]
|
|
120
|
+
|
|
121
|
+
def make_template(in_str):
|
|
122
|
+
env.add_extension(SnapshotExtension)
|
|
123
|
+
return env.from_string(in_str, globals=globals)
|
|
124
|
+
|
|
125
|
+
local.make_template = make_template
|
|
126
|
+
|
|
127
|
+
return old_from_string(*args, **kwargs)
|
|
128
|
+
|
|
129
|
+
def _unsafe_process(self, fname, in_str, config=None):
|
|
130
|
+
dbt_project = self.dbt_project_container.get_project_by_root_dir(
|
|
131
|
+
config.get_section((self.templater_selector, self.name, "project_dir"))
|
|
132
|
+
)
|
|
133
|
+
local.make_template = None
|
|
134
|
+
try:
|
|
135
|
+
if fname:
|
|
136
|
+
node = self._find_node(dbt_project, fname)
|
|
137
|
+
local.target_sql = Path(
|
|
138
|
+
os.path.relpath(fname, start=dbt_project.config.project_root)
|
|
139
|
+
)
|
|
140
|
+
compiled_node = dbt_project.compile_from_node(node)
|
|
141
|
+
else:
|
|
142
|
+
local.target_sql = in_str
|
|
143
|
+
# TRICKY: Use __wrapped__ to bypass the cache. We *must*
|
|
144
|
+
# recompile each time, because that's how we get the
|
|
145
|
+
# make_template() function.
|
|
146
|
+
compiled_node = dbt_project.compile_code.__wrapped__(
|
|
147
|
+
dbt_project, in_str
|
|
148
|
+
)
|
|
149
|
+
except Exception as err:
|
|
150
|
+
raise SQLFluffSkipFile( # pragma: no cover
|
|
151
|
+
f"Skipped file {fname} because dbt raised a fatal "
|
|
152
|
+
f"exception during compilation: {err!s}"
|
|
153
|
+
) from err
|
|
154
|
+
finally:
|
|
155
|
+
local.target_sql = None
|
|
156
|
+
|
|
157
|
+
if compiled_node.injected_code:
|
|
158
|
+
# If injected SQL is present, it contains a better picture
|
|
159
|
+
# of what will actually hit the database (e.g. with tests).
|
|
160
|
+
# However it's not always present.
|
|
161
|
+
compiled_sql = compiled_node.injected_code
|
|
162
|
+
else:
|
|
163
|
+
compiled_sql = compiled_node.compiled_code
|
|
164
|
+
|
|
165
|
+
raw_sql = compiled_node.raw_code
|
|
166
|
+
|
|
167
|
+
if not compiled_sql: # pragma: no cover
|
|
168
|
+
raise SQLTemplaterError(
|
|
169
|
+
"dbt templater compilation failed silently, check your "
|
|
170
|
+
"configuration by running `dbt compile` directly."
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
if fname:
|
|
174
|
+
with open(fname) as source_dbt_model:
|
|
175
|
+
source_dbt_sql = source_dbt_model.read()
|
|
176
|
+
else:
|
|
177
|
+
source_dbt_sql = in_str
|
|
178
|
+
|
|
179
|
+
if not source_dbt_sql.rstrip().endswith("-%}"):
|
|
180
|
+
n_trailing_newlines = len(source_dbt_sql) - len(source_dbt_sql.rstrip("\n"))
|
|
181
|
+
else:
|
|
182
|
+
# Source file ends with right whitespace stripping, so there's
|
|
183
|
+
# no need to preserve/restore trailing newlines.
|
|
184
|
+
n_trailing_newlines = 0
|
|
185
|
+
|
|
186
|
+
templater_logger.debug(
|
|
187
|
+
" Trailing newline count in source dbt model: %r",
|
|
188
|
+
n_trailing_newlines,
|
|
189
|
+
)
|
|
190
|
+
templater_logger.debug(" Raw SQL before compile: %r", source_dbt_sql)
|
|
191
|
+
templater_logger.debug(" Node raw SQL: %r", raw_sql)
|
|
192
|
+
templater_logger.debug(" Node compiled SQL: %r", compiled_sql)
|
|
193
|
+
|
|
194
|
+
# Adjust for dbt Jinja removing trailing newlines. For more details on
|
|
195
|
+
# this, see the similar code in sqlfluff-templater.dbt.
|
|
196
|
+
compiled_node.raw_sql = source_dbt_sql
|
|
197
|
+
compiled_sql = compiled_sql + "\n" * n_trailing_newlines
|
|
198
|
+
raw_sliced, sliced_file, templated_sql = self.slice_file(
|
|
199
|
+
source_dbt_sql,
|
|
200
|
+
compiled_sql,
|
|
201
|
+
config=config,
|
|
202
|
+
make_template=local.make_template,
|
|
203
|
+
append_to_templated="\n" if n_trailing_newlines else "",
|
|
204
|
+
)
|
|
205
|
+
return (
|
|
206
|
+
TemplatedFile(
|
|
207
|
+
source_str=source_dbt_sql,
|
|
208
|
+
templated_str=templated_sql,
|
|
209
|
+
fname=fname,
|
|
210
|
+
sliced_file=sliced_file,
|
|
211
|
+
raw_sliced=raw_sliced,
|
|
212
|
+
),
|
|
213
|
+
# No violations returned in this way.
|
|
214
|
+
[],
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# Monkeypatch Environment.from_string(). DCIDbtTemplater uses this to
|
|
219
|
+
# intercept Jinja compilation and capture a template trace.
|
|
220
|
+
old_from_string = Environment.from_string
|
|
221
|
+
Environment.from_string = DCIDbtTemplater.from_string
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class SnapshotExtension(StandaloneTag):
|
|
225
|
+
"""Dummy "snapshot" tags so raw dbt templates will parse.
|
|
226
|
+
|
|
227
|
+
For more context, see sqlfluff-templater-dbt.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
tags = {"snapshot", "endsnapshot"}
|
|
231
|
+
|
|
232
|
+
def render(self, format_string=None):
|
|
233
|
+
"""Dummy method that renders the tag."""
|
|
234
|
+
return ""
|