nesso-cli 0.10.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. nesso_cli-0.10.10/PKG-INFO +106 -0
  2. nesso_cli-0.10.10/README.md +37 -0
  3. nesso_cli-0.10.10/pyproject.toml +117 -0
  4. nesso_cli-0.10.10/src/nesso_cli/__init__.py +14 -0
  5. nesso_cli-0.10.10/src/nesso_cli/jobs/.gitkeep +0 -0
  6. nesso_cli-0.10.10/src/nesso_cli/jobs/main.py +3 -0
  7. nesso_cli-0.10.10/src/nesso_cli/models/__init__.py +0 -0
  8. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/__init__.py +0 -0
  9. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/__init__.py +2 -0
  10. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/LICENSE.md +26 -0
  11. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/__init__.py +25 -0
  12. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/dbt_templater/templater.py +234 -0
  13. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/project.py +6294 -0
  14. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/sqlfluff_util.py +142 -0
  15. nesso_cli-0.10.10/src/nesso_cli/models/_vendored/dbt_core_interface/state.py +1 -0
  16. nesso_cli-0.10.10/src/nesso_cli/models/base_model.py +347 -0
  17. nesso_cli-0.10.10/src/nesso_cli/models/common.py +529 -0
  18. nesso_cli-0.10.10/src/nesso_cli/models/config.py +49 -0
  19. nesso_cli-0.10.10/src/nesso_cli/models/context.py +20 -0
  20. nesso_cli-0.10.10/src/nesso_cli/models/init.py +594 -0
  21. nesso_cli-0.10.10/src/nesso_cli/models/main.py +121 -0
  22. nesso_cli-0.10.10/src/nesso_cli/models/metadata.py +24 -0
  23. nesso_cli-0.10.10/src/nesso_cli/models/model.py +129 -0
  24. nesso_cli-0.10.10/src/nesso_cli/models/models.py +947 -0
  25. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/dbt_project.yml +3 -0
  26. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/codegen_helpers.sql +285 -0
  27. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/create_description_markdown.sql +21 -0
  28. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/dbt_profiler.sql +368 -0
  29. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_base_model.sql +45 -0
  30. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_model_yaml_boilerplate.sql +362 -0
  31. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_seed_yaml_boilerplate.sql +72 -0
  32. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/generate_source_yaml_boilerplate.sql +149 -0
  33. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_custom_schema.sql +8 -0
  34. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_source_pii_columns.sql +19 -0
  35. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/get_table_columns.sql +23 -0
  36. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/hash_column.sql +15 -0
  37. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/hash_source_pii_columns.sql +13 -0
  38. nesso_cli-0.10.10/src/nesso_cli/models/nesso_macros/macros/redshift_external_tables_fix.sql +36 -0
  39. nesso_cli-0.10.10/src/nesso_cli/models/seed.py +264 -0
  40. nesso_cli-0.10.10/src/nesso_cli/models/source.py +489 -0
  41. nesso_cli-0.10.10/src/nesso_cli/models/templates/.gitignore +158 -0
  42. nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/extensions.list +40 -0
  43. nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/install_extensions.sh +1 -0
  44. nesso_cli-0.10.10/src/nesso_cli/models/templates/.vscode/settings.json +27 -0
  45. nesso_cli-0.10.10/src/nesso_cli/models/templates/CONTRIBUTING.md +9 -0
  46. nesso_cli-0.10.10/src/nesso_cli/models/templates/README.md +45 -0
  47. nesso_cli-0.10.10/src/nesso_cli/models/templates/config.yml +10 -0
  48. nesso_cli-0.10.10/src/nesso_cli/models/templates/dbt_project.yml +51 -0
  49. nesso_cli-0.10.10/src/nesso_cli/models/templates/packages.yml +20 -0
  50. nesso_cli-0.10.10/src/nesso_cli/models/templates/prepare.sh +50 -0
  51. nesso_cli-0.10.10/src/nesso_cli/models/templates/requirements.txt +1 -0
  52. nesso_cli-0.10.10/src/nesso_cli/models/templates/{{ bronze_schema }}.yml +6 -0
  53. nesso_cli-0.10.10/src/nesso_cli/models/tests/.env.example +1 -0
  54. nesso_cli-0.10.10/src/nesso_cli/models/tests/compose.yml +17 -0
  55. nesso_cli-0.10.10/src/nesso_cli/models/tests/conftest.py +612 -0
  56. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/.nesso/config.yml +10 -0
  57. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/dbt_project.yml +33 -0
  58. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/packages.yml +8 -0
  59. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/profiles.yml +23 -0
  60. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/average salary test.xlsx +0 -0
  61. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/countries_example.csv +9 -0
  62. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/dbt_project.yml +33 -0
  63. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/packages.yml +7 -0
  64. nesso_cli-0.10.10/src/nesso_cli/models/tests/dbt_projects/trino/profiles.yml +12 -0
  65. nesso_cli-0.10.10/src/nesso_cli/models/tests/functional/test_da_workflow.py +239 -0
  66. nesso_cli-0.10.10/src/nesso_cli/models/tests/pytest.ini +6 -0
  67. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_base_model.py +517 -0
  68. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_common.py +263 -0
  69. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_dbt_macros.py +65 -0
  70. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_init.py +426 -0
  71. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_main.py +49 -0
  72. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_metadata.py +13 -0
  73. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_model.py +157 -0
  74. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_models.py +256 -0
  75. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_seed.py +324 -0
  76. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_source.py +553 -0
  77. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_update.py +626 -0
  78. nesso_cli-0.10.10/src/nesso_cli/models/tests/test_yaml_generators.py +24 -0
  79. nesso_cli-0.10.10/src/nesso_cli/models/tests/validate_tables.py +337 -0
  80. nesso_cli-0.10.10/src/nesso_cli/models/update.py +128 -0
  81. nesso_cli-0.10.10/src/nesso_cli/models/yaml_generators.py +45 -0
  82. nesso_cli-0.10.10/src/nesso_cli/nesso_cli.py +39 -0
@@ -0,0 +1,106 @@
1
+ Metadata-Version: 2.1
2
+ Name: nesso_cli
3
+ Version: 0.10.10
4
+ Summary: A CLI tool for managing data models.
5
+ Keywords: cli,dbt,dyvenia,data,nesso,models
6
+ Author-email: Michał Zawadzki <mzawadzki@dyvenia.com>
7
+ Requires-Python: >=3.10, <3.13
8
+ Description-Content-Type: text/markdown
9
+ Requires-Dist: loguru == 0.7.2
10
+ Requires-Dist: python-dotenv == 1.0.0
11
+ Requires-Dist: ruamel.yaml == 0.18.5
12
+ Requires-Dist: pydantic >=1.10.11,<2
13
+ Requires-Dist: typer>=0.9.0
14
+ Requires-Dist: nesso_cli[trino] ; extra == "all"
15
+ Requires-Dist: nesso_cli[redshift] ; extra == "all"
16
+ Requires-Dist: nesso_cli[databricks] ; extra == "all"
17
+ Requires-Dist: nesso_cli[sqlserver] ; extra == "all"
18
+ Requires-Dist: nesso_cli[postgres] ; extra == "all"
19
+ Requires-Dist: nesso_cli[duckdb] ; extra == "all"
20
+ Requires-Dist: dbt-databricks == 1.3.2 ; extra == "databricks"
21
+ Requires-Dist: nesso_cli[test] ; extra == "dev"
22
+ Requires-Dist: nesso_cli[docs] ; extra == "dev"
23
+ Requires-Dist: dbt-osmosis>=0.12.4 ; extra == "dev"
24
+ Requires-Dist: sqlfluff>=2.3.5 ; extra == "dev"
25
+ Requires-Dist: mkdocstrings[python] == 0.24.0 ; extra == "docs"
26
+ Requires-Dist: mkdocs-material[imaging] == 9.4.11 ; extra == "docs"
27
+ Requires-Dist: mkdocs-include-dir-to-nav == 1.2.0 ; extra == "docs"
28
+ Requires-Dist: neoteroi-mkdocs == 1.0.4 ; extra == "docs"
29
+ Requires-Dist: mkdocs-jupyter == 0.24.6 ; extra == "docs"
30
+ Requires-Dist: mike == 2.0.0 ; extra == "docs"
31
+ Requires-Dist: mkdocs-table-reader-plugin == 2.0.3 ; extra == "docs"
32
+ Requires-Dist: mkdocs-include-markdown-plugin == 6.0.4 ; extra == "docs"
33
+ Requires-Dist: mkdocs-git-revision-date-plugin == 0.3.2 ; extra == "docs"
34
+ Requires-Dist: mkdocs-glightbox>=0.3.5 ; extra == "docs"
35
+ Requires-Dist: mkdocs-material-extensions==1.3.1 ; extra == "docs"
36
+ Requires-Dist: dbt-duckdb == 1.7.4 ; extra == "duckdb"
37
+ Requires-Dist: duckdb == 0.10.1 ; extra == "duckdb"
38
+ Requires-Dist: dbt-postgres == 1.7.12 ; extra == "postgres"
39
+ Requires-Dist: dbt-redshift == 1.3.0 ; extra == "redshift"
40
+ Requires-Dist: dyvenia-dbt-sqlserver == 1.3.2 ; extra == "sqlserver"
41
+ Requires-Dist: pytest == 7.4.3 ; extra == "test"
42
+ Requires-Dist: pytest-cov == 4.1.0 ; extra == "test"
43
+ Requires-Dist: pytest-env == 1.1.1 ; extra == "test"
44
+ Requires-Dist: coverage == 7.3.2 ; extra == "test"
45
+ Requires-Dist: black == 22.12.0 ; extra == "test"
46
+ Requires-Dist: mypy == 1.7.0 ; extra == "test"
47
+ Requires-Dist: sqlalchemy == 1.4.50 ; extra == "test"
48
+ Requires-Dist: faker == 20.1.0 ; extra == "test"
49
+ Requires-Dist: getkey == 0.6.5 ; extra == "test"
50
+ Requires-Dist: pandas == 1.5.3 ; extra == "test"
51
+ Requires-Dist: mock == 5.1.0 ; extra == "test"
52
+ Requires-Dist: genbadge[coverage] == 1.1.0 ; extra == "test"
53
+ Requires-Dist: flake8 == 6.1.0 ; extra == "test"
54
+ Requires-Dist: Flake8-pyproject == 1.2.3 ; extra == "test"
55
+ Requires-Dist: tomlkit == 0.12.3 ; extra == "test"
56
+ Requires-Dist: interrogate[png] == 1.5.0 ; extra == "test"
57
+ Requires-Dist: dbt-postgres == 1.7.12 ; extra == "test"
58
+ Requires-Dist: dbt-trino == 1.3.2 ; extra == "trino"
59
+ Provides-Extra: all
60
+ Provides-Extra: databricks
61
+ Provides-Extra: dev
62
+ Provides-Extra: docs
63
+ Provides-Extra: duckdb
64
+ Provides-Extra: postgres
65
+ Provides-Extra: redshift
66
+ Provides-Extra: sqlserver
67
+ Provides-Extra: test
68
+ Provides-Extra: trino
69
+
70
+ # nesso-cli
71
+
72
+ ![coverage](coverage/coverage-badge.svg)![docs_coverage](coverage/docstring_coverage.svg)
73
+
74
+ ---
75
+ **Documentation**: 📚 [dyvenia docs (internal)][mkdocs page]
76
+
77
+ **Source Code**: 💾 [dyvenia/nesso-cli][github page]
78
+
79
+ ---
80
+
81
+ <!-- body-begin -->
82
+
83
+ The [CLI](https://www.w3schools.com/whatis/whatis_cli.asp) interface of the [nesso data platform].
84
+
85
+ ## Features
86
+
87
+ - [x] simplify and automate data modelling
88
+ - [x] simplify and automate metadata generation
89
+ - [x] manage nesso project configuration
90
+ - [ ] simplify and automate job scheduling (coming soon!)
91
+
92
+ ## Where does nesso-cli fit in?
93
+
94
+ Currently, nesso-cli contains a single module, `models` (`nesso models`), which is used for the T in ELTC (Extract, Load, Transform, Catalog), sitting between data ingestion (`viadot`) and metadata ingestion (`luma-cli`):
95
+
96
+ ![Where does nesso-cli fit](docs/_static/where_nesso_cli_fits.png)
97
+
98
+ In the future, nesso-cli will include additional modules to allow interacting with different components of the nesso data platform through a unified interface.
99
+
100
+ The next planned module is `jobs`, which will allow creating and scheduling EL and ELTC jobs via a simple CLI interface. Currently, this is done by creating jobs manually in Python and then manually scheduling them in Prefect. We hope to replace this tedious and error-prone (though repeatable) process with simple commands, such as `nesso jobs deployment create --job my_job --schedule "0 0 * * *"`, as well as interactive commands which will guide user through a set of limited choices, such as `nesso jobs job create`.
101
+
102
+ [github page]: https://github.com/dyvenia/nesso-cli
103
+ [mkdocs page]: https://nesso-cli.docs.dyvenia.com/
104
+ [nesso data platform]: https://nesso.docs.dyvenia.com/
105
+
106
+ <!-- body-end -->
@@ -0,0 +1,37 @@
1
+ # nesso-cli
2
+
3
+ ![coverage](coverage/coverage-badge.svg)![docs_coverage](coverage/docstring_coverage.svg)
4
+
5
+ ---
6
+ **Documentation**: 📚 [dyvenia docs (internal)][mkdocs page]
7
+
8
+ **Source Code**: 💾 [dyvenia/nesso-cli][github page]
9
+
10
+ ---
11
+
12
+ <!-- body-begin -->
13
+
14
+ The [CLI](https://www.w3schools.com/whatis/whatis_cli.asp) interface of the [nesso data platform].
15
+
16
+ ## Features
17
+
18
+ - [x] simplify and automate data modelling
19
+ - [x] simplify and automate metadata generation
20
+ - [x] manage nesso project configuration
21
+ - [ ] simplify and automate job scheduling (coming soon!)
22
+
23
+ ## Where does nesso-cli fit in?
24
+
25
+ Currently, nesso-cli contains a single module, `models` (`nesso models`), which is used for the T in ELTC (Extract, Load, Transform, Catalog), sitting between data ingestion (`viadot`) and metadata ingestion (`luma-cli`):
26
+
27
+ ![Where does nesso-cli fit](docs/_static/where_nesso_cli_fits.png)
28
+
29
+ In the future, nesso-cli will include additional modules to allow interacting with different components of the nesso data platform through a unified interface.
30
+
31
+ The next planned module is `jobs`, which will allow creating and scheduling EL and ELTC jobs via a simple CLI interface. Currently, this is done by creating jobs manually in Python and then manually scheduling them in Prefect. We hope to replace this tedious and error-prone (though repeatable) process with simple commands, such as `nesso jobs deployment create --job my_job --schedule "0 0 * * *"`, as well as interactive commands which will guide user through a set of limited choices, such as `nesso jobs job create`.
32
+
33
+ [github page]: https://github.com/dyvenia/nesso-cli
34
+ [mkdocs page]: https://nesso-cli.docs.dyvenia.com/
35
+ [nesso data platform]: https://nesso.docs.dyvenia.com/
36
+
37
+ <!-- body-end -->
@@ -0,0 +1,117 @@
1
+ [project]
2
+ name = "nesso_cli"
3
+ description = "A CLI tool for managing data models."
4
+ readme = "README.md"
5
+ version = "0.10.10"
6
+ authors = [{ name = "Michał Zawadzki", email = "mzawadzki@dyvenia.com" }]
7
+ classifiers = []
8
+ keywords = ["cli", "dbt", "dyvenia", "data", "nesso", "models"]
9
+ requires-python = ">=3.10, <3.13"
10
+ dependencies = [
11
+ "loguru == 0.7.2",
12
+ "python-dotenv == 1.0.0",
13
+ "ruamel.yaml == 0.18.5",
14
+ "pydantic >=1.10.11,<2",
15
+ "typer>=0.9.0",
16
+ # Below fork of the dbt-core-interface is vendored till https://github.com/z3z1ma/dbt-core-interface/pull/117 is merged.
17
+ # "dbt-core-interface @ git+https://github.com/trymzet/dbt-core-interface",
18
+ ]
19
+
20
+ [project.optional-dependencies]
21
+ test = [
22
+ "pytest == 7.4.3",
23
+ "pytest-cov == 4.1.0",
24
+ "pytest-env == 1.1.1",
25
+ "coverage == 7.3.2",
26
+ "black == 22.12.0",
27
+ "mypy == 1.7.0",
28
+ "sqlalchemy == 1.4.50",
29
+ "faker == 20.1.0",
30
+ "getkey == 0.6.5",
31
+ "pandas == 1.5.3",
32
+ "mock == 5.1.0",
33
+ "genbadge[coverage] == 1.1.0",
34
+ "flake8 == 6.1.0",
35
+ "Flake8-pyproject == 1.2.3",
36
+ "tomlkit == 0.12.3",
37
+ "interrogate[png] == 1.5.0",
38
+ "dbt-postgres == 1.7.12",
39
+ ]
40
+ docs = [
41
+ "mkdocstrings[python] == 0.24.0",
42
+ "mkdocs-material[imaging] == 9.4.11",
43
+ "mkdocs-include-dir-to-nav == 1.2.0",
44
+ "neoteroi-mkdocs == 1.0.4",
45
+ "mkdocs-jupyter == 0.24.6",
46
+ "mike == 2.0.0",
47
+ "mkdocs-table-reader-plugin == 2.0.3",
48
+ "mkdocs-include-markdown-plugin == 6.0.4",
49
+ "mkdocs-git-revision-date-plugin == 0.3.2",
50
+ "mkdocs-glightbox>=0.3.5",
51
+ "mkdocs-material-extensions==1.3.1",
52
+ ]
53
+ dev = [
54
+ "nesso_cli[test]",
55
+ "nesso_cli[docs]",
56
+ "dbt-osmosis>=0.12.4",
57
+ "sqlfluff>=2.3.5",
58
+ ]
59
+
60
+ trino = ["dbt-trino == 1.3.2"]
61
+ redshift = ["dbt-redshift == 1.3.0"]
62
+ databricks = ["dbt-databricks == 1.3.2"]
63
+ # Workaround till https://github.com/dbt-msft/dbt-sqlserver/pull/460 is merged.
64
+ sqlserver = ["dyvenia-dbt-sqlserver == 1.3.2"]
65
+ postgres = ["dbt-postgres == 1.7.12"]
66
+ duckdb = ["dbt-duckdb == 1.7.4", "duckdb == 0.10.1"]
67
+
68
+ all = [
69
+ "nesso_cli[trino]",
70
+ "nesso_cli[redshift]",
71
+ "nesso_cli[databricks]",
72
+ "nesso_cli[sqlserver]",
73
+ "nesso_cli[postgres]",
74
+ "nesso_cli[duckdb]",
75
+ ]
76
+
77
+ [project.scripts]
78
+ nesso = "nesso_cli.nesso_cli:cli"
79
+
80
+ [tool.setuptools]
81
+ packages = ["cli"]
82
+
83
+ [tool.black]
84
+ line-length = 88
85
+ exclude = '''
86
+ /(
87
+ \.git
88
+ | \.mypy_cache
89
+ | \.tox
90
+ | venv
91
+ | \.venv
92
+ | _build
93
+ | buck-out
94
+ | build
95
+ | dist
96
+ )/
97
+ '''
98
+
99
+ [tool.isort]
100
+ # make it compatible with Black
101
+ profile = "black"
102
+
103
+ [tool.flake8]
104
+ extend-ignore = "F722"
105
+ exclude = "_vendored"
106
+
107
+ max-line-length = 88
108
+ [tool.interrogate]
109
+ fail-under = 40
110
+ quiet = false
111
+ exclude = ["src/nesso_cli/models/tests/"]
112
+ generate-badge = "coverage/docstring_coverage.svg"
113
+
114
+
115
+ [build-system]
116
+ build-backend = "flit_core.buildapi"
117
+ requires = ["flit_core >=3.8,<4"]
@@ -0,0 +1,14 @@
1
+ """Description placeholder (required by Flit)."""
2
+
3
+ import importlib.metadata
4
+
5
+ from .jobs import main # noqa
6
+ from .models import base_model, common, main, model, seed, source # noqa
7
+
8
+ # This allows specifying the version in a single place (`pyproject.toml`).
9
+ # See https://gist.github.com/benkehoe/066a73903e84576a8d6d911cfedc2df6.
10
+ try:
11
+ # __package__ allows for the case where __name__ is "__main__"
12
+ __version__ = importlib.metadata.version(__package__ or __name__)
13
+ except importlib.metadata.PackageNotFoundError:
14
+ __version__ = "0.0.0"
File without changes
@@ -0,0 +1,3 @@
1
+ import typer
2
+
3
+ app = typer.Typer()
File without changes
@@ -0,0 +1,2 @@
1
+ """Dbt Core Interface."""
2
+ from nesso_cli.models._vendored.dbt_core_interface.project import * # noqa: F401, F403
@@ -0,0 +1,26 @@
1
+ Portions of this software are adapted from the sqlfluff-templater-dbt
2
+ package. The original license and copyright are reproduced below.
3
+
4
+ ==
5
+
6
+ MIT License
7
+
8
+ Copyright (c) 2018 Alan Cruickshank
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
@@ -0,0 +1,25 @@
1
+ """Defines the hook endpoints for the dbt templater plugin."""
2
+ import logging
3
+
4
+ from sqlfluff.core.plugin import hookimpl
5
+
6
+ from nesso_cli.models._vendored.dbt_core_interface.dbt_templater.templater import (
7
+ DCIDbtTemplater,
8
+ )
9
+
10
+ LOGGER = logging.getLogger(__name__)
11
+
12
+
13
+ @hookimpl
14
+ def get_templaters():
15
+ """Get templaters."""
16
+ def create_templater(**kwargs):
17
+ import nesso_cli.models.dbt_core_interface.state
18
+ assert nesso_cli.models.dbt_core_interface.state.dbt_project_container is not None, "dbt_project_container is None"
19
+ return DCIDbtTemplater(
20
+ dbt_project_container=nesso_cli.models.dbt_core_interface.state.dbt_project_container,
21
+ **kwargs
22
+ )
23
+
24
+ create_templater.name = DCIDbtTemplater.name
25
+ return [create_templater]
@@ -0,0 +1,234 @@
1
+ """Specialized dbt templater for embedding in the dbt-core-interface server."""
2
+
3
+ import logging
4
+ import os.path
5
+ import threading
6
+ from pathlib import Path
7
+ from typing import Optional
8
+
9
+ from dbt.version import get_installed_version
10
+
11
+ try:
12
+ from dbt.exceptions import (
13
+ CompilationException as DbtCompilationException,
14
+ )
15
+ except ImportError:
16
+ from dbt.exceptions import (
17
+ CompilationError as DbtCompilationException,
18
+ )
19
+ from jinja2 import Environment
20
+ from jinja2_simple_tags import StandaloneTag
21
+ from sqlfluff.core.errors import SQLFluffSkipFile, SQLTemplaterError
22
+ from sqlfluff.core.templaters.base import TemplatedFile, large_file_check
23
+ from sqlfluff.core.templaters.jinja import JinjaTemplater
24
+
25
+ from nesso_cli.models._vendored.dbt_core_interface.project import DbtProject
26
+
27
+ templater_logger = logging.getLogger(__name__)
28
+
29
+ DBT_VERSION = get_installed_version()
30
+ DBT_VERSION_TUPLE = (int(DBT_VERSION.major), int(DBT_VERSION.minor))
31
+
32
+ if DBT_VERSION_TUPLE >= (1, 3):
33
+ COMPILED_SQL_ATTRIBUTE = "compiled_code"
34
+ RAW_SQL_ATTRIBUTE = "raw_code"
35
+ else:
36
+ COMPILED_SQL_ATTRIBUTE = "compiled_sql"
37
+ RAW_SQL_ATTRIBUTE = "raw_sql"
38
+
39
+ local = threading.local()
40
+
41
+
42
+ class DCIDbtTemplater(JinjaTemplater):
43
+ """dbt templater for dbt-core-interface, based on sqlfluff-templater-dbt."""
44
+
45
+ # Same templater name as sqlfluff-templater-dbt. It is functionally
46
+ # equivalent to that templater, but optimized for dbt-core-interface.
47
+ # The two templaters cannot be installed in the same virtualenv.
48
+ name = "dbt"
49
+
50
+ def __init__(self, **kwargs):
51
+ self.dbt_project_container = kwargs.pop("dbt_project_container")
52
+ super().__init__(**kwargs)
53
+
54
+ def config_pairs(self): # pragma: no cover
55
+ """Returns info about the given templater for output by the cli."""
56
+ return [("templater", self.name), ("dbt", DBT_VERSION.to_version_string())]
57
+
58
+ @large_file_check
59
+ def process(self, *, fname: str, in_str=None, config, **kwargs):
60
+ """Compile a dbt model and return the compiled SQL."""
61
+ try:
62
+ return self._unsafe_process(
63
+ os.path.abspath(fname) if in_str is None else None, in_str, config
64
+ )
65
+ except DbtCompilationException as e:
66
+ if e.node:
67
+ return None, [
68
+ SQLTemplaterError(
69
+ f"dbt compilation error on file '{e.node.original_file_path}', " f"{e.msg}"
70
+ )
71
+ ]
72
+ else:
73
+ raise # pragma: no cover
74
+ # If a SQLFluff error is raised, just pass it through
75
+ except SQLTemplaterError as e: # pragma: no cover
76
+ return None, [e]
77
+
78
+ def _find_node(self, project: DbtProject, fname: str):
79
+ expected_node_path = os.path.relpath(fname, start=os.path.abspath(project.config.project_root))
80
+ node = project.get_node_by_path(expected_node_path)
81
+ if node:
82
+ return node
83
+ skip_reason = self._find_skip_reason(project, expected_node_path)
84
+ if skip_reason:
85
+ raise SQLFluffSkipFile(f"Skipped file {fname} because it is {skip_reason}")
86
+ raise SQLFluffSkipFile(f"File {fname} was not found in dbt project") # pragma: no cover
87
+
88
+ @staticmethod
89
+ def _find_skip_reason(project: DbtProject, expected_node_path: str) -> Optional[str]:
90
+ """Return string reason if model okay to skip, otherwise None."""
91
+ # Scan macros.
92
+ for macro in project.dbt.macros.values():
93
+ if macro.original_file_path == expected_node_path:
94
+ return "a macro"
95
+
96
+ # Scan disabled nodes.
97
+ for nodes in project.dbt.disabled.values():
98
+ for node in nodes:
99
+ if node.original_file_path == expected_node_path:
100
+ return "disabled"
101
+ return None
102
+
103
+ @staticmethod
104
+ def from_string(*args, **kwargs):
105
+ """Replaces (via monkeypatch) the jinja2.Environment function."""
106
+ globals = kwargs.get("globals")
107
+ if globals and hasattr(local, "target_sql"):
108
+ model = globals.get("model")
109
+ if model:
110
+ # Is it processing the node we're interested in?
111
+ if isinstance(local.target_sql, Path):
112
+ the_one = str(local.target_sql) == model.get("original_file_path")
113
+ else:
114
+ the_one = local.target_sql == args[1]
115
+ if the_one:
116
+ # Yes. Capture the important arguments and create
117
+ # a make_template() function.
118
+ env = args[0]
119
+ globals = args[2] if len(args) >= 3 else kwargs["globals"]
120
+
121
+ def make_template(in_str):
122
+ env.add_extension(SnapshotExtension)
123
+ return env.from_string(in_str, globals=globals)
124
+
125
+ local.make_template = make_template
126
+
127
+ return old_from_string(*args, **kwargs)
128
+
129
+ def _unsafe_process(self, fname, in_str, config=None):
130
+ dbt_project = self.dbt_project_container.get_project_by_root_dir(
131
+ config.get_section((self.templater_selector, self.name, "project_dir"))
132
+ )
133
+ local.make_template = None
134
+ try:
135
+ if fname:
136
+ node = self._find_node(dbt_project, fname)
137
+ local.target_sql = Path(
138
+ os.path.relpath(fname, start=dbt_project.config.project_root)
139
+ )
140
+ compiled_node = dbt_project.compile_from_node(node)
141
+ else:
142
+ local.target_sql = in_str
143
+ # TRICKY: Use __wrapped__ to bypass the cache. We *must*
144
+ # recompile each time, because that's how we get the
145
+ # make_template() function.
146
+ compiled_node = dbt_project.compile_code.__wrapped__(
147
+ dbt_project, in_str
148
+ )
149
+ except Exception as err:
150
+ raise SQLFluffSkipFile( # pragma: no cover
151
+ f"Skipped file {fname} because dbt raised a fatal "
152
+ f"exception during compilation: {err!s}"
153
+ ) from err
154
+ finally:
155
+ local.target_sql = None
156
+
157
+ if compiled_node.injected_code:
158
+ # If injected SQL is present, it contains a better picture
159
+ # of what will actually hit the database (e.g. with tests).
160
+ # However it's not always present.
161
+ compiled_sql = compiled_node.injected_code
162
+ else:
163
+ compiled_sql = compiled_node.compiled_code
164
+
165
+ raw_sql = compiled_node.raw_code
166
+
167
+ if not compiled_sql: # pragma: no cover
168
+ raise SQLTemplaterError(
169
+ "dbt templater compilation failed silently, check your "
170
+ "configuration by running `dbt compile` directly."
171
+ )
172
+
173
+ if fname:
174
+ with open(fname) as source_dbt_model:
175
+ source_dbt_sql = source_dbt_model.read()
176
+ else:
177
+ source_dbt_sql = in_str
178
+
179
+ if not source_dbt_sql.rstrip().endswith("-%}"):
180
+ n_trailing_newlines = len(source_dbt_sql) - len(source_dbt_sql.rstrip("\n"))
181
+ else:
182
+ # Source file ends with right whitespace stripping, so there's
183
+ # no need to preserve/restore trailing newlines.
184
+ n_trailing_newlines = 0
185
+
186
+ templater_logger.debug(
187
+ " Trailing newline count in source dbt model: %r",
188
+ n_trailing_newlines,
189
+ )
190
+ templater_logger.debug(" Raw SQL before compile: %r", source_dbt_sql)
191
+ templater_logger.debug(" Node raw SQL: %r", raw_sql)
192
+ templater_logger.debug(" Node compiled SQL: %r", compiled_sql)
193
+
194
+ # Adjust for dbt Jinja removing trailing newlines. For more details on
195
+ # this, see the similar code in sqlfluff-templater.dbt.
196
+ compiled_node.raw_sql = source_dbt_sql
197
+ compiled_sql = compiled_sql + "\n" * n_trailing_newlines
198
+ raw_sliced, sliced_file, templated_sql = self.slice_file(
199
+ source_dbt_sql,
200
+ compiled_sql,
201
+ config=config,
202
+ make_template=local.make_template,
203
+ append_to_templated="\n" if n_trailing_newlines else "",
204
+ )
205
+ return (
206
+ TemplatedFile(
207
+ source_str=source_dbt_sql,
208
+ templated_str=templated_sql,
209
+ fname=fname,
210
+ sliced_file=sliced_file,
211
+ raw_sliced=raw_sliced,
212
+ ),
213
+ # No violations returned in this way.
214
+ [],
215
+ )
216
+
217
+
218
+ # Monkeypatch Environment.from_string(). DCIDbtTemplater uses this to
219
+ # intercept Jinja compilation and capture a template trace.
220
+ old_from_string = Environment.from_string
221
+ Environment.from_string = DCIDbtTemplater.from_string
222
+
223
+
224
+ class SnapshotExtension(StandaloneTag):
225
+ """Dummy "snapshot" tags so raw dbt templates will parse.
226
+
227
+ For more context, see sqlfluff-templater-dbt.
228
+ """
229
+
230
+ tags = {"snapshot", "endsnapshot"}
231
+
232
+ def render(self, format_string=None):
233
+ """Dummy method that renders the tag."""
234
+ return ""