nesso-cli 0.16.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/PKG-INFO +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/reference/reference.md +1 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/.nesso/config.yml +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/pyproject.toml +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/base_model.py +39 -14
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/config.py +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/generate_source_yaml_boilerplate.sql +2 -2
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/resources.py +9 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/source.py +20 -3
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/.nesso/config.yml +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/pytest.ini +1 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_base_model.py +1 -5
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_source.py +2 -10
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_update.py +31 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/update.py +11 -5
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/uv.lock +1 -1
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.github/workflows/build.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.github/workflows/cd.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.github/workflows/ci.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.github/workflows/notify-labeled-devops.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.github/workflows/static-site-prepare.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.gitignore +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.pre-commit-config.yaml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.prettierignore +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.python-version +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.secrets.baseline +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.vscode/extensions.list +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.vscode/install_extensions.sh +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.vscode/launch.json +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/.vscode/settings.json +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/CHANGELOG.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/CONTRIBUTING.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/README.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/coverage/coverage-badge.svg +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/coverage/docstring_coverage.svg +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/_static/data_architecture_vision_copied_from_nesso_docs.drawio.png +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/_static/logo.png +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/_static/where_nesso_cli_fits.png +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/advanced_usage/introduction.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/advanced_usage/scheduling.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/advanced_usage/seeds.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/architecture/architecture.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/getting_started/getting_started.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/index.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/integrations/luma.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/overrides/main.html +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/reference/glossary.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/reference/introduction.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/reference/security.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/reference/supported_dbs.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/tutorials/create_tutorial_tables.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/tutorials/creating_a_nesso_models_project.ipynb +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/tutorials/nesso_models.ipynb +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/tutorials/nesso_project_user_onboarding.ipynb +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/base_models.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/dbt_adapters_nesso_features.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/introduction.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/model_maintenance.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/models.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/sources.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/user_guide/workflows.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/workflows/data_analyst.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/docs/workflows/data_engineer.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/.gitignore +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/README.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/create_tutorial_tables.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/dbt_project.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/install.sh +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/models/sources/staging/staging.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/packages.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/profiles.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/requirements.txt +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/mkdocs.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/__init__.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/jobs/.gitkeep +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/jobs/main.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/__init__.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/common.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/context.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/init.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/main.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/metadata.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/model.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/models.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/dbt_project.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/codegen_helpers.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/dbt_profiler.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/generate_seed_yaml_boilerplate.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/get_custom_schema.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/get_source_pii_columns.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/get_table_columns.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/hash_column.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/hash_source_pii_columns.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/redshift_external_tables_fix.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/seed.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.gitignore +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.vscode/extensions.list +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.vscode/install_extensions.sh +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.vscode/settings.json +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/CONTRIBUTING.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/README.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/config.yml.j2 +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/dbt_project.yml.j2 +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/packages.yml.j2 +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/prepare.sh +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/requirements.txt +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/source_docs.md +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/{{ bronze_schema }}.yml.j2 +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/conftest.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/.nesso/config.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/dbt_project.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/macros/test_macros.sql +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/package-lock.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/packages.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/profiles.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/seeds/average salary test.xlsx +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/seeds/countries_example.csv +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/dbt_project.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/packages.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/profiles.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/average salary test.xlsx +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/seeds/countries_example.csv +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/dbt_project.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/packages.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/profiles.yml +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/functional/test_da_workflow.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_common.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_config.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_init.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_main.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_metadata.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_model.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_models.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_resources.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/test_seed.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/validate_tables.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/nesso_cli.py +0 -0
- {nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/py.typed +0 -0
|
@@ -527,6 +527,7 @@ $ nesso models source rm [OPTIONS] TABLE_NAME
|
|
|
527
527
|
|
|
528
528
|
**Options**:
|
|
529
529
|
|
|
530
|
+
* `-sp, --silver-schrmema-prefix TEXT`: The prefix for the corresponding base model in the silver schema. Defaults to `config.silver_schema_prefix`.
|
|
530
531
|
* `-b, --remove-base-model`: Whether to remove the corresponding base model.
|
|
531
532
|
* `-e, --env TEXT`: The environment to use. [default: dev]
|
|
532
533
|
* `--help`: Show this message and exit.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name = "nesso_cli"
|
|
3
3
|
description = "A CLI tool for managing data models."
|
|
4
4
|
readme = "README.md"
|
|
5
|
-
version = "0.
|
|
5
|
+
version = "0.18.0"
|
|
6
6
|
authors = [{ name = "Michał Zawadzki", email = "mzawadzki@dyvenia.com" }]
|
|
7
7
|
classifiers = []
|
|
8
8
|
keywords = ["cli", "dbt", "dyvenia", "data", "nesso", "models"]
|
|
@@ -10,6 +10,7 @@ import typer
|
|
|
10
10
|
|
|
11
11
|
from nesso_cli.models import context
|
|
12
12
|
from nesso_cli.models.common import (
|
|
13
|
+
check_if_relation_exists,
|
|
13
14
|
convert_list_of_options_to_dict,
|
|
14
15
|
drop,
|
|
15
16
|
get_current_dbt_project_obj,
|
|
@@ -139,6 +140,13 @@ def bootstrap(
|
|
|
139
140
|
help="The directory where the silver schema models are stored.",
|
|
140
141
|
),
|
|
141
142
|
] = config.silver_schema_dir,
|
|
143
|
+
silver_schema_prefix: Annotated[
|
|
144
|
+
str,
|
|
145
|
+
typer.Option(
|
|
146
|
+
"--silver-schema-prefix",
|
|
147
|
+
help="The prefix to add to the base model name if it's not already present.",
|
|
148
|
+
),
|
|
149
|
+
] = config.silver_schema_prefix,
|
|
142
150
|
subdir: Annotated[
|
|
143
151
|
str | None,
|
|
144
152
|
typer.Option(
|
|
@@ -153,13 +161,11 @@ def bootstrap(
|
|
|
153
161
|
If silver schema prefix is not specified at the beginning of the model name, it
|
|
154
162
|
will be added automatically.
|
|
155
163
|
"""
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
base_model_prefix = f"{base_model_prefix}_"
|
|
159
|
-
|
|
160
|
-
if not base_model_name.startswith(base_model_prefix):
|
|
161
|
-
base_model_name = f"{base_model_prefix}{base_model_name}"
|
|
164
|
+
if silver_schema_prefix and not silver_schema_prefix.endswith("_"):
|
|
165
|
+
silver_schema_prefix = f"{silver_schema_prefix}_"
|
|
162
166
|
|
|
167
|
+
if not base_model_name.startswith(silver_schema_prefix):
|
|
168
|
+
base_model_name = f"{silver_schema_prefix}{base_model_name}"
|
|
163
169
|
dbt_project_dir = Path(context.get("PROJECT_DIR"))
|
|
164
170
|
|
|
165
171
|
base_dir = (
|
|
@@ -216,6 +222,13 @@ def bootstrap_yaml(
|
|
|
216
222
|
help="The directory where the silver schema models are stored.",
|
|
217
223
|
),
|
|
218
224
|
] = config.silver_schema_dir,
|
|
225
|
+
silver_schema_prefix: Annotated[
|
|
226
|
+
str,
|
|
227
|
+
typer.Option(
|
|
228
|
+
"--silver-schema-prefix",
|
|
229
|
+
help="The prefix to add to the base model name if it's not already present.",
|
|
230
|
+
),
|
|
231
|
+
] = config.silver_schema_prefix,
|
|
219
232
|
subdir: Annotated[
|
|
220
233
|
str | None,
|
|
221
234
|
typer.Option(
|
|
@@ -232,7 +245,13 @@ def bootstrap_yaml(
|
|
|
232
245
|
it will be added automatically.
|
|
233
246
|
"""
|
|
234
247
|
meta = convert_list_of_options_to_dict(ctx.args)
|
|
235
|
-
dbt_model = NessoDBTModel(
|
|
248
|
+
dbt_model = NessoDBTModel(
|
|
249
|
+
name=base_model_name,
|
|
250
|
+
env=env,
|
|
251
|
+
base=True,
|
|
252
|
+
base_model_prefix=silver_schema_prefix,
|
|
253
|
+
**meta,
|
|
254
|
+
)
|
|
236
255
|
dbt_project_dir = Path(context.get("PROJECT_DIR"))
|
|
237
256
|
base_dir = (
|
|
238
257
|
dbt_project_dir / "models" / silver_schema_dir / (subdir or dbt_model.name)
|
|
@@ -247,13 +266,19 @@ def bootstrap_yaml(
|
|
|
247
266
|
)
|
|
248
267
|
|
|
249
268
|
# First, materialize the model.
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
269
|
+
table_exists = check_if_relation_exists(
|
|
270
|
+
name=dbt_model.name,
|
|
271
|
+
schema=dbt_model.node.schema,
|
|
272
|
+
target=env,
|
|
273
|
+
)
|
|
274
|
+
if not table_exists:
|
|
275
|
+
dbt_project = get_current_dbt_project_obj(target=env)
|
|
276
|
+
result = dbt_project.run("--select", dbt_model.name)
|
|
277
|
+
|
|
278
|
+
if not result.success:
|
|
279
|
+
exception = result.exception or "unknown error"
|
|
280
|
+
msg = f"dbt run failed: {exception}"
|
|
281
|
+
raise RuntimeError(msg)
|
|
257
282
|
|
|
258
283
|
# Now that model is materialized, we can generate the YAML.
|
|
259
284
|
dbt_model.to_yaml(yaml_path)
|
|
@@ -142,9 +142,9 @@
|
|
|
142
142
|
{% do sources_yaml.append(" meta:") %}
|
|
143
143
|
{% do sources_yaml.append(" owners:") %}
|
|
144
144
|
{% do sources_yaml.append(" - type: Technical owner") %}
|
|
145
|
-
{% do sources_yaml.append(" email: ''
|
|
145
|
+
{% do sources_yaml.append(" email: ''") %}
|
|
146
146
|
{% do sources_yaml.append(" - type: Business owner") %}
|
|
147
|
-
{% do sources_yaml.append(" email: ''
|
|
147
|
+
{% do sources_yaml.append(" email: ''" ) %}
|
|
148
148
|
{% do sources_yaml.append(" domains: " ~ domains) %}
|
|
149
149
|
{% if include_sla %} {% do sources_yaml.append(' SLA: "24 hours"') %} {% endif %}
|
|
150
150
|
{% do sources_yaml.append(" tags: " ~ tags) %}
|
|
@@ -41,6 +41,7 @@ class NessoDBTResource:
|
|
|
41
41
|
dbt_project: DbtProject | None = None,
|
|
42
42
|
env: str = config.default_env,
|
|
43
43
|
base: bool = False,
|
|
44
|
+
base_model_prefix: str | None = None,
|
|
44
45
|
comments: list[tuple[str, str]] | None = None,
|
|
45
46
|
**meta,
|
|
46
47
|
) -> None:
|
|
@@ -70,8 +71,11 @@ class NessoDBTResource:
|
|
|
70
71
|
("unique", "- unique"),
|
|
71
72
|
("not_null", "- not_null"),
|
|
72
73
|
]
|
|
73
|
-
|
|
74
|
+
|
|
75
|
+
if base_model_prefix is None:
|
|
74
76
|
base_model_prefix = config.silver_schema_prefix
|
|
77
|
+
|
|
78
|
+
if base:
|
|
75
79
|
if base_model_prefix and not base_model_prefix.endswith("_"):
|
|
76
80
|
base_model_prefix = f"{base_model_prefix}_"
|
|
77
81
|
|
|
@@ -234,6 +238,7 @@ class NessoDBTModel(NessoDBTResource):
|
|
|
234
238
|
dbt_project: DbtProject | None = None,
|
|
235
239
|
env: str = config.default_env,
|
|
236
240
|
base: bool = False,
|
|
241
|
+
base_model_prefix: str | None = None,
|
|
237
242
|
comments: list[tuple[str, str]] | None = None,
|
|
238
243
|
**meta,
|
|
239
244
|
) -> None:
|
|
@@ -262,6 +267,7 @@ class NessoDBTModel(NessoDBTResource):
|
|
|
262
267
|
dbt_project,
|
|
263
268
|
env,
|
|
264
269
|
base,
|
|
270
|
+
base_model_prefix,
|
|
265
271
|
comments,
|
|
266
272
|
**meta,
|
|
267
273
|
)
|
|
@@ -529,6 +535,8 @@ Possible causes:
|
|
|
529
535
|
|
|
530
536
|
# Inherit non-meta config fields.
|
|
531
537
|
tags.extend(dependency_config.tags)
|
|
538
|
+
# Deduplicate tags while preserving order.
|
|
539
|
+
tags = list(dict.fromkeys(tags))
|
|
532
540
|
if isinstance(dependency, Model):
|
|
533
541
|
# NOTE: this logic is quite naive - we simply inherit from the last
|
|
534
542
|
# encountered upstream model.
|
|
@@ -441,8 +441,16 @@ def rm(
|
|
|
441
441
|
help="The directory where source property files are stored.",
|
|
442
442
|
),
|
|
443
443
|
] = config.bronze_schema_dir,
|
|
444
|
+
silver_schema_prefix: Annotated[
|
|
445
|
+
str,
|
|
446
|
+
typer.Option(
|
|
447
|
+
"--silver-schema-prefix",
|
|
448
|
+
"-sp",
|
|
449
|
+
help="The prefix of the corresponding base model to remove.",
|
|
450
|
+
),
|
|
451
|
+
] = config.silver_schema_prefix,
|
|
444
452
|
remove_base_model: Annotated[
|
|
445
|
-
bool
|
|
453
|
+
bool,
|
|
446
454
|
typer.Option(
|
|
447
455
|
"--remove-base-model",
|
|
448
456
|
"-b",
|
|
@@ -472,13 +480,22 @@ def rm(
|
|
|
472
480
|
|
|
473
481
|
source_cfg["tables"] = [t for t in source_cfg["tables"] if t["name"] != table_name]
|
|
474
482
|
|
|
483
|
+
if not source_cfg["tables"]:
|
|
484
|
+
# If there are no tables left in the source, remove the entire source definition.
|
|
485
|
+
cfg["sources"] = [s for s in cfg["sources"] if s["name"] != source_name]
|
|
486
|
+
|
|
475
487
|
with source_props_path.open("w") as file:
|
|
476
488
|
yaml.dump(cfg, file)
|
|
477
489
|
|
|
478
490
|
# Remove the base model.
|
|
479
491
|
if remove_base_model:
|
|
480
|
-
|
|
481
|
-
|
|
492
|
+
prefix = f"{silver_schema_prefix}_" if silver_schema_prefix else ""
|
|
493
|
+
base_model_name = prefix + table_name
|
|
494
|
+
base_model_rm(
|
|
495
|
+
name=base_model_name,
|
|
496
|
+
drop_relation=True,
|
|
497
|
+
env=env,
|
|
498
|
+
)
|
|
482
499
|
|
|
483
500
|
return True
|
|
484
501
|
|
|
@@ -218,8 +218,6 @@ def test_base_model_bootstrap_yaml_no_prefix(
|
|
|
218
218
|
assert not check_if_base_model_exists(TEST_TABLE_ACCOUNT)
|
|
219
219
|
|
|
220
220
|
# Create and materialize the base model.
|
|
221
|
-
config.silver_schema_prefix = ""
|
|
222
|
-
|
|
223
221
|
base_model_dir_path = BASE_MODELS_DIR_PATH / TEST_TABLE_ACCOUNT
|
|
224
222
|
base_model_sql_path = base_model_dir_path / f"{TEST_TABLE_ACCOUNT}.sql"
|
|
225
223
|
base_model_sql_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -238,7 +236,7 @@ def test_base_model_bootstrap_yaml_no_prefix(
|
|
|
238
236
|
mock_ctx = Mock()
|
|
239
237
|
mock_ctx.args = []
|
|
240
238
|
|
|
241
|
-
bootstrap_yaml(ctx=mock_ctx, base_model_name=TEST_TABLE_ACCOUNT)
|
|
239
|
+
bootstrap_yaml(ctx=mock_ctx, base_model_name=TEST_TABLE_ACCOUNT, silver_schema_prefix="")
|
|
242
240
|
|
|
243
241
|
base_model_yaml_path = base_model_dir_path / f"{TEST_TABLE_ACCOUNT}.yml"
|
|
244
242
|
with open(base_model_yaml_path) as f:
|
|
@@ -248,8 +246,6 @@ def test_base_model_bootstrap_yaml_no_prefix(
|
|
|
248
246
|
expected_schema = copy.deepcopy(TestData.base_model_props_no_overrides)
|
|
249
247
|
expected_schema["models"][0]["name"] = "test_table_account"
|
|
250
248
|
|
|
251
|
-
config.silver_schema_prefix = "int"
|
|
252
|
-
|
|
253
249
|
assert check_if_base_model_exists(TEST_TABLE_ACCOUNT)
|
|
254
250
|
assert schema == expected_schema
|
|
255
251
|
|
|
@@ -16,6 +16,7 @@ import nesso_cli.models.context as context
|
|
|
16
16
|
from nesso_cli.models.models import Source
|
|
17
17
|
from nesso_cli.models.main import app
|
|
18
18
|
from nesso_cli.models.source import add as source_add
|
|
19
|
+
from nesso_cli.models.source import rm as source_rm
|
|
19
20
|
from nesso_cli.models.source import check_if_source_exists, check_if_source_table_exists
|
|
20
21
|
|
|
21
22
|
|
|
@@ -402,17 +403,8 @@ def test_rm_drop_base_table(
|
|
|
402
403
|
assert setup_base_model.exists()
|
|
403
404
|
|
|
404
405
|
# Remove the source table & base model.
|
|
405
|
-
|
|
406
|
-
app,
|
|
407
|
-
[
|
|
408
|
-
"source",
|
|
409
|
-
"rm",
|
|
410
|
-
TEST_TABLE_ACCOUNT,
|
|
411
|
-
"--remove-base-model",
|
|
412
|
-
],
|
|
413
|
-
)
|
|
406
|
+
source_rm(TEST_TABLE_ACCOUNT, remove_base_model=True)
|
|
414
407
|
|
|
415
|
-
assert result.exit_code == 0
|
|
416
408
|
assert not check_if_source_table_exists(TEST_SOURCE, TEST_TABLE_ACCOUNT)
|
|
417
409
|
assert not check_if_base_model_exists(TEST_TABLE_ACCOUNT_BASE_MODEL)
|
|
418
410
|
assert not check_if_relation_exists(
|
|
@@ -378,6 +378,37 @@ def test_update_base_model_create_new_column(
|
|
|
378
378
|
assert yaml_columns_metadata == COLUMNS_AFTER_INSERT
|
|
379
379
|
|
|
380
380
|
|
|
381
|
+
def test_update_base_model_create_new_column_with_silver_schema_prefix_option(
|
|
382
|
+
setup_base_model,
|
|
383
|
+
setup_models_in_db,
|
|
384
|
+
TEST_TABLE_ACCOUNT,
|
|
385
|
+
TEST_TABLE_ACCOUNT_BASE_MODEL,
|
|
386
|
+
):
|
|
387
|
+
# Assumptions.
|
|
388
|
+
yaml_columns_metadata = DBTProperties(
|
|
389
|
+
file_path=setup_base_model
|
|
390
|
+
).get_yaml_table_columns(TEST_TABLE_ACCOUNT_BASE_MODEL)
|
|
391
|
+
assert yaml_columns_metadata != COLUMNS_AFTER_INSERT
|
|
392
|
+
|
|
393
|
+
# Derive the expected prefix from the fully-qualified base model fixture and
|
|
394
|
+
# pass it without a trailing underscore to test normalization.
|
|
395
|
+
silver_schema_prefix = TEST_TABLE_ACCOUNT_BASE_MODEL.removesuffix(
|
|
396
|
+
TEST_TABLE_ACCOUNT
|
|
397
|
+
).rstrip("_")
|
|
398
|
+
|
|
399
|
+
# Call the function directly using an unprefixed base model name.
|
|
400
|
+
update_base_model(
|
|
401
|
+
base_model=TEST_TABLE_ACCOUNT,
|
|
402
|
+
silver_schema_prefix=silver_schema_prefix,
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
yaml_columns_metadata = DBTProperties(
|
|
406
|
+
file_path=setup_base_model
|
|
407
|
+
).get_yaml_table_columns(TEST_TABLE_ACCOUNT_BASE_MODEL)
|
|
408
|
+
|
|
409
|
+
assert yaml_columns_metadata == COLUMNS_AFTER_INSERT
|
|
410
|
+
|
|
411
|
+
|
|
381
412
|
@pytest.mark.parametrize("setup_models_in_db", [UPDATE_COLUMN], indirect=True)
|
|
382
413
|
def test_update_base_model_update_column(
|
|
383
414
|
setup_base_model,
|
|
@@ -77,6 +77,13 @@ def base_model(
|
|
|
77
77
|
help="The name of the base_model to update.", show_default=False
|
|
78
78
|
),
|
|
79
79
|
],
|
|
80
|
+
silver_schema_prefix: Annotated[
|
|
81
|
+
str,
|
|
82
|
+
typer.Option(
|
|
83
|
+
"--silver-schema-prefix",
|
|
84
|
+
help="The prefix to add to the base model name if it's not already present.",
|
|
85
|
+
),
|
|
86
|
+
] = config.silver_schema_prefix,
|
|
80
87
|
env: options.environment = config.default_env,
|
|
81
88
|
) -> None:
|
|
82
89
|
"""Update the base model YAML file.
|
|
@@ -84,12 +91,11 @@ def base_model(
|
|
|
84
91
|
If silver schema prefix is not specified at the beginning of the model name, it
|
|
85
92
|
will be added automatically.
|
|
86
93
|
"""
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
base_model_prefix = f"{base_model_prefix}_"
|
|
94
|
+
if silver_schema_prefix and not silver_schema_prefix.endswith("_"):
|
|
95
|
+
silver_schema_prefix = f"{silver_schema_prefix}_"
|
|
90
96
|
|
|
91
|
-
if not base_model.startswith(
|
|
92
|
-
base_model = f"{
|
|
97
|
+
if not base_model.startswith(silver_schema_prefix):
|
|
98
|
+
base_model = f"{silver_schema_prefix}{base_model}"
|
|
93
99
|
|
|
94
100
|
base_model_dir = _get_default_base_dir_path(base_model)
|
|
95
101
|
yaml_file_path = base_model_dir / f"{base_model}.yml"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/examples/nesso_models/models/sources/staging/staging.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/codegen_helpers.sql
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/dbt_profiler.sql
RENAMED
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/get_custom_schema.sql
RENAMED
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/get_table_columns.sql
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/nesso_macros/macros/hash_column.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.vscode/extensions.list
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/.vscode/install_extensions.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/templates/{{ bronze_schema }}.yml.j2
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/dbt_project.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/packages.yml
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/duckdb/profiles.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/packages.yml
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/postgres/profiles.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/dbt_project.yml
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/packages.yml
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/dbt_projects/trino/profiles.yml
RENAMED
|
File without changes
|
{nesso_cli-0.16.0 → nesso_cli-0.18.0}/src/nesso_cli/models/tests/functional/test_da_workflow.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|