datasecops-cli 0.4.3__tar.gz → 0.4.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/CHANGELOG.md +14 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/PKG-INFO +1 -1
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/pyproject.toml +1 -1
- datasecops_cli-0.4.5/src/datasecops_cli/__init__.py +1 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/config.py +7 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/main.py +21 -6
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/configuration.py +40 -2
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/downloads.py +4 -2
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/project_config.py +9 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/bootstrap_service.py +5 -2
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_project_generator.py +19 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_runner.py +6 -4
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/download_service.py +42 -1
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_main.py +118 -1
- datasecops_cli-0.4.3/src/datasecops_cli/__init__.py +0 -1
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.github/workflows/auto-tag.yml +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.gitignore +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/DEVELOPMENT.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/LICENSE +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/README.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/getting-started.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/legacy.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/mcp-servers.json +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/setup.ps1 +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/setup.sh +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/development.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/skill_service.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/snowflake_service.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/upstream_service.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/display.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/file_utils.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/connection.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/server.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/__init__.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_config.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_models.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_version.py +0 -0
- {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_yaml_utils.py +0 -0
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the DataSecOps CLI are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.4.5] - 2026-05-18
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- **Package downloads now respect profile selection** — `datasecops download packages` and the interactive downloads menu now only include dbt packages enabled on the active profile, instead of downloading all packages from the global catalog. This matches the native app's Streamlit UI behavior.
|
|
10
|
+
- **Upstream package exclusion** — packages already provided by upstream projects are automatically excluded from `packages.yml`, preventing duplicate dependencies when using multi-project setups.
|
|
11
|
+
- **`generate_packages_yml()` filtering** — the dbt project generator now also filters packages by profile, fixing the same issue in the bootstrap/init path.
|
|
12
|
+
|
|
13
|
+
## [0.4.4] - 2026-05-18
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **dbt engine toggle** — new `dbt_engine` field in `.datasecops.yml` to switch between `dbtf` (dbt Fusion) and `dbt` (dbt Core). Defaults to `dbtf`. Toggleable via Configure menu `[6] dbt engine`. All dbt commands use the configured engine binary.
|
|
18
|
+
|
|
5
19
|
## [0.4.3] - 2026-05-17
|
|
6
20
|
|
|
7
21
|
### Added
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.5"
|
|
@@ -36,7 +36,14 @@ class Config:
|
|
|
36
36
|
app_database=raw.get("app_database", ""),
|
|
37
37
|
profile_name=raw.get("profile_name", ""),
|
|
38
38
|
cortex_connection=raw.get("cortex_connection", ""),
|
|
39
|
+
dbt_engine=raw.get("dbt_engine", "dbtf"),
|
|
39
40
|
)
|
|
41
|
+
|
|
42
|
+
# Validate dbt_engine
|
|
43
|
+
if self.datasecops.dbt_engine not in self.datasecops.VALID_ENGINES:
|
|
44
|
+
from datasecops_cli.utilities.display import warning_line
|
|
45
|
+
warning_line(f"Invalid dbt_engine '{self.datasecops.dbt_engine}' in .datasecops.yml — falling back to dbtf")
|
|
46
|
+
self.datasecops.dbt_engine = "dbtf"
|
|
40
47
|
|
|
41
48
|
if not self.datasecops.connection_name or not self.datasecops.app_database:
|
|
42
49
|
error_line(".datasecops.yml is missing connection_name or app_database.")
|
|
@@ -473,11 +473,17 @@ def _run_interactive(config: Config):
|
|
|
473
473
|
sf_service = _connect_and_load(config)
|
|
474
474
|
|
|
475
475
|
try:
|
|
476
|
-
# Check for dbt
|
|
477
|
-
|
|
476
|
+
# Check for configured dbt engine on PATH
|
|
477
|
+
dbt_engine = config.datasecops.dbt_engine or "dbtf"
|
|
478
|
+
if not shutil.which(dbt_engine):
|
|
478
479
|
from datasecops_cli.utilities.display import warning_line
|
|
479
|
-
warning_line("
|
|
480
|
-
|
|
480
|
+
warning_line(f"{dbt_engine} not found on PATH — dbt commands will not work.")
|
|
481
|
+
if dbt_engine == "dbtf":
|
|
482
|
+
warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
|
|
483
|
+
else:
|
|
484
|
+
warning_line("Install dbt Core: uv pip install dbt-core dbt-snowflake")
|
|
485
|
+
else:
|
|
486
|
+
info_line(f"dbt engine: {config.datasecops.get_engine_label()}")
|
|
481
487
|
|
|
482
488
|
# Check upstream project versions
|
|
483
489
|
from datasecops_cli.services.upstream_service import check_upstream_versions
|
|
@@ -492,7 +498,8 @@ def _run_interactive(config: Config):
|
|
|
492
498
|
dbt_runner = DbtRunner(
|
|
493
499
|
project_dir=config.dbt_project_dir,
|
|
494
500
|
profiles_dir=config.get_dbt_profiles_dir(),
|
|
495
|
-
target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev"
|
|
501
|
+
target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev",
|
|
502
|
+
engine=config.datasecops.dbt_engine or "dbtf"
|
|
496
503
|
)
|
|
497
504
|
|
|
498
505
|
try:
|
|
@@ -558,7 +565,10 @@ def _run_download(config: Config, items: list[str],
|
|
|
558
565
|
if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
|
|
559
566
|
failed = True
|
|
560
567
|
elif item == "packages":
|
|
561
|
-
if not download_service.download_dbt_packages(
|
|
568
|
+
if not download_service.download_dbt_packages(
|
|
569
|
+
config.dbt_project_dir, profile=config.profile,
|
|
570
|
+
all_profiles=config.all_profiles
|
|
571
|
+
):
|
|
562
572
|
failed = True
|
|
563
573
|
elif item == "macros":
|
|
564
574
|
if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
|
|
@@ -685,6 +695,8 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
685
695
|
project_settings=config.project_settings,
|
|
686
696
|
profile=config.profile,
|
|
687
697
|
source_control=config.source_control,
|
|
698
|
+
datasecops_config=config.datasecops,
|
|
699
|
+
project_dir=config.project_dir,
|
|
688
700
|
)
|
|
689
701
|
config_menu.show()
|
|
690
702
|
|
|
@@ -695,6 +707,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
695
707
|
project_settings=config.project_settings,
|
|
696
708
|
profile=config.profile,
|
|
697
709
|
source_control=config.source_control,
|
|
710
|
+
all_profiles=config.all_profiles,
|
|
698
711
|
)
|
|
699
712
|
dl_menu.show()
|
|
700
713
|
|
|
@@ -713,6 +726,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
713
726
|
project_dir=config.project_dir,
|
|
714
727
|
project_settings=config.project_settings,
|
|
715
728
|
profile=config.profile,
|
|
729
|
+
all_profiles=config.all_profiles,
|
|
716
730
|
)
|
|
717
731
|
bootstrap.run(platform=platform, install_skills=install_skills,
|
|
718
732
|
skill_targets=skill_targets, run_deps=run_deps)
|
|
@@ -775,6 +789,7 @@ def _run_bootstrap(config: Config):
|
|
|
775
789
|
project_dir=config.project_dir,
|
|
776
790
|
project_settings=config.project_settings,
|
|
777
791
|
profile=config.profile,
|
|
792
|
+
all_profiles=config.all_profiles,
|
|
778
793
|
)
|
|
779
794
|
success = bootstrap.run(platform=platform, install_skills=install_skills,
|
|
780
795
|
skill_targets=skill_targets, run_deps=run_deps)
|
|
@@ -5,7 +5,7 @@ import shutil
|
|
|
5
5
|
import subprocess
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
8
|
-
from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings, SourceControl
|
|
8
|
+
from datasecops_cli.models.project_config import DatasecopsConfig, ProjectProfile, ProjectSettings, SourceControl
|
|
9
9
|
from datasecops_cli.services.download_service import DownloadService
|
|
10
10
|
from datasecops_cli.services.linting_service import LintingService
|
|
11
11
|
from datasecops_cli.utilities.display import (
|
|
@@ -14,13 +14,15 @@ from datasecops_cli.utilities.display import (
|
|
|
14
14
|
info_line, success_line, warning_line, error_line,
|
|
15
15
|
get_input_string, select_from_list, select_multiple_from_list
|
|
16
16
|
)
|
|
17
|
+
from datasecops_cli.utilities.yaml_utils import write_datasecops_config
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
class ConfigurationMenu:
|
|
20
21
|
def __init__(self, download_service: DownloadService, linting_service: LintingService,
|
|
21
22
|
profile_name: str, dbt_project_dir: Path,
|
|
22
23
|
project_settings: ProjectSettings = None, profile: ProjectProfile = None,
|
|
23
|
-
source_control: SourceControl = None
|
|
24
|
+
source_control: SourceControl = None,
|
|
25
|
+
datasecops_config: DatasecopsConfig = None, project_dir: Path = None):
|
|
24
26
|
self.downloads = download_service
|
|
25
27
|
self.linting = linting_service
|
|
26
28
|
self.profile_name = profile_name
|
|
@@ -28,6 +30,8 @@ class ConfigurationMenu:
|
|
|
28
30
|
self.project_settings = project_settings
|
|
29
31
|
self.profile = profile
|
|
30
32
|
self.source_control = source_control
|
|
33
|
+
self.datasecops_config = datasecops_config or DatasecopsConfig()
|
|
34
|
+
self.project_dir = project_dir or Path.cwd()
|
|
31
35
|
|
|
32
36
|
def show(self) -> None:
|
|
33
37
|
self._menu()
|
|
@@ -45,17 +49,21 @@ class ConfigurationMenu:
|
|
|
45
49
|
complete_action()
|
|
46
50
|
elif option == 5:
|
|
47
51
|
self._cortex_upgrade()
|
|
52
|
+
elif option == 6:
|
|
53
|
+
self._toggle_dbt_engine()
|
|
48
54
|
self._menu()
|
|
49
55
|
option = get_input_number("Choose an option: ")
|
|
50
56
|
|
|
51
57
|
def _menu(self) -> None:
|
|
52
58
|
clear()
|
|
59
|
+
engine_label = self.datasecops_config.get_engine_label()
|
|
53
60
|
section_header("Configuration", self.profile_name)
|
|
54
61
|
menu_option(1, "install dbt - Install dbt-core & dbt-snowflake from framework versions")
|
|
55
62
|
menu_option(2, "install lint - Install SQLFluff from framework versions")
|
|
56
63
|
menu_option(3, "mcp servers - Configure MCP servers for AI tools")
|
|
57
64
|
menu_option(4, "new project - Initialize a new dbt project with framework profiles")
|
|
58
65
|
menu_option(5, "cortex update - Update Cortex Code to the latest version")
|
|
66
|
+
menu_option(6, f"dbt engine - Switch dbt engine (current: {engine_label})")
|
|
59
67
|
menu_option(0, "back - Return to main menu")
|
|
60
68
|
|
|
61
69
|
def _install_dbt_requirements(self) -> None:
|
|
@@ -250,3 +258,33 @@ class ConfigurationMenu:
|
|
|
250
258
|
except FileNotFoundError:
|
|
251
259
|
error_line("Cortex Code CLI not found.")
|
|
252
260
|
complete_action()
|
|
261
|
+
|
|
262
|
+
def _toggle_dbt_engine(self) -> None:
|
|
263
|
+
"""Toggle between dbt Fusion and dbt Core."""
|
|
264
|
+
display_action_header("Switch dbt Engine")
|
|
265
|
+
current = self.datasecops_config.dbt_engine or "dbtf"
|
|
266
|
+
info_line(f"Current engine: {self.datasecops_config.get_engine_label()} ({current})")
|
|
267
|
+
info_line("")
|
|
268
|
+
menu_option(1, "dbt Fusion - dbtf (recommended)")
|
|
269
|
+
menu_option(2, "dbt Core - dbt")
|
|
270
|
+
menu_option(0, "cancel")
|
|
271
|
+
option = get_input_number("Choose an option: ")
|
|
272
|
+
|
|
273
|
+
if option == 1:
|
|
274
|
+
new_engine = "dbtf"
|
|
275
|
+
elif option == 2:
|
|
276
|
+
new_engine = "dbt"
|
|
277
|
+
else:
|
|
278
|
+
return
|
|
279
|
+
|
|
280
|
+
if new_engine == current:
|
|
281
|
+
info_line("No change.")
|
|
282
|
+
complete_action()
|
|
283
|
+
return
|
|
284
|
+
|
|
285
|
+
self.datasecops_config.dbt_engine = new_engine
|
|
286
|
+
config_data = self.datasecops_config.model_dump(exclude={"VALID_ENGINES"})
|
|
287
|
+
write_datasecops_config(self.project_dir, config_data)
|
|
288
|
+
success_line(f"dbt engine switched to {self.datasecops_config.get_engine_label()} ({new_engine})")
|
|
289
|
+
warning_line("Restart the CLI for the change to take effect.")
|
|
290
|
+
complete_action()
|
|
@@ -19,7 +19,7 @@ class DownloadsMenu:
|
|
|
19
19
|
def __init__(self, download_service: DownloadService, skill_service: SkillService,
|
|
20
20
|
dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
|
|
21
21
|
project_settings: ProjectSettings = None, profile: ProjectProfile = None,
|
|
22
|
-
source_control: SourceControl = None):
|
|
22
|
+
source_control: SourceControl = None, all_profiles: list[ProjectProfile] = None):
|
|
23
23
|
self.downloads = download_service
|
|
24
24
|
self.skills = skill_service
|
|
25
25
|
self.dbt = dbt_runner
|
|
@@ -28,6 +28,7 @@ class DownloadsMenu:
|
|
|
28
28
|
self.project_settings = project_settings
|
|
29
29
|
self.profile = profile
|
|
30
30
|
self.source_control = source_control
|
|
31
|
+
self.all_profiles = all_profiles
|
|
31
32
|
|
|
32
33
|
def show(self) -> None:
|
|
33
34
|
self._menu()
|
|
@@ -46,7 +47,8 @@ class DownloadsMenu:
|
|
|
46
47
|
complete_action()
|
|
47
48
|
elif option == 3:
|
|
48
49
|
display_action_header("Download dbt Packages")
|
|
49
|
-
self.downloads.download_dbt_packages(self.dbt_project_dir
|
|
50
|
+
self.downloads.download_dbt_packages(self.dbt_project_dir, profile=self.profile,
|
|
51
|
+
all_profiles=self.all_profiles)
|
|
50
52
|
if get_input_true_false("Run dbt deps now?"):
|
|
51
53
|
self.dbt.deps()
|
|
52
54
|
complete_action()
|
|
@@ -7,6 +7,15 @@ class DatasecopsConfig(BaseModel):
|
|
|
7
7
|
app_database: str = ""
|
|
8
8
|
profile_name: str = ""
|
|
9
9
|
cortex_connection: str = ""
|
|
10
|
+
dbt_engine: str = "dbtf" # "dbtf" for dbt Fusion, "dbt" for dbt Core
|
|
11
|
+
|
|
12
|
+
VALID_ENGINES: dict = {"dbtf": "dbt Fusion", "dbt": "dbt Core"}
|
|
13
|
+
|
|
14
|
+
model_config = {"arbitrary_types_allowed": True}
|
|
15
|
+
|
|
16
|
+
def get_engine_label(self) -> str:
|
|
17
|
+
"""Return human-readable label for the current dbt engine."""
|
|
18
|
+
return self.VALID_ENGINES.get(self.dbt_engine, f"unknown ({self.dbt_engine})")
|
|
10
19
|
|
|
11
20
|
class DbtTarget(BaseModel):
|
|
12
21
|
target_name: str = ""
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
@@ -14,11 +14,13 @@ class BootstrapService:
|
|
|
14
14
|
"""Bootstraps a new dbt project with all framework configuration."""
|
|
15
15
|
|
|
16
16
|
def __init__(self, snowflake_service: SnowflakeService, project_dir: Path,
|
|
17
|
-
project_settings: ProjectSettings, profile: ProjectProfile
|
|
17
|
+
project_settings: ProjectSettings, profile: ProjectProfile,
|
|
18
|
+
all_profiles: list[ProjectProfile] = None):
|
|
18
19
|
self.sf = snowflake_service
|
|
19
20
|
self.project_dir = project_dir
|
|
20
21
|
self.project_settings = project_settings
|
|
21
22
|
self.profile = profile
|
|
23
|
+
self.all_profiles = all_profiles or []
|
|
22
24
|
self.download_service = DownloadService(snowflake_service, project_dir)
|
|
23
25
|
self.skill_service = SkillService(snowflake_service)
|
|
24
26
|
|
|
@@ -92,7 +94,8 @@ class BootstrapService:
|
|
|
92
94
|
# Step 5: Generate packages.yml
|
|
93
95
|
info_line("")
|
|
94
96
|
info_line("[5] Generating packages.yml...")
|
|
95
|
-
if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile
|
|
97
|
+
if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile,
|
|
98
|
+
all_profiles=self.all_profiles):
|
|
96
99
|
steps_passed += 1
|
|
97
100
|
else:
|
|
98
101
|
info_line(" (skipped - no dbt packages config in native app)")
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_project_generator.py
RENAMED
|
@@ -165,17 +165,36 @@ def generate_profiles_yml(
|
|
|
165
165
|
def generate_packages_yml(
|
|
166
166
|
profile: ProjectProfile,
|
|
167
167
|
packages_config: dict,
|
|
168
|
+
all_profiles: list[ProjectProfile] = None,
|
|
168
169
|
) -> str:
|
|
169
170
|
"""Generate packages.yml from profile's packages and upstream projects.
|
|
170
171
|
|
|
171
172
|
Args:
|
|
172
173
|
profile: The project profile.
|
|
173
174
|
packages_config: The DBT_PACKAGES framework config.
|
|
175
|
+
all_profiles: All project profiles (used for upstream package exclusion).
|
|
174
176
|
|
|
175
177
|
Returns:
|
|
176
178
|
The YAML content as a string.
|
|
177
179
|
"""
|
|
178
180
|
packages = packages_config.get("packages", [])
|
|
181
|
+
|
|
182
|
+
# Filter to only packages enabled on the profile
|
|
183
|
+
if profile.dbt_packages:
|
|
184
|
+
from datasecops_cli.services.download_service import _collect_upstream_packages
|
|
185
|
+
|
|
186
|
+
enabled = set(profile.dbt_packages)
|
|
187
|
+
|
|
188
|
+
# Exclude packages already provided by upstream projects
|
|
189
|
+
if profile.upstream_projects and all_profiles:
|
|
190
|
+
profile_map = {p.profile_name: p for p in all_profiles}
|
|
191
|
+
upstream_pkgs = _collect_upstream_packages(
|
|
192
|
+
profile.upstream_projects, profile_map
|
|
193
|
+
)
|
|
194
|
+
enabled -= upstream_pkgs
|
|
195
|
+
|
|
196
|
+
packages = [p for p in packages if p.get("name") in enabled]
|
|
197
|
+
|
|
179
198
|
pkg_list: list[dict[str, Any]] = []
|
|
180
199
|
|
|
181
200
|
for pkg in packages:
|
|
@@ -7,12 +7,14 @@ from datasecops_cli.utilities.display import info_line, error_line, success_line
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class DbtRunner:
|
|
10
|
-
"""Runs dbt commands via subprocess (dbt Fusion)."""
|
|
10
|
+
"""Runs dbt commands via subprocess (dbt Fusion or dbt Core)."""
|
|
11
11
|
|
|
12
|
-
def __init__(self, project_dir: Path, profiles_dir: Path, target: str = "dev"
|
|
12
|
+
def __init__(self, project_dir: Path, profiles_dir: Path, target: str = "dev",
|
|
13
|
+
engine: str = "dbtf"):
|
|
13
14
|
self.project_dir = project_dir
|
|
14
15
|
self.profiles_dir = profiles_dir
|
|
15
16
|
self.target = target
|
|
17
|
+
self.engine = engine # "dbtf" or "dbt"
|
|
16
18
|
|
|
17
19
|
def _default_args(self) -> list[str]:
|
|
18
20
|
return [
|
|
@@ -21,7 +23,7 @@ class DbtRunner:
|
|
|
21
23
|
]
|
|
22
24
|
|
|
23
25
|
def _run_command(self, command: str, extra_args: list[str] = None) -> subprocess.CompletedProcess:
|
|
24
|
-
cmd = [
|
|
26
|
+
cmd = [self.engine, command] + (extra_args or []) + self._default_args()
|
|
25
27
|
info_line(f"Running: {' '.join(cmd)}")
|
|
26
28
|
result = subprocess.run(cmd, capture_output=False)
|
|
27
29
|
if result.returncode != 0:
|
|
@@ -94,7 +96,7 @@ class DbtRunner:
|
|
|
94
96
|
return self._run_command("docs", ["generate", f"--target={self.target}"])
|
|
95
97
|
|
|
96
98
|
def docs_serve(self) -> subprocess.Popen:
|
|
97
|
-
cmd = [
|
|
99
|
+
cmd = [self.engine, "docs", "serve"] + self._default_args()
|
|
98
100
|
info_line(f"Running: {' '.join(cmd)}")
|
|
99
101
|
return subprocess.Popen(cmd)
|
|
100
102
|
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/download_service.py
RENAMED
|
@@ -10,6 +10,30 @@ from datasecops_cli.utilities.display import info_line, success_line, error_line
|
|
|
10
10
|
from datasecops_cli.utilities.file_utils import write_file, ensure_dir
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
def _collect_upstream_packages(
|
|
14
|
+
upstream_names: list[str],
|
|
15
|
+
profile_map: dict[str, ProjectProfile],
|
|
16
|
+
visited: set[str] | None = None,
|
|
17
|
+
) -> set[str]:
|
|
18
|
+
"""Recursively collect dbt package names from all upstream projects."""
|
|
19
|
+
if visited is None:
|
|
20
|
+
visited = set()
|
|
21
|
+
result: set[str] = set()
|
|
22
|
+
for name in upstream_names:
|
|
23
|
+
if name in visited:
|
|
24
|
+
continue
|
|
25
|
+
visited.add(name)
|
|
26
|
+
upstream = profile_map.get(name)
|
|
27
|
+
if upstream:
|
|
28
|
+
result.update(upstream.dbt_packages)
|
|
29
|
+
result.update(
|
|
30
|
+
_collect_upstream_packages(
|
|
31
|
+
upstream.upstream_projects or [], profile_map, visited
|
|
32
|
+
)
|
|
33
|
+
)
|
|
34
|
+
return result
|
|
35
|
+
|
|
36
|
+
|
|
13
37
|
class DownloadService:
|
|
14
38
|
"""Downloads configurations from the native app."""
|
|
15
39
|
|
|
@@ -279,7 +303,8 @@ class DownloadService:
|
|
|
279
303
|
success_line(f"Downloaded {count} script(s) to {scripts_dir}")
|
|
280
304
|
return True
|
|
281
305
|
|
|
282
|
-
def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None
|
|
306
|
+
def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None,
|
|
307
|
+
all_profiles: list[ProjectProfile] = None) -> bool:
|
|
283
308
|
info_line("Downloading dbt package versions...")
|
|
284
309
|
raw = self.sf.get_framework_config("DBT_PACKAGES")
|
|
285
310
|
if not raw:
|
|
@@ -287,6 +312,22 @@ class DownloadService:
|
|
|
287
312
|
return False
|
|
288
313
|
|
|
289
314
|
packages = raw.get("packages", [])
|
|
315
|
+
|
|
316
|
+
# Filter to only packages enabled on the active profile
|
|
317
|
+
if profile and profile.dbt_packages:
|
|
318
|
+
enabled = set(profile.dbt_packages)
|
|
319
|
+
|
|
320
|
+
# Exclude packages already provided by upstream projects
|
|
321
|
+
if profile.upstream_projects and all_profiles:
|
|
322
|
+
profile_map = {p.profile_name: p for p in all_profiles}
|
|
323
|
+
upstream_pkgs = _collect_upstream_packages(
|
|
324
|
+
profile.upstream_projects, profile_map
|
|
325
|
+
)
|
|
326
|
+
enabled -= upstream_pkgs
|
|
327
|
+
|
|
328
|
+
packages = [p for p in packages if p.get("name") in enabled]
|
|
329
|
+
info_line(f" Filtered to {len(packages)} package(s) for profile '{profile.profile_name}'")
|
|
330
|
+
|
|
290
331
|
pkg_list = []
|
|
291
332
|
for pkg in packages:
|
|
292
333
|
source = pkg.get("source", "git")
|
|
@@ -5,6 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
|
|
7
7
|
from datasecops_cli.config import Config
|
|
8
|
+
from datasecops_cli.models.project_config import ProjectProfile
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class TestBuildParser:
|
|
@@ -64,6 +65,11 @@ class TestRunDownload:
|
|
|
64
65
|
config.project_dir = tmp_path
|
|
65
66
|
config.dbt_project_dir = tmp_path
|
|
66
67
|
config.profile_name = "test_profile"
|
|
68
|
+
config.profile = ProjectProfile(
|
|
69
|
+
profile_name="test_profile",
|
|
70
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
71
|
+
)
|
|
72
|
+
config.all_profiles = [config.profile]
|
|
67
73
|
# source_control defaults to GitHub
|
|
68
74
|
return config
|
|
69
75
|
|
|
@@ -119,7 +125,9 @@ class TestRunDownload:
|
|
|
119
125
|
_run_download(config, ["packages"])
|
|
120
126
|
|
|
121
127
|
assert exc.value.code == 0
|
|
122
|
-
mock_ds.download_dbt_packages.assert_called_once_with(
|
|
128
|
+
mock_ds.download_dbt_packages.assert_called_once_with(
|
|
129
|
+
tmp_path, profile=config.profile, all_profiles=config.all_profiles
|
|
130
|
+
)
|
|
123
131
|
|
|
124
132
|
@patch("datasecops_cli.main._connect_and_load")
|
|
125
133
|
def test_download_macros(self, mock_connect, tmp_path):
|
|
@@ -300,3 +308,112 @@ class TestRunDownload:
|
|
|
300
308
|
_run_download(config, ["install-dbt"])
|
|
301
309
|
|
|
302
310
|
assert exc.value.code == 1
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class TestDownloadPackageFiltering:
|
|
314
|
+
"""Tests for dbt package filtering by profile."""
|
|
315
|
+
|
|
316
|
+
def test_download_dbt_packages_filters_by_profile(self, tmp_path):
|
|
317
|
+
"""Only packages in profile.dbt_packages should appear in packages.yml."""
|
|
318
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
319
|
+
|
|
320
|
+
mock_sf = MagicMock()
|
|
321
|
+
mock_sf.get_framework_config.return_value = {
|
|
322
|
+
"packages": [
|
|
323
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
324
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
325
|
+
{"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
|
|
326
|
+
]
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
profile = ProjectProfile(
|
|
330
|
+
profile_name="test",
|
|
331
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
335
|
+
service.download_dbt_packages(tmp_path, profile=profile)
|
|
336
|
+
|
|
337
|
+
import yaml
|
|
338
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
339
|
+
assert len(result["packages"]) == 2
|
|
340
|
+
urls = [p.get("git") for p in result["packages"]]
|
|
341
|
+
assert "https://github.com/dbt-labs/dbt-utils.git" in urls
|
|
342
|
+
assert "https://github.com/calogica/dbt-expectations.git" in urls
|
|
343
|
+
assert "https://github.com/Snowflake-Labs/dbt_constraints.git" not in urls
|
|
344
|
+
|
|
345
|
+
def test_download_dbt_packages_excludes_upstream(self, tmp_path):
|
|
346
|
+
"""Packages provided by upstream projects should be excluded."""
|
|
347
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
348
|
+
|
|
349
|
+
mock_sf = MagicMock()
|
|
350
|
+
mock_sf.get_framework_config.return_value = {
|
|
351
|
+
"packages": [
|
|
352
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
353
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
354
|
+
]
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
upstream_profile = ProjectProfile(
|
|
358
|
+
profile_name="upstream_project",
|
|
359
|
+
dbt_packages=["dbt_utils"],
|
|
360
|
+
)
|
|
361
|
+
profile = ProjectProfile(
|
|
362
|
+
profile_name="test",
|
|
363
|
+
dbt_packages=["dbt_utils", "dbt_expectations"],
|
|
364
|
+
upstream_projects=["upstream_project"],
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
368
|
+
service.download_dbt_packages(tmp_path, profile=profile, all_profiles=[profile, upstream_profile])
|
|
369
|
+
|
|
370
|
+
import yaml
|
|
371
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
372
|
+
# dbt_utils excluded (provided by upstream), only dbt_expectations + local upstream dep
|
|
373
|
+
git_packages = [p for p in result["packages"] if "git" in p]
|
|
374
|
+
local_packages = [p for p in result["packages"] if "local" in p]
|
|
375
|
+
assert len(git_packages) == 1
|
|
376
|
+
assert git_packages[0]["git"] == "https://github.com/calogica/dbt-expectations.git"
|
|
377
|
+
assert len(local_packages) == 1
|
|
378
|
+
assert local_packages[0]["local"] == "local_packages/upstream_project"
|
|
379
|
+
|
|
380
|
+
def test_download_dbt_packages_no_profile_downloads_all(self, tmp_path):
|
|
381
|
+
"""When no profile is passed, all packages should be included."""
|
|
382
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
383
|
+
|
|
384
|
+
mock_sf = MagicMock()
|
|
385
|
+
mock_sf.get_framework_config.return_value = {
|
|
386
|
+
"packages": [
|
|
387
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
388
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
389
|
+
{"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
|
|
390
|
+
]
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
394
|
+
service.download_dbt_packages(tmp_path, profile=None)
|
|
395
|
+
|
|
396
|
+
import yaml
|
|
397
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
398
|
+
assert len(result["packages"]) == 3
|
|
399
|
+
|
|
400
|
+
def test_download_dbt_packages_empty_dbt_packages_downloads_all(self, tmp_path):
|
|
401
|
+
"""When profile.dbt_packages is empty, all packages should be included."""
|
|
402
|
+
from datasecops_cli.services.download_service import DownloadService
|
|
403
|
+
|
|
404
|
+
mock_sf = MagicMock()
|
|
405
|
+
mock_sf.get_framework_config.return_value = {
|
|
406
|
+
"packages": [
|
|
407
|
+
{"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
|
|
408
|
+
{"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
|
|
409
|
+
]
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
profile = ProjectProfile(profile_name="test", dbt_packages=[])
|
|
413
|
+
|
|
414
|
+
service = DownloadService(mock_sf, tmp_path)
|
|
415
|
+
service.download_dbt_packages(tmp_path, profile=profile)
|
|
416
|
+
|
|
417
|
+
import yaml
|
|
418
|
+
result = yaml.safe_load((tmp_path / "packages.yml").read_text())
|
|
419
|
+
assert len(result["packages"]) == 2
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.4.3"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/directory_scaffolder.py
RENAMED
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/linting_service.py
RENAMED
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/snowflake_service.py
RENAMED
|
File without changes
|
{datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/upstream_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|