datasecops-cli 0.4.3__tar.gz → 0.4.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/CHANGELOG.md +14 -0
  2. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/PKG-INFO +1 -1
  3. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/pyproject.toml +1 -1
  4. datasecops_cli-0.4.5/src/datasecops_cli/__init__.py +1 -0
  5. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/config.py +7 -0
  6. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/main.py +21 -6
  7. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/configuration.py +40 -2
  8. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/downloads.py +4 -2
  9. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/project_config.py +9 -0
  10. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/bootstrap_service.py +5 -2
  11. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_project_generator.py +19 -0
  12. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_runner.py +6 -4
  13. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/download_service.py +42 -1
  14. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_main.py +118 -1
  15. datasecops_cli-0.4.3/src/datasecops_cli/__init__.py +0 -1
  16. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.github/workflows/auto-tag.yml +0 -0
  17. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.github/workflows/publish-cli.yml +0 -0
  18. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/.gitignore +0 -0
  19. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/DEVELOPMENT.md +0 -0
  20. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/LICENSE +0 -0
  21. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/README.md +0 -0
  22. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/getting-started.md +0 -0
  23. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/legacy.md +0 -0
  24. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/legacy_plan_of_action.md +0 -0
  25. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/docs/mcp-server.md +0 -0
  26. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/mcp-servers.json +0 -0
  27. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/setup.ps1 +0 -0
  28. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/setup.sh +0 -0
  29. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/__init__.py +0 -0
  30. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/development.py +0 -0
  31. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/git_operations.py +0 -0
  32. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/__init__.py +0 -0
  33. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/models/git_helpers.py +0 -0
  34. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/__init__.py +0 -0
  35. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
  36. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/git_service.py +0 -0
  37. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/linting_service.py +0 -0
  38. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/skill_service.py +0 -0
  39. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/snowflake_service.py +0 -0
  40. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/services/upstream_service.py +0 -0
  41. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/__init__.py +0 -0
  42. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/display.py +0 -0
  43. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/file_utils.py +0 -0
  44. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  45. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/__init__.py +0 -0
  46. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/__main__.py +0 -0
  47. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/connection.py +0 -0
  48. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/src/datasecops_mcp/server.py +0 -0
  49. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/__init__.py +0 -0
  50. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_config.py +0 -0
  51. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_file_utils.py +0 -0
  52. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_models.py +0 -0
  53. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_version.py +0 -0
  54. {datasecops_cli-0.4.3 → datasecops_cli-0.4.5}/tests/test_yaml_utils.py +0 -0
@@ -2,6 +2,20 @@
2
2
 
3
3
  All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
+ ## [0.4.5] - 2026-05-18
6
+
7
+ ### Fixed
8
+
9
+ - **Package downloads now respect profile selection** — `datasecops download packages` and the interactive downloads menu now only include dbt packages enabled on the active profile, instead of downloading all packages from the global catalog. This matches the native app's Streamlit UI behavior.
10
+ - **Upstream package exclusion** — packages already provided by upstream projects are automatically excluded from `packages.yml`, preventing duplicate dependencies when using multi-project setups.
11
+ - **`generate_packages_yml()` filtering** — the dbt project generator now also filters packages by profile, fixing the same issue in the bootstrap/init path.
12
+
13
+ ## [0.4.4] - 2026-05-18
14
+
15
+ ### Added
16
+
17
+ - **dbt engine toggle** — new `dbt_engine` field in `.datasecops.yml` to switch between `dbtf` (dbt Fusion) and `dbt` (dbt Core). Defaults to `dbtf`. Toggleable via Configure menu `[6] dbt engine`. All dbt commands use the configured engine binary.
18
+
5
19
  ## [0.4.3] - 2026-05-17
6
20
 
7
21
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasecops-cli
3
- Version: 0.4.3
3
+ Version: 0.4.5
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.4.3"
7
+ version = "0.4.5"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1 @@
1
+ __version__ = "0.4.5"
@@ -36,7 +36,14 @@ class Config:
36
36
  app_database=raw.get("app_database", ""),
37
37
  profile_name=raw.get("profile_name", ""),
38
38
  cortex_connection=raw.get("cortex_connection", ""),
39
+ dbt_engine=raw.get("dbt_engine", "dbtf"),
39
40
  )
41
+
42
+ # Validate dbt_engine
43
+ if self.datasecops.dbt_engine not in self.datasecops.VALID_ENGINES:
44
+ from datasecops_cli.utilities.display import warning_line
45
+ warning_line(f"Invalid dbt_engine '{self.datasecops.dbt_engine}' in .datasecops.yml — falling back to dbtf")
46
+ self.datasecops.dbt_engine = "dbtf"
40
47
 
41
48
  if not self.datasecops.connection_name or not self.datasecops.app_database:
42
49
  error_line(".datasecops.yml is missing connection_name or app_database.")
@@ -473,11 +473,17 @@ def _run_interactive(config: Config):
473
473
  sf_service = _connect_and_load(config)
474
474
 
475
475
  try:
476
- # Check for dbt Fusion (dbtf) on PATH
477
- if not shutil.which("dbtf"):
476
+ # Check for configured dbt engine on PATH
477
+ dbt_engine = config.datasecops.dbt_engine or "dbtf"
478
+ if not shutil.which(dbt_engine):
478
479
  from datasecops_cli.utilities.display import warning_line
479
- warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
480
- warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
480
+ warning_line(f"{dbt_engine} not found on PATH — dbt commands will not work.")
481
+ if dbt_engine == "dbtf":
482
+ warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
483
+ else:
484
+ warning_line("Install dbt Core: uv pip install dbt-core dbt-snowflake")
485
+ else:
486
+ info_line(f"dbt engine: {config.datasecops.get_engine_label()}")
481
487
 
482
488
  # Check upstream project versions
483
489
  from datasecops_cli.services.upstream_service import check_upstream_versions
@@ -492,7 +498,8 @@ def _run_interactive(config: Config):
492
498
  dbt_runner = DbtRunner(
493
499
  project_dir=config.dbt_project_dir,
494
500
  profiles_dir=config.get_dbt_profiles_dir(),
495
- target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev"
501
+ target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev",
502
+ engine=config.datasecops.dbt_engine or "dbtf"
496
503
  )
497
504
 
498
505
  try:
@@ -558,7 +565,10 @@ def _run_download(config: Config, items: list[str],
558
565
  if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
559
566
  failed = True
560
567
  elif item == "packages":
561
- if not download_service.download_dbt_packages(config.dbt_project_dir):
568
+ if not download_service.download_dbt_packages(
569
+ config.dbt_project_dir, profile=config.profile,
570
+ all_profiles=config.all_profiles
571
+ ):
562
572
  failed = True
563
573
  elif item == "macros":
564
574
  if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
@@ -685,6 +695,8 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
685
695
  project_settings=config.project_settings,
686
696
  profile=config.profile,
687
697
  source_control=config.source_control,
698
+ datasecops_config=config.datasecops,
699
+ project_dir=config.project_dir,
688
700
  )
689
701
  config_menu.show()
690
702
 
@@ -695,6 +707,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
695
707
  project_settings=config.project_settings,
696
708
  profile=config.profile,
697
709
  source_control=config.source_control,
710
+ all_profiles=config.all_profiles,
698
711
  )
699
712
  dl_menu.show()
700
713
 
@@ -713,6 +726,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
713
726
  project_dir=config.project_dir,
714
727
  project_settings=config.project_settings,
715
728
  profile=config.profile,
729
+ all_profiles=config.all_profiles,
716
730
  )
717
731
  bootstrap.run(platform=platform, install_skills=install_skills,
718
732
  skill_targets=skill_targets, run_deps=run_deps)
@@ -775,6 +789,7 @@ def _run_bootstrap(config: Config):
775
789
  project_dir=config.project_dir,
776
790
  project_settings=config.project_settings,
777
791
  profile=config.profile,
792
+ all_profiles=config.all_profiles,
778
793
  )
779
794
  success = bootstrap.run(platform=platform, install_skills=install_skills,
780
795
  skill_targets=skill_targets, run_deps=run_deps)
@@ -5,7 +5,7 @@ import shutil
5
5
  import subprocess
6
6
  from pathlib import Path
7
7
 
8
- from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings, SourceControl
8
+ from datasecops_cli.models.project_config import DatasecopsConfig, ProjectProfile, ProjectSettings, SourceControl
9
9
  from datasecops_cli.services.download_service import DownloadService
10
10
  from datasecops_cli.services.linting_service import LintingService
11
11
  from datasecops_cli.utilities.display import (
@@ -14,13 +14,15 @@ from datasecops_cli.utilities.display import (
14
14
  info_line, success_line, warning_line, error_line,
15
15
  get_input_string, select_from_list, select_multiple_from_list
16
16
  )
17
+ from datasecops_cli.utilities.yaml_utils import write_datasecops_config
17
18
 
18
19
 
19
20
  class ConfigurationMenu:
20
21
  def __init__(self, download_service: DownloadService, linting_service: LintingService,
21
22
  profile_name: str, dbt_project_dir: Path,
22
23
  project_settings: ProjectSettings = None, profile: ProjectProfile = None,
23
- source_control: SourceControl = None):
24
+ source_control: SourceControl = None,
25
+ datasecops_config: DatasecopsConfig = None, project_dir: Path = None):
24
26
  self.downloads = download_service
25
27
  self.linting = linting_service
26
28
  self.profile_name = profile_name
@@ -28,6 +30,8 @@ class ConfigurationMenu:
28
30
  self.project_settings = project_settings
29
31
  self.profile = profile
30
32
  self.source_control = source_control
33
+ self.datasecops_config = datasecops_config or DatasecopsConfig()
34
+ self.project_dir = project_dir or Path.cwd()
31
35
 
32
36
  def show(self) -> None:
33
37
  self._menu()
@@ -45,17 +49,21 @@ class ConfigurationMenu:
45
49
  complete_action()
46
50
  elif option == 5:
47
51
  self._cortex_upgrade()
52
+ elif option == 6:
53
+ self._toggle_dbt_engine()
48
54
  self._menu()
49
55
  option = get_input_number("Choose an option: ")
50
56
 
51
57
  def _menu(self) -> None:
52
58
  clear()
59
+ engine_label = self.datasecops_config.get_engine_label()
53
60
  section_header("Configuration", self.profile_name)
54
61
  menu_option(1, "install dbt - Install dbt-core & dbt-snowflake from framework versions")
55
62
  menu_option(2, "install lint - Install SQLFluff from framework versions")
56
63
  menu_option(3, "mcp servers - Configure MCP servers for AI tools")
57
64
  menu_option(4, "new project - Initialize a new dbt project with framework profiles")
58
65
  menu_option(5, "cortex update - Update Cortex Code to the latest version")
66
+ menu_option(6, f"dbt engine - Switch dbt engine (current: {engine_label})")
59
67
  menu_option(0, "back - Return to main menu")
60
68
 
61
69
  def _install_dbt_requirements(self) -> None:
@@ -250,3 +258,33 @@ class ConfigurationMenu:
250
258
  except FileNotFoundError:
251
259
  error_line("Cortex Code CLI not found.")
252
260
  complete_action()
261
+
262
+ def _toggle_dbt_engine(self) -> None:
263
+ """Toggle between dbt Fusion and dbt Core."""
264
+ display_action_header("Switch dbt Engine")
265
+ current = self.datasecops_config.dbt_engine or "dbtf"
266
+ info_line(f"Current engine: {self.datasecops_config.get_engine_label()} ({current})")
267
+ info_line("")
268
+ menu_option(1, "dbt Fusion - dbtf (recommended)")
269
+ menu_option(2, "dbt Core - dbt")
270
+ menu_option(0, "cancel")
271
+ option = get_input_number("Choose an option: ")
272
+
273
+ if option == 1:
274
+ new_engine = "dbtf"
275
+ elif option == 2:
276
+ new_engine = "dbt"
277
+ else:
278
+ return
279
+
280
+ if new_engine == current:
281
+ info_line("No change.")
282
+ complete_action()
283
+ return
284
+
285
+ self.datasecops_config.dbt_engine = new_engine
286
+ config_data = self.datasecops_config.model_dump(exclude={"VALID_ENGINES"})
287
+ write_datasecops_config(self.project_dir, config_data)
288
+ success_line(f"dbt engine switched to {self.datasecops_config.get_engine_label()} ({new_engine})")
289
+ warning_line("Restart the CLI for the change to take effect.")
290
+ complete_action()
@@ -19,7 +19,7 @@ class DownloadsMenu:
19
19
  def __init__(self, download_service: DownloadService, skill_service: SkillService,
20
20
  dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
21
21
  project_settings: ProjectSettings = None, profile: ProjectProfile = None,
22
- source_control: SourceControl = None):
22
+ source_control: SourceControl = None, all_profiles: list[ProjectProfile] = None):
23
23
  self.downloads = download_service
24
24
  self.skills = skill_service
25
25
  self.dbt = dbt_runner
@@ -28,6 +28,7 @@ class DownloadsMenu:
28
28
  self.project_settings = project_settings
29
29
  self.profile = profile
30
30
  self.source_control = source_control
31
+ self.all_profiles = all_profiles
31
32
 
32
33
  def show(self) -> None:
33
34
  self._menu()
@@ -46,7 +47,8 @@ class DownloadsMenu:
46
47
  complete_action()
47
48
  elif option == 3:
48
49
  display_action_header("Download dbt Packages")
49
- self.downloads.download_dbt_packages(self.dbt_project_dir)
50
+ self.downloads.download_dbt_packages(self.dbt_project_dir, profile=self.profile,
51
+ all_profiles=self.all_profiles)
50
52
  if get_input_true_false("Run dbt deps now?"):
51
53
  self.dbt.deps()
52
54
  complete_action()
@@ -7,6 +7,15 @@ class DatasecopsConfig(BaseModel):
7
7
  app_database: str = ""
8
8
  profile_name: str = ""
9
9
  cortex_connection: str = ""
10
+ dbt_engine: str = "dbtf" # "dbtf" for dbt Fusion, "dbt" for dbt Core
11
+
12
+ VALID_ENGINES: dict = {"dbtf": "dbt Fusion", "dbt": "dbt Core"}
13
+
14
+ model_config = {"arbitrary_types_allowed": True}
15
+
16
+ def get_engine_label(self) -> str:
17
+ """Return human-readable label for the current dbt engine."""
18
+ return self.VALID_ENGINES.get(self.dbt_engine, f"unknown ({self.dbt_engine})")
10
19
 
11
20
  class DbtTarget(BaseModel):
12
21
  target_name: str = ""
@@ -14,11 +14,13 @@ class BootstrapService:
14
14
  """Bootstraps a new dbt project with all framework configuration."""
15
15
 
16
16
  def __init__(self, snowflake_service: SnowflakeService, project_dir: Path,
17
- project_settings: ProjectSettings, profile: ProjectProfile):
17
+ project_settings: ProjectSettings, profile: ProjectProfile,
18
+ all_profiles: list[ProjectProfile] = None):
18
19
  self.sf = snowflake_service
19
20
  self.project_dir = project_dir
20
21
  self.project_settings = project_settings
21
22
  self.profile = profile
23
+ self.all_profiles = all_profiles or []
22
24
  self.download_service = DownloadService(snowflake_service, project_dir)
23
25
  self.skill_service = SkillService(snowflake_service)
24
26
 
@@ -92,7 +94,8 @@ class BootstrapService:
92
94
  # Step 5: Generate packages.yml
93
95
  info_line("")
94
96
  info_line("[5] Generating packages.yml...")
95
- if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile):
97
+ if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile,
98
+ all_profiles=self.all_profiles):
96
99
  steps_passed += 1
97
100
  else:
98
101
  info_line(" (skipped - no dbt packages config in native app)")
@@ -165,17 +165,36 @@ def generate_profiles_yml(
165
165
  def generate_packages_yml(
166
166
  profile: ProjectProfile,
167
167
  packages_config: dict,
168
+ all_profiles: list[ProjectProfile] = None,
168
169
  ) -> str:
169
170
  """Generate packages.yml from profile's packages and upstream projects.
170
171
 
171
172
  Args:
172
173
  profile: The project profile.
173
174
  packages_config: The DBT_PACKAGES framework config.
175
+ all_profiles: All project profiles (used for upstream package exclusion).
174
176
 
175
177
  Returns:
176
178
  The YAML content as a string.
177
179
  """
178
180
  packages = packages_config.get("packages", [])
181
+
182
+ # Filter to only packages enabled on the profile
183
+ if profile.dbt_packages:
184
+ from datasecops_cli.services.download_service import _collect_upstream_packages
185
+
186
+ enabled = set(profile.dbt_packages)
187
+
188
+ # Exclude packages already provided by upstream projects
189
+ if profile.upstream_projects and all_profiles:
190
+ profile_map = {p.profile_name: p for p in all_profiles}
191
+ upstream_pkgs = _collect_upstream_packages(
192
+ profile.upstream_projects, profile_map
193
+ )
194
+ enabled -= upstream_pkgs
195
+
196
+ packages = [p for p in packages if p.get("name") in enabled]
197
+
179
198
  pkg_list: list[dict[str, Any]] = []
180
199
 
181
200
  for pkg in packages:
@@ -7,12 +7,14 @@ from datasecops_cli.utilities.display import info_line, error_line, success_line
7
7
 
8
8
 
9
9
  class DbtRunner:
10
- """Runs dbt commands via subprocess (dbt Fusion)."""
10
+ """Runs dbt commands via subprocess (dbt Fusion or dbt Core)."""
11
11
 
12
- def __init__(self, project_dir: Path, profiles_dir: Path, target: str = "dev"):
12
+ def __init__(self, project_dir: Path, profiles_dir: Path, target: str = "dev",
13
+ engine: str = "dbtf"):
13
14
  self.project_dir = project_dir
14
15
  self.profiles_dir = profiles_dir
15
16
  self.target = target
17
+ self.engine = engine # "dbtf" or "dbt"
16
18
 
17
19
  def _default_args(self) -> list[str]:
18
20
  return [
@@ -21,7 +23,7 @@ class DbtRunner:
21
23
  ]
22
24
 
23
25
  def _run_command(self, command: str, extra_args: list[str] = None) -> subprocess.CompletedProcess:
24
- cmd = ["dbtf", command] + (extra_args or []) + self._default_args()
26
+ cmd = [self.engine, command] + (extra_args or []) + self._default_args()
25
27
  info_line(f"Running: {' '.join(cmd)}")
26
28
  result = subprocess.run(cmd, capture_output=False)
27
29
  if result.returncode != 0:
@@ -94,7 +96,7 @@ class DbtRunner:
94
96
  return self._run_command("docs", ["generate", f"--target={self.target}"])
95
97
 
96
98
  def docs_serve(self) -> subprocess.Popen:
97
- cmd = ["dbtf", "docs", "serve"] + self._default_args()
99
+ cmd = [self.engine, "docs", "serve"] + self._default_args()
98
100
  info_line(f"Running: {' '.join(cmd)}")
99
101
  return subprocess.Popen(cmd)
100
102
 
@@ -10,6 +10,30 @@ from datasecops_cli.utilities.display import info_line, success_line, error_line
10
10
  from datasecops_cli.utilities.file_utils import write_file, ensure_dir
11
11
 
12
12
 
13
+ def _collect_upstream_packages(
14
+ upstream_names: list[str],
15
+ profile_map: dict[str, ProjectProfile],
16
+ visited: set[str] | None = None,
17
+ ) -> set[str]:
18
+ """Recursively collect dbt package names from all upstream projects."""
19
+ if visited is None:
20
+ visited = set()
21
+ result: set[str] = set()
22
+ for name in upstream_names:
23
+ if name in visited:
24
+ continue
25
+ visited.add(name)
26
+ upstream = profile_map.get(name)
27
+ if upstream:
28
+ result.update(upstream.dbt_packages)
29
+ result.update(
30
+ _collect_upstream_packages(
31
+ upstream.upstream_projects or [], profile_map, visited
32
+ )
33
+ )
34
+ return result
35
+
36
+
13
37
  class DownloadService:
14
38
  """Downloads configurations from the native app."""
15
39
 
@@ -279,7 +303,8 @@ class DownloadService:
279
303
  success_line(f"Downloaded {count} script(s) to {scripts_dir}")
280
304
  return True
281
305
 
282
- def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None) -> bool:
306
+ def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None,
307
+ all_profiles: list[ProjectProfile] = None) -> bool:
283
308
  info_line("Downloading dbt package versions...")
284
309
  raw = self.sf.get_framework_config("DBT_PACKAGES")
285
310
  if not raw:
@@ -287,6 +312,22 @@ class DownloadService:
287
312
  return False
288
313
 
289
314
  packages = raw.get("packages", [])
315
+
316
+ # Filter to only packages enabled on the active profile
317
+ if profile and profile.dbt_packages:
318
+ enabled = set(profile.dbt_packages)
319
+
320
+ # Exclude packages already provided by upstream projects
321
+ if profile.upstream_projects and all_profiles:
322
+ profile_map = {p.profile_name: p for p in all_profiles}
323
+ upstream_pkgs = _collect_upstream_packages(
324
+ profile.upstream_projects, profile_map
325
+ )
326
+ enabled -= upstream_pkgs
327
+
328
+ packages = [p for p in packages if p.get("name") in enabled]
329
+ info_line(f" Filtered to {len(packages)} package(s) for profile '{profile.profile_name}'")
330
+
290
331
  pkg_list = []
291
332
  for pkg in packages:
292
333
  source = pkg.get("source", "git")
@@ -5,6 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
7
7
  from datasecops_cli.config import Config
8
+ from datasecops_cli.models.project_config import ProjectProfile
8
9
 
9
10
 
10
11
  class TestBuildParser:
@@ -64,6 +65,11 @@ class TestRunDownload:
64
65
  config.project_dir = tmp_path
65
66
  config.dbt_project_dir = tmp_path
66
67
  config.profile_name = "test_profile"
68
+ config.profile = ProjectProfile(
69
+ profile_name="test_profile",
70
+ dbt_packages=["dbt_utils", "dbt_expectations"],
71
+ )
72
+ config.all_profiles = [config.profile]
67
73
  # source_control defaults to GitHub
68
74
  return config
69
75
 
@@ -119,7 +125,9 @@ class TestRunDownload:
119
125
  _run_download(config, ["packages"])
120
126
 
121
127
  assert exc.value.code == 0
122
- mock_ds.download_dbt_packages.assert_called_once_with(tmp_path)
128
+ mock_ds.download_dbt_packages.assert_called_once_with(
129
+ tmp_path, profile=config.profile, all_profiles=config.all_profiles
130
+ )
123
131
 
124
132
  @patch("datasecops_cli.main._connect_and_load")
125
133
  def test_download_macros(self, mock_connect, tmp_path):
@@ -300,3 +308,112 @@ class TestRunDownload:
300
308
  _run_download(config, ["install-dbt"])
301
309
 
302
310
  assert exc.value.code == 1
311
+
312
+
313
+ class TestDownloadPackageFiltering:
314
+ """Tests for dbt package filtering by profile."""
315
+
316
+ def test_download_dbt_packages_filters_by_profile(self, tmp_path):
317
+ """Only packages in profile.dbt_packages should appear in packages.yml."""
318
+ from datasecops_cli.services.download_service import DownloadService
319
+
320
+ mock_sf = MagicMock()
321
+ mock_sf.get_framework_config.return_value = {
322
+ "packages": [
323
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
324
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
325
+ {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
326
+ ]
327
+ }
328
+
329
+ profile = ProjectProfile(
330
+ profile_name="test",
331
+ dbt_packages=["dbt_utils", "dbt_expectations"],
332
+ )
333
+
334
+ service = DownloadService(mock_sf, tmp_path)
335
+ service.download_dbt_packages(tmp_path, profile=profile)
336
+
337
+ import yaml
338
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
339
+ assert len(result["packages"]) == 2
340
+ urls = [p.get("git") for p in result["packages"]]
341
+ assert "https://github.com/dbt-labs/dbt-utils.git" in urls
342
+ assert "https://github.com/calogica/dbt-expectations.git" in urls
343
+ assert "https://github.com/Snowflake-Labs/dbt_constraints.git" not in urls
344
+
345
+ def test_download_dbt_packages_excludes_upstream(self, tmp_path):
346
+ """Packages provided by upstream projects should be excluded."""
347
+ from datasecops_cli.services.download_service import DownloadService
348
+
349
+ mock_sf = MagicMock()
350
+ mock_sf.get_framework_config.return_value = {
351
+ "packages": [
352
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
353
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
354
+ ]
355
+ }
356
+
357
+ upstream_profile = ProjectProfile(
358
+ profile_name="upstream_project",
359
+ dbt_packages=["dbt_utils"],
360
+ )
361
+ profile = ProjectProfile(
362
+ profile_name="test",
363
+ dbt_packages=["dbt_utils", "dbt_expectations"],
364
+ upstream_projects=["upstream_project"],
365
+ )
366
+
367
+ service = DownloadService(mock_sf, tmp_path)
368
+ service.download_dbt_packages(tmp_path, profile=profile, all_profiles=[profile, upstream_profile])
369
+
370
+ import yaml
371
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
372
+ # dbt_utils excluded (provided by upstream), only dbt_expectations + local upstream dep
373
+ git_packages = [p for p in result["packages"] if "git" in p]
374
+ local_packages = [p for p in result["packages"] if "local" in p]
375
+ assert len(git_packages) == 1
376
+ assert git_packages[0]["git"] == "https://github.com/calogica/dbt-expectations.git"
377
+ assert len(local_packages) == 1
378
+ assert local_packages[0]["local"] == "local_packages/upstream_project"
379
+
380
+ def test_download_dbt_packages_no_profile_downloads_all(self, tmp_path):
381
+ """When no profile is passed, all packages should be included."""
382
+ from datasecops_cli.services.download_service import DownloadService
383
+
384
+ mock_sf = MagicMock()
385
+ mock_sf.get_framework_config.return_value = {
386
+ "packages": [
387
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
388
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
389
+ {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
390
+ ]
391
+ }
392
+
393
+ service = DownloadService(mock_sf, tmp_path)
394
+ service.download_dbt_packages(tmp_path, profile=None)
395
+
396
+ import yaml
397
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
398
+ assert len(result["packages"]) == 3
399
+
400
+ def test_download_dbt_packages_empty_dbt_packages_downloads_all(self, tmp_path):
401
+ """When profile.dbt_packages is empty, all packages should be included."""
402
+ from datasecops_cli.services.download_service import DownloadService
403
+
404
+ mock_sf = MagicMock()
405
+ mock_sf.get_framework_config.return_value = {
406
+ "packages": [
407
+ {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
408
+ {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
409
+ ]
410
+ }
411
+
412
+ profile = ProjectProfile(profile_name="test", dbt_packages=[])
413
+
414
+ service = DownloadService(mock_sf, tmp_path)
415
+ service.download_dbt_packages(tmp_path, profile=profile)
416
+
417
+ import yaml
418
+ result = yaml.safe_load((tmp_path / "packages.yml").read_text())
419
+ assert len(result["packages"]) == 2
@@ -1 +0,0 @@
1
- __version__ = "0.4.3"
File without changes
File without changes
File without changes
File without changes