datasecops-cli 0.4.4__tar.gz → 0.4.5__tar.gz

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.4.5] - 2026-05-18
+
+### Fixed
+
+- **Package downloads now respect profile selection** — `datasecops download packages` and the interactive downloads menu now only include dbt packages enabled on the active profile, instead of downloading all packages from the global catalog. This matches the native app's Streamlit UI behavior.
+- **Upstream package exclusion** — packages already provided by upstream projects are automatically excluded from `packages.yml`, preventing duplicate dependencies when using multi-project setups.
+- **`generate_packages_yml()` filtering** — the dbt project generator now also filters packages by profile, fixing the same issue in the bootstrap/init path.
+
 ## [0.4.4] - 2026-05-18
 
 ### Added

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.4.4
+Version: 0.4.5
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.4.4"
+version = "0.4.5"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

datasecops_cli-0.4.5/src/datasecops_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.5"

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/src/datasecops_cli/main.py

@@ -565,7 +565,10 @@ def _run_download(config: Config, items: list[str],
                 if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
                     failed = True
             elif item == "packages":
-                if not download_service.download_dbt_packages(config.dbt_project_dir):
+                if not download_service.download_dbt_packages(
+                    config.dbt_project_dir, profile=config.profile,
+                    all_profiles=config.all_profiles
+                ):
                     failed = True
             elif item == "macros":
                 if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
@@ -704,6 +707,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
                 project_settings=config.project_settings,
                 profile=config.profile,
                 source_control=config.source_control,
+                all_profiles=config.all_profiles,
             )
             dl_menu.show()
 
@@ -722,6 +726,7 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
                 project_dir=config.project_dir,
                 project_settings=config.project_settings,
                 profile=config.profile,
+                all_profiles=config.all_profiles,
             )
             bootstrap.run(platform=platform, install_skills=install_skills,
                          skill_targets=skill_targets, run_deps=run_deps)
@@ -784,6 +789,7 @@ def _run_bootstrap(config: Config):
             project_dir=config.project_dir,
             project_settings=config.project_settings,
             profile=config.profile,
+            all_profiles=config.all_profiles,
         )
         success = bootstrap.run(platform=platform, install_skills=install_skills,
                                skill_targets=skill_targets, run_deps=run_deps)

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/src/datasecops_cli/menus/downloads.py

@@ -19,7 +19,7 @@ class DownloadsMenu:
     def __init__(self, download_service: DownloadService, skill_service: SkillService,
                  dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
                  project_settings: ProjectSettings = None, profile: ProjectProfile = None,
-                 source_control: SourceControl = None):
+                 source_control: SourceControl = None, all_profiles: list[ProjectProfile] = None):
         self.downloads = download_service
         self.skills = skill_service
         self.dbt = dbt_runner
@@ -28,6 +28,7 @@ class DownloadsMenu:
         self.project_settings = project_settings
         self.profile = profile
         self.source_control = source_control
+        self.all_profiles = all_profiles
     
     def show(self) -> None:
         self._menu()
@@ -46,7 +47,8 @@ class DownloadsMenu:
                 complete_action()
             elif option == 3:
                 display_action_header("Download dbt Packages")
-                self.downloads.download_dbt_packages(self.dbt_project_dir)
+                self.downloads.download_dbt_packages(self.dbt_project_dir, profile=self.profile,
+                                                     all_profiles=self.all_profiles)
                 if get_input_true_false("Run dbt deps now?"):
                     self.dbt.deps()
                 complete_action()

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/src/datasecops_cli/services/bootstrap_service.py

@@ -14,11 +14,13 @@ class BootstrapService:
     """Bootstraps a new dbt project with all framework configuration."""
 
     def __init__(self, snowflake_service: SnowflakeService, project_dir: Path,
-                 project_settings: ProjectSettings, profile: ProjectProfile):
+                 project_settings: ProjectSettings, profile: ProjectProfile,
+                 all_profiles: list[ProjectProfile] = None):
         self.sf = snowflake_service
         self.project_dir = project_dir
         self.project_settings = project_settings
         self.profile = profile
+        self.all_profiles = all_profiles or []
         self.download_service = DownloadService(snowflake_service, project_dir)
         self.skill_service = SkillService(snowflake_service)
 
@@ -92,7 +94,8 @@ class BootstrapService:
         # Step 5: Generate packages.yml
         info_line("")
         info_line("[5] Generating packages.yml...")
-        if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile):
+        if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile,
+                                                       all_profiles=self.all_profiles):
             steps_passed += 1
         else:
             info_line("  (skipped - no dbt packages config in native app)")

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/src/datasecops_cli/services/dbt_project_generator.py

@@ -165,17 +165,36 @@ def generate_profiles_yml(
 def generate_packages_yml(
     profile: ProjectProfile,
     packages_config: dict,
+    all_profiles: list[ProjectProfile] = None,
 ) -> str:
     """Generate packages.yml from profile's packages and upstream projects.
 
     Args:
         profile: The project profile.
         packages_config: The DBT_PACKAGES framework config.
+        all_profiles: All project profiles (used for upstream package exclusion).
 
     Returns:
         The YAML content as a string.
     """
     packages = packages_config.get("packages", [])
+
+    # Filter to only packages enabled on the profile
+    if profile.dbt_packages:
+        from datasecops_cli.services.download_service import _collect_upstream_packages
+
+        enabled = set(profile.dbt_packages)
+
+        # Exclude packages already provided by upstream projects
+        if profile.upstream_projects and all_profiles:
+            profile_map = {p.profile_name: p for p in all_profiles}
+            upstream_pkgs = _collect_upstream_packages(
+                profile.upstream_projects, profile_map
+            )
+            enabled -= upstream_pkgs
+
+        packages = [p for p in packages if p.get("name") in enabled]
+
     pkg_list: list[dict[str, Any]] = []
 
     for pkg in packages:

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/src/datasecops_cli/services/download_service.py

@@ -10,6 +10,30 @@ from datasecops_cli.utilities.display import info_line, success_line, error_line
 from datasecops_cli.utilities.file_utils import write_file, ensure_dir
 
 
+def _collect_upstream_packages(
+    upstream_names: list[str],
+    profile_map: dict[str, ProjectProfile],
+    visited: set[str] | None = None,
+) -> set[str]:
+    """Recursively collect dbt package names from all upstream projects."""
+    if visited is None:
+        visited = set()
+    result: set[str] = set()
+    for name in upstream_names:
+        if name in visited:
+            continue
+        visited.add(name)
+        upstream = profile_map.get(name)
+        if upstream:
+            result.update(upstream.dbt_packages)
+            result.update(
+                _collect_upstream_packages(
+                    upstream.upstream_projects or [], profile_map, visited
+                )
+            )
+    return result
+
+
 class DownloadService:
     """Downloads configurations from the native app."""
 
@@ -279,7 +303,8 @@ class DownloadService:
         success_line(f"Downloaded {count} script(s) to {scripts_dir}")
         return True
 
-    def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None) -> bool:
+    def download_dbt_packages(self, dbt_project_dir: Path, profile: ProjectProfile = None,
+                              all_profiles: list[ProjectProfile] = None) -> bool:
         info_line("Downloading dbt package versions...")
         raw = self.sf.get_framework_config("DBT_PACKAGES")
         if not raw:
@@ -287,6 +312,22 @@ class DownloadService:
             return False
         
         packages = raw.get("packages", [])
+
+        # Filter to only packages enabled on the active profile
+        if profile and profile.dbt_packages:
+            enabled = set(profile.dbt_packages)
+
+            # Exclude packages already provided by upstream projects
+            if profile.upstream_projects and all_profiles:
+                profile_map = {p.profile_name: p for p in all_profiles}
+                upstream_pkgs = _collect_upstream_packages(
+                    profile.upstream_projects, profile_map
+                )
+                enabled -= upstream_pkgs
+
+            packages = [p for p in packages if p.get("name") in enabled]
+            info_line(f"  Filtered to {len(packages)} package(s) for profile '{profile.profile_name}'")
+
         pkg_list = []
         for pkg in packages:
             source = pkg.get("source", "git")

{datasecops_cli-0.4.4 → datasecops_cli-0.4.5}/tests/test_main.py

@@ -5,6 +5,7 @@ from pathlib import Path
 
 from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
 from datasecops_cli.config import Config
+from datasecops_cli.models.project_config import ProjectProfile
 
 
 class TestBuildParser:
@@ -64,6 +65,11 @@ class TestRunDownload:
         config.project_dir = tmp_path
         config.dbt_project_dir = tmp_path
         config.profile_name = "test_profile"
+        config.profile = ProjectProfile(
+            profile_name="test_profile",
+            dbt_packages=["dbt_utils", "dbt_expectations"],
+        )
+        config.all_profiles = [config.profile]
         # source_control defaults to GitHub
         return config
 
@@ -119,7 +125,9 @@ class TestRunDownload:
                 _run_download(config, ["packages"])
 
             assert exc.value.code == 0
-            mock_ds.download_dbt_packages.assert_called_once_with(tmp_path)
+            mock_ds.download_dbt_packages.assert_called_once_with(
+                tmp_path, profile=config.profile, all_profiles=config.all_profiles
+            )
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_download_macros(self, mock_connect, tmp_path):
@@ -300,3 +308,112 @@ class TestRunDownload:
                 _run_download(config, ["install-dbt"])
 
             assert exc.value.code == 1
+
+
+class TestDownloadPackageFiltering:
+    """Tests for dbt package filtering by profile."""
+
+    def test_download_dbt_packages_filters_by_profile(self, tmp_path):
+        """Only packages in profile.dbt_packages should appear in packages.yml."""
+        from datasecops_cli.services.download_service import DownloadService
+
+        mock_sf = MagicMock()
+        mock_sf.get_framework_config.return_value = {
+            "packages": [
+                {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
+                {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
+                {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
+            ]
+        }
+
+        profile = ProjectProfile(
+            profile_name="test",
+            dbt_packages=["dbt_utils", "dbt_expectations"],
+        )
+
+        service = DownloadService(mock_sf, tmp_path)
+        service.download_dbt_packages(tmp_path, profile=profile)
+
+        import yaml
+        result = yaml.safe_load((tmp_path / "packages.yml").read_text())
+        assert len(result["packages"]) == 2
+        urls = [p.get("git") for p in result["packages"]]
+        assert "https://github.com/dbt-labs/dbt-utils.git" in urls
+        assert "https://github.com/calogica/dbt-expectations.git" in urls
+        assert "https://github.com/Snowflake-Labs/dbt_constraints.git" not in urls
+
+    def test_download_dbt_packages_excludes_upstream(self, tmp_path):
+        """Packages provided by upstream projects should be excluded."""
+        from datasecops_cli.services.download_service import DownloadService
+
+        mock_sf = MagicMock()
+        mock_sf.get_framework_config.return_value = {
+            "packages": [
+                {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
+                {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
+            ]
+        }
+
+        upstream_profile = ProjectProfile(
+            profile_name="upstream_project",
+            dbt_packages=["dbt_utils"],
+        )
+        profile = ProjectProfile(
+            profile_name="test",
+            dbt_packages=["dbt_utils", "dbt_expectations"],
+            upstream_projects=["upstream_project"],
+        )
+
+        service = DownloadService(mock_sf, tmp_path)
+        service.download_dbt_packages(tmp_path, profile=profile, all_profiles=[profile, upstream_profile])
+
+        import yaml
+        result = yaml.safe_load((tmp_path / "packages.yml").read_text())
+        # dbt_utils excluded (provided by upstream), only dbt_expectations + local upstream dep
+        git_packages = [p for p in result["packages"] if "git" in p]
+        local_packages = [p for p in result["packages"] if "local" in p]
+        assert len(git_packages) == 1
+        assert git_packages[0]["git"] == "https://github.com/calogica/dbt-expectations.git"
+        assert len(local_packages) == 1
+        assert local_packages[0]["local"] == "local_packages/upstream_project"
+
+    def test_download_dbt_packages_no_profile_downloads_all(self, tmp_path):
+        """When no profile is passed, all packages should be included."""
+        from datasecops_cli.services.download_service import DownloadService
+
+        mock_sf = MagicMock()
+        mock_sf.get_framework_config.return_value = {
+            "packages": [
+                {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
+                {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
+                {"name": "dbt_constraints", "source": "git", "url": "https://github.com/Snowflake-Labs/dbt_constraints.git", "latest_version": "0.6.0"},
+            ]
+        }
+
+        service = DownloadService(mock_sf, tmp_path)
+        service.download_dbt_packages(tmp_path, profile=None)
+
+        import yaml
+        result = yaml.safe_load((tmp_path / "packages.yml").read_text())
+        assert len(result["packages"]) == 3
+
+    def test_download_dbt_packages_empty_dbt_packages_downloads_all(self, tmp_path):
+        """When profile.dbt_packages is empty, all packages should be included."""
+        from datasecops_cli.services.download_service import DownloadService
+
+        mock_sf = MagicMock()
+        mock_sf.get_framework_config.return_value = {
+            "packages": [
+                {"name": "dbt_utils", "source": "git", "url": "https://github.com/dbt-labs/dbt-utils.git", "latest_version": "1.0.0"},
+                {"name": "dbt_expectations", "source": "git", "url": "https://github.com/calogica/dbt-expectations.git", "latest_version": "0.10.0"},
+            ]
+        }
+
+        profile = ProjectProfile(profile_name="test", dbt_packages=[])
+
+        service = DownloadService(mock_sf, tmp_path)
+        service.download_dbt_packages(tmp_path, profile=profile)
+
+        import yaml
+        result = yaml.safe_load((tmp_path / "packages.yml").read_text())
+        assert len(result["packages"]) == 2

datasecops_cli-0.4.4/src/datasecops_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.4"