datasecops-cli 0.3.0__tar.gz → 0.3.1__tar.gz

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/CHANGELOG.md

@@ -7,10 +7,17 @@ All notable changes to the DataSecOps CLI are documented in this file.
 ### Added
 
 - **`datasecops setup` subcommand** — pure-Python replacement for `setup.ps1` / `setup.sh`. Discovers Snowflake connections from `~/.snowflake/connections.toml`, prompts for connection and app database, and writes `.datasecops.yml`. No platform-specific scripts needed.
-- **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one
+- **Prerequisite checks during setup** — checks for `dbtf`, `gh`, and `az` on PATH and verifies authentication status (`gh auth status`, `az account show`)
+- **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one. Selected profile is saved back to `.datasecops.yml`.
 - **Auto-setup on first run** — running `datasecops` without a `.datasecops.yml` now offers to run setup interactively instead of just exiting with an error
+- **`--connection`, `--app-database`, `--profile` flags on `download`** — allows CI/CD pipelines to run without a committed `.datasecops.yml` by passing connection details as CLI flags (e.g. `datasecops download sqlfluff -c ci -d MY_APP_DB -p my_project`)
+- **Default app database value** — setup now defaults to `DATA_ENGINEERS_DATASECOPS_FRAMEWORK` (press Enter to accept)
 - **dbt project directory resolution from framework** — `dbt_project_dir` is now re-resolved using the framework's `project_dir` setting after native app config is loaded
 
+### Changed
+
+- **Non-interactive mode safety** — `_connect_and_load` accepts an `interactive` flag; the `download` subcommand passes `interactive=False` so it never prompts (missing config exits 1, multiple profiles use the first one silently)
+
 ## [0.2.9] - 2026-05-14
 
 ### Added

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.3.0
+Version: 0.3.1
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.3.0"
+version = "0.3.1"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/src/datasecops_cli/main.py

@@ -46,6 +46,18 @@ def _build_parser() -> argparse.ArgumentParser:
         choices=DOWNLOAD_ITEMS,
         help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
     )
+    dl.add_argument(
+        "--connection", "-c",
+        help="Snowflake connection name (overrides .datasecops.yml)",
+    )
+    dl.add_argument(
+        "--app-database", "-d",
+        help="Native app database name (overrides .datasecops.yml)",
+    )
+    dl.add_argument(
+        "--profile", "-p",
+        help="Project profile name (overrides .datasecops.yml)",
+    )
 
     return parser
 
@@ -62,7 +74,9 @@ def main():
     elif args.command == "bootstrap":
         _run_bootstrap(config)
     elif args.command == "download":
-        _run_download(config, args.items)
+        _run_download(config, args.items,
+                      connection=args.connection, app_database=args.app_database,
+                      profile=args.profile)
     else:
         _run_interactive(config)
 
@@ -70,11 +84,40 @@ def main():
 def _run_setup(project_dir: Path):
     """Interactive setup: create .datasecops.yml by prompting for connection and app database."""
     from datasecops_cli.utilities.display import (
-        get_input_string, select_from_list
+        get_input_string, get_input_string_or_default, select_from_list, warning_line
     )
 
     section_header("DataSecOps Setup")
 
+    # --- Check prerequisites ---
+    info_line("Checking prerequisites...")
+
+    if shutil.which("dbtf"):
+        success_line("dbt Fusion (dbtf) found")
+    else:
+        warning_line("dbt Fusion (dbtf) not found — dbt commands will not work")
+        warning_line("  Install: https://docs.getdbt.com/docs/core/installation")
+
+    if shutil.which("gh"):
+        result = subprocess.run(["gh", "auth", "status"], capture_output=True, text=True)
+        if result.returncode == 0:
+            success_line("GitHub CLI (gh) found and authenticated")
+        else:
+            warning_line("GitHub CLI (gh) found but not logged in — run: gh auth login")
+    else:
+        warning_line("GitHub CLI (gh) not found — install from https://cli.github.com if using GitHub")
+
+    if shutil.which("az"):
+        result = subprocess.run(["az", "account", "show"], capture_output=True, text=True)
+        if result.returncode == 0:
+            success_line("Azure CLI (az) found and authenticated")
+        else:
+            warning_line("Azure CLI (az) found but not logged in — run: az login")
+    else:
+        warning_line("Azure CLI (az) not found — install from https://aka.ms/installazurecli if using Azure DevOps")
+
+    info_line("")
+
     # --- Discover Snowflake connections ---
     connections_file = Path.home() / ".snowflake" / "connections.toml"
     connection_names = []
@@ -96,12 +139,10 @@ def _run_setup(project_dir: Path):
         sys.exit(1)
 
     # --- App database ---
-    app_database = get_input_string(
-        "Enter native app database name (e.g. DATA_ENGINEERS_DATASECOPS_FRAMEWORK): "
+    app_database = get_input_string_or_default(
+        "Enter native app database name",
+        "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
     )
-    if not app_database:
-        error_line("App database name is required")
-        sys.exit(1)
 
     # --- Write config ---
     config_data = {
@@ -124,15 +165,15 @@ def _offer_setup(project_dir: Path):
     _run_setup(project_dir)
 
 
-def _connect_and_load(config: Config) -> SnowflakeService:
+def _connect_and_load(config: Config, interactive: bool = True) -> SnowflakeService:
     """Load config, connect to Snowflake, and load native app settings.
     
     Returns the connected SnowflakeService, or calls sys.exit on failure.
-    If .datasecops.yml is missing and a setup script exists, offers to run it.
+    If .datasecops.yml is missing and interactive is True, offers to run setup.
     """
     if not config.load():
         # Check if the failure is due to missing .datasecops.yml
-        if not (config.project_dir / ".datasecops.yml").exists():
+        if not (config.project_dir / ".datasecops.yml").exists() and interactive:
             _offer_setup(config.project_dir)
             # Retry after setup
             if not config.load():
@@ -157,16 +198,29 @@ def _connect_and_load(config: Config) -> SnowflakeService:
 
     # If profile_name was not explicitly set and there are multiple profiles, ask the user
     if not profile_was_set and len(config.all_profiles) > 1:
-        from datasecops_cli.utilities.display import select_from_list
-        names = [p.profile_name for p in config.all_profiles]
-        info_line(f"Found {len(names)} project profiles:")
-        selected = select_from_list(names, "profile", add_back=False)
-        config.profile = None
-        for p in config.all_profiles:
-            if p.profile_name == selected:
-                config.profile = p
-                config.profile_name = selected
-                break
+        if interactive:
+            from datasecops_cli.utilities.display import select_from_list
+            names = [p.profile_name for p in config.all_profiles]
+            info_line(f"Found {len(names)} project profiles:")
+            selected = select_from_list(names, "profile", add_back=False)
+            config.profile = None
+            for p in config.all_profiles:
+                if p.profile_name == selected:
+                    config.profile = p
+                    config.profile_name = selected
+                    break
+            # Save the selected profile back to .datasecops.yml
+            if config.profile:
+                config_data = {
+                    "connection_name": config.datasecops.connection_name,
+                    "app_database": config.datasecops.app_database,
+                    "profile_name": config.profile_name,
+                }
+                write_datasecops_config(config.project_dir, config_data)
+                success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
+        else:
+            # Non-interactive: use the auto-selected first profile
+            info_line(f"Using default profile: {config.profile_name}")
 
     # Re-resolve dbt_project_dir using framework project_dir setting
     framework_dbt_dir = config.project_dir / config.project_settings.project_dir
@@ -216,9 +270,28 @@ def _run_interactive(config: Config):
         sf_service.close()
 
 
-def _run_download(config: Config, items: list[str]):
+def _run_download(config: Config, items: list[str],
+                   connection: str = None, app_database: str = None,
+                   profile: str = None):
     """Run non-interactive downloads for CI/CD pipelines."""
-    sf_service = _connect_and_load(config)
+    # If CLI flags provided, write/override .datasecops.yml on the fly
+    if connection or app_database:
+        from datasecops_cli.utilities.yaml_utils import read_datasecops_config
+        existing = read_datasecops_config(config.project_dir) or {}
+        config_data = {
+            "connection_name": connection or existing.get("connection_name", ""),
+            "app_database": app_database or existing.get("app_database", ""),
+            "profile_name": profile or existing.get("profile_name", ""),
+        }
+        write_datasecops_config(config.project_dir, config_data)
+        info_line(f"Wrote .datasecops.yml (connection={config_data['connection_name']}, app_database={config_data['app_database']})")
+    elif profile:
+        from datasecops_cli.utilities.yaml_utils import read_datasecops_config
+        existing = read_datasecops_config(config.project_dir) or {}
+        existing["profile_name"] = profile
+        write_datasecops_config(config.project_dir, existing)
+
+    sf_service = _connect_and_load(config, interactive=False)
 
     try:
         download_service = DownloadService(sf_service, config.project_dir)
@@ -240,7 +313,7 @@ def _run_download(config: Config, items: list[str]):
             elif item == "pipelines":
                 platform = config.source_control.source_control_platform.lower()
                 info_line(f"Platform: {platform}")
-                if not download_service.download_pipelines(platform=platform):
+                if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
                     failed = True
             elif item == "packages":
                 if not download_service.download_dbt_packages(config.dbt_project_dir):

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/src/datasecops_cli/menus/downloads.py

@@ -38,7 +38,7 @@ class DownloadsMenu:
                 display_action_header("Download Pipeline Files")
                 platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
                 info_line(f"Platform: {platform}")
-                self.downloads.download_pipelines(platform=platform)
+                self.downloads.download_pipelines(platform=platform, profile_name=self.profile_name)
                 complete_action()
             elif option == 3:
                 display_action_header("Download dbt Packages")

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/src/datasecops_cli/models/project_config.py

@@ -65,6 +65,7 @@ class ProjectProfile(BaseModel):
     target_database: str = ""
     dbt_version: str = "1.9"
     dbt_packages: list[str] = Field(default_factory=list)
+    pipeline_ids: list[str] = Field(default_factory=list)
 
 class DbtPackage(BaseModel):
     name: str = ""

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/src/datasecops_cli/services/download_service.py

@@ -220,9 +220,13 @@ class DownloadService:
             error_line("No active dbt versions found in framework configuration")
         return dbt_packages
     
-    def download_pipelines(self, platform: str = "github") -> bool:
+    def download_pipelines(self, platform: str = "github", profile_name: str = "") -> bool:
         info_line(f"Downloading {platform} pipeline configurations...")
-        raw = self.sf.get_framework_config("PIPELINES")
+        if profile_name:
+            info_line(f"Filtering pipelines for profile: {profile_name}")
+            raw = self.sf.get_pipelines_for_profile(profile_name)
+        else:
+            raw = self.sf.get_framework_config("PIPELINES")
         if not raw:
             error_line("No pipeline configuration found in native app")
             return False

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/src/datasecops_cli/services/snowflake_service.py

@@ -59,7 +59,10 @@ class SnowflakeService:
     
     def get_project_profiles(self) -> Optional[list[dict]]:
         return self.call_procedure("get_project_profiles")
-    
+
+    def get_pipelines_for_profile(self, profile_name: str) -> Optional[dict]:
+        return self.call_procedure("get_pipelines_for_profile", profile_name)
+
     def get_account_info(self) -> dict:
         rows = self.execute_query("SELECT CURRENT_ACCOUNT_NAME() as account, CURRENT_USER() as user_name")
         return rows[0] if rows else {}

{datasecops_cli-0.3.0 → datasecops_cli-0.3.1}/tests/test_main.py

@@ -102,7 +102,7 @@ class TestRunDownload:
                 _run_download(config, ["pipelines"])
 
             assert exc.value.code == 0
-            mock_ds.download_pipelines.assert_called_once_with(platform="github")
+            mock_ds.download_pipelines.assert_called_once_with(platform="github", profile_name="test_profile")
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_download_packages(self, mock_connect, tmp_path):
@@ -219,7 +219,7 @@ class TestRunDownload:
                 _run_download(config, ["pipelines"])
 
             assert exc.value.code == 0
-            mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
+            mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops", profile_name="test_profile")
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_install_sqlfluff(self, mock_connect, tmp_path):