datasecops-cli 0.2.9__tar.gz → 0.3.1__tar.gz

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.3.0] - 2026-05-14
+
+### Added
+
+- **`datasecops setup` subcommand** — pure-Python replacement for `setup.ps1` / `setup.sh`. Discovers Snowflake connections from `~/.snowflake/connections.toml`, prompts for connection and app database, and writes `.datasecops.yml`. No platform-specific scripts needed.
+- **Prerequisite checks during setup** — checks for `dbtf`, `gh`, and `az` on PATH and verifies authentication status (`gh auth status`, `az account show`)
+- **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one. Selected profile is saved back to `.datasecops.yml`.
+- **Auto-setup on first run** — running `datasecops` without a `.datasecops.yml` now offers to run setup interactively instead of just exiting with an error
+- **`--connection`, `--app-database`, `--profile` flags on `download`** — allows CI/CD pipelines to run without a committed `.datasecops.yml` by passing connection details as CLI flags (e.g. `datasecops download sqlfluff -c ci -d MY_APP_DB -p my_project`)
+- **Default app database value** — setup now defaults to `DATA_ENGINEERS_DATASECOPS_FRAMEWORK` (press Enter to accept)
+- **dbt project directory resolution from framework** — `dbt_project_dir` is now re-resolved using the framework's `project_dir` setting after native app config is loaded
+
+### Changed
+
+- **Non-interactive mode safety** — `_connect_and_load` accepts an `interactive` flag; the `download` subcommand passes `interactive=False` so it never prompts (missing config exits 1, multiple profiles use the first one silently)
+
 ## [0.2.9] - 2026-05-14
 
 ### Added

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.2.9
+Version: 0.3.1
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE
@@ -61,30 +61,19 @@ Optional:
 
 ## Quick Start
 
-### 1. Run the setup script
-
-The setup script creates a virtual environment, installs the CLI, and writes your local configuration.
-
-**Linux / macOS:**
+### 1. Configure your project
 
 ```bash
-chmod +x setup.sh && ./setup.sh
+datasecops setup
 ```
 
-**Windows (PowerShell):**
+This prompts for your Snowflake connection name and native app database, then writes `.datasecops.yml`.
 
-```powershell
-.\setup.ps1
-```
+If you skip this step, running `datasecops` will offer to run setup automatically.
 
-You'll be prompted for your Snowflake connection name and the native app database name.
-
-### 2. Activate the virtual environment and run
+### 2. Run the CLI
 
 ```bash
-source .venv/bin/activate   # Linux/macOS
-.\.venv\Scripts\Activate.ps1  # Windows
-
 datasecops
 ```
 

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/README.md

@@ -41,30 +41,19 @@ Optional:
 
 ## Quick Start
 
-### 1. Run the setup script
-
-The setup script creates a virtual environment, installs the CLI, and writes your local configuration.
-
-**Linux / macOS:**
+### 1. Configure your project
 
 ```bash
-chmod +x setup.sh && ./setup.sh
+datasecops setup
 ```
 
-**Windows (PowerShell):**
+This prompts for your Snowflake connection name and native app database, then writes `.datasecops.yml`.
 
-```powershell
-.\setup.ps1
-```
+If you skip this step, running `datasecops` will offer to run setup automatically.
 
-You'll be prompted for your Snowflake connection name and the native app database name.
-
-### 2. Activate the virtual environment and run
+### 2. Run the CLI
 
 ```bash
-source .venv/bin/activate   # Linux/macOS
-.\.venv\Scripts\Activate.ps1  # Windows
-
 datasecops
 ```
 

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.2.9"
+version = "0.3.1"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/src/datasecops_cli/main.py

@@ -1,5 +1,7 @@
 import argparse
+import os
 import shutil
+import subprocess
 import sys
 from pathlib import Path
 
@@ -18,6 +20,7 @@ from datasecops_cli.utilities.display import (
     clear, section_header, menu_option, get_input_number,
     info_line, error_line, success_line
 )
+from datasecops_cli.utilities.yaml_utils import write_datasecops_config
 
 
 DOWNLOAD_ITEMS = [
@@ -33,6 +36,7 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     sub = parser.add_subparsers(dest="command")
 
+    sub.add_parser("setup", help="Configure .datasecops.yml for this project")
     sub.add_parser("bootstrap", help="Set up a new dbt project with all framework config")
 
     dl = sub.add_parser("download", help="Download framework config non-interactively (for CI/CD)")
@@ -42,6 +46,18 @@ def _build_parser() -> argparse.ArgumentParser:
         choices=DOWNLOAD_ITEMS,
         help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
     )
+    dl.add_argument(
+        "--connection", "-c",
+        help="Snowflake connection name (overrides .datasecops.yml)",
+    )
+    dl.add_argument(
+        "--app-database", "-d",
+        help="Native app database name (overrides .datasecops.yml)",
+    )
+    dl.add_argument(
+        "--profile", "-p",
+        help="Project profile name (overrides .datasecops.yml)",
+    )
 
     return parser
 
@@ -53,24 +69,121 @@ def main():
 
     config = Config()
 
-    if args.command == "bootstrap":
+    if args.command == "setup":
+        _run_setup(config.project_dir)
+    elif args.command == "bootstrap":
         _run_bootstrap(config)
     elif args.command == "download":
-        _run_download(config, args.items)
+        _run_download(config, args.items,
+                      connection=args.connection, app_database=args.app_database,
+                      profile=args.profile)
     else:
         _run_interactive(config)
 
 
-def _connect_and_load(config: Config) -> SnowflakeService:
+def _run_setup(project_dir: Path):
+    """Interactive setup: create .datasecops.yml by prompting for connection and app database."""
+    from datasecops_cli.utilities.display import (
+        get_input_string, get_input_string_or_default, select_from_list, warning_line
+    )
+
+    section_header("DataSecOps Setup")
+
+    # --- Check prerequisites ---
+    info_line("Checking prerequisites...")
+
+    if shutil.which("dbtf"):
+        success_line("dbt Fusion (dbtf) found")
+    else:
+        warning_line("dbt Fusion (dbtf) not found — dbt commands will not work")
+        warning_line("  Install: https://docs.getdbt.com/docs/core/installation")
+
+    if shutil.which("gh"):
+        result = subprocess.run(["gh", "auth", "status"], capture_output=True, text=True)
+        if result.returncode == 0:
+            success_line("GitHub CLI (gh) found and authenticated")
+        else:
+            warning_line("GitHub CLI (gh) found but not logged in — run: gh auth login")
+    else:
+        warning_line("GitHub CLI (gh) not found — install from https://cli.github.com if using GitHub")
+
+    if shutil.which("az"):
+        result = subprocess.run(["az", "account", "show"], capture_output=True, text=True)
+        if result.returncode == 0:
+            success_line("Azure CLI (az) found and authenticated")
+        else:
+            warning_line("Azure CLI (az) found but not logged in — run: az login")
+    else:
+        warning_line("Azure CLI (az) not found — install from https://aka.ms/installazurecli if using Azure DevOps")
+
+    info_line("")
+
+    # --- Discover Snowflake connections ---
+    connections_file = Path.home() / ".snowflake" / "connections.toml"
+    connection_names = []
+    if connections_file.exists():
+        import re
+        for line in connections_file.read_text(encoding="utf-8").splitlines():
+            m = re.match(r"^\[([^\]]+)\]", line)
+            if m:
+                connection_names.append(m.group(1))
+
+    if connection_names:
+        connection_name = select_from_list(connection_names, "Snowflake connection", add_back=False)
+    else:
+        info_line("No connections found in ~/.snowflake/connections.toml")
+        connection_name = get_input_string("Enter Snowflake connection name: ")
+
+    if not connection_name:
+        error_line("Connection name is required")
+        sys.exit(1)
+
+    # --- App database ---
+    app_database = get_input_string_or_default(
+        "Enter native app database name",
+        "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
+    )
+
+    # --- Write config ---
+    config_data = {
+        "connection_name": connection_name,
+        "app_database": app_database,
+        "profile_name": "",
+    }
+    write_datasecops_config(project_dir, config_data)
+    success_line(f".datasecops.yml written to {project_dir / '.datasecops.yml'}")
+
+
+def _offer_setup(project_dir: Path):
+    """Offer to run setup when .datasecops.yml is missing."""
+    from datasecops_cli.utilities.display import get_input_true_false
+
+    info_line("No .datasecops.yml found in this project.")
+    if not get_input_true_false("Run setup now?"):
+        sys.exit(1)
+
+    _run_setup(project_dir)
+
+
+def _connect_and_load(config: Config, interactive: bool = True) -> SnowflakeService:
     """Load config, connect to Snowflake, and load native app settings.
     
     Returns the connected SnowflakeService, or calls sys.exit on failure.
+    If .datasecops.yml is missing and interactive is True, offers to run setup.
     """
     if not config.load():
-        sys.exit(1)
+        # Check if the failure is due to missing .datasecops.yml
+        if not (config.project_dir / ".datasecops.yml").exists() and interactive:
+            _offer_setup(config.project_dir)
+            # Retry after setup
+            if not config.load():
+                sys.exit(1)
+        else:
+            sys.exit(1)
 
     sf_config = config.datasecops
     sf_service = SnowflakeService(sf_config)
+    profile_was_set = bool(config.profile_name)
 
     try:
         info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
@@ -83,6 +196,37 @@ def _connect_and_load(config: Config) -> SnowflakeService:
     info_line("Loading framework configuration...")
     config.load_from_native_app(sf_service)
 
+    # If profile_name was not explicitly set and there are multiple profiles, ask the user
+    if not profile_was_set and len(config.all_profiles) > 1:
+        if interactive:
+            from datasecops_cli.utilities.display import select_from_list
+            names = [p.profile_name for p in config.all_profiles]
+            info_line(f"Found {len(names)} project profiles:")
+            selected = select_from_list(names, "profile", add_back=False)
+            config.profile = None
+            for p in config.all_profiles:
+                if p.profile_name == selected:
+                    config.profile = p
+                    config.profile_name = selected
+                    break
+            # Save the selected profile back to .datasecops.yml
+            if config.profile:
+                config_data = {
+                    "connection_name": config.datasecops.connection_name,
+                    "app_database": config.datasecops.app_database,
+                    "profile_name": config.profile_name,
+                }
+                write_datasecops_config(config.project_dir, config_data)
+                success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
+        else:
+            # Non-interactive: use the auto-selected first profile
+            info_line(f"Using default profile: {config.profile_name}")
+
+    # Re-resolve dbt_project_dir using framework project_dir setting
+    framework_dbt_dir = config.project_dir / config.project_settings.project_dir
+    if (framework_dbt_dir / "dbt_project.yml").exists():
+        config.dbt_project_dir = framework_dbt_dir
+
     if not config.profile:
         error_line(f"Profile '{config.profile_name}' not found in native app")
         sf_service.close()
@@ -126,9 +270,28 @@ def _run_interactive(config: Config):
         sf_service.close()
 
 
-def _run_download(config: Config, items: list[str]):
+def _run_download(config: Config, items: list[str],
+                   connection: str = None, app_database: str = None,
+                   profile: str = None):
     """Run non-interactive downloads for CI/CD pipelines."""
-    sf_service = _connect_and_load(config)
+    # If CLI flags provided, write/override .datasecops.yml on the fly
+    if connection or app_database:
+        from datasecops_cli.utilities.yaml_utils import read_datasecops_config
+        existing = read_datasecops_config(config.project_dir) or {}
+        config_data = {
+            "connection_name": connection or existing.get("connection_name", ""),
+            "app_database": app_database or existing.get("app_database", ""),
+            "profile_name": profile or existing.get("profile_name", ""),
+        }
+        write_datasecops_config(config.project_dir, config_data)
+        info_line(f"Wrote .datasecops.yml (connection={config_data['connection_name']}, app_database={config_data['app_database']})")
+    elif profile:
+        from datasecops_cli.utilities.yaml_utils import read_datasecops_config
+        existing = read_datasecops_config(config.project_dir) or {}
+        existing["profile_name"] = profile
+        write_datasecops_config(config.project_dir, existing)
+
+    sf_service = _connect_and_load(config, interactive=False)
 
     try:
         download_service = DownloadService(sf_service, config.project_dir)
@@ -150,7 +313,7 @@ def _run_download(config: Config, items: list[str]):
             elif item == "pipelines":
                 platform = config.source_control.source_control_platform.lower()
                 info_line(f"Platform: {platform}")
-                if not download_service.download_pipelines(platform=platform):
+                if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
                     failed = True
             elif item == "packages":
                 if not download_service.download_dbt_packages(config.dbt_project_dir):

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/src/datasecops_cli/menus/downloads.py

@@ -38,7 +38,7 @@ class DownloadsMenu:
                 display_action_header("Download Pipeline Files")
                 platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
                 info_line(f"Platform: {platform}")
-                self.downloads.download_pipelines(platform=platform)
+                self.downloads.download_pipelines(platform=platform, profile_name=self.profile_name)
                 complete_action()
             elif option == 3:
                 display_action_header("Download dbt Packages")

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/src/datasecops_cli/models/project_config.py

@@ -65,6 +65,7 @@ class ProjectProfile(BaseModel):
     target_database: str = ""
     dbt_version: str = "1.9"
     dbt_packages: list[str] = Field(default_factory=list)
+    pipeline_ids: list[str] = Field(default_factory=list)
 
 class DbtPackage(BaseModel):
     name: str = ""

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/src/datasecops_cli/services/download_service.py

@@ -220,9 +220,13 @@ class DownloadService:
             error_line("No active dbt versions found in framework configuration")
         return dbt_packages
     
-    def download_pipelines(self, platform: str = "github") -> bool:
+    def download_pipelines(self, platform: str = "github", profile_name: str = "") -> bool:
         info_line(f"Downloading {platform} pipeline configurations...")
-        raw = self.sf.get_framework_config("PIPELINES")
+        if profile_name:
+            info_line(f"Filtering pipelines for profile: {profile_name}")
+            raw = self.sf.get_pipelines_for_profile(profile_name)
+        else:
+            raw = self.sf.get_framework_config("PIPELINES")
         if not raw:
             error_line("No pipeline configuration found in native app")
             return False

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/src/datasecops_cli/services/snowflake_service.py

@@ -59,7 +59,10 @@ class SnowflakeService:
     
     def get_project_profiles(self) -> Optional[list[dict]]:
         return self.call_procedure("get_project_profiles")
-    
+
+    def get_pipelines_for_profile(self, profile_name: str) -> Optional[dict]:
+        return self.call_procedure("get_pipelines_for_profile", profile_name)
+
     def get_account_info(self) -> dict:
         rows = self.execute_query("SELECT CURRENT_ACCOUNT_NAME() as account, CURRENT_USER() as user_name")
         return rows[0] if rows else {}

{datasecops_cli-0.2.9 → datasecops_cli-0.3.1}/tests/test_main.py

@@ -102,7 +102,7 @@ class TestRunDownload:
                 _run_download(config, ["pipelines"])
 
             assert exc.value.code == 0
-            mock_ds.download_pipelines.assert_called_once_with(platform="github")
+            mock_ds.download_pipelines.assert_called_once_with(platform="github", profile_name="test_profile")
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_download_packages(self, mock_connect, tmp_path):
@@ -219,7 +219,7 @@ class TestRunDownload:
                 _run_download(config, ["pipelines"])
 
             assert exc.value.code == 0
-            mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
+            mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops", profile_name="test_profile")
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_install_sqlfluff(self, mock_connect, tmp_path):