datasecops-cli 0.2.8__tar.gz → 0.3.0__tar.gz

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.3.0] - 2026-05-14
+
+### Added
+
+- **`datasecops setup` subcommand** — pure-Python replacement for `setup.ps1` / `setup.sh`. Discovers Snowflake connections from `~/.snowflake/connections.toml`, prompts for connection and app database, and writes `.datasecops.yml`. No platform-specific scripts needed.
+- **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one
+- **Auto-setup on first run** — running `datasecops` without a `.datasecops.yml` now offers to run setup interactively instead of just exiting with an error
+- **dbt project directory resolution from framework** — `dbt_project_dir` is now re-resolved using the framework's `project_dir` setting after native app config is loaded
+
+## [0.2.9] - 2026-05-14
+
+### Added
+
+- **`install-sqlfluff` and `install-dbt` download items** — `datasecops download install-sqlfluff` and `datasecops download install-dbt` fetch framework-pinned package versions from the native app and install them via `uv pip install`. `datasecops download all` now includes both installs.
+
 ## [0.2.8] - 2026-05-14
 
 ### Added

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.2.8
+Version: 0.3.0
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE
@@ -61,30 +61,19 @@ Optional:
 
 ## Quick Start
 
-### 1. Run the setup script
-
-The setup script creates a virtual environment, installs the CLI, and writes your local configuration.
-
-**Linux / macOS:**
+### 1. Configure your project
 
 ```bash
-chmod +x setup.sh && ./setup.sh
+datasecops setup
 ```
 
-**Windows (PowerShell):**
-
-```powershell
-.\setup.ps1
-```
+This prompts for your Snowflake connection name and native app database, then writes `.datasecops.yml`.
 
-You'll be prompted for your Snowflake connection name and the native app database name.
+If you skip this step, running `datasecops` will offer to run setup automatically.
 
-### 2. Activate the virtual environment and run
+### 2. Run the CLI
 
 ```bash
-source .venv/bin/activate   # Linux/macOS
-.\.venv\Scripts\Activate.ps1  # Windows
-
 datasecops
 ```
 
@@ -106,11 +95,18 @@ datasecops download sqlfluff
 datasecops download sqlfluff packages
 datasecops download pipelines macros
 
-# Download everything
+# Install framework-pinned package versions
+datasecops download install-sqlfluff
+datasecops download install-dbt
+
+# Download config and install packages together
+datasecops download sqlfluff install-sqlfluff
+
+# Download and install everything
 datasecops download all
 ```
 
-Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `all`
+Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
 
 The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
 

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/README.md

@@ -41,30 +41,19 @@ Optional:
 
 ## Quick Start
 
-### 1. Run the setup script
-
-The setup script creates a virtual environment, installs the CLI, and writes your local configuration.
-
-**Linux / macOS:**
+### 1. Configure your project
 
 ```bash
-chmod +x setup.sh && ./setup.sh
+datasecops setup
 ```
 
-**Windows (PowerShell):**
-
-```powershell
-.\setup.ps1
-```
+This prompts for your Snowflake connection name and native app database, then writes `.datasecops.yml`.
 
-You'll be prompted for your Snowflake connection name and the native app database name.
+If you skip this step, running `datasecops` will offer to run setup automatically.
 
-### 2. Activate the virtual environment and run
+### 2. Run the CLI
 
 ```bash
-source .venv/bin/activate   # Linux/macOS
-.\.venv\Scripts\Activate.ps1  # Windows
-
 datasecops
 ```
 
@@ -86,11 +75,18 @@ datasecops download sqlfluff
 datasecops download sqlfluff packages
 datasecops download pipelines macros
 
-# Download everything
+# Install framework-pinned package versions
+datasecops download install-sqlfluff
+datasecops download install-dbt
+
+# Download config and install packages together
+datasecops download sqlfluff install-sqlfluff
+
+# Download and install everything
 datasecops download all
 ```
 
-Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `all`
+Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
 
 The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
 

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.2.8"
+version = "0.3.0"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/src/datasecops_cli/main.py

@@ -1,5 +1,7 @@
 import argparse
+import os
 import shutil
+import subprocess
 import sys
 from pathlib import Path
 
@@ -18,9 +20,13 @@ from datasecops_cli.utilities.display import (
     clear, section_header, menu_option, get_input_number,
     info_line, error_line, success_line
 )
+from datasecops_cli.utilities.yaml_utils import write_datasecops_config
 
 
-DOWNLOAD_ITEMS = ["sqlfluff", "pipelines", "packages", "macros", "all"]
+DOWNLOAD_ITEMS = [
+    "sqlfluff", "pipelines", "packages", "macros",
+    "install-sqlfluff", "install-dbt", "all",
+]
 
 
 def _build_parser() -> argparse.ArgumentParser:
@@ -30,6 +36,7 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     sub = parser.add_subparsers(dest="command")
 
+    sub.add_parser("setup", help="Configure .datasecops.yml for this project")
     sub.add_parser("bootstrap", help="Set up a new dbt project with all framework config")
 
     dl = sub.add_parser("download", help="Download framework config non-interactively (for CI/CD)")
@@ -37,7 +44,7 @@ def _build_parser() -> argparse.ArgumentParser:
         "items",
         nargs="+",
         choices=DOWNLOAD_ITEMS,
-        help="Item(s) to download: sqlfluff, pipelines, packages, macros, or all",
+        help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
     )
 
     return parser
@@ -50,7 +57,9 @@ def main():
 
     config = Config()
 
-    if args.command == "bootstrap":
+    if args.command == "setup":
+        _run_setup(config.project_dir)
+    elif args.command == "bootstrap":
         _run_bootstrap(config)
     elif args.command == "download":
         _run_download(config, args.items)
@@ -58,16 +67,82 @@ def main():
         _run_interactive(config)
 
 
+def _run_setup(project_dir: Path):
+    """Interactive setup: create .datasecops.yml by prompting for connection and app database."""
+    from datasecops_cli.utilities.display import (
+        get_input_string, select_from_list
+    )
+
+    section_header("DataSecOps Setup")
+
+    # --- Discover Snowflake connections ---
+    connections_file = Path.home() / ".snowflake" / "connections.toml"
+    connection_names = []
+    if connections_file.exists():
+        import re
+        for line in connections_file.read_text(encoding="utf-8").splitlines():
+            m = re.match(r"^\[([^\]]+)\]", line)
+            if m:
+                connection_names.append(m.group(1))
+
+    if connection_names:
+        connection_name = select_from_list(connection_names, "Snowflake connection", add_back=False)
+    else:
+        info_line("No connections found in ~/.snowflake/connections.toml")
+        connection_name = get_input_string("Enter Snowflake connection name: ")
+
+    if not connection_name:
+        error_line("Connection name is required")
+        sys.exit(1)
+
+    # --- App database ---
+    app_database = get_input_string(
+        "Enter native app database name (e.g. DATA_ENGINEERS_DATASECOPS_FRAMEWORK): "
+    )
+    if not app_database:
+        error_line("App database name is required")
+        sys.exit(1)
+
+    # --- Write config ---
+    config_data = {
+        "connection_name": connection_name,
+        "app_database": app_database,
+        "profile_name": "",
+    }
+    write_datasecops_config(project_dir, config_data)
+    success_line(f".datasecops.yml written to {project_dir / '.datasecops.yml'}")
+
+
+def _offer_setup(project_dir: Path):
+    """Offer to run setup when .datasecops.yml is missing."""
+    from datasecops_cli.utilities.display import get_input_true_false
+
+    info_line("No .datasecops.yml found in this project.")
+    if not get_input_true_false("Run setup now?"):
+        sys.exit(1)
+
+    _run_setup(project_dir)
+
+
 def _connect_and_load(config: Config) -> SnowflakeService:
     """Load config, connect to Snowflake, and load native app settings.
     
     Returns the connected SnowflakeService, or calls sys.exit on failure.
+    If .datasecops.yml is missing and a setup script exists, offers to run it.
     """
     if not config.load():
-        sys.exit(1)
+        # Check if the failure is due to missing .datasecops.yml
+        if not (config.project_dir / ".datasecops.yml").exists():
+            _offer_setup(config.project_dir)
+            # Retry after setup
+            if not config.load():
+                sys.exit(1)
+        else:
+            sys.exit(1)
 
     sf_config = config.datasecops
     sf_service = SnowflakeService(sf_config)
+    profile_was_set = bool(config.profile_name)
 
     try:
         info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
@@ -80,6 +155,24 @@ def _connect_and_load(config: Config) -> SnowflakeService:
     info_line("Loading framework configuration...")
     config.load_from_native_app(sf_service)
 
+    # If profile_name was not explicitly set and there are multiple profiles, ask the user
+    if not profile_was_set and len(config.all_profiles) > 1:
+        from datasecops_cli.utilities.display import select_from_list
+        names = [p.profile_name for p in config.all_profiles]
+        info_line(f"Found {len(names)} project profiles:")
+        selected = select_from_list(names, "profile", add_back=False)
+        config.profile = None
+        for p in config.all_profiles:
+            if p.profile_name == selected:
+                config.profile = p
+                config.profile_name = selected
+                break
+
+    # Re-resolve dbt_project_dir using framework project_dir setting
+    framework_dbt_dir = config.project_dir / config.project_settings.project_dir
+    if (framework_dbt_dir / "dbt_project.yml").exists():
+        config.dbt_project_dir = framework_dbt_dir
+
     if not config.profile:
         error_line(f"Profile '{config.profile_name}' not found in native app")
         sf_service.close()
@@ -129,10 +222,12 @@ def _run_download(config: Config, items: list[str]):
 
     try:
         download_service = DownloadService(sf_service, config.project_dir)
+        linting_service = LintingService(config.dbt_project_dir)
         profiles_dir = str(config.get_dbt_profiles_dir())
 
         if "all" in items:
-            items = ["sqlfluff", "pipelines", "packages", "macros"]
+            items = ["sqlfluff", "pipelines", "packages", "macros",
+                     "install-sqlfluff", "install-dbt"]
 
         failed = False
         for item in items:
@@ -153,6 +248,20 @@ def _run_download(config: Config, items: list[str]):
             elif item == "macros":
                 if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
                     failed = True
+            elif item == "install-sqlfluff":
+                packages = download_service.get_sqlfluff_requirements()
+                if packages:
+                    if not linting_service.install_requirements(packages):
+                        failed = True
+                else:
+                    failed = True
+            elif item == "install-dbt":
+                packages = download_service.get_dbt_requirements()
+                if packages:
+                    if not linting_service.install_requirements(packages):
+                        failed = True
+                else:
+                    failed = True
 
         sys.exit(1 if failed else 0)
     finally:

{datasecops_cli-0.2.8 → datasecops_cli-0.3.0}/tests/test_main.py

@@ -73,7 +73,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_sqlfluff_config.return_value = True
 
@@ -92,7 +93,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_pipelines.return_value = True
 
@@ -108,7 +110,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_dbt_packages.return_value = True
 
@@ -124,7 +127,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_macros.return_value = True
 
@@ -140,12 +144,17 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService") as MockLS:
             mock_ds = MockDS.return_value
+            mock_ls = MockLS.return_value
             mock_ds.download_sqlfluff_config.return_value = True
             mock_ds.download_pipelines.return_value = True
             mock_ds.download_dbt_packages.return_value = True
             mock_ds.download_macros.return_value = True
+            mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0"]
+            mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
+            mock_ls.install_requirements.return_value = True
 
             with pytest.raises(SystemExit) as exc:
                 _run_download(config, ["all"])
@@ -155,6 +164,9 @@ class TestRunDownload:
             mock_ds.download_pipelines.assert_called_once()
             mock_ds.download_dbt_packages.assert_called_once()
             mock_ds.download_macros.assert_called_once()
+            mock_ds.get_sqlfluff_requirements.assert_called_once()
+            mock_ds.get_dbt_requirements.assert_called_once()
+            assert mock_ls.install_requirements.call_count == 2
 
     @patch("datasecops_cli.main._connect_and_load")
     def test_download_failure_exits_1(self, mock_connect, tmp_path):
@@ -162,7 +174,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_sqlfluff_config.return_value = False  # failure
 
@@ -178,7 +191,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_sqlfluff_config.return_value = True
             mock_ds.download_dbt_packages.return_value = False
@@ -196,7 +210,8 @@ class TestRunDownload:
         mock_sf = MagicMock()
         mock_connect.return_value = mock_sf
 
-        with patch("datasecops_cli.main.DownloadService") as MockDS:
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService"):
             mock_ds = MockDS.return_value
             mock_ds.download_pipelines.return_value = True
 
@@ -205,3 +220,83 @@ class TestRunDownload:
 
             assert exc.value.code == 0
             mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
+
+    @patch("datasecops_cli.main._connect_and_load")
+    def test_install_sqlfluff(self, mock_connect, tmp_path):
+        """install-sqlfluff fetches pinned versions and installs them."""
+        config = self._make_config(tmp_path)
+        mock_connect.return_value = MagicMock()
+
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService") as MockLS:
+            mock_ds = MockDS.return_value
+            mock_ls = MockLS.return_value
+            mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
+            mock_ls.install_requirements.return_value = True
+
+            with pytest.raises(SystemExit) as exc:
+                _run_download(config, ["install-sqlfluff"])
+
+            assert exc.value.code == 0
+            mock_ds.get_sqlfluff_requirements.assert_called_once()
+            mock_ls.install_requirements.assert_called_once_with(
+                ["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
+            )
+
+    @patch("datasecops_cli.main._connect_and_load")
+    def test_install_dbt(self, mock_connect, tmp_path):
+        """install-dbt fetches pinned versions and installs them."""
+        config = self._make_config(tmp_path)
+        mock_connect.return_value = MagicMock()
+
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService") as MockLS:
+            mock_ds = MockDS.return_value
+            mock_ls = MockLS.return_value
+            mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
+            mock_ls.install_requirements.return_value = True
+
+            with pytest.raises(SystemExit) as exc:
+                _run_download(config, ["install-dbt"])
+
+            assert exc.value.code == 0
+            mock_ds.get_dbt_requirements.assert_called_once()
+            mock_ls.install_requirements.assert_called_once_with(
+                ["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
+            )
+
+    @patch("datasecops_cli.main._connect_and_load")
+    def test_install_sqlfluff_no_versions_exits_1(self, mock_connect, tmp_path):
+        """install-sqlfluff fails if no versions returned from framework."""
+        config = self._make_config(tmp_path)
+        mock_connect.return_value = MagicMock()
+
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService") as MockLS:
+            mock_ds = MockDS.return_value
+            mock_ls = MockLS.return_value
+            mock_ds.get_sqlfluff_requirements.return_value = []
+
+            with pytest.raises(SystemExit) as exc:
+                _run_download(config, ["install-sqlfluff"])
+
+            assert exc.value.code == 1
+            mock_ls.install_requirements.assert_not_called()
+
+    @patch("datasecops_cli.main._connect_and_load")
+    def test_install_dbt_pip_failure_exits_1(self, mock_connect, tmp_path):
+        """install-dbt exits 1 if uv pip install fails."""
+        config = self._make_config(tmp_path)
+        mock_connect.return_value = MagicMock()
+
+        with patch("datasecops_cli.main.DownloadService") as MockDS, \
+             patch("datasecops_cli.main.LintingService") as MockLS:
+            mock_ds = MockDS.return_value
+            mock_ls = MockLS.return_value
+            mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
+            mock_ls.install_requirements.return_value = False  # pip failure
+
+            with pytest.raises(SystemExit) as exc:
+                _run_download(config, ["install-dbt"])
+
+            assert exc.value.code == 1