datasecops-cli 0.2.0__tar.gz → 0.2.2__tar.gz

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/CHANGELOG.md

@@ -2,6 +2,33 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.2.2] - 2026-05-10
+
+### Fixed
+
+- **Full SQLFluff config generation** — `download_sqlfluff_config()` now generates all sections from the native app config: core, templater, indentation, layout types, global rules, and all rule bundles (aliasing, ambiguous, capitalisation, convention, jinja, layout, references, structure)
+- Previously only `core` and `indentation` sections were emitted, missing ~90% of the config
+
+### Added
+
+- **Setup script connection selector** — `setup.ps1` and `setup.sh` now parse `~/.snowflake/connections.toml`, display a numbered list of connections with the default highlighted, and let the user select by number instead of typing the name manually
+- Rule code to section name mapping (`RULE_SECTION_MAP`) for all sqlfluff rule bundles (AL01-AL09, AM01-AM08, CP01-CP05, CV01-CV12, JJ01, LT01-LT15, RF01-RF06, ST01-ST11)
+- Value formatting helper (`_format_value`) for correct INI output of bools, lists, nulls, and empty strings
+
+## [0.2.1] - 2026-05-10
+
+### Fixed
+
+- **SQLFluff config now includes `[sqlfluff:templater:dbt]` section** with `project_dir` and `profiles_dir` when downloaded from the framework, eliminating the need for `${DBT_PROFILES_DIR}` environment variable substitution
+- **Auto-download `.sqlfluff` before linting** — the lint menu now automatically downloads the config from the framework if it doesn't exist locally, instead of failing silently or erroring
+- **CI lint action handles missing `.sqlfluffconfig`** — the `tasks/lint-sql` action now only runs `envsubst` if `.sqlfluffconfig` exists, and accepts a pre-generated `.sqlfluff` file directly
+
+### Changed
+
+- `download_sqlfluff_config()` accepts an optional `profiles_dir` parameter to embed in the generated config
+- `DevelopmentMenu` and `DownloadsMenu` pass the resolved `profiles_dir` through to the download service
+- `.sqlfluff` no longer needs to be committed to client repos — it is generated on-the-fly from the framework
+
 ## [0.2.0] - 2026-05-09
 
 ### Added

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.2.0
+Version: 0.2.2
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.2.0"
+version = "0.2.2"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/setup.ps1

@@ -48,18 +48,60 @@ Write-Host "--- Snowflake Connection ---" -ForegroundColor Green
 Write-Host ""
 
 $connectionsFile = Join-Path $env:USERPROFILE ".snowflake\connections.toml"
+$connectionNames = @()
+$defaultConnection = $null
+
 if (Test-Path $connectionsFile) {
+    # Parse default connection name
+    $defaultLine = Get-Content $connectionsFile | Select-String '^default_connection_name\s*=' | Select-Object -First 1
+    if ($defaultLine) {
+        $defaultConnection = ($defaultLine -replace '.*=\s*', '').Trim().Trim('"').Trim("'")
+    }
+
+    # Parse connection section names
+    $connectionNames = @(Get-Content $connectionsFile | Select-String '^\[([^\]]+)\]' | ForEach-Object { $_.Matches[0].Groups[1].Value })
+}
+
+if ($connectionNames.Count -gt 0) {
     Write-Host "Available connections:" -ForegroundColor Cyan
-    Get-Content $connectionsFile | Select-String '^\[' | ForEach-Object { $_.Line -replace '[\[\]]', '' }
+    for ($i = 0; $i -lt $connectionNames.Count; $i++) {
+        $marker = ""
+        if ($connectionNames[$i] -eq $defaultConnection) { $marker = " (default)" }
+        Write-Host "  [$($i + 1)] $($connectionNames[$i])$marker"
+    }
     Write-Host ""
-}
 
-$connectionName = Read-Host "Enter connection name"
-if ([string]::IsNullOrWhiteSpace($connectionName)) {
-    Write-Host "ERROR: Connection name is required" -ForegroundColor Red
-    exit 1
+    $defaultIndex = -1
+    if ($defaultConnection) {
+        $defaultIndex = [array]::IndexOf($connectionNames, $defaultConnection)
+    }
+
+    if ($defaultIndex -ge 0) {
+        $prompt = "Select connection [1-$($connectionNames.Count)] (default: $($defaultIndex + 1))"
+    } else {
+        $prompt = "Select connection [1-$($connectionNames.Count)]"
+    }
+
+    $selection = Read-Host $prompt
+    if ([string]::IsNullOrWhiteSpace($selection) -and $defaultIndex -ge 0) {
+        $connectionName = $defaultConnection
+    } elseif ($selection -match '^\d+$' -and [int]$selection -ge 1 -and [int]$selection -le $connectionNames.Count) {
+        $connectionName = $connectionNames[[int]$selection - 1]
+    } else {
+        Write-Host "ERROR: Invalid selection" -ForegroundColor Red
+        exit 1
+    }
+} else {
+    Write-Host "No connections found in $connectionsFile" -ForegroundColor Yellow
+    $connectionName = Read-Host "Enter connection name"
+    if ([string]::IsNullOrWhiteSpace($connectionName)) {
+        Write-Host "ERROR: Connection name is required" -ForegroundColor Red
+        exit 1
+    }
 }
 
+Write-Host "[OK] Using connection: $connectionName" -ForegroundColor Green
+
 # --- Step 3: Enter native app database name ---
 
 Write-Host ""

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/setup.sh

@@ -46,18 +46,57 @@ echo "--- Snowflake Connection ---"
 echo ""
 
 CONNECTIONS_FILE="$HOME/.snowflake/connections.toml"
+CONNECTION_NAMES=()
+DEFAULT_CONNECTION=""
+
 if [ -f "$CONNECTIONS_FILE" ]; then
+    # Parse default connection name
+    DEFAULT_CONNECTION=$(grep '^default_connection_name' "$CONNECTIONS_FILE" | sed 's/.*=\s*//;s/[" ]//g' | head -1)
+
+    # Parse connection section names
+    while IFS= read -r name; do
+        CONNECTION_NAMES+=("$name")
+    done < <(grep '^\[' "$CONNECTIONS_FILE" | sed 's/\[//;s/\]//')
+fi
+
+if [ ${#CONNECTION_NAMES[@]} -gt 0 ]; then
     echo "Available connections:"
-    grep '^\[' "$CONNECTIONS_FILE" | sed 's/\[//;s/\]//' | nl -ba
+    DEFAULT_INDEX=-1
+    for i in "${!CONNECTION_NAMES[@]}"; do
+        marker=""
+        if [ "${CONNECTION_NAMES[$i]}" = "$DEFAULT_CONNECTION" ]; then
+            marker=" (default)"
+            DEFAULT_INDEX=$i
+        fi
+        echo "  [$((i + 1))] ${CONNECTION_NAMES[$i]}${marker}"
+    done
     echo ""
-fi
 
-read -rp "Enter connection name: " CONNECTION_NAME
-if [ -z "$CONNECTION_NAME" ]; then
-    echo "ERROR: Connection name is required"
-    exit 1
+    if [ $DEFAULT_INDEX -ge 0 ]; then
+        read -rp "Select connection [1-${#CONNECTION_NAMES[@]}] (default: $((DEFAULT_INDEX + 1))): " SELECTION
+    else
+        read -rp "Select connection [1-${#CONNECTION_NAMES[@]}]: " SELECTION
+    fi
+
+    if [ -z "$SELECTION" ] && [ $DEFAULT_INDEX -ge 0 ]; then
+        CONNECTION_NAME="$DEFAULT_CONNECTION"
+    elif [[ "$SELECTION" =~ ^[0-9]+$ ]] && [ "$SELECTION" -ge 1 ] && [ "$SELECTION" -le ${#CONNECTION_NAMES[@]} ]; then
+        CONNECTION_NAME="${CONNECTION_NAMES[$((SELECTION - 1))]}"
+    else
+        echo "ERROR: Invalid selection"
+        exit 1
+    fi
+else
+    echo "No connections found in $CONNECTIONS_FILE"
+    read -rp "Enter connection name: " CONNECTION_NAME
+    if [ -z "$CONNECTION_NAME" ]; then
+        echo "ERROR: Connection name is required"
+        exit 1
+    fi
 fi
 
+echo "[OK] Using connection: $CONNECTION_NAME"
+
 # --- Step 3: Enter native app database name ---
 
 echo ""

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/src/datasecops_cli/main.py

@@ -88,12 +88,13 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
     
     while option != 0:
         if option == 1:
+            profiles_dir = str(config.get_dbt_profiles_dir())
             if git_service:
                 dev_menu = DevelopmentMenu(dbt_runner, linting_service, git_service, profile_name,
-                                          download_service=download_service)
+                                          download_service=download_service, profiles_dir=profiles_dir)
             else:
                 dev_menu = DevelopmentMenu(dbt_runner, linting_service, None, profile_name,
-                                          download_service=download_service)
+                                          download_service=download_service, profiles_dir=profiles_dir)
             dev_menu.show()
         
         elif option == 2 and git_service:

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/src/datasecops_cli/menus/development.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from datasecops_cli.services.dbt_runner import DbtRunner
 from datasecops_cli.services.linting_service import LintingService
 from datasecops_cli.services.download_service import DownloadService
@@ -12,12 +14,14 @@ from datasecops_cli.utilities.display import (
 class DevelopmentMenu:
     def __init__(self, dbt_runner: DbtRunner, linting_service: LintingService, 
                  git_service: GitService, profile_name: str,
-                 download_service: DownloadService = None):
+                 download_service: DownloadService = None,
+                 profiles_dir: str = None):
         self.dbt = dbt_runner
         self.linting = linting_service
         self.git = git_service
         self.profile_name = profile_name
         self.downloads = download_service
+        self.profiles_dir = profiles_dir
     
     def show(self) -> None:
         self._menu()
@@ -136,6 +140,17 @@ class DevelopmentMenu:
                 self.dbt.test(select=select)
         complete_action()
     
+    def _ensure_sqlfluff_config(self) -> bool:
+        """Download .sqlfluff from the framework if it doesn't exist locally."""
+        config_path = self.linting.project_dir / ".sqlfluff"
+        if config_path.exists():
+            return True
+        if not self.downloads:
+            error_line("No .sqlfluff config found and download service not available")
+            return False
+        info_line("No .sqlfluff config found — downloading from framework...")
+        return self.downloads.download_sqlfluff_config(profiles_dir=self.profiles_dir)
+    
     def _lint_menu(self) -> None:
         clear()
         display_action_header("SQLFluff Linting Options")
@@ -147,6 +162,9 @@ class DevelopmentMenu:
         menu_option(6, "install       - Install SQLFluff requirements from framework")
         menu_option(0, "back          - Return to development menu")
         option = get_input_number("Choose an option: ")
+        if option in (1, 2, 3, 4, 5) and not self._ensure_sqlfluff_config():
+            complete_action()
+            return
         if option == 1:
             changed = [f.file for f in self.git.get_changed_files()] if self.git else []
             self.linting.lint_modified(fix=False, changed_files=changed)

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/src/datasecops_cli/menus/downloads.py

@@ -29,7 +29,8 @@ class DownloadsMenu:
         while option != 0:
             if option == 1:
                 display_action_header("Download SQLFluff Config")
-                self.downloads.download_sqlfluff_config()
+                profiles_dir = str(Path(self.project_settings.profile_dir).expanduser()) if self.project_settings else None
+                self.downloads.download_sqlfluff_config(profiles_dir=profiles_dir)
                 complete_action()
             elif option == 2:
                 display_action_header("Download Pipeline Files")

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/src/datasecops_cli/services/bootstrap_service.py

@@ -68,7 +68,8 @@ class BootstrapService:
         # Step 3: Download SQLFluff config
         info_line("")
         info_line("[3] Downloading SQLFluff configuration...")
-        if self.download_service.download_sqlfluff_config():
+        profiles_dir = str(Path(self.project_settings.profile_dir).expanduser())
+        if self.download_service.download_sqlfluff_config(profiles_dir=profiles_dir):
             steps_passed += 1
         else:
             info_line("  (skipped - no SQLFluff config in native app)")

{datasecops_cli-0.2.0 → datasecops_cli-0.2.2}/src/datasecops_cli/services/download_service.py

@@ -12,37 +12,149 @@ from datasecops_cli.utilities.file_utils import write_file, ensure_dir
 
 class DownloadService:
     """Downloads configurations from the native app."""
+
+    # Maps rule codes to their sqlfluff INI section names.
+    RULE_SECTION_MAP: dict[str, str] = {
+        # Aliasing
+        "AL01": "aliasing.table", "AL02": "aliasing.column", "AL03": "aliasing.expression",
+        "AL04": "aliasing.unique.table", "AL05": "aliasing.unused", "AL06": "aliasing.length",
+        "AL07": "aliasing.forbid", "AL08": "aliasing.unique.column", "AL09": "aliasing.self_alias",
+        # Ambiguous
+        "AM01": "ambiguous.distinct", "AM02": "ambiguous.union", "AM03": "ambiguous.order_by",
+        "AM04": "ambiguous.column_count", "AM05": "ambiguous.join",
+        "AM06": "ambiguous.column_references", "AM07": "ambiguous.set_columns",
+        "AM08": "ambiguous.union_type",
+        # Capitalisation
+        "CP01": "capitalisation.keywords", "CP02": "capitalisation.identifiers",
+        "CP03": "capitalisation.functions", "CP04": "capitalisation.literals",
+        "CP05": "capitalisation.types",
+        # Convention
+        "CV01": "convention.not_equal", "CV02": "convention.coalesce",
+        "CV03": "convention.select_trailing_comma", "CV04": "convention.count_rows",
+        "CV05": "convention.is_null", "CV06": "convention.terminator",
+        "CV07": "convention.statement_brackets", "CV08": "convention.left_join",
+        "CV09": "convention.blocked_words", "CV10": "convention.quoted_literals",
+        "CV11": "convention.casting_style", "CV12": "convention.semicolon",
+        # Jinja
+        "JJ01": "jinja.padding",
+        # Layout
+        "LT01": "layout.spacing", "LT02": "layout.indent", "LT03": "layout.operators",
+        "LT04": "layout.commas", "LT05": "layout.long_lines", "LT06": "layout.functions",
+        "LT07": "layout.cte_bracket", "LT08": "layout.cte_newline",
+        "LT09": "layout.select_targets", "LT10": "layout.select_modifiers",
+        "LT11": "layout.set_operators", "LT12": "layout.end-of-file",
+        "LT13": "layout.start_of_file", "LT14": "layout.one_line", "LT15": "layout.newlines",
+        # References
+        "RF01": "references.from", "RF02": "references.qualification",
+        "RF03": "references.consistent", "RF04": "references.keywords",
+        "RF05": "references.special_chars", "RF06": "references.quoting",
+        # Structure
+        "ST01": "structure.else_null", "ST02": "structure.simple_case",
+        "ST03": "structure.unused_cte", "ST04": "structure.nested_case",
+        "ST05": "structure.subquery", "ST06": "structure.column_order",
+        "ST07": "structure.using", "ST08": "structure.distinct",
+        "ST09": "structure.join_condition_order", "ST10": "structure.cte_definition_order",
+        "ST11": "structure.test_query",
+    }
+
+    # Core sub-keys that map to specific INI sections.
+    CORE_SECTION_MAP: dict[str, str] = {
+        "sqlfluff": "sqlfluff",
+        "dbt": "sqlfluff:templater:dbt",
+        "jinja": "sqlfluff:templater:jinja",
+        "templater": "sqlfluff:templater",
+    }
     
     def __init__(self, snowflake_service: SnowflakeService, project_dir: Path):
         self.sf = snowflake_service
         self.project_dir = project_dir
+
+    @staticmethod
+    def _format_value(val) -> str:
+        """Format a JSON value for sqlfluff INI output."""
+        if isinstance(val, bool):
+            return str(val)
+        if isinstance(val, list):
+            if not val:
+                return "None"
+            return ", ".join(str(v) for v in val)
+        if val is None or val == "":
+            return "None"
+        return str(val)
+
+    @staticmethod
+    def _emit_section(lines: list[str], header: str, options: dict) -> None:
+        """Append an INI section with its options to lines."""
+        lines.append(f"[{header}]")
+        for key, val in options.items():
+            lines.append(f"{key} = {DownloadService._format_value(val)}")
+        lines.append("")
     
-    def download_sqlfluff_config(self) -> bool:
+    def download_sqlfluff_config(self, profiles_dir: str = None) -> bool:
         info_line("Downloading SQLFluff configuration...")
         raw = self.sf.get_framework_config("SQLFLUFF_RULES")
         if not raw:
             error_line("No SQLFluff configuration found in native app")
             return False
         
-        lines = ["[sqlfluff]", "dialect = snowflake", "templater = dbt", ""]
-        
+        lines: list[str] = []
+
+        # --- Core sections ([sqlfluff], [sqlfluff:templater:dbt], etc.) ---
         core = raw.get("core", {})
-        for code, state in core.items():
-            if isinstance(state, dict) and state.get("enabled", True):
-                opts = state.get("options", {})
-                for opt_key, opt_val in opts.items():
-                    lines.append(f"{opt_key} = {opt_val}")
-        
-        lines.append("")
-        lines.append("[sqlfluff:indentation]")
+        for key, section_header in self.CORE_SECTION_MAP.items():
+            entry = core.get(key)
+            if not isinstance(entry, dict) or not entry.get("enabled", True):
+                continue
+            opts = dict(entry.get("options", {}))
+            if key == "dbt":
+                # Override profiles_dir / project_dir with local values
+                opts["project_dir"] = "."
+                if profiles_dir:
+                    opts["profiles_dir"] = profiles_dir
+                elif "profiles_dir" in opts:
+                    del opts["profiles_dir"]
+            if opts:
+                self._emit_section(lines, section_header, opts)
+
+        # --- Indentation ---
         indentation = raw.get("indentation", {})
-        for code, state in indentation.items():
-            if isinstance(state, dict):
-                opts = state.get("options", {})
-                for opt_key, opt_val in opts.items():
-                    lines.append(f"{opt_key} = {opt_val}")
-        
-        content = "\n".join(lines) + "\n"
+        for _code, entry in indentation.items():
+            if isinstance(entry, dict) and entry.get("enabled", True):
+                opts = entry.get("options", {})
+                if opts:
+                    self._emit_section(lines, "sqlfluff:indentation", opts)
+
+        # --- Layout sections ([sqlfluff:layout:type:<code>]) ---
+        layout = raw.get("layout", {})
+        for code, entry in layout.items():
+            if not isinstance(entry, dict) or not entry.get("enabled", True):
+                continue
+            opts = entry.get("options", {})
+            if opts:
+                self._emit_section(lines, f"sqlfluff:layout:type:{code}", opts)
+
+        # --- Global rules ([sqlfluff:rules]) ---
+        rules = raw.get("rules", {})
+        for _code, entry in rules.items():
+            if isinstance(entry, dict) and entry.get("enabled", True):
+                opts = entry.get("options", {})
+                if opts:
+                    self._emit_section(lines, "sqlfluff:rules", opts)
+
+        # --- Rule bundle sections (rules_aliasing, rules_capitalisation, etc.) ---
+        for section_key, entries in raw.items():
+            if not section_key.startswith("rules_") or not isinstance(entries, dict):
+                continue
+            for code, entry in entries.items():
+                if not isinstance(entry, dict) or not entry.get("enabled", True):
+                    continue
+                section_name = self.RULE_SECTION_MAP.get(code)
+                if not section_name:
+                    continue
+                opts = entry.get("options", {})
+                self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
+
+        content = "\n".join(lines)
         dest = self.project_dir / ".sqlfluff"
         write_file(dest, content)
         success_line(f"SQLFluff config written to {dest}")