datasecops-cli 0.2.1__tar.gz → 0.2.3__tar.gz

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.2.3] - 2026-05-11
+
+### Fixed
+
+- **Stripped invalid `dbt_skip_compilation_error` option** from the generated `[sqlfluff:templater:dbt]` section — this native app field is not a valid sqlfluff config key and caused parse errors
+- **`profiles_dir = ${DBT_PROFILES_DIR}` no longer leaks** into generated config — the raw value from the native app is now always replaced with the resolved local path or removed entirely
+- **Empty string values** (e.g. `warnings = ""`) now emit `None` instead of blank, matching sqlfluff's expected INI format
+
+### Added
+
+- **Auto-install `sqlfluff-templater-dbt`** — the lint menu now checks for missing sqlfluff packages before linting and installs them automatically
+
+## [0.2.2] - 2026-05-10
+
+### Fixed
+
+- **Full SQLFluff config generation** — `download_sqlfluff_config()` now generates all sections from the native app config: core, templater, indentation, layout types, global rules, and all rule bundles (aliasing, ambiguous, capitalisation, convention, jinja, layout, references, structure)
+- Previously only `core` and `indentation` sections were emitted, missing ~90% of the config
+
+### Added
+
+- **Setup script connection selector** — `setup.ps1` and `setup.sh` now parse `~/.snowflake/connections.toml`, display a numbered list of connections with the default highlighted, and let the user select by number instead of typing the name manually
+- Rule code to section name mapping (`RULE_SECTION_MAP`) for all sqlfluff rule bundles (AL01-AL09, AM01-AM08, CP01-CP05, CV01-CV12, JJ01, LT01-LT15, RF01-RF06, ST01-ST11)
+- Value formatting helper (`_format_value`) for correct INI output of bools, lists, nulls, and empty strings
+
 ## [0.2.1] - 2026-05-10
 
 ### Fixed

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.2.1
+Version: 0.2.3
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.2.1"
+version = "0.2.3"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/setup.ps1

@@ -48,18 +48,60 @@ Write-Host "--- Snowflake Connection ---" -ForegroundColor Green
 Write-Host ""
 
 $connectionsFile = Join-Path $env:USERPROFILE ".snowflake\connections.toml"
+$connectionNames = @()
+$defaultConnection = $null
+
 if (Test-Path $connectionsFile) {
+    # Parse default connection name
+    $defaultLine = Get-Content $connectionsFile | Select-String '^default_connection_name\s*=' | Select-Object -First 1
+    if ($defaultLine) {
+        $defaultConnection = ($defaultLine -replace '.*=\s*', '').Trim().Trim('"').Trim("'")
+    }
+
+    # Parse connection section names
+    $connectionNames = @(Get-Content $connectionsFile | Select-String '^\[([^\]]+)\]' | ForEach-Object { $_.Matches[0].Groups[1].Value })
+}
+
+if ($connectionNames.Count -gt 0) {
     Write-Host "Available connections:" -ForegroundColor Cyan
-    Get-Content $connectionsFile | Select-String '^\[' | ForEach-Object { $_.Line -replace '[\[\]]', '' }
+    for ($i = 0; $i -lt $connectionNames.Count; $i++) {
+        $marker = ""
+        if ($connectionNames[$i] -eq $defaultConnection) { $marker = " (default)" }
+        Write-Host "  [$($i + 1)] $($connectionNames[$i])$marker"
+    }
     Write-Host ""
-}
 
-$connectionName = Read-Host "Enter connection name"
-if ([string]::IsNullOrWhiteSpace($connectionName)) {
-    Write-Host "ERROR: Connection name is required" -ForegroundColor Red
-    exit 1
+    $defaultIndex = -1
+    if ($defaultConnection) {
+        $defaultIndex = [array]::IndexOf($connectionNames, $defaultConnection)
+    }
+
+    if ($defaultIndex -ge 0) {
+        $prompt = "Select connection [1-$($connectionNames.Count)] (default: $($defaultIndex + 1))"
+    } else {
+        $prompt = "Select connection [1-$($connectionNames.Count)]"
+    }
+
+    $selection = Read-Host $prompt
+    if ([string]::IsNullOrWhiteSpace($selection) -and $defaultIndex -ge 0) {
+        $connectionName = $defaultConnection
+    } elseif ($selection -match '^\d+$' -and [int]$selection -ge 1 -and [int]$selection -le $connectionNames.Count) {
+        $connectionName = $connectionNames[[int]$selection - 1]
+    } else {
+        Write-Host "ERROR: Invalid selection" -ForegroundColor Red
+        exit 1
+    }
+} else {
+    Write-Host "No connections found in $connectionsFile" -ForegroundColor Yellow
+    $connectionName = Read-Host "Enter connection name"
+    if ([string]::IsNullOrWhiteSpace($connectionName)) {
+        Write-Host "ERROR: Connection name is required" -ForegroundColor Red
+        exit 1
+    }
 }
 
+Write-Host "[OK] Using connection: $connectionName" -ForegroundColor Green
+
 # --- Step 3: Enter native app database name ---
 
 Write-Host ""

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/setup.sh

@@ -46,18 +46,57 @@ echo "--- Snowflake Connection ---"
 echo ""
 
 CONNECTIONS_FILE="$HOME/.snowflake/connections.toml"
+CONNECTION_NAMES=()
+DEFAULT_CONNECTION=""
+
 if [ -f "$CONNECTIONS_FILE" ]; then
+    # Parse default connection name
+    DEFAULT_CONNECTION=$(grep '^default_connection_name' "$CONNECTIONS_FILE" | sed 's/.*=\s*//;s/[" ]//g' | head -1)
+
+    # Parse connection section names
+    while IFS= read -r name; do
+        CONNECTION_NAMES+=("$name")
+    done < <(grep '^\[' "$CONNECTIONS_FILE" | sed 's/\[//;s/\]//')
+fi
+
+if [ ${#CONNECTION_NAMES[@]} -gt 0 ]; then
     echo "Available connections:"
-    grep '^\[' "$CONNECTIONS_FILE" | sed 's/\[//;s/\]//' | nl -ba
+    DEFAULT_INDEX=-1
+    for i in "${!CONNECTION_NAMES[@]}"; do
+        marker=""
+        if [ "${CONNECTION_NAMES[$i]}" = "$DEFAULT_CONNECTION" ]; then
+            marker=" (default)"
+            DEFAULT_INDEX=$i
+        fi
+        echo "  [$((i + 1))] ${CONNECTION_NAMES[$i]}${marker}"
+    done
     echo ""
-fi
 
-read -rp "Enter connection name: " CONNECTION_NAME
-if [ -z "$CONNECTION_NAME" ]; then
-    echo "ERROR: Connection name is required"
-    exit 1
+    if [ $DEFAULT_INDEX -ge 0 ]; then
+        read -rp "Select connection [1-${#CONNECTION_NAMES[@]}] (default: $((DEFAULT_INDEX + 1))): " SELECTION
+    else
+        read -rp "Select connection [1-${#CONNECTION_NAMES[@]}]: " SELECTION
+    fi
+
+    if [ -z "$SELECTION" ] && [ $DEFAULT_INDEX -ge 0 ]; then
+        CONNECTION_NAME="$DEFAULT_CONNECTION"
+    elif [[ "$SELECTION" =~ ^[0-9]+$ ]] && [ "$SELECTION" -ge 1 ] && [ "$SELECTION" -le ${#CONNECTION_NAMES[@]} ]; then
+        CONNECTION_NAME="${CONNECTION_NAMES[$((SELECTION - 1))]}"
+    else
+        echo "ERROR: Invalid selection"
+        exit 1
+    fi
+else
+    echo "No connections found in $CONNECTIONS_FILE"
+    read -rp "Enter connection name: " CONNECTION_NAME
+    if [ -z "$CONNECTION_NAME" ]; then
+        echo "ERROR: Connection name is required"
+        exit 1
+    fi
 fi
 
+echo "[OK] Using connection: $CONNECTION_NAME"
+
 # --- Step 3: Enter native app database name ---
 
 echo ""

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/src/datasecops_cli/menus/development.py

@@ -141,15 +141,16 @@ class DevelopmentMenu:
         complete_action()
     
     def _ensure_sqlfluff_config(self) -> bool:
-        """Download .sqlfluff from the framework if it doesn't exist locally."""
+        """Download .sqlfluff from the framework if it doesn't exist locally, and ensure templater is installed."""
         config_path = self.linting.project_dir / ".sqlfluff"
-        if config_path.exists():
-            return True
-        if not self.downloads:
-            error_line("No .sqlfluff config found and download service not available")
-            return False
-        info_line("No .sqlfluff config found — downloading from framework...")
-        return self.downloads.download_sqlfluff_config(profiles_dir=self.profiles_dir)
+        if not config_path.exists():
+            if not self.downloads:
+                error_line("No .sqlfluff config found and download service not available")
+                return False
+            info_line("No .sqlfluff config found — downloading from framework...")
+            if not self.downloads.download_sqlfluff_config(profiles_dir=self.profiles_dir):
+                return False
+        return self.linting.ensure_templater_installed()
     
     def _lint_menu(self) -> None:
         clear()

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/src/datasecops_cli/services/download_service.py

@@ -12,10 +12,88 @@ from datasecops_cli.utilities.file_utils import write_file, ensure_dir
 
 class DownloadService:
     """Downloads configurations from the native app."""
+
+    # Maps rule codes to their sqlfluff INI section names.
+    RULE_SECTION_MAP: dict[str, str] = {
+        # Aliasing
+        "AL01": "aliasing.table", "AL02": "aliasing.column", "AL03": "aliasing.expression",
+        "AL04": "aliasing.unique.table", "AL05": "aliasing.unused", "AL06": "aliasing.length",
+        "AL07": "aliasing.forbid", "AL08": "aliasing.unique.column", "AL09": "aliasing.self_alias",
+        # Ambiguous
+        "AM01": "ambiguous.distinct", "AM02": "ambiguous.union", "AM03": "ambiguous.order_by",
+        "AM04": "ambiguous.column_count", "AM05": "ambiguous.join",
+        "AM06": "ambiguous.column_references", "AM07": "ambiguous.set_columns",
+        "AM08": "ambiguous.union_type",
+        # Capitalisation
+        "CP01": "capitalisation.keywords", "CP02": "capitalisation.identifiers",
+        "CP03": "capitalisation.functions", "CP04": "capitalisation.literals",
+        "CP05": "capitalisation.types",
+        # Convention
+        "CV01": "convention.not_equal", "CV02": "convention.coalesce",
+        "CV03": "convention.select_trailing_comma", "CV04": "convention.count_rows",
+        "CV05": "convention.is_null", "CV06": "convention.terminator",
+        "CV07": "convention.statement_brackets", "CV08": "convention.left_join",
+        "CV09": "convention.blocked_words", "CV10": "convention.quoted_literals",
+        "CV11": "convention.casting_style", "CV12": "convention.semicolon",
+        # Jinja
+        "JJ01": "jinja.padding",
+        # Layout
+        "LT01": "layout.spacing", "LT02": "layout.indent", "LT03": "layout.operators",
+        "LT04": "layout.commas", "LT05": "layout.long_lines", "LT06": "layout.functions",
+        "LT07": "layout.cte_bracket", "LT08": "layout.cte_newline",
+        "LT09": "layout.select_targets", "LT10": "layout.select_modifiers",
+        "LT11": "layout.set_operators", "LT12": "layout.end-of-file",
+        "LT13": "layout.start_of_file", "LT14": "layout.one_line", "LT15": "layout.newlines",
+        # References
+        "RF01": "references.from", "RF02": "references.qualification",
+        "RF03": "references.consistent", "RF04": "references.keywords",
+        "RF05": "references.special_chars", "RF06": "references.quoting",
+        # Structure
+        "ST01": "structure.else_null", "ST02": "structure.simple_case",
+        "ST03": "structure.unused_cte", "ST04": "structure.nested_case",
+        "ST05": "structure.subquery", "ST06": "structure.column_order",
+        "ST07": "structure.using", "ST08": "structure.distinct",
+        "ST09": "structure.join_condition_order", "ST10": "structure.cte_definition_order",
+        "ST11": "structure.test_query",
+    }
+
+    # Core sub-keys that map to specific INI sections.
+    CORE_SECTION_MAP: dict[str, str] = {
+        "sqlfluff": "sqlfluff",
+        "dbt": "sqlfluff:templater:dbt",
+        "jinja": "sqlfluff:templater:jinja",
+        "templater": "sqlfluff:templater",
+    }
     
     def __init__(self, snowflake_service: SnowflakeService, project_dir: Path):
         self.sf = snowflake_service
         self.project_dir = project_dir
+
+    # Options to strip from specific core sections (not valid sqlfluff config keys).
+    CORE_STRIP_OPTIONS: dict[str, set[str]] = {
+        "dbt": {"dbt_skip_compilation_error"},
+    }
+
+    @staticmethod
+    def _format_value(val) -> str:
+        """Format a JSON value for sqlfluff INI output."""
+        if isinstance(val, bool):
+            return str(val)
+        if isinstance(val, list):
+            if not val:
+                return "None"
+            return ", ".join(str(v) for v in val)
+        if val is None or val == "":
+            return "None"
+        return str(val)
+
+    @staticmethod
+    def _emit_section(lines: list[str], header: str, options: dict) -> None:
+        """Append an INI section with its options to lines."""
+        lines.append(f"[{header}]")
+        for key, val in options.items():
+            lines.append(f"{key} = {DownloadService._format_value(val)}")
+        lines.append("")
     
     def download_sqlfluff_config(self, profiles_dir: str = None) -> bool:
         info_line("Downloading SQLFluff configuration...")
@@ -24,31 +102,67 @@ class DownloadService:
             error_line("No SQLFluff configuration found in native app")
             return False
         
-        lines = ["[sqlfluff]", "dialect = snowflake", "templater = dbt", ""]
-        
+        lines: list[str] = []
+
+        # --- Core sections ([sqlfluff], [sqlfluff:templater:dbt], etc.) ---
         core = raw.get("core", {})
-        for code, state in core.items():
-            if isinstance(state, dict) and state.get("enabled", True):
-                opts = state.get("options", {})
-                for opt_key, opt_val in opts.items():
-                    lines.append(f"{opt_key} = {opt_val}")
-        
-        lines.append("")
-        lines.append("[sqlfluff:templater:dbt]")
-        lines.append("project_dir = .")
-        if profiles_dir:
-            lines.append(f"profiles_dir = {profiles_dir}")
-        
-        lines.append("")
-        lines.append("[sqlfluff:indentation]")
+        for key, section_header in self.CORE_SECTION_MAP.items():
+            entry = core.get(key)
+            if not isinstance(entry, dict) or not entry.get("enabled", True):
+                continue
+            opts = dict(entry.get("options", {}))
+            # Strip options that are not valid sqlfluff config keys
+            for strip_key in self.CORE_STRIP_OPTIONS.get(key, set()):
+                opts.pop(strip_key, None)
+            if key == "dbt":
+                # Override profiles_dir / project_dir with local values
+                opts["project_dir"] = "."
+                if profiles_dir:
+                    opts["profiles_dir"] = profiles_dir
+                else:
+                    opts.pop("profiles_dir", None)
+            if opts:
+                self._emit_section(lines, section_header, opts)
+
+        # --- Indentation ---
         indentation = raw.get("indentation", {})
-        for code, state in indentation.items():
-            if isinstance(state, dict):
-                opts = state.get("options", {})
-                for opt_key, opt_val in opts.items():
-                    lines.append(f"{opt_key} = {opt_val}")
-        
-        content = "\n".join(lines) + "\n"
+        for _code, entry in indentation.items():
+            if isinstance(entry, dict) and entry.get("enabled", True):
+                opts = entry.get("options", {})
+                if opts:
+                    self._emit_section(lines, "sqlfluff:indentation", opts)
+
+        # --- Layout sections ([sqlfluff:layout:type:<code>]) ---
+        layout = raw.get("layout", {})
+        for code, entry in layout.items():
+            if not isinstance(entry, dict) or not entry.get("enabled", True):
+                continue
+            opts = entry.get("options", {})
+            if opts:
+                self._emit_section(lines, f"sqlfluff:layout:type:{code}", opts)
+
+        # --- Global rules ([sqlfluff:rules]) ---
+        rules = raw.get("rules", {})
+        for _code, entry in rules.items():
+            if isinstance(entry, dict) and entry.get("enabled", True):
+                opts = entry.get("options", {})
+                if opts:
+                    self._emit_section(lines, "sqlfluff:rules", opts)
+
+        # --- Rule bundle sections (rules_aliasing, rules_capitalisation, etc.) ---
+        for section_key, entries in raw.items():
+            if not section_key.startswith("rules_") or not isinstance(entries, dict):
+                continue
+            for code, entry in entries.items():
+                if not isinstance(entry, dict) or not entry.get("enabled", True):
+                    continue
+                section_name = self.RULE_SECTION_MAP.get(code)
+                if not section_name:
+                    continue
+                opts = entry.get("options", {})
+                self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
+
+        content = "\n".join(lines)
         dest = self.project_dir / ".sqlfluff"
         write_file(dest, content)
         success_line(f"SQLFluff config written to {dest}")

{datasecops_cli-0.2.1 → datasecops_cli-0.2.3}/src/datasecops_cli/services/linting_service.py

@@ -43,6 +43,15 @@ class LintingService:
             error_line(f"pip install failed with exit code {result.returncode}")
             return False
     
+    def ensure_templater_installed(self) -> bool:
+        """Check if sqlfluff-templater-dbt is installed; install it if missing."""
+        versions = self.get_installed_versions()
+        missing = [pkg for pkg in ["sqlfluff", "sqlfluff-templater-dbt"] if not versions.get(pkg)]
+        if not missing:
+            return True
+        warning_line(f"Missing packages: {', '.join(missing)}")
+        return self.install_requirements(missing)
+    
     def lint_file(self, file_path: str = None, fix: bool = False) -> subprocess.CompletedProcess:
         action = "fix" if fix else "lint"
         cmd = ["sqlfluff", action]