datasecops-cli 0.2.7__tar.gz → 0.2.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/CHANGELOG.md +21 -0
  2. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/PKG-INFO +61 -8
  3. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/README.md +60 -7
  4. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/getting-started.md +1 -1
  5. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/pyproject.toml +1 -1
  6. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/setup.ps1 +6 -5
  7. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/setup.sh +4 -4
  8. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/main.py +132 -48
  9. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/development.py +1 -1
  10. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/downloads.py +6 -3
  11. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/bootstrap_service.py +2 -2
  12. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/dbt_runner.py +2 -2
  13. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/download_service.py +8 -3
  14. datasecops_cli-0.2.9/tests/test_main.py +302 -0
  15. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/.github/workflows/publish-cli.yml +0 -0
  16. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/.gitignore +0 -0
  17. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/DEVELOPMENT.md +0 -0
  18. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/LICENSE +0 -0
  19. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/legacy.md +0 -0
  20. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/legacy_plan_of_action.md +0 -0
  21. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/mcp-server.md +0 -0
  22. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/mcp-servers.json +0 -0
  23. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/__init__.py +0 -0
  24. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/config.py +0 -0
  25. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/__init__.py +0 -0
  26. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/git_operations.py +0 -0
  27. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/__init__.py +0 -0
  28. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/git_helpers.py +0 -0
  29. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/project_config.py +0 -0
  30. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/__init__.py +0 -0
  31. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/git_service.py +0 -0
  32. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/linting_service.py +0 -0
  33. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/skill_service.py +0 -0
  34. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/snowflake_service.py +0 -0
  35. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/__init__.py +0 -0
  36. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/display.py +0 -0
  37. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/file_utils.py +0 -0
  38. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  39. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/__init__.py +0 -0
  40. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/__main__.py +0 -0
  41. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/connection.py +0 -0
  42. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/server.py +0 -0
  43. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/__init__.py +0 -0
  44. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_config.py +0 -0
  45. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_file_utils.py +0 -0
  46. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_models.py +0 -0
  47. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_version.py +0 -0
  48. {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_yaml_utils.py +0 -0
@@ -2,6 +2,27 @@
2
2
 
3
3
  All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
+ ## [0.2.9] - 2026-05-14
6
+
7
+ ### Added
8
+
9
+ - **`install-sqlfluff` and `install-dbt` download items** — `datasecops download install-sqlfluff` and `datasecops download install-dbt` fetch framework-pinned package versions from the native app and install them via `uv pip install`. `datasecops download all` now includes both installs.
10
+
11
+ ## [0.2.8] - 2026-05-14
12
+
13
+ ### Added
14
+
15
+ - **Non-interactive `download` subcommand** — `datasecops download <items>` downloads framework config without prompts, designed for CI/CD pipelines. Supports `sqlfluff`, `pipelines`, `packages`, `macros`, or `all`
16
+ - **Runtime `dbtf` check** — the CLI now warns at startup if dbt Fusion (`dbtf`) is not found on PATH
17
+
18
+ ### Changed
19
+
20
+ - **Platform auto-detected from framework config** — pipeline downloads, bootstrap, and the downloads menu now read `source_control_platform` from the native app instead of prompting the user to select GitHub or Azure DevOps
21
+ - **Case-insensitive platform matching** — `download_pipelines()` now compares platform values case-insensitively, fixing a bug where pipelines stored as `"GitHub"` would not match the lowercase `"github"` filter
22
+ - **All dbt commands use `dbtf`** — `DbtRunner` and setup scripts now consistently use the `dbtf` binary (dbt Fusion) instead of `dbt`
23
+ - **Refactored CLI entry point** — `main()` now uses `argparse` with subcommands (`bootstrap`, `download`) instead of manual `sys.argv` parsing. Shared connection/config logic extracted to `_connect_and_load()`
24
+ - **Development menu option 14** relabelled to clarify it installs dbt-core/dbt-snowflake for linting, not dbt Fusion
25
+
5
26
  ## [0.2.7] - 2026-05-12
6
27
 
7
28
  ### Changed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasecops-cli
3
- Version: 0.2.7
3
+ Version: 0.2.9
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -55,7 +55,7 @@ Requires Python 3.10 or later.
55
55
 
56
56
  Optional:
57
57
 
58
- - **dbt Fusion** (or dbt-core with dbt-snowflake) for dbt commands
58
+ - **dbt Fusion** required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation
59
59
  - **Cortex Code** for skill downloads
60
60
  - **Node.js 18+** for GitHub/Azure DevOps MCP servers
61
61
 
@@ -96,6 +96,65 @@ datasecops
96
96
  | **Git** | Branch create/checkout/delete, commit & push, rebase, squash, deploy to environment branches, cherry-pick |
97
97
  | **Downloads** | SQLFluff config, CI/CD pipelines (GitHub Actions / Azure DevOps), dbt packages, Cortex Code skills |
98
98
 
99
+ ## Non-Interactive Mode (CI/CD)
100
+
101
+ The `download` subcommand lets you pull framework config in CI/CD pipelines without interactive prompts:
102
+
103
+ ```bash
104
+ # Download specific items
105
+ datasecops download sqlfluff
106
+ datasecops download sqlfluff packages
107
+ datasecops download pipelines macros
108
+
109
+ # Install framework-pinned package versions
110
+ datasecops download install-sqlfluff
111
+ datasecops download install-dbt
112
+
113
+ # Download config and install packages together
114
+ datasecops download sqlfluff install-sqlfluff
115
+
116
+ # Download and install everything
117
+ datasecops download all
118
+ ```
119
+
120
+ Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
121
+
122
+ The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
123
+
124
+ ### Pipeline Setup
125
+
126
+ Your pipeline needs two things:
127
+
128
+ 1. **A `.datasecops.yml`** in the repo (already committed — contains no secrets):
129
+
130
+ ```yaml
131
+ connection_name: "ci"
132
+ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
133
+ ```
134
+
135
+ 2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
136
+
137
+ ```yaml
138
+ # GitHub Actions example
139
+ - name: Configure Snowflake connection
140
+ run: |
141
+ mkdir -p ~/.snowflake
142
+ cat > ~/.snowflake/connections.toml << EOF
143
+ [ci]
144
+ account = "${{ vars.SNOWFLAKE_ACCOUNT }}"
145
+ user = "${{ vars.SNOWFLAKE_USER }}"
146
+ authenticator = "snowflake_jwt"
147
+ private_key_file = "/tmp/rsa_key.p8"
148
+ warehouse = "CI_WH"
149
+ role = "CI_ROLE"
150
+ EOF
151
+
152
+ - name: Download SQLFluff config
153
+ run: datasecops download sqlfluff
154
+ ```
155
+
156
+ The exit code is `0` on success, `1` if any download fails.
157
+
99
158
  ## MCP Server
100
159
 
101
160
  The package includes an MCP (Model Context Protocol) server that exposes your framework's governance configuration to AI coding assistants. Instead of static skill files, the MCP server gives AI tools live access to your native app's current rules.
@@ -154,12 +213,6 @@ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
154
213
 
155
214
  Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
156
215
 
157
- ## Documentation
158
-
159
- - [Getting Started Guide](docs/getting-started.md) — install CLI, configure MCP servers for VS Code/Cursor/Cortex Code with Snowflake, dbt, and GitHub/Azure DevOps
160
- - [MCP Server Reference](docs/mcp-server.md) — full tool documentation, architecture, and usage examples
161
- - [Development Guide](DEVELOPMENT.md) — project structure, setup scripts, native app API reference, and publishing details
162
-
163
216
  ## License
164
217
 
165
218
  MIT
@@ -35,7 +35,7 @@ Requires Python 3.10 or later.
35
35
 
36
36
  Optional:
37
37
 
38
- - **dbt Fusion** (or dbt-core with dbt-snowflake) for dbt commands
38
+ - **dbt Fusion** required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation
39
39
  - **Cortex Code** for skill downloads
40
40
  - **Node.js 18+** for GitHub/Azure DevOps MCP servers
41
41
 
@@ -76,6 +76,65 @@ datasecops
76
76
  | **Git** | Branch create/checkout/delete, commit & push, rebase, squash, deploy to environment branches, cherry-pick |
77
77
  | **Downloads** | SQLFluff config, CI/CD pipelines (GitHub Actions / Azure DevOps), dbt packages, Cortex Code skills |
78
78
 
79
+ ## Non-Interactive Mode (CI/CD)
80
+
81
+ The `download` subcommand lets you pull framework config in CI/CD pipelines without interactive prompts:
82
+
83
+ ```bash
84
+ # Download specific items
85
+ datasecops download sqlfluff
86
+ datasecops download sqlfluff packages
87
+ datasecops download pipelines macros
88
+
89
+ # Install framework-pinned package versions
90
+ datasecops download install-sqlfluff
91
+ datasecops download install-dbt
92
+
93
+ # Download config and install packages together
94
+ datasecops download sqlfluff install-sqlfluff
95
+
96
+ # Download and install everything
97
+ datasecops download all
98
+ ```
99
+
100
+ Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
101
+
102
+ The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
103
+
104
+ ### Pipeline Setup
105
+
106
+ Your pipeline needs two things:
107
+
108
+ 1. **A `.datasecops.yml`** in the repo (already committed — contains no secrets):
109
+
110
+ ```yaml
111
+ connection_name: "ci"
112
+ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
113
+ ```
114
+
115
+ 2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
116
+
117
+ ```yaml
118
+ # GitHub Actions example
119
+ - name: Configure Snowflake connection
120
+ run: |
121
+ mkdir -p ~/.snowflake
122
+ cat > ~/.snowflake/connections.toml << EOF
123
+ [ci]
124
+ account = "${{ vars.SNOWFLAKE_ACCOUNT }}"
125
+ user = "${{ vars.SNOWFLAKE_USER }}"
126
+ authenticator = "snowflake_jwt"
127
+ private_key_file = "/tmp/rsa_key.p8"
128
+ warehouse = "CI_WH"
129
+ role = "CI_ROLE"
130
+ EOF
131
+
132
+ - name: Download SQLFluff config
133
+ run: datasecops download sqlfluff
134
+ ```
135
+
136
+ The exit code is `0` on success, `1` if any download fails.
137
+
79
138
  ## MCP Server
80
139
 
81
140
  The package includes an MCP (Model Context Protocol) server that exposes your framework's governance configuration to AI coding assistants. Instead of static skill files, the MCP server gives AI tools live access to your native app's current rules.
@@ -134,12 +193,6 @@ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
134
193
 
135
194
  Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
136
195
 
137
- ## Documentation
138
-
139
- - [Getting Started Guide](docs/getting-started.md) — install CLI, configure MCP servers for VS Code/Cursor/Cortex Code with Snowflake, dbt, and GitHub/Azure DevOps
140
- - [MCP Server Reference](docs/mcp-server.md) — full tool documentation, architecture, and usage examples
141
- - [Development Guide](DEVELOPMENT.md) — project structure, setup scripts, native app API reference, and publishing details
142
-
143
196
  ## License
144
197
 
145
198
  MIT
@@ -14,7 +14,7 @@ Before you begin, ensure you have:
14
14
  - **Git** configured with access to your repository
15
15
 
16
16
  Optional:
17
- - **dbt Fusion** (or dbt-core with dbt-snowflake) for dbt commands
17
+ - **dbt Fusion** required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation. The CLI invokes `dbtf` for all dbt operations.
18
18
  - **A GitHub Personal Access Token** (for GitHub MCP server)
19
19
  - **An Azure DevOps PAT** (for Azure DevOps MCP server)
20
20
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.2.7"
7
+ version = "0.2.9"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -16,13 +16,14 @@ try {
16
16
  $env:Path = "$env:USERPROFILE\.local\bin;$env:Path"
17
17
  }
18
18
 
19
- # Check for dbt
19
+ # Check for dbt Fusion (dbtf)
20
20
  try {
21
- $null = Get-Command dbt -ErrorAction Stop
22
- Write-Host "[OK] dbt found" -ForegroundColor Green
21
+ $null = Get-Command dbtf -ErrorAction Stop
22
+ Write-Host "[OK] dbt Fusion (dbtf) found" -ForegroundColor Green
23
23
  } catch {
24
- Write-Host "WARNING: dbt not found on PATH." -ForegroundColor Yellow
25
- Write-Host " Install dbt Fusion or: pip install dbt-snowflake" -ForegroundColor Yellow
24
+ Write-Host "WARNING: dbt Fusion (dbtf) not found on PATH." -ForegroundColor Yellow
25
+ Write-Host " Install dbt Fusion: https://docs.getdbt.com/docs/core/installation" -ForegroundColor Yellow
26
+ Write-Host " The CLI uses 'dbtf' for all dbt commands." -ForegroundColor Yellow
26
27
  }
27
28
 
28
29
  # Check for cortex (optional)
@@ -16,12 +16,12 @@ if ! command -v uv &> /dev/null; then
16
16
  fi
17
17
  echo "[OK] uv $(uv --version 2>/dev/null || echo 'installed')"
18
18
 
19
- # Check for dbt
20
- if ! command -v dbt &> /dev/null; then
19
+ # Check for dbt Fusion (dbtf)
20
+ if ! command -v dbtf &> /dev/null; then
21
21
  echo ""
22
- echo "WARNING: dbt not found on PATH."
22
+ echo "WARNING: dbt Fusion (dbtf) not found on PATH."
23
23
  echo " Install dbt Fusion: https://docs.getdbt.com/docs/core/installation"
24
- echo " Or install via pip: pip install dbt-snowflake"
24
+ echo " The CLI uses 'dbtf' for all dbt commands."
25
25
  echo ""
26
26
  fi
27
27
 
@@ -1,3 +1,5 @@
1
+ import argparse
2
+ import shutil
1
3
  import sys
2
4
  from pathlib import Path
3
5
 
@@ -18,22 +20,58 @@ from datasecops_cli.utilities.display import (
18
20
  )
19
21
 
20
22
 
23
+ DOWNLOAD_ITEMS = [
24
+ "sqlfluff", "pipelines", "packages", "macros",
25
+ "install-sqlfluff", "install-dbt", "all",
26
+ ]
27
+
28
+
29
+ def _build_parser() -> argparse.ArgumentParser:
30
+ parser = argparse.ArgumentParser(
31
+ prog="datasecops",
32
+ description="DataSecOps Framework CLI for Snowflake Native App",
33
+ )
34
+ sub = parser.add_subparsers(dest="command")
35
+
36
+ sub.add_parser("bootstrap", help="Set up a new dbt project with all framework config")
37
+
38
+ dl = sub.add_parser("download", help="Download framework config non-interactively (for CI/CD)")
39
+ dl.add_argument(
40
+ "items",
41
+ nargs="+",
42
+ choices=DOWNLOAD_ITEMS,
43
+ help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
44
+ )
45
+
46
+ return parser
47
+
48
+
21
49
  def main():
22
50
  """Main entry point for the datasecops CLI."""
51
+ parser = _build_parser()
52
+ args = parser.parse_args()
53
+
23
54
  config = Config()
24
55
 
25
- # Handle 'bootstrap' subcommand
26
- if len(sys.argv) > 1 and sys.argv[1] == "bootstrap":
56
+ if args.command == "bootstrap":
27
57
  _run_bootstrap(config)
28
- return
58
+ elif args.command == "download":
59
+ _run_download(config, args.items)
60
+ else:
61
+ _run_interactive(config)
29
62
 
63
+
64
+ def _connect_and_load(config: Config) -> SnowflakeService:
65
+ """Load config, connect to Snowflake, and load native app settings.
66
+
67
+ Returns the connected SnowflakeService, or calls sys.exit on failure.
68
+ """
30
69
  if not config.load():
31
70
  sys.exit(1)
32
-
33
- # Connect to Snowflake
71
+
34
72
  sf_config = config.datasecops
35
73
  sf_service = SnowflakeService(sf_config)
36
-
74
+
37
75
  try:
38
76
  info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
39
77
  sf_service.connect()
@@ -41,38 +79,101 @@ def main():
41
79
  except Exception as e:
42
80
  error_line(f"Failed to connect to Snowflake: {e}")
43
81
  sys.exit(1)
44
-
82
+
83
+ info_line("Loading framework configuration...")
84
+ config.load_from_native_app(sf_service)
85
+
86
+ if not config.profile:
87
+ error_line(f"Profile '{config.profile_name}' not found in native app")
88
+ sf_service.close()
89
+ sys.exit(1)
90
+
91
+ success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
92
+ return sf_service
93
+
94
+
95
+ def _run_interactive(config: Config):
96
+ """Run the interactive menu-driven CLI."""
97
+ sf_service = _connect_and_load(config)
98
+
45
99
  try:
46
- # Load config from native app
47
- info_line("Loading framework configuration...")
48
- config.load_from_native_app(sf_service)
49
-
50
- if not config.profile:
51
- error_line(f"Profile '{config.profile_name}' not found in native app")
52
- sys.exit(1)
53
-
54
- success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
55
-
100
+ # Check for dbt Fusion (dbtf) on PATH
101
+ if not shutil.which("dbtf"):
102
+ from datasecops_cli.utilities.display import warning_line
103
+ warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
104
+ warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
105
+
56
106
  # Initialize services
57
107
  dbt_runner = DbtRunner(
58
108
  project_dir=config.dbt_project_dir,
59
109
  profiles_dir=config.get_dbt_profiles_dir(),
60
110
  target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev"
61
111
  )
62
-
112
+
63
113
  try:
64
114
  git_service = GitService(config.project_dir)
65
115
  except Exception:
66
116
  git_service = None
67
-
117
+
68
118
  linting_service = LintingService(config.dbt_project_dir)
69
119
  download_service = DownloadService(sf_service, config.project_dir)
70
120
  skill_service = SkillService(sf_service)
71
-
121
+
72
122
  # Main menu loop
73
- _main_menu(config, dbt_runner, git_service, linting_service,
123
+ _main_menu(config, dbt_runner, git_service, linting_service,
74
124
  download_service, skill_service, sf_service)
75
-
125
+ finally:
126
+ sf_service.close()
127
+
128
+
129
+ def _run_download(config: Config, items: list[str]):
130
+ """Run non-interactive downloads for CI/CD pipelines."""
131
+ sf_service = _connect_and_load(config)
132
+
133
+ try:
134
+ download_service = DownloadService(sf_service, config.project_dir)
135
+ linting_service = LintingService(config.dbt_project_dir)
136
+ profiles_dir = str(config.get_dbt_profiles_dir())
137
+
138
+ if "all" in items:
139
+ items = ["sqlfluff", "pipelines", "packages", "macros",
140
+ "install-sqlfluff", "install-dbt"]
141
+
142
+ failed = False
143
+ for item in items:
144
+ info_line("")
145
+ if item == "sqlfluff":
146
+ if not download_service.download_sqlfluff_config(
147
+ profiles_dir=profiles_dir, dbt_project_dir=config.dbt_project_dir
148
+ ):
149
+ failed = True
150
+ elif item == "pipelines":
151
+ platform = config.source_control.source_control_platform.lower()
152
+ info_line(f"Platform: {platform}")
153
+ if not download_service.download_pipelines(platform=platform):
154
+ failed = True
155
+ elif item == "packages":
156
+ if not download_service.download_dbt_packages(config.dbt_project_dir):
157
+ failed = True
158
+ elif item == "macros":
159
+ if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
160
+ failed = True
161
+ elif item == "install-sqlfluff":
162
+ packages = download_service.get_sqlfluff_requirements()
163
+ if packages:
164
+ if not linting_service.install_requirements(packages):
165
+ failed = True
166
+ else:
167
+ failed = True
168
+ elif item == "install-dbt":
169
+ packages = download_service.get_dbt_requirements()
170
+ if packages:
171
+ if not linting_service.install_requirements(packages):
172
+ failed = True
173
+ else:
174
+ failed = True
175
+
176
+ sys.exit(1 if failed else 0)
76
177
  finally:
77
178
  sf_service.close()
78
179
 
@@ -110,12 +211,14 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
110
211
  profile_name, config.dbt_project_dir,
111
212
  project_settings=config.project_settings,
112
213
  profile=config.profile,
214
+ source_control=config.source_control,
113
215
  )
114
216
  dl_menu.show()
115
217
 
116
218
  elif option == 4:
117
- from datasecops_cli.utilities.display import select_from_list, get_input_true_false
118
- platform = select_from_list(["github", "azuredevops"], "source control platform", add_back=False)
219
+ from datasecops_cli.utilities.display import get_input_true_false
220
+ platform = config.source_control.source_control_platform.lower()
221
+ info_line(f"Platform: {platform}")
119
222
  install_skills = get_input_true_false("Install Cortex Code skills?")
120
223
  run_deps = get_input_true_false("Run dbt deps after downloading packages?")
121
224
  bootstrap = BootstrapService(
@@ -145,35 +248,16 @@ def _show_main_menu(profile_name: str, git_service: GitService = None):
145
248
 
146
249
  def _run_bootstrap(config: Config):
147
250
  """Run the bootstrap command to initialise a new project."""
148
- from datasecops_cli.utilities.display import select_from_list, get_input_true_false
251
+ from datasecops_cli.utilities.display import get_input_true_false
149
252
 
150
- if not config.load():
151
- sys.exit(1)
152
-
153
- sf_config = config.datasecops
154
- sf_service = SnowflakeService(sf_config)
253
+ sf_service = _connect_and_load(config)
155
254
 
156
255
  try:
157
- info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
158
- sf_service.connect()
159
- success_line("Connected")
160
- except Exception as e:
161
- error_line(f"Failed to connect to Snowflake: {e}")
162
- sys.exit(1)
163
-
164
- try:
165
- info_line("Loading framework configuration...")
166
- config.load_from_native_app(sf_service)
167
-
168
- if not config.profile:
169
- error_line(f"Profile '{config.profile_name}' not found in native app")
170
- sys.exit(1)
171
-
172
- success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
173
256
  info_line("")
174
257
 
175
- # Ask for platform
176
- platform = select_from_list(["github", "azuredevops"], "source control platform", add_back=False)
258
+ # Use platform from framework config
259
+ platform = config.source_control.source_control_platform.lower()
260
+ info_line(f"Platform: {platform}")
177
261
 
178
262
  # Ask about optional steps
179
263
  install_skills = get_input_true_false("Install Cortex Code skills?")
@@ -96,7 +96,7 @@ class DevelopmentMenu:
96
96
  menu_option(11, "clean - Clean dbt target")
97
97
  menu_option(12, "debug - Debug dbt connection")
98
98
  menu_option(13, "list - List dbt resources")
99
- menu_option(14, "install dbt - Install dbt-core & dbt-snowflake from framework")
99
+ menu_option(14, "install lint - Install dbt-core & dbt-snowflake for linting (not dbt Fusion)")
100
100
  menu_option(0, "back - Return to main menu")
101
101
 
102
102
  def _run_menu(self) -> None:
@@ -1,6 +1,6 @@
1
1
  from pathlib import Path
2
2
 
3
- from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings
3
+ from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings, SourceControl
4
4
  from datasecops_cli.services.download_service import DownloadService
5
5
  from datasecops_cli.services.skill_service import SkillService
6
6
  from datasecops_cli.services.dbt_runner import DbtRunner
@@ -14,7 +14,8 @@ from datasecops_cli.utilities.display import (
14
14
  class DownloadsMenu:
15
15
  def __init__(self, download_service: DownloadService, skill_service: SkillService,
16
16
  dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
17
- project_settings: ProjectSettings = None, profile: ProjectProfile = None):
17
+ project_settings: ProjectSettings = None, profile: ProjectProfile = None,
18
+ source_control: SourceControl = None):
18
19
  self.downloads = download_service
19
20
  self.skills = skill_service
20
21
  self.dbt = dbt_runner
@@ -22,6 +23,7 @@ class DownloadsMenu:
22
23
  self.dbt_project_dir = dbt_project_dir
23
24
  self.project_settings = project_settings
24
25
  self.profile = profile
26
+ self.source_control = source_control
25
27
 
26
28
  def show(self) -> None:
27
29
  self._menu()
@@ -34,7 +36,8 @@ class DownloadsMenu:
34
36
  complete_action()
35
37
  elif option == 2:
36
38
  display_action_header("Download Pipeline Files")
37
- platform = select_from_list(["github", "azuredevops"], "platform", add_back=False)
39
+ platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
40
+ info_line(f"Platform: {platform}")
38
41
  self.downloads.download_pipelines(platform=platform)
39
42
  complete_action()
40
43
  elif option == 3:
@@ -138,7 +138,7 @@ class BootstrapService:
138
138
  info_line("Your project is ready. Next steps:")
139
139
  info_line(f" cd {dbt_project_dir}")
140
140
  info_line(" datasecops — run the framework CLI")
141
- info_line(" dbt debug — verify Snowflake connection")
142
- info_line(" dbt run — run your first models")
141
+ info_line(" dbtf debug — verify Snowflake connection")
142
+ info_line(" dbtf run — run your first models")
143
143
  info_line("")
144
144
  return True
@@ -21,7 +21,7 @@ class DbtRunner:
21
21
  ]
22
22
 
23
23
  def _run_command(self, command: str, extra_args: list[str] = None) -> subprocess.CompletedProcess:
24
- cmd = ["dbt", command] + (extra_args or []) + self._default_args()
24
+ cmd = ["dbtf", command] + (extra_args or []) + self._default_args()
25
25
  info_line(f"Running: {' '.join(cmd)}")
26
26
  result = subprocess.run(cmd, capture_output=False)
27
27
  if result.returncode != 0:
@@ -94,7 +94,7 @@ class DbtRunner:
94
94
  return self._run_command("docs", ["generate", f"--target={self.target}"])
95
95
 
96
96
  def docs_serve(self) -> subprocess.Popen:
97
- cmd = ["dbt", "docs", "serve"] + self._default_args()
97
+ cmd = ["dbtf", "docs", "serve"] + self._default_args()
98
98
  info_line(f"Running: {' '.join(cmd)}")
99
99
  return subprocess.Popen(cmd)
100
100
 
@@ -159,7 +159,12 @@ class DownloadService:
159
159
  section_name = self.RULE_SECTION_MAP.get(code)
160
160
  if not section_name:
161
161
  continue
162
- opts = entry.get("options", {})
162
+ opts = dict(entry.get("options", {}))
163
+ # For aliasing.length, 0 means "no limit" which sqlfluff expects as None
164
+ if section_name == "aliasing.length":
165
+ for len_key in ("min_alias_length", "max_alias_length"):
166
+ if len_key in opts and opts[len_key] == 0:
167
+ opts[len_key] = None
163
168
  self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
164
169
 
165
170
  content = "\n".join(lines)
@@ -225,12 +230,12 @@ class DownloadService:
225
230
  pipelines = raw.get("pipelines", [])
226
231
  count = 0
227
232
  for pipe in pipelines:
228
- if pipe.get("platform", "github") != platform or not pipe.get("enabled", True):
233
+ if pipe.get("platform", "github").lower() != platform.lower() or not pipe.get("enabled", True):
229
234
  continue
230
235
  filename = pipe.get("filename", "")
231
236
  yaml_content = pipe.get("yaml_content", "")
232
237
  if filename and yaml_content:
233
- if platform == "github":
238
+ if platform.lower() == "github":
234
239
  dest = self.project_dir / ".github" / "workflows" / filename
235
240
  else:
236
241
  dest = self.project_dir / filename
@@ -0,0 +1,302 @@
1
+ """Tests for datasecops_cli.main argument parsing and download dispatch."""
2
+ import pytest
3
+ from unittest.mock import MagicMock, patch, call
4
+ from pathlib import Path
5
+
6
+ from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
7
+ from datasecops_cli.config import Config
8
+
9
+
10
+ class TestBuildParser:
11
+ """Tests for the argparse configuration."""
12
+
13
+ def test_no_args_gives_no_command(self):
14
+ parser = _build_parser()
15
+ args = parser.parse_args([])
16
+ assert args.command is None
17
+
18
+ def test_bootstrap_command(self):
19
+ parser = _build_parser()
20
+ args = parser.parse_args(["bootstrap"])
21
+ assert args.command == "bootstrap"
22
+
23
+ def test_download_single_item(self):
24
+ parser = _build_parser()
25
+ args = parser.parse_args(["download", "sqlfluff"])
26
+ assert args.command == "download"
27
+ assert args.items == ["sqlfluff"]
28
+
29
+ def test_download_multiple_items(self):
30
+ parser = _build_parser()
31
+ args = parser.parse_args(["download", "sqlfluff", "packages", "macros"])
32
+ assert args.command == "download"
33
+ assert args.items == ["sqlfluff", "packages", "macros"]
34
+
35
+ def test_download_all(self):
36
+ parser = _build_parser()
37
+ args = parser.parse_args(["download", "all"])
38
+ assert args.command == "download"
39
+ assert args.items == ["all"]
40
+
41
+ def test_download_invalid_item_exits(self):
42
+ parser = _build_parser()
43
+ with pytest.raises(SystemExit):
44
+ parser.parse_args(["download", "invalid"])
45
+
46
+ def test_download_no_items_exits(self):
47
+ parser = _build_parser()
48
+ with pytest.raises(SystemExit):
49
+ parser.parse_args(["download"])
50
+
51
+ def test_all_download_items_accepted(self):
52
+ parser = _build_parser()
53
+ for item in DOWNLOAD_ITEMS:
54
+ args = parser.parse_args(["download", item])
55
+ assert args.items == [item]
56
+
57
+
58
+ class TestRunDownload:
59
+ """Tests for _run_download dispatch logic using mocked services."""
60
+
61
+ def _make_config(self, tmp_path):
62
+ """Create a Config with minimal valid state."""
63
+ config = Config()
64
+ config.project_dir = tmp_path
65
+ config.dbt_project_dir = tmp_path
66
+ config.profile_name = "test_profile"
67
+ # source_control defaults to GitHub
68
+ return config
69
+
70
+ @patch("datasecops_cli.main._connect_and_load")
71
+ def test_download_sqlfluff(self, mock_connect, tmp_path):
72
+ config = self._make_config(tmp_path)
73
+ mock_sf = MagicMock()
74
+ mock_connect.return_value = mock_sf
75
+
76
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
77
+ patch("datasecops_cli.main.LintingService"):
78
+ mock_ds = MockDS.return_value
79
+ mock_ds.download_sqlfluff_config.return_value = True
80
+
81
+ with pytest.raises(SystemExit) as exc:
82
+ _run_download(config, ["sqlfluff"])
83
+
84
+ assert exc.value.code == 0
85
+ mock_ds.download_sqlfluff_config.assert_called_once()
86
+ mock_ds.download_pipelines.assert_not_called()
87
+ mock_ds.download_dbt_packages.assert_not_called()
88
+ mock_ds.download_macros.assert_not_called()
89
+
90
+ @patch("datasecops_cli.main._connect_and_load")
91
+ def test_download_pipelines(self, mock_connect, tmp_path):
92
+ config = self._make_config(tmp_path)
93
+ mock_sf = MagicMock()
94
+ mock_connect.return_value = mock_sf
95
+
96
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
97
+ patch("datasecops_cli.main.LintingService"):
98
+ mock_ds = MockDS.return_value
99
+ mock_ds.download_pipelines.return_value = True
100
+
101
+ with pytest.raises(SystemExit) as exc:
102
+ _run_download(config, ["pipelines"])
103
+
104
+ assert exc.value.code == 0
105
+ mock_ds.download_pipelines.assert_called_once_with(platform="github")
106
+
107
+ @patch("datasecops_cli.main._connect_and_load")
108
+ def test_download_packages(self, mock_connect, tmp_path):
109
+ config = self._make_config(tmp_path)
110
+ mock_sf = MagicMock()
111
+ mock_connect.return_value = mock_sf
112
+
113
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
114
+ patch("datasecops_cli.main.LintingService"):
115
+ mock_ds = MockDS.return_value
116
+ mock_ds.download_dbt_packages.return_value = True
117
+
118
+ with pytest.raises(SystemExit) as exc:
119
+ _run_download(config, ["packages"])
120
+
121
+ assert exc.value.code == 0
122
+ mock_ds.download_dbt_packages.assert_called_once_with(tmp_path)
123
+
124
+ @patch("datasecops_cli.main._connect_and_load")
125
+ def test_download_macros(self, mock_connect, tmp_path):
126
+ config = self._make_config(tmp_path)
127
+ mock_sf = MagicMock()
128
+ mock_connect.return_value = mock_sf
129
+
130
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
131
+ patch("datasecops_cli.main.LintingService"):
132
+ mock_ds = MockDS.return_value
133
+ mock_ds.download_macros.return_value = True
134
+
135
+ with pytest.raises(SystemExit) as exc:
136
+ _run_download(config, ["macros"])
137
+
138
+ assert exc.value.code == 0
139
+ mock_ds.download_macros.assert_called_once_with("test_profile", tmp_path)
140
+
141
+ @patch("datasecops_cli.main._connect_and_load")
142
+ def test_download_all_expands_to_all_items(self, mock_connect, tmp_path):
143
+ config = self._make_config(tmp_path)
144
+ mock_sf = MagicMock()
145
+ mock_connect.return_value = mock_sf
146
+
147
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
148
+ patch("datasecops_cli.main.LintingService") as MockLS:
149
+ mock_ds = MockDS.return_value
150
+ mock_ls = MockLS.return_value
151
+ mock_ds.download_sqlfluff_config.return_value = True
152
+ mock_ds.download_pipelines.return_value = True
153
+ mock_ds.download_dbt_packages.return_value = True
154
+ mock_ds.download_macros.return_value = True
155
+ mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0"]
156
+ mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
157
+ mock_ls.install_requirements.return_value = True
158
+
159
+ with pytest.raises(SystemExit) as exc:
160
+ _run_download(config, ["all"])
161
+
162
+ assert exc.value.code == 0
163
+ mock_ds.download_sqlfluff_config.assert_called_once()
164
+ mock_ds.download_pipelines.assert_called_once()
165
+ mock_ds.download_dbt_packages.assert_called_once()
166
+ mock_ds.download_macros.assert_called_once()
167
+ mock_ds.get_sqlfluff_requirements.assert_called_once()
168
+ mock_ds.get_dbt_requirements.assert_called_once()
169
+ assert mock_ls.install_requirements.call_count == 2
170
+
171
+ @patch("datasecops_cli.main._connect_and_load")
172
+ def test_download_failure_exits_1(self, mock_connect, tmp_path):
173
+ config = self._make_config(tmp_path)
174
+ mock_sf = MagicMock()
175
+ mock_connect.return_value = mock_sf
176
+
177
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
178
+ patch("datasecops_cli.main.LintingService"):
179
+ mock_ds = MockDS.return_value
180
+ mock_ds.download_sqlfluff_config.return_value = False # failure
181
+
182
+ with pytest.raises(SystemExit) as exc:
183
+ _run_download(config, ["sqlfluff"])
184
+
185
+ assert exc.value.code == 1
186
+
187
+ @patch("datasecops_cli.main._connect_and_load")
188
+ def test_download_partial_failure_exits_1(self, mock_connect, tmp_path):
189
+ """If one item fails but another succeeds, exit code is still 1."""
190
+ config = self._make_config(tmp_path)
191
+ mock_sf = MagicMock()
192
+ mock_connect.return_value = mock_sf
193
+
194
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
195
+ patch("datasecops_cli.main.LintingService"):
196
+ mock_ds = MockDS.return_value
197
+ mock_ds.download_sqlfluff_config.return_value = True
198
+ mock_ds.download_dbt_packages.return_value = False
199
+
200
+ with pytest.raises(SystemExit) as exc:
201
+ _run_download(config, ["sqlfluff", "packages"])
202
+
203
+ assert exc.value.code == 1
204
+
205
+ @patch("datasecops_cli.main._connect_and_load")
206
+ def test_download_pipelines_uses_configured_platform(self, mock_connect, tmp_path):
207
+ """Pipeline download reads platform from source_control config."""
208
+ config = self._make_config(tmp_path)
209
+ config.source_control.source_control_platform = "AzureDevOps"
210
+ mock_sf = MagicMock()
211
+ mock_connect.return_value = mock_sf
212
+
213
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
214
+ patch("datasecops_cli.main.LintingService"):
215
+ mock_ds = MockDS.return_value
216
+ mock_ds.download_pipelines.return_value = True
217
+
218
+ with pytest.raises(SystemExit) as exc:
219
+ _run_download(config, ["pipelines"])
220
+
221
+ assert exc.value.code == 0
222
+ mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
223
+
224
+ @patch("datasecops_cli.main._connect_and_load")
225
+ def test_install_sqlfluff(self, mock_connect, tmp_path):
226
+ """install-sqlfluff fetches pinned versions and installs them."""
227
+ config = self._make_config(tmp_path)
228
+ mock_connect.return_value = MagicMock()
229
+
230
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
231
+ patch("datasecops_cli.main.LintingService") as MockLS:
232
+ mock_ds = MockDS.return_value
233
+ mock_ls = MockLS.return_value
234
+ mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
235
+ mock_ls.install_requirements.return_value = True
236
+
237
+ with pytest.raises(SystemExit) as exc:
238
+ _run_download(config, ["install-sqlfluff"])
239
+
240
+ assert exc.value.code == 0
241
+ mock_ds.get_sqlfluff_requirements.assert_called_once()
242
+ mock_ls.install_requirements.assert_called_once_with(
243
+ ["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
244
+ )
245
+
246
+ @patch("datasecops_cli.main._connect_and_load")
247
+ def test_install_dbt(self, mock_connect, tmp_path):
248
+ """install-dbt fetches pinned versions and installs them."""
249
+ config = self._make_config(tmp_path)
250
+ mock_connect.return_value = MagicMock()
251
+
252
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
253
+ patch("datasecops_cli.main.LintingService") as MockLS:
254
+ mock_ds = MockDS.return_value
255
+ mock_ls = MockLS.return_value
256
+ mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
257
+ mock_ls.install_requirements.return_value = True
258
+
259
+ with pytest.raises(SystemExit) as exc:
260
+ _run_download(config, ["install-dbt"])
261
+
262
+ assert exc.value.code == 0
263
+ mock_ds.get_dbt_requirements.assert_called_once()
264
+ mock_ls.install_requirements.assert_called_once_with(
265
+ ["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
266
+ )
267
+
268
+ @patch("datasecops_cli.main._connect_and_load")
269
+ def test_install_sqlfluff_no_versions_exits_1(self, mock_connect, tmp_path):
270
+ """install-sqlfluff fails if no versions returned from framework."""
271
+ config = self._make_config(tmp_path)
272
+ mock_connect.return_value = MagicMock()
273
+
274
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
275
+ patch("datasecops_cli.main.LintingService") as MockLS:
276
+ mock_ds = MockDS.return_value
277
+ mock_ls = MockLS.return_value
278
+ mock_ds.get_sqlfluff_requirements.return_value = []
279
+
280
+ with pytest.raises(SystemExit) as exc:
281
+ _run_download(config, ["install-sqlfluff"])
282
+
283
+ assert exc.value.code == 1
284
+ mock_ls.install_requirements.assert_not_called()
285
+
286
+ @patch("datasecops_cli.main._connect_and_load")
287
+ def test_install_dbt_pip_failure_exits_1(self, mock_connect, tmp_path):
288
+ """install-dbt exits 1 if uv pip install fails."""
289
+ config = self._make_config(tmp_path)
290
+ mock_connect.return_value = MagicMock()
291
+
292
+ with patch("datasecops_cli.main.DownloadService") as MockDS, \
293
+ patch("datasecops_cli.main.LintingService") as MockLS:
294
+ mock_ds = MockDS.return_value
295
+ mock_ls = MockLS.return_value
296
+ mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
297
+ mock_ls.install_requirements.return_value = False # pip failure
298
+
299
+ with pytest.raises(SystemExit) as exc:
300
+ _run_download(config, ["install-dbt"])
301
+
302
+ assert exc.value.code == 1
File without changes