datasecops-cli 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/CHANGELOG.md +21 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/PKG-INFO +61 -8
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/README.md +60 -7
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/getting-started.md +1 -1
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/pyproject.toml +1 -1
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/setup.ps1 +6 -5
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/setup.sh +4 -4
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/main.py +132 -48
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/development.py +1 -1
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/downloads.py +6 -3
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/bootstrap_service.py +2 -2
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/dbt_runner.py +2 -2
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/download_service.py +8 -3
- datasecops_cli-0.2.9/tests/test_main.py +302 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/.gitignore +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/DEVELOPMENT.md +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/LICENSE +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/legacy.md +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/mcp-servers.json +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/config.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/models/project_config.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/skill_service.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/snowflake_service.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/display.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/file_utils.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/connection.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_mcp/server.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/__init__.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_config.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_models.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_version.py +0 -0
- {datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/tests/test_yaml_utils.py +0 -0
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the DataSecOps CLI are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.2.9] - 2026-05-14
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **`install-sqlfluff` and `install-dbt` download items** — `datasecops download install-sqlfluff` and `datasecops download install-dbt` fetch framework-pinned package versions from the native app and install them via `uv pip install`. `datasecops download all` now includes both installs.
|
|
10
|
+
|
|
11
|
+
## [0.2.8] - 2026-05-14
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- **Non-interactive `download` subcommand** — `datasecops download <items>` downloads framework config without prompts, designed for CI/CD pipelines. Supports `sqlfluff`, `pipelines`, `packages`, `macros`, or `all`
|
|
16
|
+
- **Runtime `dbtf` check** — the CLI now warns at startup if dbt Fusion (`dbtf`) is not found on PATH
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- **Platform auto-detected from framework config** — pipeline downloads, bootstrap, and the downloads menu now read `source_control_platform` from the native app instead of prompting the user to select GitHub or Azure DevOps
|
|
21
|
+
- **Case-insensitive platform matching** — `download_pipelines()` now compares platform values case-insensitively, fixing a bug where pipelines stored as `"GitHub"` would not match the lowercase `"github"` filter
|
|
22
|
+
- **All dbt commands use `dbtf`** — `DbtRunner` and setup scripts now consistently use the `dbtf` binary (dbt Fusion) instead of `dbt`
|
|
23
|
+
- **Refactored CLI entry point** — `main()` now uses `argparse` with subcommands (`bootstrap`, `download`) instead of manual `sys.argv` parsing. Shared connection/config logic extracted to `_connect_and_load()`
|
|
24
|
+
- **Development menu option 14** relabelled to clarify it installs dbt-core/dbt-snowflake for linting, not dbt Fusion
|
|
25
|
+
|
|
5
26
|
## [0.2.7] - 2026-05-12
|
|
6
27
|
|
|
7
28
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datasecops-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: DataSecOps Framework CLI for Snowflake Native App
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -55,7 +55,7 @@ Requires Python 3.10 or later.
|
|
|
55
55
|
|
|
56
56
|
Optional:
|
|
57
57
|
|
|
58
|
-
- **dbt Fusion**
|
|
58
|
+
- **dbt Fusion** — required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation
|
|
59
59
|
- **Cortex Code** for skill downloads
|
|
60
60
|
- **Node.js 18+** for GitHub/Azure DevOps MCP servers
|
|
61
61
|
|
|
@@ -96,6 +96,65 @@ datasecops
|
|
|
96
96
|
| **Git** | Branch create/checkout/delete, commit & push, rebase, squash, deploy to environment branches, cherry-pick |
|
|
97
97
|
| **Downloads** | SQLFluff config, CI/CD pipelines (GitHub Actions / Azure DevOps), dbt packages, Cortex Code skills |
|
|
98
98
|
|
|
99
|
+
## Non-Interactive Mode (CI/CD)
|
|
100
|
+
|
|
101
|
+
The `download` subcommand lets you pull framework config in CI/CD pipelines without interactive prompts:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
# Download specific items
|
|
105
|
+
datasecops download sqlfluff
|
|
106
|
+
datasecops download sqlfluff packages
|
|
107
|
+
datasecops download pipelines macros
|
|
108
|
+
|
|
109
|
+
# Install framework-pinned package versions
|
|
110
|
+
datasecops download install-sqlfluff
|
|
111
|
+
datasecops download install-dbt
|
|
112
|
+
|
|
113
|
+
# Download config and install packages together
|
|
114
|
+
datasecops download sqlfluff install-sqlfluff
|
|
115
|
+
|
|
116
|
+
# Download and install everything
|
|
117
|
+
datasecops download all
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
|
|
121
|
+
|
|
122
|
+
The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
|
|
123
|
+
|
|
124
|
+
### Pipeline Setup
|
|
125
|
+
|
|
126
|
+
Your pipeline needs two things:
|
|
127
|
+
|
|
128
|
+
1. **A `.datasecops.yml`** in the repo (already committed — contains no secrets):
|
|
129
|
+
|
|
130
|
+
```yaml
|
|
131
|
+
connection_name: "ci"
|
|
132
|
+
app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
|
|
136
|
+
|
|
137
|
+
```yaml
|
|
138
|
+
# GitHub Actions example
|
|
139
|
+
- name: Configure Snowflake connection
|
|
140
|
+
run: |
|
|
141
|
+
mkdir -p ~/.snowflake
|
|
142
|
+
cat > ~/.snowflake/connections.toml << EOF
|
|
143
|
+
[ci]
|
|
144
|
+
account = "${{ vars.SNOWFLAKE_ACCOUNT }}"
|
|
145
|
+
user = "${{ vars.SNOWFLAKE_USER }}"
|
|
146
|
+
authenticator = "snowflake_jwt"
|
|
147
|
+
private_key_file = "/tmp/rsa_key.p8"
|
|
148
|
+
warehouse = "CI_WH"
|
|
149
|
+
role = "CI_ROLE"
|
|
150
|
+
EOF
|
|
151
|
+
|
|
152
|
+
- name: Download SQLFluff config
|
|
153
|
+
run: datasecops download sqlfluff
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The exit code is `0` on success, `1` if any download fails.
|
|
157
|
+
|
|
99
158
|
## MCP Server
|
|
100
159
|
|
|
101
160
|
The package includes an MCP (Model Context Protocol) server that exposes your framework's governance configuration to AI coding assistants. Instead of static skill files, the MCP server gives AI tools live access to your native app's current rules.
|
|
@@ -154,12 +213,6 @@ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
|
|
|
154
213
|
|
|
155
214
|
Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
|
|
156
215
|
|
|
157
|
-
## Documentation
|
|
158
|
-
|
|
159
|
-
- [Getting Started Guide](docs/getting-started.md) — install CLI, configure MCP servers for VS Code/Cursor/Cortex Code with Snowflake, dbt, and GitHub/Azure DevOps
|
|
160
|
-
- [MCP Server Reference](docs/mcp-server.md) — full tool documentation, architecture, and usage examples
|
|
161
|
-
- [Development Guide](DEVELOPMENT.md) — project structure, setup scripts, native app API reference, and publishing details
|
|
162
|
-
|
|
163
216
|
## License
|
|
164
217
|
|
|
165
218
|
MIT
|
|
@@ -35,7 +35,7 @@ Requires Python 3.10 or later.
|
|
|
35
35
|
|
|
36
36
|
Optional:
|
|
37
37
|
|
|
38
|
-
- **dbt Fusion**
|
|
38
|
+
- **dbt Fusion** — required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation
|
|
39
39
|
- **Cortex Code** for skill downloads
|
|
40
40
|
- **Node.js 18+** for GitHub/Azure DevOps MCP servers
|
|
41
41
|
|
|
@@ -76,6 +76,65 @@ datasecops
|
|
|
76
76
|
| **Git** | Branch create/checkout/delete, commit & push, rebase, squash, deploy to environment branches, cherry-pick |
|
|
77
77
|
| **Downloads** | SQLFluff config, CI/CD pipelines (GitHub Actions / Azure DevOps), dbt packages, Cortex Code skills |
|
|
78
78
|
|
|
79
|
+
## Non-Interactive Mode (CI/CD)
|
|
80
|
+
|
|
81
|
+
The `download` subcommand lets you pull framework config in CI/CD pipelines without interactive prompts:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
# Download specific items
|
|
85
|
+
datasecops download sqlfluff
|
|
86
|
+
datasecops download sqlfluff packages
|
|
87
|
+
datasecops download pipelines macros
|
|
88
|
+
|
|
89
|
+
# Install framework-pinned package versions
|
|
90
|
+
datasecops download install-sqlfluff
|
|
91
|
+
datasecops download install-dbt
|
|
92
|
+
|
|
93
|
+
# Download config and install packages together
|
|
94
|
+
datasecops download sqlfluff install-sqlfluff
|
|
95
|
+
|
|
96
|
+
# Download and install everything
|
|
97
|
+
datasecops download all
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
|
|
101
|
+
|
|
102
|
+
The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
|
|
103
|
+
|
|
104
|
+
### Pipeline Setup
|
|
105
|
+
|
|
106
|
+
Your pipeline needs two things:
|
|
107
|
+
|
|
108
|
+
1. **A `.datasecops.yml`** in the repo (already committed — contains no secrets):
|
|
109
|
+
|
|
110
|
+
```yaml
|
|
111
|
+
connection_name: "ci"
|
|
112
|
+
app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
# GitHub Actions example
|
|
119
|
+
- name: Configure Snowflake connection
|
|
120
|
+
run: |
|
|
121
|
+
mkdir -p ~/.snowflake
|
|
122
|
+
cat > ~/.snowflake/connections.toml << EOF
|
|
123
|
+
[ci]
|
|
124
|
+
account = "${{ vars.SNOWFLAKE_ACCOUNT }}"
|
|
125
|
+
user = "${{ vars.SNOWFLAKE_USER }}"
|
|
126
|
+
authenticator = "snowflake_jwt"
|
|
127
|
+
private_key_file = "/tmp/rsa_key.p8"
|
|
128
|
+
warehouse = "CI_WH"
|
|
129
|
+
role = "CI_ROLE"
|
|
130
|
+
EOF
|
|
131
|
+
|
|
132
|
+
- name: Download SQLFluff config
|
|
133
|
+
run: datasecops download sqlfluff
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
The exit code is `0` on success, `1` if any download fails.
|
|
137
|
+
|
|
79
138
|
## MCP Server
|
|
80
139
|
|
|
81
140
|
The package includes an MCP (Model Context Protocol) server that exposes your framework's governance configuration to AI coding assistants. Instead of static skill files, the MCP server gives AI tools live access to your native app's current rules.
|
|
@@ -134,12 +193,6 @@ app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
|
|
|
134
193
|
|
|
135
194
|
Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
|
|
136
195
|
|
|
137
|
-
## Documentation
|
|
138
|
-
|
|
139
|
-
- [Getting Started Guide](docs/getting-started.md) — install CLI, configure MCP servers for VS Code/Cursor/Cortex Code with Snowflake, dbt, and GitHub/Azure DevOps
|
|
140
|
-
- [MCP Server Reference](docs/mcp-server.md) — full tool documentation, architecture, and usage examples
|
|
141
|
-
- [Development Guide](DEVELOPMENT.md) — project structure, setup scripts, native app API reference, and publishing details
|
|
142
|
-
|
|
143
196
|
## License
|
|
144
197
|
|
|
145
198
|
MIT
|
|
@@ -14,7 +14,7 @@ Before you begin, ensure you have:
|
|
|
14
14
|
- **Git** configured with access to your repository
|
|
15
15
|
|
|
16
16
|
Optional:
|
|
17
|
-
- **dbt Fusion**
|
|
17
|
+
- **dbt Fusion** — required for dbt commands (`dbtf`). Install from https://docs.getdbt.com/docs/core/installation. The CLI invokes `dbtf` for all dbt operations.
|
|
18
18
|
- **A GitHub Personal Access Token** (for GitHub MCP server)
|
|
19
19
|
- **An Azure DevOps PAT** (for Azure DevOps MCP server)
|
|
20
20
|
|
|
@@ -16,13 +16,14 @@ try {
|
|
|
16
16
|
$env:Path = "$env:USERPROFILE\.local\bin;$env:Path"
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
# Check for dbt
|
|
19
|
+
# Check for dbt Fusion (dbtf)
|
|
20
20
|
try {
|
|
21
|
-
$null = Get-Command
|
|
22
|
-
Write-Host "[OK] dbt found" -ForegroundColor Green
|
|
21
|
+
$null = Get-Command dbtf -ErrorAction Stop
|
|
22
|
+
Write-Host "[OK] dbt Fusion (dbtf) found" -ForegroundColor Green
|
|
23
23
|
} catch {
|
|
24
|
-
Write-Host "WARNING: dbt not found on PATH." -ForegroundColor Yellow
|
|
25
|
-
Write-Host " Install dbt Fusion
|
|
24
|
+
Write-Host "WARNING: dbt Fusion (dbtf) not found on PATH." -ForegroundColor Yellow
|
|
25
|
+
Write-Host " Install dbt Fusion: https://docs.getdbt.com/docs/core/installation" -ForegroundColor Yellow
|
|
26
|
+
Write-Host " The CLI uses 'dbtf' for all dbt commands." -ForegroundColor Yellow
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
# Check for cortex (optional)
|
|
@@ -16,12 +16,12 @@ if ! command -v uv &> /dev/null; then
|
|
|
16
16
|
fi
|
|
17
17
|
echo "[OK] uv $(uv --version 2>/dev/null || echo 'installed')"
|
|
18
18
|
|
|
19
|
-
# Check for dbt
|
|
20
|
-
if ! command -v
|
|
19
|
+
# Check for dbt Fusion (dbtf)
|
|
20
|
+
if ! command -v dbtf &> /dev/null; then
|
|
21
21
|
echo ""
|
|
22
|
-
echo "WARNING: dbt not found on PATH."
|
|
22
|
+
echo "WARNING: dbt Fusion (dbtf) not found on PATH."
|
|
23
23
|
echo " Install dbt Fusion: https://docs.getdbt.com/docs/core/installation"
|
|
24
|
-
echo "
|
|
24
|
+
echo " The CLI uses 'dbtf' for all dbt commands."
|
|
25
25
|
echo ""
|
|
26
26
|
fi
|
|
27
27
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import shutil
|
|
1
3
|
import sys
|
|
2
4
|
from pathlib import Path
|
|
3
5
|
|
|
@@ -18,22 +20,58 @@ from datasecops_cli.utilities.display import (
|
|
|
18
20
|
)
|
|
19
21
|
|
|
20
22
|
|
|
23
|
+
DOWNLOAD_ITEMS = [
|
|
24
|
+
"sqlfluff", "pipelines", "packages", "macros",
|
|
25
|
+
"install-sqlfluff", "install-dbt", "all",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
30
|
+
parser = argparse.ArgumentParser(
|
|
31
|
+
prog="datasecops",
|
|
32
|
+
description="DataSecOps Framework CLI for Snowflake Native App",
|
|
33
|
+
)
|
|
34
|
+
sub = parser.add_subparsers(dest="command")
|
|
35
|
+
|
|
36
|
+
sub.add_parser("bootstrap", help="Set up a new dbt project with all framework config")
|
|
37
|
+
|
|
38
|
+
dl = sub.add_parser("download", help="Download framework config non-interactively (for CI/CD)")
|
|
39
|
+
dl.add_argument(
|
|
40
|
+
"items",
|
|
41
|
+
nargs="+",
|
|
42
|
+
choices=DOWNLOAD_ITEMS,
|
|
43
|
+
help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return parser
|
|
47
|
+
|
|
48
|
+
|
|
21
49
|
def main():
|
|
22
50
|
"""Main entry point for the datasecops CLI."""
|
|
51
|
+
parser = _build_parser()
|
|
52
|
+
args = parser.parse_args()
|
|
53
|
+
|
|
23
54
|
config = Config()
|
|
24
55
|
|
|
25
|
-
|
|
26
|
-
if len(sys.argv) > 1 and sys.argv[1] == "bootstrap":
|
|
56
|
+
if args.command == "bootstrap":
|
|
27
57
|
_run_bootstrap(config)
|
|
28
|
-
|
|
58
|
+
elif args.command == "download":
|
|
59
|
+
_run_download(config, args.items)
|
|
60
|
+
else:
|
|
61
|
+
_run_interactive(config)
|
|
29
62
|
|
|
63
|
+
|
|
64
|
+
def _connect_and_load(config: Config) -> SnowflakeService:
|
|
65
|
+
"""Load config, connect to Snowflake, and load native app settings.
|
|
66
|
+
|
|
67
|
+
Returns the connected SnowflakeService, or calls sys.exit on failure.
|
|
68
|
+
"""
|
|
30
69
|
if not config.load():
|
|
31
70
|
sys.exit(1)
|
|
32
|
-
|
|
33
|
-
# Connect to Snowflake
|
|
71
|
+
|
|
34
72
|
sf_config = config.datasecops
|
|
35
73
|
sf_service = SnowflakeService(sf_config)
|
|
36
|
-
|
|
74
|
+
|
|
37
75
|
try:
|
|
38
76
|
info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
|
|
39
77
|
sf_service.connect()
|
|
@@ -41,38 +79,101 @@ def main():
|
|
|
41
79
|
except Exception as e:
|
|
42
80
|
error_line(f"Failed to connect to Snowflake: {e}")
|
|
43
81
|
sys.exit(1)
|
|
44
|
-
|
|
82
|
+
|
|
83
|
+
info_line("Loading framework configuration...")
|
|
84
|
+
config.load_from_native_app(sf_service)
|
|
85
|
+
|
|
86
|
+
if not config.profile:
|
|
87
|
+
error_line(f"Profile '{config.profile_name}' not found in native app")
|
|
88
|
+
sf_service.close()
|
|
89
|
+
sys.exit(1)
|
|
90
|
+
|
|
91
|
+
success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
|
|
92
|
+
return sf_service
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _run_interactive(config: Config):
|
|
96
|
+
"""Run the interactive menu-driven CLI."""
|
|
97
|
+
sf_service = _connect_and_load(config)
|
|
98
|
+
|
|
45
99
|
try:
|
|
46
|
-
#
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
sys.exit(1)
|
|
53
|
-
|
|
54
|
-
success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
|
|
55
|
-
|
|
100
|
+
# Check for dbt Fusion (dbtf) on PATH
|
|
101
|
+
if not shutil.which("dbtf"):
|
|
102
|
+
from datasecops_cli.utilities.display import warning_line
|
|
103
|
+
warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
|
|
104
|
+
warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
|
|
105
|
+
|
|
56
106
|
# Initialize services
|
|
57
107
|
dbt_runner = DbtRunner(
|
|
58
108
|
project_dir=config.dbt_project_dir,
|
|
59
109
|
profiles_dir=config.get_dbt_profiles_dir(),
|
|
60
110
|
target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev"
|
|
61
111
|
)
|
|
62
|
-
|
|
112
|
+
|
|
63
113
|
try:
|
|
64
114
|
git_service = GitService(config.project_dir)
|
|
65
115
|
except Exception:
|
|
66
116
|
git_service = None
|
|
67
|
-
|
|
117
|
+
|
|
68
118
|
linting_service = LintingService(config.dbt_project_dir)
|
|
69
119
|
download_service = DownloadService(sf_service, config.project_dir)
|
|
70
120
|
skill_service = SkillService(sf_service)
|
|
71
|
-
|
|
121
|
+
|
|
72
122
|
# Main menu loop
|
|
73
|
-
_main_menu(config, dbt_runner, git_service, linting_service,
|
|
123
|
+
_main_menu(config, dbt_runner, git_service, linting_service,
|
|
74
124
|
download_service, skill_service, sf_service)
|
|
75
|
-
|
|
125
|
+
finally:
|
|
126
|
+
sf_service.close()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _run_download(config: Config, items: list[str]):
|
|
130
|
+
"""Run non-interactive downloads for CI/CD pipelines."""
|
|
131
|
+
sf_service = _connect_and_load(config)
|
|
132
|
+
|
|
133
|
+
try:
|
|
134
|
+
download_service = DownloadService(sf_service, config.project_dir)
|
|
135
|
+
linting_service = LintingService(config.dbt_project_dir)
|
|
136
|
+
profiles_dir = str(config.get_dbt_profiles_dir())
|
|
137
|
+
|
|
138
|
+
if "all" in items:
|
|
139
|
+
items = ["sqlfluff", "pipelines", "packages", "macros",
|
|
140
|
+
"install-sqlfluff", "install-dbt"]
|
|
141
|
+
|
|
142
|
+
failed = False
|
|
143
|
+
for item in items:
|
|
144
|
+
info_line("")
|
|
145
|
+
if item == "sqlfluff":
|
|
146
|
+
if not download_service.download_sqlfluff_config(
|
|
147
|
+
profiles_dir=profiles_dir, dbt_project_dir=config.dbt_project_dir
|
|
148
|
+
):
|
|
149
|
+
failed = True
|
|
150
|
+
elif item == "pipelines":
|
|
151
|
+
platform = config.source_control.source_control_platform.lower()
|
|
152
|
+
info_line(f"Platform: {platform}")
|
|
153
|
+
if not download_service.download_pipelines(platform=platform):
|
|
154
|
+
failed = True
|
|
155
|
+
elif item == "packages":
|
|
156
|
+
if not download_service.download_dbt_packages(config.dbt_project_dir):
|
|
157
|
+
failed = True
|
|
158
|
+
elif item == "macros":
|
|
159
|
+
if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
|
|
160
|
+
failed = True
|
|
161
|
+
elif item == "install-sqlfluff":
|
|
162
|
+
packages = download_service.get_sqlfluff_requirements()
|
|
163
|
+
if packages:
|
|
164
|
+
if not linting_service.install_requirements(packages):
|
|
165
|
+
failed = True
|
|
166
|
+
else:
|
|
167
|
+
failed = True
|
|
168
|
+
elif item == "install-dbt":
|
|
169
|
+
packages = download_service.get_dbt_requirements()
|
|
170
|
+
if packages:
|
|
171
|
+
if not linting_service.install_requirements(packages):
|
|
172
|
+
failed = True
|
|
173
|
+
else:
|
|
174
|
+
failed = True
|
|
175
|
+
|
|
176
|
+
sys.exit(1 if failed else 0)
|
|
76
177
|
finally:
|
|
77
178
|
sf_service.close()
|
|
78
179
|
|
|
@@ -110,12 +211,14 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
110
211
|
profile_name, config.dbt_project_dir,
|
|
111
212
|
project_settings=config.project_settings,
|
|
112
213
|
profile=config.profile,
|
|
214
|
+
source_control=config.source_control,
|
|
113
215
|
)
|
|
114
216
|
dl_menu.show()
|
|
115
217
|
|
|
116
218
|
elif option == 4:
|
|
117
|
-
from datasecops_cli.utilities.display import
|
|
118
|
-
platform =
|
|
219
|
+
from datasecops_cli.utilities.display import get_input_true_false
|
|
220
|
+
platform = config.source_control.source_control_platform.lower()
|
|
221
|
+
info_line(f"Platform: {platform}")
|
|
119
222
|
install_skills = get_input_true_false("Install Cortex Code skills?")
|
|
120
223
|
run_deps = get_input_true_false("Run dbt deps after downloading packages?")
|
|
121
224
|
bootstrap = BootstrapService(
|
|
@@ -145,35 +248,16 @@ def _show_main_menu(profile_name: str, git_service: GitService = None):
|
|
|
145
248
|
|
|
146
249
|
def _run_bootstrap(config: Config):
|
|
147
250
|
"""Run the bootstrap command to initialise a new project."""
|
|
148
|
-
from datasecops_cli.utilities.display import
|
|
251
|
+
from datasecops_cli.utilities.display import get_input_true_false
|
|
149
252
|
|
|
150
|
-
|
|
151
|
-
sys.exit(1)
|
|
152
|
-
|
|
153
|
-
sf_config = config.datasecops
|
|
154
|
-
sf_service = SnowflakeService(sf_config)
|
|
253
|
+
sf_service = _connect_and_load(config)
|
|
155
254
|
|
|
156
255
|
try:
|
|
157
|
-
info_line(f"Connecting to Snowflake ({sf_config.connection_name})...")
|
|
158
|
-
sf_service.connect()
|
|
159
|
-
success_line("Connected")
|
|
160
|
-
except Exception as e:
|
|
161
|
-
error_line(f"Failed to connect to Snowflake: {e}")
|
|
162
|
-
sys.exit(1)
|
|
163
|
-
|
|
164
|
-
try:
|
|
165
|
-
info_line("Loading framework configuration...")
|
|
166
|
-
config.load_from_native_app(sf_service)
|
|
167
|
-
|
|
168
|
-
if not config.profile:
|
|
169
|
-
error_line(f"Profile '{config.profile_name}' not found in native app")
|
|
170
|
-
sys.exit(1)
|
|
171
|
-
|
|
172
|
-
success_line(f"Profile: {config.profile.project_name} ({config.profile_name})")
|
|
173
256
|
info_line("")
|
|
174
257
|
|
|
175
|
-
#
|
|
176
|
-
platform =
|
|
258
|
+
# Use platform from framework config
|
|
259
|
+
platform = config.source_control.source_control_platform.lower()
|
|
260
|
+
info_line(f"Platform: {platform}")
|
|
177
261
|
|
|
178
262
|
# Ask about optional steps
|
|
179
263
|
install_skills = get_input_true_false("Install Cortex Code skills?")
|
|
@@ -96,7 +96,7 @@ class DevelopmentMenu:
|
|
|
96
96
|
menu_option(11, "clean - Clean dbt target")
|
|
97
97
|
menu_option(12, "debug - Debug dbt connection")
|
|
98
98
|
menu_option(13, "list - List dbt resources")
|
|
99
|
-
menu_option(14, "install
|
|
99
|
+
menu_option(14, "install lint - Install dbt-core & dbt-snowflake for linting (not dbt Fusion)")
|
|
100
100
|
menu_option(0, "back - Return to main menu")
|
|
101
101
|
|
|
102
102
|
def _run_menu(self) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
|
|
3
|
-
from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings
|
|
3
|
+
from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings, SourceControl
|
|
4
4
|
from datasecops_cli.services.download_service import DownloadService
|
|
5
5
|
from datasecops_cli.services.skill_service import SkillService
|
|
6
6
|
from datasecops_cli.services.dbt_runner import DbtRunner
|
|
@@ -14,7 +14,8 @@ from datasecops_cli.utilities.display import (
|
|
|
14
14
|
class DownloadsMenu:
|
|
15
15
|
def __init__(self, download_service: DownloadService, skill_service: SkillService,
|
|
16
16
|
dbt_runner: DbtRunner, profile_name: str, dbt_project_dir: Path,
|
|
17
|
-
project_settings: ProjectSettings = None, profile: ProjectProfile = None
|
|
17
|
+
project_settings: ProjectSettings = None, profile: ProjectProfile = None,
|
|
18
|
+
source_control: SourceControl = None):
|
|
18
19
|
self.downloads = download_service
|
|
19
20
|
self.skills = skill_service
|
|
20
21
|
self.dbt = dbt_runner
|
|
@@ -22,6 +23,7 @@ class DownloadsMenu:
|
|
|
22
23
|
self.dbt_project_dir = dbt_project_dir
|
|
23
24
|
self.project_settings = project_settings
|
|
24
25
|
self.profile = profile
|
|
26
|
+
self.source_control = source_control
|
|
25
27
|
|
|
26
28
|
def show(self) -> None:
|
|
27
29
|
self._menu()
|
|
@@ -34,7 +36,8 @@ class DownloadsMenu:
|
|
|
34
36
|
complete_action()
|
|
35
37
|
elif option == 2:
|
|
36
38
|
display_action_header("Download Pipeline Files")
|
|
37
|
-
platform =
|
|
39
|
+
platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
|
|
40
|
+
info_line(f"Platform: {platform}")
|
|
38
41
|
self.downloads.download_pipelines(platform=platform)
|
|
39
42
|
complete_action()
|
|
40
43
|
elif option == 3:
|
{datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
@@ -138,7 +138,7 @@ class BootstrapService:
|
|
|
138
138
|
info_line("Your project is ready. Next steps:")
|
|
139
139
|
info_line(f" cd {dbt_project_dir}")
|
|
140
140
|
info_line(" datasecops — run the framework CLI")
|
|
141
|
-
info_line("
|
|
142
|
-
info_line("
|
|
141
|
+
info_line(" dbtf debug — verify Snowflake connection")
|
|
142
|
+
info_line(" dbtf run — run your first models")
|
|
143
143
|
info_line("")
|
|
144
144
|
return True
|
|
@@ -21,7 +21,7 @@ class DbtRunner:
|
|
|
21
21
|
]
|
|
22
22
|
|
|
23
23
|
def _run_command(self, command: str, extra_args: list[str] = None) -> subprocess.CompletedProcess:
|
|
24
|
-
cmd = ["
|
|
24
|
+
cmd = ["dbtf", command] + (extra_args or []) + self._default_args()
|
|
25
25
|
info_line(f"Running: {' '.join(cmd)}")
|
|
26
26
|
result = subprocess.run(cmd, capture_output=False)
|
|
27
27
|
if result.returncode != 0:
|
|
@@ -94,7 +94,7 @@ class DbtRunner:
|
|
|
94
94
|
return self._run_command("docs", ["generate", f"--target={self.target}"])
|
|
95
95
|
|
|
96
96
|
def docs_serve(self) -> subprocess.Popen:
|
|
97
|
-
cmd = ["
|
|
97
|
+
cmd = ["dbtf", "docs", "serve"] + self._default_args()
|
|
98
98
|
info_line(f"Running: {' '.join(cmd)}")
|
|
99
99
|
return subprocess.Popen(cmd)
|
|
100
100
|
|
{datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/download_service.py
RENAMED
|
@@ -159,7 +159,12 @@ class DownloadService:
|
|
|
159
159
|
section_name = self.RULE_SECTION_MAP.get(code)
|
|
160
160
|
if not section_name:
|
|
161
161
|
continue
|
|
162
|
-
opts = entry.get("options", {})
|
|
162
|
+
opts = dict(entry.get("options", {}))
|
|
163
|
+
# For aliasing.length, 0 means "no limit" which sqlfluff expects as None
|
|
164
|
+
if section_name == "aliasing.length":
|
|
165
|
+
for len_key in ("min_alias_length", "max_alias_length"):
|
|
166
|
+
if len_key in opts and opts[len_key] == 0:
|
|
167
|
+
opts[len_key] = None
|
|
163
168
|
self._emit_section(lines, f"sqlfluff:rules:{section_name}", opts)
|
|
164
169
|
|
|
165
170
|
content = "\n".join(lines)
|
|
@@ -225,12 +230,12 @@ class DownloadService:
|
|
|
225
230
|
pipelines = raw.get("pipelines", [])
|
|
226
231
|
count = 0
|
|
227
232
|
for pipe in pipelines:
|
|
228
|
-
if pipe.get("platform", "github") != platform or not pipe.get("enabled", True):
|
|
233
|
+
if pipe.get("platform", "github").lower() != platform.lower() or not pipe.get("enabled", True):
|
|
229
234
|
continue
|
|
230
235
|
filename = pipe.get("filename", "")
|
|
231
236
|
yaml_content = pipe.get("yaml_content", "")
|
|
232
237
|
if filename and yaml_content:
|
|
233
|
-
if platform == "github":
|
|
238
|
+
if platform.lower() == "github":
|
|
234
239
|
dest = self.project_dir / ".github" / "workflows" / filename
|
|
235
240
|
else:
|
|
236
241
|
dest = self.project_dir / filename
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"""Tests for datasecops_cli.main argument parsing and download dispatch."""
|
|
2
|
+
import pytest
|
|
3
|
+
from unittest.mock import MagicMock, patch, call
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from datasecops_cli.main import _build_parser, _run_download, DOWNLOAD_ITEMS
|
|
7
|
+
from datasecops_cli.config import Config
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestBuildParser:
|
|
11
|
+
"""Tests for the argparse configuration."""
|
|
12
|
+
|
|
13
|
+
def test_no_args_gives_no_command(self):
|
|
14
|
+
parser = _build_parser()
|
|
15
|
+
args = parser.parse_args([])
|
|
16
|
+
assert args.command is None
|
|
17
|
+
|
|
18
|
+
def test_bootstrap_command(self):
|
|
19
|
+
parser = _build_parser()
|
|
20
|
+
args = parser.parse_args(["bootstrap"])
|
|
21
|
+
assert args.command == "bootstrap"
|
|
22
|
+
|
|
23
|
+
def test_download_single_item(self):
|
|
24
|
+
parser = _build_parser()
|
|
25
|
+
args = parser.parse_args(["download", "sqlfluff"])
|
|
26
|
+
assert args.command == "download"
|
|
27
|
+
assert args.items == ["sqlfluff"]
|
|
28
|
+
|
|
29
|
+
def test_download_multiple_items(self):
|
|
30
|
+
parser = _build_parser()
|
|
31
|
+
args = parser.parse_args(["download", "sqlfluff", "packages", "macros"])
|
|
32
|
+
assert args.command == "download"
|
|
33
|
+
assert args.items == ["sqlfluff", "packages", "macros"]
|
|
34
|
+
|
|
35
|
+
def test_download_all(self):
|
|
36
|
+
parser = _build_parser()
|
|
37
|
+
args = parser.parse_args(["download", "all"])
|
|
38
|
+
assert args.command == "download"
|
|
39
|
+
assert args.items == ["all"]
|
|
40
|
+
|
|
41
|
+
def test_download_invalid_item_exits(self):
|
|
42
|
+
parser = _build_parser()
|
|
43
|
+
with pytest.raises(SystemExit):
|
|
44
|
+
parser.parse_args(["download", "invalid"])
|
|
45
|
+
|
|
46
|
+
def test_download_no_items_exits(self):
|
|
47
|
+
parser = _build_parser()
|
|
48
|
+
with pytest.raises(SystemExit):
|
|
49
|
+
parser.parse_args(["download"])
|
|
50
|
+
|
|
51
|
+
def test_all_download_items_accepted(self):
|
|
52
|
+
parser = _build_parser()
|
|
53
|
+
for item in DOWNLOAD_ITEMS:
|
|
54
|
+
args = parser.parse_args(["download", item])
|
|
55
|
+
assert args.items == [item]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class TestRunDownload:
|
|
59
|
+
"""Tests for _run_download dispatch logic using mocked services."""
|
|
60
|
+
|
|
61
|
+
def _make_config(self, tmp_path):
|
|
62
|
+
"""Create a Config with minimal valid state."""
|
|
63
|
+
config = Config()
|
|
64
|
+
config.project_dir = tmp_path
|
|
65
|
+
config.dbt_project_dir = tmp_path
|
|
66
|
+
config.profile_name = "test_profile"
|
|
67
|
+
# source_control defaults to GitHub
|
|
68
|
+
return config
|
|
69
|
+
|
|
70
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
71
|
+
def test_download_sqlfluff(self, mock_connect, tmp_path):
|
|
72
|
+
config = self._make_config(tmp_path)
|
|
73
|
+
mock_sf = MagicMock()
|
|
74
|
+
mock_connect.return_value = mock_sf
|
|
75
|
+
|
|
76
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
77
|
+
patch("datasecops_cli.main.LintingService"):
|
|
78
|
+
mock_ds = MockDS.return_value
|
|
79
|
+
mock_ds.download_sqlfluff_config.return_value = True
|
|
80
|
+
|
|
81
|
+
with pytest.raises(SystemExit) as exc:
|
|
82
|
+
_run_download(config, ["sqlfluff"])
|
|
83
|
+
|
|
84
|
+
assert exc.value.code == 0
|
|
85
|
+
mock_ds.download_sqlfluff_config.assert_called_once()
|
|
86
|
+
mock_ds.download_pipelines.assert_not_called()
|
|
87
|
+
mock_ds.download_dbt_packages.assert_not_called()
|
|
88
|
+
mock_ds.download_macros.assert_not_called()
|
|
89
|
+
|
|
90
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
91
|
+
def test_download_pipelines(self, mock_connect, tmp_path):
|
|
92
|
+
config = self._make_config(tmp_path)
|
|
93
|
+
mock_sf = MagicMock()
|
|
94
|
+
mock_connect.return_value = mock_sf
|
|
95
|
+
|
|
96
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
97
|
+
patch("datasecops_cli.main.LintingService"):
|
|
98
|
+
mock_ds = MockDS.return_value
|
|
99
|
+
mock_ds.download_pipelines.return_value = True
|
|
100
|
+
|
|
101
|
+
with pytest.raises(SystemExit) as exc:
|
|
102
|
+
_run_download(config, ["pipelines"])
|
|
103
|
+
|
|
104
|
+
assert exc.value.code == 0
|
|
105
|
+
mock_ds.download_pipelines.assert_called_once_with(platform="github")
|
|
106
|
+
|
|
107
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
108
|
+
def test_download_packages(self, mock_connect, tmp_path):
|
|
109
|
+
config = self._make_config(tmp_path)
|
|
110
|
+
mock_sf = MagicMock()
|
|
111
|
+
mock_connect.return_value = mock_sf
|
|
112
|
+
|
|
113
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
114
|
+
patch("datasecops_cli.main.LintingService"):
|
|
115
|
+
mock_ds = MockDS.return_value
|
|
116
|
+
mock_ds.download_dbt_packages.return_value = True
|
|
117
|
+
|
|
118
|
+
with pytest.raises(SystemExit) as exc:
|
|
119
|
+
_run_download(config, ["packages"])
|
|
120
|
+
|
|
121
|
+
assert exc.value.code == 0
|
|
122
|
+
mock_ds.download_dbt_packages.assert_called_once_with(tmp_path)
|
|
123
|
+
|
|
124
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
125
|
+
def test_download_macros(self, mock_connect, tmp_path):
|
|
126
|
+
config = self._make_config(tmp_path)
|
|
127
|
+
mock_sf = MagicMock()
|
|
128
|
+
mock_connect.return_value = mock_sf
|
|
129
|
+
|
|
130
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
131
|
+
patch("datasecops_cli.main.LintingService"):
|
|
132
|
+
mock_ds = MockDS.return_value
|
|
133
|
+
mock_ds.download_macros.return_value = True
|
|
134
|
+
|
|
135
|
+
with pytest.raises(SystemExit) as exc:
|
|
136
|
+
_run_download(config, ["macros"])
|
|
137
|
+
|
|
138
|
+
assert exc.value.code == 0
|
|
139
|
+
mock_ds.download_macros.assert_called_once_with("test_profile", tmp_path)
|
|
140
|
+
|
|
141
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
142
|
+
def test_download_all_expands_to_all_items(self, mock_connect, tmp_path):
|
|
143
|
+
config = self._make_config(tmp_path)
|
|
144
|
+
mock_sf = MagicMock()
|
|
145
|
+
mock_connect.return_value = mock_sf
|
|
146
|
+
|
|
147
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
148
|
+
patch("datasecops_cli.main.LintingService") as MockLS:
|
|
149
|
+
mock_ds = MockDS.return_value
|
|
150
|
+
mock_ls = MockLS.return_value
|
|
151
|
+
mock_ds.download_sqlfluff_config.return_value = True
|
|
152
|
+
mock_ds.download_pipelines.return_value = True
|
|
153
|
+
mock_ds.download_dbt_packages.return_value = True
|
|
154
|
+
mock_ds.download_macros.return_value = True
|
|
155
|
+
mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0"]
|
|
156
|
+
mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
|
|
157
|
+
mock_ls.install_requirements.return_value = True
|
|
158
|
+
|
|
159
|
+
with pytest.raises(SystemExit) as exc:
|
|
160
|
+
_run_download(config, ["all"])
|
|
161
|
+
|
|
162
|
+
assert exc.value.code == 0
|
|
163
|
+
mock_ds.download_sqlfluff_config.assert_called_once()
|
|
164
|
+
mock_ds.download_pipelines.assert_called_once()
|
|
165
|
+
mock_ds.download_dbt_packages.assert_called_once()
|
|
166
|
+
mock_ds.download_macros.assert_called_once()
|
|
167
|
+
mock_ds.get_sqlfluff_requirements.assert_called_once()
|
|
168
|
+
mock_ds.get_dbt_requirements.assert_called_once()
|
|
169
|
+
assert mock_ls.install_requirements.call_count == 2
|
|
170
|
+
|
|
171
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
172
|
+
def test_download_failure_exits_1(self, mock_connect, tmp_path):
|
|
173
|
+
config = self._make_config(tmp_path)
|
|
174
|
+
mock_sf = MagicMock()
|
|
175
|
+
mock_connect.return_value = mock_sf
|
|
176
|
+
|
|
177
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
178
|
+
patch("datasecops_cli.main.LintingService"):
|
|
179
|
+
mock_ds = MockDS.return_value
|
|
180
|
+
mock_ds.download_sqlfluff_config.return_value = False # failure
|
|
181
|
+
|
|
182
|
+
with pytest.raises(SystemExit) as exc:
|
|
183
|
+
_run_download(config, ["sqlfluff"])
|
|
184
|
+
|
|
185
|
+
assert exc.value.code == 1
|
|
186
|
+
|
|
187
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
188
|
+
def test_download_partial_failure_exits_1(self, mock_connect, tmp_path):
|
|
189
|
+
"""If one item fails but another succeeds, exit code is still 1."""
|
|
190
|
+
config = self._make_config(tmp_path)
|
|
191
|
+
mock_sf = MagicMock()
|
|
192
|
+
mock_connect.return_value = mock_sf
|
|
193
|
+
|
|
194
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
195
|
+
patch("datasecops_cli.main.LintingService"):
|
|
196
|
+
mock_ds = MockDS.return_value
|
|
197
|
+
mock_ds.download_sqlfluff_config.return_value = True
|
|
198
|
+
mock_ds.download_dbt_packages.return_value = False
|
|
199
|
+
|
|
200
|
+
with pytest.raises(SystemExit) as exc:
|
|
201
|
+
_run_download(config, ["sqlfluff", "packages"])
|
|
202
|
+
|
|
203
|
+
assert exc.value.code == 1
|
|
204
|
+
|
|
205
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
206
|
+
def test_download_pipelines_uses_configured_platform(self, mock_connect, tmp_path):
|
|
207
|
+
"""Pipeline download reads platform from source_control config."""
|
|
208
|
+
config = self._make_config(tmp_path)
|
|
209
|
+
config.source_control.source_control_platform = "AzureDevOps"
|
|
210
|
+
mock_sf = MagicMock()
|
|
211
|
+
mock_connect.return_value = mock_sf
|
|
212
|
+
|
|
213
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
214
|
+
patch("datasecops_cli.main.LintingService"):
|
|
215
|
+
mock_ds = MockDS.return_value
|
|
216
|
+
mock_ds.download_pipelines.return_value = True
|
|
217
|
+
|
|
218
|
+
with pytest.raises(SystemExit) as exc:
|
|
219
|
+
_run_download(config, ["pipelines"])
|
|
220
|
+
|
|
221
|
+
assert exc.value.code == 0
|
|
222
|
+
mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
|
|
223
|
+
|
|
224
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
225
|
+
def test_install_sqlfluff(self, mock_connect, tmp_path):
|
|
226
|
+
"""install-sqlfluff fetches pinned versions and installs them."""
|
|
227
|
+
config = self._make_config(tmp_path)
|
|
228
|
+
mock_connect.return_value = MagicMock()
|
|
229
|
+
|
|
230
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
231
|
+
patch("datasecops_cli.main.LintingService") as MockLS:
|
|
232
|
+
mock_ds = MockDS.return_value
|
|
233
|
+
mock_ls = MockLS.return_value
|
|
234
|
+
mock_ds.get_sqlfluff_requirements.return_value = ["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
|
|
235
|
+
mock_ls.install_requirements.return_value = True
|
|
236
|
+
|
|
237
|
+
with pytest.raises(SystemExit) as exc:
|
|
238
|
+
_run_download(config, ["install-sqlfluff"])
|
|
239
|
+
|
|
240
|
+
assert exc.value.code == 0
|
|
241
|
+
mock_ds.get_sqlfluff_requirements.assert_called_once()
|
|
242
|
+
mock_ls.install_requirements.assert_called_once_with(
|
|
243
|
+
["sqlfluff==3.4.0", "sqlfluff-templater-dbt==3.4.0"]
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
247
|
+
def test_install_dbt(self, mock_connect, tmp_path):
|
|
248
|
+
"""install-dbt fetches pinned versions and installs them."""
|
|
249
|
+
config = self._make_config(tmp_path)
|
|
250
|
+
mock_connect.return_value = MagicMock()
|
|
251
|
+
|
|
252
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
253
|
+
patch("datasecops_cli.main.LintingService") as MockLS:
|
|
254
|
+
mock_ds = MockDS.return_value
|
|
255
|
+
mock_ls = MockLS.return_value
|
|
256
|
+
mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
|
|
257
|
+
mock_ls.install_requirements.return_value = True
|
|
258
|
+
|
|
259
|
+
with pytest.raises(SystemExit) as exc:
|
|
260
|
+
_run_download(config, ["install-dbt"])
|
|
261
|
+
|
|
262
|
+
assert exc.value.code == 0
|
|
263
|
+
mock_ds.get_dbt_requirements.assert_called_once()
|
|
264
|
+
mock_ls.install_requirements.assert_called_once_with(
|
|
265
|
+
["dbt-core==1.9.0", "dbt-snowflake==1.9.0"]
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
269
|
+
def test_install_sqlfluff_no_versions_exits_1(self, mock_connect, tmp_path):
|
|
270
|
+
"""install-sqlfluff fails if no versions returned from framework."""
|
|
271
|
+
config = self._make_config(tmp_path)
|
|
272
|
+
mock_connect.return_value = MagicMock()
|
|
273
|
+
|
|
274
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
275
|
+
patch("datasecops_cli.main.LintingService") as MockLS:
|
|
276
|
+
mock_ds = MockDS.return_value
|
|
277
|
+
mock_ls = MockLS.return_value
|
|
278
|
+
mock_ds.get_sqlfluff_requirements.return_value = []
|
|
279
|
+
|
|
280
|
+
with pytest.raises(SystemExit) as exc:
|
|
281
|
+
_run_download(config, ["install-sqlfluff"])
|
|
282
|
+
|
|
283
|
+
assert exc.value.code == 1
|
|
284
|
+
mock_ls.install_requirements.assert_not_called()
|
|
285
|
+
|
|
286
|
+
@patch("datasecops_cli.main._connect_and_load")
|
|
287
|
+
def test_install_dbt_pip_failure_exits_1(self, mock_connect, tmp_path):
|
|
288
|
+
"""install-dbt exits 1 if uv pip install fails."""
|
|
289
|
+
config = self._make_config(tmp_path)
|
|
290
|
+
mock_connect.return_value = MagicMock()
|
|
291
|
+
|
|
292
|
+
with patch("datasecops_cli.main.DownloadService") as MockDS, \
|
|
293
|
+
patch("datasecops_cli.main.LintingService") as MockLS:
|
|
294
|
+
mock_ds = MockDS.return_value
|
|
295
|
+
mock_ls = MockLS.return_value
|
|
296
|
+
mock_ds.get_dbt_requirements.return_value = ["dbt-core==1.9.0"]
|
|
297
|
+
mock_ls.install_requirements.return_value = False # pip failure
|
|
298
|
+
|
|
299
|
+
with pytest.raises(SystemExit) as exc:
|
|
300
|
+
_run_download(config, ["install-dbt"])
|
|
301
|
+
|
|
302
|
+
assert exc.value.code == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/linting_service.py
RENAMED
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.2.7 → datasecops_cli-0.2.9}/src/datasecops_cli/services/snowflake_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|