datasecops-cli 0.3.3__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datasecops_cli-0.4.0/.github/workflows/auto-tag.yml +113 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/CHANGELOG.md +46 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/PKG-INFO +2 -2
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/README.md +1 -1
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/pyproject.toml +1 -1
- datasecops_cli-0.4.0/src/datasecops_cli/__init__.py +1 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/main.py +22 -9
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/downloads.py +13 -5
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/bootstrap_service.py +19 -13
- datasecops_cli-0.4.0/src/datasecops_cli/services/dbt_project_generator.py +211 -0
- datasecops_cli-0.4.0/src/datasecops_cli/services/directory_scaffolder.py +81 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/download_service.py +98 -57
- datasecops_cli-0.4.0/src/datasecops_cli/services/skill_service.py +167 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/snowflake_service.py +15 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/display.py +21 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/file_utils.py +8 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/connection.py +36 -3
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/server.py +77 -0
- datasecops_cli-0.3.3/src/datasecops_cli/__init__.py +0 -1
- datasecops_cli-0.3.3/src/datasecops_cli/services/skill_service.py +0 -86
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/.gitignore +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/DEVELOPMENT.md +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/LICENSE +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/getting-started.md +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/legacy.md +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/mcp-servers.json +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/setup.ps1 +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/setup.sh +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/config.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/development.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/project_config.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/dbt_runner.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/upstream_service.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/__init__.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_config.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_main.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_models.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_version.py +0 -0
- {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_yaml_utils.py +0 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
name: Tag and Deploy
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
id-token: write
|
|
9
|
+
contents: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
auto-tag:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
outputs:
|
|
15
|
+
tag: ${{ steps.tag.outputs.tag }}
|
|
16
|
+
created: ${{ steps.tag.outputs.created }}
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- name: Checkout repository
|
|
20
|
+
uses: actions/checkout@v4
|
|
21
|
+
|
|
22
|
+
- name: Extract version from pyproject.toml
|
|
23
|
+
id: version
|
|
24
|
+
run: |
|
|
25
|
+
set -euo pipefail
|
|
26
|
+
VERSION=$(python3 -c "
|
|
27
|
+
import tomllib, pathlib
|
|
28
|
+
data = tomllib.loads(pathlib.Path('pyproject.toml').read_text())
|
|
29
|
+
print(data['project']['version'])
|
|
30
|
+
")
|
|
31
|
+
if [ -z "$VERSION" ]; then
|
|
32
|
+
echo "Failed to extract version from pyproject.toml" >&2
|
|
33
|
+
exit 1
|
|
34
|
+
fi
|
|
35
|
+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
|
36
|
+
echo "Detected version: $VERSION"
|
|
37
|
+
|
|
38
|
+
- name: Create and push tag
|
|
39
|
+
id: tag
|
|
40
|
+
run: |
|
|
41
|
+
TAG="cli-v${{ steps.version.outputs.version }}"
|
|
42
|
+
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
|
|
43
|
+
if git ls-remote --tags origin "refs/tags/$TAG" | grep -q "$TAG"; then
|
|
44
|
+
echo "Tag $TAG already exists, skipping."
|
|
45
|
+
echo "created=false" >> "$GITHUB_OUTPUT"
|
|
46
|
+
exit 0
|
|
47
|
+
fi
|
|
48
|
+
git tag "$TAG"
|
|
49
|
+
PUSH_OUTPUT=$(git push origin "$TAG" 2>&1) && {
|
|
50
|
+
echo "Created and pushed tag: $TAG"
|
|
51
|
+
echo "created=true" >> "$GITHUB_OUTPUT"
|
|
52
|
+
} || {
|
|
53
|
+
if echo "$PUSH_OUTPUT" | grep -qi "already exists"; then
|
|
54
|
+
echo "Tag $TAG was created by another run, skipping."
|
|
55
|
+
echo "created=false" >> "$GITHUB_OUTPUT"
|
|
56
|
+
else
|
|
57
|
+
echo "Tag push failed unexpectedly:"
|
|
58
|
+
echo "$PUSH_OUTPUT"
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
test:
|
|
64
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
65
|
+
needs: auto-tag
|
|
66
|
+
if: needs.auto-tag.outputs.created == 'true'
|
|
67
|
+
runs-on: ubuntu-latest
|
|
68
|
+
strategy:
|
|
69
|
+
matrix:
|
|
70
|
+
python-version: ['3.11', '3.12', '3.13']
|
|
71
|
+
|
|
72
|
+
steps:
|
|
73
|
+
- name: Checkout repository
|
|
74
|
+
uses: actions/checkout@v4
|
|
75
|
+
with:
|
|
76
|
+
ref: ${{ needs.auto-tag.outputs.tag }}
|
|
77
|
+
|
|
78
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
79
|
+
uses: actions/setup-python@v5
|
|
80
|
+
with:
|
|
81
|
+
python-version: ${{ matrix.python-version }}
|
|
82
|
+
|
|
83
|
+
- name: Install package with test dependencies
|
|
84
|
+
run: pip install -e ".[test]"
|
|
85
|
+
|
|
86
|
+
- name: Run tests
|
|
87
|
+
run: pytest --tb=short
|
|
88
|
+
|
|
89
|
+
build-and-publish:
|
|
90
|
+
name: Build and publish to PyPI
|
|
91
|
+
needs: [auto-tag, test]
|
|
92
|
+
if: needs.auto-tag.outputs.created == 'true'
|
|
93
|
+
runs-on: ubuntu-latest
|
|
94
|
+
|
|
95
|
+
steps:
|
|
96
|
+
- name: Checkout repository
|
|
97
|
+
uses: actions/checkout@v4
|
|
98
|
+
with:
|
|
99
|
+
ref: ${{ needs.auto-tag.outputs.tag }}
|
|
100
|
+
|
|
101
|
+
- name: Set up Python
|
|
102
|
+
uses: actions/setup-python@v5
|
|
103
|
+
with:
|
|
104
|
+
python-version: '3.11'
|
|
105
|
+
|
|
106
|
+
- name: Install build tools
|
|
107
|
+
run: pip install build
|
|
108
|
+
|
|
109
|
+
- name: Build package
|
|
110
|
+
run: python -m build
|
|
111
|
+
|
|
112
|
+
- name: Publish to PyPI
|
|
113
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -2,6 +2,52 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the DataSecOps CLI are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.4.0] - 2026-05-16
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **Framework-aligned project scaffolding** — `datasecops bootstrap` now generates `dbt_project.yml` directly from the native app configuration instead of running `dbtf init`. The generated project matches the `setup-dbt-project` Cortex Code skill specification, including:
|
|
10
|
+
- Model sections based on profile's `MODEL_TYPES` (sources, dimensional_modelling, domains, cortex_models)
|
|
11
|
+
- Object sections with doc node colors (stored_procedures, tasks, UDFs)
|
|
12
|
+
- `dispatch` with framework package search order
|
|
13
|
+
- `on-run-end` with `clean_objects` macro
|
|
14
|
+
- `quoting` and `vars` configuration
|
|
15
|
+
- **Directory structure scaffolding** — bootstrap creates the full directory tree with `.gitkeep` files in empty model/object subdirectories (models/sources/, models/dm/dims/, objects/stored_procedures/, etc.)
|
|
16
|
+
- **`.gitignore` generation** — bootstrap creates a `.gitignore` with standard dbt exclusions including `profiles.yml`
|
|
17
|
+
- **Macro category subdirectories** — macros are now written to `macros/{category}/{file_name}` (e.g., `macros/schema/generate_schema_name.sql`) matching the framework's category organisation
|
|
18
|
+
- **Upstream project support in packages.yml** — profiles with `upstream_projects` configured now generate `local: "local_packages/{name}"` entries in `packages.yml`
|
|
19
|
+
- **profiles.yml database suffixes** — each target's database name now includes the branch suffix in uppercase (e.g., `MY_DB_PROD`) matching the skill specification
|
|
20
|
+
- **New MCP tools** — 5 new tools exposed via the MCP server:
|
|
21
|
+
- `get_model_definitions` — model types with paths, prefixes, and metadata fields
|
|
22
|
+
- `get_object_definitions` — object types with paths, prefixes, and options
|
|
23
|
+
- `get_source_definitions` — configured source systems and their schemas
|
|
24
|
+
- `get_access_groups` — RBAC access groups with owners
|
|
25
|
+
- `get_support_contacts` — team contact details
|
|
26
|
+
- **New native app stored procedures** — `api.get_model_definitions()`, `api.get_object_definitions()`, `api.get_source_definitions()`, `api.get_access_groups()`, `api.get_support_contacts()` for CLI/MCP consumption
|
|
27
|
+
- **New SnowflakeService methods** — both CLI and MCP Snowflake services expose the new stored procedures
|
|
28
|
+
- **Multi-IDE skill installation** — skills now install to Cursor (`.cursor/rules/{id}.mdc` with frontmatter) and Claude Code (`.claude/commands/{id}.md`) in addition to Cortex Code. Bootstrap and downloads menu prompt users to select target IDE(s) via a multi-select prompt (`[1] Cortex Code`, `[2] Cursor`, `[3] Claude Code`, `[a] All`)
|
|
29
|
+
- **`select_multiple_from_list` display helper** — new utility for comma-separated multi-select input with "all" shortcut
|
|
30
|
+
|
|
31
|
+
### Changed
|
|
32
|
+
|
|
33
|
+
- **Bootstrap no longer requires `dbtf` for init** — project scaffolding is now pure Python, removing the dependency on having dbt Fusion installed before bootstrapping
|
|
34
|
+
- **`init_dbt_project()` rewritten** — replaced `subprocess.run(["dbtf", "init"])` with framework-specific `dbt_project_generator` and `directory_scaffolder` services
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- **Version drift** — `__init__.py` version now matches `pyproject.toml` (was stuck at `0.1.0`)
|
|
39
|
+
- **Path safety** — replaced unsafe `lstrip("./")` with `removeprefix("./")` in bootstrap and download services
|
|
40
|
+
- **Path traversal prevention** — macro category and file names sanitized with `Path.name` before writing
|
|
41
|
+
- **MCP SQL safety** — added `_ALLOWED_PROCEDURES` frozenset to `MCPSnowflakeService._call_procedure` to prevent arbitrary procedure calls
|
|
42
|
+
- **Narrowed exception handling** — replaced bare `Exception` with `OSError` in skill installation; failed installs no longer added to success list
|
|
43
|
+
- **Cursor `.mdc` frontmatter** — fixed `globs:` to `globs: []` for valid YAML
|
|
44
|
+
|
|
45
|
+
## [0.3.4] - 2026-05-15
|
|
46
|
+
|
|
47
|
+
### Added
|
|
48
|
+
|
|
49
|
+
- **`scripts` download item** — `datasecops download scripts` downloads pipeline scripts from the native app to `.datasecops/scripts/`. Included in `datasecops download all`.
|
|
50
|
+
|
|
5
51
|
## [0.3.3] - 2026-05-15
|
|
6
52
|
|
|
7
53
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datasecops-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: DataSecOps Framework CLI for Snowflake Native App
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -105,7 +105,7 @@ datasecops download sqlfluff install-sqlfluff
|
|
|
105
105
|
datasecops download all
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
|
|
108
|
+
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `scripts`, `install-sqlfluff`, `install-dbt`, `all`
|
|
109
109
|
|
|
110
110
|
The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
|
|
111
111
|
|
|
@@ -86,7 +86,7 @@ datasecops download sqlfluff install-sqlfluff
|
|
|
86
86
|
datasecops download all
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
-
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
|
|
89
|
+
Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `scripts`, `install-sqlfluff`, `install-dbt`, `all`
|
|
90
90
|
|
|
91
91
|
The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
|
|
92
92
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.0"
|
|
@@ -24,7 +24,7 @@ from datasecops_cli.utilities.yaml_utils import write_datasecops_config
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
DOWNLOAD_ITEMS = [
|
|
27
|
-
"sqlfluff", "pipelines", "packages", "macros",
|
|
27
|
+
"sqlfluff", "pipelines", "packages", "macros", "scripts",
|
|
28
28
|
"install-sqlfluff", "install-dbt", "all",
|
|
29
29
|
]
|
|
30
30
|
|
|
@@ -44,7 +44,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
44
44
|
"items",
|
|
45
45
|
nargs="+",
|
|
46
46
|
choices=DOWNLOAD_ITEMS,
|
|
47
|
-
help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
|
|
47
|
+
help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, scripts, install-sqlfluff, install-dbt, or all",
|
|
48
48
|
)
|
|
49
49
|
dl.add_argument(
|
|
50
50
|
"--connection", "-c",
|
|
@@ -309,7 +309,7 @@ def _run_download(config: Config, items: list[str],
|
|
|
309
309
|
profiles_dir = str(config.get_dbt_profiles_dir())
|
|
310
310
|
|
|
311
311
|
if "all" in items:
|
|
312
|
-
items = ["sqlfluff", "pipelines", "packages", "macros",
|
|
312
|
+
items = ["sqlfluff", "pipelines", "packages", "macros", "scripts",
|
|
313
313
|
"install-sqlfluff", "install-dbt"]
|
|
314
314
|
|
|
315
315
|
failed = False
|
|
@@ -331,6 +331,9 @@ def _run_download(config: Config, items: list[str],
|
|
|
331
331
|
elif item == "macros":
|
|
332
332
|
if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
|
|
333
333
|
failed = True
|
|
334
|
+
elif item == "scripts":
|
|
335
|
+
if not download_service.download_scripts():
|
|
336
|
+
failed = True
|
|
334
337
|
elif item == "install-sqlfluff":
|
|
335
338
|
packages = download_service.get_sqlfluff_requirements()
|
|
336
339
|
if packages:
|
|
@@ -420,10 +423,14 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
420
423
|
dl_menu.show()
|
|
421
424
|
|
|
422
425
|
elif option == 4:
|
|
423
|
-
from datasecops_cli.utilities.display import get_input_true_false
|
|
426
|
+
from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
|
|
424
427
|
platform = config.source_control.source_control_platform.lower()
|
|
425
428
|
info_line(f"Platform: {platform}")
|
|
426
|
-
install_skills = get_input_true_false("Install
|
|
429
|
+
install_skills = get_input_true_false("Install AI coding skills?")
|
|
430
|
+
skill_targets = None
|
|
431
|
+
if install_skills:
|
|
432
|
+
info_line("Which AI coding tools should receive the skills?")
|
|
433
|
+
skill_targets = select_multiple_from_list(SkillService.TARGETS, "targets")
|
|
427
434
|
run_deps = get_input_true_false("Run dbt deps after downloading packages?")
|
|
428
435
|
bootstrap = BootstrapService(
|
|
429
436
|
snowflake_service=sf_service,
|
|
@@ -431,7 +438,8 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
431
438
|
project_settings=config.project_settings,
|
|
432
439
|
profile=config.profile,
|
|
433
440
|
)
|
|
434
|
-
bootstrap.run(platform=platform, install_skills=install_skills,
|
|
441
|
+
bootstrap.run(platform=platform, install_skills=install_skills,
|
|
442
|
+
skill_targets=skill_targets, run_deps=run_deps)
|
|
435
443
|
|
|
436
444
|
elif option == 5 and has_outdated:
|
|
437
445
|
_update_upstream_packages(config, dbt_runner, upstream_statuses)
|
|
@@ -465,7 +473,7 @@ def _show_main_menu(profile_name: str, git_service: GitService = None,
|
|
|
465
473
|
|
|
466
474
|
def _run_bootstrap(config: Config):
|
|
467
475
|
"""Run the bootstrap command to initialise a new project."""
|
|
468
|
-
from datasecops_cli.utilities.display import get_input_true_false
|
|
476
|
+
from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
|
|
469
477
|
|
|
470
478
|
sf_service = _connect_and_load(config)
|
|
471
479
|
|
|
@@ -477,7 +485,11 @@ def _run_bootstrap(config: Config):
|
|
|
477
485
|
info_line(f"Platform: {platform}")
|
|
478
486
|
|
|
479
487
|
# Ask about optional steps
|
|
480
|
-
install_skills = get_input_true_false("Install
|
|
488
|
+
install_skills = get_input_true_false("Install AI coding skills?")
|
|
489
|
+
skill_targets = None
|
|
490
|
+
if install_skills:
|
|
491
|
+
info_line("Which AI coding tools should receive the skills?")
|
|
492
|
+
skill_targets = select_multiple_from_list(SkillService.TARGETS, "targets")
|
|
481
493
|
run_deps = get_input_true_false("Run dbt deps after downloading packages?")
|
|
482
494
|
|
|
483
495
|
bootstrap = BootstrapService(
|
|
@@ -486,7 +498,8 @@ def _run_bootstrap(config: Config):
|
|
|
486
498
|
project_settings=config.project_settings,
|
|
487
499
|
profile=config.profile,
|
|
488
500
|
)
|
|
489
|
-
success = bootstrap.run(platform=platform, install_skills=install_skills,
|
|
501
|
+
success = bootstrap.run(platform=platform, install_skills=install_skills,
|
|
502
|
+
skill_targets=skill_targets, run_deps=run_deps)
|
|
490
503
|
sys.exit(0 if success else 1)
|
|
491
504
|
|
|
492
505
|
finally:
|
|
@@ -7,7 +7,7 @@ from datasecops_cli.services.dbt_runner import DbtRunner
|
|
|
7
7
|
from datasecops_cli.utilities.display import (
|
|
8
8
|
clear, section_header, display_action_header, menu_option,
|
|
9
9
|
get_input_number, get_input_true_false, complete_action,
|
|
10
|
-
info_line, select_from_list
|
|
10
|
+
info_line, select_from_list, select_multiple_from_list
|
|
11
11
|
)
|
|
12
12
|
|
|
13
13
|
|
|
@@ -74,7 +74,7 @@ class DownloadsMenu:
|
|
|
74
74
|
|
|
75
75
|
def _skills_menu(self) -> None:
|
|
76
76
|
clear()
|
|
77
|
-
display_action_header("
|
|
77
|
+
display_action_header("AI Coding Skills")
|
|
78
78
|
menu_option(1, "list - List available skills")
|
|
79
79
|
menu_option(2, "install - Install a specific skill")
|
|
80
80
|
menu_option(3, "install all - Install all available skills")
|
|
@@ -89,11 +89,19 @@ class DownloadsMenu:
|
|
|
89
89
|
names = {s.name: s for s in available}
|
|
90
90
|
selected = select_from_list(list(names.keys()), "skill")
|
|
91
91
|
if selected != "back" and selected in names:
|
|
92
|
-
self.
|
|
92
|
+
targets = self._select_skill_targets()
|
|
93
|
+
self.skills.install_skill(names[selected], targets=targets)
|
|
93
94
|
else:
|
|
94
95
|
info_line("No skills available")
|
|
95
96
|
elif option == 3:
|
|
96
|
-
self.
|
|
97
|
+
targets = self._select_skill_targets()
|
|
98
|
+
self.skills.install_all(targets=targets)
|
|
97
99
|
elif option == 4:
|
|
98
|
-
self.
|
|
100
|
+
targets = self._select_skill_targets()
|
|
101
|
+
self.skills.update_skills(targets=targets)
|
|
99
102
|
complete_action()
|
|
103
|
+
|
|
104
|
+
def _select_skill_targets(self) -> list[str]:
|
|
105
|
+
"""Prompt user to select which IDE(s) to install skills for."""
|
|
106
|
+
info_line("Which AI coding tools should receive the skills?")
|
|
107
|
+
return select_multiple_from_list(SkillService.TARGETS, "targets")
|
{datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
@@ -23,12 +23,13 @@ class BootstrapService:
|
|
|
23
23
|
self.skill_service = SkillService(snowflake_service)
|
|
24
24
|
|
|
25
25
|
def run(self, platform: str = "github", install_skills: bool = True,
|
|
26
|
-
run_deps: bool = True) -> bool:
|
|
26
|
+
skill_targets: list[str] = None, run_deps: bool = True) -> bool:
|
|
27
27
|
"""Run the full bootstrap process.
|
|
28
28
|
|
|
29
29
|
Args:
|
|
30
30
|
platform: Source control platform - "github" or "azuredevops".
|
|
31
|
-
install_skills: Whether to install
|
|
31
|
+
install_skills: Whether to install AI coding skills.
|
|
32
|
+
skill_targets: List of IDE targets (e.g. ["Cortex Code", "Cursor", "Claude Code"]).
|
|
32
33
|
run_deps: Whether to run dbt deps after downloading packages.
|
|
33
34
|
|
|
34
35
|
Returns True if all steps succeeded.
|
|
@@ -42,16 +43,20 @@ class BootstrapService:
|
|
|
42
43
|
steps_passed = 0
|
|
43
44
|
total_steps = 6 + (1 if install_skills else 0) + (1 if run_deps else 0)
|
|
44
45
|
|
|
45
|
-
# Step 1:
|
|
46
|
-
info_line("[1]
|
|
46
|
+
# Step 1: Scaffold dbt project (dbt_project.yml, profiles.yml, .gitignore, directory structure)
|
|
47
|
+
info_line("[1] Scaffolding dbt project...")
|
|
47
48
|
if not self.download_service.init_dbt_project(self.project_settings, self.profile):
|
|
48
|
-
error_line("Bootstrap failed at:
|
|
49
|
+
error_line("Bootstrap failed at: scaffold dbt project")
|
|
49
50
|
return False
|
|
50
51
|
steps_passed += 1
|
|
51
52
|
|
|
52
|
-
# Resolve the dbt project directory after
|
|
53
|
+
# Resolve the dbt project directory after scaffolding
|
|
53
54
|
dbt_project_dir = self.project_dir
|
|
54
|
-
for candidate in [
|
|
55
|
+
for candidate in [
|
|
56
|
+
self.project_dir / self.project_settings.project_dir.removeprefix("./"),
|
|
57
|
+
self.project_dir / "dbt",
|
|
58
|
+
self.project_dir,
|
|
59
|
+
]:
|
|
55
60
|
if (candidate / "dbt_project.yml").exists():
|
|
56
61
|
dbt_project_dir = candidate
|
|
57
62
|
break
|
|
@@ -84,10 +89,10 @@ class BootstrapService:
|
|
|
84
89
|
info_line(" (skipped - no pipeline config in native app)")
|
|
85
90
|
steps_passed += 1
|
|
86
91
|
|
|
87
|
-
# Step 5:
|
|
92
|
+
# Step 5: Generate packages.yml
|
|
88
93
|
info_line("")
|
|
89
|
-
info_line("[5]
|
|
90
|
-
if self.download_service.download_dbt_packages(dbt_project_dir):
|
|
94
|
+
info_line("[5] Generating packages.yml...")
|
|
95
|
+
if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile):
|
|
91
96
|
steps_passed += 1
|
|
92
97
|
else:
|
|
93
98
|
info_line(" (skipped - no dbt packages config in native app)")
|
|
@@ -123,12 +128,13 @@ class BootstrapService:
|
|
|
123
128
|
info_line(" (install with: uv pip install " + " ".join(dbt_requirements) + ")")
|
|
124
129
|
steps_passed += 1
|
|
125
130
|
|
|
126
|
-
# Step 8: Install
|
|
131
|
+
# Step 8: Install AI coding skills
|
|
127
132
|
if install_skills:
|
|
128
133
|
info_line("")
|
|
129
134
|
step_num += 1
|
|
130
|
-
|
|
131
|
-
|
|
135
|
+
targets_label = ", ".join(skill_targets) if skill_targets else "all"
|
|
136
|
+
info_line(f"[{step_num}] Installing AI coding skills ({targets_label})...")
|
|
137
|
+
count = self.skill_service.install_all(targets=skill_targets)
|
|
132
138
|
steps_passed += 1
|
|
133
139
|
|
|
134
140
|
# Done
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Generate dbt_project.yml matching the DataSecOps Framework skill specification."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import yaml
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# Model type keys that map to dbt_project.yml model sections
|
|
11
|
+
MODEL_SECTION_DEFAULTS: dict[str, dict[str, Any]] = {
|
|
12
|
+
"sources": {"tags": "ods", "materialized": "view"},
|
|
13
|
+
"domains": {"tags": "domains", "materialized": "view"},
|
|
14
|
+
"dimensional_modelling": {"tags": "dm", "materialized": "view"},
|
|
15
|
+
"cortex_models": {"tags": "cortex", "materialized": "view"},
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
# Object type keys that map to dbt_project.yml model sections with doc colors
|
|
19
|
+
OBJECT_SECTION_DEFAULTS: dict[str, dict[str, Any]] = {
|
|
20
|
+
"stored_procedures": {"tags": "stored_procedures", "docs": {"node_color": "DarkOrchid"}},
|
|
21
|
+
"tasks": {"tags": "tasks", "docs": {"node_color": "Coral"}},
|
|
22
|
+
"user_defined_functions": {"tags": "user_defined_functions", "docs": {"node_color": "DarkCyan"}},
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _snake_case(name: str) -> str:
|
|
27
|
+
"""Convert a project name to snake_case."""
|
|
28
|
+
return re.sub(r"[\s\-]+", "_", name).lower()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def generate_dbt_project_yml(
|
|
32
|
+
profile: ProjectProfile,
|
|
33
|
+
project_settings: ProjectSettings,
|
|
34
|
+
) -> str:
|
|
35
|
+
"""Generate a dbt_project.yml string matching the framework skill specification.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
profile: The project profile from the native app.
|
|
39
|
+
project_settings: The project settings from the native app.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
The YAML content as a string.
|
|
43
|
+
"""
|
|
44
|
+
project_name = _snake_case(profile.project_name)
|
|
45
|
+
dbt_version = profile.dbt_version.lstrip("v")
|
|
46
|
+
default_target = project_settings.get_default_target()
|
|
47
|
+
target_warehouse = default_target.target_warehouse if default_target else "DEV_WH"
|
|
48
|
+
|
|
49
|
+
# Build the models section based on profile's model_types
|
|
50
|
+
models_section: dict[str, Any] = {
|
|
51
|
+
"+persist_docs": {"relation": True, "columns": True},
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
for model_type in profile.model_types:
|
|
55
|
+
defaults = MODEL_SECTION_DEFAULTS.get(model_type)
|
|
56
|
+
if defaults:
|
|
57
|
+
# Map model type keys to shorter dbt_project.yml section names
|
|
58
|
+
key_map = {"dimensional_modelling": "dm", "cortex_models": "cortex"}
|
|
59
|
+
section_key = key_map.get(model_type, model_type)
|
|
60
|
+
section = {f"+tags": defaults["tags"], "+materialized": defaults["materialized"]}
|
|
61
|
+
models_section[section_key] = section
|
|
62
|
+
|
|
63
|
+
# Add object sections
|
|
64
|
+
for obj_key, obj_defaults in OBJECT_SECTION_DEFAULTS.items():
|
|
65
|
+
section: dict[str, Any] = {f"+tags": obj_defaults["tags"]}
|
|
66
|
+
if "docs" in obj_defaults:
|
|
67
|
+
section["+docs"] = obj_defaults["docs"]
|
|
68
|
+
models_section[obj_key] = section
|
|
69
|
+
|
|
70
|
+
project = {
|
|
71
|
+
"name": project_name,
|
|
72
|
+
"version": "1.0.0",
|
|
73
|
+
"config-version": 2,
|
|
74
|
+
"profile": profile.profile_name,
|
|
75
|
+
"model-paths": ["models", "objects"],
|
|
76
|
+
"analysis-paths": ["analyses"],
|
|
77
|
+
"test-paths": ["tests"],
|
|
78
|
+
"seed-paths": ["seeds"],
|
|
79
|
+
"macro-paths": ["macros"],
|
|
80
|
+
"snapshot-paths": ["snapshots"],
|
|
81
|
+
"clean-targets": ["target", "dbt_packages"],
|
|
82
|
+
"require-dbt-version": f">={dbt_version}",
|
|
83
|
+
"models": {project_name: models_section},
|
|
84
|
+
"quoting": {"database": False, "schema": False, "identifier": False},
|
|
85
|
+
"dispatch": [
|
|
86
|
+
{
|
|
87
|
+
"macro_namespace": "dbt",
|
|
88
|
+
"search_order": [
|
|
89
|
+
project_name,
|
|
90
|
+
"dbt_dataengineers_utils",
|
|
91
|
+
"dbt_dataengineers_materializations",
|
|
92
|
+
"dbt",
|
|
93
|
+
],
|
|
94
|
+
}
|
|
95
|
+
],
|
|
96
|
+
"on-run-start": [],
|
|
97
|
+
"on-run-end": [
|
|
98
|
+
"{{ dbt_dataengineers_utils.clean_objects(database=target.database, "
|
|
99
|
+
"clean_targets=['local-dev', 'test', 'prod'], "
|
|
100
|
+
"object_types=['schemas', 'functions_and_procedures', 'tasks', 'streams', "
|
|
101
|
+
"'stages', 'tables_and_views', 'alerts', 'file_formats', 'semantic_views', 'agents']) }}"
|
|
102
|
+
],
|
|
103
|
+
"vars": {"dbt_project_target_warehouse": target_warehouse},
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return yaml.dump(project, default_flow_style=False, sort_keys=False, allow_unicode=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def generate_profiles_yml(
|
|
110
|
+
profile: ProjectProfile,
|
|
111
|
+
project_settings: ProjectSettings,
|
|
112
|
+
account: str,
|
|
113
|
+
user: str = "",
|
|
114
|
+
) -> str:
|
|
115
|
+
"""Generate profiles.yml from framework targets.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
profile: The project profile.
|
|
119
|
+
project_settings: The project settings with targets.
|
|
120
|
+
account: Snowflake account name.
|
|
121
|
+
user: Snowflake user name.
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
The YAML content as a string.
|
|
125
|
+
"""
|
|
126
|
+
outputs: dict[str, Any] = {}
|
|
127
|
+
|
|
128
|
+
# Add local-dev target
|
|
129
|
+
outputs["local-dev"] = {
|
|
130
|
+
"type": "snowflake",
|
|
131
|
+
"account": account,
|
|
132
|
+
"authenticator": "externalbrowser",
|
|
133
|
+
"role": project_settings.targets[0].target_role if project_settings.targets else "DEVELOPERS",
|
|
134
|
+
"warehouse": project_settings.targets[0].target_warehouse if project_settings.targets else "DEV_WH",
|
|
135
|
+
"database": profile.target_database,
|
|
136
|
+
"schema": project_settings.targets[0].target_schema if project_settings.targets else "PUBLIC",
|
|
137
|
+
"threads": 4,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
# Add all framework targets
|
|
141
|
+
for target in project_settings.targets:
|
|
142
|
+
branch_suffix = target.branch_name.upper()
|
|
143
|
+
outputs[target.target_name] = {
|
|
144
|
+
"type": "snowflake",
|
|
145
|
+
"account": account,
|
|
146
|
+
"user": user or "{{ env_var('SNOWFLAKE_USER') }}",
|
|
147
|
+
"role": target.target_role,
|
|
148
|
+
"warehouse": target.target_warehouse,
|
|
149
|
+
"database": f"{profile.target_database}_{branch_suffix}",
|
|
150
|
+
"schema": target.target_schema,
|
|
151
|
+
"threads": 4,
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
default_target = project_settings.get_default_target()
|
|
155
|
+
profiles_data = {
|
|
156
|
+
profile.profile_name: {
|
|
157
|
+
"target": default_target.target_name if default_target else "local-dev",
|
|
158
|
+
"outputs": outputs,
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return yaml.dump(profiles_data, default_flow_style=False, sort_keys=False)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def generate_packages_yml(
|
|
166
|
+
profile: ProjectProfile,
|
|
167
|
+
packages_config: dict,
|
|
168
|
+
) -> str:
|
|
169
|
+
"""Generate packages.yml from profile's packages and upstream projects.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
profile: The project profile.
|
|
173
|
+
packages_config: The DBT_PACKAGES framework config.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
The YAML content as a string.
|
|
177
|
+
"""
|
|
178
|
+
packages = packages_config.get("packages", [])
|
|
179
|
+
pkg_list: list[dict[str, Any]] = []
|
|
180
|
+
|
|
181
|
+
for pkg in packages:
|
|
182
|
+
source = pkg.get("source", "git")
|
|
183
|
+
url = pkg.get("url", "")
|
|
184
|
+
version = pkg.get("latest_version", "")
|
|
185
|
+
if source == "git" and url:
|
|
186
|
+
entry: dict[str, Any] = {"git": url}
|
|
187
|
+
if version:
|
|
188
|
+
entry["revision"] = version
|
|
189
|
+
pkg_list.append(entry)
|
|
190
|
+
elif source == "package" and url:
|
|
191
|
+
entry = {"package": url}
|
|
192
|
+
if version:
|
|
193
|
+
entry["version"] = version
|
|
194
|
+
pkg_list.append(entry)
|
|
195
|
+
|
|
196
|
+
# Add upstream project dependencies as local packages
|
|
197
|
+
for upstream_name in profile.upstream_projects:
|
|
198
|
+
pkg_list.append({"local": f"local_packages/{upstream_name}"})
|
|
199
|
+
|
|
200
|
+
return yaml.dump({"packages": pkg_list}, default_flow_style=False, sort_keys=False)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def generate_gitignore() -> str:
|
|
204
|
+
"""Generate a .gitignore for dbt projects."""
|
|
205
|
+
return """target/
|
|
206
|
+
dbt_packages/
|
|
207
|
+
logs/
|
|
208
|
+
*.pyc
|
|
209
|
+
__pycache__/
|
|
210
|
+
profiles.yml
|
|
211
|
+
"""
|