datasecops-cli 0.3.3__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. datasecops_cli-0.4.0/.github/workflows/auto-tag.yml +113 -0
  2. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/CHANGELOG.md +46 -0
  3. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/PKG-INFO +2 -2
  4. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/README.md +1 -1
  5. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/pyproject.toml +1 -1
  6. datasecops_cli-0.4.0/src/datasecops_cli/__init__.py +1 -0
  7. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/main.py +22 -9
  8. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/downloads.py +13 -5
  9. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/bootstrap_service.py +19 -13
  10. datasecops_cli-0.4.0/src/datasecops_cli/services/dbt_project_generator.py +211 -0
  11. datasecops_cli-0.4.0/src/datasecops_cli/services/directory_scaffolder.py +81 -0
  12. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/download_service.py +98 -57
  13. datasecops_cli-0.4.0/src/datasecops_cli/services/skill_service.py +167 -0
  14. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/snowflake_service.py +15 -0
  15. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/display.py +21 -0
  16. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/file_utils.py +8 -0
  17. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/connection.py +36 -3
  18. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/server.py +77 -0
  19. datasecops_cli-0.3.3/src/datasecops_cli/__init__.py +0 -1
  20. datasecops_cli-0.3.3/src/datasecops_cli/services/skill_service.py +0 -86
  21. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/.github/workflows/publish-cli.yml +0 -0
  22. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/.gitignore +0 -0
  23. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/DEVELOPMENT.md +0 -0
  24. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/LICENSE +0 -0
  25. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/getting-started.md +0 -0
  26. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/legacy.md +0 -0
  27. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/legacy_plan_of_action.md +0 -0
  28. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/docs/mcp-server.md +0 -0
  29. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/mcp-servers.json +0 -0
  30. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/setup.ps1 +0 -0
  31. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/setup.sh +0 -0
  32. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/config.py +0 -0
  33. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/__init__.py +0 -0
  34. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/development.py +0 -0
  35. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/menus/git_operations.py +0 -0
  36. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/__init__.py +0 -0
  37. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/git_helpers.py +0 -0
  38. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/models/project_config.py +0 -0
  39. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/__init__.py +0 -0
  40. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/dbt_runner.py +0 -0
  41. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/git_service.py +0 -0
  42. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/linting_service.py +0 -0
  43. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/services/upstream_service.py +0 -0
  44. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/__init__.py +0 -0
  45. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  46. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/__init__.py +0 -0
  47. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/src/datasecops_mcp/__main__.py +0 -0
  48. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/__init__.py +0 -0
  49. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_config.py +0 -0
  50. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_file_utils.py +0 -0
  51. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_main.py +0 -0
  52. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_models.py +0 -0
  53. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_version.py +0 -0
  54. {datasecops_cli-0.3.3 → datasecops_cli-0.4.0}/tests/test_yaml_utils.py +0 -0
@@ -0,0 +1,113 @@
1
+ name: Tag and Deploy
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+
7
+ permissions:
8
+ id-token: write
9
+ contents: write
10
+
11
+ jobs:
12
+ auto-tag:
13
+ runs-on: ubuntu-latest
14
+ outputs:
15
+ tag: ${{ steps.tag.outputs.tag }}
16
+ created: ${{ steps.tag.outputs.created }}
17
+
18
+ steps:
19
+ - name: Checkout repository
20
+ uses: actions/checkout@v4
21
+
22
+ - name: Extract version from pyproject.toml
23
+ id: version
24
+ run: |
25
+ set -euo pipefail
26
+ VERSION=$(python3 -c "
27
+ import tomllib, pathlib
28
+ data = tomllib.loads(pathlib.Path('pyproject.toml').read_text())
29
+ print(data['project']['version'])
30
+ ")
31
+ if [ -z "$VERSION" ]; then
32
+ echo "Failed to extract version from pyproject.toml" >&2
33
+ exit 1
34
+ fi
35
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
36
+ echo "Detected version: $VERSION"
37
+
38
+ - name: Create and push tag
39
+ id: tag
40
+ run: |
41
+ TAG="cli-v${{ steps.version.outputs.version }}"
42
+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"
43
+ if git ls-remote --tags origin "refs/tags/$TAG" | grep -q "$TAG"; then
44
+ echo "Tag $TAG already exists, skipping."
45
+ echo "created=false" >> "$GITHUB_OUTPUT"
46
+ exit 0
47
+ fi
48
+ git tag "$TAG"
49
+ PUSH_OUTPUT=$(git push origin "$TAG" 2>&1) && {
50
+ echo "Created and pushed tag: $TAG"
51
+ echo "created=true" >> "$GITHUB_OUTPUT"
52
+ } || {
53
+ if echo "$PUSH_OUTPUT" | grep -qi "already exists"; then
54
+ echo "Tag $TAG was created by another run, skipping."
55
+ echo "created=false" >> "$GITHUB_OUTPUT"
56
+ else
57
+ echo "Tag push failed unexpectedly:"
58
+ echo "$PUSH_OUTPUT"
59
+ exit 1
60
+ fi
61
+ }
62
+
63
+ test:
64
+ name: Test (Python ${{ matrix.python-version }})
65
+ needs: auto-tag
66
+ if: needs.auto-tag.outputs.created == 'true'
67
+ runs-on: ubuntu-latest
68
+ strategy:
69
+ matrix:
70
+ python-version: ['3.11', '3.12', '3.13']
71
+
72
+ steps:
73
+ - name: Checkout repository
74
+ uses: actions/checkout@v4
75
+ with:
76
+ ref: ${{ needs.auto-tag.outputs.tag }}
77
+
78
+ - name: Set up Python ${{ matrix.python-version }}
79
+ uses: actions/setup-python@v5
80
+ with:
81
+ python-version: ${{ matrix.python-version }}
82
+
83
+ - name: Install package with test dependencies
84
+ run: pip install -e ".[test]"
85
+
86
+ - name: Run tests
87
+ run: pytest --tb=short
88
+
89
+ build-and-publish:
90
+ name: Build and publish to PyPI
91
+ needs: [auto-tag, test]
92
+ if: needs.auto-tag.outputs.created == 'true'
93
+ runs-on: ubuntu-latest
94
+
95
+ steps:
96
+ - name: Checkout repository
97
+ uses: actions/checkout@v4
98
+ with:
99
+ ref: ${{ needs.auto-tag.outputs.tag }}
100
+
101
+ - name: Set up Python
102
+ uses: actions/setup-python@v5
103
+ with:
104
+ python-version: '3.11'
105
+
106
+ - name: Install build tools
107
+ run: pip install build
108
+
109
+ - name: Build package
110
+ run: python -m build
111
+
112
+ - name: Publish to PyPI
113
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -2,6 +2,52 @@
2
2
 
3
3
  All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
+ ## [0.4.0] - 2026-05-16
6
+
7
+ ### Added
8
+
9
+ - **Framework-aligned project scaffolding** — `datasecops bootstrap` now generates `dbt_project.yml` directly from the native app configuration instead of running `dbtf init`. The generated project matches the `setup-dbt-project` Cortex Code skill specification, including:
10
+ - Model sections based on profile's `MODEL_TYPES` (sources, dimensional_modelling, domains, cortex_models)
11
+ - Object sections with doc node colors (stored_procedures, tasks, UDFs)
12
+ - `dispatch` with framework package search order
13
+ - `on-run-end` with `clean_objects` macro
14
+ - `quoting` and `vars` configuration
15
+ - **Directory structure scaffolding** — bootstrap creates the full directory tree with `.gitkeep` files in empty model/object subdirectories (models/sources/, models/dm/dims/, objects/stored_procedures/, etc.)
16
+ - **`.gitignore` generation** — bootstrap creates a `.gitignore` with standard dbt exclusions including `profiles.yml`
17
+ - **Macro category subdirectories** — macros are now written to `macros/{category}/{file_name}` (e.g., `macros/schema/generate_schema_name.sql`) matching the framework's category organisation
18
+ - **Upstream project support in packages.yml** — profiles with `upstream_projects` configured now generate `local: "local_packages/{name}"` entries in `packages.yml`
19
+ - **profiles.yml database suffixes** — each target's database name now includes the branch suffix in uppercase (e.g., `MY_DB_PROD`) matching the skill specification
20
+ - **New MCP tools** — 5 new tools exposed via the MCP server:
21
+ - `get_model_definitions` — model types with paths, prefixes, and metadata fields
22
+ - `get_object_definitions` — object types with paths, prefixes, and options
23
+ - `get_source_definitions` — configured source systems and their schemas
24
+ - `get_access_groups` — RBAC access groups with owners
25
+ - `get_support_contacts` — team contact details
26
+ - **New native app stored procedures** — `api.get_model_definitions()`, `api.get_object_definitions()`, `api.get_source_definitions()`, `api.get_access_groups()`, `api.get_support_contacts()` for CLI/MCP consumption
27
+ - **New SnowflakeService methods** — both CLI and MCP Snowflake services expose the new stored procedures
28
+ - **Multi-IDE skill installation** — skills now install to Cursor (`.cursor/rules/{id}.mdc` with frontmatter) and Claude Code (`.claude/commands/{id}.md`) in addition to Cortex Code. Bootstrap and downloads menu prompt users to select target IDE(s) via a multi-select prompt (`[1] Cortex Code`, `[2] Cursor`, `[3] Claude Code`, `[a] All`)
29
+ - **`select_multiple_from_list` display helper** — new utility for comma-separated multi-select input with "all" shortcut
30
+
31
+ ### Changed
32
+
33
+ - **Bootstrap no longer requires `dbtf` for init** — project scaffolding is now pure Python, removing the dependency on having dbt Fusion installed before bootstrapping
34
+ - **`init_dbt_project()` rewritten** — replaced `subprocess.run(["dbtf", "init"])` with framework-specific `dbt_project_generator` and `directory_scaffolder` services
35
+
36
+ ### Fixed
37
+
38
+ - **Version drift** — `__init__.py` version now matches `pyproject.toml` (was stuck at `0.1.0`)
39
+ - **Path safety** — replaced unsafe `lstrip("./")` with `removeprefix("./")` in bootstrap and download services
40
+ - **Path traversal prevention** — macro category and file names sanitized with `Path.name` before writing
41
+ - **MCP SQL safety** — added `_ALLOWED_PROCEDURES` frozenset to `MCPSnowflakeService._call_procedure` to prevent arbitrary procedure calls
42
+ - **Narrowed exception handling** — replaced bare `Exception` with `OSError` in skill installation; failed installs no longer added to success list
43
+ - **Cursor `.mdc` frontmatter** — fixed `globs:` to `globs: []` for valid YAML
44
+
45
+ ## [0.3.4] - 2026-05-15
46
+
47
+ ### Added
48
+
49
+ - **`scripts` download item** — `datasecops download scripts` downloads pipeline scripts from the native app to `.datasecops/scripts/`. Included in `datasecops download all`.
50
+
5
51
  ## [0.3.3] - 2026-05-15
6
52
 
7
53
  ### Changed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasecops-cli
3
- Version: 0.3.3
3
+ Version: 0.4.0
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -105,7 +105,7 @@ datasecops download sqlfluff install-sqlfluff
105
105
  datasecops download all
106
106
  ```
107
107
 
108
- Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
108
+ Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `scripts`, `install-sqlfluff`, `install-dbt`, `all`
109
109
 
110
110
  The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
111
111
 
@@ -86,7 +86,7 @@ datasecops download sqlfluff install-sqlfluff
86
86
  datasecops download all
87
87
  ```
88
88
 
89
- Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `install-sqlfluff`, `install-dbt`, `all`
89
+ Available items: `sqlfluff`, `pipelines`, `packages`, `macros`, `scripts`, `install-sqlfluff`, `install-dbt`, `all`
90
90
 
91
91
  The pipeline platform (GitHub / Azure DevOps) is auto-detected from the native app's source control configuration.
92
92
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.3.3"
7
+ version = "0.4.0"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1 @@
1
+ __version__ = "0.4.0"
@@ -24,7 +24,7 @@ from datasecops_cli.utilities.yaml_utils import write_datasecops_config
24
24
 
25
25
 
26
26
  DOWNLOAD_ITEMS = [
27
- "sqlfluff", "pipelines", "packages", "macros",
27
+ "sqlfluff", "pipelines", "packages", "macros", "scripts",
28
28
  "install-sqlfluff", "install-dbt", "all",
29
29
  ]
30
30
 
@@ -44,7 +44,7 @@ def _build_parser() -> argparse.ArgumentParser:
44
44
  "items",
45
45
  nargs="+",
46
46
  choices=DOWNLOAD_ITEMS,
47
- help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
47
+ help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, scripts, install-sqlfluff, install-dbt, or all",
48
48
  )
49
49
  dl.add_argument(
50
50
  "--connection", "-c",
@@ -309,7 +309,7 @@ def _run_download(config: Config, items: list[str],
309
309
  profiles_dir = str(config.get_dbt_profiles_dir())
310
310
 
311
311
  if "all" in items:
312
- items = ["sqlfluff", "pipelines", "packages", "macros",
312
+ items = ["sqlfluff", "pipelines", "packages", "macros", "scripts",
313
313
  "install-sqlfluff", "install-dbt"]
314
314
 
315
315
  failed = False
@@ -331,6 +331,9 @@ def _run_download(config: Config, items: list[str],
331
331
  elif item == "macros":
332
332
  if not download_service.download_macros(config.profile_name, config.dbt_project_dir):
333
333
  failed = True
334
+ elif item == "scripts":
335
+ if not download_service.download_scripts():
336
+ failed = True
334
337
  elif item == "install-sqlfluff":
335
338
  packages = download_service.get_sqlfluff_requirements()
336
339
  if packages:
@@ -420,10 +423,14 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
420
423
  dl_menu.show()
421
424
 
422
425
  elif option == 4:
423
- from datasecops_cli.utilities.display import get_input_true_false
426
+ from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
424
427
  platform = config.source_control.source_control_platform.lower()
425
428
  info_line(f"Platform: {platform}")
426
- install_skills = get_input_true_false("Install Cortex Code skills?")
429
+ install_skills = get_input_true_false("Install AI coding skills?")
430
+ skill_targets = None
431
+ if install_skills:
432
+ info_line("Which AI coding tools should receive the skills?")
433
+ skill_targets = select_multiple_from_list(SkillService.TARGETS, "targets")
427
434
  run_deps = get_input_true_false("Run dbt deps after downloading packages?")
428
435
  bootstrap = BootstrapService(
429
436
  snowflake_service=sf_service,
@@ -431,7 +438,8 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
431
438
  project_settings=config.project_settings,
432
439
  profile=config.profile,
433
440
  )
434
- bootstrap.run(platform=platform, install_skills=install_skills, run_deps=run_deps)
441
+ bootstrap.run(platform=platform, install_skills=install_skills,
442
+ skill_targets=skill_targets, run_deps=run_deps)
435
443
 
436
444
  elif option == 5 and has_outdated:
437
445
  _update_upstream_packages(config, dbt_runner, upstream_statuses)
@@ -465,7 +473,7 @@ def _show_main_menu(profile_name: str, git_service: GitService = None,
465
473
 
466
474
  def _run_bootstrap(config: Config):
467
475
  """Run the bootstrap command to initialise a new project."""
468
- from datasecops_cli.utilities.display import get_input_true_false
476
+ from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
469
477
 
470
478
  sf_service = _connect_and_load(config)
471
479
 
@@ -477,7 +485,11 @@ def _run_bootstrap(config: Config):
477
485
  info_line(f"Platform: {platform}")
478
486
 
479
487
  # Ask about optional steps
480
- install_skills = get_input_true_false("Install Cortex Code skills?")
488
+ install_skills = get_input_true_false("Install AI coding skills?")
489
+ skill_targets = None
490
+ if install_skills:
491
+ info_line("Which AI coding tools should receive the skills?")
492
+ skill_targets = select_multiple_from_list(SkillService.TARGETS, "targets")
481
493
  run_deps = get_input_true_false("Run dbt deps after downloading packages?")
482
494
 
483
495
  bootstrap = BootstrapService(
@@ -486,7 +498,8 @@ def _run_bootstrap(config: Config):
486
498
  project_settings=config.project_settings,
487
499
  profile=config.profile,
488
500
  )
489
- success = bootstrap.run(platform=platform, install_skills=install_skills, run_deps=run_deps)
501
+ success = bootstrap.run(platform=platform, install_skills=install_skills,
502
+ skill_targets=skill_targets, run_deps=run_deps)
490
503
  sys.exit(0 if success else 1)
491
504
 
492
505
  finally:
@@ -7,7 +7,7 @@ from datasecops_cli.services.dbt_runner import DbtRunner
7
7
  from datasecops_cli.utilities.display import (
8
8
  clear, section_header, display_action_header, menu_option,
9
9
  get_input_number, get_input_true_false, complete_action,
10
- info_line, select_from_list
10
+ info_line, select_from_list, select_multiple_from_list
11
11
  )
12
12
 
13
13
 
@@ -74,7 +74,7 @@ class DownloadsMenu:
74
74
 
75
75
  def _skills_menu(self) -> None:
76
76
  clear()
77
- display_action_header("Cortex Code Skills")
77
+ display_action_header("AI Coding Skills")
78
78
  menu_option(1, "list - List available skills")
79
79
  menu_option(2, "install - Install a specific skill")
80
80
  menu_option(3, "install all - Install all available skills")
@@ -89,11 +89,19 @@ class DownloadsMenu:
89
89
  names = {s.name: s for s in available}
90
90
  selected = select_from_list(list(names.keys()), "skill")
91
91
  if selected != "back" and selected in names:
92
- self.skills.install_skill(names[selected])
92
+ targets = self._select_skill_targets()
93
+ self.skills.install_skill(names[selected], targets=targets)
93
94
  else:
94
95
  info_line("No skills available")
95
96
  elif option == 3:
96
- self.skills.install_all()
97
+ targets = self._select_skill_targets()
98
+ self.skills.install_all(targets=targets)
97
99
  elif option == 4:
98
- self.skills.update_skills()
100
+ targets = self._select_skill_targets()
101
+ self.skills.update_skills(targets=targets)
99
102
  complete_action()
103
+
104
+ def _select_skill_targets(self) -> list[str]:
105
+ """Prompt user to select which IDE(s) to install skills for."""
106
+ info_line("Which AI coding tools should receive the skills?")
107
+ return select_multiple_from_list(SkillService.TARGETS, "targets")
@@ -23,12 +23,13 @@ class BootstrapService:
23
23
  self.skill_service = SkillService(snowflake_service)
24
24
 
25
25
  def run(self, platform: str = "github", install_skills: bool = True,
26
- run_deps: bool = True) -> bool:
26
+ skill_targets: list[str] = None, run_deps: bool = True) -> bool:
27
27
  """Run the full bootstrap process.
28
28
 
29
29
  Args:
30
30
  platform: Source control platform - "github" or "azuredevops".
31
- install_skills: Whether to install Cortex Code skills.
31
+ install_skills: Whether to install AI coding skills.
32
+ skill_targets: List of IDE targets (e.g. ["Cortex Code", "Cursor", "Claude Code"]).
32
33
  run_deps: Whether to run dbt deps after downloading packages.
33
34
 
34
35
  Returns True if all steps succeeded.
@@ -42,16 +43,20 @@ class BootstrapService:
42
43
  steps_passed = 0
43
44
  total_steps = 6 + (1 if install_skills else 0) + (1 if run_deps else 0)
44
45
 
45
- # Step 1: Init dbt project
46
- info_line("[1] Initialising dbt project...")
46
+ # Step 1: Scaffold dbt project (dbt_project.yml, profiles.yml, .gitignore, directory structure)
47
+ info_line("[1] Scaffolding dbt project...")
47
48
  if not self.download_service.init_dbt_project(self.project_settings, self.profile):
48
- error_line("Bootstrap failed at: init dbt project")
49
+ error_line("Bootstrap failed at: scaffold dbt project")
49
50
  return False
50
51
  steps_passed += 1
51
52
 
52
- # Resolve the dbt project directory after init
53
+ # Resolve the dbt project directory after scaffolding
53
54
  dbt_project_dir = self.project_dir
54
- for candidate in [self.project_dir / "dbt", self.project_dir]:
55
+ for candidate in [
56
+ self.project_dir / self.project_settings.project_dir.removeprefix("./"),
57
+ self.project_dir / "dbt",
58
+ self.project_dir,
59
+ ]:
55
60
  if (candidate / "dbt_project.yml").exists():
56
61
  dbt_project_dir = candidate
57
62
  break
@@ -84,10 +89,10 @@ class BootstrapService:
84
89
  info_line(" (skipped - no pipeline config in native app)")
85
90
  steps_passed += 1
86
91
 
87
- # Step 5: Download dbt packages
92
+ # Step 5: Generate packages.yml
88
93
  info_line("")
89
- info_line("[5] Downloading dbt packages...")
90
- if self.download_service.download_dbt_packages(dbt_project_dir):
94
+ info_line("[5] Generating packages.yml...")
95
+ if self.download_service.download_dbt_packages(dbt_project_dir, profile=self.profile):
91
96
  steps_passed += 1
92
97
  else:
93
98
  info_line(" (skipped - no dbt packages config in native app)")
@@ -123,12 +128,13 @@ class BootstrapService:
123
128
  info_line(" (install with: uv pip install " + " ".join(dbt_requirements) + ")")
124
129
  steps_passed += 1
125
130
 
126
- # Step 8: Install Cortex Code skills
131
+ # Step 8: Install AI coding skills
127
132
  if install_skills:
128
133
  info_line("")
129
134
  step_num += 1
130
- info_line(f"[{step_num}] Installing Cortex Code skills...")
131
- count = self.skill_service.install_all()
135
+ targets_label = ", ".join(skill_targets) if skill_targets else "all"
136
+ info_line(f"[{step_num}] Installing AI coding skills ({targets_label})...")
137
+ count = self.skill_service.install_all(targets=skill_targets)
132
138
  steps_passed += 1
133
139
 
134
140
  # Done
@@ -0,0 +1,211 @@
1
+ """Generate dbt_project.yml matching the DataSecOps Framework skill specification."""
2
+
3
+ import re
4
+ import yaml
5
+ from typing import Any
6
+
7
+ from datasecops_cli.models.project_config import ProjectProfile, ProjectSettings
8
+
9
+
10
+ # Model type keys that map to dbt_project.yml model sections
11
+ MODEL_SECTION_DEFAULTS: dict[str, dict[str, Any]] = {
12
+ "sources": {"tags": "ods", "materialized": "view"},
13
+ "domains": {"tags": "domains", "materialized": "view"},
14
+ "dimensional_modelling": {"tags": "dm", "materialized": "view"},
15
+ "cortex_models": {"tags": "cortex", "materialized": "view"},
16
+ }
17
+
18
+ # Object type keys that map to dbt_project.yml model sections with doc colors
19
+ OBJECT_SECTION_DEFAULTS: dict[str, dict[str, Any]] = {
20
+ "stored_procedures": {"tags": "stored_procedures", "docs": {"node_color": "DarkOrchid"}},
21
+ "tasks": {"tags": "tasks", "docs": {"node_color": "Coral"}},
22
+ "user_defined_functions": {"tags": "user_defined_functions", "docs": {"node_color": "DarkCyan"}},
23
+ }
24
+
25
+
26
+ def _snake_case(name: str) -> str:
27
+ """Convert a project name to snake_case."""
28
+ return re.sub(r"[\s\-]+", "_", name).lower()
29
+
30
+
31
+ def generate_dbt_project_yml(
32
+ profile: ProjectProfile,
33
+ project_settings: ProjectSettings,
34
+ ) -> str:
35
+ """Generate a dbt_project.yml string matching the framework skill specification.
36
+
37
+ Args:
38
+ profile: The project profile from the native app.
39
+ project_settings: The project settings from the native app.
40
+
41
+ Returns:
42
+ The YAML content as a string.
43
+ """
44
+ project_name = _snake_case(profile.project_name)
45
+ dbt_version = profile.dbt_version.lstrip("v")
46
+ default_target = project_settings.get_default_target()
47
+ target_warehouse = default_target.target_warehouse if default_target else "DEV_WH"
48
+
49
+ # Build the models section based on profile's model_types
50
+ models_section: dict[str, Any] = {
51
+ "+persist_docs": {"relation": True, "columns": True},
52
+ }
53
+
54
+ for model_type in profile.model_types:
55
+ defaults = MODEL_SECTION_DEFAULTS.get(model_type)
56
+ if defaults:
57
+ # Map model type keys to shorter dbt_project.yml section names
58
+ key_map = {"dimensional_modelling": "dm", "cortex_models": "cortex"}
59
+ section_key = key_map.get(model_type, model_type)
60
+ section = {f"+tags": defaults["tags"], "+materialized": defaults["materialized"]}
61
+ models_section[section_key] = section
62
+
63
+ # Add object sections
64
+ for obj_key, obj_defaults in OBJECT_SECTION_DEFAULTS.items():
65
+ section: dict[str, Any] = {f"+tags": obj_defaults["tags"]}
66
+ if "docs" in obj_defaults:
67
+ section["+docs"] = obj_defaults["docs"]
68
+ models_section[obj_key] = section
69
+
70
+ project = {
71
+ "name": project_name,
72
+ "version": "1.0.0",
73
+ "config-version": 2,
74
+ "profile": profile.profile_name,
75
+ "model-paths": ["models", "objects"],
76
+ "analysis-paths": ["analyses"],
77
+ "test-paths": ["tests"],
78
+ "seed-paths": ["seeds"],
79
+ "macro-paths": ["macros"],
80
+ "snapshot-paths": ["snapshots"],
81
+ "clean-targets": ["target", "dbt_packages"],
82
+ "require-dbt-version": f">={dbt_version}",
83
+ "models": {project_name: models_section},
84
+ "quoting": {"database": False, "schema": False, "identifier": False},
85
+ "dispatch": [
86
+ {
87
+ "macro_namespace": "dbt",
88
+ "search_order": [
89
+ project_name,
90
+ "dbt_dataengineers_utils",
91
+ "dbt_dataengineers_materializations",
92
+ "dbt",
93
+ ],
94
+ }
95
+ ],
96
+ "on-run-start": [],
97
+ "on-run-end": [
98
+ "{{ dbt_dataengineers_utils.clean_objects(database=target.database, "
99
+ "clean_targets=['local-dev', 'test', 'prod'], "
100
+ "object_types=['schemas', 'functions_and_procedures', 'tasks', 'streams', "
101
+ "'stages', 'tables_and_views', 'alerts', 'file_formats', 'semantic_views', 'agents']) }}"
102
+ ],
103
+ "vars": {"dbt_project_target_warehouse": target_warehouse},
104
+ }
105
+
106
+ return yaml.dump(project, default_flow_style=False, sort_keys=False, allow_unicode=True)
107
+
108
+
109
+ def generate_profiles_yml(
110
+ profile: ProjectProfile,
111
+ project_settings: ProjectSettings,
112
+ account: str,
113
+ user: str = "",
114
+ ) -> str:
115
+ """Generate profiles.yml from framework targets.
116
+
117
+ Args:
118
+ profile: The project profile.
119
+ project_settings: The project settings with targets.
120
+ account: Snowflake account name.
121
+ user: Snowflake user name.
122
+
123
+ Returns:
124
+ The YAML content as a string.
125
+ """
126
+ outputs: dict[str, Any] = {}
127
+
128
+ # Add local-dev target
129
+ outputs["local-dev"] = {
130
+ "type": "snowflake",
131
+ "account": account,
132
+ "authenticator": "externalbrowser",
133
+ "role": project_settings.targets[0].target_role if project_settings.targets else "DEVELOPERS",
134
+ "warehouse": project_settings.targets[0].target_warehouse if project_settings.targets else "DEV_WH",
135
+ "database": profile.target_database,
136
+ "schema": project_settings.targets[0].target_schema if project_settings.targets else "PUBLIC",
137
+ "threads": 4,
138
+ }
139
+
140
+ # Add all framework targets
141
+ for target in project_settings.targets:
142
+ branch_suffix = target.branch_name.upper()
143
+ outputs[target.target_name] = {
144
+ "type": "snowflake",
145
+ "account": account,
146
+ "user": user or "{{ env_var('SNOWFLAKE_USER') }}",
147
+ "role": target.target_role,
148
+ "warehouse": target.target_warehouse,
149
+ "database": f"{profile.target_database}_{branch_suffix}",
150
+ "schema": target.target_schema,
151
+ "threads": 4,
152
+ }
153
+
154
+ default_target = project_settings.get_default_target()
155
+ profiles_data = {
156
+ profile.profile_name: {
157
+ "target": default_target.target_name if default_target else "local-dev",
158
+ "outputs": outputs,
159
+ }
160
+ }
161
+
162
+ return yaml.dump(profiles_data, default_flow_style=False, sort_keys=False)
163
+
164
+
165
+ def generate_packages_yml(
166
+ profile: ProjectProfile,
167
+ packages_config: dict,
168
+ ) -> str:
169
+ """Generate packages.yml from profile's packages and upstream projects.
170
+
171
+ Args:
172
+ profile: The project profile.
173
+ packages_config: The DBT_PACKAGES framework config.
174
+
175
+ Returns:
176
+ The YAML content as a string.
177
+ """
178
+ packages = packages_config.get("packages", [])
179
+ pkg_list: list[dict[str, Any]] = []
180
+
181
+ for pkg in packages:
182
+ source = pkg.get("source", "git")
183
+ url = pkg.get("url", "")
184
+ version = pkg.get("latest_version", "")
185
+ if source == "git" and url:
186
+ entry: dict[str, Any] = {"git": url}
187
+ if version:
188
+ entry["revision"] = version
189
+ pkg_list.append(entry)
190
+ elif source == "package" and url:
191
+ entry = {"package": url}
192
+ if version:
193
+ entry["version"] = version
194
+ pkg_list.append(entry)
195
+
196
+ # Add upstream project dependencies as local packages
197
+ for upstream_name in profile.upstream_projects:
198
+ pkg_list.append({"local": f"local_packages/{upstream_name}"})
199
+
200
+ return yaml.dump({"packages": pkg_list}, default_flow_style=False, sort_keys=False)
201
+
202
+
203
+ def generate_gitignore() -> str:
204
+ """Generate a .gitignore for dbt projects."""
205
+ return """target/
206
+ dbt_packages/
207
+ logs/
208
+ *.pyc
209
+ __pycache__/
210
+ profiles.yml
211
+ """