datasecops-cli 0.7.0b1__tar.gz → 0.7.0b4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.gitignore +2 -1
  2. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/CHANGELOG.md +24 -0
  3. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/DEVELOPMENT.md +2 -2
  4. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/PKG-INFO +3 -3
  5. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/README.md +2 -2
  6. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/getting-started.md +44 -1
  7. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/pyproject.toml +1 -1
  8. datasecops_cli-0.7.0b4/src/datasecops_cli/__init__.py +1 -0
  9. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/config.py +1 -0
  10. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/main.py +118 -92
  11. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/project_config.py +1 -0
  12. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/bootstrap_service.py +3 -1
  13. datasecops_cli-0.7.0b4/src/datasecops_cli/services/dbt_project_generator.py +364 -0
  14. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/download_service.py +12 -8
  15. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/snowflake_service.py +15 -0
  16. datasecops_cli-0.7.0b4/tests/test_dbt_project_generator_profiles.py +150 -0
  17. datasecops_cli-0.7.0b1/src/datasecops_cli/__init__.py +0 -1
  18. datasecops_cli-0.7.0b1/src/datasecops_cli/services/dbt_project_generator.py +0 -230
  19. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/auto-tag.yml +0 -0
  20. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/publish-cli.yml +0 -0
  21. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/test.yml +0 -0
  22. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/AGENTS.md +0 -0
  23. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/LICENSE +0 -0
  24. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/dcm.md +0 -0
  25. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/git-operations.md +0 -0
  26. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/legacy.md +0 -0
  27. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/legacy_plan_of_action.md +0 -0
  28. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/mcp-server.md +0 -0
  29. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/mcp-servers.json +0 -0
  30. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/scripts/vendor_model.py +0 -0
  31. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/setup.ps1 +0 -0
  32. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/setup.sh +0 -0
  33. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/__init__.py +0 -0
  34. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/configuration.py +0 -0
  35. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/dcm_operations.py +0 -0
  36. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/development.py +0 -0
  37. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/downloads.py +0 -0
  38. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/git_operations.py +0 -0
  39. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/__init__.py +0 -0
  40. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/git_helpers.py +0 -0
  41. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/__init__.py +0 -0
  42. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dbt_runner.py +0 -0
  43. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_orchestrator.py +0 -0
  44. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_pipelines.py +0 -0
  45. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_registry.py +0 -0
  46. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_service.py +0 -0
  47. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
  48. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/git_service.py +0 -0
  49. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/linting_service.py +0 -0
  50. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/model_commands.py +0 -0
  51. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/skill_service.py +0 -0
  52. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/upstream_service.py +0 -0
  53. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/__init__.py +0 -0
  54. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/display.py +0 -0
  55. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/file_utils.py +0 -0
  56. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/mcp.py +0 -0
  57. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  58. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/__init__.py +0 -0
  59. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/__main__.py +0 -0
  60. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/connection.py +0 -0
  61. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/server.py +0 -0
  62. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/__init__.py +0 -0
  63. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/convert.py +0 -0
  64. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/diagram.py +0 -0
  65. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/expand.py +0 -0
  66. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/extract.py +0 -0
  67. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/render.py +0 -0
  68. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/schema.py +0 -0
  69. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/scope.py +0 -0
  70. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/scripts.py +0 -0
  71. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/starter.py +0 -0
  72. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/role.sql +0 -0
  73. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/role_guard.sql +0 -0
  74. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/warehouse.sql +0 -0
  75. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/validate.py +0 -0
  76. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/__init__.py +0 -0
  77. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_config.py +0 -0
  78. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dbt_runner.py +0 -0
  79. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_model.py +0 -0
  80. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_orchestrator.py +0 -0
  81. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_service.py +0 -0
  82. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_download_service.py +0 -0
  83. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_file_utils.py +0 -0
  84. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_linting_service.py +0 -0
  85. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_main.py +0 -0
  86. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_model_commands.py +0 -0
  87. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_models.py +0 -0
  88. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_version.py +0 -0
  89. {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_yaml_utils.py +0 -0
@@ -219,4 +219,5 @@ __marimo__/
219
219
 
220
220
 
221
221
  legacy_framework/
222
- plans/
222
+ plans/
223
+ .cortex/
@@ -4,6 +4,30 @@ All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ### Fixed
8
+
9
+ - **`profiles.yml` deployment targets were being written to a developer's own `~/.dbt/profiles.yml`** — CI/deployment targets (`dev`/`test`/`prod`) belong to the native app's CI pipeline, not a developer's machine, so they're no longer written there at all. Only the `local-dev` target is ever added or refreshed; any other targets already present in the file (added manually, or by a framework target sharing the profile's name) are left untouched. Removed the now-unused CI-identity profile generation (`ProjectSettings.ci_username`, the old `generate_profiles_yml`).
10
+ - **`local-dev` used a hardcoded `authenticator: externalbrowser` and a shared database instead of the developer's own identity** — `local-dev` now reuses whatever Snowflake CLI connection is already configured for the project in `~/.snowflake/connections.toml` (account, user, and whatever auth method is already in use — key pair, SSO, password, etc.), read directly via `snowflake.connector.config_manager` without establishing a connection. `role`/`warehouse`/`schema` still always come from the project's dbt targets config. The developer is prompted once for their name (persisted to `.datasecops.yml`), used to build a personal sandpit database (`<target_database>SANDPIT_<name>`) so multiple developers never collide.
11
+ - **`profiles.yml` is no longer written inside the dbt project folder** — dbt never reads a `profiles.yml` there (it reads `--profiles-dir`, i.e. `~/.dbt` per the native app's `profile_dir` setting), so the project-local copy was always a dead reference file. `datasecops setup`/`bootstrap`/`new project` now write/merge the `local-dev` target directly into `~/.dbt/profiles.yml` via a shared `write_profiles_yml()` helper, creating the folder/file if needed.
12
+
13
+ ## [0.7.0b2] - 2026-10-26
14
+
15
+ ### Fixed
16
+
17
+ - **Default app database renamed to `MONITORIAL_ORBIT`** — the `datasecops setup` suggested app-database name (and matching docs examples) was `DATA_ENGINEERS_DATASECOPS_FRAMEWORK`; it now matches the native app's actual installed name.
18
+ - **`dbt_project.yml` generator matches the source-based project spec** — added `docs-paths`, fixed `analysis-paths` (was `analyses`), added top-level `seeds:`/`tests:` blocks, `+static_analysis`/`+meta`/`+tags` at the model root, per-object-type `+static_analysis` (stored procedures/UDFs off, tasks baseline), the commented-out post-hook/on-run-start/on-run-end guidance, and the data-governance `vars` block.
19
+ - **`profiles.yml` deploy targets now run as the framework's CI identity** — every target except `local-dev` previously used the local developer's Snowflake user with each target's own role/warehouse. They now use the CI Username/Role/Warehouse from the native app's Deployment settings (new `ProjectSettings.ci_username`/`ci_role`/`ci_warehouse`, falling back to the previous behavior when unset), matching what the CI pipeline actually deploys with.
20
+ - **`datasecops setup` could silently default to GitHub for a non-GitHub platform** — `PROJECT`/`SOURCE_CONTROL`/`WORK_ITEMS` framework config and profile/`profiles.yml` resolution were one single `try/except`; any failure after connecting (including just the `SOURCE_CONTROL` fetch itself) discarded already-fetched config and silently fell back to `SourceControl()`'s default platform (`GitHub`) with no indication why, so the MCP setup step could prompt for the wrong PAT. Each framework config fetch is now independent, so one failing can't blank out another; a missing/failed `SOURCE_CONTROL` config, or one missing `source_control_platform` specifically, now warns explicitly which platform it's defaulting to.
21
+ - **`local-dev` could be overwritten by a same-named framework target, or by the CI identity on a plain `dbt run`** — `generate_profiles_yml` no longer lets a framework target literally named `local-dev` clobber the dedicated external-browser output, and the profile's default target is now always `local-dev` (previously it followed whatever the native app marked as its own default, which could be a deploy-only target running as the CI identity). `_run_interactive` and bootstrap's `dbt deps` step now always run as `local-dev` instead of trusting the framework's default target.
22
+ - **Re-running `datasecops setup` for an already-known profile never refreshed its deployment targets** — a CI identity change in the native app had no effect until `profiles.yml` was deleted. Setup now always refreshes the CI/deployment targets (while preserving an existing `local-dev` target if already customized), and skips writing `profiles.yml` entirely if the `PROJECT` framework config couldn't be loaded (previously it wrote incorrect defaults).
23
+ - **Hardened `dbt_project.yml` generation against unsafe characters and a YAML boolean-coercion bug** — project/profile name, dbt version and warehouse name are now validated against a safe identifier pattern before being substituted into the raw YAML template (closing a potential injection via a newline or quote in native-app-configured values), and `static_analysis: off` is now quoted so YAML doesn't resolve it to the boolean `false`.
24
+
25
+ ### Docs
26
+
27
+ - **Documented setting up `~/.dbt/profiles.yml`** — `datasecops bootstrap` writes a generated `profiles.yml` inside the dbt project folder as a reference only; dbt itself reads from `~/.dbt` (per the native app's `profile_dir` setting), which getting-started.md now explains, including the minimal `local-dev` block a developer needs there.
28
+
29
+ ## [0.7.0b1] - 2026-10-26
30
+
7
31
  ### Added
8
32
 
9
33
  - **Shared DCM model (`datasecops_model`)** — one versioned JSON document (`config/dcm_model.json`, schema `datasecops.dcm/v1`) describing every DCM project (type, dir, `depends_on`, targets with deploy roles), the security model (role groups with environment fan-out, SCIM/UGE prefixes, diagram layout) and the warehouses (the `warehouse()` macro arguments, grants, application grants, budgets). The same package is vendored into the DataSecOps native app (`scripts/vendor_model.py`), so the app and the CLI validate, render and draw identically.
@@ -38,7 +38,7 @@ chmod +x setup.sh
38
38
  The script will prompt you for:
39
39
 
40
40
  - **Connection name** — the name of your Snowflake connection from `~/.snowflake/connections.toml`
41
- - **Native app database name** — the database where the DataSecOps Framework app is installed (e.g. `DATA_ENGINEERS_DATASECOPS_FRAMEWORK`)
41
+ - **Native app database name** — the database where the DataSecOps Framework app is installed (e.g. `MONITORIAL_ORBIT`)
42
42
 
43
43
  ### 2. Activate the virtual environment
44
44
 
@@ -120,7 +120,7 @@ Created by the setup script in your project root. Contains:
120
120
 
121
121
  ```yaml
122
122
  connection_name: "my_connection" # Snowflake connection name
123
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK" # Native app database
123
+ app_database: "MONITORIAL_ORBIT" # Native app database
124
124
  profile_name: "" # Auto-detected from dbt_project.yml
125
125
  ```
126
126
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: datasecops-cli
3
- Version: 0.7.0b1
3
+ Version: 0.7.0b4
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -120,7 +120,7 @@ Your pipeline needs two things:
120
120
 
121
121
  ```yaml
122
122
  connection_name: "ci"
123
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
123
+ app_database: "MONITORIAL_ORBIT"
124
124
  ```
125
125
 
126
126
  2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
@@ -264,7 +264,7 @@ The CLI reads from a `.datasecops.yml` file in your project root (created by the
264
264
 
265
265
  ```yaml
266
266
  connection_name: "my_connection"
267
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
267
+ app_database: "MONITORIAL_ORBIT"
268
268
  ```
269
269
 
270
270
  Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
@@ -100,7 +100,7 @@ Your pipeline needs two things:
100
100
 
101
101
  ```yaml
102
102
  connection_name: "ci"
103
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
103
+ app_database: "MONITORIAL_ORBIT"
104
104
  ```
105
105
 
106
106
  2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
@@ -244,7 +244,7 @@ The CLI reads from a `.datasecops.yml` file in your project root (created by the
244
244
 
245
245
  ```yaml
246
246
  connection_name: "my_connection"
247
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
247
+ app_database: "MONITORIAL_ORBIT"
248
248
  ```
249
249
 
250
250
  Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
@@ -59,7 +59,7 @@ This creates `.datasecops.yml` in your project root:
59
59
 
60
60
  ```yaml
61
61
  connection_name: "my_connection"
62
- app_database: "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
62
+ app_database: "MONITORIAL_ORBIT"
63
63
  ```
64
64
 
65
65
  If you prefer to create it manually, the two required fields are:
@@ -91,6 +91,49 @@ You should see the main menu, confirming the CLI can reach the native app.
91
91
 
92
92
  ---
93
93
 
94
+ ## Step 3b: Set up your local dbt profile (`~/.dbt/profiles.yml`)
95
+
96
+ Running `datasecops setup` or `datasecops bootstrap` (or **Configuration → new project** from the
97
+ main menu) writes a `local-dev` target for your profile directly to `~/.dbt/profiles.yml` (per the
98
+ native app's `PROJECT.profile_dir` setting, used as dbt's `--profiles-dir`) — not inside the dbt
99
+ project folder, since dbt never reads a `profiles.yml` there. The `.dbt` folder and `profiles.yml`
100
+ file are created if they don't exist yet; if a profile entry already exists, only its `local-dev`
101
+ target is refreshed — any other targets already in the file (e.g. CI/deployment targets you or a
102
+ pipeline added) are left untouched. **Only `local-dev` is ever written here** — deployment targets
103
+ (`dev`/`test`/`prod`) belong to the native app's CI pipeline, never to a developer's machine.
104
+
105
+ The `local-dev` target reuses whatever Snowflake CLI connection you already have configured in
106
+ `~/.snowflake/connections.toml` for the `connection_name` in your `.datasecops.yml` — account, user,
107
+ and whatever auth method you already use (external browser, key pair, password, etc.) are copied
108
+ across as-is, so there's nothing extra to configure. You'll be asked for your name once (used to
109
+ build your personal sandpit database, e.g. `jon` → `SDS_SANDPIT_JON`), and it's saved to
110
+ `.datasecops.yml` so you won't be asked again. `role`/`warehouse`/`schema` always come from the
111
+ project's dbt targets config, never from connections.toml.
112
+
113
+ The resulting `~/.dbt/profiles.yml` looks like:
114
+
115
+ ```yaml
116
+ <profile_name>: # matches the `profile:` field in dbt_project.yml
117
+ target: local-dev
118
+ outputs:
119
+ local-dev:
120
+ type: snowflake
121
+ account: "<from your connections.toml connection>"
122
+ user: "<from your connections.toml connection>"
123
+ authenticator: "<from your connections.toml connection, if set>"
124
+ private_key_path: "<from your connections.toml connection, if set>"
125
+ role: DEVELOPERS
126
+ warehouse: DEV_WH
127
+ database: "<target_database>SANDPIT_<your name>"
128
+ schema: PUBLIC
129
+ threads: 4
130
+ ```
131
+
132
+ If you work on multiple framework projects, each profile (one per project, named after its
133
+ `profile:` field) gets its own block in the same `~/.dbt/profiles.yml` file.
134
+
135
+ ---
136
+
94
137
  ## Step 4: Install MCP servers
95
138
 
96
139
  ### 4a. DataSecOps Framework MCP Server (included)
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.7.0b1"
7
+ version = "0.7.0b4"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1 @@
1
+ __version__ = "0.7.0b4"
@@ -38,6 +38,7 @@ class Config:
38
38
  app_database=raw.get("app_database", ""),
39
39
  profile_name=raw.get("profile_name", ""),
40
40
  cortex_connection=raw.get("cortex_connection", ""),
41
+ sandpit_name=raw.get("sandpit_name", ""),
41
42
  dbt_engine=raw.get("dbt_engine", "dbt_core"),
42
43
  dcm_enabled=raw.get("dcm_enabled", False),
43
44
  dcm_security_dir=raw.get("dcm_security_dir", "security"),
@@ -475,9 +475,12 @@ def _run_setup(project_dir: Path):
475
475
  # --- App database ---
476
476
  app_database = get_input_string_or_default(
477
477
  "Enter native app database name",
478
- "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
478
+ "MONITORIAL_ORBIT"
479
479
  )
480
480
 
481
+ # --- Sandpit name (for your personal local-dev database) ---
482
+ sandpit_name = get_input_string("Enter your name (used to build your personal sandpit database, e.g. 'jon'): ")
483
+
481
484
  # --- CoCo agent connection ---
482
485
  cortex_connection = ""
483
486
  if shutil.which("cortex") and connection_names:
@@ -496,104 +499,115 @@ def _run_setup(project_dir: Path):
496
499
  source_control = SourceControl()
497
500
  work_items = WorkItems()
498
501
  matched_profile = None
502
+ account_identifier = ""
503
+ project_settings_loaded = False
499
504
 
505
+ connected = False
500
506
  try:
501
507
  temp_sf.connect()
502
508
  success_line("Connected")
509
+ connected = True
510
+ except Exception as e:
511
+ warning_line(f"Could not connect to Snowflake: {e}")
512
+ warning_line("You can set profile_name manually in .datasecops.yml")
513
+ warning_line("You may need to create profiles.yml manually")
503
514
 
504
- # Load framework PROJECT settings (for profile_dir, targets)
505
- raw_project = temp_sf.get_framework_config("PROJECT")
506
- if raw_project:
507
- project_settings = ProjectSettings(**{k: v for k, v in raw_project.items() if k in ProjectSettings.model_fields})
515
+ profile_name = ""
516
+ if connected:
517
+ # Each framework config code is fetched independently: a failure (or a missing
518
+ # config row) must not discard config that was already fetched successfully, and
519
+ # must not silently fall back to a default (e.g. GitHub) without saying so.
520
+ try:
521
+ raw_project = temp_sf.get_framework_config("PROJECT")
522
+ if raw_project:
523
+ project_settings = ProjectSettings(**{k: v for k, v in raw_project.items() if k in ProjectSettings.model_fields})
524
+ project_settings_loaded = True
525
+ else:
526
+ warning_line("No PROJECT framework config found in native app - using defaults")
527
+ except Exception as e:
528
+ warning_line(f"Could not load PROJECT framework config: {e}")
508
529
 
509
- # Load SOURCE_CONTROL settings (for source_control_platform)
510
- raw_sc = temp_sf.get_framework_config("SOURCE_CONTROL")
511
- if raw_sc:
512
- source_control = SourceControl(**{k: v for k, v in raw_sc.items() if k in SourceControl.model_fields})
530
+ try:
531
+ raw_sc = temp_sf.get_framework_config("SOURCE_CONTROL")
532
+ if raw_sc:
533
+ if "source_control_platform" not in raw_sc:
534
+ warning_line(f"SOURCE_CONTROL framework config has no source_control_platform - "
535
+ f"defaulting to {source_control.source_control_platform}")
536
+ source_control = SourceControl(**{k: v for k, v in raw_sc.items() if k in SourceControl.model_fields})
537
+ else:
538
+ warning_line(f"No SOURCE_CONTROL framework config found in native app - "
539
+ f"defaulting source control platform to {source_control.source_control_platform}")
540
+ except Exception as e:
541
+ warning_line(f"Could not load SOURCE_CONTROL framework config "
542
+ f"(defaulting source control platform to {source_control.source_control_platform}): {e}")
513
543
 
514
- # Load WORK_ITEMS settings (for ticket system and ticket_number_required)
515
- raw_wi = temp_sf.get_framework_config("WORK_ITEMS")
516
- if raw_wi:
517
- work_items = WorkItems(**{k: v for k, v in raw_wi.items() if k in WorkItems.model_fields})
544
+ try:
545
+ raw_wi = temp_sf.get_framework_config("WORK_ITEMS")
546
+ if raw_wi:
547
+ work_items = WorkItems(**{k: v for k, v in raw_wi.items() if k in WorkItems.model_fields})
548
+ except Exception as e:
549
+ warning_line(f"Could not load WORK_ITEMS framework config: {e}")
518
550
 
519
551
  # Get account identifier for MCP URL construction
520
- account_identifier = ""
521
552
  try:
522
553
  org_rows = temp_sf.execute_query("SELECT CURRENT_ORGANIZATION_NAME() || '-' || CURRENT_ACCOUNT_NAME() AS acct")
523
554
  account_identifier = org_rows[0]["ACCT"].lower() if org_rows else ""
524
555
  except Exception as e:
525
556
  warning_line(f"Could not determine account identifier (AI Agent MCP will be skipped): {e}")
526
557
 
527
- # --- Resolve profile name ---
528
- from datasecops_cli.utilities.yaml_utils import get_profile_name
529
- profile_name = get_profile_name(project_dir)
530
- if profile_name:
531
- success_line(f"Profile found in dbt_project.yml: {profile_name}")
532
- else:
533
- info_line("No dbt_project.yml found. Fetching project profiles...")
534
- profiles_data = temp_sf.get_project_profiles()
535
- if profiles_data:
536
- profile_names = [p.get("PROFILE_NAME", p.get("profile_name", "")) for p in profiles_data]
537
- profile_names = [n for n in profile_names if n]
538
- if len(profile_names) == 1:
539
- profile_name = profile_names[0]
540
- success_line(f"Auto-selected profile: {profile_name}")
541
- elif len(profile_names) > 1:
542
- info_line(f"Found {len(profile_names)} project profiles:")
543
- profile_name = select_from_list(profile_names, "profile", add_back=False)
558
+ try:
559
+ # --- Resolve profile name ---
560
+ from datasecops_cli.utilities.yaml_utils import get_profile_name
561
+ profile_name = get_profile_name(project_dir)
562
+ if profile_name:
563
+ success_line(f"Profile found in dbt_project.yml: {profile_name}")
564
+ else:
565
+ info_line("No dbt_project.yml found. Fetching project profiles...")
566
+ profiles_data = temp_sf.get_project_profiles()
567
+ if profiles_data:
568
+ profile_names = [p.get("PROFILE_NAME", p.get("profile_name", "")) for p in profiles_data]
569
+ profile_names = [n for n in profile_names if n]
570
+ if len(profile_names) == 1:
571
+ profile_name = profile_names[0]
572
+ success_line(f"Auto-selected profile: {profile_name}")
573
+ elif len(profile_names) > 1:
574
+ info_line(f"Found {len(profile_names)} project profiles:")
575
+ profile_name = select_from_list(profile_names, "profile", add_back=False)
576
+ else:
577
+ warning_line("No profiles found in native app")
544
578
  else:
545
579
  warning_line("No profiles found in native app")
546
- else:
547
- warning_line("No profiles found in native app")
548
-
549
- # Find the full profile object for profiles.yml generation
550
- if profile_name:
551
- profiles_data = temp_sf.get_project_profiles()
552
- if profiles_data:
553
- for p in profiles_data:
554
- pn = p.get("PROFILE_NAME", p.get("profile_name", ""))
555
- if pn == profile_name:
556
- matched_profile = ProjectProfile(**{k.lower(): v for k, v in p.items()})
557
- break
558
-
559
- # --- Ensure profiles.yml has the profile entry ---
560
- if profile_name and matched_profile:
561
- import yaml
562
- from datasecops_cli.utilities.file_utils import ensure_dir, write_file
563
- from datasecops_cli.services.dbt_project_generator import generate_profiles_yml
564
-
565
- profiles_dir = Path(project_settings.profile_dir).expanduser()
566
- profiles_path = profiles_dir / "profiles.yml"
567
-
568
- # Read existing profiles.yml or start fresh
569
- existing_profiles = {}
570
- if profiles_path.exists():
571
- with open(profiles_path, "r", encoding="utf-8") as f:
572
- existing_profiles = yaml.safe_load(f) or {}
573
580
 
574
- if profile_name in existing_profiles:
575
- success_line(f"Profile '{profile_name}' already exists in {profiles_path}")
576
- else:
577
- info_line(f"Adding profile '{profile_name}' to {profiles_path}...")
578
- account_info = temp_sf.get_account_info()
579
- account = account_info.get("ACCOUNT", account_info.get("account", ""))
580
- user = account_info.get("USER_NAME", account_info.get("user_name", ""))
581
- new_profiles_yaml = generate_profiles_yml(matched_profile, project_settings, account, user)
582
- new_profiles = yaml.safe_load(new_profiles_yaml) or {}
583
- existing_profiles.update(new_profiles)
584
- ensure_dir(profiles_dir)
585
- write_file(profiles_path, yaml.dump(existing_profiles, default_flow_style=False, sort_keys=False))
586
- success_line(f"Profile '{profile_name}' written to {profiles_path}")
581
+ # Find the full profile object for profiles.yml generation
582
+ if profile_name:
583
+ profiles_data = temp_sf.get_project_profiles()
584
+ if profiles_data:
585
+ for p in profiles_data:
586
+ pn = p.get("PROFILE_NAME", p.get("profile_name", ""))
587
+ if pn == profile_name:
588
+ matched_profile = ProjectProfile(**{k.lower(): v for k, v in p.items()})
589
+ break
590
+
591
+ # --- Ensure profiles.yml has the profile entry ---
592
+ if profile_name and matched_profile and not project_settings_loaded:
593
+ warning_line(f"Skipping profiles.yml for '{profile_name}': PROJECT framework config "
594
+ "was not loaded, so the local-dev role/warehouse aren't known")
595
+ elif profile_name and matched_profile:
596
+ from datasecops_cli.services.dbt_project_generator import write_profiles_yml
597
+ from datasecops_cli.services.snowflake_service import get_connection_params
598
+
599
+ profiles_dir = Path(project_settings.profile_dir).expanduser()
600
+ connection_params = get_connection_params(connection_name)
601
+ status = write_profiles_yml(profiles_dir, profile_name, matched_profile, project_settings, connection_params, sandpit_name)
602
+ success_line(f"Profile '{profile_name}' {status} in {profiles_dir / 'profiles.yml'}")
603
+ except Exception as e:
604
+ warning_line(f"Could not resolve the project profile / write profiles.yml: {e}")
605
+ warning_line("You can set profile_name manually in .datasecops.yml")
606
+ warning_line("You may need to create profiles.yml manually")
607
+ from datasecops_cli.utilities.yaml_utils import get_profile_name
608
+ profile_name = get_profile_name(project_dir) or ""
587
609
 
588
- except Exception as e:
589
- warning_line(f"Could not connect to Snowflake: {e}")
590
- warning_line("You can set profile_name manually in .datasecops.yml")
591
- warning_line("You may need to create profiles.yml manually")
592
- # Fall back to reading profile from dbt_project.yml if available
593
- from datasecops_cli.utilities.yaml_utils import get_profile_name
594
- profile_name = get_profile_name(project_dir) or ""
595
- finally:
596
- temp_sf.close()
610
+ temp_sf.close()
597
611
 
598
612
  # --- Write config ---
599
613
  config_data = {
@@ -601,6 +615,7 @@ def _run_setup(project_dir: Path):
601
615
  "app_database": app_database,
602
616
  "profile_name": profile_name,
603
617
  "cortex_connection": cortex_connection,
618
+ "sandpit_name": sandpit_name,
604
619
  }
605
620
  write_datasecops_config(project_dir, config_data)
606
621
  success_line(f".datasecops.yml written to {project_dir / '.datasecops.yml'}")
@@ -796,11 +811,11 @@ def _connect_and_load(config: Config, interactive: bool = True,
796
811
  break
797
812
  # Save the selected profile back to .datasecops.yml
798
813
  if config.profile:
799
- config_data = {
800
- "connection_name": config.datasecops.connection_name,
801
- "app_database": config.datasecops.app_database,
802
- "profile_name": config.profile_name,
803
- }
814
+ from datasecops_cli.utilities.yaml_utils import read_datasecops_config
815
+ config_data = read_datasecops_config(config.project_dir) or {}
816
+ config_data["connection_name"] = config.datasecops.connection_name
817
+ config_data["app_database"] = config.datasecops.app_database
818
+ config_data["profile_name"] = config.profile_name
804
819
  write_datasecops_config(config.project_dir, config_data)
805
820
  success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
806
821
  else:
@@ -808,11 +823,11 @@ def _connect_and_load(config: Config, interactive: bool = True,
808
823
  info_line(f"Using default profile: {config.profile_name}")
809
824
  elif not profile_was_set and len(config.all_profiles) == 1:
810
825
  # Single profile auto-selected — persist it so future runs don't re-prompt
811
- config_data = {
812
- "connection_name": config.datasecops.connection_name,
813
- "app_database": config.datasecops.app_database,
814
- "profile_name": config.profile_name,
815
- }
826
+ from datasecops_cli.utilities.yaml_utils import read_datasecops_config
827
+ config_data = read_datasecops_config(config.project_dir) or {}
828
+ config_data["connection_name"] = config.datasecops.connection_name
829
+ config_data["app_database"] = config.datasecops.app_database
830
+ config_data["profile_name"] = config.profile_name
816
831
  write_datasecops_config(config.project_dir, config_data)
817
832
  success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
818
833
 
@@ -860,7 +875,9 @@ def _run_interactive(config: Config):
860
875
  dbt_runner = DbtRunner(
861
876
  project_dir=config.dbt_project_dir,
862
877
  profiles_dir=config.get_dbt_profiles_dir(),
863
- target=config.project_settings.get_default_target().target_name if config.get_default_target() else "dev",
878
+ # Always the developer's own identity, never the framework's CI/deployment
879
+ # default target (which may be a deploy-only target running as the CI identity).
880
+ target="local-dev",
864
881
  engine=config.datasecops.dbt_engine or "dbt_core"
865
882
  )
866
883
 
@@ -1230,7 +1247,7 @@ def _show_main_menu(profile_name: str, git_service: GitService = None,
1230
1247
 
1231
1248
  def _run_bootstrap(config: Config):
1232
1249
  """Run the bootstrap command to initialise a new project."""
1233
- from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
1250
+ from datasecops_cli.utilities.display import get_input_string, get_input_true_false, select_multiple_from_list
1234
1251
 
1235
1252
  sf_service = _connect_and_load(config)
1236
1253
 
@@ -1241,6 +1258,15 @@ def _run_bootstrap(config: Config):
1241
1258
  platform = config.source_control.source_control_platform.lower()
1242
1259
  info_line(f"Platform: {platform}")
1243
1260
 
1261
+ # Sandpit name (for your personal local-dev database) — ask once, then persist
1262
+ if not config.datasecops.sandpit_name:
1263
+ from datasecops_cli.utilities.yaml_utils import read_datasecops_config, write_datasecops_config
1264
+ config.datasecops.sandpit_name = get_input_string(
1265
+ "Enter your name (used to build your personal sandpit database, e.g. 'jon'): ")
1266
+ config_data = read_datasecops_config(config.project_dir) or {}
1267
+ config_data["sandpit_name"] = config.datasecops.sandpit_name
1268
+ write_datasecops_config(config.project_dir, config_data)
1269
+
1244
1270
  # Ask about optional steps
1245
1271
  install_skills = get_input_true_false("Install AI coding skills?")
1246
1272
  skill_targets = None
@@ -10,6 +10,7 @@ class DatasecopsConfig(BaseModel):
10
10
  app_database: str = ""
11
11
  profile_name: str = ""
12
12
  cortex_connection: str = ""
13
+ sandpit_name: str = "" # personal name used to build your local-dev sandpit database (e.g. "jon")
13
14
  dbt_engine: str = "dbt_core" # "dbt2" for dbt 2.0 (pip install dbt), "dbt_core" for dbt Core 1.x (pip install dbt-core + dbt-snowflake)
14
15
  dcm_enabled: bool = False
15
16
  dcm_security_dir: str = "security"
@@ -112,7 +112,9 @@ class BootstrapService:
112
112
  dbt_runner = DbtRunner(
113
113
  project_dir=dbt_project_dir,
114
114
  profiles_dir=Path(self.project_settings.profile_dir).expanduser(),
115
- target=self.project_settings.get_default_target().target_name if self.project_settings.get_default_target() else "dev",
115
+ # Always the developer's own identity, never the framework's CI/deployment
116
+ # default target.
117
+ target="local-dev",
116
118
  engine=self.datasecops_config.dbt_engine,
117
119
  )
118
120
  result = dbt_runner.deps()