datasecops-cli 0.7.0b1__tar.gz → 0.7.0b4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.gitignore +2 -1
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/CHANGELOG.md +24 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/DEVELOPMENT.md +2 -2
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/PKG-INFO +3 -3
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/README.md +2 -2
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/getting-started.md +44 -1
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/pyproject.toml +1 -1
- datasecops_cli-0.7.0b4/src/datasecops_cli/__init__.py +1 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/config.py +1 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/main.py +118 -92
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/project_config.py +1 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/bootstrap_service.py +3 -1
- datasecops_cli-0.7.0b4/src/datasecops_cli/services/dbt_project_generator.py +364 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/download_service.py +12 -8
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/snowflake_service.py +15 -0
- datasecops_cli-0.7.0b4/tests/test_dbt_project_generator_profiles.py +150 -0
- datasecops_cli-0.7.0b1/src/datasecops_cli/__init__.py +0 -1
- datasecops_cli-0.7.0b1/src/datasecops_cli/services/dbt_project_generator.py +0 -230
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/auto-tag.yml +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/.github/workflows/test.yml +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/AGENTS.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/LICENSE +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/dcm.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/git-operations.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/legacy.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/mcp-servers.json +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/scripts/vendor_model.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/setup.ps1 +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/setup.sh +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/configuration.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/dcm_operations.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/development.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/downloads.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dbt_runner.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_orchestrator.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_pipelines.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_registry.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/dcm_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/model_commands.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/skill_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/upstream_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/display.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/file_utils.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/mcp.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/connection.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_mcp/server.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/convert.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/diagram.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/expand.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/extract.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/render.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/schema.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/scope.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/scripts.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/starter.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/role.sql +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/role_guard.sql +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/templates/macros/warehouse.sql +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_model/validate.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/__init__.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_config.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dbt_runner.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_model.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_orchestrator.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_dcm_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_download_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_linting_service.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_main.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_model_commands.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_models.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_version.py +0 -0
- {datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/tests/test_yaml_utils.py +0 -0
|
@@ -4,6 +4,30 @@ All notable changes to the DataSecOps CLI are documented in this file.
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- **`profiles.yml` deployment targets were being written to a developer's own `~/.dbt/profiles.yml`** — CI/deployment targets (`dev`/`test`/`prod`) belong to the native app's CI pipeline, not a developer's machine, so they're no longer written there at all. Only the `local-dev` target is ever added or refreshed; any other targets already present in the file (added manually, or by a framework target sharing the profile's name) are left untouched. Removed the now-unused CI-identity profile generation (`ProjectSettings.ci_username`, the old `generate_profiles_yml`).
|
|
10
|
+
- **`local-dev` used a hardcoded `authenticator: externalbrowser` and a shared database instead of the developer's own identity** — `local-dev` now reuses whatever Snowflake CLI connection is already configured for the project in `~/.snowflake/connections.toml` (account, user, and whatever auth method is already in use — key pair, SSO, password, etc.), read directly via `snowflake.connector.config_manager` without establishing a connection. `role`/`warehouse`/`schema` still always come from the project's dbt targets config. The developer is prompted once for their name (persisted to `.datasecops.yml`), used to build a personal sandpit database (`<target_database>SANDPIT_<name>`) so multiple developers never collide.
|
|
11
|
+
- **`profiles.yml` is no longer written inside the dbt project folder** — dbt never reads a `profiles.yml` there (it reads `--profiles-dir`, i.e. `~/.dbt` per the native app's `profile_dir` setting), so the project-local copy was always a dead reference file. `datasecops setup`/`bootstrap`/`new project` now write/merge the `local-dev` target directly into `~/.dbt/profiles.yml` via a shared `write_profiles_yml()` helper, creating the folder/file if needed.
|
|
12
|
+
|
|
13
|
+
## [0.7.0b2] - 2026-10-26
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- **Default app database renamed to `MONITORIAL_ORBIT`** — the `datasecops setup` suggested app-database name (and matching docs examples) was `DATA_ENGINEERS_DATASECOPS_FRAMEWORK`; it now matches the native app's actual installed name.
|
|
18
|
+
- **`dbt_project.yml` generator matches the source-based project spec** — added `docs-paths`, fixed `analysis-paths` (was `analyses`), added top-level `seeds:`/`tests:` blocks, `+static_analysis`/`+meta`/`+tags` at the model root, per-object-type `+static_analysis` (stored procedures/UDFs off, tasks baseline), the commented-out post-hook/on-run-start/on-run-end guidance, and the data-governance `vars` block.
|
|
19
|
+
- **`profiles.yml` deploy targets now run as the framework's CI identity** — every target except `local-dev` previously used the local developer's Snowflake user with each target's own role/warehouse. They now use the CI Username/Role/Warehouse from the native app's Deployment settings (new `ProjectSettings.ci_username`/`ci_role`/`ci_warehouse`, falling back to the previous behavior when unset), matching what the CI pipeline actually deploys with.
|
|
20
|
+
- **`datasecops setup` could silently default to GitHub for a non-GitHub platform** — `PROJECT`/`SOURCE_CONTROL`/`WORK_ITEMS` framework config and profile/`profiles.yml` resolution were one single `try/except`; any failure after connecting (including just the `SOURCE_CONTROL` fetch itself) discarded already-fetched config and silently fell back to `SourceControl()`'s default platform (`GitHub`) with no indication why, so the MCP setup step could prompt for the wrong PAT. Each framework config fetch is now independent, so one failing can't blank out another; a missing/failed `SOURCE_CONTROL` config, or one missing `source_control_platform` specifically, now warns explicitly which platform it's defaulting to.
|
|
21
|
+
- **`local-dev` could be overwritten by a same-named framework target, or by the CI identity on a plain `dbt run`** — `generate_profiles_yml` no longer lets a framework target literally named `local-dev` clobber the dedicated external-browser output, and the profile's default target is now always `local-dev` (previously it followed whatever the native app marked as its own default, which could be a deploy-only target running as the CI identity). `_run_interactive` and bootstrap's `dbt deps` step now always run as `local-dev` instead of trusting the framework's default target.
|
|
22
|
+
- **Re-running `datasecops setup` for an already-known profile never refreshed its deployment targets** — a CI identity change in the native app had no effect until `profiles.yml` was deleted. Setup now always refreshes the CI/deployment targets (while preserving an existing `local-dev` target if already customized), and skips writing `profiles.yml` entirely if the `PROJECT` framework config couldn't be loaded (previously it wrote incorrect defaults).
|
|
23
|
+
- **Hardened `dbt_project.yml` generation against unsafe characters and a YAML boolean-coercion bug** — project/profile name, dbt version and warehouse name are now validated against a safe identifier pattern before being substituted into the raw YAML template (closing a potential injection via a newline or quote in native-app-configured values), and `static_analysis: off` is now quoted so YAML doesn't resolve it to the boolean `false`.
|
|
24
|
+
|
|
25
|
+
### Docs
|
|
26
|
+
|
|
27
|
+
- **Documented setting up `~/.dbt/profiles.yml`** — `datasecops bootstrap` writes a generated `profiles.yml` inside the dbt project folder as a reference only; dbt itself reads from `~/.dbt` (per the native app's `profile_dir` setting), which getting-started.md now explains, including the minimal `local-dev` block a developer needs there.
|
|
28
|
+
|
|
29
|
+
## [0.7.0b1] - 2026-10-26
|
|
30
|
+
|
|
7
31
|
### Added
|
|
8
32
|
|
|
9
33
|
- **Shared DCM model (`datasecops_model`)** — one versioned JSON document (`config/dcm_model.json`, schema `datasecops.dcm/v1`) describing every DCM project (type, dir, `depends_on`, targets with deploy roles), the security model (role groups with environment fan-out, SCIM/UGE prefixes, diagram layout) and the warehouses (the `warehouse()` macro arguments, grants, application grants, budgets). The same package is vendored into the DataSecOps native app (`scripts/vendor_model.py`), so the app and the CLI validate, render and draw identically.
|
|
@@ -38,7 +38,7 @@ chmod +x setup.sh
|
|
|
38
38
|
The script will prompt you for:
|
|
39
39
|
|
|
40
40
|
- **Connection name** — the name of your Snowflake connection from `~/.snowflake/connections.toml`
|
|
41
|
-
- **Native app database name** — the database where the DataSecOps Framework app is installed (e.g. `
|
|
41
|
+
- **Native app database name** — the database where the DataSecOps Framework app is installed (e.g. `MONITORIAL_ORBIT`)
|
|
42
42
|
|
|
43
43
|
### 2. Activate the virtual environment
|
|
44
44
|
|
|
@@ -120,7 +120,7 @@ Created by the setup script in your project root. Contains:
|
|
|
120
120
|
|
|
121
121
|
```yaml
|
|
122
122
|
connection_name: "my_connection" # Snowflake connection name
|
|
123
|
-
app_database: "
|
|
123
|
+
app_database: "MONITORIAL_ORBIT" # Native app database
|
|
124
124
|
profile_name: "" # Auto-detected from dbt_project.yml
|
|
125
125
|
```
|
|
126
126
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: datasecops-cli
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.0b4
|
|
4
4
|
Summary: DataSecOps Framework CLI for Snowflake Native App
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -120,7 +120,7 @@ Your pipeline needs two things:
|
|
|
120
120
|
|
|
121
121
|
```yaml
|
|
122
122
|
connection_name: "ci"
|
|
123
|
-
app_database: "
|
|
123
|
+
app_database: "MONITORIAL_ORBIT"
|
|
124
124
|
```
|
|
125
125
|
|
|
126
126
|
2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
|
|
@@ -264,7 +264,7 @@ The CLI reads from a `.datasecops.yml` file in your project root (created by the
|
|
|
264
264
|
|
|
265
265
|
```yaml
|
|
266
266
|
connection_name: "my_connection"
|
|
267
|
-
app_database: "
|
|
267
|
+
app_database: "MONITORIAL_ORBIT"
|
|
268
268
|
```
|
|
269
269
|
|
|
270
270
|
Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
|
|
@@ -100,7 +100,7 @@ Your pipeline needs two things:
|
|
|
100
100
|
|
|
101
101
|
```yaml
|
|
102
102
|
connection_name: "ci"
|
|
103
|
-
app_database: "
|
|
103
|
+
app_database: "MONITORIAL_ORBIT"
|
|
104
104
|
```
|
|
105
105
|
|
|
106
106
|
2. **A Snowflake connection** in `~/.snowflake/connections.toml` for the CI service account:
|
|
@@ -244,7 +244,7 @@ The CLI reads from a `.datasecops.yml` file in your project root (created by the
|
|
|
244
244
|
|
|
245
245
|
```yaml
|
|
246
246
|
connection_name: "my_connection"
|
|
247
|
-
app_database: "
|
|
247
|
+
app_database: "MONITORIAL_ORBIT"
|
|
248
248
|
```
|
|
249
249
|
|
|
250
250
|
Project profiles, linting rules, pipeline templates, and deployment targets are all managed centrally in the native app and pulled down by the CLI.
|
|
@@ -59,7 +59,7 @@ This creates `.datasecops.yml` in your project root:
|
|
|
59
59
|
|
|
60
60
|
```yaml
|
|
61
61
|
connection_name: "my_connection"
|
|
62
|
-
app_database: "
|
|
62
|
+
app_database: "MONITORIAL_ORBIT"
|
|
63
63
|
```
|
|
64
64
|
|
|
65
65
|
If you prefer to create it manually, the two required fields are:
|
|
@@ -91,6 +91,49 @@ You should see the main menu, confirming the CLI can reach the native app.
|
|
|
91
91
|
|
|
92
92
|
---
|
|
93
93
|
|
|
94
|
+
## Step 3b: Set up your local dbt profile (`~/.dbt/profiles.yml`)
|
|
95
|
+
|
|
96
|
+
Running `datasecops setup` or `datasecops bootstrap` (or **Configuration → new project** from the
|
|
97
|
+
main menu) writes a `local-dev` target for your profile directly to `~/.dbt/profiles.yml` (per the
|
|
98
|
+
native app's `PROJECT.profile_dir` setting, used as dbt's `--profiles-dir`) — not inside the dbt
|
|
99
|
+
project folder, since dbt never reads a `profiles.yml` there. The `.dbt` folder and `profiles.yml`
|
|
100
|
+
file are created if they don't exist yet; if a profile entry already exists, only its `local-dev`
|
|
101
|
+
target is refreshed — any other targets already in the file (e.g. CI/deployment targets you or a
|
|
102
|
+
pipeline added) are left untouched. **Only `local-dev` is ever written here** — deployment targets
|
|
103
|
+
(`dev`/`test`/`prod`) belong to the native app's CI pipeline, never to a developer's machine.
|
|
104
|
+
|
|
105
|
+
The `local-dev` target reuses whatever Snowflake CLI connection you already have configured in
|
|
106
|
+
`~/.snowflake/connections.toml` for the `connection_name` in your `.datasecops.yml` — account, user,
|
|
107
|
+
and whatever auth method you already use (external browser, key pair, password, etc.) are copied
|
|
108
|
+
across as-is, so there's nothing extra to configure. You'll be asked for your name once (used to
|
|
109
|
+
build your personal sandpit database, e.g. `jon` → `SDS_SANDPIT_JON`), and it's saved to
|
|
110
|
+
`.datasecops.yml` so you won't be asked again. `role`/`warehouse`/`schema` always come from the
|
|
111
|
+
project's dbt targets config, never from connections.toml.
|
|
112
|
+
|
|
113
|
+
The resulting `~/.dbt/profiles.yml` looks like:
|
|
114
|
+
|
|
115
|
+
```yaml
|
|
116
|
+
<profile_name>: # matches the `profile:` field in dbt_project.yml
|
|
117
|
+
target: local-dev
|
|
118
|
+
outputs:
|
|
119
|
+
local-dev:
|
|
120
|
+
type: snowflake
|
|
121
|
+
account: "<from your connections.toml connection>"
|
|
122
|
+
user: "<from your connections.toml connection>"
|
|
123
|
+
authenticator: "<from your connections.toml connection, if set>"
|
|
124
|
+
private_key_path: "<from your connections.toml connection, if set>"
|
|
125
|
+
role: DEVELOPERS
|
|
126
|
+
warehouse: DEV_WH
|
|
127
|
+
database: "<target_database>SANDPIT_<your name>"
|
|
128
|
+
schema: PUBLIC
|
|
129
|
+
threads: 4
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
If you work on multiple framework projects, each profile (one per project, named after its
|
|
133
|
+
`profile:` field) gets its own block in the same `~/.dbt/profiles.yml` file.
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
94
137
|
## Step 4: Install MCP servers
|
|
95
138
|
|
|
96
139
|
### 4a. DataSecOps Framework MCP Server (included)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.7.0b4"
|
|
@@ -38,6 +38,7 @@ class Config:
|
|
|
38
38
|
app_database=raw.get("app_database", ""),
|
|
39
39
|
profile_name=raw.get("profile_name", ""),
|
|
40
40
|
cortex_connection=raw.get("cortex_connection", ""),
|
|
41
|
+
sandpit_name=raw.get("sandpit_name", ""),
|
|
41
42
|
dbt_engine=raw.get("dbt_engine", "dbt_core"),
|
|
42
43
|
dcm_enabled=raw.get("dcm_enabled", False),
|
|
43
44
|
dcm_security_dir=raw.get("dcm_security_dir", "security"),
|
|
@@ -475,9 +475,12 @@ def _run_setup(project_dir: Path):
|
|
|
475
475
|
# --- App database ---
|
|
476
476
|
app_database = get_input_string_or_default(
|
|
477
477
|
"Enter native app database name",
|
|
478
|
-
"
|
|
478
|
+
"MONITORIAL_ORBIT"
|
|
479
479
|
)
|
|
480
480
|
|
|
481
|
+
# --- Sandpit name (for your personal local-dev database) ---
|
|
482
|
+
sandpit_name = get_input_string("Enter your name (used to build your personal sandpit database, e.g. 'jon'): ")
|
|
483
|
+
|
|
481
484
|
# --- CoCo agent connection ---
|
|
482
485
|
cortex_connection = ""
|
|
483
486
|
if shutil.which("cortex") and connection_names:
|
|
@@ -496,104 +499,115 @@ def _run_setup(project_dir: Path):
|
|
|
496
499
|
source_control = SourceControl()
|
|
497
500
|
work_items = WorkItems()
|
|
498
501
|
matched_profile = None
|
|
502
|
+
account_identifier = ""
|
|
503
|
+
project_settings_loaded = False
|
|
499
504
|
|
|
505
|
+
connected = False
|
|
500
506
|
try:
|
|
501
507
|
temp_sf.connect()
|
|
502
508
|
success_line("Connected")
|
|
509
|
+
connected = True
|
|
510
|
+
except Exception as e:
|
|
511
|
+
warning_line(f"Could not connect to Snowflake: {e}")
|
|
512
|
+
warning_line("You can set profile_name manually in .datasecops.yml")
|
|
513
|
+
warning_line("You may need to create profiles.yml manually")
|
|
503
514
|
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
515
|
+
profile_name = ""
|
|
516
|
+
if connected:
|
|
517
|
+
# Each framework config code is fetched independently: a failure (or a missing
|
|
518
|
+
# config row) must not discard config that was already fetched successfully, and
|
|
519
|
+
# must not silently fall back to a default (e.g. GitHub) without saying so.
|
|
520
|
+
try:
|
|
521
|
+
raw_project = temp_sf.get_framework_config("PROJECT")
|
|
522
|
+
if raw_project:
|
|
523
|
+
project_settings = ProjectSettings(**{k: v for k, v in raw_project.items() if k in ProjectSettings.model_fields})
|
|
524
|
+
project_settings_loaded = True
|
|
525
|
+
else:
|
|
526
|
+
warning_line("No PROJECT framework config found in native app - using defaults")
|
|
527
|
+
except Exception as e:
|
|
528
|
+
warning_line(f"Could not load PROJECT framework config: {e}")
|
|
508
529
|
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
530
|
+
try:
|
|
531
|
+
raw_sc = temp_sf.get_framework_config("SOURCE_CONTROL")
|
|
532
|
+
if raw_sc:
|
|
533
|
+
if "source_control_platform" not in raw_sc:
|
|
534
|
+
warning_line(f"SOURCE_CONTROL framework config has no source_control_platform - "
|
|
535
|
+
f"defaulting to {source_control.source_control_platform}")
|
|
536
|
+
source_control = SourceControl(**{k: v for k, v in raw_sc.items() if k in SourceControl.model_fields})
|
|
537
|
+
else:
|
|
538
|
+
warning_line(f"No SOURCE_CONTROL framework config found in native app - "
|
|
539
|
+
f"defaulting source control platform to {source_control.source_control_platform}")
|
|
540
|
+
except Exception as e:
|
|
541
|
+
warning_line(f"Could not load SOURCE_CONTROL framework config "
|
|
542
|
+
f"(defaulting source control platform to {source_control.source_control_platform}): {e}")
|
|
513
543
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
544
|
+
try:
|
|
545
|
+
raw_wi = temp_sf.get_framework_config("WORK_ITEMS")
|
|
546
|
+
if raw_wi:
|
|
547
|
+
work_items = WorkItems(**{k: v for k, v in raw_wi.items() if k in WorkItems.model_fields})
|
|
548
|
+
except Exception as e:
|
|
549
|
+
warning_line(f"Could not load WORK_ITEMS framework config: {e}")
|
|
518
550
|
|
|
519
551
|
# Get account identifier for MCP URL construction
|
|
520
|
-
account_identifier = ""
|
|
521
552
|
try:
|
|
522
553
|
org_rows = temp_sf.execute_query("SELECT CURRENT_ORGANIZATION_NAME() || '-' || CURRENT_ACCOUNT_NAME() AS acct")
|
|
523
554
|
account_identifier = org_rows[0]["ACCT"].lower() if org_rows else ""
|
|
524
555
|
except Exception as e:
|
|
525
556
|
warning_line(f"Could not determine account identifier (AI Agent MCP will be skipped): {e}")
|
|
526
557
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
558
|
+
try:
|
|
559
|
+
# --- Resolve profile name ---
|
|
560
|
+
from datasecops_cli.utilities.yaml_utils import get_profile_name
|
|
561
|
+
profile_name = get_profile_name(project_dir)
|
|
562
|
+
if profile_name:
|
|
563
|
+
success_line(f"Profile found in dbt_project.yml: {profile_name}")
|
|
564
|
+
else:
|
|
565
|
+
info_line("No dbt_project.yml found. Fetching project profiles...")
|
|
566
|
+
profiles_data = temp_sf.get_project_profiles()
|
|
567
|
+
if profiles_data:
|
|
568
|
+
profile_names = [p.get("PROFILE_NAME", p.get("profile_name", "")) for p in profiles_data]
|
|
569
|
+
profile_names = [n for n in profile_names if n]
|
|
570
|
+
if len(profile_names) == 1:
|
|
571
|
+
profile_name = profile_names[0]
|
|
572
|
+
success_line(f"Auto-selected profile: {profile_name}")
|
|
573
|
+
elif len(profile_names) > 1:
|
|
574
|
+
info_line(f"Found {len(profile_names)} project profiles:")
|
|
575
|
+
profile_name = select_from_list(profile_names, "profile", add_back=False)
|
|
576
|
+
else:
|
|
577
|
+
warning_line("No profiles found in native app")
|
|
544
578
|
else:
|
|
545
579
|
warning_line("No profiles found in native app")
|
|
546
|
-
else:
|
|
547
|
-
warning_line("No profiles found in native app")
|
|
548
|
-
|
|
549
|
-
# Find the full profile object for profiles.yml generation
|
|
550
|
-
if profile_name:
|
|
551
|
-
profiles_data = temp_sf.get_project_profiles()
|
|
552
|
-
if profiles_data:
|
|
553
|
-
for p in profiles_data:
|
|
554
|
-
pn = p.get("PROFILE_NAME", p.get("profile_name", ""))
|
|
555
|
-
if pn == profile_name:
|
|
556
|
-
matched_profile = ProjectProfile(**{k.lower(): v for k, v in p.items()})
|
|
557
|
-
break
|
|
558
|
-
|
|
559
|
-
# --- Ensure profiles.yml has the profile entry ---
|
|
560
|
-
if profile_name and matched_profile:
|
|
561
|
-
import yaml
|
|
562
|
-
from datasecops_cli.utilities.file_utils import ensure_dir, write_file
|
|
563
|
-
from datasecops_cli.services.dbt_project_generator import generate_profiles_yml
|
|
564
|
-
|
|
565
|
-
profiles_dir = Path(project_settings.profile_dir).expanduser()
|
|
566
|
-
profiles_path = profiles_dir / "profiles.yml"
|
|
567
|
-
|
|
568
|
-
# Read existing profiles.yml or start fresh
|
|
569
|
-
existing_profiles = {}
|
|
570
|
-
if profiles_path.exists():
|
|
571
|
-
with open(profiles_path, "r", encoding="utf-8") as f:
|
|
572
|
-
existing_profiles = yaml.safe_load(f) or {}
|
|
573
580
|
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
581
|
+
# Find the full profile object for profiles.yml generation
|
|
582
|
+
if profile_name:
|
|
583
|
+
profiles_data = temp_sf.get_project_profiles()
|
|
584
|
+
if profiles_data:
|
|
585
|
+
for p in profiles_data:
|
|
586
|
+
pn = p.get("PROFILE_NAME", p.get("profile_name", ""))
|
|
587
|
+
if pn == profile_name:
|
|
588
|
+
matched_profile = ProjectProfile(**{k.lower(): v for k, v in p.items()})
|
|
589
|
+
break
|
|
590
|
+
|
|
591
|
+
# --- Ensure profiles.yml has the profile entry ---
|
|
592
|
+
if profile_name and matched_profile and not project_settings_loaded:
|
|
593
|
+
warning_line(f"Skipping profiles.yml for '{profile_name}': PROJECT framework config "
|
|
594
|
+
"was not loaded, so the local-dev role/warehouse aren't known")
|
|
595
|
+
elif profile_name and matched_profile:
|
|
596
|
+
from datasecops_cli.services.dbt_project_generator import write_profiles_yml
|
|
597
|
+
from datasecops_cli.services.snowflake_service import get_connection_params
|
|
598
|
+
|
|
599
|
+
profiles_dir = Path(project_settings.profile_dir).expanduser()
|
|
600
|
+
connection_params = get_connection_params(connection_name)
|
|
601
|
+
status = write_profiles_yml(profiles_dir, profile_name, matched_profile, project_settings, connection_params, sandpit_name)
|
|
602
|
+
success_line(f"Profile '{profile_name}' {status} in {profiles_dir / 'profiles.yml'}")
|
|
603
|
+
except Exception as e:
|
|
604
|
+
warning_line(f"Could not resolve the project profile / write profiles.yml: {e}")
|
|
605
|
+
warning_line("You can set profile_name manually in .datasecops.yml")
|
|
606
|
+
warning_line("You may need to create profiles.yml manually")
|
|
607
|
+
from datasecops_cli.utilities.yaml_utils import get_profile_name
|
|
608
|
+
profile_name = get_profile_name(project_dir) or ""
|
|
587
609
|
|
|
588
|
-
|
|
589
|
-
warning_line(f"Could not connect to Snowflake: {e}")
|
|
590
|
-
warning_line("You can set profile_name manually in .datasecops.yml")
|
|
591
|
-
warning_line("You may need to create profiles.yml manually")
|
|
592
|
-
# Fall back to reading profile from dbt_project.yml if available
|
|
593
|
-
from datasecops_cli.utilities.yaml_utils import get_profile_name
|
|
594
|
-
profile_name = get_profile_name(project_dir) or ""
|
|
595
|
-
finally:
|
|
596
|
-
temp_sf.close()
|
|
610
|
+
temp_sf.close()
|
|
597
611
|
|
|
598
612
|
# --- Write config ---
|
|
599
613
|
config_data = {
|
|
@@ -601,6 +615,7 @@ def _run_setup(project_dir: Path):
|
|
|
601
615
|
"app_database": app_database,
|
|
602
616
|
"profile_name": profile_name,
|
|
603
617
|
"cortex_connection": cortex_connection,
|
|
618
|
+
"sandpit_name": sandpit_name,
|
|
604
619
|
}
|
|
605
620
|
write_datasecops_config(project_dir, config_data)
|
|
606
621
|
success_line(f".datasecops.yml written to {project_dir / '.datasecops.yml'}")
|
|
@@ -796,11 +811,11 @@ def _connect_and_load(config: Config, interactive: bool = True,
|
|
|
796
811
|
break
|
|
797
812
|
# Save the selected profile back to .datasecops.yml
|
|
798
813
|
if config.profile:
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
814
|
+
from datasecops_cli.utilities.yaml_utils import read_datasecops_config
|
|
815
|
+
config_data = read_datasecops_config(config.project_dir) or {}
|
|
816
|
+
config_data["connection_name"] = config.datasecops.connection_name
|
|
817
|
+
config_data["app_database"] = config.datasecops.app_database
|
|
818
|
+
config_data["profile_name"] = config.profile_name
|
|
804
819
|
write_datasecops_config(config.project_dir, config_data)
|
|
805
820
|
success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
|
|
806
821
|
else:
|
|
@@ -808,11 +823,11 @@ def _connect_and_load(config: Config, interactive: bool = True,
|
|
|
808
823
|
info_line(f"Using default profile: {config.profile_name}")
|
|
809
824
|
elif not profile_was_set and len(config.all_profiles) == 1:
|
|
810
825
|
# Single profile auto-selected — persist it so future runs don't re-prompt
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
826
|
+
from datasecops_cli.utilities.yaml_utils import read_datasecops_config
|
|
827
|
+
config_data = read_datasecops_config(config.project_dir) or {}
|
|
828
|
+
config_data["connection_name"] = config.datasecops.connection_name
|
|
829
|
+
config_data["app_database"] = config.datasecops.app_database
|
|
830
|
+
config_data["profile_name"] = config.profile_name
|
|
816
831
|
write_datasecops_config(config.project_dir, config_data)
|
|
817
832
|
success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
|
|
818
833
|
|
|
@@ -860,7 +875,9 @@ def _run_interactive(config: Config):
|
|
|
860
875
|
dbt_runner = DbtRunner(
|
|
861
876
|
project_dir=config.dbt_project_dir,
|
|
862
877
|
profiles_dir=config.get_dbt_profiles_dir(),
|
|
863
|
-
|
|
878
|
+
# Always the developer's own identity, never the framework's CI/deployment
|
|
879
|
+
# default target (which may be a deploy-only target running as the CI identity).
|
|
880
|
+
target="local-dev",
|
|
864
881
|
engine=config.datasecops.dbt_engine or "dbt_core"
|
|
865
882
|
)
|
|
866
883
|
|
|
@@ -1230,7 +1247,7 @@ def _show_main_menu(profile_name: str, git_service: GitService = None,
|
|
|
1230
1247
|
|
|
1231
1248
|
def _run_bootstrap(config: Config):
|
|
1232
1249
|
"""Run the bootstrap command to initialise a new project."""
|
|
1233
|
-
from datasecops_cli.utilities.display import get_input_true_false, select_multiple_from_list
|
|
1250
|
+
from datasecops_cli.utilities.display import get_input_string, get_input_true_false, select_multiple_from_list
|
|
1234
1251
|
|
|
1235
1252
|
sf_service = _connect_and_load(config)
|
|
1236
1253
|
|
|
@@ -1241,6 +1258,15 @@ def _run_bootstrap(config: Config):
|
|
|
1241
1258
|
platform = config.source_control.source_control_platform.lower()
|
|
1242
1259
|
info_line(f"Platform: {platform}")
|
|
1243
1260
|
|
|
1261
|
+
# Sandpit name (for your personal local-dev database) — ask once, then persist
|
|
1262
|
+
if not config.datasecops.sandpit_name:
|
|
1263
|
+
from datasecops_cli.utilities.yaml_utils import read_datasecops_config, write_datasecops_config
|
|
1264
|
+
config.datasecops.sandpit_name = get_input_string(
|
|
1265
|
+
"Enter your name (used to build your personal sandpit database, e.g. 'jon'): ")
|
|
1266
|
+
config_data = read_datasecops_config(config.project_dir) or {}
|
|
1267
|
+
config_data["sandpit_name"] = config.datasecops.sandpit_name
|
|
1268
|
+
write_datasecops_config(config.project_dir, config_data)
|
|
1269
|
+
|
|
1244
1270
|
# Ask about optional steps
|
|
1245
1271
|
install_skills = get_input_true_false("Install AI coding skills?")
|
|
1246
1272
|
skill_targets = None
|
{datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/models/project_config.py
RENAMED
|
@@ -10,6 +10,7 @@ class DatasecopsConfig(BaseModel):
|
|
|
10
10
|
app_database: str = ""
|
|
11
11
|
profile_name: str = ""
|
|
12
12
|
cortex_connection: str = ""
|
|
13
|
+
sandpit_name: str = "" # personal name used to build your local-dev sandpit database (e.g. "jon")
|
|
13
14
|
dbt_engine: str = "dbt_core" # "dbt2" for dbt 2.0 (pip install dbt), "dbt_core" for dbt Core 1.x (pip install dbt-core + dbt-snowflake)
|
|
14
15
|
dcm_enabled: bool = False
|
|
15
16
|
dcm_security_dir: str = "security"
|
{datasecops_cli-0.7.0b1 → datasecops_cli-0.7.0b4}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
@@ -112,7 +112,9 @@ class BootstrapService:
|
|
|
112
112
|
dbt_runner = DbtRunner(
|
|
113
113
|
project_dir=dbt_project_dir,
|
|
114
114
|
profiles_dir=Path(self.project_settings.profile_dir).expanduser(),
|
|
115
|
-
|
|
115
|
+
# Always the developer's own identity, never the framework's CI/deployment
|
|
116
|
+
# default target.
|
|
117
|
+
target="local-dev",
|
|
116
118
|
engine=self.datasecops_config.dbt_engine,
|
|
117
119
|
)
|
|
118
120
|
result = dbt_runner.deps()
|