datasecops-cli 0.7.0b4__tar.gz → 0.7.0b5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/CHANGELOG.md +7 -0
  2. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/PKG-INFO +28 -1
  3. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/README.md +27 -0
  4. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/pyproject.toml +1 -1
  5. datasecops_cli-0.7.0b5/src/datasecops_cli/__init__.py +1 -0
  6. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/config.py +5 -2
  7. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/main.py +51 -1
  8. datasecops_cli-0.7.0b5/src/datasecops_cli/services/pipeline_env.py +93 -0
  9. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/snowflake_service.py +15 -1
  10. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_mcp/connection.py +4 -1
  11. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_config.py +7 -0
  12. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_main.py +40 -1
  13. datasecops_cli-0.7.0b5/tests/test_pipeline_env.py +108 -0
  14. datasecops_cli-0.7.0b4/src/datasecops_cli/__init__.py +0 -1
  15. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/.github/workflows/auto-tag.yml +0 -0
  16. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/.github/workflows/publish-cli.yml +0 -0
  17. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/.github/workflows/test.yml +0 -0
  18. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/.gitignore +0 -0
  19. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/AGENTS.md +0 -0
  20. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/DEVELOPMENT.md +0 -0
  21. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/LICENSE +0 -0
  22. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/dcm.md +0 -0
  23. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/getting-started.md +0 -0
  24. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/git-operations.md +0 -0
  25. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/legacy.md +0 -0
  26. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/legacy_plan_of_action.md +0 -0
  27. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/docs/mcp-server.md +0 -0
  28. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/mcp-servers.json +0 -0
  29. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/scripts/vendor_model.py +0 -0
  30. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/setup.ps1 +0 -0
  31. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/setup.sh +0 -0
  32. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/__init__.py +0 -0
  33. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/configuration.py +0 -0
  34. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/dcm_operations.py +0 -0
  35. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/development.py +0 -0
  36. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/downloads.py +0 -0
  37. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/menus/git_operations.py +0 -0
  38. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/models/__init__.py +0 -0
  39. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/models/git_helpers.py +0 -0
  40. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/models/project_config.py +0 -0
  41. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/__init__.py +0 -0
  42. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/bootstrap_service.py +0 -0
  43. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dbt_project_generator.py +0 -0
  44. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dbt_runner.py +0 -0
  45. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dcm_orchestrator.py +0 -0
  46. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dcm_pipelines.py +0 -0
  47. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dcm_registry.py +0 -0
  48. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/dcm_service.py +0 -0
  49. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/directory_scaffolder.py +0 -0
  50. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/download_service.py +0 -0
  51. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/git_service.py +0 -0
  52. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/linting_service.py +0 -0
  53. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/model_commands.py +0 -0
  54. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/skill_service.py +0 -0
  55. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/services/upstream_service.py +0 -0
  56. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/utilities/__init__.py +0 -0
  57. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/utilities/display.py +0 -0
  58. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/utilities/file_utils.py +0 -0
  59. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/utilities/mcp.py +0 -0
  60. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  61. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_mcp/__init__.py +0 -0
  62. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_mcp/__main__.py +0 -0
  63. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_mcp/server.py +0 -0
  64. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/__init__.py +0 -0
  65. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/convert.py +0 -0
  66. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/diagram.py +0 -0
  67. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/expand.py +0 -0
  68. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/extract.py +0 -0
  69. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/render.py +0 -0
  70. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/schema.py +0 -0
  71. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/scope.py +0 -0
  72. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/scripts.py +0 -0
  73. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/starter.py +0 -0
  74. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/templates/macros/role.sql +0 -0
  75. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/templates/macros/role_guard.sql +0 -0
  76. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/templates/macros/warehouse.sql +0 -0
  77. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/src/datasecops_model/validate.py +0 -0
  78. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/__init__.py +0 -0
  79. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_dbt_project_generator_profiles.py +0 -0
  80. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_dbt_runner.py +0 -0
  81. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_dcm_model.py +0 -0
  82. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_dcm_orchestrator.py +0 -0
  83. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_dcm_service.py +0 -0
  84. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_download_service.py +0 -0
  85. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_file_utils.py +0 -0
  86. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_linting_service.py +0 -0
  87. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_model_commands.py +0 -0
  88. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_models.py +0 -0
  89. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_version.py +0 -0
  90. {datasecops_cli-0.7.0b4 → datasecops_cli-0.7.0b5}/tests/test_yaml_utils.py +0 -0
@@ -4,11 +4,18 @@ All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ## [0.7.0b5] - 2026-10-26
8
+
9
+ ### Added
10
+
11
+ - **`datasecops pipeline env`** — resolves the dbt target for a pipeline run from the native app's dbt Targets (`--branch` matches a target's Branch, then a target named like the branch; `--target` picks one by name) and sets `TARGET`, `SF_ROLE`, `SF_WAREHOUSE` and `TARGET_SCHEMA` (`$GITHUB_ENV` on GitHub Actions, `##vso` on Azure DevOps, `KEY=value` otherwise). Generated pipelines use it instead of hard-coding the CI role and warehouse.
12
+
7
13
  ### Fixed
8
14
 
9
15
  - **`profiles.yml` deployment targets were being written to a developer's own `~/.dbt/profiles.yml`** — CI/deployment targets (`dev`/`test`/`prod`) belong to the native app's CI pipeline, not a developer's machine, so they're no longer written there at all. Only the `local-dev` target is ever added or refreshed; any other targets already present in the file (added manually, or by a framework target sharing the profile's name) are left untouched. Removed the now-unused CI-identity profile generation (`ProjectSettings.ci_username`, the old `generate_profiles_yml`).
10
16
  - **`local-dev` used a hardcoded `authenticator: externalbrowser` and a shared database instead of the developer's own identity** — `local-dev` now reuses whatever Snowflake CLI connection is already configured for the project in `~/.snowflake/connections.toml` (account, user, and whatever auth method is already in use — key pair, SSO, password, etc.), read directly via `snowflake.connector.config_manager` without establishing a connection. `role`/`warehouse`/`schema` still always come from the project's dbt targets config. The developer is prompted once for their name (persisted to `.datasecops.yml`), used to build a personal sandpit database (`<target_database>SANDPIT_<name>`) so multiple developers never collide.
11
17
  - **`profiles.yml` is no longer written inside the dbt project folder** — dbt never reads a `profiles.yml` there (it reads `--profiles-dir`, i.e. `~/.dbt` per the native app's `profile_dir` setting), so the project-local copy was always a dead reference file. `datasecops setup`/`bootstrap`/`new project` now write/merge the `local-dev` target directly into `~/.dbt/profiles.yml` via a shared `write_profiles_yml()` helper, creating the folder/file if needed.
18
+ - **`app_database` is validated before it is used in SQL** — it is interpolated into the native app `CALL` statements (CLI and MCP server), so a value that isn't a single plain or double-quoted identifier is now rejected instead of being able to change the statement.
12
19
 
13
20
  ## [0.7.0b2] - 2026-10-26
14
21
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: datasecops-cli
3
- Version: 0.7.0b4
3
+ Version: 0.7.0b5
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -146,6 +146,33 @@ Your pipeline needs two things:
146
146
 
147
147
  The exit code is `0` on success, `1` if any download fails.
148
148
 
149
+ ### Environment settings for the branch
150
+
151
+ `datasecops pipeline env` reads the dbt Targets from the native app (**Settings › Data › dbt
152
+ Targets**) and sets `TARGET`, `SF_ROLE`, `SF_WAREHOUSE` and `TARGET_SCHEMA` for the target mapped to
153
+ the branch, so pipelines don't hard-code them:
154
+
155
+ ```bash
156
+ datasecops pipeline env --branch "$GITHUB_REF_NAME" # GitHub Actions: target whose Branch is this branch
157
+ datasecops pipeline env --branch "$BUILD_SOURCEBRANCH" # Azure DevOps (the full ref, not BUILD_SOURCEBRANCHNAME)
158
+ datasecops pipeline env --target unit-test # a named target
159
+ ```
160
+
161
+ Pass the branch through the runner's environment variable, not a `${{ ... }}` / `$(...)` expression:
162
+ those are substituted into the script before the shell runs, so a branch name containing `$(...)`
163
+ would execute.
164
+
165
+ - `--branch` matches a target's **Branch**, then a target named like the branch; `refs/heads/` is
166
+ ignored and a pull request's merge ref resolves to its source branch (`$GITHUB_HEAD_REF` on GitHub,
167
+ `$SYSTEM_PULLREQUEST_SOURCEBRANCH` on Azure DevOps). On Azure DevOps pass `$BUILD_SOURCEBRANCH`:
168
+ `$BUILD_SOURCEBRANCHNAME` is only the last part of the ref (`release/prod` -> `prod`).
169
+ Exact names win; a case-insensitive match is used only when it is unambiguous.
170
+ - GitHub Actions: the values are appended to `$GITHUB_ENV`. Azure DevOps: they are set with
171
+ `##vso[task.setvariable]`. Elsewhere (or `--format shell`) they are printed as shell-quoted `KEY=value` lines.
172
+ - A target with no role, warehouse or schema leaves that variable as it is.
173
+ - The command connects with `.datasecops.yml`'s connection, so the CI user needs the `orbit_developer`
174
+ application role. It exits `1` when no target matches.
175
+
149
176
  ## DCM projects
150
177
 
151
178
  ```bash
@@ -126,6 +126,33 @@ Your pipeline needs two things:
126
126
 
127
127
  The exit code is `0` on success, `1` if any download fails.
128
128
 
129
+ ### Environment settings for the branch
130
+
131
+ `datasecops pipeline env` reads the dbt Targets from the native app (**Settings › Data › dbt
132
+ Targets**) and sets `TARGET`, `SF_ROLE`, `SF_WAREHOUSE` and `TARGET_SCHEMA` for the target mapped to
133
+ the branch, so pipelines don't hard-code them:
134
+
135
+ ```bash
136
+ datasecops pipeline env --branch "$GITHUB_REF_NAME" # GitHub Actions: target whose Branch is this branch
137
+ datasecops pipeline env --branch "$BUILD_SOURCEBRANCH" # Azure DevOps (the full ref, not BUILD_SOURCEBRANCHNAME)
138
+ datasecops pipeline env --target unit-test # a named target
139
+ ```
140
+
141
+ Pass the branch through the runner's environment variable, not a `${{ ... }}` / `$(...)` expression:
142
+ those are substituted into the script before the shell runs, so a branch name containing `$(...)`
143
+ would execute.
144
+
145
+ - `--branch` matches a target's **Branch**, then a target named like the branch; `refs/heads/` is
146
+ ignored and a pull request's merge ref resolves to its source branch (`$GITHUB_HEAD_REF` on GitHub,
147
+ `$SYSTEM_PULLREQUEST_SOURCEBRANCH` on Azure DevOps). On Azure DevOps pass `$BUILD_SOURCEBRANCH`:
148
+ `$BUILD_SOURCEBRANCHNAME` is only the last part of the ref (`release/prod` -> `prod`).
149
+ Exact names win; a case-insensitive match is used only when it is unambiguous.
150
+ - GitHub Actions: the values are appended to `$GITHUB_ENV`. Azure DevOps: they are set with
151
+ `##vso[task.setvariable]`. Elsewhere (or `--format shell`) they are printed as shell-quoted `KEY=value` lines.
152
+ - A target with no role, warehouse or schema leaves that variable as it is.
153
+ - The command connects with `.datasecops.yml`'s connection, so the CI user needs the `orbit_developer`
154
+ application role. It exits `1` when no target matches.
155
+
129
156
  ## DCM projects
130
157
 
131
158
  ```bash
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.7.0b4"
7
+ version = "0.7.0b5"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1 @@
1
+ __version__ = "0.7.0b5"
@@ -23,8 +23,9 @@ class Config:
23
23
  self.dbt_project_dir: Path = Path.cwd()
24
24
  self.profile_name: str = ""
25
25
 
26
- def load(self, project_dir: Path = None) -> bool:
27
- """Load configuration from .datasecops.yml and dbt_project.yml."""
26
+ def load(self, project_dir: Path = None, connection_name: Optional[str] = None) -> bool:
27
+ """Load configuration from .datasecops.yml and dbt_project.yml. `connection_name`
28
+ (a --connection option) overrides the file's and may stand in for a missing one."""
28
29
  self.project_dir = project_dir or Path.cwd()
29
30
 
30
31
  # Load .datasecops.yml
@@ -54,6 +55,8 @@ class Config:
54
55
  warning_line(f"Invalid dbt_engine '{self.datasecops.dbt_engine}' in .datasecops.yml — falling back to dbt_core")
55
56
  self.datasecops.dbt_engine = "dbt_core"
56
57
 
58
+ if connection_name:
59
+ self.datasecops.connection_name = connection_name
57
60
  if not self.datasecops.connection_name or not self.datasecops.app_database:
58
61
  error_line(".datasecops.yml is missing connection_name or app_database.")
59
62
  return False
@@ -17,7 +17,8 @@ from datasecops_cli.services.download_service import DownloadService
17
17
  from datasecops_cli.services.skill_service import SkillService
18
18
  from datasecops_cli.services.bootstrap_service import BootstrapService
19
19
  from datasecops_cli.services.dcm_service import DcmService
20
- from datasecops_cli.services import dcm_registry, model_commands
20
+ from datasecops_cli.services import dcm_registry, model_commands, pipeline_env
21
+ from datasecops_cli.models.project_config import DbtTarget
21
22
  from datasecops_cli.menus.development import DevelopmentMenu
22
23
  from datasecops_cli.menus.git_operations import GitOperationsMenu
23
24
  from datasecops_cli.menus.downloads import DownloadsMenu
@@ -239,6 +240,55 @@ def dcm_pipelines_cmd(platform, names, force, projects, connection):
239
240
  sf_service.close()
240
241
 
241
242
 
243
+ # ── pipeline commands ──────────────────────────────────────────────────────
244
+
245
+
246
+ @cli.group()
247
+ def pipeline():
248
+ """Helpers for CI/CD pipelines."""
249
+
250
+
251
+ @pipeline.command("env")
252
+ @click.option("--branch", "-b", default=None,
253
+ help="Branch being built: picks the dbt target mapped to it (a GitHub pull request's "
254
+ "<n>/merge ref resolves to its source branch)")
255
+ @click.option("--target", "-t", default=None, help="dbt target name (instead of --branch)")
256
+ @click.option("--connection", "-c", default=None, help="Snowflake connection name (overrides .datasecops.yml)")
257
+ @click.option("--format", "fmt", type=click.Choice(["auto", *pipeline_env.PLATFORMS]), default="auto",
258
+ show_default=True, help="auto: GitHub when $GITHUB_ENV is set, Azure DevOps when $TF_BUILD is set")
259
+ def pipeline_env_cmd(branch: Optional[str], target: Optional[str], connection: Optional[str], fmt: str):
260
+ """Set TARGET, SF_ROLE, SF_WAREHOUSE and TARGET_SCHEMA from the app's dbt Targets.
261
+
262
+ GitHub Actions: appended to $GITHUB_ENV. Azure DevOps: set with ##vso logging commands.
263
+ Otherwise printed as KEY=value. Empty role / warehouse / schema values are left unset.
264
+ """
265
+ if bool(branch) == bool(target):
266
+ raise click.UsageError("give exactly one of --branch or --target")
267
+ config = Config()
268
+ if not config.load(connection_name=connection):
269
+ sys.exit(1)
270
+ sf_service = SnowflakeService(config.datasecops)
271
+ try:
272
+ raw = sf_service.get_framework_config("PROJECT") or {}
273
+ except Exception as e: # noqa: BLE001 - connector errors
274
+ error_line(f"Failed to read the dbt targets from {config.datasecops.app_database}: {e}")
275
+ sys.exit(1)
276
+ finally:
277
+ sf_service.close()
278
+ targets = [DbtTarget(**{k: v for k, v in t.items() if k in DbtTarget.model_fields})
279
+ for t in raw.get("targets") or []]
280
+ try:
281
+ resolved = pipeline_env.resolve_target(targets, branch=branch, target=target)
282
+ except pipeline_env.TargetNotFound as e:
283
+ error_line(str(e))
284
+ sys.exit(1)
285
+ values = pipeline_env.env_values(resolved)
286
+ platform = pipeline_env.detect_platform() if fmt == "auto" else fmt
287
+ for line in pipeline_env.export(values, platform):
288
+ click.echo(line)
289
+ click.echo("dbt environment: " + ", ".join(f"{k}={v}" for k, v in values.items()), err=True)
290
+
291
+
242
292
  # ── model commands ─────────────────────────────────────────────────────────
243
293
 
244
294
 
@@ -0,0 +1,93 @@
1
+ """Resolve a pipeline run's dbt environment (target, role, warehouse, schema) from the dbt Targets.
2
+
3
+ The dbt Targets (Settings > Data > dbt Targets in the native app) map branches to targets and give
4
+ each target its role, warehouse and schema. Pipelines call `datasecops pipeline env` instead of
5
+ hard-coding them, so a change in the app applies without regenerating the pipelines.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import re
11
+ import shlex
12
+ from typing import Optional
13
+
14
+ from datasecops_cli.models.project_config import DbtTarget
15
+
16
+ PLATFORMS = ("github", "azure", "shell")
17
+
18
+
19
+ class TargetNotFound(ValueError):
20
+ pass
21
+
22
+
23
+ def _pick(entries: list[DbtTarget], key, value: str) -> Optional[DbtTarget]:
24
+ """An exact match first; a case-insensitive one only when it is unambiguous."""
25
+ exact = [t for t in entries if key(t) and key(t) == value]
26
+ if exact:
27
+ return exact[0]
28
+ loose = [t for t in entries if key(t) and key(t).lower() == value.lower()]
29
+ return loose[0] if len(loose) == 1 else None
30
+
31
+
32
+ def branch_name(branch: str) -> str:
33
+ """The branch to look up: without refs/heads/, and for a pull request's merge ref (GitHub
34
+ `<number>/merge`, Azure DevOps `refs/pull/<number>/merge`) the pull request's source branch
35
+ ($GITHUB_HEAD_REF / $SYSTEM_PULLREQUEST_SOURCEBRANCH)."""
36
+ name = branch.removeprefix("refs/heads/")
37
+ if re.fullmatch(r"(refs/pull/)?\d+/merge", name):
38
+ source = os.environ.get("GITHUB_HEAD_REF") or os.environ.get("SYSTEM_PULLREQUEST_SOURCEBRANCH")
39
+ if source:
40
+ return source.removeprefix("refs/heads/")
41
+ return name
42
+
43
+
44
+ def resolve_target(targets: list[DbtTarget], branch: Optional[str] = None,
45
+ target: Optional[str] = None) -> DbtTarget:
46
+ """`target` matches a target name; `branch` matches a target's branch, then a target named
47
+ like the branch (the convention before branches were mapped). Exact matches win; a
48
+ case-insensitive match is used only when exactly one target has it."""
49
+ if not target and not branch:
50
+ raise TargetNotFound("give --branch or --target")
51
+ if target:
52
+ found = _pick(targets, lambda t: t.target_name, target)
53
+ else:
54
+ name = branch_name(branch)
55
+ found = _pick(targets, lambda t: t.branch_name, name) or _pick(targets, lambda t: t.target_name, name)
56
+ if found is None:
57
+ known = ", ".join(f"{t.target_name} (branch {t.branch_name})" if t.branch_name else t.target_name
58
+ for t in targets) or "none"
59
+ wanted = f"target '{target}'" if target else f"branch '{branch}'"
60
+ raise TargetNotFound(f"no dbt target for {wanted}; configured targets: {known}")
61
+ return found
62
+
63
+
64
+ def env_values(t: DbtTarget) -> dict[str, str]:
65
+ """Variables for the target. An empty role / warehouse / schema is left out, so the
66
+ pipeline's own (CI) value stays in place."""
67
+ values = {"TARGET": t.target_name, "SF_ROLE": t.target_role,
68
+ "SF_WAREHOUSE": t.target_warehouse, "TARGET_SCHEMA": t.target_schema}
69
+ return {k: v for k, v in values.items() if v}
70
+
71
+
72
+ def detect_platform() -> str:
73
+ if os.environ.get("GITHUB_ENV"):
74
+ return "github"
75
+ if os.environ.get("TF_BUILD"):
76
+ return "azure"
77
+ return "shell"
78
+
79
+
80
+ def export(values: dict[str, str], platform: str) -> list[str]:
81
+ """Write the values for the platform: GitHub appends them to $GITHUB_ENV, Azure DevOps sets
82
+ them with logging commands, otherwise shell-quoted KEY=value lines. Returns the lines to print."""
83
+ for k, v in values.items():
84
+ if "\n" in v or "\r" in v:
85
+ raise ValueError(f"{k} contains a newline")
86
+ if platform == "github":
87
+ with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as f:
88
+ f.writelines(f"{k}={v}\n" for k, v in values.items())
89
+ return []
90
+ if platform == "azure":
91
+ # The agent decodes %AZP25 / %0A / %0D in logging commands: encode % so values stay literal.
92
+ return [f"##vso[task.setvariable variable={k}]{v.replace('%', '%AZP25')}" for k, v in values.items()]
93
+ return [f"{k}={shlex.quote(v)}" for k, v in values.items()]
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import re
2
3
  from typing import Any, Optional
3
4
  import snowflake.connector
4
5
 
@@ -31,6 +32,19 @@ def get_connection_params(connection_name: str) -> dict[str, Any]:
31
32
  return {}
32
33
 
33
34
 
35
+ _PLAIN_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*")
36
+ _QUOTED_IDENTIFIER = re.compile(r'"(?:[^"]|"")+"')
37
+
38
+
39
+ def app_database_identifier(name: str) -> str:
40
+ """The app database as a SQL identifier: a plain name or one double-quoted name. It is
41
+ interpolated into CALL statements, so anything else (dots, spaces, SQL) is rejected."""
42
+ name = (name or "").strip()
43
+ if _PLAIN_IDENTIFIER.fullmatch(name) or _QUOTED_IDENTIFIER.fullmatch(name):
44
+ return name
45
+ raise ValueError(f"app_database must be a single Snowflake identifier, got {name!r}")
46
+
47
+
34
48
  class SnowflakeService:
35
49
  """Connects to Snowflake and calls native app API procedures."""
36
50
 
@@ -65,7 +79,7 @@ class SnowflakeService:
65
79
  def call_procedure(self, proc_name: str, *args) -> Any:
66
80
  if proc_name not in ALLOWED_PROCEDURES:
67
81
  raise ValueError(f"Unknown native app procedure: {proc_name}")
68
- db = self.config.app_database
82
+ db = app_database_identifier(self.config.app_database)
69
83
  arg_placeholders = ", ".join(["%s"] * len(args))
70
84
  sql = f"CALL {db}.api.{proc_name}({arg_placeholders})"
71
85
  rows = self.execute_query(sql, list(args))
@@ -11,13 +11,16 @@ from typing import Any, Optional
11
11
  import snowflake.connector
12
12
  import yaml
13
13
 
14
+ from datasecops_cli.services.snowflake_service import app_database_identifier
15
+
14
16
 
15
17
  class MCPSnowflakeService:
16
18
  """Lightweight Snowflake service for the MCP server."""
17
19
 
18
20
  def __init__(self, connection_name: str, app_database: str):
19
21
  self.connection_name = connection_name
20
- self.app_database = app_database
22
+ # Interpolated into CALL statements: must be a single identifier.
23
+ self.app_database = app_database_identifier(app_database)
21
24
  self._conn: Optional[snowflake.connector.SnowflakeConnection] = None
22
25
 
23
26
  @property
@@ -42,6 +42,13 @@ class TestConfigLoad:
42
42
  result = config.load(project_dir=tmp_path)
43
43
  assert result is False
44
44
 
45
+ def test_load_connection_override_fills_missing_connection_name(self, tmp_path):
46
+ (tmp_path / ".datasecops.yml").write_text("connection_name: ''\napp_database: MYDB\n", encoding="utf-8")
47
+
48
+ config = Config()
49
+ assert config.load(project_dir=tmp_path, connection_name="ci") is True
50
+ assert config.datasecops.connection_name == "ci"
51
+
45
52
  def test_load_missing_app_database(self, tmp_path):
46
53
  """Config.load() fails when app_database is empty string."""
47
54
  (tmp_path / ".datasecops.yml").write_text(
@@ -7,7 +7,7 @@ from click.testing import CliRunner
7
7
 
8
8
  from datasecops_cli.main import cli, _run_download, DOWNLOAD_ITEMS
9
9
  from datasecops_cli.config import Config
10
- from datasecops_cli.models.project_config import ProjectProfile
10
+ from datasecops_cli.models.project_config import DbtTarget, ProjectProfile
11
11
 
12
12
 
13
13
  class TestCliParsing:
@@ -508,3 +508,42 @@ class TestDownloadPackageFiltering:
508
508
  import yaml
509
509
  result = yaml.safe_load((tmp_path / "packages.yml").read_text())
510
510
  assert len(result["packages"]) == 2
511
+
512
+
513
+ class TestPipelineEnvCommand:
514
+ TARGETS = [
515
+ DbtTarget(target_name="unit-test", target_role="DATAOPS_ADMIN", target_warehouse="DEV_WH"),
516
+ DbtTarget(target_name="prd", branch_name="main", target_role="DATAOPS_ADMIN", target_warehouse="PRD_WH",
517
+ target_schema="CORE"),
518
+ ]
519
+
520
+ def _invoke(self, args, project=None):
521
+ sf = MagicMock()
522
+ sf.get_framework_config.return_value = project if project is not None else {
523
+ "targets": [t.model_dump() for t in self.TARGETS]}
524
+ with patch("datasecops_cli.main.Config") as config_cls, \
525
+ patch("datasecops_cli.main.SnowflakeService", return_value=sf):
526
+ config_cls.return_value.load.return_value = True
527
+ self.config = config_cls.return_value
528
+ result = CliRunner().invoke(cli, ["pipeline", "env", *args])
529
+ return result, sf
530
+
531
+ def test_prints_shell_values_for_the_branch(self):
532
+ result, sf = self._invoke(["--branch", "main", "--format", "shell"])
533
+ assert result.exit_code == 0, result.output
534
+ assert "TARGET=prd\nSF_ROLE=DATAOPS_ADMIN\nSF_WAREHOUSE=PRD_WH\nTARGET_SCHEMA=CORE\n" in result.output
535
+ sf.get_framework_config.assert_called_once_with("PROJECT")
536
+ sf.close.assert_called_once()
537
+
538
+ def test_unknown_branch_fails(self):
539
+ result, _ = self._invoke(["--branch", "feature/x", "--format", "shell"])
540
+ assert result.exit_code == 1
541
+
542
+ def test_needs_exactly_one_of_branch_and_target(self):
543
+ assert self._invoke(["--format", "shell"])[0].exit_code == 2
544
+ assert self._invoke(["--branch", "main", "--target", "prd"])[0].exit_code == 2
545
+
546
+ def test_connection_option_is_passed_to_config_load(self):
547
+ result, _ = self._invoke(["--target", "prd", "--format", "shell", "--connection", "ci"])
548
+ assert result.exit_code == 0, result.output
549
+ self.config.load.assert_called_once_with(connection_name="ci")
@@ -0,0 +1,108 @@
1
+ """Tests for datasecops_cli.services.pipeline_env (the `pipeline env` Click wiring is in test_main.py)."""
2
+ import pytest
3
+
4
+ from datasecops_cli.models.project_config import DbtTarget
5
+ from datasecops_cli.services import pipeline_env
6
+ from datasecops_cli.services.pipeline_env import TargetNotFound, env_values, export, resolve_target
7
+
8
+ TARGETS = [
9
+ DbtTarget(target_name="unit-test", target_role="DATAOPS_ADMIN", target_warehouse="DEV_WH"),
10
+ DbtTarget(target_name="dev", branch_name="develop", target_role="DEVELOPERS", target_warehouse="DEV_WH"),
11
+ DbtTarget(target_name="prd", branch_name="main", target_role="DATAOPS_ADMIN", target_warehouse="PRD_WH",
12
+ target_schema="CORE"),
13
+ DbtTarget(target_name="test", target_role="", target_warehouse=""),
14
+ ]
15
+
16
+
17
+ class TestResolveTarget:
18
+ def test_branch_uses_the_mapped_branch(self):
19
+ assert resolve_target(TARGETS, branch="main").target_name == "prd"
20
+ assert resolve_target(TARGETS, branch="refs/heads/develop").target_name == "dev"
21
+
22
+ def test_branch_falls_back_to_a_target_named_like_it(self):
23
+ assert resolve_target(TARGETS, branch="TEST").target_name == "test"
24
+
25
+ def test_explicit_target(self):
26
+ assert resolve_target(TARGETS, target="Unit-Test").target_name == "unit-test"
27
+
28
+ def test_unknown_branch_lists_the_targets(self):
29
+ with pytest.raises(TargetNotFound, match="prd \\(branch main\\)"):
30
+ resolve_target(TARGETS, branch="feature/x")
31
+
32
+ def test_exact_case_wins_and_ambiguous_case_matches_nothing(self):
33
+ targets = [DbtTarget(target_name="Prod", target_role="A"), DbtTarget(target_name="prod", target_role="B")]
34
+ assert resolve_target(targets, target="Prod").target_role == "A"
35
+ assert resolve_target(targets, target="prod").target_role == "B"
36
+ with pytest.raises(TargetNotFound):
37
+ resolve_target(targets, target="PROD")
38
+
39
+ def test_pull_request_merge_ref_uses_the_source_branch(self, monkeypatch):
40
+ monkeypatch.setenv("GITHUB_HEAD_REF", "develop")
41
+ assert resolve_target(TARGETS, branch="42/merge").target_name == "dev"
42
+ assert resolve_target(TARGETS, branch="refs/pull/42/merge").target_name == "dev"
43
+
44
+ def test_azure_pull_request_ref_uses_the_source_branch(self, monkeypatch):
45
+ monkeypatch.delenv("GITHUB_HEAD_REF", raising=False)
46
+ monkeypatch.setenv("SYSTEM_PULLREQUEST_SOURCEBRANCH", "refs/heads/develop")
47
+ assert resolve_target(TARGETS, branch="refs/pull/7/merge").target_name == "dev"
48
+
49
+ def test_branch_with_slashes_keeps_its_full_name(self):
50
+ targets = [DbtTarget(target_name="prod", branch_name="release/prod"), DbtTarget(target_name="other", branch_name="prod")]
51
+ assert resolve_target(targets, branch="refs/heads/release/prod").target_name == "prod"
52
+
53
+ def test_needs_branch_or_target(self):
54
+ with pytest.raises(TargetNotFound):
55
+ resolve_target(TARGETS)
56
+
57
+
58
+ class TestExport:
59
+ def test_values_leave_out_empty_settings(self):
60
+ assert env_values(TARGETS[2]) == {"TARGET": "prd", "SF_ROLE": "DATAOPS_ADMIN",
61
+ "SF_WAREHOUSE": "PRD_WH", "TARGET_SCHEMA": "CORE"}
62
+ assert env_values(TARGETS[3]) == {"TARGET": "test", "TARGET_SCHEMA": "PUBLIC"}
63
+
64
+ def test_azure_and_shell_lines(self):
65
+ values = {"TARGET": "prd", "SF_ROLE": "R"}
66
+ assert export(values, "azure") == ["##vso[task.setvariable variable=TARGET]prd",
67
+ "##vso[task.setvariable variable=SF_ROLE]R"]
68
+ assert export(values, "shell") == ["TARGET=prd", "SF_ROLE=R"]
69
+
70
+ def test_shell_values_are_quoted(self):
71
+ assert export({"SF_ROLE": "$(touch x) ok"}, "shell") == ["SF_ROLE='$(touch x) ok'"]
72
+
73
+ def test_azure_encodes_percent_so_escapes_stay_literal(self):
74
+ assert export({"TARGET": "build%0Aarchive"}, "azure") == [
75
+ "##vso[task.setvariable variable=TARGET]build%AZP250Aarchive"]
76
+
77
+ def test_github_appends_to_github_env(self, tmp_path, monkeypatch):
78
+ env_file = tmp_path / "github_env"
79
+ env_file.write_text("EXISTING=1\n")
80
+ monkeypatch.setenv("GITHUB_ENV", str(env_file))
81
+ assert export({"TARGET": "prd"}, "github") == []
82
+ assert env_file.read_text() == "EXISTING=1\nTARGET=prd\n"
83
+
84
+ def test_rejects_newlines(self):
85
+ with pytest.raises(ValueError):
86
+ export({"SF_ROLE": "A\nB=1"}, "shell")
87
+
88
+ def test_detect_platform(self, monkeypatch):
89
+ monkeypatch.delenv("GITHUB_ENV", raising=False)
90
+ monkeypatch.delenv("TF_BUILD", raising=False)
91
+ assert pipeline_env.detect_platform() == "shell"
92
+ monkeypatch.setenv("TF_BUILD", "True")
93
+ assert pipeline_env.detect_platform() == "azure"
94
+ monkeypatch.setenv("GITHUB_ENV", "/tmp/x")
95
+ assert pipeline_env.detect_platform() == "github"
96
+
97
+
98
+ class TestAppDatabaseIdentifier:
99
+ @pytest.mark.parametrize("name", ["MONITORIAL_ORBIT", '"MONITORIAL_ORBIT"', '"My ""App"""', "app$1"])
100
+ def test_accepts_one_identifier(self, name):
101
+ from datasecops_cli.services.snowflake_service import app_database_identifier
102
+ assert app_database_identifier(name) == name
103
+
104
+ @pytest.mark.parametrize("name", ["", "DB.api.x", "DB; drop table t", "DB --", '"A"."B"', "1DB"])
105
+ def test_rejects_anything_else(self, name):
106
+ from datasecops_cli.services.snowflake_service import app_database_identifier
107
+ with pytest.raises(ValueError):
108
+ app_database_identifier(name)
@@ -1 +0,0 @@
1
- __version__ = "0.7.0b4"