datasecops-cli 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/CHANGELOG.md +16 -1
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/PKG-INFO +1 -2
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/pyproject.toml +1 -2
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/main.py +156 -29
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/downloads.py +1 -1
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/project_config.py +2 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/download_service.py +6 -2
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/snowflake_service.py +4 -1
- datasecops_cli-0.3.2/src/datasecops_cli/services/upstream_service.py +158 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/display.py +14 -2
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_main.py +2 -2
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/.github/workflows/publish-cli.yml +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/.gitignore +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/DEVELOPMENT.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/LICENSE +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/README.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/getting-started.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/legacy.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/legacy_plan_of_action.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/mcp-server.md +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/mcp-servers.json +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/setup.ps1 +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/setup.sh +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/config.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/development.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/git_operations.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/git_helpers.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/bootstrap_service.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/dbt_runner.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/git_service.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/linting_service.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/skill_service.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/file_utils.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/__main__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/connection.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/server.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/__init__.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_config.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_file_utils.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_models.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_version.py +0 -0
- {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_yaml_utils.py +0 -0
|
@@ -2,15 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the DataSecOps CLI are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.3.2] - 2026-05-15
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **Upstream project version tracking** — if a profile has `upstream_projects` configured in the native app, the main menu header shows each upstream project's current pinned version vs the latest available tag (fetched via `git ls-remote`). Outdated dependencies are highlighted in yellow.
|
|
10
|
+
- **Auto-update upstream packages** — when outdated upstream packages are detected, a new menu option `[5] update pkgs` appears, which updates `packages.yml` revisions to the latest tags and optionally runs `dbtf deps`
|
|
11
|
+
- **`upstream_projects` field on `ProjectProfile`** — new model field to support upstream dependency tracking from the native app
|
|
12
|
+
|
|
5
13
|
## [0.3.0] - 2026-05-14
|
|
6
14
|
|
|
7
15
|
### Added
|
|
8
16
|
|
|
9
17
|
- **`datasecops setup` subcommand** — pure-Python replacement for `setup.ps1` / `setup.sh`. Discovers Snowflake connections from `~/.snowflake/connections.toml`, prompts for connection and app database, and writes `.datasecops.yml`. No platform-specific scripts needed.
|
|
10
|
-
- **
|
|
18
|
+
- **Prerequisite checks during setup** — checks for `dbtf`, `gh`, and `az` on PATH and verifies authentication status (`gh auth status`, `az account show`)
|
|
19
|
+
- **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one. Selected profile is saved back to `.datasecops.yml`.
|
|
11
20
|
- **Auto-setup on first run** — running `datasecops` without a `.datasecops.yml` now offers to run setup interactively instead of just exiting with an error
|
|
21
|
+
- **`--connection`, `--app-database`, `--profile` flags on `download`** — allows CI/CD pipelines to run without a committed `.datasecops.yml` by passing connection details as CLI flags (e.g. `datasecops download sqlfluff -c ci -d MY_APP_DB -p my_project`)
|
|
22
|
+
- **Default app database value** — setup now defaults to `DATA_ENGINEERS_DATASECOPS_FRAMEWORK` (press Enter to accept)
|
|
12
23
|
- **dbt project directory resolution from framework** — `dbt_project_dir` is now re-resolved using the framework's `project_dir` setting after native app config is loaded
|
|
13
24
|
|
|
25
|
+
### Changed
|
|
26
|
+
|
|
27
|
+
- **Non-interactive mode safety** — `_connect_and_load` accepts an `interactive` flag; the `download` subcommand passes `interactive=False` so it never prompts (missing config exits 1, multiple profiles use the first one silently)
|
|
28
|
+
|
|
14
29
|
## [0.2.9] - 2026-05-14
|
|
15
30
|
|
|
16
31
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datasecops-cli
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: DataSecOps Framework CLI for Snowflake Native App
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -10,7 +10,6 @@ Requires-Dist: gitpython>=3.1
|
|
|
10
10
|
Requires-Dist: pydantic>=2.0
|
|
11
11
|
Requires-Dist: pyyaml>=6.0
|
|
12
12
|
Requires-Dist: snowflake-connector-python>=3.0
|
|
13
|
-
Requires-Dist: sqlfluff>=3.0
|
|
14
13
|
Provides-Extra: mcp
|
|
15
14
|
Requires-Dist: mcp>=1.0; extra == 'mcp'
|
|
16
15
|
Provides-Extra: test
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "datasecops-cli"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "DataSecOps Framework CLI for Snowflake Native App"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -14,7 +14,6 @@ dependencies = [
|
|
|
14
14
|
"snowflake-connector-python>=3.0",
|
|
15
15
|
"colorama>=0.4",
|
|
16
16
|
"pyyaml>=6.0",
|
|
17
|
-
"sqlfluff>=3.0",
|
|
18
17
|
"pydantic>=2.0",
|
|
19
18
|
]
|
|
20
19
|
|
|
@@ -46,6 +46,18 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
46
46
|
choices=DOWNLOAD_ITEMS,
|
|
47
47
|
help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
|
|
48
48
|
)
|
|
49
|
+
dl.add_argument(
|
|
50
|
+
"--connection", "-c",
|
|
51
|
+
help="Snowflake connection name (overrides .datasecops.yml)",
|
|
52
|
+
)
|
|
53
|
+
dl.add_argument(
|
|
54
|
+
"--app-database", "-d",
|
|
55
|
+
help="Native app database name (overrides .datasecops.yml)",
|
|
56
|
+
)
|
|
57
|
+
dl.add_argument(
|
|
58
|
+
"--profile", "-p",
|
|
59
|
+
help="Project profile name (overrides .datasecops.yml)",
|
|
60
|
+
)
|
|
49
61
|
|
|
50
62
|
return parser
|
|
51
63
|
|
|
@@ -62,7 +74,9 @@ def main():
|
|
|
62
74
|
elif args.command == "bootstrap":
|
|
63
75
|
_run_bootstrap(config)
|
|
64
76
|
elif args.command == "download":
|
|
65
|
-
_run_download(config, args.items
|
|
77
|
+
_run_download(config, args.items,
|
|
78
|
+
connection=args.connection, app_database=args.app_database,
|
|
79
|
+
profile=args.profile)
|
|
66
80
|
else:
|
|
67
81
|
_run_interactive(config)
|
|
68
82
|
|
|
@@ -70,11 +84,40 @@ def main():
|
|
|
70
84
|
def _run_setup(project_dir: Path):
|
|
71
85
|
"""Interactive setup: create .datasecops.yml by prompting for connection and app database."""
|
|
72
86
|
from datasecops_cli.utilities.display import (
|
|
73
|
-
get_input_string, select_from_list
|
|
87
|
+
get_input_string, get_input_string_or_default, select_from_list, warning_line
|
|
74
88
|
)
|
|
75
89
|
|
|
76
90
|
section_header("DataSecOps Setup")
|
|
77
91
|
|
|
92
|
+
# --- Check prerequisites ---
|
|
93
|
+
info_line("Checking prerequisites...")
|
|
94
|
+
|
|
95
|
+
if shutil.which("dbtf"):
|
|
96
|
+
success_line("dbt Fusion (dbtf) found")
|
|
97
|
+
else:
|
|
98
|
+
warning_line("dbt Fusion (dbtf) not found — dbt commands will not work")
|
|
99
|
+
warning_line(" Install: https://docs.getdbt.com/docs/core/installation")
|
|
100
|
+
|
|
101
|
+
if shutil.which("gh"):
|
|
102
|
+
result = subprocess.run(["gh", "auth", "status"], capture_output=True, text=True)
|
|
103
|
+
if result.returncode == 0:
|
|
104
|
+
success_line("GitHub CLI (gh) found and authenticated")
|
|
105
|
+
else:
|
|
106
|
+
warning_line("GitHub CLI (gh) found but not logged in — run: gh auth login")
|
|
107
|
+
else:
|
|
108
|
+
warning_line("GitHub CLI (gh) not found — install from https://cli.github.com if using GitHub")
|
|
109
|
+
|
|
110
|
+
if shutil.which("az"):
|
|
111
|
+
result = subprocess.run(["az", "account", "show"], capture_output=True, text=True)
|
|
112
|
+
if result.returncode == 0:
|
|
113
|
+
success_line("Azure CLI (az) found and authenticated")
|
|
114
|
+
else:
|
|
115
|
+
warning_line("Azure CLI (az) found but not logged in — run: az login")
|
|
116
|
+
else:
|
|
117
|
+
warning_line("Azure CLI (az) not found — install from https://aka.ms/installazurecli if using Azure DevOps")
|
|
118
|
+
|
|
119
|
+
info_line("")
|
|
120
|
+
|
|
78
121
|
# --- Discover Snowflake connections ---
|
|
79
122
|
connections_file = Path.home() / ".snowflake" / "connections.toml"
|
|
80
123
|
connection_names = []
|
|
@@ -96,12 +139,10 @@ def _run_setup(project_dir: Path):
|
|
|
96
139
|
sys.exit(1)
|
|
97
140
|
|
|
98
141
|
# --- App database ---
|
|
99
|
-
app_database =
|
|
100
|
-
"Enter native app database name
|
|
142
|
+
app_database = get_input_string_or_default(
|
|
143
|
+
"Enter native app database name",
|
|
144
|
+
"DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
|
|
101
145
|
)
|
|
102
|
-
if not app_database:
|
|
103
|
-
error_line("App database name is required")
|
|
104
|
-
sys.exit(1)
|
|
105
146
|
|
|
106
147
|
# --- Write config ---
|
|
107
148
|
config_data = {
|
|
@@ -124,15 +165,15 @@ def _offer_setup(project_dir: Path):
|
|
|
124
165
|
_run_setup(project_dir)
|
|
125
166
|
|
|
126
167
|
|
|
127
|
-
def _connect_and_load(config: Config) -> SnowflakeService:
|
|
168
|
+
def _connect_and_load(config: Config, interactive: bool = True) -> SnowflakeService:
|
|
128
169
|
"""Load config, connect to Snowflake, and load native app settings.
|
|
129
170
|
|
|
130
171
|
Returns the connected SnowflakeService, or calls sys.exit on failure.
|
|
131
|
-
If .datasecops.yml is missing and
|
|
172
|
+
If .datasecops.yml is missing and interactive is True, offers to run setup.
|
|
132
173
|
"""
|
|
133
174
|
if not config.load():
|
|
134
175
|
# Check if the failure is due to missing .datasecops.yml
|
|
135
|
-
if not (config.project_dir / ".datasecops.yml").exists():
|
|
176
|
+
if not (config.project_dir / ".datasecops.yml").exists() and interactive:
|
|
136
177
|
_offer_setup(config.project_dir)
|
|
137
178
|
# Retry after setup
|
|
138
179
|
if not config.load():
|
|
@@ -157,16 +198,29 @@ def _connect_and_load(config: Config) -> SnowflakeService:
|
|
|
157
198
|
|
|
158
199
|
# If profile_name was not explicitly set and there are multiple profiles, ask the user
|
|
159
200
|
if not profile_was_set and len(config.all_profiles) > 1:
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
201
|
+
if interactive:
|
|
202
|
+
from datasecops_cli.utilities.display import select_from_list
|
|
203
|
+
names = [p.profile_name for p in config.all_profiles]
|
|
204
|
+
info_line(f"Found {len(names)} project profiles:")
|
|
205
|
+
selected = select_from_list(names, "profile", add_back=False)
|
|
206
|
+
config.profile = None
|
|
207
|
+
for p in config.all_profiles:
|
|
208
|
+
if p.profile_name == selected:
|
|
209
|
+
config.profile = p
|
|
210
|
+
config.profile_name = selected
|
|
211
|
+
break
|
|
212
|
+
# Save the selected profile back to .datasecops.yml
|
|
213
|
+
if config.profile:
|
|
214
|
+
config_data = {
|
|
215
|
+
"connection_name": config.datasecops.connection_name,
|
|
216
|
+
"app_database": config.datasecops.app_database,
|
|
217
|
+
"profile_name": config.profile_name,
|
|
218
|
+
}
|
|
219
|
+
write_datasecops_config(config.project_dir, config_data)
|
|
220
|
+
success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
|
|
221
|
+
else:
|
|
222
|
+
# Non-interactive: use the auto-selected first profile
|
|
223
|
+
info_line(f"Using default profile: {config.profile_name}")
|
|
170
224
|
|
|
171
225
|
# Re-resolve dbt_project_dir using framework project_dir setting
|
|
172
226
|
framework_dbt_dir = config.project_dir / config.project_settings.project_dir
|
|
@@ -193,6 +247,15 @@ def _run_interactive(config: Config):
|
|
|
193
247
|
warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
|
|
194
248
|
warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
|
|
195
249
|
|
|
250
|
+
# Check upstream project versions
|
|
251
|
+
from datasecops_cli.services.upstream_service import check_upstream_versions
|
|
252
|
+
upstream_statuses = []
|
|
253
|
+
if config.profile and config.profile.upstream_projects:
|
|
254
|
+
info_line("Checking upstream project versions...")
|
|
255
|
+
upstream_statuses = check_upstream_versions(
|
|
256
|
+
config.profile, config.all_profiles, config.dbt_project_dir
|
|
257
|
+
)
|
|
258
|
+
|
|
196
259
|
# Initialize services
|
|
197
260
|
dbt_runner = DbtRunner(
|
|
198
261
|
project_dir=config.dbt_project_dir,
|
|
@@ -211,14 +274,34 @@ def _run_interactive(config: Config):
|
|
|
211
274
|
|
|
212
275
|
# Main menu loop
|
|
213
276
|
_main_menu(config, dbt_runner, git_service, linting_service,
|
|
214
|
-
download_service, skill_service, sf_service
|
|
277
|
+
download_service, skill_service, sf_service,
|
|
278
|
+
upstream_statuses=upstream_statuses)
|
|
215
279
|
finally:
|
|
216
280
|
sf_service.close()
|
|
217
281
|
|
|
218
282
|
|
|
219
|
-
def _run_download(config: Config, items: list[str]
|
|
283
|
+
def _run_download(config: Config, items: list[str],
|
|
284
|
+
connection: str = None, app_database: str = None,
|
|
285
|
+
profile: str = None):
|
|
220
286
|
"""Run non-interactive downloads for CI/CD pipelines."""
|
|
221
|
-
|
|
287
|
+
# If CLI flags provided, write/override .datasecops.yml on the fly
|
|
288
|
+
if connection or app_database:
|
|
289
|
+
from datasecops_cli.utilities.yaml_utils import read_datasecops_config
|
|
290
|
+
existing = read_datasecops_config(config.project_dir) or {}
|
|
291
|
+
config_data = {
|
|
292
|
+
"connection_name": connection or existing.get("connection_name", ""),
|
|
293
|
+
"app_database": app_database or existing.get("app_database", ""),
|
|
294
|
+
"profile_name": profile or existing.get("profile_name", ""),
|
|
295
|
+
}
|
|
296
|
+
write_datasecops_config(config.project_dir, config_data)
|
|
297
|
+
info_line(f"Wrote .datasecops.yml (connection={config_data['connection_name']}, app_database={config_data['app_database']})")
|
|
298
|
+
elif profile:
|
|
299
|
+
from datasecops_cli.utilities.yaml_utils import read_datasecops_config
|
|
300
|
+
existing = read_datasecops_config(config.project_dir) or {}
|
|
301
|
+
existing["profile_name"] = profile
|
|
302
|
+
write_datasecops_config(config.project_dir, existing)
|
|
303
|
+
|
|
304
|
+
sf_service = _connect_and_load(config, interactive=False)
|
|
222
305
|
|
|
223
306
|
try:
|
|
224
307
|
download_service = DownloadService(sf_service, config.project_dir)
|
|
@@ -240,7 +323,7 @@ def _run_download(config: Config, items: list[str]):
|
|
|
240
323
|
elif item == "pipelines":
|
|
241
324
|
platform = config.source_control.source_control_platform.lower()
|
|
242
325
|
info_line(f"Platform: {platform}")
|
|
243
|
-
if not download_service.download_pipelines(platform=platform):
|
|
326
|
+
if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
|
|
244
327
|
failed = True
|
|
245
328
|
elif item == "packages":
|
|
246
329
|
if not download_service.download_dbt_packages(config.dbt_project_dir):
|
|
@@ -268,13 +351,44 @@ def _run_download(config: Config, items: list[str]):
|
|
|
268
351
|
sf_service.close()
|
|
269
352
|
|
|
270
353
|
|
|
354
|
+
def _update_upstream_packages(config: Config, dbt_runner: DbtRunner,
|
|
355
|
+
upstream_statuses: list):
|
|
356
|
+
"""Update packages.yml with latest upstream versions and optionally run deps."""
|
|
357
|
+
from datasecops_cli.services.upstream_service import update_packages_yml
|
|
358
|
+
from datasecops_cli.utilities.display import display_action_header, get_input_true_false, complete_action
|
|
359
|
+
|
|
360
|
+
display_action_header("Update Upstream Packages")
|
|
361
|
+
|
|
362
|
+
outdated = [s for s in upstream_statuses if s.needs_update and s.latest_tag != "unknown"]
|
|
363
|
+
if not outdated:
|
|
364
|
+
info_line("No packages to update")
|
|
365
|
+
complete_action()
|
|
366
|
+
return
|
|
367
|
+
|
|
368
|
+
for s in outdated:
|
|
369
|
+
info_line(f" {s.profile_name}: {s.current_version} → {s.latest_tag}")
|
|
370
|
+
|
|
371
|
+
info_line("")
|
|
372
|
+
count = update_packages_yml(config.dbt_project_dir, upstream_statuses)
|
|
373
|
+
if count:
|
|
374
|
+
success_line(f"Updated {count} package(s) in packages.yml")
|
|
375
|
+
if get_input_true_false("Run dbtf deps now?"):
|
|
376
|
+
dbt_runner.deps()
|
|
377
|
+
else:
|
|
378
|
+
info_line("No changes made to packages.yml")
|
|
379
|
+
|
|
380
|
+
complete_action()
|
|
381
|
+
|
|
382
|
+
|
|
271
383
|
def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
272
384
|
linting_service: LintingService, download_service: DownloadService,
|
|
273
|
-
skill_service: SkillService, sf_service: SnowflakeService
|
|
385
|
+
skill_service: SkillService, sf_service: SnowflakeService,
|
|
386
|
+
upstream_statuses: list = None):
|
|
274
387
|
"""Main menu loop."""
|
|
275
388
|
profile_name = config.profile_name
|
|
389
|
+
has_outdated = upstream_statuses and any(s.needs_update for s in upstream_statuses)
|
|
276
390
|
|
|
277
|
-
_show_main_menu(profile_name, git_service)
|
|
391
|
+
_show_main_menu(profile_name, git_service, upstream_statuses)
|
|
278
392
|
option = get_input_number("Choose an option: ")
|
|
279
393
|
|
|
280
394
|
while option != 0:
|
|
@@ -318,21 +432,34 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
|
|
|
318
432
|
profile=config.profile,
|
|
319
433
|
)
|
|
320
434
|
bootstrap.run(platform=platform, install_skills=install_skills, run_deps=run_deps)
|
|
435
|
+
|
|
436
|
+
elif option == 5 and has_outdated:
|
|
437
|
+
_update_upstream_packages(config, dbt_runner, upstream_statuses)
|
|
438
|
+
# Refresh upstream statuses after update
|
|
439
|
+
from datasecops_cli.services.upstream_service import check_upstream_versions
|
|
440
|
+
upstream_statuses = check_upstream_versions(
|
|
441
|
+
config.profile, config.all_profiles, config.dbt_project_dir
|
|
442
|
+
)
|
|
443
|
+
has_outdated = any(s.needs_update for s in upstream_statuses)
|
|
321
444
|
|
|
322
|
-
_show_main_menu(profile_name, git_service)
|
|
445
|
+
_show_main_menu(profile_name, git_service, upstream_statuses)
|
|
323
446
|
option = get_input_number("Choose an option: ")
|
|
324
447
|
|
|
325
448
|
info_line("Exiting DataSecOps Framework CLI")
|
|
326
449
|
|
|
327
450
|
|
|
328
|
-
def _show_main_menu(profile_name: str, git_service: GitService = None
|
|
451
|
+
def _show_main_menu(profile_name: str, git_service: GitService = None,
|
|
452
|
+
upstream_statuses: list = None):
|
|
329
453
|
clear()
|
|
330
454
|
branch = git_service.get_current_branch() if git_service else None
|
|
331
|
-
section_header("DataSecOps Framework CLI", profile_name, branch
|
|
455
|
+
section_header("DataSecOps Framework CLI", profile_name, branch,
|
|
456
|
+
upstream_info=upstream_statuses)
|
|
332
457
|
menu_option(1, "development - dbt Development Commands")
|
|
333
458
|
menu_option(2, "git - Source Control Operations")
|
|
334
459
|
menu_option(3, "downloads - Download Configs & Skills")
|
|
335
460
|
menu_option(4, "bootstrap - Set up a new dbt project with all framework config")
|
|
461
|
+
if upstream_statuses and any(s.needs_update for s in upstream_statuses):
|
|
462
|
+
menu_option(5, "update pkgs - Update upstream packages to latest versions")
|
|
336
463
|
menu_option(0, "exit - Exit")
|
|
337
464
|
|
|
338
465
|
|
|
@@ -38,7 +38,7 @@ class DownloadsMenu:
|
|
|
38
38
|
display_action_header("Download Pipeline Files")
|
|
39
39
|
platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
|
|
40
40
|
info_line(f"Platform: {platform}")
|
|
41
|
-
self.downloads.download_pipelines(platform=platform)
|
|
41
|
+
self.downloads.download_pipelines(platform=platform, profile_name=self.profile_name)
|
|
42
42
|
complete_action()
|
|
43
43
|
elif option == 3:
|
|
44
44
|
display_action_header("Download dbt Packages")
|
|
@@ -61,10 +61,12 @@ class ProjectProfile(BaseModel):
|
|
|
61
61
|
project_description: str = ""
|
|
62
62
|
model_types: list[str] = Field(default_factory=list)
|
|
63
63
|
downstream_projects: list[str] = Field(default_factory=list)
|
|
64
|
+
upstream_projects: list[str] = Field(default_factory=list)
|
|
64
65
|
git_url: str = ""
|
|
65
66
|
target_database: str = ""
|
|
66
67
|
dbt_version: str = "1.9"
|
|
67
68
|
dbt_packages: list[str] = Field(default_factory=list)
|
|
69
|
+
pipeline_ids: list[str] = Field(default_factory=list)
|
|
68
70
|
|
|
69
71
|
class DbtPackage(BaseModel):
|
|
70
72
|
name: str = ""
|
{datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/download_service.py
RENAMED
|
@@ -220,9 +220,13 @@ class DownloadService:
|
|
|
220
220
|
error_line("No active dbt versions found in framework configuration")
|
|
221
221
|
return dbt_packages
|
|
222
222
|
|
|
223
|
-
def download_pipelines(self, platform: str = "github") -> bool:
|
|
223
|
+
def download_pipelines(self, platform: str = "github", profile_name: str = "") -> bool:
|
|
224
224
|
info_line(f"Downloading {platform} pipeline configurations...")
|
|
225
|
-
|
|
225
|
+
if profile_name:
|
|
226
|
+
info_line(f"Filtering pipelines for profile: {profile_name}")
|
|
227
|
+
raw = self.sf.get_pipelines_for_profile(profile_name)
|
|
228
|
+
else:
|
|
229
|
+
raw = self.sf.get_framework_config("PIPELINES")
|
|
226
230
|
if not raw:
|
|
227
231
|
error_line("No pipeline configuration found in native app")
|
|
228
232
|
return False
|
{datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/snowflake_service.py
RENAMED
|
@@ -59,7 +59,10 @@ class SnowflakeService:
|
|
|
59
59
|
|
|
60
60
|
def get_project_profiles(self) -> Optional[list[dict]]:
|
|
61
61
|
return self.call_procedure("get_project_profiles")
|
|
62
|
-
|
|
62
|
+
|
|
63
|
+
def get_pipelines_for_profile(self, profile_name: str) -> Optional[dict]:
|
|
64
|
+
return self.call_procedure("get_pipelines_for_profile", profile_name)
|
|
65
|
+
|
|
63
66
|
def get_account_info(self) -> dict:
|
|
64
67
|
rows = self.execute_query("SELECT CURRENT_ACCOUNT_NAME() as account, CURRENT_USER() as user_name")
|
|
65
68
|
return rows[0] if rows else {}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Service for checking upstream project versions against packages.yml."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from datasecops_cli.models.project_config import ProjectProfile
|
|
10
|
+
from datasecops_cli.utilities.display import info_line, warning_line
|
|
11
|
+
from datasecops_cli.utilities.yaml_utils import read_yaml, write_yaml
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class UpstreamStatus:
|
|
16
|
+
"""Version status of an upstream project dependency."""
|
|
17
|
+
profile_name: str
|
|
18
|
+
git_url: str
|
|
19
|
+
latest_tag: str = ""
|
|
20
|
+
current_version: str = ""
|
|
21
|
+
needs_update: bool = False
|
|
22
|
+
error: str = ""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_latest_tag(git_url: str, timeout: int = 10) -> Optional[str]:
|
|
26
|
+
"""Fetch the latest tag from a remote git repo without cloning."""
|
|
27
|
+
try:
|
|
28
|
+
result = subprocess.run(
|
|
29
|
+
["git", "ls-remote", "--tags", "--sort=-v:refname", git_url],
|
|
30
|
+
capture_output=True, text=True, timeout=timeout,
|
|
31
|
+
)
|
|
32
|
+
if result.returncode != 0:
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
for line in result.stdout.strip().splitlines():
|
|
36
|
+
ref = line.split("\t")[-1]
|
|
37
|
+
# Skip ^{} dereferenced tags
|
|
38
|
+
if ref.endswith("^{}"):
|
|
39
|
+
continue
|
|
40
|
+
tag = ref.replace("refs/tags/", "")
|
|
41
|
+
# Skip pre-release tags
|
|
42
|
+
if any(pre in tag.lower() for pre in ("alpha", "beta", "rc", "dev")):
|
|
43
|
+
continue
|
|
44
|
+
return tag
|
|
45
|
+
return None
|
|
46
|
+
except (subprocess.TimeoutExpired, FileNotFoundError):
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def read_packages_yml(dbt_project_dir: Path) -> Optional[dict]:
|
|
51
|
+
"""Read packages.yml from the dbt project directory."""
|
|
52
|
+
return read_yaml(dbt_project_dir / "packages.yml")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def find_pinned_version(packages_data: dict, git_url: str) -> str:
|
|
56
|
+
"""Find the pinned revision/version for a git URL in packages.yml."""
|
|
57
|
+
if not packages_data:
|
|
58
|
+
return ""
|
|
59
|
+
for pkg in packages_data.get("packages", []):
|
|
60
|
+
pkg_url = pkg.get("git", "")
|
|
61
|
+
if pkg_url and _urls_match(pkg_url, git_url):
|
|
62
|
+
return pkg.get("revision", "")
|
|
63
|
+
return ""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _urls_match(url1: str, url2: str) -> bool:
|
|
67
|
+
"""Compare two git URLs, ignoring trailing .git and protocol differences."""
|
|
68
|
+
def normalize(url: str) -> str:
|
|
69
|
+
url = url.rstrip("/").removesuffix(".git").lower()
|
|
70
|
+
url = re.sub(r"^https?://", "", url)
|
|
71
|
+
url = re.sub(r"^git@([^:]+):", r"\1/", url)
|
|
72
|
+
return url
|
|
73
|
+
return normalize(url1) == normalize(url2)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def check_upstream_versions(
|
|
77
|
+
profile: ProjectProfile,
|
|
78
|
+
all_profiles: list[ProjectProfile],
|
|
79
|
+
dbt_project_dir: Path,
|
|
80
|
+
) -> list[UpstreamStatus]:
|
|
81
|
+
"""Check version status of all upstream project dependencies."""
|
|
82
|
+
if not profile.upstream_projects:
|
|
83
|
+
return []
|
|
84
|
+
|
|
85
|
+
# Build lookup of profiles by name
|
|
86
|
+
profile_map = {p.profile_name: p for p in all_profiles}
|
|
87
|
+
|
|
88
|
+
# Read current packages.yml
|
|
89
|
+
packages_data = read_packages_yml(dbt_project_dir)
|
|
90
|
+
|
|
91
|
+
results = []
|
|
92
|
+
for upstream_name in profile.upstream_projects:
|
|
93
|
+
upstream_profile = profile_map.get(upstream_name)
|
|
94
|
+
if not upstream_profile:
|
|
95
|
+
results.append(UpstreamStatus(
|
|
96
|
+
profile_name=upstream_name,
|
|
97
|
+
git_url="",
|
|
98
|
+
error=f"Profile '{upstream_name}' not found",
|
|
99
|
+
))
|
|
100
|
+
continue
|
|
101
|
+
|
|
102
|
+
git_url = upstream_profile.git_url
|
|
103
|
+
if not git_url:
|
|
104
|
+
results.append(UpstreamStatus(
|
|
105
|
+
profile_name=upstream_name,
|
|
106
|
+
git_url="",
|
|
107
|
+
error="No git_url configured",
|
|
108
|
+
))
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
latest_tag = get_latest_tag(git_url)
|
|
112
|
+
current_version = find_pinned_version(packages_data, git_url)
|
|
113
|
+
|
|
114
|
+
needs_update = False
|
|
115
|
+
if latest_tag and current_version:
|
|
116
|
+
needs_update = latest_tag.lstrip("v") != current_version.lstrip("v")
|
|
117
|
+
elif latest_tag and not current_version:
|
|
118
|
+
needs_update = True
|
|
119
|
+
|
|
120
|
+
results.append(UpstreamStatus(
|
|
121
|
+
profile_name=upstream_name,
|
|
122
|
+
git_url=git_url,
|
|
123
|
+
latest_tag=latest_tag or "unknown",
|
|
124
|
+
current_version=current_version or "not pinned",
|
|
125
|
+
needs_update=needs_update,
|
|
126
|
+
))
|
|
127
|
+
|
|
128
|
+
return results
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def update_packages_yml(
|
|
132
|
+
dbt_project_dir: Path,
|
|
133
|
+
upstream_statuses: list[UpstreamStatus],
|
|
134
|
+
) -> int:
|
|
135
|
+
"""Update packages.yml with latest versions for outdated upstream packages.
|
|
136
|
+
|
|
137
|
+
Returns the number of packages updated.
|
|
138
|
+
"""
|
|
139
|
+
packages_data = read_packages_yml(dbt_project_dir)
|
|
140
|
+
if not packages_data:
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
updated = 0
|
|
144
|
+
for status in upstream_statuses:
|
|
145
|
+
if not status.needs_update or status.latest_tag == "unknown":
|
|
146
|
+
continue
|
|
147
|
+
for pkg in packages_data.get("packages", []):
|
|
148
|
+
pkg_url = pkg.get("git", "")
|
|
149
|
+
if pkg_url and _urls_match(pkg_url, status.git_url):
|
|
150
|
+
pkg["revision"] = status.latest_tag
|
|
151
|
+
updated += 1
|
|
152
|
+
break
|
|
153
|
+
|
|
154
|
+
if updated:
|
|
155
|
+
dest = dbt_project_dir / "packages.yml"
|
|
156
|
+
write_yaml(dest, packages_data)
|
|
157
|
+
|
|
158
|
+
return updated
|
|
@@ -16,7 +16,8 @@ def print_long_line(color: str = Fore.GREEN) -> None:
|
|
|
16
16
|
print_detail("=" * 76, color)
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
def section_header(message: str, active_profile: str = None, current_branch: str = None
|
|
19
|
+
def section_header(message: str, active_profile: str = None, current_branch: str = None,
|
|
20
|
+
upstream_info: list = None) -> None:
|
|
20
21
|
print_long_line(Fore.GREEN)
|
|
21
22
|
print_detail(message, Fore.GREEN)
|
|
22
23
|
print_long_line(Fore.GREEN)
|
|
@@ -24,7 +25,18 @@ def section_header(message: str, active_profile: str = None, current_branch: str
|
|
|
24
25
|
print_detail(f" Profile: {active_profile}", Fore.GREEN)
|
|
25
26
|
if current_branch:
|
|
26
27
|
print_detail(f" Branch: {current_branch}", Fore.GREEN)
|
|
27
|
-
if
|
|
28
|
+
if upstream_info:
|
|
29
|
+
for status in upstream_info:
|
|
30
|
+
if status.error:
|
|
31
|
+
print_detail(f" Upstream: {status.profile_name} — {status.error}", Fore.YELLOW)
|
|
32
|
+
elif status.needs_update:
|
|
33
|
+
print_detail(
|
|
34
|
+
f" Upstream: {status.profile_name} ({status.current_version} → {status.latest_tag} available)",
|
|
35
|
+
Fore.YELLOW,
|
|
36
|
+
)
|
|
37
|
+
else:
|
|
38
|
+
print_detail(f" Upstream: {status.profile_name} ({status.current_version} ✓)", Fore.GREEN)
|
|
39
|
+
if active_profile or current_branch or upstream_info:
|
|
28
40
|
print_long_line(Fore.GREEN)
|
|
29
41
|
|
|
30
42
|
|
|
@@ -102,7 +102,7 @@ class TestRunDownload:
|
|
|
102
102
|
_run_download(config, ["pipelines"])
|
|
103
103
|
|
|
104
104
|
assert exc.value.code == 0
|
|
105
|
-
mock_ds.download_pipelines.assert_called_once_with(platform="github")
|
|
105
|
+
mock_ds.download_pipelines.assert_called_once_with(platform="github", profile_name="test_profile")
|
|
106
106
|
|
|
107
107
|
@patch("datasecops_cli.main._connect_and_load")
|
|
108
108
|
def test_download_packages(self, mock_connect, tmp_path):
|
|
@@ -219,7 +219,7 @@ class TestRunDownload:
|
|
|
219
219
|
_run_download(config, ["pipelines"])
|
|
220
220
|
|
|
221
221
|
assert exc.value.code == 0
|
|
222
|
-
mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
|
|
222
|
+
mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops", profile_name="test_profile")
|
|
223
223
|
|
|
224
224
|
@patch("datasecops_cli.main._connect_and_load")
|
|
225
225
|
def test_install_sqlfluff(self, mock_connect, tmp_path):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/bootstrap_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/linting_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|