datasecops-cli 0.3.0__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/CHANGELOG.md +16 -1
  2. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/PKG-INFO +1 -2
  3. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/pyproject.toml +1 -2
  4. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/main.py +156 -29
  5. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/downloads.py +1 -1
  6. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/project_config.py +2 -0
  7. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/download_service.py +6 -2
  8. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/snowflake_service.py +4 -1
  9. datasecops_cli-0.3.2/src/datasecops_cli/services/upstream_service.py +158 -0
  10. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/display.py +14 -2
  11. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_main.py +2 -2
  12. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/.github/workflows/publish-cli.yml +0 -0
  13. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/.gitignore +0 -0
  14. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/DEVELOPMENT.md +0 -0
  15. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/LICENSE +0 -0
  16. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/README.md +0 -0
  17. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/getting-started.md +0 -0
  18. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/legacy.md +0 -0
  19. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/legacy_plan_of_action.md +0 -0
  20. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/docs/mcp-server.md +0 -0
  21. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/mcp-servers.json +0 -0
  22. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/setup.ps1 +0 -0
  23. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/setup.sh +0 -0
  24. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/__init__.py +0 -0
  25. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/config.py +0 -0
  26. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/__init__.py +0 -0
  27. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/development.py +0 -0
  28. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/menus/git_operations.py +0 -0
  29. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/__init__.py +0 -0
  30. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/models/git_helpers.py +0 -0
  31. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/__init__.py +0 -0
  32. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/bootstrap_service.py +0 -0
  33. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/dbt_runner.py +0 -0
  34. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/git_service.py +0 -0
  35. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/linting_service.py +0 -0
  36. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/services/skill_service.py +0 -0
  37. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/__init__.py +0 -0
  38. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/file_utils.py +0 -0
  39. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/yaml_utils.py +0 -0
  40. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/__init__.py +0 -0
  41. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/__main__.py +0 -0
  42. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/connection.py +0 -0
  43. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/src/datasecops_mcp/server.py +0 -0
  44. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/__init__.py +0 -0
  45. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_config.py +0 -0
  46. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_file_utils.py +0 -0
  47. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_models.py +0 -0
  48. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_version.py +0 -0
  49. {datasecops_cli-0.3.0 → datasecops_cli-0.3.2}/tests/test_yaml_utils.py +0 -0
@@ -2,15 +2,30 @@
2
2
 
3
3
  All notable changes to the DataSecOps CLI are documented in this file.
4
4
 
5
+ ## [0.3.2] - 2026-05-15
6
+
7
+ ### Added
8
+
9
+ - **Upstream project version tracking** — if a profile has `upstream_projects` configured in the native app, the main menu header shows each upstream project's current pinned version vs the latest available tag (fetched via `git ls-remote`). Outdated dependencies are highlighted in yellow.
10
+ - **Auto-update upstream packages** — when outdated upstream packages are detected, a new menu option `[5] update pkgs` appears, which updates `packages.yml` revisions to the latest tags and optionally runs `dbtf deps`
11
+ - **`upstream_projects` field on `ProjectProfile`** — new model field to support upstream dependency tracking from the native app
12
+
5
13
  ## [0.3.0] - 2026-05-14
6
14
 
7
15
  ### Added
8
16
 
9
17
  - **`datasecops setup` subcommand** — pure-Python replacement for `setup.ps1` / `setup.sh`. Discovers Snowflake connections from `~/.snowflake/connections.toml`, prompts for connection and app database, and writes `.datasecops.yml`. No platform-specific scripts needed.
10
- - **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one
18
+ - **Prerequisite checks during setup** — checks for `dbtf`, `gh`, and `az` on PATH and verifies authentication status (`gh auth status`, `az account show`)
19
+ - **Interactive profile selection** — when `profile_name` is not set (no `dbt_project.yml` and no profile in `.datasecops.yml`), the CLI prompts the user to choose from available profiles in the native app instead of silently picking the first one. Selected profile is saved back to `.datasecops.yml`.
11
20
  - **Auto-setup on first run** — running `datasecops` without a `.datasecops.yml` now offers to run setup interactively instead of just exiting with an error
21
+ - **`--connection`, `--app-database`, `--profile` flags on `download`** — allows CI/CD pipelines to run without a committed `.datasecops.yml` by passing connection details as CLI flags (e.g. `datasecops download sqlfluff -c ci -d MY_APP_DB -p my_project`)
22
+ - **Default app database value** — setup now defaults to `DATA_ENGINEERS_DATASECOPS_FRAMEWORK` (press Enter to accept)
12
23
  - **dbt project directory resolution from framework** — `dbt_project_dir` is now re-resolved using the framework's `project_dir` setting after native app config is loaded
13
24
 
25
+ ### Changed
26
+
27
+ - **Non-interactive mode safety** — `_connect_and_load` accepts an `interactive` flag; the `download` subcommand passes `interactive=False` so it never prompts (missing config exits 1, multiple profiles use the first one silently)
28
+
14
29
  ## [0.2.9] - 2026-05-14
15
30
 
16
31
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datasecops-cli
3
- Version: 0.3.0
3
+ Version: 0.3.2
4
4
  Summary: DataSecOps Framework CLI for Snowflake Native App
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -10,7 +10,6 @@ Requires-Dist: gitpython>=3.1
10
10
  Requires-Dist: pydantic>=2.0
11
11
  Requires-Dist: pyyaml>=6.0
12
12
  Requires-Dist: snowflake-connector-python>=3.0
13
- Requires-Dist: sqlfluff>=3.0
14
13
  Provides-Extra: mcp
15
14
  Requires-Dist: mcp>=1.0; extra == 'mcp'
16
15
  Provides-Extra: test
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "datasecops-cli"
7
- version = "0.3.0"
7
+ version = "0.3.2"
8
8
  description = "DataSecOps Framework CLI for Snowflake Native App"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -14,7 +14,6 @@ dependencies = [
14
14
  "snowflake-connector-python>=3.0",
15
15
  "colorama>=0.4",
16
16
  "pyyaml>=6.0",
17
- "sqlfluff>=3.0",
18
17
  "pydantic>=2.0",
19
18
  ]
20
19
 
@@ -46,6 +46,18 @@ def _build_parser() -> argparse.ArgumentParser:
46
46
  choices=DOWNLOAD_ITEMS,
47
47
  help="Item(s) to download/install: sqlfluff, pipelines, packages, macros, install-sqlfluff, install-dbt, or all",
48
48
  )
49
+ dl.add_argument(
50
+ "--connection", "-c",
51
+ help="Snowflake connection name (overrides .datasecops.yml)",
52
+ )
53
+ dl.add_argument(
54
+ "--app-database", "-d",
55
+ help="Native app database name (overrides .datasecops.yml)",
56
+ )
57
+ dl.add_argument(
58
+ "--profile", "-p",
59
+ help="Project profile name (overrides .datasecops.yml)",
60
+ )
49
61
 
50
62
  return parser
51
63
 
@@ -62,7 +74,9 @@ def main():
62
74
  elif args.command == "bootstrap":
63
75
  _run_bootstrap(config)
64
76
  elif args.command == "download":
65
- _run_download(config, args.items)
77
+ _run_download(config, args.items,
78
+ connection=args.connection, app_database=args.app_database,
79
+ profile=args.profile)
66
80
  else:
67
81
  _run_interactive(config)
68
82
 
@@ -70,11 +84,40 @@ def main():
70
84
  def _run_setup(project_dir: Path):
71
85
  """Interactive setup: create .datasecops.yml by prompting for connection and app database."""
72
86
  from datasecops_cli.utilities.display import (
73
- get_input_string, select_from_list
87
+ get_input_string, get_input_string_or_default, select_from_list, warning_line
74
88
  )
75
89
 
76
90
  section_header("DataSecOps Setup")
77
91
 
92
+ # --- Check prerequisites ---
93
+ info_line("Checking prerequisites...")
94
+
95
+ if shutil.which("dbtf"):
96
+ success_line("dbt Fusion (dbtf) found")
97
+ else:
98
+ warning_line("dbt Fusion (dbtf) not found — dbt commands will not work")
99
+ warning_line(" Install: https://docs.getdbt.com/docs/core/installation")
100
+
101
+ if shutil.which("gh"):
102
+ result = subprocess.run(["gh", "auth", "status"], capture_output=True, text=True)
103
+ if result.returncode == 0:
104
+ success_line("GitHub CLI (gh) found and authenticated")
105
+ else:
106
+ warning_line("GitHub CLI (gh) found but not logged in — run: gh auth login")
107
+ else:
108
+ warning_line("GitHub CLI (gh) not found — install from https://cli.github.com if using GitHub")
109
+
110
+ if shutil.which("az"):
111
+ result = subprocess.run(["az", "account", "show"], capture_output=True, text=True)
112
+ if result.returncode == 0:
113
+ success_line("Azure CLI (az) found and authenticated")
114
+ else:
115
+ warning_line("Azure CLI (az) found but not logged in — run: az login")
116
+ else:
117
+ warning_line("Azure CLI (az) not found — install from https://aka.ms/installazurecli if using Azure DevOps")
118
+
119
+ info_line("")
120
+
78
121
  # --- Discover Snowflake connections ---
79
122
  connections_file = Path.home() / ".snowflake" / "connections.toml"
80
123
  connection_names = []
@@ -96,12 +139,10 @@ def _run_setup(project_dir: Path):
96
139
  sys.exit(1)
97
140
 
98
141
  # --- App database ---
99
- app_database = get_input_string(
100
- "Enter native app database name (e.g. DATA_ENGINEERS_DATASECOPS_FRAMEWORK): "
142
+ app_database = get_input_string_or_default(
143
+ "Enter native app database name",
144
+ "DATA_ENGINEERS_DATASECOPS_FRAMEWORK"
101
145
  )
102
- if not app_database:
103
- error_line("App database name is required")
104
- sys.exit(1)
105
146
 
106
147
  # --- Write config ---
107
148
  config_data = {
@@ -124,15 +165,15 @@ def _offer_setup(project_dir: Path):
124
165
  _run_setup(project_dir)
125
166
 
126
167
 
127
- def _connect_and_load(config: Config) -> SnowflakeService:
168
+ def _connect_and_load(config: Config, interactive: bool = True) -> SnowflakeService:
128
169
  """Load config, connect to Snowflake, and load native app settings.
129
170
 
130
171
  Returns the connected SnowflakeService, or calls sys.exit on failure.
131
- If .datasecops.yml is missing and a setup script exists, offers to run it.
172
+ If .datasecops.yml is missing and interactive is True, offers to run setup.
132
173
  """
133
174
  if not config.load():
134
175
  # Check if the failure is due to missing .datasecops.yml
135
- if not (config.project_dir / ".datasecops.yml").exists():
176
+ if not (config.project_dir / ".datasecops.yml").exists() and interactive:
136
177
  _offer_setup(config.project_dir)
137
178
  # Retry after setup
138
179
  if not config.load():
@@ -157,16 +198,29 @@ def _connect_and_load(config: Config) -> SnowflakeService:
157
198
 
158
199
  # If profile_name was not explicitly set and there are multiple profiles, ask the user
159
200
  if not profile_was_set and len(config.all_profiles) > 1:
160
- from datasecops_cli.utilities.display import select_from_list
161
- names = [p.profile_name for p in config.all_profiles]
162
- info_line(f"Found {len(names)} project profiles:")
163
- selected = select_from_list(names, "profile", add_back=False)
164
- config.profile = None
165
- for p in config.all_profiles:
166
- if p.profile_name == selected:
167
- config.profile = p
168
- config.profile_name = selected
169
- break
201
+ if interactive:
202
+ from datasecops_cli.utilities.display import select_from_list
203
+ names = [p.profile_name for p in config.all_profiles]
204
+ info_line(f"Found {len(names)} project profiles:")
205
+ selected = select_from_list(names, "profile", add_back=False)
206
+ config.profile = None
207
+ for p in config.all_profiles:
208
+ if p.profile_name == selected:
209
+ config.profile = p
210
+ config.profile_name = selected
211
+ break
212
+ # Save the selected profile back to .datasecops.yml
213
+ if config.profile:
214
+ config_data = {
215
+ "connection_name": config.datasecops.connection_name,
216
+ "app_database": config.datasecops.app_database,
217
+ "profile_name": config.profile_name,
218
+ }
219
+ write_datasecops_config(config.project_dir, config_data)
220
+ success_line(f"Saved profile_name '{config.profile_name}' to .datasecops.yml")
221
+ else:
222
+ # Non-interactive: use the auto-selected first profile
223
+ info_line(f"Using default profile: {config.profile_name}")
170
224
 
171
225
  # Re-resolve dbt_project_dir using framework project_dir setting
172
226
  framework_dbt_dir = config.project_dir / config.project_settings.project_dir
@@ -193,6 +247,15 @@ def _run_interactive(config: Config):
193
247
  warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
194
248
  warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
195
249
 
250
+ # Check upstream project versions
251
+ from datasecops_cli.services.upstream_service import check_upstream_versions
252
+ upstream_statuses = []
253
+ if config.profile and config.profile.upstream_projects:
254
+ info_line("Checking upstream project versions...")
255
+ upstream_statuses = check_upstream_versions(
256
+ config.profile, config.all_profiles, config.dbt_project_dir
257
+ )
258
+
196
259
  # Initialize services
197
260
  dbt_runner = DbtRunner(
198
261
  project_dir=config.dbt_project_dir,
@@ -211,14 +274,34 @@ def _run_interactive(config: Config):
211
274
 
212
275
  # Main menu loop
213
276
  _main_menu(config, dbt_runner, git_service, linting_service,
214
- download_service, skill_service, sf_service)
277
+ download_service, skill_service, sf_service,
278
+ upstream_statuses=upstream_statuses)
215
279
  finally:
216
280
  sf_service.close()
217
281
 
218
282
 
219
- def _run_download(config: Config, items: list[str]):
283
+ def _run_download(config: Config, items: list[str],
284
+ connection: str = None, app_database: str = None,
285
+ profile: str = None):
220
286
  """Run non-interactive downloads for CI/CD pipelines."""
221
- sf_service = _connect_and_load(config)
287
+ # If CLI flags provided, write/override .datasecops.yml on the fly
288
+ if connection or app_database:
289
+ from datasecops_cli.utilities.yaml_utils import read_datasecops_config
290
+ existing = read_datasecops_config(config.project_dir) or {}
291
+ config_data = {
292
+ "connection_name": connection or existing.get("connection_name", ""),
293
+ "app_database": app_database or existing.get("app_database", ""),
294
+ "profile_name": profile or existing.get("profile_name", ""),
295
+ }
296
+ write_datasecops_config(config.project_dir, config_data)
297
+ info_line(f"Wrote .datasecops.yml (connection={config_data['connection_name']}, app_database={config_data['app_database']})")
298
+ elif profile:
299
+ from datasecops_cli.utilities.yaml_utils import read_datasecops_config
300
+ existing = read_datasecops_config(config.project_dir) or {}
301
+ existing["profile_name"] = profile
302
+ write_datasecops_config(config.project_dir, existing)
303
+
304
+ sf_service = _connect_and_load(config, interactive=False)
222
305
 
223
306
  try:
224
307
  download_service = DownloadService(sf_service, config.project_dir)
@@ -240,7 +323,7 @@ def _run_download(config: Config, items: list[str]):
240
323
  elif item == "pipelines":
241
324
  platform = config.source_control.source_control_platform.lower()
242
325
  info_line(f"Platform: {platform}")
243
- if not download_service.download_pipelines(platform=platform):
326
+ if not download_service.download_pipelines(platform=platform, profile_name=config.profile_name):
244
327
  failed = True
245
328
  elif item == "packages":
246
329
  if not download_service.download_dbt_packages(config.dbt_project_dir):
@@ -268,13 +351,44 @@ def _run_download(config: Config, items: list[str]):
268
351
  sf_service.close()
269
352
 
270
353
 
354
+ def _update_upstream_packages(config: Config, dbt_runner: DbtRunner,
355
+ upstream_statuses: list):
356
+ """Update packages.yml with latest upstream versions and optionally run deps."""
357
+ from datasecops_cli.services.upstream_service import update_packages_yml
358
+ from datasecops_cli.utilities.display import display_action_header, get_input_true_false, complete_action
359
+
360
+ display_action_header("Update Upstream Packages")
361
+
362
+ outdated = [s for s in upstream_statuses if s.needs_update and s.latest_tag != "unknown"]
363
+ if not outdated:
364
+ info_line("No packages to update")
365
+ complete_action()
366
+ return
367
+
368
+ for s in outdated:
369
+ info_line(f" {s.profile_name}: {s.current_version} → {s.latest_tag}")
370
+
371
+ info_line("")
372
+ count = update_packages_yml(config.dbt_project_dir, upstream_statuses)
373
+ if count:
374
+ success_line(f"Updated {count} package(s) in packages.yml")
375
+ if get_input_true_false("Run dbtf deps now?"):
376
+ dbt_runner.deps()
377
+ else:
378
+ info_line("No changes made to packages.yml")
379
+
380
+ complete_action()
381
+
382
+
271
383
  def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
272
384
  linting_service: LintingService, download_service: DownloadService,
273
- skill_service: SkillService, sf_service: SnowflakeService):
385
+ skill_service: SkillService, sf_service: SnowflakeService,
386
+ upstream_statuses: list = None):
274
387
  """Main menu loop."""
275
388
  profile_name = config.profile_name
389
+ has_outdated = upstream_statuses and any(s.needs_update for s in upstream_statuses)
276
390
 
277
- _show_main_menu(profile_name, git_service)
391
+ _show_main_menu(profile_name, git_service, upstream_statuses)
278
392
  option = get_input_number("Choose an option: ")
279
393
 
280
394
  while option != 0:
@@ -318,21 +432,34 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
318
432
  profile=config.profile,
319
433
  )
320
434
  bootstrap.run(platform=platform, install_skills=install_skills, run_deps=run_deps)
435
+
436
+ elif option == 5 and has_outdated:
437
+ _update_upstream_packages(config, dbt_runner, upstream_statuses)
438
+ # Refresh upstream statuses after update
439
+ from datasecops_cli.services.upstream_service import check_upstream_versions
440
+ upstream_statuses = check_upstream_versions(
441
+ config.profile, config.all_profiles, config.dbt_project_dir
442
+ )
443
+ has_outdated = any(s.needs_update for s in upstream_statuses)
321
444
 
322
- _show_main_menu(profile_name, git_service)
445
+ _show_main_menu(profile_name, git_service, upstream_statuses)
323
446
  option = get_input_number("Choose an option: ")
324
447
 
325
448
  info_line("Exiting DataSecOps Framework CLI")
326
449
 
327
450
 
328
- def _show_main_menu(profile_name: str, git_service: GitService = None):
451
+ def _show_main_menu(profile_name: str, git_service: GitService = None,
452
+ upstream_statuses: list = None):
329
453
  clear()
330
454
  branch = git_service.get_current_branch() if git_service else None
331
- section_header("DataSecOps Framework CLI", profile_name, branch)
455
+ section_header("DataSecOps Framework CLI", profile_name, branch,
456
+ upstream_info=upstream_statuses)
332
457
  menu_option(1, "development - dbt Development Commands")
333
458
  menu_option(2, "git - Source Control Operations")
334
459
  menu_option(3, "downloads - Download Configs & Skills")
335
460
  menu_option(4, "bootstrap - Set up a new dbt project with all framework config")
461
+ if upstream_statuses and any(s.needs_update for s in upstream_statuses):
462
+ menu_option(5, "update pkgs - Update upstream packages to latest versions")
336
463
  menu_option(0, "exit - Exit")
337
464
 
338
465
 
@@ -38,7 +38,7 @@ class DownloadsMenu:
38
38
  display_action_header("Download Pipeline Files")
39
39
  platform = self.source_control.source_control_platform.lower() if self.source_control else "github"
40
40
  info_line(f"Platform: {platform}")
41
- self.downloads.download_pipelines(platform=platform)
41
+ self.downloads.download_pipelines(platform=platform, profile_name=self.profile_name)
42
42
  complete_action()
43
43
  elif option == 3:
44
44
  display_action_header("Download dbt Packages")
@@ -61,10 +61,12 @@ class ProjectProfile(BaseModel):
61
61
  project_description: str = ""
62
62
  model_types: list[str] = Field(default_factory=list)
63
63
  downstream_projects: list[str] = Field(default_factory=list)
64
+ upstream_projects: list[str] = Field(default_factory=list)
64
65
  git_url: str = ""
65
66
  target_database: str = ""
66
67
  dbt_version: str = "1.9"
67
68
  dbt_packages: list[str] = Field(default_factory=list)
69
+ pipeline_ids: list[str] = Field(default_factory=list)
68
70
 
69
71
  class DbtPackage(BaseModel):
70
72
  name: str = ""
@@ -220,9 +220,13 @@ class DownloadService:
220
220
  error_line("No active dbt versions found in framework configuration")
221
221
  return dbt_packages
222
222
 
223
- def download_pipelines(self, platform: str = "github") -> bool:
223
+ def download_pipelines(self, platform: str = "github", profile_name: str = "") -> bool:
224
224
  info_line(f"Downloading {platform} pipeline configurations...")
225
- raw = self.sf.get_framework_config("PIPELINES")
225
+ if profile_name:
226
+ info_line(f"Filtering pipelines for profile: {profile_name}")
227
+ raw = self.sf.get_pipelines_for_profile(profile_name)
228
+ else:
229
+ raw = self.sf.get_framework_config("PIPELINES")
226
230
  if not raw:
227
231
  error_line("No pipeline configuration found in native app")
228
232
  return False
@@ -59,7 +59,10 @@ class SnowflakeService:
59
59
 
60
60
  def get_project_profiles(self) -> Optional[list[dict]]:
61
61
  return self.call_procedure("get_project_profiles")
62
-
62
+
63
+ def get_pipelines_for_profile(self, profile_name: str) -> Optional[dict]:
64
+ return self.call_procedure("get_pipelines_for_profile", profile_name)
65
+
63
66
  def get_account_info(self) -> dict:
64
67
  rows = self.execute_query("SELECT CURRENT_ACCOUNT_NAME() as account, CURRENT_USER() as user_name")
65
68
  return rows[0] if rows else {}
@@ -0,0 +1,158 @@
1
+ """Service for checking upstream project versions against packages.yml."""
2
+
3
+ import re
4
+ import subprocess
5
+ from dataclasses import dataclass, field
6
+ from pathlib import Path
7
+ from typing import Optional
8
+
9
+ from datasecops_cli.models.project_config import ProjectProfile
10
+ from datasecops_cli.utilities.display import info_line, warning_line
11
+ from datasecops_cli.utilities.yaml_utils import read_yaml, write_yaml
12
+
13
+
14
+ @dataclass
15
+ class UpstreamStatus:
16
+ """Version status of an upstream project dependency."""
17
+ profile_name: str
18
+ git_url: str
19
+ latest_tag: str = ""
20
+ current_version: str = ""
21
+ needs_update: bool = False
22
+ error: str = ""
23
+
24
+
25
+ def get_latest_tag(git_url: str, timeout: int = 10) -> Optional[str]:
26
+ """Fetch the latest tag from a remote git repo without cloning."""
27
+ try:
28
+ result = subprocess.run(
29
+ ["git", "ls-remote", "--tags", "--sort=-v:refname", git_url],
30
+ capture_output=True, text=True, timeout=timeout,
31
+ )
32
+ if result.returncode != 0:
33
+ return None
34
+
35
+ for line in result.stdout.strip().splitlines():
36
+ ref = line.split("\t")[-1]
37
+ # Skip ^{} dereferenced tags
38
+ if ref.endswith("^{}"):
39
+ continue
40
+ tag = ref.replace("refs/tags/", "")
41
+ # Skip pre-release tags
42
+ if any(pre in tag.lower() for pre in ("alpha", "beta", "rc", "dev")):
43
+ continue
44
+ return tag
45
+ return None
46
+ except (subprocess.TimeoutExpired, FileNotFoundError):
47
+ return None
48
+
49
+
50
+ def read_packages_yml(dbt_project_dir: Path) -> Optional[dict]:
51
+ """Read packages.yml from the dbt project directory."""
52
+ return read_yaml(dbt_project_dir / "packages.yml")
53
+
54
+
55
+ def find_pinned_version(packages_data: dict, git_url: str) -> str:
56
+ """Find the pinned revision/version for a git URL in packages.yml."""
57
+ if not packages_data:
58
+ return ""
59
+ for pkg in packages_data.get("packages", []):
60
+ pkg_url = pkg.get("git", "")
61
+ if pkg_url and _urls_match(pkg_url, git_url):
62
+ return pkg.get("revision", "")
63
+ return ""
64
+
65
+
66
+ def _urls_match(url1: str, url2: str) -> bool:
67
+ """Compare two git URLs, ignoring trailing .git and protocol differences."""
68
+ def normalize(url: str) -> str:
69
+ url = url.rstrip("/").removesuffix(".git").lower()
70
+ url = re.sub(r"^https?://", "", url)
71
+ url = re.sub(r"^git@([^:]+):", r"\1/", url)
72
+ return url
73
+ return normalize(url1) == normalize(url2)
74
+
75
+
76
+ def check_upstream_versions(
77
+ profile: ProjectProfile,
78
+ all_profiles: list[ProjectProfile],
79
+ dbt_project_dir: Path,
80
+ ) -> list[UpstreamStatus]:
81
+ """Check version status of all upstream project dependencies."""
82
+ if not profile.upstream_projects:
83
+ return []
84
+
85
+ # Build lookup of profiles by name
86
+ profile_map = {p.profile_name: p for p in all_profiles}
87
+
88
+ # Read current packages.yml
89
+ packages_data = read_packages_yml(dbt_project_dir)
90
+
91
+ results = []
92
+ for upstream_name in profile.upstream_projects:
93
+ upstream_profile = profile_map.get(upstream_name)
94
+ if not upstream_profile:
95
+ results.append(UpstreamStatus(
96
+ profile_name=upstream_name,
97
+ git_url="",
98
+ error=f"Profile '{upstream_name}' not found",
99
+ ))
100
+ continue
101
+
102
+ git_url = upstream_profile.git_url
103
+ if not git_url:
104
+ results.append(UpstreamStatus(
105
+ profile_name=upstream_name,
106
+ git_url="",
107
+ error="No git_url configured",
108
+ ))
109
+ continue
110
+
111
+ latest_tag = get_latest_tag(git_url)
112
+ current_version = find_pinned_version(packages_data, git_url)
113
+
114
+ needs_update = False
115
+ if latest_tag and current_version:
116
+ needs_update = latest_tag.lstrip("v") != current_version.lstrip("v")
117
+ elif latest_tag and not current_version:
118
+ needs_update = True
119
+
120
+ results.append(UpstreamStatus(
121
+ profile_name=upstream_name,
122
+ git_url=git_url,
123
+ latest_tag=latest_tag or "unknown",
124
+ current_version=current_version or "not pinned",
125
+ needs_update=needs_update,
126
+ ))
127
+
128
+ return results
129
+
130
+
131
+ def update_packages_yml(
132
+ dbt_project_dir: Path,
133
+ upstream_statuses: list[UpstreamStatus],
134
+ ) -> int:
135
+ """Update packages.yml with latest versions for outdated upstream packages.
136
+
137
+ Returns the number of packages updated.
138
+ """
139
+ packages_data = read_packages_yml(dbt_project_dir)
140
+ if not packages_data:
141
+ return 0
142
+
143
+ updated = 0
144
+ for status in upstream_statuses:
145
+ if not status.needs_update or status.latest_tag == "unknown":
146
+ continue
147
+ for pkg in packages_data.get("packages", []):
148
+ pkg_url = pkg.get("git", "")
149
+ if pkg_url and _urls_match(pkg_url, status.git_url):
150
+ pkg["revision"] = status.latest_tag
151
+ updated += 1
152
+ break
153
+
154
+ if updated:
155
+ dest = dbt_project_dir / "packages.yml"
156
+ write_yaml(dest, packages_data)
157
+
158
+ return updated
@@ -16,7 +16,8 @@ def print_long_line(color: str = Fore.GREEN) -> None:
16
16
  print_detail("=" * 76, color)
17
17
 
18
18
 
19
- def section_header(message: str, active_profile: str = None, current_branch: str = None) -> None:
19
+ def section_header(message: str, active_profile: str = None, current_branch: str = None,
20
+ upstream_info: list = None) -> None:
20
21
  print_long_line(Fore.GREEN)
21
22
  print_detail(message, Fore.GREEN)
22
23
  print_long_line(Fore.GREEN)
@@ -24,7 +25,18 @@ def section_header(message: str, active_profile: str = None, current_branch: str
24
25
  print_detail(f" Profile: {active_profile}", Fore.GREEN)
25
26
  if current_branch:
26
27
  print_detail(f" Branch: {current_branch}", Fore.GREEN)
27
- if active_profile or current_branch:
28
+ if upstream_info:
29
+ for status in upstream_info:
30
+ if status.error:
31
+ print_detail(f" Upstream: {status.profile_name} — {status.error}", Fore.YELLOW)
32
+ elif status.needs_update:
33
+ print_detail(
34
+ f" Upstream: {status.profile_name} ({status.current_version} → {status.latest_tag} available)",
35
+ Fore.YELLOW,
36
+ )
37
+ else:
38
+ print_detail(f" Upstream: {status.profile_name} ({status.current_version} ✓)", Fore.GREEN)
39
+ if active_profile or current_branch or upstream_info:
28
40
  print_long_line(Fore.GREEN)
29
41
 
30
42
 
@@ -102,7 +102,7 @@ class TestRunDownload:
102
102
  _run_download(config, ["pipelines"])
103
103
 
104
104
  assert exc.value.code == 0
105
- mock_ds.download_pipelines.assert_called_once_with(platform="github")
105
+ mock_ds.download_pipelines.assert_called_once_with(platform="github", profile_name="test_profile")
106
106
 
107
107
  @patch("datasecops_cli.main._connect_and_load")
108
108
  def test_download_packages(self, mock_connect, tmp_path):
@@ -219,7 +219,7 @@ class TestRunDownload:
219
219
  _run_download(config, ["pipelines"])
220
220
 
221
221
  assert exc.value.code == 0
222
- mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops")
222
+ mock_ds.download_pipelines.assert_called_once_with(platform="azuredevops", profile_name="test_profile")
223
223
 
224
224
  @patch("datasecops_cli.main._connect_and_load")
225
225
  def test_install_sqlfluff(self, mock_connect, tmp_path):
File without changes
File without changes
File without changes
File without changes