datasecops-cli 0.3.1__tar.gz → 0.3.2__tar.gz

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to the DataSecOps CLI are documented in this file.
 
+## [0.3.2] - 2026-05-15
+
+### Added
+
+- **Upstream project version tracking** — if a profile has `upstream_projects` configured in the native app, the main menu header shows each upstream project's current pinned version vs the latest available tag (fetched via `git ls-remote`). Outdated dependencies are highlighted in yellow.
+- **Auto-update upstream packages** — when outdated upstream packages are detected, a new menu option `[5] update pkgs` appears, which updates `packages.yml` revisions to the latest tags and optionally runs `dbtf deps`
+- **`upstream_projects` field on `ProjectProfile`** — new model field to support upstream dependency tracking from the native app
+
 ## [0.3.0] - 2026-05-14
 
 ### Added

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasecops-cli
-Version: 0.3.1
+Version: 0.3.2
 Summary: DataSecOps Framework CLI for Snowflake Native App
 License-Expression: MIT
 License-File: LICENSE
@@ -10,7 +10,6 @@ Requires-Dist: gitpython>=3.1
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: snowflake-connector-python>=3.0
-Requires-Dist: sqlfluff>=3.0
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.0; extra == 'mcp'
 Provides-Extra: test

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasecops-cli"
-version = "0.3.1"
+version = "0.3.2"
 description = "DataSecOps Framework CLI for Snowflake Native App"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -14,7 +14,6 @@ dependencies = [
     "snowflake-connector-python>=3.0",
     "colorama>=0.4",
     "pyyaml>=6.0",
-    "sqlfluff>=3.0",
     "pydantic>=2.0",
 ]
 

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/src/datasecops_cli/main.py

@@ -247,6 +247,15 @@ def _run_interactive(config: Config):
             warning_line("dbt Fusion (dbtf) not found on PATH — dbt commands will not work.")
             warning_line("Install dbt Fusion: https://docs.getdbt.com/docs/core/installation")
 
+        # Check upstream project versions
+        from datasecops_cli.services.upstream_service import check_upstream_versions
+        upstream_statuses = []
+        if config.profile and config.profile.upstream_projects:
+            info_line("Checking upstream project versions...")
+            upstream_statuses = check_upstream_versions(
+                config.profile, config.all_profiles, config.dbt_project_dir
+            )
+
         # Initialize services
         dbt_runner = DbtRunner(
             project_dir=config.dbt_project_dir,
@@ -265,7 +274,8 @@ def _run_interactive(config: Config):
 
         # Main menu loop
         _main_menu(config, dbt_runner, git_service, linting_service,
-                    download_service, skill_service, sf_service)
+                    download_service, skill_service, sf_service,
+                    upstream_statuses=upstream_statuses)
     finally:
         sf_service.close()
 
@@ -341,13 +351,44 @@ def _run_download(config: Config, items: list[str],
         sf_service.close()
 
 
+def _update_upstream_packages(config: Config, dbt_runner: DbtRunner,
+                               upstream_statuses: list):
+    """Update packages.yml with latest upstream versions and optionally run deps."""
+    from datasecops_cli.services.upstream_service import update_packages_yml
+    from datasecops_cli.utilities.display import display_action_header, get_input_true_false, complete_action
+
+    display_action_header("Update Upstream Packages")
+
+    outdated = [s for s in upstream_statuses if s.needs_update and s.latest_tag != "unknown"]
+    if not outdated:
+        info_line("No packages to update")
+        complete_action()
+        return
+
+    for s in outdated:
+        info_line(f"  {s.profile_name}: {s.current_version} → {s.latest_tag}")
+
+    info_line("")
+    count = update_packages_yml(config.dbt_project_dir, upstream_statuses)
+    if count:
+        success_line(f"Updated {count} package(s) in packages.yml")
+        if get_input_true_false("Run dbtf deps now?"):
+            dbt_runner.deps()
+    else:
+        info_line("No changes made to packages.yml")
+
+    complete_action()
+
+
 def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
                linting_service: LintingService, download_service: DownloadService,
-               skill_service: SkillService, sf_service: SnowflakeService):
+               skill_service: SkillService, sf_service: SnowflakeService,
+               upstream_statuses: list = None):
     """Main menu loop."""
     profile_name = config.profile_name
+    has_outdated = upstream_statuses and any(s.needs_update for s in upstream_statuses)
     
-    _show_main_menu(profile_name, git_service)
+    _show_main_menu(profile_name, git_service, upstream_statuses)
     option = get_input_number("Choose an option: ")
     
     while option != 0:
@@ -391,21 +432,34 @@ def _main_menu(config: Config, dbt_runner: DbtRunner, git_service: GitService,
                 profile=config.profile,
             )
             bootstrap.run(platform=platform, install_skills=install_skills, run_deps=run_deps)
+
+        elif option == 5 and has_outdated:
+            _update_upstream_packages(config, dbt_runner, upstream_statuses)
+            # Refresh upstream statuses after update
+            from datasecops_cli.services.upstream_service import check_upstream_versions
+            upstream_statuses = check_upstream_versions(
+                config.profile, config.all_profiles, config.dbt_project_dir
+            )
+            has_outdated = any(s.needs_update for s in upstream_statuses)
         
-        _show_main_menu(profile_name, git_service)
+        _show_main_menu(profile_name, git_service, upstream_statuses)
         option = get_input_number("Choose an option: ")
     
     info_line("Exiting DataSecOps Framework CLI")
 
 
-def _show_main_menu(profile_name: str, git_service: GitService = None):
+def _show_main_menu(profile_name: str, git_service: GitService = None,
+                    upstream_statuses: list = None):
     clear()
     branch = git_service.get_current_branch() if git_service else None
-    section_header("DataSecOps Framework CLI", profile_name, branch)
+    section_header("DataSecOps Framework CLI", profile_name, branch,
+                   upstream_info=upstream_statuses)
     menu_option(1, "development   - dbt Development Commands")
     menu_option(2, "git           - Source Control Operations")
     menu_option(3, "downloads     - Download Configs & Skills")
     menu_option(4, "bootstrap     - Set up a new dbt project with all framework config")
+    if upstream_statuses and any(s.needs_update for s in upstream_statuses):
+        menu_option(5, "update pkgs   - Update upstream packages to latest versions")
     menu_option(0, "exit          - Exit")
 
 

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/src/datasecops_cli/models/project_config.py

@@ -61,6 +61,7 @@ class ProjectProfile(BaseModel):
     project_description: str = ""
     model_types: list[str] = Field(default_factory=list)
     downstream_projects: list[str] = Field(default_factory=list)
+    upstream_projects: list[str] = Field(default_factory=list)
     git_url: str = ""
     target_database: str = ""
     dbt_version: str = "1.9"

datasecops_cli-0.3.2/src/datasecops_cli/services/upstream_service.py

@@ -0,0 +1,158 @@
+"""Service for checking upstream project versions against packages.yml."""
+
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from datasecops_cli.models.project_config import ProjectProfile
+from datasecops_cli.utilities.display import info_line, warning_line
+from datasecops_cli.utilities.yaml_utils import read_yaml, write_yaml
+
+
+@dataclass
+class UpstreamStatus:
+    """Version status of an upstream project dependency."""
+    profile_name: str
+    git_url: str
+    latest_tag: str = ""
+    current_version: str = ""
+    needs_update: bool = False
+    error: str = ""
+
+
+def get_latest_tag(git_url: str, timeout: int = 10) -> Optional[str]:
+    """Fetch the latest tag from a remote git repo without cloning."""
+    try:
+        result = subprocess.run(
+            ["git", "ls-remote", "--tags", "--sort=-v:refname", git_url],
+            capture_output=True, text=True, timeout=timeout,
+        )
+        if result.returncode != 0:
+            return None
+
+        for line in result.stdout.strip().splitlines():
+            ref = line.split("\t")[-1]
+            # Skip ^{} dereferenced tags
+            if ref.endswith("^{}"):
+                continue
+            tag = ref.replace("refs/tags/", "")
+            # Skip pre-release tags
+            if any(pre in tag.lower() for pre in ("alpha", "beta", "rc", "dev")):
+                continue
+            return tag
+        return None
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        return None
+
+
+def read_packages_yml(dbt_project_dir: Path) -> Optional[dict]:
+    """Read packages.yml from the dbt project directory."""
+    return read_yaml(dbt_project_dir / "packages.yml")
+
+
+def find_pinned_version(packages_data: dict, git_url: str) -> str:
+    """Find the pinned revision/version for a git URL in packages.yml."""
+    if not packages_data:
+        return ""
+    for pkg in packages_data.get("packages", []):
+        pkg_url = pkg.get("git", "")
+        if pkg_url and _urls_match(pkg_url, git_url):
+            return pkg.get("revision", "")
+    return ""
+
+
+def _urls_match(url1: str, url2: str) -> bool:
+    """Compare two git URLs, ignoring trailing .git and protocol differences."""
+    def normalize(url: str) -> str:
+        url = url.rstrip("/").removesuffix(".git").lower()
+        url = re.sub(r"^https?://", "", url)
+        url = re.sub(r"^git@([^:]+):", r"\1/", url)
+        return url
+    return normalize(url1) == normalize(url2)
+
+
+def check_upstream_versions(
+    profile: ProjectProfile,
+    all_profiles: list[ProjectProfile],
+    dbt_project_dir: Path,
+) -> list[UpstreamStatus]:
+    """Check version status of all upstream project dependencies."""
+    if not profile.upstream_projects:
+        return []
+
+    # Build lookup of profiles by name
+    profile_map = {p.profile_name: p for p in all_profiles}
+
+    # Read current packages.yml
+    packages_data = read_packages_yml(dbt_project_dir)
+
+    results = []
+    for upstream_name in profile.upstream_projects:
+        upstream_profile = profile_map.get(upstream_name)
+        if not upstream_profile:
+            results.append(UpstreamStatus(
+                profile_name=upstream_name,
+                git_url="",
+                error=f"Profile '{upstream_name}' not found",
+            ))
+            continue
+
+        git_url = upstream_profile.git_url
+        if not git_url:
+            results.append(UpstreamStatus(
+                profile_name=upstream_name,
+                git_url="",
+                error="No git_url configured",
+            ))
+            continue
+
+        latest_tag = get_latest_tag(git_url)
+        current_version = find_pinned_version(packages_data, git_url)
+
+        needs_update = False
+        if latest_tag and current_version:
+            needs_update = latest_tag.lstrip("v") != current_version.lstrip("v")
+        elif latest_tag and not current_version:
+            needs_update = True
+
+        results.append(UpstreamStatus(
+            profile_name=upstream_name,
+            git_url=git_url,
+            latest_tag=latest_tag or "unknown",
+            current_version=current_version or "not pinned",
+            needs_update=needs_update,
+        ))
+
+    return results
+
+
+def update_packages_yml(
+    dbt_project_dir: Path,
+    upstream_statuses: list[UpstreamStatus],
+) -> int:
+    """Update packages.yml with latest versions for outdated upstream packages.
+    
+    Returns the number of packages updated.
+    """
+    packages_data = read_packages_yml(dbt_project_dir)
+    if not packages_data:
+        return 0
+
+    updated = 0
+    for status in upstream_statuses:
+        if not status.needs_update or status.latest_tag == "unknown":
+            continue
+        for pkg in packages_data.get("packages", []):
+            pkg_url = pkg.get("git", "")
+            if pkg_url and _urls_match(pkg_url, status.git_url):
+                pkg["revision"] = status.latest_tag
+                updated += 1
+                break
+
+    if updated:
+        dest = dbt_project_dir / "packages.yml"
+        write_yaml(dest, packages_data)
+
+    return updated

{datasecops_cli-0.3.1 → datasecops_cli-0.3.2}/src/datasecops_cli/utilities/display.py

@@ -16,7 +16,8 @@ def print_long_line(color: str = Fore.GREEN) -> None:
     print_detail("=" * 76, color)
 
 
-def section_header(message: str, active_profile: str = None, current_branch: str = None) -> None:
+def section_header(message: str, active_profile: str = None, current_branch: str = None,
+                   upstream_info: list = None) -> None:
     print_long_line(Fore.GREEN)
     print_detail(message, Fore.GREEN)
     print_long_line(Fore.GREEN)
@@ -24,7 +25,18 @@ def section_header(message: str, active_profile: str = None, current_branch: str
         print_detail(f"  Profile: {active_profile}", Fore.GREEN)
     if current_branch:
         print_detail(f"  Branch:  {current_branch}", Fore.GREEN)
-    if active_profile or current_branch:
+    if upstream_info:
+        for status in upstream_info:
+            if status.error:
+                print_detail(f"  Upstream: {status.profile_name} — {status.error}", Fore.YELLOW)
+            elif status.needs_update:
+                print_detail(
+                    f"  Upstream: {status.profile_name} ({status.current_version} → {status.latest_tag} available)",
+                    Fore.YELLOW,
+                )
+            else:
+                print_detail(f"  Upstream: {status.profile_name} ({status.current_version} ✓)", Fore.GREEN)
+    if active_profile or current_branch or upstream_info:
         print_long_line(Fore.GREEN)