datasecops-cli 0.1.0__py3-none-any.whl

datasecops_cli/models/git_helpers.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional
+
+@dataclass
+class GitBranchState:
+    is_dirty: bool = False
+    commits_behind_main: int = 0
+    syncs_ahead: int = 0
+    syncs_behind: int = 0
+    uncommitted_file_count: int = 0
+    last_reset: Optional[datetime] = None
+    remote_exists: bool = True
+    
+    def should_reset(self) -> bool:
+        if self.last_reset is None:
+            return True
+        return (datetime.now() - self.last_reset).total_seconds() > 60
+
+@dataclass
+class GitCommitHelper:
+    file: str = ""
+    checked: bool = True
+
+@dataclass 
+class GitBranchComparison:
+    local: str = ""
+    remote: str = ""
+    is_current: bool = False

datasecops_cli/models/project_config.py

@@ -0,0 +1,87 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+
+class DatasecopsConfig(BaseModel):
+    """Local .datasecops.yml config."""
+    connection_name: str = ""
+    app_database: str = ""
+    profile_name: str = ""
+
+class DbtTarget(BaseModel):
+    target_name: str = ""
+    branch_name: str = ""
+    target_role: str = ""
+    target_warehouse: str = ""
+    target_schema: str = "PUBLIC"
+    is_default: bool = False
+
+class ProjectSettings(BaseModel):
+    project_dir: str = "./dbt"
+    profile_dir: str = "~/.dbt"
+    execution_mode: str = "dbt_cli"
+    targets: list[DbtTarget] = Field(default_factory=lambda: [
+        DbtTarget(target_name="dev", branch_name="dev", target_role="DEVELOPERS", target_warehouse="DEV_WH", is_default=True),
+        DbtTarget(target_name="test", branch_name="test", target_role="DATAOPS_ADMIN", target_warehouse="DATAOPS_WH"),
+        DbtTarget(target_name="prod", branch_name="prod", target_role="DATAOPS_ADMIN", target_warehouse="DATAOPS_WH"),
+    ])
+    sources_database_prefix: list[str] = Field(default_factory=list)
+    
+    def get_default_target(self) -> Optional[DbtTarget]:
+        for t in self.targets:
+            if t.is_default:
+                return t
+        return self.targets[0] if self.targets else None
+    
+    def get_deployment_branches(self) -> list[str]:
+        return [t.branch_name for t in self.targets if t.branch_name]
+
+class BranchType(BaseModel):
+    name: str = ""
+    purpose: str = ""
+    lifecycle: str = ""
+    use_case: str = ""
+
+class EnvironmentBranch(BaseModel):
+    branch_name: str = ""
+    purpose: str = ""
+    merge_into_using: str = ""
+    auto_released_to_environment: str = ""
+
+class SourceControl(BaseModel):
+    branch_types: list[BranchType] = Field(default_factory=list)
+    ticket_number_required: bool = False
+    branch_format: str = "{branch_type}/{branch_name}"
+    source_control_platform: str = "GitHub"
+    environments: list[EnvironmentBranch] = Field(default_factory=list)
+
+class ProjectProfile(BaseModel):
+    project_id: int = 0
+    profile_name: str = ""
+    project_name: str = ""
+    project_description: str = ""
+    model_types: list[str] = Field(default_factory=list)
+    downstream_projects: list[str] = Field(default_factory=list)
+    git_url: str = ""
+    target_database: str = ""
+    dbt_version: str = "1.9"
+    dbt_packages: list[str] = Field(default_factory=list)
+
+class DbtPackage(BaseModel):
+    name: str = ""
+    source: str = "git"
+    url: str = ""
+    description: str = ""
+    latest_version: str = ""
+
+class CortexSkillFile(BaseModel):
+    filename: str = ""
+    content: str = ""
+
+class CortexSkill(BaseModel):
+    skill_id: str = ""
+    name: str = ""
+    description: str = ""
+    version: str = "0.1.0"
+    category: str = "general"
+    files: list[CortexSkillFile] = Field(default_factory=list)
+    enabled: bool = True

datasecops_cli/services/__init__.py

@@ -0,0 +1 @@
+

datasecops_cli/services/dbt_runner.py

@@ -0,0 +1,130 @@
+import subprocess
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from datasecops_cli.utilities.display import info_line, error_line, success_line
+
+
+class DbtRunner:
+    """Runs dbt commands via subprocess (dbt Fusion)."""
+    
+    def __init__(self, project_dir: Path, profiles_dir: Path, target: str = "dev"):
+        self.project_dir = project_dir
+        self.profiles_dir = profiles_dir
+        self.target = target
+    
+    def _default_args(self) -> list[str]:
+        return [
+            f"--project-dir={self.project_dir}",
+            f"--profiles-dir={self.profiles_dir}",
+        ]
+    
+    def _run_command(self, command: str, extra_args: list[str] = None) -> subprocess.CompletedProcess:
+        cmd = ["dbt", command] + (extra_args or []) + self._default_args()
+        info_line(f"Running: {' '.join(cmd)}")
+        result = subprocess.run(cmd, capture_output=False)
+        if result.returncode != 0:
+            error_line(f"dbt {command} failed with exit code {result.returncode}")
+        else:
+            success_line(f"dbt {command} completed successfully")
+        return result
+    
+    def run(self, select: str = None, exclude: str = None, full_refresh: bool = False,
+            modified_only: bool = False) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if modified_only and (self.project_dir / "manifest.json").exists():
+            args += ["--select", "state:modified+", "--state", str(self.project_dir)]
+        elif select:
+            args += ["--select", select]
+        if exclude:
+            args += ["--exclude", exclude]
+        if full_refresh:
+            args.append("--full-refresh")
+        result = self._run_command("run", args)
+        self._copy_manifest()
+        return result
+    
+    def build(self, select: str = None, exclude: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        if exclude:
+            args += ["--exclude", exclude]
+        result = self._run_command("build", args)
+        self._copy_manifest()
+        return result
+    
+    def test(self, select: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        return self._run_command("test", args)
+    
+    def compile(self, select: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        return self._run_command("compile", args)
+    
+    def deps(self) -> subprocess.CompletedProcess:
+        return self._run_command("deps")
+    
+    def seed(self, select: str = None, full_refresh: bool = False) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        if full_refresh:
+            args.append("--full-refresh")
+        return self._run_command("seed", args)
+    
+    def snapshot(self, select: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        return self._run_command("snapshot", args)
+    
+    def source_freshness(self, select: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        return self._run_command("source", ["freshness"] + args)
+    
+    def docs_generate(self) -> subprocess.CompletedProcess:
+        return self._run_command("docs", ["generate", f"--target={self.target}"])
+    
+    def docs_serve(self) -> subprocess.Popen:
+        cmd = ["dbt", "docs", "serve"] + self._default_args()
+        info_line(f"Running: {' '.join(cmd)}")
+        return subprocess.Popen(cmd)
+    
+    def clean(self) -> subprocess.CompletedProcess:
+        return self._run_command("clean")
+    
+    def debug(self) -> subprocess.CompletedProcess:
+        return self._run_command("debug", [f"--target={self.target}"])
+    
+    def list_models(self, select: str = None, resource_type: str = None) -> subprocess.CompletedProcess:
+        args = [f"--target={self.target}"]
+        if select:
+            args += ["--select", select]
+        if resource_type:
+            args += ["--resource-type", resource_type]
+        return self._run_command("list", args)
+    
+    def retry(self) -> subprocess.CompletedProcess:
+        result = self._run_command("retry", [f"--target={self.target}"])
+        self._copy_manifest()
+        return result
+    
+    def run_operation(self, macro: str, args_str: str = None) -> subprocess.CompletedProcess:
+        cmd_args = [f"--target={self.target}"]
+        if args_str:
+            cmd_args += ["--args", args_str]
+        return self._run_command("run-operation", [macro] + cmd_args)
+    
+    def _copy_manifest(self) -> None:
+        manifest = self.project_dir / "target" / "manifest.json"
+        dest = self.project_dir / "manifest.json"
+        if manifest.exists():
+            shutil.copy2(manifest, dest)

datasecops_cli/services/download_service.py

@@ -0,0 +1,103 @@
+import json
+import yaml
+from pathlib import Path
+from typing import Optional
+
+from datasecops_cli.services.snowflake_service import SnowflakeService
+from datasecops_cli.utilities.display import info_line, success_line, error_line
+from datasecops_cli.utilities.file_utils import write_file, ensure_dir
+
+
+class DownloadService:
+    """Downloads configurations from the native app."""
+    
+    def __init__(self, snowflake_service: SnowflakeService, project_dir: Path):
+        self.sf = snowflake_service
+        self.project_dir = project_dir
+    
+    def download_sqlfluff_config(self) -> bool:
+        info_line("Downloading SQLFluff configuration...")
+        raw = self.sf.get_framework_config("SQLFLUFF_RULES")
+        if not raw:
+            error_line("No SQLFluff configuration found in native app")
+            return False
+        
+        lines = ["[sqlfluff]", "dialect = snowflake", "templater = dbt", ""]
+        
+        core = raw.get("core", {})
+        for code, state in core.items():
+            if isinstance(state, dict) and state.get("enabled", True):
+                opts = state.get("options", {})
+                for opt_key, opt_val in opts.items():
+                    lines.append(f"{opt_key} = {opt_val}")
+        
+        lines.append("")
+        lines.append("[sqlfluff:indentation]")
+        indentation = raw.get("indentation", {})
+        for code, state in indentation.items():
+            if isinstance(state, dict):
+                opts = state.get("options", {})
+                for opt_key, opt_val in opts.items():
+                    lines.append(f"{opt_key} = {opt_val}")
+        
+        content = "\n".join(lines) + "\n"
+        dest = self.project_dir / ".sqlfluff"
+        write_file(dest, content)
+        success_line(f"SQLFluff config written to {dest}")
+        return True
+    
+    def download_pipelines(self, platform: str = "github") -> bool:
+        info_line(f"Downloading {platform} pipeline configurations...")
+        raw = self.sf.get_framework_config("PIPELINES")
+        if not raw:
+            error_line("No pipeline configuration found in native app")
+            return False
+        
+        pipelines = raw.get("pipelines", [])
+        count = 0
+        for pipe in pipelines:
+            if pipe.get("platform", "github") != platform or not pipe.get("enabled", True):
+                continue
+            filename = pipe.get("filename", "")
+            yaml_content = pipe.get("yaml_content", "")
+            if filename and yaml_content:
+                if platform == "github":
+                    dest = self.project_dir / ".github" / "workflows" / filename
+                else:
+                    dest = self.project_dir / filename
+                write_file(dest, yaml_content)
+                info_line(f"  Written: {dest}")
+                count += 1
+        
+        success_line(f"Downloaded {count} pipeline file(s)")
+        return True
+    
+    def download_dbt_packages(self, dbt_project_dir: Path) -> bool:
+        info_line("Downloading dbt package versions...")
+        raw = self.sf.get_framework_config("DBT_PACKAGES")
+        if not raw:
+            error_line("No dbt packages configuration found in native app")
+            return False
+        
+        packages = raw.get("packages", [])
+        pkg_list = []
+        for pkg in packages:
+            source = pkg.get("source", "git")
+            url = pkg.get("url", "")
+            version = pkg.get("latest_version", "")
+            if source == "git" and url:
+                entry = {"git": url}
+                if version:
+                    entry["revision"] = version
+                pkg_list.append(entry)
+            elif source == "package" and url:
+                entry = {"package": url}
+                if version:
+                    entry["version"] = version
+                pkg_list.append(entry)
+        
+        dest = dbt_project_dir / "packages.yml"
+        content = yaml.dump({"packages": pkg_list}, default_flow_style=False, sort_keys=False)
+        write_file(dest, content)
+        success_line(f"packages.yml written to {dest}")
+        return True

datasecops_cli/services/git_service.py

@@ -0,0 +1,183 @@
+from pathlib import Path
+from typing import Optional
+from git import Repo, GitCommandError
+
+from datasecops_cli.models.git_helpers import GitCommitHelper
+from datasecops_cli.utilities.display import info_line, error_line, success_line, warning_line
+
+
+class GitService:
+    """Git operations via GitPython."""
+    
+    def __init__(self, repo_path: Path):
+        self.repo = Repo(repo_path)
+    
+    def get_current_branch(self) -> str:
+        try:
+            return self.repo.active_branch.name
+        except TypeError:
+            return "DETACHED HEAD"
+    
+    def is_dirty(self) -> bool:
+        return self.repo.is_dirty(untracked_files=True)
+    
+    def get_uncommitted_file_count(self) -> int:
+        return len(self.repo.index.diff(None)) + len(self.repo.untracked_files)
+    
+    def get_changed_files(self) -> list[GitCommitHelper]:
+        return [GitCommitHelper(file=d.a_path) for d in self.repo.index.diff(None)]
+    
+    def get_new_files(self) -> list[GitCommitHelper]:
+        return [GitCommitHelper(file=f) for f in self.repo.untracked_files]
+    
+    def get_local_branches(self) -> dict[str, str]:
+        return {b.name: b.name for b in self.repo.branches}
+    
+    def get_remote_branches(self) -> dict[str, str]:
+        self.repo.remotes.origin.fetch()
+        return {
+            ref.remote_head: ref.remote_head
+            for ref in self.repo.remotes.origin.refs
+            if ref.remote_head != "HEAD"
+        }
+    
+    def create_branch(self, branch_type: str, ticket: str, name: str) -> None:
+        branch_name = f"{branch_type}/{ticket}_{name}" if ticket else f"{branch_type}/{name}"
+        info_line(f"Creating branch: {branch_name}")
+        self.repo.remotes.origin.fetch()
+        new_branch = self.repo.create_head(branch_name, "origin/main")
+        new_branch.checkout()
+        self.repo.remotes.origin.push(branch_name, set_upstream=True)
+        success_line(f"Branch {branch_name} created and checked out")
+    
+    def checkout_branch(self, remote_name: str) -> None:
+        info_line(f"Checking out: {remote_name}")
+        self.repo.remotes.origin.fetch()
+        try:
+            self.repo.git.checkout(remote_name)
+        except GitCommandError:
+            self.repo.git.checkout("-b", remote_name, f"origin/{remote_name}")
+        success_line(f"Switched to {remote_name}")
+    
+    def switch_branch(self, name: str) -> None:
+        self.repo.branches[name].checkout()
+        success_line(f"Switched to {name}")
+    
+    def delete_branch(self, name: str) -> None:
+        if name == self.get_current_branch():
+            error_line("Cannot delete the current branch")
+            return
+        self.repo.delete_head(name, force=True)
+        try:
+            self.repo.remotes.origin.push(refspec=f":{name}")
+        except GitCommandError:
+            pass
+        success_line(f"Deleted branch {name}")
+    
+    def commit_changes(self, message: str) -> None:
+        self.repo.git.add(A=True)
+        self.repo.index.commit(message)
+        success_line(f"Committed: {message}")
+    
+    def push_branch(self) -> None:
+        branch = self.get_current_branch()
+        info_line(f"Pushing {branch}...")
+        self.repo.remotes.origin.push(branch)
+        success_line(f"Pushed {branch}")
+    
+    def pull_branch(self) -> None:
+        branch = self.get_current_branch()
+        info_line(f"Pulling {branch}...")
+        self.repo.remotes.origin.pull(branch)
+        success_line(f"Pulled {branch}")
+    
+    def push_branch_to_destination(self, destination: str, force: bool = False) -> None:
+        current = self.get_current_branch()
+        info_line(f"Pushing {current} to {destination}...")
+        refspec = f"{current}:{destination}"
+        self.repo.remotes.origin.push(refspec=refspec, force=force)
+        success_line(f"Pushed to {destination}")
+    
+    def rebase_with_main(self) -> bool:
+        try:
+            self.repo.remotes.origin.fetch()
+            self.repo.git.rebase("origin/main")
+            success_line("Rebase with main completed")
+            return True
+        except GitCommandError as e:
+            error_line(f"Rebase conflict: {e}")
+            warning_line("Resolve conflicts then use 'rebase continue' or 'rebase abort'")
+            return False
+    
+    def rebase_continue(self) -> None:
+        self.repo.git.rebase("--continue")
+        success_line("Rebase continued")
+    
+    def rebase_abort(self) -> None:
+        self.repo.git.rebase("--abort")
+        success_line("Rebase aborted")
+    
+    def squash_and_rebase(self) -> bool:
+        try:
+            self.repo.remotes.origin.fetch()
+            main_commit = self.repo.commit("origin/main")
+            current = self.get_current_branch()
+            self.repo.git.reset("--soft", main_commit.hexsha)
+            self.repo.index.commit(f"squash: {current}")
+            self.repo.git.rebase("origin/main")
+            success_line("Squash and rebase completed")
+            return True
+        except GitCommandError as e:
+            error_line(f"Squash rebase failed: {e}")
+            return False
+    
+    def squash_merge_into_test(self) -> bool:
+        current = self.get_current_branch()
+        try:
+            self.repo.remotes.origin.fetch()
+            self.repo.git.checkout("test")
+            self.repo.remotes.origin.pull("test")
+            self.repo.git.merge("--squash", current)
+            self.repo.index.commit(f"squash merge: {current} into test")
+            self.repo.remotes.origin.push("test")
+            self.repo.git.checkout(current)
+            success_line(f"Squash merged {current} into test")
+            return True
+        except GitCommandError as e:
+            error_line(f"Squash merge failed: {e}")
+            try:
+                self.repo.git.checkout(current)
+            except Exception:
+                pass
+            return False
+    
+    def cherry_pick_from_test(self, commit_sha: str) -> bool:
+        try:
+            self.repo.git.cherry_pick(commit_sha)
+            success_line(f"Cherry-picked {commit_sha[:8]}")
+            return True
+        except GitCommandError as e:
+            error_line(f"Cherry-pick failed: {e}")
+            return False
+    
+    def get_test_commits(self, limit: int = 20) -> list[dict]:
+        try:
+            self.repo.remotes.origin.fetch()
+            commits = list(self.repo.iter_commits("origin/test", max_count=limit))
+            return [{"sha": c.hexsha, "message": c.message.strip(), "author": str(c.author), "date": str(c.committed_datetime)} for c in commits]
+        except Exception:
+            return []
+    
+    def prune_remote_branches(self) -> None:
+        self.repo.remotes.origin.fetch(prune=True)
+        success_line("Pruned remote branches")
+    
+    def reset_to_main(self) -> None:
+        self.repo.remotes.origin.fetch()
+        self.repo.git.checkout("main")
+        self.repo.git.reset("--hard", "origin/main")
+        # Delete all local branches except main
+        for branch in self.repo.branches:
+            if branch.name != "main":
+                self.repo.delete_head(branch, force=True)
+        success_line("Reset to main completed")

datasecops_cli/services/linting_service.py

@@ -0,0 +1,47 @@
+import subprocess
+from pathlib import Path
+
+from datasecops_cli.utilities.display import info_line, error_line, success_line
+
+
+class LintingService:
+    """SQLFluff linting operations."""
+    
+    def __init__(self, project_dir: Path):
+        self.project_dir = project_dir
+    
+    def lint_file(self, file_path: str = None, fix: bool = False) -> subprocess.CompletedProcess:
+        action = "fix" if fix else "lint"
+        cmd = ["sqlfluff", action]
+        if file_path:
+            cmd.append(file_path)
+        else:
+            cmd.append(str(self.project_dir / "models"))
+        
+        config_path = self.project_dir / ".sqlfluff"
+        if config_path.exists():
+            cmd += ["--config", str(config_path)]
+        
+        info_line(f"Running: sqlfluff {action}")
+        result = subprocess.run(cmd, capture_output=False, cwd=str(self.project_dir))
+        if result.returncode == 0:
+            success_line(f"SQLFluff {action} completed — no issues found")
+        elif result.returncode == 1 and not fix:
+            info_line(f"SQLFluff {action} found linting issues")
+        else:
+            error_line(f"SQLFluff {action} failed with exit code {result.returncode}")
+        return result
+    
+    def lint_modified(self, fix: bool = False, changed_files: list[str] = None) -> None:
+        if not changed_files:
+            info_line("No modified SQL files to lint")
+            return
+        
+        sql_files = [f for f in changed_files if f.endswith(".sql")]
+        if not sql_files:
+            info_line("No modified SQL files to lint")
+            return
+        
+        info_line(f"Linting {len(sql_files)} modified SQL file(s)...")
+        for f in sql_files:
+            self.lint_file(f, fix=fix)

datasecops_cli/services/skill_service.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+from typing import Optional
+
+from datasecops_cli.services.snowflake_service import SnowflakeService
+from datasecops_cli.models.project_config import CortexSkill, CortexSkillFile
+from datasecops_cli.utilities.display import info_line, success_line, error_line, warning_line
+from datasecops_cli.utilities.file_utils import write_file, ensure_dir, get_cortex_skills_dir
+
+
+class SkillService:
+    """Manages Cortex Code skill installation."""
+    
+    def __init__(self, snowflake_service: SnowflakeService):
+        self.sf = snowflake_service
+    
+    def get_available_skills(self) -> list[CortexSkill]:
+        raw = self.sf.get_framework_config("CORTEX_SKILLS")
+        if not raw:
+            return []
+        skills_data = raw.get("skills", [])
+        result = []
+        for s in skills_data:
+            if isinstance(s, dict):
+                files = [CortexSkillFile(**f) for f in s.get("files", []) if isinstance(f, dict)]
+                skill = CortexSkill(
+                    skill_id=s.get("skill_id", ""),
+                    name=s.get("name", ""),
+                    description=s.get("description", ""),
+                    version=s.get("version", "0.1.0"),
+                    category=s.get("category", "general"),
+                    files=files,
+                    enabled=s.get("enabled", True),
+                )
+                if skill.enabled:
+                    result.append(skill)
+        return result
+    
+    def list_skills(self) -> None:
+        skills = self.get_available_skills()
+        if not skills:
+            info_line("No skills available in the native app")
+            return
+        
+        info_line(f"Available Cortex Code skills ({len(skills)}):")
+        for s in skills:
+            installed = self._is_installed(s.skill_id)
+            status = "installed" if installed else "not installed"
+            info_line(f"  [{s.category}] {s.name} v{s.version} — {status}")
+            if s.description:
+                info_line(f"           {s.description}")
+    
+    def install_skill(self, skill: CortexSkill, target_dir: Path = None) -> bool:
+        target = target_dir or get_cortex_skills_dir()
+        skill_dir = target / skill.skill_id
+        ensure_dir(skill_dir)
+        
+        for f in skill.files:
+            if f.filename and f.content:
+                write_file(skill_dir / f.filename, f.content)
+        
+        success_line(f"Installed skill: {skill.name} v{skill.version} -> {skill_dir}")
+        return True
+    
+    def install_all(self, target_dir: Path = None) -> int:
+        skills = self.get_available_skills()
+        count = 0
+        for skill in skills:
+            if self.install_skill(skill, target_dir):
+                count += 1
+        success_line(f"Installed {count} skill(s)")
+        return count
+    
+    def update_skills(self, target_dir: Path = None) -> int:
+        skills = self.get_available_skills()
+        target = target_dir or get_cortex_skills_dir()
+        count = 0
+        for skill in skills:
+            if self._is_installed(skill.skill_id, target):
+                self.install_skill(skill, target)
+                count += 1
+        success_line(f"Updated {count} skill(s)")
+        return count
+    
+    def _is_installed(self, skill_id: str, target_dir: Path = None) -> bool:
+        target = target_dir or get_cortex_skills_dir()
+        return (target / skill_id).exists()