odibi 2.5.0__py3-none-any.whl

odibi/cli/doctor.py

@@ -0,0 +1,161 @@
+import importlib
+import os
+from typing import Optional
+
+from odibi.pipeline import PipelineManager
+
+
+def check_dependencies() -> bool:
+    """Check installed dependencies."""
+    print("Checking dependencies...")
+    deps = [
+        ("pandas", "Pandas"),
+        ("duckdb", "DuckDB (Local SQL Engine)"),
+        ("pyspark", "PySpark (Distributed Engine)"),
+        ("fastapi", "FastAPI (UI Backend)"),
+        ("uvicorn", "Uvicorn (UI Server)"),
+        ("openlineage.client", "OpenLineage (Governance)"),
+        ("azure.storage.blob", "Azure Blob Storage"),
+        ("delta", "Delta Lake"),
+    ]
+
+    all_good = True
+    for module, name in deps:
+        try:
+            mod = importlib.import_module(module)
+            version = getattr(mod, "__version__", "installed")
+            print(f"  [OK] {name}: {version}")
+        except ImportError:
+            # Define optional vs required
+            optional = ["pyspark", "openlineage.client", "azure.storage.blob", "delta"]
+
+            if module in optional:
+                print(f"  [OPTIONAL] {name}: Not installed")
+            else:
+                print(f"  [MISSING] {name}")
+                all_good = False
+
+    return all_good
+
+
+def check_config() -> Optional[PipelineManager]:
+    """Check configuration file."""
+    print("\nChecking configuration...")
+    config_files = ["odibi.yaml", "project.yaml", "odibi.yml", "project.yml"]
+    found_config = None
+    for f in config_files:
+        if os.path.exists(f):
+            found_config = f
+            break
+
+    if not found_config:
+        print("  [FAIL] No configuration file found (odibi.yaml or project.yaml)")
+        return None
+
+    print(f"  [OK] Found {found_config}")
+
+    try:
+        # We use PipelineManager to parse and validate
+        manager = PipelineManager.from_yaml(found_config)
+        print("  [OK] Configuration is valid")
+        return manager
+    except Exception as e:
+        print(f"  [FAIL] Invalid configuration: {e}")
+        return None
+
+
+def check_connections(manager: PipelineManager) -> bool:
+    """Check connections."""
+    print("\nChecking connections...")
+    if not manager:
+        print("  [SKIP] Skipping connection checks (no config)")
+        return False
+
+    all_good = True
+    for name, conn in manager.connections.items():
+        try:
+            conn.validate()
+            print(f"  [OK] {name} ({conn.__class__.__name__})")
+        except Exception as e:
+            print(f"  [FAIL] {name}: {e}")
+            all_good = False
+
+    return all_good
+
+
+def check_system_catalog(manager: PipelineManager) -> bool:
+    """Check system catalog connectivity and integrity."""
+    print("\nChecking System Catalog...")
+
+    if not manager or not manager.catalog_manager:
+        print("  [SKIP] System Catalog not configured (Local Only Mode)")
+        return True
+
+    print(f"  [INFO] Catalog Path: {manager.catalog_manager.base_path}")
+
+    all_good = True
+    required_tables = [
+        "meta_tables",
+        "meta_runs",
+        "meta_patterns",
+        "meta_metrics",
+        "meta_state",
+        "meta_pipelines",
+        "meta_nodes",
+    ]
+
+    for table in required_tables:
+        path = manager.catalog_manager.tables.get(table)
+        # Access internal method for check (acceptable for doctor/diagnostic tool)
+        exists = manager.catalog_manager._table_exists(path)
+        if exists:
+            print(f"  [OK] {table}")
+        else:
+            print(f"  [FAIL] {table} not found at {path}")
+            all_good = False
+
+    return all_good
+
+
+def doctor_command(args) -> int:
+    """Run doctor command."""
+    try:
+        version = __import__("odibi").__version__
+    except Exception:
+        version = "unknown"
+
+    print(f"Odibi Doctor (v{version})")
+    print("=" * 40)
+
+    deps_ok = check_dependencies()
+    manager = check_config()
+    conns_ok = check_connections(manager)
+
+    catalog_ok = True
+    if manager:
+        catalog_ok = check_system_catalog(manager)
+
+    if not deps_ok:
+        print("\n[WARNING] Some required dependencies are missing.")
+        return 1
+
+    if not manager:
+        print("\n[WARNING] Configuration issues found.")
+        return 1
+
+    if not conns_ok:
+        print("\n[WARNING] Some connections failed validation.")
+        return 1
+
+    if not catalog_ok:
+        print("\n[WARNING] System Catalog issues found.")
+        return 1
+
+    print("\n[SUCCESS] You are ready to run pipelines!")
+    return 0
+
+
+def add_doctor_parser(subparsers):
+    """Add doctor parser."""
+    parser = subparsers.add_parser("doctor", help="Check environment health")
+    return parser

odibi/cli/export.py

@@ -0,0 +1,66 @@
+import os
+
+from odibi.config import load_config_from_file
+from odibi.orchestration.airflow import AirflowExporter
+
+
+def add_export_parser(subparsers):
+    export_parser = subparsers.add_parser("export", help="Export pipeline to orchestration code")
+    export_parser.add_argument(
+        "--target", choices=["airflow", "dagster"], required=True, help="Export target"
+    )
+    export_parser.add_argument("--pipeline", help="Pipeline name (required for Airflow)")
+    export_parser.add_argument("--out", required=True, help="Output file path")
+    export_parser.add_argument("--config", default="odibi.yaml", help="Path to odibi.yaml")
+
+
+def export_command(args):
+    """Export pipeline to orchestration code."""
+    config_path = args.config
+    if not os.path.exists(config_path):
+        print(f"Error: Config file '{config_path}' not found.")
+        return 1
+
+    try:
+        project_config = load_config_from_file(config_path)
+    except Exception as e:
+        print(f"Error loading config: {e}")
+        return 1
+
+    if args.target == "airflow":
+        if not args.pipeline:
+            print("Error: --pipeline is required for Airflow export.")
+            return 1
+
+        exporter = AirflowExporter(project_config)
+        try:
+            code = exporter.generate_code(args.pipeline)
+            with open(args.out, "w", encoding="utf-8") as f:
+                f.write(code)
+            print(f"Successfully exported Airflow DAG to {args.out}")
+        except ValueError as e:
+            print(f"Error: {e}")
+            return 1
+
+    elif args.target == "dagster":
+        # Generate a definitions.py scaffolding
+        content = f"""
+# Generated by Odibi
+import os
+from dagster import Definitions
+from odibi.config import load_config_from_file
+from odibi.orchestration.dagster import DagsterFactory
+
+# Load project config
+# Ensure odibi.yaml is in the same directory or adjust path
+config_path = os.path.join(os.path.dirname(__file__), "{config_path}")
+project_config = load_config_from_file(config_path)
+
+# Create definitions
+defs = DagsterFactory(project_config).create_definitions()
+"""
+        with open(args.out, "w", encoding="utf-8") as f:
+            f.write(content)
+        print(f"Successfully created Dagster definitions at {args.out}")
+
+    return 0

odibi/cli/graph.py

@@ -0,0 +1,150 @@
+"""
+Graph CLI Command
+=================
+
+Visualizes the pipeline dependency graph.
+"""
+
+from odibi.graph import DependencyGraph
+from odibi.pipeline import PipelineManager
+
+
+def graph_command(args):
+    """
+    Handle graph subcommand.
+
+    Args:
+        args: Parsed command-line arguments
+
+    Returns:
+        Exit code
+    """
+    try:
+        # Load pipeline manager
+        env = getattr(args, "env", None)
+        manager = PipelineManager.from_yaml(args.config, env=env)
+
+        # Determine which pipeline to graph
+        pipeline_name = args.pipeline
+        if not pipeline_name:
+            # Default to first pipeline if not specified
+            pipeline_names = manager.list_pipelines()
+            if not pipeline_names:
+                print("❌ No pipelines found in configuration")
+                return 1
+            pipeline_name = pipeline_names[0]
+
+        # Get the pipeline
+        try:
+            pipeline = manager.get_pipeline(pipeline_name)
+        except ValueError:
+            print(f"❌ Pipeline '{pipeline_name}' not found")
+            return 1
+
+        # Generate visualization
+        if args.format == "ascii":
+            print(pipeline.visualize())
+        elif args.format == "dot":
+            print(_generate_dot(pipeline.graph, pipeline_name, manager.catalog_manager))
+        elif args.format == "mermaid":
+            print(_generate_mermaid(pipeline.graph, pipeline_name, manager.catalog_manager))
+
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error generating graph: {e}")
+        if args.verbose:
+            import traceback
+
+            traceback.print_exc()
+        return 1
+
+
+def _generate_dot(graph: DependencyGraph, pipeline_name: str, catalog_manager=None) -> str:
+    """Generate DOT (Graphviz) representation."""
+    lines = []
+    lines.append(f'digraph "{pipeline_name}" {{')
+    lines.append("    rankdir=LR;")
+    lines.append('    node [shape=box, style=rounded, fontname="Helvetica"];')
+    lines.append('    edge [fontname="Helvetica"];')
+    lines.append("")
+
+    for node_name in graph.nodes:
+        # Add node
+        node = graph.nodes[node_name]
+        op_type = "unknown"
+        if node.read:
+            op_type = "read"
+            color = "lightblue"
+        elif node.write:
+            op_type = "write"
+            color = "lightgreen"
+        elif node.transform:
+            op_type = "transform"
+            color = "lightyellow"
+
+        # Enrich with Catalog Stats
+        stats_text = ""
+        if catalog_manager:
+            try:
+                avg_rows = catalog_manager.get_average_volume(node_name)
+                if avg_rows is not None:
+                    stats_text = f"\\n~{int(avg_rows)} rows"
+            except Exception:
+                pass
+
+        label = f"{node_name}\\n({op_type}){stats_text}"
+        lines.append(f'    "{node_name}" [label="{label}", style="filled", fillcolor="{color}"];')
+
+        # Add edges
+        for dep in node.depends_on:
+            lines.append(f'    "{dep}" -> "{node_name}";')
+
+    lines.append("}")
+    return "\n".join(lines)
+
+
+def _generate_mermaid(graph: DependencyGraph, pipeline_name: str, catalog_manager=None) -> str:
+    """Generate Mermaid diagram."""
+    lines = []
+    lines.append("graph LR")
+
+    # Define styles
+    lines.append("    classDef read fill:lightblue,stroke:#333,stroke-width:1px;")
+    lines.append("    classDef write fill:lightgreen,stroke:#333,stroke-width:1px;")
+    lines.append("    classDef transform fill:lightyellow,stroke:#333,stroke-width:1px;")
+
+    for node_name in graph.nodes:
+        node = graph.nodes[node_name]
+        style_class = "transform"
+
+        # Node styling based on type
+        if node.read:
+            shape = "(("  # Circle
+            end_shape = "))"
+            style_class = "read"
+        elif node.write:
+            shape = "[/"  # Parallelogram
+            end_shape = "/]"
+            style_class = "write"
+        else:
+            shape = "["  # Box
+            end_shape = "]"
+
+        # Enrich with Catalog Stats
+        label = node_name
+        if catalog_manager:
+            try:
+                avg_rows = catalog_manager.get_average_volume(node_name)
+                if avg_rows is not None:
+                    label = f"{node_name}<br/>~{int(avg_rows)} rows"
+            except Exception:
+                pass
+
+        lines.append(f'    {node_name}{shape}"{label}"{end_shape}:::{style_class}')
+
+        # Edges
+        for dep in node.depends_on:
+            lines.append(f"    {dep} --> {node_name}")
+
+    return "\n".join(lines)

odibi/cli/init_pipeline.py

@@ -0,0 +1,242 @@
+import logging
+import os
+import shutil
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Map template names to their relative paths in the repo
+TEMPLATE_MAP = {
+    "kitchen": "examples/templates/kitchen_sink.odibi.yaml",
+    "full": "examples/templates/template_full.yaml",
+    "local": "examples/templates/simple_local.yaml",
+    "local-medallion": "examples/templates/simple_local.yaml",
+    "azure": "examples/templates/azure_spark.yaml",
+}
+
+
+def add_init_parser(subparsers):
+    """Add arguments for init-pipeline command."""
+    parser = subparsers.add_parser(
+        "init-pipeline",
+        aliases=["create", "init", "generate-project"],
+        help="Initialize a new Odibi project from a template",
+    )
+    parser.add_argument("name", help="Name of the project directory to create")
+    parser.add_argument(
+        "--template",
+        choices=list(TEMPLATE_MAP.keys()),
+        default=None,
+        help="Template to use (default: prompt user)",
+    )
+    # Add --force to overwrite existing directory
+    parser.add_argument(
+        "--force", action="store_true", help="Overwrite existing directory if it exists"
+    )
+
+
+def init_pipeline_command(args):
+    """Execute the init-pipeline command."""
+    project_name = args.name
+    template_name = args.template
+    force = args.force
+
+    # Interactive Prompt
+    if template_name is None:
+        print("\nSelect a project template:")
+        templates = list(TEMPLATE_MAP.keys())
+        for i, t in enumerate(templates):
+            print(f"  {i + 1}. {t}")
+
+        try:
+            choice = input(f"\nEnter number (default: 1 [{templates[0]}]): ").strip()
+            if not choice:
+                template_name = templates[0]
+            else:
+                idx = int(choice) - 1
+                if 0 <= idx < len(templates):
+                    template_name = templates[idx]
+                else:
+                    logger.error("Invalid selection.")
+                    return 1
+        except (ValueError, EOFError, KeyboardInterrupt):
+            # Fallback for non-interactive
+            template_name = "local"
+            logger.info(f"Using default template: {template_name}")
+
+    # 1. Determine Target Path
+    target_dir = Path(os.getcwd()) / project_name
+
+    if target_dir.exists():
+        if not force:
+            logger.error(f"Directory '{project_name}' already exists. Use --force to overwrite.")
+            return 1
+        else:
+            logger.warning(f"Overwriting existing directory '{project_name}'...")
+            shutil.rmtree(target_dir)
+
+    # 2. Find Template File
+    # Assuming we are running from within the installed package or repo
+    # Try to find the repo root relative to this file
+    # This file is in odibi/cli/init_pipeline.py
+    # Repo root is ../../../
+
+    current_file = Path(__file__).resolve()
+    repo_root = current_file.parent.parent.parent
+
+    template_rel_path = TEMPLATE_MAP[template_name]
+    source_path = repo_root / template_rel_path
+
+    if not source_path.exists():
+        # Fallback: check if we are installed and templates are packaged (not likely in this env but good practice)
+        # For now, just fail if not found in repo structure
+        logger.error(f"Template file not found at: {source_path}")
+        logger.error(
+            "Ensure you are running Odibi from the repository root or templates are correctly installed."
+        )
+        return 1
+
+    # 3. Create Project Structure
+    try:
+        os.makedirs(target_dir)
+
+        # Copy the template to odibi.yaml
+        target_file = target_dir / "odibi.yaml"
+        shutil.copy2(source_path, target_file)
+
+        # Create standard directories
+        os.makedirs(target_dir / "data", exist_ok=True)
+        os.makedirs(target_dir / "data/raw", exist_ok=True)
+        os.makedirs(target_dir / "logs", exist_ok=True)
+        os.makedirs(target_dir / ".github/workflows", exist_ok=True)
+
+        # Create sample data for local template
+        if template_name in ["local", "local-medallion"]:
+            with open(target_dir / "data/raw/sample_data.csv", "w") as f:
+                f.write("id,name,value,updated_at\n")
+                f.write("1,Item A,100,2023-01-01 10:00:00\n")
+                f.write("2,Item B,200,2023-01-01 11:00:00\n")
+                f.write("1,Item A (Old),90,2023-01-01 09:00:00\n")
+
+        # Create Dockerfile
+        dockerfile_content = """FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies if needed (e.g., for pyodbc)
+# RUN apt-get update && apt-get install -y unixodbc-dev
+
+# Install Odibi
+RUN pip install odibi[all]
+
+# Copy project files
+COPY . /app
+
+# Default command
+CMD ["odibi", "run", "odibi.yaml"]
+"""
+        with open(target_dir / "Dockerfile", "w") as f:
+            f.write(dockerfile_content)
+
+        # Create GitHub CI Workflow
+        ci_yaml_content = """name: Odibi CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.11"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install odibi[all] pytest
+
+    - name: Check Health (Doctor)
+      run: odibi doctor
+
+    - name: Validate Configuration
+      run: odibi validate odibi.yaml
+
+    - name: Run Unit Tests
+      run: odibi test
+
+    # Optional: Dry Run
+    # - name: Dry Run Pipeline
+    #   run: odibi run odibi.yaml --dry-run
+"""
+        with open(target_dir / ".github/workflows/ci.yaml", "w") as f:
+            f.write(ci_yaml_content)
+
+        # Create .dockerignore
+        with open(target_dir / ".dockerignore", "w") as f:
+            f.write("data/\nlogs/\n.git/\n__pycache__/\n*.pyc\n")
+
+        # Create .gitignore
+        with open(target_dir / ".gitignore", "w") as f:
+            f.write("data/\nlogs/\n__pycache__/\n*.pyc\n.odibi/\nstories/\n")
+
+        # Generate README.md
+        readme_content = f"""# {project_name}
+
+A data engineering project built with [Odibi](https://github.com/henryodibi11/Odibi).
+
+## Getting Started
+
+### Prerequisites
+- Python 3.9+
+- Odibi (`pip install odibi`)
+
+### Project Structure
+- `odibi.yaml`: Main pipeline configuration
+- `data/`: Local data storage
+- `stories/`: Execution reports (HTML)
+
+### Commands
+
+**1. Validate Configuration**
+```bash
+odibi validate odibi.yaml
+```
+
+**2. Check Health**
+```bash
+odibi doctor
+```
+
+**3. Run Pipeline**
+```bash
+odibi run odibi.yaml
+```
+
+**4. View Results**
+```bash
+odibi ui
+```
+
+## CI/CD
+A GitHub Actions workflow is included in `.github/workflows/ci.yaml` that validates the project on every push.
+"""
+        with open(target_dir / "README.md", "w") as f:
+            f.write(readme_content)
+
+        logger.info(f"Created new project '{project_name}' using '{template_name}' template.")
+        logger.info(f"Location: {target_dir}")
+        logger.info(f"Next step: cd {project_name} && odibi run odibi.yaml")
+
+        return 0
+
+    except Exception as e:
+        logger.error(f"Failed to create project: {str(e)}")
+        return 1