getdango 0.0.1__py3-none-any.whl

dango/__init__.py

@@ -0,0 +1,12 @@
+"""
+Dango - Local-first data platform
+
+Laptop to cloud data stack in one weekend.
+"""
+
+__version__ = "0.0.1"
+__author__ = "Dango Team"
+__license__ = "Apache-2.0"
+
+# Package metadata
+__all__ = ["__version__", "__author__", "__license__"]

dango/cli/__init__.py

@@ -0,0 +1,5 @@
+"""
+Dango CLI module
+
+Provides command-line interface for Dango data platform.
+"""

dango/cli/db_helpers.py

@@ -0,0 +1,123 @@
+"""
+Database Helper Functions
+
+Utilities for matching tables to source configurations, used by db status and db clean commands.
+"""
+
+from typing import Dict, Set, Tuple
+from dango.config import DangoConfig
+from dango.ingestion.sources.registry import get_source_metadata
+
+
+def build_schema_table_mapping(config: DangoConfig) -> Tuple[Dict[str, Set[str]], Dict[str, str]]:
+    """
+    Build mapping of schemas to expected tables based on source configurations.
+
+    For multi-resource sources (Stripe, Shopify, etc.):
+      - Schema: raw_{source_name} (e.g., raw_stripe_test)
+      - Tables: endpoint names (e.g., charge, customer, subscription)
+
+    For single-resource sources (CSV, etc.):
+      - Schema: raw
+      - Tables: source names (e.g., orders)
+
+    Args:
+        config: Dango configuration with source definitions
+
+    Returns:
+        Tuple of:
+          - schema_to_tables: Dict[schema_name, Set[table_names]]
+          - source_to_schema: Dict[source_name, schema_name]
+    """
+    schema_to_tables = {}  # schema → set of table names
+    source_to_schema = {}  # source_name → schema_name (for staging lookup)
+
+    for source in config.sources.sources:
+        source_name = source.name.lower()
+
+        # Get source metadata to check if multi-resource
+        metadata = get_source_metadata(source.type.value)
+        is_multi_resource = metadata.get("multi_resource", False) if metadata else False
+
+        if is_multi_resource:
+            # Multi-resource source: one schema per source, tables are endpoint names
+            schema_name = f"raw_{source_name}"
+            source_to_schema[source_name] = schema_name
+
+            # Get source-specific config
+            source_config = getattr(source, source.type.value, None)
+            if source_config:
+                source_dict = source_config.model_dump() if hasattr(source_config, 'model_dump') else {}
+                endpoints = source_dict.get('endpoints') or source_dict.get('resources') or source_dict.get('tables')
+
+                if endpoints:
+                    if schema_name not in schema_to_tables:
+                        schema_to_tables[schema_name] = set()
+                    for endpoint in endpoints:
+                        schema_to_tables[schema_name].add(endpoint.lower())
+        else:
+            # Single-resource source: 'raw' schema, table name is source name
+            if 'raw' not in schema_to_tables:
+                schema_to_tables['raw'] = set()
+            schema_to_tables['raw'].add(source_name)
+            source_to_schema[source_name] = 'raw'
+
+    return schema_to_tables, source_to_schema
+
+
+def is_table_configured(
+    schema: str,
+    table: str,
+    schema_to_tables: Dict[str, Set[str]],
+    source_to_schema: Dict[str, str]
+) -> bool:
+    """
+    Check if a table is configured in sources.yml
+
+    Args:
+        schema: Table schema name
+        table: Table name
+        schema_to_tables: Mapping from schema to expected tables
+        source_to_schema: Mapping from source name to schema
+
+    Returns:
+        True if table is configured, False if orphaned
+    """
+    # Skip dlt internal tables (always considered configured)
+    if table.startswith('_dlt_'):
+        return True
+
+    # Raw tables: check schema-specific expected tables
+    if schema == 'raw' or schema.startswith('raw_'):
+        expected_in_schema = schema_to_tables.get(schema, set())
+        return table in expected_in_schema
+
+    # Staging tables: stg_{source_name}__{endpoint} or stg_{source_name}
+    elif schema == 'staging':
+        if table.startswith('stg_'):
+            # Try to match against source schemas
+            for source_name, raw_schema in source_to_schema.items():
+                # Check if staging table belongs to this source
+                if table.startswith(f"stg_{source_name}__") or table == f"stg_{source_name}":
+                    # Extract endpoint/table name
+                    if "__" in table:
+                        endpoint = table.split("__", 1)[1]
+                    else:
+                        endpoint = source_name
+
+                    # Check if this endpoint exists in the raw schema
+                    expected_in_raw = schema_to_tables.get(raw_schema, set())
+                    if endpoint in expected_in_raw or source_name in expected_in_raw:
+                        return True
+            return False
+        else:
+            # Other staging tables - assume configured
+            return True
+
+    # Intermediate and marts tables - always assume configured
+    # (these are custom models, not auto-generated)
+    elif schema in ('intermediate', 'marts'):
+        return True
+
+    # Unknown schema - assume not configured
+    return False

dango/cli/env_helpers.py

@@ -0,0 +1,336 @@
+"""
+.env File Helpers
+
+Utilities for creating, validating, and managing .env files for credentials.
+"""
+
+import os
+import platform
+import subprocess
+from pathlib import Path
+from typing import List, Dict, Set, Optional, Tuple
+from rich.console import Console
+
+console = Console()
+
+
+def create_env_template(
+    env_file: Path,
+    env_vars: List[Dict[str, str]],
+    backup: bool = True
+) -> None:
+    """
+    Create or update .env file with helpful templates for required variables.
+
+    Uses atomic write pattern to prevent data loss on failure.
+
+    Args:
+        env_file: Path to .env file
+        env_vars: List of env var configs with name, help, format, example
+        backup: Whether to backup existing .env before modifying
+    """
+    env_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Read existing content (or start with header for new file)
+    if env_file.exists():
+        original_content = env_file.read_text()
+        existing_lines = original_content.splitlines()
+    else:
+        original_content = ""
+        existing_lines = [
+            "# Dango Data Platform - Environment Variables",
+            "# Add your credentials below",
+            ""
+        ]
+
+    # Backup existing file if requested
+    if backup and env_file.exists():
+        backup_file = env_file.with_suffix('.env.backup')
+        backup_file.write_text(original_content)
+
+    # Parse existing vars
+    existing_vars = set()
+    for line in existing_lines:
+        if '=' in line and not line.strip().startswith('#'):
+            var_name = line.split('=')[0].strip()
+            existing_vars.add(var_name)
+
+    # Build new vars section (only if not already present)
+    new_content = []
+    for var_config in env_vars:
+        var_name = var_config.get('name')
+
+        if var_name not in existing_vars:
+            # Add section header with source information
+            source_name = var_config.get('source_name')
+            source_type = var_config.get('source_type')
+
+            if source_name and source_type:
+                new_content.append(f"\n# {var_config.get('display_name', var_name)}")
+                new_content.append(f"# For source: '{source_name}' ({source_type})")
+            else:
+                new_content.append(f"\n# {var_config.get('display_name', var_name)}")
+
+            # Add help text
+            if 'help' in var_config:
+                new_content.append(f"# {var_config['help']}")
+
+            # Add format hint
+            if 'format' in var_config:
+                new_content.append(f"# Format: {var_config['format']}")
+
+            # Add example
+            if 'example' in var_config:
+                new_content.append(f"# Example: {var_config['example']}")
+
+            # Add empty var line
+            new_content.append(f"{var_name}=")
+            new_content.append("")  # Blank line
+
+    # Atomic write: temp file + rename
+    if new_content or not env_file.exists():
+        temp_file = env_file.with_suffix('.env.tmp')
+        try:
+            # Write combined content to temp file
+            with open(temp_file, 'w') as f:
+                if existing_lines:
+                    f.write('\n'.join(existing_lines))
+                if new_content:
+                    f.write('\n'.join(new_content))
+
+            # Atomic rename
+            temp_file.replace(env_file)
+
+        except Exception as e:
+            # Clean up temp file and restore original if it existed
+            try:
+                if temp_file.exists():
+                    temp_file.unlink()
+            except:
+                pass
+            raise Exception(f"Failed to update .env file: {e}")
+
+
+def validate_env_file(
+    env_file: Path,
+    required_vars: List[str]
+) -> Tuple[bool, List[str]]:
+    """
+    Validate that .env file contains all required variables with non-empty values.
+
+    Args:
+        env_file: Path to .env file
+        required_vars: List of required variable names
+
+    Returns:
+        Tuple of (all_valid, missing_vars)
+    """
+    if not env_file.exists():
+        return False, required_vars
+
+    # Parse .env file
+    env_values = {}
+    for line in env_file.read_text().splitlines():
+        line = line.strip()
+        if line and not line.startswith('#') and '=' in line:
+            key, value = line.split('=', 1)
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            env_values[key] = value
+
+    # Check required vars
+    missing = []
+    for var in required_vars:
+        if var not in env_values or not env_values[var]:
+            missing.append(var)
+
+    return (len(missing) == 0, missing)
+
+
+def open_file_in_default_app(filepath: Path) -> bool:
+    """
+    Open file in OS default application (non-blocking).
+
+    Args:
+        filepath: Path to file to open
+
+    Returns:
+        True if opened successfully, False otherwise
+    """
+    try:
+        system = platform.system()
+
+        if system == "Darwin":  # macOS
+            subprocess.Popen(["open", str(filepath)])
+        elif system == "Windows":
+            subprocess.Popen(["start", str(filepath)], shell=True)
+        elif system == "Linux":
+            subprocess.Popen(["xdg-open", str(filepath)])
+        else:
+            return False
+
+        return True
+    except Exception:
+        return False
+
+
+def guide_env_setup(
+    env_file: Path,
+    required_vars: List[Dict[str, str]],
+    source_name: str,
+    setup_guide: List[str] = None
+) -> bool:
+    """
+    Guide user through .env setup with optional file opening and validation.
+
+    Args:
+        env_file: Path to .env file
+        required_vars: List of required var configs (with name, help, etc.)
+        source_name: Name of source being configured
+        setup_guide: Optional list of setup instructions to display
+
+    Returns:
+        True if validated successfully, False otherwise
+    """
+    from inquirer import prompt, Confirm
+    from inquirer.themes import GreenPassion
+
+    console.print(f"\n[bold cyan]📝 Credentials Setup[/bold cyan]")
+    console.print(f"File: [dim]{env_file.absolute()}[/dim]\n")
+
+    # Show detailed setup guide if provided
+    if setup_guide:
+        console.print("[bold]How to get your credentials:[/bold]")
+        for step in setup_guide:
+            console.print(f"  {step}")
+        console.print()
+
+    # Show required credentials
+    console.print("[bold]Required credentials:[/bold]")
+    for var_config in required_vars:
+        var_name = var_config.get('name')
+        help_text = var_config.get('help', '')
+        console.print(f"  • {var_name}")
+        if help_text:
+            console.print(f"    [dim]{help_text}[/dim]")
+
+    # Offer to open file
+    console.print()
+    questions = [
+        Confirm(
+            'open_file',
+            message="Would you like me to open .env in your editor?",
+            default=True
+        )
+    ]
+    answers = prompt(questions, theme=GreenPassion())
+
+    if answers and answers['open_file']:
+        if open_file_in_default_app(env_file):
+            console.print("[green]✅ Opened .env in your default editor[/green]")
+            console.print("\n[bold cyan]Next steps:[/bold cyan]")
+            console.print("  1. Find the section with empty values (e.g., STRIPE_API_KEY=)")
+            console.print("  2. Paste your credential after the = sign")
+            console.print("  3. [bold]SAVE THE FILE[/bold] (Cmd+S or Ctrl+S)")
+            console.print("  4. Return here and press Enter")
+        else:
+            console.print("[yellow]⚠️  Couldn't open automatically[/yellow]")
+            console.print(f"Please manually open: {env_file}")
+            console.print("\n[bold cyan]Then:[/bold cyan]")
+            console.print("  1. Find the section with empty values")
+            console.print("  2. Paste your credentials after the = sign")
+            console.print("  3. [bold]SAVE THE FILE[/bold]")
+
+    # Wait for user to finish
+    console.print("\n[bold yellow]⚠️  Press Enter AFTER you've saved the .env file[/bold yellow]")
+    input()
+
+    # Validate
+    var_names = [v.get('name') for v in required_vars]
+    is_valid, missing = validate_env_file(env_file, var_names)
+
+    return handle_validation_result(
+        is_valid, missing, env_file, source_name
+    )
+
+
+def handle_validation_result(
+    is_valid: bool,
+    missing_vars: List[str],
+    env_file: Path,
+    source_name: str
+) -> bool:
+    """
+    Handle validation results with retry logic and graceful failure.
+
+    Args:
+        is_valid: Whether all required vars are present
+        missing_vars: List of missing variable names
+        env_file: Path to .env file
+        source_name: Name of source being configured
+
+    Returns:
+        True if validation passed or user chose to skip
+    """
+    from inquirer import prompt, List as InquirerList
+    from inquirer.themes import GreenPassion
+
+    if is_valid:
+        console.print("\n[green]✅ All credentials validated successfully![/green]")
+        return True
+
+    # Validation failed
+    console.print("\n[yellow]⚠️  Credential Validation Failed[/yellow]")
+    console.print("\nMissing or empty variables:")
+    for var in missing_vars:
+        console.print(f"  • {var}")
+
+    console.print(f"\n[bold]Note:[/bold] Source '{source_name}' has been saved to sources.yml")
+    console.print("However, it won't work until you add valid credentials.\n")
+
+    # Offer retry
+    questions = [
+        InquirerList(
+            'action',
+            message="What would you like to do?",
+            choices=[
+                'Edit .env again and retry validation',
+                'Skip validation (I\'ll add credentials later)',
+                'Cancel source setup'
+            ],
+        )
+    ]
+    answers = prompt(questions, theme=GreenPassion())
+
+    if not answers:
+        return False
+
+    action = answers['action']
+
+    if action == 'Edit .env again and retry validation':
+        # Reopen file
+        if open_file_in_default_app(env_file):
+            console.print("[green]✅ Reopened .env[/green]")
+
+        console.print("\n[bold]Press Enter when ready to retry validation...[/bold]")
+        input()
+
+        # Retry validation
+        is_valid_retry, missing_retry = validate_env_file(env_file, missing_vars)
+        return handle_validation_result(
+            is_valid_retry, missing_retry, env_file, source_name
+        )
+
+    elif action == 'Skip validation (I\'ll add credentials later)':
+        console.print("\n[yellow]⚠️  Skipping validation[/yellow]")
+        console.print("\n[cyan]Next steps:[/cyan]")
+        console.print(f"  1. Edit .env and add missing credentials")
+        console.print(f"  2. Validate with: [bold]dango config validate[/bold]")
+        console.print(f"  3. Or try syncing: [bold]dango sync --source {source_name}[/bold]\n")
+        return True  # Allow wizard to complete
+
+    else:  # Cancel
+        console.print("\n[red]Source setup cancelled[/red]")
+        console.print(f"The source config was saved but is incomplete.")
+        console.print(f"To remove it, edit .dango/sources.yml\n")
+        return False