dataface 0.1.2__py3-none-any.whl

dataface/core/compile/sources.py

@@ -0,0 +1,518 @@
+"""Source detection utilities for database and dbt profiles.
+
+Stage: COMPILE
+Purpose: Centralized utilities for detecting database types and reading dbt profiles.
+
+Entry Points:
+    - detect_dbt_database_type(profiles_path, profile_name, target_name) -> dict
+    - detect_dbt_connections(project_path) -> list[dict]
+    - detect_database_type_from_registry(registry) -> dict
+    - detect_dbt_connection_string(cwd) -> tuple[str, str] | None
+    - DB_INFO_MAP: Database type information dictionary
+
+This module is the single canonical home for all database/source detection logic.
+Surfaces (playground, cloud, CLI) import from here rather than implementing their own.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+from urllib.parse import quote_plus
+
+if TYPE_CHECKING:
+    from dataface.core.execute.adapters.adapter_registry import AdapterRegistry
+
+logger = logging.getLogger(__name__)
+
+# Single source of truth for database-specific SQL syntax and introspection
+DB_INFO_MAP: dict[str, dict[str, str]] = {
+    "duckdb": {
+        "type": "duckdb",
+        "engine": "DuckDB",
+        "dialect": "DuckDB SQL",
+        "introspection": (
+            "List tables: `SHOW TABLES;` | "
+            "Describe table: `DESCRIBE table_name;` or `PRAGMA table_info('table_name');`"
+        ),
+        "notes": (
+            "DuckDB does NOT support `SHOW COLUMNS FROM` or `PRAGMA show_columns()`. "
+            "Use DESCRIBE or PRAGMA table_info instead."
+        ),
+    },
+    "postgresql": {
+        "type": "postgresql",
+        "engine": "PostgreSQL",
+        "dialect": "PostgreSQL SQL",
+        "introspection": (
+            "List tables: `SELECT table_name FROM information_schema.tables "
+            "WHERE table_schema = 'public';` | "
+            "Describe table: `SELECT column_name, data_type FROM information_schema.columns "
+            "WHERE table_name = 'table_name';`"
+        ),
+        "notes": "PostgreSQL supports full information_schema queries.",
+    },
+    "snowflake": {
+        "type": "snowflake",
+        "engine": "Snowflake",
+        "dialect": "Snowflake SQL",
+        "introspection": (
+            "List tables: `SHOW TABLES;` | "
+            "Describe table: `DESCRIBE TABLE table_name;`"
+        ),
+        "notes": "Snowflake has its own SHOW/DESCRIBE commands and information_schema.",
+    },
+    "bigquery": {
+        "type": "bigquery",
+        "engine": "BigQuery",
+        "dialect": "BigQuery SQL",
+        "introspection": (
+            "List tables: `SELECT table_name FROM project.dataset.INFORMATION_SCHEMA.TABLES;` | "
+            "Describe table: `SELECT column_name, data_type FROM "
+            "project.dataset.INFORMATION_SCHEMA.COLUMNS WHERE table_name = 'table_name';`"
+        ),
+        "notes": "BigQuery uses backtick-quoted identifiers for projects/datasets.",
+    },
+    "redshift": {
+        "type": "redshift",
+        "engine": "Amazon Redshift",
+        "dialect": "Redshift SQL",
+        "introspection": (
+            "List tables: `SELECT tablename FROM pg_tables WHERE schemaname = 'public';` | "
+            "Describe table: `SELECT column_name, data_type FROM information_schema.columns "
+            "WHERE table_name = 'table_name';`"
+        ),
+        "notes": "Redshift supports PostgreSQL-style information_schema.",
+    },
+    "mysql": {
+        "type": "mysql",
+        "engine": "MySQL",
+        "dialect": "MySQL SQL",
+        "introspection": (
+            "List tables: `SHOW TABLES;` | "
+            "Describe table: `DESCRIBE table_name;` or `SHOW COLUMNS FROM table_name;`"
+        ),
+        "notes": "MySQL supports SHOW commands and information_schema.",
+    },
+    "databricks": {
+        "type": "databricks",
+        "engine": "Databricks",
+        "dialect": "Databricks SQL (Spark SQL)",
+        "introspection": (
+            "List tables: `SHOW TABLES;` | "
+            "Describe table: `DESCRIBE TABLE table_name;`"
+        ),
+        "notes": "Databricks supports Spark SQL syntax with SHOW/DESCRIBE commands.",
+    },
+    "athena": {
+        "type": "athena",
+        "engine": "AWS Athena",
+        "dialect": "Athena SQL (Presto/Trino)",
+        "introspection": (
+            "List tables: `SHOW TABLES IN database_name;` | "
+            "Describe table: `DESCRIBE table_name;`"
+        ),
+        "notes": "Athena uses Presto/Trino syntax.",
+    },
+    "sqlserver": {
+        "type": "sqlserver",
+        "engine": "Microsoft SQL Server",
+        "dialect": "T-SQL",
+        "introspection": (
+            "List tables: `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES;` | "
+            "Describe table: `SELECT COLUMN_NAME, DATA_TYPE FROM "
+            "INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'table_name';`"
+        ),
+        "notes": "SQL Server supports information_schema and sp_help procedures.",
+    },
+}
+
+
+def get_database_info(db_type: str) -> dict[str, str] | None:
+    """Get database info for a given type.
+
+    Args:
+        db_type: Database type string (e.g., "duckdb", "postgres", "snowflake")
+
+    Returns:
+        Database info dict with type, engine, dialect, introspection, notes
+        Returns None if db_type is not recognized
+    """
+    normalized_type = db_type.lower().strip()
+    # Normalize common aliases
+    if normalized_type == "postgres":
+        normalized_type = "postgresql"
+    return DB_INFO_MAP.get(normalized_type)
+
+
+def detect_dbt_database_type(
+    profiles_path: Path,
+    profile_name: str,
+    target_name: str | None = None,
+    *,
+    fallback_type: str | None = None,
+) -> dict[str, Any] | None:
+    """Detect database type from dbt profiles.yml file.
+
+    Reads the profiles.yml configuration and extracts database type information
+    for the specified profile and target.
+
+    Args:
+        profiles_path: Path to profiles.yml file
+        profile_name: Profile name to read (e.g., "my_project")
+        target_name: Target name to read (e.g., "dev", "prod"). If None, uses
+                     the default target or first available.
+        fallback_type: If set and the profile is not found but the file exists,
+                       return info for this database type instead of None.
+
+    Returns:
+        Dict with database type info including:
+            - type: Database type (e.g., "duckdb", "postgresql")
+            - engine: Human-readable engine name
+            - dialect: SQL dialect name
+            - introspection: Introspection query examples
+            - notes: Important notes about SQL syntax
+            - description: Auto-generated description
+            - profile_name: The profile name used
+            - target_name: The target name used
+        Returns None if profile not found or file doesn't exist
+        (unless fallback_type is set and the file exists)
+
+    Example:
+        >>> info = detect_dbt_database_type(
+        ...     Path("profiles.yml"), "my_project", "dev"
+        ... )
+        >>> info["engine"]
+        'DuckDB'
+    """
+    if not profiles_path.exists():
+        return None
+
+    import yaml
+
+    try:
+        with open(profiles_path) as f:
+            profiles = yaml.safe_load(f)
+
+        if not profiles or profile_name not in profiles:
+            return _fallback_info(fallback_type) if fallback_type else None
+
+        profile = profiles[profile_name]
+        if not isinstance(profile, dict) or "outputs" not in profile:
+            return _fallback_info(fallback_type) if fallback_type else None
+
+        outputs = profile["outputs"]
+        if not outputs:
+            return _fallback_info(fallback_type) if fallback_type else None
+
+        # Use specified target, default target, or first available
+        actual_target = target_name
+        if actual_target is None or actual_target not in outputs:
+            # Try default target from profile
+            actual_target = profile.get("target")
+            if actual_target is None or actual_target not in outputs:
+                # Fall back to first available target
+                actual_target = next(iter(outputs.keys()))
+
+        target_config = outputs[actual_target]
+        db_type = target_config.get("type", "").lower()
+
+        db_info = get_database_info(db_type)
+        if db_info:
+            result = db_info.copy()
+            result["description"] = f"SQL queries execute via {db_info['engine']}"
+            result["profile_name"] = profile_name
+            result["target_name"] = actual_target
+            return result
+
+        # Unknown type - return basic info
+        return {
+            "type": db_type or "unknown",
+            "engine": db_type.title() if db_type else "Unknown",
+            "dialect": "SQL",
+            "introspection": "",
+            "notes": f"Unknown database type: {db_type}",
+            "description": f"SQL queries execute via unknown database type: {db_type}",
+            "profile_name": profile_name,
+            "target_name": actual_target,
+        }
+
+    except (OSError, yaml.YAMLError, AttributeError, KeyError, TypeError) as e:
+        logger.debug(f"Failed to read or parse profiles.yml: {e}")
+        return _fallback_info(fallback_type) if fallback_type else None
+
+
+def _fallback_info(fallback_type: str) -> dict[str, Any]:
+    """Build a fallback database info dict for a given type."""
+    db_info = get_database_info(fallback_type)
+    if db_info:
+        result = db_info.copy()
+        result["description"] = (
+            f"SQL queries execute via {db_info['engine']} (fallback)"
+        )
+        return result
+    return {
+        "type": fallback_type,
+        "engine": fallback_type.title(),
+        "dialect": "SQL",
+        "introspection": "",
+        "notes": f"Fallback database type: {fallback_type}",
+        "description": f"SQL queries execute via {fallback_type} (fallback)",
+    }
+
+
+def detect_dbt_connections(project_path: Path) -> list[dict[str, Any]]:
+    """Detect connections from dbt profiles.yml.
+
+    Searches for profiles.yml in the project directory or ~/.dbt and extracts
+    all connection configurations for all profiles and targets.
+
+    Args:
+        project_path: Path to the project directory
+
+    Returns:
+        List of connection configuration dicts, each containing:
+            - name: Connection name (format: "profile_name (target_name)")
+            - type: Database type
+            - host: Database host
+            - port: Database port
+            - database: Database name
+            - schema: Schema name
+            - username: Username
+            - account: Snowflake account (if applicable)
+            - project: BigQuery project (if applicable)
+
+    Example:
+        >>> connections = detect_dbt_connections(Path("/my/project"))
+        >>> connections[0]
+        {'name': 'my_project (dev)', 'type': 'duckdb', ...}
+    """
+    import yaml
+
+    # Look for profiles.yml in project or ~/.dbt
+    profiles_path = project_path / "profiles.yml"
+    if not profiles_path.exists():
+        profiles_path = Path.home() / ".dbt" / "profiles.yml"
+
+    if not profiles_path.exists():
+        return []
+
+    try:
+        profiles = yaml.safe_load(profiles_path.read_text())
+    except (OSError, yaml.YAMLError):
+        return []  # Can't read or invalid YAML - no connections to detect
+
+    if not profiles:
+        return []
+
+    connections: list[dict[str, Any]] = []
+    for profile_name, profile in profiles.items():
+        if not isinstance(profile, dict):
+            continue
+
+        for target_name, target in profile.get("outputs", {}).items():
+            if not isinstance(target, dict):
+                continue
+
+            connections.append(
+                {
+                    "name": f"{profile_name} ({target_name})",
+                    "type": target.get("type"),
+                    "host": target.get("host", target.get("server", "")),
+                    "port": target.get("port"),
+                    "database": target.get("database", target.get("dbname", "")),
+                    "schema": target.get("schema", ""),
+                    "username": target.get("user", target.get("username", "")),
+                    "account": target.get("account", ""),  # Snowflake
+                    "project": target.get("project", ""),  # BigQuery
+                }
+            )
+
+    return connections
+
+
+def detect_database_type_from_registry(
+    registry: AdapterRegistry,
+    *,
+    default_profile_name: str = "default",
+    default_target_name: str = "dev",
+) -> dict[str, str]:
+    """Detect database type information from an adapter registry.
+
+    Inspects registered SQL adapters (DbtAdapter, SqlAdapter) to determine
+    the database type by reading their dbt profiles.yml configuration.
+
+    Args:
+        registry: AdapterRegistry instance to inspect.
+        default_profile_name: Profile name to use when adapter doesn't specify one.
+        default_target_name: Target name to use when adapter doesn't specify one.
+
+    Returns:
+        Dict with database type information. Always returns a dict — never None.
+        Keys: type, engine, dialect, notes (and optionally profile_name, target_name).
+    """
+    from dataface.core.execute.adapters.dbt_adapter import DbtAdapter
+    from dataface.core.execute.adapters.sql_adapter import SqlAdapter
+
+    sql_adapters = registry.get_adapters_for_type("sql")
+
+    for adapter in sql_adapters:
+        if isinstance(adapter, DbtAdapter):
+            dbt_path = Path(adapter.dbt_project_path)
+            profiles_path = dbt_path / "profiles.yml"
+            profile_name = adapter.profile_name or default_profile_name
+            target_name = adapter.target_name or default_target_name
+
+            db_info = detect_dbt_database_type(
+                profiles_path, profile_name, target_name, fallback_type="duckdb"
+            )
+            if db_info:
+                db_info["profile_name"] = profile_name
+                db_info["target_name"] = target_name
+                return db_info
+
+            # DbtAdapter but can't determine type — default to DuckDB
+            info = DB_INFO_MAP["duckdb"].copy()
+            info["description"] = "SQL queries execute via DuckDB (default)"
+            return info
+
+        elif isinstance(adapter, SqlAdapter):
+            if adapter.dbt_project_path:
+                dbt_path = Path(adapter.dbt_project_path)
+                profiles_path = dbt_path / "profiles.yml"
+
+                db_info = detect_dbt_database_type(
+                    profiles_path,
+                    default_profile_name,
+                    default_target_name,
+                    fallback_type="duckdb",
+                )
+                if db_info:
+                    return db_info
+
+            # SqlAdapter without dbt project
+            return {
+                "type": "unknown",
+                "engine": "Unknown",
+                "description": "SQL adapter configured but database type unknown",
+                "dialect": "SQL",
+                "notes": "Database type could not be determined. Use generic SQL syntax.",
+            }
+
+    return {
+        "type": "none",
+        "engine": "None",
+        "description": "No SQL adapter available",
+        "dialect": "N/A",
+        "notes": "",
+    }
+
+
+def detect_dbt_connection_string(cwd: Path) -> tuple[str, str] | None:
+    """Auto-detect database connection from dbt project files.
+
+    Walks up from *cwd* to find ``dbt_project.yml``, reads the matching
+    ``profiles.yml``, and builds a connection string + dialect pair.
+
+    Args:
+        cwd: Directory to start searching from.
+
+    Returns:
+        ``(connection_string, dialect)`` or ``None`` if no dbt project found.
+    """
+    import yaml
+
+    # Walk up to find dbt_project.yml
+    search_path = cwd
+    dbt_project_path = None
+    for _ in range(10):
+        candidate = search_path / "dbt_project.yml"
+        if candidate.exists():
+            dbt_project_path = candidate
+            break
+        if search_path.parent == search_path:
+            break
+        search_path = search_path.parent
+
+    if not dbt_project_path:
+        return None
+
+    project_dir = dbt_project_path.parent
+
+    try:
+        with open(dbt_project_path) as f:
+            dbt_project = yaml.safe_load(f)
+        profile_name = dbt_project.get("profile", "default")
+    except (FileNotFoundError, yaml.YAMLError, TypeError):
+        return None
+
+    # Find profiles.yml
+    profiles_path = project_dir / "profiles.yml"
+    if not profiles_path.exists():
+        profiles_path = Path.home() / ".dbt" / "profiles.yml"
+    if not profiles_path.exists():
+        return None
+
+    try:
+        with open(profiles_path) as f:
+            profiles = yaml.safe_load(f)
+
+        profile = profiles.get(profile_name, {})
+        target_name = profile.get("target", "dev")
+        outputs = profile.get("outputs", {})
+        target = outputs.get(target_name, {})
+        db_type = target.get("type", "duckdb")
+
+        if db_type == "duckdb":
+            path = target.get("path", ":memory:")
+            if not path.startswith(":") and not Path(path).is_absolute():
+                path = str(project_dir / path)
+            return (path, "duckdb")
+        elif db_type == "postgres":
+            host = target.get("host", "localhost")
+            port = target.get("port", 5432)
+            user = quote_plus(target.get("user", "postgres"))
+            password = quote_plus(target.get("password", ""))
+            dbname = target.get("dbname", "postgres")
+            return (
+                f"postgresql://{user}:{password}@{host}:{port}/{dbname}",
+                "postgres",
+            )
+        elif db_type == "bigquery":
+            project = target.get("project", target.get("database", ""))
+            if project:
+                return (f"bigquery://{project}", "bigquery")
+        elif db_type == "snowflake":
+            account = target.get("account", "")
+            database = target.get("database", "")
+            db_schema = target.get("schema", "")
+            warehouse = target.get("warehouse", "")
+            if account:
+                conn = f"snowflake://{account}/{database}/{db_schema}"
+                if warehouse:
+                    conn += f"?warehouse={warehouse}"
+                return (conn, "snowflake")
+        elif db_type == "databricks":
+            host = target.get("host", "")
+            http_path = target.get("http_path", "")
+            token = target.get("token", "")
+            if host:
+                conn = f"databricks://{host}/{http_path}"
+                if token:
+                    conn += f"?token={token}"
+                return (conn, "databricks")
+    except (FileNotFoundError, yaml.YAMLError, TypeError, AttributeError):
+        return None
+
+    return None
+
+
+__all__ = [
+    "DB_INFO_MAP",
+    "detect_database_type_from_registry",
+    "detect_dbt_connection_string",
+    "detect_dbt_connections",
+    "detect_dbt_database_type",
+    "get_database_info",
+]

dataface/core/compile/sql_authoring_lint.py

@@ -0,0 +1,56 @@
+"""Compile-time authoring checks for SQL query strings."""
+
+from __future__ import annotations
+
+import sqlglot
+import sqlglot.errors
+
+from dataface.core.execute.sql_guard import sqlglot_dialect
+
+
+def _sql_parses(sql: str, dialect: str | None) -> bool:
+    """Return True if sqlglot.parse succeeds without error, False otherwise."""
+    try:
+        sqlglot.parse(sql, read=sqlglot_dialect(dialect))
+    except (sqlglot.errors.ParseError, sqlglot.errors.TokenError):
+        return False
+    return True
+
+
+def has_literal_escaped_newlines(sql: str, *, dialect: str | None) -> bool:
+    """Return True if sql contains literal \\n that is the cause of a parse failure.
+
+    Three-step parse-replace-parse heuristic:
+    1. If '\\n' not in sql → False (fast path, no parse needed)
+    2. If sqlglot.parse(sql) succeeds → False (\\n is inside a SQL string literal
+       or otherwise accepted — step 1 guards the cost of this parse)
+    3. If sqlglot.parse(sql.replace('\\n', newline)) succeeds → True
+       (the replacement fixed the failure: author used single-quoted YAML instead of
+       a block scalar, so \\n was preserved literally instead of becoming a real newline)
+    4. Else → False (genuine syntax/Jinja issue unrelated to \\n; step 4 is the
+       absence of step 3)
+
+    Note: SQL with Jinja expressions ({{ ref('orders') }}) bypasses detection — both
+    step 2 and step 3 fail to parse Jinja skeletons, so the function returns False
+    and the existing UnparseableSqlError path handles it at runtime. This is intentional.
+
+    Args:
+        sql: Raw SQL string after YAML parsing.
+        dialect: Dataface dialect name (e.g. "duckdb", "sqlserver"), or None for the default.
+            Normalised to the sqlglot equivalent via sqlglot_dialect().
+
+    Returns:
+        True if the literal-\\n authoring error is detected, False otherwise.
+    """
+    # Step 1: fast exit — no literal \n present, skip all parsing.
+    if "\\n" not in sql:
+        return False
+
+    # Step 2: parses fine as-is — \n is inside a SQL string literal or similar.
+    if _sql_parses(sql, dialect):
+        return False
+
+    # Step 3: parses after replacing literal \n with a real newline — authoring mistake.
+    # Step 4 is the implicit False when step 3 also fails (genuine syntax/Jinja issue).
+    # Author wrote sql: 'SELECT\n...' in YAML single-quotes instead of sql: | block scalar.
+    return _sql_parses(sql.replace("\\n", "\n"), dialect)