dataface 0.1.2__py3-none-any.whl

dataface/core/execute/cache_backend.py

@@ -0,0 +1,135 @@
+"""Pluggable cache-backend Protocol for query result storage.
+
+Stage: EXECUTE (Cache)
+Purpose: Define the abstract surface that any query-result cache backend must
+implement. The only current implementation is DuckDBCache. Future alternatives
+(Redis, S3, in-memory) can plug in by satisfying this Protocol.
+
+Cache keys are always (source_hash, query_hash, variables_hash):
+  - source_hash  — stable identity of the data source (connection string, project, etc.)
+  - query_hash   — hash of the SQL/query after static resolution
+  - variables_hash — hash of the variables that materially affect the result
+
+face_slug and query_name are METADATA, not key components. They are passed
+as keyword-only arguments to put/put_failure so backends can build internal
+indexes for {{ results.X }} cross-query references and snapshot history.
+
+Backend selection: today DuckDBCache is the only implementation. A future
+DFT_CACHE_BACKEND env-var hook can route to alternatives when a second
+backend exists. This Protocol is also the seam for a future
+`pip install dataface[duckdb]` extras-packaging effort.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class CachedQueryFailure(Exception):
+    """A query failure retrieved from the cache.
+
+    Raised instead of re-running a query that already failed within the TTL window.
+    Consumers can distinguish cached failures from fresh failures by catching this type.
+    """
+
+    error_class: str
+    error_message: str
+    traceback: str | None
+    failed_at: datetime
+
+
+@runtime_checkable
+class QueryResultCache(Protocol):
+    """Pluggable backend for persistent query-result caching.
+
+    All operations are keyed by (source_hash, query_hash, variables_hash).
+    face_slug and query_name are metadata passed on writes; they are NOT key
+    components. This keeps the key free of app-layer noise so two faces with
+    identical queries against identical sources share one cache entry.
+
+    Implementations may use any storage: DuckDB, Redis, S3, in-memory, etc.
+    The only current implementation is DuckDBCache.
+    """
+
+    def get(
+        self, source_hash: str, query_hash: str, variables_hash: str
+    ) -> list[dict[str, Any]] | None:
+        """Return cached rows, or None on miss."""
+        ...
+
+    def put(
+        self,
+        source_hash: str,
+        query_hash: str,
+        variables_hash: str,
+        data: list[dict[str, Any]],
+        *,
+        face_slug: str,
+        query_name: str,
+        variables: dict[str, Any],
+    ) -> None:
+        """Store rows under the given key; record face/query as metadata."""
+        ...
+
+    def get_failure(
+        self, source_hash: str, query_hash: str, variables_hash: str
+    ) -> CachedQueryFailure | None:
+        """Return a cached failure if one exists and is within TTL, else None."""
+        ...
+
+    def put_failure(
+        self,
+        source_hash: str,
+        query_hash: str,
+        variables_hash: str,
+        exception: Exception,
+        *,
+        face_slug: str,
+        query_name: str,
+    ) -> None:
+        """Store a query failure under the given key; record face/query as metadata."""
+        ...
+
+    def clear(self, source_hash: str, query_hash: str, variables_hash: str) -> None:
+        """Remove both success and failure entries for the given key."""
+        ...
+
+    def close(self) -> None:
+        """Release any held resources (connections, file handles, etc.)."""
+        ...
+
+    def supports_results_refs(self) -> bool:
+        """Return True if this backend can execute {{ results.X }} SQL queries.
+
+        DuckDBCache returns True (it has an embedded SQL engine).
+        Backends without a SQL engine return False; callers raise a clear error
+        ("{{ results.X }} requires a SQL-engine cache backend") in that case.
+        """
+        ...
+
+    def rewrite_results_refs(self, face_slug: str, sql: str) -> str:
+        """Rewrite {{ results.X }} references to backend-specific view/table names.
+
+        Args:
+            face_slug: Face identifier (for view name prefix)
+            sql: SQL containing {{ results.X }} references
+
+        Returns:
+            SQL with references substituted for backend-native names
+        """
+        ...
+
+    def execute_results_query(self, face_slug: str, sql: str) -> list[dict[str, Any]]:
+        """Execute a SQL query against cached result views.
+
+        Args:
+            face_slug: Face identifier
+            sql: SQL with {{ results.X }} already rewritten
+
+        Returns:
+            Result rows
+        """
+        ...

dataface/core/execute/cache_keys.py

@@ -0,0 +1,66 @@
+"""Cache key helpers — produce identity payloads for source-aware hashing.
+
+Stage: EXECUTE (Cache)
+Purpose: Reduce a source reference to its cache identity, stripping secrets.
+
+Secrets are NEVER included in cache hash inputs. Two users connecting to the
+same source name with different credentials must produce different hashes
+(so they don't share cache entries when their DB-side row-level access
+differs); but a password rotation alone must NOT invalidate the cache.
+
+`source_identity(source)` is the single transformation. The executor calls
+it (via `compute_source_hash`) after resolving named source references
+through the project's `face.sources` map.
+
+See also:
+    - dataface/core/execute/duckdb_cache.py — `compute_source_hash` consumer
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# Field names that must NEVER appear in cache hash inputs. Match by case-
+# insensitive exact name. Conservative: anything that looks secret-shaped
+# is dropped. Adding a new auth field elsewhere in the codebase? Add the
+# name here.
+SECRET_FIELDS: frozenset[str] = frozenset(
+    {
+        "password",
+        "passwd",
+        "secret",
+        "token",
+        "api_key",
+        "apikey",
+        "access_key",
+        "access_token",
+        "private_key",
+        "client_secret",
+        "auth",
+        "authorization",
+    }
+)
+
+
+def source_identity(
+    source: str | dict[str, Any] | None,
+) -> str | dict[str, Any] | None:
+    """Reduce a source reference to its identity for cache hashing.
+
+    String names and ``None`` pass through unchanged — they carry no
+    secrets and the caller hashes them as-is. Dict configs (from inline
+    YAML or resolved from `face.sources`) have every secret-shaped field
+    dropped, so passwords never reach cache table names or hash inputs.
+    Identity-bearing fields (user, host, project, dataset, keyfile, etc.)
+    are preserved so two users with the same source name but different
+    credentials get distinct cache entries.
+
+    Args:
+        source: ``str`` name reference, ``dict`` config, or ``None``.
+
+    Returns:
+        Same type, with secrets stripped from dicts.
+    """
+    if source is None or isinstance(source, str):
+        return source
+    return {k: v for k, v in source.items() if k.lower() not in SECRET_FIELDS}

dataface/core/execute/dbt_jinja.py

@@ -0,0 +1,21 @@
+"""Shared dbt Jinja detection patterns.
+
+Single source of truth for identifying SQL that contains dbt-specific
+Jinja calls (ref(), source()). Used by both DbtAdapter and SqlAdapter
+to route queries correctly.
+
+The patterns handle:
+  - Standard spacing: {{ ref( and {{ source(
+  - No spacing:       {{ref( and {{source(
+  - Whitespace trim:  {{- ref( and {{- source(
+"""
+
+import re
+
+_DBT_REF_RE = re.compile(r"\{\{-?\s*ref\(")
+_DBT_SOURCE_RE = re.compile(r"\{\{-?\s*source\(")
+
+
+def has_dbt_jinja(sql: str) -> bool:
+    """Return True if sql contains a dbt ref() or source() Jinja call."""
+    return bool(_DBT_REF_RE.search(sql) or _DBT_SOURCE_RE.search(sql))

dataface/core/execute/dialects/__init__.py

@@ -0,0 +1,121 @@
+"""SQL dialect registry for database-specific behavior.
+
+This module provides a registry of SQL dialects for different database backends.
+Use get_dialect() to obtain the appropriate dialect instance for a given
+database profile type.
+
+Example:
+    >>> from dataface.core.execute.dialects import get_dialect
+    >>> dialect = get_dialect('postgres')
+    >>> dialect.param(1)
+    '$1'
+    >>> dialect = get_dialect('mysql')
+    >>> dialect.param(1)
+    '%s'
+"""
+
+from dataface.core.execute.dialects.athena import AthenaDialect
+from dataface.core.execute.dialects.base import VALID_OPERATORS, SQLDialect
+from dataface.core.execute.dialects.bigquery import BigQueryDialect
+from dataface.core.execute.dialects.databricks import DatabricksDialect
+from dataface.core.execute.dialects.duckdb import DuckDBDialect
+from dataface.core.execute.dialects.mysql import MySQLDialect
+from dataface.core.execute.dialects.postgres import PostgresDialect
+from dataface.core.execute.dialects.redshift import RedshiftDialect
+from dataface.core.execute.dialects.snowflake import SnowflakeDialect
+from dataface.core.execute.dialects.sqlserver import SQLServerDialect
+
+# Singleton instances for each dialect
+_postgres_dialect = PostgresDialect()
+_duckdb_dialect = DuckDBDialect()
+_mysql_dialect = MySQLDialect()
+_snowflake_dialect = SnowflakeDialect()
+_bigquery_dialect = BigQueryDialect()
+_redshift_dialect = RedshiftDialect()
+_sqlserver_dialect = SQLServerDialect()
+_databricks_dialect = DatabricksDialect()
+_athena_dialect = AthenaDialect()
+
+# Registry mapping profile types to dialect instances
+DIALECTS: dict[str, SQLDialect] = {
+    # PostgreSQL (default)
+    "postgres": _postgres_dialect,
+    "postgresql": _postgres_dialect,
+    # DuckDB
+    "duckdb": _duckdb_dialect,
+    # MySQL / MariaDB
+    "mysql": _mysql_dialect,
+    "mariadb": _mysql_dialect,
+    # Snowflake
+    "snowflake": _snowflake_dialect,
+    # BigQuery
+    "bigquery": _bigquery_dialect,
+    # Redshift
+    "redshift": _redshift_dialect,
+    # SQL Server
+    "sqlserver": _sqlserver_dialect,
+    "mssql": _sqlserver_dialect,
+    # Databricks
+    "databricks": _databricks_dialect,
+    "spark": _databricks_dialect,
+    # Athena
+    "athena": _athena_dialect,
+    "presto": _athena_dialect,
+    "trino": _athena_dialect,
+}
+
+# Default dialect when profile type is unknown
+DEFAULT_DIALECT = _postgres_dialect
+
+
+def get_dialect(profile_type: str) -> SQLDialect:
+    """Get the appropriate SQL dialect for a database profile type.
+
+    Returns a singleton dialect instance for the given profile type.
+    Falls back to PostgreSQL dialect for unknown types.
+
+    Args:
+        profile_type: Database type string (e.g., 'postgres', 'mysql', 'snowflake')
+
+    Returns:
+        SQLDialect instance for the specified database type
+
+    Example:
+        >>> dialect = get_dialect('postgres')
+        >>> dialect.param(1)
+        '$1'
+        >>> dialect = get_dialect('unknown')  # Falls back to PostgreSQL
+        >>> dialect.name
+        'postgres'
+    """
+    return DIALECTS.get(profile_type.lower(), DEFAULT_DIALECT)
+
+
+def list_dialects() -> list[str]:
+    """List all supported dialect profile types.
+
+    Returns:
+        List of profile type strings that are supported.
+    """
+    return sorted(set(DIALECTS.keys()))
+
+
+__all__ = [
+    # Base class and constants
+    "SQLDialect",
+    "VALID_OPERATORS",
+    # Dialect implementations
+    "PostgresDialect",
+    "DuckDBDialect",
+    "MySQLDialect",
+    "SnowflakeDialect",
+    "BigQueryDialect",
+    "RedshiftDialect",
+    "SQLServerDialect",
+    "DatabricksDialect",
+    "AthenaDialect",
+    # Registry
+    "DIALECTS",
+    "get_dialect",
+    "list_dialects",
+]

dataface/core/execute/dialects/athena.py

@@ -0,0 +1,75 @@
+"""Athena dialect implementation.
+
+Athena uses:
+- ? for positional parameters
+- NO temp table support (raises NotImplementedError)
+"""
+
+from dataface.core.execute.dialects.base import SQLDialect
+
+
+class AthenaDialect(SQLDialect):
+    """Athena dialect with ? parameter style and no temp table support.
+
+    AWS Athena is based on Presto/Trino and does not support temporary tables.
+    Calling temp_table() will raise NotImplementedError.
+
+    Example:
+        >>> dialect = AthenaDialect()
+        >>> dialect.param()
+        '?'
+        >>> dialect.supports_temp_tables
+        False
+        >>> dialect.filter('category', '=', 1)
+        'category = ?'
+    """
+
+    name = "athena"
+
+    @property
+    def supports_temp_tables(self) -> bool:
+        """Athena does not support temporary tables.
+
+        Returns:
+            Always False for Athena.
+        """
+        return False
+
+    def param(self, index: int) -> str:
+        """Generate Athena parameter placeholder.
+
+        Athena uses ? for positional parameters.
+
+        Args:
+            index: 1-based parameter index (ignored for positional)
+
+        Returns:
+            Parameter placeholder '?'
+        """
+        return "?"
+
+    def temp_table(self, name: str, sql: str) -> str:
+        """Athena does not support temporary tables.
+
+        Args:
+            name: Name of the temporary table
+            sql: SQL query to populate the table
+
+        Raises:
+            NotImplementedError: Always, as Athena doesn't support temp tables
+        """
+        raise NotImplementedError(
+            "Athena does not support temporary tables. "
+            "Consider using CTEs or materializing to S3."
+        )
+
+    def drop_temp_table(self, name: str) -> str:
+        """Athena does not support temporary tables.
+
+        Args:
+            name: Name of the temporary table
+
+        Raises:
+            NotImplementedError: Always, as Athena doesn't support temp tables
+        """
+        raise NotImplementedError("Athena does not support temporary tables.")