dataface 0.1.2__py3-none-any.whl

dataface/core/errors/structured.py

@@ -0,0 +1,92 @@
+"""StructuredError wire shape and build_structured helper."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from dataface.core.errors.registry import ErrorCode
+
+
+class NextCommand(BaseModel):
+    label: str
+    command: str
+
+
+class StructuredError(BaseModel):
+    code: str
+    message: str
+    domain: str
+    fields: dict[str, Any] = {}
+    file: str | None = None
+    line: int | None = None
+    column: int | None = None
+    field_path: str | None = None
+    hint: str | None = None
+    doc_url: str
+    docs_topic: str | None = None
+    next_commands: list[NextCommand] = []
+    # Nested when __cause__ is itself a DatafaceError (preserves code/domain/
+    # doc_url through the chain); a bare string when __cause__ is a foreign
+    # exception type. None when there is no cause.
+    cause: StructuredError | str | None = None
+
+
+def build_structured(exc: Any, *, file: str | None = None) -> StructuredError:
+    """Build a StructuredError from any DatafaceError subclass.
+
+    The exc must have a populated .code attribute (an ErrorCode instance).
+    Raises RuntimeError if .code is None — that path should never happen once
+    every subclass __init__ stamps the legacy fallback code; raise (not
+    assert) so the guard survives `python -O`.
+    """
+    ec = getattr(exc, "code", None)
+    if not isinstance(ec, ErrorCode):
+        raise RuntimeError(
+            f"build_structured called on {type(exc).__name__!r} whose .code is "
+            f"{type(ec).__name__!r}, not an ErrorCode. Every DatafaceError "
+            "subclass must stamp a code at construction."
+        )
+
+    fields: dict[str, Any] = getattr(exc, "fields", {})
+
+    hint: str | None = None
+    if ec.hint_generator is not None:
+        hint = ec.hint_generator(**fields)
+
+    next_commands: list[NextCommand] = []
+    if file:
+        next_commands.append(
+            NextCommand(label="Validate", command=f"dft validate {file}")
+        )
+
+    cause: StructuredError | str | None = None
+    raw_cause = exc.__cause__
+    if raw_cause is not None:
+        # Recurse for our own DatafaceError chain so structured consumers
+        # keep code/domain/doc_url; flatten foreign exceptions to a string.
+        # Use isinstance(ErrorCode) — duck-typing on `.code is not None`
+        # would falsely recurse into foreign exceptions like
+        # urllib.error.HTTPError that carry a non-ErrorCode `.code` attr.
+        raw_code = getattr(raw_cause, "code", None)
+        if isinstance(raw_code, ErrorCode):
+            cause = build_structured(raw_cause)
+        else:
+            cause = str(raw_cause)
+
+    return StructuredError(
+        code=ec.code,
+        message=str(exc),
+        domain=ec.domain,
+        fields=fields,
+        file=file,
+        line=getattr(exc, "line", None),
+        column=getattr(exc, "column", None),
+        field_path=getattr(exc, "field_path", None),
+        hint=hint,
+        doc_url=ec.doc_url,
+        docs_topic=ec.docs_topic,
+        next_commands=next_commands,
+        cause=cause,
+    )

dataface/core/execute/__init__.py

@@ -0,0 +1,91 @@
+"""Query execution module.
+
+Stage: EXECUTE (Service)
+Purpose: Execute queries for compiled datafaces.
+
+This is a SERVICE module, not a sequential stage. It's called lazily
+by the render module when charts need data.
+
+Entry Points:
+    - Executor: Main executor class
+    - Executor.execute_query(query_name, variables) -> List[Dict]
+    - Executor.execute_chart(chart, variables) -> List[Dict]
+
+Inputs:
+    - Face: Face with compiled queries
+    - Query name and variables
+
+Outputs:
+    - List[Dict[str, Any]]: Query results (list of row dictionaries)
+
+Dependencies:
+    - dataface.compile (for Face, AnyQuery types)
+
+This module can import from compile/ (needs types) but should NOT
+import from render/ (render imports us, not vice versa).
+"""
+
+from dataface.core.execute.batch import (
+    TEMP_TABLE_PREFIX,
+    BatchStatement,
+    CrossSourceReferenceError,
+    DependencyGraph,
+    TempTableNotSupportedError,
+    build_dependency_graph,
+    collect_face_queries,
+    create_batch_execution_plan,
+    extract_query_refs,
+    generate_batch_sql,
+    group_by_source,
+    topological_sort,
+    validate_cross_source_references,
+)
+from dataface.core.execute.cache_backend import CachedQueryFailure, QueryResultCache
+from dataface.core.execute.cache_keys import SECRET_FIELDS, source_identity
+from dataface.core.execute.duckdb_cache import (
+    DuckDBCache,
+    compute_query_hash,
+    compute_source_hash,
+    compute_variables_hash,
+)
+from dataface.core.execute.errors import (
+    AdapterError,
+    ExecutionError,
+    QueryError,
+    QueryTimeoutError,
+)
+from dataface.core.execute.executor import Executor
+
+__all__ = [
+    # Executor
+    "Executor",
+    # Cache Protocol
+    "QueryResultCache",
+    # DuckDB Cache
+    "CachedQueryFailure",
+    "DuckDBCache",
+    "SECRET_FIELDS",
+    "compute_query_hash",
+    "compute_source_hash",
+    "compute_variables_hash",
+    "source_identity",
+    # Errors
+    "ExecutionError",
+    "QueryError",
+    "QueryTimeoutError",
+    "AdapterError",
+    "CrossSourceReferenceError",
+    "TempTableNotSupportedError",
+    # Batch execution
+    "BatchStatement",
+    "DependencyGraph",
+    "TEMP_TABLE_PREFIX",
+    "build_dependency_graph",
+    "collect_face_queries",
+    "create_batch_execution_plan",
+    "extract_query_refs",
+    "generate_batch_sql",
+    "group_by_source",
+    "topological_sort",
+    "validate_cross_source_references",
+]

dataface/core/execute/adapters/__init__.py

@@ -0,0 +1,49 @@
+"""Data source adapters.
+
+Stage: EXECUTE
+Purpose: Provide adapters for different data sources.
+
+Available Adapters:
+    - SqlAdapter: Raw SQL queries
+    - CsvAdapter: CSV file queries
+    - HttpAdapter: REST API queries
+    - DbtAdapter: dbt-integrated SQL queries
+    - MetricFlowAdapter: dbt Semantic Layer queries
+"""
+
+from dataface.core.execute.adapters.adapter_registry import (
+    LOCAL_AUTHORING_REGISTRY_KWARGS,
+    AdapterRegistry,
+    build_adapter_registry,
+)
+from dataface.core.execute.adapters.base import (
+    BaseAdapter,
+    QueryResult,
+    handle_adapter_error,
+)
+from dataface.core.execute.adapters.csv_adapter import CsvAdapter
+from dataface.core.execute.adapters.dbt_adapter import DbtAdapter
+from dataface.core.execute.adapters.http_adapter import HttpAdapter
+from dataface.core.execute.adapters.metricflow_adapter import MetricFlowAdapter
+from dataface.core.execute.adapters.schema_resolver_adapter import SchemaResolverAdapter
+from dataface.core.execute.adapters.sql_adapter import SqlAdapter
+from dataface.core.execute.adapters.values_adapter import ValuesAdapter
+
+__all__ = [
+    # Base
+    "BaseAdapter",
+    "QueryResult",
+    "handle_adapter_error",
+    # Registry
+    "AdapterRegistry",
+    "LOCAL_AUTHORING_REGISTRY_KWARGS",
+    "build_adapter_registry",
+    # Adapters
+    "SqlAdapter",
+    "CsvAdapter",
+    "HttpAdapter",
+    "DbtAdapter",
+    "MetricFlowAdapter",
+    "ValuesAdapter",
+    "SchemaResolverAdapter",
+]

dataface/core/execute/adapters/adapter_registry.py

@@ -0,0 +1,400 @@
+"""Registry for query adapters.
+
+Stage: EXECUTE
+Purpose: Manage and select appropriate adapters for query execution.
+
+The AdapterRegistry maintains a list of available adapters and uses
+type-based routing via the unified query interface.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from dataface.core.compile.errors import DatafaceError
+from dataface.core.compile.models.face.compiled import VariableValues
+from dataface.core.compile.models.query.compiled import AnyQuery
+from dataface.core.errors.codes_execute import (
+    DF_EXECUTE_NO_DEFAULT_SOURCE,
+    DF_EXECUTE_SOURCE_NOT_FOUND,
+    DF_EXECUTE_SOURCE_NOT_FOUND_EMPTY,
+)
+from dataface.core.execute.adapters.base import BaseAdapter, QueryParams, QueryResult
+from dataface.core.execute.source_registry import SourceRegistry
+from dataface.core.execute.source_resolver import DbtContext, DefaultSourceResolver
+
+if TYPE_CHECKING:
+    from dataface.core.compile.models.face.compiled import Face
+    from dataface.core.execute.source_resolver import SourceResolver
+
+# Approved local-dev surfaces that need non-exclusive DuckDB locks AND SQL
+# read_csv/read_json_auto (see m3-duckdb-concurrency-and-file-sources initiative).
+# Grep this symbol before adding allow_external_access_in_readonly=True elsewhere.
+LOCAL_AUTHORING_REGISTRY_KWARGS: dict[str, Any] = {
+    "read_only": True,
+    "allow_external_access_in_readonly": True,
+    "duckdb_config": {"enable_external_access": True},
+}
+
+
+def build_adapter_registry(
+    project_root: Path,
+    *,
+    read_only: bool = True,
+    allowed_types: set[str] | None = None,
+    dbt_project_path: Path | None = None,
+    connection_string: str | None = None,
+    profile_type: str = "duckdb",
+    target: str | None = None,
+    duckdb_config: dict[str, Any] | None = None,
+    resolver: SourceResolver | None = None,
+    allow_external_access_in_readonly: bool = False,
+) -> AdapterRegistry:
+    """Build an AdapterRegistry with standard adapters.
+
+    This is the single canonical way to create an adapter registry.
+    All entry points should use this instead of hand-wiring adapters.
+
+    Args:
+        project_root: Root directory for resolving relative file paths.
+        read_only: Whether SQL adapters should be read-only (default True).
+        allowed_types: If set, only register adapters whose supported_types
+            intersect with this set. None means register all.
+        dbt_project_path: Explicit dbt project path. If None, auto-discovers
+            by looking for dbt_project.yml in/above project_root.
+        connection_string: Override SQL connection string.
+        profile_type: SQL dialect (default "duckdb").
+        target: dbt target name override.
+        duckdb_config: Optional DuckDB config dict passed to the default
+            connection. When read_only=True (the default) and
+            allow_external_access_in_readonly=False, enable_external_access is
+            hard-forced to False regardless of what this dict contains. To combine
+            read_only=True with external file access (read_csv, httpfs, etc.), set
+            both allow_external_access_in_readonly=True AND
+            duckdb_config={"enable_external_access": True}. The default path
+            (read_only=False) also allows external access without the flag.
+        resolver: Source resolver instance. Defaults to DefaultSourceResolver when
+            None. Pass an AllowlistedSourceResolver for deployed-mode enforcement.
+        allow_external_access_in_readonly: Security opt-in; passed through to
+            SqlAdapter. See SqlAdapter docstring for full semantics. Default False
+            preserves existing behavior. Approved callsites passing True are listed
+            on LOCAL_AUTHORING_REGISTRY_KWARGS in this module (playground, MCP,
+            chat, serve).
+    """
+    from dataface.core.execute.adapters.csv_adapter import CsvAdapter
+    from dataface.core.execute.adapters.dbt_adapter import DbtAdapter
+    from dataface.core.execute.adapters.http_adapter import HttpAdapter
+    from dataface.core.execute.adapters.metricflow_adapter import MetricFlowAdapter
+    from dataface.core.execute.adapters.sql_adapter import SqlAdapter
+    from dataface.core.execute.adapters.values_adapter import ValuesAdapter
+
+    registry = AdapterRegistry(project_root=project_root, resolver=resolver)
+
+    def _should_register(types: set[str]) -> bool:
+        return allowed_types is None or bool(types & allowed_types)
+
+    # Auto-discover dbt project if not explicitly provided
+    resolved_dbt_path = dbt_project_path
+    if resolved_dbt_path is None:
+        candidate = project_root.resolve()
+        while True:
+            if (candidate / "dbt_project.yml").exists():
+                resolved_dbt_path = candidate
+                break
+            parent = candidate.parent
+            if parent == candidate:
+                break
+            candidate = parent
+
+    # DbtAdapter first (handles SQL queries with dbt features, takes priority)
+    if resolved_dbt_path is not None and _should_register({"sql"}):
+        registry.register(
+            DbtAdapter(
+                dbt_project_path=resolved_dbt_path,
+                target_name=target,
+            )
+        )
+
+    # SqlAdapter (fallback for plain SQL)
+    if _should_register({"sql"}):
+        registry.register(
+            SqlAdapter(
+                dbt_project_path=str(resolved_dbt_path) if resolved_dbt_path else None,
+                connection_string=connection_string,
+                project_root=project_root,
+                profile_type=profile_type,
+                read_only=read_only,
+                duckdb_config=duckdb_config,
+                allow_external_access_in_readonly=allow_external_access_in_readonly,
+            )
+        )
+
+    # Lazy adapters — zero cost until actually called
+    if _should_register({"metricflow"}):
+        registry.register(MetricFlowAdapter())
+    if _should_register({"http"}):
+        registry.register(HttpAdapter())
+
+    # File-based adapters
+    if _should_register({"csv"}):
+        registry.register(CsvAdapter(project_root=project_root))
+    if _should_register({"values"}):
+        registry.register(ValuesAdapter())
+
+    if _should_register({"schema_resolver"}):
+        from dataface.core.execute.adapters.schema_resolver_adapter import (
+            SchemaResolverAdapter,
+        )
+
+        registry.register(SchemaResolverAdapter(registry))
+
+    return registry
+
+
+class AdapterRegistry:
+    """Registry for managing and selecting query adapters.
+
+    The registry uses type-based routing from the unified query interface.
+    Adapters declare their supported types via the `supported_types` property,
+    and the registry routes queries to the appropriate adapter.
+
+    Priority order:
+    1. First registered adapter that supports the query type wins
+    2. dbt adapter is registered first for SQL (with dbt features)
+    3. Fallback SQL adapter for plain SQL without dbt
+
+    Example:
+        >>> from dataface.core.execute.adapters import build_adapter_registry
+        >>> registry = build_adapter_registry(Path("."))
+        >>> result = registry.execute(SqlQuery(sql="SELECT 1"))
+    """
+
+    def __init__(
+        self,
+        project_root: Path | None = None,
+        resolver: SourceResolver | None = None,
+    ) -> None:
+        """Initialize adapter registry.
+
+        Use ``build_adapter_registry()`` to get a fully configured registry.
+
+        Args:
+            project_root: Root directory for resolving relative file paths.
+            resolver: Source resolver instance. Defaults to DefaultSourceResolver.
+        """
+        self._adapters: list[BaseAdapter] = []
+        self._type_index: dict[str, list[BaseAdapter]] = {}
+        self._project_root = project_root
+        self._sources = SourceRegistry(project_root=project_root)
+        self._resolver: SourceResolver = (
+            resolver if resolver is not None else DefaultSourceResolver()
+        )
+
+    @property
+    def project_root(self) -> Path | None:
+        """Return the registry's configured project root, if any."""
+        return self._project_root
+
+    def register(self, adapter: BaseAdapter) -> None:
+        """Register an adapter.
+
+        Adapters are indexed by their supported types for fast lookup.
+
+        Args:
+            adapter: Adapter instance to register
+        """
+        self._adapters.append(adapter)
+
+        # Update type index
+        for query_type in adapter.supported_types:
+            if query_type not in self._type_index:
+                self._type_index[query_type] = []
+            self._type_index[query_type].append(adapter)
+
+    def close(self) -> None:
+        """Close adapters that hold external resources (e.g. DuckDB file handles)."""
+        for adapter in self._adapters:
+            close = getattr(adapter, "close", None)
+            if close is not None:
+                close()
+
+    def get_adapters_for_type(self, query_type: str) -> list[BaseAdapter]:
+        """Get all adapters that support a given query type.
+
+        Args:
+            query_type: Query type string (e.g., "sql", "csv")
+
+        Returns:
+            List of adapters supporting this type (in registration order)
+        """
+        return self._type_index.get(query_type, [])
+
+    def get_adapter(self, query: AnyQuery) -> BaseAdapter | None:
+        """Get an adapter that can execute the given query.
+
+        Uses type-based routing for fast lookup, then checks additional
+        eligibility via can_execute().
+
+        Args:
+            query: AnyQuery object
+
+        Returns:
+            Adapter instance or None if no adapter can execute the query
+        """
+        # Fast path: lookup by type
+        candidates = self.get_adapters_for_type(query.query_type)
+
+        for adapter in candidates:
+            if adapter.can_execute(query):
+                return adapter
+
+        # Fallback: check all adapters (for adapters with custom can_execute)
+        for adapter in self._adapters:
+            if adapter not in candidates and adapter.can_execute(query):
+                return adapter
+
+        return None
+
+    def register_source(self, name: str, config: dict[str, Any]) -> None:
+        """Register a synthetic source. Idempotent: first registration wins."""
+        self._sources.register(name, config)
+
+    def _derive_dbt_context(self) -> DbtContext | None:
+        from dataface.core.execute.adapters.dbt_adapter import DbtAdapter
+
+        for adapter in self._adapters:
+            if isinstance(adapter, DbtAdapter):
+                return DbtContext()
+        return None
+
+    def execute(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None = None,
+        params: QueryParams = None,
+        *,
+        face: Face | None = None,
+        dbt_context: DbtContext | None = None,
+    ) -> QueryResult:
+        """Execute a query using the appropriate adapter.
+
+        Args:
+            query: AnyQuery object
+            variables: Variable values for query resolution
+            params: Optional pre-computed parameter values for parameterized execution.
+                When provided, the adapter should use these params directly instead
+                of re-processing variables. This is used by batch execution where
+                parameterization happens before adapter execution.
+            face: Compiled face; used to extract face-level sources for resolution.
+            dbt_context: Explicit dbt context override. When None, derived from
+                registered adapters.
+
+        Returns:
+            QueryResult with data or error
+        """
+        adapter = self.get_adapter(query)
+        if not adapter:
+            return QueryResult(
+                data=[],
+                error=f"No adapter found for query type: {query.query_type}",
+            )
+
+        authored = getattr(query, "source", None)
+        face_sources: dict[str, dict[str, Any]] = (
+            face.sources if face is not None else {}
+        )
+        project_sources = self._sources.project_sources_config()
+        if dbt_context is None:
+            dbt_context = self._derive_dbt_context()
+
+        try:
+            source_config = self._resolver.resolve(
+                authored=authored,
+                face_sources=face_sources,
+                project_sources=project_sources,
+                dbt_context=dbt_context,
+            )
+        except DatafaceError as exc:
+            return QueryResult(data=[], error=str(exc))
+
+        return adapter.execute(query, variables, params, source_config=source_config)
+
+    @property
+    def adapters(self) -> list[BaseAdapter]:
+        """Get all registered adapters."""
+        return list(self._adapters)
+
+    @property
+    def supported_types(self) -> set[str]:
+        """Get all query types supported by registered adapters.
+
+        Returns:
+            Set of query type strings
+        """
+        return set(self._type_index.keys())
+
+    def list_sql_sources(self) -> list[dict[str, Any]]:
+        """Return configured SQL sources from the project source registry."""
+        sources: dict[str, dict[str, Any]] = {}
+        for name, config in self._sources.all().items():
+            source_type = str(config.get("type", "unknown"))
+            source_info: dict[str, Any] = {"name": name, "type": source_type}
+            db_path = config.get("path")
+            if db_path:
+                source_info["path"] = str(db_path)
+            if source_type == "duckdb" and str(db_path) == ":memory:":
+                source_info["in_memory"] = True
+            sources[name] = source_info
+        return [sources[name] for name in sorted(sources)]
+
+    def resolve_source_config(self, source: str | None = None) -> dict[str, Any]:
+        """Resolve a full source_config dict for use with InspectConnection.
+
+        Returns the warehouse-specific dict (type + credentials).
+
+        Args:
+            source: Named source profile. When None, falls back to the
+                registry's default SqlAdapter — but only if it's a DuckDB
+                adapter (the connection_string is :memory: or a .duckdb path).
+                For non-DuckDB SqlAdapters, raises DatafaceError because
+                reconstructing config from (connection_string, profile_type)
+                drops credentials, host, port, keyfile_json, etc.
+
+        Returns:
+            source_config dict with at least a 'type' key.
+
+        Raises:
+            DatafaceError: DF-EXECUTE-NO-DEFAULT-SOURCE if source is None and
+                the default adapter is non-DuckDB; DF-EXECUTE-SOURCE-NOT-FOUND
+                if the named source is missing but others are configured;
+                DF-EXECUTE-SOURCE-NOT-FOUND-EMPTY if no sources are configured.
+        """
+        from dataface.core.execute.adapters.sql_adapter import SqlAdapter
+
+        if source is None:
+            # No-source fallback only works for DuckDB SqlAdapters — they have
+            # no credentials, host, port, keyfile_json to lose.
+            for adapter in self._adapters:
+                if isinstance(adapter, SqlAdapter) and adapter.profile_type == "duckdb":
+                    # Inline DuckDB source config: matches what source_config_from_url
+                    # returns for the duckdb branch ({"type": "duckdb", "path": ...}).
+                    # Handles all DuckDB URL shapes: file paths, :memory:, md: (MotherDuck).
+                    # The loop already filters to profile_type=="duckdb" so no dialect
+                    # ambiguity; the inline form avoids importing from the private package.
+                    return {
+                        "type": "duckdb",
+                        "path": adapter.connection_string,
+                    }
+            raise DatafaceError.from_code(DF_EXECUTE_NO_DEFAULT_SOURCE)
+
+        config = self._sources.get(source)
+        if config and config.get("type"):
+            return config
+
+        available = ", ".join(item["name"] for item in self.list_sql_sources())
+        if available:
+            raise DatafaceError.from_code(
+                DF_EXECUTE_SOURCE_NOT_FOUND, source=source, available=available
+            )
+        raise DatafaceError.from_code(DF_EXECUTE_SOURCE_NOT_FOUND_EMPTY, source=source)