dataface 0.1.2__py3-none-any.whl

dataface/core/execute/adapters/dbt_utils.py

@@ -0,0 +1,150 @@
+"""Shared dbt utilities for SQL adapters."""
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from dataface.core.execute.adapters.base import ResolvedRelation, SchemaStatus
+
+DBT_PROJECT_DB_NAMES = ["sample.duckdb", "dataface_examples.duckdb", "dev.duckdb"]
+logger = logging.getLogger(__name__)
+
+
+def load_dbt_manifest(project_path: Path) -> dict[str, Any] | None:
+    """Load a dbt manifest from target/ or the committed snapshot fallback."""
+    manifest_path = project_path / "target" / "manifest.json"
+    snapshot_path = project_path / "manifest.snapshot.json"
+
+    manifest_to_load = None
+    if manifest_path.exists():
+        manifest_to_load = manifest_path
+    elif snapshot_path.exists():
+        manifest_to_load = snapshot_path
+
+    if manifest_to_load is None:
+        return None
+
+    try:
+        with open(manifest_to_load) as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError) as e:
+        logger.debug("Failed to load dbt manifest from %s: %s", manifest_to_load, e)
+        return None
+
+
+def classify_schema_status(
+    dev_schema: str,
+    prod_schema: str | None,
+    *,
+    prod_manifest_available: bool = True,
+) -> SchemaStatus:
+    """Compare dev vs prod schema for a resolved dbt relation."""
+    if not prod_manifest_available:
+        return "unknown"
+    if prod_schema is None:
+        return "changed"
+    if dev_schema.lower() == prod_schema.lower():
+        return "unchanged"
+    return "changed"
+
+
+def _find_model_node(
+    manifest: dict[str, Any] | None, model_name: str
+) -> dict[str, Any] | None:
+    if not manifest or "nodes" not in manifest:
+        return None
+    for node in manifest["nodes"].values():
+        if node.get("resource_type") == "model" and node.get("name") == model_name:
+            return node
+    return None
+
+
+def _find_source_node(
+    manifest: dict[str, Any] | None, source_name: str, table_name: str
+) -> dict[str, Any] | None:
+    if not manifest or "sources" not in manifest:
+        return None
+    for node in manifest["sources"].values():
+        if node.get("source_name") == source_name and node.get("name") == table_name:
+            return node
+    return None
+
+
+def _node_schema_and_sql(node: dict[str, Any], fallback_name: str) -> tuple[str, str]:
+    """Return (logical_schema_for_provenance, sql_fragment)."""
+    schema = str(node.get("schema", "main"))
+    if node.get("relation_name"):
+        rel = str(node["relation_name"])
+        return schema, rel
+    alias = str(node.get("alias", fallback_name))
+    return schema, f"{schema}.{alias}"
+
+
+def resolve_dbt_refs_with_provenance(
+    sql: str,
+    manifest: dict[str, Any],
+    prod_manifest: dict[str, Any] | None,
+) -> tuple[str, list[ResolvedRelation]]:
+    """Resolve ref/source in SQL and record dev-vs-prod schema status per relation."""
+    relations: list[ResolvedRelation] = []
+    prod_available = prod_manifest is not None
+
+    def resolve_ref(match: re.Match[str]) -> str:
+        model_name = match.group(1)
+        node = _find_model_node(manifest, model_name)
+        if not node:
+            return model_name
+        schema, fragment = _node_schema_and_sql(node, model_name)
+        prod_node = _find_model_node(prod_manifest, model_name)
+        prod_schema = (
+            _node_schema_and_sql(prod_node, model_name)[0] if prod_node else None
+        )
+        status = classify_schema_status(
+            schema,
+            prod_schema,
+            prod_manifest_available=prod_available,
+        )
+        relations.append(
+            ResolvedRelation(ref_name=model_name, schema=schema, status=status)
+        )
+        return fragment
+
+    sql = re.sub(r"\{\{\s*ref\(['\"]([^'\"]+)['\"]\)\s*\}\}", resolve_ref, sql)
+
+    def resolve_source(match: re.Match[str]) -> str:
+        source_name = match.group(1)
+        table_name = match.group(2)
+        node = _find_source_node(manifest, source_name, table_name)
+        if not node:
+            return f"{source_name}.{table_name}"
+        schema, fragment = _node_schema_and_sql(node, table_name)
+        prod_node = _find_source_node(prod_manifest, source_name, table_name)
+        prod_schema = (
+            _node_schema_and_sql(prod_node, table_name)[0] if prod_node else None
+        )
+        status = classify_schema_status(
+            schema,
+            prod_schema,
+            prod_manifest_available=prod_available,
+        )
+        ref_name = f"{source_name}.{table_name}"
+        relations.append(
+            ResolvedRelation(ref_name=ref_name, schema=schema, status=status)
+        )
+        return fragment
+
+    sql = re.sub(
+        r"\{\{\s*source\(['\"]([^'\"]+)['\"],\s*['\"]([^'\"]+)['\"]\)\s*\}\}",
+        resolve_source,
+        sql,
+    )
+
+    return sql, relations
+
+
+def resolve_dbt_refs(sql: str, manifest: dict[str, Any]) -> str:
+    """Resolve refs/sources in SQL using the dbt manifest."""
+    resolved_sql, _ = resolve_dbt_refs_with_provenance(sql, manifest, None)
+    return resolved_sql

dataface/core/execute/adapters/http_adapter.py

@@ -0,0 +1,224 @@
+"""HTTP adapter for executing REST API queries.
+
+Stage: EXECUTE
+Purpose: Execute queries against HTTP/REST API endpoints.
+
+This adapter allows fetching data from external APIs, supporting
+GET, POST, PUT, DELETE, and PATCH methods with headers, params, and body.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+if TYPE_CHECKING:
+    from dataface.core.compile.models.source import SourceConfig
+
+from dataface.core.compile.jinja import resolve_jinja_template
+from dataface.core.compile.models.face.compiled import VariableValues
+from dataface.core.compile.models.query.compiled import (
+    AnyQuery,
+    is_http_query,
+)
+from dataface.core.execute.adapters.base import (
+    BaseAdapter,
+    QueryParams,
+    QueryResult,
+    handle_adapter_error,
+)
+
+
+def _resolve_json_path(data: Any, json_path: str) -> list[Any]:
+    """Extract a list from data using a dot-notation JSONPath expression.
+
+    Supports $.key and $.key.subkey format. Raises ValueError if the path
+    is missing or the resolved value is not a list.
+    """
+    path = json_path.lstrip("$").lstrip(".")
+    keys = path.split(".") if path else []
+
+    node: Any = data
+    traversed: list[str] = []
+    for key in keys:
+        if not isinstance(node, dict) or key not in node:
+            location = ".".join(traversed) or "$"
+            raise ValueError(
+                f"json_path '{json_path}': key '{key}' not found at '{location}'"
+            )
+        node = node[key]
+        traversed.append(key)
+
+    if not isinstance(node, list):
+        raise ValueError(
+            f"json_path '{json_path}': expected a list, got {type(node).__name__}"
+        )
+    return node
+
+
+class HttpAdapter(BaseAdapter):
+    """Adapter for executing HTTP/REST API queries.
+
+    Supported query types: http
+
+    Fetches data from REST API endpoints with support for:
+    - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
+    - Custom headers
+    - Query parameters
+    - Request body (JSON)
+
+    Example:
+        >>> adapter = HttpAdapter()
+        >>> query = HttpQuery(
+        ...     url="https://api.example.com/users",
+        ...     headers={"Authorization": "Bearer {{ token }}"}
+        ... )
+        >>> result = adapter.execute(query, {"token": "xyz"})
+    """
+
+    def __init__(self, timeout: int = 30):
+        """Initialize HTTP adapter.
+
+        Args:
+            timeout: Request timeout in seconds
+        """
+        self.timeout = timeout
+
+    @property
+    def supported_types(self) -> set[str]:
+        """Return supported query types."""
+        return {"http"}
+
+    def _execute(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None = None,
+        params: QueryParams = None,
+        source_config: SourceConfig | None = None,
+    ) -> QueryResult:
+        """Execute an HTTP query.
+
+        Uses type guard for type-safe field access.
+
+        Args:
+            query: AnyQuery object (HttpQuery expected)
+            variables: Variable values for Jinja resolution
+            params: Not used for HTTP queries (HTTP doesn't use SQL parameterization)
+
+        Returns:
+            QueryResult with data or error
+        """
+        if not is_http_query(query):
+            return QueryResult(
+                data=[],
+                error=f"Expected HTTP query, got {query.query_type}",
+            )
+
+        url = query.url
+
+        # Resolve Jinja templates in URL and other fields
+        try:
+            resolved_url = resolve_jinja_template(url, variables)
+            method = (query.method or "GET").upper()
+            headers = query.headers or {}
+            params_dict: dict[str, Any] = query.params or {}
+            body = query.body
+
+            # Resolve templates in headers, params, body
+            resolved_headers: dict[str, Any] = {}
+            for k, v in headers.items():
+                if isinstance(v, str):
+                    resolved_headers[k] = resolve_jinja_template(v, variables)
+                else:
+                    resolved_headers[k] = v
+
+            resolved_params: dict[str, Any] = {}
+            for k, v in params_dict.items():
+                if isinstance(v, str):
+                    resolved_params[k] = resolve_jinja_template(v, variables)
+                else:
+                    resolved_params[k] = v
+
+            resolved_body: Any = None
+            if body:
+                if isinstance(body, str):
+                    resolved_body = resolve_jinja_template(body, variables)
+                elif isinstance(body, dict):
+                    resolved_body = {}
+                    for k, v in body.items():
+                        if isinstance(v, str):
+                            resolved_body[k] = resolve_jinja_template(v, variables)
+                        else:
+                            resolved_body[k] = v
+                else:
+                    resolved_body = body
+
+        except (ValueError, KeyError, TypeError) as e:
+            return handle_adapter_error("HTTP query template resolution", e)
+
+        # Execute HTTP request
+        try:
+            with httpx.Client(timeout=self.timeout) as client:
+                if method == "GET":
+                    response = client.get(
+                        resolved_url, headers=resolved_headers, params=resolved_params
+                    )
+                elif method == "POST":
+                    response = client.post(
+                        resolved_url,
+                        headers=resolved_headers,
+                        json=resolved_body,
+                        params=resolved_params,
+                    )
+                elif method == "PUT":
+                    response = client.put(
+                        resolved_url,
+                        headers=resolved_headers,
+                        json=resolved_body,
+                        params=resolved_params,
+                    )
+                elif method == "DELETE":
+                    response = client.delete(
+                        resolved_url, headers=resolved_headers, params=resolved_params
+                    )
+                elif method == "PATCH":
+                    response = client.patch(
+                        resolved_url,
+                        headers=resolved_headers,
+                        json=resolved_body,
+                        params=resolved_params,
+                    )
+                else:
+                    return QueryResult(
+                        data=[],
+                        error=f"Unsupported HTTP method: {method}",
+                    )
+
+                response.raise_for_status()
+                data = response.json()
+
+                if query.json_path is not None:
+                    try:
+                        result_data = _resolve_json_path(data, query.json_path)
+                    except ValueError as e:
+                        return QueryResult(data=[], error=str(e))
+                elif isinstance(data, list):
+                    result_data = data
+                else:
+                    return QueryResult(
+                        data=[],
+                        error=(
+                            "HTTP response is a JSON object. "
+                            "Set json_path to extract the data array "
+                            "(e.g., json_path: $.data)."
+                        ),
+                    )
+
+                return QueryResult(data=result_data)
+
+        except httpx.HTTPError as e:
+            return handle_adapter_error("HTTP request", e)
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            return handle_adapter_error("HTTP query execution", e)

dataface/core/execute/adapters/metricflow_adapter.py

@@ -0,0 +1,94 @@
+"""MetricFlow adapter for executing dbt Semantic Layer queries.
+
+Stage: EXECUTE
+Purpose: Execute queries against dbt's Semantic Layer using MetricFlow.
+
+This adapter enables querying dbt metrics and dimensions using the
+MetricFlow query engine, supporting the dbt Semantic Layer.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dataface.core.compile.models.source import SourceConfig
+
+from dataface.core.compile.jinja import resolve_query_filters
+from dataface.core.compile.models.face.compiled import VariableValues
+from dataface.core.compile.models.query.compiled import (
+    AnyQuery,
+    is_metricflow_query,
+)
+from dataface.core.execute.adapters.base import BaseAdapter, QueryParams, QueryResult
+
+
+class MetricFlowAdapter(BaseAdapter):
+    """Adapter for executing MetricFlow (dbt Semantic Layer) queries.
+
+    Supported query types: metricflow
+
+    Executes queries against dbt's Semantic Layer using MetricFlow.
+    In a full implementation, this uses the MetricFlow Python API or CLI.
+
+    Example:
+        >>> adapter = MetricFlowAdapter()
+        >>> query = MetricFlowQuery(
+        ...     metrics=["revenue", "orders"],
+        ...     dimensions=["date", "region"]
+        ... )
+        >>> result = adapter.execute(query)
+    """
+
+    def __init__(self, config_path: str | None = None):
+        """Initialize MetricFlow adapter.
+
+        Args:
+            config_path: Optional path to MetricFlow configuration
+        """
+        self.config_path = config_path
+
+    @property
+    def supported_types(self) -> set[str]:
+        """Return supported query types."""
+        return {"metricflow"}
+
+    def _execute(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None = None,
+        params: QueryParams = None,
+        source_config: SourceConfig | None = None,
+    ) -> QueryResult:
+        """Execute a MetricFlow query.
+
+        Uses type guard for type-safe field access.
+
+        Args:
+            query: AnyQuery object (MetricFlowQuery expected)
+            variables: Variable values for filter resolution
+            params: Not used for MetricFlow queries (handled differently)
+
+        Returns:
+            QueryResult with data or error
+        """
+        # Type guard ensures query.metrics is List[str]
+        if not is_metricflow_query(query):
+            return QueryResult(
+                data=[],
+                error=f"Expected MetricFlow query, got {query.query_type}",
+            )
+
+        # Resolve filters with variables (for future use)
+        filters = query.filters or {}
+        _ = resolve_query_filters(filters, variables)
+
+        # In a real implementation, this would:
+        # 1. Construct MetricFlow query object
+        # 2. Apply filters
+        # 3. Execute via MetricFlow API/CLI
+        # 4. Convert results to list of dicts
+        return QueryResult(
+            data=[],
+            error="MetricFlow execution not yet implemented - requires MetricFlow setup",
+        )

dataface/core/execute/adapters/schema_resolver_adapter.py

@@ -0,0 +1,144 @@
+"""Schema resolver adapter — queries the LayeredSchemaResolver in-process.
+
+Stage: EXECUTE
+Purpose: Dispatch face queries to the LayeredSchemaResolver without a subprocess.
+
+Routing by field population:
+- source only        → list_schemas(source)
+- source + schema    → list_tables(source, schema)
+- source + schema + table        → profile_table → column rows
+- source + schema + table + column → profile_column → single column row
+
+Field-prerequisite and Jinja-rejection invariants are enforced at compile time
+by SchemaResolverQuery's Pydantic validators — the adapter trusts them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from dataface.core.compile.models.source import SourceConfig
+
+from dataface.core.compile.models.face.compiled import VariableValues
+from dataface.core.compile.models.query.compiled import (
+    AnyQuery,
+    is_schema_resolver_query,
+)
+from dataface.core.execute.adapters.base import BaseAdapter, QueryParams, QueryResult
+
+if TYPE_CHECKING:
+    from dataface.core.execute.adapters import AdapterRegistry
+
+
+class SchemaResolverAdapter(BaseAdapter):
+    """Adapter for in-process schema resolver queries.
+
+    Supported query types: schema_resolver
+
+    Dispatches to list_schemas / list_tables / profile_table / profile_column
+    on the LayeredSchemaResolver based on which fields are populated.
+    """
+
+    def __init__(self, adapter_registry: AdapterRegistry) -> None:
+        self._registry = adapter_registry
+
+    @property
+    def supported_types(self) -> set[str]:
+        return {"schema_resolver"}
+
+    def _execute(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None = None,
+        params: QueryParams = None,
+        source_config: SourceConfig | None = None,
+    ) -> QueryResult:
+        if not is_schema_resolver_query(query):
+            return QueryResult(
+                data=[],
+                error=f"Expected schema_resolver query, got {query.query_type}",
+            )
+
+        from dataface.core.inspect.cache_factory import build_resolver  # noqa: PLC0415
+
+        resolver = build_resolver(self._registry)
+
+        source = query.source
+        schema = query.schema_name
+        table = query.table
+        column = query.column
+
+        if schema is None:
+            envelope = resolver.list_schemas(source)
+            schemas: dict[str, Any] = (
+                envelope.get("sources", {}).get(source, {}).get("schemas", {})
+            )
+            rows = [{**m, "name": n} for n, m in schemas.items()]
+            return QueryResult(data=query.apply_limit(rows))
+
+        if table is None:
+            envelope = resolver.list_tables(source, schema)
+            source_schemas: dict[str, Any] = (
+                envelope.get("sources", {}).get(source, {}).get("schemas", {})
+            )
+            if schema not in source_schemas:
+                return QueryResult(
+                    data=[],
+                    error=f"Schema {schema!r} not found in source {source!r}",
+                )
+            tables: dict[str, Any] = source_schemas[schema].get("tables", {})
+            rows = [{**m, "name": n} for n, m in tables.items()]
+            return QueryResult(data=query.apply_limit(rows))
+
+        if column is None:
+            envelope = resolver.profile_table(source, schema, table)
+            all_tables: dict[str, Any] = (
+                envelope.get("sources", {})
+                .get(source, {})
+                .get("schemas", {})
+                .get(schema, {})
+                .get("tables", {})
+            )
+            if table not in all_tables:
+                return QueryResult(
+                    data=[],
+                    error=f"Table {schema}.{table} not found in source {source!r}",
+                )
+            columns: dict[str, Any] = all_tables[table].get("columns") or {}
+            rows = [{**m, "name": n} for n, m in columns.items()]
+            return QueryResult(data=query.apply_limit(rows))
+
+        # column specified → profile_column
+        envelope = resolver.profile_column(source, schema, table, column)
+        col_map: dict[str, Any] = (
+            envelope.get("sources", {})
+            .get(source, {})
+            .get("schemas", {})
+            .get(schema, {})
+            .get("tables", {})
+            .get(table, {})
+            .get("columns")
+            or {}
+        )
+        if column not in col_map:
+            # The resolver drops the schemas key for both "table missing" and
+            # "column missing" — probe profile_table to emit the right message.
+            table_env = resolver.profile_table(source, schema, table)
+            table_found = table in (
+                table_env.get("sources", {})
+                .get(source, {})
+                .get("schemas", {})
+                .get(schema, {})
+                .get("tables", {})
+            )
+            if not table_found:
+                return QueryResult(
+                    data=[],
+                    error=f"Table {schema}.{table} not found in source {source!r}",
+                )
+            return QueryResult(
+                data=[],
+                error=f"Column {column!r} not found in {schema}.{table}",
+            )
+        return QueryResult(data=[{**col_map[column], "name": column}])