dataface 0.1.2__py3-none-any.whl

dataface/core/compile/models/source.py

@@ -0,0 +1,611 @@
+"""Source configuration types for database connections and data sources.
+
+Stage: COMPILE
+Purpose: Define Pydantic models for source configurations.
+
+Sources represent "where data comes from" - databases, CSV files, HTTP APIs, etc.
+Field names match dbt's profiles.yml exactly where applicable.
+
+Design Principles:
+- Match dbt field names exactly for database types
+- Support env_var() syntax like dbt profiles
+- Use discriminated union via 'type' field
+- DB source configs inherit extra="allow" from _DbSourceShim (ADR-009 passthrough contract)
+- File/HTTP/dbt_profile configs inherit extra="forbid" from BaseSourceConfig (closed contracts)
+
+Dependencies:
+    - pydantic (BaseModel, Field, field_validator)
+
+Read-only posture per warehouse:
+    DuckDB opens connections with read_only=True in-process (see SqlAdapter.read_only).
+    Every other warehouse lacks a native read-only driver flag. For those, the operator
+    must bind SELECT-only credentials at the database/IAM layer:
+
+    | Warehouse       | In-process flag | Operator posture                                      |
+    |-----------------|-----------------|-------------------------------------------------------|
+    | DuckDB          | read_only=True  | Handled in-process by SqlAdapter                      |
+    | Postgres        | none            | Bind a SELECT-only role to the dbt profile credential |
+    | MySQL/MariaDB   | none            | Bind a SELECT-only user to the dbt profile credential |
+    | Snowflake       | none            | Bind a SELECT-only role to the Snowflake user/key-pair|
+    | BigQuery        | none            | Bind credentials to roles/bigquery.dataViewer         |
+    | Databricks      | none            | Unity Catalog: bind service principal to SELECT-only  |
+    | Redshift        | none            | IAM/grants: bind to a SELECT-only IAM role or DB user |
+    | SQL Server      | none            | Login-level SELECT-only grants                        |
+
+    sql_guard (allowlist enforcement) provides the in-process defense for all warehouses.
+    SELECT-only credentials are the connection-level defense for non-DuckDB warehouses.
+
+See also:
+    - config.py: Project-level configuration
+    - models/query/compiled.py: Query interface types
+    - normalizer.py: Resolves source references
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+# ============================================================================
+# BIGQUERY AUTH HELPERS
+# ============================================================================
+
+
+def infer_bq_method(
+    keyfile: str | None, keyfile_json: dict[str, Any] | None
+) -> Literal["oauth", "service-account", "service-account-json"]:
+    """Return the dbt-bigquery method for the given credential shape."""
+    if keyfile_json is not None:
+        return "service-account-json"
+    if keyfile is not None:
+        return "service-account"
+    return "oauth"
+
+
+# ============================================================================
+# ENVIRONMENT VARIABLE RESOLUTION
+# ============================================================================
+
+
+def resolve_env_var(value: str) -> str:
+    """Resolve {{ env_var('VAR') }} or {{ env_var('VAR', 'default') }} syntax.
+
+    Matches dbt's env_var() function behavior.
+
+    Args:
+        value: String that may contain env_var() expressions
+
+    Returns:
+        String with env_var() resolved to actual values
+
+    Raises:
+        ValueError: If required env var is not set
+    """
+    if not isinstance(value, str):
+        return value
+
+    # Pattern: {{ env_var('VAR') }} or {{ env_var('VAR', 'default') }}
+    pattern = r"\{\{\s*env_var\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*['\"]([^'\"]*)['\"])?\s*\)\s*\}\}"
+
+    def replace_env_var(match: re.Match[str]) -> str:
+        var_name = match.group(1)
+        default_value = match.group(2)
+
+        env_value = os.environ.get(var_name)
+        if env_value is not None:
+            return env_value
+        elif default_value is not None:
+            return default_value
+        else:
+            raise ValueError(
+                f"Environment variable '{var_name}' is not set and no default provided"
+            )
+
+    return re.sub(pattern, replace_env_var, value)
+
+
+def resolve_env_vars_in_dict(data: dict[str, Any]) -> dict[str, Any]:
+    """Recursively resolve env_var() expressions in a dictionary.
+
+    Args:
+        data: Dictionary that may contain env_var() expressions
+
+    Returns:
+        Dictionary with all env_var() expressions resolved
+    """
+    result: dict[str, Any] = {}
+    for key, value in data.items():
+        if isinstance(value, str):
+            result[key] = resolve_env_var(value)
+        elif isinstance(value, dict):
+            result[key] = resolve_env_vars_in_dict(value)
+        else:
+            result[key] = value
+    return result
+
+
+# ============================================================================
+# BASE SOURCE CONFIG
+# ============================================================================
+
+
+class BaseSourceConfig(BaseModel):
+    """Closed-contract root for all source configurations.
+
+    extra="forbid" — file/HTTP/dbt_profile source configs have defined Dataface
+    contracts; unknown fields are validation errors, not silent pass-throughs.
+    DB source configs widen to extra="allow" via _DbSourceShim (ADR-009).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def _resolve_all_env_vars(cls, data: Any) -> Any:
+        """Resolve {{ env_var('VAR') }} in all string fields before Pydantic validates."""
+        if isinstance(data, dict):
+            return resolve_env_vars_in_dict(data)
+        return data
+
+
+class _DbSourceShim(BaseSourceConfig):
+    """Passthrough shim for DB source configs.
+
+    extra="allow" — dbt profiles.yml passthrough contract (ADR-009): dbt can
+    add or rename target fields across versions; Dataface reads whatever the
+    user's profile contains without raising a ValidationError. Only DB source
+    configs (Postgres, Snowflake, BigQuery, Redshift, MySQL, DuckDB) inherit
+    from this shim. File/HTTP/dbt_profile configs inherit BaseSourceConfig directly.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+
+# ============================================================================
+# DATABASE SOURCE CONFIGS
+# ============================================================================
+
+
+class PostgresSourceConfig(_DbSourceShim):
+    """Postgres source configuration.
+
+    Field names match dbt profiles.yml exactly.
+
+    Read-only posture: no native driver flag. Bind a SELECT-only role to the
+    dbt profile credential at the database level.
+    """
+
+    type: Literal["postgres"] = Field(description="Source type identifier.")
+    host: str = Field(description="Database host name or IP address.")
+    port: int = Field(default=5432, description="Database port number.")
+    dbname: str = Field(description="Database name (matches dbt 'dbname').")
+    schema_: str = Field(
+        alias="schema", default="public", description="Default schema for queries."
+    )
+    user: str = Field(description="Database user name.")
+    password: str = Field(description="Database password.")
+
+
+class SnowflakeSourceConfig(_DbSourceShim):
+    """Snowflake source configuration.
+
+    Field names match dbt profiles.yml exactly.
+
+    Read-only posture: no native driver flag. Bind a SELECT-only role to the
+    Snowflake user or key-pair in the dbt profile.
+    """
+
+    type: Literal["snowflake"] = Field(description="Source type identifier.")
+    account: str = Field(
+        description="Snowflake account identifier (e.g. xy12345.us-east-1)."
+    )
+    user: str = Field(description="Snowflake user name.")
+    password: str | None = Field(
+        default=None,
+        description="Snowflake password. Omit when using OAuth or key-pair auth.",
+    )
+    database: str = Field(description="Snowflake database name.")
+    warehouse: str = Field(description="Snowflake virtual warehouse name.")
+    schema_: str = Field(
+        alias="schema", default="PUBLIC", description="Default schema for queries."
+    )
+    role: str | None = Field(
+        default=None, description="Snowflake role to assume for the session."
+    )
+
+
+class BigQuerySourceConfig(_DbSourceShim):
+    """BigQuery source configuration.
+
+    Field names match dbt profiles.yml exactly.
+
+    Read-only posture: no native driver flag. Bind credentials to
+    roles/bigquery.dataViewer (or equivalent) at the IAM layer.
+    """
+
+    type: Literal["bigquery"] = Field(description="Source type identifier.")
+    project: str = Field(description="GCP project ID.")
+    dataset: str = Field(description="BigQuery dataset name (equivalent to schema).")
+    keyfile: str | None = Field(
+        default=None, description="Path to service account JSON key file."
+    )
+    keyfile_json: dict[str, Any] | None = Field(
+        default=None, description="Inline service account JSON dict."
+    )
+    location: str | None = Field(
+        default=None, description="Dataset location (e.g. US, EU)."
+    )
+    method: Literal["oauth", "service-account", "service-account-json"] | None = Field(
+        default=None,
+        description=(
+            "dbt-bigquery authentication method. "
+            "Inferred from keyfile/keyfile_json when omitted: "
+            "'service-account-json' if keyfile_json set, "
+            "'service-account' if keyfile set, "
+            "'oauth' (Application Default Credentials) otherwise."
+        ),
+    )
+
+    @model_validator(mode="after")
+    def _resolve_auth(self) -> BigQuerySourceConfig:
+        """Validate credential fields and infer method when not explicitly set."""
+        if self.keyfile is not None and self.keyfile_json is not None:
+            raise ValueError(
+                "BigQuery credentials must use either 'keyfile' (path to JSON file) "
+                "or 'keyfile_json' (inline dict), not both."
+            )
+        if self.method is None:
+            self.method = infer_bq_method(self.keyfile, self.keyfile_json)
+        return self
+
+
+class RedshiftSourceConfig(_DbSourceShim):
+    """Redshift source configuration.
+
+    Field names match dbt profiles.yml exactly.
+
+    Read-only posture: no native driver flag. Bind to a SELECT-only IAM role
+    or database user at the Redshift/IAM layer.
+    """
+
+    type: Literal["redshift"] = Field(description="Source type identifier.")
+    host: str = Field(description="Redshift cluster host name.")
+    port: int = Field(default=5439, description="Redshift port number.")
+    dbname: str = Field(description="Redshift database name.")
+    schema_: str = Field(
+        alias="schema", default="public", description="Default schema for queries."
+    )
+    user: str = Field(description="Redshift user name.")
+    password: str = Field(description="Redshift password.")
+
+
+class MySQLSourceConfig(_DbSourceShim):
+    """MySQL source configuration.
+
+    Field names match dbt profiles.yml exactly.
+
+    Read-only posture: no native driver flag. Bind a SELECT-only user to the
+    dbt profile credential at the database level.
+    """
+
+    type: Literal["mysql"] = Field(description="Source type identifier.")
+    host: str = Field(description="MySQL host name or IP address.")
+    port: int = Field(default=3306, description="MySQL port number.")
+    database: str = Field(description="MySQL database name.")
+    schema_: str = Field(
+        alias="schema", default="", description="Default schema for queries."
+    )
+    user: str = Field(description="MySQL user name.")
+    password: str = Field(description="MySQL password.")
+
+
+class DuckDBSourceConfig(_DbSourceShim):
+    """DuckDB source configuration.
+
+    Read-only posture: handled in-process. SqlAdapter opens file-based DuckDB
+    connections with read_only=True and forces enable_external_access=False.
+    Callers that need external access (e.g. the playground for local dev) can
+    pass allow_external_access_in_readonly=True on SqlAdapter /
+    build_adapter_registry together with duckdb_config={"enable_external_access":
+    True} to explicitly opt back in.
+
+    Example:
+        sources:
+          analytics:
+            type: duckdb
+            path: ./data/analytics.duckdb
+    """
+
+    type: Literal["duckdb"] = Field(description="Source type identifier.")
+    path: str = Field(
+        default=":memory:",
+        description="DuckDB file path or ':memory:' for an in-memory database.",
+    )
+    schema_: str | None = Field(
+        alias="schema",
+        default=None,
+        description="Default schema for unqualified table names (sets search_path).",
+    )
+
+    @field_validator("schema_", mode="before")
+    @classmethod
+    def validate_schema_identifier(cls, v: Any) -> Any:
+        if v is None:
+            return v
+        if not isinstance(v, str) or not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", v):
+            raise ValueError(
+                f"schema must be a valid SQL identifier (letters, digits, underscores; "
+                f"no leading digit): got {v!r}"
+            )
+        return v
+
+    @model_validator(mode="before")
+    @classmethod
+    def reject_database_key(cls, data: Any) -> Any:
+        """Reject the legacy 'database' key at parse time.
+
+        The runtime already raises on 'database', so reject it here too so
+        the two layers agree and the error fires as early as possible.
+        """
+        if isinstance(data, dict) and "database" in data:
+            raise ValueError(
+                "DuckDB source config uses 'path', not 'database'. "
+                "Replace 'database' with 'path' "
+                '(e.g. {"type": "duckdb", "path": "db.duckdb"}).'
+            )
+        return data
+
+
+# ============================================================================
+# FILE SOURCE CONFIGS
+# ============================================================================
+
+
+class CsvSourceConfig(BaseSourceConfig):
+    """CSV file source configuration."""
+
+    type: Literal["csv"] = Field(description="Source type identifier.")
+    file: str | None = Field(default=None, description="Local path to the CSV file.")
+    url: str | None = Field(default=None, description="Remote URL of the CSV file.")
+    headers: dict[str, str] | None = Field(
+        default=None, description="HTTP headers for authenticated URLs."
+    )
+    delimiter: str = Field(default=",", description="Field delimiter character.")
+    encoding: str = Field(default="utf-8", description="File encoding.")
+
+    @model_validator(mode="after")
+    def validate_file_or_url(self) -> CsvSourceConfig:
+        """Ensure either file or url is provided."""
+        if not self.file and not self.url:
+            raise ValueError("CsvSourceConfig requires either 'file' or 'url'")
+        return self
+
+    @field_validator("file", "url", mode="before")
+    @classmethod
+    def resolve_env_vars(cls, v: Any) -> Any:
+        """Resolve env_var() syntax in string fields."""
+        if isinstance(v, str):
+            return resolve_env_var(v)
+        return v
+
+
+class ParquetSourceConfig(BaseSourceConfig):
+    """Parquet file source configuration."""
+
+    type: Literal["parquet"] = Field(description="Source type identifier.")
+    file: str | None = Field(
+        default=None, description="Local path to the Parquet file."
+    )
+    url: str | None = Field(default=None, description="Remote URL of the Parquet file.")
+    headers: dict[str, str] | None = Field(
+        default=None, description="HTTP headers for authenticated URLs."
+    )
+
+    @model_validator(mode="after")
+    def validate_file_or_url(self) -> ParquetSourceConfig:
+        """Ensure either file or url is provided."""
+        if not self.file and not self.url:
+            raise ValueError("ParquetSourceConfig requires either 'file' or 'url'")
+        return self
+
+
+class JsonSourceConfig(BaseSourceConfig):
+    """JSON file source configuration."""
+
+    type: Literal["json"] = Field(description="Source type identifier.")
+    file: str | None = Field(default=None, description="Local path to the JSON file.")
+    url: str | None = Field(default=None, description="Remote URL of the JSON file.")
+    headers: dict[str, str] | None = Field(
+        default=None, description="HTTP headers for authenticated URLs."
+    )
+    json_path: str | None = Field(
+        default=None, description="JSONPath expression to extract the data array."
+    )
+
+    @model_validator(mode="after")
+    def validate_file_or_url(self) -> JsonSourceConfig:
+        """Ensure either file or url is provided."""
+        if not self.file and not self.url:
+            raise ValueError("JsonSourceConfig requires either 'file' or 'url'")
+        return self
+
+
+# ============================================================================
+# HTTP/API SOURCE CONFIG
+# ============================================================================
+
+
+class HttpSourceConfig(BaseSourceConfig):
+    """HTTP/REST API source configuration."""
+
+    type: Literal["http"] = Field(description="Source type identifier.")
+    url: str = Field(description="Base URL for HTTP requests.")
+    headers: dict[str, str] | None = Field(
+        default=None, description="Default HTTP headers (e.g. Authorization)."
+    )
+
+    @field_validator("url", mode="before")
+    @classmethod
+    def resolve_url_env_var(cls, v: Any) -> Any:
+        """Resolve env_var() syntax in URL."""
+        if isinstance(v, str):
+            return resolve_env_var(v)
+        return v
+
+    @field_validator("headers", mode="before")
+    @classmethod
+    def resolve_headers_env_vars(cls, v: Any) -> Any:
+        """Resolve env_var() syntax in headers."""
+        if isinstance(v, dict):
+            return resolve_env_vars_in_dict(v)
+        return v
+
+
+# ============================================================================
+# DBT PROFILE SOURCE CONFIG
+# ============================================================================
+
+
+class DbtProfileSourceConfig(BaseSourceConfig):
+    """Reference to a dbt profile (for backward compatibility).
+
+    Instead of defining connection details, references an existing
+    dbt profile from profiles.yml.
+    """
+
+    type: Literal["dbt_profile"] = Field(description="Source type identifier.")
+    profile: str = Field(description="dbt profile name from profiles.yml.")
+    target: str | None = Field(
+        default=None,
+        description="dbt target to use; defaults to the profile's default target.",
+    )
+
+
+# ============================================================================
+# UNION TYPE
+# ============================================================================
+
+
+# Union of all source config types
+SourceConfig = (
+    PostgresSourceConfig
+    | SnowflakeSourceConfig
+    | BigQuerySourceConfig
+    | RedshiftSourceConfig
+    | MySQLSourceConfig
+    | DuckDBSourceConfig
+    | CsvSourceConfig
+    | ParquetSourceConfig
+    | JsonSourceConfig
+    | HttpSourceConfig
+    | DbtProfileSourceConfig
+)
+
+
+# Valid source type strings
+VALID_SOURCE_TYPES = {
+    "postgres",
+    "snowflake",
+    "bigquery",
+    "redshift",
+    "mysql",
+    "duckdb",
+    "csv",
+    "parquet",
+    "json",
+    "http",
+    "dbt_profile",
+}
+
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+
+def parse_source_config(data: dict[str, Any]) -> SourceConfig:
+    """Parse a source configuration dictionary into the appropriate type.
+
+    Uses the 'type' field to determine which Pydantic model to use.
+
+    Args:
+        data: Source configuration dictionary
+
+    Returns:
+        Appropriate SourceConfig subclass instance
+
+    Raises:
+        ValueError: If type is missing or invalid
+    """
+    if "type" not in data:
+        raise ValueError("Source configuration must have a 'type' field")
+
+    source_type = data["type"]
+
+    type_map = {
+        "postgres": PostgresSourceConfig,
+        "snowflake": SnowflakeSourceConfig,
+        "bigquery": BigQuerySourceConfig,
+        "redshift": RedshiftSourceConfig,
+        "mysql": MySQLSourceConfig,
+        "duckdb": DuckDBSourceConfig,
+        "csv": CsvSourceConfig,
+        "parquet": ParquetSourceConfig,
+        "json": JsonSourceConfig,
+        "http": HttpSourceConfig,
+        "dbt_profile": DbtProfileSourceConfig,
+    }
+
+    if source_type not in type_map:
+        raise ValueError(
+            f"Unknown source type: '{source_type}'. "
+            f"Valid types: {', '.join(sorted(type_map.keys()))}"
+        )
+
+    return type_map[source_type](**data)
+
+
+def is_database_source(config: SourceConfig) -> bool:
+    """Check if source config is a database type.
+
+    Args:
+        config: Source configuration
+
+    Returns:
+        True if source is a database connection
+    """
+    return config.type in {
+        "postgres",
+        "snowflake",
+        "bigquery",
+        "redshift",
+        "mysql",
+        "duckdb",
+    }
+
+
+def is_file_source(config: SourceConfig) -> bool:
+    """Check if source config is a file type.
+
+    Args:
+        config: Source configuration
+
+    Returns:
+        True if source is a file-based source
+    """
+    return config.type in {"csv", "parquet", "json"}
+
+
+def is_api_source(config: SourceConfig) -> bool:
+    """Check if source config is an API type.
+
+    Args:
+        config: Source configuration
+
+    Returns:
+        True if source is an HTTP API
+    """
+    return config.type == "http"