dataface 0.1.2__py3-none-any.whl

dataface/core/compile/parser.py

@@ -0,0 +1,228 @@
+"""YAML parsing module.
+
+Stage: COMPILE (Step 1 of 4)
+Purpose: Parse YAML strings into AuthoredFace input types.
+
+Entry Points:
+    - parse_yaml(content: str) -> AuthoredFace
+
+Inputs:
+    - YAML string (face definition)
+
+Outputs:
+    - AuthoredFace (input type with optional fields)
+
+Dependencies:
+    - yaml (PyYAML)
+    - .types (AuthoredFace)
+
+Errors:
+    - ParseError: Invalid YAML syntax (with line numbers, context, suggestions)
+
+See also:
+    - docs/docs/contributing/architecture.md for the pipeline overview
+    - compile/validator.py for the next step
+
+Refs #94
+"""
+
+import re
+from typing import Any
+
+import yaml
+
+from dataface.core.compile.errors import ParseError
+from dataface.core.compile.models.face.authored import AuthoredFace
+from dataface.core.compile.models.refs import normalize_query_value
+from dataface.core.errors.codes_compile import DF_COMPILE_EXTRA_FIELD
+
+
+def parse_yaml(content: str) -> AuthoredFace:
+    """Parse YAML content into a AuthoredFace object.
+
+    Stage: COMPILE (Step 1 of 4: Parsing)
+
+    This is the first step of compilation. It converts a raw YAML string
+    into a structured AuthoredFace object. Only basic syntax validation happens
+    here - schema validation is the next step.
+
+    Enhanced error messages include:
+    - Line numbers where errors occur
+    - YAML context showing the problematic snippet
+    - Helpful suggestions ("Did you mean?")
+
+    Args:
+        content: Raw YAML string to parse
+
+    Returns:
+        AuthoredFace object with parsed structure
+
+    Raises:
+        ParseError: If YAML syntax is invalid or parsing fails
+
+    Example:
+        >>> yaml_content = '''
+        ... title: My Dataface
+        ... queries:
+        ...   sales: SELECT * FROM sales
+        ... charts:
+        ...   revenue:
+        ...     query: sales
+        ...     type: line
+        ... rows:
+        ...   - revenue
+        ... '''
+        >>> face = parse_yaml(yaml_content)
+        >>> face.title
+        'My Dataface'
+    """
+    # Step 1a: Parse YAML to dict
+    try:
+        parsed_data = yaml.safe_load(content)
+    except yaml.YAMLError as e:
+        # Extract line number from YAML error and provide context
+        line_num = _extract_line_from_yaml_error(str(e))
+        context = _get_yaml_context_for_error(content, line_num) if line_num else None
+        suggestion = _get_yaml_parse_suggestion(str(e))
+        raise ParseError(
+            f"Invalid YAML syntax: {e}",
+            line=line_num,
+            context=context,
+            suggestion=suggestion,
+        ) from e
+
+    # Step 1b: Handle empty content
+    if parsed_data is None:
+        raise ParseError("Empty YAML document")
+
+    if not isinstance(parsed_data, dict):
+        raise ParseError(f"YAML must be a mapping, got {type(parsed_data).__name__}")
+
+    # Step 1c: Normalize queries (infer types)
+    if "queries" in parsed_data and isinstance(parsed_data["queries"], dict):
+        parsed_data["queries"] = _normalize_query_definitions(parsed_data["queries"])
+
+    # Step 1d: Convert to AuthoredFace with enhanced error messages
+    from pydantic import ValidationError as PydanticValidationError
+
+    try:
+        return AuthoredFace(**parsed_data)
+    except PydanticValidationError as e:
+        # Enhance with line-number context and suggestions first.
+        try:
+            from dataface.core.compile.yaml_error_formatter import (
+                format_validation_error,
+            )
+
+            enhanced_msg = format_validation_error(e, content)
+            err = ParseError(enhanced_msg)
+        except ParseError:
+            raise
+        except ImportError:
+            err = ParseError(f"Failed to parse face structure: {e}")
+
+        # Stamp the typed code when the primary failure is an extra field — the
+        # enhanced message is preserved; only the code changes from DF-UNKNOWN-INTERNAL.
+        pydantic_errors = e.errors()
+        if pydantic_errors and pydantic_errors[0].get("type") == "extra_forbidden":
+            err.code = DF_COMPILE_EXTRA_FIELD
+
+        raise err from e
+    except (TypeError, ValueError) as e:
+        # Other parsing errors (wrong types, invalid values)
+        raise ParseError(f"Failed to parse face structure: {e}") from e
+
+
+def _extract_line_from_yaml_error(error_msg: str) -> int | None:
+    """Extract line number from a YAML error message.
+
+    Args:
+        error_msg: The YAML error message
+
+    Returns:
+        Line number or None if not found
+    """
+    # PyYAML errors often contain "line X"
+    match = re.search(r"line\s+(\d+)", error_msg, re.I)
+    if match:
+        return int(match.group(1))
+    return None
+
+
+def _get_yaml_context_for_error(
+    content: str,
+    line_num: int,
+    context_lines: int = 2,
+) -> str:
+    """Get YAML context around an error line.
+
+    Args:
+        content: Full YAML content
+        line_num: Line number of the error (1-indexed)
+        context_lines: Number of lines to show before/after
+
+    Returns:
+        Formatted context string
+    """
+    from dataface.core.compile.yaml_error_formatter import get_yaml_context
+
+    return get_yaml_context(content, line_num, context_lines)
+
+
+def _get_yaml_parse_suggestion(error_msg: str) -> str | None:
+    """Get a helpful suggestion for a YAML parse error.
+
+    Args:
+        error_msg: The error message
+
+    Returns:
+        Suggestion string or None
+    """
+    error_lower = error_msg.lower()
+
+    if "indent" in error_lower:
+        return "💡 Check your indentation - YAML requires consistent spacing (typically 2 spaces)"
+    elif "expected" in error_lower and "block" in error_lower:
+        return "💡 This often happens with incorrect indentation or missing colons"
+    elif "mapping" in error_lower:
+        return "💡 Check for missing colons after key names or incorrect nesting"
+    elif "duplicate" in error_lower:
+        return "💡 You have duplicate keys - each key name must be unique at the same level"
+    elif "found character" in error_lower:
+        return (
+            "💡 Check for special characters that need quoting, or invalid YAML syntax"
+        )
+
+    return None
+
+
+def _normalize_query_definitions(queries: dict[str, Any]) -> dict[str, Any]:
+    """Normalize query definitions to consistent format.
+
+    Converts shorthand query formats to full format:
+    - String: "SELECT ..." -> {"type": "sql", "sql": "SELECT ..."}
+    - Dict without type: Infers type from keys
+
+    Delegates to normalize_query_value (models/refs.py) — the shared helper
+    used by both this function and AuthoredFace._normalize_queries.
+
+    Args:
+        queries: Raw query definitions from YAML
+
+    Returns:
+        Normalized query definitions with type field
+    """
+    return {
+        name: normalize_query_value(query_def) for name, query_def in queries.items()
+    }
+
+
+_SQL_PREFIX_RE = re.compile(
+    r"^\s*(SELECT|WITH|PRAGMA|INSERT|UPDATE|DELETE|CREATE)\b",
+    re.IGNORECASE,
+)
+
+
+def looks_like_sql(s: str) -> bool:
+    """Return True when a string starts like a raw SQL statement."""
+    return bool(_SQL_PREFIX_RE.match(s))

dataface/core/compile/schema.py

@@ -0,0 +1,20 @@
+"""Schema documentation generation for AI integrations.
+
+Delegates to the two-layer IR pipeline:
+  introspect() → AuthorableSchema → render_prompt() → str
+"""
+
+from dataface.core.compile.introspection import introspect
+from dataface.core.compile.schema_renderers.prompt import render_prompt
+
+
+def get_schema_for_prompt() -> str:
+    """Get complete schema documentation for AI prompts."""
+    return render_prompt(introspect())
+
+
+__all__ = ["get_schema_for_prompt"]
+
+
+if __name__ == "__main__":
+    print(get_schema_for_prompt())

dataface/core/compile/schema_renderers/json_schema.py

@@ -0,0 +1,163 @@
+"""JSON Schema renderer — Layer 2 of the two-layer schema IR.
+
+Converts an AuthorableSchema IR to a draft-07 JSON Schema dict.
+Callers: IDE schema generation, validation tooling.
+
+Entry point: render_json_schema(schema: AuthorableSchema) -> dict
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any
+
+from pydantic_core import PydanticUndefined
+
+from dataface.core.compile.introspection import AuthorableSchema, SchemaField
+
+_PRIMITIVES: dict[str, dict[str, Any]] = {
+    "str": {"type": "string"},
+    "int": {"type": "integer"},
+    "float": {"type": "number"},
+    "bool": {"type": "boolean"},
+    "None": {"type": "null"},
+}
+
+
+def _schema_for_token(token: str) -> dict[str, Any]:
+    """Map a single non-union type token to a JSON Schema dict."""
+    if token in _PRIMITIVES:
+        return _PRIMITIVES[token]
+    if token.startswith("list["):
+        return {"type": "array"}
+    if token.startswith("dict["):
+        return {"type": "object"}
+    return {}
+
+
+def _type_schema(field: SchemaField) -> dict[str, Any]:
+    """Build the JSON Schema type object for a single field."""
+    nullable = "None" in field.type_repr
+
+    extra = [_PRIMITIVES[t] for t in field.extra_union_types if t in _PRIMITIVES]
+
+    if field.enum_values is not None:
+        base: dict[str, Any] = {"enum": field.enum_values}
+        branches: list[dict[str, Any]] = [base] + extra
+        if nullable:
+            branches.append({"type": "null"})
+        return {"anyOf": branches} if len(branches) > 1 else branches[0]
+
+    if field.nested_models:
+        refs: list[dict[str, Any]] = [
+            {"$ref": f"#/$defs/{n}"} for n in field.nested_models
+        ]
+        inner: dict[str, Any] = refs[0] if len(refs) == 1 else {"anyOf": refs}
+        if field.container == "list":
+            arr: dict[str, Any] = {"type": "array", "items": inner}
+            parts: list[dict[str, Any]] = [arr] + extra
+            if nullable:
+                parts.append({"type": "null"})
+            return {"anyOf": parts} if len(parts) > 1 else parts[0]
+        if field.container == "dict":
+            obj: dict[str, Any] = {"type": "object", "additionalProperties": inner}
+            parts = [obj] + extra
+            if nullable:
+                parts.append({"type": "null"})
+            return {"anyOf": parts} if len(parts) > 1 else parts[0]
+        all_branches: list[dict[str, Any]] = refs + extra
+        if nullable:
+            all_branches.append({"type": "null"})
+        return {"anyOf": all_branches} if len(all_branches) > 1 else all_branches[0]
+
+    return _map_type_repr(field.type_repr)
+
+
+def _map_type_repr(type_repr: str) -> dict[str, Any]:
+    """Map a type_repr string to a JSON Schema type dict."""
+    nullable = " | None" in type_repr
+    base = type_repr.replace(" | None", "").strip()
+
+    if base in _PRIMITIVES:
+        schema: dict[str, Any] = _PRIMITIVES[base]
+    elif base.startswith("list["):
+        schema = {"type": "array"}
+    elif base.startswith("dict["):
+        schema = {"type": "object"}
+    elif " | " in base:
+        parts = [p.strip() for p in base.split(" | ")]
+        schemas = [s for p in parts if (s := _schema_for_token(p))]
+        schema = {"anyOf": schemas} if schemas else {}
+    else:
+        schema = {}
+
+    if not nullable or not schema:
+        return schema
+    if "anyOf" in schema:
+        schema = dict(schema)
+        schema["anyOf"] = list(schema["anyOf"]) + [{"type": "null"}]
+        return schema
+    return {"anyOf": [schema, {"type": "null"}]}
+
+
+def _model_to_def(model_name: str, ir: AuthorableSchema) -> dict[str, Any]:
+    """Build a JSON Schema $defs entry for one AuthorableModel."""
+    model = ir.models[model_name]
+    required = [f.name for f in model.fields if f.required and not f.is_extra_key]
+    properties: dict[str, Any] = {}
+    additional_properties: dict[str, Any] | None = None
+    for field in model.fields:
+        prop: dict[str, Any] = _type_schema(field)
+        if field.description:
+            prop = dict(prop)
+            prop["description"] = field.description
+        if (
+            not field.required
+            and field.default is not dataclasses.MISSING
+            and field.default is not None
+            and field.default is not PydanticUndefined
+        ):
+            prop = dict(prop)
+            prop["default"] = field.default
+        if field.is_extra_key:
+            additional_properties = prop
+        else:
+            properties[field.name] = prop
+
+    defn: dict[str, Any] = {"type": "object", "properties": properties}
+    if model.doc:
+        defn["description"] = model.doc
+    if required:
+        defn["required"] = required
+    if additional_properties is not None:
+        defn["additionalProperties"] = additional_properties
+    return defn
+
+
+def render_json_schema(schema: AuthorableSchema) -> dict[str, Any]:
+    """Render an AuthorableSchema IR as a draft-07 JSON Schema.
+
+    Produces a self-contained schema with the root model's properties
+    inlined and all reachable authored models in $defs.
+    """
+    root = _model_to_def(schema.root, schema)
+    defs = {
+        name: _model_to_def(name, schema)
+        for name in schema.models
+        if name != schema.root
+    }
+
+    result: dict[str, Any] = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": schema.root,
+    }
+    if root.get("description"):
+        result["description"] = root["description"]
+    result["type"] = "object"
+    result["properties"] = root.get("properties", {})
+    if root.get("required"):
+        result["required"] = root["required"]
+    if defs:
+        result["$defs"] = defs
+
+    return result

dataface/core/compile/schema_renderers/prompt.py

@@ -0,0 +1,152 @@
+"""Prompt renderer — Layer 2 of the two-layer schema IR.
+
+Converts an AuthorableSchema IR to a concise text reference for AI prompts.
+Replaces the hand-written schema.get_schema_for_prompt().
+
+Entry point: render_prompt(schema: AuthorableSchema) -> str
+"""
+
+from __future__ import annotations
+
+import re
+
+from dataface.core.compile.introspection import AuthorableSchema, SchemaField
+
+# Source connector configs are leaf models with no cross-references; emit them last.
+_SOURCE_CONFIG_NAMES = [
+    "PostgresSourceConfig",
+    "SnowflakeSourceConfig",
+    "BigQuerySourceConfig",
+    "RedshiftSourceConfig",
+    "MySQLSourceConfig",
+    "DuckDBSourceConfig",
+    "CsvSourceConfig",
+    "ParquetSourceConfig",
+    "JsonSourceConfig",
+    "HttpSourceConfig",
+    "DbtProfileSourceConfig",
+]
+
+
+def _display_name(class_name: str) -> str:
+    """Strip 'Authored' prefix and 'Patch' suffix for user-facing model headings."""
+    name = class_name
+    if name.startswith("Authored"):
+        name = name[len("Authored") :]
+    if name.endswith("Patch"):
+        name = name[: -len("Patch")]
+    return name
+
+
+def _bfs(
+    start: str, schema: AuthorableSchema, seen: set[str], order: list[str]
+) -> None:
+    queue = [start]
+    while queue:
+        name = queue.pop(0)
+        if name in seen or name not in schema.models:
+            continue
+        seen.add(name)
+        order.append(name)
+        for field in schema.models[name].fields:
+            for nested in field.nested_models:
+                if nested not in seen:
+                    queue.append(nested)
+
+
+def _ordered_model_names(schema: AuthorableSchema) -> list[str]:
+    """Return model names in a sensible top-down order.
+
+    1. BFS from root (AuthoredFace + all models reachable via type annotations)
+    2. BFS from AuthoredChart (not reachable via AuthoredFace annotations)
+    3. PaginationConfig
+    4. Source connector configs (grouped, leaf-level)
+    5. Any remaining models
+    """
+    order: list[str] = []
+    seen: set[str] = set()
+
+    _bfs(schema.root, schema, seen, order)
+    _bfs("AuthoredChart", schema, seen, order)
+    _bfs("PaginationConfig", schema, seen, order)
+    for name in _SOURCE_CONFIG_NAMES:
+        _bfs(name, schema, seen, order)
+    for name in schema.models:
+        if name not in seen:
+            order.append(name)
+
+    return order
+
+
+def _type_cell(field: SchemaField) -> str:
+    if field.enum_values is not None:
+        return "enum: " + ", ".join(f'"{v}"' for v in field.enum_values)
+    t = re.sub(r"\s*\|\s*None\b", "", field.type_repr)
+    t = re.sub(r"\bNone\s*\|\s*", "", t)
+    for model_name in field.nested_models:
+        display = _display_name(model_name)
+        # Use word-boundary replacement to avoid matching model_name as a substring
+        # of a longer class name (e.g. "BarChart" inside "SparkBarChart").
+        t = re.sub(
+            r"\b" + re.escape(model_name) + r"\b",
+            f"[{display}](#{_anchor(display)})",
+            t,
+        )
+    return t.replace("|", "\\|")
+
+
+def _anchor(display_name: str) -> str:
+    return display_name.lower()
+
+
+def _render_model(name: str, schema: AuthorableSchema) -> str:
+    model = schema.models[name]
+    display = _display_name(name)
+    lines = [f'<a id="{_anchor(display)}"></a>', f"## {display}"]
+    if model.doc:
+        lines.append(model.doc)
+    lines.append("")
+
+    required = [f for f in model.fields if f.required]
+    optional = [f for f in model.fields if not f.required]
+    fields = required + optional
+
+    show_optional = any(not f.required for f in fields)
+
+    header = "| Field | Type |"
+    sep = "|-------|------|"
+    if show_optional:
+        header += " Optional |"
+        sep += ":--------:|"
+    header += " Description |"
+    sep += "-------------|"
+
+    lines.append(header)
+    lines.append(sep)
+
+    for field in fields:
+        desc = (field.description or "").replace("|", "\\|")
+        row = f"| `{field.name}` | {_type_cell(field)} |"
+        if show_optional:
+            row += f" {'✓' if not field.required else ''} |"
+        row += f" {desc} |"
+        lines.append(row)
+
+    return "\n".join(lines)
+
+
+def render_prompt(schema: AuthorableSchema) -> str:
+    """Render an AuthorableSchema IR as a YAML schema reference table.
+
+    Suitable for inclusion in AI prompts. Each authored model is rendered as a
+    markdown table with required fields first, then optional fields.
+    Models are ordered top-down: root first, then referenced sub-models (BFS),
+    then chart patch models, then source connector configs.
+    """
+    order = _ordered_model_names(schema)
+    parts = ["# Dataface YAML Schema Reference", ""]
+    for i, name in enumerate(order):
+        if i > 0:
+            parts.append("")
+        parts.append(_render_model(name, schema))
+    return "\n".join(parts)