dataface 0.1.2__py3-none-any.whl

dataface/core/compile/compiler.py

@@ -0,0 +1,904 @@
+"""AuthoredFace compiler module.
+
+Stage: COMPILE
+Purpose: Compile YAML face definitions into Face objects
+ready for execution and rendering.
+
+Entry Points:
+    - compile(yaml_content: str) -> CompileResult
+    - compile_file(file_path: Path) -> CompileResult
+
+This is the main orchestrator for compilation. It:
+1. Parses YAML to AuthoredFace (parser.py)
+2. Validates the face structure (validator.py)
+3. Normalizes references and adds metadata (normalizer.py)
+
+Layout dimensions are calculated later in the render pipeline (after query
+execution) so table heights can use actual row counts.
+
+Dependencies:
+    - .parser (parse_yaml)
+    - .validator (validate_face)
+    - .normalizer (normalize_face)
+    - .jinja (detect_query_dependencies)
+    - .errors (CompilationError, etc.)
+
+See also:
+    - docs/docs/contributing/architecture.md for the pipeline overview
+    - execute/executor.py for the next stage
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import yaml
+from pydantic import ValidationError as PydanticValidationError
+
+from dataface.core.compile.models.face.compiled import Face
+from dataface.core.compile.models.query.compiled import AnyQuery
+from dataface.core.errors.structured import StructuredError
+from dataface.core.render.warnings import unreferenced_chart
+from dataface.core.render.warnings.base import RenderWarning
+from dataface.core.render.warnings.from_query_diagnostic import from_query_diagnostic
+
+logger = logging.getLogger(__name__)
+from dataface.core.compile.errors import (
+    CompilationError,
+    JinjaError,
+    ParseError,
+    ReferenceError,
+)
+from dataface.core.compile.jinja import detect_query_dependencies
+from dataface.core.compile.meta import get_meta_for_face, merge_meta_with_face
+from dataface.core.compile.models.face.authored import AuthoredFace
+from dataface.core.compile.models.refs import ChartRef, QueryRef, VariableRef
+from dataface.core.compile.models.variable.authored import Variable
+from dataface.core.compile.normalizer import normalize_face, normalize_query
+from dataface.core.compile.parser import parse_yaml
+from dataface.core.compile.validator import validate_face
+from dataface.core.compile.yaml_error_formatter import (
+    format_validation_errors_structured,
+)
+
+if TYPE_CHECKING:
+    from dataface.core.inspect.query_validator import (
+        QueryDiagnostic,
+        RelationshipContext,
+    )
+
+
+def _to_plain_dict(obj: Any) -> Any:
+    """Recursively convert DotDict (or any dict subclass) to plain dict."""
+    if isinstance(obj, dict):
+        return {k: _to_plain_dict(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_to_plain_dict(v) for v in obj]
+    return obj
+
+
+def resolve_project_source_paths(
+    sources: dict[str, dict[str, Any]],
+    project_dir: Path,
+) -> dict[str, dict[str, Any]]:
+    """Resolve relative file/database paths in project-level source configs."""
+    resolved_sources: dict[str, dict[str, Any]] = {}
+
+    for name, config in sources.items():
+        resolved_config = dict(config)
+        source_type = resolved_config.get("type")
+
+        if source_type == "duckdb":
+            path_value = resolved_config.get("path")
+            if (
+                isinstance(path_value, str)
+                and path_value != ":memory:"
+                and not Path(path_value).is_absolute()
+            ):
+                resolved_config["path"] = str((project_dir / path_value).resolve())
+        elif source_type in {"csv", "parquet", "json"}:
+            file_value = resolved_config.get("file")
+            if isinstance(file_value, str) and not Path(file_value).is_absolute():
+                resolved_config["file"] = str((project_dir / file_value).resolve())
+
+        resolved_sources[name] = resolved_config
+
+    return resolved_sources
+
+
+@dataclass
+class CompileResult:
+    """Result of compilation.
+
+    Contains the compiled face (if successful), any errors encountered,
+    and warnings that don't prevent compilation.
+
+    Attributes:
+        face: Compiled face (None if errors occurred)
+        errors: Structured compile errors (list[StructuredError] — union-aware collapse applied)
+        warnings: Non-fatal warnings as ``RenderWarning`` objects with stable
+            SCREAMING_SNAKE_CASE codes. Each emitter (orphan-chart check,
+            validate_compiled_queries) owns its code on a module under
+            ``dataface.core.render.warnings``.
+        suppressed_warnings: ``RenderWarning`` entries that matched a
+            suppression layer (query-level ``ignore:`` or meta.yaml lint
+            config). Kept separate so consumers can surface "would have
+            warned" without re-running validation.
+        diagnostics: Structured query diagnostics from validate_compiled_queries.
+            Includes all severity levels; ``warnings`` only includes warning+error.
+            Infrastructure for structured consumers (UI, AI agents).
+        query_registry: All normalized queries (for executor)
+        sources: Source configurations (for resolving named source references)
+
+    Example:
+        >>> result = compile(yaml_content)
+        >>> if result.success:
+        ...     print(f"Compiled: {result.face.title}")
+        ... else:
+        ...     for error in result.errors:
+        ...         print(f"Error: {error}")
+    """
+
+    face: Face | None = None
+    errors: list[StructuredError] = field(default_factory=list)
+    warnings: list[RenderWarning] = field(default_factory=list)
+    suppressed_warnings: list[RenderWarning] = field(default_factory=list)
+    diagnostics: list[QueryDiagnostic] = field(default_factory=list)
+    query_registry: dict[str, AnyQuery] = field(default_factory=dict)
+    meta_config: Any = None
+
+    @property
+    def success(self) -> bool:
+        """Check if compilation was successful."""
+        return self.face is not None and len(self.errors) == 0
+
+
+def compile(
+    yaml_content: str,
+    options: dict[str, Any] | None = None,
+    base_dir: Path | None = None,
+    project_dir: Path | None = None,
+) -> CompileResult:
+    """Compile YAML content to a Face.
+
+    Stage: COMPILE (Full Pipeline)
+
+    This is the main entry point for compilation. It orchestrates:
+    1. PARSE: YAML string → AuthoredFace object
+    2. VALIDATE: Check structure and references
+    3. NORMALIZE: Resolve references, add metadata
+
+    Layout sizing happens later in the render pipeline (data-aware).
+
+    Args:
+        yaml_content: YAML string to compile
+        options: Optional compilation options
+        base_dir: Base directory for resolving file references
+        project_dir: Project directory for resolving project-level sources
+
+    Returns:
+        CompileResult with compiled face or errors
+
+    Example:
+        >>> yaml_content = '''
+        ... title: My Dataface
+        ... queries:
+        ...   users: SELECT * FROM users
+        ... charts:
+        ...   user_count:
+        ...     query: users
+        ...     type: kpi
+        ...     value: count
+        ... rows:
+        ...   - user_count
+        ... '''
+        >>> result = compile(yaml_content)
+        >>> if result.success:
+        ...     face = result.face
+        ...     print(face.title)  # "My Dataface"
+    """
+    options = options or {}
+    errors: list[StructuredError] = []
+    warnings: list[RenderWarning] = []
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 1: Parse YAML
+    # ════════════════════════════════════════════════════════════════════
+    # Convert YAML string to AuthoredFace object. Handles syntax errors.
+    try:
+        face = parse_yaml(yaml_content)
+    except ParseError as e:
+        # When the parse error was caused by a PydanticValidationError, use the
+        # union-aware collapse to produce structured errors from the raw Pydantic
+        # error list. Other parse errors (bad YAML syntax, empty doc) produce one
+        # structured error from the CompilationError shape.
+        if isinstance(e.__cause__, PydanticValidationError):
+            return CompileResult(
+                errors=format_validation_errors_structured(e.__cause__, yaml_content)
+            )
+        return CompileResult(errors=[e.to_structured()])
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 2: Validate Structure
+    # ════════════════════════════════════════════════════════════════════
+    # Check references and semantic constraints.
+    validation_errors = validate_face(face)
+    if validation_errors:
+        return CompileResult(errors=[e.to_structured() for e in validation_errors])
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 3: Get Default Source and Extract Named Sources
+    # ════════════════════════════════════════════════════════════════════
+    # Get project-level sources when available so named source references and
+    # project defaults behave consistently for file-based compiles.
+    project_sources = None
+    if project_dir is not None:
+        from dataface.core.compile.config import get_project_sources
+
+        project_sources = get_project_sources(project_dir)
+
+    # Get the face's default source to pass to query normalization.
+    default_source = face.get_default_source()
+    if default_source is None and project_sources is not None:
+        default_source = project_sources.default
+
+    # Extract named source configurations from the sources section
+    # These are used to resolve source references in queries (e.g., source: profiles)
+    sources_registry = {}
+    if project_sources is not None:
+        assert project_dir is not None
+        sources_registry.update(
+            resolve_project_source_paths(project_sources.sources, project_dir)
+        )
+    sources_registry.update(_extract_sources(face))
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 4: Build Query Registry
+    # ════════════════════════════════════════════════════════════════════
+    # Collect and normalize all queries before normalization.
+    # This allows charts to reference queries from any part of the face.
+    try:
+        query_registry = build_query_registry(
+            face, base_dir, default_source=default_source
+        )
+    except CompilationError as e:
+        return CompileResult(errors=[e.to_structured()])
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 5: Detect Circular Dependencies
+    # ════════════════════════════════════════════════════════════════════
+    # Check for {{ queries.* }} cycles in SQL.
+    if query_registry:
+        try:
+            detect_query_dependencies(query_registry)
+        except JinjaError as e:
+            return CompileResult(errors=[e.to_structured()])
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 5b: Build Chart Registry
+    # ════════════════════════════════════════════════════════════════════
+    # Collect all charts from the entire face tree (global namespace).
+    # Charts are global like queries - any layout can reference any chart.
+    try:
+        chart_registry = build_chart_registry(face, base_dir=base_dir)
+    except CompilationError as e:
+        return CompileResult(errors=[e.to_structured()])
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 6: Normalize (Transform to Face)
+    # ════════════════════════════════════════════════════════════════════
+    # Resolve references, add metadata, create unified layout.
+    # Note: Variable registry is built by renderer at render time (not needed here)
+    try:
+        compiled = normalize_face(
+            face,
+            query_registry=query_registry,
+            chart_registry=chart_registry,
+            base_path=base_dir,
+        )
+    except ReferenceError as e:
+        return CompileResult(errors=[e.to_structured()])
+    except CompilationError as e:
+        return CompileResult(errors=[e.to_structured()])
+    except PydanticValidationError as e:
+        return CompileResult(
+            errors=format_validation_errors_structured(e, yaml_content)
+        )
+
+    compiled.sources = _to_plain_dict(sources_registry)
+
+    # Orphan charts: defined but never referenced from any layout in the tree.
+    # Warn rather than error — a content-only face still renders, the warning
+    # surfaces lazy authoring. Render-side has the harder check for the
+    # specific case of "face has charts but layout is empty" (Case A) which
+    # would produce a silently empty dashboard.
+    for chart_id in _collect_orphan_charts(compiled):
+        warnings.append(
+            RenderWarning(
+                code=unreferenced_chart.CODE,
+                chart=chart_id,
+                message=(
+                    f"Chart '{chart_id}' is defined but not referenced in any layout."
+                ),
+                fix=(
+                    "Add it to `rows:`/`cols:`/`grid:`/`tabs:` "
+                    "or delete the definition."
+                ),
+            )
+        )
+
+    # Layout dimensions are calculated later in the render pipeline
+    # (after query execution) so table heights can use actual row counts.
+
+    return CompileResult(
+        face=compiled,
+        errors=errors,
+        warnings=warnings,
+        query_registry=query_registry,
+        meta_config=options.get("meta_config"),
+    )
+
+
+def _collect_orphan_charts(face: Face) -> list[str]:
+    """Return chart IDs defined somewhere in the face tree but never placed
+    in any layout. Walks nested faces; chart references resolve globally
+    (a parent-defined chart referenced from a nested face is not an orphan).
+    """
+    defined: set[str] = set()
+    referenced: set[str] = set()
+
+    def walk(f: Face) -> None:
+        if f.charts:
+            defined.update(f.charts.keys())
+        for item in f.layout.items:
+            if item.type == "chart" and item.chart and item.chart.id:
+                referenced.add(item.chart.id)
+            elif item.type == "face" and item.face is not None:
+                walk(item.face)
+
+    walk(face)
+    return sorted(defined - referenced)
+
+
+def _collect_suppressed_codes(
+    result: CompileResult,
+    meta_config: Any = None,
+) -> dict[str, set[str]]:
+    """Build per-query suppression sets from query ignore + meta.yaml lint config.
+
+    Returns a dict mapping query_name → set of suppressed diagnostic codes.
+    """
+    per_query: dict[str, set[str]] = {}
+    # Layer 2: per-query ignore field
+    for name, query in result.query_registry.items():
+        codes: set[str] = set()
+        if query.ignore:
+            codes.update(query.ignore)
+        per_query[name] = codes
+
+    # Layer 3: meta.yaml lint config
+    if meta_config is not None:
+        global_ignore = set(getattr(meta_config, "lint_ignore", []))
+        per_query_meta = getattr(meta_config, "lint_ignore_queries", {})
+        for name in per_query:
+            per_query[name] |= global_ignore
+            if name in per_query_meta:
+                per_query[name].update(per_query_meta[name])
+
+    return per_query
+
+
+def validate_compiled_queries(
+    result: CompileResult,
+    relationship_context: RelationshipContext | None = None,
+) -> None:
+    """Run query validation on all SQL queries in a compile result (mutates in place).
+
+    Calls ``validate_query`` for each SQL query, appends structured
+    ``QueryDiagnostic`` objects to ``result.diagnostics``, and emits
+    ``RenderWarning`` entries onto ``result.warnings`` for findings at
+    warning severity or above. Each warning carries a stable code owned by
+    a module under ``dataface.core.render.warnings`` (see
+    ``from_query_diagnostic`` for the mapping).
+
+    Honors diagnostic suppressions from:
+    - SQL-inline ``-- dft:ignore`` comments (handled by validate_query)
+    - Per-query ``ignore`` field
+    - meta.yaml ``lint`` config
+
+    Args:
+        result: CompileResult to enrich.
+        relationship_context: Optional RelationshipContext for severity calibration.
+    """
+    if not result.success or not result.query_registry:
+        return
+
+    from dataface.core.compile.models.query.compiled import is_sql_query
+    from dataface.core.inspect.query_validator import validate_query
+
+    suppressed = _collect_suppressed_codes(result, result.meta_config)
+
+    for name, query in result.query_registry.items():
+        if not is_sql_query(query):
+            continue
+        query_suppress = suppressed.get(name, set())
+        diags, suppressed_diags = validate_query(
+            query.sql,
+            relationship_context=relationship_context,
+            suppress=query_suppress,
+            return_suppressed=True,
+        )
+        for d in diags:
+            result.diagnostics.append(d)
+            if d.severity in ("error", "warning"):
+                result.warnings.append(from_query_diagnostic(name, d))
+        for d in suppressed_diags:
+            result.suppressed_warnings.append(from_query_diagnostic(name, d))
+
+
+def compile_file(
+    file_path: Path,
+    options: dict[str, Any] | None = None,
+    apply_meta: bool = True,
+    root_path: Path | None = None,
+) -> CompileResult:
+    """Compile a YAML file to a Face.
+
+    Convenience function that reads a file and compiles it. Optionally
+    applies meta.yaml cascading configuration from parent directories.
+
+    Args:
+        file_path: Path to YAML file
+        options: Optional compilation options
+        apply_meta: If True, resolve and apply meta.yaml chain (default: True)
+        root_path: Project root for meta resolution (default: auto-detect)
+
+    Returns:
+        CompileResult with compiled face or errors
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+
+    Example:
+        >>> result = compile_file(Path("face.yml"))
+        >>> if result.success:
+        ...     print(result.face.title)
+    """
+
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    if not file_path.exists():
+        return CompileResult(
+            errors=[CompilationError(f"File not found: {file_path}").to_structured()]
+        )
+
+    # Markdown report files: translate to YAML before compiling
+    if file_path.suffix.lower() == ".md":
+        from dataface.core.compile.markdown import is_markdown_face, markdown_to_yaml
+
+        if not is_markdown_face(file_path):
+            return CompileResult(
+                errors=[
+                    CompilationError(
+                        "Not a Dataface face file: .md files require YAML frontmatter "
+                        "with at least one of: queries, charts, variables, source, sources"
+                    ).to_structured()
+                ]
+            )
+
+        try:
+            raw_text = file_path.read_text()
+            yaml_content = markdown_to_yaml(raw_text)
+        except (OSError, ValueError) as e:
+            return CompileResult(
+                errors=[CompilationError(f"Markdown parse error: {e}").to_structured()]
+            )
+    else:
+        try:
+            yaml_content = file_path.read_text()
+        except OSError as e:
+            return CompileResult(
+                errors=[CompilationError(f"Failed to read file: {e}").to_structured()]
+            )
+
+    # Apply meta.yaml chain if enabled
+    meta_config = None
+    if apply_meta:
+        try:
+            # Parse YAML to dict first
+            face_data = yaml.safe_load(yaml_content) or {}
+
+            if not isinstance(face_data, dict):
+                return CompileResult(
+                    errors=[
+                        CompilationError(
+                            f"Face YAML must be a mapping at the top level, got {type(face_data).__name__}"
+                        ).to_structured()
+                    ]
+                )
+
+            # Resolve meta chain and merge with face
+            meta_config = get_meta_for_face(file_path, root_path=root_path)
+            if not meta_config.is_empty():
+                merged_data = merge_meta_with_face(meta_config, face_data)
+                # Re-serialize to YAML for the compile function
+                yaml_content = yaml.dump(
+                    merged_data, default_flow_style=False, sort_keys=False
+                )
+        except CompilationError as e:
+            return CompileResult(errors=[e.to_structured()])
+        except (yaml.YAMLError, OSError) as e:
+            # Meta resolution errors are warnings, not failures
+            # Continue with original content, but log the issue for debugging
+            logger.warning(
+                "Meta resolution failed for %s, continuing with original content: %s",
+                file_path,
+                e,
+            )
+
+    project_dir = root_path
+    if project_dir is None:
+        from dataface.core.project_roots import discover_render_context
+
+        compile_boundary = Path(file_path.anchor) if file_path.anchor else None
+        project_dir, _ = discover_render_context(file_path.parent, compile_boundary)
+
+    # Thread meta lint config through to compile for query diagnostics
+    compile_options = dict(options) if options else {}
+    if meta_config is not None:
+        compile_options["meta_config"] = meta_config
+
+    result = compile(
+        yaml_content,
+        options=compile_options,
+        base_dir=file_path.parent,
+        project_dir=project_dir,
+    )
+    # Stamp file path and add validate next_command on each compile error so
+    # UI surfaces and CLI can surface "dft validate <file>" without re-deriving the path.
+    if result.errors:
+        file_str = str(file_path)
+        result.errors = [_stamp_error_file(e, file_str) for e in result.errors]
+    return result
+
+
+def _stamp_error_file(err: StructuredError, file_str: str) -> StructuredError:
+    """Return a copy of err with file set and a Validate next_command added."""
+    from dataface.core.errors.structured import NextCommand
+
+    next_commands = list(err.next_commands)
+    if not any(nc.label == "Validate" for nc in next_commands):
+        next_commands.append(
+            NextCommand(label="Validate", command=f"dft validate {file_str}")
+        )
+    return err.model_copy(update={"file": file_str, "next_commands": next_commands})
+
+
+def _extract_nested_faces(face: AuthoredFace) -> list[AuthoredFace]:
+    """Extract all nested faces from a face's layout.
+
+    Recursively traverses rows, cols, grid, and tabs to find all nested faces.
+
+    Args:
+        face: AuthoredFace to extract nested faces from
+
+    Returns:
+        List of nested AuthoredFace objects found in the layout
+    """
+    from dataface.core.compile.models.face.authored import (
+        ForeachItem,
+        TabItem,
+    )
+
+    nested_faces: list[AuthoredFace] = []
+    items: list[Any] = []
+
+    # Collect items from rows and cols
+    if face.rows:
+        items.extend(face.rows)
+    if face.cols:
+        items.extend(face.cols)
+
+    # Collect items from grid (always GridLayout now — dict form removed)
+    if face.grid:
+        items.extend([gi.item for gi in face.grid.items])
+
+    # Collect items from tabs (always TabLayout now — dict form removed)
+    if face.tabs:
+        items.extend(face.tabs.items)
+
+    # Process all items to find nested faces
+    for item in items:
+        if isinstance(item, AuthoredFace):
+            nested_faces.append(item)
+        elif isinstance(item, TabItem):
+            # Tab items contain their own rows/cols — treat as an inline face
+            tab_face = AuthoredFace.model_construct(
+                rows=item.rows,
+                cols=item.cols,
+                grid=item.grid,
+                tabs=item.tabs,
+            )
+            nested_faces.append(tab_face)
+        elif isinstance(item, ForeachItem):
+            # Recurse into foreach items to find any nested faces
+            for foreach_subitem in item.foreach.items:
+                if isinstance(foreach_subitem, AuthoredFace):
+                    nested_faces.append(foreach_subitem)
+
+    return nested_faces
+
+
+def _build_registry(
+    face: AuthoredFace,
+    registry_type: str,
+    registry: dict[str, Any] | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """Build a registry for queries or charts recursively.
+
+    This unified function traverses the entire face tree to collect definitions
+    of the specified type. Each face processes its own definitions, then recursively
+    processes all nested faces and merges their results.
+
+    Note: Variables are built at render time, not compile time, so they don't use
+    this registry building function.
+
+    Args:
+        face: AuthoredFace to process
+        registry_type: Type of registry to build ("queries" or "charts")
+        registry: Existing registry to add to (will be created if None)
+        **kwargs: Additional arguments specific to registry type:
+            - For "queries": base_dir, default_source
+            - For "charts": base_dir
+
+    Returns:
+        Complete registry dictionary
+
+    Raises:
+        CompilationError: If duplicate names found or invalid registry_type
+    """
+    if registry is None:
+        registry = {}
+
+    # Process definitions at this face level
+    if registry_type == "queries":
+        _process_queries_for_registry(face, registry, **kwargs)
+    elif registry_type == "charts":
+        _process_charts_for_registry(face, registry, **kwargs)
+    else:
+        raise CompilationError(f"Invalid registry type: {registry_type}")
+
+    # Recursively process nested faces
+    nested_faces = _extract_nested_faces(face)
+    for nested_face in nested_faces:
+        _build_registry(nested_face, registry_type, registry, **kwargs)
+
+    return registry
+
+
+def _process_queries_for_registry(
+    face: AuthoredFace,
+    registry: dict[str, AnyQuery],
+    base_dir: Path | None = None,
+    default_source: str | None = None,
+) -> None:
+    """Process queries from a face and add to registry.
+
+    Args:
+        face: AuthoredFace to process queries from
+        registry: Registry to add queries to
+        base_dir: Base directory for cross-file references
+        default_source: Default source to apply to queries without explicit source
+
+    Raises:
+        CompilationError: If duplicate query names found
+    """
+    # Get default source from this face level (may override parent)
+    face_default_source = face.get_default_source()
+    effective_default_source = face_default_source or default_source
+
+    # Process queries at this level
+    for name, query_def in (face.queries or {}).items():
+        if name in registry:
+            raise CompilationError(
+                f"Duplicate query name '{name}'. Query names must be unique "
+                "within a file (and those that are imported)."
+            )
+
+        # Handle cross-file references
+        if isinstance(query_def, QueryRef):
+            registry[name] = load_from_reference(query_def, base_dir=base_dir)
+        else:
+            registry[name] = normalize_query(
+                name, query_def, default_source=effective_default_source
+            )
+
+
+def _process_charts_for_registry(
+    face: AuthoredFace,
+    registry: dict[str, Any],
+    base_dir: Path | None = None,
+    **kwargs: Any,
+) -> None:
+    """Process charts from a face and add to registry.
+
+    Args:
+        face: AuthoredFace to process charts from
+        registry: Registry to add charts to
+        base_dir: Base directory for cross-file references
+        **kwargs: Additional arguments (e.g., query_registry for future use)
+
+    Raises:
+        CompilationError: If duplicate chart names found
+    """
+    # Process charts at this level
+    for name, chart_def in (face.charts or {}).items():
+        if name in registry:
+            raise CompilationError(
+                f"Duplicate chart name '{name}'. Chart names must be unique "
+                "within a file (and those that are imported)."
+            )
+
+        # Handle cross-file references
+        if isinstance(chart_def, ChartRef):
+            registry[name] = load_from_reference(chart_def, base_dir=base_dir)
+        else:
+            registry[name] = chart_def
+
+
+def build_query_registry(
+    face: AuthoredFace,
+    base_dir: Path | None = None,
+    registry: dict[str, AnyQuery] | None = None,
+    default_source: str | None = None,
+) -> dict[str, AnyQuery]:
+    """Build complete query registry from face and nested faces.
+
+    Traverses the entire face structure to collect all query definitions,
+    including from nested faces.
+
+    Args:
+        face: AuthoredFace to process
+        base_dir: Base directory for cross-file references
+        registry: Existing registry to add to
+        default_source: Default source to apply to queries without explicit source
+
+    Returns:
+        Complete query registry
+
+    Raises:
+        CompilationError: If duplicate query names found
+    """
+    return _build_registry(
+        face,
+        "queries",
+        registry=registry,
+        base_dir=base_dir,
+        default_source=default_source,
+    )
+
+
+def build_chart_registry(
+    face: AuthoredFace,
+    base_dir: Path | None = None,
+    registry: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build complete chart registry from face and nested faces.
+
+    Traverses the entire face structure to collect all chart definitions,
+    including from nested faces. Charts are global - any layout can reference any chart.
+
+    Args:
+        face: AuthoredFace to process
+        base_dir: Base directory for cross-file references
+        registry: Existing registry to add to
+
+    Returns:
+        Complete chart registry (raw chart definitions, not normalized)
+
+    Raises:
+        CompilationError: If duplicate chart names found
+    """
+    return _build_registry(face, "charts", registry=registry, base_dir=base_dir)
+
+
+def load_from_reference(
+    reference: VariableRef | QueryRef | ChartRef,
+    base_dir: Path | None = None,
+) -> Any:
+    """Load an item from a typed cross-file reference.
+
+    Args:
+        reference: A typed ref model (VariableRef, QueryRef, or ChartRef).
+            The grammar has already been validated by the Pydantic model.
+        base_dir: Base directory for resolving paths
+
+    Returns:
+        - QueryRef → AnyQuery
+        - ChartRef → chart dict
+        - VariableRef → Variable
+
+    Raises:
+        CompilationError: If reference cannot be resolved
+    """
+    if isinstance(reference, VariableRef):
+        section_name = "variables"
+    elif isinstance(reference, QueryRef):
+        section_name = "queries"
+    elif isinstance(reference, ChartRef):
+        section_name = "charts"
+    else:
+        raise CompilationError(f"Unexpected reference type: {type(reference)!r}")
+
+    # Grammar already validated by the typed model; split is deterministic.
+    file_path_str, item_name = reference.ref.rsplit(f".{section_name}.", 1)
+
+    # Add extension if needed
+    if not (file_path_str.endswith(".yml") or file_path_str.endswith(".yaml")):
+        file_path_str += ".yml"
+
+    full_path = base_dir / file_path_str if base_dir else Path(file_path_str)
+
+    if not full_path.exists():
+        raise CompilationError(f"Referenced file not found: {file_path_str}")
+
+    try:
+        with open(full_path) as f:
+            content = yaml.safe_load(f)
+    except (OSError, yaml.YAMLError) as e:
+        raise CompilationError(f"Failed to load {file_path_str}: {e}") from e
+
+    if not content or section_name not in content:
+        raise CompilationError(f"No {section_name} found in {file_path_str}")
+
+    section = content[section_name]
+    singular = {"queries": "query", "charts": "chart", "variables": "variable"}.get(
+        section_name, section_name[:-1]
+    )
+
+    if not isinstance(section, dict) or item_name not in section:
+        raise CompilationError(
+            f"{singular.capitalize()} '{item_name}' not found in {file_path_str}"
+        )
+
+    item_def = section[item_name]
+
+    if section_name == "queries":
+        return normalize_query(item_name, item_def)
+    if section_name == "charts":
+        return item_def
+    # section_name == "variables"
+    if isinstance(item_def, Variable):
+        return item_def
+    if isinstance(item_def, dict):
+        return Variable(**item_def)
+    raise CompilationError(
+        f"Invalid variable definition for '{item_name}' in {file_path_str}"
+    )
+
+
+def _extract_sources(face: AuthoredFace) -> dict[str, dict]:
+    """Extract named source configurations from AuthoredFace.
+
+    Extracts named source definitions (excluding 'default') into a dict.
+
+    Args:
+        face: AuthoredFace with optional sources section
+
+    Returns:
+        Dict mapping source name to source config dict
+    """
+    if not face.sources:
+        return {}
+
+    # Extract named sources (everything except 'default')
+    return {
+        k: v
+        for k, v in face.sources.model_dump(exclude_none=True).items()
+        if k != "default"
+    }