weevr-core 1.18.0__py3-none-any.whl

weevr/config/__init__.py

@@ -0,0 +1,265 @@
+"""Configuration loading and validation."""
+
+from pathlib import Path
+from typing import Any
+
+from pydantic import ValidationError
+
+from weevr.config.inheritance import apply_inheritance
+from weevr.config.locations import ConfigLocation, LocalConfigLocation, make_location
+from weevr.config.macros import expand_foreach
+from weevr.config.parser import (
+    detect_config_type,
+    detect_config_type_from_extension,
+    extract_config_version,
+    parse_yaml,
+    validate_config_version,
+)
+from weevr.config.resolver import (
+    build_param_context,
+    resolve_declared_params,
+    resolve_references,
+    resolve_variables,
+    validate_params,
+)
+from weevr.config.validation import validate_schema
+from weevr.errors import ConfigError, ModelValidationError
+from weevr.model import Loom, Thread, Weave
+
+
+def _derive_config_name(path: ConfigLocation) -> str:
+    """Derive the component name from a config location.
+
+    Returns the filename stem — the filename without the typed extension.
+    For example, ``dim_customer.thread`` returns ``'dim_customer'``.
+    """
+    return path.stem
+
+
+def load_config(
+    path: str | Path | ConfigLocation,
+    runtime_params: dict[str, Any] | None = None,
+    project_root: Path | ConfigLocation | None = None,
+) -> Thread | Weave | Loom | dict[str, Any]:
+    """Load and validate a weevr configuration file.
+
+    This function orchestrates the full config loading pipeline:
+    1. Parse YAML file
+    2. Extract and validate config_version
+    3. Detect config type (extension-based for components, content-based for params)
+    4. Validate schema with Pydantic
+    5. Build parameter context (runtime > defaults)
+    6. Resolve variable references (${var} and ${var:-default})
+    7. Resolve references to child configs (threads, weaves)
+    8. Apply inheritance cascade (loom -> weave -> thread)
+    9. Validate name against filename stem
+    10. Hydrate into typed domain model (thread, weave, loom only)
+
+    Args:
+        path: Path or :class:`ConfigLocation` for the config file (thread,
+            weave, or loom). Local paths are wrapped automatically; remote
+            URIs must be supplied as a :class:`RemoteConfigLocation` so the
+            caller can attach a SparkSession.
+        runtime_params: Optional runtime parameter overrides.
+        project_root: The ``.weevr`` project directory. Required for configs
+            that reference other files. Accepts a :class:`ConfigLocation` or
+            a bare :class:`pathlib.Path`.
+
+    Returns:
+        A frozen, typed domain model instance (Thread, Weave, or Loom) for
+        thread/weave/loom config types. Returns a plain dict for params configs.
+
+    Raises:
+        ConfigParseError: YAML syntax errors, file not found
+        ConfigVersionError: Unsupported config version
+        ConfigSchemaError: Schema validation failures
+        ConfigError: Extension or name validation failures
+        VariableResolutionError: Unresolved variables without defaults
+        ReferenceResolutionError: Missing referenced files, circular dependencies
+        ModelValidationError: Semantic validation failures during model hydration
+    """
+    file_path = make_location(path)
+
+    # Step 1: Parse YAML
+    raw = parse_yaml(file_path)
+
+    # Step 2: Extract and validate version
+    version = extract_config_version(raw)
+
+    # Step 3: Detect config type — extension-based for typed files, content-based for .yaml
+    ext_type = detect_config_type_from_extension(file_path)
+    config_type = ext_type if ext_type is not None else detect_config_type(raw)
+
+    # Step 4: Validate schema
+    validate_config_version(version, config_type)
+    validated = validate_schema(raw, config_type)
+    config_dict = validated.model_dump(exclude_unset=True)
+
+    # Step 5: Build parameter context (no more param_file)
+    declared_param_specs = config_dict.get("params")
+    resolved_declared_params = resolve_declared_params(
+        declared_param_specs,
+        runtime_params,
+        file_path=str(file_path),
+    )
+    # Type validation only — required-missing already raised above.
+    validate_params(declared_param_specs, resolved_declared_params)
+    context = build_param_context(
+        runtime_params,
+        config_dict.get("defaults"),
+        entry_params=resolved_declared_params or None,
+    )
+
+    # Step 6: Resolve variables
+    resolved = resolve_variables(config_dict, context)
+
+    # Step 7: Resolve references to child configs
+    if project_root is None:
+        effective_root: ConfigLocation = file_path.parent
+    elif isinstance(project_root, ConfigLocation):
+        effective_root = project_root
+    else:
+        effective_root = LocalConfigLocation(Path(project_root))
+    resolved_with_refs = resolve_references(
+        resolved,
+        config_type,
+        effective_root,
+        runtime_params,
+    )
+
+    # Step 8: Apply inheritance for child configs
+    if config_type == "loom" and "_resolved_weaves" in resolved_with_refs:
+        loom_defaults = resolved_with_refs.get("defaults")
+        loom_audit_templates = resolved_with_refs.get("audit_templates")
+        loom_connections = resolved_with_refs.get("connections")
+        for weave in resolved_with_refs["_resolved_weaves"]:
+            if "_resolved_threads" in weave:
+                weave_defaults = weave.get("defaults")
+                weave_audit_templates = weave.get("audit_templates")
+                weave_connections = weave.get("connections")
+                for i, thread in enumerate(weave["_resolved_threads"]):
+                    merged = apply_inheritance(
+                        loom_defaults,
+                        weave_defaults,
+                        thread,
+                        loom_audit_templates=loom_audit_templates,
+                        weave_audit_templates=weave_audit_templates,
+                        loom_connections=loom_connections,
+                        weave_connections=weave_connections,
+                    )
+                    weave["_resolved_threads"][i] = merged
+
+    elif config_type == "weave" and "_resolved_threads" in resolved_with_refs:
+        weave_defaults = resolved_with_refs.get("defaults")
+        weave_audit_templates = resolved_with_refs.get("audit_templates")
+        weave_connections = resolved_with_refs.get("connections")
+        for i, thread in enumerate(resolved_with_refs["_resolved_threads"]):
+            merged = apply_inheritance(
+                None,
+                weave_defaults,
+                thread,
+                weave_audit_templates=weave_audit_templates,
+                weave_connections=weave_connections,
+            )
+            resolved_with_refs["_resolved_threads"][i] = merged
+
+    # Step 8b: Expand foreach macros in thread steps
+    if config_type == "thread" and isinstance(resolved_with_refs.get("steps"), list):
+        resolved_with_refs["steps"] = expand_foreach(resolved_with_refs["steps"])
+
+    # Step 9: Name validation and injection
+    if config_type in ("thread", "weave", "loom") and ext_type is not None:
+        stem = _derive_config_name(file_path)
+        declared_name = resolved_with_refs.get("name", "")
+        if declared_name and declared_name != stem:
+            raise ConfigError(
+                f"Declared name '{declared_name}' does not match filename stem '{stem}'",
+                file_path=str(file_path),
+            )
+        if not declared_name:
+            resolved_with_refs["name"] = stem
+
+        # Compute qualified key for standalone files
+        if project_root is not None and ext_type is not None:
+            if file_path.is_relative_to(effective_root):
+                # Strip the project_root prefix to produce a relative key.
+                # For local locations this matches the previous Path.resolve()
+                # / Path.relative_to behavior; for remote locations it is a
+                # normalized URI prefix strip.
+                if isinstance(file_path, LocalConfigLocation) and isinstance(
+                    effective_root, LocalConfigLocation
+                ):
+                    rel = file_path.path.resolve().relative_to(effective_root.path.resolve())
+                    resolved_with_refs["qualified_key"] = str(rel)
+                else:
+                    file_str = str(file_path)
+                    root_str = str(effective_root).rstrip("/")
+                    resolved_with_refs["qualified_key"] = (
+                        file_str[len(root_str) + 1 :]
+                        if file_str.startswith(root_str + "/")
+                        else file_str
+                    )
+            else:
+                resolved_with_refs["qualified_key"] = str(file_path)
+    elif not resolved_with_refs.get("name"):
+        resolved_with_refs["name"] = file_path.stem
+
+    # Step 10: Hydrate into typed domain model
+    model_map: dict[str, type[Thread | Weave | Loom]] = {
+        "thread": Thread,
+        "weave": Weave,
+        "loom": Loom,
+    }
+    if config_type in model_map:
+        try:
+            return model_map[config_type].model_validate(resolved_with_refs)
+        except ValidationError as exc:
+            raise ModelValidationError(
+                f"Model hydration failed for {config_type}: {exc}",
+                cause=exc,
+                file_path=str(path),
+            ) from exc
+
+    # params config type returns a plain dict
+    return resolved_with_refs
+
+
+# Federated namespace: weevr.config splits across two wheels — pure
+# parser/resolver/validation/inheritance modules in weevr-core (which
+# ships this __init__.py) and Spark-bound config helpers (fabric.py,
+# paths.py) contributed by the weevr engine wheel. In editable workspace
+# installs each wheel's src/ is a separate sys.path entry, so we extend
+# __path__ to pull engine-side contributions into the import-time view.
+# In a real install both wheels' files merge into a single
+# site-packages/weevr/config/ directory and the loop is a no-op. The
+# basename(p) == "src" gate scopes the scan to src-layout package roots.
+def _extend_namespace_path() -> None:
+    """Append sibling weevr.config/ directories on sys.path to ``__path__``."""
+    import os.path as osp
+    import sys
+
+    components = __name__.split(".")
+    src_entries_seen = 0
+    candidates_added = 0
+    for entry in sys.path:
+        if not entry or osp.basename(entry) != "src":
+            continue
+        src_entries_seen += 1
+        candidate = osp.join(entry, *components)
+        if osp.isdir(candidate) and candidate not in __path__:
+            __path__.append(candidate)
+            candidates_added += 1
+    if src_entries_seen > 0 and candidates_added == 0:
+        import warnings
+
+        warnings.warn(
+            f"weevr federated namespace {__name__!r}: no sibling "
+            "contributor found on sys.path. If this is an editable "
+            "workspace install, run `uv sync --dev` from the repo root "
+            "to install both wheels.",
+            stacklevel=2,
+        )
+
+
+_extend_namespace_path()
+del _extend_namespace_path

weevr/config/audit_templates.py

@@ -0,0 +1,314 @@
+"""Pure-Python helpers for resolving and merging audit column templates.
+
+This is the canonical home for the audit-template merge primitives used by
+``weevr.config.inheritance``. The historical home in ``weevr.operations.audit``
+re-exports these symbols for back-compat with external callers; core modules
+(notably ``weevr.config.inheritance``) MUST import from here directly so the
+config layer stays Spark-free.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+import re
+from dataclasses import dataclass
+
+from weevr.errors.exceptions import ConfigError
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "BUILTIN_AUDIT_PRESETS",
+    "AuditContext",
+    "CONTEXT_VAR_PATTERN",
+    "resolve_audit_columns",
+    "apply_audit_exclusions",
+    "resolve_template_columns",
+    "merge_audit_columns_with_templates",
+    "resolve_context_variables",
+]
+
+# ---------------------------------------------------------------------------
+# Built-in audit presets
+# ---------------------------------------------------------------------------
+
+#: Built-in audit column presets keyed by name.
+#:
+#: ``fabric`` — nine columns aligned with Microsoft Fabric pipeline parameters.
+#: ``minimal`` — three lightweight columns for simple run tracking.
+BUILTIN_AUDIT_PRESETS: dict[str, dict[str, str]] = {
+    "fabric": {
+        "_batch_id": "${param.batch_id}",
+        "_batch_version": "${param.batch_version}",
+        "_batch_source": "${param.batch_source}",
+        "_batch_process_ts": "current_timestamp()",
+        "_pipeline_id": "${param.pipeline_id}",
+        "_pipeline_name": "${param.pipeline_name}",
+        "_workspace_id": "${param.workspace_id}",
+        "_spark_app_id": "spark_context().applicationId",
+        "_task_ts": "current_timestamp()",
+    },
+    "minimal": {
+        "_weevr_loaded_at": "current_timestamp()",
+        "_weevr_run_id": "${param.run_id}",
+        "_weevr_thread": "${thread.name}",
+    },
+}
+
+
+@dataclass(frozen=True)
+class AuditContext:
+    """Execution context for resolving audit column variables.
+
+    Attributes:
+        thread_name: Thread name (e.g. ``stg_orders``).
+        thread_qualified_key: Fully qualified key (e.g. ``staging.stg_orders``).
+        thread_source: Primary source alias (first declared source).
+        thread_sources_json: JSON array of all sources with type metadata.
+        weave_name: Weave name (e.g. ``staging``).
+        loom_name: Loom name (e.g. ``my-project``).
+        run_timestamp: ISO 8601 UTC timestamp of execution start.
+        run_id: UUID4 identifier unique per execution.
+    """
+
+    thread_name: str
+    thread_qualified_key: str
+    thread_source: str | None
+    thread_sources_json: str
+    weave_name: str
+    loom_name: str
+    run_timestamp: str = ""
+    run_id: str = ""
+
+
+# Pattern matching ${namespace.property} context variables.
+CONTEXT_VAR_PATTERN = re.compile(r"\$\{(thread|weave|loom|run)\.([a-z_]+)\}")
+
+
+def resolve_audit_columns(
+    loom_audit: dict[str, str] | None,
+    weave_audit: dict[str, str] | None,
+    thread_audit: dict[str, str] | None,
+) -> dict[str, str]:
+    """Merge audit columns additively: loom -> weave -> thread.
+
+    Lower levels extend the column set. Same-named columns at a lower
+    level override the expression from a higher level.
+
+    Args:
+        loom_audit: Audit columns from loom defaults.
+        weave_audit: Audit columns from weave defaults.
+        thread_audit: Audit columns from thread target.
+
+    Returns:
+        Merged audit column dict. Empty dict if none defined.
+    """
+    merged: dict[str, str] = {}
+    if loom_audit:
+        merged.update(loom_audit)
+    if weave_audit:
+        merged.update(weave_audit)
+    if thread_audit:
+        merged.update(thread_audit)
+    return merged
+
+
+def apply_audit_exclusions(
+    columns: dict[str, str],
+    exclude_patterns: list[str] | None,
+) -> dict[str, str]:
+    """Remove audit columns whose names match any of the given glob patterns.
+
+    Args:
+        columns: Resolved audit column definitions as ``{name: expression}``.
+        exclude_patterns: List of glob patterns (``fnmatch`` syntax). ``None``
+            or an empty list is a no-op.
+
+    Returns:
+        Copy of ``columns`` with matching entries removed.
+    """
+    if not exclude_patterns:
+        return columns
+
+    result: dict[str, str] = {}
+    for name, expression in columns.items():
+        matched = False
+        for pattern in exclude_patterns:
+            if fnmatch.fnmatch(name, pattern):
+                logger.debug("Audit column %r excluded by pattern %r", name, pattern)
+                matched = True
+                break
+        if not matched:
+            result[name] = expression
+
+    return result
+
+
+def resolve_template_columns(
+    template_names: list[str],
+    user_templates: dict[str, dict[str, str]] | None = None,
+) -> dict[str, str]:
+    """Resolve template names to merged column definitions.
+
+    Looks up each name in user-defined templates first, then built-in
+    presets. Logs a warning when a user-defined template shadows a
+    built-in preset. Raises ConfigError if a name is not found.
+
+    Args:
+        template_names: Ordered list of template names to resolve.
+        user_templates: User-defined templates keyed by name.
+
+    Returns:
+        Merged column dict. Later names override earlier on collision.
+
+    Raises:
+        ConfigError: If a template name is not found in user-defined
+            or built-in presets.
+    """
+    resolved_user = user_templates or {}
+    merged: dict[str, str] = {}
+
+    for name in template_names:
+        if name in resolved_user:
+            if name in BUILTIN_AUDIT_PRESETS:
+                logger.warning(
+                    "User-defined template %r shadows built-in preset with the same name",
+                    name,
+                )
+            merged.update(resolved_user[name])
+        elif name in BUILTIN_AUDIT_PRESETS:
+            merged.update(BUILTIN_AUDIT_PRESETS[name])
+        else:
+            available = sorted(set(resolved_user) | set(BUILTIN_AUDIT_PRESETS))
+            raise ConfigError(
+                f"Audit template {name!r} not found. Available templates: {', '.join(available)}"
+            )
+
+    return merged
+
+
+def merge_audit_columns_with_templates(
+    *,
+    loom_templates: dict[str, dict[str, str]] | None = None,
+    loom_template_refs: list[str] | None = None,
+    loom_inline: dict[str, str] | None = None,
+    weave_templates: dict[str, dict[str, str]] | None = None,
+    weave_template_refs: list[str] | None = None,
+    weave_inline: dict[str, str] | None = None,
+    thread_templates: dict[str, dict[str, str]] | None = None,
+    thread_template_refs: list[str] | None = None,
+    thread_inline: dict[str, str] | None = None,
+    thread_inherit: bool = True,
+    thread_exclude: list[str] | None = None,
+) -> dict[str, str]:
+    """Merge audit columns from templates, inline definitions, and inheritance.
+
+    Resolution order:
+    1. Collect user-defined templates from all levels.
+    2. If thread_inherit is False, skip loom and weave template refs and inline.
+    3. Resolve template refs at each level via resolve_template_columns().
+    4. Merge: loom template cols → loom inline → weave template cols
+       → weave inline → thread template cols → thread inline.
+    5. Apply apply_audit_exclusions() with thread_exclude.
+
+    User-defined template definitions from all levels are always available for
+    name resolution, even when thread_inherit is False. Inheritance only controls
+    whether loom and weave template refs and inline columns contribute to the
+    merged output.
+
+    Args:
+        loom_templates: User-defined templates declared at the loom level.
+        loom_template_refs: Template names to resolve at the loom level.
+        loom_inline: Explicit audit columns defined inline at the loom level.
+        weave_templates: User-defined templates declared at the weave level.
+        weave_template_refs: Template names to resolve at the weave level.
+        weave_inline: Explicit audit columns defined inline at the weave level.
+        thread_templates: User-defined templates declared at the thread level.
+        thread_template_refs: Template names to resolve at the thread level.
+        thread_inline: Explicit audit columns defined inline at the thread level.
+        thread_inherit: When False, loom and weave template refs and inline
+            columns are excluded from the merged result. Defaults to True.
+        thread_exclude: Glob patterns applied after all merging. Matching
+            column names are removed from the final result.
+
+    Returns:
+        Merged audit column dict. Empty dict if nothing is defined.
+    """
+    # Merge all user-defined templates across levels so they are available for
+    # any level's template refs regardless of inheritance setting.
+    all_user_templates: dict[str, dict[str, str]] = {}
+    if loom_templates:
+        all_user_templates.update(loom_templates)
+    if weave_templates:
+        all_user_templates.update(weave_templates)
+    if thread_templates:
+        all_user_templates.update(thread_templates)
+
+    merged: dict[str, str] = {}
+
+    if thread_inherit:
+        if loom_template_refs:
+            merged.update(resolve_template_columns(loom_template_refs, all_user_templates))
+        if loom_inline:
+            merged.update(loom_inline)
+        if weave_template_refs:
+            merged.update(resolve_template_columns(weave_template_refs, all_user_templates))
+        if weave_inline:
+            merged.update(weave_inline)
+
+    if thread_template_refs:
+        merged.update(resolve_template_columns(thread_template_refs, all_user_templates))
+    if thread_inline:
+        merged.update(thread_inline)
+
+    return apply_audit_exclusions(merged, thread_exclude)
+
+
+def resolve_context_variables(expression: str, context: AuditContext) -> str:
+    """Substitute ${namespace.property} variables in an expression string.
+
+    Args:
+        expression: Spark SQL expression with context variable placeholders.
+        context: Audit context providing variable values.
+
+    Returns:
+        Expression with context variables replaced by their values.
+    """
+    lookup: dict[str, dict[str, str | None]] = {
+        "thread": {
+            "name": context.thread_name,
+            "qualified_key": context.thread_qualified_key,
+            "source": context.thread_source,
+            "sources": context.thread_sources_json,
+        },
+        "weave": {
+            "name": context.weave_name,
+        },
+        "loom": {
+            "name": context.loom_name,
+        },
+        "run": {
+            "timestamp": context.run_timestamp,
+            "id": context.run_id,
+        },
+    }
+
+    def _replace(match: re.Match[str]) -> str:
+        namespace = match.group(1)
+        prop = match.group(2)
+        ns_lookup = lookup.get(namespace, {})
+        if prop not in ns_lookup:
+            return match.group(0)  # Unknown property — leave unresolved
+        value = ns_lookup[prop]
+        if value is None:
+            logger.warning(
+                "Audit column context variable ${%s.%s} resolved to None, "
+                "substituting empty string",
+                namespace,
+                prop,
+            )
+            return ""
+        return value
+
+    return CONTEXT_VAR_PATTERN.sub(_replace, expression)

weevr/config/duration.py

@@ -0,0 +1,5 @@
+"""Duration string parser — re-exported from weevr.duration for backward compatibility."""
+
+from weevr.duration import parse_duration
+
+__all__ = ["parse_duration"]