continuo-python-runtime 0.1.0__py3-none-any.whl

continuo_python_runtime/harness.py

@@ -0,0 +1,208 @@
+"""Container entrypoint: dispatches a single node's script and writes its output.
+
+``run_node`` is the sole write sink for a python-model node. It resolves the
+node from the contract, loads and executes its script inside a
+:class:`~continuo_python_runtime.context.RunContext`, conforms the result to
+the declared schema, and writes it through the runtime adapter. Exactly one
+sentinel-framed result block is printed to stdout per run; every other
+diagnostic goes to stderr via ``logging``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import logging
+import os
+import sys
+import uuid
+from collections.abc import Mapping
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+from continuo_validation_contract.port import (  # type: ignore[import-untyped]
+    discover_runtime_adapter,
+)
+from continuo_validation_contract.result import result_block  # type: ignore[import-untyped]
+
+from continuo_python_runtime.conform import conform, to_arrow
+from continuo_python_runtime.context import RunContext
+from continuo_python_runtime.contract.loader import load_contract_dir
+from continuo_python_runtime.contract.model import Node
+from continuo_python_runtime.contract.paths import resolve_script_path
+from continuo_python_runtime.errors import ContractError, HarnessError, LoadError, ScriptError
+
+logger = logging.getLogger("continuo_python_runtime.harness")
+
+
+def _require_env(env: Mapping[str, str], key: str) -> str:
+    """Return env[key], raising ContractError if it is missing or empty."""
+    value = env.get(key)
+    if not value:
+        raise ContractError(f"missing required environment variable {key!r}")
+    return value
+
+
+def select_node(nodes: list[Node], node_id: str) -> Node:
+    """Select the node matching the trailing ``schema.table`` of ``node_id``.
+
+    ``node_id`` is split on ``.``; the trailing two segments are taken as
+    ``(schema, table)`` and matched against the declared nodes.
+
+    Raises:
+        ContractError: If ``node_id`` has fewer than 2 dot-separated
+            segments, or no declared node matches.
+    """
+    segments = node_id.split(".")
+    if len(segments) < 2:
+        raise ContractError(
+            f"node_id {node_id!r} must have at least 2 dot-separated segments (schema.table)"
+        )
+    schema, table = segments[-2], segments[-1]
+    for node in nodes:
+        if node.schema == schema and node.table == table:
+            return node
+
+    available = sorted(f"{n.schema}.{n.table}" for n in nodes)
+    raise ContractError(
+        f"no node matches {node_id!r} (schema={schema!r}, table={table!r}); "
+        f"available relations: {available}"
+    )
+
+
+def load_script(node: Node, repo_root: Path) -> ModuleType:
+    """Import ``node.script`` (relative to ``repo_root``) and return the module.
+
+    Raises:
+        ContractError: If the script path is absolute, escapes the
+            repository root, or does not exist.
+        ScriptError: If the module has no callable ``run``.
+    """
+    script_path = resolve_script_path(node.script, repo_root, context=node.relation)
+
+    module_name = f"_continuo_node_{uuid.uuid4().hex}"
+    spec = importlib.util.spec_from_file_location(module_name, script_path)
+    if spec is None or spec.loader is None:
+        raise ContractError(f"cannot load script: {node.script}")
+
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except Exception as exc:
+        raise ScriptError(f"script import failed: {exc}") from exc
+
+    if not callable(getattr(module, "run", None)):
+        raise ScriptError(f"script {node.script} has no callable 'run'")
+
+    return module
+
+
+def build_adapter() -> Any:
+    """Discover and construct the single installed runtime adapter.
+
+    Raises:
+        LoadError: If any of the adapter's ``required_env()`` vars are unset
+            or empty in ``os.environ``.
+    """
+    _, cls = discover_runtime_adapter()
+    required = cls.required_env()
+    missing = [key for key in required if not os.environ.get(key)]
+    if missing:
+        raise LoadError(f"missing required warehouse env: {sorted(missing)}")
+    return cls.from_env()
+
+
+def _execute_script(module: ModuleType, ctx: RunContext) -> Any:
+    """Run ``module.run(ctx)`` with stdout redirected to stderr.
+
+    Raises:
+        HarnessError: Propagated as-is (e.g. a ``ReadError`` from ``ctx.read``).
+        ScriptError: Wraps any other exception raised by user code.
+    """
+    with contextlib.redirect_stdout(sys.stderr):
+        try:
+            return module.run(ctx)
+        except HarnessError:
+            raise
+        except Exception as exc:
+            raise ScriptError(f"run() raised {exc.__class__.__name__}: {exc}") from exc
+
+
+def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
+    """Run a single node end-to-end and print exactly one sentinel result block.
+
+    Returns 0 on success, 1 on any :class:`HarnessError`.
+    """
+    node_id = env.get("NODE_ID") or ""
+    active_adapter: Any = None
+    try:
+        node_id = _require_env(env, "NODE_ID")
+        table_name = _require_env(env, "TABLE_NAME")
+        target_schema = _require_env(env, "TARGET_SCHEMA")
+
+        contract_dir = Path(env.get("CONTRACT_DIR") or "/app/contracts")
+        app_root = Path(env["APP_ROOT"]) if env.get("APP_ROOT") else contract_dir.parent
+
+        logger.info("running node %s -> %s.%s", node_id, target_schema, table_name)
+
+        nodes = load_contract_dir(contract_dir)
+        node = select_node(nodes, node_id)
+
+        with contextlib.redirect_stdout(sys.stderr):
+            module = load_script(node, app_root)
+
+        if adapter is not None:
+            active_adapter = adapter
+        else:
+            try:
+                active_adapter = build_adapter()
+            except HarnessError:
+                raise
+            except Exception as exc:
+                raise LoadError(f"adapter construction failed: {exc}") from exc
+
+        ctx = RunContext(node, active_adapter)
+        raw_result = _execute_script(module, ctx)
+
+        table = to_arrow(raw_result)
+        conformed = conform(table, node.output_columns, node.extra_columns)
+
+        columns = [
+            {"name": c.name, "type": c.type, "nullable": c.nullable}
+            for c in node.output_columns
+        ]
+        try:
+            active_adapter.ensure_table(target_schema, table_name, columns)
+            active_adapter.load(target_schema, table_name, conformed)
+        except HarnessError:
+            raise
+        except Exception as exc:
+            raise LoadError(f"failed to write {target_schema}.{table_name}: {exc}") from exc
+
+        print(result_block("success", message=f"rows={conformed.num_rows}", unique_id=node_id))
+        return 0
+    except HarnessError as err:
+        print(
+            result_block(
+                "error", message=err.sentinel_message(), failures=1, unique_id=node_id
+            )
+        )
+        return 1
+    except Exception as exc:
+        logger.exception("unexpected failure running node %s", node_id)
+        print(
+            result_block(
+                "error",
+                message=f"ScriptError: unexpected failure: {exc}",
+                failures=1,
+                unique_id=node_id,
+            )
+        )
+        return 1
+    finally:
+        if active_adapter is not None:
+            try:
+                active_adapter.close()
+            except Exception:
+                logger.warning("adapter.close() failed", exc_info=True)

continuo_python_runtime/hashing.py

@@ -0,0 +1,34 @@
+import copy
+import hashlib
+import json
+
+
+def canonical_entry(entry: dict) -> dict:
+    """
+    Deep-copy the entry, drop content_hash field, and whitespace-normalize
+    every value in reads via " ".join(sql.split()).
+    """
+    canonical = copy.deepcopy(entry)
+
+    # Drop content_hash if present
+    canonical.pop("content_hash", None)
+
+    # Whitespace-normalize every value in reads
+    if "reads" in canonical and isinstance(canonical["reads"], dict):
+        for key, value in canonical["reads"].items():
+            if isinstance(value, str):
+                canonical["reads"][key] = " ".join(value.split())
+
+    return canonical
+
+
+def content_hash(entry: dict, script_bytes: bytes) -> str:
+    """
+    Compute content hash using the formula:
+    "sha256:" + sha256(json.dumps(canonical_entry(entry), sort_keys=True, separators=(",", ":")).encode() + b"\x00" + script_bytes).hexdigest()
+    """
+    canonical = canonical_entry(entry)
+    json_str = json.dumps(canonical, sort_keys=True, separators=(",", ":"))
+    data = json_str.encode() + b"\x00" + script_bytes
+    hash_digest = hashlib.sha256(data).hexdigest()
+    return "sha256:" + hash_digest

continuo_python_runtime/lint.py

@@ -0,0 +1,295 @@
+"""Script linting for forbidden imports, SQL literals, and data-access calls."""
+
+import ast
+import re
+from pathlib import Path
+
+# Forbidden warehouse driver modules (check root module of imports)
+FORBIDDEN_DRIVERS = {
+    "psycopg2",
+    "sqlalchemy",
+    "trino",
+    "snowflake",
+    "pyodbc",
+    "duckdb",
+    "sqlite3",
+    "pymysql",
+    "mysql",
+    "clickhouse_driver",
+    "pyhive",
+}
+
+# SQL pattern: detects select, insert, update, delete, create table statements
+SQL_PATTERN = re.compile(
+    r"(?is)\b(select\s.+?\sfrom\s|insert\s+into\s|update\s.+?\sset\s|delete\s+from\s|create\s+table\s)"
+)
+
+# Forbidden data-access method calls
+FORBIDDEN_CALLS = {"read_sql", "read_sql_query", "read_sql_table", "execute", "read_database"}
+
+
+def _reconstruct_joined_str(node: ast.JoinedStr) -> tuple[str, set[int]]:
+    """Reconstruct text from a JoinedStr (f-string) by joining Constant parts
+    with space placeholders.
+
+    Returns the reconstructed text together with the ids of the Constant
+    nodes that were actually folded into it (the JoinedStr's direct literal
+    parts). Constants nested inside a ``FormattedValue`` expression (e.g.
+    ``f'{"select ... from ..."}'``) are NEVER consumed here — they are
+    replaced with a space placeholder and remain visible to the plain-Constant
+    pass in ``lint_source``.
+    """
+    parts = []
+    consumed: set[int] = set()
+    for value in node.values:
+        if isinstance(value, ast.Constant) and isinstance(value.value, str):
+            parts.append(value.value)
+            consumed.add(id(value))
+        else:
+            # FormattedValue or other node type - use space placeholder.
+            # Its contents (including any nested Constants) are intentionally
+            # left unconsumed.
+            parts.append(" ")
+    return "".join(parts), consumed
+
+
+def _reconstruct_binop_concat(node: ast.BinOp) -> tuple[str, set[int]] | None:
+    """Reconstruct string from BinOp(Add) with string Constant/JoinedStr leaves.
+
+    Returns the reconstructed text together with the ids of the Constant
+    nodes actually folded into it: leaf string Constants and the literal
+    fragments of any JoinedStr operand (including nested JoinedStr/BinOp
+    fragments). Constants that live inside a FormattedValue expression are
+    never included, matching ``_reconstruct_joined_str``.
+    """
+    if not isinstance(node.op, ast.Add):
+        return None
+
+    # Collect all parts of the binary operation tree
+    def collect_parts(n: ast.expr) -> tuple[list[str], set[int]] | None:
+        if isinstance(n, ast.Constant) and isinstance(n.value, str):
+            return [n.value], {id(n)}
+        elif isinstance(n, ast.JoinedStr):
+            text, consumed = _reconstruct_joined_str(n)
+            return [text], consumed
+        elif isinstance(n, ast.BinOp) and isinstance(n.op, ast.Add):
+            left = collect_parts(n.left)
+            right = collect_parts(n.right)
+            if left is not None and right is not None:
+                return left[0] + right[0], left[1] | right[1]
+            return None
+        else:
+            return None
+
+    result = collect_parts(node)
+    if result is None:
+        return None
+    parts, consumed = result
+    return "".join(parts), consumed
+
+
+def _docstring_constant_ids(tree: ast.Module) -> set[int]:
+    """Return ids of Constant nodes that are docstrings.
+
+    A str Constant is a docstring if it is the value of an ``ast.Expr``
+    statement appearing as the FIRST statement of a Module/ClassDef/
+    FunctionDef/AsyncFunctionDef body.
+    """
+    ids: set[int] = set()
+    for node in ast.walk(tree):
+        if isinstance(
+            node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
+        ):
+            body = node.body
+            if (
+                body
+                and isinstance(body[0], ast.Expr)
+                and isinstance(body[0].value, ast.Constant)
+                and isinstance(body[0].value.value, str)
+            ):
+                ids.add(id(body[0].value))
+    return ids
+
+
+def lint_source(source: str, filename: str) -> list[str]:
+    """Lint a Python source code string for violations.
+
+    Returns a list of violations in the format:
+    "<filename>:<lineno>: <rule text>"
+
+    Rules:
+    - L1: forbidden warehouse driver import
+    - L2: SQL string literal (including in f-strings and concatenated strings)
+    - L3: forbidden data-access call (attribute or imported function)
+    - L4: private/protected attribute access (any ``ast.Attribute`` whose
+      ``attr`` starts with ``_``), closing bypasses like ``ctx._fetch``.
+      ``__name__``/``__main__`` are ``ast.Name`` nodes, not attributes, so
+      they are never matched and need no exemption. Exempted: attribute
+      access on ``self``/``cls`` (e.g. ``self._helper()``), so a script's own
+      class-private helpers aren't flagged.
+
+    Docstrings (the first statement of a Module/ClassDef/FunctionDef/
+    AsyncFunctionDef body, when it is a bare string Expr) are exempt from the
+    L2 constant pass, since prose commonly contains words like "select" and
+    "from". This is a best-effort, position-based exemption: the same prose
+    assigned to a variable is still flagged.
+    """
+    violations = []
+
+    # Try to parse the source code
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as e:
+        return [f"{filename}:{e.lineno}: syntax error: {e.msg}"]
+
+    # Track imported data-access functions and their local aliases
+    imported_forbidden_calls: dict[str, int] = {}  # {local_name: lineno}
+
+    # First pass: collect information about imports
+    for node in ast.walk(tree):
+        # L1: Check for forbidden imports
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                root_module = alias.name.split(".")[0]
+                if root_module in FORBIDDEN_DRIVERS:
+                    violations.append(
+                        f"{filename}:{node.lineno}: forbidden warehouse driver import '{root_module}'"
+                    )
+
+        elif isinstance(node, ast.ImportFrom):
+            if node.module:
+                root_module = node.module.split(".")[0]
+                if root_module in FORBIDDEN_DRIVERS:
+                    violations.append(
+                        f"{filename}:{node.lineno}: forbidden warehouse driver import '{root_module}'"
+                    )
+
+            # Track imports of forbidden data-access functions
+            if node.names:
+                for alias in node.names:
+                    # alias.name is the original name, alias.asname is the local alias (or None)
+                    if alias.name in FORBIDDEN_CALLS:
+                        local_name = alias.asname if alias.asname else alias.name
+                        imported_forbidden_calls[local_name] = node.lineno
+                        violations.append(
+                            f"{filename}:{node.lineno}: forbidden data-access import '{alias.name}'"
+                        )
+
+    # Track constants consumed in compound expressions (to avoid double-reporting)
+    consumed_constants: set[int] = set()
+
+    # Docstrings are exempt from the L2 constant pass (prose false positives)
+    docstring_constants = _docstring_constant_ids(tree)
+
+    # Second pass: check for SQL and L3 violations
+    for node in ast.walk(tree):
+        # L2: Check for SQL in JoinedStr (f-strings)
+        if isinstance(node, ast.JoinedStr):
+            reconstructed, consumed = _reconstruct_joined_str(node)
+            if SQL_PATTERN.search(reconstructed):
+                snippet = reconstructed[:40]
+                if len(reconstructed) > 40:
+                    snippet += "..."
+                violations.append(
+                    f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
+                )
+            # Only the literal fragments actually folded into the
+            # reconstruction are consumed; FormattedValue contents are not.
+            consumed_constants |= consumed
+
+        # L2: Check for SQL in BinOp concatenation
+        elif isinstance(node, ast.BinOp):
+            result = _reconstruct_binop_concat(node)
+            if result is not None:
+                reconstructed_text, consumed = result
+                if SQL_PATTERN.search(reconstructed_text):
+                    snippet = reconstructed_text[:40]
+                    if len(reconstructed_text) > 40:
+                        snippet += "..."
+                    violations.append(
+                        f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
+                    )
+                # Only the leaf Constants actually folded into the
+                # reconstruction are consumed - never descendants of a
+                # FormattedValue expression.
+                consumed_constants |= consumed
+
+        # L2: Check for SQL in plain string literals (skip if consumed by
+        # compound reconstruction, or if it's a docstring)
+        elif isinstance(node, ast.Constant) and isinstance(node.value, str):
+            if id(node) not in consumed_constants and id(node) not in docstring_constants:
+                if SQL_PATTERN.search(node.value):
+                    snippet = node.value[:40]
+                    if len(node.value) > 40:
+                        snippet += "..."
+                    violations.append(
+                        f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
+                    )
+
+        # L3: Check for forbidden data-access calls
+        elif isinstance(node, ast.Call):
+            # Attribute access: pd.read_sql(...)
+            if isinstance(node.func, ast.Attribute):
+                if node.func.attr in FORBIDDEN_CALLS:
+                    violations.append(
+                        f"{filename}:{node.lineno}: forbidden data-access call '{node.func.attr}'"
+                    )
+
+            # Name reference: read_sql_table(...) or rs(...) where rs is an alias
+            elif isinstance(node.func, ast.Name):
+                if node.func.id in imported_forbidden_calls:
+                    violations.append(
+                        f"{filename}:{node.lineno}: forbidden data-access call '{node.func.id}'"
+                    )
+
+        # L4: Check for private/protected attribute access
+        elif isinstance(node, ast.Attribute):
+            is_self_or_cls = isinstance(node.value, ast.Name) and node.value.id in (
+                "self",
+                "cls",
+            )
+            if node.attr.startswith("_") and not is_self_or_cls:
+                violations.append(
+                    f"{filename}:{node.lineno}: private attribute access '{node.attr}'"
+                )
+
+    return violations
+
+
+def lint_paths(paths: list[Path]) -> list[str]:
+    """Lint Python files in the given paths.
+
+    For each path:
+    - If it's a directory, recursively find all .py files (sorted)
+    - If it's a file, lint it directly
+    - If path does not exist, record a violation instead of crashing
+
+    Returns aggregate violations in order.
+    """
+    violations = []
+
+    # Collect all Python files to lint
+    files_to_lint = []
+    for path in paths:
+        path = Path(path)
+        if not path.exists():
+            violations.append(f"{path}: path does not exist")
+        elif path.is_dir():
+            # Recursively find all .py files, sorted
+            py_files = sorted(path.rglob("*.py"))
+            files_to_lint.extend(py_files)
+        else:
+            # Single file
+            files_to_lint.append(path)
+
+    # Lint each file
+    for filepath in files_to_lint:
+        try:
+            source = filepath.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError) as exc:
+            violations.append(f"{filepath}: unreadable: {exc}")
+            continue
+        file_violations = lint_source(source, str(filepath))
+        violations.extend(file_violations)
+
+    return violations