continuo-python-runtime 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

continuo_python_runtime/cli.py

@@ -4,6 +4,7 @@ import argparse
 import logging
 import os
 import sys
+from importlib import metadata
 from pathlib import Path
 
 from continuo_python_runtime.contract.loader import load_contract_dir
@@ -28,6 +29,11 @@ def main(argv: list[str] | None = None) -> int:
     )
 
     parser = argparse.ArgumentParser(prog="continuo-runtime")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"continuo-runtime {metadata.version('continuo-python-runtime')}",
+    )
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     dialect_help = (

continuo_python_runtime/content_check.py

@@ -0,0 +1,73 @@
+"""The harness's start check: the image must hold what the release hashed.
+
+continuo sets ``CONTENT_HASH`` to the ``content_hash`` the release promoted for
+the node. The check recomputes the node's hash from the files in the image with
+the helpers ``merge`` uses (``gather_hash_inputs`` and ``hash_parts``), so the
+two can never disagree, and refuses to run a node whose image reproduces a
+different one: the image holds other code, imports, assets or contract than the
+release.
+"""
+
+import logging
+from collections.abc import Mapping
+from importlib import metadata
+from pathlib import Path
+
+from continuo_python_runtime.contract.model import Node
+from continuo_python_runtime.errors import ContractError, ImageContractMismatch
+from continuo_python_runtime.hash_inputs import gather_hash_inputs
+from continuo_python_runtime.hashing import hash_parts
+
+logger = logging.getLogger("continuo_python_runtime.content_check")
+
+_ENV_KEY = "CONTENT_HASH"
+
+
+def check_image_contract(node: Node, app_root: Path, env: Mapping[str, str]) -> None:
+    """Fail unless the image reproduces the ``content_hash`` in ``CONTENT_HASH``.
+
+    An unset or empty ``CONTENT_HASH`` logs a warning and skips the check. Any
+    other value is compared as an exact string, so a value without the
+    ``sha256:`` prefix, in upper case or with surrounding whitespace is a
+    mismatch.
+
+    Raises:
+        ImageContractMismatch: The recomputed ``content_hash`` differs, or a
+            file the release hashed is missing from the image.
+    """
+    released = env.get(_ENV_KEY)
+    if not released:
+        logger.warning(
+            "%s is not set: skipping the check that the image holds the release's content for %s",
+            _ENV_KEY, node.relation,
+        )
+        return
+
+    try:
+        inputs = gather_hash_inputs(node, app_root)
+        parts = hash_parts(node, inputs.primary, inputs.units)
+    except ContractError as exc:
+        raise ImageContractMismatch(
+            _message(node, f"the image cannot reproduce {_ENV_KEY} {released}: {exc}")
+        ) from exc
+
+    if parts["content_hash"] != released:
+        detail = (
+            f"the image's content_hash {parts['content_hash']} "
+            f"(source_hash {parts['source_hash']}, "
+            f"shared_code_hash {parts['shared_code_hash']}, "
+            f"config_hash {parts['config_hash']}, "
+            f"units {sorted(inputs.units)})"
+        )
+        raise ImageContractMismatch(_message(node, f"{_ENV_KEY} {released} differs from {detail}"))
+
+    logger.info("image holds the release's content for %s (%s)", node.relation, released)
+
+
+def _message(node: Node, detail: str) -> str:
+    return (
+        f"the image does not hold what the release hashed for {node.relation}: "
+        f"{detail}. "
+        f"This image runs runtime {metadata.version('continuo-python-runtime')}. "
+        "Fix: rebuild the image from the released commit, with CI on the runtime of the image's FROM tag."
+    )

continuo_python_runtime/contract/kinds.py

@@ -112,14 +112,22 @@ class KindRules:
     # The write modes a node of this kind may declare. A new kind gets replace
     # only unless its entry opts into more.
     mode_types: frozenset[str] = frozenset({DEFAULT_MODE_TYPE})
+    # Whether a node of this kind may list the repository files its script reads.
+    assets_allowed: bool = False
 
 
 RULES: dict[str, KindRules] = {
-    "python-node": KindRules(script_required=True, reads=SqlReads(), mode_types=MODE_TYPES),
+    "python-node": KindRules(
+        script_required=True, reads=SqlReads(), mode_types=MODE_TYPES, assets_allowed=True
+    ),
     # A csv drop is a snapshot of its source: replace only (spec D2).
     "python-csv": KindRules(script_required=False, reads=CsvRead()),
     "python-api": KindRules(
-        script_required=True, reads=NoReads(), secret_allowed=True, mode_types=MODE_TYPES
+        script_required=True,
+        reads=NoReads(),
+        secret_allowed=True,
+        mode_types=MODE_TYPES,
+        assets_allowed=True,
     ),
 }
 

continuo_python_runtime/contract/loader.py

@@ -7,13 +7,14 @@ structural or semantic violation.
 
 from __future__ import annotations
 
-from pathlib import Path
+import re
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 import yaml
 from sqlglot import Dialect
 
-from continuo_python_runtime.contract.kinds import rules_for
+from continuo_python_runtime.contract.kinds import KindRules, rules_for
 from continuo_python_runtime.contract.model import (
     CRITICALITIES,
     DEFAULT_KIND,
@@ -45,6 +46,7 @@ _ALLOWED_KEYS = {
     "mode_type",
     "primary_key",
     "timeout_minutes",
+    "assets",
 }
 
 # Bounds of a node's declared task timeout, in minutes (one minute to one week).
@@ -210,6 +212,52 @@ def _validate_primary_key(
     return tuple(raw_key)
 
 
+_DRIVE_PATH = re.compile(r"^[A-Za-z]:")
+
+
+def _validate_assets(
+    raw: dict[str, Any], rules: KindRules, kind: str, label: str
+) -> tuple[str, ...]:
+    """Validate ``assets``: repo-relative paths of the files a script reads.
+
+    A kind without a script has nothing that reads a file, so it refuses the
+    key. Each path is written with forward slashes, relative to the repository
+    root and without a ``..`` segment; it is normalised (``./data//ref.json``
+    is ``data/ref.json``) and keeps its declared position. A repeat after
+    normalising is refused. Whether the file exists is checked where a
+    repository root is known.
+    """
+    if "assets" not in raw:
+        return ()
+    if not rules.assets_allowed:
+        raise ContractError(
+            f"{label}: 'assets' is not allowed for kind {kind} (it has no script to read files)"
+        )
+    raw_assets = raw["assets"]
+    if not isinstance(raw_assets, list):
+        raise ContractError(f"{label}: 'assets' must be a list of repository-relative paths")
+    assets: list[str] = []
+    for entry in raw_assets:
+        if not isinstance(entry, str) or not entry.strip():
+            raise ContractError(f"{label}: 'assets' entry {entry!r} must be a non-empty string")
+        if "\\" in entry:
+            raise ContractError(f"{label}: 'assets' entry {entry!r} must use forward slashes")
+        path = PurePosixPath(entry)
+        if path.is_absolute() or _DRIVE_PATH.match(entry):
+            raise ContractError(
+                f"{label}: 'assets' entry {entry!r} must be relative to the repository root"
+            )
+        if ".." in path.parts:
+            raise ContractError(f"{label}: 'assets' entry {entry!r} must not contain '..'")
+        normalised = path.as_posix()
+        if normalised == ".":
+            raise ContractError(f"{label}: 'assets' entry {entry!r} must name a file")
+        if normalised in assets:
+            raise ContractError(f"{label}: 'assets' names {normalised!r} more than once")
+        assets.append(normalised)
+    return tuple(assets)
+
+
 def parse_node(
     raw: dict[str, Any],
     source: str,
@@ -362,6 +410,7 @@ def parse_node(
         columns.append(Column(name=name, type=col_type, nullable=nullable))
 
     primary_key = _validate_primary_key(raw, mode_type, columns, label)
+    assets = _validate_assets(raw, rules, kind, label)
 
     description = raw.get("description", "")
     if not isinstance(description, str):
@@ -391,6 +440,7 @@ def parse_node(
         mode_type=mode_type,
         primary_key=primary_key,
         timeout_minutes=timeout_minutes,
+        assets=assets,
     )
 
 

continuo_python_runtime/contract/merge.py

@@ -1,15 +1,13 @@
 """Contract v1 merger: wire contract builder."""
 
-from collections.abc import Callable
 from pathlib import Path
 
 import yaml
 
-from continuo_python_runtime.closure import resolve_closure
 from continuo_python_runtime.contract.loader import load_contract_dir
 from continuo_python_runtime.contract.model import CONTRACT_VERSION, DEFAULT_MODE_TYPE, Node
-from continuo_python_runtime.contract.paths import resolve_script_path
 from continuo_python_runtime.errors import ContractError
+from continuo_python_runtime.hash_inputs import gather_hash_inputs
 from continuo_python_runtime.hashing import hash_parts
 from continuo_python_runtime.lint import lint_source
 
@@ -21,9 +19,9 @@ def node_entry(node: Node) -> dict:
     dicts. The entry carries no hash fields: build_wire_contract adds all
     four (source_hash, shared_code_hash, config_hash, content_hash). The
     nullable field is always present in output_columns. ``secret_ref``,
-    ``mode_type`` and ``primary_key`` appear only when they differ from their
-    default, so a node that does not use them has the entry (and hashes) it
-    had before they existed.
+    ``mode_type``, ``primary_key`` and ``assets`` appear only when they differ
+    from their default, so the entry of a node that does not use them stays
+    minimal.
     """
     entry = {
         "schema": node.schema,
@@ -46,26 +44,24 @@ def node_entry(node: Node) -> dict:
         "extra_columns": node.extra_columns,
         "config": dict(node.config),
     }
-    # Only a kind that may name a Secret ever carries one; emitting the key
-    # only when set keeps every other node's config_hash unchanged.
+    # Only a kind that may name a Secret ever carries one; the key is emitted
+    # only when set.
     if node.secret_ref:
         entry["secret_ref"] = node.secret_ref
-    # Same reasoning: every replace node's entry, config_hash and content_hash
-    # stay byte-identical, so contract_version stays 1.
+    # Same reasoning: a replace node's entry carries no mode_type.
     if node.mode_type != DEFAULT_MODE_TYPE:
         entry["mode_type"] = node.mode_type
     if node.mode_type == "upsert":
         entry["primary_key"] = list(node.primary_key)
+    if node.assets:
+        entry["assets"] = list(node.assets)
     return entry
 
 
 def _lint_node_closure(
     node: Node,
     repo_root: Path,
-    script_path: Path,
-    script_bytes: bytes,
-    closure: list[Path],
-    member_bytes: list[bytes],
+    code_files: tuple[tuple[Path, bytes], ...],
 ) -> None:
     """Lint the node's script and every resolved closure member.
 
@@ -73,7 +69,8 @@ def _lint_node_closure(
     in-repo import closure — is held to the same lint rules
     (``continuo_python_runtime.lint.lint_source``: L1 forbidden driver
     imports, L2 SQL string literals, L3 forbidden data-access calls, L4
-    private-attribute access, L5 dynamic-import constructs) that CI's
+    private-attribute access, L5 dynamic-import constructs, L6 constructs
+    that run code the hash cannot see) that CI's
     ``continuo-runtime lint`` applies to ``scripts/``. Enforcing it again
     here, at merge time, means a helper file outside ``scripts/`` — which
     the CI lint step never looks at — cannot become a side channel around
@@ -107,10 +104,8 @@ def _lint_node_closure(
     catches only ``HarnessError``/``OSError``.
     """
     repo_root_resolved = repo_root.resolve()
-    files = [(script_path, script_bytes), *zip(closure, member_bytes, strict=True)]
-
     violations: list[str] = []
-    for path, data in files:
+    for path, data in code_files:
         rel = path.relative_to(repo_root_resolved)
         try:
             text = data.decode("utf-8-sig")
@@ -125,30 +120,6 @@ def _lint_node_closure(
         )
 
 
-def _script_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
-    """A script node's source is its script plus its in-repo import closure,
-    every file of which is held to the closure lint."""
-    script_path = resolve_script_path(node.script, repo_root, context=node.relation)
-    script_bytes = script_path.read_bytes()
-    closure = resolve_closure(script_path, repo_root)
-    member_bytes = [member.read_bytes() for member in closure]
-    _lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
-    return script_bytes, member_bytes
-
-
-def _csv_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
-    """A csv node has no script and no import closure: its source is the uri.
-    New file content at the same uri is new data, not a new node version."""
-    return node.reads["csv"].encode(), []
-
-
-_HASH_INPUTS: dict[str, Callable[[Node, Path], tuple[bytes, list[bytes]]]] = {
-    "python-node": _script_hash_inputs,
-    "python-csv": _csv_hash_inputs,
-    "python-api": _script_hash_inputs,
-}
-
-
 def build_wire_contract(
     contract_dir: Path, repo_root: Path, service: str, *, dialect: str | None = None
 ) -> dict:
@@ -160,16 +131,26 @@ def build_wire_contract(
     .load_contract_dir` so every declared read is checked against that sqlglot
     dialect (``None`` -- the default -- uses sqlglot's dialect-neutral parser).
 
-    Raises ContractError if any script file is missing.
+    Raises ContractError if any script file is missing, or if the repository
+    root holds a ``sitecustomize.py`` or ``usercustomize.py``: Python imports
+    either at startup, before the harness, so nothing in a node's import
+    closure reaches it.
     """
     nodes = load_contract_dir(contract_dir, dialect=dialect)
 
+    for hook in ("sitecustomize.py", "usercustomize.py"):
+        if (repo_root / hook).exists():
+            raise ContractError(
+                f"{hook} at the repository root runs before the harness and is not in the hash"
+            )
+
     wire_nodes = []
     for node in nodes:
         entry = node_entry(node)
 
-        primary_bytes, member_bytes = _HASH_INPUTS[node.kind](node, repo_root)
-        entry.update(hash_parts(entry, primary_bytes, member_bytes))
+        inputs = gather_hash_inputs(node, repo_root)
+        _lint_node_closure(node, repo_root, inputs.code_files)
+        entry.update(hash_parts(node, inputs.primary, inputs.units))
         # The one wire key outside config_hash: it is written after the hashes
         # are computed, so changing it alone never changes content_hash.
         if node.timeout_minutes is not None:

continuo_python_runtime/contract/model.py

@@ -46,6 +46,7 @@ class Node:
     mode_type: str = DEFAULT_MODE_TYPE
     primary_key: tuple[str, ...] = ()
     timeout_minutes: int | None = None
+    assets: tuple[str, ...] = ()
 
     @property
     def relation(self) -> str:

continuo_python_runtime/contract/paths.py

@@ -36,3 +36,36 @@ def resolve_script_path(script: str, repo_root: Path, *, context: str) -> Path:
         raise ContractError(f"{context}: script not found: {script}")
 
     return script_path
+
+
+def resolve_asset_path(asset: str, repo_root: Path, *, context: str) -> Path:
+    """Resolve ``asset`` (relative to ``repo_root``) and validate it.
+
+    Rejects absolute paths, paths that escape ``repo_root`` once symlinks
+    resolve, paths that do not exist and paths that are not a regular file.
+
+    Args:
+        asset: The asset path as declared in the contract, relative to
+            ``repo_root``.
+        repo_root: The repository root the asset path is resolved against.
+        context: A label prefixed to every error message (e.g. a node's
+            relation) to identify which node's asset failed.
+
+    Raises:
+        ContractError: If the path is absolute, escapes the repository root,
+            does not exist or is not a file.
+    """
+    if Path(asset).is_absolute():
+        raise ContractError(
+            f"{context}: asset path {asset!r} must be relative to the repository root"
+        )
+
+    asset_path = (repo_root / asset).resolve()
+    if not asset_path.is_relative_to(repo_root.resolve()):
+        raise ContractError(f"{context}: asset path {asset!r} escapes the repository root")
+    if not asset_path.exists():
+        raise ContractError(f"{context}: asset not found: {asset}")
+    if not asset_path.is_file():
+        raise ContractError(f"{context}: asset {asset!r} is not a file")
+
+    return asset_path

continuo_python_runtime/errors.py

@@ -6,7 +6,13 @@ remediation classifier can key off it without parsing free text.
 
 
 class HarnessError(Exception):
-    """Base for all runtime failures the harness converts to a sentinel block."""
+    """Base for all runtime failures the harness converts to a sentinel block.
+
+    ``reason`` is a machine-readable code the block carries beside the message;
+    empty for the errors that have none.
+    """
+
+    reason: str = ""
 
     @property
     def error_class(self) -> str:
@@ -20,6 +26,12 @@ class ContractError(HarnessError):
     """Contract missing, invalid, node not found, or script missing."""
 
 
+class ImageContractMismatch(ContractError):
+    """The image does not hold the contract, script, imports or assets the release hashed."""
+
+    reason = "image_contract_mismatch"
+
+
 class ReadError(HarnessError):
     """Unknown read name, or a declared read failed at the warehouse."""
 

continuo_python_runtime/harness.py

@@ -32,6 +32,7 @@ from continuo_engine_contract.port import (  # type: ignore[import-untyped]
 from continuo_engine_contract.result import result_block  # type: ignore[import-untyped]
 
 from continuo_python_runtime.conform import conform, to_arrow
+from continuo_python_runtime.content_check import check_image_contract
 from continuo_python_runtime.context import RunContext
 from continuo_python_runtime.contract.loader import load_contract_dir
 from continuo_python_runtime.contract.model import Node
@@ -345,6 +346,7 @@ def run_node(
         # Every other loader validation still runs at container start.
         nodes = load_contract_dir(contract_dir, check_reads=False)
         node = select_node(nodes, node_id)
+        check_image_contract(node, app_root, env)
 
         if adapter is not None:
             active_adapter = adapter
@@ -392,9 +394,21 @@ def run_node(
         print(result_block("success", message=f"rows={conformed.num_rows}", unique_id=node_id))
         return 0
     except HarnessError as err:
+        # Logged as well as printed: the executor cuts the result block out of
+        # the pod log before storing it, so the log holds only what is logged.
+        # The traceback is kept when the error wraps another exception (a
+        # driver error, the script's own exception), whose frames locate it.
+        logger.error(
+            "node %s failed: %s", node_id, err.sentinel_message(),
+            exc_info=err.__cause__ is not None,
+        )
         print(
             result_block(
-                "error", message=err.sentinel_message(), failures=1, unique_id=node_id
+                "error",
+                message=err.sentinel_message(),
+                failures=1,
+                unique_id=node_id,
+                reason=err.reason,
             )
         )
         return 1

continuo_python_runtime/hash_inputs.py

@@ -0,0 +1,63 @@
+"""The bytes each node kind hashes, gathered from the repository.
+
+A script kind hashes its script, its in-repo import closure and the assets it
+declares (a file that is both is one unit). A csv node has
+no script and no closure: its source is its uri, so new file content at the same
+uri is new data, not a new node version.
+"""
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from continuo_python_runtime.closure import resolve_closure
+from continuo_python_runtime.contract.model import Node
+from continuo_python_runtime.contract.paths import resolve_asset_path, resolve_script_path
+
+
+@dataclass(frozen=True)
+class HashInputs:
+    """What a node's hash parts are computed from.
+
+    ``primary`` is the script's bytes (a csv node's uri, encoded). ``units`` maps
+    each closure file's and each declared asset's repo-relative POSIX path to
+    its bytes. ``code_files`` holds every file that executes for the node, the
+    script first, for the lint.
+    """
+
+    primary: bytes
+    units: dict[str, bytes]
+    code_files: tuple[tuple[Path, bytes], ...]
+
+
+def _script_inputs(node: Node, repo_root: Path) -> HashInputs:
+    script_path = resolve_script_path(node.script, repo_root, context=node.relation)
+    script_bytes = script_path.read_bytes()
+    root = repo_root.resolve()
+    members = [(member, member.read_bytes()) for member in resolve_closure(script_path, repo_root)]
+    units = {member.relative_to(root).as_posix(): data for member, data in members}
+    # Assets are data the script reads, not code that runs, so they are hashed
+    # but not part of code_files (which the lint reads).
+    for asset in node.assets:
+        units[asset] = resolve_asset_path(asset, repo_root, context=node.relation).read_bytes()
+    return HashInputs(
+        primary=script_bytes,
+        units=units,
+        code_files=((script_path, script_bytes), *members),
+    )
+
+
+def _csv_inputs(node: Node, repo_root: Path) -> HashInputs:
+    return HashInputs(node.reads["csv"].encode(), {}, ())
+
+
+HASH_INPUTS: dict[str, Callable[[Node, Path], HashInputs]] = {
+    "python-node": _script_inputs,
+    "python-csv": _csv_inputs,
+    "python-api": _script_inputs,
+}
+
+
+def gather_hash_inputs(node: Node, repo_root: Path) -> HashInputs:
+    """Read the files *node* hashes, by its kind."""
+    return HASH_INPUTS[node.kind](node, repo_root)