continuo-python-runtime 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

continuo_python_runtime/hashing.py

@@ -1,94 +1,93 @@
 """Content hashing: three independent parts folded into one content_hash.
 
-The three parts (``source_hash``, ``shared_code_hash``, ``config_hash``) are
-computed independently from already-gathered bytes/dicts and folded together
-with ``content_hash_fold``. This module is pure computation — no filesystem
-access, no knowledge of ``closure.py`` — so the known-vector tests can call
-``content_hash_fold`` on bare strings with no entry or file involved.
-
-The fold formula and the ``shared_code_hash`` sorted-digest fold are wire
-contracts mirrored byte-for-byte from continuo's manifest-controller
-(``content_hash_fold``) and its dbt-side shared-code hashing. Every byte here
-is load-bearing: manifest-controller independently recomputes the fold from
-the parts this repo emits and rejects the release artifact if the two
-results differ.
+A Python node's ``content_hash`` folds three parts: ``source_hash`` (the node's
+own script bytes), ``shared_code_hash`` (the import closure, each file with its
+repo-relative path) and ``config_hash`` (the node's code projection). The parts
+are computed from already-gathered bytes and a parsed node, so this module does
+no filesystem access and knows nothing about ``closure.py``.
+
+``shared_code_hash``, ``config_hash`` and the fold are computed by
+continuo-contracts, the library continuo's topology-controller pins and
+recomputes the config part with. Nothing here canonicalises or normalises: the
+library does, so every producer of a hash writes the same bytes. The fold is
+the three-part fold that older topology-controllers check on the wire, so an
+entry this module writes passes their re-computation unchanged.
+
+The library refuses a few ``config`` values the contract loader accepts (an
+integer beyond 2**53 - 1, NaN or an infinity, nesting deeper than 99 levels, a
+lone surrogate). Each refusal reaches the caller as a ``ContractError`` naming
+the node.
 """
 
-from collections.abc import Iterable
-from copy import deepcopy
+from collections.abc import Mapping
 from hashlib import sha256
-from json import dumps
 
-HASH_PART_FIELDS = ("source_hash", "shared_code_hash", "config_hash")
-CONTENT_HASH_FIELD = "content_hash"
-# A wire key that is not part of the config_hash basis: it tunes how the node
-# runs (the task timeout) and says nothing about what the node computes, so
-# changing it alone is not a code change.
-NON_HASHED_FIELDS = ("timeout_minutes",)
+from continuo_contracts.errors import ContractsError
+from continuo_contracts.hashing import fold3, shared_hash
+from continuo_contracts.python import PythonColumn, PythonNode, projection_hash
 
+from continuo_python_runtime.contract.model import Node
+from continuo_python_runtime.errors import ContractError
 
-def canonical_entry(entry: dict) -> dict:
-    """Deep-copy *entry*, drop the four hash fields, whitespace-normalize reads.
 
-    Drops ``content_hash`` and all three of ``HASH_PART_FIELDS`` from the
-    basis, and whitespace-normalizes every ``str`` value in ``reads`` via
-    ``" ".join(sql.split())``. ``timeout_minutes`` is the one other wire key
-    left out of the basis (``NON_HASHED_FIELDS``). Everything else is
-    preserved verbatim, including ``config``.
-    """
-    canonical = deepcopy(entry)
-
-    canonical.pop(CONTENT_HASH_FIELD, None)
-    for field in (*HASH_PART_FIELDS, *NON_HASHED_FIELDS):
-        canonical.pop(field, None)
-
-    reads = canonical.get("reads")
-    if isinstance(reads, dict):
-        for key, value in reads.items():
-            if isinstance(value, str):
-                reads[key] = " ".join(value.split())
+def source_hash(primary: bytes) -> str:
+    """Bare hex `sha256(primary)` — the node's own script bytes (a csv node's uri)."""
+    return sha256(primary).hexdigest()
 
-    return canonical
 
+def to_python_node(node: Node) -> PythonNode:
+    """The library's view of *node*: the fields its code projection is made of."""
+    return PythonNode(
+        schema=node.schema,
+        table=node.table,
+        kind=node.kind,
+        script=node.script,
+        reads=dict(node.reads),
+        output_columns=tuple(
+            PythonColumn(column.name, column.type, column.nullable)
+            for column in node.output_columns
+        ),
+        extra_columns=node.extra_columns,
+        config=dict(node.config),
+        secret_ref=node.secret_ref,
+        mode_type=node.mode_type,
+        primary_key=tuple(node.primary_key),
+        assets=node.assets,
+    )
 
-def canonical_json(entry: dict) -> str:
-    """`json.dumps(canonical_entry(entry), sort_keys=True, separators=(",", ":"))`."""
-    return dumps(canonical_entry(entry), sort_keys=True, separators=(",", ":"))
 
+def config_hash(node: Node) -> str:
+    """Bare hex SHA-256 of the RFC 8785 bytes of the node's code projection."""
+    try:
+        return projection_hash(to_python_node(node))
+    except ContractsError as exc:
+        raise ContractError(str(exc)) from exc
 
-def source_hash(script_bytes: bytes) -> str:
-    """Bare hex `sha256(script_bytes)` — the node's own script, byte-for-byte."""
-    return sha256(script_bytes).hexdigest()
 
+def shared_code_hash(units: Mapping[str, bytes]) -> str:
+    """`""` when there are no units, else bare hex over each unit's path and digest.
 
-def shared_code_hash(member_bytes: Iterable[bytes]) -> str:
-    """`""` when the closure is empty, else bare hex of the sorted-digest fold."""
-    unit_hashes = sorted(sha256(member).hexdigest() for member in member_bytes)
-    if not unit_hashes:
-        return ""
-    return sha256("".join(unit_hashes).encode()).hexdigest()
-
-
-def config_hash(entry: dict) -> str:
-    """Bare hex `sha256(canonical_json(entry).encode())`."""
-    return sha256(canonical_json(entry).encode()).hexdigest()
+    *units* maps a repo-relative POSIX path to the file's bytes.
+    """
+    try:
+        return shared_hash(units)
+    except ContractsError as exc:
+        raise ContractError(str(exc)) from exc
 
 
 def content_hash_fold(source: str, shared: str, config: str) -> str:
-    """`"sha256:" + sha256(f"{source}|{shared}|{config}".encode()).hexdigest()`."""
-    return "sha256:" + sha256(f"{source}|{shared}|{config}".encode()).hexdigest()
+    """`"sha256:"` plus the hex SHA-256 of the three parts joined by `|`."""
+    return fold3(source, shared, config)
 
 
-def hash_parts(
-    entry: dict, script_bytes: bytes, member_bytes: Iterable[bytes]
-) -> dict[str, str]:
+def hash_parts(node: Node, primary: bytes, units: Mapping[str, bytes]) -> dict[str, str]:
     """Return all four fields: the three parts plus their fold."""
-    s = source_hash(script_bytes)
-    sh = shared_code_hash(member_bytes)
-    c = config_hash(entry)
+    source = source_hash(primary)
+    shared = shared_code_hash(units)
+    config = config_hash(node)
     return {
-        "source_hash": s,
-        "shared_code_hash": sh,
-        "config_hash": c,
-        "content_hash": content_hash_fold(s, sh, c),
+        "source_hash": source,
+        "shared_code_hash": shared,
+        "config_hash": config,
+        "content_hash": content_hash_fold(source, shared, config),
     }

continuo_python_runtime/lint.py

@@ -1,5 +1,5 @@
-"""Script linting for forbidden imports, SQL literals, data-access calls, and
-dynamic-import constructs."""
+"""Script linting for forbidden imports, SQL literals, data-access calls,
+dynamic-import constructs, and code the content hash cannot see."""
 
 import ast
 import re
@@ -30,6 +30,16 @@ SQL_PATTERN = re.compile(
 # Forbidden data-access method calls
 FORBIDDEN_CALLS = {"read_sql", "read_sql_query", "read_sql_table", "execute", "read_database"}
 
+# Modules whose import runs code the import closure never reaches.
+_STARTUP_HOOK_MODULES = {"sitecustomize", "usercustomize"}
+_STARTUP_HOOK_FILES = {f"{module}.py" for module in _STARTUP_HOOK_MODULES}
+
+# ``list`` methods that change ``sys.path`` in place.
+_PATH_MUTATORS = {"append", "insert", "extend", "remove", "pop", "clear", "reverse", "sort"}
+
+# Names ``pickle`` exports that rebuild objects, and so run code, from bytes.
+_PICKLE_LOADERS = {"load", "loads", "Unpickler"}
+
 
 def _reconstruct_joined_str(node: ast.JoinedStr) -> tuple[str, set[int]]:
     """Reconstruct text from a JoinedStr (f-string) by joining Constant parts
@@ -114,6 +124,166 @@ def _docstring_constant_ids(tree: ast.Module) -> set[int]:
     return ids
 
 
+def _module_aliases(tree: ast.AST, module: str) -> set[str]:
+    """Local names bound to *module* by ``import module`` / ``import module as x``."""
+    return {
+        alias.asname or alias.name
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Import)
+        for alias in node.names
+        if alias.name == module
+    }
+
+
+def _from_imports(tree: ast.AST, module: str) -> dict[str, str]:
+    """``{local name: imported name}`` for every ``from module import name [as x]``."""
+    return {
+        alias.asname or alias.name: alias.name
+        for node in ast.walk(tree)
+        if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == module
+        for alias in node.names
+    }
+
+
+def _assignment_targets(node: ast.AST) -> list[ast.expr]:
+    """Every expression *node* (an assignment or ``del``) writes to, with
+    tuple and list targets unpacked."""
+    if isinstance(node, ast.Assign):
+        pending = list(node.targets)
+    elif isinstance(node, (ast.AugAssign, ast.AnnAssign)):
+        pending = [node.target]
+    elif isinstance(node, ast.Delete):
+        pending = list(node.targets)
+    else:
+        return []
+    targets: list[ast.expr] = []
+    while pending:
+        target = pending.pop()
+        if isinstance(target, (ast.Tuple, ast.List)):
+            pending.extend(target.elts)
+        elif isinstance(target, ast.Starred):
+            pending.append(target.value)
+        else:
+            targets.append(target)
+    return targets
+
+
+def unhashed_code_violations(tree: ast.AST) -> list[tuple[int, str]]:
+    """Return ``(lineno, construct)`` for every construct in *tree* that runs
+    code the content hash does not cover.
+
+    The hash folds the script and the files it imports, found by reading
+    imports. A node that executes anything else, or changes where imports
+    resolve, can change its behaviour without changing its hash. Flagged
+    constructs, by name:
+
+    - ``runpy``: any import of it, which runs a file by path.
+    - ``sys.path``: a call that mutates it (``append``, ``insert``, ``extend``,
+      ``remove``, ``pop``, ``clear``, ``reverse``, ``sort``), an assignment,
+      augmented assignment or ``del`` of it or of a subscript of it, and
+      ``site.addsitedir``. ``import sys as s`` and ``from sys import path``
+      are followed.
+    - ``sitecustomize`` and ``usercustomize``: any import of either.
+    - ``compile``: a call of the builtin, by its bare name (a ``compile``
+      imported from a module, such as ``re.compile``, is not the builtin).
+    - ``FunctionType``: ``types.FunctionType`` and the name imported from
+      ``types``.
+    - ``pickle.load``, ``pickle.loads`` and ``pickle.Unpickler``: attributes of
+      ``pickle`` or of an alias of it, and the names imported from it.
+
+    Names are resolved by the file's imports alone, not by scope or by
+    data flow: ``p = sys.path`` followed by ``p.append(...)`` is not seen.
+
+    Sorted by lineno, then construct.
+    """
+    sys_names = _module_aliases(tree, "sys")
+    site_names = _module_aliases(tree, "site")
+    types_names = _module_aliases(tree, "types")
+    pickle_names = _module_aliases(tree, "pickle")
+    sys_path_names = {local for local, name in _from_imports(tree, "sys").items() if name == "path"}
+    addsitedir_names = {
+        local for local, name in _from_imports(tree, "site").items() if name == "addsitedir"
+    }
+    # ``from re import compile`` shadows the builtin.
+    imported_names = {
+        alias.asname or alias.name
+        for node in ast.walk(tree)
+        if isinstance(node, ast.ImportFrom)
+        for alias in node.names
+    }
+
+    def module_attribute(node: ast.expr, names: set[str], attr: str) -> bool:
+        return (
+            isinstance(node, ast.Attribute)
+            and node.attr == attr
+            and isinstance(node.value, ast.Name)
+            and node.value.id in names
+        )
+
+    def is_sys_path(node: ast.expr) -> bool:
+        return module_attribute(node, sys_names, "path") or (
+            isinstance(node, ast.Name) and node.id in sys_path_names
+        )
+
+    def writes_sys_path(target: ast.expr, *, rebinding: bool) -> bool:
+        """Whether writing to *target* changes ``sys.path``. Rebinding a bare
+        name imported from ``sys`` only changes the local name."""
+        if isinstance(target, ast.Subscript):
+            return is_sys_path(target.value)
+        if rebinding:
+            return module_attribute(target, sys_names, "path")
+        return is_sys_path(target)
+
+    violations: list[tuple[int, str]] = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                root = alias.name.split(".")[0]
+                if root == "runpy" or root in _STARTUP_HOOK_MODULES:
+                    violations.append((node.lineno, root))
+        elif isinstance(node, ast.ImportFrom):
+            from_root = node.module.split(".")[0] if node.module and node.level == 0 else None
+            if from_root == "runpy" or from_root in _STARTUP_HOOK_MODULES:
+                violations.append((node.lineno, from_root))
+            elif from_root == "types" and any(a.name == "FunctionType" for a in node.names):
+                violations.append((node.lineno, "FunctionType"))
+            elif from_root == "pickle":
+                violations.extend(
+                    (node.lineno, f"pickle.{a.name}")
+                    for a in node.names
+                    if a.name in _PICKLE_LOADERS
+                )
+        elif isinstance(node, ast.Call):
+            func = node.func
+            if isinstance(func, ast.Attribute) and func.attr in _PATH_MUTATORS and is_sys_path(
+                func.value
+            ):
+                violations.append((node.lineno, "sys.path"))
+            elif module_attribute(func, site_names, "addsitedir") or (
+                isinstance(func, ast.Name) and func.id in addsitedir_names
+            ):
+                violations.append((node.lineno, "sys.path"))
+            elif (
+                isinstance(func, ast.Name)
+                and func.id == "compile"
+                and "compile" not in imported_names
+            ):
+                violations.append((node.lineno, "compile"))
+        elif isinstance(node, ast.Attribute):
+            if module_attribute(node, types_names, "FunctionType"):
+                violations.append((node.lineno, "FunctionType"))
+            elif node.attr in _PICKLE_LOADERS and module_attribute(node, pickle_names, node.attr):
+                violations.append((node.lineno, f"pickle.{node.attr}"))
+        elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Delete)):
+            rebinding = not isinstance(node, ast.AugAssign)
+            if any(
+                writes_sys_path(target, rebinding=rebinding)
+                for target in _assignment_targets(node)
+            ):
+                violations.append((node.lineno, "sys.path"))
+    return sorted(violations)
+
+
 def lint_source(source: str, filename: str) -> list[str]:
     """Lint a Python source code string for violations.
 
@@ -137,6 +307,14 @@ def lint_source(source: str, filename: str) -> list[str]:
       import a module whose edits never re-fingerprint it. Detection is
       delegated to ``continuo_python_runtime.closure.dynamic_import_violations``
       so the predicate is defined in exactly one place.
+    - L6: unhashed-code construct (``runpy``, a change to ``sys.path``, a
+      ``sitecustomize``/``usercustomize`` module, ``compile()``,
+      ``types.FunctionType``, ``pickle.load``/``loads``/``Unpickler``), and a
+      file named ``sitecustomize.py`` or ``usercustomize.py``. The import
+      closure follows imports only, so code these constructs run is not in
+      ``content_hash``. Detection is
+      ``unhashed_code_violations``. A file a script reads belongs in the
+      node's ``assets``, where it is hashed.
 
     Docstrings (the first statement of a Module/ClassDef/FunctionDef/
     AsyncFunctionDef body, when it is a bare string Expr) are exempt from the
@@ -148,11 +326,16 @@ def lint_source(source: str, filename: str) -> list[str]:
     """
     entries: list[tuple[int, str]] = []
 
+    # L6: a startup hook runs before the harness, whatever it contains.
+    if Path(filename).name in _STARTUP_HOOK_FILES:
+        entries.append((1, f"unhashed code construct '{Path(filename).stem}'"))
+
     # Try to parse the source code
     try:
         tree = ast.parse(source)
     except SyntaxError as e:
-        return [f"{filename}:{e.lineno}: syntax error: {e.msg}"]
+        entries.append((e.lineno or 1, f"syntax error: {e.msg}"))
+        return [f"{filename}:{lineno}: {message}" for lineno, message in entries]
 
     # Track imported data-access functions and their local aliases
     imported_forbidden_calls: dict[str, int] = {}  # {local_name: lineno}
@@ -262,10 +445,15 @@ def lint_source(source: str, filename: str) -> list[str]:
     for lineno, construct in dynamic_import_violations(tree):
         entries.append((lineno, f"dynamic import construct '{construct}'"))
 
-    # Stable sort by line number: this is what puts L5 hits "alongside" the
-    # L1-L4 hits found above instead of trailing as a separate block, while
+    # L6: constructs that run code the hash cannot see.
+    for lineno, construct in unhashed_code_violations(tree):
+        entries.append((lineno, f"unhashed code construct '{construct}'"))
+
+    # Stable sort by line number: this is what puts L5 and L6 hits "alongside"
+    # the L1-L4 hits found above instead of trailing as a separate block, while
     # violations on the same line keep the order the passes above found them
-    # (L1 first pass, then L2-L4 second pass, then L5).
+    # (the file-name L6 entry, at line 1, first; then L1 from the first pass,
+    # then L2-L4 from the second pass, then L5, then the construct L6 hits).
     entries.sort(key=lambda entry: entry[0])
     return [f"{filename}:{lineno}: {message}" for lineno, message in entries]