quantlab-factor-agent 0.1.0__py3-none-any.whl

factor_agent/gate/__init__.py

@@ -0,0 +1,359 @@
+"""The Gate: deterministic checks that every candidate passes before it is evaluated.
+
+``Gate.check`` never reads data; what candidates may write with is its
+``Vocabulary``, from the campaign's field and operator files. A bad candidate comes back ``Rejected``
+with readable reasons; only an infrastructure failure raises. The checks
+run in this order:
+
+1. static checks on the code (syntax allow-list, names in the vocabulary, ``alpha`` assigned);
+2. the graph build in a sandboxed subprocess (timeout from when the sandbox is
+   ready, memory limit, bubblewrap filesystem isolation where it is installed);
+3. the stage rules of an intraday family's candidate (``stages.split``):
+   an intraday field reaches the daily stage only through a session
+   aggregation, the intraday stage reads no daily field, no aggregation of
+   a daily value or of an aggregation, each operator is used only in its
+   stages, each window and lag is one of its stage's set; each refusal
+   says how to write it instead (``Rejected.hints``);
+4. the graph's size, parameter values, dimensions and complexity, all reported together;
+5. originality against the zoo: not a rewrite of a zoo factor, not one with
+   other windows or parameters only, and no common subtree with one larger
+   than the limit. The reasons name the zoo factor and quote the subtree;
+6. not a duplicate of a candidate this gate passed before: the same canonical
+   graph up to what keeps or reverses its ranking (``Graph.ranking``): a
+   factor and its negation predict alike, in opposite directions.
+"""
+
+import ctypes
+import ctypes.util
+import importlib
+import json
+import os
+import re
+import select
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from dataclasses import dataclass, field
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import psutil
+
+from factor_agent.candidate import Candidate
+from factor_agent.gate import _constraints, _originality, stages
+from factor_agent.gate._checks import static_reasons
+from factor_agent.gate.graph import Graph
+
+if TYPE_CHECKING:  # the sandbox imports this package; it must not load pandas through the config
+    from factor_agent.config import GateConfig
+    from factor_agent.gate.vocabulary import Vocabulary
+    from factor_agent.zoo import Zoo
+
+#: Characters of the subprocess's standard error quoted when it crashes.
+_STDERR_TAIL = 2000
+
+#: Seconds between two looks at the sandbox's memory.
+_POLL_SECONDS = 0.02
+
+#: The interpreter arguments that run the sandbox subprocess.
+SANDBOX_ARGS = ("-m", "factor_agent.gate._sandbox")
+
+#: The line the sandbox writes once it has started, before it runs the candidate's code.
+READY = b"ready\n"
+
+#: Seconds the sandbox may take to be ready (its imports and the vocabulary's rebuild); a graph build's time counts from then.
+STARTUP_SECONDS = 60.0
+
+#: The reason given when a graph build runs out of memory.
+MEMORY_REASON = "graph build used more than the {memory_mb} MB memory limit"
+
+
+@dataclass(frozen=True)
+class Passed:
+    """A candidate whose graph was built: the graph, its fingerprint and its statistics."""
+
+    graph: Graph
+    fingerprint: str
+    stats: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Rejected:
+    """A candidate refused by the gate, with the reasons sent back to the proposer.
+
+    ``hints`` say how to write it instead: the vocabulary's fixes for a
+    writing error (a static check or the graph build), or those of the
+    stage rules it breaks; they go with the reasons into the rewrite
+    request only.
+    """
+
+    reasons: tuple[str, ...]
+    hints: tuple[str, ...] = ()
+
+
+class Gate:
+    """Check candidates; see the module docstring for the order of the checks.
+
+    A gate remembers the ranking of every candidate it passed and rejects
+    a later candidate that only keeps or reverses one.
+
+    Parameters
+    ----------
+    config : GateConfig
+        The sandbox limits and the graph constraints.
+    vocabulary : Vocabulary
+        The fields, operators and parameter values candidates may use;
+        the sandbox rebuilds the same one from its sources.
+    seen : iterable of Graph, optional
+        The graphs of candidates passed before, such as those of a campaign
+        that is resumed; a candidate that keeps or reverses one's ranking
+        (``Graph.ranking``) is a duplicate.
+    """
+
+    def __init__(self, config: "GateConfig", vocabulary: "Vocabulary", seen=()):
+        self.config = config
+        self.vocabulary = vocabulary
+        self._seen: set[str] = {graph.ranking() for graph in seen}
+
+    def check(self, candidate: Candidate, zoo: "Zoo") -> Passed | Rejected:
+        """Return ``Passed`` with the candidate's graph, or ``Rejected`` with why not.
+
+        ``zoo`` holds the factors the candidate must be original against.
+        A writing error carries the hints of the vocabulary's fixes it
+        matches, a broken stage rule its own (``Rejected.hints``).
+
+        Raises
+        ------
+        RuntimeError
+            If the sandbox cannot be started or replies with something
+            that is not its protocol.
+        """
+        reasons = static_reasons(candidate.code, self.vocabulary.names)
+        if reasons:
+            return self._written_badly(reasons)
+        reply = self._build(candidate.code)
+        if isinstance(reply, Rejected):
+            return self._written_badly(reply.reasons)
+        graph = Graph.from_json(reply, self.vocabulary.classes)
+        staged = stages.split(graph, self.vocabulary)
+        if staged.reasons:
+            return Rejected(staged.reasons, staged.hints)
+        reasons, stats = _constraints.check(graph, self.config, self.vocabulary)
+        if reasons:
+            return Rejected(tuple(reasons))
+        reasons = _originality.reasons(graph, zoo, self.config.max_common_subtree_nodes, self.vocabulary)
+        if reasons:
+            return Rejected(tuple(reasons))
+        ranking = graph.ranking()
+        if ranking in self._seen:
+            return Rejected((
+                "the graph is a duplicate: it keeps or reverses the ranking of an earlier candidate (the same "
+                "canonical graph up to a rank, a constant offset or a scale, a sign flip included); "
+                "propose a different mechanism",
+            ))
+        self._seen.add(ranking)
+        return Passed(graph, graph.fingerprint(), stats)
+
+    def withdraw(self, graph: Graph) -> None:
+        """Forget that ``graph`` passed, so a rewrite of its candidate is not refused as its duplicate.
+
+        A candidate the reviewer sends back is rewritten, and its rewrite
+        often keeps the graph: the rewrite continues the same candidate, so
+        the version it replaces gives up its ranking. A candidate the
+        reviewer rejects for good keeps it, as a resume restores it.
+        """
+        self._seen.discard(graph.ranking())
+
+    def _written_badly(self, reasons) -> Rejected:
+        """A writing error's refusal, with the hints of the fixes its reasons match."""
+        return Rejected(tuple(reasons), self.vocabulary.fix_hints(reasons))
+
+    def _build(self, code: str) -> list | Rejected:
+        """Build the graph in the sandbox subprocess; return its nodes or the rejection."""
+        request = json.dumps({"code": code, "memory_mb": self.config.memory_mb, "vocabulary": self.vocabulary.sources})
+        # The request (the vocabulary's files included) is larger than a pipe holds, so it is read from a file.
+        with tempfile.TemporaryDirectory(prefix="factor-agent-gate-") as scratch, tempfile.TemporaryFile("w+") as stdin:
+            stdin.write(request)
+            stdin.flush()
+            stdin.seek(0)
+            command = [sys.executable, "-I", *SANDBOX_ARGS]
+            if shutil.which("bwrap") and _bubblewrap_works():
+                command = _bubblewrap(command, self.vocabulary.sources["operator_modules"])
+            try:
+                process = subprocess.Popen(
+                    command, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    cwd=scratch, env={"PATH": os.defpath},
+                )
+            except OSError as exc:
+                raise RuntimeError(f"Gate: cannot start the sandbox: {exc}") from exc
+            outcome = self._watch(process)
+        if isinstance(outcome, Rejected):
+            return outcome
+        stdout, stderr = outcome
+        if process.returncode != 0:
+            tail = stderr[-_STDERR_TAIL:].strip()
+            if "ModuleNotFoundError" in tail and "factor_agent" in tail:
+                raise RuntimeError(f"Gate: the sandbox cannot import factor_agent:\n{tail}")
+            if "MemoryError" in tail:
+                return Rejected((MEMORY_REASON.format(memory_mb=self.config.memory_mb),))
+            return Rejected((f"graph build crashed (exit {process.returncode}): {_last_line(tail)}",))
+        try:
+            message = json.loads(stdout)
+        except json.JSONDecodeError as exc:
+            raise RuntimeError(f"Gate: the sandbox replied {stdout[:200]!r}") from exc
+        if "reasons" in message:
+            return Rejected(tuple(message["reasons"]))
+        return message["graph"]
+
+    def _watch(self, process: subprocess.Popen) -> tuple[str, str] | Rejected:
+        """Wait for the reply, killing the sandbox past the time or memory limit.
+
+        The time limit counts from the sandbox's ``READY``: its start, the
+        imports and the vocabulary's rebuild, is not the candidate's.
+
+        Raises
+        ------
+        RuntimeError
+            If the sandbox is not ready within ``STARTUP_SECONDS``: a broken
+            environment, not a bad candidate.
+        """
+        limit = self.config.memory_mb * 2**20
+        watched = psutil.Process(process.pid)
+        output = _Output(process)
+        deadline, ready = time.monotonic() + STARTUP_SECONDS, False
+        while True:
+            output.read(_POLL_SECONDS)
+            if output.closed:
+                process.wait()
+                return output.stdout, output.stderr
+            if output.ready and not ready:
+                deadline, ready = time.monotonic() + self.config.timeout_seconds, True
+            try:
+                used = sum(_memory(p) for p in [watched, *watched.children(recursive=True)])
+            except psutil.Error:
+                continue
+            if used > limit:
+                outcome = Rejected((MEMORY_REASON.format(memory_mb=self.config.memory_mb),))
+            elif time.monotonic() > deadline and ready:
+                outcome = Rejected((f"graph build timed out after {self.config.timeout_seconds:g} s",))
+            elif time.monotonic() > deadline:
+                outcome = RuntimeError(f"Gate: the sandbox was not ready within {STARTUP_SECONDS:g} s")
+            else:
+                continue
+            process.kill()
+            process.wait()
+            if isinstance(outcome, RuntimeError):
+                raise outcome
+            return outcome
+
+
+class _Output:
+    """The sandbox's standard output and error, read to their end as they come, so neither pipe fills."""
+
+    def __init__(self, process: subprocess.Popen):
+        self._stdout, self._stderr = process.stdout.fileno(), process.stderr.fileno()
+        self._open = [self._stdout, self._stderr]
+        self._read = {fd: b"" for fd in self._open}
+
+    def read(self, timeout: float) -> None:
+        """Read what the sandbox writes within ``timeout`` seconds."""
+        for fd in select.select(self._open, [], [], timeout)[0]:
+            chunk = os.read(fd, 65536)
+            self._read[fd] += chunk
+            if not chunk:
+                self._open.remove(fd)
+
+    @property
+    def closed(self) -> bool:
+        """Whether the sandbox has closed both pipes: it has exited."""
+        return not self._open
+
+    @property
+    def ready(self) -> bool:
+        """Whether the sandbox has written ``READY``: it has started."""
+        return READY in self._read[self._stdout]
+
+    @property
+    def stdout(self) -> str:
+        """The standard output after ``READY``, all of it if the sandbox exited before."""
+        before, ready, after = self._read[self._stdout].partition(READY)
+        return (after if ready else before).decode()
+
+    @property
+    def stderr(self) -> str:
+        """The standard error."""
+        return self._read[self._stderr].decode(errors="replace")
+
+
+def _memory(process: psutil.Process) -> int:
+    """Return the bytes ``process`` uses.
+
+    On macOS this is its physical footprint, which counts the pages the
+    memory compressor holds; the resident set alone falls as the compressor works.
+    """
+    if sys.platform != "darwin":
+        return process.memory_info().rss
+    usage = (ctypes.c_uint64 * 40)()  # struct rusage_info_v2
+    if _libproc().proc_pid_rusage(process.pid, 2, ctypes.byref(usage)) != 0:
+        raise psutil.NoSuchProcess(process.pid)
+    return usage[9]  # ri_phys_footprint, after the 16-byte uuid and seven counters
+
+
+@cache
+def _libproc() -> ctypes.CDLL:
+    """macOS's libproc, loaded once."""
+    return ctypes.CDLL(ctypes.util.find_library("proc"))
+
+
+def _last_line(stderr: str) -> str:
+    """The last line of a crash's stderr, its exception, with no file path, so a replay elsewhere reads the same."""
+    lines = [line for line in stderr.splitlines() if line.strip()]
+    return re.sub(r"(?<![\w.])(?:/[\w.-]+){2,}/", "", lines[-1].strip()) if lines else "no output"
+
+
+@cache
+def _bubblewrap_works() -> bool:
+    """Check once that the installed bubblewrap can run the sandbox's imports.
+
+    Raises
+    ------
+    RuntimeError
+        If it cannot (for example where unprivileged user namespaces are
+        disabled): every build would crash, which is a broken sandbox, not
+        a bad candidate.
+    """
+    probe = _bubblewrap([sys.executable, "-I", "-c", "import factor_agent.gate.graph"])
+    done = subprocess.run(probe, capture_output=True, text=True, timeout=60, env={"PATH": os.defpath})
+    if done.returncode != 0:
+        raise RuntimeError(f"Gate: bubblewrap is installed but cannot run the sandbox:\n{done.stderr[-_STDERR_TAIL:]}")
+    return True
+
+
+def _bubblewrap(command: list[str], modules=()) -> list[str]:
+    """Wrap ``command`` in bubblewrap.
+
+    The sandbox gets no network, a private /tmp, and read-only access to the
+    system directories, the Python installation and the packages it imports
+    (the vocabulary's ``modules`` among them), nothing else. A package
+    imported from outside the Python installation (an editable checkout) is
+    bound alone, so the rest of its checkout, credentials included, stays
+    out of view.
+    """
+    prefixes = {Path(p).resolve() for p in (sys.prefix, sys.base_prefix, sys.exec_prefix)}
+    visible = set(prefixes) | {Path(p) for p in ("/usr", "/lib", "/lib64", "/bin", "/etc/ld.so.cache")}
+    for name in dict.fromkeys(["factor_agent", "quantlab", "KunQuant", *(m.split(".")[0] for m in modules)]):
+        source = Path(importlib.import_module(name).__file__).resolve()
+        package = source.parent if source.name == "__init__.py" else source
+        if not any(package.is_relative_to(prefix) for prefix in prefixes):
+            visible.add(package)
+    binds = []
+    for path in sorted(visible):
+        binds += ["--ro-bind-try", str(path), str(path)]
+    return [
+        "bwrap", "--unshare-all", "--die-with-parent", "--new-session",
+        *binds, "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--chdir", "/tmp",
+        *command,
+    ]

factor_agent/gate/_canonical.py

@@ -0,0 +1,331 @@
+"""Canonical hashes of a graph's nodes: what the fingerprint and duplicate checks compare.
+
+Each node's hash is a Merkle hash of its operator, attributes, inputs and
+enclosing loop, computed after three rewrites:
+
+- constant folding: an elementwise operator whose inputs are all constants
+  becomes the constant it computes;
+- constant arithmetic in one form: ``x + c``, ``Add(x, ConstantOp(c))`` and
+  ``x - (-c)`` are all ``AddConst(x, c)``, ``x / c`` is ``MulConst(x, 1/c)``,
+  nested constant offsets and scales merge, and ``x + 0`` and ``x * 1`` are ``x``;
+- commutative ordering: the inputs of a commutative operator are sorted;
+- division guards: ``Select(d > eps, n / d, nan)``, also on ``Abs(d)``, with
+  ``d <= eps`` and the NaN first, and with the denominator itself guarded
+  (``n / Select(d > eps, d, 1)``, as quantlab's ``FundamentalAlpha`` writes
+  its ratios), is ``n / d`` for any positive ``eps`` up to ``GUARD_EPSILON``.
+
+A repeated subexpression hashes the same wherever it appears, so the
+fingerprint does not depend on how the code shared intermediate values.
+Multiplying by zero is never folded: ``0 * x`` is NaN where ``x`` is, which
+the NaN-preserving stand-ins rely on.
+"""
+
+import hashlib
+import json
+import math
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from factor_agent.gate.graph import Graph
+
+#: Operators whose inputs may be reordered.
+COMMUTATIVE = frozenset({
+    "Add", "Mul", "Max", "Min", "Equals", "And", "Or", "WindowedCorrelation", "WindowedCovariance", "EWCov",
+    "SessionCorr",
+})
+
+#: Elementwise operators folded when every input is a constant.
+_FOLD = {
+    "Add": lambda a, b: a + b,
+    "Sub": lambda a, b: a - b,
+    "Mul": lambda a, b: a * b,
+    "Div": lambda a, b: a / b,
+    "Max": max,
+    "Min": min,
+    "Pow": lambda a, b: a ** b,
+    "Abs": abs,
+    "Sign": lambda a: float((a > 0) - (a < 0)),
+    "Exp": math.exp,
+    "Log": math.log,
+    "Sqrt": math.sqrt,
+}
+
+#: Significant digits a constant keeps, so 1/3 * 3 and 1 hash alike.
+_DIGITS = 12
+
+#: The largest threshold a division guard may test the denominator against.
+GUARD_EPSILON = 1e-6
+
+
+class _Const:
+    """A node that folded to a constant."""
+
+    def __init__(self, value: float):
+        self.value = value
+
+    def key(self):
+        return ["const", "nan" if math.isnan(self.value) else _rounded(self.value)]
+
+
+class _Node:
+    """A node's canonical form: operator, attributes, input terms, loop term."""
+
+    def __init__(self, op: str, attrs: dict, inputs: list, parent):
+        self.op = op
+        self.attrs = attrs
+        self.inputs = inputs
+        self.parent = parent
+        self.hash = _sha([op, sorted(attrs.items()), [_key(t) for t in inputs], None if parent is None else _key(parent)])
+
+    def key(self):
+        return self.hash
+
+
+def _rounded(value: float) -> float:
+    return float(f"{value:.{_DIGITS}g}")
+
+
+def _sha(value) -> str:
+    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
+
+
+def _key(term):
+    return term.key()
+
+
+def _same(one, other) -> bool:
+    return (one is None and other is None) or (one is not None and other is not None and _key(one) == _key(other))
+
+
+def _number(value):
+    """Attribute values as JSON numbers; 5 and 5.0 are the same window."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return value
+    return float(value)
+
+
+def canonical_hashes(graph: "Graph") -> list[str]:
+    """Return the canonical hash of every node of ``graph``, in node order; the last is the factor's."""
+    return [_sha(["term", _key(t)]) for t in canonical_terms(graph)]
+
+
+#: Operators whose result does not depend on the order of their inputs, to the last bit, NaN included.
+#: ``Max`` and ``Min`` are not among them: a SIMD maximum returns its second input when one is NaN.
+_EXACTLY_COMMUTATIVE = frozenset({"Add", "Mul", "Equals", "And", "Or", "SessionCorr"})
+
+
+def computation_hashes(graph: "Graph") -> list[str]:
+    """Return a hash of what every node of ``graph`` computes, in node order; the last is the factor's.
+
+    Two nodes hash alike only when they compute the same values to the last
+    bit: the hash is of the operator, attributes, inputs and loop as
+    written, without the canonical hash's rewrites (``x + 1e8 - 1e8`` is not
+    ``x`` in float32, nor ``x / 3`` ``x * (1/3)``), the inputs of an
+    operator whose result does not depend on their order sorted. A repeated
+    subexpression hashes the same wherever it appears.
+    """
+    hashes: list[str] = []
+    for node in graph.nodes:
+        inputs = [hashes[i] for i in node.inputs]
+        if node.op in _EXACTLY_COMMUTATIVE:
+            inputs.sort()
+        parent = None if node.parent is None else hashes[node.parent]
+        hashes.append(_sha(["computation", node.op, sorted(node.attrs), inputs, parent]))
+    return hashes
+
+
+def effective_parameters(graph: "Graph", lags_below: frozenset[str] = frozenset()) -> list[tuple[str, str, float]]:
+    """Return ``(operator, kind, value)`` for every value the factor uses once constants are merged.
+
+    Kinds are ``"constant"`` (a merged offset or a folded constant),
+    ``"scale"`` (a merged multiplier; dividing by ``c`` is scaling by ``1/c``)
+    and ``"lag"`` (a merged chain of ``BackRef``). Chaining allowed values
+    (``x * 0.25 * 1.5``, ``BackRef(BackRef(x, 5), 10)``) shows up here as the
+    value it really is. The lags below an operator of ``lags_below`` (a
+    session aggregation, whose inputs count bars, not days) are left out.
+    """
+    found, seen, stack = [], set(), [(canonical_terms(graph)[-1], True)]
+    while stack:
+        term, lags = stack.pop()
+        if isinstance(term, _Const):
+            found.append(("ConstantOp", "constant", term.value))
+            continue
+        if (term.hash, lags) in seen:
+            continue
+        seen.add((term.hash, lags))
+        if term.op in ("AddConst", "SubConstSwap", "DivConstSwap"):
+            found.append((term.op, "constant", term.attrs["value"]))
+        elif term.op == "MulConst":
+            found.append((term.op, "scale", term.attrs["value"]))
+        elif term.op == "BackRef" and lags:
+            found.append((term.op, "lag", term.attrs["window"]))
+        below = lags and term.op not in lags_below
+        stack += [(t, below) for t in term.inputs + ([] if term.parent is None else [term.parent])]
+    return found
+
+
+def canonical_terms(graph: "Graph") -> list:
+    """Return every node's canonical term, in node order; the last is the factor's."""
+    terms = []
+    for node in graph.nodes:
+        attrs = dict(node.attrs)
+        inputs = [terms[i] for i in node.inputs]
+        parent = None if node.parent is None else terms[node.parent]
+        terms.append(_canonical(node.op, attrs, inputs, parent))
+    return terms
+
+
+def _canonical(op: str, attrs: dict, inputs: list, parent):
+    """Return one node's canonical term, given its inputs' and loop's terms."""
+    if op == "ConstantOp":
+        value = attrs["value"]
+        return _Const(float("nan") if value == "nan" else float(value))
+    swap = bool(attrs.pop("swap", False))
+    attrs = {k: _number(v) for k, v in attrs.items()}
+
+    if op in ("AddConst", "SubConst", "MulConst", "DivConst"):
+        x, c = inputs[0], attrs["value"]
+        return _with_constant(op, x, c, swap, parent)
+    if op in _FOLD and inputs and all(isinstance(t, _Const) for t in inputs):
+        try:
+            return _Const(float(_FOLD[op](*(t.value for t in inputs))))
+        except (ArithmeticError, ValueError):
+            pass
+    if op in ("Add", "Sub", "Mul", "Div") and len(inputs) == 2:
+        left, right = inputs
+        if isinstance(right, _Const) and not isinstance(left, _Const):
+            return _with_constant(op + "Const", left, right.value, False, parent)
+        if isinstance(left, _Const) and not isinstance(right, _Const):
+            swapped = op in ("Sub", "Div")
+            return _with_constant(op + "Const", right, left.value, swapped, parent)
+    if op == "BackRef" and isinstance(inputs[0], _Node) and inputs[0].op == "BackRef" and _same(inputs[0].parent, parent):
+        return _Node(op, {"window": inputs[0].attrs["window"] + attrs["window"]}, inputs[0].inputs, parent)
+    if op == "Select":
+        ratio = _unguarded(inputs, parent)
+        if ratio is not None:
+            return ratio
+    if op in COMMUTATIVE:
+        inputs = sorted(inputs, key=lambda t: json.dumps(_key(t)))
+    return _Node(op, attrs, inputs, parent)
+
+
+def _guard(condition) -> tuple[object, bool] | None:
+    """``(d, holds)`` when ``condition`` tests ``d`` (or ``Abs(d)``) against a small positive threshold.
+
+    ``holds`` is True for ``d > eps`` (the ratio is valid where it holds),
+    False for ``d <= eps``; None for any other condition.
+    """
+    if not isinstance(condition, _Node):
+        return None
+    tested, threshold, holds = None, None, None
+    if condition.op in ("GreaterThan", "LessEqual") and isinstance(condition.inputs[1], _Const):
+        tested, threshold, holds = condition.inputs[0], condition.inputs[1].value, condition.op == "GreaterThan"
+    elif condition.op in ("LessThan", "GreaterEqual") and isinstance(condition.inputs[0], _Const):
+        tested, threshold, holds = condition.inputs[1], condition.inputs[0].value, condition.op == "LessThan"
+    elif condition.op in ("GreaterThanConst", "LessEqualConst") and not condition.attrs.get("swap"):
+        tested, threshold, holds = condition.inputs[0], condition.attrs["value"], condition.op == "GreaterThanConst"
+    if tested is None or not 0 < threshold <= GUARD_EPSILON:
+        return None
+    if isinstance(tested, _Node) and tested.op == "Abs":
+        tested = tested.inputs[0]
+    return tested, holds
+
+
+def _unguarded(inputs: list, parent):
+    """``n / d`` for a guarded ratio ``Select(guard on d, n / d, nan)`` (either order); None for another Select."""
+    if len(inputs) != 3:
+        return None
+    guard = _guard(inputs[0])
+    if guard is None:
+        return None
+    d, holds = guard
+    ratio, fill = (inputs[1], inputs[2]) if holds else (inputs[2], inputs[1])
+    if not (isinstance(fill, _Const) and math.isnan(fill.value)) or not isinstance(ratio, _Node):
+        return None
+    if ratio.op == "Div":
+        numerator, denominator = ratio.inputs
+    elif ratio.op == "DivConstSwap":
+        numerator, [denominator] = None, ratio.inputs
+    else:
+        return None
+    inner = _guard(denominator.inputs[0]) if isinstance(denominator, _Node) and denominator.op == "Select" else None
+    if inner is not None and inner[1] and _same(inner[0], d) and _same(denominator.inputs[1], d):
+        denominator = d
+    if not _same(denominator, d) or not _same(ratio.parent, parent):
+        return None
+    if numerator is None:
+        return _Node("DivConstSwap", ratio.attrs, [d], parent)
+    return _canonical("Div", {}, [numerator, d], parent)
+
+
+def _with_constant(op: str, x, c: float, swap: bool, parent):
+    """``x op c`` (``c op x`` when ``swap``) as ``AddConst``/``MulConst``, or a swapped form."""
+    if isinstance(x, _Const):
+        a, b = (c, x.value) if swap else (x.value, c)
+        try:
+            return _Const(float(_FOLD[op.removesuffix("Const")](a, b)))
+        except (ArithmeticError, ValueError):
+            pass
+    if op == "SubConst" and not swap:
+        op, c = "AddConst", -c
+    elif op == "DivConst" and not swap and c != 0:
+        op, c = "MulConst", 1.0 / c
+    if op in ("SubConst", "DivConst"):  # c - x, c / x
+        return _Node(op + "Swap", {"value": _number(c)}, [x], parent)
+    if isinstance(x, _Node) and x.op == op and _same(x.parent, parent):
+        inner = x.attrs["value"]
+        c = inner + c if op == "AddConst" else inner * c
+        x = x.inputs[0]
+    if (op == "AddConst" and c == 0) or (op == "MulConst" and c == 1):
+        return x
+    return _Node(op, {"value": _rounded(c)}, [x], parent)
+
+
+#: Root operators a cross-sectional rank IC cannot tell from their input.
+_RANK_PRESERVING = frozenset({"Rank", "Scale", "CrossSectionalZScore"})
+
+
+def ranked_root(term):
+    """Return ``term`` without the rank-preserving operators at its root.
+
+    A cross-sectional rank, scale or z-score, a constant offset and a
+    nonzero scale keep or reverse the factor's cross-sectional order, so
+    ``Rank(x) * 2 - 1`` and ``1 - x`` are ``x`` to the originality check
+    (the sign is the factor's direction, read from its training IC, not a new factor).
+    """
+    while isinstance(term, _Node) and (
+        term.op in _RANK_PRESERVING
+        or term.op in ("AddConst", "SubConstSwap")
+        or (term.op == "MulConst" and term.attrs["value"] != 0)
+    ):
+        term = term.inputs[0]
+    return term
+
+
+def term_key(term) -> str:
+    """Return a canonical term's identity as a string."""
+    return json.dumps(_key(term))
+
+
+def shape_key(term) -> str:
+    """Return the hash of a canonical term with every numeric attribute erased.
+
+    Two terms with the same shape differ at most in their windows, lags and
+    constants.
+    """
+    memo: dict[int, str] = {}
+
+    def shape(t) -> str:
+        if isinstance(t, _Const):
+            return "const"
+        if id(t) not in memo:
+            attrs = sorted(
+                (k, "#" if isinstance(v, (int, float)) and not isinstance(v, bool) else v) for k, v in t.attrs.items()
+            )
+            inputs = [shape(i) for i in t.inputs]
+            if t.op in COMMUTATIVE:
+                inputs.sort()
+            memo[id(t)] = _sha([t.op, attrs, inputs, None if t.parent is None else shape(t.parent)])
+        return memo[id(t)]
+
+    return shape(term)