openscript 0.4.0__py3-none-any.whl

openscript/verify_code.py

@@ -0,0 +1,290 @@
+"""Checks 2 to 7 of ``compiled-program.md`` section 3.5, over one instruction list.
+
+Verification is not optional and not a debug mode. An engine that skips it can
+be handed a malformed list and will read past the end of an array or jump into
+the middle of an expression, and the failure will look like a wrong number
+rather than a broken program. What these checks buy is what the interpreter is
+then allowed to assume and therefore not to test: a verified program cannot
+underflow the stack, cannot jump out of bounds, cannot address a slot that does
+not exist and cannot loop without charging the budget.
+
+Check 6 is the one that looks small and is not. The target of every backward
+jump must be a ``TICK``, so no cycle in the control flow graph can run without
+charging the loop budget, and the whole of that guarantee is checkable by
+looking at one instruction rather than by analysing the graph.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from .diagnostics import at_instruction
+from .opcodes import (
+    INSTRUCTION_TAGS,
+    TABLE_NAMES,
+    depth_change,
+    is_opcode,
+    operand_count,
+    tables_for,
+    targets,
+)
+from .verify_shape import ShapeCheck
+
+
+@dataclass
+class ListLimits:
+    """How big each table an operand may index is, for the list being walked."""
+
+    slots: int
+    cells: int
+    states: int
+    registers: int
+    channels: int
+    libFunctions: int
+    callSites: int
+    loops: int
+    consts: int
+    bindings: int
+    argc_of: Callable[[int], int] = field(default=lambda site: 0)
+
+    def size(self, table: str) -> Optional[int]:
+        return getattr(self, table, None)
+
+
+def check_list(
+    shape: ShapeCheck,
+    listing: str,
+    code: Sequence[Any],
+    limits: ListLimits,
+    terminator: str,
+) -> bool:
+    """Walks one instruction list: checks 2, 3, 4, 5 and 6.
+
+    ``terminator`` is ``HALT`` for the bar's own code and ``RET`` for a function
+    body or a read's body, which is check 4, and the depth at it must be zero,
+    which is check 5.
+    """
+    if len(code) == 0:
+        return shape.fail(listing, "an instruction list cannot be empty")
+
+    # Check 2, and every operand in range.
+    for at, instruction in enumerate(code):
+        where = at_instruction(listing, at)
+        if not shape.array(instruction, where):
+            return False
+        opcode = instruction[0] if instruction else None
+        if not is_opcode(opcode):
+            return shape.fail(where, f"{opcode} is not an opcode this engine implements")
+        operands = instruction[1:]
+        if len(operands) != operand_count(opcode):
+            return shape.fail(
+                where,
+                f"{opcode} takes {operand_count(opcode)} operands and carries {len(operands)}",
+            )
+        if not _check_operands(shape, where, opcode, operands, limits, len(code)):
+            return False
+
+        # Check 6: the target of a backward jump is a TICK.
+        for position in targets(opcode):
+            target = operands[position]
+            if target > at:
+                continue
+            landing = code[target]
+            if not isinstance(landing, list) or not landing or landing[0] != "TICK":
+                return shape.fail(
+                    where,
+                    f"it jumps back to instruction {target}, which is not a TICK, so the loop "
+                    "would run without charging the budget",
+                )
+
+    # Check 4.
+    last = code[-1]
+    if not isinstance(last, list) or not last or last[0] != terminator:
+        return shape.fail(
+            at_instruction(listing, len(code) - 1),
+            f"the last instruction of this list must be {terminator}",
+        )
+    for at in range(len(code) - 1):
+        if code[at][0] == "HALT":
+            return shape.fail(at_instruction(listing, at), "HALT is only ever the last instruction")
+
+    return _check_depths(shape, listing, code, limits, terminator)
+
+
+def _check_operands(
+    shape: ShapeCheck,
+    where: str,
+    opcode: str,
+    operands: Sequence[Any],
+    limits: ListLimits,
+    length: int,
+) -> bool:
+    tables = tables_for(opcode)
+    for at, operand in enumerate(operands):
+        if not shape.whole(operand, f"{where} operand {at}"):
+            return False
+        table = tables[at] if at < len(tables) else None
+        if table is None or table == "count":
+            if operand < 0:
+                return shape.fail(f"{where} operand {at}", "a count cannot be negative")
+            continue
+        if table == "target":
+            # Check 3. A target past the terminator is out of the list.
+            if operand < 0 or operand >= length:
+                return shape.fail(where, f"the jump target {operand} is outside a list of {length}")
+            continue
+        # A CALL_LIB with no state carries -1 rather than an index.
+        if opcode == "CALL_LIB" and table == "states" and operand == -1:
+            continue
+        size = limits.size(table)
+        if size is None:
+            continue
+        if operand < 0 or operand >= size:
+            named = TABLE_NAMES.get(table, table)
+            return shape.fail(
+                f"{where} operand {at}", f"{operand} is outside {named}, which holds {size}"
+            )
+    return True
+
+
+def _check_depths(
+    shape: ShapeCheck,
+    listing: str,
+    code: Sequence[Any],
+    limits: ListLimits,
+    terminator: str,
+) -> bool:
+    """Check 5: the depth agrees on every path, never goes below zero, is zero at the end.
+
+    A disagreement at a join is a corrupt program and not a program with an
+    unusual shape: every instruction has a fixed effect, so two paths reaching
+    one instruction with different depths means one of them was built wrong.
+    """
+    depths: List[Optional[int]] = [None] * len(code)
+    depths[0] = 0
+    pending = [0]
+
+    def reach(index: int, depth: int, origin: int) -> bool:
+        if depth < 0:
+            return shape.fail(at_instruction(listing, origin), "it takes the stack below zero")
+        known = depths[index]
+        if known is None:
+            depths[index] = depth
+            pending.append(index)
+            return True
+        if known != depth:
+            return shape.fail(
+                at_instruction(listing, index),
+                f"two paths reach it with the stack {known} and {depth} deep",
+            )
+        return True
+
+    while pending:
+        index = pending.pop()
+        instruction = code[index]
+        depth = depths[index]
+        opcode = instruction[0]
+        operands = instruction[1:]
+        after = depth + depth_change(opcode, operands, limits.argc_of)
+        if opcode == terminator:
+            if after != 0:
+                return shape.fail(
+                    at_instruction(listing, index),
+                    f"the stack is {after} deep at {terminator} and must be empty",
+                )
+            continue
+        for position in targets(opcode):
+            if not reach(operands[position], after, index):
+                return False
+        if opcode not in ("JUMP", "RET", "HALT"):
+            if not reach(index + 1, after, index):
+                return False
+
+    return True
+
+
+#: How high a write count is carried before it stops mattering.
+#:
+#: "More than once" is the whole of what check 7 needs to know, and without a
+#: ceiling a loop around an ``EMIT`` would never settle.
+_CAP = 2
+
+
+def check_once(shape: ShapeCheck, code: Sequence[Any], once: Sequence[bool]) -> bool:
+    """Check 7: every channel declared ``once`` is written exactly once on every path.
+
+    That is how a plot column is guaranteed a value, or an explicit absence, for
+    every bar.
+    """
+    channels = len(once)
+    if channels == 0:
+        return True
+    least: List[Optional[List[int]]] = [None] * len(code)
+    most: List[Optional[List[int]]] = [None] * len(code)
+    least[0] = [0] * channels
+    most[0] = [0] * channels
+    pending = [0]
+
+    def merge(index: int, low: Sequence[int], high: Sequence[int]) -> None:
+        if index < 0 or index >= len(code):
+            return
+        known_low, known_high = least[index], most[index]
+        if known_low is None or known_high is None:
+            least[index] = list(low)
+            most[index] = list(high)
+            pending.append(index)
+            return
+        changed = False
+        for channel in range(channels):
+            lower = min(known_low[channel], low[channel])
+            upper = min(_CAP, max(known_high[channel], high[channel]))
+            if lower != known_low[channel]:
+                known_low[channel] = lower
+                changed = True
+            if upper != known_high[channel]:
+                known_high[channel] = upper
+                changed = True
+        if changed:
+            pending.append(index)
+
+    end = -1
+    while pending:
+        index = pending.pop()
+        instruction = code[index]
+        low, high = least[index], most[index]
+        opcode = instruction[0]
+        if opcode == "HALT":
+            end = index
+            continue
+        next_low, next_high = list(low), list(high)
+        if opcode == "EMIT":
+            channel = instruction[1]
+            next_low[channel] = min(_CAP, next_low[channel] + 1)
+            next_high[channel] = min(_CAP, next_high[channel] + 1)
+        for position in targets(opcode):
+            merge(instruction[position + 1], next_low, next_high)
+        if opcode not in ("JUMP", "RET"):
+            merge(index + 1, next_low, next_high)
+
+    if end < 0:
+        return shape.fail("code", "no path reaches HALT")
+    low, high = least[end], most[end]
+    for channel in range(channels):
+        if not once[channel]:
+            continue
+        if low[channel] != 1 or high[channel] != 1:
+            return shape.fail(
+                f"channels[{channel}]",
+                f"it is declared once and is written {low[channel]} to {high[channel]} times "
+                "on a path to HALT",
+            )
+    return True
+
+
+def missing_tag(lists: Sequence[Sequence[Any]], requires: Sequence[str]) -> Optional[Dict[str, str]]:
+    """Check 8's program half: an instruction whose tag the program never declared."""
+    for code in lists:
+        for instruction in code:
+            tag = INSTRUCTION_TAGS.get(instruction[0])
+            if tag is not None and tag not in requires:
+                return {"opcode": instruction[0], "tag": tag}
+    return None

openscript/verify_requests.py

@@ -0,0 +1,271 @@
+"""Check 1 over the tables a machine indexes, and over ``requests`` and its bodies.
+
+A read's body (section 2.16.1) holds the tables a program holds for the machine
+of section 3, counted from zero, so it needs the same walk the program's own
+tables get and for the same reason: every later step indexes into them, and an
+index out of range that is not caught here is a read past the end of an array
+during a bar.
+
+**A body that was not walked is the hole verification exists to close.** It is
+reached by the same interpreter, with the same instruction list, over another
+instrument's history. Verifying the program and not the expression it carries
+would leave exactly the failure section 3.5 is written to prevent, one level
+down where nobody would look for it.
+
+The walk is shared rather than written twice, because two copies would be two
+places for a table to gain a field and only one of them to learn about it.
+"""
+
+from dataclasses import dataclass
+from typing import Any, List, Sequence, Set
+
+from .verify_shape import ShapeCheck
+
+READS = ("timeframe", "symbol")
+MODES = ("confirmed", "developing", "lookahead")
+CHART_FACTS = ("symbol", "exchange", "interval")
+
+#: The five kinds section 2.10 declares, not the ones an engine happens to run.
+#:
+#: A verifier that listed only the kinds it executes would refuse a well formed
+#: program with a message blaming the compiler that wrote it, and would pre-empt
+#: the capability refusal that exists to say which feature is missing.
+#: Verification answers whether this is a valid program; the capability check
+#: answers whether this engine can run it.
+REGISTER_KINDS = ("bar", "computed", "argument", "request", "input")
+CELL_KINDS = ("var", "live")
+LOOP_KINDS = ("for", "forIn", "while")
+
+
+@dataclass(frozen=True)
+class MachineTables:
+    """The tables one machine indexes, wherever in the program they sit."""
+
+    series: List[Any]
+    cells: List[Any]
+    states: List[Any]
+    functions: List[Any]
+    call_sites: List[Any]
+    loops: List[Any]
+    slots: int
+    lib_functions: int
+
+
+def check_machine_tables(shape: ShapeCheck, prefix: str, tables: MachineTables) -> bool:
+    for at, register in enumerate(tables.series):
+        path = f"{prefix}series[{at}]"
+        if not shape.object(register, path):
+            return False
+        if register.get("id") != at:
+            return shape.fail(f"{path}.id", "a register id is its position")
+        if not shape.one(register.get("kind"), f"{path}.kind", REGISTER_KINDS):
+            return False
+        if register["kind"] == "bar" and not shape.string(register.get("field"), f"{path}.field"):
+            return False
+
+    for at, cell in enumerate(tables.cells):
+        path = f"{prefix}cells[{at}]"
+        if not shape.object(cell, path):
+            return False
+        if not shape.one(cell.get("kind"), f"{path}.kind", CELL_KINDS):
+            return False
+
+    for at, state in enumerate(tables.states):
+        path = f"{prefix}states[{at}]"
+        if not shape.object(state, path):
+            return False
+        if not shape.index(state.get("fn"), f"{path}.fn", tables.lib_functions, "lib.functions"):
+            return False
+
+    for at, one in enumerate(tables.functions):
+        path = f"{prefix}functions[{at}]"
+        if not shape.object(one, path):
+            return False
+        if not shape.string(one.get("name"), f"{path}.name"):
+            return False
+        if not shape.whole(one.get("params"), f"{path}.params"):
+            return False
+        if not shape.whole(one.get("slots"), f"{path}.slots"):
+            return False
+        if not shape.array(one.get("code"), f"{path}.code"):
+            return False
+        if one["params"] > one["slots"]:
+            return shape.fail(f"{path}.slots", "a frame cannot be smaller than its parameter list")
+
+    for at, site in enumerate(tables.call_sites):
+        path = f"{prefix}callSites[{at}]"
+        if not shape.object(site, path):
+            return False
+        if not shape.index(site.get("fn"), f"{path}.fn", len(tables.functions), "functions"):
+            return False
+        for name in ("argc", "cellBase", "stateBase"):
+            if not shape.whole(site.get(name), f"{path}.{name}"):
+                return False
+        if not shape.array(site.get("series"), f"{path}.series"):
+            return False
+        if site["argc"] != tables.functions[site["fn"]]["params"]:
+            return shape.fail(
+                f"{path}.argc", "a site passes exactly the function's parameter count"
+            )
+        for which, bound in enumerate(site["series"]):
+            if bound == -1:
+                continue
+            if not shape.index(bound, f"{path}.series[{which}]", len(tables.series), "series"):
+                return False
+
+    for at, loop in enumerate(tables.loops):
+        path = f"{prefix}loops[{at}]"
+        if not shape.object(loop, path):
+            return False
+        if not shape.one(loop.get("kind"), f"{path}.kind", LOOP_KINDS):
+            return False
+        for name in ("line", "col"):
+            if not shape.whole(loop.get(name), f"{path}.{name}"):
+                return False
+
+    return True
+
+
+def check_requests(
+    shape: ShapeCheck,
+    prefix: str,
+    requests: Sequence[Any],
+    registers: int,
+    lib_functions: int,
+    keys: Set[str],
+) -> bool:
+    """``requests`` and every body inside it, section 2.16.
+
+    ``series`` is the register of the enclosing machine that the read's value
+    lands in, so it is checked against the table the caller passes; everything
+    inside ``body`` is the body's own and is checked against the body's.
+    """
+    return _walk(shape, prefix, requests, registers, lib_functions, keys, set())
+
+
+def _walk(
+    shape: ShapeCheck,
+    prefix: str,
+    requests: Sequence[Any],
+    registers: int,
+    lib_functions: int,
+    keys: Set[str],
+    seen: Set[int],
+) -> bool:
+    for at, request in enumerate(requests):
+        path = f"{prefix}requests[{at}]"
+        if not shape.object(request, path):
+            return False
+        if not shape.whole(request.get("id"), f"{path}.id"):
+            return False
+        # The id is this read's handle and is what req.isReady is handed, so two
+        # reads sharing one would answer each other's question.
+        if request["id"] in seen:
+            handle = request["id"]
+            return shape.fail(f"{path}.id", f"{handle} is already another read's handle")
+        seen.add(request["id"])
+        if not shape.one(request.get("read"), f"{path}.read", READS):
+            return False
+        if not shape.one(request.get("mode"), f"{path}.mode", MODES):
+            return False
+        if not shape.index(request.get("series"), f"{path}.series", registers, "series"):
+            return False
+        warmup = request.get("warmup")
+        if warmup is not None and not shape.whole(warmup, f"{path}.warmup"):
+            return False
+        for name in ("symbol", "exchange", "timeframe"):
+            if not _check_identity(shape, request.get(name), f"{path}.{name}", keys):
+                return False
+        if not _check_body(shape, f"{path}.body.", request.get("body"), lib_functions, keys, seen):
+            return False
+    return True
+
+
+#: The tables a read's body carries, section 2.16.1. Not the program's tables:
+#: a body holds neither the constant pool nor the library manifest, because
+#: neither holds anything that depends on a bar.
+BODY_TABLES = (
+    "inputs",
+    "series",
+    "cells",
+    "states",
+    "functions",
+    "callSites",
+    "loops",
+    "requests",
+    "code",
+    "pos",
+    "fnPos",
+)
+
+
+def _check_body(
+    shape: ShapeCheck,
+    prefix: str,
+    raw: Any,
+    lib_functions: int,
+    keys: Set[str],
+    seen: Set[int],
+) -> bool:
+    at = prefix[:-1]
+    if not shape.object(raw, at):
+        return False
+    for name in BODY_TABLES:
+        if not shape.array(raw.get(name), f"{prefix}{name}"):
+            return False
+    if not shape.object(raw.get("frame"), f"{prefix}frame"):
+        return False
+    if not shape.whole(raw["frame"].get("slots"), f"{prefix}frame.slots"):
+        return False
+
+    series = raw["series"]
+    tables = MachineTables(
+        series=series,
+        cells=raw["cells"],
+        states=raw["states"],
+        functions=raw["functions"],
+        call_sites=raw["callSites"],
+        loops=raw["loops"],
+        slots=raw["frame"]["slots"],
+        lib_functions=lib_functions,
+    )
+    if not check_machine_tables(shape, prefix, tables):
+        return False
+
+    for which, one in enumerate(raw["inputs"]):
+        path = f"{prefix}inputs[{which}]"
+        if not shape.object(one, path):
+            return False
+        if not shape.string(one.get("input"), f"{path}.input"):
+            return False
+        if one["input"] not in keys:
+            named = one["input"]
+            return shape.fail(path, f"it names the input {named}, which inputs[] does not declare")
+        if not shape.index(one.get("series"), f"{path}.series", len(series), "the body's series"):
+            return False
+
+    return _walk(shape, prefix, raw["requests"], len(series), lib_functions, keys, seen)
+
+
+def _check_identity(shape: ShapeCheck, value: Any, path: str, keys: Set[str]) -> bool:
+    """One of the three forms a request's identity field takes, section 2.16.
+
+    A value, the setting that supplies it, or one of the three chart facts that
+    identify a chart rather than describe it. The rest of the chart namespace
+    describes the chart, and a request built from one of those would name
+    nothing.
+    """
+    if value is None or isinstance(value, str):
+        return True
+    if not shape.object(value, path):
+        return False
+    if "input" in value:
+        if not shape.string(value["input"], f"{path}.input"):
+            return False
+        if value["input"] not in keys:
+            named = value["input"]
+            return shape.fail(path, f"it names the input {named}, which inputs[] does not declare")
+        return True
+    if "chart" not in value:
+        return shape.fail(path, "an object here names an input or a chart fact")
+    return shape.one(value["chart"], f"{path}.chart", CHART_FACTS)