opencode-pyneruntime 6.6.4__py3-none-any.whl

pynecore/testing/broker_lab/runner.py

@@ -0,0 +1,372 @@
+"""Scenario execution over the real broker engine and SQLite store."""
+
+import asyncio
+import socket
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+from pynecore import lib
+from pynecore.core.broker.position import BrokerPosition
+from pynecore.core.broker.run_identity import RunIdentity
+from pynecore.core.broker.storage import BrokerStore, RunContext
+from pynecore.core.broker.sync_engine import OrderSyncEngine
+from pynecore.lib.strategy import Order, _order_type_close, _order_type_entry, oca
+
+from .model import Scenario, ScenarioInvariantError, ScenarioResult, Step, VenueProfile
+from .scheduler import DeterministicScheduler
+
+
+@dataclass
+class RunRuntime:
+    """Resources belonging to one logical bot run."""
+
+    name: str
+    store: BrokerStore
+    store_ctx: RunContext
+    broker: Any
+    position: BrokerPosition
+    engine: OrderSyncEngine
+
+
+class _NetworkGuard:
+    def __enter__(self):
+        self._create_connection = socket.create_connection
+        self._connect = socket.socket.connect
+        self._connect_ex = socket.socket.connect_ex
+
+        def blocked(*_args: Any, **_kwargs: Any):
+            raise RuntimeError("offline broker lab blocked a network attempt")
+
+        setattr(socket, "create_connection", blocked)
+        setattr(socket.socket, "connect", blocked)
+        setattr(socket.socket, "connect_ex", blocked)
+        return self
+
+    def __exit__(self, *_exc: Any) -> None:
+        setattr(socket, "create_connection", self._create_connection)
+        setattr(socket.socket, "connect", self._connect)
+        setattr(socket.socket, "connect_ex", self._connect_ex)
+
+
+class ScenarioRunner:
+    """Execute, check, and minimize deterministic broker scenarios."""
+
+    def __init__(self, *, artifact_root: Path | None = None) -> None:
+        self.artifact_root = artifact_root
+        self.profile: VenueProfile
+        self.runs: dict[str, RunRuntime] = {}
+        self.now_ms = 1_700_000_000_000
+        self.scheduler = DeterministicScheduler(self.now_ms)
+        self._tmp: tempfile.TemporaryDirectory[str] | None = None
+        self._root: Path | None = None
+        self._script_sentinel = object()
+        self._old_script: Any = self._script_sentinel
+
+    @property
+    def workdir(self) -> Path:
+        """Return the isolated filesystem root of the active scenario."""
+        if self._root is None:
+            raise RuntimeError(
+                "scenario workdir is only available while a scenario is running"
+            )
+        return self._root
+
+    def run(self, scenario: Scenario, *, minimize: bool = True) -> ScenarioResult:
+        result = self._execute(scenario)
+        if result.passed or not minimize or len(scenario.steps) < 2:
+            return result
+        target = (result.violation or "").split(":", 1)[0]
+        steps = list(scenario.steps)
+        changed = True
+        while changed:
+            changed = False
+            for index in range(len(steps)):
+                candidate = steps[:index] + steps[index + 1 :]
+                if not candidate:
+                    continue
+                trial = self._execute(
+                    Scenario(
+                        name=scenario.name,
+                        profile_factory=scenario.profile_factory,
+                        steps=tuple(candidate),
+                        runs=scenario.runs,
+                        seed=scenario.seed,
+                        tags=scenario.tags,
+                        expected_violation=scenario.expected_violation,
+                    )
+                )
+                if (
+                    not trial.passed
+                    and (trial.violation or "").split(":", 1)[0] == target
+                ):
+                    steps = candidate
+                    changed = True
+                    break
+        return ScenarioResult(
+            name=result.name,
+            passed=False,
+            seed=result.seed,
+            executed_steps=result.executed_steps,
+            violation=result.violation,
+            minimized_steps=tuple(steps),
+            artifact_dir=result.artifact_dir,
+        )
+
+    def _execute(self, scenario: Scenario) -> ScenarioResult:
+        executed: list[Step] = []
+        violation: str | None = None
+        artifact_dir: Path | None = None
+        try:
+            self._start(scenario)
+            artifact_dir = self._root if self.artifact_root is not None else None
+            with _NetworkGuard():
+                for step in scenario.steps:
+                    self.apply(step)
+                    executed.append(step)
+                    if step.check_invariants:
+                        self.check_invariants()
+        except Exception as exc:
+            violation = f"{type(exc).__name__}: {exc}"
+        finally:
+            self.close()
+        if scenario.expected_violation is not None:
+            if violation is None:
+                violation = (
+                    f"expected violation was not raised: {scenario.expected_violation}"
+                )
+            elif scenario.expected_violation in violation:
+                violation = None
+        return ScenarioResult(
+            name=scenario.name,
+            passed=violation is None,
+            seed=scenario.seed,
+            executed_steps=tuple(executed),
+            violation=violation,
+            artifact_dir=artifact_dir,
+        )
+
+    def _start(self, scenario: Scenario) -> None:
+        self.profile = scenario.profile_factory()
+        self.runs = {}
+        self.now_ms = 1_700_000_000_000
+        self.scheduler = DeterministicScheduler(self.now_ms)
+        self._old_script = getattr(lib, "_script", self._script_sentinel)
+        lib._script = SimpleNamespace(initial_capital=1_000_000.0)
+        if self.artifact_root is None:
+            self._tmp = tempfile.TemporaryDirectory(prefix="pyne-broker-lab-")
+            self._root = Path(self._tmp.name)
+        else:
+            self._root = self.artifact_root / scenario.name
+            self._root.mkdir(parents=True, exist_ok=True)
+        for run_name in scenario.runs:
+            self._open_run(run_name)
+
+    def _open_run(self, run_name: str) -> None:
+        assert self._root is not None
+        store = BrokerStore(
+            self._root / "broker.sqlite",
+            plugin_name=self.profile.plugin_name,
+        )
+        identity = RunIdentity(
+            strategy_id="broker_lab",
+            symbol=self.profile.symbol,
+            timeframe=self.profile.timeframe,
+            account_id=self.profile.account_id,
+            label=run_name,
+        )
+        store_ctx = store.open_run(identity, script_source="// offline broker lab")
+        broker = self.profile.create_broker(run_name, store_ctx)
+        position = BrokerPosition()
+        engine = OrderSyncEngine(
+            broker=broker,
+            position=position,
+            symbol=self.profile.symbol,
+            run_tag=store_ctx.run_tag,
+            mintick=0.01,
+            store_ctx=store_ctx,
+            reconcile_every_n_syncs=1,
+        )
+        engine.reconcile()
+        self.runs[run_name] = RunRuntime(
+            name=run_name,
+            store=store,
+            store_ctx=store_ctx,
+            broker=broker,
+            position=position,
+            engine=engine,
+        )
+
+    def apply(self, step: Step) -> None:
+        if step.run not in self.runs:
+            raise KeyError(f"unknown run {step.run!r}")
+        runtime = self.runs[step.run]
+        values = step.values
+        if step.kind == "entry":
+            pine_id = str(values.get("id", "L"))
+            side = str(values.get("side", "buy"))
+            qty = float(values.get("qty", 1.0))
+            runtime.position.entry_orders[pine_id] = Order(
+                pine_id,
+                qty if side == "buy" else -qty,
+                order_type=_order_type_entry,
+                limit=values.get("limit"),
+                stop=values.get("stop"),
+                oca_name=values.get("oca_name"),
+                oca_type=getattr(oca, str(values.get("oca_type", "none"))),
+            )
+        elif step.kind == "amend":
+            pine_id = str(values.get("id", "L"))
+            old = runtime.position.entry_orders.get(pine_id)
+            if old is None:
+                raise ValueError(f"cannot amend unknown entry {pine_id!r}")
+            runtime.position.entry_orders[pine_id] = Order(
+                pine_id,
+                old.size,
+                order_type=_order_type_entry,
+                limit=values.get("limit", old.limit),
+                stop=values.get("stop", old.stop),
+                oca_name=values.get("oca_name", old.oca_name),
+                oca_type=getattr(oca, str(values.get("oca_type", old.oca_type))),
+            )
+        elif step.kind == "amend_exit":
+            exit_id = str(values.get("id", "X"))
+            from_entry = values.get("from_entry")
+            key = (exit_id, from_entry)
+            old = runtime.position.exit_orders.get(key)
+            if old is None:
+                raise ValueError(f"cannot amend unknown exit {key!r}")
+            order = Order(
+                from_entry,
+                old.size,
+                order_type=_order_type_close,
+                exit_id=exit_id,
+                limit=values.get("limit", old.limit),
+                stop=values.get("stop", old.stop),
+                trail_price=values.get("trail_price", old.trail_price),
+                trail_offset=values.get("trail_offset", old.trail_offset),
+            )
+            order.from_entry_na = from_entry is None
+            order.rest_leg = bool(values.get("rest_leg", old.rest_leg))
+            runtime.position.exit_orders[key] = order
+        elif step.kind in ("exit", "close", "close_percent"):
+            is_exit = step.kind == "exit"
+            pine_id = str(values.get("id", "X" if is_exit else "L"))
+            exit_id = pine_id if is_exit else f"Close entry(s) order {pine_id}"
+            from_entry = values.get("from_entry")
+            side = str(values.get("side", "sell"))
+            if step.kind == "close_percent":
+                percent = float(values["percent"])
+                if percent <= 0.0 or percent > 100.0:
+                    raise ValueError("close percent must be in (0, 100]")
+                qty = abs(runtime.position.size) * percent / 100.0
+            else:
+                qty = float(values.get("qty", abs(runtime.position.size) or 1.0))
+            order = Order(
+                from_entry if is_exit else pine_id,
+                qty if side == "buy" else -qty,
+                order_type=_order_type_close,
+                exit_id=exit_id,
+                limit=values.get("limit"),
+                stop=values.get("stop"),
+                trail_price=values.get("trail_price"),
+                trail_offset=values.get("trail_offset"),
+            )
+            order.from_entry_na = from_entry is None
+            order.rest_leg = bool(values.get("rest_leg", False))
+            key = (exit_id, from_entry if is_exit else pine_id)
+            runtime.position.exit_orders[key] = order
+        elif step.kind == "cancel":
+            cancel_id = values.get("id")
+            runtime.position.entry_orders.pop(cancel_id, None)
+            for key in list(runtime.position.exit_orders):
+                if cancel_id in key:
+                    runtime.position.exit_orders.pop(key)
+        elif step.kind == "cancel_all":
+            runtime.position.entry_orders.clear()
+            runtime.position.exit_orders.clear()
+        elif step.kind == "sync":
+            self.scheduler.advance(int(values.get("advance_ms", 1_000)))
+            self.now_ms = self.scheduler.now_ms
+            runtime.engine.sync(
+                self.now_ms, last_price=float(values.get("last_price", 100.0))
+            )
+        elif step.kind == "sync_expect_error":
+            self.scheduler.advance(int(values.get("advance_ms", 1_000)))
+            self.now_ms = self.scheduler.now_ms
+            expected = str(values["type"])
+            try:
+                runtime.engine.sync(
+                    self.now_ms,
+                    last_price=float(values.get("last_price", 100.0)),
+                )
+            except Exception as exc:
+                if type(exc).__name__ != expected:
+                    raise AssertionError(
+                        f"expected sync error {expected}, got {type(exc).__name__}: {exc}"
+                    ) from exc
+            else:
+                raise AssertionError(
+                    f"expected sync error {expected}, but sync succeeded"
+                )
+        elif step.kind == "advance":
+            self.scheduler.advance(int(values.get("ms", 1_000)))
+            self.now_ms = self.scheduler.now_ms
+        elif step.kind == "pump_watch":
+
+            async def pump_one() -> None:
+                stream = runtime.broker.watch_orders()
+                timeout_seconds = float(values.get("timeout_seconds", 1.0))
+                try:
+                    event = await asyncio.wait_for(
+                        stream.__anext__(),
+                        timeout=timeout_seconds,
+                    )
+                except TimeoutError as exc:
+                    raise AssertionError(
+                        "pump_watch received no broker event within "
+                        f"{timeout_seconds:g}s"
+                    ) from exc
+                finally:
+                    await stream.aclose()
+                runtime.engine.on_order_event(event)
+
+            asyncio.run(pump_one())
+            runtime.engine.apply_async_events()
+        elif step.kind == "restart":
+            runtime.store_ctx.close()
+            runtime.store.close()
+            del self.runs[step.run]
+            self._open_run(step.run)
+            self.runs[step.run].engine.settle_restart_state(self.now_ms)
+        elif step.kind == "shutdown":
+            runtime.store_ctx.close()
+            runtime.store.close()
+        elif not self.profile.handle_step(self, step):
+            raise ValueError(f"unsupported scenario step {step.kind!r}")
+
+    def check_invariants(self) -> None:
+        violations = list(self.profile.check_invariants(self))
+        if violations:
+            raise ScenarioInvariantError("; ".join(violations))
+
+    def close(self) -> None:
+        for runtime in list(self.runs.values()):
+            try:
+                runtime.store_ctx.close()
+            except Exception:
+                pass
+            runtime.store.close()
+        self.runs.clear()
+        if hasattr(self, "profile"):
+            self.profile.close()
+        if self._old_script is self._script_sentinel:
+            if hasattr(lib, "_script"):
+                delattr(lib, "_script")
+        else:
+            lib._script = self._old_script
+        if self._tmp is not None:
+            self._tmp.cleanup()
+            self._tmp = None

pynecore/testing/broker_lab/scheduler.py

@@ -0,0 +1,50 @@
+"""Deterministic virtual-time scheduling for offline broker scenarios."""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from heapq import heappop, heappush
+
+
+@dataclass(order=True)
+class ScheduledEvent:
+    """One callback ordered by virtual deadline and insertion sequence."""
+
+    deadline_ms: int
+    sequence: int
+    callback: Callable[[], None] = field(compare=False)
+
+
+class DeterministicScheduler:
+    """Advance callbacks without sleeping or consulting wall-clock time."""
+
+    def __init__(self, start_ms: int = 1_700_000_000_000) -> None:
+        self.now_ms = start_ms
+        self._sequence = 0
+        self._events: list[ScheduledEvent] = []
+
+    def schedule(self, delay_ms: int, callback: Callable[[], None]) -> int:
+        if delay_ms < 0:
+            raise ValueError("delay_ms must not be negative")
+        self._sequence += 1
+        heappush(
+            self._events,
+            ScheduledEvent(self.now_ms + delay_ms, self._sequence, callback),
+        )
+        return self._sequence
+
+    def advance(self, milliseconds: int) -> None:
+        if milliseconds < 0:
+            raise ValueError("milliseconds must not be negative")
+        target = self.now_ms + milliseconds
+        while self._events and self._events[0].deadline_ms <= target:
+            event = heappop(self._events)
+            self.now_ms = event.deadline_ms
+            event.callback()
+        self.now_ms = target
+
+    def run_ready(self) -> None:
+        self.advance(0)
+
+    @property
+    def pending(self) -> int:
+        return len(self._events)

pynecore/testing/broker_lab/subprocess.py

@@ -0,0 +1,73 @@
+"""Isolated subprocess helpers for opt-in CLI lifecycle scenarios."""
+
+import os
+import subprocess
+import sys
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+
+@dataclass(frozen=True)
+class SubprocessResult:
+    """Bounded child-process result captured by the broker lab."""
+
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+def run_subprocess(
+    args: Sequence[str],
+    *,
+    cwd: Path,
+    timeout: float = 10.0,
+    env: Mapping[str, str] | None = None,
+) -> SubprocessResult:
+    """Run a child with a hard external timeout and captured output."""
+    child_env = os.environ.copy()
+    if env:
+        child_env.update(env)
+    try:
+        completed = subprocess.run(
+            list(args),
+            cwd=cwd,
+            env=child_env,
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise TimeoutError(f"broker-lab subprocess exceeded {timeout:.1f}s: {args!r}") from exc
+    return SubprocessResult(completed.returncode, completed.stdout, completed.stderr)
+
+
+@contextmanager
+def temporary_entry_point(
+    *,
+    group: str,
+    name: str,
+    target: str,
+) -> Iterator[Path]:
+    """Inject disposable ``.dist-info`` metadata without production hooks."""
+    with TemporaryDirectory(prefix="pyne-broker-lab-entrypoint-") as temp:
+        root = Path(temp)
+        dist_info = root / "offline_broker_lab-0.dist-info"
+        dist_info.mkdir()
+        (dist_info / "METADATA").write_text(
+            "Metadata-Version: 2.1\nName: offline-broker-lab\nVersion: 0\n",
+            encoding="utf-8",
+        )
+        (dist_info / "entry_points.txt").write_text(
+            f"[{group}]\n{name} = {target}\n",
+            encoding="utf-8",
+        )
+        yield root
+
+
+def python_executable() -> str:
+    """Return the interpreter running the opt-in lab."""
+    return sys.executable

pynecore/transformers/builtin_shadow.py

@@ -0,0 +1,136 @@
+from typing import Any, cast
+import ast
+import importlib
+import json
+from pathlib import Path
+
+# Pine namespaces whose pynecore module has a different name
+_NAMESPACE_RENAMES = {
+    'str': 'string',
+}
+
+
+class BuiltinShadowTransformer(ast.NodeTransformer):
+    """
+    Resolve workdir library imports whose alias shadows a built-in namespace.
+
+    Pine resolves ``ta.x`` after ``import user/somelib/1 as ta`` against the
+    library's exports first and falls back to the built-in ``ta.*`` namespace
+    for everything else, so e.g. ``ta.valuewhen`` keeps working when the
+    library does not export it. The compiled ``import lib.user.somelib.v1 as
+    ta`` would route every access to the library module, so attribute reads
+    the library cannot serve are rewritten here to the canonical built-in form
+    (``lib.ta.valuewhen``), which the downstream transformers (import
+    normalizer, module properties, isolation, series) handle natively.
+
+    Which members the library serves is runtime knowledge: the library module
+    is imported at transform time (the same pattern callee resolution uses in
+    function isolation). Its ``__all__`` is the membership test — matching
+    TV, where non-exported library names are not reachable through the alias —
+    with a ``hasattr`` fallback for hand-written libraries without ``__all__``.
+    If the library cannot be imported, the alias is left untouched and the
+    script fails at its own import statement exactly as before.
+    """
+
+    def __init__(self):
+        # Structure: module -> name -> {"type": "property"|"variable"}
+        try:
+            with open(Path(__file__).parent / "module_properties.json") as f:
+                self.registry: dict[str, dict[str, Any]] = json.load(f)
+        except (IOError, json.JSONDecodeError) as e:
+            raise RuntimeError(f"Failed to load module properties config: {e}")
+
+        # alias -> (library module, exported names or None, builtin namespace)
+        self.aliases: dict[str, tuple[Any, frozenset[str] | None, str]] = {}
+
+    def visit_Module(self, node: ast.Module) -> ast.Module:
+        for stmt in node.body:
+            if not isinstance(stmt, ast.Import):
+                continue
+            for alias in stmt.names:
+                if not alias.asname or not alias.name.startswith('lib.'):
+                    continue
+                name = alias.asname
+                # --strict compilation suffixes every global binding
+                if name.endswith('__global__'):
+                    name = name[:-len('__global__')]
+                # A builtin-named alias is emitted under its canonical image
+                # (``import ... as ta`` compiles to ``as ta__ren__``); the
+                # image can only come from the bare name (a Pine ``ta__ren__``
+                # alias compiles to ``ta__ren____ren__``), so stripping the
+                # suffix recovers the source alias.
+                if name.endswith('__ren__'):
+                    name = name[:-len('__ren__')]
+                namespace = _NAMESPACE_RENAMES.get(name, name)
+                if f'lib.{namespace}' not in self.registry:
+                    continue
+                try:
+                    lib_module = importlib.import_module(alias.name)
+                except Exception:  # noqa: BLE001 - the script's own import will report it
+                    continue
+                exported = getattr(lib_module, '__all__', None)
+                members = frozenset(exported) if exported is not None else None
+                self.aliases[alias.asname] = (lib_module, members, namespace)
+
+        if not self.aliases:
+            return node
+        return cast(ast.Module, self.generic_visit(node))
+
+    def _visit_scope(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda) -> ast.AST:
+        """Visit a function-like scope, masking aliases shadowed by parameters."""
+        args = node.args
+        params = {a.arg for a in (*args.posonlyargs, *args.args, *args.kwonlyargs)}
+        if args.vararg:
+            params.add(args.vararg.arg)
+        if args.kwarg:
+            params.add(args.kwarg.arg)
+        masked = {name: self.aliases.pop(name) for name in params & self.aliases.keys()}
+        try:
+            return self.generic_visit(node)
+        finally:
+            self.aliases.update(masked)
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST:
+        return self._visit_scope(node)
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST:
+        return self._visit_scope(node)
+
+    def visit_Lambda(self, node: ast.Lambda) -> ast.AST:
+        return self._visit_scope(node)
+
+    def visit_Attribute(self, node: ast.Attribute) -> ast.AST:
+        node = cast(ast.Attribute, self.generic_visit(node))
+        if not (isinstance(node.ctx, ast.Load) and isinstance(node.value, ast.Name)):
+            return node
+        entry = self.aliases.get(node.value.id)
+        if entry is None:
+            return node
+        lib_module, members, namespace = entry
+
+        # The library serves the name -> the access stays on the library
+        if node.attr in members if members is not None else hasattr(lib_module, node.attr):
+            return node
+
+        # Built-in namespace fallback -> canonical lib.<namespace>.<attr> form;
+        # the nested-key check covers sub-namespaces (e.g. strategy.commission)
+        attr = node.attr
+        if (attr not in self.registry[f'lib.{namespace}']
+                and f'lib.{namespace}.{attr}' not in self.registry
+                and attr.endswith('__ren__')):
+            # A trigger-named member is emitted under its canonical image
+            # (``math.max`` compiles to ``max__ren__``); when the library does
+            # not serve it, the built-in namespace knows the bare name only
+            attr = attr[:-len('__ren__')]
+        if (attr in self.registry[f'lib.{namespace}']
+                or f'lib.{namespace}.{attr}' in self.registry):
+            return ast.copy_location(
+                ast.Attribute(
+                    value=ast.Attribute(
+                        value=ast.Name(id='lib', ctx=ast.Load()),
+                        attr=namespace, ctx=ast.Load()),
+                    attr=attr, ctx=node.ctx),
+                node)
+
+        # In neither -> leave it to fail at runtime, as before
+        return node