opencode-pyneruntime 6.6.4__py3-none-any.whl

pynecore/core/htf_aggregator.py

@@ -0,0 +1,181 @@
+"""
+Higher-timeframe bar aggregator for live ``request.security()`` developing bars.
+
+The chart process aggregates its own (lower-TF) OHLCV stream into a developing
+HTF bar for each ``lookahead_on`` security context. The aggregated developing
+bar is then transported to the security child via the SyncBlock so the child
+can run its ``main()`` function under ``barstate.isconfirmed=False`` using the
+chart-derived OHLCV — mirroring the TradingView live ``lookahead_on`` semantics.
+
+Aggregation rules per HTF period (chart bars whose ``get_bar_time(chart_ts)``
+maps to the same HTF open):
+
+    open    = first chart bar's open
+    high    = max chart high seen so far
+    low     = min chart low seen so far
+    close   = latest chart close
+    volume  = sum of chart volumes (one contribution per chart bar)
+
+Live providers (e.g. CCXT ``watch_ohlcv``) emit repeated intra-bar updates
+carrying the *cumulative* candle volume. ``update()`` is therefore keyed on
+``chart_time_ms``: when the same chart bar fires again, the prior
+contribution of that bar is subtracted from the HTF developing volume and
+replaced with the latest cumulative reading — preventing double counting.
+
+A period closes following TradingView's ``lookahead_off`` merge rule: the
+confirmed chart bar whose close instant reaches the HTF period end completes
+the bar, so the closed HTF bar is delivered together with the period's LAST
+chart bar — not with the next period's first one. When no such completing bar
+arrives (data gap, or the chart bar span is unknown — D/W/M charts), the close
+falls back to the first chart bar that crosses into a new HTF period.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from .resampler import Resampler
+
+if TYPE_CHECKING:
+    from zoneinfo import ZoneInfo
+
+
+@dataclass
+class DevelopingBar:
+    """In-progress aggregated OHLCV for one HTF period.
+
+    ``last_chart_time_ms`` / ``last_chart_volume`` track the most recently
+    folded chart bar so repeated intra-bar updates from a live provider
+    (which carry the running cumulative candle volume) can replace their
+    own prior contribution instead of stacking it.
+    """
+    period_start: int  # HTF period open time in ms
+    open: float
+    high: float
+    low: float
+    close: float
+    volume: float
+    last_chart_time_ms: int = 0
+    last_chart_volume: float = 0.0
+
+    def update(self, chart_time_ms: int, chart_high: float, chart_low: float,
+               chart_close: float, chart_volume: float) -> None:
+        if chart_high > self.high:
+            self.high = chart_high
+        if chart_low < self.low:
+            self.low = chart_low
+        self.close = chart_close
+        if chart_time_ms == self.last_chart_time_ms:
+            # Same chart bar re-firing (intra-bar tick): live providers send
+            # the running cumulative candle volume, so the previous reading
+            # must be subtracted before adding the new one.
+            self.volume += chart_volume - self.last_chart_volume
+        else:
+            self.volume += chart_volume
+            self.last_chart_time_ms = chart_time_ms
+        self.last_chart_volume = chart_volume
+
+
+class HTFAggregator:
+    """Per-sec_id aggregator of chart OHLCV into a developing HTF bar."""
+
+    __slots__ = ('_timeframe', '_tz', '_resampler', '_state', '_session_starts',
+                 '_chart_span_ms')
+
+    def __init__(self, timeframe: str, tz: 'ZoneInfo',
+                 session_starts: 'list | None' = None,
+                 chart_span_ms: int = 0):
+        self._timeframe = timeframe
+        self._tz = tz
+        self._resampler = Resampler.get_resampler(timeframe)
+        self._state: DevelopingBar | None = None
+        # Intraday session anchoring (TradingView aligns HTF bars to the session
+        # open). ``None`` keeps the pure clock-floor — see ``Resampler.get_bar_time``.
+        self._session_starts = session_starts
+        # Chart bar span in ms for the close-instant period completion check.
+        # 0 (D/W/M charts: no fixed arithmetic span) disables it — the period
+        # then closes on the first chart bar of the next period instead.
+        self._chart_span_ms = chart_span_ms
+
+    @property
+    def timeframe(self) -> str:
+        return self._timeframe
+
+    @property
+    def current(self) -> DevelopingBar | None:
+        """Currently developing HTF bar, or None if no chart bar seen yet."""
+        return self._state
+
+    def update(
+        self,
+        chart_time_ms: int,
+        chart_open: float, chart_high: float, chart_low: float,
+        chart_close: float, chart_volume: float,
+        chart_confirmed: bool = True,
+    ) -> tuple[bool, DevelopingBar | None, DevelopingBar | None]:
+        """
+        Fold one chart bar into the developing HTF bar.
+
+        :param chart_time_ms: Chart bar open timestamp in ms.
+        :param chart_open: Chart bar open price.
+        :param chart_high: Chart bar high price.
+        :param chart_low: Chart bar low price.
+        :param chart_close: Chart bar close price.
+        :param chart_volume: Chart bar volume (cumulative within the bar for
+            live providers; the aggregator deduplicates same-bar updates).
+        :param chart_confirmed: Whether the folded chart bar is final
+            (``barstate.isconfirmed``). Only a confirmed chart bar may complete
+            the period — a developing intra-bar tick must not freeze a
+            non-final HTF close.
+        :return: ``(is_new_period, developing_or_none, closed_or_none)``
+
+            - ``is_new_period`` — True iff the chart bar opens a fresh HTF period.
+            - ``developing_or_none`` — the accumulated HTF bar including this
+              chart bar, or None when this chart bar just completed the period
+              (no fresh developing bar exists until the next chart bar).
+            - ``closed_or_none`` — the completed period's final accumulated bar:
+              delivered with the confirmed chart bar whose close instant reaches
+              the period end (TV ``lookahead_off`` merge rule), or — when no
+              such bar arrived — with the first chart bar of the next period.
+        """
+        period_start = self._resampler.get_bar_time(
+            chart_time_ms, self._tz, self._session_starts)
+
+        closed: DevelopingBar | None = None
+        state = self._state
+        is_new = state is None or period_start != state.period_start
+        if is_new:
+            if state is not None:
+                # The period was left without its completing chart bar (data
+                # gap) — deliver its accumulated final form one bar late.
+                closed = state
+            state = DevelopingBar(
+                period_start=period_start,
+                open=chart_open, high=chart_high, low=chart_low,
+                close=chart_close, volume=chart_volume,
+                last_chart_time_ms=chart_time_ms,
+                last_chart_volume=chart_volume,
+            )
+            self._state = state
+        else:
+            state.update(chart_time_ms, chart_high, chart_low,
+                         chart_close, chart_volume)
+
+        # TV ``lookahead_off`` merge rule: the confirmed chart bar whose close
+        # instant reaches the period end completes the HTF bar — deliver it
+        # now, one chart bar earlier than the next-period fallback above.
+        # Skipped when a gap-closed bar is already pending (one closed bar per
+        # update; the fresh period then closes via the fallback) or when the
+        # chart bar span is unknown (``chart_span_ms == 0``).
+        if (closed is None and chart_confirmed and self._chart_span_ms
+                and self._resampler.get_bar_time(
+                    chart_time_ms + self._chart_span_ms, self._tz,
+                    self._session_starts) != period_start):
+            self._state = None
+            return is_new, None, state
+
+        return is_new, state, closed
+
+    def reset(self) -> None:
+        """Discard any in-flight developing bar (used on context teardown)."""
+        self._state = None

pynecore/core/import_hook.py

@@ -0,0 +1,358 @@
+from typing import cast
+import os
+import sys
+import hashlib
+import importlib.util
+import importlib.machinery
+import re
+from pathlib import Path
+
+
+# Module-level constant the transform pipeline bakes into every transformed module
+# (see ``PyneLoader.source_to_code``). Its presence — together with a matching
+# pipeline hash in ``co_consts`` — certifies a loaded code object as current
+# pipeline output, so foreign or stale bytecode can be told apart and dropped.
+_PYNE_SENTINEL = '__pyne_transformed__'
+
+# A module is Pyne code only when its docstring STARTS with ``@pyne``. Matching the
+# raw source head mirrors the strict docstring check in ``source_to_code`` without
+# paying for a full parse on every import. Leading comment lines are skipped so a
+# PEP 723 ``# /// script`` metadata block before the docstring does not hide it.
+_PYNE_HEAD_RE = re.compile(
+    rb'^(?:\s*#[^\r\n]*(?:\r?\n|$))*\s*[rRbBuUfF]*("""|\'\'\'|"|\')\s*@pyne(?:\s|\1|$)')
+
+
+def _source_starts_with_pyne(head: bytes) -> bool:
+    """Return whether a source head is a Pyne module (docstring begins with ``@pyne``).
+
+    :param head: First bytes of the source file.
+    :return: Whether the module should carry the transform sentinel.
+    """
+    return _PYNE_HEAD_RE.match(head) is not None
+
+
+def _cache_from_source(source_path: Path) -> Path:
+    """Return the cached ``.pyc`` path CPython uses for a given ``.py`` source.
+
+    Delegates to :func:`importlib.util.cache_from_source` instead of hand-building
+    ``<dir>/__pycache__/<stem>.<tag>.pyc`` so the result matches CPython exactly:
+    it honours ``sys.pycache_prefix`` / ``PYTHONPYCACHEPREFIX`` (which mirrors the
+    cache under a separate tree rather than a sibling ``__pycache__``) and the
+    active optimization level (``.opt-1`` / ``.opt-2`` under ``-O`` / ``-OO``).
+    Stale-bytecode invalidation must target the exact file CPython reads back, so
+    a mismatch here would silently leave the cache untouched and the bug unfixed.
+
+    :param source_path: Path to the ``.py`` source file.
+    :return: Path to the corresponding cached bytecode file.
+    """
+    return Path(importlib.util.cache_from_source(str(source_path)))
+
+
+_transform_pipeline_hash: str | None = None
+
+
+def _get_transform_pipeline_hash() -> str:
+    """Return a content digest identifying the current AST transform pipeline.
+
+    Transformed bytecode is only valid for the exact pipeline that produced it, yet
+    CPython validates a ``.pyc`` solely against its source ``.py`` mtime/size — it
+    cannot tell a transformed module from one compiled without the import hook
+    (``pip``'s post-install ``compileall``, an IDE, a packaging step) or one left
+    over by an older PyneCore. This digest is baked into every transformed module as
+    ``__pyne_transformed__`` and re-checked on load; a missing or mismatched value
+    forces a retransform.
+
+    Hashing the pipeline *contents* — this module plus every file under
+    ``transformers/`` (``module_properties.json`` shapes the output yet has no
+    bytecode of its own) — keeps the check deterministic and immune to file mtimes,
+    cache markers and read-only install locations.
+
+    :return: Hex digest pinning the transform pipeline.
+    """
+    global _transform_pipeline_hash
+    if _transform_pipeline_hash is None:
+        files = [Path(__file__)]  # this module pins the transformer pipeline order
+        transformers_dir = Path(__file__).parent.parent / "transformers"
+        try:
+            files.extend(transformers_dir.iterdir())
+        except OSError:
+            pass
+        digest = hashlib.sha256()
+        for f in sorted(files, key=lambda p: p.name):
+            try:
+                if f.is_file():
+                    digest.update(f.name.encode('utf-8'))
+                    digest.update(f.read_bytes())
+            except OSError:
+                pass
+        _transform_pipeline_hash = digest.hexdigest()[:16]
+    return _transform_pipeline_hash
+
+
+class PyneLoader(importlib.machinery.SourceFileLoader):
+    """Loader that handles AST transformation"""
+
+    def get_code(self, fullname: str):
+        """Retransform cached bytecode not produced by the current transform pipeline.
+
+        CPython validates a cached ``.pyc`` only against its source ``.py`` mtime and
+        size, so it cannot distinguish a transformed ``@pyne`` module from one compiled
+        without the import hook (``pip``'s post-install ``compileall``, an IDE, a
+        packaging step) or one left over by an older pipeline — all of them load as
+        "valid" and silently run the wrong bytecode. Every transformed module carries a
+        ``__pyne_transformed__ = <pipeline hash>`` sentinel baked into its code object;
+        if the loaded bytecode lacks it or the hash is stale, the ``.pyc`` is dropped and
+        the source is retransformed. The check is content-based, so it holds regardless
+        of file mtimes, cache markers or a read-only install location.
+
+        :param fullname: Fully-qualified module name being loaded.
+        :return: The compiled code object (retransformed if the cache was foreign or stale).
+        """
+        source_path = self.get_filename(fullname)
+        code = super().get_code(fullname)
+
+        try:
+            with open(source_path, 'rb') as f:
+                # Large enough to cover a PEP 723 metadata block before the docstring
+                head = f.read(4096)
+        except OSError:
+            head = b''
+
+        # Only transformed modules carry the sentinel; leave everything else untouched.
+        # ``get_code`` is typed Optional, but a real source file always yields a code
+        # object — the ``None`` guard just narrows the type for the checks below.
+        if code is None or not _source_starts_with_pyne(head):
+            return code
+
+        pipeline_hash = _get_transform_pipeline_hash()
+        if _PYNE_SENTINEL in code.co_names and pipeline_hash in code.co_consts:
+            return code
+
+        # Foreign or stale bytecode slipped past CPython's mtime/size check — drop it
+        # and let the loader recompile, refreshing the cache when the dir is writable.
+        try:
+            _cache_from_source(Path(source_path)).unlink()
+        except OSError:
+            pass  # no cached bytecode, or a read-only cache dir: nothing to drop
+        code = super().get_code(fullname)
+        if code is None or (_PYNE_SENTINEL in code.co_names and pipeline_hash in code.co_consts):
+            return code
+
+        # The stale ``.pyc`` could not be removed (read-only / locked cache) and still
+        # masks the source. Compile straight from source so the correct bytecode runs
+        # regardless; caching is skipped this load — correctness wins over the cache.
+        return self.source_to_code(self.get_data(source_path), source_path)
+
+    # noinspection PyMethodOverriding
+    def source_to_code(self, data: bytes | str, path: str, *, _optimize: int = -1):
+        """Transform source to code if needed"""
+        path: Path = Path(path)
+
+        # Fast prefilter: require @pyne as a standalone token, not just any substring.
+        # Compiled Pyne code always has it as the first non-whitespace content of the
+        # module docstring, either multi-line (`"""\n@pyne\n…"""`) or single-line
+        # (`"""@pyne"""`); the latter puts the closing quote right after the token, so a
+        # quote must terminate the match alongside whitespace / end-of-input. A loose
+        # check would AST-transform ordinary modules that merely *mention* @pyne in a
+        # docstring (e.g. standalone.py); the strict docstring check below still gates it.
+        data_str = data.decode('utf-8') if isinstance(data, bytes) else data
+        if not re.search(r'@pyne(\s|["\']|$)', data_str):
+            return compile(data, path, 'exec', optimize=_optimize)
+
+        import ast
+
+        tree = ast.parse(data_str)
+
+        # Strict check: the module docstring must START with @pyne (whitespace-stripped),
+        # followed by whitespace or end of string. Substring matches don't count — they
+        # would catch innocuous mentions inside docstrings of non-script library modules.
+        is_pyne_module = False
+        if (tree.body and isinstance(tree.body[0], ast.Expr) and
+                isinstance(cast(ast.Expr, tree.body[0]).value, ast.Constant) and
+                isinstance(cast(ast.Constant, cast(ast.Expr, tree.body[0]).value).value, str)):
+            docstring = cast(str, cast(ast.Constant, cast(ast.Expr, tree.body[0]).value).value)
+            is_pyne_module = re.match(r'\s*@pyne(\s|$)', docstring) is not None
+
+        if is_pyne_module:
+
+            # Remove test cases from the output, because they can coorupt the output
+            transformed = tree
+            # Source path for the transformers (SecurityTransformer hashes it into
+            # the per-module sec ids, so security contexts stay unique across the
+            # script and its imported library modules). Resolved so the chart
+            # process and its security children derive identical ids.
+            transformed._module_file_path = str(path.resolve())  # type: ignore[attr-defined]
+            transformed.body = [node for node in transformed.body
+                                if not (isinstance(node, ast.FunctionDef)
+                                        and node.name.startswith('__test_') and node.name.endswith('__'))]
+
+            # Transform AST - lazy import transformers only when needed
+            from pynecore.transformers.import_lifter import ImportLifterTransformer
+            from pynecore.transformers.type_checking_stripper import TypeCheckingStripperTransformer
+            from pynecore.transformers.builtin_shadow import BuiltinShadowTransformer
+            from pynecore.transformers.import_normalizer import ImportNormalizerTransformer
+            from pynecore.transformers.dynamic_default import DynamicDefaultTransformer
+            from pynecore.transformers.inline_series_hoist import InlineSeriesHoistTransformer
+            from pynecore.transformers.security import SecurityTransformer
+            from pynecore.transformers.security_instantiation import (
+                SecurityInstantiationTransformer,
+            )
+            from pynecore.transformers.persistent_series import PersistentSeriesTransformer
+            from pynecore.transformers.lib_series import LibrarySeriesTransformer
+            from pynecore.transformers.closure_arguments_transformer import ClosureArgumentsTransformer
+            from pynecore.transformers.function_isolation import FunctionIsolationTransformer
+            from pynecore.transformers.module_property import ModulePropertyTransformer
+            from pynecore.transformers.series import SeriesTransformer
+            from pynecore.transformers.script_requirements import ScriptRequirementsTransformer
+            from pynecore.transformers.unused_series_detector import UnusedSeriesDetectorTransformer
+            from pynecore.transformers.persistent import PersistentTransformer
+            from pynecore.transformers.input_transformer import InputTransformer
+            from pynecore.transformers.safe_convert_transformer import SafeConvertTransformer
+            from pynecore.transformers.safe_division_transformer import SafeDivisionTransformer
+            from pynecore.transformers.ne_guard import NeGuardTransformer
+            from pynecore.transformers.slot_layout import ModuleLayout, apply_layout
+            from pynecore.transformers.locations import fix_locations
+
+            # Shared slot allocator of the module (see slot_layout.py); the
+            # state-contributing transformers fill it, apply_layout emits it
+            slot_layout = ModuleLayout()
+
+            transformed = ImportLifterTransformer().visit(transformed)
+            transformed = TypeCheckingStripperTransformer().visit(transformed)
+            # The builtin-namespace fallback must run before import normalization
+            # so the lib.<ns>.<name> chains it emits get their imports added there
+            transformed = BuiltinShadowTransformer().visit(transformed)
+            transformed = ImportNormalizerTransformer().visit(transformed)
+            # Per-call evaluation of lib.*-referencing parameter defaults; must
+            # precede the series/isolation passes so the moved expressions get
+            # their series slots and call-site anchors like any body statement
+            transformed = DynamicDefaultTransformer().visit(transformed)
+            # Lazy-context history hoist must run before call-site anchoring:
+            # the hoisted statements are the anchorable call sites
+            transformed = InlineSeriesHoistTransformer().visit(transformed)
+            # Pine instantiation semantics: clone security-bearing functions
+            # per call site so each call site gets its own security contexts
+            transformed = SecurityInstantiationTransformer().visit(transformed)
+            transformed = SecurityTransformer().visit(transformed)
+            transformed = PersistentSeriesTransformer().visit(transformed)
+            transformed = LibrarySeriesTransformer().visit(transformed)
+            transformed = ModulePropertyTransformer().visit(transformed)
+            transformed = ClosureArgumentsTransformer().visit(transformed)
+            transformed = UnusedSeriesDetectorTransformer().optimize(transformed)
+            transformed = SeriesTransformer(slot_layout).visit(transformed)
+            transformed = PersistentTransformer(slot_layout).visit(transformed)
+            # Call-site classification needs the var/series slots, so the
+            # isolation transformer must run after Persistent and Series
+            transformed = FunctionIsolationTransformer(slot_layout).visit(transformed)
+            transformed = ScriptRequirementsTransformer().visit(transformed)
+            transformed = InputTransformer().visit(transformed)
+            transformed = SafeConvertTransformer().visit(transformed)
+            transformed = SafeDivisionTransformer().visit(transformed)
+            # After SafeDivision so wrapped operands (safe_div calls) are bound
+            # once by the walrus instead of evaluating twice. Only user/compiled
+            # scripts get the guard: pynecore's own lib modules use the raw
+            # ``x != x`` nan idiom deliberately, and wrapping it would invert it
+            if not path.is_relative_to(Path(__file__).parent.parent):
+                transformed = NeGuardTransformer().visit(transformed)
+            transformed = apply_layout(transformed, slot_layout)
+
+            # Debugger-safe variant of ast.fix_missing_locations: synthetic
+            # nodes get point anchors, so no prologue bytecode maps onto the
+            # function's last line (see transformers/locations.py)
+            fix_locations(transformed)
+
+            # Debug output if requested. The pretty dump and the saved copy go
+            # through the display rewrite (named index constants instead of
+            # literal slot indexes); the RAW dump stays the exact emission —
+            # the AST golden tests compare against it.
+            if os.environ.get('PYNE_AST_DEBUG'):
+                from pynecore.transformers.display_rewrite import display_dump
+                print("-" * 100)
+                print(f"Transformed {path}:")
+                try:
+                    from rich.syntax import Syntax  # type: ignore
+                    from rich import print as rprint  # type: ignore
+                    rprint(Syntax(display_dump(transformed, slot_layout), "python",
+                                  word_wrap=True, line_numbers=False))
+                except ImportError:
+                    print(display_dump(transformed, slot_layout))
+                print("-" * 100)
+            elif raw_filter := os.environ.get('PYNE_AST_DEBUG_RAW'):
+                # '1' dumps every transformed module; any other value is a source
+                # path filter so a capture is not polluted by modules imported
+                # during the transform (callee resolution imports lib submodules)
+                if raw_filter == '1' or Path(raw_filter).resolve() == path.resolve():
+                    print(ast.unparse(transformed))
+
+            if os.environ.get('PYNE_AST_SAVE'):
+                from pynecore.transformers.display_rewrite import display_dump
+                Path("/tmp/pyne").mkdir(parents=True, exist_ok=True)
+
+                with open(f"/tmp/pyne/{path.stem}.py", "w") as f:
+                    f.write(display_dump(transformed, slot_layout))
+
+            # Bake a pipeline-identity sentinel into the module body so a loaded code
+            # object can be distinguished from foreign or stale bytecode (see get_code).
+            # It must survive into the .pyc, so it is a plain assignment the compiler
+            # marshals like any other constant — no .pyc-format surgery needed. Added
+            # after the debug/save dumps above so those keep showing the semantic
+            # transform, free of this loader-level bookkeeping.
+            sentinel = ast.Assign(
+                targets=[ast.Name(id=_PYNE_SENTINEL, ctx=ast.Store())],
+                value=ast.Constant(value=_get_transform_pipeline_hash()),
+            )
+            # is_pyne_module guarantees body[0] is the module docstring; keep it first,
+            # and stay after any ``from __future__`` imports (which must lead the module).
+            insert_at = 1
+            while (insert_at < len(transformed.body)
+                   and isinstance(transformed.body[insert_at], ast.ImportFrom)
+                   and cast(ast.ImportFrom, transformed.body[insert_at]).module == '__future__'):
+                insert_at += 1
+            transformed.body.insert(insert_at, sentinel)
+            ast.fix_missing_locations(transformed)
+
+            tree = transformed
+
+        # Let Python handle bytecode caching
+        return compile(tree, path, 'exec', optimize=_optimize)
+
+
+class PyneImportHook:
+    """Import hook that uses PyneLoader"""
+
+    # noinspection PyMethodMayBeStatic,PyUnusedLocal
+    def find_spec(self, fullname: str, path, target=None):
+        """Find and create module spec"""
+        if path is None:
+            path = sys.path
+
+        if "." in fullname:
+            *_, name = fullname.split(".")
+        else:
+            name = fullname
+
+        for entry in path:
+            if entry == "":
+                entry = "."
+
+            # Check both module.py and module/__init__.py
+            candidates = [
+                Path(entry) / f"{name}.py",
+                Path(entry) / name / "__init__.py"
+            ]
+
+            for py_path in candidates:
+                if py_path.exists():
+                    # Stale/foreign bytecode is handled content-based in
+                    # ``PyneLoader.get_code`` (via the transform sentinel), so there is
+                    # no per-path cache bookkeeping to do here.
+                    return importlib.util.spec_from_file_location(
+                        fullname,
+                        py_path,
+                        loader=PyneLoader(fullname, str(py_path))
+                    )
+        return None
+
+
+# Install the import hook
+sys.meta_path.insert(0, PyneImportHook())