pineforge-data 0.2.0__py3-none-any.whl

pineforge_data/docker_runtime.py

@@ -0,0 +1,226 @@
+"""Run backtests through the published pineforge-release container."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from .backtest import BacktestOptions, JsonValue
+from .models import Bar, Instrument
+from .release_contract import (
+    DEFAULT_RELEASE_IMAGE,
+    ReleaseContractError,
+    parse_release_report,
+    release_environment,
+    release_response,
+    write_release_inputs,
+)
+
+PullPolicy = Literal["always", "missing", "never"]
+
+
+class DockerPrerequisiteError(RuntimeError):
+    """Docker or the configured release image is unavailable."""
+
+
+class DockerExecutionError(RuntimeError):
+    """The isolated release-container backtest failed."""
+
+
+def _completed_error(completed: subprocess.CompletedProcess[str]) -> str:
+    return completed.stderr.strip() or completed.stdout.strip() or "command failed without output"
+
+
+@dataclass(slots=True)
+class DockerBacktestRuntime:
+    """Pull and execute an immutable pineforge-release image."""
+
+    image: str = DEFAULT_RELEASE_IMAGE
+    pull_policy: PullPolicy = "missing"
+    timeout_seconds: float = 300.0
+
+    def __post_init__(self) -> None:
+        if not self.image.strip():
+            raise ValueError("image must not be empty")
+        if self.pull_policy not in ("always", "missing", "never"):
+            raise ValueError(f"invalid pull policy: {self.pull_policy}")
+        if self.timeout_seconds <= 0:
+            raise ValueError("timeout_seconds must be positive")
+
+    def resolved_image(self) -> str:
+        return self.image
+
+    def _check_docker(self) -> None:
+        if shutil.which("docker") is None:
+            raise DockerPrerequisiteError(
+                "Docker is required but the `docker` command was not found"
+            )
+        completed = subprocess.run(
+            ["docker", "info", "--format", "{{.ServerVersion}}"],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        if completed.returncode != 0:
+            raise DockerPrerequisiteError(
+                f"Docker daemon is unavailable: {_completed_error(completed)}"
+            )
+
+    def _image_exists(self) -> bool:
+        completed = subprocess.run(
+            ["docker", "image", "inspect", self.image],
+            text=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        return completed.returncode == 0
+
+    def _pull_image(self) -> None:
+        completed = subprocess.run(
+            ["docker", "pull", self.image],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        if completed.returncode != 0:
+            raise DockerPrerequisiteError(
+                f"failed to pull pineforge-release image: {_completed_error(completed)}"
+            )
+
+    def _image_identity(self) -> dict[str, object]:
+        completed = subprocess.run(
+            ["docker", "image", "inspect", self.image],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        if completed.returncode != 0:
+            return {}
+        try:
+            values = json.loads(completed.stdout)
+            value = values[0]
+            labels = value.get("Config", {}).get("Labels", {})
+            digests = value.get("RepoDigests", [])
+        except (IndexError, AttributeError, json.JSONDecodeError, TypeError):
+            return {}
+        if not isinstance(labels, dict) or not isinstance(digests, list):
+            return {}
+        return {
+            "resolved_digest": str(digests[0]) if digests else None,
+            "release_version": labels.get("org.opencontainers.image.version"),
+            "engine_version": labels.get("io.pineforge.engine.version"),
+            "codegen_version": labels.get("io.pineforge.codegen.version"),
+        }
+
+    def ensure_image(self) -> str:
+        """Apply the configured pull policy and return the immutable image ref."""
+
+        self._check_docker()
+        exists = self._image_exists()
+        if self.pull_policy == "always" or (self.pull_policy == "missing" and not exists):
+            self._pull_image()
+            exists = True
+        if not exists:
+            raise DockerPrerequisiteError(
+                f"pineforge-release image is not available locally: {self.image}"
+            )
+        return self.image
+
+    @staticmethod
+    def _remove_timed_out_container(cid_path: Path) -> None:
+        if not cid_path.is_file():
+            return
+        container_id = cid_path.read_text(encoding="utf-8").strip()
+        if container_id:
+            subprocess.run(
+                ["docker", "rm", "--force", container_id],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=False,
+            )
+
+    def run(
+        self,
+        pine_source: str,
+        bars: Sequence[Bar],
+        *,
+        instrument: Instrument,
+        source: str,
+        options: BacktestOptions,
+        strategy_params: Mapping[str, JsonValue] | None = None,
+        strategy_overrides: Mapping[str, JsonValue] | None = None,
+    ) -> dict[str, object]:
+        """Run PineScript and normalized OHLCV in pineforge-release."""
+
+        image = self.ensure_image()
+        with tempfile.TemporaryDirectory(prefix="pineforge-data-") as temporary:
+            workspace = Path(temporary)
+            write_release_inputs(workspace, pine_source, bars, instrument)
+            environment = release_environment(
+                "/in",
+                instrument,
+                options,
+                strategy_params,
+                strategy_overrides,
+            )
+            environment["PINEFORGE_DATA_SOURCE"] = source
+            cid_path = workspace / "container.cid"
+            command = [
+                "docker",
+                "run",
+                "--rm",
+                "--cidfile",
+                str(cid_path),
+                "--network",
+                "none",
+                "--read-only",
+                "--tmpfs",
+                "/tmp:rw,exec,nosuid,nodev,size=512m",
+                "--cap-drop",
+                "ALL",
+                "--security-opt",
+                "no-new-privileges",
+                "--mount",
+                f"type=bind,src={workspace},dst=/in,readonly",
+            ]
+            for key, value in sorted(environment.items()):
+                command.extend(("--env", f"{key}={value}"))
+            command.append(image)
+            try:
+                completed = subprocess.run(
+                    command,
+                    text=True,
+                    capture_output=True,
+                    check=False,
+                    timeout=self.timeout_seconds,
+                    env=os.environ.copy(),
+                )
+            except subprocess.TimeoutExpired as exc:
+                self._remove_timed_out_container(cid_path)
+                raise DockerExecutionError(
+                    f"pineforge-release exceeded {self.timeout_seconds:g} seconds"
+                ) from exc
+        if completed.returncode != 0:
+            phase = {2: "input", 3: "compile", 4: "backtest", 5: "transpile"}.get(
+                completed.returncode, "container"
+            )
+            raise DockerExecutionError(f"{phase} failed: {_completed_error(completed)}")
+        try:
+            report = parse_release_report(completed.stdout)
+        except ReleaseContractError as exc:
+            raise DockerExecutionError(str(exc)) from exc
+        return release_response(
+            report,
+            release_image=image,
+            mode="local-container",
+            request_id=None,
+            runtime_metadata=self._image_identity(),
+        )

pineforge_data/engine.py

@@ -0,0 +1,185 @@
+"""Dependency-free interoperability with PineForge's streaming C ABI."""
+
+from __future__ import annotations
+
+import ctypes
+from collections.abc import Sequence
+
+from .errors import EngineStreamError
+from .models import Bar, Instrument, TradeTick
+
+
+class PfBar(ctypes.Structure):
+    """``ctypes`` mirror of PineForge's ``pf_bar_t``."""
+
+    _fields_ = [
+        ("open", ctypes.c_double),
+        ("high", ctypes.c_double),
+        ("low", ctypes.c_double),
+        ("close", ctypes.c_double),
+        ("volume", ctypes.c_double),
+        ("timestamp", ctypes.c_int64),
+    ]
+
+
+class PfTradeTick(ctypes.Structure):
+    """``ctypes`` mirror of PineForge's ``pf_trade_tick_t``."""
+
+    _fields_ = [
+        ("timestamp", ctypes.c_int64),
+        ("sequence", ctypes.c_uint64),
+        ("price", ctypes.c_double),
+        ("quantity", ctypes.c_double),
+    ]
+
+
+def _require_instrument(
+    records: Sequence[Bar] | Sequence[TradeTick], expected: Instrument | None
+) -> None:
+    if not records:
+        return
+    reference = expected or records[0].instrument
+    if any(record.instrument != reference for record in records):
+        raise ValueError("all records must belong to the expected instrument")
+
+
+def pack_bars(bars: Sequence[Bar], *, instrument: Instrument | None = None) -> ctypes.Array[PfBar]:
+    """Pack normalized bars into one contiguous ``pf_bar_t`` array."""
+
+    _require_instrument(bars, instrument)
+    array_type = PfBar * len(bars)
+    return array_type(
+        *(
+            PfBar(bar.open, bar.high, bar.low, bar.close, bar.volume, bar.timestamp_ms)
+            for bar in bars
+        )
+    )
+
+
+def pack_trade_ticks(
+    ticks: Sequence[TradeTick], *, instrument: Instrument | None = None
+) -> ctypes.Array[PfTradeTick]:
+    """Pack normalized trades into one contiguous ``pf_trade_tick_t`` array."""
+
+    _require_instrument(ticks, instrument)
+    array_type = PfTradeTick * len(ticks)
+    return array_type(
+        *(
+            PfTradeTick(tick.timestamp_ms, tick.sequence, tick.price, tick.quantity)
+            for tick in ticks
+        )
+    )
+
+
+class EngineStreamSink:
+    """Send normalized records to an existing PineForge strategy instance.
+
+    The caller owns both ``library`` and ``state``. This adapter configures and
+    invokes only the streaming functions; it never creates or frees a strategy.
+    """
+
+    def __init__(
+        self,
+        library: ctypes.CDLL,
+        state: int | ctypes.c_void_p,
+        instrument: Instrument,
+    ) -> None:
+        if not state:
+            raise ValueError("state must be a non-null PineForge strategy handle")
+        self._library = library
+        self._state = state
+        self.instrument = instrument
+        self._configure_signatures()
+
+    def _configure_signatures(self) -> None:
+        self._library.strategy_get_last_error.argtypes = [ctypes.c_void_p]
+        self._library.strategy_get_last_error.restype = ctypes.c_char_p
+        self._library.strategy_stream_begin.argtypes = [
+            ctypes.c_void_p,
+            ctypes.POINTER(PfBar),
+            ctypes.c_int,
+            ctypes.c_char_p,
+            ctypes.c_char_p,
+        ]
+        self._library.strategy_stream_begin.restype = ctypes.c_int
+        self._library.strategy_stream_push_tick.argtypes = [
+            ctypes.c_void_p,
+            ctypes.POINTER(PfTradeTick),
+        ]
+        self._library.strategy_stream_push_tick.restype = ctypes.c_int
+        self._library.strategy_stream_push_ticks.argtypes = [
+            ctypes.c_void_p,
+            ctypes.POINTER(PfTradeTick),
+            ctypes.c_int,
+        ]
+        self._library.strategy_stream_push_ticks.restype = ctypes.c_int
+        self._library.strategy_stream_advance_time.argtypes = [ctypes.c_void_p, ctypes.c_int64]
+        self._library.strategy_stream_advance_time.restype = ctypes.c_int
+        self._library.strategy_stream_end.argtypes = [ctypes.c_void_p, ctypes.c_int]
+        self._library.strategy_stream_end.restype = ctypes.c_int
+
+    def _check(self, status: int, operation: str) -> None:
+        if status == 0:
+            return
+        raw = self._library.strategy_get_last_error(self._state)
+        detail = raw.decode("utf-8", "replace") if raw else "unknown engine error"
+        raise EngineStreamError(f"{operation}: {detail}")
+
+    def begin(
+        self,
+        warmup_bars: Sequence[Bar],
+        *,
+        input_timeframe: str,
+        script_timeframe: str,
+    ) -> None:
+        """Warm the strategy on confirmed bars and start realtime streaming."""
+
+        if not warmup_bars:
+            raise ValueError("warmup_bars must not be empty")
+        if not input_timeframe or not script_timeframe:
+            raise ValueError("input_timeframe and script_timeframe must not be empty")
+        packed = pack_bars(warmup_bars, instrument=self.instrument)
+        status = self._library.strategy_stream_begin(
+            self._state,
+            packed,
+            len(packed),
+            input_timeframe.encode(),
+            script_timeframe.encode(),
+        )
+        self._check(status, "stream begin")
+
+    def push_tick(self, tick: TradeTick) -> None:
+        """Push one normalized executed trade."""
+
+        if tick.instrument != self.instrument:
+            raise ValueError("tick does not belong to the sink instrument")
+        packed = PfTradeTick(tick.timestamp_ms, tick.sequence, tick.price, tick.quantity)
+        status = self._library.strategy_stream_push_tick(self._state, ctypes.byref(packed))
+        self._check(status, "stream tick")
+
+    def push_ticks(self, ticks: Sequence[TradeTick]) -> None:
+        """Push normalized executed trades in one contiguous ABI call."""
+
+        if not ticks:
+            return
+        packed = pack_trade_ticks(ticks, instrument=self.instrument)
+        status = self._library.strategy_stream_push_ticks(self._state, packed, len(packed))
+        self._check(status, "stream ticks")
+
+    def advance_time(self, timestamp_ms: int) -> None:
+        """Advance the stream clock so elapsed input bars can close."""
+
+        if timestamp_ms < 0:
+            raise ValueError("timestamp_ms must be non-negative")
+        status = self._library.strategy_stream_advance_time(self._state, timestamp_ms)
+        self._check(status, "stream advance")
+
+    def end(self, *, finalize_partial_input_bar: bool = False) -> None:
+        """End the stream, optionally dispatching its partial input bar."""
+
+        status = self._library.strategy_stream_end(self._state, int(finalize_partial_input_bar))
+        self._check(status, "stream end")
+
+
+if ctypes.sizeof(PfBar) != 48 or ctypes.sizeof(PfTradeTick) != 32:
+    raise RuntimeError("unsupported platform ABI layout for PineForge streaming records")

pineforge_data/errors.py

@@ -0,0 +1,5 @@
+"""Exceptions raised at PineForge integration boundaries."""
+
+
+class EngineStreamError(RuntimeError):
+    """A PineForge streaming C ABI call failed."""

pineforge_data/models.py

@@ -0,0 +1,199 @@
+"""Provider-neutral records shared by adapters and engine sinks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from math import isfinite
+
+
+def _non_empty(value: str, field: str) -> None:
+    if not value.strip():
+        raise ValueError(f"{field} must not be empty")
+
+
+def _non_negative(value: int, field: str) -> None:
+    if value < 0:
+        raise ValueError(f"{field} must be non-negative")
+
+
+def _unix_ms(value: int, field: str) -> None:
+    _non_negative(value, field)
+    if value > 2**63 - 1:
+        raise ValueError(f"{field} must fit a signed 64-bit integer")
+
+
+def _optional_text(value: str, field: str) -> None:
+    if value and not value.strip():
+        raise ValueError(f"{field} must not contain only whitespace")
+
+
+class AssetClass(StrEnum):
+    """Broad economic asset class, independent of execution venue."""
+
+    UNKNOWN = "unknown"
+    CRYPTO = "crypto"
+    EQUITY = "equity"
+    FOREX = "forex"
+    COMMODITY = "commodity"
+    INDEX = "index"
+    FUND = "fund"
+    BOND = "bond"
+
+
+class MarketType(StrEnum):
+    """Trading-market structure for an instrument listing."""
+
+    UNKNOWN = "unknown"
+    SPOT = "spot"
+    CASH = "cash"
+    SWAP = "swap"
+    FUTURE = "future"
+    OPTION = "option"
+    CFD = "cfd"
+
+
+class OptionType(StrEnum):
+    CALL = "call"
+    PUT = "put"
+
+
+@dataclass(frozen=True, slots=True)
+class ContractSpec:
+    """Normalized derivative terms supplied by a market catalog."""
+
+    contract_size: float | None = None
+    linear: bool | None = None
+    inverse: bool | None = None
+    expiry_ms: int | None = None
+    strike: float | None = None
+    option_type: OptionType | None = None
+
+    def __post_init__(self) -> None:
+        if self.contract_size is not None and (
+            not isfinite(self.contract_size) or self.contract_size <= 0
+        ):
+            raise ValueError("contract_size must be finite and positive")
+        if self.linear is True and self.inverse is True:
+            raise ValueError("a contract cannot be both linear and inverse")
+        if self.expiry_ms is not None:
+            _unix_ms(self.expiry_ms, "expiry_ms")
+        if self.strike is not None and (not isfinite(self.strike) or self.strike <= 0):
+            raise ValueError("strike must be finite and positive")
+
+
+@dataclass(frozen=True, slots=True)
+class Instrument:
+    """A normalized market instrument with provider identity kept explicit."""
+
+    symbol: str
+    venue: str = ""
+    timezone: str = "UTC"
+    session: str = "24x7"
+    volume_unit: str = "base"
+    asset_class: AssetClass = AssetClass.UNKNOWN
+    market_type: MarketType = MarketType.UNKNOWN
+    base: str = ""
+    quote: str = ""
+    settle: str = ""
+    provider_id: str = ""
+    contract: ContractSpec | None = None
+
+    def __post_init__(self) -> None:
+        _non_empty(self.symbol, "symbol")
+        _non_empty(self.timezone, "timezone")
+        _non_empty(self.session, "session")
+        _non_empty(self.volume_unit, "volume_unit")
+        _optional_text(self.venue, "venue")
+        _optional_text(self.base, "base")
+        _optional_text(self.quote, "quote")
+        _optional_text(self.settle, "settle")
+        _optional_text(self.provider_id, "provider_id")
+
+
+@dataclass(frozen=True, slots=True)
+class MarketListing:
+    """One instrument as listed by one provider-bound venue."""
+
+    instrument: Instrument
+    active: bool | None = None
+    margin_supported: bool | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Bar:
+    """One confirmed, provider-normalized OHLCV bar."""
+
+    instrument: Instrument
+    timestamp_ms: int
+    open: float
+    high: float
+    low: float
+    close: float
+    volume: float
+    source: str
+
+    def __post_init__(self) -> None:
+        _unix_ms(self.timestamp_ms, "timestamp_ms")
+        _non_empty(self.source, "source")
+        prices = (self.open, self.high, self.low, self.close)
+        if not all(isfinite(value) and value > 0 for value in prices):
+            raise ValueError("OHLC prices must be finite and positive")
+        if self.high < max(self.open, self.close, self.low):
+            raise ValueError("high must be greater than or equal to OHLC values")
+        if self.low > min(self.open, self.close, self.high):
+            raise ValueError("low must be less than or equal to OHLC values")
+        if not isfinite(self.volume) or self.volume < 0:
+            raise ValueError("volume must be finite and non-negative")
+
+
+@dataclass(frozen=True, slots=True)
+class TradeTick:
+    """One normalized executed trade ready for the PineForge stream ABI."""
+
+    instrument: Instrument
+    timestamp_ms: int
+    sequence: int
+    price: float
+    quantity: float
+    source: str
+
+    def __post_init__(self) -> None:
+        _unix_ms(self.timestamp_ms, "timestamp_ms")
+        _non_negative(self.sequence, "sequence")
+        if self.sequence > 2**64 - 1:
+            raise ValueError("sequence must fit an unsigned 64-bit integer")
+        _non_empty(self.source, "source")
+        if not isfinite(self.price) or self.price <= 0:
+            raise ValueError("price must be finite and positive")
+        if not isfinite(self.quantity) or self.quantity < 0:
+            raise ValueError("quantity must be finite and non-negative")
+
+
+@dataclass(frozen=True, slots=True)
+class MacroObservation:
+    """A vintage-aware macro value safe to align without revised-data lookahead."""
+
+    key: str
+    currency: str
+    period_end_ms: int
+    released_at_ms: int
+    vintage_at_ms: int
+    value: float
+    unit: str
+    source: str
+
+    def __post_init__(self) -> None:
+        _non_empty(self.key, "key")
+        _non_empty(self.currency, "currency")
+        _non_empty(self.unit, "unit")
+        _non_empty(self.source, "source")
+        _unix_ms(self.period_end_ms, "period_end_ms")
+        _unix_ms(self.released_at_ms, "released_at_ms")
+        _unix_ms(self.vintage_at_ms, "vintage_at_ms")
+        if self.released_at_ms < self.period_end_ms:
+            raise ValueError("released_at_ms must not precede period_end_ms")
+        if self.vintage_at_ms < self.released_at_ms:
+            raise ValueError("vintage_at_ms must not precede released_at_ms")
+        if not isfinite(self.value):
+            raise ValueError("value must be finite")

pineforge_data/providers/__init__.py

@@ -0,0 +1,69 @@
+"""Public provider protocols and registry."""
+
+from .base import (
+    HistoricalBarProvider,
+    LiveTradeProvider,
+    MacroDataProvider,
+    MarketCatalogProvider,
+    MarketDataProvider,
+    MarketNotFoundError,
+)
+from .ccxt import (
+    CcxtCapabilityError,
+    CcxtDataError,
+    CcxtDependencyError,
+    CcxtError,
+    CcxtProvider,
+)
+from .local import CsvBarProvider, SqliteBarProvider
+from .registry import (
+    ENTRY_POINT_GROUP,
+    ProviderFactory,
+    ProviderNotFoundError,
+    ProviderRegistry,
+    ProviderRegistryError,
+    create_provider,
+    default_registry,
+)
+from .sqlalchemy import SqlAlchemyBarProvider, SqlAlchemyDependencyError
+from .tabular import (
+    BarColumnMapping,
+    SchemaMappingError,
+    SourceColumn,
+    TabularBarProvider,
+    TabularDataError,
+    TabularSchema,
+    TimestampUnit,
+)
+
+__all__ = [
+    "ENTRY_POINT_GROUP",
+    "BarColumnMapping",
+    "CcxtCapabilityError",
+    "CcxtDataError",
+    "CcxtDependencyError",
+    "CcxtError",
+    "CcxtProvider",
+    "CsvBarProvider",
+    "HistoricalBarProvider",
+    "LiveTradeProvider",
+    "MacroDataProvider",
+    "MarketCatalogProvider",
+    "MarketDataProvider",
+    "MarketNotFoundError",
+    "ProviderFactory",
+    "ProviderNotFoundError",
+    "ProviderRegistry",
+    "ProviderRegistryError",
+    "SchemaMappingError",
+    "SourceColumn",
+    "SqlAlchemyBarProvider",
+    "SqlAlchemyDependencyError",
+    "SqliteBarProvider",
+    "TabularBarProvider",
+    "TabularDataError",
+    "TabularSchema",
+    "TimestampUnit",
+    "create_provider",
+    "default_registry",
+]