pineforge-data 0.2.0__py3-none-any.whl

pineforge_data/providers/registry.py

@@ -0,0 +1,252 @@
+"""Provider factories and third-party entry-point discovery."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from importlib.metadata import EntryPoint, entry_points
+from typing import Protocol, cast
+
+from ..models import Instrument
+from .base import MarketDataProvider
+from .ccxt import CcxtProvider
+from .local import CsvBarProvider, SqliteBarProvider
+from .sqlalchemy import SqlAlchemyBarProvider
+
+ENTRY_POINT_GROUP = "pineforge_data.providers"
+
+
+class ProviderRegistryError(RuntimeError):
+    """A provider factory cannot be registered or loaded."""
+
+
+class ProviderNotFoundError(ProviderRegistryError):
+    """No built-in or installed provider has the requested name."""
+
+
+class ProviderFactory(Protocol):
+    def __call__(self, venue: str, config: Mapping[str, object]) -> MarketDataProvider: ...
+
+
+def _ccxt_factory(venue: str, config: Mapping[str, object]) -> MarketDataProvider:
+    return CcxtProvider(venue, config=config)
+
+
+def _unknown_config(config: Mapping[str, object], allowed: set[str]) -> None:
+    unknown = sorted(config.keys() - allowed)
+    if unknown:
+        raise ValueError(f"unknown provider configuration key(s): {', '.join(unknown)}")
+
+
+def _required_text(config: Mapping[str, object], key: str) -> str:
+    value = config.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"provider configuration {key!r} must be a non-empty string")
+    return value
+
+
+def _optional_text(config: Mapping[str, object], key: str) -> str | None:
+    value = config.get(key)
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"provider configuration {key!r} must be a non-empty string")
+    return value
+
+
+def _column_overrides(config: Mapping[str, object]) -> Mapping[str, str] | None:
+    value = config.get("columns")
+    if value is None:
+        return None
+    if not isinstance(value, Mapping):
+        raise ValueError("provider configuration 'columns' must be an object")
+    overrides: dict[str, str] = {}
+    allowed = {"timestamp", "open", "high", "low", "close", "volume", "symbol", "timeframe"}
+    for field, column in value.items():
+        if not isinstance(field, str) or not isinstance(column, str):
+            raise ValueError("provider column mappings must have string keys and values")
+        if field not in allowed:
+            raise ValueError(f"unknown canonical column field: {field}")
+        if not column:
+            raise ValueError(f"provider column mapping for {field!r} must not be empty")
+        overrides[field] = column
+    return overrides
+
+
+def _instrument(venue: str, config: Mapping[str, object]) -> Instrument | None:
+    symbol = _optional_text(config, "symbol")
+    return None if symbol is None else Instrument(symbol, venue=venue, provider_id=symbol)
+
+
+_COMMON_TABULAR_KEYS = {
+    "columns",
+    "timestamp_unit",
+    "timestamp_timezone",
+    "symbol",
+    "timeframe",
+}
+
+
+def _csv_factory(venue: str, config: Mapping[str, object]) -> MarketDataProvider:
+    _unknown_config(config, _COMMON_TABULAR_KEYS | {"path", "encoding", "delimiter"})
+    return CsvBarProvider(
+        _required_text(config, "path"),
+        venue=venue,
+        mapping=_column_overrides(config),
+        timestamp_unit=_optional_text(config, "timestamp_unit") or "milliseconds",
+        timestamp_timezone=_optional_text(config, "timestamp_timezone") or "UTC",
+        instrument=_instrument(venue, config),
+        timeframe=_optional_text(config, "timeframe"),
+        encoding=_optional_text(config, "encoding") or "utf-8-sig",
+        delimiter=_optional_text(config, "delimiter") or ",",
+    )
+
+
+def _sqlite_factory(venue: str, config: Mapping[str, object]) -> MarketDataProvider:
+    _unknown_config(config, _COMMON_TABULAR_KEYS | {"path", "table"})
+    return SqliteBarProvider(
+        _required_text(config, "path"),
+        _required_text(config, "table"),
+        venue=venue,
+        mapping=_column_overrides(config),
+        timestamp_unit=_optional_text(config, "timestamp_unit") or "milliseconds",
+        timestamp_timezone=_optional_text(config, "timestamp_timezone") or "UTC",
+        instrument=_instrument(venue, config),
+        timeframe=_optional_text(config, "timeframe"),
+    )
+
+
+def _sqlalchemy_factory(venue: str, config: Mapping[str, object]) -> MarketDataProvider:
+    _unknown_config(
+        config,
+        _COMMON_TABULAR_KEYS | {"url", "url_env", "table", "schema", "engine_options"},
+    )
+    url = _optional_text(config, "url")
+    url_env = _optional_text(config, "url_env")
+    if url is not None and url_env is not None:
+        raise ValueError("configure only one of 'url' and 'url_env'")
+    if url_env is not None:
+        url = os.environ.get(url_env)
+        if not url:
+            raise ValueError(f"database URL environment variable is unset: {url_env}")
+    if url is None:
+        raise ValueError("provider configuration requires 'url' or 'url_env'")
+    engine_options = config.get("engine_options")
+    normalized_engine_options: Mapping[str, object] | None = None
+    if engine_options is not None:
+        if not isinstance(engine_options, Mapping) or not all(
+            isinstance(key, str) for key in engine_options
+        ):
+            raise ValueError("provider configuration 'engine_options' must be an object")
+        normalized_engine_options = cast(Mapping[str, object], engine_options)
+    return SqlAlchemyBarProvider(
+        url,
+        _required_text(config, "table"),
+        venue=venue,
+        schema=_optional_text(config, "schema"),
+        mapping=_column_overrides(config),
+        timestamp_unit=_optional_text(config, "timestamp_unit") or "milliseconds",
+        timestamp_timezone=_optional_text(config, "timestamp_timezone") or "UTC",
+        instrument=_instrument(venue, config),
+        timeframe=_optional_text(config, "timeframe"),
+        engine_options=normalized_engine_options,
+    )
+
+
+class ProviderRegistry:
+    """Resolve built-ins and externally installed provider factories by name."""
+
+    def __init__(self, *, include_builtin: bool = True) -> None:
+        self._factories: dict[str, ProviderFactory] = {}
+        if include_builtin:
+            self.register("ccxt", _ccxt_factory)
+            self.register("csv", _csv_factory)
+            self.register("sqlite", _sqlite_factory)
+            self.register("sqlalchemy", _sqlalchemy_factory)
+
+    @staticmethod
+    def _normalize_name(name: str) -> str:
+        normalized = name.strip().casefold()
+        if not normalized:
+            raise ValueError("provider name must not be empty")
+        return normalized
+
+    def register(self, name: str, factory: ProviderFactory, *, replace: bool = False) -> None:
+        """Register an in-process provider factory."""
+
+        normalized = self._normalize_name(name)
+        if normalized in self._factories and not replace:
+            raise ProviderRegistryError(f"provider already registered: {normalized}")
+        self._factories[normalized] = factory
+
+    def _matching_entry_point(self, name: str) -> EntryPoint | None:
+        matches = [
+            candidate
+            for candidate in entry_points().select(group=ENTRY_POINT_GROUP)
+            if candidate.name.casefold() == name
+        ]
+        if len(matches) > 1:
+            packages = ", ".join(sorted(candidate.value for candidate in matches))
+            raise ProviderRegistryError(
+                f"multiple entry points registered for provider {name!r}: {packages}"
+            )
+        return matches[0] if matches else None
+
+    def _load_external(self, name: str) -> ProviderFactory | None:
+        entry_point = self._matching_entry_point(name)
+        if entry_point is None:
+            return None
+        candidate = entry_point.load()
+        if not callable(candidate):
+            raise ProviderRegistryError(
+                f"provider entry point {entry_point.value!r} is not callable"
+            )
+        factory = cast(ProviderFactory, candidate)
+        self._factories[name] = factory
+        return factory
+
+    def create(
+        self,
+        name: str,
+        venue: str,
+        *,
+        config: Mapping[str, object] | None = None,
+    ) -> MarketDataProvider:
+        """Create one provider bound to a venue or broker environment."""
+
+        normalized = self._normalize_name(name)
+        factory = self._factories.get(normalized) or self._load_external(normalized)
+        if factory is None:
+            available = ", ".join(self.names()) or "none"
+            raise ProviderNotFoundError(
+                f"unknown provider {name!r}; available providers: {available}"
+            )
+        provider = factory(venue, config or {})
+        if not isinstance(provider, MarketDataProvider):
+            raise ProviderRegistryError(
+                f"provider {normalized!r} does not implement MarketDataProvider"
+            )
+        return provider
+
+    def names(self) -> tuple[str, ...]:
+        """List built-in, registered, and advertised provider names."""
+
+        advertised = {
+            candidate.name.casefold()
+            for candidate in entry_points().select(group=ENTRY_POINT_GROUP)
+        }
+        return tuple(sorted(self._factories.keys() | advertised))
+
+
+default_registry = ProviderRegistry()
+
+
+def create_provider(
+    name: str,
+    venue: str,
+    *,
+    config: Mapping[str, object] | None = None,
+) -> MarketDataProvider:
+    """Create a provider from the process-wide registry."""
+
+    return default_registry.create(name, venue, config=config)

pineforge_data/providers/sqlalchemy.py

@@ -0,0 +1,138 @@
+"""SQLAlchemy Core provider for reflected user-owned database tables."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from importlib import import_module
+from threading import Lock
+from types import ModuleType
+from typing import Any, cast
+
+from ..models import Instrument
+from ..requests import BarRequest
+from .tabular import (
+    BarColumnMapping,
+    ColumnMappingInput,
+    SourceColumn,
+    TabularBarProvider,
+    TabularRow,
+    TabularSchema,
+    TimestampUnit,
+    numeric_source_bounds,
+)
+
+
+class SqlAlchemyDependencyError(RuntimeError):
+    """SQLAlchemy support was requested without the optional dependency."""
+
+
+def _load_sqlalchemy() -> ModuleType:
+    try:
+        return import_module("sqlalchemy")
+    except ModuleNotFoundError as exc:
+        if exc.name == "sqlalchemy" or (exc.name and exc.name.startswith("sqlalchemy.")):
+            raise SqlAlchemyDependencyError(
+                "SQLAlchemy is not installed; install pineforge-data[database]"
+            ) from exc
+        raise
+
+
+class SqlAlchemyBarProvider(TabularBarProvider):
+    """Reflect and query one table or view through a synchronous SQLAlchemy engine."""
+
+    def __init__(
+        self,
+        url: str,
+        table: str,
+        *,
+        venue: str = "database",
+        schema: str | None = None,
+        mapping: ColumnMappingInput = None,
+        timestamp_unit: TimestampUnit | str = TimestampUnit.MILLISECONDS,
+        timestamp_timezone: str = "UTC",
+        instrument: Instrument | None = None,
+        timeframe: str | None = None,
+        engine_options: Mapping[str, object] | None = None,
+    ) -> None:
+        super().__init__(
+            venue=venue,
+            mapping=mapping,
+            timestamp_unit=timestamp_unit,
+            timestamp_timezone=timestamp_timezone,
+            instrument=instrument,
+            timeframe=timeframe,
+        )
+        if not url.strip():
+            raise ValueError("SQLAlchemy URL must not be empty")
+        if not table:
+            raise ValueError("table must not be empty")
+        if schema is not None and not schema:
+            raise ValueError("schema must be None or a non-empty string")
+
+        self._sa = _load_sqlalchemy()
+        options = dict(engine_options or {})
+        options.setdefault("hide_parameters", True)
+        self._engine: Any = self._sa.create_engine(url, **options)
+        self._table: Any | None = None
+        self._reflection_lock = Lock()
+        self.table = table
+        self.schema_name = schema
+        self.name = f"sqlalchemy:{venue}"
+
+    def _reflected_table(self) -> Any:
+        with self._reflection_lock:
+            if self._table is None:
+                metadata = self._sa.MetaData()
+                self._table = self._sa.Table(
+                    self.table,
+                    metadata,
+                    schema=self.schema_name,
+                    autoload_with=self._engine,
+                )
+            return self._table
+
+    def _inspect_schema_sync(self) -> TabularSchema:
+        table = self._reflected_table()
+        qualified_table = f"{self.schema_name}.{self.table}" if self.schema_name else self.table
+        return TabularSchema(
+            source=f"sqlalchemy:{self.venue}#{qualified_table}",
+            columns=tuple(
+                SourceColumn(
+                    name=str(column.name),
+                    data_type=str(column.type),
+                    nullable=bool(column.nullable),
+                )
+                for column in table.columns
+            ),
+        )
+
+    def _read_rows_sync(
+        self, mapping: BarColumnMapping, request: BarRequest
+    ) -> tuple[TabularRow, ...]:
+        table = self._reflected_table()
+        columns = [table.c[column] for column in mapping.columns]
+        statement = self._sa.select(*columns)
+        if mapping.symbol is not None:
+            statement = statement.where(table.c[mapping.symbol] == request.instrument.symbol)
+        if mapping.timeframe is not None:
+            statement = statement.where(table.c[mapping.timeframe] == request.timeframe)
+        bounds = numeric_source_bounds(request.start_ms, request.end_ms, self.timestamp_unit)
+        if bounds is not None:
+            statement = statement.where(table.c[mapping.timestamp] >= bounds[0])
+            statement = statement.where(table.c[mapping.timestamp] < bounds[1])
+        statement = statement.order_by(table.c[mapping.timestamp])
+
+        with self._engine.connect() as connection:
+            result = connection.execute(statement).mappings().all()
+        return tuple(
+            cast(TabularRow, {str(key): value for key, value in row.items()}) for row in result
+        )
+
+    def _distinct_values_sync(self, column: str) -> tuple[object, ...]:
+        table = self._reflected_table()
+        statement = self._sa.select(table.c[column]).distinct().order_by(table.c[column])
+        with self._engine.connect() as connection:
+            return tuple(connection.execute(statement).scalars().all())
+
+    def _close_sync(self) -> None:
+        self._engine.dispose()