nlalpha-data 0.3.0__py3-none-any.whl

nlalpha_data/discovery.py

@@ -0,0 +1,129 @@
+"""Discover installed data sources and retain their initialization outcomes."""
+
+from importlib.metadata import entry_points, version
+from importlib import resources
+import json
+import logging
+import os
+from pathlib import Path
+
+from nlalpha.core.description import markdown_sections
+
+from nlalpha_data.loader import DataLoader, DataProviderRecord
+from nlalpha_data.local_provider import LocalFileDataProvider
+
+logger = logging.getLogger("nlalpha_data.discovery")
+
+
+def _read_guidance(path: str | None) -> str | None:
+    """Read and validate one provider's explicitly declared Markdown resource."""
+    if path is None:
+        return None
+    if not isinstance(path, str) or not path or path != path.strip() or not Path(path).is_absolute():
+        raise TypeError("get_prompt_path must return an absolute trimmed file path or None")
+    source = Path(path)
+    if not source.is_file():
+        raise FileNotFoundError(f"Declared data provider guidance is not a file: {path}")
+    guidance = source.read_text(encoding="utf-8")
+    if not markdown_sections(guidance):
+        raise ValueError("Declared data provider guidance must contain Markdown text")
+    return guidance
+
+
+def discover_data_providers(include_local: bool = True, local_data_dir: str = "workspace/data") -> list[DataProviderRecord]:
+    """Record every installed source while isolating plugin-specific failures."""
+    points = entry_points().select(group="nlalpha.data")
+    records = []
+    if include_local:
+        record = DataProviderRecord("local", "nlalpha-data", version("nlalpha-data"))
+        records.append(record)
+        try:
+            record.guidance = _read_guidance(str(resources.files("nlalpha_data").joinpath("knowledge", "local-provider.md")))
+            raw = os.environ.get("NLALPHA_LOCAL_PRICE_ADJUSTMENTS")
+            adjustments = json.loads(raw) if raw is not None else None
+            if raw is not None and not isinstance(adjustments, dict):
+                raise ValueError("NLALPHA_LOCAL_PRICE_ADJUSTMENTS must be a JSON object.")
+            provider = LocalFileDataProvider(local_data_dir, price_adjustments=adjustments)
+            available = provider.is_available()
+            record.manifest = {"provider_id": "local", "provider_name": "Local files", "available": available}
+            if available:
+                record.provider = provider
+            else:
+                record.status, record.reason = "unavailable", f"Local data directory is unavailable: {local_data_dir}"
+        except Exception as exc:
+            record.status, record.reason = "load_failed", str(exc)
+    for point in points:
+        distribution = point.dist
+        record = DataProviderRecord(None, distribution.metadata["Name"] if distribution else None,
+                                    distribution.version if distribution else None,
+                                    entry_point=point.name, entry_point_value=point.value)
+        if distribution is not None:
+            for name, value in (("Name", record.package), ("version", record.version)):
+                if not isinstance(value, str) or not value or value != value.strip():
+                    raise ValueError(f"distribution {name} must be a nonempty trimmed string")
+        records.append(record)
+        stage = "load"
+        try:
+            plugin = point.load()()
+            for method in ("get_manifest", "get_provider", "is_available", "get_prompt_path"):
+                if not callable(getattr(plugin, method, None)):
+                    raise TypeError(f"Data plugin requires {method}().")
+            stage = "get_manifest"
+            manifest = plugin.get_manifest()
+            if not isinstance(manifest, dict):
+                raise TypeError("Data manifest must be an object.")
+            for name in ("provider_id", "provider_name"):
+                value = manifest.get(name)
+                if not isinstance(value, str) or not value or value != value.strip():
+                    raise ValueError(f"Data manifest {name} must be a nonempty trimmed string.")
+            if type(manifest.get("available")) is not bool:
+                raise ValueError("Data manifest available must be a boolean.")
+            record.manifest = dict(manifest)
+            record.provider_id = manifest["provider_id"]
+            stage = "get_prompt_path"
+            record.guidance = _read_guidance(plugin.get_prompt_path())
+            stage = "is_available"
+            available = plugin.is_available()
+            if type(available) is not bool:
+                raise TypeError("Data plugin is_available() must return a boolean.")
+            if available != manifest["available"]:
+                raise ValueError("Data plugin availability contradicts its manifest.")
+            if not available:
+                record.status = "unavailable"
+                record.reason = manifest.get("unavailable_reason") or "Installed data source is unavailable or missing configuration."
+                continue
+            stage = "get_provider"
+            provider = plugin.get_provider()
+            for method in ("load_bars", "describe_request", "is_available", "cache_identity"):
+                if not callable(getattr(provider, method, None)):
+                    raise TypeError(f"Data provider requires {method}().")
+            stage = "provider.is_available"
+            provider_available = provider.is_available()
+            if type(provider_available) is not bool:
+                raise TypeError("Data provider is_available() must return a boolean.")
+            if not provider_available:
+                record.status = "unavailable"
+                record.reason = "Provider reports unavailable despite the plugin and manifest declaring availability."
+                continue
+            record.provider = provider
+        except Exception as exc:
+            record.status, record.reason = "load_failed", f"{stage}: {exc}"
+    identities = {}
+    for record in records:
+        if record.provider_id is not None:
+            identities.setdefault(record.provider_id, []).append(record)
+    for identity, duplicates in identities.items():
+        if len(duplicates) > 1:
+            for record in duplicates:
+                record.provider = None
+                record.status, record.reason = "load_failed", f"Duplicate provider_id: {identity}"
+    for record in records:
+        if record.status != "available":
+            logger.warning("Data source %s %s: %s", record.entry_point or record.provider_id, record.status, record.reason)
+    return records
+
+
+def get_default_data_provider(cache_dir: str | None = None, local_data_dir: str = "workspace/data") -> DataLoader:
+    """Assemble an application-owned loader with the host's source selection."""
+    return DataLoader(discover_data_providers(local_data_dir=local_data_dir), cache_dir=cache_dir,
+                      default_provider_id=os.environ.get("NLALPHA_DATA_PROVIDER"))

nlalpha_data/knowledge/loading.md

@@ -0,0 +1,17 @@
+## Data loading and coverage guide
+
+Choose `provider_id` explicitly when multiple sources are installed, or configure `NLALPHA_DATA_PROVIDER`. A requested `price_adjustment` must be established before loading; unknown evidence cannot satisfy forward/backward/unadjusted requirements. Responses bind `source` to the loaded artifact, whose `nlalpha.market.source.v1` metadata records per-symbol evidence and cache use.
+
+`check_data_coverage` summarizes the requested symbol, interval, and bounds with returned bar count and observed endpoints. Current provider availability does not guarantee that a specific instrument, range, or interval is supported or complete. `get_data_quality` checks OHLC range consistency and zero-volume bars; it is a limited diagnostic, not a full market-data audit. Read each response's `error`, available fields, and scope; absent evidence remains unknown.
+
+Keep data work within the user's request. `load_bars` returns a reference, bar-count summary, and time/unit contract metadata; it does not return bars or per-day observed session dates. Do not load unrelated ranges, create extra `data_ref` values, or start a real backtest when the user only requested a data check. Keep caller IDs and artifact ownership details internal in user-facing responses.
+
+`load_bars` returns a caller-scoped `data_ref` plus `time_contract`, `volume_unit`, and `bar_contract` when loading succeeds. Treat a reference as immutable evidence for that loaded data and its declared contracts. The canonical time contract `nlalpha.market.time.v1` records UTC bounds and precision plus per-symbol source timezone, original timestamp semantics, calendar IDs, session-date availability, and time/calendar library provenance where available. Read this metadata with the bars when a question depends on source time or session labels.
+
+Provider conversion happens before storage and is declared by `volume_unit` and `bar_contract`. The current bar contract labels volume as `instrument_quantity`; do not convert those values a second time. If an older reference has a different or absent contract, reload the bars to obtain a current reference rather than rewriting or reusing the old reference as if it had new units. A null or absent unit is unknown, not zero.
+
+Keep source time, analysis time, and display time distinct. Source timezone and timestamp semantics describe the origin; canonical bars preserve resolved `open_time` and `close_time` bounds in UTC and original source metadata. An explicit `open_time`/`close_time` pair represents actual endpoints whose width may differ from the interval label. For raw timestamps, verify whether the source timestamp is aware or naive and establish its endpoint/session meaning from source evidence; naive values need an explicitly verified source timezone. Do not infer meaning from the timestamp's appearance, local date, or interval. Conflicting declarations against a canonical time contract are invalid.
+
+Backtest `execution.analysis_timezone` controls local Gregorian month/year analysis and defaults to UTC; it does not change source instants. A requested `display_timezone` only adds formatted presentation values. Neither changes the stored UTC timestamps, source timezone, session date, or bar bounds. Missing source timezone, endpoint semantics, calendar, or session bounds leaves the affected conclusion unknown.
+
+Calendar tools report calendar sessions, not data coverage. For an exact missing-session count, retrieve actual observed session dates only through an authorized artifact-reading capability that is callable in the current tool catalog, then compare those dates with sessions from the applicable calendar and session schedule. `bar_count` alone cannot establish which dates are absent, and intraday rows cannot be compared directly to a daily session count. If that observation-reading capability, the required per-day observations, or an instrument-appropriate session definition is unavailable, state that the exact missing count is unknown.

nlalpha_data/knowledge/local-provider.md

@@ -0,0 +1,17 @@
+## File lookup and row contract
+
+The local provider resolves its configured data directory and searches for one symbol file in this order: `<symbol>.csv`, `<symbol>.xlsx`, then `<symbol>.xls`. Forward slashes and backslashes in the symbol are replaced with underscores for the filename. CSV rows are read with the standard library; Excel requires pandas. The first matching file is used, and a missing file raises an error. A directory being available says nothing about whether it contains a file for the requested symbol.
+
+OHLCV columns accept the provider's declared English and Chinese aliases. Rows may describe bounds with both `open_time` and `close_time`, one `datetime` value, or a date label. A single timestamp requires explicit `open` or `close` semantics from the source contract and a fixed interval. Date-only rows require `trading_date` semantics and an explicit calendar. Source time declarations may come from the provider's `time_contracts` constructor mapping or a matching `<symbol>.time.json` sidecar; conflicting declarations fail. The sidecar accepts `timezone`, `timestamp_semantics`, and `calendar_id`.
+
+The returned Bars retain UTC bounds and source timezone, offset, timestamp semantics, session date, and calendar identity when established. Explicit bounds are preserved; one timestamp is expanded using the declared interval, and trading-date rows use calendar session bounds. Invalid timestamps, missing OHLCV values, non-finite numbers, and unsupported calendar assumptions fail rather than being guessed.
+
+## Price adjustment and host declarations
+
+The local provider reports price adjustment as `unknown` unless the host supplies a symbol-specific declaration when constructing it. `nlalpha-data` reads `NLALPHA_LOCAL_PRICE_ADJUSTMENTS` as a JSON object for this declaration. The value is recorded as host evidence; it is not inferred from prices. If a request explicitly requires a price adjustment that the declaration cannot establish, the data loader fails the request.
+
+## Availability, cache identity, and source selection
+
+The provider is available when its configured data path exists and is a directory. This does not prove a requested symbol file exists or that its rows cover the requested interval. Cache identity includes the resolved data directory, time-contract configuration, and price-adjustment declarations, so cached bars remain bound to the source configuration that produced them.
+
+`DataLoader` selects one provider using an explicit provider ID, the configured default, or the single-available-provider rule. Ambiguous selection and source errors fail explicitly. The local provider does not fetch or substitute data from another source.

nlalpha_data/loader.py

@@ -0,0 +1,210 @@
+"""Load bars from one explicitly selected provider with request-bound provenance."""
+
+from dataclasses import dataclass, field
+from hashlib import sha256
+import json
+from pathlib import Path
+from typing import Any
+
+from nlalpha.core.contracts.market import Bar
+from nlalpha.core.errors import DataError, InvalidRequestError, NlAlphaError, INVALID_REQUEST
+from nlalpha_data.base import DataProvider, validate_request_description
+from nlalpha_data.parquet_store import ParquetBarStore
+
+
+class DataSelectionError(NlAlphaError):
+    """Report an invalid provider or price-adjustment selection."""
+
+    error_type = INVALID_REQUEST
+
+
+@dataclass
+class DataProviderRecord:
+    """Retain one installed provider's identity and discovery outcome."""
+
+    provider_id: str | None
+    package: str | None
+    version: str | None
+    provider: DataProvider | None = None
+    manifest: dict = field(default_factory=dict)
+    status: str = "available"
+    reason: str | None = None
+    entry_point: str | None = None
+    entry_point_value: str | None = None
+    guidance: str | None = None
+
+    def summary(self) -> dict:
+        """Return the serializable discovery record."""
+        return {name: getattr(self, name) for name in (
+            "provider_id", "package", "version", "manifest", "status", "reason", "entry_point", "entry_point_value",
+        )}
+
+
+@dataclass(frozen=True)
+class LoadedBars:
+    """Bind returned bars to the source evidence used for this request."""
+
+    bars: list[Bar]
+    provider_id: str
+    package: str | None
+    version: str | None
+    price_adjustment: str
+    evidence: dict[str, str]
+    cache_hit: bool
+
+    @property
+    def source(self) -> dict[str, Any]:
+        """Return a fresh provenance snapshot for serialization."""
+        return {"provider_id": self.provider_id, "package": self.package, "version": self.version,
+                "price_adjustment": self.price_adjustment, "evidence": dict(self.evidence), "cache_hit": self.cache_hit}
+
+
+class DataLoader:
+    """Select one provider and isolate caches by its declared request source."""
+
+    def __init__(self, records: list[DataProviderRecord], *, cache_dir: str | Path | None = None,
+                 default_provider_id: str | None = None):
+        self.records = list(records)
+        self.cache_dir = Path(cache_dir) if cache_dir is not None else None
+        self.default_provider_id = default_provider_id
+        self._stores: dict[str, ParquetBarStore] = {}
+
+    def list_providers(self) -> list[dict]:
+        """List only providers usable by this loader."""
+        return [record.summary() for record in self.records if record.status == "available"]
+
+    def diagnostics(self) -> list[dict]:
+        """List installed providers with configuration or loading failures."""
+        return [record.summary() for record in self.records if record.status != "available"]
+
+    def get_record(self, provider_id: str) -> DataProviderRecord:
+        """Resolve one cached source identity and expose failed installations precisely."""
+        if not isinstance(provider_id, str) or not provider_id or provider_id != provider_id.strip():
+            raise InvalidRequestError("provider_id must be a nonempty trimmed string", stage="request")
+        matches = [record for record in self.records if record.provider_id == provider_id]
+        if len(matches) == 1 and matches[0].status == "available":
+            return matches[0]
+        faults = matches or [record for record in self.records if record.entry_point == provider_id]
+        if faults:
+            raise NlAlphaError(
+                f"Installed data provider is unavailable: {provider_id}",
+                details={"diagnostics": [record.summary() for record in faults]}, stage="load",
+            )
+        raise InvalidRequestError(
+            f"Unknown data provider: {provider_id}",
+            details={"available_providers": [record.provider_id for record in self.records if record.status == "available"]},
+            stage="request",
+        )
+
+    def _select(self, provider_id: str | None) -> DataProviderRecord:
+        available = {record.provider_id: record for record in self.records if record.status == "available"}
+        selected = provider_id if provider_id is not None else self.default_provider_id
+        if selected is not None and (not isinstance(selected, str) or not selected or selected != selected.strip()):
+            raise DataSelectionError("provider_id must be a nonempty trimmed string.")
+        if not available:
+            raise DataError("No data providers are available.", details={"diagnostics": self.diagnostics()})
+        if selected is None:
+            if len(available) != 1:
+                raise DataSelectionError("Select provider_id when multiple data providers are available.",
+                                         details={"available_provider_ids": sorted(available)})
+            return next(iter(available.values()))
+        if selected not in available:
+            raise DataSelectionError(f"Data provider {selected!r} is not available.",
+                                     details={"available_provider_ids": sorted(available)})
+        return available[selected]
+
+    async def load_bars(self, symbol: str, interval: str, start: str, end: str, *,
+                        provider_id: str | None = None, price_adjustment: str | None = None) -> LoadedBars:
+        """Load one source after validating its request-specific price evidence."""
+        if price_adjustment is not None and (not isinstance(price_adjustment, str) or price_adjustment not in {"unadjusted", "forward", "backward"}):
+            raise DataSelectionError("price_adjustment must be unadjusted, forward, or backward.")
+        record = self._select(provider_id)
+        declaration = validate_request_description(
+            await record.provider.describe_request(symbol, interval, price_adjustment)
+        )
+        if price_adjustment is not None and declaration["price_adjustment"] != price_adjustment:
+            raise DataError("Selected source cannot establish the requested price adjustment.",
+                            details={"requested_price_adjustment": price_adjustment, "source": declaration,
+                                     "provider_id": record.provider_id})
+        source = {"provider_id": record.provider_id, "package": record.package, "version": record.version, **declaration}
+        store = None
+        if self.cache_dir is not None:
+            identity = record.provider.cache_identity()
+            if not isinstance(identity, dict) or not identity:
+                raise DataError("Data provider cache_identity() must return a nonempty JSON object.")
+            try:
+                cache_key = json.dumps({"source": source, "configuration": identity}, sort_keys=True,
+                                       separators=(",", ":"), allow_nan=False)
+            except (ValueError, TypeError) as exc:
+                raise DataError("Data provider cache_identity() must contain canonical JSON values.") from exc
+            fingerprint = sha256(cache_key.encode()).hexdigest()
+            store = self._stores.get(fingerprint)
+            if store is None:
+                store = ParquetBarStore(self.cache_dir / fingerprint, source=source)
+                self._stores[fingerprint] = store
+            try:
+                bars = await store.load(symbol, interval, start, end)
+            except FileNotFoundError:
+                pass
+            else:
+                return LoadedBars(bars=bars, cache_hit=True, **source)
+        bars = await record.provider.load_bars(symbol, interval, start, end, price_adjustment=price_adjustment)
+        if store is not None and bars:
+            await store.save(bars, symbol=symbol, interval=interval, start=start, end=end)
+        return LoadedBars(bars=bars, cache_hit=False, **source)
+
+    @staticmethod
+    def _source(record: DataProviderRecord) -> dict[str, Any]:
+        """Return the provider identity attached to pass-through reference data."""
+        return {"provider_id": record.provider_id, "package": record.package, "version": record.version}
+
+    def _provider_call(self, method_name: str) -> tuple[DataProviderRecord, Any]:
+        """Resolve one required provider method or expose its absence."""
+        record = self._select(None)
+        method = getattr(record.provider, method_name, None)
+        if not callable(method):
+            raise DataError(f"Selected data provider does not expose {method_name}.")
+        return record, method
+
+    async def list_instruments(self, asset: str, *, market: str | None = None,
+                               status: str | None = None) -> tuple[list[dict], dict[str, Any]]:
+        """List one asset class's instrument master with its source identity."""
+        record, method = self._provider_call("list_instruments")
+        rows = await method(asset, market=market, status=status)
+        return rows, self._source(record)
+
+    async def get_trading_calendar(self, exchange: str, start: str, end: str) -> tuple[list[dict], dict[str, Any]]:
+        """Return one exchange calendar with its source identity."""
+        record, method = self._provider_call("get_trading_calendar")
+        rows = await method(exchange, start, end)
+        return rows, self._source(record)
+
+    async def get_daily_basic(self, symbol: str, start: str, end: str) -> tuple[list[dict], dict[str, Any]]:
+        """Return daily fundamental metrics with their source identity."""
+        record, method = self._provider_call("get_daily_basic")
+        rows = await method(symbol, start, end)
+        return rows, self._source(record)
+
+    async def get_price_limits(self, symbol: str, start: str, end: str) -> tuple[list[dict], dict[str, Any]]:
+        """Return daily price limits with their source identity."""
+        record, method = self._provider_call("get_price_limits")
+        rows = await method(symbol, start, end)
+        return rows, self._source(record)
+
+    async def get_suspensions(self, symbol: str, start: str, end: str) -> tuple[list[dict], dict[str, Any]]:
+        """Return daily suspension rows with their source identity."""
+        record, method = self._provider_call("get_suspensions")
+        rows = await method(symbol, start, end)
+        return rows, self._source(record)
+
+    async def get_index_weights(self, index_code: str, start: str, end: str) -> tuple[list[dict], dict[str, Any]]:
+        """Return index constituent weights with their source identity."""
+        record, method = self._provider_call("get_index_weights")
+        rows = await method(index_code, start, end)
+        return rows, self._source(record)
+
+    async def aclose(self) -> None:
+        """Close all cache resources owned by this loader."""
+        for store in self._stores.values():
+            await store.aclose()
+        self._stores.clear()