bagelquant-data 0.1.0__tar.gz

bagelquant_data-0.1.0/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.3
+Name: bagelquant-data
+Version: 0.1.0
+Summary: Unified data layer for the BagelQuant ecosystem
+Author: BagelQuant
+License: Apache-2.0
+Requires-Dist: polars>=1.35.0
+Requires-Dist: pyarrow>=22.0.0
+Requires-Dist: tqdm>=4.67.0
+Requires-Dist: pandas>=3.0.0 ; extra == 'tushare'
+Requires-Dist: tushare>=1.4.21 ; extra == 'tushare'
+Requires-Python: >=3.13
+Provides-Extra: tushare
+Description-Content-Type: text/markdown
+
+# BagelQuant Data
+
+`bagelquant-data` is a Polars-native, source-agnostic data lake framework for
+quantitative research.
+
+- Polars is the dataframe engine.
+- Parquet is the canonical analytical storage format.
+- SQLite stores mutable metadata, manifests, run state, and source/dataset
+  registration.
+- Tushare is implemented as the first source adapter under
+  `bagelquant_data.sources.tushare`.
+- Non-reference research extraction returns one field at a time as
+  `time | asset_id | value`.
+
+```python
+import polars as pl
+
+from bagelquant_data import DataLake, DatasetSpec
+
+lake = DataLake.open("data")
+spec = DatasetSpec(
+    name="daily",
+    source="custom",
+    source_dataset="daily",
+    category="market",
+    field_mapping={"ts_code": "ts_code", "trade_date": "trade_date"},
+    required_columns=("asset_id", "time"),
+    primary_key=("asset_id", "time"),
+    asset_column="ts_code",
+    time_column="trade_date",
+    partition_strategy="year_month",
+    deduplication="primary_key_last",
+    sort_columns=("time", "asset_id"),
+)
+lake.ingest_frame(
+    spec,
+    pl.DataFrame(
+        {
+            "trade_date": ["2024-01-02"],
+            "ts_code": ["000001.SZ"],
+            "close": [100.0],
+        }
+    ),
+)
+
+close = lake.query.field("daily", "close", source="custom", collect=True)
+print(close)  # time, asset_id, close
+```
+
+Documentation is available in two languages:
+
+- English: `docs/en/index.md`
+- Chinese: `docs/cn/index.md`
+
+## Development
+
+```bash
+uv run pytest
+```

bagelquant_data-0.1.0/README.md

@@ -0,0 +1,59 @@
+# BagelQuant Data
+
+`bagelquant-data` is a Polars-native, source-agnostic data lake framework for
+quantitative research.
+
+- Polars is the dataframe engine.
+- Parquet is the canonical analytical storage format.
+- SQLite stores mutable metadata, manifests, run state, and source/dataset
+  registration.
+- Tushare is implemented as the first source adapter under
+  `bagelquant_data.sources.tushare`.
+- Non-reference research extraction returns one field at a time as
+  `time | asset_id | value`.
+
+```python
+import polars as pl
+
+from bagelquant_data import DataLake, DatasetSpec
+
+lake = DataLake.open("data")
+spec = DatasetSpec(
+    name="daily",
+    source="custom",
+    source_dataset="daily",
+    category="market",
+    field_mapping={"ts_code": "ts_code", "trade_date": "trade_date"},
+    required_columns=("asset_id", "time"),
+    primary_key=("asset_id", "time"),
+    asset_column="ts_code",
+    time_column="trade_date",
+    partition_strategy="year_month",
+    deduplication="primary_key_last",
+    sort_columns=("time", "asset_id"),
+)
+lake.ingest_frame(
+    spec,
+    pl.DataFrame(
+        {
+            "trade_date": ["2024-01-02"],
+            "ts_code": ["000001.SZ"],
+            "close": [100.0],
+        }
+    ),
+)
+
+close = lake.query.field("daily", "close", source="custom", collect=True)
+print(close)  # time, asset_id, close
+```
+
+Documentation is available in two languages:
+
+- English: `docs/en/index.md`
+- Chinese: `docs/cn/index.md`
+
+## Development
+
+```bash
+uv run pytest
+```

bagelquant_data-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "bagelquant-data"
+version = "0.1.0"
+description = "Unified data layer for the BagelQuant ecosystem"
+readme = "README.md"
+requires-python = ">=3.13"
+license = { text = "Apache-2.0" }
+authors = [{ name = "BagelQuant" }]
+dependencies = [
+    "polars>=1.35.0",
+    "pyarrow>=22.0.0",
+    "tqdm>=4.67.0",
+]
+
+[project.scripts]
+bagelquant-data = "bagelquant_data.cli.main:main"
+
+[project.optional-dependencies]
+tushare = [
+    "pandas>=3.0.0",
+    "tushare>=1.4.21",
+]
+
+[dependency-groups]
+dev = [
+    "pandas>=3.0.0",
+    "pre-commit>=4.5.0",
+    "pyright>=1.1.410",
+    "pytest>=9.0.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.0,<0.9.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+pythonpath = [".", "src"]
+testpaths = ["tests"]

bagelquant_data-0.1.0/src/bagelquant_data/__init__.py

@@ -0,0 +1,32 @@
+"""Source-agnostic data lake framework for BagelQuant research."""
+
+from bagelquant_data.core import (
+    BagelQuantDataError,
+    DataSource,
+    DatasetNotFoundError,
+    DatasetSpec,
+    DatasetSpecError,
+    DuplicateResolutionError,
+    SourceNotFoundError,
+    ValidationError,
+    stable_bucket,
+)
+from bagelquant_data.finance import FinancialFieldKind, FinancialFieldSpec
+from bagelquant_data.management import DataLake
+from bagelquant_data.sources.tushare import TushareSource
+
+__all__ = [
+    "BagelQuantDataError",
+    "DataLake",
+    "DataSource",
+    "DatasetNotFoundError",
+    "DatasetSpec",
+    "DatasetSpecError",
+    "DuplicateResolutionError",
+    "FinancialFieldKind",
+    "FinancialFieldSpec",
+    "SourceNotFoundError",
+    "TushareSource",
+    "ValidationError",
+    "stable_bucket",
+]

bagelquant_data-0.1.0/src/bagelquant_data/cli/main.py

@@ -0,0 +1,30 @@
+"""Thin CLI for the Python-first data lake API."""
+
+from __future__ import annotations
+
+import argparse
+
+from bagelquant_data import DataLake
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="bagelquant-data")
+    parser.add_argument("--root", default="data")
+    sub = parser.add_subparsers(dest="command", required=True)
+    sub.add_parser("status")
+    datasets = sub.add_parser("dataset-list")
+    datasets.add_argument("--source")
+    sub.add_parser("source-list")
+    args = parser.parse_args(argv)
+    lake = DataLake.open(args.root)
+    if args.command == "status":
+        print(lake.status.summary())
+    elif args.command == "dataset-list":
+        print(lake.datasets.list(args.source))
+    elif args.command == "source-list":
+        print(lake.sources.list())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bagelquant_data-0.1.0/src/bagelquant_data/core/__init__.py

@@ -0,0 +1,37 @@
+"""Core source-agnostic framework primitives."""
+
+from bagelquant_data.core.dataset import DatasetSpec
+from bagelquant_data.core.exceptions import (
+    BagelQuantDataError,
+    ConfigurationError,
+    DatasetNotFoundError,
+    DatasetSpecError,
+    DestructiveOperationError,
+    DuplicateResolutionError,
+    SourceNotFoundError,
+    ValidationError,
+)
+from bagelquant_data.core.hashing import frame_content_hash, stable_bucket, stable_record_hash
+from bagelquant_data.core.registry import FrameworkRegistries, Registry, default_registries
+from bagelquant_data.core.request import RequestContext
+from bagelquant_data.core.source import DataSource
+
+__all__ = [
+    "BagelQuantDataError",
+    "ConfigurationError",
+    "DataSource",
+    "DatasetNotFoundError",
+    "DatasetSpec",
+    "DatasetSpecError",
+    "DestructiveOperationError",
+    "DuplicateResolutionError",
+    "FrameworkRegistries",
+    "Registry",
+    "RequestContext",
+    "SourceNotFoundError",
+    "ValidationError",
+    "default_registries",
+    "frame_content_hash",
+    "stable_bucket",
+    "stable_record_hash",
+]

bagelquant_data-0.1.0/src/bagelquant_data/core/dataset.py

@@ -0,0 +1,170 @@
+"""Dataset specification model."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from bagelquant_data.core.exceptions import DatasetSpecError
+
+
+@dataclass(frozen=True, slots=True)
+class DatasetSpec:
+    """Declarative canonical dataset behavior."""
+
+    name: str
+    source: str
+    source_dataset: str
+    category: str
+    field_mapping: dict[str, str]
+    required_columns: tuple[str, ...]
+    primary_key: tuple[str, ...] | None = None
+    business_key: tuple[str, ...] | None = None
+    asset_column: str | None = None
+    time_column: str | None = None
+    period_column: str | None = None
+    request_planner: str = "snapshot"
+    request_options: dict[str, Any] = field(default_factory=dict)
+    normalizer: str = "standard"
+    deduplication: str = "exact_record_hash"
+    partition_strategy: str = "single_file"
+    partition_options: dict[str, Any] = field(default_factory=dict)
+    update_mode: str = "upsert"
+    sort_columns: tuple[str, ...] = ()
+    point_in_time: bool = False
+    reference: bool = False
+    enabled: bool = True
+
+    @classmethod
+    def from_mapping(cls, value: dict[str, Any]) -> "DatasetSpec":
+        """Build a specification from parsed config data."""
+
+        required = ("name", "source", "source_dataset", "category")
+        missing = [key for key in required if key not in value]
+        if missing:
+            raise DatasetSpecError(f"Dataset spec missing required keys: {missing}")
+        return cls(
+            name=str(value["name"]),
+            source=str(value["source"]),
+            source_dataset=str(value["source_dataset"]),
+            category=str(value["category"]),
+            field_mapping=dict(value.get("field_mapping") or {}),
+            required_columns=_tuple(value.get("required_columns")),
+            primary_key=_optional_tuple(value.get("primary_key")),
+            business_key=_optional_tuple(value.get("business_key")),
+            asset_column=_optional_str(value.get("asset_column")),
+            time_column=_optional_str(value.get("time_column")),
+            period_column=_optional_str(value.get("period_column")),
+            request_planner=str(value.get("request_planner") or "snapshot"),
+            request_options=dict(value.get("request_options") or {}),
+            normalizer=str(value.get("normalizer") or "standard"),
+            deduplication=str(value.get("deduplication") or "exact_record_hash"),
+            partition_strategy=str(value.get("partition_strategy") or "single_file"),
+            partition_options=dict(value.get("partition_options") or {}),
+            update_mode=str(value.get("update_mode") or "upsert"),
+            sort_columns=_tuple(value.get("sort_columns")),
+            point_in_time=bool(value.get("point_in_time", False)),
+            reference=bool(value.get("reference", False)),
+            enabled=bool(value.get("enabled", True)),
+        )
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> "DatasetSpec":
+        """Load a dataset spec from a small YAML file.
+
+        The project intentionally avoids a YAML runtime dependency. This parser
+        supports the simple mappings and lists used by bundled dataset specs.
+        """
+
+        return cls.from_mapping(_parse_simple_yaml(Path(path).read_text()))
+
+    @property
+    def key(self) -> tuple[str, str]:
+        """Return the `(source, name)` lookup key."""
+
+        return (self.source, self.name)
+
+
+def _tuple(value: Any) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        return (value,)
+    return tuple(str(item) for item in value)
+
+
+def _optional_tuple(value: Any) -> tuple[str, ...] | None:
+    result = _tuple(value)
+    return result or None
+
+
+def _optional_str(value: Any) -> str | None:
+    return None if value is None else str(value)
+
+
+def _parse_simple_yaml(text: str) -> dict[str, Any]:
+    rows = [line.rstrip() for line in text.splitlines()]
+    root: dict[str, Any] = {}
+    stack: list[tuple[int, dict[str, Any] | list[Any]]] = [(-1, root)]
+    for raw in rows:
+        if not raw.strip() or raw.lstrip().startswith("#"):
+            continue
+        indent = len(raw) - len(raw.lstrip(" "))
+        line = raw.strip()
+        while stack and indent <= stack[-1][0]:
+            stack.pop()
+        parent = stack[-1][1]
+        if line.startswith("- "):
+            if not isinstance(parent, list):
+                raise DatasetSpecError(f"Unsupported YAML list location: {line}")
+            parent.append(_yaml_scalar(line[2:]))
+            continue
+        key, sep, value = line.partition(":")
+        if not sep:
+            raise DatasetSpecError(f"Unsupported YAML line: {line}")
+        key = key.strip()
+        value = value.strip()
+        if value == "":
+            container: dict[str, Any] | list[Any]
+            next_list = _next_content_is_list(rows, raw)
+            container = [] if next_list else {}
+            if isinstance(parent, dict):
+                parent[key] = container
+            else:
+                raise DatasetSpecError(f"Unsupported nested YAML key: {key}")
+            stack.append((indent, container))
+        elif isinstance(parent, dict):
+            parent[key] = _yaml_scalar(value)
+    return root
+
+
+def _next_content_is_list(rows: list[str], current: str) -> bool:
+    index = rows.index(current)
+    current_indent = len(current) - len(current.lstrip(" "))
+    for row in rows[index + 1 :]:
+        if not row.strip() or row.lstrip().startswith("#"):
+            continue
+        indent = len(row) - len(row.lstrip(" "))
+        return indent > current_indent and row.strip().startswith("- ")
+    return False
+
+
+def _yaml_scalar(value: str) -> Any:
+    if value in {"true", "True"}:
+        return True
+    if value in {"false", "False"}:
+        return False
+    if value in {"null", "None", "~"}:
+        return None
+    if value.startswith("[") and value.endswith("]"):
+        inner = value[1:-1].strip()
+        return [] if not inner else [_yaml_scalar(part.strip()) for part in inner.split(",")]
+    if (value.startswith('"') and value.endswith('"')) or (
+        value.startswith("'") and value.endswith("'")
+    ):
+        return value[1:-1]
+    try:
+        return int(value)
+    except ValueError:
+        return value

bagelquant_data-0.1.0/src/bagelquant_data/core/deduplication.py

@@ -0,0 +1,40 @@
+"""Deduplication strategies."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+import polars as pl
+
+from bagelquant_data.core.dataset import DatasetSpec
+
+
+class DeduplicationStrategy(Protocol):
+    """Deduplicate records for a dataset."""
+
+    def apply(self, frame: pl.LazyFrame, spec: DatasetSpec) -> pl.LazyFrame:
+        """Return deduplicated records."""
+        ...
+
+
+class NoDeduplication:
+    """Leave records unchanged."""
+
+    def apply(self, frame: pl.LazyFrame, spec: DatasetSpec) -> pl.LazyFrame:
+        return frame
+
+
+class ExactRecordHashDeduplication:
+    """Drop exact duplicate rows."""
+
+    def apply(self, frame: pl.LazyFrame, spec: DatasetSpec) -> pl.LazyFrame:
+        return frame.unique(maintain_order=True)
+
+
+class PrimaryKeyLastDeduplication:
+    """Keep the last row for each primary key."""
+
+    def apply(self, frame: pl.LazyFrame, spec: DatasetSpec) -> pl.LazyFrame:
+        if not spec.primary_key:
+            return frame.unique(maintain_order=True)
+        return frame.unique(subset=list(spec.primary_key), keep="last", maintain_order=True)

bagelquant_data-0.1.0/src/bagelquant_data/core/exceptions.py

@@ -0,0 +1,39 @@
+"""Framework exceptions."""
+
+from __future__ import annotations
+
+
+class BagelQuantDataError(Exception):
+    """Base package error."""
+
+
+class ConfigurationError(BagelQuantDataError):
+    """Configuration is invalid or incomplete."""
+
+
+class DatasetSpecError(ConfigurationError):
+    """Dataset specification is invalid."""
+
+
+class DatasetNotFoundError(BagelQuantDataError):
+    """Requested dataset is not registered or has no canonical data."""
+
+
+class SourceNotFoundError(BagelQuantDataError):
+    """Requested source is not registered."""
+
+
+class DataSourceError(BagelQuantDataError):
+    """Source adapter failed."""
+
+
+class ValidationError(BagelQuantDataError):
+    """Data failed validation."""
+
+
+class DuplicateResolutionError(BagelQuantDataError):
+    """A single-value panel cannot be produced without resolving duplicates."""
+
+
+class DestructiveOperationError(BagelQuantDataError):
+    """A destructive operation was requested without explicit confirmation."""

bagelquant_data-0.1.0/src/bagelquant_data/core/hashing.py

@@ -0,0 +1,32 @@
+"""Stable hashing helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterable
+
+import polars as pl
+
+
+def stable_bucket(asset_id: str, bucket_count: int) -> int:
+    """Return a deterministic asset bucket."""
+
+    digest = hashlib.blake2b(asset_id.encode("utf-8"), digest_size=8).digest()
+    return int.from_bytes(digest, byteorder="big") % bucket_count
+
+
+def stable_record_hash(values: dict[str, object]) -> str:
+    """Hash a record using stable JSON encoding."""
+
+    payload = json.dumps(values, sort_keys=True, default=str, separators=(",", ":"))
+    return hashlib.blake2b(payload.encode("utf-8"), digest_size=16).hexdigest()
+
+
+def frame_content_hash(frame: pl.DataFrame, columns: Iterable[str] | None = None) -> str:
+    """Hash a dataframe deterministically after sorting selected columns."""
+
+    selected = list(columns or frame.columns)
+    rows = frame.select(selected).sort(selected).to_dicts()
+    payload = json.dumps(rows, sort_keys=True, default=str, separators=(",", ":"))
+    return hashlib.blake2b(payload.encode("utf-8"), digest_size=16).hexdigest()

bagelquant_data-0.1.0/src/bagelquant_data/core/normalization.py

@@ -0,0 +1,67 @@
+"""Canonical normalization."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+import polars as pl
+
+from bagelquant_data.core.dataset import DatasetSpec
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizeContext:
+    """Normalization context."""
+
+    source: str
+    dataset: str
+    run_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizeResult:
+    """Accepted and rejected normalized records."""
+
+    accepted: pl.LazyFrame
+    rejected: pl.LazyFrame
+
+
+class Normalizer(Protocol):
+    """Dataset normalizer protocol."""
+
+    def normalize(
+        self, frame: pl.LazyFrame, spec: DatasetSpec, context: NormalizeContext
+    ) -> NormalizeResult:
+        """Normalize source rows."""
+        ...
+
+
+class StandardNormalizer:
+    """Map configured source fields into canonical columns."""
+
+    def normalize(
+        self, frame: pl.LazyFrame, spec: DatasetSpec, context: NormalizeContext
+    ) -> NormalizeResult:
+        lf = frame.rename(spec.field_mapping)
+        expressions: list[pl.Expr] = [
+            pl.lit(context.source).alias("source"),
+            pl.lit(spec.source_dataset).alias("source_dataset"),
+        ]
+        if spec.asset_column and spec.asset_column in lf.collect_schema().names():
+            expressions.append(pl.col(spec.asset_column).cast(pl.String).alias("asset_id"))
+        if spec.time_column and spec.time_column in lf.collect_schema().names():
+            expressions.append(_date_expr(spec.time_column).alias("time"))
+        if spec.period_column and spec.period_column in lf.collect_schema().names():
+            expressions.append(_date_expr(spec.period_column).alias("period"))
+        accepted = lf.with_columns(expressions)
+        rejected = accepted.filter(pl.lit(False))
+        return NormalizeResult(accepted=accepted, rejected=rejected)
+
+
+def _date_expr(column: str) -> pl.Expr:
+    return (
+        pl.when(pl.col(column).cast(pl.String).str.len_chars() == 8)
+        .then(pl.col(column).cast(pl.String).str.strptime(pl.Date, "%Y%m%d", strict=False))
+        .otherwise(pl.col(column).cast(pl.Date, strict=False))
+    )