quant-research-dataset 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. quant_research_dataset/__init__.py +67 -0
  2. quant_research_dataset/_version.py +1 -0
  3. quant_research_dataset/adapters/__init__.py +12 -0
  4. quant_research_dataset/adapters/base.py +63 -0
  5. quant_research_dataset/adapters/us_fundamentals.py +60 -0
  6. quant_research_dataset/adapters/us_options.py +35 -0
  7. quant_research_dataset/adapters/us_stocks.py +48 -0
  8. quant_research_dataset/analytics.py +73 -0
  9. quant_research_dataset/build.py +316 -0
  10. quant_research_dataset/catalog.py +176 -0
  11. quant_research_dataset/cli.py +169 -0
  12. quant_research_dataset/config.py +107 -0
  13. quant_research_dataset/errors.py +26 -0
  14. quant_research_dataset/links.py +152 -0
  15. quant_research_dataset/manifest.py +202 -0
  16. quant_research_dataset/markets/__init__.py +50 -0
  17. quant_research_dataset/markets/cn_hk.py +73 -0
  18. quant_research_dataset/markets/us.py +236 -0
  19. quant_research_dataset/publish.py +147 -0
  20. quant_research_dataset/py.typed +0 -0
  21. quant_research_dataset/query.py +349 -0
  22. quant_research_dataset/registry/cn_hk/equity/ah_pairs.yaml +21 -0
  23. quant_research_dataset/registry/cn_hk/equity/daily_bars.yaml +30 -0
  24. quant_research_dataset/registry/cn_hk/fx/daily.yaml +25 -0
  25. quant_research_dataset/registry/us/equity/stock_daily.yaml +64 -0
  26. quant_research_dataset/registry/us/etf/bars_1min.yaml +27 -0
  27. quant_research_dataset/registry/us/etf/bars_5min.yaml +32 -0
  28. quant_research_dataset/registry/us/etf/daily_bars.yaml +31 -0
  29. quant_research_dataset/registry/us/fundamentals/annual.yaml +90 -0
  30. quant_research_dataset/registry/us/fundamentals/quarterly.yaml +93 -0
  31. quant_research_dataset/registry/us/index/spx_daily.yaml +25 -0
  32. quant_research_dataset/registry/us/options/spx_option_prices.yaml +54 -0
  33. quant_research_dataset/registry/us/options/spx_vol_surface.yaml +29 -0
  34. quant_research_dataset/registry/us/rates/zero_curve.yaml +23 -0
  35. quant_research_dataset/registry.py +335 -0
  36. quant_research_dataset/sql.py +54 -0
  37. quant_research_dataset/storage.py +139 -0
  38. quant_research_dataset-0.1.0.dist-info/METADATA +254 -0
  39. quant_research_dataset-0.1.0.dist-info/RECORD +42 -0
  40. quant_research_dataset-0.1.0.dist-info/WHEEL +4 -0
  41. quant_research_dataset-0.1.0.dist-info/entry_points.txt +2 -0
  42. quant_research_dataset-0.1.0.dist-info/licenses/LICENSE +24 -0
@@ -0,0 +1,67 @@
1
+ """quant-research-dataset: one API for research-grade market data across markets.
2
+
3
+ >>> import quant_research_dataset as qrd
4
+ >>> qrd.list_datasets(market="us") # doctest: +SKIP
5
+ >>> df = qrd.load("us/equity/stock_daily", start="2024-01-01",
6
+ ... columns=["date", "permno", "ret"], common_only=True) # doctest: +SKIP
7
+
8
+ Generic API
9
+ :func:`load`, :func:`list_datasets`, :func:`describe`, :func:`download`
10
+ Per-market shortcuts
11
+ :mod:`qrd.us <quant_research_dataset.markets.us>`,
12
+ :mod:`qrd.cn_hk <quant_research_dataset.markets.cn_hk>`
13
+ Research helpers
14
+ :func:`quantile_portfolios`, :func:`performance_summary`
15
+ """
16
+
17
+ from ._version import __version__
18
+ from .analytics import performance_summary, quantile_portfolios
19
+ from .config import configure, get_settings
20
+ from .errors import ConfigError, DatasetNotFoundError, ManifestError, QRDError, RegistryError
21
+ from .markets import cn_hk, get_prices, us
22
+ from .markets.us import (
23
+ get_etf_bars,
24
+ get_fundamentals,
25
+ get_spx_options,
26
+ get_stock_daily,
27
+ get_vol_surface,
28
+ merge_fundamentals,
29
+ zero_rate,
30
+ )
31
+ from .query import describe, download, files_for, get_spec, list_datasets, load
32
+ from .registry import DatasetSpec, Registry, default_registry, load_registry
33
+ from .sql import duckdb_connect
34
+
35
+ __all__ = [
36
+ "ConfigError",
37
+ "DatasetNotFoundError",
38
+ "DatasetSpec",
39
+ "ManifestError",
40
+ "QRDError",
41
+ "Registry",
42
+ "RegistryError",
43
+ "__version__",
44
+ "cn_hk",
45
+ "configure",
46
+ "default_registry",
47
+ "describe",
48
+ "download",
49
+ "duckdb_connect",
50
+ "files_for",
51
+ "get_etf_bars",
52
+ "get_fundamentals",
53
+ "get_prices",
54
+ "get_settings",
55
+ "get_spec",
56
+ "get_spx_options",
57
+ "get_stock_daily",
58
+ "get_vol_surface",
59
+ "list_datasets",
60
+ "load",
61
+ "load_registry",
62
+ "merge_fundamentals",
63
+ "performance_summary",
64
+ "quantile_portfolios",
65
+ "us",
66
+ "zero_rate",
67
+ ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,12 @@
1
+ """Market-specific logic, kept out of the generic loader.
2
+
3
+ An adapter can add a pushed-down row filter (e.g. "common stocks only"),
4
+ derived columns (e.g. option mid prices) or point-in-time fields to a dataset
5
+ at load time. Datasets pick an adapter with the ``adapter:`` key in their
6
+ registry YAML; datasets without one use the base :class:`Adapter`.
7
+ """
8
+
9
+ from . import us_fundamentals, us_options, us_stocks # noqa: F401 (registers adapters)
10
+ from .base import Adapter, get_adapter, register_adapter
11
+
12
+ __all__ = ["Adapter", "get_adapter", "register_adapter"]
@@ -0,0 +1,63 @@
1
+ """Adapter base class and lookup."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar, TypeVar
6
+
7
+ import pandas as pd
8
+ import pyarrow.compute as pc
9
+
10
+ from ..errors import QRDError
11
+ from ..registry import DatasetSpec
12
+
13
+ _ADAPTERS: dict[str, type[Adapter]] = {}
14
+ AdapterType = TypeVar("AdapterType", bound="type[Adapter]")
15
+
16
+
17
+ class Adapter:
18
+ """Load-time hooks for a dataset.
19
+
20
+ Subclasses declare the keyword options they accept in ``options`` (name
21
+ to default value). :func:`quant_research_dataset.load` forwards extra
22
+ keyword arguments to the adapter and rejects options it does not declare.
23
+ """
24
+
25
+ name: ClassVar[str] = "default"
26
+ options: ClassVar[dict[str, Any]] = {}
27
+
28
+ def resolve_options(self, spec: DatasetSpec, given: dict[str, Any]) -> dict[str, Any]:
29
+ unknown = sorted(set(given) - set(self.options))
30
+ if unknown:
31
+ allowed = ", ".join(sorted(self.options)) or "none"
32
+ raise QRDError(f"{spec.id} does not accept options {unknown} (allowed: {allowed})")
33
+ return {**self.options, **given}
34
+
35
+ def row_filter(self, spec: DatasetSpec, options: dict[str, Any]) -> pc.Expression | None:
36
+ """An Arrow filter pushed down into the parquet scan, or ``None``."""
37
+ return None
38
+
39
+ def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
40
+ """Columns :meth:`postprocess` needs even if the caller did not ask for them."""
41
+ return []
42
+
43
+ def postprocess(
44
+ self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
45
+ ) -> pd.DataFrame:
46
+ """Add derived columns after the scan."""
47
+ return df
48
+
49
+
50
+ def register_adapter(cls: AdapterType) -> AdapterType:
51
+ """Class decorator that makes an adapter available by its ``name``."""
52
+ _ADAPTERS[cls.name] = cls
53
+ return cls
54
+
55
+
56
+ register_adapter(Adapter)
57
+
58
+
59
+ def get_adapter(name: str) -> Adapter:
60
+ try:
61
+ return _ADAPTERS[name]()
62
+ except KeyError:
63
+ raise QRDError(f"unknown adapter {name!r}; known: {sorted(_ADAPTERS)}") from None
@@ -0,0 +1,60 @@
1
+ """Compustat fundamentals adapter: point-in-time availability dates."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar
6
+
7
+ import pandas as pd
8
+
9
+ from ..registry import DatasetSpec
10
+ from .base import Adapter, register_adapter
11
+
12
+ #: Conservative reporting lags used when the report date is unknown.
13
+ DEFAULT_LAG_DAYS = {"quarterly": 90, "annual": 180}
14
+
15
+
16
+ def available_date(
17
+ df: pd.DataFrame,
18
+ frequency: str,
19
+ lag_days: int | None = None,
20
+ report_date_col: str = "rdq",
21
+ ) -> pd.Series:
22
+ """Date from which a fundamentals row was known to investors.
23
+
24
+ Uses the earnings report date (``rdq``) when there is one, otherwise the
25
+ fiscal period end (``datadate``) plus a fixed lag: 90 days for quarterly
26
+ and 180 days for annual data by default. Joining on this date instead of
27
+ ``datadate`` avoids look-ahead bias in backtests.
28
+ """
29
+ lag = pd.Timedelta(days=lag_days if lag_days is not None else DEFAULT_LAG_DAYS[frequency])
30
+ fallback = pd.to_datetime(df["datadate"]) + lag
31
+ if report_date_col in df.columns:
32
+ return pd.to_datetime(df[report_date_col]).fillna(fallback).rename("available_date")
33
+ return fallback.rename("available_date")
34
+
35
+
36
+ @register_adapter
37
+ class FundamentalsAdapter(Adapter):
38
+ """Options:
39
+
40
+ ``point_in_time``
41
+ Add an ``available_date`` column (see :func:`available_date`).
42
+ ``lag_days``
43
+ Lag used when the report date is missing. Defaults to 90 days
44
+ (quarterly) or 180 days (annual).
45
+ """
46
+
47
+ name: ClassVar[str] = "us_fundamentals"
48
+ options: ClassVar[dict[str, Any]] = {"point_in_time": False, "lag_days": None}
49
+
50
+ def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
51
+ if not options["point_in_time"]:
52
+ return []
53
+ return ["datadate", *(["rdq"] if "rdq" in spec.column_names else [])]
54
+
55
+ def postprocess(
56
+ self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
57
+ ) -> pd.DataFrame:
58
+ if options["point_in_time"]:
59
+ df = df.assign(available_date=available_date(df, spec.frequency, options["lag_days"]))
60
+ return df
@@ -0,0 +1,35 @@
1
+ """OptionMetrics adapter: derived option fields."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar
6
+
7
+ import pandas as pd
8
+
9
+ from ..registry import DatasetSpec
10
+ from .base import Adapter, register_adapter
11
+
12
+
13
+ @register_adapter
14
+ class OptionsAdapter(Adapter):
15
+ """Options:
16
+
17
+ ``derived``
18
+ Add ``mid`` (bid/offer midpoint), ``spread`` (offer minus bid) and
19
+ ``dte`` (calendar days to expiry). On by default.
20
+ """
21
+
22
+ name: ClassVar[str] = "us_options"
23
+ options: ClassVar[dict[str, Any]] = {"derived": True}
24
+
25
+ def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
26
+ return ["best_bid", "best_offer", "date", "exdate"] if options["derived"] else []
27
+
28
+ def postprocess(
29
+ self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
30
+ ) -> pd.DataFrame:
31
+ if not options["derived"]:
32
+ return df
33
+ bid, ask = df["best_bid"], df["best_offer"]
34
+ dte = (pd.to_datetime(df["exdate"]) - pd.to_datetime(df["date"])).dt.days
35
+ return df.assign(mid=(bid + ask) / 2.0, spread=ask - bid, dte=dte.astype("int32"))
@@ -0,0 +1,48 @@
1
+ """CRSP (US stocks) adapter."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from functools import reduce
6
+ from typing import Any, ClassVar
7
+
8
+ import pyarrow.compute as pc
9
+
10
+ from ..registry import DatasetSpec
11
+ from .base import Adapter, register_adapter
12
+
13
+ #: The usual "ordinary common stock" screen in CRSP CIZ flags: common shares
14
+ #: of US-incorporated companies, listed on NYSE, NYSE American or Nasdaq,
15
+ #: trading regular way and active. It is the CIZ equivalent of the classic
16
+ #: ``shrcd in (10, 11)`` and ``exchcd in (1, 2, 3)`` filter.
17
+ COMMON_STOCK_SCREEN: dict[str, tuple[str, ...]] = {
18
+ "share_type": ("NS",),
19
+ "security_type": ("EQTY",),
20
+ "security_sub_type": ("COM",),
21
+ "us_inc_flg": ("Y",),
22
+ "issuer_type": ("CORP", "ACOR"),
23
+ "primary_exch": ("N", "A", "Q"),
24
+ "conditional_type": ("RW",),
25
+ "trading_status_flg": ("A",),
26
+ }
27
+
28
+
29
+ def common_stock_expression() -> pc.Expression:
30
+ """Arrow filter for :data:`COMMON_STOCK_SCREEN`."""
31
+ parts = [pc.field(col).isin(list(values)) for col, values in COMMON_STOCK_SCREEN.items()]
32
+ return reduce(lambda a, b: a & b, parts)
33
+
34
+
35
+ @register_adapter
36
+ class StockAdapter(Adapter):
37
+ """Options:
38
+
39
+ ``common_only``
40
+ Keep only ordinary common shares (see :data:`COMMON_STOCK_SCREEN`),
41
+ the standard universe for cross-sectional equity research.
42
+ """
43
+
44
+ name: ClassVar[str] = "us_stocks"
45
+ options: ClassVar[dict[str, Any]] = {"common_only": False}
46
+
47
+ def row_filter(self, spec: DatasetSpec, options: dict[str, Any]) -> pc.Expression | None:
48
+ return common_stock_expression() if options["common_only"] else None
@@ -0,0 +1,73 @@
1
+ """Small, dependency-free research helpers: portfolio sorts and performance stats."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+
8
+
9
+ def quantile_portfolios(
10
+ df: pd.DataFrame,
11
+ signal: str,
12
+ ret: str = "ret",
13
+ date: str = "date",
14
+ n: int = 10,
15
+ weight: str | None = None,
16
+ min_obs: int = 20,
17
+ ) -> pd.DataFrame:
18
+ """Sort assets into ``n`` quantiles of ``signal`` each date and average their returns.
19
+
20
+ ``df`` holds one row per asset and date, where ``signal`` is known at
21
+ ``date`` and ``ret`` is the return over the *next* holding period (shift
22
+ returns yourself so there is no look-ahead). Returns a wide frame indexed
23
+ by date with columns ``1..n`` and ``long_short`` (quantile n minus 1).
24
+ ``weight`` (e.g. lagged market cap) gives value-weighted portfolios.
25
+ Dates with fewer than ``min_obs`` valid rows are skipped.
26
+ """
27
+ cols = [date, signal, ret] + ([weight] if weight else [])
28
+ data = df[cols].dropna()
29
+ counts = data.groupby(date)[signal].transform("size")
30
+ data = data[counts >= max(min_obs, n)]
31
+ if data.empty:
32
+ return pd.DataFrame(columns=[*range(1, n + 1), "long_short"])
33
+ ranks = data.groupby(date)[signal].rank(method="first")
34
+ sizes = data.groupby(date)[signal].transform("size")
35
+ data = data.assign(q=np.ceil(ranks * n / sizes).astype(int).clip(1, n))
36
+ if weight:
37
+ w = data[weight].clip(lower=0)
38
+ num = (data[ret] * w).groupby([data[date], data["q"]]).sum()
39
+ den = w.groupby([data[date], data["q"]]).sum()
40
+ port = (num / den).unstack("q")
41
+ else:
42
+ port = data.groupby([date, "q"])[ret].mean().unstack("q")
43
+ port.columns = [int(c) for c in port.columns]
44
+ port["long_short"] = port[n] - port[1]
45
+ port.columns.name = None
46
+ return port
47
+
48
+
49
+ def performance_summary(
50
+ returns: pd.Series | pd.DataFrame, periods_per_year: int = 252
51
+ ) -> pd.DataFrame:
52
+ """Annualized return, volatility, Sharpe ratio (zero risk-free rate), max drawdown, hit rate."""
53
+ frame = returns.to_frame() if isinstance(returns, pd.Series) else returns
54
+ rows = {}
55
+ for name, r in frame.items():
56
+ r = r.dropna()
57
+ if r.empty:
58
+ continue
59
+ wealth = (1 + r).cumprod()
60
+ years = len(r) / periods_per_year
61
+ ann_ret = wealth.iloc[-1] ** (1 / years) - 1 if years > 0 else np.nan
62
+ vol = r.std(ddof=1) * np.sqrt(periods_per_year)
63
+ rows[name] = {
64
+ "ann_return": ann_ret,
65
+ "ann_vol": vol,
66
+ "sharpe": r.mean() / r.std(ddof=1) * np.sqrt(periods_per_year)
67
+ if r.std(ddof=1) > 0
68
+ else np.nan,
69
+ "max_drawdown": (wealth / wealth.cummax() - 1).min(),
70
+ "hit_rate": (r > 0).mean(),
71
+ "n_periods": len(r),
72
+ }
73
+ return pd.DataFrame(rows).T
@@ -0,0 +1,316 @@
1
+ """Build prepared, partitioned parquet files from raw source files.
2
+
3
+ This is maintainer tooling: it turns the raw downloads (WRDS parquet/CSV,
4
+ vendor CSVs) into the standard layout the client reads::
5
+
6
+ <out>/<market>/<asset>/<name>/year=2015/part-0.parquet
7
+ <out>/<market>/<asset>/<name>/year=2015/month=08/part-0.parquet
8
+ <out>/<market>/<asset>/<name>/data.parquet (unpartitioned)
9
+ <out>/manifest.json
10
+
11
+ Everything is driven by the registry YAML: which raw files to read, how to
12
+ rename and type each column, how to partition and sort. Raw files are only
13
+ ever read, never modified.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import csv
19
+ import datetime as dt
20
+ import logging
21
+ import re
22
+ import shutil
23
+ from collections.abc import Iterable, Iterator
24
+ from dataclasses import dataclass
25
+ from pathlib import Path
26
+ from typing import Any
27
+
28
+ import pyarrow as pa
29
+ import pyarrow.compute as pc
30
+ import pyarrow.csv as pacsv
31
+ import pyarrow.dataset as ds
32
+ import pyarrow.parquet as pq
33
+
34
+ from .errors import QRDError
35
+ from .links import Enricher, make_enricher
36
+ from .manifest import Manifest, build_manifest
37
+ from .registry import DatasetSpec, Registry, default_registry
38
+
39
+ log = logging.getLogger(__name__)
40
+
41
+ ROW_GROUP_SIZE = 131_072
42
+ COMPRESSION = "zstd"
43
+ COMPRESSION_LEVEL = 6
44
+
45
+
46
+ # --------------------------------------------------------------------- sources
47
+ def source_files(spec: DatasetSpec, source_root: Path) -> list[Path]:
48
+ """Resolve the raw files for ``spec`` (``build.source`` may hold globs)."""
49
+ files: list[Path] = []
50
+ for pattern in spec.build.source:
51
+ matches = sorted(source_root.glob(pattern))
52
+ if not matches:
53
+ raise QRDError(f"{spec.id}: no source files match {pattern!r} under {source_root}")
54
+ files += matches
55
+ return files
56
+
57
+
58
+ def _read_csv(path: Path, spec: DatasetSpec) -> pa.Table:
59
+ # Read every column as text and convert explicitly later. This keeps
60
+ # identifiers such as gvkey ("001004") and CUSIPs intact.
61
+ with path.open(encoding="utf-8-sig", newline="") as fh:
62
+ header = next(csv.reader(fh))
63
+ convert = pacsv.ConvertOptions(
64
+ column_types={name: pa.string() for name in header},
65
+ strings_can_be_null=True,
66
+ null_values=[""],
67
+ )
68
+ table = pacsv.read_csv(path, convert_options=convert)
69
+ pattern = spec.build.options.get("id_from_filename")
70
+ if pattern:
71
+ if spec.id_column is None:
72
+ raise QRDError(f"{spec.id}: id_from_filename needs an id_column")
73
+ m = re.search(pattern, path.name)
74
+ if not m:
75
+ raise QRDError(f"{spec.id}: {path.name} does not match id_from_filename {pattern!r}")
76
+ target = spec.column(spec.id_column).source_name
77
+ table = table.append_column(target, pa.array([m.group("id")] * table.num_rows, pa.string()))
78
+ return table
79
+
80
+
81
+ def read_source(spec: DatasetSpec, source_root: Path) -> ds.Dataset:
82
+ """Open the raw source as an Arrow dataset (CSV files are read eagerly)."""
83
+ files = source_files(spec, source_root)
84
+ if spec.build.format == "parquet":
85
+ return ds.dataset([str(f) for f in files], format="parquet")
86
+ if spec.build.format == "csv":
87
+ tables = [_read_csv(f, spec) for f in files]
88
+ return ds.dataset(pa.concat_tables(tables, promote_options="default"))
89
+ raise QRDError(f"{spec.id}: unsupported source format {spec.build.format!r}")
90
+
91
+
92
+ # ------------------------------------------------------------------ conversion
93
+ def _to_naive_or_zoned(
94
+ arr: pa.ChunkedArray, target: pa.DataType, tz: str | None
95
+ ) -> pa.ChunkedArray:
96
+ """Convert a timestamp array to ``target``, localising naive values to ``tz``."""
97
+ src_tz = getattr(arr.type, "tz", None)
98
+ tgt_tz = getattr(target, "tz", None)
99
+ if src_tz is None and tgt_tz is not None:
100
+ arr = pc.assume_timezone(arr, timezone=tz or "UTC")
101
+ return arr.cast(target)
102
+
103
+
104
+ def convert_column(
105
+ arr: pa.ChunkedArray, target: pa.DataType, options: dict[str, Any]
106
+ ) -> pa.ChunkedArray:
107
+ """Convert a raw column to its registry type."""
108
+ if pa.types.is_dictionary(arr.type):
109
+ arr = arr.cast(arr.type.value_type)
110
+ if arr.type == target:
111
+ return arr
112
+ src = arr.type
113
+ tz = options.get("timezone")
114
+ if pa.types.is_string(src) or pa.types.is_large_string(src):
115
+ if pa.types.is_date32(target):
116
+ return pc.strptime(arr, format="%Y-%m-%d", unit="s").cast(pa.date32())
117
+ if pa.types.is_timestamp(target):
118
+ fmt = options.get("timestamp_format", "%Y-%m-%d %H:%M:%S")
119
+ return _to_naive_or_zoned(pc.strptime(arr, format=fmt, unit="us"), target, tz)
120
+ if pa.types.is_integer(target):
121
+ as_float = arr.cast(pa.float64())
122
+ return as_float.cast(target) # safe cast: fails on fractional values
123
+ return arr.cast(target)
124
+ if pa.types.is_timestamp(src):
125
+ if pa.types.is_date32(target):
126
+ return arr.cast(pa.date32())
127
+ if pa.types.is_timestamp(target):
128
+ return _to_naive_or_zoned(arr, target, tz)
129
+ return arr.cast(target)
130
+
131
+
132
+ def standardize(raw: pa.Table, spec: DatasetSpec, enrich: Enricher | None = None) -> pa.Table:
133
+ """Select, rename, type, filter, enrich and sort raw columns into the registry schema."""
134
+ opts = spec.build.options
135
+ columns: dict[str, pa.ChunkedArray] = {}
136
+ for col in spec.columns:
137
+ if col.derived: # filled in by the enrichment step below
138
+ columns[col.name] = pa.chunked_array([pa.nulls(raw.num_rows, col.arrow_type)])
139
+ continue
140
+ if col.source_name not in raw.column_names:
141
+ raise QRDError(f"{spec.id}: source column {col.source_name!r} not found")
142
+ arr = raw.column(col.source_name)
143
+ for token in opts.get("null_tokens", {}).get(col.source_name, []):
144
+ arr = pc.if_else(pc.equal(arr, token), pa.scalar(None, arr.type), arr)
145
+ arr = convert_column(arr, col.arrow_type, opts)
146
+ scale = opts.get("scale", {}).get(col.name)
147
+ if scale is not None:
148
+ arr = pc.multiply(arr, pa.scalar(float(scale))).cast(col.arrow_type)
149
+ columns[col.name] = arr
150
+ table = pa.table(columns, schema=spec.schema)
151
+
152
+ if spec.id_column and (spec.build.include_ids or spec.build.exclude_ids):
153
+ ids = table.column(spec.id_column)
154
+ if spec.build.include_ids:
155
+ table = table.filter(pc.is_in(ids, pa.array(spec.build.include_ids)))
156
+ if spec.build.exclude_ids:
157
+ table = table.filter(pc.invert(pc.is_in(ids, pa.array(spec.build.exclude_ids))))
158
+ if enrich is not None:
159
+ table = enrich(table).select(spec.column_names).cast(spec.schema)
160
+ if spec.sort_by and table.num_rows:
161
+ table = table.sort_by([(c, "ascending") for c in spec.sort_by])
162
+ return table
163
+
164
+
165
+ # ------------------------------------------------------------------ partitions
166
+ @dataclass(frozen=True)
167
+ class Period:
168
+ year: int
169
+ month: int | None = None
170
+
171
+ @property
172
+ def start(self) -> dt.date:
173
+ return dt.date(self.year, self.month or 1, 1)
174
+
175
+ @property
176
+ def stop(self) -> dt.date: # exclusive
177
+ if self.month is None:
178
+ return dt.date(self.year + 1, 1, 1)
179
+ return dt.date(self.year + (self.month == 12), self.month % 12 + 1, 1)
180
+
181
+ def relpath(self) -> str:
182
+ if self.month is None:
183
+ return f"year={self.year}/part-0.parquet"
184
+ return f"year={self.year}/month={self.month:02d}/part-0.parquet"
185
+
186
+
187
+ def periods_between(first: dt.date, last: dt.date, partitioning: str) -> list[Period]:
188
+ out: list[Period] = []
189
+ for year in range(first.year, last.year + 1):
190
+ if partitioning == "year":
191
+ out.append(Period(year))
192
+ else:
193
+ lo = first.month if year == first.year else 1
194
+ hi = last.month if year == last.year else 12
195
+ out += [Period(year, m) for m in range(lo, hi + 1)]
196
+ return out
197
+
198
+
199
+ def _as_date(value: Any) -> dt.date:
200
+ if isinstance(value, dt.datetime):
201
+ return value.date()
202
+ if isinstance(value, dt.date):
203
+ return value
204
+ return dt.date.fromisoformat(str(value)[:10])
205
+
206
+
207
+ def _bound(value: dt.date, typ: pa.DataType) -> pa.Scalar:
208
+ if pa.types.is_timestamp(typ):
209
+ tz = dt.timezone.utc if typ.tz else None
210
+ return pa.scalar(dt.datetime(value.year, value.month, value.day, tzinfo=tz), type=typ)
211
+ return pa.scalar(value, type=typ)
212
+
213
+
214
+ def iter_partitions(
215
+ spec: DatasetSpec, source: ds.Dataset, enrich: Enricher | None = None
216
+ ) -> Iterator[tuple[str, pa.Table]]:
217
+ """Yield ``(relative path, standardized table)`` for each output file."""
218
+ if spec.partitioning == "none":
219
+ yield "data.parquet", standardize(source.to_table(), spec, enrich)
220
+ return
221
+
222
+ assert spec.date_column is not None
223
+ raw_date = spec.column(spec.date_column).source_name
224
+ raw_type = source.schema.field(raw_date).type
225
+ if pa.types.is_date(raw_type) or pa.types.is_timestamp(raw_type):
226
+ # Large typed sources: scan one period at a time to bound memory.
227
+ mm = pc.min_max(source.to_table(columns=[raw_date]).column(0))
228
+ for period in periods_between(
229
+ _as_date(mm["min"].as_py()), _as_date(mm["max"].as_py()), spec.partitioning
230
+ ):
231
+ expr = (pc.field(raw_date) >= _bound(period.start, raw_type)) & (
232
+ pc.field(raw_date) < _bound(period.stop, raw_type)
233
+ )
234
+ table = standardize(source.to_table(filter=expr), spec, enrich)
235
+ if table.num_rows:
236
+ yield period.relpath(), table
237
+ return
238
+
239
+ # Small text sources: standardize once, then split in memory.
240
+ table = standardize(source.to_table(), spec, enrich)
241
+ dates = table.column(spec.date_column)
242
+ years = pc.year(dates)
243
+ months = pc.month(dates) if spec.partitioning == "year_month" else None
244
+ month_list = months.to_pylist() if months is not None else [None] * len(years)
245
+ keys = sorted(set(zip(years.to_pylist(), month_list, strict=True)))
246
+ for year, month in keys:
247
+ mask = pc.equal(years, year)
248
+ if months is not None:
249
+ mask = pc.and_(mask, pc.equal(months, month))
250
+ yield Period(year, month).relpath(), table.filter(mask)
251
+
252
+
253
+ def write_table(table: pa.Table, path: Path, row_group_size: int = ROW_GROUP_SIZE) -> None:
254
+ path.parent.mkdir(parents=True, exist_ok=True)
255
+ pq.write_table(
256
+ table,
257
+ path,
258
+ compression=COMPRESSION,
259
+ compression_level=COMPRESSION_LEVEL,
260
+ row_group_size=row_group_size,
261
+ write_statistics=True,
262
+ )
263
+
264
+
265
+ def build_dataset(spec: DatasetSpec, source_root: str | Path, out_root: str | Path) -> list[Path]:
266
+ """Build one dataset. Its previous output folder is replaced."""
267
+ source_root, out_root = Path(source_root).resolve(), Path(out_root).resolve()
268
+ if (
269
+ out_root == source_root
270
+ or source_root in out_root.parents
271
+ or out_root in source_root.parents
272
+ ):
273
+ raise QRDError("out_root and source_root must not contain each other")
274
+ target = out_root / spec.id
275
+ tmp = target.with_name(target.name + ".tmp")
276
+ if tmp.exists():
277
+ shutil.rmtree(tmp)
278
+ written: list[Path] = []
279
+ rows = 0
280
+ for rel, table in iter_partitions(
281
+ spec, read_source(spec, source_root), make_enricher(spec, source_root)
282
+ ):
283
+ write_table(table, tmp / rel, int(spec.build.options.get("row_group_size", ROW_GROUP_SIZE)))
284
+ written.append(target / rel)
285
+ rows += table.num_rows
286
+ if not written:
287
+ raise QRDError(f"{spec.id}: the source produced no rows")
288
+ if target.exists():
289
+ shutil.rmtree(target)
290
+ tmp.rename(target)
291
+ log.info("built %s: %d files, %d rows", spec.id, len(written), rows)
292
+ return written
293
+
294
+
295
+ def build_all(
296
+ source_root: str | Path,
297
+ out_root: str | Path,
298
+ datasets: Iterable[str] | None = None,
299
+ registry: Registry | None = None,
300
+ version: str = "dev",
301
+ ) -> Manifest:
302
+ """Build the selected datasets (all by default) and write ``manifest.json``.
303
+
304
+ The manifest always covers every dataset present under ``out_root``, so
305
+ datasets can be rebuilt one at a time.
306
+ """
307
+ registry = registry or default_registry()
308
+ specs = [registry.get(d) for d in datasets] if datasets else list(registry)
309
+ out = Path(out_root)
310
+ out.mkdir(parents=True, exist_ok=True)
311
+ for spec in specs:
312
+ log.info("building %s", spec.id)
313
+ build_dataset(spec, source_root, out)
314
+ manifest = build_manifest(out, registry, version=version)
315
+ manifest.save(out)
316
+ return manifest