quant-research-dataset 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quant_research_dataset/__init__.py +67 -0
- quant_research_dataset/_version.py +1 -0
- quant_research_dataset/adapters/__init__.py +12 -0
- quant_research_dataset/adapters/base.py +63 -0
- quant_research_dataset/adapters/us_fundamentals.py +60 -0
- quant_research_dataset/adapters/us_options.py +35 -0
- quant_research_dataset/adapters/us_stocks.py +48 -0
- quant_research_dataset/analytics.py +73 -0
- quant_research_dataset/build.py +316 -0
- quant_research_dataset/catalog.py +176 -0
- quant_research_dataset/cli.py +169 -0
- quant_research_dataset/config.py +107 -0
- quant_research_dataset/errors.py +26 -0
- quant_research_dataset/links.py +152 -0
- quant_research_dataset/manifest.py +202 -0
- quant_research_dataset/markets/__init__.py +50 -0
- quant_research_dataset/markets/cn_hk.py +73 -0
- quant_research_dataset/markets/us.py +236 -0
- quant_research_dataset/publish.py +147 -0
- quant_research_dataset/py.typed +0 -0
- quant_research_dataset/query.py +349 -0
- quant_research_dataset/registry/cn_hk/equity/ah_pairs.yaml +21 -0
- quant_research_dataset/registry/cn_hk/equity/daily_bars.yaml +30 -0
- quant_research_dataset/registry/cn_hk/fx/daily.yaml +25 -0
- quant_research_dataset/registry/us/equity/stock_daily.yaml +64 -0
- quant_research_dataset/registry/us/etf/bars_1min.yaml +27 -0
- quant_research_dataset/registry/us/etf/bars_5min.yaml +32 -0
- quant_research_dataset/registry/us/etf/daily_bars.yaml +31 -0
- quant_research_dataset/registry/us/fundamentals/annual.yaml +90 -0
- quant_research_dataset/registry/us/fundamentals/quarterly.yaml +93 -0
- quant_research_dataset/registry/us/index/spx_daily.yaml +25 -0
- quant_research_dataset/registry/us/options/spx_option_prices.yaml +54 -0
- quant_research_dataset/registry/us/options/spx_vol_surface.yaml +29 -0
- quant_research_dataset/registry/us/rates/zero_curve.yaml +23 -0
- quant_research_dataset/registry.py +335 -0
- quant_research_dataset/sql.py +54 -0
- quant_research_dataset/storage.py +139 -0
- quant_research_dataset-0.1.0.dist-info/METADATA +254 -0
- quant_research_dataset-0.1.0.dist-info/RECORD +42 -0
- quant_research_dataset-0.1.0.dist-info/WHEEL +4 -0
- quant_research_dataset-0.1.0.dist-info/entry_points.txt +2 -0
- quant_research_dataset-0.1.0.dist-info/licenses/LICENSE +24 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""quant-research-dataset: one API for research-grade market data across markets.
|
|
2
|
+
|
|
3
|
+
>>> import quant_research_dataset as qrd
|
|
4
|
+
>>> qrd.list_datasets(market="us") # doctest: +SKIP
|
|
5
|
+
>>> df = qrd.load("us/equity/stock_daily", start="2024-01-01",
|
|
6
|
+
... columns=["date", "permno", "ret"], common_only=True) # doctest: +SKIP
|
|
7
|
+
|
|
8
|
+
Generic API
|
|
9
|
+
:func:`load`, :func:`list_datasets`, :func:`describe`, :func:`download`
|
|
10
|
+
Per-market shortcuts
|
|
11
|
+
:mod:`qrd.us <quant_research_dataset.markets.us>`,
|
|
12
|
+
:mod:`qrd.cn_hk <quant_research_dataset.markets.cn_hk>`
|
|
13
|
+
Research helpers
|
|
14
|
+
:func:`quantile_portfolios`, :func:`performance_summary`
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from ._version import __version__
|
|
18
|
+
from .analytics import performance_summary, quantile_portfolios
|
|
19
|
+
from .config import configure, get_settings
|
|
20
|
+
from .errors import ConfigError, DatasetNotFoundError, ManifestError, QRDError, RegistryError
|
|
21
|
+
from .markets import cn_hk, get_prices, us
|
|
22
|
+
from .markets.us import (
|
|
23
|
+
get_etf_bars,
|
|
24
|
+
get_fundamentals,
|
|
25
|
+
get_spx_options,
|
|
26
|
+
get_stock_daily,
|
|
27
|
+
get_vol_surface,
|
|
28
|
+
merge_fundamentals,
|
|
29
|
+
zero_rate,
|
|
30
|
+
)
|
|
31
|
+
from .query import describe, download, files_for, get_spec, list_datasets, load
|
|
32
|
+
from .registry import DatasetSpec, Registry, default_registry, load_registry
|
|
33
|
+
from .sql import duckdb_connect
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"ConfigError",
|
|
37
|
+
"DatasetNotFoundError",
|
|
38
|
+
"DatasetSpec",
|
|
39
|
+
"ManifestError",
|
|
40
|
+
"QRDError",
|
|
41
|
+
"Registry",
|
|
42
|
+
"RegistryError",
|
|
43
|
+
"__version__",
|
|
44
|
+
"cn_hk",
|
|
45
|
+
"configure",
|
|
46
|
+
"default_registry",
|
|
47
|
+
"describe",
|
|
48
|
+
"download",
|
|
49
|
+
"duckdb_connect",
|
|
50
|
+
"files_for",
|
|
51
|
+
"get_etf_bars",
|
|
52
|
+
"get_fundamentals",
|
|
53
|
+
"get_prices",
|
|
54
|
+
"get_settings",
|
|
55
|
+
"get_spec",
|
|
56
|
+
"get_spx_options",
|
|
57
|
+
"get_stock_daily",
|
|
58
|
+
"get_vol_surface",
|
|
59
|
+
"list_datasets",
|
|
60
|
+
"load",
|
|
61
|
+
"load_registry",
|
|
62
|
+
"merge_fundamentals",
|
|
63
|
+
"performance_summary",
|
|
64
|
+
"quantile_portfolios",
|
|
65
|
+
"us",
|
|
66
|
+
"zero_rate",
|
|
67
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Market-specific logic, kept out of the generic loader.
|
|
2
|
+
|
|
3
|
+
An adapter can add a pushed-down row filter (e.g. "common stocks only"),
|
|
4
|
+
derived columns (e.g. option mid prices) or point-in-time fields to a dataset
|
|
5
|
+
at load time. Datasets pick an adapter with the ``adapter:`` key in their
|
|
6
|
+
registry YAML; datasets without one use the base :class:`Adapter`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from . import us_fundamentals, us_options, us_stocks # noqa: F401 (registers adapters)
|
|
10
|
+
from .base import Adapter, get_adapter, register_adapter
|
|
11
|
+
|
|
12
|
+
__all__ = ["Adapter", "get_adapter", "register_adapter"]
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Adapter base class and lookup."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar, TypeVar
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import pyarrow.compute as pc
|
|
9
|
+
|
|
10
|
+
from ..errors import QRDError
|
|
11
|
+
from ..registry import DatasetSpec
|
|
12
|
+
|
|
13
|
+
_ADAPTERS: dict[str, type[Adapter]] = {}
|
|
14
|
+
AdapterType = TypeVar("AdapterType", bound="type[Adapter]")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Adapter:
|
|
18
|
+
"""Load-time hooks for a dataset.
|
|
19
|
+
|
|
20
|
+
Subclasses declare the keyword options they accept in ``options`` (name
|
|
21
|
+
to default value). :func:`quant_research_dataset.load` forwards extra
|
|
22
|
+
keyword arguments to the adapter and rejects options it does not declare.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
name: ClassVar[str] = "default"
|
|
26
|
+
options: ClassVar[dict[str, Any]] = {}
|
|
27
|
+
|
|
28
|
+
def resolve_options(self, spec: DatasetSpec, given: dict[str, Any]) -> dict[str, Any]:
|
|
29
|
+
unknown = sorted(set(given) - set(self.options))
|
|
30
|
+
if unknown:
|
|
31
|
+
allowed = ", ".join(sorted(self.options)) or "none"
|
|
32
|
+
raise QRDError(f"{spec.id} does not accept options {unknown} (allowed: {allowed})")
|
|
33
|
+
return {**self.options, **given}
|
|
34
|
+
|
|
35
|
+
def row_filter(self, spec: DatasetSpec, options: dict[str, Any]) -> pc.Expression | None:
|
|
36
|
+
"""An Arrow filter pushed down into the parquet scan, or ``None``."""
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
|
|
40
|
+
"""Columns :meth:`postprocess` needs even if the caller did not ask for them."""
|
|
41
|
+
return []
|
|
42
|
+
|
|
43
|
+
def postprocess(
|
|
44
|
+
self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
|
|
45
|
+
) -> pd.DataFrame:
|
|
46
|
+
"""Add derived columns after the scan."""
|
|
47
|
+
return df
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def register_adapter(cls: AdapterType) -> AdapterType:
|
|
51
|
+
"""Class decorator that makes an adapter available by its ``name``."""
|
|
52
|
+
_ADAPTERS[cls.name] = cls
|
|
53
|
+
return cls
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
register_adapter(Adapter)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_adapter(name: str) -> Adapter:
|
|
60
|
+
try:
|
|
61
|
+
return _ADAPTERS[name]()
|
|
62
|
+
except KeyError:
|
|
63
|
+
raise QRDError(f"unknown adapter {name!r}; known: {sorted(_ADAPTERS)}") from None
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Compustat fundamentals adapter: point-in-time availability dates."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from ..registry import DatasetSpec
|
|
10
|
+
from .base import Adapter, register_adapter
|
|
11
|
+
|
|
12
|
+
#: Conservative reporting lags used when the report date is unknown.
|
|
13
|
+
DEFAULT_LAG_DAYS = {"quarterly": 90, "annual": 180}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def available_date(
|
|
17
|
+
df: pd.DataFrame,
|
|
18
|
+
frequency: str,
|
|
19
|
+
lag_days: int | None = None,
|
|
20
|
+
report_date_col: str = "rdq",
|
|
21
|
+
) -> pd.Series:
|
|
22
|
+
"""Date from which a fundamentals row was known to investors.
|
|
23
|
+
|
|
24
|
+
Uses the earnings report date (``rdq``) when there is one, otherwise the
|
|
25
|
+
fiscal period end (``datadate``) plus a fixed lag: 90 days for quarterly
|
|
26
|
+
and 180 days for annual data by default. Joining on this date instead of
|
|
27
|
+
``datadate`` avoids look-ahead bias in backtests.
|
|
28
|
+
"""
|
|
29
|
+
lag = pd.Timedelta(days=lag_days if lag_days is not None else DEFAULT_LAG_DAYS[frequency])
|
|
30
|
+
fallback = pd.to_datetime(df["datadate"]) + lag
|
|
31
|
+
if report_date_col in df.columns:
|
|
32
|
+
return pd.to_datetime(df[report_date_col]).fillna(fallback).rename("available_date")
|
|
33
|
+
return fallback.rename("available_date")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@register_adapter
|
|
37
|
+
class FundamentalsAdapter(Adapter):
|
|
38
|
+
"""Options:
|
|
39
|
+
|
|
40
|
+
``point_in_time``
|
|
41
|
+
Add an ``available_date`` column (see :func:`available_date`).
|
|
42
|
+
``lag_days``
|
|
43
|
+
Lag used when the report date is missing. Defaults to 90 days
|
|
44
|
+
(quarterly) or 180 days (annual).
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
name: ClassVar[str] = "us_fundamentals"
|
|
48
|
+
options: ClassVar[dict[str, Any]] = {"point_in_time": False, "lag_days": None}
|
|
49
|
+
|
|
50
|
+
def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
|
|
51
|
+
if not options["point_in_time"]:
|
|
52
|
+
return []
|
|
53
|
+
return ["datadate", *(["rdq"] if "rdq" in spec.column_names else [])]
|
|
54
|
+
|
|
55
|
+
def postprocess(
|
|
56
|
+
self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
|
|
57
|
+
) -> pd.DataFrame:
|
|
58
|
+
if options["point_in_time"]:
|
|
59
|
+
df = df.assign(available_date=available_date(df, spec.frequency, options["lag_days"]))
|
|
60
|
+
return df
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""OptionMetrics adapter: derived option fields."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from ..registry import DatasetSpec
|
|
10
|
+
from .base import Adapter, register_adapter
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@register_adapter
|
|
14
|
+
class OptionsAdapter(Adapter):
|
|
15
|
+
"""Options:
|
|
16
|
+
|
|
17
|
+
``derived``
|
|
18
|
+
Add ``mid`` (bid/offer midpoint), ``spread`` (offer minus bid) and
|
|
19
|
+
``dte`` (calendar days to expiry). On by default.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
name: ClassVar[str] = "us_options"
|
|
23
|
+
options: ClassVar[dict[str, Any]] = {"derived": True}
|
|
24
|
+
|
|
25
|
+
def extra_columns(self, spec: DatasetSpec, options: dict[str, Any]) -> list[str]:
|
|
26
|
+
return ["best_bid", "best_offer", "date", "exdate"] if options["derived"] else []
|
|
27
|
+
|
|
28
|
+
def postprocess(
|
|
29
|
+
self, df: pd.DataFrame, spec: DatasetSpec, options: dict[str, Any]
|
|
30
|
+
) -> pd.DataFrame:
|
|
31
|
+
if not options["derived"]:
|
|
32
|
+
return df
|
|
33
|
+
bid, ask = df["best_bid"], df["best_offer"]
|
|
34
|
+
dte = (pd.to_datetime(df["exdate"]) - pd.to_datetime(df["date"])).dt.days
|
|
35
|
+
return df.assign(mid=(bid + ask) / 2.0, spread=ask - bid, dte=dte.astype("int32"))
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""CRSP (US stocks) adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from functools import reduce
|
|
6
|
+
from typing import Any, ClassVar
|
|
7
|
+
|
|
8
|
+
import pyarrow.compute as pc
|
|
9
|
+
|
|
10
|
+
from ..registry import DatasetSpec
|
|
11
|
+
from .base import Adapter, register_adapter
|
|
12
|
+
|
|
13
|
+
#: The usual "ordinary common stock" screen in CRSP CIZ flags: common shares
|
|
14
|
+
#: of US-incorporated companies, listed on NYSE, NYSE American or Nasdaq,
|
|
15
|
+
#: trading regular way and active. It is the CIZ equivalent of the classic
|
|
16
|
+
#: ``shrcd in (10, 11)`` and ``exchcd in (1, 2, 3)`` filter.
|
|
17
|
+
COMMON_STOCK_SCREEN: dict[str, tuple[str, ...]] = {
|
|
18
|
+
"share_type": ("NS",),
|
|
19
|
+
"security_type": ("EQTY",),
|
|
20
|
+
"security_sub_type": ("COM",),
|
|
21
|
+
"us_inc_flg": ("Y",),
|
|
22
|
+
"issuer_type": ("CORP", "ACOR"),
|
|
23
|
+
"primary_exch": ("N", "A", "Q"),
|
|
24
|
+
"conditional_type": ("RW",),
|
|
25
|
+
"trading_status_flg": ("A",),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def common_stock_expression() -> pc.Expression:
|
|
30
|
+
"""Arrow filter for :data:`COMMON_STOCK_SCREEN`."""
|
|
31
|
+
parts = [pc.field(col).isin(list(values)) for col, values in COMMON_STOCK_SCREEN.items()]
|
|
32
|
+
return reduce(lambda a, b: a & b, parts)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@register_adapter
|
|
36
|
+
class StockAdapter(Adapter):
|
|
37
|
+
"""Options:
|
|
38
|
+
|
|
39
|
+
``common_only``
|
|
40
|
+
Keep only ordinary common shares (see :data:`COMMON_STOCK_SCREEN`),
|
|
41
|
+
the standard universe for cross-sectional equity research.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
name: ClassVar[str] = "us_stocks"
|
|
45
|
+
options: ClassVar[dict[str, Any]] = {"common_only": False}
|
|
46
|
+
|
|
47
|
+
def row_filter(self, spec: DatasetSpec, options: dict[str, Any]) -> pc.Expression | None:
|
|
48
|
+
return common_stock_expression() if options["common_only"] else None
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Small, dependency-free research helpers: portfolio sorts and performance stats."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def quantile_portfolios(
|
|
10
|
+
df: pd.DataFrame,
|
|
11
|
+
signal: str,
|
|
12
|
+
ret: str = "ret",
|
|
13
|
+
date: str = "date",
|
|
14
|
+
n: int = 10,
|
|
15
|
+
weight: str | None = None,
|
|
16
|
+
min_obs: int = 20,
|
|
17
|
+
) -> pd.DataFrame:
|
|
18
|
+
"""Sort assets into ``n`` quantiles of ``signal`` each date and average their returns.
|
|
19
|
+
|
|
20
|
+
``df`` holds one row per asset and date, where ``signal`` is known at
|
|
21
|
+
``date`` and ``ret`` is the return over the *next* holding period (shift
|
|
22
|
+
returns yourself so there is no look-ahead). Returns a wide frame indexed
|
|
23
|
+
by date with columns ``1..n`` and ``long_short`` (quantile n minus 1).
|
|
24
|
+
``weight`` (e.g. lagged market cap) gives value-weighted portfolios.
|
|
25
|
+
Dates with fewer than ``min_obs`` valid rows are skipped.
|
|
26
|
+
"""
|
|
27
|
+
cols = [date, signal, ret] + ([weight] if weight else [])
|
|
28
|
+
data = df[cols].dropna()
|
|
29
|
+
counts = data.groupby(date)[signal].transform("size")
|
|
30
|
+
data = data[counts >= max(min_obs, n)]
|
|
31
|
+
if data.empty:
|
|
32
|
+
return pd.DataFrame(columns=[*range(1, n + 1), "long_short"])
|
|
33
|
+
ranks = data.groupby(date)[signal].rank(method="first")
|
|
34
|
+
sizes = data.groupby(date)[signal].transform("size")
|
|
35
|
+
data = data.assign(q=np.ceil(ranks * n / sizes).astype(int).clip(1, n))
|
|
36
|
+
if weight:
|
|
37
|
+
w = data[weight].clip(lower=0)
|
|
38
|
+
num = (data[ret] * w).groupby([data[date], data["q"]]).sum()
|
|
39
|
+
den = w.groupby([data[date], data["q"]]).sum()
|
|
40
|
+
port = (num / den).unstack("q")
|
|
41
|
+
else:
|
|
42
|
+
port = data.groupby([date, "q"])[ret].mean().unstack("q")
|
|
43
|
+
port.columns = [int(c) for c in port.columns]
|
|
44
|
+
port["long_short"] = port[n] - port[1]
|
|
45
|
+
port.columns.name = None
|
|
46
|
+
return port
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def performance_summary(
|
|
50
|
+
returns: pd.Series | pd.DataFrame, periods_per_year: int = 252
|
|
51
|
+
) -> pd.DataFrame:
|
|
52
|
+
"""Annualized return, volatility, Sharpe ratio (zero risk-free rate), max drawdown, hit rate."""
|
|
53
|
+
frame = returns.to_frame() if isinstance(returns, pd.Series) else returns
|
|
54
|
+
rows = {}
|
|
55
|
+
for name, r in frame.items():
|
|
56
|
+
r = r.dropna()
|
|
57
|
+
if r.empty:
|
|
58
|
+
continue
|
|
59
|
+
wealth = (1 + r).cumprod()
|
|
60
|
+
years = len(r) / periods_per_year
|
|
61
|
+
ann_ret = wealth.iloc[-1] ** (1 / years) - 1 if years > 0 else np.nan
|
|
62
|
+
vol = r.std(ddof=1) * np.sqrt(periods_per_year)
|
|
63
|
+
rows[name] = {
|
|
64
|
+
"ann_return": ann_ret,
|
|
65
|
+
"ann_vol": vol,
|
|
66
|
+
"sharpe": r.mean() / r.std(ddof=1) * np.sqrt(periods_per_year)
|
|
67
|
+
if r.std(ddof=1) > 0
|
|
68
|
+
else np.nan,
|
|
69
|
+
"max_drawdown": (wealth / wealth.cummax() - 1).min(),
|
|
70
|
+
"hit_rate": (r > 0).mean(),
|
|
71
|
+
"n_periods": len(r),
|
|
72
|
+
}
|
|
73
|
+
return pd.DataFrame(rows).T
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""Build prepared, partitioned parquet files from raw source files.
|
|
2
|
+
|
|
3
|
+
This is maintainer tooling: it turns the raw downloads (WRDS parquet/CSV,
|
|
4
|
+
vendor CSVs) into the standard layout the client reads::
|
|
5
|
+
|
|
6
|
+
<out>/<market>/<asset>/<name>/year=2015/part-0.parquet
|
|
7
|
+
<out>/<market>/<asset>/<name>/year=2015/month=08/part-0.parquet
|
|
8
|
+
<out>/<market>/<asset>/<name>/data.parquet (unpartitioned)
|
|
9
|
+
<out>/manifest.json
|
|
10
|
+
|
|
11
|
+
Everything is driven by the registry YAML: which raw files to read, how to
|
|
12
|
+
rename and type each column, how to partition and sort. Raw files are only
|
|
13
|
+
ever read, never modified.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import csv
|
|
19
|
+
import datetime as dt
|
|
20
|
+
import logging
|
|
21
|
+
import re
|
|
22
|
+
import shutil
|
|
23
|
+
from collections.abc import Iterable, Iterator
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
import pyarrow as pa
|
|
29
|
+
import pyarrow.compute as pc
|
|
30
|
+
import pyarrow.csv as pacsv
|
|
31
|
+
import pyarrow.dataset as ds
|
|
32
|
+
import pyarrow.parquet as pq
|
|
33
|
+
|
|
34
|
+
from .errors import QRDError
|
|
35
|
+
from .links import Enricher, make_enricher
|
|
36
|
+
from .manifest import Manifest, build_manifest
|
|
37
|
+
from .registry import DatasetSpec, Registry, default_registry
|
|
38
|
+
|
|
39
|
+
log = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
ROW_GROUP_SIZE = 131_072
|
|
42
|
+
COMPRESSION = "zstd"
|
|
43
|
+
COMPRESSION_LEVEL = 6
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# --------------------------------------------------------------------- sources
|
|
47
|
+
def source_files(spec: DatasetSpec, source_root: Path) -> list[Path]:
|
|
48
|
+
"""Resolve the raw files for ``spec`` (``build.source`` may hold globs)."""
|
|
49
|
+
files: list[Path] = []
|
|
50
|
+
for pattern in spec.build.source:
|
|
51
|
+
matches = sorted(source_root.glob(pattern))
|
|
52
|
+
if not matches:
|
|
53
|
+
raise QRDError(f"{spec.id}: no source files match {pattern!r} under {source_root}")
|
|
54
|
+
files += matches
|
|
55
|
+
return files
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _read_csv(path: Path, spec: DatasetSpec) -> pa.Table:
|
|
59
|
+
# Read every column as text and convert explicitly later. This keeps
|
|
60
|
+
# identifiers such as gvkey ("001004") and CUSIPs intact.
|
|
61
|
+
with path.open(encoding="utf-8-sig", newline="") as fh:
|
|
62
|
+
header = next(csv.reader(fh))
|
|
63
|
+
convert = pacsv.ConvertOptions(
|
|
64
|
+
column_types={name: pa.string() for name in header},
|
|
65
|
+
strings_can_be_null=True,
|
|
66
|
+
null_values=[""],
|
|
67
|
+
)
|
|
68
|
+
table = pacsv.read_csv(path, convert_options=convert)
|
|
69
|
+
pattern = spec.build.options.get("id_from_filename")
|
|
70
|
+
if pattern:
|
|
71
|
+
if spec.id_column is None:
|
|
72
|
+
raise QRDError(f"{spec.id}: id_from_filename needs an id_column")
|
|
73
|
+
m = re.search(pattern, path.name)
|
|
74
|
+
if not m:
|
|
75
|
+
raise QRDError(f"{spec.id}: {path.name} does not match id_from_filename {pattern!r}")
|
|
76
|
+
target = spec.column(spec.id_column).source_name
|
|
77
|
+
table = table.append_column(target, pa.array([m.group("id")] * table.num_rows, pa.string()))
|
|
78
|
+
return table
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def read_source(spec: DatasetSpec, source_root: Path) -> ds.Dataset:
|
|
82
|
+
"""Open the raw source as an Arrow dataset (CSV files are read eagerly)."""
|
|
83
|
+
files = source_files(spec, source_root)
|
|
84
|
+
if spec.build.format == "parquet":
|
|
85
|
+
return ds.dataset([str(f) for f in files], format="parquet")
|
|
86
|
+
if spec.build.format == "csv":
|
|
87
|
+
tables = [_read_csv(f, spec) for f in files]
|
|
88
|
+
return ds.dataset(pa.concat_tables(tables, promote_options="default"))
|
|
89
|
+
raise QRDError(f"{spec.id}: unsupported source format {spec.build.format!r}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------ conversion
|
|
93
|
+
def _to_naive_or_zoned(
|
|
94
|
+
arr: pa.ChunkedArray, target: pa.DataType, tz: str | None
|
|
95
|
+
) -> pa.ChunkedArray:
|
|
96
|
+
"""Convert a timestamp array to ``target``, localising naive values to ``tz``."""
|
|
97
|
+
src_tz = getattr(arr.type, "tz", None)
|
|
98
|
+
tgt_tz = getattr(target, "tz", None)
|
|
99
|
+
if src_tz is None and tgt_tz is not None:
|
|
100
|
+
arr = pc.assume_timezone(arr, timezone=tz or "UTC")
|
|
101
|
+
return arr.cast(target)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def convert_column(
|
|
105
|
+
arr: pa.ChunkedArray, target: pa.DataType, options: dict[str, Any]
|
|
106
|
+
) -> pa.ChunkedArray:
|
|
107
|
+
"""Convert a raw column to its registry type."""
|
|
108
|
+
if pa.types.is_dictionary(arr.type):
|
|
109
|
+
arr = arr.cast(arr.type.value_type)
|
|
110
|
+
if arr.type == target:
|
|
111
|
+
return arr
|
|
112
|
+
src = arr.type
|
|
113
|
+
tz = options.get("timezone")
|
|
114
|
+
if pa.types.is_string(src) or pa.types.is_large_string(src):
|
|
115
|
+
if pa.types.is_date32(target):
|
|
116
|
+
return pc.strptime(arr, format="%Y-%m-%d", unit="s").cast(pa.date32())
|
|
117
|
+
if pa.types.is_timestamp(target):
|
|
118
|
+
fmt = options.get("timestamp_format", "%Y-%m-%d %H:%M:%S")
|
|
119
|
+
return _to_naive_or_zoned(pc.strptime(arr, format=fmt, unit="us"), target, tz)
|
|
120
|
+
if pa.types.is_integer(target):
|
|
121
|
+
as_float = arr.cast(pa.float64())
|
|
122
|
+
return as_float.cast(target) # safe cast: fails on fractional values
|
|
123
|
+
return arr.cast(target)
|
|
124
|
+
if pa.types.is_timestamp(src):
|
|
125
|
+
if pa.types.is_date32(target):
|
|
126
|
+
return arr.cast(pa.date32())
|
|
127
|
+
if pa.types.is_timestamp(target):
|
|
128
|
+
return _to_naive_or_zoned(arr, target, tz)
|
|
129
|
+
return arr.cast(target)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def standardize(raw: pa.Table, spec: DatasetSpec, enrich: Enricher | None = None) -> pa.Table:
|
|
133
|
+
"""Select, rename, type, filter, enrich and sort raw columns into the registry schema."""
|
|
134
|
+
opts = spec.build.options
|
|
135
|
+
columns: dict[str, pa.ChunkedArray] = {}
|
|
136
|
+
for col in spec.columns:
|
|
137
|
+
if col.derived: # filled in by the enrichment step below
|
|
138
|
+
columns[col.name] = pa.chunked_array([pa.nulls(raw.num_rows, col.arrow_type)])
|
|
139
|
+
continue
|
|
140
|
+
if col.source_name not in raw.column_names:
|
|
141
|
+
raise QRDError(f"{spec.id}: source column {col.source_name!r} not found")
|
|
142
|
+
arr = raw.column(col.source_name)
|
|
143
|
+
for token in opts.get("null_tokens", {}).get(col.source_name, []):
|
|
144
|
+
arr = pc.if_else(pc.equal(arr, token), pa.scalar(None, arr.type), arr)
|
|
145
|
+
arr = convert_column(arr, col.arrow_type, opts)
|
|
146
|
+
scale = opts.get("scale", {}).get(col.name)
|
|
147
|
+
if scale is not None:
|
|
148
|
+
arr = pc.multiply(arr, pa.scalar(float(scale))).cast(col.arrow_type)
|
|
149
|
+
columns[col.name] = arr
|
|
150
|
+
table = pa.table(columns, schema=spec.schema)
|
|
151
|
+
|
|
152
|
+
if spec.id_column and (spec.build.include_ids or spec.build.exclude_ids):
|
|
153
|
+
ids = table.column(spec.id_column)
|
|
154
|
+
if spec.build.include_ids:
|
|
155
|
+
table = table.filter(pc.is_in(ids, pa.array(spec.build.include_ids)))
|
|
156
|
+
if spec.build.exclude_ids:
|
|
157
|
+
table = table.filter(pc.invert(pc.is_in(ids, pa.array(spec.build.exclude_ids))))
|
|
158
|
+
if enrich is not None:
|
|
159
|
+
table = enrich(table).select(spec.column_names).cast(spec.schema)
|
|
160
|
+
if spec.sort_by and table.num_rows:
|
|
161
|
+
table = table.sort_by([(c, "ascending") for c in spec.sort_by])
|
|
162
|
+
return table
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ------------------------------------------------------------------ partitions
|
|
166
|
+
@dataclass(frozen=True)
|
|
167
|
+
class Period:
|
|
168
|
+
year: int
|
|
169
|
+
month: int | None = None
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def start(self) -> dt.date:
|
|
173
|
+
return dt.date(self.year, self.month or 1, 1)
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def stop(self) -> dt.date: # exclusive
|
|
177
|
+
if self.month is None:
|
|
178
|
+
return dt.date(self.year + 1, 1, 1)
|
|
179
|
+
return dt.date(self.year + (self.month == 12), self.month % 12 + 1, 1)
|
|
180
|
+
|
|
181
|
+
def relpath(self) -> str:
|
|
182
|
+
if self.month is None:
|
|
183
|
+
return f"year={self.year}/part-0.parquet"
|
|
184
|
+
return f"year={self.year}/month={self.month:02d}/part-0.parquet"
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def periods_between(first: dt.date, last: dt.date, partitioning: str) -> list[Period]:
|
|
188
|
+
out: list[Period] = []
|
|
189
|
+
for year in range(first.year, last.year + 1):
|
|
190
|
+
if partitioning == "year":
|
|
191
|
+
out.append(Period(year))
|
|
192
|
+
else:
|
|
193
|
+
lo = first.month if year == first.year else 1
|
|
194
|
+
hi = last.month if year == last.year else 12
|
|
195
|
+
out += [Period(year, m) for m in range(lo, hi + 1)]
|
|
196
|
+
return out
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _as_date(value: Any) -> dt.date:
|
|
200
|
+
if isinstance(value, dt.datetime):
|
|
201
|
+
return value.date()
|
|
202
|
+
if isinstance(value, dt.date):
|
|
203
|
+
return value
|
|
204
|
+
return dt.date.fromisoformat(str(value)[:10])
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _bound(value: dt.date, typ: pa.DataType) -> pa.Scalar:
|
|
208
|
+
if pa.types.is_timestamp(typ):
|
|
209
|
+
tz = dt.timezone.utc if typ.tz else None
|
|
210
|
+
return pa.scalar(dt.datetime(value.year, value.month, value.day, tzinfo=tz), type=typ)
|
|
211
|
+
return pa.scalar(value, type=typ)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def iter_partitions(
|
|
215
|
+
spec: DatasetSpec, source: ds.Dataset, enrich: Enricher | None = None
|
|
216
|
+
) -> Iterator[tuple[str, pa.Table]]:
|
|
217
|
+
"""Yield ``(relative path, standardized table)`` for each output file."""
|
|
218
|
+
if spec.partitioning == "none":
|
|
219
|
+
yield "data.parquet", standardize(source.to_table(), spec, enrich)
|
|
220
|
+
return
|
|
221
|
+
|
|
222
|
+
assert spec.date_column is not None
|
|
223
|
+
raw_date = spec.column(spec.date_column).source_name
|
|
224
|
+
raw_type = source.schema.field(raw_date).type
|
|
225
|
+
if pa.types.is_date(raw_type) or pa.types.is_timestamp(raw_type):
|
|
226
|
+
# Large typed sources: scan one period at a time to bound memory.
|
|
227
|
+
mm = pc.min_max(source.to_table(columns=[raw_date]).column(0))
|
|
228
|
+
for period in periods_between(
|
|
229
|
+
_as_date(mm["min"].as_py()), _as_date(mm["max"].as_py()), spec.partitioning
|
|
230
|
+
):
|
|
231
|
+
expr = (pc.field(raw_date) >= _bound(period.start, raw_type)) & (
|
|
232
|
+
pc.field(raw_date) < _bound(period.stop, raw_type)
|
|
233
|
+
)
|
|
234
|
+
table = standardize(source.to_table(filter=expr), spec, enrich)
|
|
235
|
+
if table.num_rows:
|
|
236
|
+
yield period.relpath(), table
|
|
237
|
+
return
|
|
238
|
+
|
|
239
|
+
# Small text sources: standardize once, then split in memory.
|
|
240
|
+
table = standardize(source.to_table(), spec, enrich)
|
|
241
|
+
dates = table.column(spec.date_column)
|
|
242
|
+
years = pc.year(dates)
|
|
243
|
+
months = pc.month(dates) if spec.partitioning == "year_month" else None
|
|
244
|
+
month_list = months.to_pylist() if months is not None else [None] * len(years)
|
|
245
|
+
keys = sorted(set(zip(years.to_pylist(), month_list, strict=True)))
|
|
246
|
+
for year, month in keys:
|
|
247
|
+
mask = pc.equal(years, year)
|
|
248
|
+
if months is not None:
|
|
249
|
+
mask = pc.and_(mask, pc.equal(months, month))
|
|
250
|
+
yield Period(year, month).relpath(), table.filter(mask)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def write_table(table: pa.Table, path: Path, row_group_size: int = ROW_GROUP_SIZE) -> None:
|
|
254
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
255
|
+
pq.write_table(
|
|
256
|
+
table,
|
|
257
|
+
path,
|
|
258
|
+
compression=COMPRESSION,
|
|
259
|
+
compression_level=COMPRESSION_LEVEL,
|
|
260
|
+
row_group_size=row_group_size,
|
|
261
|
+
write_statistics=True,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def build_dataset(spec: DatasetSpec, source_root: str | Path, out_root: str | Path) -> list[Path]:
|
|
266
|
+
"""Build one dataset. Its previous output folder is replaced."""
|
|
267
|
+
source_root, out_root = Path(source_root).resolve(), Path(out_root).resolve()
|
|
268
|
+
if (
|
|
269
|
+
out_root == source_root
|
|
270
|
+
or source_root in out_root.parents
|
|
271
|
+
or out_root in source_root.parents
|
|
272
|
+
):
|
|
273
|
+
raise QRDError("out_root and source_root must not contain each other")
|
|
274
|
+
target = out_root / spec.id
|
|
275
|
+
tmp = target.with_name(target.name + ".tmp")
|
|
276
|
+
if tmp.exists():
|
|
277
|
+
shutil.rmtree(tmp)
|
|
278
|
+
written: list[Path] = []
|
|
279
|
+
rows = 0
|
|
280
|
+
for rel, table in iter_partitions(
|
|
281
|
+
spec, read_source(spec, source_root), make_enricher(spec, source_root)
|
|
282
|
+
):
|
|
283
|
+
write_table(table, tmp / rel, int(spec.build.options.get("row_group_size", ROW_GROUP_SIZE)))
|
|
284
|
+
written.append(target / rel)
|
|
285
|
+
rows += table.num_rows
|
|
286
|
+
if not written:
|
|
287
|
+
raise QRDError(f"{spec.id}: the source produced no rows")
|
|
288
|
+
if target.exists():
|
|
289
|
+
shutil.rmtree(target)
|
|
290
|
+
tmp.rename(target)
|
|
291
|
+
log.info("built %s: %d files, %d rows", spec.id, len(written), rows)
|
|
292
|
+
return written
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def build_all(
|
|
296
|
+
source_root: str | Path,
|
|
297
|
+
out_root: str | Path,
|
|
298
|
+
datasets: Iterable[str] | None = None,
|
|
299
|
+
registry: Registry | None = None,
|
|
300
|
+
version: str = "dev",
|
|
301
|
+
) -> Manifest:
|
|
302
|
+
"""Build the selected datasets (all by default) and write ``manifest.json``.
|
|
303
|
+
|
|
304
|
+
The manifest always covers every dataset present under ``out_root``, so
|
|
305
|
+
datasets can be rebuilt one at a time.
|
|
306
|
+
"""
|
|
307
|
+
registry = registry or default_registry()
|
|
308
|
+
specs = [registry.get(d) for d in datasets] if datasets else list(registry)
|
|
309
|
+
out = Path(out_root)
|
|
310
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
311
|
+
for spec in specs:
|
|
312
|
+
log.info("building %s", spec.id)
|
|
313
|
+
build_dataset(spec, source_root, out)
|
|
314
|
+
manifest = build_manifest(out, registry, version=version)
|
|
315
|
+
manifest.save(out)
|
|
316
|
+
return manifest
|