firstrate-data 0.1.0__py3-none-any.whl

firstrate_data/__init__.py

@@ -0,0 +1,38 @@
+from firstrate_data.domain import (
+    Adjustment,
+    AssetType,
+    BarType,
+    ContinuousFuturesAdjustment,
+    ContractFiles,
+    DelistedArchive,
+    DelistedUpdate,
+    EquitiesAdjustment,
+    OtherData,
+    Period,
+    TickerListing,
+    Timeframe,
+    TradingHours,
+    Unadjusted,
+)
+from firstrate_data.download.client import Client
+from firstrate_data.store.store import Ingested, Store
+
+__all__ = [
+    "Adjustment",
+    "AssetType",
+    "BarType",
+    "Client",
+    "ContinuousFuturesAdjustment",
+    "ContractFiles",
+    "DelistedArchive",
+    "DelistedUpdate",
+    "EquitiesAdjustment",
+    "Ingested",
+    "OtherData",
+    "Period",
+    "Store",
+    "TickerListing",
+    "Timeframe",
+    "TradingHours",
+    "Unadjusted",
+]

firstrate_data/config.py

@@ -0,0 +1,49 @@
+import os
+from pathlib import Path
+
+from dotenv import find_dotenv, load_dotenv
+
+DEFAULT_BASE_URL = "https://firstratedata.com/api"
+_BASE_URL_KEY = "FIRSTRATE_BASE_URL"
+
+
+class MissingSettingError(KeyError):
+    pass
+
+
+def _find_and_load_dotenv_cwd() -> None:
+    load_dotenv(find_dotenv(usecwd=True))
+
+
+def _raise_if_missing(env_variable: str) -> str:
+    _find_and_load_dotenv_cwd()
+    value = os.getenv(env_variable)
+    if value:
+        return value
+    msg = f"set {env_variable} environment variable"
+    raise MissingSettingError(msg)
+
+
+# ----------------------------------------------------------------------
+# Store config
+# ----------------------------------------------------------------------
+
+
+def firstrate_data_path() -> Path:
+    return Path(_raise_if_missing("FIRSTRATE_DATA_PATH"))
+
+
+# ----------------------------------------------------------------------
+# Client config
+# ----------------------------------------------------------------------
+
+
+def firstrate_user_id() -> str:
+    return _raise_if_missing("FIRSTRATE_USERID")
+
+
+def base_url() -> str:
+    """Return the vendor's API root."""
+    # FIRSTRATE_BASE_URL lets a test point the client at a stand-in server.
+    _find_and_load_dotenv_cwd()
+    return os.getenv(_BASE_URL_KEY, DEFAULT_BASE_URL)

firstrate_data/domain/__init__.py

@@ -0,0 +1,33 @@
+from firstrate_data.domain.bar_type import BarType
+from firstrate_data.domain.enums import (
+    Adjustment,
+    AssetType,
+    ContinuousFuturesAdjustment,
+    ContractFiles,
+    DelistedArchive,
+    DelistedUpdate,
+    EquitiesAdjustment,
+    OtherData,
+    Period,
+    Timeframe,
+    TradingHours,
+    Unadjusted,
+)
+from firstrate_data.domain.ticker_listing import TickerListing
+
+__all__ = [
+    "Adjustment",
+    "AssetType",
+    "BarType",
+    "ContinuousFuturesAdjustment",
+    "ContractFiles",
+    "DelistedArchive",
+    "DelistedUpdate",
+    "EquitiesAdjustment",
+    "OtherData",
+    "Period",
+    "TickerListing",
+    "Timeframe",
+    "TradingHours",
+    "Unadjusted",
+]

firstrate_data/domain/bar_type.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass, fields, replace
+from typing import Self
+
+from firstrate_data.domain.enums import Adjustment, AssetType, Timeframe
+
+
+@dataclass(frozen=True, slots=True)
+class BarType:
+    # declared in the same order used as the store's paths.
+    asset_type: AssetType | None = None
+    adjustment: Adjustment | None = None
+    timeframe: Timeframe | None = None
+    ticker: str | None = None
+
+    def from_ticker(self, ticker: str | None) -> Self:
+        """Return a copy of this `BarType` with `ticker` swapped in."""
+        return replace(self, ticker=ticker)
+
+    def levels(self) -> dict[str, str | None]:
+        """Every level of the tree, in the tree's order. None where unstated."""
+        return {
+            f.name: None if (value := getattr(self, f.name)) is None else str(value)
+            for f in fields(self)
+        }
+
+    def stated_levels(self) -> dict[str, str]:
+        """The levels this bar type names, in the tree's order."""
+        return {
+            name: value for name, value in self.levels().items() if value is not None
+        }
+
+    @classmethod
+    def fields(cls) -> list[str]:
+        return [f.name for f in fields(cls)]

firstrate_data/domain/enums.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from enum import StrEnum, auto
+from typing import Self
+
+
+class AssetType(StrEnum):
+    STOCK = auto()
+    ETF = auto()
+    INDEX = auto()
+    FUTURES = auto()
+    CRYPTO = auto()
+    FX = auto()
+    OPTIONS = auto()
+
+    def timezone(self) -> str:
+        if self is AssetType.CRYPTO:
+            return "UTC"
+        return "America/New_York"
+
+
+# adjustments ----------------------------------------------------------
+
+
+class Adjustment(StrEnum):
+    @property
+    def changes_past(self) -> bool:
+        """Return `True` for all adjustments which are not `UNADJUSTED`.
+
+        A series holding these kind of adjustments need to be fully replaced
+        when a new adjustment comes.
+        """
+        return self.name != "UNADJUSTED"
+
+
+class Unadjusted(Adjustment):
+    UNADJUSTED = "UNADJUSTED"
+
+
+class EquitiesAdjustment(Adjustment):
+    SPLIT = "adj_split"
+    SPLIT_AND_DIVIDEND = "adj_splitdiv"
+    UNADJUSTED = "UNADJUSTED"
+
+
+class ContinuousFuturesAdjustment(Adjustment):
+    RATIO = "contin_adj_ratio"
+    ABSOLUTE = "contin_adj_absolute"
+    UNADJUSTED = "contin_UNadj"
+
+
+class Timeframe(StrEnum):
+    MIN_1 = "1min"
+    MIN_5 = "5min"
+    MIN_30 = "30min"
+    HOUR_1 = "1hour"
+    DAY_1 = "1day"
+
+    def is_higher_than(self, other_timeframe: Self) -> bool:
+        order = list(Timeframe)
+        return order.index(self) > order.index(other_timeframe)
+
+
+# ----------------------------------------------------------------------
+
+
+class TradingHours(StrEnum):
+    ALL = auto()
+    REGULAR = auto()
+
+
+class Period(StrEnum):
+    FULL = auto()
+    MONTH = auto()
+    WEEK = auto()
+    DAY = auto()
+
+
+class OtherData(StrEnum):
+    SPLITS = auto()
+    DIVIDENDS = auto()
+    COMPANY_PROFILES = auto()
+    CONTRACT_DATES = "contin_audit"
+
+
+class ContractFiles(StrEnum):
+    """Which half of the individual-contract dataset a request names.
+
+    The vendor splits it in two: the archive freezes everything up to 2025,
+    and the update carries the contracts trading from 2026, refreshed daily.
+    They name different contracts, so neither contains the other.
+    """
+
+    ARCHIVE = auto()
+    UPDATE = auto()
+
+
+# ----------------------------------------------------------------------
+# delisted data
+# ----------------------------------------------------------------------
+
+
+class DelistedArchive(StrEnum):
+    """One slice of the pre-2026 delisted history, downloaded on its own."""
+
+    # the docs list accepted values 1-5 but say "the four historical archives"
+    # in the same breath. The values win: a fifth archive that doesn't exist
+    # fails on the request, while omitting one that does loses data unseen.
+    ARCHIVE_1 = "1"
+    ARCHIVE_2 = "2"
+    ARCHIVE_3 = "3"
+    ARCHIVE_4 = "4"
+    ARCHIVE_5 = "5"
+
+
+class DelistedUpdate(StrEnum):
+    """The 2026+ delisted data, refreshed each Sunday at 11 PM, Eastern."""
+
+    WEEK = auto()
+    YEAR = auto()

firstrate_data/domain/ticker_listing.py

@@ -0,0 +1,78 @@
+import csv
+from dataclasses import dataclass
+from datetime import date
+from typing import Self
+
+_DELISTED_SUFFIX = "-DELISTED"
+
+
+@dataclass(frozen=True, slots=True)
+class TickerListing:
+    # example structure of a ticker listing file: https://firstratedata.com/api_ticker_files/etf_ticker_dates_listing.txt
+    # Ticker,Name,First Date,Last Date
+    # AAA,Listed Funds Trust Aaf First Priority Clo Bond ETF,2020-09-09,2026-08-26
+    ticker: str
+    full_name: str
+    start_date: date
+    end_date: date
+    is_delisted: bool
+
+    @classmethod
+    def from_csv(cls, csv_body: str) -> list[Self]:
+        # get one TickerListing for each row of file_body; the vendor heads the
+        # file with "Ticker,Name,First Date,Last Date"
+        rows = csv.reader(csv_body.splitlines())
+        listed_tickers = [
+            cls._from_row(row)
+            for row in rows
+            if any(row) and row[0].strip().casefold() != "ticker"
+        ]
+        if not listed_tickers:
+            msg = f"ticker_listing answered with no rows: {csv_body}"
+            raise ValueError(msg)
+        return listed_tickers
+
+    # ------------------------------------------------------------------
+    # helpers
+    # ------------------------------------------------------------------
+
+    _ROW_FIELDS = 4
+
+    @classmethod
+    def _from_row(cls, row: list[str]) -> Self:
+        if len(row) < cls._ROW_FIELDS:
+            msg = (
+                f"ticker_listing row {','.join(row)!r} is not "
+                f"{{ticker}},{{name}},{{startDate}},{{endDate}}"
+            )
+            raise ValueError(
+                msg,
+            )
+
+        # from both ends rather than by position: an unquoted comma in a name
+        # ("Dow Jones Industrial Average, Total Return") splits into extra fields.
+        # Those fields belong to the name.
+        symbol, *full_name, start_date, end_date = row
+        is_delisted = symbol.endswith(_DELISTED_SUFFIX)
+        return cls(
+            symbol.removesuffix(_DELISTED_SUFFIX),
+            # strip the name whole rather than field by field: the space after
+            # the comma in "S&P 500, Total Return" belongs to the name
+            ",".join(full_name).strip(),
+            cls._listing_date_from_str(start_date.strip(), row),
+            cls._listing_date_from_str(end_date.strip(), row),
+            is_delisted,
+        )
+
+    @staticmethod
+    def _listing_date_from_str(field: str, row: list[str]) -> date:
+        try:
+            return date.fromisoformat(field)
+        except ValueError as unreadable:
+            msg = (
+                f"ticker_listing row {','.join(row)!r} carries {field!r} "
+                "where a date belongs"
+            )
+            raise ValueError(
+                msg,
+            ) from unreadable

firstrate_data/download/bundles.py

@@ -0,0 +1,201 @@
+from collections.abc import Callable, Iterable, Iterator
+from dataclasses import dataclass
+from string import ascii_uppercase
+
+from firstrate_data.domain import AssetType, BarType
+from firstrate_data.domain.enums import (
+    ContinuousFuturesAdjustment,
+    ContractFiles,
+    DelistedArchive,
+    EquitiesAdjustment,
+    OtherData,
+    Period,
+    Timeframe,
+    Unadjusted,
+)
+from firstrate_data.download.requests import (
+    BarsRequest,
+    ContractBarsRequest,
+    DelistedBarsRequest,
+    NotOfferedError,
+    OtherDataRequest,
+    Request,
+)
+
+
+@dataclass(frozen=True)
+class BundleConfig:
+    """Enough for Indices, FX and Crypto.
+
+    [Index docs](https://firstratedata.com/_readme/index.txt)
+    [FX docs](https://firstratedata.com/_readme/fx.txt)
+    [Crypto docs](https://firstratedata.com/_readme/crypto.txt)
+    """
+
+    asset_type: AssetType
+    # None -> all the available timeframes
+    timeframes: Iterable[Timeframe] | None
+
+
+@dataclass(frozen=True)
+class EquitiesBundleConfig(BundleConfig):
+    """Use this for ETFs.
+
+    [docs](https://firstratedata.com/_readme/etf.txt).
+    """
+
+    adjustment: EquitiesAdjustment
+    # None behaves like ["A", ..., "Z"]
+    ticker_range: Iterable[str] | None
+    include_splits: bool
+    include_dividends: bool
+
+
+UNADJUSTED_DELISTED_TIMEFRAMES = (Timeframe.DAY_1, Timeframe.MIN_1)
+
+
+@dataclass(frozen=True)
+class StocksBundleConfig(EquitiesBundleConfig):
+    """[Stocks docs](https://firstratedata.com/_readme/stock.txt)."""
+
+    include_company_profiles: bool
+    # false -> no archives are downloaded, true -> all of them,
+    # otherwise pass an iterable to select them.
+    include_delisted_archives: bool | Iterable[DelistedArchive]
+
+
+@dataclass(frozen=True)
+class FuturesBundleConfig(BundleConfig):
+    """[Future docs](https://firstratedata.com/_readme/futures.txt)."""
+
+    adjustment: ContinuousFuturesAdjustment
+    include_individual_contracts: bool | Iterable[Timeframe]
+    # Downloads a folder with one .txt per contract.
+    # Each row follows the YYYY-MM-DD,CONTRACT_CODE format.
+    # Useful to know rollover dates used by FirstRate in contract construction.
+    include_contract_dates: bool
+
+
+def bundle_requests(config: BundleConfig) -> Iterator[Request]:
+    """Yield every request the bundle downloads, in download order."""
+    for timeframe in config.timeframes or tuple(Timeframe):
+        yield from _bar_requests(config, timeframe)
+    yield from _meta_requests(config)
+
+
+def _bar_requests(config: BundleConfig, timeframe: Timeframe) -> Iterator[Request]:
+    if isinstance(config, FuturesBundleConfig):
+        yield from _futures_requests(config, timeframe)
+    elif isinstance(config, EquitiesBundleConfig):
+        yield from _equities_requests(config, timeframe)
+    else:
+        # UNADJUSTED is the store's word for it, not the vendor's: index, FX
+        # and crypto carry no adjustment on the wire, and the bar type path
+        # names one at every level
+        yield from _offered(
+            lambda: BarsRequest(
+                BarType(
+                    config.asset_type,
+                    timeframe=timeframe,
+                    adjustment=Unadjusted.UNADJUSTED,
+                ),
+                Period.FULL,
+            ),
+        )
+
+
+def _equities_requests(
+    config: EquitiesBundleConfig,
+    timeframe: Timeframe,
+) -> Iterator[Request]:
+    bar_type = BarType(
+        config.asset_type,
+        timeframe=timeframe,
+        adjustment=config.adjustment,
+    )
+    for letter in config.ticker_range or ascii_uppercase:
+        yield from _offered(
+            lambda letter=letter: BarsRequest(
+                bar_type,
+                Period.FULL,
+                ticker_range=letter,
+            ),
+        )
+
+    if not isinstance(config, StocksBundleConfig):
+        return
+    for archive in _delisted_archives(selection=config.include_delisted_archives):
+        yield from _offered(
+            lambda archive=archive: DelistedBarsRequest(bar_type, selector=archive),
+        )
+
+
+def _futures_requests(
+    config: FuturesBundleConfig,
+    timeframe: Timeframe,
+) -> Iterator[Request]:
+    yield from _offered(
+        lambda: BarsRequest(
+            BarType(
+                AssetType.FUTURES,
+                timeframe=timeframe,
+                adjustment=config.adjustment,
+            ),
+            Period.FULL,
+        ),
+    )
+
+    if not _wants_contracts(timeframe, selection=config.include_individual_contracts):
+        return
+    # both halves name different contracts, so neither one contains the other
+    for half in ContractFiles:
+        yield ContractBarsRequest(
+            BarType(AssetType.FUTURES, timeframe=timeframe),
+            contract_files=half,
+        )
+
+
+def _meta_requests(config: BundleConfig) -> Iterator[Request]:
+    # ponytail: include_company_profiles is unserved -- the vendor documents no
+    # endpoint for it. Yield an OtherDataRequest here once one exists.
+    if isinstance(config, EquitiesBundleConfig):
+        if config.include_splits:
+            yield OtherDataRequest(config.asset_type, OtherData.SPLITS)
+        if config.include_dividends:
+            yield OtherDataRequest(config.asset_type, OtherData.DIVIDENDS)
+    if isinstance(config, FuturesBundleConfig) and config.include_contract_dates:
+        yield OtherDataRequest(AssetType.FUTURES, OtherData.CONTRACT_DATES)
+
+
+def _delisted_archives(
+    *,
+    selection: bool | Iterable[DelistedArchive],
+) -> tuple[DelistedArchive, ...]:
+    if selection is True:
+        return tuple(DelistedArchive)
+    if selection is False:
+        return ()
+    return tuple(selection)
+
+
+def _wants_contracts(
+    timeframe: Timeframe,
+    *,
+    selection: bool | Iterable[Timeframe],
+) -> bool:
+    if isinstance(selection, bool):
+        return selection
+    return timeframe in tuple(selection)
+
+
+def _offered(build: Callable[[], Request]) -> Iterator[Request]:
+    """Yield the request, or nothing when the vendor doesn't serve it.
+
+    A bundle names timeframes and adjustments across a whole universe, and the
+    vendor serves some pairs and not others. The pairs it refuses are dropped
+    so the rest of the bundle still comes down.
+    """
+    try:
+        yield build()
+    except NotOfferedError:
+        return