kuznets 1.0__py3-none-any.whl

kuznets/__init__.py

@@ -0,0 +1,74 @@
+from importlib.metadata import version
+from pathlib import Path
+import sys
+
+__version__ = version("kuznets")
+
+from kuznets.config import options
+from kuznets.data import (
+    DataReader,
+    Options,
+    get_data_alphavantage,
+    get_data_econdb,
+    get_data_famafrench,
+    get_data_fred,
+    get_data_moex,
+    get_data_quandl,
+    get_data_stooq,
+    get_data_tiingo,
+    get_data_yahoo,
+    get_data_yahoo_actions,
+    get_data_yahoo_fundamentals,
+    get_iex_data_tiingo,
+    get_nasdaq_symbols,
+    get_quote_yahoo,
+)
+
+PKG = Path(__file__).parent
+
+__all__ = [
+    "__version__",
+    "options",
+    "get_data_econdb",
+    "get_data_famafrench",
+    "get_data_yahoo",
+    "get_data_yahoo_actions",
+    "get_data_yahoo_fundamentals",
+    "get_quote_yahoo",
+    "get_nasdaq_symbols",
+    "get_data_quandl",
+    "get_data_moex",
+    "get_data_fred",
+    "get_data_stooq",
+    "DataReader",
+    "Options",
+    "get_data_tiingo",
+    "get_iex_data_tiingo",
+    "get_data_alphavantage",
+    "test",
+]
+
+
+def test(extra_args=None):
+    """
+    Run the test suite
+
+    Parameters
+    ----------
+    extra_args : {str, List[str]}
+        A string or list of strings to pass to pytest. Default is ["--only-stable",
+        "--skip-requires-api-key"]
+    """
+    try:
+        import pytest
+    except ImportError as err:
+        raise ImportError("Need pytest>=5.0.1 to run tests") from err
+    cmd = ["--only-stable", "--skip-requires-api-key"]
+    if extra_args:
+        if not isinstance(extra_args, list):
+            extra_args = [extra_args]
+        cmd = extra_args
+    cmd += [str(PKG)]
+    joined = " ".join(cmd)
+    print(f"running: pytest {joined}")
+    sys.exit(pytest.main(cmd))

kuznets/_output.py

@@ -0,0 +1,337 @@
+from collections.abc import Sequence
+import datetime
+import importlib.util
+
+import narwhals.stable.v2 as nw
+from narwhals.stable.v2.typing import IntoFrame
+import pandas as pd
+
+PANDAS = "pandas"
+
+# Canonical backend name -> (module probed for availability, pip extra that installs it). pandas is
+# absent here because it is a hard dependency and needs no availability check.
+_OPTIONAL_BACKENDS = {
+    "polars": ("polars", "polars"),
+    "pyarrow": ("pyarrow", "pyarrow"),
+    "dask": ("dask.dataframe", "dask"),
+}
+_ALIASES = {"arrow": "pyarrow"}
+
+
+def validate_output_type(output_type: str) -> str:
+    """Canonicalize an ``output_type`` value and verify its backend is importable.
+
+    Call before issuing any network request so an invalid value or missing backend fails fast.
+
+    Parameters
+    ----------
+    output_type : str
+        Requested output backend, case-insensitive. One of 'pandas', 'polars', 'pyarrow' (alias
+        'arrow'), or 'dask'.
+
+    Returns
+    -------
+    str
+        The canonical backend name.
+
+    Raises
+    ------
+    TypeError
+        If ``output_type`` is not a str.
+    ValueError
+        If ``output_type`` does not name a recognized backend.
+    ImportError
+        If the backend is recognized but its package is not installed.
+    """
+    if not isinstance(output_type, str):
+        raise TypeError(f"output_type must be a str, got {type(output_type).__name__}")
+    lowered = output_type.lower()
+    canonical = _ALIASES.get(lowered, lowered)
+    if canonical == PANDAS:
+        return canonical
+    if canonical not in _OPTIONAL_BACKENDS:
+        valid = ", ".join(repr(name) for name in sorted([PANDAS, *_OPTIONAL_BACKENDS, *_ALIASES]))
+        raise ValueError(f"output_type={output_type!r} is not supported; choose one of {valid}")
+    _require_backend(canonical)
+    return canonical
+
+
+def _require_backend(canonical: str) -> None:
+    """Raise a helpful ImportError if the package backing *canonical* is not installed."""
+    module, extra = _OPTIONAL_BACKENDS[canonical]
+    package = module.partition(".")[0]
+    try:
+        missing = importlib.util.find_spec(module) is None
+    except ModuleNotFoundError:
+        # find_spec on a dotted name (dask.dataframe) raises when the parent package is absent.
+        missing = True
+    if missing:
+        raise ImportError(
+            f"output_type={canonical!r} requires the optional dependency {package!r}. "
+            f"Install it with 'pip install {package}' or 'pip install kuznets[{extra}]'."
+        )
+
+
+def detach_index(df: pd.DataFrame) -> tuple[pd.DataFrame, list[str]]:
+    """Move a frame's index into ordinary columns.
+
+    An unnamed datetime-like index is named ``Date``; any other unnamed index is named ``index``;
+    unnamed MultiIndex levels get ``level_{position}`` names. The returned names let a pandas
+    presenter restore today's index via :func:`attach_index`.
+
+    Parameters
+    ----------
+    df : DataFrame
+        Frame whose index carries meaning (dates, symbols, dimension levels).
+
+    Returns
+    -------
+    tuple of (DataFrame, list of str)
+        The index-free frame and the column names the index became, in level order.
+    """
+    index = df.index
+    if isinstance(index, pd.MultiIndex):
+        names = [name if name is not None else f"level_{position}" for position, name in enumerate(index.names)]
+        index = index.set_names(names)
+    else:
+        if index.name is None:
+            fallback = "Date" if isinstance(index, pd.DatetimeIndex | pd.PeriodIndex) else "index"
+            index = index.rename(fallback)
+        names = [index.name]
+    return df.set_axis(index, axis=0).reset_index(), names
+
+
+def attach_index(df: pd.DataFrame, index_cols: list[str]) -> pd.DataFrame:
+    """Restore columns produced by :func:`detach_index` as the frame's index.
+
+    Parameters
+    ----------
+    df : DataFrame
+        Tidy frame containing every column named in ``index_cols``.
+    index_cols : list of str
+        Column names to promote, in level order.
+
+    Returns
+    -------
+    DataFrame
+        Frame indexed by ``index_cols`` (a MultiIndex when more than one name is given).
+    """
+    return df.set_index(index_cols[0] if len(index_cols) == 1 else index_cols)
+
+
+def from_pandas(df: pd.DataFrame, output_type: str):
+    """Convert a tidy pandas frame to the requested backend.
+
+    The tidy contract is enforced, not repaired: an index that carries data (MultiIndex, named
+    index, or datetime-like index) raises, because the narwhals backends disagree on whether to
+    keep or drop it -- callers must run :func:`detach_index` first. A leftover positional index
+    (e.g. non-contiguous after a boolean filter) is silently discarded. Non-string column labels
+    are cast to str, and all-null object columns to float64, since polars and pyarrow accept
+    neither.
+
+    Parameters
+    ----------
+    df : DataFrame
+        Tidy frame: data in columns only.
+    output_type : str
+        Canonical backend name from :func:`validate_output_type`.
+
+    Returns
+    -------
+    DataFrame or Table
+        ``df`` unchanged for 'pandas'; otherwise a native frame of the requested backend
+        (``polars.DataFrame``, ``pyarrow.Table``, or a dask collection).
+    """
+    if output_type == PANDAS:
+        return df
+    if isinstance(df.index, pd.MultiIndex) or isinstance(df.columns, pd.MultiIndex):
+        raise TypeError("from_pandas requires a tidy frame with no MultiIndex; call detach_index first")
+    if df.index.name is not None or isinstance(df.index, pd.DatetimeIndex | pd.PeriodIndex):
+        raise TypeError(
+            f"from_pandas would silently drop the meaningful index {df.index.name!r} "
+            f"({type(df.index).__name__}); call detach_index first"
+        )
+    prepared = df if isinstance(df.index, pd.RangeIndex) else df.reset_index(drop=True)
+    if not all(isinstance(label, str) for label in prepared.columns):
+        prepared = prepared.set_axis([str(label) for label in prepared.columns], axis=1)
+    all_null_object = [
+        name for name in prepared.columns if prepared[name].dtype == object and prepared[name].isna().all()
+    ]
+    if all_null_object:
+        prepared = prepared.astype(dict.fromkeys(all_null_object, "float64"))
+    frame = nw.from_native(prepared, eager_only=True)
+    try:
+        if output_type == "polars":
+            return frame.to_polars()
+        if output_type == "pyarrow":
+            return frame.to_arrow()
+        return frame.lazy(backend="dask").to_native()
+    except ModuleNotFoundError as exc:
+        # Catches transitive gaps find_spec cannot see, e.g. polars needing pyarrow for from_pandas.
+        _require_backend(output_type)
+        raise ImportError(f"output_type={output_type!r} conversion failed: {exc}") from exc
+
+
+def make_frame(records: list[dict] | dict[str, list], output_type: str, schema: dict | None = None):
+    """Build a native frame of the requested backend directly from parsed records.
+
+    Keys missing from a record become nulls; column order follows first appearance across the
+    records.
+
+    Parameters
+    ----------
+    records : list of dict or dict of str to list
+        Row records as parsed from a JSON or SDMX payload, or ready-made columns.
+    output_type : str
+        Canonical backend name from :func:`validate_output_type`.
+    schema : dict mapping str to narwhals dtype, optional
+        Column dtypes to impose, guaranteeing identical schemas on every backend. When omitted each
+        backend infers its own dtypes. Default None.
+
+    Returns
+    -------
+    DataFrame or Table
+        A native frame of the requested backend.
+    """
+    if isinstance(records, dict):
+        columns = records
+    else:
+        keys = {}
+        for record in records:
+            for key in record:
+                keys[key] = None
+        columns = {key: [record.get(key) for record in records] for key in keys}
+    if not columns and schema is not None:
+        columns = {name: [] for name in schema}
+    if output_type == "dask":
+        # narwhals from_dict is eager-only; build in pandas and hand off lazily.
+        return nw.from_dict(columns, schema=schema, backend=PANDAS).lazy(backend="dask").to_native()
+    return nw.from_dict(columns, schema=schema, backend=output_type).to_native()
+
+
+def observation_schema(dimension_names) -> dict:
+    """Narwhals schema for long-form observation records: string dimensions plus a float64 value.
+
+    Parameters
+    ----------
+    dimension_names : iterable of str
+        Names of the dimension columns.
+
+    Returns
+    -------
+    dict
+        Mapping of column name to narwhals dtype, ready for :func:`make_frame`.
+    """
+    return {name: nw.String() for name in dimension_names} | {"value": nw.Float64()}
+
+
+def filter_date_range(
+    frame: IntoFrame,
+    column: str | None = None,
+    start: str | datetime.date | datetime.datetime | pd.Timestamp | None = None,
+    end: str | datetime.date | datetime.datetime | pd.Timestamp | None = None,
+):
+    """Keep rows whose ``column`` value lies in the inclusive range [start, end].
+
+    Inclusivity on both endpoints matches pandas ``truncate`` and label slicing. A column that is
+    not datetime- or date-typed -- e.g. non-calendar period codes like '2013-S1' -- comes back
+    unfiltered, leaving only the server-side query bounds in effect.
+
+    Parameters
+    ----------
+    frame : DataFrame or Table
+        Any narwhals-compatible native frame, eager or lazy.
+    column : str, optional
+        Name of the date column to filter on. When omitted, the frame's single datetime- or
+        date-typed column is used; a frame with none comes back unfiltered, and one with several
+        raises ValueError. Default None.
+    start : datetime-like, optional
+        Inclusive lower bound; no lower bound when None. Default None.
+    end : datetime-like, optional
+        Inclusive upper bound; no upper bound when None. Default None.
+
+    Returns
+    -------
+    DataFrame or Table
+        The filtered frame in its original backend.
+    """
+    if start is None and end is None:
+        return frame
+    ndf = nw.from_native(frame)
+    schema = ndf.collect_schema()
+    if column is None:
+        datetime_columns = [name for name, dtype in schema.items() if dtype == nw.Datetime or dtype == nw.Date]
+        if not datetime_columns:
+            return frame
+        if len(datetime_columns) > 1:
+            raise ValueError(f"multiple datetime columns {datetime_columns}; pass column= explicitly")
+        column = datetime_columns[0]
+    dtype = schema[column]
+    if dtype == nw.Date:
+        lower = None if start is None else pd.Timestamp(start).date()
+        upper = None if end is None else pd.Timestamp(end).date()
+    elif dtype == nw.Datetime:
+        lower = None if start is None else pd.Timestamp(start).to_pydatetime()
+        upper = None if end is None else pd.Timestamp(end).to_pydatetime()
+    else:
+        return frame
+    target = nw.col(column)
+    if lower is not None and upper is not None:
+        condition = target.is_between(lower, upper, closed="both")
+    elif lower is not None:
+        condition = target >= lower
+    else:
+        condition = target <= upper
+    return ndf.filter(condition).to_native()
+
+
+def to_datetime_col(frame: IntoFrame, column: str):
+    """Cast a string ``column`` to datetime, keeping the strings when they are not calendar dates.
+
+    The datetime format is inferred by the backend. Values that defeat inference leave the frame
+    unchanged, as does a column that is not string-typed.
+
+    Parameters
+    ----------
+    frame : DataFrame or Table
+        Any narwhals-compatible native frame.
+    column : str
+        Name of the column to cast.
+
+    Returns
+    -------
+    DataFrame or Table
+        The frame in its original backend, with ``column`` cast when possible.
+    """
+    ndf = nw.from_native(frame)
+    if ndf.collect_schema()[column] != nw.String:
+        return frame
+    try:
+        return ndf.with_columns(nw.col(column).str.to_datetime()).to_native()
+    except Exception:
+        # Backends raise different error types for unparseable dates; keeping the strings is the
+        # designed fallback, not a swallowed failure.
+        return frame
+
+
+def concat_frames(frames: Sequence[IntoFrame]):
+    """Concatenate native frames of the same backend vertically.
+
+    Parameters
+    ----------
+    frames : sequence of DataFrame or Table
+        Frames of one backend with matching schemas.
+
+    Returns
+    -------
+    DataFrame or Table
+        The stacked frame in the shared backend.
+
+    Raises
+    ------
+    ValueError
+        If ``frames`` is empty.
+    """
+    if not frames:
+        raise ValueError("concat_frames requires at least one frame")
+    return nw.concat([nw.from_native(frame) for frame in frames], how="vertical").to_native()

kuznets/_testing.py

@@ -0,0 +1,29 @@
+"""
+Utilities for testing purposes.
+"""
+
+import wrapt
+
+
+def skip_on_exception(exp):
+    """
+    Skip a test if a specific Exception is raised. This is because the Exception is raised for
+    reasons beyond our control (e.g. flakey 3rd-party API).
+
+    a signature-preserving decorator
+
+    Parameters
+    ----------
+    exp : The Exception under which to execute try-except.
+    """
+
+    from pytest import skip
+
+    @wrapt.decorator
+    def wrapper(wrapped, instance, args, kwargs):
+        try:
+            return wrapped(*args, **kwargs)
+        except exp as e:
+            skip(str(e))
+
+    return wrapper

kuznets/_utils.py

@@ -0,0 +1,128 @@
+import datetime as dt
+from importlib.metadata import PackageNotFoundError, version
+
+from pandas import Timestamp, to_datetime
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from kuznets.compat import is_number
+
+try:
+    DEFAULT_USER_AGENT = f"kuznets/{version('kuznets')}"
+except PackageNotFoundError:  # pragma: no cover
+    DEFAULT_USER_AGENT = "kuznets"
+
+# Transient statuses worth retrying. Other 4xx (e.g. 404) won't recover, so they fall straight
+# through to the caller. 429 and 503 carry a ``Retry-After`` that the Retry strategy honors.
+RETRYABLE_STATUS_CODES = (413, 429, 500, 502, 503, 504)
+
+
+class SymbolWarning(UserWarning):
+    pass
+
+
+class RemoteDataError(IOError):
+    pass
+
+
+def _sanitize_dates(
+    start: str | int | dt.date | dt.datetime | Timestamp,
+    end: str | int | dt.date | dt.datetime | Timestamp,
+) -> tuple[Timestamp, Timestamp]:
+    """
+    Return (timestamp_start, timestamp_end) tuple.
+
+    If start is None, default is 5 years before the current date. If end is None, default is today.
+
+    Parameters
+    ----------
+    start : str, int, date, datetime, or Timestamp
+        Desired start date.
+    end : str, int, date, datetime, or Timestamp
+        Desired end date.
+
+    Returns
+    -------
+    start : Timestamp
+        Sanitized start date.
+    end : Timestamp
+        Sanitized end date.
+    """
+    if is_number(start):
+        # regard int as year
+        start = dt.datetime(start, 1, 1)
+
+    if is_number(end):
+        end = dt.datetime(end, 1, 1)
+
+    if start is None:
+        # default to 5 years before today
+        today = dt.date.today()
+        start = today - dt.timedelta(days=365 * 5)
+    if end is None:
+        # default to today
+        end = dt.date.today()
+    try:
+        start = to_datetime(start)
+        end = to_datetime(end)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("Invalid date format.") from exc
+    if start > end:
+        raise ValueError("start must be an earlier date than end")
+    return start, end
+
+
+def _init_session(
+    session: requests.Session | None,
+    retry_count: int = 3,
+    pause: float = 0.1,
+    headers: dict | None = None,
+) -> requests.Session:
+    """
+    Initialize a requests session with a retry strategy.
+
+    Mount an :class:`~urllib3.util.Retry`-backed adapter so urllib3 handles retry counting,
+    exponential backoff, and ``Retry-After`` for transient failures. ``raise_on_status`` is left
+    off so the exhausted response flows back to :meth:`~kuznets.base._BaseReader._get_response`,
+    which raises a ``RemoteDataError`` carrying the response body.
+
+    Parameters
+    ----------
+    session : Session or None
+        ``requests.sessions.Session`` instance to be used, or ``None`` to create a new session.
+    retry_count : int, optional
+        Maximum number of retries for transient failures. Default 3.
+    pause : float, optional
+        Backoff factor, in seconds, between retries. The nth retry waits ``pause * 2 ** (n - 1)``
+        seconds. Default 0.1.
+    headers : dict, optional
+        Headers to apply to the session, taking precedence over the defaults.
+
+    Returns
+    -------
+    session : Session
+        The initialized session.
+    """
+    if session is None:
+        session = requests.Session()
+        # Identify ourselves so hosts can throttle politely rather than blocking the anonymous
+        # ``python-requests`` agent that requests sets by default.
+        session.headers["User-Agent"] = DEFAULT_USER_AGENT
+    elif not isinstance(session, requests.Session):
+        raise TypeError("session must be a request.Session")
+
+    if headers:
+        session.headers.update(headers)
+
+    retry = Retry(
+        total=retry_count,
+        backoff_factor=pause,
+        status_forcelist=RETRYABLE_STATUS_CODES,
+        respect_retry_after_header=True,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session

kuznets/av/__init__.py

@@ -0,0 +1,111 @@
+import pandas as pd
+
+from kuznets._utils import RemoteDataError
+from kuznets.base import _BaseReader
+from kuznets.config import get_api_key
+
+AV_BASE_URL = "https://www.alphavantage.co/query"
+
+
+class AlphaVantage(_BaseReader):
+    """Base class for all Alpha Vantage queries."""
+
+    _format = "json"
+
+    def __init__(
+        self,
+        symbols: str | list[str] | None = None,
+        start=None,
+        end=None,
+        retry_count: int | None = None,
+        pause: float | None = None,
+        session=None,
+        api_key: str | None = None,
+        output_type: str = "pandas",
+    ) -> None:
+        """
+        Initialize the reader.
+
+        Parameters
+        ----------
+        symbols : str or list of str, optional
+            String symbol or list of symbols.
+        start : str, int, date, datetime, or Timestamp, optional
+            Starting date.
+        end : str, int, date, datetime, or Timestamp, optional
+            Ending date.
+        retry_count : int, optional
+            Number of times to retry query request. Falls back to the configured default.
+        pause : float, optional
+            Time, in seconds, of the pause between retries. Falls back to the configured default.
+        session : Session, optional
+            ``requests.sessions.Session`` instance to be used.
+        api_key : str, optional
+            Alpha Vantage API key. Resolved through :func:`kuznets.config.get_api_key`
+            (argument, ``options.api_keys['alphavantage']``, ``ALPHAVANTAGE_API_KEY``, then the
+            config file). The API key is *required*.
+        output_type : str, optional
+            Backend of the returned data: 'pandas', 'polars', 'pyarrow' (alias 'arrow'), or 'dask'.
+            Backends other than pandas must be installed separately. Default 'pandas'.
+
+        Notes
+        -----
+        See `Alpha Vantage <https://www.alphavantage.co/>`__
+        """
+        super().__init__(
+            symbols=symbols,
+            start=start,
+            end=end,
+            retry_count=retry_count,
+            pause=pause,
+            session=session,
+            output_type=output_type,
+        )
+        self.api_key = get_api_key("alphavantage", api_key)
+
+    @property
+    def url(self) -> str:
+        """API URL."""
+        return AV_BASE_URL
+
+    @property
+    def params(self) -> dict:
+        """Parameters to use in API calls."""
+        return {"function": self.function, "apikey": self.api_key}
+
+    @property
+    def function(self) -> str:
+        """Alpha Vantage endpoint function. Must be overridden in subclass."""
+        raise NotImplementedError
+
+    @property
+    def data_key(self) -> str:
+        """Key of data returned from Alpha Vantage. Must be overridden in subclass."""
+        raise NotImplementedError
+
+    def _read_lines(self, out: dict) -> pd.DataFrame:
+        """Parse Alpha Vantage JSON response.
+
+        Parameters
+        ----------
+        out : dict
+            Parsed JSON response.
+
+        Returns
+        -------
+        df : DataFrame
+        """
+        try:
+            df = pd.DataFrame.from_dict(out[self.data_key], orient="index")
+        except KeyError as exc:
+            if "Error Message" in out:
+                raise ValueError(
+                    f"The requested symbol {self.symbols} could not be retrieved. Check valid ticker."
+                ) from exc
+            else:
+                raise RemoteDataError(
+                    f" Their was an issue from the data vendor side, here is their response: {out}"
+                ) from exc
+        df = df[sorted(df.columns)]
+        df.columns = [id[3:] for id in df.columns]
+        return df