numerai-tools 0.5.0.dev13__tar.gz → 0.6.0.dev0__tar.gz

{numerai_tools-0.5.0.dev13 → numerai_tools-0.6.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: numerai-tools
-Version: 0.5.0.dev13
+Version: 0.6.0.dev0
 Summary: A collection of open-source tools to help interact with Numerai, model data, and automate submissions.
 License: MIT
 Author: Numerai Engineering

numerai_tools-0.6.0.dev0/numerai_tools/data.py

@@ -0,0 +1,133 @@
+from typing import List, Union, Optional
+
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import OneHotEncoder  # type: ignore
+
+from numerai_tools.scoring import tie_kept_rank
+
+DEFAULT_BINS = (0.0, 0.25, 0.5, 0.75, 1.0)
+DEFAULT_QUANTILES = (0.05, 0.25, 0.75, 0.95)
+
+
+def one_hot_encode(
+    df: pd.DataFrame, columns: List[str], dtype: type = np.float64
+) -> pd.DataFrame:
+    """One-hot encodes specified columns in a pandas dataframe.
+    Each column i should have x_i discrete values (eg. categories, bucket values, etc.)
+    and will be converted to x_i columns that each have 0s for rows that don't have
+    the associated value and 1s for rows that do have that value.
+
+    Arguments:
+        df: pd.DataFrame - the data with columns to one-hot encode
+        columns: List[str] - list of columns names to replace w/ one-hot encoding
+        dtype: type = np.float64 - the target datatype for the resulting columns
+
+    Returns:
+        pd.DataFrame - original data, but specified cols replaced w/ one-hot encoding
+    """
+    for col in columns:
+        encoder = OneHotEncoder(dtype=dtype)
+        one_hot = encoder.fit_transform(df[[col]])
+        one_hot = pd.DataFrame(
+            one_hot.toarray(),
+            columns=encoder.get_feature_names_out(),
+            index=df.index,
+        )
+        df = df.join(one_hot).drop(columns=col)
+    return df
+
+
+def balanced_rank_transform(
+    df: pd.DataFrame,
+    cols: List[str],
+    rank_group: Optional[str] = None,
+    rank_filter: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Perform a balanced rank transformation on specified columns of a DataFrame,
+    optionally within groups and with a filter.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame containing the data to be ranked.
+    cols : list of str
+        List of column names to apply the rank transformation to.
+    rank_group : str
+        Column name to group by before ranking.
+    rank_filter : str, optional
+        Column name to filter rows before ranking. Only rows where this column is True
+        will be ranked. If None, no filtering is applied.
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with the same index as the input, containing the ranked columns.
+    """
+    if rank_filter is not None:
+        df = df.loc[df[rank_filter]]
+    else:
+        df = df
+    if rank_group is not None:
+        df = df.groupby(rank_group, group_keys=False).apply(
+            lambda d: tie_kept_rank(d[cols])
+        )
+    else:
+        df = tie_kept_rank(df[cols])
+    return df[cols]
+
+
+def quantile_bin(
+    data: Union[pd.Series, pd.DataFrame],
+    bins: tuple[float, ...] = DEFAULT_BINS,
+    quantiles: tuple[float, ...] = DEFAULT_QUANTILES,
+) -> pd.DataFrame:
+    """
+    Bin a Series or DataFrame into discrete quantile-based bins.
+    Handles identical-value columns by assigning all values to the lowest bin.
+
+    Parameters
+    ----------
+    data : pd.Series or pd.DataFrame
+        Data to bin.
+    bins : list of float
+        Values to assign to each bin.
+    quantiles : list of float
+        Quantile thresholds to use for binning (len = number of bins - 1)
+
+    Returns
+    -------
+    pd.DataFrame
+        Binned values, same shape as input.
+    """
+    assert len(bins), "Invalid bins! Must not be empty."
+    assert len(quantiles), "Invalid quantiles! Must not be empty."
+    assert len(quantiles) == (
+        len(bins) - 1
+    ), "Invalid quantiles! Length must be 1 less than bins."
+
+    if isinstance(data, pd.Series):
+        data = data.to_frame(name="value")
+
+    binned = data.copy()
+    for col in binned.columns:
+        s = binned[col].astype(float)
+
+        # handle all-identical values
+        if s.nunique() <= 1:
+            binned[col] = 0.0
+            continue
+
+        # calculate quantile thresholds
+        q = s.quantile(quantiles)
+
+        # assign bins according to quantiles
+        s.loc[s <= q[quantiles[0]]] = bins[0]
+        for i in range(1, len(bins) - 1):
+            s.loc[(s > q[quantiles[i - 1]]) & (s <= q[quantiles[i]])] = bins[i]
+        s.loc[s >= q[quantiles[-1]]] = bins[-1]
+
+        binned[col] = s.astype(float)
+
+    return binned

numerai_tools-0.6.0.dev0/numerai_tools/indexing.py

@@ -0,0 +1,106 @@
+from typing import List, Tuple, cast, Any
+
+import numpy as np
+import pandas as pd
+
+# leaving this here for backwards compatibility
+from numerai_tools.typing import S1, S2
+
+
+# sometimes when we match up the target/prediction indices,
+# changes in stock universe causes some stocks to enter / leave,
+# this ensures we don't filter too much
+DEFAULT_MAX_FILTERED_INDEX_RATIO = 0.2
+
+
+def filter_sort_index(
+    s1: S1, s2: S2, max_filtered_ratio: float = DEFAULT_MAX_FILTERED_INDEX_RATIO
+) -> Tuple[S1, S2]:
+    """Filters the indices of the given series to match each other,
+    then sorts the indices, then checks that we didn't filter too many indices
+    before returning the filtered and sorted series.
+
+    Arguments:
+        s1: Union[pd.DataFrame, pd.Series] - the first dataset to filter and sort
+        s2: Union[pd.DataFrame, pd.Series] - the second dataset to filter and sort
+
+    Returns:
+        Tuple[
+            Union[pd.DataFrame, pd.Series],
+            Union[pd.DataFrame, pd.Series],
+        ] - the filtered and sorted datasets
+    """
+    ids = s1.dropna().index.intersection(s2.dropna().index)
+    # ensure we didn't filter too many ids
+    assert len(ids) / len(s1) >= (1 - max_filtered_ratio), (
+        "s1 does not have enough overlapping ids with s2,"
+        f" must have >= {round(1-max_filtered_ratio,2)*100}% overlapping ids"
+    )
+    assert len(ids) / len(s2) >= (1 - max_filtered_ratio), (
+        "s2 does not have enough overlapping ids with s1,"
+        f" must have >= {round(1-max_filtered_ratio,2)*100}% overlapping ids"
+    )
+    return cast(S1, s1.loc[ids].sort_index()), cast(S2, s2.loc[ids].sort_index())
+
+
+def filter_sort_index_many(
+    inputs: List[Any],
+    max_filtered_ratio: float = DEFAULT_MAX_FILTERED_INDEX_RATIO,
+) -> List[Any]:
+    """Filters the indices of the given list of series to match each other,
+    then sorts the indices, then checks that we didn't filter too many indices
+    before returning the filtered and sorted series.
+
+    Arguments:
+        inputs: List[Union[pd.DataFrame, pd.Series]] - the list of datasets to filter and sort
+
+    Returns:
+        List[Union[pd.DataFrame, pd.Series]] - the filtered and sorted datasets
+    """
+    assert len(inputs) > 0, "List must contain at least one element"
+    ids = inputs[0].dropna().index
+    for i in range(1, len(inputs)):
+        ids = ids.intersection(inputs[i].dropna().index)
+    result = [inputs[i].loc[ids].sort_index() for i in range(len(inputs))]
+    # ensure we didn't filter too many ids
+    for i in range(len(result)):
+        assert len(result[i]) / len(inputs[i]) >= (1 - max_filtered_ratio), (
+            f"inputs[{i}] does not have enough overlapping ids with the others,"
+            f" must have >= {round(1-max_filtered_ratio,2)*100}% overlapping ids"
+        )
+    return result
+
+
+def filter_sort_top_bottom(
+    s: pd.Series, top_bottom: int
+) -> Tuple[pd.Series, pd.Series]:
+    """Filters the series according to the top n and bottom n values
+    then sorts the index and returns two filtered and sorted series
+    for the top and bottom values respectively.
+
+    Arguments:
+        s: pd.Series - the data to filter and sort
+        top_bottom: int - the number of top n and bottom n values to keep
+
+    Returns:
+        Tuple[pd.Series, pd.Series] - the filtered and sorted top and bottom series respectively
+    """
+    tb_idx = np.argsort(s, kind="stable")
+    bot = s.iloc[tb_idx[:top_bottom]]
+    top = s.iloc[tb_idx[-top_bottom:]]
+    return top.sort_index(), bot.sort_index()
+
+
+def filter_sort_top_bottom_concat(s: pd.Series, top_bottom: int) -> pd.Series:
+    """Similar to filter_sort_top_bottom, but concatenates the top and bottom series
+    into 1 series and then sorts the index.
+
+    Arguments:
+        s: pd.Series - the data to filter and sort
+        top_bottom: int - the number of top n and bottom n values to keep
+
+    Returns:
+        pd.Series - the concatenated and sorted series of top and bottom values
+    """
+    top, bot = filter_sort_top_bottom(s, top_bottom)
+    return pd.concat([top, bot]).sort_index()

numerai_tools-0.6.0.dev0/numerai_tools/math.py

@@ -0,0 +1,263 @@
+from typing import Optional, cast, Literal
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+# leaving this here for backwards compatibility
+from numerai_tools.typing import S1
+
+from numerai_tools.indexing import (
+    filter_sort_index,
+    filter_sort_top_bottom_concat,
+)
+
+
+RANK_METHOD_TYPE = Literal["average", "min", "max", "first", "dense"]
+
+
+def rank_series(s: pd.Series, method: RANK_METHOD_TYPE = "average") -> pd.Series:
+    """Percentile rank a pandas Series, centering values around 0.5.
+
+    Arguments:
+        s: pd.Series - the data to rank
+        method: str - the pandas ranking method to use, options:
+            'average' (default) - keeps ties
+            'first' - breaks ties by index
+
+    Returns:
+        pd.Series - the ranked Series
+    """
+    assert np.array_equal(s.index.sort_values(), s.index), "unsorted index found"
+    # Ensure denominator is at least 1 to avoid division by zero
+    denom = max(int(s.count()), 1)
+    return (s.rank(method=method) - 0.5) / denom
+
+
+def rank(s: S1, method: RANK_METHOD_TYPE = "average") -> S1:
+    """Percentile rank each columns or series, centering values around 0.5
+
+    Arguments:
+        s: pd.DataFrame | pd.Series - the data to rank
+        method: str - the pandas ranking method to use, options:
+            'average' (default) - keeps ties
+            'first' - breaks ties by index
+
+    Returns:
+        pd.DataFrame | pd.Series - the ranked input data
+    """
+    if isinstance(s, pd.Series):
+        return cast(S1, rank_series(s, method))
+    else:
+        return s.apply(lambda series: rank(series, method=method))
+
+
+def tie_broken_rank(df: pd.DataFrame) -> pd.DataFrame:
+    """Rank columns, breaking ties by index."""
+    return rank(df, "first")
+
+
+def tie_kept_rank(s: S1) -> S1:
+    """Rank columns, but keep ties."""
+    return cast(S1, rank(s, "average"))
+
+
+def min_max_normalize(s: pd.Series) -> pd.Series:
+    """Scale a series to be between 0 and 1."""
+    return (s - s.min()) / (s.max() - s.min())
+
+
+def variance_normalize(df: pd.DataFrame) -> pd.DataFrame:
+    """Scale a df such that all columns have std == 1."""
+    return df / np.std(df, axis=0)
+
+
+def weight_normalize(s: S1) -> S1:
+    """Scale a input such that all columns have absolute value sum == 1."""
+    return cast(S1, s / s.abs().sum(axis=0))
+
+
+def center(s: S1) -> S1:
+    """Shift the input such that all columns have mean == 0."""
+    return cast(S1, s - s.mean())
+
+
+def standardize(df: pd.DataFrame) -> pd.DataFrame:
+    """Scale a df such that all columns have mean == 0 and std == 1."""
+    return variance_normalize(center(df))
+
+
+def validate_indices(live_targets: pd.Series, predictions: pd.Series) -> None:
+    # ensure the ids are equivalent and sorted
+    assert np.array_equal(predictions.index, live_targets.index.sort_values())
+    assert np.array_equal(live_targets.index, live_targets.index.sort_values())
+    assert np.array_equal(predictions.index, predictions.index.sort_values())
+    # ensure no nans
+    assert not predictions.isna().any()
+    assert not live_targets.isna().any()
+
+
+def correlation(live_targets: pd.Series, predictions: pd.Series) -> float:
+    validate_indices(live_targets, predictions)
+    # calculate correlation coefficient
+    return np.corrcoef(live_targets, predictions)[0, 1]
+
+
+def tie_broken_rank_correlation(target: pd.Series, predictions: pd.Series) -> float:
+    # percentile rank the predictions and get the correlation with the target
+    ranked_predictions = tie_broken_rank(predictions.to_frame())[predictions.name]
+    return correlation(target, ranked_predictions)
+
+
+def spearman_correlation(target: pd.Series, predictions: pd.Series) -> float:
+    validate_indices(target, predictions)
+    return target.corr(predictions, method="spearman")
+
+
+def pearson_correlation(
+    target: pd.Series, predictions: pd.Series, top_bottom: Optional[int] = None
+) -> float:
+    if top_bottom is not None and top_bottom > 0:
+        predictions = filter_sort_top_bottom_concat(predictions, top_bottom)
+        target, predictions = filter_sort_index(
+            target, predictions, (1 - top_bottom / len(target))
+        )
+    validate_indices(target, predictions)
+    return target.corr(predictions, method="pearson")
+
+
+def sharpe_ratio(s: pd.Series) -> float:
+    # calculate the sharpe ratio of a series
+    return np.mean(s) / np.std(s)
+
+
+def gaussian(df: pd.DataFrame) -> pd.DataFrame:
+    """Gaussianize each column of a pandas DataFrame using a normal percent point func.
+    Effectively scales each column such that mean == 0 and std == 1.
+
+    Arguments:
+        df: pd.DataFrame - the data to gaussianize
+
+    Returns:
+        pd.DataFrame - the gaussianized data
+    """
+    assert np.array_equal(df.index.sort_values(), df.index)
+    return df.apply(lambda series: cast(np.ndarray, stats.norm.ppf(series)))
+
+
+def power(df: pd.DataFrame, p: float) -> pd.DataFrame:
+    """Raise given predictions series to the given power.
+
+    Arguments:
+        df: pd.DataFrame - the data to raise to the given power
+        p: float - the power to which we exponentiate the data
+
+    Returns:
+        pd.DataFrame - the predictions raised to the given power,
+            each column should be at least 90% correlated with the original data
+    """
+    assert not df.isna().any().any(), "Data contains NaNs"
+    assert np.array_equal(df.index.sort_values(), df.index), "Index is not sorted"
+    result = cast(pd.DataFrame, np.sign(df) * np.abs(df) ** p)
+    assert ((result.std() == 0) | (result.corrwith(df) >= 0.9)).all()
+    return result
+
+
+def tie_kept_rank__gaussianize__pow_1_5(df: pd.DataFrame) -> pd.DataFrame:
+    """Perform the 3 functions in order on the given pandas DataFrame.
+    Will tie-kept rank then gaussianize then exponentiate to the 1.5 power.
+
+    Arguments:
+        df: pd.DataFrame - the data to transform
+
+    Returns:
+        pd.DataFrame - the resulting data after applying the 3 functions
+    """
+    return power(gaussian(tie_kept_rank(df)), 1.5)
+
+
+def tie_kept_rank__gaussianize__neutralize__variance_normalize(
+    df: pd.DataFrame, neutralizers: pd.DataFrame
+) -> pd.DataFrame:
+    """Perform the 4 functions in order on the given pandas DataFrame.
+    1. tie-kept rank each column
+    2. gaussianize each column
+    3. neutralize each column to the neutralizers
+    4. variance normalize each column
+
+    Arguments:
+        df: pd.DataFrame - the data to transform
+
+    Returns:
+        pd.DataFrame - the resulting data after applying the 3 functions
+    """
+    return variance_normalize(neutralize(gaussian(tie_kept_rank(df)), neutralizers))
+
+
+def orthogonalize(v: np.ndarray, u: np.ndarray) -> np.ndarray:
+    """Orthogonalizes v with respect to u by projecting v onto u,
+    then subtracting that projection from v.
+
+    This will reach the same result as the neutralize
+    function when v and u are centered single column vectors,
+    but this is much faster.
+
+    Arguments:
+        v: np.ndarray - the vector to orthogonalize
+        u: np.ndarray - the vector orthogonalize v
+
+    Returns:
+        np.ndarray - the orthogonalized vector v
+    """
+    return v - np.outer(u, (v.T @ u) / (u.T @ u))
+
+
+def stake_weight(
+    predictions: pd.DataFrame,
+    stakes: pd.Series,
+) -> pd.Series:
+    """Create a stake-weighted meta model from the given predictions and stakes.
+
+    Arguments:
+        predictions: pd.DataFrame - the predictions to weight
+        stakes: pd.Series - the stakes to use as weights
+
+    Returns:
+        pd.Series - the stake-weighted meta model
+    """
+    return (predictions[stakes.index] * stakes).sum(axis=1) / stakes.sum()
+
+
+def neutralize(
+    df: pd.DataFrame,
+    neutralizers: pd.DataFrame,
+    proportion: float = 1.0,
+) -> pd.DataFrame:
+    """Neutralize each column of a given DataFrame by each feature in a given
+    neutralizers DataFrame. Neutralization uses least-squares regression to
+    find the orthogonal projection of each column onto the neutralizers, then
+    subtracts the result from the original predictions.
+
+    Arguments:
+        df: pd.DataFrame - the data with columns to neutralize
+        neutralizers: pd.DataFrame - the neutralizer data with features as columns
+        proportion: float - the degree to which neutralization occurs
+
+    Returns:
+        pd.DataFrame - the neutralized data
+    """
+    assert not df.isna().any().any(), "Data contains NaNs"
+    assert not neutralizers.isna().any().any(), "Neutralizers contain NaNs"
+    assert len(df.index) == len(neutralizers.index), "Indices don't match"
+    assert (df.index == neutralizers.index).all(), "Indices don't match"
+    df[df.columns[df.std() == 0]] = np.nan
+    df_arr = df.values
+    neutralizer_arr = neutralizers.values
+    neutralizer_arr = np.hstack(
+        # add a column of 1s to the neutralizer array in case neutralizer_arr is a single column
+        (neutralizer_arr, np.array([1] * len(neutralizer_arr)).reshape(-1, 1))
+    )
+    least_squares = np.linalg.lstsq(neutralizer_arr, df_arr, rcond=1e-6)[0]
+    adjustments = proportion * neutralizer_arr.dot(least_squares)
+    neutral = df_arr - adjustments
+    return pd.DataFrame(neutral, index=df.index, columns=df.columns)