skfolio 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

skfolio/datasets/__init__.py

@@ -8,6 +8,7 @@ from skfolio.datasets._base import (
     load_ftse100_dataset,
     load_nasdaq_dataset,
     load_sp500_dataset,
+    load_sp500_implied_vol_dataset,
     load_sp500_index,
 )
 
@@ -17,4 +18,5 @@ __all__ = [
     "load_ftse100_dataset",
     "load_sp500_dataset",
     "load_sp500_index",
+    "load_sp500_implied_vol_dataset",
 ]

skfolio/datasets/_base.py

@@ -392,3 +392,54 @@ def load_nasdaq_dataset(data_home=None, download_if_missing=True) -> pd.DataFram
         data_filename, data_home=data_home, download_if_missing=download_if_missing
     )
     return df
+
+
+def load_sp500_implied_vol_dataset(
+    data_home=None, download_if_missing=True
+) -> pd.DataFrame:
+    """Load the 3 months ATM implied volatility of the 20 assets from the
+    SP500 dataset.
+
+    This dataset is composed of the 3 months ATM implied volatility of 20 assets
+    from the S&P 500 composition starting from 2010-01-04 up to 2022-12-28.
+
+    The data comes from the Yahoo public API option chains.
+
+    ==============   ==================
+    Observations     3270
+    Assets           20
+    ==============   ==================
+
+    Parameters
+    ----------
+    data_home : str, optional
+        Specify another download and cache folder for the datasets.
+        By default, all skfolio data is stored in `~/skfolio_data` subfolders.
+
+    download_if_missing : bool, default=True
+        If False, raise an OSError if the data is not locally available
+        instead of trying to download the data from the source site.
+
+    Returns
+    -------
+    df : DataFrame of shape (n_observations, n_assets)
+        Implied volatility DataFrame
+
+    Examples
+    --------
+    >>> from skfolio.datasets import load_sp500_implied_vol_dataset
+    >>> implied_vol = load_sp500_implied_vol_dataset()
+    >>> implied_vol.head()
+                    AAPL       AMD       BAC  ...       UNH       WMT       XOM
+    Date                                      ...
+    2010-01-04  0.364353  0.572056  0.382926  ...  0.362751  0.171737  0.201485
+    2010-01-05  0.371865  0.568791  0.374699  ...  0.368504  0.174764  0.203852
+    2010-01-06  0.356746  0.558054  0.349220  ...  0.368514  0.171892  0.197475
+    2010-01-07  0.361084  0.560475  0.354942  ...  0.355792  0.169083  0.200046
+    2010-01-08  0.348085  0.543932  0.360345  ...  0.351130  0.170897  0.204832
+    """
+    data_filename = "sp500_implied_vol_dataset"
+    df = download_dataset(
+        data_filename, data_home=data_home, download_if_missing=download_if_missing
+    )
+    return df

skfolio/distance/_distance.py

@@ -9,7 +9,8 @@ import numpy.typing as npt
 import pandas as pd
 import scipy.spatial.distance as scd
 import scipy.stats as sct
-import sklearn.metrics as skm
+import sklearn.metrics as skmc
+import sklearn.utils.metadata_routing as skm
 
 from skfolio.distance._base import BaseDistance
 from skfolio.moments import BaseCovariance, GerberCovariance
@@ -300,7 +301,15 @@ class CovarianceDistance(BaseDistance):
         self.absolute = absolute
         self.power = power
 
-    def fit(self, X: npt.ArrayLike, y=None) -> "CovarianceDistance":
+    def get_metadata_routing(self):
+        # noinspection PyTypeChecker
+        router = skm.MetadataRouter(owner=self.__class__.__name__).add(
+            covariance_estimator=self.covariance_estimator,
+            method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def fit(self, X: npt.ArrayLike, y=None, **fit_params) -> "CovarianceDistance":
         """Fit the Covariance Distance estimator.
 
         Parameters
@@ -316,13 +325,15 @@ class CovarianceDistance(BaseDistance):
         self : CovarianceDistance
             Fitted estimator.
         """
+        routed_params = skm.process_routing(self, "fit", **fit_params)
+
         # fitting estimators
         self.covariance_estimator_ = check_estimator(
             self.covariance_estimator,
             default=GerberCovariance(),
             check_type=BaseCovariance,
         )
-        self.covariance_estimator_.fit(X)
+        self.covariance_estimator_.fit(X, y, **routed_params.covariance_estimator.fit)
 
         # we validate and convert to numpy after all models have been fitted to keep the
         # features names information.
@@ -512,7 +523,7 @@ class MutualInformation(BaseDistance):
             x = X[:, i]
             y = X[:, j]
             contingency = np.histogram2d(x, y, bins=n_bins)[0]
-            mutual_information = skm.mutual_info_score(
+            mutual_information = skmc.mutual_info_score(
                 None, None, contingency=contingency
             )
             entropy_x = sct.entropy(np.histogram(x, n_bins)[0])

skfolio/model_selection/_combinatorial.py

@@ -18,7 +18,7 @@ import numpy as np
 import numpy.typing as npt
 import pandas as pd
 import plotly.graph_objects as go
-import sklearn.model_selection as skm
+import sklearn.model_selection as sks
 import sklearn.utils as sku
 
 import skfolio.typing as skt
@@ -39,7 +39,7 @@ class BaseCombinatorialCV(ABC):
         """Return the path id of each test sets in each split"""
         pass
 
-    __repr__ = skm.BaseCrossValidator.__repr__
+    __repr__ = sks.BaseCrossValidator.__repr__
 
 
 # TODO: review params and function naming

skfolio/model_selection/_validation.py

@@ -12,8 +12,10 @@ import numpy as np
 import numpy.typing as npt
 import sklearn as sk
 import sklearn.base as skb
-import sklearn.model_selection as skm
+import sklearn.exceptions as ske
+import sklearn.model_selection as sks
 import sklearn.utils as sku
+import sklearn.utils.metadata_routing as skm
 import sklearn.utils.parallel as skp
 
 from skfolio.model_selection._combinatorial import BaseCombinatorialCV
@@ -22,16 +24,26 @@ from skfolio.portfolio import MultiPeriodPortfolio
 from skfolio.utils.tools import fit_and_predict, safe_split
 
 
+def _routing_enabled():
+    """Return whether metadata routing is enabled.
+    Returns
+    -------
+    enabled : bool
+        Whether metadata routing is enabled. If the config is not set, it
+        defaults to False.
+    """
+    return sk.get_config().get("enable_metadata_routing", False)
+
+
 def cross_val_predict(
     estimator: skb.BaseEstimator,
     X: npt.ArrayLike,
     y: npt.ArrayLike = None,
-    groups: np.ndarray | None = None,
-    cv: skm.BaseCrossValidator | BaseCombinatorialCV | int | None = None,
+    cv: sks.BaseCrossValidator | BaseCombinatorialCV | int | None = None,
     n_jobs: int | None = None,
     method: str = "predict",
     verbose: int = 0,
-    fit_params: dict | None = None,
+    params: dict | None = None,
     pre_dispatch: str = "2*n_jobs",
     column_indices: np.ndarray | None = None,
     portfolio_params: dict | None = None,
@@ -65,11 +77,6 @@ def cross_val_predict(
         Target data (optional).
         For example, the price returns of the factors.
 
-    groups : array-like of shape (n_observations,), optional
-        Group labels for the samples used while splitting the dataset into
-        train/test set. Only used in conjunction with a "Group" `cv`
-        instance (e.g., `GroupKFold`).
-
     cv : int | cross-validation generator, optional
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
@@ -90,8 +97,8 @@ def cross_val_predict(
     verbose : int, default=0
         The verbosity level.
 
-    fit_params : dict, optional
-        Parameters to pass to the fit method of the estimator.
+    params : dict, optional
+        Parameters to pass to the underlying estimator's ``fit`` and the CV splitter.
 
     pre_dispatch : int or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
@@ -121,10 +128,57 @@ def cross_val_predict(
     predictions : MultiPeriodPortfolio | Population
         This is the result of calling `predict`
     """
+    params = {} if params is None else params
+
     X, y = safe_split(X, y, indices=column_indices, axis=1)
-    X, y, groups = sku.indexable(X, y, groups)
-    cv = skm.check_cv(cv, y)
-    splits = list(cv.split(X, y, groups))
+    X, y = sku.indexable(X, y)
+
+    if _routing_enabled():
+        # For estimators, a MetadataRouter is created in get_metadata_routing
+        # methods. For these router methods, we create the router to use
+        # `process_routing` on it.
+        # noinspection PyTypeChecker
+        router = (
+            skm.MetadataRouter(owner="cross_validate")
+            .add(
+                splitter=cv,
+                method_mapping=skm.MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                estimator=estimator,
+                method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
+            )
+        )
+        try:
+            routed_params = skm.process_routing(router, "fit", **params)
+        except ske.UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            unrequested_params = sorted(e.unrequested_params)
+            raise ske.UnsetMetadataPassedError(
+                message=(
+                    f"{unrequested_params} are passed to `cross_val_predict` but are"
+                    " not explicitly set as requested or not requested for"
+                    f" cross_validate's estimator: {estimator.__class__.__name__} Call"
+                    " `.set_fit_request({{metadata}}=True)` on the estimator for"
+                    f" each metadata in {unrequested_params} that you want to use and"
+                    " `metadata=False` for not using it. See the Metadata Routing User"
+                    " guide <https://scikit-learn.org/stable/metadata_routing.html>"
+                    " for more information."
+                ),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            ) from None
+    else:
+        routed_params = sku.Bunch()
+        routed_params.splitter = sku.Bunch(split={})
+        routed_params.estimator = sku.Bunch(fit=params)
+
+    cv = sks.check_cv(cv, y)
+    splits = list(cv.split(X, y, **routed_params.splitter.split))
+
     portfolio_params = {} if portfolio_params is None else portfolio_params.copy()
 
     # We ensure that the folds are not shuffled
@@ -148,6 +202,7 @@ def cross_val_predict(
     # and that it is pickle-able.
     parallel = skp.Parallel(n_jobs=n_jobs, verbose=verbose, pre_dispatch=pre_dispatch)
     # TODO remove when https://github.com/joblib/joblib/issues/1071 is fixed
+    # noinspection PyCallingNonCallable
     predictions = parallel(
         skp.delayed(fit_and_predict)(
             sk.clone(estimator),
@@ -155,7 +210,7 @@ def cross_val_predict(
             y,
             train=train,
             test=test,
-            fit_params=fit_params,
+            fit_params=routed_params.estimator.fit,
             method=method,
         )
         for train, test in splits

skfolio/model_selection/_walk_forward.py

@@ -12,11 +12,11 @@ from collections.abc import Iterator
 
 import numpy as np
 import numpy.typing as npt
-import sklearn.model_selection as skm
+import sklearn.model_selection as sks
 import sklearn.utils as sku
 
 
-class WalkForward(skm.BaseCrossValidator):
+class WalkForward(sks.BaseCrossValidator):
     """Walk Forward cross-validator.
 
     Provides train/test indices to split time series data samples in a walk forward
@@ -196,7 +196,7 @@ class WalkForward(skm.BaseCrossValidator):
                 )
             test_start = test_end
 
-    def get_n_splits(self, X: npt.ArrayLike, y=None, groups=None) -> int:
+    def get_n_splits(self, X=None, y=None, groups=None) -> int:
         """Returns the number of splitting iterations in the cross-validator
 
         Parameters

skfolio/moments/__init__.py

@@ -9,6 +9,7 @@ from skfolio.moments.covariance import (
     EmpiricalCovariance,
     GerberCovariance,
     GraphicalLassoCV,
+    ImpliedCovariance,
     LedoitWolf,
     ShrunkCovariance,
 )
@@ -38,4 +39,5 @@ __all__ = [
     "OAS",
     "ShrunkCovariance",
     "GraphicalLassoCV",
+    "ImpliedCovariance",
 ]

skfolio/moments/covariance/__init__.py

@@ -3,17 +3,16 @@
 from skfolio.moments.covariance._base import (
     BaseCovariance,
 )
-from skfolio.moments.covariance._covariance import (
-    OAS,
-    DenoiseCovariance,
-    DetoneCovariance,
-    EWCovariance,
-    EmpiricalCovariance,
-    GerberCovariance,
-    GraphicalLassoCV,
-    LedoitWolf,
-    ShrunkCovariance,
-)
+from skfolio.moments.covariance._denoise_covariance import DenoiseCovariance
+from skfolio.moments.covariance._detone_covariance import DetoneCovariance
+from skfolio.moments.covariance._empirical_covariance import EmpiricalCovariance
+from skfolio.moments.covariance._ew_covariance import EWCovariance
+from skfolio.moments.covariance._gerber_covariance import GerberCovariance
+from skfolio.moments.covariance._graphical_lasso_cv import GraphicalLassoCV
+from skfolio.moments.covariance._implied_covariance import ImpliedCovariance
+from skfolio.moments.covariance._ledoit_wolf import LedoitWolf
+from skfolio.moments.covariance._oas import OAS
+from skfolio.moments.covariance._shrunk_covariance import ShrunkCovariance
 
 __all__ = [
     "BaseCovariance",
@@ -26,4 +25,5 @@ __all__ = [
     "OAS",
     "ShrunkCovariance",
     "GraphicalLassoCV",
+    "ImpliedCovariance",
 ]

skfolio/moments/covariance/_base.py

@@ -22,19 +22,19 @@ class BaseCovariance(skb.BaseEstimator, ABC):
 
     Parameters
     ----------
-    nearest : bool, default=False
+    nearest : bool, default=True
         If this is set to True, the covariance is replaced by the nearest covariance
         matrix that is positive definite and with a Cholesky decomposition than can be
-        computed. The variance is left unchanged. A covariance matrix is in theory PSD.
-        However, due to floating-point inaccuracies, we can end up with a covariance
-        matrix that is slightly non-PSD or where Cholesky decomposition is failing.
-        This often occurs in high dimensional problems.
-        For more details, see :func:`~skfolio.units.stats.cov_nearest`.
-        The default is `False`.
+        computed. The variance is left unchanged.
+        A covariance matrix that is not positive definite often occurs in high
+        dimensional problems. It can be due to multicollinearity, floating-point
+        inaccuracies, or when the number of observations is smaller than the number of
+        assets. For more details, see :func:`~skfolio.utils.stats.cov_nearest`.
+        The default is `True`.
 
     higham : bool, default=False
         If this is set to True, the Higham & Nick (2002) algorithm is used to find the
-        nearest PSD covariance, otherwise the eigenvalues are clipped to a threshold
+        nearest PD covariance, otherwise the eigenvalues are clipped to a threshold
         above zeros (1e-13). The default is `False` and use the clipping method as the
         Higham & Nick algorithm can be slow for large datasets.
 
@@ -59,7 +59,7 @@ class BaseCovariance(skb.BaseEstimator, ABC):
     @abstractmethod
     def __init__(
         self,
-        nearest: bool = False,
+        nearest: bool = True,
         higham: bool = False,
         higham_max_iteration: int = 100,
     ):
@@ -103,6 +103,7 @@ class BaseCovariance(skb.BaseEstimator, ABC):
                 covariance,
                 higham=self.higham,
                 higham_max_iteration=self.higham_max_iteration,
+                warn=True,
             )
         # set covariance
         self.covariance_ = covariance

skfolio/moments/covariance/_denoise_covariance.py

@@ -0,0 +1,181 @@
+"""Covariance Denoising Estimators."""
+
+# Copyright (c) 2023
+# Author: Hugo Delatte <delatte.hugo@gmail.com>
+# License: BSD 3 clause
+# Implementation derived from:
+# scikit-learn, Copyright (c) 2007-2010 David Cournapeau, Fabian Pedregosa, Olivier
+# Grisel Licensed under BSD 3 clause.
+
+import numpy as np
+import numpy.typing as npt
+import scipy.optimize as sco
+import sklearn.neighbors as skn
+import sklearn.utils.metadata_routing as skm
+
+from skfolio.moments.covariance._base import BaseCovariance
+from skfolio.moments.covariance._empirical_covariance import EmpiricalCovariance
+from skfolio.utils.stats import corr_to_cov, cov_to_corr
+from skfolio.utils.tools import check_estimator
+
+
+class DenoiseCovariance(BaseCovariance):
+    """Covariance Denoising estimator.
+
+    The goal of Covariance Denoising is to reduce the noise and enhance the signal of
+    the empirical covariance matrix [1]_.
+    It reduces the ill-conditioning of the traditional covariance estimate by
+    differentiating the eigenvalues associated with noise from the eigenvalues
+    associated with signal.
+    Denoising replaces the eigenvalues of the eigenvectors classified as random by
+    Marčenko-Pastur with a constant eigenvalue.
+
+    Parameters
+    ----------
+    covariance_estimator : BaseCovariance, optional
+        :ref:`Covariance estimator <covariance_estimator>` to estimate the covariance
+        matrix that will be denoised.
+        The default (`None`) is to use :class:`~skfolio.moments.EmpiricalCovariance`.
+
+    nearest : bool, default=True
+        If this is set to True, the covariance is replaced by the nearest covariance
+        matrix that is positive definite and with a Cholesky decomposition than can be
+        computed. The variance is left unchanged.
+        A covariance matrix that is not positive definite often occurs in high
+        dimensional problems. It can be due to multicollinearity, floating-point
+        inaccuracies, or when the number of observations is smaller than the number of
+        assets. For more details, see :func:`~skfolio.utils.stats.cov_nearest`.
+        The default is `True`.
+
+    higham : bool, default=False
+        If this is set to True, the Higham & Nick (2002) algorithm is used to find the
+        nearest PD covariance, otherwise the eigenvalues are clipped to a threshold
+        above zeros (1e-13). The default is `False` and use the clipping method as the
+        Higham & Nick algorithm can be slow for large datasets.
+
+    higham_max_iteration : int, default=100
+        Maximum number of iteration of the Higham & Nick (2002) algorithm.
+        The default value is `100`.
+
+    Attributes
+    ----------
+    covariance_ : ndarray of shape (n_assets, n_assets)
+        Estimated covariance.
+
+    covariance_estimator_ : BaseCovariance
+        Fitted `covariance_estimator`.
+
+    n_features_in_ : int
+        Number of assets seen during `fit`.
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of assets seen during `fit`. Defined only when `X`
+        has assets names that are all strings.
+
+    References
+    ----------
+    .. [1]  "Machine Learning for Asset Managers".
+        Elements in Quantitative Finance.
+        Lòpez de Prado (2020).
+    """
+
+    covariance_estimator_: BaseCovariance
+
+    def __init__(
+        self,
+        covariance_estimator: BaseCovariance | None = None,
+        nearest: bool = True,
+        higham: bool = False,
+        higham_max_iteration: int = 100,
+    ):
+        super().__init__(
+            nearest=nearest,
+            higham=higham,
+            higham_max_iteration=higham_max_iteration,
+        )
+        self.covariance_estimator = covariance_estimator
+
+    def get_metadata_routing(self):
+        # noinspection PyTypeChecker
+        router = skm.MetadataRouter(owner=self.__class__.__name__).add(
+            covariance_estimator=self.covariance_estimator,
+            method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def fit(self, X: npt.ArrayLike, y=None, **fit_params) -> "DenoiseCovariance":
+        """Fit the Covariance Denoising estimator.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_observations, n_assets)
+          Price returns of the assets.
+
+        y : Ignored
+           Not used, present for API consistency by convention.
+
+        **fit_params : dict
+            Parameters to pass to the underlying estimators.
+            Only available if `enable_metadata_routing=True`, which can be
+            set by using ``sklearn.set_config(enable_metadata_routing=True)``.
+            See :ref:`Metadata Routing User Guide <metadata_routing>` for
+            more details.
+
+        Returns
+        -------
+        self : DenoiseCovariance
+           Fitted estimator.
+        """
+        routed_params = skm.process_routing(self, "fit", **fit_params)
+
+        # fitting estimators
+        self.covariance_estimator_ = check_estimator(
+            self.covariance_estimator,
+            default=EmpiricalCovariance(),
+            check_type=BaseCovariance,
+        )
+        # noinspection PyArgumentList
+        self.covariance_estimator_.fit(X, y, **routed_params.covariance_estimator.fit)
+
+        # we validate and convert to numpy after all models have been fitted to keep
+        # features names information.
+        X = self._validate_data(X)
+        n_observations, n_assets = X.shape
+        q = n_observations / n_assets
+        corr, std = cov_to_corr(self.covariance_estimator_.covariance_)
+        e_val, e_vec = np.linalg.eigh(corr)
+        indices = e_val.argsort()[::-1]
+        e_val, e_vec = e_val[indices], e_vec[:, indices]
+
+        def _marchenko(x_var):
+            e_min, e_max = (
+                x_var * (1 - (1.0 / q) ** 0.5) ** 2,
+                x_var * (1 + (1.0 / q) ** 0.5) ** 2,
+            )
+            e_val_lin = np.linspace(e_min, e_max, 1000)
+            pdf_0 = (
+                q
+                / (2 * np.pi * x_var * e_val_lin)
+                * ((e_max - e_val_lin) * (e_val_lin - e_min)) ** 0.5
+            )
+            kde = skn.KernelDensity(kernel="gaussian", bandwidth=0.01).fit(
+                e_val.reshape(-1, 1)
+            )
+            # noinspection PyUnresolvedReferences
+            pdf_1 = np.exp(kde.score_samples(pdf_0.reshape(-1, 1)))
+            return np.sum((pdf_1 - pdf_0) ** 2)
+
+        # noinspection PyTypeChecker
+        res = sco.minimize(_marchenko, x0=0.5, bounds=((1e-5, 1 - 1e-5),))
+
+        var = res["x"][0]
+        n_facts = e_val.shape[0] - e_val[::-1].searchsorted(
+            var * (1 + (1.0 / q) ** 0.5) ** 2
+        )
+        e_val_ = e_val.copy()
+        e_val_[n_facts:] = e_val_[n_facts:].sum() / float(e_val_.shape[0] - n_facts)
+        corr = e_vec @ np.diag(e_val_) @ e_vec.T
+        corr, _ = cov_to_corr(corr)
+        covariance = corr_to_cov(corr, std)
+        self._set_covariance(covariance)
+        return self