PyIAML 1.0.0__py3-none-any.whl

iaml/iaml_pipeline.py

@@ -0,0 +1,600 @@
+"""Based on Scikit-learn Pipeline but for IAML Pipelines !
+Apply preprocessing in construction order, then predict from Candidate instance.
+"""
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+import pickle
+
+from copy import deepcopy
+from hashlib import md5
+
+import numpy as np
+import shap
+import pandas as pd
+
+from sklearn.pipeline import Pipeline
+
+from .dataset import Dataset
+from .void_step import VoidStep
+from .explanation import Explanation
+from .cache import Cache
+from .reference import Reference
+
+if TYPE_CHECKING:
+    from .metric import Metric
+    from .step import Step
+
+class IAMLPipeline(Pipeline):
+    """Based on Scikit-learn Pipeline but for IAML Pipelines !
+    Apply preprocessing in construction order, then predict from Candidate instance.
+    
+    :param list[tuple[str, Step]], optional steps: Ordered list of IAML.Steps. Defaults to None.
+    :param pd.DataFrame, optional original_dataset: Untransformed dataset to use as a masker 
+                for the SHAP explainer which will be used to explain the model later on. 
+                Defaults to None. If not provided, the prediction dataset will be used as the 
+                masker, which may impact the accuracy of the explanations.
+    :param str, optional estimator_type: Type of estimator. Must be one of 'classifier', 
+        'survival', 'regressor'.
+    """
+
+    def __init__(
+        self,
+        steps: list[tuple[str, Step]] = None,
+        original_dataset: pd.DataFrame = None,
+        estimator_type: str = None) -> None:
+        if steps is None:
+            steps = []
+
+        self.original_dataset: pd.DataFrame = original_dataset
+        """Original dataset used for this pipeline"""
+
+        self._preprocessing_steps: list[tuple[str, object]] = []
+        """Transformers and resamplers in their construction order."""
+
+        self.predictor: tuple[str, object] = None
+        """Predictor that'll be used in this pipeline"""
+
+        self.metrics: list[Metric] = []
+        """List of metrics that'll be computed in this pipeline"""
+
+        self._fingerprint_cache: str | None = None
+        """Cached fingerprint of training steps."""
+
+        self._transform_fingerprint_cache: str | None = None
+        """Cached fingerprint of transforms/resamplers."""
+
+        self._fingerprint_cache_version: tuple | None = None
+        """Cached config versions for training steps."""
+
+        self._transform_fingerprint_cache_version: tuple | None = None
+        """Cached config versions for transforms/resamplers."""
+
+        self._trained_columns: list[str] | None = None
+        """Columns seen by the predictor during fit (after transforms)."""
+
+        if estimator_type not in ['classifier', 'regressor', 'survival']:
+            raise ValueError(f"Estimator type ({estimator_type}) must be classifier, \
+                survival or regressor")
+        self.__estimator_type: str = estimator_type
+        """Estimator type"""
+
+        super().__init__(steps)
+
+    @property
+    def _estimator_type(self) -> str:
+        """Expose the estimator type to scikit-learn.
+        
+        :return: estimator type.
+        """
+        return self.__estimator_type
+
+    @property
+    def estimator_type(self) -> str:
+        """Return the estimator type configured for this pipeline.
+        
+        :return: estimator type.
+        """
+        return self.__estimator_type
+
+    @property
+    def transformers(self) -> list[tuple[str, object]]:
+        """Transform steps in execution order, excluding training-only resamplers."""
+        return [step for step in self._preprocessing_steps
+                if callable(getattr(step[1], 'transform', None))]
+
+    @property
+    def resamplers(self) -> list[tuple[str, object]]:
+        """Resampling steps in execution order."""
+        return [step for step in self._preprocessing_steps
+                if not callable(getattr(step[1], 'transform', None))
+                and callable(getattr(step[1], 'resample', None))]
+
+    @property
+    def steps(self) -> list[tuple[str, object]]:
+        """Get prediction steps, excluding training-only resamplers.
+        
+        :return: list of steps
+        """
+        return [item for item in [*self.transformers, self.predictor] if item is not None]
+
+    @property
+    def training_steps(self) -> list[tuple[str, object]]:
+        """Steps used to fit the pipeline, preserving preprocessing order.
+
+        :return: list of steps
+        """
+        return [item for item in [*self._preprocessing_steps, self.predictor] \
+            if item is not None]
+
+    @steps.setter
+    def steps(self, values: list[tuple[str, object]]) -> list[tuple[str, object]]:
+        """Set pipeline steps
+        
+        :param list[tuple[str, object]] values: List of steps to add.
+        :return: List of new steps.
+        """
+        self._preprocessing_steps = []
+        self.predictor = None
+        self._invalidate_fingerprint_cache()
+
+        for value in values:
+            self.__add_step(value)
+
+        return self.steps
+
+    def __add_step(self, step: tuple[str, object]) -> None:
+        """Add a step to the pipeline steps
+        
+        :param tuple[str,object] step: The step to add.
+        """
+        _, instance = step
+        self._invalidate_fingerprint_cache()
+        if hasattr(instance, 'predict') and callable(instance.predict):
+            self.predictor = step
+        elif callable(getattr(instance, 'transform', None)) \
+                or callable(getattr(instance, 'resample', None)):
+            self._preprocessing_steps.append(step)
+
+    def replace_step(self, old: 'Step', new: 'Step') -> bool:
+        """Replace a step in the pipeline by another (by id)
+
+        :param Step old: The step to replace.
+        :param Step new: The new step.
+        :return: Was replaced ?
+        """
+        for idx, step in enumerate(self._preprocessing_steps):
+            if old is step[1]:
+                self._preprocessing_steps[idx] = (new.name, new)
+                self._invalidate_fingerprint_cache()
+                return True
+        if self.predictor is not None and old is self.predictor[1]:
+            self.predictor = (new.name, new)
+            self._invalidate_fingerprint_cache()
+            return True
+
+        return False
+
+    def remove_step(self, to_remove: 'Step') -> bool:
+        """Remove a step from the pipeline (by object id)
+
+        :param Step to_remove: Step to remove.
+        :return: Step was removed ?
+        """
+        for idx, step in enumerate(self._preprocessing_steps):
+            if to_remove is step[1]:
+                del self._preprocessing_steps[idx]
+                self._invalidate_fingerprint_cache()
+                return True
+        if self.predictor is not None and to_remove is self.predictor[1]:
+            self.predictor = None
+            self._invalidate_fingerprint_cache()
+            return True
+
+        return False
+
+    def fit(
+        self,
+        X: pd.DataFrame,
+        y: pd.DataFrame = None,
+        only_predictor: bool = False,
+        groups_columns: list[str] = None,
+        metrics: list[Metric] = None,
+        **kwargs: dict) -> 'IAMLPipeline':
+        """Fit Pipeline on new data (or with new parameters)
+        
+        :param pd.DataFrame X: Candidate features.
+        :param pd.DataFrame, optional y: label to predict. Default to None.
+        :param bool, optional only_predictor: Run predictions only. Default to False.
+        :param list[str], optional groups_columns: Columns name to use in splitting. 
+            Default to None.
+        :param list[Metric], optional metrics: List of Metrics to compute. Default to None.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :return: Fitted IAMLPipeline.
+        """
+        self.metrics = metrics
+
+        if groups_columns is None:
+            groups_columns = []
+
+        if not only_predictor:
+            X, y = self.fit_transform(X, y, groups_columns=groups_columns, **kwargs)
+            # Reset groups_columns as returned X is aldready pruned from groups columns
+            # This way we avoid caching KeyError in dataset init
+            groups_columns = []
+        dataset = Dataset(X, y, groups_columns=groups_columns)
+
+        if self.predictor[1].suitable(dataset):
+            if isinstance(dataset.X, pd.DataFrame):
+                self._trained_columns = list(dataset.X.columns)
+            self.predictor[1].fit(dataset, **kwargs)
+        else:
+            self.predictor = None
+
+        return self
+
+    def fit_transform(
+        self,
+        X: pd.DataFrame,
+        y: pd.DataFrame = None,
+        groups_columns: list[str] = None,
+        **kwargs: dict) -> 'IAMLPipeline':
+        """Fit Pipeline and transform data 
+        
+        :param pd.DataFrame X: Candidate features.
+        :param pd.DataFrame, optional y: Label to predict. Default to None.
+        :param list[str], optional groups_columns: Columns name to use in splitting. 
+            Default to None.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :return: Fitted IAMLPipeline.
+        """
+        if groups_columns is None:
+            groups_columns = []
+
+        dataset = Dataset(X, y, groups_columns=groups_columns)
+
+        # Cached fits can replace steps, and unsuitable steps can be removed.
+        for _, step in list(self._preprocessing_steps):
+            # FIT
+            if 'Step' in map(lambda s: s.__name__, step.__class__.__mro__):
+                fit_key = f"fit_{step.fingerprint()}"
+                fit_data_key = dataset.fingerprint()
+                from_cache = Cache().from_cache(fit_key, fit_data_key)
+                if from_cache is not None:
+                    fitted_step, dataset = from_cache
+                    self.replace_step(step, fitted_step)
+                    step = fitted_step
+                else:
+                    if step.suitable(dataset):
+                        step.fit(dataset)
+                        # Copy together to preserve shared references to training data
+                        # (e.g. a target encoder's out-of-fold training transform).
+                        Cache().add_to_cache(fit_key, fit_data_key, (step, dataset))
+                    else:
+                        if step.is_interchangeable:
+                            old_step = step
+                            step = VoidStep(step_to_mimic=step)
+                            self.replace_step(old_step, step)
+                        else:
+                            self.remove_step(step)
+                            continue
+            else:
+                step.fit(dataset.X, dataset.y, **kwargs)
+
+            # APPLY TRANSFORM / RESAMPLE
+            apply_key = f"apply_{step.fingerprint()}"
+            # Freeze before a step can mutate X or y in place.
+            apply_data_key = dataset.fingerprint()
+            dataset_from_cache = Cache().from_cache(apply_key, apply_data_key)
+
+            if dataset_from_cache is not None:
+                dataset = dataset_from_cache
+            else:
+                if hasattr(step, 'transform'):
+                    dataset.transform(step.transform)
+                elif hasattr(step, 'resample'):
+                    dataset = dataset.resample(step.resample)
+                Cache().add_to_cache(apply_key, apply_data_key, dataset)
+
+        return dataset.X, dataset.y
+
+    @property
+    def explanations(self) -> list[str]:
+        """Get explanations from all pipeline steps
+
+        :return: List of markdown explanations
+        """
+        return [ e for _, step in self.training_steps if (e := step.explain()) is not None ]
+
+    @property
+    def model(self) -> Step:
+        """Shortcut to get the prediction model of IAMLPipeline 
+
+        :return: Prediction model of the pipeline (or None)
+        """
+        return self.predictor
+
+    def add_transform(self, instance: Step) -> None:
+        """Add transform Step to the Pipeline
+
+        :param Step instance: Step to add (must implement transform).
+        :raise ValueError: Step must implement transform method.
+        """
+        if instance and hasattr(instance, 'transform'):
+            self._invalidate_fingerprint_cache()
+            self._preprocessing_steps.append((str(instance), instance))
+        else:
+            raise ValueError("Step must implement transform method")
+
+    def add_resample(self, instance: Step) -> None:
+        """Add resample Step to the Pipeline
+
+        :param Step instance: Step to add (must implement resample).
+        :raise ValueError: Step must implement resample method.
+        """
+        if instance and hasattr(instance, 'resample'):
+            self._invalidate_fingerprint_cache()
+            self._preprocessing_steps.append((str(instance), instance))
+        else:
+            raise ValueError("Step must implement resample method")
+
+    def set_model(self, instance: Step) -> None:
+        """
+        Set the predict model (Step) of the Pipeline
+
+        :param Step instance: Step to add (must implement predict).
+        :raise ValueError: Step must implement predict method.
+        """
+        self._invalidate_fingerprint_cache()
+        self.predictor = (str(instance), instance)
+
+    def copy(self) -> IAMLPipeline:
+        """Return a copied IAMLPipeline
+
+        :return: Copied IAMLPipeline instance.
+        """
+        return deepcopy(self)
+
+    def pickle(self) -> bytes:
+        """Serialize IAMLPipeline to bytes.
+        Can be save into a file and reload with pickle.
+
+        :return: Serialized IAMLPipeline.
+        """
+        return pickle.dumps(self)
+
+    @property
+    def have_model(self) -> bool:
+        """Does the IAMLPipeline have a model set?
+
+        :return: True if a model has been set.
+        """
+        return bool(self.predictor)
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame: # pylint: disable=arguments-differ
+        """Apply transformers without predict
+
+        :param pd.DataFrame X: candidate data.
+        :return: Transformed DF.
+        """
+        for _, step in self.transformers:
+            X = step.transform(X)
+
+        return X
+
+    def predict(self, X: pd.DataFrame, model_only: bool = False, **kwargs: dict) -> list:
+        """Run all the steps to predict labels from candidate data
+
+        :param pd.DataFrame X: Features used as candidate of the pipeline.
+        :param bool, optional model_only: True to execute only the model with already transformed 
+            data. Defaults to False.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :raise ValueError: Model must have been set before call predict.
+        :return: Predicted values
+        """
+        if not self.have_model:
+            raise ValueError("Model need to be set before predict")
+
+        if not model_only:
+            return super().predict(X, **kwargs)
+
+        if self._trained_columns and isinstance(X, pd.DataFrame):
+            X = X.reindex(columns=self._trained_columns, fill_value=0)
+        return self.predictor[1].predict(X)
+
+    def predict_survival_function(
+        self,
+        X: pd.DataFrame,
+        model_only: bool = False,
+        **kwargs) -> list:
+        """Run all the steps to predict survival function  from candidate data
+
+        :param pd.DataFrame X: Features used as candidate of the pipeline.
+        :param bool, optional model_only: True to execute only the model with already transformed 
+            data. Defaults to False.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :raise ValueError: Model must have been set before call predict.
+        :return: Predicted values
+        """
+        if not self.have_model:
+            raise ValueError("Model need to be set before predict")
+
+        if not model_only:
+            X = self.transform(X, **kwargs)
+
+        if model_only and self._trained_columns and isinstance(X, pd.DataFrame):
+            X = X.reindex(columns=self._trained_columns, fill_value=0)
+        return self.predictor[1].predict_survival_function(X)
+
+    def predict_proba(self, X: pd.DataFrame, model_only: bool = False, **kwargs) -> list:
+        """Run all the steps to predict labels from candidate data
+
+        :param pd.DataFrame X: Features used as candidate of the pipeline.
+        :param bool, optional model_only: True to execute only the model with already transformed 
+            data. Defaults to False.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :raise ValueError: Model must have been set before call predict.
+        :return: Predicted values
+        """
+        if not self.have_model:
+            raise ValueError("Model need to be set before predict")
+
+        if not model_only:
+            return super().predict_proba(X, **kwargs)
+
+        if self._trained_columns and isinstance(X, pd.DataFrame):
+            X = X.reindex(columns=self._trained_columns, fill_value=0)
+        return self.predictor[1].predict_proba(X)
+
+    def __getattribute__(self, attr: str) -> bool:
+        """Overload getattr to allow accurate hasattr on predict_proba
+
+        :param str attr: Attribute to test.
+        :raise AttributeError: predict_proba not implemented in this model.
+        :return: Does attribute is implemented.
+        """
+        if attr == 'predict_proba' \
+            and not( \
+                self.have_model and hasattr(self.predictor[1], 'predict_proba') \
+            ):
+            raise AttributeError("predict_proba not implemented in this model")
+
+        return super().__getattribute__(attr)
+
+    @property
+    def optimizable_step(self) -> list['Step']:
+        """Return steps whose parameters or choice of implementation can change.
+        
+        :return: List of optimizable step
+        """
+        return [step for _, step in self.training_steps
+                if step.optimizable or step.is_interchangeable]
+
+    def __eq__(self, other: 'IAMLPipeline') -> bool:
+        """Compare two pipelines
+
+        :param IAMLPipeline other: The pipeline to compare.
+        :return: Equal or not ?
+        """
+        if isinstance(other, IAMLPipeline):
+            return self.fingerprint() == other.fingerprint()
+        return NotImplemented
+
+    @property
+    def name(self) -> str:
+        """Return pipeline formatted name
+        
+        :return: formatted name
+        """
+        return ' '.join(x.title() for x in str(self.model[0]).split('_'))
+
+    def explain_model(self, X: pd.DataFrame, nsamples: int = 20) -> Explanation:
+        """Explains the model by computing SHAP values on the fitted model.
+        Uses the train set as the masker, and the provided set as
+        prediction.
+
+        :param pd.DataFrame X: Prediction set to compute SHAP values for.
+        :param int, optional nsamples: Number of samples to pick from the masker to pick feature 
+            data from for each row in the provided prediction dataset. More samples means more 
+            accurate SHAP values and longer computing times. Defaults to 20.
+        :raise RuntimeError: There is no model to explain.
+        :return: Model explanation, with an overview of the most important features, and graphs.
+        """
+        if not self.have_model:
+            raise RuntimeError('There is no model to explain.')
+
+        def p(pred_data):
+            df = pd.DataFrame(pred_data, columns=X.columns)
+
+            if hasattr(self, 'predict_proba'):
+                return self.predict_proba(df)[:, 1]
+
+            # when the regressor does not implement predict_proba
+            return self.predict(df)
+
+        mask_dataset = self.original_dataset if self.original_dataset is not None \
+            and not self.original_dataset.empty else X
+
+        explainer = shap.KernelExplainer(p, mask_dataset)
+        shap_values = explainer.shap_values(X, nsamples=nsamples)
+
+        shap_explanation = shap.Explanation(
+            shap_values,
+            base_values=np.tile(explainer.expected_value, (shap_values.shape[0], 1)),
+            data=X.to_numpy(),
+            feature_names=X.columns.to_list(),
+            output_names=X.columns.to_list())
+
+        return Explanation(shap_explanation)
+
+    # Implement scikit-learn estimator's methods
+    def __sklearn_is_fitted__(self):
+        return self.have_model
+
+    def __sklearn_clone__(self):
+        return deepcopy(self)
+
+    def target_type_(self) -> str:
+        """Mimic Scikit-learn API
+        Return models target type
+        """
+        return self.original_dataset.type_of_target
+
+    # Fingerprint (used by cache)
+    def fingerprint(self) -> str:
+        """Return a md5 hash that can by use to compare Pipelines 
+
+        :return: md5 sting
+        """
+        current_version = tuple(
+            (id(step), getattr(step, "_config_version", None))
+            for _, step in self.training_steps
+        )
+        if self._fingerprint_cache is None or self._fingerprint_cache_version != current_version:
+            to_hash = "\n".join([step.fingerprint() for _, step in self.training_steps])
+            self._fingerprint_cache = md5(to_hash.encode()).hexdigest()
+            self._fingerprint_cache_version = current_version
+        return self._fingerprint_cache
+
+    def transformers_resamplers_fingerprint(self) -> str:
+        """Fingerprint for transformers/resamplers only (used by Candidate)."""
+        current_version = tuple(
+            (id(step), getattr(step, "_config_version", None))
+            for _, step in self._preprocessing_steps
+        )
+        if self._transform_fingerprint_cache is None \
+            or self._transform_fingerprint_cache_version != current_version:
+            to_hash = "\n".join([
+                step.fingerprint()
+                for _, step in self._preprocessing_steps
+            ])
+            self._transform_fingerprint_cache = md5(to_hash.encode()).hexdigest()
+            self._transform_fingerprint_cache_version = current_version
+        return self._transform_fingerprint_cache
+
+    def _invalidate_fingerprint_cache(self) -> None:
+        self._fingerprint_cache = None
+        self._transform_fingerprint_cache = None
+        self._fingerprint_cache_version = None
+        self._transform_fingerprint_cache_version = None
+
+    def bibliography(self, structured: bool) -> str | list[dict]:
+        """Return a string listing all step's references or a structured list of dict.
+        
+        :param bool structured: JSON structured bibliography or not.
+        :return str | list[dict]: Bibliography.
+        """
+        references = [reference for step in self.steps
+            for reference in step[1].references] \
+            + [reference for metric in self.metrics for reference in metric.get_refs()] \
+            + [Reference({
+                'year': 2017,
+                'name': 'A Unified Approach to Interpreting Model Predictions',
+                'authors': [
+                    'Scott M. Lundberg', 'Su-In Lee'
+                ],
+                'doi': 'https://doi.org/10.48550/arXiv.1705.07874',
+                'publisher': 'arXiv preprint arXiv:1705.07874'
+            }, 'Shap')]
+
+        return Reference.bibliography(references, structured)