PyIAML 1.0.0__py3-none-any.whl

iaml/core_dispatcher.py

@@ -0,0 +1,125 @@
+"""Singleton used by IAML to dispatch cores to process"""
+import threading
+import warnings
+import multiprocess
+import multiprocess.managers
+import psutil
+from .meta_singleton import MetaSingleton
+from .logger import Logger
+
+
+class CoreDispatcher(metaclass=MetaSingleton):
+    """Singleton used by IAML to dispatch cores to process
+    
+    :param tuple, optional \\*args: Additional parameters.
+    :param dict, optional \\**kwargs: Additional parameters.
+    """
+    def __init__(self, *args, **kwargs) -> None:  #pylint: disable=unused-argument
+        self.__all_cores: set[int] = set(range(psutil.cpu_count(logical=True)))
+        """Set of all cores available"""
+
+        self.manager: multiprocess.Manager | None = None
+        """Manager used to dispatch cores to process (None when not available)."""
+
+        self._lock: threading.RLock | multiprocess.managers.SyncManager.RLockProxy
+        """Lock protecting access to booked cores."""
+
+        self.books: list | multiprocess.managers.ListProxy
+        """Booked cores registry."""
+
+        try:
+            self.manager = multiprocess.Manager()
+            self._lock = self.manager.RLock()
+            self.books = self.manager.list([])
+        except Exception as exc:  # pylint: disable=broad-except
+            warnings.warn(f"CoreDispatcher fallback to local mode (manager start failed: {exc!r})")
+            self.manager = None
+            self._lock = threading.RLock()
+            self.books = []
+
+    def reset_books(self) -> None:
+        """Set all cpu cores as available"""
+        with self._lock:
+            if hasattr(self.books, "clear"):
+                self.books.clear()
+            else:
+                self.books[:] = []
+
+    @property
+    def available_cores(self) -> set[int]:
+        """All cores minus booked ones
+        
+        :return: Set of available cpu cores
+        """
+        avail = set(self.__all_cores)
+        with self._lock:
+            for book in list(self.books):
+                avail -= set(book['cores'])
+
+        return avail
+
+    def __book_cpu(self, pids: set[int], number: int) -> None:
+        """Affiliate CPU cores to process
+
+        :param set[int] pids: pids to set affinity with.
+        :param int number: Number of cores to book.
+        :raise RuntimeError: Not enough CPU cores available.
+        """
+        self.__free_cores()
+        with self._lock:
+            if len(self.available_cores) < number:
+                raise RuntimeError('Not enough CPU cores available')
+
+            to_book = list(self.available_cores)[:number]
+            entry = {
+                'pids': pids,
+                'cores': to_book
+            }
+            self.books.extend([entry])
+
+        for pid in pids:
+            process = psutil.Process(pid)
+            try:
+                process.cpu_affinity(to_book)
+            except (psutil.AccessDenied, AttributeError, NotImplementedError):
+                Logger().warning(
+                    'Unable to set CPU affinity for process '
+                    f'{pid}. Continuing without affinity control.'
+                )
+
+    def affiliate(self, pids: list[int], core_number: int = 1) -> None:
+        """Run process with CPU affinity
+
+        :param list[int] pids:  Process to affiliate with CPU cores.
+        :param int, optional core_number: number of process to run 
+            (each process run the same target). Defaults to 1.
+        """
+        try:
+            test_process = psutil.Process(pids[0])
+        except psutil.Error:
+            Logger().warning('Unable to inspect process for CPU affinity; skipping affinity control.')
+            return
+
+        if hasattr(test_process, 'cpu_affinity'):
+            try:
+                self.__book_cpu(pids, core_number)
+            except RuntimeError as exc:
+                Logger().warning(str(exc))
+        else:
+            Logger().warning(
+                'Your OS doesn\'t support CPU affinity. We are not able to control CPU cores access'
+            )
+
+    def __free_cores(self) -> None:
+        """Free cores that are not used anymore"""
+        with self._lock:
+            new_books = []
+            for book in self.books:
+                if any(psutil.pid_exists(pid) and psutil.Process(pid).is_running()
+                    for pid in book['pids']):
+                    new_books.append(book)
+
+            self.reset_books()
+            self.books.extend(new_books)
+
+CoreDispatcher() # Run it a first time to init shared objects

iaml/data_type.py

@@ -0,0 +1,11 @@
+"""Types of data used in Dataset"""
+from enum import IntEnum
+
+
+class DataType(IntEnum):
+    """Types of data used in Dataset"""
+    CATEGORICAL = 0
+    TEXT = 1
+    SHORT_TEXT = 2
+    NUMERIC = 3
+    DATE = 4

iaml/dataset.py

@@ -0,0 +1,506 @@
+"""Encapsulate X, y data to be used by Steps
+Add features like data type detection and splitting 
+"""
+import copy
+from copy import deepcopy
+from typing import Any, Iterator, TYPE_CHECKING
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
+import numpy as np
+import pandas as pd
+
+from .cache_keys import hash_dataset
+from .data_type import DataType
+from .type_of_target import type_of_target
+from .logger import Logger
+
+
+if TYPE_CHECKING:
+    from .iaml_pipeline import IAMLPipeline
+
+
+class Dataset:
+    """Encapsulate X, y data to be used by Steps
+    Add features like data type detection and splitting
+    
+    :param pd.DataFrame X: The dataframe used in this Dataset.
+    :param list, optional y: The dataframe target. Default to None.
+    :param pd.DataFrame, optional groups: If set, will be used when splitting to avoid
+        having data rows for a similar ID in two different splits. Default to None.
+    :param list[str], optional groups_columns: Same as groups, but with column names. 
+        Default to None.
+    :param dict, optional columns_types: Specify column types instead of detecting them 
+        automaticaly. Default to None.
+    
+    """
+
+    def __init__(self,
+        X: pd.DataFrame,
+        y: list = None,
+        groups: pd.DataFrame = None,
+        groups_columns: list[str] = None,
+        columns_types: dict = None
+    ):
+        type_of_target_override = None
+        if isinstance(y, str):
+            type_of_target_override = y
+            y = []
+
+        if columns_types is None and isinstance(groups, dict) and not groups_columns:
+            if self._looks_like_columns_types(groups):
+                columns_types = groups
+                groups = None
+
+        if groups is not None and groups_columns:
+            raise ValueError("groups and groups_columns are not None. Only one must be set")
+
+        if isinstance(groups, pd.DataFrame):
+            if set(groups.columns).intersection(X.columns):
+                raise ValueError("Group columns present in dataset!")
+
+        if groups_columns is None:
+            groups_columns = []
+
+        self.__X: pd.DataFrame = X.drop(columns=groups_columns)
+        """The dataframe used in this Dataset object without groups columns if provided"""
+
+        # Make sure y is either None or single column
+        if isinstance(y, pd.DataFrame):
+            if len(y.columns) > 1:
+                raise ValueError("IAML only handle single column labels !")
+            y = y.values.ravel()
+        self.__y: np.array = np.array(y)
+        """The dataframe target"""
+
+        if groups_columns:
+            self.groups = X[groups_columns]
+        else:
+            if isinstance(groups, dict):
+                self.groups = pd.DataFrame(groups)
+            elif isinstance(groups, (pd.Series, list, np.ndarray)):
+                self.groups = pd.DataFrame(groups)
+            else:
+                self.groups = groups
+
+        if self.groups is not None and self.groups.shape[1] > 1:
+            # Create a combined group label by concatenating all columns into tuples
+            Logger().warning("You are using multiple columns as groups. \
+                Be careful, as these columns will serve as a composite key.")
+            self.groups = pd.DataFrame(pd.Series(
+                list(zip(*[self.groups[col] for col in self.groups.columns]))),
+                columns=['groups']
+                )
+
+        if self.groups is not None and len(self.groups) != len(self.__X):
+            raise ValueError("Groups must have one row per feature row")
+
+        self.columns_types: dict = self._normalize_columns_types(columns_types, self.__X)
+        """Columns types to be applied to our dataframe columns"""
+
+        self.__detect_columns_types()
+
+        self.type_of_target: str = None
+        """Type of target to predict"""
+
+        if type_of_target_override is not None:
+            self.type_of_target = type_of_target_override
+        elif y is not None:
+            self.type_of_target = type_of_target(self.__y)
+
+    @staticmethod
+    def _normalize_columns_types(columns_types: dict | None, X: pd.DataFrame) -> dict:
+        if not columns_types:
+            return {}
+
+        if all(isinstance(key, DataType) for key in columns_types.keys()):
+            normalized = {}
+            for data_type, columns in columns_types.items():
+                if columns is None:
+                    continue
+                if isinstance(columns, (str, bytes)):
+                    columns = [columns]
+                for column in columns:
+                    if column in X.columns:
+                        normalized[column] = (X[column].dtype, data_type)
+            return normalized
+
+        if all(isinstance(value, DataType) for value in columns_types.values()):
+            normalized = {}
+            for column, data_type in columns_types.items():
+                if column in X.columns:
+                    normalized[column] = (X[column].dtype, data_type)
+            return normalized
+
+        return columns_types
+
+    @staticmethod
+    def _looks_like_columns_types(columns_types: dict) -> bool:
+        if not columns_types:
+            return False
+
+        keys = columns_types.keys()
+        values = columns_types.values()
+
+        if all(isinstance(key, DataType) for key in keys):
+            return True
+
+        if all(isinstance(value, DataType) for value in values):
+            return True
+
+        if all(
+            isinstance(value, tuple)
+            and len(value) == 2
+            and isinstance(value[1], DataType)
+            for value in values
+        ):
+            return True
+
+        return False
+
+    @property
+    def features(self) -> list[str]:
+        """List columns names of X data
+
+        :return: columns names.
+        """
+        return self.X.columns.to_list()
+
+    @property
+    def X(self) -> pd.DataFrame:
+        """X data getter
+
+        :return: X data
+        """
+        return self.__X
+
+    @property
+    def y(self) -> np.ndarray:
+        """y data getter
+
+        :return: y data
+        """
+        return self.__y
+
+    def copy(self, deep: bool = True) -> 'Dataset':
+        """Copy Dataset into a new instance
+
+        :param bool, optional deep: Perform a deep copy. Defaults to True.
+        :return: Copied Dataset.
+        """
+        if deep:
+            return copy.deepcopy(self)
+        return copy.copy(self)
+
+    def decline(self, X: pd.DataFrame, y: list, groups: pd.DataFrame = None) -> 'Dataset':
+        """Create a new Dataset with columns_types based on self.
+        Avoid time consuming columns_types computing
+        
+        :param pd.DataFrame X: The dataframe to use for our newly created Dataset.
+        :param list y: The target to use for our newly created Dataset.
+        :param pd.DataFrame, optional groups: The groups to use for our newly created Dataset.
+        :return: New Dataset.
+        """
+        if groups is None:
+            groups = self.groups
+        return Dataset(X, y, groups=groups, columns_types=self.columns_types)
+
+    def sample(self, n: int) -> 'Dataset':
+        """Return a dataset with a sample of data
+
+        :param int n: Number of line in the sample dataset
+        :return: dataset with a sample of data
+        """
+        if isinstance(n, float):
+            n = int(self.X.shape[0]*n)
+
+        if n >= self.X.shape[0]:
+            return self.decline(self.X, self.y)
+
+        if self.type_of_target == 'continuous':
+            _, test_idx = next(
+                ShuffleSplit(n_splits=1, test_size=n, random_state=42
+                ).split(self.X, self.y))
+        else:
+            try:
+                _, test_idx = next(
+                    StratifiedShuffleSplit(n_splits=1, test_size=n, random_state=42
+                    ).split(self.X, self.y))
+            except ValueError:
+                _, test_idx = next(
+                    ShuffleSplit(n_splits=1, test_size=n, random_state=42
+                    ).split(self.X, self.y))
+
+        groups = self.groups.iloc[test_idx].copy() if self.has_groups else None
+        return self.decline(self.X.iloc[test_idx], self.y[test_idx], groups=groups)
+
+    def transform(self, method: callable) -> None:
+        """Apply transform method to X or y data based on the method signature
+
+        :param callable method: Callable to apply. Will be call with X or y as parameter.
+        :return: Transformed dataset.
+        """
+        self.__X = method(self.__X)
+        self.__detect_columns_types()
+
+    @property
+    def has_groups(self) -> bool:
+        """Groups exists ?
+        
+        :return: Exists ?
+        """
+        return self.groups is not None and not self.groups.empty
+
+    def resample(self, resampler: callable) -> 'Dataset':
+        """Apply a resampler on X, y and groups data. 
+
+        :param callable resampler: Resampler method.
+        :return: Resampled Dataset.
+        """
+
+        if self.has_groups:
+            # Merge groups with X
+            X = self.X.reset_index(drop=True).join(self.groups.reset_index(drop=True))
+
+            # Resampler
+            X, y = resampler(X, self.y)
+
+            # Split groups and X
+            return self.decline(X.drop(columns=self.groups.columns),
+                            y,
+                            groups=X[self.groups.columns])
+        return self.decline(*resampler(self.X, self.y))
+
+    def fingerprint(self) -> str:
+        """Hash the current features, targets, groups and interpretation metadata.
+
+        Recompute because X, y and groups are exposed as mutable objects.
+        """
+        return hash_dataset(
+            self.__X, self.__y, self.groups, self.columns_types, self.type_of_target
+        )
+
+    def split(self, splitter: callable, *args, **kwargs) -> Iterator[tuple['Dataset', 'Dataset']]:
+        """Use splitter to split dataset into a list of tuple (train set, test set) 
+
+        :param callable splitter: The splitter function to perform.
+        :param tuple, optional \\*args: Additional parameters.
+        :param dict, optional \\**kwargs: Additional parameters.
+        :return: Train set and Test set iterator.
+        """
+        # Split the dataset as many times as the splitter requires it
+        y = self.y if self.type_of_target is not None else None
+        for i_train, i_test in splitter(self.X, y, *args, **kwargs):
+            X_train = self.X.iloc[i_train].copy()
+            X_test = self.X.iloc[i_test].copy()
+
+            if self.has_groups:
+                groups = self.groups.iloc[i_train].copy()
+                test_groups = self.groups.iloc[i_test].copy()
+            else:
+                groups = None
+                test_groups = None
+
+            if y is not None:
+                y_train = self.__y[i_train].copy()
+                y_test = self.__y[i_test].copy()
+            else:
+                y_train = None
+                y_test = None
+
+            yield (self.decline(X_train, y_train, groups=groups),
+                   self.decline(X_test, y_test, groups=test_groups))
+
+    def x_with_groups(self) -> pd.DataFrame:
+        """Return X dataframe with groups columns if not None. Return X otherwise.
+        
+        :return: The X dataframe with or without the groups columns.
+        """
+        if self.has_groups:
+            return self.X.reset_index(drop=True).join(self.groups.reset_index(drop=True))
+        return self.X
+
+    def get_columns_names_by_type(self, types: list[DataType]) -> list[str]:
+        """Get names of all the columns with DataType in types
+
+        :param list[DataType] types: List of Datatype to search.
+        :return: columns names.
+        """
+        if not isinstance(types, list):
+            types = [types]
+
+        return [
+            column
+            for column, (dtype, type) in self.columns_types.items()
+            if type in types
+        ]
+
+    def __detect_data_type(self, column_name: str) -> DataType:
+        """Detect data type of a column
+
+        :param str column_name: Name of the column to analyse.
+        :return  Type of the columns.
+        """
+        column_value = self.X[column_name]
+        dtype = column_value.dtype
+        detected: DataType = None
+        if isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_bool_dtype(dtype):
+            detected = DataType.CATEGORICAL
+        elif pd.api.types.is_object_dtype(dtype) or pd.api.types.is_string_dtype(dtype):
+            if len(column_value) == 0:
+                detected = DataType.CATEGORICAL
+            else:
+                unique_count = len(column_value.unique())
+                if (unique_count / len(column_value) < 0.05 \
+                    or unique_count < 7):
+                    detected = DataType.CATEGORICAL
+                elif column_value.astype(str).apply(len).max() <= 85:
+                    detected = DataType.SHORT_TEXT
+                else:
+                    detected = DataType.TEXT
+        elif pd.api.types.is_numeric_dtype(dtype) or pd.api.types.is_timedelta64_dtype(dtype):
+            detected = DataType.NUMERIC
+        elif pd.api.types.is_datetime64_any_dtype(dtype):
+            detected = DataType.DATE
+
+        return dtype, detected
+
+    @property
+    def needed_estimator(self) -> str:
+        """Kind of estimator needed for this dataset
+        
+        :return: Estimator type.
+        """
+        if self.type_of_target == 'continuous':
+            return 'regressor'
+
+        if self.type_of_target == 'survival':
+            return 'survival'
+
+        return 'classifier'
+
+    def __detect_columns_types(self) -> None:
+        """Detect column type of all features in X"""
+        new_types = {}
+        for column in self.features:
+            if column not in self.columns_types \
+                or self.columns_types[column][0] != self.X[column].dtype:
+                new_types[column] = self.__detect_data_type(column)
+            else:
+                new_types[column] = self.columns_types[column]
+
+        self.columns_types = new_types
+
+    def to_survival(self) -> None:
+        """Turn dataframe to survival compatibility"""
+        return Dataset.fix_survival(self.X, self.y)
+
+    @staticmethod
+    def _normalize_survival_pair(value: Any) -> tuple[bool, float]:
+        """Normalize a single survival sample to a (event, time) tuple."""
+        if isinstance(value, np.void):
+            if value.dtype.names and \
+                'event' in value.dtype.names and 'time' in value.dtype.names:
+                return bool(value['event']), float(value['time'])
+            value = value.tolist()
+
+        if isinstance(value, dict):
+            if 'event' not in value or 'time' not in value:
+                raise KeyError("Survival sample dictionary must include 'event' and 'time'.")
+            return bool(value['event']), float(value['time'])
+
+        if isinstance(value, np.ndarray):
+            if value.shape == ():
+                return Dataset._normalize_survival_pair(value.item())
+            if value.ndim >= 1 and value.shape[0] >= 2:
+                return bool(value[0]), float(value[1])
+
+        if isinstance(value, (tuple, list)):
+            if len(value) < 2:
+                raise ValueError("Survival sample must provide event indicator and time.")
+            return bool(value[0]), float(value[1])
+
+        raise TypeError(f"Unsupported survival sample format: {type(value)}")
+
+    @classmethod
+    def normalize_survival_target(cls, y: Any) -> list[tuple[bool, float]]:
+        """Return survival targets as a list of (event, time) tuples."""
+        if y is None:
+            return []
+
+        if isinstance(y, pd.DataFrame):
+            if not len(y.columns):
+                return []
+            if {'event', 'time'}.issubset(y.columns):
+                iterator = zip(y['event'], y['time'])
+            elif len(y.columns) >= 2:
+                iterator = (row[:2] for row in y.itertuples(index=False, name=None))
+            else:
+                raise ValueError("Survival DataFrame must contain at least two columns.")
+            return [cls._normalize_survival_pair(sample) for sample in iterator]
+
+        if isinstance(y, pd.Series):
+            return cls.normalize_survival_target(y.to_frame())
+
+        if isinstance(y, np.ndarray):
+            if y.dtype.names and 'event' in y.dtype.names and 'time' in y.dtype.names:
+                return [cls._normalize_survival_pair((row['event'], row['time'])) for row in y]
+            if y.ndim == 0:
+                return [cls._normalize_survival_pair(y.item())]
+            if y.ndim == 1:
+                return [cls._normalize_survival_pair(sample) for sample in y.tolist()]
+            if y.ndim >= 2 and y.shape[1] >= 2:
+                return [cls._normalize_survival_pair(sample[:2]) for sample in y]
+
+        if isinstance(y, (list, tuple)):
+            return [cls._normalize_survival_pair(sample) for sample in y]
+
+        if hasattr(y, '__iter__'):
+            return cls.normalize_survival_target(list(y))
+
+        raise TypeError(f"Unsupported survival target format: {type(y)}")
+
+    @classmethod
+    def fix_survival(cls, X: pd.DataFrame, y: Any) -> tuple[pd.DataFrame, np.ndarray]:
+        """Turn dataframe to survival compatibility
+        
+        :param pd.DataFrame X: The dataframe to fix.
+        :param Any y: The dataframe target to fix.
+        :return: Fixed dataframe
+        """
+        from sksurv.util import Surv
+
+        samples = cls.normalize_survival_target(y)
+        if samples:
+            events, times = zip(*samples)
+            y_surv = Surv.from_arrays(
+                event=np.asarray(events, dtype=bool),
+                time=np.asarray(times, dtype=float)
+            )
+        else:
+            y_surv = np.array([], dtype=[('event', 'bool'), ('time', 'float')])
+        
+        X = deepcopy(X)
+        X[X.select_dtypes(include=['float64']).columns] = \
+                X.select_dtypes(include=['float64']).astype('float32')
+
+        return X, y_surv
+
+    @classmethod
+    def fix_y_survival(cls, y: Any, y_train: Any) -> list[tuple[bool, float]]:
+        """Adjust survival targets to avoid censoring beyond the training horizon."""
+        y_samples = cls.normalize_survival_target(y)
+        y_train_samples = cls.normalize_survival_target(y_train)
+
+        if not y_train_samples:
+            return y_samples
+
+        _, times = zip(*y_train_samples)
+        censure_time = max(times)
+
+        new_y: list[tuple[bool, float]] = []
+        for event, time in y_samples:
+            if time >= censure_time:
+                time = censure_time
+                event = False
+            new_y.append((event, time))
+
+        return new_y

iaml/decorators/__init__.py

@@ -0,0 +1,3 @@
+"""All IAML decorators are imported in all.py
+It allow use to import decorator individually and to avoid loop import
+"""

iaml/decorators/all.py

@@ -0,0 +1,4 @@
+"""All IAML decorators"""
+# pylint: disable=unused-import,import-error
+from .runner import runner
+from .is_step import is_step, find_steps_by_tag

iaml/decorators/is_step.py

@@ -0,0 +1,45 @@
+"""Class decorator -> Must decorate each Step inheritance"""
+from ..step import Step
+
+def is_step(*tags) -> callable:
+    """is_step is needed to declare new Step.
+    With the Step inheritance, it will setup everything to make it work smoothly
+    
+    :param tuple, optional tags: Your Step will be attached to these tags. 
+        tags are use to easily include Step into Pipeline
+    :return: Step decorator
+    """
+    def step_wrapper(cls) -> Step:
+        """Register the step with IAML.
+        Add call to Step.__init__() so the Sub Step developer have one to care about this
+        
+        :return: Edited step class
+        """
+        Step.available_steps[cls] = tags # Declare your Step to IAML
+
+        # Help Python to find parent class
+        __class__ = cls # pylint: disable=unused-variable
+
+        initial_init = cls.__init__ # Keep the __init__ you have created
+        def __init__(self, *args, **kw):
+            if cls != Step:
+                super().__init__(*args, **kw) # All parent constructor
+
+            self.tags = set(tags)
+            initial_init(self, *args, **kw) # Run your __init__
+            self.default_configuration() # Setup default configuration
+
+        cls.__init__ = __init__ # Replace your init
+
+        return cls
+
+    return step_wrapper
+
+
+def find_steps_by_tag(tag: str) -> list[Step]:
+    """Retrieves all registered steps with a specific tag.
+
+    :param str tag: Tag to search for
+    :return: Matching steps
+    """
+    return set(filter(lambda key: tag in Step.available_steps[key], Step.available_steps.keys()))