modelmatrix 0.1.0__py3-none-any.whl

modelmatrix/__init__.py

@@ -0,0 +1,26 @@
+"""
+Model matrices from tabular data, as scikit-learn transformers.
+"""
+
+from .model_spec import (ModelSpec,
+                         Feature,
+                         Contrast,
+                         Ordinal,
+                         poly,
+                         ns,
+                         bs,
+                         derived_feature,
+                         pca,
+                         contrast,
+                         ordinal,
+                         build_columns)
+from .columns import Column
+
+try:
+    from ._version import version as __version__
+except ImportError:  # not installed
+    __version__ = 'unknown'
+
+__all__ = ['ModelSpec', 'Feature', 'Column', 'Contrast', 'Ordinal',
+           'poly', 'ns', 'bs', 'derived_feature', 'pca', 'contrast',
+           'ordinal', 'build_columns']

modelmatrix/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = 'g2f08b48f3'

modelmatrix/columns.py

@@ -0,0 +1,324 @@
+
+from typing import NamedTuple, Any
+from copy import copy
+
+import numpy as np, pandas as pd
+from scipy import sparse as sp_sparse
+from sklearn.base import clone
+from sklearn.preprocessing import (OneHotEncoder,
+                                   OrdinalEncoder)
+from sklearn.utils.validation import check_is_fitted
+from sklearn.exceptions import NotFittedError
+
+class Column(NamedTuple):
+
+    """
+    A column extractor with a possible
+    encoder (following `sklearn` fit/transform template).
+    """
+
+    idx: Any
+    name: str
+    is_categorical: bool = False
+    is_ordinal: bool = False
+    columns: tuple = ()
+    encoder: Any = None
+
+    def get_columns(self, X, fit=False, sparse=False):
+
+        """
+        Extract associated column from X,
+        encoding it with `self.encoder` if not None.
+
+        Parameters
+        ----------
+
+        X : array-like
+            X on which model matrix will be evaluated.
+            If a `pd.DataFrame` or `pd.Series`, variables that are of
+            categorical dtype will be treated as categorical.
+
+        fit : bool
+            If True, fit `self.encoder` on corresponding
+            column (if it is not already fit). The encoder
+            must be fit to evaluate the columns: a `Column`
+            used in the terms of a `ModelSpec` is not itself fit
+            when the `ModelSpec` is, as a copy of it is.
+
+        sparse : bool
+            If True, the columns may be returned as a
+            `scipy.sparse` matrix, e.g. for a categorical encoding.
+
+        Returns
+        -------
+        cols : `np.ndarray`
+            Evaluated columns -- if an encoder is used,
+            several columns may be produced.
+
+        names : (str,)
+            Column names
+        """
+
+        cols = _get_column(self.idx, X) 
+        if fit:
+            self.fit_encoder(X)
+
+        if self.encoder is not None:
+            transform = self.encoder.transform
+            if sparse and hasattr(self.encoder, '_islp_sparse_transform'):
+                transform = self.encoder._islp_sparse_transform
+            try:
+                cols = transform(cols)
+            except NotFittedError as e:
+                raise NotFittedError(
+                    f"The encoder of Column '{self.name}' is not fit: "
+                    "use get_columns(X, fit=True) to fit it on X.") from e
+            except ValueError:
+                # commonly raised for transformers given pd.Seriies
+                # try casting to a 2-d ndarray
+                cols = transform(np.asarray(cols).reshape((-1,1)))
+        else:
+            cols = _check_numeric(cols, self.name)
+        if sp_sparse.issparse(cols):
+            if not sparse:
+                cols = cols.toarray()
+        else:
+            cols = np.asarray(cols)
+
+        names = self.columns
+        if hasattr(self.encoder, 'columns_'):
+            names = ['{0}[{1}]'.format(self.name, c)
+                     for c in self.encoder.columns_]
+        if not names:
+            names = ['{0}[{1}]'.format(self.name, i)
+                     for i in range(cols.shape[1])]
+        return cols, names
+
+    def fit_encoder(self, X):
+
+        """
+        Fit `self.encoder`.
+
+        Parameters
+        ----------
+        X : array-like
+            X on which encoder will be fit.
+
+        Returns
+        -------
+        None
+        """
+        cols = _get_column(self.idx, X)
+        if self.encoder is not None:
+            try:
+                check_is_fitted(self.encoder)
+            except NotFittedError:
+                self.encoder.fit(cols)
+        return np.asarray(cols)
+
+
+# private functions
+
+_NUMERIC_INFERRED = ['integer', 'floating', 'mixed-integer-float',
+                     'decimal', 'boolean', 'empty']
+
+def _check_numeric(col, name):
+    """
+    Raise a `ValueError` if `col` (a column with no encoder)
+    is not numeric: a string column must be made categorical
+    explicitly. Returns `col`, as floats if it was of object dtype.
+    """
+    values = np.asarray(col)
+    if values.dtype.kind in 'biuf':
+        return col
+    if values.dtype.kind == 'O':
+        if pd.api.types.infer_dtype(values.reshape(-1), skipna=True) in _NUMERIC_INFERRED:
+            return values.astype(float)
+    raise ValueError(
+        f"column '{name}' is not numeric; to treat it as categorical, "
+        f"convert it with `X['{name}'] = X['{name}'].astype('category')` "
+        f"or use `contrast('{name}')` in the terms")
+
+
+def _get_column(idx,
+                X,
+                loc=True):
+    """
+    Extract column `idx` from `X`
+    as a two-dimensional ndarray or a pd.DataFrame
+    """
+    if isinstance(X, np.ndarray):
+        col = X[:, [idx]]
+    elif hasattr(X, 'loc'):
+        if loc:
+            col = X.loc[:, [idx]]
+        else:   # use iloc instead
+            col = X.iloc[:, [idx]]
+    else:
+        raise ValueError('expecting an ndarray or a ' +
+                         '"loc/iloc" methods, got %s' % str(X))
+
+    return col
+
+    
+def _get_column_info(X,
+                     columns,
+                     is_categorical,
+                     is_ordinal,
+                     categorical_encoders={}
+                     ):
+
+
+    """
+    Compute a dictionary
+    of `Column` instances for each column
+    of `X`. Keys are `columns`.
+
+    Categorical columns use the encoder given for that
+    column in `categorical_encoders`, else the one under
+    'ordinal' for an ordered column if that key is present,
+    else the one under 'categorical'.
+
+    """
+
+    column_info = {}
+    for i, col in enumerate(columns):
+        if type(col) == int:
+            name = f'X{col}'
+        else:
+            name = str(col)
+        if is_categorical[i]:
+            if col in categorical_encoders:
+                key = col
+            elif is_ordinal[i] and 'ordinal' in categorical_encoders:
+                key = 'ordinal'
+            else:
+                key = 'categorical'
+            encoder = clone(categorical_encoders[key])
+            Xcol = _get_column(col, X)
+            encoder.fit(Xcol)
+            if hasattr(encoder, 'columns_'):
+                columns = ['{0}[{1}]'.format(col, c)
+                           for c in encoder.columns_]
+            else:   # only transform to count the columns
+                cols = encoder.transform(Xcol)
+                if not sp_sparse.issparse(cols):
+                    cols = np.asarray(cols)
+                if cols.ndim == 1 or cols.shape[1] == 1:
+                    columns = ['{0}'.format(col)]
+                else:
+                    columns = ['{0}[{1}]'.format(col, c)
+                               for c in range(cols.shape[1])]
+            column_info[col] = Column(col,
+                                      name,
+                                      bool(is_categorical[i]),
+                                      bool(is_ordinal[i]),
+                                      tuple(columns),
+                                      encoder)
+        else:
+            Xcol = _get_column(col, X) 
+            column_info[col] = Column(col,
+                                      name,
+                                      columns=(name,))
+    return column_info
+
+# extracted from method of BaseHistGradientBoosting from
+# https://github.com/scikit-learn/scikit-learn/blob/2beed55847ee70d363bdbfe14ee4401438fba057/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py
+# max_bins is ignored
+
+def _check_categories(categorical_features, X):
+    """Check and validate categorical features in X
+
+    Return
+    ------
+    is_categorical : ndarray of shape (n_features,) or None, dtype=bool
+        Indicates whether a feature is categorical. If no feature is
+        categorical, this is None.
+    known_categories : list of size n_features or None
+        The list contains, for each feature:
+            - an array of shape (n_categories,) with the unique cat values
+            - None if the feature is not categorical
+        None if no feature is categorical.
+    """
+    if categorical_features is None:
+        return None, None
+
+    categorical_features = np.asarray(categorical_features)
+
+    if categorical_features.size == 0:
+        return None, None
+
+    if categorical_features.dtype.kind not in ('i', 'b'):
+        raise ValueError("categorical_features must be an array-like of "
+                         "bools or array-like of ints.")
+
+    n_features = X.shape[1]
+
+    # check for categorical features as indices
+    if categorical_features.dtype.kind == 'i':
+        if (np.max(categorical_features) >= n_features
+                or np.min(categorical_features) < 0):
+            raise ValueError("categorical_features set as integer "
+                             "indices must be in [0, n_features - 1]")
+        is_categorical = np.zeros(n_features, dtype=bool)
+        is_categorical[categorical_features] = True
+    else:
+        if categorical_features.shape[0] != n_features:
+            raise ValueError("categorical_features set as a boolean mask "
+                             "must have shape (n_features,), got: "
+                             f"{categorical_features.shape}")
+        is_categorical = categorical_features
+
+    if not np.any(is_categorical):
+        return None, None
+
+    # compute the known categories in the training data. We need to do
+    # that here instead of in the BinMapper because in case of early
+    # stopping, the mapper only gets a fraction of the training data.
+    known_categories = []
+
+    for f_idx in range(n_features):
+        if is_categorical[f_idx]:
+            # _get_column returns a 2-d column: flatten it to its values
+            categories = np.array([v for v in
+                                   set(np.asarray(_get_column(f_idx,
+                                                              X,
+                                                              loc=False)).ravel())])
+            missing = []
+            for c in categories:
+                try:
+                    missing.append(np.isnan(c))
+                except TypeError:  # not a float
+                    missing.append(False)
+            missing = np.array(missing)
+            if missing.any():
+                categories = sorted(categories[~missing])
+
+        else:
+            categories = None
+        known_categories.append(categories)
+
+    return is_categorical, known_categories
+
+
+def _categorical_from_df(df):
+    """
+    Find
+    """
+    is_categorical = []
+    is_ordinal = []
+    for c in df.columns:
+        try:
+            if df[c].dtype == 'category':
+                is_categorical.append(True)
+                is_ordinal.append(df[c].cat.ordered)
+            else:
+                is_categorical.append(False)
+                is_ordinal.append(False)
+        except (TypeError, AttributeError):
+            is_categorical.append(False)
+            is_ordinal.append(False)
+    is_categorical = np.asarray(is_categorical)
+    is_ordinal = np.asarray(is_ordinal)
+
+    return is_categorical, is_ordinal