modelmatrix 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/CHANGELOG.md +23 -0
- {modelmatrix-0.2.0/modelmatrix.egg-info → modelmatrix-0.3.0}/PKG-INFO +1 -1
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/responses.rst +2 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/responses.md +44 -4
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/__init__.py +2 -1
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/_native.py +8 -2
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/_version.py +3 -3
- modelmatrix-0.3.0/modelmatrix/outcome.py +173 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/response.py +13 -10
- {modelmatrix-0.2.0 → modelmatrix-0.3.0/modelmatrix.egg-info}/PKG-INFO +1 -1
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/SOURCES.txt +2 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/scm_file_list.json +2 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/scm_version.json +2 -2
- modelmatrix-0.3.0/tests/test_outcome.py +144 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_response.py +6 -4
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/.github/workflows/build_docs.yml +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/.github/workflows/build_test.yml +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/.gitignore +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/LICENSE +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/README.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/.gitignore +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/PCA.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/anova.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/ModelSpec.rst +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/conf.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/index.rst +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/terms.rst +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/api/transforms.rst +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/comparison.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/index.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/myst.yml +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/poly.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/spec.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/docs/splines.md +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/columns.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/model_spec.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix/transforms.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/dependency_links.txt +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/requires.txt +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/modelmatrix.egg-info/top_level.txt +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/pyproject.toml +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/setup.cfg +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/_datasets.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/data/Carseats.csv +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/data/Wage.csv +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_boolean_columns.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_columns.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_model_matrix.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_model_spec.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_model_spec_hard.py +0 -0
- {modelmatrix-0.2.0 → modelmatrix-0.3.0}/tests/test_wildcard.py +0 -0
|
@@ -2,6 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.3.0 (2026-10-26)
|
|
6
|
+
|
|
7
|
+
### New features
|
|
8
|
+
|
|
9
|
+
- **`Outcome`**: a response with the other columns a model takes with
|
|
10
|
+
it, each named by its keyword, so an estimator can find them by name:
|
|
11
|
+
`Outcome(Surv('time', 'status'), offset='O', weight='w', strata='inst')`
|
|
12
|
+
gives the columns `stop`, `status`, `offset`, `weight` and `strata`. A
|
|
13
|
+
column is taken as it is (a categorical keeps its labels); a feature of
|
|
14
|
+
a single column is fit as a term, e.g. `offset(log(exposure))`.
|
|
15
|
+
`wildcard(exclude=outcome)` leaves out all of its columns, and
|
|
16
|
+
`transform(X, response=False)` builds the offset of new data. With
|
|
17
|
+
`Binomial(..., output='proportion')`, the trials are the column
|
|
18
|
+
`weight`, times any weights given.
|
|
19
|
+
- Dataframes of other backends keep a categorical column (as a polars
|
|
20
|
+
`Enum`).
|
|
21
|
+
|
|
22
|
+
### Breaking changes
|
|
23
|
+
|
|
24
|
+
- `Surv` names its columns by their role, `start`, `stop` and `status`
|
|
25
|
+
(as R's and lifelines'), not as in the data: `Surv('time', 'status')`
|
|
26
|
+
gives the columns `stop` and `status`.
|
|
27
|
+
|
|
5
28
|
## 0.2.0 (2026-10-26)
|
|
6
29
|
|
|
7
30
|
### New features
|
|
@@ -112,9 +112,10 @@ design.
|
|
|
112
112
|
### Survival responses: `Surv`
|
|
113
113
|
|
|
114
114
|
`Surv(time, status)` is R's survival response, and `Surv(start, stop,
|
|
115
|
-
status)` that of (start, stop] intervals. The response is a
|
|
116
|
-
|
|
117
|
-
`
|
|
115
|
+
status)` that of (start, stop] intervals. The response is a dataframe
|
|
116
|
+
whose columns are named by their role, whatever their names in the
|
|
117
|
+
data: `stop` (the times), `status` and, if given, `start`, as R's
|
|
118
|
+
`Surv` and lifelines' `CoxTimeVaryingFitter` name them.
|
|
118
119
|
|
|
119
120
|
```{code-cell} ipython3
|
|
120
121
|
BrainCancer = load_data('BrainCancer').dropna()
|
|
@@ -125,7 +126,7 @@ y.iloc[:5]
|
|
|
125
126
|
```
|
|
126
127
|
|
|
127
128
|
```{code-cell} ipython3
|
|
128
|
-
sm.PHReg(y['
|
|
129
|
+
sm.PHReg(y['stop'], X, status=y['status']).fit().params
|
|
129
130
|
```
|
|
130
131
|
|
|
131
132
|
### Binomial responses
|
|
@@ -167,6 +168,45 @@ of counts for each class.
|
|
|
167
168
|
Multinomial('ShelveLoc', output='onehot').fit_transform(Carseats).iloc[:5]
|
|
168
169
|
```
|
|
169
170
|
|
|
171
|
+
### Offsets, weights and strata: `Outcome`
|
|
172
|
+
|
|
173
|
+
`Outcome` puts the other columns a model takes with its response next
|
|
174
|
+
to it, each named by its keyword: an offset, weights, the strata of a
|
|
175
|
+
Cox model, or any other. A column is taken as it is (a categorical keeps
|
|
176
|
+
its labels), and a feature of a single column is fit as a term of a
|
|
177
|
+
`ModelSpec`, e.g. R's `offset(log(exposure))`:
|
|
178
|
+
|
|
179
|
+
```{code-cell} ipython3
|
|
180
|
+
from sklearn.preprocessing import FunctionTransformer
|
|
181
|
+
from modelmatrix import Outcome, derived_feature
|
|
182
|
+
|
|
183
|
+
rng = np.random.default_rng(1)
|
|
184
|
+
P = pd.DataFrame({'x': rng.standard_normal(100), 'exposure': rng.uniform(1, 5, 100)})
|
|
185
|
+
P['claims'] = rng.poisson(P['exposure'] * np.exp(0.3 * P['x']))
|
|
186
|
+
|
|
187
|
+
log_exposure = derived_feature(['exposure'], encoder=FunctionTransformer(np.log),
|
|
188
|
+
name='log(exposure)')
|
|
189
|
+
outcome = Outcome('claims', offset=log_exposure)
|
|
190
|
+
y = outcome.fit_transform(P)
|
|
191
|
+
X = MS([wildcard(exclude=outcome)], intercept=True).fit_transform(P) # leaves out exposure too
|
|
192
|
+
sm.GLM(y['claims'], X, family=sm.families.Poisson(), offset=y['offset']).fit().params
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
As the columns are named by their role, an estimator can find them by
|
|
196
|
+
name, e.g. a stratified Cox model with an offset:
|
|
197
|
+
|
|
198
|
+
```{code-cell} ipython3
|
|
199
|
+
BrainCancer['O'] = 0.1
|
|
200
|
+
outcome = Outcome(Surv('time', 'status'), offset='O', strata='sex')
|
|
201
|
+
y = outcome.fit_transform(BrainCancer)
|
|
202
|
+
y.iloc[:5]
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The offset of new data, whose response is not known, is
|
|
206
|
+
`outcome.transform(new, response=False)`. With a `Binomial` of
|
|
207
|
+
`output='proportion'`, the number of trials is the column `weight`
|
|
208
|
+
(times the weights given as `weight=`), as R's weights of a proportion.
|
|
209
|
+
|
|
170
210
|
## Missing values
|
|
171
211
|
|
|
172
212
|
As scikit-learn's transformers, a `ModelSpec` keeps every row: a row
|
|
@@ -18,6 +18,7 @@ from .model_spec import (ModelSpec,
|
|
|
18
18
|
Wildcard)
|
|
19
19
|
from .columns import Column
|
|
20
20
|
from .response import Response, Surv, Binomial, Multinomial
|
|
21
|
+
from .outcome import Outcome
|
|
21
22
|
|
|
22
23
|
try:
|
|
23
24
|
from ._version import version as __version__
|
|
@@ -27,4 +28,4 @@ except ImportError: # not installed
|
|
|
27
28
|
__all__ = ['ModelSpec', 'Feature', 'Column', 'Contrast', 'Ordinal',
|
|
28
29
|
'poly', 'ns', 'bs', 'derived_feature', 'pca', 'contrast',
|
|
29
30
|
'ordinal', 'build_columns', 'wildcard', 'Wildcard',
|
|
30
|
-
'Response', 'Surv', 'Binomial', 'Multinomial']
|
|
31
|
+
'Response', 'Surv', 'Binomial', 'Multinomial', 'Outcome']
|
|
@@ -12,6 +12,9 @@ from scipy import sparse as sp_sparse
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def _series_to_native(S, backend):
|
|
15
|
+
return _series_to_nw(S, backend).to_native()
|
|
16
|
+
|
|
17
|
+
def _series_to_nw(S, backend):
|
|
15
18
|
# a categorical is an Enum, if the backend has one, of its
|
|
16
19
|
# categories as strings
|
|
17
20
|
values, dtype = S.to_numpy(), None
|
|
@@ -19,9 +22,9 @@ def _series_to_native(S, backend):
|
|
|
19
22
|
values = np.asarray([None if pd.isna(v) else str(v) for v in S], dtype=object)
|
|
20
23
|
dtype = nw.Enum([str(c) for c in S.cat.categories])
|
|
21
24
|
try:
|
|
22
|
-
return nw.new_series(str(S.name), values, dtype=dtype, backend=backend)
|
|
25
|
+
return nw.new_series(str(S.name), values, dtype=dtype, backend=backend)
|
|
23
26
|
except NotImplementedError: # e.g. no Enum in pyarrow
|
|
24
|
-
return nw.new_series(str(S.name), values, backend=backend)
|
|
27
|
+
return nw.new_series(str(S.name), values, backend=backend)
|
|
25
28
|
|
|
26
29
|
def _from_native(X, columns=None):
|
|
27
30
|
"""
|
|
@@ -62,6 +65,9 @@ def _from_native(X, columns=None):
|
|
|
62
65
|
if duplicated:
|
|
63
66
|
raise ValueError(f'the model matrix has duplicated column names {duplicated}, '
|
|
64
67
|
'which this dataframe backend cannot hold')
|
|
68
|
+
if any(isinstance(t, pd.CategoricalDtype) for t in D.dtypes): # e.g. strata
|
|
69
|
+
return nw.from_dict({str(c): _series_to_nw(D[c].rename(str(c)), backend)
|
|
70
|
+
for c in D.columns}).to_native()
|
|
65
71
|
return nw.from_dict({str(c): D[c].to_numpy() for c in D.columns},
|
|
66
72
|
backend=backend).to_native()
|
|
67
73
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.3.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 0)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g8c1f37d25'
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Outcome
|
|
3
|
+
=======
|
|
4
|
+
|
|
5
|
+
A response with other columns a model takes with it, such as an offset,
|
|
6
|
+
weights or the strata of a Cox model, each named by its role:
|
|
7
|
+
`Outcome(Surv('time', 'status'), offset='O', weight='w')` builds a
|
|
8
|
+
dataframe with columns `stop`, `status`, `offset` and `weight`, so that
|
|
9
|
+
an estimator can find each column by its name.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from copy import deepcopy
|
|
13
|
+
|
|
14
|
+
import numpy as np, pandas as pd
|
|
15
|
+
|
|
16
|
+
from ._native import _from_native
|
|
17
|
+
from .model_spec import ModelSpec, Feature, _term_columns
|
|
18
|
+
from .response import Response, Binomial, _values, _output
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class _Column(Response):
|
|
22
|
+
|
|
23
|
+
# a column of X as it is
|
|
24
|
+
|
|
25
|
+
def __init__(self, column):
|
|
26
|
+
self.column = column
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def columns(self):
|
|
30
|
+
return (self.column,)
|
|
31
|
+
|
|
32
|
+
def _transform(self, X):
|
|
33
|
+
return _values(X, self.column).copy()
|
|
34
|
+
|
|
35
|
+
def __repr__(self):
|
|
36
|
+
return repr(self.column)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Outcome(Response):
|
|
40
|
+
|
|
41
|
+
"""
|
|
42
|
+
A response, with other columns named by their role.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
|
|
47
|
+
response : Response or column identifier
|
|
48
|
+
The response: a :py:class:`Surv`, :py:class:`Binomial` or
|
|
49
|
+
:py:class:`Multinomial`, or a column of `X` as it is.
|
|
50
|
+
|
|
51
|
+
**columns : column identifier or Feature
|
|
52
|
+
Other columns, each named by its keyword, e.g. `offset='O'` or
|
|
53
|
+
`weight='w'`: a column of `X`, as it is (a categorical, such as
|
|
54
|
+
the strata of a Cox model, keeps its labels), or a feature
|
|
55
|
+
of a single column, e.g. the log of a column, fit as a term of a
|
|
56
|
+
:py:class:`ModelSpec`.
|
|
57
|
+
|
|
58
|
+
Notes
|
|
59
|
+
-----
|
|
60
|
+
|
|
61
|
+
The columns are those of the response, then the others in the order
|
|
62
|
+
of their keywords. With a :py:class:`Binomial` of
|
|
63
|
+
`output='proportion'`, its number of trials is the column `weight`
|
|
64
|
+
(times the weights given as `weight=`, if any), as R's prior weights
|
|
65
|
+
of a proportion.
|
|
66
|
+
|
|
67
|
+
`wildcard(exclude=outcome)` leaves out every column the outcome uses,
|
|
68
|
+
so that none of them is a term of its own design. For an offset of
|
|
69
|
+
new data, without its response, use `transform(X, response=False)`.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, response, **columns):
|
|
73
|
+
self.response = response
|
|
74
|
+
self.extra = columns
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def columns(self):
|
|
78
|
+
cols = list(_as_response(self.response).columns)
|
|
79
|
+
for name, value in self.extra.items():
|
|
80
|
+
if isinstance(value, Feature):
|
|
81
|
+
used = _term_columns([value])
|
|
82
|
+
if used is None:
|
|
83
|
+
raise ValueError(f'the column {name!r} of an Outcome cannot use a wildcard')
|
|
84
|
+
cols.extend(sorted(used, key=str))
|
|
85
|
+
else:
|
|
86
|
+
cols.append(value)
|
|
87
|
+
return tuple(dict.fromkeys(cols))
|
|
88
|
+
|
|
89
|
+
def _fit(self, X):
|
|
90
|
+
self.response_ = deepcopy(_as_response(self.response))
|
|
91
|
+
self.response_._fit(X)
|
|
92
|
+
self.response_.columns_ = self.response_.columns
|
|
93
|
+
self.specs_ = {}
|
|
94
|
+
for name, value in self.extra.items():
|
|
95
|
+
if isinstance(value, Feature):
|
|
96
|
+
spec = ModelSpec([value]).fit(X)
|
|
97
|
+
if len(spec.get_feature_names_out()) != 1:
|
|
98
|
+
raise ValueError(f'the column {name!r} of an Outcome should be a single '
|
|
99
|
+
f'column: {value.name!r} has '
|
|
100
|
+
f'{len(spec.get_feature_names_out())}')
|
|
101
|
+
self.specs_[name] = spec
|
|
102
|
+
else:
|
|
103
|
+
_values(X, value) # raises if the column is not in X
|
|
104
|
+
self._build(X, response=True) # checks the names
|
|
105
|
+
|
|
106
|
+
def transform(self, X, response=True):
|
|
107
|
+
"""
|
|
108
|
+
Build the outcome from `X`, after fitting.
|
|
109
|
+
|
|
110
|
+
Parameters
|
|
111
|
+
----------
|
|
112
|
+
X : dataframe or np.ndarray
|
|
113
|
+
Data holding the columns of the outcome.
|
|
114
|
+
|
|
115
|
+
response : bool, default=True
|
|
116
|
+
Build the response? If False, only the other columns, e.g.
|
|
117
|
+
the offset of new data, whose response is not known.
|
|
118
|
+
|
|
119
|
+
Returns
|
|
120
|
+
-------
|
|
121
|
+
y : dataframe or np.ndarray
|
|
122
|
+
Of the backend of *X*, an :py:class:`np.ndarray` for an array.
|
|
123
|
+
"""
|
|
124
|
+
self._check_fitted()
|
|
125
|
+
X, to_native = _from_native(X, set(self.columns))
|
|
126
|
+
y = self._build(X, response)
|
|
127
|
+
return y if to_native is None else to_native(y)
|
|
128
|
+
|
|
129
|
+
def _transform(self, X):
|
|
130
|
+
return self._build(X, response=True)
|
|
131
|
+
|
|
132
|
+
def _build(self, X, response):
|
|
133
|
+
X_orig = X
|
|
134
|
+
if not isinstance(X, pd.DataFrame): # an array: its columns are integers
|
|
135
|
+
X = pd.DataFrame(np.asarray(X))
|
|
136
|
+
parts = []
|
|
137
|
+
weight = None
|
|
138
|
+
if response:
|
|
139
|
+
y = self.response_._transform(X)
|
|
140
|
+
y = y.to_frame() if isinstance(y, pd.Series) else y
|
|
141
|
+
if (isinstance(self.response_, Binomial) and self.response_.trials is not None
|
|
142
|
+
and self.response_.output == 'proportion'):
|
|
143
|
+
# the trials are the weights of a proportion
|
|
144
|
+
weight = y.pop(self.response_.trials)
|
|
145
|
+
parts.append(y)
|
|
146
|
+
extra = {}
|
|
147
|
+
for name, value in self.extra.items():
|
|
148
|
+
if name in self.specs_:
|
|
149
|
+
D = self.specs_[name].transform(X)
|
|
150
|
+
extra[name] = np.asarray(D, dtype=float)[:, 0]
|
|
151
|
+
else:
|
|
152
|
+
extra[name] = _values(X, value).copy() # keeps a categorical
|
|
153
|
+
if weight is not None:
|
|
154
|
+
w = weight.to_numpy()
|
|
155
|
+
extra['weight'] = w if 'weight' not in extra else w * np.asarray(extra['weight'], float)
|
|
156
|
+
parts.append(pd.DataFrame(extra, index=X.index))
|
|
157
|
+
D = pd.concat(parts, axis=1)
|
|
158
|
+
duplicated = list(D.columns[D.columns.duplicated()])
|
|
159
|
+
if duplicated:
|
|
160
|
+
raise ValueError(f'the columns of the Outcome have the same names {duplicated}: '
|
|
161
|
+
'a column of the response has the name of another')
|
|
162
|
+
return _output(D, X_orig)
|
|
163
|
+
|
|
164
|
+
def __repr__(self):
|
|
165
|
+
args = [repr(self.response)] + [f'{k}={v.name if isinstance(v, Feature) else repr(v)}'
|
|
166
|
+
for k, v in self.extra.items()]
|
|
167
|
+
return f'Outcome({", ".join(args)})'
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _as_response(response):
|
|
171
|
+
if isinstance(response, Response):
|
|
172
|
+
return response
|
|
173
|
+
return _Column(response)
|
|
@@ -71,9 +71,7 @@ class Response(object):
|
|
|
71
71
|
:py:class:`pd.DataFrame`), an :py:class:`np.ndarray` for
|
|
72
72
|
an array.
|
|
73
73
|
"""
|
|
74
|
-
|
|
75
|
-
raise NotFittedError(f'this {type(self).__name__} is not fitted yet: '
|
|
76
|
-
'call fit (or fit_transform) first')
|
|
74
|
+
self._check_fitted()
|
|
77
75
|
X, to_native = _from_native(X, set(self.columns))
|
|
78
76
|
y = self._transform(X)
|
|
79
77
|
return y if to_native is None else to_native(y)
|
|
@@ -84,6 +82,11 @@ class Response(object):
|
|
|
84
82
|
"""
|
|
85
83
|
return self.fit(X).transform(X)
|
|
86
84
|
|
|
85
|
+
def _check_fitted(self):
|
|
86
|
+
if not hasattr(self, 'columns_'):
|
|
87
|
+
raise NotFittedError(f'this {type(self).__name__} is not fitted yet: '
|
|
88
|
+
'call fit (or fit_transform) first')
|
|
89
|
+
|
|
87
90
|
def _fit(self, X):
|
|
88
91
|
for col in self.columns:
|
|
89
92
|
_values(X, col) # raises if the column is not in X
|
|
@@ -147,10 +150,10 @@ class Surv(Response):
|
|
|
147
150
|
Notes
|
|
148
151
|
-----
|
|
149
152
|
|
|
150
|
-
The response is a
|
|
151
|
-
(if given), `stop` and `status`,
|
|
152
|
-
|
|
153
|
-
|
|
153
|
+
The response is a dataframe with columns named by their role,
|
|
154
|
+
whatever their names in `X`: `start` (if given), `stop` and `status`,
|
|
155
|
+
the names of R's `Surv` and of lifelines' `CoxTimeVaryingFitter`.
|
|
156
|
+
(scikit-survival's `Surv` is another object, a structured array.)
|
|
154
157
|
"""
|
|
155
158
|
|
|
156
159
|
def __init__(self, *args, start=None, stop=None, status=None):
|
|
@@ -190,9 +193,9 @@ class Surv(Response):
|
|
|
190
193
|
if (start >= stop).any():
|
|
191
194
|
raise ValueError(f'the start {self.start!r} of Surv should be less than '
|
|
192
195
|
f'its stop {self.stop!r}')
|
|
193
|
-
D[
|
|
194
|
-
D[
|
|
195
|
-
D[
|
|
196
|
+
D['start'] = start.to_numpy()
|
|
197
|
+
D['stop'] = stop.to_numpy()
|
|
198
|
+
D['status'] = status.to_numpy()
|
|
196
199
|
index = X.index if isinstance(X, pd.DataFrame) else None
|
|
197
200
|
return _output(pd.DataFrame(D, index=index), X)
|
|
198
201
|
|
|
@@ -26,6 +26,7 @@ modelmatrix/_native.py
|
|
|
26
26
|
modelmatrix/_version.py
|
|
27
27
|
modelmatrix/columns.py
|
|
28
28
|
modelmatrix/model_spec.py
|
|
29
|
+
modelmatrix/outcome.py
|
|
29
30
|
modelmatrix/response.py
|
|
30
31
|
modelmatrix/transforms.py
|
|
31
32
|
modelmatrix.egg-info/PKG-INFO
|
|
@@ -41,6 +42,7 @@ tests/test_columns.py
|
|
|
41
42
|
tests/test_model_matrix.py
|
|
42
43
|
tests/test_model_spec.py
|
|
43
44
|
tests/test_model_spec_hard.py
|
|
45
|
+
tests/test_outcome.py
|
|
44
46
|
tests/test_response.py
|
|
45
47
|
tests/test_wildcard.py
|
|
46
48
|
tests/data/Carseats.csv
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"modelmatrix/_native.py",
|
|
27
27
|
"modelmatrix/columns.py",
|
|
28
28
|
"modelmatrix/model_spec.py",
|
|
29
|
+
"modelmatrix/outcome.py",
|
|
29
30
|
"modelmatrix/response.py",
|
|
30
31
|
"modelmatrix/transforms.py",
|
|
31
32
|
"pyproject.toml",
|
|
@@ -37,6 +38,7 @@
|
|
|
37
38
|
"tests/test_model_matrix.py",
|
|
38
39
|
"tests/test_model_spec.py",
|
|
39
40
|
"tests/test_model_spec_hard.py",
|
|
41
|
+
"tests/test_outcome.py",
|
|
40
42
|
"tests/test_response.py",
|
|
41
43
|
"tests/test_wildcard.py"
|
|
42
44
|
]
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Outcome: a response with an offset, weights, strata, ..., named by role
|
|
2
|
+
|
|
3
|
+
import numpy as np, pandas as pd
|
|
4
|
+
import pytest
|
|
5
|
+
import statsmodels.api as sm
|
|
6
|
+
from sklearn.exceptions import NotFittedError
|
|
7
|
+
from sklearn.preprocessing import FunctionTransformer
|
|
8
|
+
|
|
9
|
+
from modelmatrix import (ModelSpec as MS, Outcome, Surv, Binomial, Multinomial,
|
|
10
|
+
wildcard, derived_feature, poly)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture
|
|
14
|
+
def D():
|
|
15
|
+
rng = np.random.default_rng(2)
|
|
16
|
+
n = 60
|
|
17
|
+
exposure = rng.uniform(1, 5, n)
|
|
18
|
+
x = rng.standard_normal(n)
|
|
19
|
+
return pd.DataFrame({'time': rng.exponential(size=n) + 0.1,
|
|
20
|
+
'status': rng.binomial(1, 0.7, n),
|
|
21
|
+
'O': rng.standard_normal(n) / 4,
|
|
22
|
+
'w': rng.uniform(0.5, 2, n),
|
|
23
|
+
'inst': pd.Categorical(rng.choice(['a', 'b', 'c'], n)),
|
|
24
|
+
'exposure': exposure,
|
|
25
|
+
'x': x,
|
|
26
|
+
'claims': rng.poisson(exposure * np.exp(0.3 * x)),
|
|
27
|
+
'cases': rng.binomial(8, 0.4, n),
|
|
28
|
+
'size': np.full(n, 8)})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
log_exposure = derived_feature(['exposure'], encoder=FunctionTransformer(np.log),
|
|
32
|
+
name='log(exposure)')
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_columns_named_by_role(D):
|
|
36
|
+
Y = Outcome(Surv('time', 'status'), offset='O', weight='w', strata='inst')
|
|
37
|
+
y = Y.fit_transform(D)
|
|
38
|
+
assert list(y.columns) == ['stop', 'status', 'offset', 'weight', 'strata']
|
|
39
|
+
np.testing.assert_allclose(y['offset'], D['O'])
|
|
40
|
+
np.testing.assert_allclose(y['weight'], D['w'])
|
|
41
|
+
assert isinstance(y['strata'].dtype, pd.CategoricalDtype) # labels, not contrasts
|
|
42
|
+
assert y.index.equals(D.index)
|
|
43
|
+
assert repr(Y) == "Outcome(Surv('time', 'status'), offset='O', weight='w', strata='inst')"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_wildcard_excludes_every_column(D):
|
|
47
|
+
Y = Outcome(Surv('time', 'status'), offset='O', weight='w', strata='inst')
|
|
48
|
+
assert Y.columns == ('time', 'status', 'O', 'w', 'inst')
|
|
49
|
+
spec = MS([wildcard(exclude=Y)]).fit(D)
|
|
50
|
+
assert spec.expanded_terms_ == ['exposure', 'x', 'claims', 'cases', 'size']
|
|
51
|
+
Z = Outcome('claims', offset=log_exposure)
|
|
52
|
+
assert Z.columns == ('claims', 'exposure')
|
|
53
|
+
assert MS([wildcard(exclude=Z)]).fit(D).expanded_terms_ == [
|
|
54
|
+
'time', 'status', 'O', 'w', 'inst', 'x', 'cases', 'size']
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_poisson_offset_matches_statsmodels(D):
|
|
58
|
+
# R's glm(claims ~ x + offset(log(exposure)), family=poisson)
|
|
59
|
+
Y = Outcome('claims', offset=log_exposure)
|
|
60
|
+
y = Y.fit_transform(D)
|
|
61
|
+
assert list(y.columns) == ['claims', 'offset']
|
|
62
|
+
X = MS(['x'], intercept=True).fit_transform(D)
|
|
63
|
+
family = sm.families.Poisson()
|
|
64
|
+
fit = sm.GLM(y['claims'], X, family=family, offset=y['offset']).fit()
|
|
65
|
+
ref = sm.GLM(D['claims'], X, family=family, offset=np.log(D['exposure'])).fit()
|
|
66
|
+
np.testing.assert_allclose(fit.params, ref.params)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_offset_of_new_data(D):
|
|
70
|
+
Y = Outcome('claims', offset=log_exposure, weight='w').fit(D)
|
|
71
|
+
new = D.drop(columns='claims').iloc[:5]
|
|
72
|
+
O = Y.transform(new, response=False)
|
|
73
|
+
assert list(O.columns) == ['offset', 'weight']
|
|
74
|
+
np.testing.assert_allclose(O['offset'], np.log(new['exposure']))
|
|
75
|
+
with pytest.raises(ValueError, match="'claims' is not in X"):
|
|
76
|
+
Y.transform(new)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_stratified_cox_matches_statsmodels(D):
|
|
80
|
+
Y = Outcome(Surv('time', 'status'), offset='O', strata='inst')
|
|
81
|
+
y = Y.fit_transform(D)
|
|
82
|
+
X = MS(['x']).fit_transform(D)
|
|
83
|
+
fit = sm.PHReg(y['stop'], X, status=y['status'], strata=y['strata'],
|
|
84
|
+
offset=y['offset']).fit()
|
|
85
|
+
ref = sm.PHReg(D['time'], D[['x']], status=D['status'], strata=D['inst'],
|
|
86
|
+
offset=D['O']).fit()
|
|
87
|
+
np.testing.assert_allclose(fit.params, ref.params)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_binomial_proportion_weights(D):
|
|
91
|
+
# the trials are the weights of a proportion, times the prior weights
|
|
92
|
+
y = Outcome(Binomial('cases', trials='size', output='proportion')).fit_transform(D)
|
|
93
|
+
assert list(y.columns) == ['cases/size', 'weight']
|
|
94
|
+
np.testing.assert_allclose(y['weight'], D['size'])
|
|
95
|
+
y = Outcome(Binomial('cases', trials='size', output='proportion'), weight='w').fit_transform(D)
|
|
96
|
+
assert list(y.columns) == ['cases/size', 'weight']
|
|
97
|
+
np.testing.assert_allclose(y['weight'], D['size'] * D['w'])
|
|
98
|
+
# counts are left as they are
|
|
99
|
+
y = Outcome(Binomial('cases', trials='size'), weight='w').fit_transform(D)
|
|
100
|
+
assert list(y.columns) == ['cases', 'size-cases', 'weight']
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_other_responses(D):
|
|
104
|
+
y = Outcome(Multinomial('inst'), weight='w').fit_transform(D)
|
|
105
|
+
assert list(y.columns) == ['inst', 'weight']
|
|
106
|
+
assert isinstance(y['inst'].dtype, pd.CategoricalDtype)
|
|
107
|
+
# the response given is not modified by fit
|
|
108
|
+
response = Binomial('status')
|
|
109
|
+
Outcome(response, weight='w').fit(D)
|
|
110
|
+
assert not hasattr(response, 'levels_')
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_errors(D):
|
|
114
|
+
with pytest.raises(NotFittedError):
|
|
115
|
+
Outcome('claims', offset='O').transform(D)
|
|
116
|
+
with pytest.raises(ValueError, match='single column'):
|
|
117
|
+
Outcome('claims', offset=poly('x', 2)).fit(D)
|
|
118
|
+
with pytest.raises(ValueError, match='same names'):
|
|
119
|
+
Outcome('claims', claims='O').fit(D)
|
|
120
|
+
with pytest.raises(ValueError, match="'nope' is not in X"):
|
|
121
|
+
Outcome('claims', offset='nope').fit(D)
|
|
122
|
+
with pytest.raises(ValueError, match='wildcard'):
|
|
123
|
+
Outcome('claims', offset=derived_feature(wildcard, name='all')).columns
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_ndarray(D):
|
|
127
|
+
A = np.asarray(D[['time', 'status', 'O']], dtype=float)
|
|
128
|
+
y = Outcome(Surv(0, 1), offset=2).fit_transform(A)
|
|
129
|
+
assert isinstance(y, np.ndarray)
|
|
130
|
+
np.testing.assert_allclose(y, A)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.mark.parametrize('backend', ['polars', 'pyarrow'])
|
|
134
|
+
def test_native(D, backend):
|
|
135
|
+
lib = pytest.importorskip(backend)
|
|
136
|
+
nw = pytest.importorskip('narwhals')
|
|
137
|
+
P = lib.from_pandas(D) if backend == 'polars' else lib.Table.from_pandas(D)
|
|
138
|
+
y = Outcome(Surv('time', 'status'), offset='O', strata='inst').fit_transform(P)
|
|
139
|
+
y_nw = nw.from_native(y)
|
|
140
|
+
assert nw.get_native_namespace(y_nw) is lib
|
|
141
|
+
assert y_nw.columns == ['stop', 'status', 'offset', 'strata']
|
|
142
|
+
assert y_nw['strata'].to_list() == list(D['inst'])
|
|
143
|
+
if backend == 'polars':
|
|
144
|
+
assert y['strata'].dtype == lib.Enum(['a', 'b', 'c'])
|
|
@@ -68,15 +68,17 @@ def test_index_of_y(Carseats):
|
|
|
68
68
|
|
|
69
69
|
def test_surv(S):
|
|
70
70
|
y = Surv('time', 'status').fit_transform(S)
|
|
71
|
-
|
|
71
|
+
# named by role, whatever their names in the data
|
|
72
|
+
pd.testing.assert_frame_equal(y, S[['time', 'status']].astype(float)
|
|
73
|
+
.set_axis(['stop', 'status'], axis=1))
|
|
72
74
|
y3 = Surv('start', 'time', 'status').fit_transform(S)
|
|
73
|
-
assert list(y3.columns) == ['start', '
|
|
75
|
+
assert list(y3.columns) == ['start', 'stop', 'status']
|
|
74
76
|
y_kw = Surv(stop='time', status='status', start='start').fit_transform(S)
|
|
75
77
|
pd.testing.assert_frame_equal(y3, y_kw)
|
|
76
78
|
assert repr(Surv('start', 'time', 'status')) == "Surv('start', 'time', 'status')"
|
|
77
79
|
# a boolean status
|
|
78
80
|
yb = Surv('time', 'flag').fit_transform(S)
|
|
79
|
-
np.testing.assert_array_equal(yb['
|
|
81
|
+
np.testing.assert_array_equal(yb['status'], S['flag'].astype(float))
|
|
80
82
|
|
|
81
83
|
|
|
82
84
|
def test_surv_matches_statsmodels_PHReg(S):
|
|
@@ -84,7 +86,7 @@ def test_surv_matches_statsmodels_PHReg(S):
|
|
|
84
86
|
y = response.fit_transform(S)
|
|
85
87
|
X = MS([wildcard(exclude=[response, 'start', 'cases', 'size', 'flag', 'kind'])]).fit_transform(S)
|
|
86
88
|
assert list(X.columns) == ['x']
|
|
87
|
-
fit = sm.PHReg(y['
|
|
89
|
+
fit = sm.PHReg(y['stop'], X, status=y['status']).fit()
|
|
88
90
|
ref = sm.PHReg(S['time'], S[['x']], status=S['status']).fit()
|
|
89
91
|
np.testing.assert_allclose(fit.params, ref.params)
|
|
90
92
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|