philler 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phil/__init__.py +21 -0
- phil/gallery.py +149 -0
- phil/imputation/__init__.py +8 -0
- phil/imputation/config.py +26 -0
- phil/imputation/distribution.py +67 -0
- phil/magic/__init__.py +9 -0
- phil/magic/base.py +18 -0
- phil/magic/config.py +14 -0
- phil/magic/ect.py +58 -0
- phil/magic/rust_backend.py +48 -0
- phil/phil.py +221 -0
- phil/transformers.py +52 -0
- philler-1.0.0.dist-info/METADATA +122 -0
- philler-1.0.0.dist-info/RECORD +17 -0
- philler-1.0.0.dist-info/WHEEL +5 -0
- philler-1.0.0.dist-info/licenses/LICENSE +28 -0
- philler-1.0.0.dist-info/top_level.txt +1 -0
phil/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Phil package.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from phil.gallery import GridGallery
|
|
6
|
+
from phil.imputation import DistributionImputer, ImputationConfig, PreprocessingConfig
|
|
7
|
+
from phil.magic import ECT, ECTConfig
|
|
8
|
+
from phil.phil import Phil
|
|
9
|
+
from phil.transformers import PhilTransformer
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Phil",
|
|
14
|
+
"PhilTransformer",
|
|
15
|
+
"GridGallery",
|
|
16
|
+
"ECT",
|
|
17
|
+
"ECTConfig",
|
|
18
|
+
"ImputationConfig",
|
|
19
|
+
"PreprocessingConfig",
|
|
20
|
+
"DistributionImputer",
|
|
21
|
+
]
|
phil/gallery.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Collection of predefined configurations for Phil."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from sklearn.model_selection import ParameterGrid
|
|
8
|
+
|
|
9
|
+
from phil.imputation import ImputationConfig, PreprocessingConfig
|
|
10
|
+
from phil.magic import ECTConfig
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GridGallery:
|
|
14
|
+
_grids = {
|
|
15
|
+
"default": ImputationConfig(
|
|
16
|
+
methods=[
|
|
17
|
+
"BayesianRidge",
|
|
18
|
+
"DecisionTreeRegressor",
|
|
19
|
+
"RandomForestRegressor",
|
|
20
|
+
"GradientBoostingRegressor",
|
|
21
|
+
],
|
|
22
|
+
modules=[
|
|
23
|
+
"sklearn.linear_model",
|
|
24
|
+
"sklearn.tree",
|
|
25
|
+
"sklearn.ensemble",
|
|
26
|
+
"sklearn.ensemble",
|
|
27
|
+
],
|
|
28
|
+
grids=[
|
|
29
|
+
ParameterGrid({"alpha": [1.0, 0.1, 0.01]}),
|
|
30
|
+
ParameterGrid(
|
|
31
|
+
{"max_depth": [None, 5, 10], "min_samples_split": [2, 5]}
|
|
32
|
+
),
|
|
33
|
+
ParameterGrid({"n_estimators": [10, 50], "max_depth": [None, 5]}),
|
|
34
|
+
ParameterGrid(
|
|
35
|
+
{"learning_rate": [0.1, 0.01], "n_estimators": [50, 100]}
|
|
36
|
+
),
|
|
37
|
+
],
|
|
38
|
+
),
|
|
39
|
+
"sampling": ImputationConfig(
|
|
40
|
+
methods=["DistributionImputer"],
|
|
41
|
+
modules=["phil.imputation"],
|
|
42
|
+
grids=[ParameterGrid({"random_state": np.arange(0, 100, 1)})],
|
|
43
|
+
),
|
|
44
|
+
"finance": ImputationConfig(
|
|
45
|
+
methods=["IterativeImputer", "KNNImputer", "SimpleImputer"],
|
|
46
|
+
modules=["sklearn.impute"] * 3,
|
|
47
|
+
grids=[
|
|
48
|
+
ParameterGrid({"estimator": ["BayesianRidge"], "max_iter": [10, 50]}),
|
|
49
|
+
ParameterGrid(
|
|
50
|
+
{"n_neighbors": [3, 5, 10], "weights": ["uniform", "distance"]}
|
|
51
|
+
),
|
|
52
|
+
ParameterGrid({"strategy": ["mean", "median"]}),
|
|
53
|
+
],
|
|
54
|
+
),
|
|
55
|
+
"healthcare": ImputationConfig(
|
|
56
|
+
methods=["KNNImputer", "SimpleImputer", "IterativeImputer"],
|
|
57
|
+
modules=["sklearn.impute"] * 3,
|
|
58
|
+
grids=[
|
|
59
|
+
ParameterGrid({"n_neighbors": [5, 10], "weights": ["distance"]}),
|
|
60
|
+
ParameterGrid({"strategy": ["median", "most_frequent"]}),
|
|
61
|
+
ParameterGrid(
|
|
62
|
+
{"estimator": ["RandomForestRegressor"], "max_iter": [10, 20]}
|
|
63
|
+
),
|
|
64
|
+
],
|
|
65
|
+
),
|
|
66
|
+
"marketing": ImputationConfig(
|
|
67
|
+
methods=["SimpleImputer", "KNNImputer", "IterativeImputer"],
|
|
68
|
+
modules=["sklearn.impute"] * 3,
|
|
69
|
+
grids=[
|
|
70
|
+
ParameterGrid(
|
|
71
|
+
{
|
|
72
|
+
"strategy": ["most_frequent", "constant"],
|
|
73
|
+
"fill_value": ["unknown"],
|
|
74
|
+
}
|
|
75
|
+
),
|
|
76
|
+
ParameterGrid({"n_neighbors": [3, 5], "weights": ["uniform"]}),
|
|
77
|
+
ParameterGrid(
|
|
78
|
+
{"estimator": ["GradientBoostingRegressor"], "max_iter": [10, 30]}
|
|
79
|
+
),
|
|
80
|
+
],
|
|
81
|
+
),
|
|
82
|
+
"engineering": ImputationConfig(
|
|
83
|
+
methods=["SimpleImputer", "KNNImputer", "IterativeImputer"],
|
|
84
|
+
modules=["sklearn.impute"] * 3,
|
|
85
|
+
grids=[
|
|
86
|
+
ParameterGrid({"strategy": ["mean", "median"]}),
|
|
87
|
+
ParameterGrid({"n_neighbors": [3, 5, 7], "weights": ["distance"]}),
|
|
88
|
+
ParameterGrid(
|
|
89
|
+
{"estimator": ["DecisionTreeRegressor"], "max_iter": [10, 20]}
|
|
90
|
+
),
|
|
91
|
+
],
|
|
92
|
+
),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def get(cls, name: str) -> ImputationConfig:
|
|
97
|
+
return cls._grids.get(name, cls._grids["default"])
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class ProcessingGallery:
|
|
101
|
+
_numeric_methods = {
|
|
102
|
+
"default": PreprocessingConfig(method="StandardScaler"),
|
|
103
|
+
"finance": PreprocessingConfig(
|
|
104
|
+
method="MinMaxScaler", params={"feature_range": [(0, 1)]}
|
|
105
|
+
),
|
|
106
|
+
"healthcare": PreprocessingConfig(method="RobustScaler"),
|
|
107
|
+
"marketing": PreprocessingConfig(
|
|
108
|
+
method="PowerTransformer", params={"method": ["yeo-johnson"]}
|
|
109
|
+
),
|
|
110
|
+
"engineering": PreprocessingConfig(method="StandardScaler"),
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
_categorical_methods = {
|
|
114
|
+
"default": PreprocessingConfig(method="OneHotEncoder"),
|
|
115
|
+
"finance": PreprocessingConfig(
|
|
116
|
+
method="OneHotEncoder", params={"handle_unknown": ["ignore"]}
|
|
117
|
+
),
|
|
118
|
+
"healthcare": PreprocessingConfig(
|
|
119
|
+
method="OrdinalEncoder",
|
|
120
|
+
params={"handle_unknown": ["use_encoded_value"]},
|
|
121
|
+
),
|
|
122
|
+
"marketing": PreprocessingConfig(
|
|
123
|
+
method="OneHotEncoder",
|
|
124
|
+
params={"sparse": [False], "handle_unknown": ["ignore"]},
|
|
125
|
+
),
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def get(cls, name: str = "default") -> Dict[str, PreprocessingConfig]:
|
|
130
|
+
return {
|
|
131
|
+
"num": cls._numeric_methods.get(name, cls._numeric_methods["default"]),
|
|
132
|
+
"cat": cls._categorical_methods.get(
|
|
133
|
+
name, cls._categorical_methods["default"]
|
|
134
|
+
),
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class MagicGallery:
|
|
139
|
+
@staticmethod
|
|
140
|
+
def get(method: str) -> BaseModel:
|
|
141
|
+
if method == "ECT":
|
|
142
|
+
return ECTConfig(
|
|
143
|
+
num_thetas=64,
|
|
144
|
+
radius=1.0,
|
|
145
|
+
resolution=100,
|
|
146
|
+
scale=500,
|
|
147
|
+
seed=42,
|
|
148
|
+
)
|
|
149
|
+
raise ValueError(f"Unknown magic method: {method}")
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configuration models for Phil's imputation strategies.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict, List
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, Field
|
|
8
|
+
from sklearn.model_selection import ParameterGrid
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ImputationConfig(BaseModel):
|
|
12
|
+
"""Configuration for imputation methods and parameter grids."""
|
|
13
|
+
|
|
14
|
+
model_config = {"arbitrary_types_allowed": True}
|
|
15
|
+
|
|
16
|
+
methods: List[str] = Field(..., description="Names of imputation methods")
|
|
17
|
+
modules: List[str] = Field(..., description="Python modules containing methods")
|
|
18
|
+
grids: List[ParameterGrid] = Field(..., description="Parameter grids for methods")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PreprocessingConfig(BaseModel):
|
|
22
|
+
"""Configuration for data preprocessing steps."""
|
|
23
|
+
|
|
24
|
+
method: str
|
|
25
|
+
module: str = "sklearn.preprocessing"
|
|
26
|
+
params: Dict[str, Any] = Field(default_factory=dict)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Distribution-preserving imputation strategies.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from sklearn.base import BaseEstimator
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DistributionImputer(BaseEstimator):
|
|
11
|
+
"""Imputer that samples from empirical observed values."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, missing_values=np.nan, random_state=None, threshold=1.0):
|
|
14
|
+
if not 0 <= threshold <= 1:
|
|
15
|
+
raise ValueError("threshold must be between 0 and 1")
|
|
16
|
+
self.missing_values = missing_values
|
|
17
|
+
self.random_state = random_state
|
|
18
|
+
self.threshold = threshold
|
|
19
|
+
|
|
20
|
+
def fit(self, X, y):
|
|
21
|
+
if not isinstance(y, (np.ndarray, pd.Series)):
|
|
22
|
+
y = np.asarray(y)
|
|
23
|
+
|
|
24
|
+
if y.ndim != 1:
|
|
25
|
+
raise ValueError("DistributionImputer only supports 1D y.")
|
|
26
|
+
|
|
27
|
+
self.dtype_ = y.dtype
|
|
28
|
+
self.is_categorical_ = y.dtype.kind in "OSU"
|
|
29
|
+
|
|
30
|
+
if not self.is_categorical_:
|
|
31
|
+
y = y.astype(float, copy=True)
|
|
32
|
+
else:
|
|
33
|
+
y = y.astype(object, copy=True)
|
|
34
|
+
|
|
35
|
+
missing_mask = (y == self.missing_values) | pd.isnull(y)
|
|
36
|
+
fraction_missing = missing_mask.sum() / y.size
|
|
37
|
+
|
|
38
|
+
if fraction_missing == 1.0:
|
|
39
|
+
self.skip_imputation_ = True
|
|
40
|
+
self.distribution_ = np.array([], dtype=self.dtype_)
|
|
41
|
+
else:
|
|
42
|
+
self.skip_imputation_ = fraction_missing > self.threshold
|
|
43
|
+
if not self.skip_imputation_:
|
|
44
|
+
self.distribution_ = y[~missing_mask]
|
|
45
|
+
else:
|
|
46
|
+
self.distribution_ = np.array([], dtype=self.dtype_)
|
|
47
|
+
|
|
48
|
+
if isinstance(self.random_state, np.random.RandomState):
|
|
49
|
+
self.rng_ = self.random_state
|
|
50
|
+
else:
|
|
51
|
+
self.rng_ = np.random.RandomState(self.random_state)
|
|
52
|
+
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def predict(self, X):
|
|
56
|
+
if not hasattr(self, "distribution_"):
|
|
57
|
+
raise RuntimeError("Call fit before predict")
|
|
58
|
+
|
|
59
|
+
n_samples = X.shape[0]
|
|
60
|
+
|
|
61
|
+
if self.skip_imputation_ or self.distribution_.size == 0:
|
|
62
|
+
if self.is_categorical_:
|
|
63
|
+
return np.full(n_samples, None, dtype=object)
|
|
64
|
+
return np.full(n_samples, np.nan, dtype=float)
|
|
65
|
+
|
|
66
|
+
predictions = self.rng_.choice(self.distribution_, size=n_samples, replace=True)
|
|
67
|
+
return predictions.astype(self.dtype_)
|
phil/magic/__init__.py
ADDED
phil/magic/base.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base interfaces for descriptor generation methods.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import List
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Magic(ABC):
|
|
13
|
+
def __init__(self, config: BaseModel):
|
|
14
|
+
self.config = config
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def generate(self, data: List[np.ndarray]) -> List[np.ndarray]:
|
|
18
|
+
pass
|
phil/magic/config.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Euler Characteristic Transform (ECT) configuration.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ECTConfig(BaseModel):
|
|
9
|
+
num_thetas: int = Field(..., description="Number of angles to sample")
|
|
10
|
+
radius: float = Field(..., description="Maximum radius for filtration")
|
|
11
|
+
resolution: int = Field(..., description="Number of points per direction")
|
|
12
|
+
scale: int = Field(..., description="Scaling factor for point cloud")
|
|
13
|
+
normalize: bool = Field(True, description="Whether to normalize the ECT output")
|
|
14
|
+
seed: int = Field(0, description="Random seed for reproducibility")
|
phil/magic/ect.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Euler Characteristic Transform implementation backed by Rust.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import List
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from . import rust_backend
|
|
10
|
+
from .base import Magic
|
|
11
|
+
from .config import ECTConfig
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ECT(Magic):
|
|
15
|
+
def __init__(self, config: ECTConfig):
|
|
16
|
+
self.config = config
|
|
17
|
+
self.configure(**config.model_dump())
|
|
18
|
+
|
|
19
|
+
def configure(self, **kwargs):
|
|
20
|
+
for key, value in kwargs.items():
|
|
21
|
+
if hasattr(self.config, key):
|
|
22
|
+
setattr(self, key, value)
|
|
23
|
+
else:
|
|
24
|
+
raise ValueError(f"Invalid configuration key: {key}")
|
|
25
|
+
|
|
26
|
+
def generate(self, X: List[np.ndarray]) -> List[np.ndarray]:
|
|
27
|
+
if not isinstance(X, list):
|
|
28
|
+
raise ValueError("Input must be a list of numpy arrays")
|
|
29
|
+
if not X or any(x.size == 0 for x in X):
|
|
30
|
+
raise ValueError("Input cannot be empty")
|
|
31
|
+
|
|
32
|
+
scale = float(self.scale)
|
|
33
|
+
descriptors: List[np.ndarray] = []
|
|
34
|
+
for sample in X:
|
|
35
|
+
if np.ndim(sample) != 2:
|
|
36
|
+
raise ValueError("Each sample must be a 2D numpy array")
|
|
37
|
+
cloud = np.asarray(sample, dtype=np.float32)
|
|
38
|
+
descriptor = rust_backend.compute_ect_descriptor(
|
|
39
|
+
points=cloud,
|
|
40
|
+
num_thetas=self.num_thetas,
|
|
41
|
+
radius=self.radius,
|
|
42
|
+
resolution=self.resolution,
|
|
43
|
+
scale=scale,
|
|
44
|
+
seed=self.seed,
|
|
45
|
+
)
|
|
46
|
+
if self.normalize:
|
|
47
|
+
descriptor = self._normalize(descriptor)
|
|
48
|
+
descriptors.append(descriptor)
|
|
49
|
+
|
|
50
|
+
return descriptors
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def _normalize(arr: np.ndarray) -> np.ndarray:
|
|
54
|
+
lo = np.min(arr)
|
|
55
|
+
hi = np.max(arr)
|
|
56
|
+
if hi <= lo:
|
|
57
|
+
return np.zeros_like(arr)
|
|
58
|
+
return (arr - lo) / (hi - lo)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""ECT backend adapter backed by the `trailed` package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _load_backend():
|
|
11
|
+
try:
|
|
12
|
+
return importlib.import_module("trailed")
|
|
13
|
+
except ModuleNotFoundError as exc:
|
|
14
|
+
raise ModuleNotFoundError("No ECT backend found. Install `trailed`.") from exc
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_BACKEND = _load_backend()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def compute_ect_descriptor(
|
|
21
|
+
points: np.ndarray,
|
|
22
|
+
num_thetas: int,
|
|
23
|
+
radius: float,
|
|
24
|
+
resolution: int,
|
|
25
|
+
scale: float,
|
|
26
|
+
seed: int,
|
|
27
|
+
) -> np.ndarray:
|
|
28
|
+
"""Compute a single-sample ECT descriptor with stable output shape.
|
|
29
|
+
|
|
30
|
+
Returns an array shaped [num_thetas, resolution].
|
|
31
|
+
"""
|
|
32
|
+
points = np.asarray(points, dtype=np.float32)
|
|
33
|
+
ect = _BACKEND.compute_ect_from_numpy(
|
|
34
|
+
points=points,
|
|
35
|
+
num_thetas=num_thetas,
|
|
36
|
+
resolution=resolution,
|
|
37
|
+
radius=radius,
|
|
38
|
+
scale=scale,
|
|
39
|
+
seed=seed,
|
|
40
|
+
normalized=False,
|
|
41
|
+
parallel=True,
|
|
42
|
+
)
|
|
43
|
+
ect = np.asarray(ect, dtype=np.float32)
|
|
44
|
+
if ect.shape == (resolution, num_thetas):
|
|
45
|
+
return ect.T
|
|
46
|
+
if ect.shape == (num_thetas, resolution):
|
|
47
|
+
return ect
|
|
48
|
+
raise ValueError(f"Unexpected ECT shape from trailed backend: {ect.shape}")
|
phil/phil.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, List, Tuple
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
from sklearn.compose import ColumnTransformer
|
|
9
|
+
from sklearn.exceptions import ConvergenceWarning
|
|
10
|
+
from sklearn.impute import IterativeImputer
|
|
11
|
+
from sklearn.pipeline import Pipeline
|
|
12
|
+
|
|
13
|
+
from phil.gallery import GridGallery, MagicGallery, ProcessingGallery
|
|
14
|
+
from phil.imputation import ImputationConfig
|
|
15
|
+
from phil.magic import Magic
|
|
16
|
+
import phil.magic as METHODS
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Phil:
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
samples: int = 30,
|
|
23
|
+
param_grid: str = "default",
|
|
24
|
+
magic: str = "ECT",
|
|
25
|
+
config=None,
|
|
26
|
+
random_state=None,
|
|
27
|
+
):
|
|
28
|
+
self.config, self.magic = self._configure_magic_method(
|
|
29
|
+
magic=magic, config=config
|
|
30
|
+
)
|
|
31
|
+
self.samples = samples
|
|
32
|
+
self.param_grid = self._configure_param_grid(param_grid)
|
|
33
|
+
self.random_state = random_state
|
|
34
|
+
self.representations = []
|
|
35
|
+
self.magic_descriptors = []
|
|
36
|
+
|
|
37
|
+
def impute(self, df: pd.DataFrame, max_iter: int = 10) -> List[np.ndarray]:
|
|
38
|
+
if df.isnull().sum().sum() == 0:
|
|
39
|
+
raise ValueError("No missing values found in the input DataFrame.")
|
|
40
|
+
categorical_columns, numerical_columns = self._identify_column_types(df)
|
|
41
|
+
preprocessor = self._configure_preprocessor(
|
|
42
|
+
"default", categorical_columns, numerical_columns
|
|
43
|
+
)
|
|
44
|
+
imputers = self._create_imputers(preprocessor, max_iter)
|
|
45
|
+
self.selected_imputers = self._select_imputations(imputers)
|
|
46
|
+
return self._apply_imputations(df, self.selected_imputers)
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def _identify_column_types(df: pd.DataFrame) -> Tuple[List[str], List[str]]:
|
|
50
|
+
categorical_columns = df.select_dtypes(
|
|
51
|
+
include=["object", "category"]
|
|
52
|
+
).columns.tolist()
|
|
53
|
+
numerical_columns = df.select_dtypes(
|
|
54
|
+
include=["number", "bool"]
|
|
55
|
+
).columns.tolist()
|
|
56
|
+
return categorical_columns, numerical_columns
|
|
57
|
+
|
|
58
|
+
def _create_imputers(
|
|
59
|
+
self, preprocessor: ColumnTransformer, max_iter: int
|
|
60
|
+
) -> List[Pipeline]:
|
|
61
|
+
imputers = []
|
|
62
|
+
for method, module, params in zip(
|
|
63
|
+
self.param_grid.methods,
|
|
64
|
+
self.param_grid.modules,
|
|
65
|
+
self.param_grid.grids,
|
|
66
|
+
):
|
|
67
|
+
model = self._import_model(module, method)
|
|
68
|
+
for param_vals in params:
|
|
69
|
+
compatible_params = {
|
|
70
|
+
k: v
|
|
71
|
+
for k, v in param_vals.items()
|
|
72
|
+
if k in model.__init__.__code__.co_varnames
|
|
73
|
+
}
|
|
74
|
+
estimator = model(**compatible_params)
|
|
75
|
+
imputers.append(self._build_pipeline(preprocessor, estimator, max_iter))
|
|
76
|
+
return imputers
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def _import_model(module: str, method: str):
|
|
80
|
+
imported_module = importlib.import_module(module)
|
|
81
|
+
return getattr(imported_module, method)
|
|
82
|
+
|
|
83
|
+
def _build_pipeline(
|
|
84
|
+
self, preprocessor: ColumnTransformer, estimator, max_iter: int
|
|
85
|
+
) -> Pipeline:
|
|
86
|
+
return Pipeline(
|
|
87
|
+
[
|
|
88
|
+
("preprocessor", preprocessor),
|
|
89
|
+
(
|
|
90
|
+
"imputer",
|
|
91
|
+
IterativeImputer(
|
|
92
|
+
estimator=estimator,
|
|
93
|
+
random_state=self.random_state,
|
|
94
|
+
max_iter=max_iter,
|
|
95
|
+
),
|
|
96
|
+
),
|
|
97
|
+
]
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def _select_imputations(self, imputers: List[Pipeline]) -> List[Pipeline]:
|
|
101
|
+
np.random.seed(self.random_state)
|
|
102
|
+
selected_idxs = np.random.choice(
|
|
103
|
+
range(len(imputers)),
|
|
104
|
+
min(self.samples, len(imputers)),
|
|
105
|
+
replace=False,
|
|
106
|
+
)
|
|
107
|
+
return [imputers[idx] for idx in selected_idxs]
|
|
108
|
+
|
|
109
|
+
def _apply_imputations(
|
|
110
|
+
self, df: pd.DataFrame, imputers: List[Pipeline]
|
|
111
|
+
) -> List[np.ndarray]:
|
|
112
|
+
imputations = []
|
|
113
|
+
with warnings.catch_warnings():
|
|
114
|
+
warnings.filterwarnings("ignore", category=ConvergenceWarning)
|
|
115
|
+
for imputer in imputers:
|
|
116
|
+
imputer.fit(df)
|
|
117
|
+
imputations.append(imputer.transform(df))
|
|
118
|
+
return imputations
|
|
119
|
+
|
|
120
|
+
def generate_descriptors(self) -> List[np.ndarray]:
|
|
121
|
+
return self.magic.generate(self.representations)
|
|
122
|
+
|
|
123
|
+
def fit(self, df: pd.DataFrame, max_iter: int = 5) -> pd.DataFrame:
|
|
124
|
+
self.representations = self.impute(df, max_iter)
|
|
125
|
+
self.magic_descriptors = self.generate_descriptors()
|
|
126
|
+
|
|
127
|
+
assert len(self.representations) == len(self.magic_descriptors)
|
|
128
|
+
self.closest_index = self._select_representative(self.magic_descriptors)
|
|
129
|
+
X = self.representations[self.closest_index]
|
|
130
|
+
self.pipeline = self.selected_imputers[self.closest_index]
|
|
131
|
+
imputed_columns = self._get_imputed_columns(
|
|
132
|
+
transformer=self.pipeline["preprocessor"]
|
|
133
|
+
)
|
|
134
|
+
return pd.DataFrame(X, columns=imputed_columns)
|
|
135
|
+
|
|
136
|
+
def transform(self, df: pd.DataFrame, max_iter: int = 5) -> pd.DataFrame:
|
|
137
|
+
if not hasattr(self, "pipeline"):
|
|
138
|
+
raise RuntimeError("Pipeline not fitted. Call `fit` first.")
|
|
139
|
+
|
|
140
|
+
imputed_columns = self._get_imputed_columns(
|
|
141
|
+
transformer=self.pipeline["preprocessor"]
|
|
142
|
+
)
|
|
143
|
+
return pd.DataFrame(self.pipeline.transform(df), columns=imputed_columns)
|
|
144
|
+
|
|
145
|
+
@staticmethod
|
|
146
|
+
def _get_imputed_columns(transformer: ColumnTransformer) -> List[str]:
|
|
147
|
+
return transformer.get_feature_names_out()
|
|
148
|
+
|
|
149
|
+
@staticmethod
|
|
150
|
+
def _select_representative(descriptors: List[np.ndarray]) -> int:
|
|
151
|
+
stacked = np.stack(descriptors)
|
|
152
|
+
avg_descriptor = stacked.mean(axis=0)
|
|
153
|
+
norms = np.linalg.norm(
|
|
154
|
+
(stacked - avg_descriptor).reshape(len(descriptors), -1), axis=1
|
|
155
|
+
)
|
|
156
|
+
return int(np.argmin(norms))
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def _configure_magic_method(magic: str, config) -> Tuple[BaseModel, Magic]:
|
|
160
|
+
magic_method = getattr(METHODS, magic, None)
|
|
161
|
+
if magic_method is None:
|
|
162
|
+
raise ValueError(f"Magic method '{magic}' not found.")
|
|
163
|
+
if not isinstance(config, BaseModel):
|
|
164
|
+
config = MagicGallery.get(magic)
|
|
165
|
+
return config, magic_method(config=config)
|
|
166
|
+
|
|
167
|
+
@staticmethod
|
|
168
|
+
def _configure_param_grid(param_grid) -> ImputationConfig:
|
|
169
|
+
if isinstance(param_grid, str):
|
|
170
|
+
return GridGallery.get(param_grid)
|
|
171
|
+
if isinstance(param_grid, ImputationConfig):
|
|
172
|
+
return param_grid
|
|
173
|
+
if isinstance(param_grid, BaseModel):
|
|
174
|
+
if (
|
|
175
|
+
not hasattr(param_grid, "methods")
|
|
176
|
+
or not hasattr(param_grid, "modules")
|
|
177
|
+
or not hasattr(param_grid, "grids")
|
|
178
|
+
):
|
|
179
|
+
raise ValueError("Invalid parameter grid configuration.")
|
|
180
|
+
return ImputationConfig(
|
|
181
|
+
methods=param_grid.methods,
|
|
182
|
+
modules=param_grid.modules,
|
|
183
|
+
grids=param_grid.grids,
|
|
184
|
+
)
|
|
185
|
+
if isinstance(param_grid, dict):
|
|
186
|
+
if not all(key in param_grid for key in ["methods", "modules", "grids"]):
|
|
187
|
+
raise ValueError("Invalid parameter grid configuration.")
|
|
188
|
+
return ImputationConfig(
|
|
189
|
+
methods=param_grid["methods"],
|
|
190
|
+
modules=param_grid["modules"],
|
|
191
|
+
grids=param_grid["grids"],
|
|
192
|
+
)
|
|
193
|
+
raise ValueError("Invalid parameter grid type.")
|
|
194
|
+
|
|
195
|
+
@staticmethod
|
|
196
|
+
def _configure_preprocessor(
|
|
197
|
+
strategy: str,
|
|
198
|
+
categorical_columns: List[str],
|
|
199
|
+
numerical_columns: List[str],
|
|
200
|
+
) -> ColumnTransformer:
|
|
201
|
+
strategy = ProcessingGallery.get(strategy)
|
|
202
|
+
transformers: List[Tuple[str, Any, List[str]]] = []
|
|
203
|
+
|
|
204
|
+
for key, preprocessing_config in strategy.items():
|
|
205
|
+
try:
|
|
206
|
+
model = Phil._import_model(
|
|
207
|
+
preprocessing_config.module, preprocessing_config.method
|
|
208
|
+
)
|
|
209
|
+
except (ImportError, AttributeError) as e:
|
|
210
|
+
raise RuntimeError(
|
|
211
|
+
f"Failed to import model {preprocessing_config.method} from module {preprocessing_config.module}: {e}"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
transformer = model(**preprocessing_config.params)
|
|
215
|
+
|
|
216
|
+
if key == "num" and len(numerical_columns) > 0:
|
|
217
|
+
transformers.append((key, transformer, numerical_columns))
|
|
218
|
+
elif key == "cat" and len(categorical_columns) > 0:
|
|
219
|
+
transformers.append((key, transformer, categorical_columns))
|
|
220
|
+
|
|
221
|
+
return ColumnTransformer(transformers, verbose_feature_names_out=True)
|
phil/transformers.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Scikit-learn compatible transformers for Phil.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional, Union
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from sklearn.base import BaseEstimator, TransformerMixin
|
|
9
|
+
|
|
10
|
+
from phil.imputation import ImputationConfig
|
|
11
|
+
from phil.phil import Phil
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PhilTransformer(BaseEstimator, TransformerMixin):
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
samples: int = 30,
|
|
18
|
+
param_grid: Union[str, ImputationConfig] = "default",
|
|
19
|
+
magic: str = "ECT",
|
|
20
|
+
config: Optional[dict] = None,
|
|
21
|
+
random_state: Optional[int] = None,
|
|
22
|
+
max_iter: int = 5,
|
|
23
|
+
):
|
|
24
|
+
self.samples = samples
|
|
25
|
+
self.param_grid = param_grid
|
|
26
|
+
self.magic = magic
|
|
27
|
+
self.config = config
|
|
28
|
+
self.random_state = random_state
|
|
29
|
+
self.max_iter = max_iter
|
|
30
|
+
self.phil = None
|
|
31
|
+
|
|
32
|
+
def fit(self, X: pd.DataFrame, y: Any = None) -> "PhilTransformer":
|
|
33
|
+
self.feature_names_in_ = X.columns.tolist()
|
|
34
|
+
self.n_features_in_ = len(self.feature_names_in_)
|
|
35
|
+
|
|
36
|
+
self.phil = Phil(
|
|
37
|
+
samples=self.samples,
|
|
38
|
+
param_grid=self.param_grid,
|
|
39
|
+
magic=self.magic,
|
|
40
|
+
config=self.config,
|
|
41
|
+
random_state=self.random_state,
|
|
42
|
+
)
|
|
43
|
+
self.phil.fit(X, max_iter=self.max_iter)
|
|
44
|
+
return self
|
|
45
|
+
|
|
46
|
+
def transform(self, X: pd.DataFrame) -> pd.DataFrame:
|
|
47
|
+
if self.phil is None:
|
|
48
|
+
raise RuntimeError(
|
|
49
|
+
"This PhilTransformer instance is not fitted yet. "
|
|
50
|
+
"Call 'fit' before using this estimator."
|
|
51
|
+
)
|
|
52
|
+
return self.phil.transform(X, max_iter=self.max_iter)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: philler
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Phil: representation-guided multiverse imputation with Rust-backed ECT.
|
|
5
|
+
Author-email: Krv Labs <team@krv.ai>
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: pandas>=2.2.3
|
|
10
|
+
Requires-Dist: pydantic>=2.10.6
|
|
11
|
+
Requires-Dist: scikit-learn>=1.6.1
|
|
12
|
+
Requires-Dist: scipy>=1.15.2
|
|
13
|
+
Requires-Dist: trailed>=0.1.0
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# Phil
|
|
17
|
+
|
|
18
|
+
`Phil` is a representation-guided imputation library for missing tabular data.
|
|
19
|
+
|
|
20
|
+
It generates multiple imputations using a configurable strategy grid, computes
|
|
21
|
+
Euler Characteristic Transform (ECT) descriptors over each imputed dataset, and
|
|
22
|
+
selects the most representative imputation from the candidate set.
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install phil
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`phil` requires the `trailed` backend for ECT computation. Install it from the
|
|
31
|
+
KRV research index or provide a compatible local build.
|
|
32
|
+
|
|
33
|
+
## What Phil Does
|
|
34
|
+
|
|
35
|
+
1. **Impute** — runs a grid of imputation strategies (sklearn estimators or custom) over the input dataframe, producing a set of candidate datasets
|
|
36
|
+
2. **Describe** — computes an ECT descriptor for each candidate via the `trailed` backend
|
|
37
|
+
3. **Select** — picks the candidate closest to the mean descriptor (most representative imputation)
|
|
38
|
+
4. **Transform** — exposes the fitted pipeline for inference on new data
|
|
39
|
+
|
|
40
|
+
## Quick Start
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import pandas as pd
|
|
44
|
+
from phil import Phil
|
|
45
|
+
|
|
46
|
+
df = pd.read_csv("data_with_missing.csv")
|
|
47
|
+
|
|
48
|
+
phil = Phil(samples=30, random_state=42)
|
|
49
|
+
imputed_df = phil.fit(df)
|
|
50
|
+
|
|
51
|
+
# Apply the same fitted pipeline to new data
|
|
52
|
+
new_df = phil.transform(new_data)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### scikit-learn Pipeline Integration
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from sklearn.pipeline import Pipeline
|
|
59
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
60
|
+
from phil import PhilTransformer
|
|
61
|
+
|
|
62
|
+
pipe = Pipeline([
|
|
63
|
+
("imputer", PhilTransformer(samples=20, random_state=0)),
|
|
64
|
+
("model", RandomForestClassifier()),
|
|
65
|
+
])
|
|
66
|
+
pipe.fit(X_train, y_train)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Configuration
|
|
70
|
+
|
|
71
|
+
### Imputation grids
|
|
72
|
+
|
|
73
|
+
`Phil` ships with named grids accessible via `GridGallery`:
|
|
74
|
+
|
|
75
|
+
| Name | Methods |
|
|
76
|
+
| ------------- | ----------------------------------------------------------- |
|
|
77
|
+
| `default` | BayesianRidge, DecisionTree, RandomForest, GradientBoosting |
|
|
78
|
+
| `sampling` | DistributionImputer (empirical sampling) |
|
|
79
|
+
| `finance` | IterativeImputer, KNNImputer, SimpleImputer |
|
|
80
|
+
| `healthcare` | KNNImputer, SimpleImputer, IterativeImputer |
|
|
81
|
+
| `marketing` | SimpleImputer, KNNImputer, IterativeImputer |
|
|
82
|
+
| `engineering` | SimpleImputer, KNNImputer, IterativeImputer |
|
|
83
|
+
|
|
84
|
+
Pass a grid name or an `ImputationConfig` directly:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from phil import Phil, ImputationConfig
|
|
88
|
+
from sklearn.model_selection import ParameterGrid
|
|
89
|
+
|
|
90
|
+
config = ImputationConfig(
|
|
91
|
+
methods=["KNNImputer"],
|
|
92
|
+
modules=["sklearn.impute"],
|
|
93
|
+
grids=[ParameterGrid({"n_neighbors": [3, 5, 7]})],
|
|
94
|
+
)
|
|
95
|
+
phil = Phil(param_grid=config)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### ECT descriptor
|
|
99
|
+
|
|
100
|
+
ECT is configured via `ECTConfig`:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from phil import Phil, ECTConfig
|
|
104
|
+
|
|
105
|
+
ect_config = ECTConfig(
|
|
106
|
+
num_thetas=64,
|
|
107
|
+
radius=1.0,
|
|
108
|
+
resolution=100,
|
|
109
|
+
scale=500,
|
|
110
|
+
normalize=True,
|
|
111
|
+
seed=42,
|
|
112
|
+
)
|
|
113
|
+
phil = Phil(config=ect_config)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Development
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
uv sync --all-extras
|
|
120
|
+
uv run pytest -v
|
|
121
|
+
uv run black phil/ tests/
|
|
122
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
phil/__init__.py,sha256=9qA-x4ZrloLe5ZUatzPZbnkE0ArDyamhAnJ5KlhodLI,455
|
|
2
|
+
phil/gallery.py,sha256=QlegEYRrfaLSQc4N_wDQYrjMHcXc1YfQ1QD3-Q8TWYI,5338
|
|
3
|
+
phil/phil.py,sha256=9NdRisPYL1JNqSMCCfcGXU9b38uBGwIDsEYo7LRyb1M,8538
|
|
4
|
+
phil/transformers.py,sha256=qlfN5qq43bWLSVnOTIcJNgeLLtpM2awuVDGr8vgVxjE,1586
|
|
5
|
+
phil/imputation/__init__.py,sha256=e5kSgr2T4uhWFE8F63ixZFWuC6gPpRmXTYRSafC66Y0,215
|
|
6
|
+
phil/imputation/config.py,sha256=rqoNK5wvzH-xF29KcFBWtI2wo8X5cxOLsoimjhb6Ez8,809
|
|
7
|
+
phil/imputation/distribution.py,sha256=lDN6wevpNtlV0F6DagI7EDDLKZSYabBrTCyg9HkyMEs,2242
|
|
8
|
+
phil/magic/__init__.py,sha256=7ztowOwMwniADEBIJYxej3YwOJFgJ_b_2MjLi7B_eVg,174
|
|
9
|
+
phil/magic/base.py,sha256=RO7_Aq0jZJPlxCEPGFdXtkYbEjF5arxD9CDzicbpaWY,365
|
|
10
|
+
phil/magic/config.py,sha256=UAeL5-la6R5GGWaOxr3V8pHYptXyflwEid0EIko3jSM,590
|
|
11
|
+
phil/magic/ect.py,sha256=wXT3lVZW-q7NvgvKrCH3DZnWuwUTGugBn5f_1XEbB3U,1796
|
|
12
|
+
phil/magic/rust_backend.py,sha256=ec_umoUNf6E8O1aKOtmfVS7IEJr13pmYHcb9ATqBxto,1222
|
|
13
|
+
philler-1.0.0.dist-info/licenses/LICENSE,sha256=HfxHaB1HkeMGtfAEnffSxaXnFC6VIy4rrZnTxPD7xJo,1500
|
|
14
|
+
philler-1.0.0.dist-info/METADATA,sha256=iTHQqcj1loGh72sg6zG0at5lRjp9D1alf431zga0ohM,3396
|
|
15
|
+
philler-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
16
|
+
philler-1.0.0.dist-info/top_level.txt,sha256=a29xWUCTArptaZ9wsBRhMSJc8qd9EEapZ69mHNmvkPk,5
|
|
17
|
+
philler-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025, Krv Analytics
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phil
|