AutoCarver 7.3.0__tar.gz → 7.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/multiclass_carver.py +1 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/utils/base_carver.py +4 -6
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/binary/binary_combination_evaluators.py +7 -4
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/continuous/continuous_combination_evaluators.py +7 -2
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/continuous/continuous_target_rates.py +9 -2
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/utils/frequency_ci.py +4 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/features.py +1 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/qualitative_feature.py +19 -57
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/quantitative_feature.py +39 -54
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/utils/base_feature.py +67 -4
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/_vectorized.py +23 -8
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/base_measures.py +6 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/qualitative_measures.py +14 -3
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/quantitative_measures.py +13 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/PKG-INFO +1 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/pyproject.toml +2 -1
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/binary_carver.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/continuous_carver.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/carvers/utils/pretty_print.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/binary/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/binary/binary_target_rates.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/continuous/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/combination_evaluator.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/combinations.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/target_rate.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/testing.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/config.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/categorical_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/nested_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/ordinal_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/qualitative_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/quantitatives/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/quantitatives/continuous_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/quantitatives/quantitative_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/utils/base_discretizer.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/utils/multiprocessing.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/utils/type_discretizers.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/categorical_feature.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/nested_feature.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/ordinal_feature.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/datetime_feature.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/numerical_feature.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/utils/grouped_list.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/utils/serialization.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/classification_selector.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/filters/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/filters/base_filters.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/filters/qualitative_filters.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/filters/quantitative_filters.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/regression_selector.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/utils/base_selector.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/utils/pretty_print.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/utils/__init__.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/utils/attributes.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/utils/dependencies.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/utils/extend_docstring.py +0 -0
- {autocarver-7.3.0 → autocarver-7.3.2}/README.md +0 -0
|
@@ -51,7 +51,7 @@ class MulticlassCarver(BinaryCarver):
|
|
|
51
51
|
|
|
52
52
|
# multiclass cannot copy inplace
|
|
53
53
|
if self.config.copy:
|
|
54
|
-
print("WARNING: can't set copy=
|
|
54
|
+
print("WARNING: can't set copy=False for MulticlassCarver (no inplace DataFrame.assign).")
|
|
55
55
|
|
|
56
56
|
def _prepare_samples(self, samples: Samples) -> Samples:
|
|
57
57
|
"""Validates format and content of X and y."""
|
|
@@ -245,12 +245,10 @@ class BaseCarver(BaseDiscretizer, ABC):
|
|
|
245
245
|
if summaries.empty:
|
|
246
246
|
return summaries
|
|
247
247
|
|
|
248
|
-
#
|
|
249
|
-
#
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
excluded = {"feature", "label", "content", "target_mean", "frequency", "dropped", "dropped_reason"}
|
|
248
|
+
# per-modality stats (count, target_mean, frequency) stay columns; only per-feature
|
|
249
|
+
# metrics (sort_by association, n_mod) become index levels so they collapse to one
|
|
250
|
+
# row per feature instead of repeating across every modality.
|
|
251
|
+
excluded = {"feature", "label", "content", "target_mean", "frequency", "count", "dropped", "dropped_reason"}
|
|
254
252
|
indices = [col for col in summaries.columns if col not in excluded]
|
|
255
253
|
indices = ["feature"] + indices + ["label"]
|
|
256
254
|
return summaries.set_index(indices)
|
|
@@ -79,10 +79,13 @@ class BinaryCombinationEvaluator(CombinationEvaluator[pd.DataFrame], ABC):
|
|
|
79
79
|
if pd.notna(cramerv):
|
|
80
80
|
cramerv = round(cramerv / tol) * tol
|
|
81
81
|
|
|
82
|
-
# Tschuprow's T
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
82
|
+
# Tschuprow's T (undefined for a single modality: avoids a divide-by-zero)
|
|
83
|
+
if n_mod_x <= 1:
|
|
84
|
+
tschuprowt = np.nan
|
|
85
|
+
else:
|
|
86
|
+
tschuprowt = cramerv / np.sqrt(np.sqrt(n_mod_x - 1))
|
|
87
|
+
if pd.notna(tschuprowt):
|
|
88
|
+
tschuprowt = round(tschuprowt / tol) * tol
|
|
86
89
|
|
|
87
90
|
return {"cramerv": cramerv, "tschuprowt": tschuprowt}
|
|
88
91
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Module for continuous combination evaluators."""
|
|
2
2
|
|
|
3
3
|
import math
|
|
4
|
+
import warnings
|
|
4
5
|
from abc import ABC
|
|
5
6
|
from collections.abc import Iterable, Iterator
|
|
6
7
|
from typing import Any
|
|
@@ -64,9 +65,13 @@ class ContinuousCombinationEvaluator(CombinationEvaluator[pd.Series], ABC):
|
|
|
64
65
|
"""
|
|
65
66
|
_, _ = n_obs, tol # unused attribute
|
|
66
67
|
|
|
67
|
-
# Kruskal-Wallis' H
|
|
68
|
+
# Kruskal-Wallis' H (degenerate groups legitimately yield NaN: scipy's
|
|
69
|
+
# SmallSampleWarning / tie-correction RuntimeWarning are silenced here so
|
|
70
|
+
# end users don't see them)
|
|
68
71
|
try:
|
|
69
|
-
|
|
72
|
+
with warnings.catch_warnings():
|
|
73
|
+
warnings.simplefilter("ignore")
|
|
74
|
+
return {"kruskal": kruskal(*tuple(xagg.values))[0]}
|
|
70
75
|
except (ValueError, IndexError):
|
|
71
76
|
return {"kruskal": None}
|
|
72
77
|
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/continuous/continuous_target_rates.py
RENAMED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""set of target rates for binary classification"""
|
|
2
2
|
|
|
3
|
+
import warnings
|
|
3
4
|
from abc import ABC
|
|
4
5
|
from typing import overload
|
|
5
6
|
|
|
@@ -72,7 +73,10 @@ class TargetMean(ContinuousTargetRate):
|
|
|
72
73
|
# pandas stubs widen `Series.apply(func)` to `Series | DataFrame`
|
|
73
74
|
# because they can't tell np.mean returns a scalar; the runtime
|
|
74
75
|
# result is always a Series.
|
|
75
|
-
|
|
76
|
+
# empty modalities legitimately yield NaN: silence numpy's mean-of-empty warnings
|
|
77
|
+
with np.errstate(invalid="ignore", divide="ignore"), warnings.catch_warnings():
|
|
78
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
79
|
+
return xagg.apply(np.mean) # type: ignore
|
|
76
80
|
|
|
77
81
|
def compute_from_stats(self, *, stats: dict, index_to_groupby: dict) -> pd.DataFrame | None:
|
|
78
82
|
"""Closed-form ``(target_mean, frequency)`` per group from per-modality stats.
|
|
@@ -146,7 +150,10 @@ class TargetMedian(ContinuousTargetRate):
|
|
|
146
150
|
def _compute(self, xagg: pd.Series | pd.DataFrame) -> pd.Series:
|
|
147
151
|
"""Computes the mean target rate."""
|
|
148
152
|
# see TargetMean._compute: same pandas-stub widening on apply().
|
|
149
|
-
|
|
153
|
+
# empty modalities legitimately yield NaN: silence numpy's median-of-empty warnings
|
|
154
|
+
with np.errstate(invalid="ignore", divide="ignore"), warnings.catch_warnings():
|
|
155
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
156
|
+
return xagg.apply(np.median) # type: ignore
|
|
150
157
|
|
|
151
158
|
|
|
152
159
|
# class TargetVariance(ContinuousTargetRate):
|
|
@@ -50,7 +50,10 @@ def wilson_upper_bound(
|
|
|
50
50
|
denom = 1.0 + (z * z) / n
|
|
51
51
|
center = (phat + (z * z) / (2.0 * n)) / denom
|
|
52
52
|
half_width = (z / denom) * np.sqrt(phat * (1.0 - phat) / n + (z * z) / (4.0 * n * n))
|
|
53
|
-
|
|
53
|
+
# A proportion CI bound is by definition in [0, 1]; clamp to absorb float
|
|
54
|
+
# rounding (e.g. count == nobs cancels to exactly 1.0 in exact arithmetic but
|
|
55
|
+
# lands at 1.0000000000000002 in floating point).
|
|
56
|
+
upper = np.clip(center + half_width, 0.0, 1.0)
|
|
54
57
|
|
|
55
58
|
if np.isscalar(count):
|
|
56
59
|
return float(upper)
|
|
@@ -604,7 +604,7 @@ class Features:
|
|
|
604
604
|
# defining indices to set
|
|
605
605
|
indices = []
|
|
606
606
|
for col in summaries.columns:
|
|
607
|
-
if col not in ["feature", "label", "content", "target_mean", "frequency"]:
|
|
607
|
+
if col not in ["feature", "label", "content", "target_mean", "frequency", "count"]:
|
|
608
608
|
indices += [col]
|
|
609
609
|
indices = ["feature"] + indices + ["label"]
|
|
610
610
|
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/qualitative_feature.py
RENAMED
|
@@ -33,8 +33,6 @@ class QualitativeFeature(BaseFeature):
|
|
|
33
33
|
super()._restore_from_json(feature_json)
|
|
34
34
|
|
|
35
35
|
def fit(self, X: pd.DataFrame, y: pd.Series | None = None) -> None:
|
|
36
|
-
"""TODO fit stats"""
|
|
37
|
-
|
|
38
36
|
# checking for feature's unique non-nan values
|
|
39
37
|
sorted_unique_values = nan_unique(X[self.version], sort=True)
|
|
40
38
|
|
|
@@ -119,64 +117,28 @@ class QualitativeFeature(BaseFeature):
|
|
|
119
117
|
# adding statistics and history
|
|
120
118
|
return self._add_statistics_to_summary(summary)
|
|
121
119
|
|
|
122
|
-
def
|
|
123
|
-
|
|
120
|
+
def _resolve_grouping(
|
|
121
|
+
self, kept_label: str | float, grouped_values: list, r_value_per_label: dict
|
|
122
|
+
) -> tuple[list, str | float]:
|
|
123
|
+
"""selects the kept value and finalizes grouped values"""
|
|
124
124
|
|
|
125
|
-
#
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
# choosing which value to keep: getting group of kept_value
|
|
126
|
+
kept_value = self.values.get_group(self.value_per_label.get(kept_label))
|
|
127
|
+
# TODO force kept_value to != self.nan like in quantitative feature?
|
|
128
128
|
|
|
129
|
-
# values
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
self.values.group(grouped_values, kept_value)
|
|
129
|
+
# keeping only values not already grouped with kept_value
|
|
130
|
+
grouped_values = [
|
|
131
|
+
self.values.get_group(value) for value in grouped_values if self.values.get_group(value) != kept_value
|
|
132
|
+
]
|
|
134
133
|
|
|
135
|
-
#
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
# checking that kept values exists
|
|
144
|
-
if kept_label not in self.value_per_label:
|
|
145
|
-
raise AttributeError(f"{self} no {kept_label}, in value_per_label: {self.value_per_label}")
|
|
146
|
-
|
|
147
|
-
# checking that grouped values exists
|
|
148
|
-
for grouped_value, grouped_label in zip(grouped_values, grouped_labels):
|
|
149
|
-
if grouped_value is None:
|
|
150
|
-
print(f"{self} no {grouped_label}, in value_per_label: {self.value_per_label}")
|
|
151
|
-
|
|
152
|
-
# choosing which value to keep: getting group of kept_value
|
|
153
|
-
kept_value = self.values.get_group(kept_value)
|
|
154
|
-
# TODO force kept_value to != self.nan like in quantitative feature?
|
|
155
|
-
|
|
156
|
-
# keeping only values not already grouped with kept_value
|
|
157
|
-
grouped_values = [
|
|
158
|
-
self.values.get_group(value)
|
|
159
|
-
for value in grouped_values
|
|
160
|
-
if self.values.get_group(value) != kept_value
|
|
161
|
-
]
|
|
162
|
-
|
|
163
|
-
# deduplicating
|
|
164
|
-
grouped_values = [
|
|
165
|
-
value for num, value in enumerate(grouped_values) if value not in grouped_values[num + 1 :]
|
|
166
|
-
]
|
|
167
|
-
|
|
168
|
-
# updating values if any to group
|
|
169
|
-
if len(grouped_values) > 0:
|
|
170
|
-
# if ordinal_encoding, converting values to unique values
|
|
171
|
-
if self.ordinal_encoding:
|
|
172
|
-
r_value_per_label = {v: self.values[k] for k, v in self.value_per_label.items()}
|
|
173
|
-
grouped_values = [r_value_per_label[value] for value in grouped_values]
|
|
174
|
-
|
|
175
|
-
# grouping values
|
|
176
|
-
self.values.group(grouped_values, kept_value)
|
|
177
|
-
|
|
178
|
-
# updating statistics
|
|
179
|
-
self._update_statistics_value(kept_label, kept_value)
|
|
134
|
+
# deduplicating
|
|
135
|
+
grouped_values = [value for num, value in enumerate(grouped_values) if value not in grouped_values[num + 1 :]]
|
|
136
|
+
|
|
137
|
+
# if ordinal_encoding, converting values to unique values
|
|
138
|
+
if len(grouped_values) > 0 and self.ordinal_encoding:
|
|
139
|
+
grouped_values = [r_value_per_label[value] for value in grouped_values]
|
|
140
|
+
|
|
141
|
+
return grouped_values, kept_value
|
|
180
142
|
|
|
181
143
|
@abstractmethod
|
|
182
144
|
def _specific_formatting(self, ordered_content: list[str]) -> str:
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/quantitative_feature.py
RENAMED
|
@@ -18,57 +18,30 @@ class QuantitativeFeature(BaseFeature):
|
|
|
18
18
|
"""No-op: quantitative features cannot have a default value."""
|
|
19
19
|
_ = value
|
|
20
20
|
|
|
21
|
-
def
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
#
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
# converting labels to values
|
|
46
|
-
kept_value = self.value_per_label[kept_label]
|
|
47
|
-
grouped_values = [self.value_per_label.get(label) for label in grouped_labels]
|
|
48
|
-
|
|
49
|
-
# checking that grouped values exists
|
|
50
|
-
for grouped_value, grouped_label in zip(grouped_values, grouped_labels):
|
|
51
|
-
if grouped_value is None:
|
|
52
|
-
print(f"{self} no {grouped_label}, in value_per_label: {self.value_per_label}")
|
|
53
|
-
|
|
54
|
-
# choosing which value to keep
|
|
55
|
-
which_to_keep = [value for value in grouped_values if value != self.nan]
|
|
56
|
-
|
|
57
|
-
# keeping the largest value amongst the discarded
|
|
58
|
-
if len(which_to_keep) > 0:
|
|
59
|
-
kept_value = max(which_to_keep)
|
|
60
|
-
|
|
61
|
-
# updating values if any to group
|
|
62
|
-
if len(grouped_values) > 0:
|
|
63
|
-
# if ordinal_encoding, converting values to unique values
|
|
64
|
-
if self.ordinal_encoding:
|
|
65
|
-
r_value_per_label = {v: self.values[k] for k, v in self.value_per_label.items()}
|
|
66
|
-
grouped_values = [r_value_per_label[value] for value in grouped_values]
|
|
67
|
-
kept_value = r_value_per_label[kept_value]
|
|
68
|
-
self.values.group(grouped_values, kept_value)
|
|
69
|
-
|
|
70
|
-
# updating statistics
|
|
71
|
-
self._update_statistics_value(kept_label, kept_value)
|
|
21
|
+
def _check_empty_values(self, values: GroupedList) -> None:
|
|
22
|
+
"""checks that inf is amongst values"""
|
|
23
|
+
if values[-1] != np.inf:
|
|
24
|
+
raise ValueError(f"[{self}] Must provide values with values[-1] == numpy.inf")
|
|
25
|
+
|
|
26
|
+
def _resolve_grouping(
|
|
27
|
+
self, kept_label: str | float, grouped_values: list, r_value_per_label: dict
|
|
28
|
+
) -> tuple[list, str | float]:
|
|
29
|
+
"""selects the kept value and finalizes grouped values"""
|
|
30
|
+
|
|
31
|
+
# choosing which value to keep
|
|
32
|
+
kept_value = self.value_per_label[kept_label]
|
|
33
|
+
|
|
34
|
+
# keeping the largest value amongst the discarded
|
|
35
|
+
which_to_keep = [value for value in grouped_values if value != self.nan]
|
|
36
|
+
if len(which_to_keep) > 0:
|
|
37
|
+
kept_value = max(which_to_keep)
|
|
38
|
+
|
|
39
|
+
# if ordinal_encoding, converting values to unique values
|
|
40
|
+
if len(grouped_values) > 0 and self.ordinal_encoding:
|
|
41
|
+
grouped_values = [r_value_per_label[value] for value in grouped_values]
|
|
42
|
+
kept_value = r_value_per_label[kept_value]
|
|
43
|
+
|
|
44
|
+
return grouped_values, kept_value
|
|
72
45
|
|
|
73
46
|
def make_labels(self) -> GroupedList:
|
|
74
47
|
"""gives labels per quantile (values for continuous features)
|
|
@@ -91,6 +64,16 @@ class QuantitativeFeature(BaseFeature):
|
|
|
91
64
|
# converting quantiles in string
|
|
92
65
|
labels = format_quantiles(quantiles)
|
|
93
66
|
|
|
67
|
+
# add NaNs if there are any (grouped into an existing quantile bucket):
|
|
68
|
+
# mark the absorbing bucket's label (mirrors QualitativeFeature._format_modalities).
|
|
69
|
+
# Only relevant when nan isn't its own leader; labels is positionally aligned with
|
|
70
|
+
# the non-nan leaders of self.values (finite quantiles followed by inf).
|
|
71
|
+
if self.nan not in self.values and self.values.contains(self.nan):
|
|
72
|
+
labels = [
|
|
73
|
+
f"{label}, {self.nan}" if self.nan in self.values.get(leader) else label
|
|
74
|
+
for label, leader in zip(labels, self.values)
|
|
75
|
+
]
|
|
76
|
+
|
|
94
77
|
# converting to grouped list
|
|
95
78
|
labels = GroupedList(labels)
|
|
96
79
|
|
|
@@ -98,9 +81,6 @@ class QuantitativeFeature(BaseFeature):
|
|
|
98
81
|
if self.nan in self.values:
|
|
99
82
|
labels.append(self.nan)
|
|
100
83
|
|
|
101
|
-
# TODO add NaNs if there are any (grouped)
|
|
102
|
-
# elif self.nan in self.values.values:
|
|
103
|
-
|
|
104
84
|
return labels
|
|
105
85
|
|
|
106
86
|
def _make_summary(self):
|
|
@@ -145,6 +125,11 @@ def format_quantiles(a_list: list[float]) -> list[str]:
|
|
|
145
125
|
|
|
146
126
|
# several quantiles
|
|
147
127
|
else:
|
|
128
|
+
# collapse signed zero: an identical boundary can surface as -0.0 or 0.0
|
|
129
|
+
# depending on the quantile computation path (they compare equal but
|
|
130
|
+
# format to different strings), which would make labels run-dependent
|
|
131
|
+
a_list = [number + 0.0 for number in a_list]
|
|
132
|
+
|
|
148
133
|
# getting minimal number of decimals to differentiate labels
|
|
149
134
|
decimals_needed = min_decimals_to_differentiate(a_list, min_decimals=1)
|
|
150
135
|
|
|
@@ -267,9 +267,73 @@ class BaseFeature(ABC):
|
|
|
267
267
|
def _make_summary(self) -> list[dict]:
|
|
268
268
|
"""Returns a summary of the feature."""
|
|
269
269
|
|
|
270
|
-
@abstractmethod
|
|
271
270
|
def _specific_update(self, values: GroupedList, convert_labels: bool = False) -> None:
|
|
272
|
-
"""Update content of values
|
|
271
|
+
"""Update content of values, converting labels back to values if needed."""
|
|
272
|
+
|
|
273
|
+
# no values have been set
|
|
274
|
+
if not convert_labels and self.values.is_empty():
|
|
275
|
+
self._check_empty_values(values)
|
|
276
|
+
self.values = values
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
# values are not labels
|
|
280
|
+
if not convert_labels:
|
|
281
|
+
# updating: iterating over each grouped values
|
|
282
|
+
for kept_value, grouped_values in values.content.items():
|
|
283
|
+
self.values.group(grouped_values, kept_value)
|
|
284
|
+
return
|
|
285
|
+
|
|
286
|
+
# values are labels -> converting them back to values
|
|
287
|
+
# Snapshot the encoded-label -> leader-value map once. self.values.group()
|
|
288
|
+
# below mutates leader positions, so rebuilding this inside the loop would
|
|
289
|
+
# desync self.values[k] (IndexError on the 2nd+ grouped label).
|
|
290
|
+
r_value_per_label = self._reverse_value_per_label()
|
|
291
|
+
|
|
292
|
+
# iterating over each grouped values
|
|
293
|
+
for kept_label, grouped_labels in values.content.items():
|
|
294
|
+
self._update_grouped_label(kept_label, grouped_labels, r_value_per_label)
|
|
295
|
+
|
|
296
|
+
def _reverse_value_per_label(self) -> dict:
|
|
297
|
+
"""Maps each encoded label back to its current leader value (ordinal only)."""
|
|
298
|
+
if self.ordinal_encoding:
|
|
299
|
+
leaders = list(self.values)
|
|
300
|
+
return {v: leaders[k] for k, v in self.value_per_label.items()}
|
|
301
|
+
return {}
|
|
302
|
+
|
|
303
|
+
def _update_grouped_label(self, kept_label: str | float, grouped_labels: list, r_value_per_label: dict) -> None:
|
|
304
|
+
"""Converts one labelled group back to values and groups them."""
|
|
305
|
+
|
|
306
|
+
# checking that kept value exists
|
|
307
|
+
if kept_label not in self.value_per_label:
|
|
308
|
+
raise AttributeError(f"{self} no {kept_label}, in value_per_label: {self.value_per_label}")
|
|
309
|
+
|
|
310
|
+
# converting labels to values
|
|
311
|
+
grouped_values = [self.value_per_label.get(label) for label in grouped_labels]
|
|
312
|
+
|
|
313
|
+
# checking that grouped values exists
|
|
314
|
+
for grouped_value, grouped_label in zip(grouped_values, grouped_labels):
|
|
315
|
+
if grouped_value is None:
|
|
316
|
+
print(f"{self} no {grouped_label}, in value_per_label: {self.value_per_label}")
|
|
317
|
+
|
|
318
|
+
# feature-specific: choosing kept value and finalizing grouped values
|
|
319
|
+
grouped_values, kept_value = self._resolve_grouping(kept_label, grouped_values, r_value_per_label)
|
|
320
|
+
|
|
321
|
+
# updating values if any to group
|
|
322
|
+
if len(grouped_values) > 0:
|
|
323
|
+
self.values.group(grouped_values, kept_value)
|
|
324
|
+
|
|
325
|
+
# updating statistics
|
|
326
|
+
self._update_statistics_value(kept_label, kept_value)
|
|
327
|
+
|
|
328
|
+
def _check_empty_values(self, values: GroupedList) -> None:
|
|
329
|
+
"""Optional hook: validates values before the first assignment (no-op by default)."""
|
|
330
|
+
return
|
|
331
|
+
|
|
332
|
+
@abstractmethod
|
|
333
|
+
def _resolve_grouping(
|
|
334
|
+
self, kept_label: str | float, grouped_values: list, r_value_per_label: dict
|
|
335
|
+
) -> tuple[list, str | float]:
|
|
336
|
+
"""Selects the kept value and finalizes grouped values specifically per feature type."""
|
|
273
337
|
|
|
274
338
|
# ------------------------------------------------------------------
|
|
275
339
|
# summary
|
|
@@ -293,8 +357,7 @@ class BaseFeature(ABC):
|
|
|
293
357
|
if len(history) > 0:
|
|
294
358
|
selected: dict = {}
|
|
295
359
|
|
|
296
|
-
|
|
297
|
-
viable = history["viable"].fillna(False).astype(bool)
|
|
360
|
+
viable = history["viable"].fillna(False).astype(bool)
|
|
298
361
|
|
|
299
362
|
if viable.any():
|
|
300
363
|
dropna = history["dropna"].fillna(False)
|
|
@@ -69,13 +69,16 @@ def kruskal_h(block: pd.DataFrame, groups: pd.Series) -> tuple[np.ndarray, np.nd
|
|
|
69
69
|
n_obs = mask.sum(0).astype(float) # (P,)
|
|
70
70
|
n_groups = (n > 0).sum(0) # (P,)
|
|
71
71
|
|
|
72
|
+
tie_factors = _tie_factors(block)
|
|
72
73
|
with np.errstate(divide="ignore", invalid="ignore"):
|
|
73
74
|
term = np.nansum(np.where(n > 0, rank_sums**2 / n, np.nan), axis=0) # (P,)
|
|
74
75
|
h = 12.0 / (n_obs * (n_obs + 1)) * term - 3.0 * (n_obs + 1)
|
|
75
|
-
h = h /
|
|
76
|
+
h = h / tie_factors
|
|
76
77
|
|
|
77
|
-
# undefined where scipy.kruskal would raise / has_values would bail
|
|
78
|
-
|
|
78
|
+
# undefined where scipy.kruskal would raise / has_values would bail; a
|
|
79
|
+
# constant column has tie_factor 0 (division above -> inf), which scipy
|
|
80
|
+
# rejects ("All numbers are identical") -> treat as undefined too
|
|
81
|
+
h = np.where((n_obs > 1) & (n_groups > 1) & (tie_factors > 0), h, np.nan)
|
|
79
82
|
return h, n_obs, n_groups.astype(float)
|
|
80
83
|
|
|
81
84
|
|
|
@@ -115,12 +118,15 @@ def kruskal_h_reversed(block: pd.DataFrame, y: pd.Series) -> tuple[np.ndarray, n
|
|
|
115
118
|
continue
|
|
116
119
|
term = (rank_sums[populated] ** 2 / counts[populated]).sum()
|
|
117
120
|
h_j = 12.0 / (n * (n + 1)) * term - 3.0 * (n + 1)
|
|
118
|
-
# tie correction over the pooled y ranks
|
|
121
|
+
# tie correction over the pooled y ranks; an all-tied y gives tie_factor
|
|
122
|
+
# 0 (scipy.stats.kruskal rejects it: "All numbers are identical") -> leave
|
|
123
|
+
# h[j] undefined instead of dividing to +inf
|
|
119
124
|
_, tcounts = np.unique(ranks, return_counts=True)
|
|
120
125
|
ties = float((tcounts**3 - tcounts).sum())
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
126
|
+
tie_factor = 1.0 - ties / (n**3 - n)
|
|
127
|
+
if tie_factor == 0:
|
|
128
|
+
continue
|
|
129
|
+
h[j] = h_j / tie_factor
|
|
124
130
|
return h, n_obs, n_groups
|
|
125
131
|
|
|
126
132
|
|
|
@@ -152,7 +158,16 @@ def chi2_all(block: pd.DataFrame, y: pd.Series) -> tuple[np.ndarray, np.ndarray,
|
|
|
152
158
|
row = table.sum(1, keepdims=True)
|
|
153
159
|
col_sum = table.sum(0, keepdims=True)
|
|
154
160
|
expected = row @ col_sum / n
|
|
161
|
+
|
|
162
|
+
# Yates' continuity correction on 2x2 tables (dof == 1), matching the
|
|
163
|
+
# scalar reference (scipy.stats.chi2_contingency, correction=True by
|
|
164
|
+
# default) and the closed-form in binary_combination_evaluators.
|
|
165
|
+
obs = table
|
|
166
|
+
if m == 2 and n_y == 2:
|
|
167
|
+
diff = expected - obs
|
|
168
|
+
obs = obs + np.minimum(0.5, np.abs(diff)) * np.sign(diff)
|
|
169
|
+
|
|
155
170
|
with np.errstate(divide="ignore", invalid="ignore"):
|
|
156
|
-
contrib = np.where(expected > 0, (
|
|
171
|
+
contrib = np.where(expected > 0, (obs - expected) ** 2 / expected, 0.0)
|
|
157
172
|
chi2[j] = contrib.sum()
|
|
158
173
|
return chi2, n_obs, n_mod_x, np.full(p, float(n_y))
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
4
|
|
|
5
|
+
import numpy as np
|
|
5
6
|
import pandas as pd
|
|
6
7
|
|
|
7
8
|
from AutoCarver.features import BaseFeature
|
|
@@ -279,6 +280,10 @@ class ModeMeasure(OutlierMeasure):
|
|
|
279
280
|
Measure of ``x``'s mode
|
|
280
281
|
"""
|
|
281
282
|
_ = y
|
|
282
|
-
|
|
283
|
+
modes = x.mode(dropna=True) # computing mode
|
|
284
|
+
if len(modes) == 0: # all-NaN feature: no defined non-NaN mode
|
|
285
|
+
self.value = np.nan
|
|
286
|
+
return self.value
|
|
287
|
+
mode = modes.values[0]
|
|
283
288
|
self.value = (x == mode).mean() # Computing percentage of the mode
|
|
284
289
|
return self.value
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from math import sqrt
|
|
4
4
|
|
|
5
|
+
import numpy as np
|
|
5
6
|
import pandas as pd
|
|
6
7
|
from scipy.stats import chi2_contingency
|
|
7
8
|
|
|
@@ -40,7 +41,10 @@ class Chi2Measure(BaseMeasure):
|
|
|
40
41
|
xtab = pd.crosstab(x, y)
|
|
41
42
|
|
|
42
43
|
# computing Chi2 statistic
|
|
43
|
-
|
|
44
|
+
try:
|
|
45
|
+
self.value = chi2_contingency(xtab)[0]
|
|
46
|
+
except ValueError:
|
|
47
|
+
self.value = np.nan
|
|
44
48
|
return self.value
|
|
45
49
|
|
|
46
50
|
def compute_all(self, X, y, features) -> dict[str, dict]:
|
|
@@ -109,14 +113,21 @@ class TschuprowtMeasure(Chi2Measure):
|
|
|
109
113
|
n_mod_x, n_mod_y = x.nunique(), y.nunique()
|
|
110
114
|
|
|
111
115
|
# computing Tschuprow's T
|
|
112
|
-
|
|
116
|
+
dof_prod = (n_mod_x - 1) * (n_mod_y - 1)
|
|
117
|
+
if dof_prod < 0: # no data for x or y: Tschuprow's T is undefined
|
|
118
|
+
self.value = np.nan
|
|
119
|
+
return self.value
|
|
120
|
+
dof_mods = sqrt(dof_prod)
|
|
113
121
|
self.value = 0
|
|
114
122
|
if dof_mods > 0:
|
|
115
123
|
self.value = sqrt(chi2_value / n_obs / dof_mods)
|
|
116
124
|
return self.value
|
|
117
125
|
|
|
118
126
|
def _stat(self, chi2: float, n_obs: float, n_mod_x: float, n_mod_y: float) -> float:
|
|
119
|
-
|
|
127
|
+
dof_prod = (n_mod_x - 1) * (n_mod_y - 1)
|
|
128
|
+
if dof_prod < 0: # no data for x or y: Tschuprow's T is undefined
|
|
129
|
+
return np.nan
|
|
130
|
+
dof_mods = sqrt(dof_prod)
|
|
120
131
|
if dof_mods > 0:
|
|
121
132
|
return sqrt(chi2 / n_obs / dof_mods)
|
|
122
133
|
return 0.0
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/selectors/measures/quantitative_measures.py
RENAMED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Measures of association between a Quantitative feature and binary target."""
|
|
2
2
|
|
|
3
|
+
import warnings
|
|
3
4
|
from math import sqrt
|
|
4
5
|
|
|
5
6
|
import numpy as np
|
|
@@ -181,6 +182,12 @@ class RMeasure(BaseMeasure):
|
|
|
181
182
|
if y[~nans].nunique() != 2:
|
|
182
183
|
raise ValueError(f"[{self}] Provided y is not binary")
|
|
183
184
|
|
|
185
|
+
# a constant feature has no explained variance (centered_tss == 0): R is
|
|
186
|
+
# undefined, return NaN before fitting to avoid a divide-by-zero in statsmodels
|
|
187
|
+
if x[~nans].nunique() <= 1:
|
|
188
|
+
self.value = np.nan
|
|
189
|
+
return self.value
|
|
190
|
+
|
|
184
191
|
# grouping feature and target
|
|
185
192
|
ols_df = pd.DataFrame({"feature": x[~nans], "target": y[~nans]})
|
|
186
193
|
|
|
@@ -212,7 +219,12 @@ class CorrelationMeasure(AbsoluteMeasure):
|
|
|
212
219
|
def compute_all(self, X, y, features) -> dict[str, dict]:
|
|
213
220
|
"""Vectorized correlation of every feature with ``y`` in one ``corrwith`` call."""
|
|
214
221
|
block = X[[feature.version for feature in features]]
|
|
215
|
-
|
|
222
|
+
# degenerate columns (constant, or <=1 non-NaN overlap) legitimately yield
|
|
223
|
+
# NaN; silence the numpy/scipy RuntimeWarnings (divide, df<=0, ConstantInput)
|
|
224
|
+
# rather than surface them to end users
|
|
225
|
+
with np.errstate(divide="ignore", invalid="ignore"), warnings.catch_warnings():
|
|
226
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
227
|
+
corr = block.corrwith(y, method=self._corr_method)
|
|
216
228
|
return {feature.version: self._result(corr[feature.version]) for feature in features}
|
|
217
229
|
|
|
218
230
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: AutoCarver
|
|
3
|
-
Version: 7.3.
|
|
3
|
+
Version: 7.3.2
|
|
4
4
|
Summary: Automatic Discretization of Features with Optimal Target Association
|
|
5
5
|
Keywords: binning,discretization,feature-engineering,credit-scoring,woe,scorecards,tschuprow,tschuprow-t,cramer-v,information-value,iv,monotonic-binning,supervised-discretization,bucketization,risk-modeling,credit-risk,fraud-detection,categorical-encoding
|
|
6
6
|
Author: Mario DEFRANCE
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "AutoCarver"
|
|
3
|
-
version = "7.3.
|
|
3
|
+
version = "7.3.2"
|
|
4
4
|
description = "Automatic Discretization of Features with Optimal Target Association"
|
|
5
5
|
authors = [{ name = "Mario DEFRANCE", email = "defrancemario@gmail.com" }]
|
|
6
6
|
requires-python = ">=3.11, <3.15"
|
|
@@ -75,6 +75,7 @@ dev = [
|
|
|
75
75
|
"ruff>=0.15.12",
|
|
76
76
|
"pytest-xdist>=3.8.0",
|
|
77
77
|
"ty>=0.0.38",
|
|
78
|
+
"hypothesis>=6.0,<7",
|
|
78
79
|
]
|
|
79
80
|
|
|
80
81
|
[project.urls]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/combinations/utils/combination_evaluator.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/nested_discretizer.py
RENAMED
|
File without changes
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/discretizers/qualitatives/ordinal_discretizer.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/qualitatives/categorical_feature.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.0 → autocarver-7.3.2}/AutoCarver/features/quantitatives/numerical_feature.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|