AutoCarver 7.3.3__tar.gz → 7.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/__init__.py +4 -1
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/__init__.py +2 -1
- autocarver-7.3.4/AutoCarver/carvers/ordinal_carver.py +101 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/utils/base_carver.py +2 -1
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/__init__.py +10 -0
- autocarver-7.3.4/AutoCarver/combinations/ordinal/__init__.py +15 -0
- autocarver-7.3.4/AutoCarver/combinations/ordinal/ordinal_combination_evaluators.py +424 -0
- autocarver-7.3.4/AutoCarver/combinations/ordinal/ordinal_target_rates.py +73 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/utils/combination_evaluator.py +1 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/utils/combinations.py +25 -1
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/mcp/session.py +11 -3
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/__init__.py +2 -0
- autocarver-7.3.4/AutoCarver/selectors/ordinal_selector.py +23 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/PKG-INFO +1 -1
- {autocarver-7.3.3 → autocarver-7.3.4}/pyproject.toml +1 -1
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/binary_carver.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/continuous_carver.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/multiclass_carver.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/carvers/utils/pretty_print.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/binary/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/binary/binary_combination_evaluators.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/binary/binary_target_rates.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/continuous/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/continuous/continuous_combination_evaluators.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/continuous/continuous_target_rates.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/utils/target_rate.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/utils/testing.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/config.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/categorical_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/nested_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/ordinal_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/qualitative_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/quantitatives/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/quantitatives/continuous_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/quantitatives/quantitative_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/utils/base_discretizer.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/utils/frequency_ci.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/utils/multiprocessing.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/utils/type_discretizers.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/features.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/llm_qualifier.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/categorical_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/nested_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/ordinal_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/qualitative_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/datetime_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/numerical_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/quantitative_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/utils/base_feature.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/utils/grouped_list.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/utils/serialization.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/mcp/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/mcp/__main__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/mcp/inspection.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/mcp/server.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/classification_selector.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/filters/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/filters/base_filters.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/filters/qualitative_filters.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/filters/quantitative_filters.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/_vectorized.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/base_measures.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/qualitative_measures.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/quantitative_measures.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/regression_selector.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/utils/base_selector.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/utils/pretty_print.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/utils/__init__.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/utils/attributes.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/utils/dependencies.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/utils/extend_docstring.py +0 -0
- {autocarver-7.3.3 → autocarver-7.3.4}/README.md +0 -0
|
@@ -3,14 +3,17 @@
|
|
|
3
3
|
from AutoCarver.carvers.binary_carver import BinaryCarver
|
|
4
4
|
from AutoCarver.carvers.continuous_carver import ContinuousCarver
|
|
5
5
|
from AutoCarver.carvers.multiclass_carver import MulticlassCarver
|
|
6
|
+
from AutoCarver.carvers.ordinal_carver import OrdinalCarver
|
|
6
7
|
from AutoCarver.features import Features
|
|
7
|
-
from AutoCarver.selectors import ClassificationSelector, RegressionSelector
|
|
8
|
+
from AutoCarver.selectors import ClassificationSelector, OrdinalSelector, RegressionSelector
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"BinaryCarver",
|
|
11
12
|
"ContinuousCarver",
|
|
12
13
|
"Features",
|
|
13
14
|
"MulticlassCarver",
|
|
15
|
+
"OrdinalCarver",
|
|
14
16
|
"ClassificationSelector",
|
|
17
|
+
"OrdinalSelector",
|
|
15
18
|
"RegressionSelector",
|
|
16
19
|
]
|
|
@@ -3,5 +3,6 @@
|
|
|
3
3
|
from AutoCarver.carvers.binary_carver import BinaryCarver
|
|
4
4
|
from AutoCarver.carvers.continuous_carver import ContinuousCarver
|
|
5
5
|
from AutoCarver.carvers.multiclass_carver import MulticlassCarver
|
|
6
|
+
from AutoCarver.carvers.ordinal_carver import OrdinalCarver
|
|
6
7
|
|
|
7
|
-
__all__ = ["BinaryCarver", "ContinuousCarver", "MulticlassCarver"]
|
|
8
|
+
__all__ = ["BinaryCarver", "ContinuousCarver", "MulticlassCarver", "OrdinalCarver"]
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Tool to build optimized buckets out of Quantitative and Qualitative features
|
|
2
|
+
for ordinal targets (ordered, integer-encoded modalities).
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from AutoCarver.carvers.binary_carver import get_crosstab
|
|
9
|
+
from AutoCarver.carvers.utils.base_carver import BaseCarver, Samples
|
|
10
|
+
from AutoCarver.combinations import CombinationEvaluator, KendallTauCCombinations
|
|
11
|
+
from AutoCarver.discretizers.utils.base_discretizer import ProcessingConfig
|
|
12
|
+
from AutoCarver.features import Features
|
|
13
|
+
from AutoCarver.utils import extend_docstring
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class OrdinalCarver(BaseCarver):
|
|
17
|
+
"""Automatic carving of continuous, discrete, categorical and ordinal
|
|
18
|
+
features that maximizes association with an **ordinal** target.
|
|
19
|
+
|
|
20
|
+
The target must be **integer-encoded** with ordered levels (e.g. ``1..K``,
|
|
21
|
+
``K > 2``); the level order is taken from the ascending integer values.
|
|
22
|
+
|
|
23
|
+
For ordinal targets, Kendall's :ref:`tau_c` is the default association
|
|
24
|
+
measure to sort combinations — it rewards groupings whose order matches the
|
|
25
|
+
target's while favouring robust, parsimonious cardinality. :ref:`tau_b` and
|
|
26
|
+
the original Somers' D (:ref:`somersd`) are also available via
|
|
27
|
+
``combination_evaluator``.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
__name__ = "OrdinalCarver"
|
|
31
|
+
is_y_ordinal = True
|
|
32
|
+
|
|
33
|
+
@extend_docstring(BaseCarver.__init__, exclude=["combination_evaluator"])
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
features: Features,
|
|
37
|
+
min_freq: float,
|
|
38
|
+
max_n_mod: int,
|
|
39
|
+
*,
|
|
40
|
+
combination_evaluator: CombinationEvaluator | None = None,
|
|
41
|
+
config: ProcessingConfig | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
combination_evaluator : CombinationEvaluator, optional
|
|
47
|
+
Pre-built evaluator instance measuring association between
|
|
48
|
+
:class:`Features` and an ordinal target. Defaults to
|
|
49
|
+
:class:`KendallTauCCombinations`.
|
|
50
|
+
|
|
51
|
+
Choose from: :class:`KendallTauCCombinations` (default),
|
|
52
|
+
:class:`KendallTauBCombinations`, :class:`SomersDCombinations`.
|
|
53
|
+
"""
|
|
54
|
+
if combination_evaluator is None:
|
|
55
|
+
combination_evaluator = KendallTauCCombinations()
|
|
56
|
+
if not combination_evaluator.is_y_ordinal:
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"[{self.__name__}] {type(combination_evaluator).__name__} is not suited for ordinal targets. "
|
|
59
|
+
f"Choose from: KendallTauCCombinations, KendallTauBCombinations, SomersDCombinations."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
super().__init__(
|
|
63
|
+
features=features,
|
|
64
|
+
min_freq=min_freq,
|
|
65
|
+
max_n_mod=max_n_mod,
|
|
66
|
+
combination_evaluator=combination_evaluator,
|
|
67
|
+
config=config,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def _prepare_samples(self, samples: Samples) -> Samples:
|
|
71
|
+
"""Validates format and content of X and y."""
|
|
72
|
+
if samples.train.y is None:
|
|
73
|
+
raise ValueError(f"[{self.__name__}] y must be provided")
|
|
74
|
+
if not pd.api.types.is_numeric_dtype(samples.train.y):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"[{self.__name__}] y must be an integer-encoded ordinal Series; "
|
|
77
|
+
"integer-encode your ordered target (e.g. 1..K) before carving."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
y_values = np.unique(samples.train.y)
|
|
81
|
+
if len(y_values) <= 2:
|
|
82
|
+
raise ValueError(f"[{self.__name__}] provided y has <=2 levels, consider using BinaryCarver instead.")
|
|
83
|
+
if not np.all(np.equal(np.mod(y_values, 1), 0)):
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"[{self.__name__}] y must be integer-encoded ordered levels (e.g. 1..K); got non-integer values."
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return super()._prepare_samples(samples)
|
|
89
|
+
|
|
90
|
+
def _aggregator(self, X: pd.DataFrame, y: pd.Series) -> dict[str, pd.Series | pd.DataFrame | None]:
|
|
91
|
+
"""Computes ordered contingency tables (feature modalities × ordinal target
|
|
92
|
+
levels) for specified features, ordered according to the known labels."""
|
|
93
|
+
# checking for empty datasets (dev)
|
|
94
|
+
xtabs = {feature.version: None for feature in self.features}
|
|
95
|
+
if X is not None:
|
|
96
|
+
# crosstab for each feature — pd.crosstab emits one column per ordinal
|
|
97
|
+
# level and sorts them ascending (correct ordinal column order)
|
|
98
|
+
for feature in self.features:
|
|
99
|
+
xtabs.update({feature.version: get_crosstab(X, y, feature)})
|
|
100
|
+
|
|
101
|
+
return xtabs
|
|
@@ -90,7 +90,7 @@ def _drop_reason_from_history(history: pd.DataFrame) -> str:
|
|
|
90
90
|
of historized non-viable combinations.
|
|
91
91
|
"""
|
|
92
92
|
if history.empty:
|
|
93
|
-
return "No combination
|
|
93
|
+
return "No combination possible"
|
|
94
94
|
|
|
95
95
|
info_counts: dict[str, int] = {}
|
|
96
96
|
for _, row in history.iterrows():
|
|
@@ -121,6 +121,7 @@ class BaseCarver(BaseDiscretizer, ABC):
|
|
|
121
121
|
is_y_binary = False
|
|
122
122
|
is_y_continuous = False
|
|
123
123
|
is_y_multiclass = False
|
|
124
|
+
is_y_ordinal = False
|
|
124
125
|
|
|
125
126
|
@extend_docstring(BaseDiscretizer.__init__, exclude=["min_freq", "config"])
|
|
126
127
|
def __init__(
|
|
@@ -9,6 +9,12 @@ from AutoCarver.combinations.continuous.continuous_combination_evaluators import
|
|
|
9
9
|
ContinuousCombinationEvaluator,
|
|
10
10
|
KruskalCombinations,
|
|
11
11
|
)
|
|
12
|
+
from AutoCarver.combinations.ordinal.ordinal_combination_evaluators import (
|
|
13
|
+
KendallTauBCombinations,
|
|
14
|
+
KendallTauCCombinations,
|
|
15
|
+
OrdinalCombinationEvaluator,
|
|
16
|
+
SomersDCombinations,
|
|
17
|
+
)
|
|
12
18
|
from AutoCarver.combinations.utils.combination_evaluator import CombinationEvaluator
|
|
13
19
|
|
|
14
20
|
__all__ = [
|
|
@@ -18,4 +24,8 @@ __all__ = [
|
|
|
18
24
|
"CramervCombinations",
|
|
19
25
|
"ContinuousCombinationEvaluator",
|
|
20
26
|
"KruskalCombinations",
|
|
27
|
+
"OrdinalCombinationEvaluator",
|
|
28
|
+
"KendallTauCCombinations",
|
|
29
|
+
"KendallTauBCombinations",
|
|
30
|
+
"SomersDCombinations",
|
|
21
31
|
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""This module contains the ordinal combinations module."""
|
|
2
|
+
|
|
3
|
+
from AutoCarver.combinations.ordinal.ordinal_combination_evaluators import (
|
|
4
|
+
KendallTauBCombinations,
|
|
5
|
+
KendallTauCCombinations,
|
|
6
|
+
OrdinalCombinationEvaluator,
|
|
7
|
+
SomersDCombinations,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"OrdinalCombinationEvaluator",
|
|
12
|
+
"KendallTauCCombinations",
|
|
13
|
+
"KendallTauBCombinations",
|
|
14
|
+
"SomersDCombinations",
|
|
15
|
+
]
|
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
"""Module for ordinal combination evaluators."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from abc import ABC
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from AutoCarver.combinations.ordinal.ordinal_target_rates import OrdinalTargetRate, TargetMeanLevel
|
|
10
|
+
from AutoCarver.combinations.utils.combination_evaluator import AggregatedSample, CombinationEvaluator
|
|
11
|
+
from AutoCarver.combinations.utils.combinations import combination_formatter, group_crosstab
|
|
12
|
+
from AutoCarver.combinations.utils.target_rate import TargetRate
|
|
13
|
+
from AutoCarver.features import GroupedList
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class OrdinalCombinationEvaluator(CombinationEvaluator[pd.DataFrame], ABC):
|
|
17
|
+
"""Ordinal combination evaluator class.
|
|
18
|
+
|
|
19
|
+
The aggregation is an ordered contingency table
|
|
20
|
+
``feature-groups (rows, in target-rate order) × ordinal-target-levels
|
|
21
|
+
(cols, ascending)`` — the binary crosstab generalised from 2 columns to as
|
|
22
|
+
many columns as the target has levels. Three rank-association statistics are
|
|
23
|
+
computed per combination; concrete subclasses pick which one ranks
|
|
24
|
+
combinations via :attr:`sort_by`:
|
|
25
|
+
|
|
26
|
+
* :ref:`tau_c` — Kendall/Stuart's tau-c (**default**, rectangular-table
|
|
27
|
+
correction; self-balances to a robust, meaningful number of modalities);
|
|
28
|
+
* :ref:`tau_b` — Kendall's tau-b (matches :func:`scipy.stats.kendalltau`);
|
|
29
|
+
* :ref:`somersd` — the original asymmetric Somers' D ``D(Y|X)`` (target given
|
|
30
|
+
feature).
|
|
31
|
+
|
|
32
|
+
The symmetric Kendall taus reward a split only when it is genuinely
|
|
33
|
+
discriminative and otherwise favour fewer, more robust modalities — like
|
|
34
|
+
:class:`TschuprowtCombinations` and the Kruskal effect sizes. Somers' D is
|
|
35
|
+
asymmetric and leans strongly toward the coarsest split.
|
|
36
|
+
|
|
37
|
+
Search uses the inherited enumerate-and-score path.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
is_y_ordinal = True
|
|
41
|
+
_target_rate_classes: list[type[OrdinalTargetRate]] = [TargetMeanLevel]
|
|
42
|
+
# narrow inherited attribute: ordinal evaluators always carry an OrdinalTargetRate
|
|
43
|
+
# (enforced by _init_target_rate).
|
|
44
|
+
target_rate: OrdinalTargetRate
|
|
45
|
+
# narrow inherited `sort_by: str | None`: concrete subclasses always set a str.
|
|
46
|
+
sort_by: str
|
|
47
|
+
|
|
48
|
+
def _init_target_rate(self, target_rate: TargetRate[pd.DataFrame] | None) -> OrdinalTargetRate:
|
|
49
|
+
"""Initializes target rate."""
|
|
50
|
+
if target_rate is None:
|
|
51
|
+
return TargetMeanLevel()
|
|
52
|
+
if not isinstance(target_rate, OrdinalTargetRate):
|
|
53
|
+
raise ValueError("target_rate must be an OrdinalTargetRate")
|
|
54
|
+
return target_rate
|
|
55
|
+
|
|
56
|
+
def _grouper(self, xagg: AggregatedSample, groupby: dict) -> pd.DataFrame:
|
|
57
|
+
"""Groups a crosstab by ``groupby`` and sums column values by group.
|
|
58
|
+
|
|
59
|
+
Shares :func:`group_crosstab` with the binary path: leaders are ordered
|
|
60
|
+
by first appearance so grouping stays independent of label text.
|
|
61
|
+
"""
|
|
62
|
+
return group_crosstab(xagg, groupby)
|
|
63
|
+
|
|
64
|
+
def _association_measure(
|
|
65
|
+
self,
|
|
66
|
+
xagg: AggregatedSample | pd.Series | pd.DataFrame,
|
|
67
|
+
n_obs: int | None = None,
|
|
68
|
+
tol: float = 1e-10,
|
|
69
|
+
) -> dict[str, float | None]:
|
|
70
|
+
"""Computes Kendall's tau-b, tau-c and Somers' D between feature and target.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
xagg : pd.DataFrame
|
|
75
|
+
Ordered contingency table (rows = feature groups, cols = ordinal
|
|
76
|
+
target levels). ``n_obs`` / ``tol`` are unused (the rank statistics
|
|
77
|
+
only depend on the table's cell counts).
|
|
78
|
+
|
|
79
|
+
Returns
|
|
80
|
+
-------
|
|
81
|
+
dict[str, float | None]
|
|
82
|
+
``{"tau_b": ..., "tau_c": ..., "somersd": ...}``; any may be
|
|
83
|
+
``None`` for a degenerate table.
|
|
84
|
+
"""
|
|
85
|
+
_, _ = n_obs, tol # unused
|
|
86
|
+
return _ordinal_associations(np.asarray(xagg.values, dtype=float))
|
|
87
|
+
|
|
88
|
+
def _get_best_combination_non_nan(self) -> dict | None:
|
|
89
|
+
"""DP-based override with progressive top-K (mirrors the continuous path).
|
|
90
|
+
|
|
91
|
+
Replaces ``consecutive_combinations`` + enumerate-and-score with the
|
|
92
|
+
interval-DP in :func:`_top_k_partitions_ordinal_dp` over the additively
|
|
93
|
+
decomposable ``C−D`` numerator. Exact for tau-c (per-k constant
|
|
94
|
+
denominator); a progressively-grown top-K approximation for tau-b /
|
|
95
|
+
Somers' D, whose denominators depend on the group sizes.
|
|
96
|
+
|
|
97
|
+
The NaN path (:meth:`_get_best_combination_with_nan`) is **not** overridden:
|
|
98
|
+
it runs after this method has applied the best non-NaN grouping, so it
|
|
99
|
+
enumerates over the already-small grouped label set and the inherited
|
|
100
|
+
enumerate-and-score path is cheap there.
|
|
101
|
+
"""
|
|
102
|
+
feature_labels = self.feature.labels
|
|
103
|
+
if feature_labels is None:
|
|
104
|
+
raise RuntimeError(f"[{self.__name__}] feature labels are not populated")
|
|
105
|
+
raw_labels = GroupedList(feature_labels[:])
|
|
106
|
+
|
|
107
|
+
if self.feature.has_nan:
|
|
108
|
+
if self.feature.dropna:
|
|
109
|
+
raw_labels.remove(self.feature.nan)
|
|
110
|
+
self.samples.dropna(self.feature.nan)
|
|
111
|
+
|
|
112
|
+
if self.samples.train.shape[0] <= 1:
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
self._historize_raw_combination()
|
|
116
|
+
|
|
117
|
+
raw_index = list(raw_labels)
|
|
118
|
+
# samples.train.xagg is a crosstab DataFrame for ordinal evaluators
|
|
119
|
+
M, n_per_mod, col_sums = _dp_inputs_from_xagg(self.samples.train.xagg, raw_index) # type: ignore
|
|
120
|
+
|
|
121
|
+
# Progressive top-K with doubling, mirroring the binary/continuous DPs.
|
|
122
|
+
top_k = self.dp_top_k_initial
|
|
123
|
+
walked = 0
|
|
124
|
+
viable: dict | None = None
|
|
125
|
+
while True:
|
|
126
|
+
associations = _top_k_partitions_ordinal_dp(
|
|
127
|
+
M,
|
|
128
|
+
n_per_mod,
|
|
129
|
+
col_sums,
|
|
130
|
+
max_n_mod=self.max_n_mod,
|
|
131
|
+
raw_index=raw_index,
|
|
132
|
+
sort_by=self.sort_by,
|
|
133
|
+
top_k=top_k,
|
|
134
|
+
)
|
|
135
|
+
viable, walked = self._walk_for_viable(associations, start=walked)
|
|
136
|
+
if viable is not None:
|
|
137
|
+
break
|
|
138
|
+
if walked < top_k:
|
|
139
|
+
break # DP exhausted every consecutive partition; no viable exists
|
|
140
|
+
top_k *= 2
|
|
141
|
+
|
|
142
|
+
if viable is not None and viable.get("xagg") is None:
|
|
143
|
+
index_to_groupby = viable.get("index_to_groupby") or combination_formatter(viable["combination"])
|
|
144
|
+
viable["xagg"] = self._grouper(self.samples.train, index_to_groupby)
|
|
145
|
+
|
|
146
|
+
self._apply_best_combination(viable)
|
|
147
|
+
return viable
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class KendallTauCCombinations(OrdinalCombinationEvaluator):
|
|
151
|
+
"""Kendall's tau-c based combination evaluation toolkit (ordinal default).
|
|
152
|
+
|
|
153
|
+
Stuart's tau-c applies a ``min(r, c)`` correction tailored to **rectangular**
|
|
154
|
+
tables — exactly our shape (few feature groups × many target levels) — so its
|
|
155
|
+
magnitude stays comparable across combinations with different group counts and
|
|
156
|
+
it leans toward fewer, robust modalities, only adding one when a split is
|
|
157
|
+
genuinely meaningful.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
sort_by = "tau_c"
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class KendallTauBCombinations(OrdinalCombinationEvaluator):
|
|
164
|
+
"""Kendall's tau-b based combination evaluation toolkit.
|
|
165
|
+
|
|
166
|
+
Bit-exact with :func:`scipy.stats.kendalltau` (the ``tau-b`` variant) on the
|
|
167
|
+
grouped contingency table — pinned by parity tests. Normalised by the
|
|
168
|
+
geometric mean of both margins' untied pairs; tends to retain more modalities
|
|
169
|
+
on smoothly monotone signals than :class:`KendallTauCCombinations`.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
sort_by = "tau_b"
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class SomersDCombinations(OrdinalCombinationEvaluator):
|
|
176
|
+
"""Somers' D based combination evaluation toolkit.
|
|
177
|
+
|
|
178
|
+
The original asymmetric Somers' D ``D(Y|X)`` — concordant minus discordant
|
|
179
|
+
pairs over pairs untied on the feature ``X`` — matching
|
|
180
|
+
``scipy.stats.somersd(table).statistic``. Being asymmetric it leans strongly
|
|
181
|
+
toward the coarsest split (its maximum over groupings is typically two
|
|
182
|
+
modalities); offered for users who specifically want raw Somers' D rather
|
|
183
|
+
than the self-balancing Kendall taus.
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
sort_by = "somersd"
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _concordant_minus_discordant(values: np.ndarray) -> float:
|
|
190
|
+
"""Concordant minus discordant pairs ``C − D`` of an ordered table.
|
|
191
|
+
|
|
192
|
+
``values`` is ``(r, c)`` with rows / columns already ascending.
|
|
193
|
+
"""
|
|
194
|
+
# concordant partners of each cell: counts strictly down-right (k>i, l>j)
|
|
195
|
+
suffix = np.cumsum(np.cumsum(values[::-1, ::-1], axis=0), axis=1)[::-1, ::-1]
|
|
196
|
+
down_right = np.zeros_like(values)
|
|
197
|
+
down_right[:-1, :-1] = suffix[1:, 1:]
|
|
198
|
+
|
|
199
|
+
# discordant partners of each cell: counts strictly down-left (k>i, l<j)
|
|
200
|
+
suffix_rows_prefix_cols = np.cumsum(np.cumsum(values[::-1, :], axis=0)[::-1, :], axis=1)
|
|
201
|
+
down_left = np.zeros_like(values)
|
|
202
|
+
down_left[:-1, 1:] = suffix_rows_prefix_cols[1:, :-1]
|
|
203
|
+
|
|
204
|
+
return float((values * down_right).sum()) - float((values * down_left).sum())
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _taus_from_counts(
|
|
208
|
+
cd: float, n: float, untied_on_feature: float, untied_on_target: float, m: int
|
|
209
|
+
) -> dict[str, float | None]:
|
|
210
|
+
"""Assembles tau-b, tau-c and Somers' D from pre-computed pair counts.
|
|
211
|
+
|
|
212
|
+
Shared by the closed form (:func:`_ordinal_associations`) and the DP path
|
|
213
|
+
so both produce bit-identical values.
|
|
214
|
+
|
|
215
|
+
* ``tau_b = (C − D) / sqrt((P0 − T_X)(P0 − T_Y))`` — matches
|
|
216
|
+
``scipy.stats.kendalltau``;
|
|
217
|
+
* ``tau_c = 2·m·(C − D) / (n²·(m − 1))`` (Stuart's rectangular-table
|
|
218
|
+
correction);
|
|
219
|
+
* ``somersd = (C − D) / (P0 − T_X)`` — the original Somers' D ``D(Y|X)``.
|
|
220
|
+
|
|
221
|
+
Each measure is ``None`` when its denominator vanishes.
|
|
222
|
+
"""
|
|
223
|
+
denominator_b = math.sqrt(untied_on_feature * untied_on_target)
|
|
224
|
+
return {
|
|
225
|
+
"tau_b": cd / denominator_b if denominator_b > 0 else None,
|
|
226
|
+
"tau_c": (2.0 * m * cd) / (n * n * (m - 1)) if m > 1 else None,
|
|
227
|
+
"somersd": cd / untied_on_feature if untied_on_feature > 0 else None,
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _ordinal_associations(values: np.ndarray) -> dict[str, float | None]:
|
|
232
|
+
"""Kendall's tau-b, tau-c and Somers' D ``D(Y|X)`` for an ordered table.
|
|
233
|
+
|
|
234
|
+
``values`` is the ``(r, c)`` cell-count array with rows = ``X`` (feature
|
|
235
|
+
groups) and columns = ``Y`` (target levels), both already in ascending order.
|
|
236
|
+
Each measure is ``None`` when its denominator vanishes (degenerate table),
|
|
237
|
+
mirroring the continuous evaluator's ``None`` convention.
|
|
238
|
+
"""
|
|
239
|
+
n = float(values.sum())
|
|
240
|
+
if n < 2:
|
|
241
|
+
return {"tau_b": None, "tau_c": None, "somersd": None}
|
|
242
|
+
|
|
243
|
+
cd = _concordant_minus_discordant(values)
|
|
244
|
+
row = values.sum(axis=1)
|
|
245
|
+
col = values.sum(axis=0)
|
|
246
|
+
all_pairs = n * (n - 1) / 2.0
|
|
247
|
+
untied_on_feature = all_pairs - float((row * (row - 1) / 2.0).sum())
|
|
248
|
+
untied_on_target = all_pairs - float((col * (col - 1) / 2.0).sum())
|
|
249
|
+
m = min(int((row > 0).sum()), int((col > 0).sum()))
|
|
250
|
+
return _taus_from_counts(cd, n, untied_on_feature, untied_on_target, m)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ---------------------------------------------------------------------------
|
|
254
|
+
# Phase-B: progressive top-K interval DP over the additive C−D numerator
|
|
255
|
+
# ---------------------------------------------------------------------------
|
|
256
|
+
#
|
|
257
|
+
# C−D of a consecutive grouping decomposes additively:
|
|
258
|
+
#
|
|
259
|
+
# C−D(grouping) = TotalBetween − Σ_g WithinSegment(g)
|
|
260
|
+
#
|
|
261
|
+
# where TotalBetween (the C−D of the fully-split table) is constant and
|
|
262
|
+
# WithinSegment is prefix-summable. So an interval DP that keeps, per number of
|
|
263
|
+
# groups k, the partitions with the largest numerator (smallest Σ WithinSegment)
|
|
264
|
+
# enumerates the best candidates without materialising every consecutive
|
|
265
|
+
# partition. For tau-c the per-k denominator is constant, so numerator-optimal
|
|
266
|
+
# == metric-optimal (the DP is exact). For tau-b / Somers' D the denominator
|
|
267
|
+
# depends on the group sizes (T_X), so the kept top-K candidates are re-scored
|
|
268
|
+
# with their true denominators and ranked — exact when top_k is exhaustive, a
|
|
269
|
+
# top-K approximation otherwise.
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _dp_inputs_from_xagg(raw_xagg: pd.DataFrame, raw_index: list) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
273
|
+
"""Aligns a raw crosstab to ``raw_index`` for the DP.
|
|
274
|
+
|
|
275
|
+
Returns ``(M, n_per_mod, col_sums)`` where ``M`` is the ``(len(raw_index), c)``
|
|
276
|
+
per-modality column-count matrix (rows absent from ``raw_xagg`` are zero),
|
|
277
|
+
``n_per_mod`` the row totals and ``col_sums`` the target marginal.
|
|
278
|
+
"""
|
|
279
|
+
position = {label: i for i, label in enumerate(raw_xagg.index)}
|
|
280
|
+
values = np.asarray(raw_xagg.values, dtype=float)
|
|
281
|
+
M = np.zeros((len(raw_index), values.shape[1]))
|
|
282
|
+
for row, label in enumerate(raw_index):
|
|
283
|
+
source = position.get(label)
|
|
284
|
+
if source is not None:
|
|
285
|
+
M[row] = values[source]
|
|
286
|
+
return M, M.sum(axis=1), M.sum(axis=0)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _segment_within_costs(M: np.ndarray) -> np.ndarray:
|
|
290
|
+
"""WithinSegment ``C−D`` for every consecutive row segment.
|
|
291
|
+
|
|
292
|
+
Returns ``seg`` of shape ``(n_mod, n_mod + 1)`` where ``seg[a, b]`` is the
|
|
293
|
+
concordant−discordant count among observations whose modalities both lie in
|
|
294
|
+
the consecutive block ``[a, b)`` — the within-segment pairs that grouping
|
|
295
|
+
removes from ``C−D``. Computed in ``O(n_mod² · c)`` via the bilinearity of
|
|
296
|
+
the between-modality concordance in the two rows' column vectors.
|
|
297
|
+
"""
|
|
298
|
+
n_mod, c = M.shape
|
|
299
|
+
seg = np.zeros((n_mod, n_mod + 1))
|
|
300
|
+
for a in range(n_mod):
|
|
301
|
+
block = M[a].astype(float).copy()
|
|
302
|
+
within = 0.0
|
|
303
|
+
for b in range(a + 1, n_mod):
|
|
304
|
+
# between(block=[a,b) lower rows, row b higher): bilinear in column vectors
|
|
305
|
+
inclusive = np.cumsum(block)
|
|
306
|
+
strictly_lower = inclusive - block # target mass below each column, within the block
|
|
307
|
+
strictly_higher = block.sum() - inclusive # target mass above each column, within the block
|
|
308
|
+
within += float((M[b] * (strictly_lower - strictly_higher)).sum())
|
|
309
|
+
block = block + M[b]
|
|
310
|
+
seg[a, b + 1] = within
|
|
311
|
+
return seg
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _build_partition_dp(
|
|
315
|
+
seg: np.ndarray, *, n_mod: int, cap: int, top_k: int
|
|
316
|
+
) -> list[list[list[tuple[float, tuple[int, ...]]]]]:
|
|
317
|
+
"""dp[g][j]: up to ``top_k`` ``(sum_seg, splits)`` with the SMALLEST sum_seg
|
|
318
|
+
(largest numerator), where ``splits = (0, s_1, ..., s_{g-1}, j)`` and ``g`` is
|
|
319
|
+
the number of groups.
|
|
320
|
+
"""
|
|
321
|
+
dp: list[list[list[tuple[float, tuple[int, ...]]]]] = [[[] for _ in range(n_mod + 1)] for _ in range(cap + 1)]
|
|
322
|
+
for j in range(1, n_mod + 1):
|
|
323
|
+
dp[1][j] = [(float(seg[0, j]), (0, j))]
|
|
324
|
+
for g in range(2, cap + 1):
|
|
325
|
+
for j in range(g, n_mod + 1):
|
|
326
|
+
candidates: list[tuple[float, tuple[int, ...]]] = []
|
|
327
|
+
for i in range(g - 1, j):
|
|
328
|
+
seg_ij = float(seg[i, j])
|
|
329
|
+
for prev_sum, prev_splits in dp[g - 1][i]:
|
|
330
|
+
candidates.append((prev_sum + seg_ij, prev_splits + (j,)))
|
|
331
|
+
if candidates:
|
|
332
|
+
candidates.sort(key=lambda x: x[0]) # smallest Σ within-segment first
|
|
333
|
+
dp[g][j] = candidates[:top_k]
|
|
334
|
+
return dp
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _score_partition(
|
|
338
|
+
sum_seg: float,
|
|
339
|
+
splits: tuple[int, ...],
|
|
340
|
+
*,
|
|
341
|
+
total_between: float,
|
|
342
|
+
n_prefix: np.ndarray,
|
|
343
|
+
total_n: float,
|
|
344
|
+
all_pairs: float,
|
|
345
|
+
untied_on_target: float,
|
|
346
|
+
c_nonempty: int,
|
|
347
|
+
) -> dict:
|
|
348
|
+
"""Compute tau_b / tau_c / somersd for a single consecutive partition."""
|
|
349
|
+
cd = total_between - sum_seg
|
|
350
|
+
tied_on_feature = 0.0
|
|
351
|
+
non_empty_groups = 0
|
|
352
|
+
for g in range(len(splits) - 1):
|
|
353
|
+
size = n_prefix[splits[g + 1]] - n_prefix[splits[g]]
|
|
354
|
+
tied_on_feature += size * (size - 1) / 2.0
|
|
355
|
+
if size > 0:
|
|
356
|
+
non_empty_groups += 1
|
|
357
|
+
# m matches the closed form: min over *non-empty* grouped rows and target levels
|
|
358
|
+
m = min(non_empty_groups, c_nonempty)
|
|
359
|
+
return _taus_from_counts(cd, total_n, all_pairs - tied_on_feature, untied_on_target, m)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _top_k_partitions_ordinal_dp(
|
|
363
|
+
M: np.ndarray,
|
|
364
|
+
n_per_mod: np.ndarray,
|
|
365
|
+
col_sums: np.ndarray,
|
|
366
|
+
*,
|
|
367
|
+
max_n_mod: int,
|
|
368
|
+
raw_index: list,
|
|
369
|
+
sort_by: str,
|
|
370
|
+
top_k: int,
|
|
371
|
+
) -> list[dict]:
|
|
372
|
+
"""Top-K consecutive partitions ranked by ``sort_by`` (tau_b / tau_c / somersd).
|
|
373
|
+
|
|
374
|
+
``M`` is the ``(n_mod, c)`` per-modality column-count matrix aligned to
|
|
375
|
+
``raw_index``; ``col_sums`` is the target marginal (for ``T_Y``). Returns a
|
|
376
|
+
list of ``{combination, index_to_groupby, tau_b, tau_c, somersd}`` dicts
|
|
377
|
+
sorted by ``sort_by`` desc — same shape the streaming pipeline yields, so it
|
|
378
|
+
drops into the viability walk.
|
|
379
|
+
"""
|
|
380
|
+
n_mod = len(raw_index)
|
|
381
|
+
cap = min(max_n_mod, n_mod)
|
|
382
|
+
total_n = float(n_per_mod.sum())
|
|
383
|
+
if cap < 2 or total_n < 2:
|
|
384
|
+
return []
|
|
385
|
+
|
|
386
|
+
all_pairs = total_n * (total_n - 1) / 2.0
|
|
387
|
+
untied_on_target = all_pairs - float((col_sums * (col_sums - 1) / 2.0).sum())
|
|
388
|
+
c_nonempty = int((col_sums > 0).sum())
|
|
389
|
+
total_between = _concordant_minus_discordant(M)
|
|
390
|
+
seg = _segment_within_costs(M)
|
|
391
|
+
n_prefix = np.concatenate([[0.0], np.cumsum(n_per_mod.astype(float))])
|
|
392
|
+
|
|
393
|
+
dp = _build_partition_dp(seg, n_mod=n_mod, cap=cap, top_k=top_k)
|
|
394
|
+
|
|
395
|
+
entries: list[tuple[float, dict, tuple[int, ...]]] = []
|
|
396
|
+
for k in range(2, cap + 1):
|
|
397
|
+
for sum_seg, splits in dp[k][n_mod]:
|
|
398
|
+
metrics = _score_partition(
|
|
399
|
+
sum_seg,
|
|
400
|
+
splits,
|
|
401
|
+
total_between=total_between,
|
|
402
|
+
n_prefix=n_prefix,
|
|
403
|
+
total_n=total_n,
|
|
404
|
+
all_pairs=all_pairs,
|
|
405
|
+
untied_on_target=untied_on_target,
|
|
406
|
+
c_nonempty=c_nonempty,
|
|
407
|
+
)
|
|
408
|
+
entries.append((_sort_key(metrics.get(sort_by)), metrics, splits))
|
|
409
|
+
|
|
410
|
+
entries.sort(key=lambda e: e[0], reverse=True)
|
|
411
|
+
entries = entries[:top_k]
|
|
412
|
+
|
|
413
|
+
out: list[dict] = []
|
|
414
|
+
for _, metrics, splits in entries:
|
|
415
|
+
combination = [list(raw_index[splits[g] : splits[g + 1]]) for g in range(len(splits) - 1)]
|
|
416
|
+
out.append({"combination": combination, "index_to_groupby": combination_formatter(combination), **metrics})
|
|
417
|
+
return out
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _sort_key(value: float | None) -> float:
|
|
421
|
+
"""Sort key putting ``None`` / ``NaN`` metrics last (descending sort)."""
|
|
422
|
+
if value is None or (isinstance(value, float) and math.isnan(value)):
|
|
423
|
+
return float("-inf")
|
|
424
|
+
return float(value)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""set of target rates for ordinal targets"""
|
|
2
|
+
|
|
3
|
+
from abc import ABC
|
|
4
|
+
from typing import overload
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from AutoCarver.combinations.utils import TargetRate
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class OrdinalTargetRate(TargetRate[pd.DataFrame], ABC):
|
|
13
|
+
"""Ordinal target rate class.
|
|
14
|
+
|
|
15
|
+
Operates on an ordered contingency table ``feature-groups (rows) ×
|
|
16
|
+
ordinal-target-levels (cols)`` — the same crosstab shape the binary target
|
|
17
|
+
rates consume, only with one column per ordinal level instead of two.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
__name__ = "ordinal_target_rate"
|
|
21
|
+
|
|
22
|
+
@overload
|
|
23
|
+
def compute(self, xagg: pd.Series | pd.DataFrame) -> pd.DataFrame: ...
|
|
24
|
+
@overload
|
|
25
|
+
def compute(self, xagg: None) -> None: ...
|
|
26
|
+
def compute(self, xagg: pd.Series | pd.DataFrame | None) -> pd.DataFrame | None:
|
|
27
|
+
"""Computes the target rate.
|
|
28
|
+
|
|
29
|
+
Parameters
|
|
30
|
+
----------
|
|
31
|
+
xagg : pd.DataFrame
|
|
32
|
+
A crosstab (feature groups × ordinal target levels).
|
|
33
|
+
|
|
34
|
+
Returns
|
|
35
|
+
-------
|
|
36
|
+
pd.DataFrame
|
|
37
|
+
Per-group target rate, ``frequency`` and ``count``.
|
|
38
|
+
"""
|
|
39
|
+
# checking for an xtab
|
|
40
|
+
if xagg is not None:
|
|
41
|
+
# count + frequency per modality (count carried for CI-based viability tests)
|
|
42
|
+
count = xagg.sum(axis=1)
|
|
43
|
+
frequency = count / count.sum()
|
|
44
|
+
|
|
45
|
+
# computing target rate. `_compute` expects pd.DataFrame (Generic
|
|
46
|
+
# XAgg=DataFrame); compute()'s wide signature is for LSP matching,
|
|
47
|
+
# callers always pass a crosstab here.
|
|
48
|
+
return pd.DataFrame(
|
|
49
|
+
{self.__name__: self._compute(xagg), "frequency": frequency, "count": count} # type: ignore
|
|
50
|
+
)
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TargetMeanLevel(OrdinalTargetRate):
|
|
55
|
+
"""Mean ordinal level per modality.
|
|
56
|
+
|
|
57
|
+
The per-group rate is ``Σ_j level_j · n_gj / n_g+`` where ``level_j`` is read
|
|
58
|
+
from the (integer) crosstab columns. It is monotone in the target's order, so
|
|
59
|
+
it drives both the ``min_freq`` viability test and the train/dev
|
|
60
|
+
rank-preservation veto exactly like the binary/continuous target means.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
__name__ = "target_mean_level"
|
|
64
|
+
|
|
65
|
+
def _compute(self, xagg: pd.DataFrame) -> pd.Series:
|
|
66
|
+
"""Computes the mean ordinal level per modality."""
|
|
67
|
+
levels = np.asarray(xagg.columns, dtype=float)
|
|
68
|
+
counts = xagg.to_numpy(dtype=float)
|
|
69
|
+
totals = counts.sum(axis=1)
|
|
70
|
+
# empty modalities legitimately yield NaN: silence numpy's divide warnings
|
|
71
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
72
|
+
mean_level = (counts * levels).sum(axis=1) / totals
|
|
73
|
+
return pd.Series(mean_level, index=xagg.index)
|
|
@@ -3,12 +3,17 @@ for any task.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from collections.abc import Generator
|
|
6
|
-
from typing import Any
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
7
|
|
|
8
|
+
import numpy as np
|
|
8
9
|
import pandas as pd
|
|
9
10
|
|
|
10
11
|
from AutoCarver.features import BaseFeature, GroupedList
|
|
11
12
|
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
# typing-only: avoids a runtime import cycle (combination_evaluator imports this module)
|
|
15
|
+
from AutoCarver.combinations.utils.combination_evaluator import AggregatedSample
|
|
16
|
+
|
|
12
17
|
|
|
13
18
|
def combinations_at_index(
|
|
14
19
|
start_index: int, elements: list[Any] | GroupedList, remaining_groups: int
|
|
@@ -161,6 +166,25 @@ def xagg_apply_combination(
|
|
|
161
166
|
return combi_xagg
|
|
162
167
|
|
|
163
168
|
|
|
169
|
+
def group_crosstab(xagg: "AggregatedSample | pd.DataFrame", groupby: dict) -> "pd.DataFrame":
|
|
170
|
+
"""Sums a crosstab's rows into the groups defined by ``groupby``.
|
|
171
|
+
|
|
172
|
+
``groupby`` maps each raw modality to its group leader. Group leaders are
|
|
173
|
+
ordered by first appearance (ordinal order), keeping the result independent
|
|
174
|
+
of the cosmetic label strings — the same contract as
|
|
175
|
+
:meth:`BinaryCombinationEvaluator._grouper`, shared here so crosstab-based
|
|
176
|
+
evaluators (binary, ordinal) don't each reimplement it.
|
|
177
|
+
|
|
178
|
+
Accepts anything exposing ``.index`` and ``.groupby`` (a ``pd.DataFrame`` or
|
|
179
|
+
an :class:`AggregatedSample`); always returns a ``pd.DataFrame``.
|
|
180
|
+
"""
|
|
181
|
+
leaders = [groupby.get(index_value, index_value) for index_value in xagg.index]
|
|
182
|
+
# pass the key as an object ndarray (not a list): a list whose values happen
|
|
183
|
+
# to match column labels would be read by pandas as "group by these columns"
|
|
184
|
+
# rather than as a per-row grouping key.
|
|
185
|
+
return xagg.groupby(np.asarray(leaders, dtype=object), sort=False).sum()
|
|
186
|
+
|
|
187
|
+
|
|
164
188
|
def combination_formatter(combination: list[list[str]]) -> dict[str, str]:
|
|
165
189
|
"""Attributes the first element of a group to all elements of a group"""
|
|
166
190
|
return {modal: group[0] for group in combination for modal in group}
|
|
@@ -16,12 +16,20 @@ from pathlib import Path
|
|
|
16
16
|
|
|
17
17
|
import pandas as pd
|
|
18
18
|
|
|
19
|
-
from AutoCarver.carvers import BinaryCarver, ContinuousCarver, MulticlassCarver
|
|
19
|
+
from AutoCarver.carvers import BinaryCarver, ContinuousCarver, MulticlassCarver, OrdinalCarver
|
|
20
20
|
from AutoCarver.carvers.utils.base_carver import BaseCarver
|
|
21
21
|
from AutoCarver.features import Features, specs_to_features_kwargs
|
|
22
22
|
from AutoCarver.mcp import inspection
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
# "ordinal" is opt-in only: integer values alone can't distinguish an ordered
|
|
25
|
+
# ordinal target from an unordered multiclass one, so ``auto`` never resolves to
|
|
26
|
+
# it (see ``_resolve_task``) — the caller must request ``task="ordinal"``.
|
|
27
|
+
_CARVERS = {
|
|
28
|
+
"binary": BinaryCarver,
|
|
29
|
+
"continuous": ContinuousCarver,
|
|
30
|
+
"multiclass": MulticlassCarver,
|
|
31
|
+
"ordinal": OrdinalCarver,
|
|
32
|
+
}
|
|
25
33
|
|
|
26
34
|
|
|
27
35
|
class CarverSession:
|
|
@@ -183,7 +191,7 @@ class CarverSession:
|
|
|
183
191
|
if task in _CARVERS:
|
|
184
192
|
return task
|
|
185
193
|
if task != "auto":
|
|
186
|
-
raise ValueError(f"[session] unknown task {task!r}; use auto/binary/continuous/multiclass.")
|
|
194
|
+
raise ValueError(f"[session] unknown task {task!r}; use auto/binary/continuous/multiclass/ordinal.")
|
|
187
195
|
uniques = y.dropna().unique()
|
|
188
196
|
if len(uniques) == 2:
|
|
189
197
|
return "binary"
|
|
@@ -29,6 +29,7 @@ from AutoCarver.selectors.measures import (
|
|
|
29
29
|
TschuprowtMeasure,
|
|
30
30
|
ZscoreOutlierMeasure,
|
|
31
31
|
)
|
|
32
|
+
from AutoCarver.selectors.ordinal_selector import OrdinalSelector
|
|
32
33
|
from AutoCarver.selectors.regression_selector import RegressionSelector
|
|
33
34
|
from AutoCarver.selectors.utils.base_selector import BaseSelector
|
|
34
35
|
|
|
@@ -37,6 +38,7 @@ __all__ = [
|
|
|
37
38
|
"BaseSelector",
|
|
38
39
|
"RegressionSelector",
|
|
39
40
|
"ClassificationSelector",
|
|
41
|
+
"OrdinalSelector",
|
|
40
42
|
# filters
|
|
41
43
|
"BaseFilter",
|
|
42
44
|
"QuantitativeFilter",
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Tools to select the best Quantitative and Qualitative features for an ordinal target."""
|
|
2
|
+
|
|
3
|
+
from AutoCarver.selectors.measures import BaseMeasure, KruskalEtaSquaredMeasure, SpearmanMeasure
|
|
4
|
+
from AutoCarver.selectors.utils.base_selector import BaseSelector
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class OrdinalSelector(BaseSelector):
|
|
8
|
+
"""A pipeline of measures to perform a feature pre-selection that maximizes association
|
|
9
|
+
with an **ordinal** target.
|
|
10
|
+
|
|
11
|
+
The integer-encoded ordinal target is treated as a numeric rank, so the same
|
|
12
|
+
rank-based measures used for regression apply and stay order-aware w.r.t. the
|
|
13
|
+
target: Spearman's rho for quantitative features, Kruskal-η² (reversed) for
|
|
14
|
+
qualitative ones. No ordinal-specific selection statistic is required.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
__name__ = "OrdinalSelector"
|
|
18
|
+
# the ordinal target is ranked numerically (like regression) for measure orientation
|
|
19
|
+
_target_is_qualitative = False
|
|
20
|
+
|
|
21
|
+
def _default_measures(self) -> list[BaseMeasure]:
|
|
22
|
+
"""Spearman's rho ranks quantitative features, Kruskal-η² (reversed) ranks qualitative ones."""
|
|
23
|
+
return [SpearmanMeasure(), KruskalEtaSquaredMeasure()]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: AutoCarver
|
|
3
|
-
Version: 7.3.
|
|
3
|
+
Version: 7.3.4
|
|
4
4
|
Summary: Automatic Discretization of Features with Optimal Target Association
|
|
5
5
|
Keywords: binning,discretization,feature-engineering,credit-scoring,woe,scorecards,tschuprow,tschuprow-t,cramer-v,information-value,iv,monotonic-binning,supervised-discretization,bucketization,risk-modeling,credit-risk,fraud-detection,categorical-encoding
|
|
6
6
|
Author: Mario DEFRANCE
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "AutoCarver"
|
|
3
|
-
version = "7.3.
|
|
3
|
+
version = "7.3.4"
|
|
4
4
|
description = "Automatic Discretization of Features with Optimal Target Association"
|
|
5
5
|
authors = [{ name = "Mario DEFRANCE", email = "defrancemario@gmail.com" }]
|
|
6
6
|
requires-python = ">=3.11, <3.15"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/combinations/continuous/continuous_target_rates.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/nested_discretizer.py
RENAMED
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/discretizers/qualitatives/ordinal_discretizer.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/categorical_feature.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/qualitatives/qualitative_feature.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/numerical_feature.py
RENAMED
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/features/quantitatives/quantitative_feature.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{autocarver-7.3.3 → autocarver-7.3.4}/AutoCarver/selectors/measures/quantitative_measures.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|