retof 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retof/__init__.py +57 -0
- retof/automl/AutoML.py +206 -0
- retof/automl/__init__.py +5 -0
- retof/data/DataPreprocessor.py +144 -0
- retof/data/DataVisualizer.py +261 -0
- retof/data/__init__.py +9 -0
- retof/data/ejik.py +84 -0
- retof/experiments/ExperimentRunner.py +101 -0
- retof/experiments/__init__.py +5 -0
- retof/meta/MetaLearner.py +180 -0
- retof/meta/MetadatasetBuilder.py +40 -0
- retof/meta/__init__.py +7 -0
- retof/models/HyperparameterTuner.py +200 -0
- retof/models/ModelEvaluator.py +84 -0
- retof/models/ModelManager.py +83 -0
- retof/models/ModelTrain.py +354 -0
- retof/models/__init__.py +11 -0
- retof/utils/__init__.py +5 -0
- retof/utils/formatter.py +126 -0
- retof-0.1.0.dist-info/METADATA +770 -0
- retof-0.1.0.dist-info/RECORD +24 -0
- retof-0.1.0.dist-info/WHEEL +5 -0
- retof-0.1.0.dist-info/licenses/LICENSE +21 -0
- retof-0.1.0.dist-info/top_level.txt +1 -0
retof/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from .data import (
|
|
2
|
+
DataAnalyzer,
|
|
3
|
+
DataPreprocessor,
|
|
4
|
+
DataVisualizer,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
from .models import (
|
|
8
|
+
ModelTrain,
|
|
9
|
+
ModelEvaluator,
|
|
10
|
+
HyperparameterTuner,
|
|
11
|
+
ModelManager,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from .meta import (
|
|
15
|
+
MetaLearner,
|
|
16
|
+
MetadatasetBuilder,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
from .experiments import (
|
|
20
|
+
ExperimentRunner,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
from .automl import (
|
|
24
|
+
AutoML,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
def fit(file_path, target_column=None, target=None, tune=True):
|
|
28
|
+
if target_column is None:
|
|
29
|
+
target_column = target
|
|
30
|
+
|
|
31
|
+
if target_column is None:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
"Must provide target_column or target"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
return AutoML(
|
|
37
|
+
tune=tune,
|
|
38
|
+
use_meta_learning=False
|
|
39
|
+
).fit(
|
|
40
|
+
file_path,
|
|
41
|
+
target_column
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"DataAnalyzer",
|
|
46
|
+
"DataPreprocessor",
|
|
47
|
+
"ModelTrain",
|
|
48
|
+
"ModelEvaluator",
|
|
49
|
+
"HyperparameterTuner",
|
|
50
|
+
"ModelManager",
|
|
51
|
+
"MetaLearner",
|
|
52
|
+
"MetadatasetBuilder",
|
|
53
|
+
"ExperimentRunner",
|
|
54
|
+
"AutoML",
|
|
55
|
+
"fit",
|
|
56
|
+
"DataVisalizer",
|
|
57
|
+
]
|
retof/automl/AutoML.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from ..data.DataPreprocessor import DataPreprocessor
|
|
3
|
+
from ..data.DataVisualizer import DataVisualizer
|
|
4
|
+
from ..models.ModelTrain import ModelTrain
|
|
5
|
+
from ..models.ModelEvaluator import ModelEvaluator
|
|
6
|
+
from ..models.HyperparameterTuner import HyperparameterTuner
|
|
7
|
+
from ..experiments.ExperimentRunner import ExperimentRunner
|
|
8
|
+
from ..models.ModelManager import ModelManager
|
|
9
|
+
from ..meta.MetaLearner import MetaLearner
|
|
10
|
+
from ..meta.MetadatasetBuilder import MetadatasetBuilder
|
|
11
|
+
from ..utils.formatter import OutputFormatter
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class AutoML:
|
|
15
|
+
def __init__(self, tune=True, use_meta_learning=False):
|
|
16
|
+
self.tune = tune
|
|
17
|
+
self.use_meta_learning = use_meta_learning
|
|
18
|
+
self.model_manager = None
|
|
19
|
+
self.results = {}
|
|
20
|
+
self._fitted = False
|
|
21
|
+
self._preprocessor = None
|
|
22
|
+
self._trainer = None
|
|
23
|
+
self._evaluator = None
|
|
24
|
+
self._experiments = []
|
|
25
|
+
self.meta_learner = None
|
|
26
|
+
self.recommender = None
|
|
27
|
+
|
|
28
|
+
def fit(self, file_path, target_column):
|
|
29
|
+
self._preprocessor = DataPreprocessor(file_path, target_column)
|
|
30
|
+
self._trainer = ModelTrain(self._preprocessor.df)
|
|
31
|
+
self._evaluator = ModelEvaluator({}, None, None, None)
|
|
32
|
+
|
|
33
|
+
runner = ExperimentRunner(
|
|
34
|
+
self._preprocessor,
|
|
35
|
+
self._trainer,
|
|
36
|
+
self._evaluator,
|
|
37
|
+
hyperparameter_tuner_class=HyperparameterTuner,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
result = runner.get_results(target_column, tune=self.tune)
|
|
41
|
+
|
|
42
|
+
self.model_manager = ModelManager(
|
|
43
|
+
model=result["best_model"],
|
|
44
|
+
preprocessor=self._preprocessor,
|
|
45
|
+
target_encoder=getattr(self._trainer, 'target_encoder', None),
|
|
46
|
+
feature_names=self._trainer.feature_names,
|
|
47
|
+
target_column=target_column,
|
|
48
|
+
problem_type=getattr(self._trainer, 'problem_type', None),
|
|
49
|
+
)
|
|
50
|
+
self.results = result
|
|
51
|
+
self._fitted = True
|
|
52
|
+
|
|
53
|
+
if self.use_meta_learning:
|
|
54
|
+
self._init_meta_learning()
|
|
55
|
+
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
def _init_meta_learning(self):
|
|
59
|
+
self.meta_learner = MetaLearner(self._trainer, self._evaluator)
|
|
60
|
+
|
|
61
|
+
def add_dataset(self, file_path, target_column):
|
|
62
|
+
if not self.use_meta_learning:
|
|
63
|
+
raise RuntimeError("Meta-learning not enabled.")
|
|
64
|
+
if not self._fitted:
|
|
65
|
+
raise RuntimeError("Call fit() first.")
|
|
66
|
+
|
|
67
|
+
preprocessor = DataPreprocessor(file_path, target_column)
|
|
68
|
+
trainer = ModelTrain(preprocessor.df)
|
|
69
|
+
evaluator = ModelEvaluator({}, None, None, None)
|
|
70
|
+
runner = ExperimentRunner(
|
|
71
|
+
preprocessor,
|
|
72
|
+
trainer,
|
|
73
|
+
evaluator,
|
|
74
|
+
hyperparameter_tuner_class=HyperparameterTuner,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
result = runner.get_results(target_column, tune=False)
|
|
78
|
+
|
|
79
|
+
experiment_payload = {
|
|
80
|
+
"file_path": file_path,
|
|
81
|
+
"data": result["data"],
|
|
82
|
+
"target_column": target_column,
|
|
83
|
+
"models": runner.model_train.models,
|
|
84
|
+
"best_model_name": result["best_model_name"],
|
|
85
|
+
"best_model": result["best_model"],
|
|
86
|
+
"best_params": result.get("best_params"),
|
|
87
|
+
"evaluation_results": result["evaluation_results"],
|
|
88
|
+
}
|
|
89
|
+
self._experiments.append(experiment_payload)
|
|
90
|
+
return experiment_payload
|
|
91
|
+
|
|
92
|
+
def build_meta_dataset(self):
|
|
93
|
+
if not self.use_meta_learning or self.meta_learner is None:
|
|
94
|
+
raise RuntimeError("Meta-learning not enabled.")
|
|
95
|
+
meta_dataset = self.meta_learner.build_meta_dataset(self._experiments)
|
|
96
|
+
if len(meta_dataset) >= 2:
|
|
97
|
+
try:
|
|
98
|
+
self.recommender = self.meta_learner.train_recommender(meta_dataset)
|
|
99
|
+
except Exception:
|
|
100
|
+
pass
|
|
101
|
+
return meta_dataset
|
|
102
|
+
|
|
103
|
+
def predict(self, data):
|
|
104
|
+
if not self._fitted:
|
|
105
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
106
|
+
return self.model_manager.predict(data)
|
|
107
|
+
|
|
108
|
+
def predict_proba(self, data):
|
|
109
|
+
if not self._fitted:
|
|
110
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
111
|
+
return self.model_manager.predict_proba(data)
|
|
112
|
+
|
|
113
|
+
def save(self, file_path):
|
|
114
|
+
if not self._fitted:
|
|
115
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
116
|
+
self.model_manager.save_model(file_path)
|
|
117
|
+
|
|
118
|
+
@staticmethod
|
|
119
|
+
def load(file_path):
|
|
120
|
+
return ModelManager.load_model(file_path)
|
|
121
|
+
|
|
122
|
+
def results(self):
|
|
123
|
+
if not self._fitted:
|
|
124
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
125
|
+
return self.results
|
|
126
|
+
|
|
127
|
+
def get_model_info(self):
|
|
128
|
+
if not self._fitted:
|
|
129
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
130
|
+
return self.model_manager.get_model_info()
|
|
131
|
+
|
|
132
|
+
def print_results(self):
|
|
133
|
+
if not self._fitted:
|
|
134
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
135
|
+
OutputFormatter.print_results(
|
|
136
|
+
self.results.get("evaluation_results", {})
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def print_full(self):
|
|
140
|
+
if not self._fitted:
|
|
141
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
142
|
+
dataset_info = {
|
|
143
|
+
"rows": len(self.results["data"]),
|
|
144
|
+
"features": len(self.results["data"].columns),
|
|
145
|
+
"problem_type": getattr(self._trainer, "problem_type", "N/A"),
|
|
146
|
+
"target": self.results.get("target_column", "N/A"),
|
|
147
|
+
}
|
|
148
|
+
best_model_name = self.results.get("best_model_name", "N/A")
|
|
149
|
+
evaluation_results = self.results.get("evaluation_results", {})
|
|
150
|
+
best_model_metrics = evaluation_results.get(best_model_name, {})
|
|
151
|
+
OutputFormatter.format_full(
|
|
152
|
+
evaluation_results,
|
|
153
|
+
dataset_info,
|
|
154
|
+
best_model_name,
|
|
155
|
+
best_model_metrics,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def print_meta_dataset(self, records=None):
|
|
159
|
+
if not self.use_meta_learning:
|
|
160
|
+
raise RuntimeError("Meta-learning not enabled.")
|
|
161
|
+
if records is None:
|
|
162
|
+
records = self._experiments
|
|
163
|
+
OutputFormatter.print_meta_dataset(records)
|
|
164
|
+
|
|
165
|
+
def plot_data(self):
|
|
166
|
+
if not self._fitted:
|
|
167
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
168
|
+
visualizer = DataVisualizer(self._preprocessor.df, self._preprocessor.target_column)
|
|
169
|
+
visualizer.plot_missing_values()
|
|
170
|
+
visualizer.plot_target_distribution()
|
|
171
|
+
visualizer.plot_numeric_distributions()
|
|
172
|
+
visualizer.plot_categorical_distributions()
|
|
173
|
+
visualizer.plot_correlation_matrix()
|
|
174
|
+
visualizer.plot_feature_target_relationship()
|
|
175
|
+
visualizer.plot_outliers()
|
|
176
|
+
|
|
177
|
+
def plot_target(self):
|
|
178
|
+
if not self._fitted:
|
|
179
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
180
|
+
visualizer = DataVisualizer(self._preprocessor.df, self._preprocessor.target_column)
|
|
181
|
+
visualizer.plot_target_distribution()
|
|
182
|
+
|
|
183
|
+
def plot_correlation(self):
|
|
184
|
+
if not self._fitted:
|
|
185
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
186
|
+
visualizer = DataVisualizer(self._preprocessor.df, self._preprocessor.target_column)
|
|
187
|
+
visualizer.plot_correlation_matrix()
|
|
188
|
+
|
|
189
|
+
def plot_models(self):
|
|
190
|
+
if not self._fitted:
|
|
191
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
192
|
+
evaluation_results = self.results.get("evaluation_results", {})
|
|
193
|
+
visualizer = DataVisualizer(self._preprocessor.df, self._preprocessor.target_column)
|
|
194
|
+
visualizer.plot_model_comparison(evaluation_results)
|
|
195
|
+
|
|
196
|
+
def plot_feature_importance(self):
|
|
197
|
+
if not self._fitted:
|
|
198
|
+
raise RuntimeError("Model not trained. Call fit() first.")
|
|
199
|
+
best_model_name = self.results.get("best_model_name")
|
|
200
|
+
best_model = self.results.get("best_model")
|
|
201
|
+
if not best_model_name or best_model is None:
|
|
202
|
+
print("No best model available.")
|
|
203
|
+
return
|
|
204
|
+
feature_names = getattr(self._trainer, 'feature_names', None)
|
|
205
|
+
visualizer = DataVisualizer(self._preprocessor.df, self._preprocessor.target_column)
|
|
206
|
+
visualizer.plot_feature_importance(best_model, feature_names=feature_names)
|
retof/automl/__init__.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import numpy as np
|
|
3
|
+
from sklearn.preprocessing import StandardScaler, LabelEncoder
|
|
4
|
+
from sklearn.compose import ColumnTransformer
|
|
5
|
+
from sklearn.preprocessing import OrdinalEncoder
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DataPreprocessor:
|
|
9
|
+
def __init__(self, file_path, target_column):
|
|
10
|
+
self.data_path = file_path
|
|
11
|
+
self.target_column = target_column
|
|
12
|
+
self.df = pd.read_csv(file_path)
|
|
13
|
+
self.X = None
|
|
14
|
+
self.y = None
|
|
15
|
+
self.scaler = None
|
|
16
|
+
self.target_encoder = None
|
|
17
|
+
self.categorical_columns = []
|
|
18
|
+
self.numeric_columns = []
|
|
19
|
+
self.feature_preprocessor = None
|
|
20
|
+
self.feature_names = []
|
|
21
|
+
|
|
22
|
+
def separate_features_target(self):
|
|
23
|
+
if self.target_column not in self.df.columns:
|
|
24
|
+
raise ValueError(f"Target column '{self.target_column}' not found.")
|
|
25
|
+
self.X = self.df.drop(columns=[self.target_column]).copy()
|
|
26
|
+
self.y = self.df[self.target_column].copy()
|
|
27
|
+
|
|
28
|
+
def _get_feature_types(self):
|
|
29
|
+
if self.X is None:
|
|
30
|
+
self.separate_features_target()
|
|
31
|
+
self.numeric_columns = self.X.select_dtypes(include=['float64', 'int64']).columns.tolist()
|
|
32
|
+
self.categorical_columns = self.X.select_dtypes(include=['object', 'category']).columns.tolist()
|
|
33
|
+
|
|
34
|
+
def remove_constant_columns(self):
|
|
35
|
+
if self.X is None:
|
|
36
|
+
self.separate_features_target()
|
|
37
|
+
constant_columns = [col for col in self.X.columns if self.X[col].nunique() <= 1]
|
|
38
|
+
self.X = self.X.drop(columns=constant_columns)
|
|
39
|
+
self._get_feature_types()
|
|
40
|
+
return self.X
|
|
41
|
+
|
|
42
|
+
def remove_duplicates(self):
|
|
43
|
+
if self.X is None:
|
|
44
|
+
self.separate_features_target()
|
|
45
|
+
mask = self.X.duplicated()
|
|
46
|
+
self.X = self.X[~mask].copy()
|
|
47
|
+
self.y = self.y[~mask].copy().reset_index(drop=True)
|
|
48
|
+
return self.X
|
|
49
|
+
|
|
50
|
+
def handle_missing_values(self, strategy='fill'):
|
|
51
|
+
if self.X is None:
|
|
52
|
+
self.separate_features_target()
|
|
53
|
+
if strategy == 'drop':
|
|
54
|
+
mask = self.X.isnull().any(axis=1)
|
|
55
|
+
self.X = self.X[~mask].copy()
|
|
56
|
+
self.y = self.y[~mask].copy().reset_index(drop=True)
|
|
57
|
+
elif strategy == 'fill':
|
|
58
|
+
for col in self.numeric_columns:
|
|
59
|
+
if col in self.X.columns:
|
|
60
|
+
self.X[col] = self.X[col].fillna(self.X[col].median())
|
|
61
|
+
for col in self.categorical_columns:
|
|
62
|
+
if col in self.X.columns:
|
|
63
|
+
mode_val = self.X[col].mode()
|
|
64
|
+
if not mode_val.empty:
|
|
65
|
+
self.X[col] = self.X[col].fillna(mode_val.iloc[0])
|
|
66
|
+
return self.X
|
|
67
|
+
|
|
68
|
+
def encode_target(self):
|
|
69
|
+
if self.y is None:
|
|
70
|
+
self.separate_features_target()
|
|
71
|
+
if self.y.dtype == 'object' or self.y.dtype.name == 'category':
|
|
72
|
+
self.target_encoder = LabelEncoder()
|
|
73
|
+
self.y = pd.Series(
|
|
74
|
+
self.target_encoder.fit_transform(self.y),
|
|
75
|
+
index=self.y.index,
|
|
76
|
+
name=self.y.name
|
|
77
|
+
)
|
|
78
|
+
return self.y
|
|
79
|
+
|
|
80
|
+
def get_processed_data(self):
|
|
81
|
+
if self.X is None:
|
|
82
|
+
self.separate_features_target()
|
|
83
|
+
return self.X, self.y
|
|
84
|
+
|
|
85
|
+
def process(self):
|
|
86
|
+
self.separate_features_target()
|
|
87
|
+
self._get_feature_types()
|
|
88
|
+
self.remove_constant_columns()
|
|
89
|
+
self.remove_duplicates()
|
|
90
|
+
self.handle_missing_values(strategy='fill')
|
|
91
|
+
self.encode_target()
|
|
92
|
+
|
|
93
|
+
num_cols = [c for c in self.numeric_columns if c in self.X.columns]
|
|
94
|
+
cat_cols = [c for c in self.categorical_columns if c in self.X.columns]
|
|
95
|
+
|
|
96
|
+
numeric_transformer = StandardScaler()
|
|
97
|
+
categorical_transformer = OrdinalEncoder(
|
|
98
|
+
handle_unknown='use_encoded_value',
|
|
99
|
+
unknown_value=-1
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
self.feature_preprocessor = ColumnTransformer(
|
|
103
|
+
transformers=[
|
|
104
|
+
('num', numeric_transformer, num_cols),
|
|
105
|
+
('cat', categorical_transformer, cat_cols),
|
|
106
|
+
],
|
|
107
|
+
remainder='drop'
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
self.feature_preprocessor.fit(self.X)
|
|
111
|
+
|
|
112
|
+
X_transformed = self.feature_preprocessor.transform(self.X)
|
|
113
|
+
raw_names = self.feature_preprocessor.get_feature_names_out()
|
|
114
|
+
feature_names = [name.split('__', 1)[-1] for name in raw_names]
|
|
115
|
+
|
|
116
|
+
processed_df = pd.DataFrame(X_transformed, columns=feature_names, index=self.X.index)
|
|
117
|
+
processed_df[self.target_column] = self.y.values
|
|
118
|
+
self.df = processed_df.reset_index(drop=True)
|
|
119
|
+
|
|
120
|
+
self.X = processed_df.drop(columns=[self.target_column])
|
|
121
|
+
self.feature_names = list(self.X.columns)
|
|
122
|
+
|
|
123
|
+
return self.df
|
|
124
|
+
|
|
125
|
+
def transform(self, X):
|
|
126
|
+
if self.feature_preprocessor is None:
|
|
127
|
+
raise RuntimeError("Preprocessor not fitted. Call process() first.")
|
|
128
|
+
X_transformed = self.feature_preprocessor.transform(X)
|
|
129
|
+
raw_names = self.feature_preprocessor.get_feature_names_out()
|
|
130
|
+
feature_names = [name.split('__', 1)[-1] for name in raw_names]
|
|
131
|
+
return pd.DataFrame(X_transformed, columns=feature_names, index=X.index)
|
|
132
|
+
|
|
133
|
+
def load_dataset(self, data_path, target_column):
|
|
134
|
+
self.data_path = data_path
|
|
135
|
+
self.target_column = target_column
|
|
136
|
+
self.df = pd.read_csv(data_path)
|
|
137
|
+
self.X = None
|
|
138
|
+
self.y = None
|
|
139
|
+
self.scaler = None
|
|
140
|
+
self.target_encoder = None
|
|
141
|
+
self.categorical_columns = []
|
|
142
|
+
self.numeric_columns = []
|
|
143
|
+
self.feature_preprocessor = None
|
|
144
|
+
self.feature_names = []
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# src/retof/data/DataVisualizer.py
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DataVisualizer:
|
|
7
|
+
def __init__(self, df, target_column=None):
|
|
8
|
+
self.df = df
|
|
9
|
+
self.target_column = target_column
|
|
10
|
+
|
|
11
|
+
def _get_plt(self):
|
|
12
|
+
try:
|
|
13
|
+
import matplotlib.pyplot as plt
|
|
14
|
+
return plt
|
|
15
|
+
except ImportError:
|
|
16
|
+
raise ImportError("matplotlib is required. Install with: pip install matplotlib")
|
|
17
|
+
|
|
18
|
+
def _get_sns(self):
|
|
19
|
+
try:
|
|
20
|
+
import seaborn as sns
|
|
21
|
+
return sns
|
|
22
|
+
except ImportError:
|
|
23
|
+
raise ImportError("seaborn is required. Install with: pip install seaborn")
|
|
24
|
+
|
|
25
|
+
def plot_missing_values(self, figsize=(10, 6)):
|
|
26
|
+
plt = self._get_plt()
|
|
27
|
+
missing = self.df.isnull().sum()
|
|
28
|
+
missing = missing[missing > 0].sort_values(ascending=True)
|
|
29
|
+
if missing.empty:
|
|
30
|
+
print("No missing values found.")
|
|
31
|
+
return
|
|
32
|
+
fig, ax = plt.subplots(figsize=figsize)
|
|
33
|
+
ax.barh(missing.index, missing.values, color='steelblue')
|
|
34
|
+
ax.set_xlabel('Missing Values Count')
|
|
35
|
+
ax.set_title('Missing Values per Column')
|
|
36
|
+
ax.grid(axis='x', alpha=0.3)
|
|
37
|
+
plt.tight_layout()
|
|
38
|
+
plt.show()
|
|
39
|
+
|
|
40
|
+
def plot_target_distribution(self, figsize=(10, 6)):
|
|
41
|
+
if self.target_column is None or self.target_column not in self.df.columns:
|
|
42
|
+
print("Target column not specified.")
|
|
43
|
+
return
|
|
44
|
+
plt = self._get_plt()
|
|
45
|
+
target = self.df[self.target_column].dropna()
|
|
46
|
+
fig, ax = plt.subplots(figsize=figsize)
|
|
47
|
+
if pd.api.types.is_numeric_dtype(target) and target.nunique() > 20:
|
|
48
|
+
ax.hist(target, bins=30, color='steelblue', edgecolor='black', alpha=0.7)
|
|
49
|
+
ax.set_xlabel(self.target_column)
|
|
50
|
+
ax.set_ylabel('Frequency')
|
|
51
|
+
else:
|
|
52
|
+
target.value_counts().plot(kind='bar', ax=ax, color='steelblue')
|
|
53
|
+
ax.set_xlabel(self.target_column)
|
|
54
|
+
ax.set_ylabel('Count')
|
|
55
|
+
ax.set_title(f'Target Distribution: {self.target_column}')
|
|
56
|
+
ax.grid(axis='y', alpha=0.3)
|
|
57
|
+
plt.tight_layout()
|
|
58
|
+
plt.show()
|
|
59
|
+
|
|
60
|
+
def plot_numeric_distributions(self, figsize=(16, 10)):
|
|
61
|
+
plt = self._get_plt()
|
|
62
|
+
numeric_cols = self.df.select_dtypes(include=['float64', 'int64']).columns
|
|
63
|
+
if self.target_column in numeric_cols:
|
|
64
|
+
numeric_cols = numeric_cols.drop(self.target_column)
|
|
65
|
+
if len(numeric_cols) == 0:
|
|
66
|
+
print("No numeric columns.")
|
|
67
|
+
return
|
|
68
|
+
n_cols = min(4, len(numeric_cols))
|
|
69
|
+
n_rows = (len(numeric_cols) + n_cols - 1) // n_cols
|
|
70
|
+
fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)
|
|
71
|
+
axes = axes.flatten() if len(numeric_cols) > 1 else [axes]
|
|
72
|
+
for idx, col in enumerate(numeric_cols):
|
|
73
|
+
self.df[col].hist(bins=30, ax=axes[idx], color='steelblue', edgecolor='black', alpha=0.7)
|
|
74
|
+
axes[idx].set_title(f'Distribution of {col}')
|
|
75
|
+
axes[idx].set_xlabel(col)
|
|
76
|
+
axes[idx].set_ylabel('Frequency')
|
|
77
|
+
axes[idx].grid(axis='y', alpha=0.3)
|
|
78
|
+
for idx in range(len(numeric_cols), len(axes)):
|
|
79
|
+
fig.delaxes(axes[idx])
|
|
80
|
+
plt.tight_layout()
|
|
81
|
+
plt.show()
|
|
82
|
+
|
|
83
|
+
def plot_categorical_distributions(self, figsize=(16, 10)):
|
|
84
|
+
plt = self._get_plt()
|
|
85
|
+
cat_cols = self.df.select_dtypes(include=['object', 'category']).columns
|
|
86
|
+
if self.target_column in cat_cols:
|
|
87
|
+
cat_cols = cat_cols.drop(self.target_column)
|
|
88
|
+
if len(cat_cols) == 0:
|
|
89
|
+
print("No categorical columns.")
|
|
90
|
+
return
|
|
91
|
+
n_cols = min(4, len(cat_cols))
|
|
92
|
+
n_rows = (len(cat_cols) + n_cols - 1) // n_cols
|
|
93
|
+
fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)
|
|
94
|
+
axes = axes.flatten() if len(cat_cols) > 1 else [axes]
|
|
95
|
+
for idx, col in enumerate(cat_cols):
|
|
96
|
+
self.df[col].value_counts().plot(kind='bar', ax=axes[idx], color='steelblue')
|
|
97
|
+
axes[idx].set_title(f'Distribution of {col}')
|
|
98
|
+
axes[idx].set_xlabel(col)
|
|
99
|
+
axes[idx].set_ylabel('Count')
|
|
100
|
+
axes[idx].grid(axis='y', alpha=0.3)
|
|
101
|
+
for idx in range(len(cat_cols), len(axes)):
|
|
102
|
+
fig.delaxes(axes[idx])
|
|
103
|
+
plt.tight_layout()
|
|
104
|
+
plt.show()
|
|
105
|
+
|
|
106
|
+
def plot_correlation_matrix(self, figsize=(12, 10)):
|
|
107
|
+
plt = self._get_plt()
|
|
108
|
+
sns = self._get_sns()
|
|
109
|
+
numeric_df = self.df.select_dtypes(include=['float64', 'int64'])
|
|
110
|
+
if self.target_column in numeric_df.columns:
|
|
111
|
+
numeric_df = numeric_df.drop(columns=[self.target_column])
|
|
112
|
+
if numeric_df.shape[1] < 2:
|
|
113
|
+
print("Not enough numeric features for correlation matrix.")
|
|
114
|
+
return
|
|
115
|
+
corr = numeric_df.corr()
|
|
116
|
+
fig, ax = plt.subplots(figsize=figsize)
|
|
117
|
+
sns.heatmap(corr, annot=True, fmt='.2f', cmap='coolwarm', center=0, ax=ax)
|
|
118
|
+
ax.set_title('Correlation Matrix')
|
|
119
|
+
plt.tight_layout()
|
|
120
|
+
plt.show()
|
|
121
|
+
|
|
122
|
+
def plot_feature_target_relationship(self, figsize=(16, 10)):
|
|
123
|
+
plt = self._get_plt()
|
|
124
|
+
if self.target_column is None or self.target_column not in self.df.columns:
|
|
125
|
+
print("Target column not specified.")
|
|
126
|
+
return
|
|
127
|
+
numeric_cols = self.df.select_dtypes(include=['float64', 'int64']).columns
|
|
128
|
+
numeric_cols = numeric_cols.drop(self.target_column, errors='ignore')
|
|
129
|
+
if len(numeric_cols) == 0:
|
|
130
|
+
print("No numeric features for relationship plot.")
|
|
131
|
+
return
|
|
132
|
+
n_cols = min(4, len(numeric_cols))
|
|
133
|
+
n_rows = (len(numeric_cols) + n_cols - 1) // n_cols
|
|
134
|
+
fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)
|
|
135
|
+
axes = axes.flatten() if len(numeric_cols) > 1 else [axes]
|
|
136
|
+
for idx, col in enumerate(numeric_cols):
|
|
137
|
+
axes[idx].scatter(self.df[col], self.df[self.target_column], alpha=0.5, color='steelblue')
|
|
138
|
+
axes[idx].set_xlabel(col)
|
|
139
|
+
axes[idx].set_ylabel(self.target_column)
|
|
140
|
+
axes[idx].set_title(f'{col} vs {self.target_column}')
|
|
141
|
+
axes[idx].grid(alpha=0.3)
|
|
142
|
+
for idx in range(len(numeric_cols), len(axes)):
|
|
143
|
+
fig.delaxes(axes[idx])
|
|
144
|
+
plt.tight_layout()
|
|
145
|
+
plt.show()
|
|
146
|
+
|
|
147
|
+
def plot_outliers(self, figsize=(16, 10)):
|
|
148
|
+
plt = self._get_plt()
|
|
149
|
+
numeric_cols = self.df.select_dtypes(include=['float64', 'int64']).columns
|
|
150
|
+
if self.target_column in numeric_cols:
|
|
151
|
+
numeric_cols = numeric_cols.drop(self.target_column)
|
|
152
|
+
if len(numeric_cols) == 0:
|
|
153
|
+
print("No numeric columns for outlier detection.")
|
|
154
|
+
return
|
|
155
|
+
n_cols = min(4, len(numeric_cols))
|
|
156
|
+
n_rows = (len(numeric_cols) + n_cols - 1) // n_cols
|
|
157
|
+
fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)
|
|
158
|
+
axes = axes.flatten() if len(numeric_cols) > 1 else [axes]
|
|
159
|
+
for idx, col in enumerate(numeric_cols):
|
|
160
|
+
axes[idx].boxplot(self.df[col].dropna(), vert=True, patch_artist=True,
|
|
161
|
+
boxprops=dict(facecolor='steelblue', alpha=0.7))
|
|
162
|
+
axes[idx].set_title(f'Outliers in {col}')
|
|
163
|
+
axes[idx].set_ylabel(col)
|
|
164
|
+
axes[idx].grid(axis='y', alpha=0.3)
|
|
165
|
+
for idx in range(len(numeric_cols), len(axes)):
|
|
166
|
+
fig.delaxes(axes[idx])
|
|
167
|
+
plt.tight_layout()
|
|
168
|
+
plt.show()
|
|
169
|
+
|
|
170
|
+
def plot_model_comparison(self, results):
|
|
171
|
+
plt = self._get_plt()
|
|
172
|
+
if not results:
|
|
173
|
+
print("No results to plot.")
|
|
174
|
+
return
|
|
175
|
+
|
|
176
|
+
valid_results = {k: v for k, v in results.items() if isinstance(v, dict) and 'error' not in v}
|
|
177
|
+
if not valid_results:
|
|
178
|
+
print("No valid model results to plot.")
|
|
179
|
+
return
|
|
180
|
+
|
|
181
|
+
first_metrics = next(iter(valid_results.values()))
|
|
182
|
+
is_classification = 'accuracy' in first_metrics
|
|
183
|
+
|
|
184
|
+
models = list(valid_results.keys())
|
|
185
|
+
|
|
186
|
+
if is_classification:
|
|
187
|
+
accuracies = [valid_results[m].get('accuracy', 0) for m in models]
|
|
188
|
+
f1_scores = [valid_results[m].get('f1_macro', 0) for m in models]
|
|
189
|
+
x = range(len(models))
|
|
190
|
+
fig, ax = plt.subplots(figsize=(12, 6))
|
|
191
|
+
width = 0.35
|
|
192
|
+
ax.bar([i - width/2 for i in x], accuracies, width, label='Accuracy', color='steelblue')
|
|
193
|
+
ax.bar([i + width/2 for i in x], f1_scores, width, label='F1 Macro', color='darkorange')
|
|
194
|
+
ax.set_ylabel('Score')
|
|
195
|
+
ax.set_title('Model Comparison')
|
|
196
|
+
ax.set_xticks(x)
|
|
197
|
+
ax.set_xticklabels(models, rotation=45, ha='right')
|
|
198
|
+
ax.set_ylim(0, 1.1)
|
|
199
|
+
ax.legend()
|
|
200
|
+
ax.grid(axis='y', alpha=0.3)
|
|
201
|
+
plt.tight_layout()
|
|
202
|
+
plt.show()
|
|
203
|
+
else:
|
|
204
|
+
r2_scores = [valid_results[m].get('r2', 0) for m in models]
|
|
205
|
+
rmse_scores = [valid_results[m].get('rmse', 0) for m in models]
|
|
206
|
+
x = range(len(models))
|
|
207
|
+
fig, ax = plt.subplots(figsize=(12, 6))
|
|
208
|
+
ax.bar(x, r2_scores, color='steelblue', label='R²')
|
|
209
|
+
ax.set_ylabel('R² Score')
|
|
210
|
+
ax.set_title('Model Comparison (R²)')
|
|
211
|
+
ax.set_xticks(x)
|
|
212
|
+
ax.set_xticklabels(models, rotation=45, ha='right')
|
|
213
|
+
ax.set_ylim(0, 1.1)
|
|
214
|
+
ax.grid(axis='y', alpha=0.3)
|
|
215
|
+
plt.tight_layout()
|
|
216
|
+
plt.show()
|
|
217
|
+
|
|
218
|
+
def plot_feature_importance(self, model, feature_names=None):
|
|
219
|
+
plt = self._get_plt()
|
|
220
|
+
if feature_names is None:
|
|
221
|
+
feature_names = self.df.drop(columns=[self.target_column]).columns if self.target_column in self.df.columns else self.df.columns
|
|
222
|
+
|
|
223
|
+
if hasattr(model, 'feature_importances_'):
|
|
224
|
+
importances = model.feature_importances_
|
|
225
|
+
elif hasattr(model, 'coef_'):
|
|
226
|
+
coef = model.coef_
|
|
227
|
+
if coef.ndim == 2:
|
|
228
|
+
importances = np.mean(np.abs(coef), axis=0)
|
|
229
|
+
else:
|
|
230
|
+
importances = np.abs(coef)
|
|
231
|
+
else:
|
|
232
|
+
print("Model does not support feature importance.")
|
|
233
|
+
return
|
|
234
|
+
|
|
235
|
+
indices = importances.argsort()[::-1]
|
|
236
|
+
fig, ax = plt.subplots(figsize=(10, 6))
|
|
237
|
+
ax.bar(range(len(importances)), importances[indices], color='steelblue')
|
|
238
|
+
ax.set_xticks(range(len(importances)))
|
|
239
|
+
ax.set_xticklabels([feature_names[i] for i in indices], rotation=45, ha='right')
|
|
240
|
+
ax.set_ylabel('Importance')
|
|
241
|
+
ax.set_title('Feature Importance')
|
|
242
|
+
ax.grid(axis='y', alpha=0.3)
|
|
243
|
+
plt.tight_layout()
|
|
244
|
+
plt.show()
|
|
245
|
+
|
|
246
|
+
def generate_report(self):
|
|
247
|
+
print("=== Dataset Summary ===")
|
|
248
|
+
print(f"Shape: {self.df.shape}")
|
|
249
|
+
print(f"Columns: {list(self.df.columns)}")
|
|
250
|
+
print(f"Missing values: {int(self.df.isnull().sum().sum())}")
|
|
251
|
+
print(f"Duplicate rows: {int(self.df.duplicated().sum())}")
|
|
252
|
+
print(f"Numeric features: {len(self.df.select_dtypes(include=['float64', 'int64']).columns)}")
|
|
253
|
+
print(f"Categorical features: {len(self.df.select_dtypes(include=['object', 'category']).columns)}")
|
|
254
|
+
if self.target_column and self.target_column in self.df.columns:
|
|
255
|
+
target = self.df[self.target_column]
|
|
256
|
+
print(f"Target column: {self.target_column}")
|
|
257
|
+
print(f"Target unique values: {target.nunique()}")
|
|
258
|
+
if pd.api.types.is_object_dtype(target) or target.nunique() <= 20:
|
|
259
|
+
print(f"Target distribution:\n{target.value_counts()}")
|
|
260
|
+
else:
|
|
261
|
+
print(f"Target stats:\n{target.describe()}")
|