pythios 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythios/__init__.py +78 -0
- pythios/_utils.py +155 -0
- pythios/data.py +572 -0
- pythios/embeddings.py +29 -0
- pythios/evaluate.py +76 -0
- pythios/models.py +142 -0
- pythios/preprocess.py +96 -0
- pythios/tools.py +656 -0
- pythios/unsupervised.py +58 -0
- pythios/visualize.py +48 -0
- pythios-0.1.0.dist-info/METADATA +2408 -0
- pythios-0.1.0.dist-info/RECORD +15 -0
- pythios-0.1.0.dist-info/WHEEL +5 -0
- pythios-0.1.0.dist-info/licenses/LICENCE +0 -0
- pythios-0.1.0.dist-info/top_level.txt +1 -0
pythios/data.py
ADDED
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
"""Data class — universal entry point for Pythios."""
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import numpy as np
|
|
5
|
+
from ._utils import lazy_import, detect_task_type, format_table
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Data:
|
|
9
|
+
"""
|
|
10
|
+
Universal container for data flowing through the Pythios pipeline.
|
|
11
|
+
|
|
12
|
+
Wraps a pandas DataFrame and adds:
|
|
13
|
+
- Knowledge of the target column
|
|
14
|
+
- Auto-detected task type: "binary", "multiclass", or "regression"
|
|
15
|
+
- Train/test split (after .split() is called)
|
|
16
|
+
- A clean .describe() summary
|
|
17
|
+
|
|
18
|
+
The Data object is the currency of Pythios — every class
|
|
19
|
+
(Preprocess, Models, Visualize, etc.) takes a Data object as input.
|
|
20
|
+
|
|
21
|
+
Examples:
|
|
22
|
+
>>> d = Data.from_csv("titanic.csv", target="Survived")
|
|
23
|
+
>>> d.describe()
|
|
24
|
+
>>> d.split(test=0.2)
|
|
25
|
+
>>> print(d.X_train.shape)
|
|
26
|
+
|
|
27
|
+
>>> df = pd.read_csv("data.csv")
|
|
28
|
+
>>> d = Data.from_dataframe(df, target="price")
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, df: pd.DataFrame, target: str = None):
|
|
32
|
+
"""
|
|
33
|
+
Create a Data object from a pandas DataFrame.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
df: pandas DataFrame to wrap
|
|
37
|
+
target: Name of the target column (optional)
|
|
38
|
+
"""
|
|
39
|
+
self._df = df.copy()
|
|
40
|
+
self.target = target
|
|
41
|
+
|
|
42
|
+
if target:
|
|
43
|
+
if target not in self._df.columns:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"Target column '{target}' not found. "
|
|
46
|
+
f"Available columns: {self._df.columns.tolist()}"
|
|
47
|
+
)
|
|
48
|
+
self.task_type = detect_task_type(self._df[target])
|
|
49
|
+
else:
|
|
50
|
+
self.task_type = None
|
|
51
|
+
|
|
52
|
+
# These are set after .split() is called
|
|
53
|
+
self.X_train = None
|
|
54
|
+
self.X_test = None
|
|
55
|
+
self.y_train = None
|
|
56
|
+
self.y_test = None
|
|
57
|
+
self._is_split = False
|
|
58
|
+
|
|
59
|
+
# ──────────────────────────────────────────────────────────────────
|
|
60
|
+
# CONSTRUCTORS
|
|
61
|
+
# ──────────────────────────────────────────────────────────────────
|
|
62
|
+
|
|
63
|
+
@classmethod
|
|
64
|
+
def from_csv(
|
|
65
|
+
cls,
|
|
66
|
+
path: str,
|
|
67
|
+
target: str,
|
|
68
|
+
sep: str = ",",
|
|
69
|
+
encoding: str = "utf-8"
|
|
70
|
+
) -> "Data":
|
|
71
|
+
"""
|
|
72
|
+
Load a CSV file and return a Data object.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
path: Path to the CSV file
|
|
76
|
+
target: Name of target column
|
|
77
|
+
sep: Column separator (default: ",")
|
|
78
|
+
encoding: File encoding (default: "utf-8")
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
Data object
|
|
82
|
+
|
|
83
|
+
Example:
|
|
84
|
+
>>> d = Data.from_csv("titanic.csv", target="Survived")
|
|
85
|
+
"""
|
|
86
|
+
df = pd.read_csv(path, sep=sep, encoding=encoding)
|
|
87
|
+
return cls(df, target=target)
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def from_dataframe(cls, df: pd.DataFrame, target: str) -> "Data":
|
|
91
|
+
"""
|
|
92
|
+
Create a Data object from an existing pandas DataFrame.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
df: pandas DataFrame
|
|
96
|
+
target: Name of target column
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
Data object
|
|
100
|
+
|
|
101
|
+
Example:
|
|
102
|
+
>>> d = Data.from_dataframe(df, target="price")
|
|
103
|
+
"""
|
|
104
|
+
return cls(df, target=target)
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def from_url(cls, url: str, target: str, sep: str = ",") -> "Data":
|
|
108
|
+
"""
|
|
109
|
+
Download a CSV from a URL and return a Data object.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
url: URL pointing to a CSV file
|
|
113
|
+
target: Name of target column
|
|
114
|
+
sep: Column separator (default: ",")
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
Data object
|
|
118
|
+
|
|
119
|
+
Example:
|
|
120
|
+
>>> d = Data.from_url("https://example.com/data.csv", target="label")
|
|
121
|
+
"""
|
|
122
|
+
requests = lazy_import("requests")
|
|
123
|
+
response = requests.get(url)
|
|
124
|
+
response.raise_for_status()
|
|
125
|
+
|
|
126
|
+
from io import StringIO
|
|
127
|
+
df = pd.read_csv(StringIO(response.text), sep=sep)
|
|
128
|
+
return cls(df, target=target)
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def from_excel(cls, path: str, target: str, sheet=0) -> "Data":
|
|
132
|
+
"""
|
|
133
|
+
Load an Excel file and return a Data object.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
path: Path to .xlsx or .xls file
|
|
137
|
+
target: Name of target column
|
|
138
|
+
sheet: Sheet index or name (default: 0)
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
Data object
|
|
142
|
+
|
|
143
|
+
Example:
|
|
144
|
+
>>> d = Data.from_excel("data.xlsx", target="price")
|
|
145
|
+
"""
|
|
146
|
+
df = pd.read_excel(path, sheet_name=sheet)
|
|
147
|
+
return cls(df, target=target)
|
|
148
|
+
@classmethod
|
|
149
|
+
def from_sklearn(cls, name: str, target: str = None) -> "Data":
|
|
150
|
+
"""
|
|
151
|
+
Load a built-in sklearn dataset by name — no file download needed.
|
|
152
|
+
|
|
153
|
+
Available datasets:
|
|
154
|
+
Classification: "iris", "wine", "breast_cancer", "digits"
|
|
155
|
+
Regression: "diabetes", "california_housing", "linnerud"
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
name: Dataset name (case-insensitive)
|
|
159
|
+
target: Name to give the target column (optional, defaults to "target")
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
Data object
|
|
163
|
+
|
|
164
|
+
Example:
|
|
165
|
+
>>> d = Data.from_sklearn("iris")
|
|
166
|
+
>>> d = Data.from_sklearn("diabetes", target="disease_progression")
|
|
167
|
+
"""
|
|
168
|
+
from sklearn import datasets
|
|
169
|
+
|
|
170
|
+
loaders = {
|
|
171
|
+
"iris": datasets.load_iris,
|
|
172
|
+
"wine": datasets.load_wine,
|
|
173
|
+
"breast_cancer": datasets.load_breast_cancer,
|
|
174
|
+
"digits": datasets.load_digits,
|
|
175
|
+
"diabetes": datasets.load_diabetes,
|
|
176
|
+
"california_housing": datasets.fetch_california_housing,
|
|
177
|
+
"linnerud": datasets.load_linnerud,
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
name_lower = name.lower().strip()
|
|
181
|
+
|
|
182
|
+
if name_lower not in loaders:
|
|
183
|
+
available = ", ".join(loaders.keys())
|
|
184
|
+
raise ValueError(
|
|
185
|
+
f"Unknown dataset '{name}'. "
|
|
186
|
+
f"Available datasets: {available}"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# Load the dataset
|
|
190
|
+
dataset = loaders[name_lower]()
|
|
191
|
+
|
|
192
|
+
# Build DataFrame from features
|
|
193
|
+
if hasattr(dataset, "feature_names"):
|
|
194
|
+
df = pd.DataFrame(dataset.data, columns=dataset.feature_names)
|
|
195
|
+
else:
|
|
196
|
+
n_features = dataset.data.shape[1]
|
|
197
|
+
df = pd.DataFrame(dataset.data, columns=[f"feature_{i}" for i in range(n_features)])
|
|
198
|
+
|
|
199
|
+
# Add target column
|
|
200
|
+
target_col = target or "target"
|
|
201
|
+
|
|
202
|
+
if hasattr(dataset, "target"):
|
|
203
|
+
target_values = dataset.target
|
|
204
|
+
# Linnerud exposes three regression targets; Data intentionally
|
|
205
|
+
# models one target column, so use the first documented response.
|
|
206
|
+
if getattr(target_values, "ndim", 1) > 1:
|
|
207
|
+
target_values = target_values[:, 0]
|
|
208
|
+
# For classification, use string class names if available
|
|
209
|
+
if hasattr(dataset, "target_names") and target_values.dtype.kind == "i":
|
|
210
|
+
# Check it's actually a classification task (small number of unique values)
|
|
211
|
+
if len(np.unique(target_values)) <= 20:
|
|
212
|
+
df[target_col] = pd.Categorical.from_codes(
|
|
213
|
+
target_values, dataset.target_names
|
|
214
|
+
)
|
|
215
|
+
else:
|
|
216
|
+
df[target_col] = target_values
|
|
217
|
+
else:
|
|
218
|
+
df[target_col] = target_values
|
|
219
|
+
|
|
220
|
+
print(f"✓ Loaded sklearn dataset '{name_lower}' — {df.shape[0]} rows, {df.shape[1]} cols")
|
|
221
|
+
|
|
222
|
+
return cls(df, target=target_col)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@classmethod
|
|
226
|
+
def from_tab(cls, path: str, target: str) -> "Data":
|
|
227
|
+
"""
|
|
228
|
+
Load an Orange .tab file and return a Data object.
|
|
229
|
+
|
|
230
|
+
Orange .tab files have a specific 3-row header:
|
|
231
|
+
Row 1: Column names
|
|
232
|
+
Row 2: Column types (continuous, discrete, string)
|
|
233
|
+
Row 3: Column roles (class, meta, ignore, or blank)
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
path: Path to the .tab file
|
|
237
|
+
target: Name of the target column
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
Data object
|
|
241
|
+
|
|
242
|
+
Example:
|
|
243
|
+
>>> d = Data.from_tab("iris.tab", target="iris")
|
|
244
|
+
"""
|
|
245
|
+
# Read the three header rows first
|
|
246
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
247
|
+
col_names = f.readline().strip().split("\t") # row 1: names
|
|
248
|
+
col_types = f.readline().strip().split("\t") # row 2: types
|
|
249
|
+
col_roles = f.readline().strip().split("\t") # row 3: roles
|
|
250
|
+
|
|
251
|
+
# Read the actual data (everything after the 3-row header)
|
|
252
|
+
df = pd.read_csv(path, sep="\t", skiprows=3, header=None, names=col_names)
|
|
253
|
+
|
|
254
|
+
# Drop columns marked as "meta" or "ignore" in row 3
|
|
255
|
+
cols_to_drop = [
|
|
256
|
+
col_names[i]
|
|
257
|
+
for i, role in enumerate(col_roles)
|
|
258
|
+
if role.strip().lower() in ("meta", "ignore")
|
|
259
|
+
and col_names[i] != target
|
|
260
|
+
]
|
|
261
|
+
if cols_to_drop:
|
|
262
|
+
df = df.drop(columns=cols_to_drop, errors="ignore")
|
|
263
|
+
print(f" Dropped {len(cols_to_drop)} meta/ignore column(s): {cols_to_drop}")
|
|
264
|
+
|
|
265
|
+
# Cast continuous columns to float
|
|
266
|
+
for i, (col, col_type) in enumerate(zip(col_names, col_types)):
|
|
267
|
+
if col not in df.columns:
|
|
268
|
+
continue
|
|
269
|
+
if col_type.strip().lower() == "continuous":
|
|
270
|
+
df[col] = pd.to_numeric(df[col], errors="coerce")
|
|
271
|
+
|
|
272
|
+
print(f"✓ Loaded .tab file '{path}' — {df.shape[0]} rows, {df.shape[1]} cols")
|
|
273
|
+
|
|
274
|
+
return cls(df, target=target)
|
|
275
|
+
# ──────────────────────────────────────────────────────────────────
|
|
276
|
+
# INSPECTION
|
|
277
|
+
# ──────────────────────────────────────────────────────────────────
|
|
278
|
+
|
|
279
|
+
def describe(self):
|
|
280
|
+
"""
|
|
281
|
+
Print a detailed summary of the dataset.
|
|
282
|
+
|
|
283
|
+
Shows: row/column counts, target and task type, missing values,
|
|
284
|
+
and a per-column breakdown of dtype, % missing, unique count,
|
|
285
|
+
and any notable flags.
|
|
286
|
+
|
|
287
|
+
Example:
|
|
288
|
+
>>> d.describe()
|
|
289
|
+
|
|
290
|
+
Pythios · Data Summary
|
|
291
|
+
─────────────────────────────────────────────────────────
|
|
292
|
+
Rows: 891 Columns: 12 Target: Survived (binary)
|
|
293
|
+
Missing values: 3 column(s) affected
|
|
294
|
+
|
|
295
|
+
Column Type Missing Unique Notes
|
|
296
|
+
─────────────────────────────────────────────────────────
|
|
297
|
+
Age float64 19.9% 88
|
|
298
|
+
Cabin object 77.1% 147 high cardinality
|
|
299
|
+
Survived int64 0.0% 2 ← TARGET
|
|
300
|
+
"""
|
|
301
|
+
print("\nPythios · Data Summary")
|
|
302
|
+
print("─" * 80)
|
|
303
|
+
print(f" Rows: {len(self._df):,} Columns: {len(self._df.columns)}")
|
|
304
|
+
|
|
305
|
+
if self.target:
|
|
306
|
+
print(f" Target: {self.target} ({self.task_type})")
|
|
307
|
+
|
|
308
|
+
# Missing value overview
|
|
309
|
+
missing_counts = self._df.isnull().sum()
|
|
310
|
+
n_cols_missing = (missing_counts > 0).sum()
|
|
311
|
+
if n_cols_missing > 0:
|
|
312
|
+
print(f" Missing values: {n_cols_missing} column(s) affected")
|
|
313
|
+
|
|
314
|
+
print("\n Column Type Missing Unique Notes")
|
|
315
|
+
print("─" * 80)
|
|
316
|
+
|
|
317
|
+
rows = []
|
|
318
|
+
for col in self._df.columns:
|
|
319
|
+
dtype_str = str(self._df[col].dtype)
|
|
320
|
+
n_missing = self._df[col].isnull().sum()
|
|
321
|
+
pct_missing = 100 * n_missing / len(self._df)
|
|
322
|
+
n_unique = self._df[col].nunique()
|
|
323
|
+
|
|
324
|
+
# Notes
|
|
325
|
+
if col == self.target:
|
|
326
|
+
notes = "← TARGET"
|
|
327
|
+
elif pct_missing > 50:
|
|
328
|
+
notes = "mostly missing"
|
|
329
|
+
elif pct_missing > 20:
|
|
330
|
+
notes = "high missing"
|
|
331
|
+
elif n_unique == 1:
|
|
332
|
+
notes = "constant value"
|
|
333
|
+
elif n_unique > 100 and dtype_str == "object":
|
|
334
|
+
notes = "high cardinality"
|
|
335
|
+
else:
|
|
336
|
+
notes = ""
|
|
337
|
+
|
|
338
|
+
rows.append([col, dtype_str, f"{pct_missing:.1f}%", str(n_unique), notes])
|
|
339
|
+
|
|
340
|
+
format_table(["Column", "Type", "Missing", "Unique", "Notes"], rows)
|
|
341
|
+
|
|
342
|
+
def missing_report(self) -> pd.DataFrame:
|
|
343
|
+
"""
|
|
344
|
+
Return a DataFrame showing missing values per column.
|
|
345
|
+
|
|
346
|
+
Returns:
|
|
347
|
+
DataFrame with Column, Missing_Count, Missing_Percent columns,
|
|
348
|
+
sorted by Missing_Count descending.
|
|
349
|
+
|
|
350
|
+
Example:
|
|
351
|
+
>>> d.missing_report()
|
|
352
|
+
"""
|
|
353
|
+
counts = self._df.isnull().sum()
|
|
354
|
+
pcts = 100 * counts / len(self._df)
|
|
355
|
+
|
|
356
|
+
report = pd.DataFrame({
|
|
357
|
+
"Column": counts.index,
|
|
358
|
+
"Missing_Count": counts.values,
|
|
359
|
+
"Missing_Percent": pcts.round(2).values,
|
|
360
|
+
})
|
|
361
|
+
|
|
362
|
+
return (
|
|
363
|
+
report[report["Missing_Count"] > 0]
|
|
364
|
+
.sort_values("Missing_Count", ascending=False)
|
|
365
|
+
.reset_index(drop=True)
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
# ──────────────────────────────────────────────────────────────────
|
|
369
|
+
# TRAIN / TEST SPLIT
|
|
370
|
+
# ──────────────────────────────────────────────────────────────────
|
|
371
|
+
|
|
372
|
+
def split(
|
|
373
|
+
self,
|
|
374
|
+
test: float = 0.2,
|
|
375
|
+
seed: int = 42,
|
|
376
|
+
stratify: bool = True,
|
|
377
|
+
verbose: bool = True,
|
|
378
|
+
):
|
|
379
|
+
"""
|
|
380
|
+
Split the data into training and test sets.
|
|
381
|
+
|
|
382
|
+
For classification tasks, stratification is applied by default
|
|
383
|
+
so class proportions are preserved in both splits.
|
|
384
|
+
|
|
385
|
+
Sets the following attributes:
|
|
386
|
+
X_train, X_test: Feature DataFrames
|
|
387
|
+
y_train, y_test: Target Series
|
|
388
|
+
|
|
389
|
+
Args:
|
|
390
|
+
test: Fraction of data for test set (default: 0.2)
|
|
391
|
+
seed: Random seed for reproducibility (default: 42)
|
|
392
|
+
stratify: Stratify by target for classification (default: True)
|
|
393
|
+
|
|
394
|
+
Raises:
|
|
395
|
+
ValueError: If .split() has already been called
|
|
396
|
+
|
|
397
|
+
Example:
|
|
398
|
+
>>> d.split(test=0.2)
|
|
399
|
+
✓ Split 891 rows → 712 train (79.9%), 179 test (20.1%)
|
|
400
|
+
"""
|
|
401
|
+
if self._is_split:
|
|
402
|
+
raise ValueError(
|
|
403
|
+
"Data has already been split. "
|
|
404
|
+
"Call .copy() first if you want to re-split."
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
from sklearn.model_selection import train_test_split
|
|
408
|
+
|
|
409
|
+
# Separate features and target
|
|
410
|
+
if self.target:
|
|
411
|
+
X = self._df.drop(columns=[self.target])
|
|
412
|
+
y = self._df[self.target]
|
|
413
|
+
else:
|
|
414
|
+
X = self._df
|
|
415
|
+
y = None
|
|
416
|
+
|
|
417
|
+
# Stratify only for classification
|
|
418
|
+
stratify_col = None
|
|
419
|
+
if stratify and self.task_type in ["binary", "multiclass"] and y is not None:
|
|
420
|
+
stratify_col = y
|
|
421
|
+
|
|
422
|
+
# Perform the split
|
|
423
|
+
if y is not None:
|
|
424
|
+
X_train, X_test, y_train, y_test = train_test_split(
|
|
425
|
+
X, y,
|
|
426
|
+
test_size=test,
|
|
427
|
+
random_state=seed,
|
|
428
|
+
stratify=stratify_col
|
|
429
|
+
)
|
|
430
|
+
else:
|
|
431
|
+
X_train, X_test = train_test_split(
|
|
432
|
+
X,
|
|
433
|
+
test_size=test,
|
|
434
|
+
random_state=seed,
|
|
435
|
+
)
|
|
436
|
+
y_train, y_test = None, None
|
|
437
|
+
|
|
438
|
+
# Store with clean index
|
|
439
|
+
self.X_train = X_train.reset_index(drop=True)
|
|
440
|
+
self.X_test = X_test.reset_index(drop=True)
|
|
441
|
+
self.y_train = y_train.reset_index(drop=True) if y_train is not None else None
|
|
442
|
+
self.y_test = y_test.reset_index(drop=True) if y_test is not None else None
|
|
443
|
+
self._is_split = True
|
|
444
|
+
|
|
445
|
+
n_total = len(self.X_train) + len(self.X_test)
|
|
446
|
+
if verbose:
|
|
447
|
+
print(
|
|
448
|
+
f"✓ Split {n_total:,} rows → "
|
|
449
|
+
f"{len(self.X_train):,} train ({100*len(self.X_train)/n_total:.1f}%), "
|
|
450
|
+
f"{len(self.X_test):,} test ({100*len(self.X_test)/n_total:.1f}%)"
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
# ──────────────────────────────────────────────────────────────────
|
|
454
|
+
# PROPERTIES
|
|
455
|
+
# ──────────────────────────────────────────────────────────────────
|
|
456
|
+
|
|
457
|
+
@property
|
|
458
|
+
def shape(self) -> tuple:
|
|
459
|
+
"""Return (rows, columns) of the full dataset."""
|
|
460
|
+
return self._df.shape
|
|
461
|
+
|
|
462
|
+
@property
|
|
463
|
+
def dtypes(self) -> pd.Series:
|
|
464
|
+
"""Return data types of all columns."""
|
|
465
|
+
return self._df.dtypes
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def columns(self) -> list:
|
|
469
|
+
"""Return list of column names."""
|
|
470
|
+
return self._df.columns.tolist()
|
|
471
|
+
|
|
472
|
+
# ──────────────────────────────────────────────────────────────────
|
|
473
|
+
# UTILITY METHODS
|
|
474
|
+
# ──────────────────────────────────────────────────────────────────
|
|
475
|
+
|
|
476
|
+
def head(self, n: int = 5) -> pd.DataFrame:
|
|
477
|
+
"""Return the first n rows."""
|
|
478
|
+
return self._df.head(n)
|
|
479
|
+
|
|
480
|
+
def tail(self, n: int = 5) -> pd.DataFrame:
|
|
481
|
+
"""Return the last n rows."""
|
|
482
|
+
return self._df.tail(n)
|
|
483
|
+
|
|
484
|
+
def copy(self) -> "Data":
|
|
485
|
+
"""
|
|
486
|
+
Return a deep copy of this Data object.
|
|
487
|
+
|
|
488
|
+
Copies the DataFrame, target, task type, and split if it exists.
|
|
489
|
+
|
|
490
|
+
Returns:
|
|
491
|
+
New independent Data object
|
|
492
|
+
|
|
493
|
+
Example:
|
|
494
|
+
>>> d2 = d.copy()
|
|
495
|
+
"""
|
|
496
|
+
new = Data(self._df.copy(), target=self.target)
|
|
497
|
+
|
|
498
|
+
if self._is_split:
|
|
499
|
+
new.X_train = self.X_train.copy()
|
|
500
|
+
new.X_test = self.X_test.copy()
|
|
501
|
+
new.y_train = self.y_train.copy() if self.y_train is not None else None
|
|
502
|
+
new.y_test = self.y_test.copy() if self.y_test is not None else None
|
|
503
|
+
new._is_split = True
|
|
504
|
+
|
|
505
|
+
return new
|
|
506
|
+
|
|
507
|
+
def drop_columns(self, cols: list) -> "Data":
|
|
508
|
+
"""
|
|
509
|
+
Return a new Data object with specified columns removed.
|
|
510
|
+
|
|
511
|
+
Args:
|
|
512
|
+
cols: List of column names to drop
|
|
513
|
+
|
|
514
|
+
Returns:
|
|
515
|
+
New Data object
|
|
516
|
+
|
|
517
|
+
Example:
|
|
518
|
+
>>> d2 = d.drop_columns(["Cabin", "Name", "Ticket"])
|
|
519
|
+
"""
|
|
520
|
+
# The target is part of the Data contract and is never removed here.
|
|
521
|
+
safe_cols = [c for c in cols if c != self.target]
|
|
522
|
+
new_df = self._df.drop(columns=safe_cols, errors="ignore")
|
|
523
|
+
return Data(new_df, target=self.target)
|
|
524
|
+
|
|
525
|
+
def select_columns(self, cols: list) -> "Data":
|
|
526
|
+
"""
|
|
527
|
+
Return a new Data object with only the specified columns.
|
|
528
|
+
|
|
529
|
+
Args:
|
|
530
|
+
cols: List of column names to keep
|
|
531
|
+
|
|
532
|
+
Returns:
|
|
533
|
+
New Data object
|
|
534
|
+
|
|
535
|
+
Example:
|
|
536
|
+
>>> d2 = d.select_columns(["Age", "Fare", "Survived"])
|
|
537
|
+
"""
|
|
538
|
+
new_df = self._df[cols]
|
|
539
|
+
new_target = self.target if self.target in cols else None
|
|
540
|
+
return Data(new_df, target=new_target)
|
|
541
|
+
|
|
542
|
+
def sample(self, n: int = 1000, seed: int = 42) -> "Data":
|
|
543
|
+
"""
|
|
544
|
+
Return a new Data object with a random sample of n rows.
|
|
545
|
+
|
|
546
|
+
Useful for quick prototyping on large datasets.
|
|
547
|
+
|
|
548
|
+
Args:
|
|
549
|
+
n: Number of rows to sample (capped at dataset size)
|
|
550
|
+
seed: Random seed for reproducibility
|
|
551
|
+
|
|
552
|
+
Returns:
|
|
553
|
+
New Data object
|
|
554
|
+
|
|
555
|
+
Example:
|
|
556
|
+
>>> d_small = d.sample(500)
|
|
557
|
+
"""
|
|
558
|
+
n = min(n, len(self._df))
|
|
559
|
+
sampled = self._df.sample(n=n, random_state=seed)
|
|
560
|
+
return Data(sampled, target=self.target)
|
|
561
|
+
|
|
562
|
+
def __repr__(self) -> str:
|
|
563
|
+
split_str = " [split]" if self._is_split else ""
|
|
564
|
+
return (
|
|
565
|
+
f"Data(shape={self.shape}, "
|
|
566
|
+
f"target='{self.target}', "
|
|
567
|
+
f"task={self.task_type})"
|
|
568
|
+
f"{split_str}"
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
def __str__(self) -> str:
|
|
572
|
+
return f"Pythios Data — {self.shape[0]:,} rows × {self.shape[1]} cols"
|
pythios/embeddings.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Optional text, image, and vector-search helpers."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
from ._utils import lazy_import
|
|
4
|
+
class Embeddings:
|
|
5
|
+
def __init__(self): self._text_model=None; self._text_model_name=None; self._image_model=None; self._image_model_name=None; self._index=None; self.dim=None
|
|
6
|
+
def text(self,model_name="all-MiniLM-L6-v2"): self._text_model_name=model_name; return self
|
|
7
|
+
def encode(self,texts):
|
|
8
|
+
if self._text_model is None: self._text_model=lazy_import("sentence_transformers").SentenceTransformer(self._text_model_name or "all-MiniLM-L6-v2")
|
|
9
|
+
result=self._text_model.encode(texts,convert_to_numpy=True,show_progress_bar=len(texts)>100); self.dim=result.shape[1]; return result
|
|
10
|
+
def image(self,model_name="resnet50"): self._image_model_name=model_name; return self
|
|
11
|
+
def encode_images(self,paths):
|
|
12
|
+
torch=lazy_import("torch"); tv=lazy_import("torchvision"); Image=lazy_import("PIL.Image","Pillow"); transforms=tv.transforms
|
|
13
|
+
if self._image_model is None:
|
|
14
|
+
self._image_model=getattr(tv.models,self._image_model_name or "resnet50")(weights="DEFAULT"); self._image_model=__import__("torch").nn.Sequential(*list(self._image_model.children())[:-1]); self._image_model.eval()
|
|
15
|
+
prep=transforms.Compose([transforms.Resize(256),transforms.CenterCrop(224),transforms.ToTensor(),transforms.Normalize([.485,.456,.406],[.229,.224,.225])])
|
|
16
|
+
with torch.no_grad(): result=self._image_model(torch.stack([prep(Image.open(p).convert("RGB")) for p in paths])).flatten(1).cpu().numpy()
|
|
17
|
+
self.dim=result.shape[1]; return result
|
|
18
|
+
def similarity(self,a,b): return float(np.dot(a/(np.linalg.norm(a)+1e-9),b/(np.linalg.norm(b)+1e-9)))
|
|
19
|
+
def similarity_matrix(self,vectors):
|
|
20
|
+
v=vectors/(np.linalg.norm(vectors,axis=1,keepdims=True)+1e-9); return v@v.T
|
|
21
|
+
def search(self,query,corpus_vectors,top_k=5):
|
|
22
|
+
q=self.encode([query])[0] if isinstance(query,str) else np.asarray(query); scores=corpus_vectors@q/(np.linalg.norm(corpus_vectors,axis=1)*np.linalg.norm(q)+1e-9); return [(int(i),float(scores[i])) for i in np.argsort(scores)[::-1][:top_k]]
|
|
23
|
+
def build_index(self,vectors):
|
|
24
|
+
faiss=lazy_import("faiss","faiss-cpu"); self._index=faiss.IndexFlatL2(vectors.shape[1]); self._index.add(vectors.astype("float32")); self.dim=vectors.shape[1]; print(f"✓ Faiss index built — {len(vectors)} vectors, dim={self.dim}"); return self
|
|
25
|
+
def search_index(self,query_vec,top_k=5):
|
|
26
|
+
if self._index is None: raise RuntimeError("Build an index first")
|
|
27
|
+
distances,indices=self._index.search(np.asarray(query_vec,dtype="float32").reshape(1,-1),top_k); return [(int(i),float(d)) for i,d in zip(indices[0],distances[0])]
|
|
28
|
+
def save_index(self,path): lazy_import("faiss","faiss-cpu").write_index(self._index,path)
|
|
29
|
+
def load_index(self,path): self._index=lazy_import("faiss","faiss-cpu").read_index(path); self.dim=self._index.d; return self
|
pythios/evaluate.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Model evaluation and diagnostic plots."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
class Evaluate:
|
|
4
|
+
def __init__(self,model): self._model=model; self._data=model._data
|
|
5
|
+
def metrics(self,on="test"):
|
|
6
|
+
X=getattr(self._data,"X_"+on); y=getattr(self._data,"y_"+on); pred=self._model.predict(X)
|
|
7
|
+
from sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score,roc_auc_score,matthews_corrcoef,mean_absolute_error,mean_squared_error,r2_score
|
|
8
|
+
if self._data.task_type=="binary":
|
|
9
|
+
classes = list(getattr(self._model.current_model, "classes_", []))
|
|
10
|
+
binary_average = "binary" if set(np.unique(y)).issubset({0, 1}) else None
|
|
11
|
+
metric_kwargs = {"average": binary_average, "zero_division": 0} if binary_average else {"average": "weighted", "zero_division": 0}
|
|
12
|
+
if binary_average:
|
|
13
|
+
metric_kwargs["pos_label"] = 1
|
|
14
|
+
try: auc=roc_auc_score(y,self._model.predict_proba(X)[:,1])
|
|
15
|
+
except Exception: auc=roc_auc_score(y,pred)
|
|
16
|
+
out={"accuracy":accuracy_score(y,pred),"f1":f1_score(y,pred,**metric_kwargs),"precision":precision_score(y,pred,**metric_kwargs),"recall":recall_score(y,pred,**metric_kwargs),"auc_roc":auc,"mcc":matthews_corrcoef(y,pred)}
|
|
17
|
+
elif self._data.task_type=="multiclass": out={"accuracy":accuracy_score(y,pred),"macro_f1":f1_score(y,pred,average="macro"),"weighted_f1":f1_score(y,pred,average="weighted")}
|
|
18
|
+
else: out={"mae":mean_absolute_error(y,pred),"rmse":np.sqrt(mean_squared_error(y,pred)),"r2":r2_score(y,pred)}
|
|
19
|
+
if self._data.task_type == "binary":
|
|
20
|
+
print("\nPythios · Metrics (binary classification)")
|
|
21
|
+
print("─" * 42)
|
|
22
|
+
print(f" Accuracy {out['accuracy']:.3f} Precision {out['precision']:.3f}")
|
|
23
|
+
print(f" F1 Score {out['f1']:.3f} Recall {out['recall']:.3f}")
|
|
24
|
+
print(f" AUC-ROC {out['auc_roc']:.3f} MCC {out['mcc']:.3f}")
|
|
25
|
+
else:
|
|
26
|
+
print("\n".join(f"{k}: {v:.4f}" for k,v in out.items()))
|
|
27
|
+
return out
|
|
28
|
+
def overfit_check(self):
|
|
29
|
+
train=self.metrics("train"); test=self.metrics("test")
|
|
30
|
+
for k in train:
|
|
31
|
+
if abs(train[k]-test[k])>.1: print(f"⚠ Large train/test gap for {k}: {train[k]-test[k]:.3f}")
|
|
32
|
+
return {"train":train,"test":test}
|
|
33
|
+
def confusion_matrix(self,normalise=True):
|
|
34
|
+
import matplotlib.pyplot as plt; from sklearn.metrics import confusion_matrix
|
|
35
|
+
y=self._data.y_test; pred=self._model.predict(self._data.X_test); cm=confusion_matrix(y,pred,normalize="true" if normalise else None); plt.imshow(cm,cmap="Blues"); plt.colorbar();
|
|
36
|
+
for (i,j),v in np.ndenumerate(cm): plt.text(j,i,f"{v:.2f}" if normalise else int(v),ha="center",va="center")
|
|
37
|
+
plt.title(f"Confusion Matrix — {self._model.model_name}"); plt.xlabel("Predicted"); plt.ylabel("Actual"); plt.show()
|
|
38
|
+
def roc_curve(self):
|
|
39
|
+
if self._data.task_type=="regression": raise ValueError("ROC curves require classification")
|
|
40
|
+
import matplotlib.pyplot as plt; from sklearn.metrics import roc_curve,auc
|
|
41
|
+
if self._data.task_type=="binary":
|
|
42
|
+
score=self._model.predict_proba(self._data.X_test)[:,1]; fpr,tpr,_=roc_curve(self._data.y_test,score); plt.plot(fpr,tpr,label=f"AUC={auc(fpr,tpr):.3f}")
|
|
43
|
+
else:
|
|
44
|
+
from sklearn.preprocessing import label_binarize
|
|
45
|
+
from sklearn.metrics import roc_curve
|
|
46
|
+
y=label_binarize(self._data.y_test,classes=self._model.current_model.classes_); scores=self._model.predict_proba(self._data.X_test)
|
|
47
|
+
for i in range(y.shape[1]): fpr,tpr,_=roc_curve(y[:,i],scores[:,i]); plt.plot(fpr,tpr,label=str(i))
|
|
48
|
+
plt.legend(); plt.xlabel("False positive rate"); plt.ylabel("True positive rate"); plt.show()
|
|
49
|
+
def precision_recall_curve(self):
|
|
50
|
+
if self._data.task_type!="binary": raise ValueError("Precision-recall curve is binary only")
|
|
51
|
+
import matplotlib.pyplot as plt; from sklearn.metrics import precision_recall_curve
|
|
52
|
+
p,r,_=precision_recall_curve(self._data.y_test,self._model.predict_proba(self._data.X_test)[:,1]); plt.plot(r,p); plt.xlabel("Recall"); plt.ylabel("Precision"); plt.show()
|
|
53
|
+
def residuals(self):
|
|
54
|
+
if self._data.task_type!="regression": raise ValueError("Residuals require regression")
|
|
55
|
+
import matplotlib.pyplot as plt; pred=self._model.predict(self._data.X_test); y=self._data.y_test; fig,ax=plt.subplots(1,2); ax[0].scatter(y,pred); ax[0].plot([y.min(),y.max()],[y.min(),y.max()]); ax[1].hist(y-pred); plt.show()
|
|
56
|
+
def feature_importance(self):
|
|
57
|
+
import pandas as pd; import matplotlib.pyplot as plt
|
|
58
|
+
model=self._model.current_model; names=self._data.X_test.columns
|
|
59
|
+
if hasattr(model,"feature_importances_"): values=model.feature_importances_
|
|
60
|
+
else:
|
|
61
|
+
from sklearn.inspection import permutation_importance
|
|
62
|
+
values=permutation_importance(model,self._data.X_test,self._data.y_test,n_repeats=10,random_state=42).importances_mean
|
|
63
|
+
out=pd.DataFrame({"Feature":names,"Importance":values}).sort_values("Importance",ascending=False).reset_index(drop=True); out.head(20).sort_values("Importance").plot.barh(x="Feature",y="Importance",legend=False); plt.show(); return out
|
|
64
|
+
def cross_validate(self,k=5,stratify=True):
|
|
65
|
+
from sklearn.model_selection import cross_val_score,StratifiedKFold,KFold
|
|
66
|
+
cv=StratifiedKFold(k,shuffle=True,random_state=42) if stratify and self._data.task_type in ("binary","multiclass") else KFold(k,shuffle=True,random_state=42); scoring="roc_auc" if self._data.task_type=="binary" else ("f1_macro" if self._data.task_type=="multiclass" else "r2"); scores=cross_val_score(self._model.current_model,self._data._df.drop(columns=[self._data.target]),self._data._df[self._data.target],cv=cv,scoring=scoring); print(f"{scoring}: {scores.mean():.4f} ± {scores.std():.4f}"); return {scoring:scores}
|
|
67
|
+
def shap(self,sample=None):
|
|
68
|
+
shap=__import__("pythios._utils",fromlist=["lazy_import"]).lazy_import("shap"); X=self._data.X_test.sample(min(sample,len(self._data.X_test)),random_state=42) if sample else self._data.X_test; values=shap.Explainer(self._model.current_model,X)(X); shap.plots.beeswarm(values); return values
|
|
69
|
+
def shap_waterfall(self,index=0):
|
|
70
|
+
shap=__import__("pythios._utils",fromlist=["lazy_import"]).lazy_import("shap"); values=shap.Explainer(self._model.current_model,self._data.X_test)(self._data.X_test); shap.plots.waterfall(values[index])
|
|
71
|
+
def shap_dependence(self,feature):
|
|
72
|
+
shap=__import__("pythios._utils",fromlist=["lazy_import"]).lazy_import("shap"); shap.dependence_plot(feature,shap.TreeExplainer(self._model.current_model)(self._data.X_test).values,self._data.X_test)
|
|
73
|
+
def report(self,save=None):
|
|
74
|
+
text=f"Model: {self._model.model_name}\nTask: {self._data.task_type}\nTest set size: {len(self._data.X_test)}\nMetrics:\n"+"\n".join(f" {k}: {v:.4f}" for k,v in self.metrics().items())
|
|
75
|
+
if save: open(save,"w",encoding="utf-8").write(text)
|
|
76
|
+
print(text); return text
|