autoforge-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoforge_engine-0.1.0.dist-info/METADATA +105 -0
- autoforge_engine-0.1.0.dist-info/RECORD +32 -0
- autoforge_engine-0.1.0.dist-info/WHEEL +5 -0
- autoforge_engine-0.1.0.dist-info/entry_points.txt +2 -0
- autoforge_engine-0.1.0.dist-info/licenses/LICENSE +0 -0
- autoforge_engine-0.1.0.dist-info/top_level.txt +1 -0
- modelforge/artifact_manager.py +485 -0
- modelforge/automl.py +1472 -0
- modelforge/cli.py +1258 -0
- modelforge/column_intelligence.py +404 -0
- modelforge/config.py +580 -0
- modelforge/cross_validation.py +749 -0
- modelforge/data_audit.py +392 -0
- modelforge/data_loader.py +76 -0
- modelforge/evaluation.py +397 -0
- modelforge/experiment_tracker.py +490 -0
- modelforge/explainability.py +346 -0
- modelforge/feature_engineering.py +393 -0
- modelforge/feature_selection.py +528 -0
- modelforge/hyperparameter_optimization.py +593 -0
- modelforge/model_registry.py +684 -0
- modelforge/model_screening.py +531 -0
- modelforge/persistence.py +456 -0
- modelforge/pipeline_generator.py +278 -0
- modelforge/prediction_validator.py +316 -0
- modelforge/preprocessing.py +179 -0
- modelforge/profiler.py +85 -0
- modelforge/ranking.py +351 -0
- modelforge/reproducibility.py +295 -0
- modelforge/reproducibility_integration.py +192 -0
- modelforge/run_manager.py +200 -0
- modelforge/target_selector.py +108 -0
modelforge/cli.py
ADDED
|
@@ -0,0 +1,1258 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
|
|
7
|
+
from modelforge import AutoML
|
|
8
|
+
from modelforge.config import ModelForgeConfig
|
|
9
|
+
from modelforge.model_registry import ModelRegistry
|
|
10
|
+
from modelforge.persistence import ModelPersistence
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(
|
|
14
|
+
name="modelforge",
|
|
15
|
+
help=(
|
|
16
|
+
"ModelForge - Transparent, "
|
|
17
|
+
"local-first AutoML framework."
|
|
18
|
+
),
|
|
19
|
+
no_args_is_help=True,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
experiments_app = typer.Typer(
|
|
23
|
+
name="experiments",
|
|
24
|
+
help="Manage ModelForge experiments.",
|
|
25
|
+
no_args_is_help=True,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
app.add_typer(
|
|
29
|
+
experiments_app,
|
|
30
|
+
name="experiments",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
console = Console()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _safe_value(value, default=""):
|
|
37
|
+
"""Convert experiment values into clean CLI strings."""
|
|
38
|
+
if value is None:
|
|
39
|
+
return default
|
|
40
|
+
|
|
41
|
+
if isinstance(value, (dict, list, tuple)):
|
|
42
|
+
return default
|
|
43
|
+
|
|
44
|
+
return str(value)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _experiment_target(experiment):
|
|
48
|
+
"""Extract the target name from an experiment record."""
|
|
49
|
+
target = experiment.get("target")
|
|
50
|
+
|
|
51
|
+
if isinstance(target, dict):
|
|
52
|
+
return target.get("target", "")
|
|
53
|
+
|
|
54
|
+
return target or ""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _experiment_task_type(experiment):
|
|
58
|
+
"""Extract task type from an experiment record."""
|
|
59
|
+
task_type = experiment.get("task_type")
|
|
60
|
+
|
|
61
|
+
if task_type:
|
|
62
|
+
return task_type
|
|
63
|
+
|
|
64
|
+
target = experiment.get("target")
|
|
65
|
+
|
|
66
|
+
if isinstance(target, dict):
|
|
67
|
+
return target.get("task_type", "")
|
|
68
|
+
|
|
69
|
+
return ""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _experiment_best_model(experiment):
|
|
73
|
+
"""Extract best model from an experiment record."""
|
|
74
|
+
return experiment.get("best_model", "")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _experiment_run_id(experiment):
|
|
78
|
+
"""
|
|
79
|
+
Extract run ID from an experiment record.
|
|
80
|
+
|
|
81
|
+
New records store run_id directly.
|
|
82
|
+
Older records may store it inside run_summary.
|
|
83
|
+
"""
|
|
84
|
+
run_id = experiment.get("run_id")
|
|
85
|
+
|
|
86
|
+
if run_id:
|
|
87
|
+
return run_id
|
|
88
|
+
|
|
89
|
+
run_summary = experiment.get("run_summary")
|
|
90
|
+
|
|
91
|
+
if isinstance(run_summary, dict):
|
|
92
|
+
return run_summary.get("run_id", "")
|
|
93
|
+
|
|
94
|
+
return ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _experiment_status(experiment):
|
|
98
|
+
"""
|
|
99
|
+
Extract experiment/run status.
|
|
100
|
+
|
|
101
|
+
New records may contain status directly.
|
|
102
|
+
Failed records can also expose status through run_summary.
|
|
103
|
+
"""
|
|
104
|
+
status = experiment.get("status")
|
|
105
|
+
|
|
106
|
+
if status:
|
|
107
|
+
return status
|
|
108
|
+
|
|
109
|
+
run_summary = experiment.get("run_summary")
|
|
110
|
+
|
|
111
|
+
if isinstance(run_summary, dict):
|
|
112
|
+
return run_summary.get("status", "")
|
|
113
|
+
|
|
114
|
+
return ""
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@app.command()
|
|
118
|
+
def train(
|
|
119
|
+
data: str = typer.Option(
|
|
120
|
+
...,
|
|
121
|
+
"--data",
|
|
122
|
+
"-d",
|
|
123
|
+
help="Path to the training dataset.",
|
|
124
|
+
),
|
|
125
|
+
target: str | None = typer.Option(
|
|
126
|
+
None,
|
|
127
|
+
"--target",
|
|
128
|
+
"-t",
|
|
129
|
+
help=(
|
|
130
|
+
"Target column. "
|
|
131
|
+
"Can also be supplied through --config."
|
|
132
|
+
),
|
|
133
|
+
),
|
|
134
|
+
output: str = typer.Option(
|
|
135
|
+
"model.joblib",
|
|
136
|
+
"--output",
|
|
137
|
+
"--save",
|
|
138
|
+
"-o",
|
|
139
|
+
help=(
|
|
140
|
+
"Path where the best model will be saved. "
|
|
141
|
+
"--save is an alias for --output."
|
|
142
|
+
),
|
|
143
|
+
),
|
|
144
|
+
experiment_directory: str = typer.Option(
|
|
145
|
+
".modelforge/experiments",
|
|
146
|
+
"--experiment-directory",
|
|
147
|
+
help="Directory where experiment artifacts are stored.",
|
|
148
|
+
),
|
|
149
|
+
task_type: str | None = typer.Option(
|
|
150
|
+
None,
|
|
151
|
+
"--task-type",
|
|
152
|
+
help=(
|
|
153
|
+
"Optional: regression or classification. "
|
|
154
|
+
"Can also be supplied through --config."
|
|
155
|
+
),
|
|
156
|
+
),
|
|
157
|
+
objective: str | None = typer.Option(
|
|
158
|
+
None,
|
|
159
|
+
"--objective",
|
|
160
|
+
help=(
|
|
161
|
+
"Ranking objective: balanced, "
|
|
162
|
+
"performance, error, or speed."
|
|
163
|
+
),
|
|
164
|
+
),
|
|
165
|
+
cv: int | None = typer.Option(
|
|
166
|
+
None,
|
|
167
|
+
"--cv",
|
|
168
|
+
help="Number of cross-validation folds.",
|
|
169
|
+
),
|
|
170
|
+
test_size: float | None = typer.Option(
|
|
171
|
+
None,
|
|
172
|
+
"--test-size",
|
|
173
|
+
help="Holdout test-set proportion.",
|
|
174
|
+
),
|
|
175
|
+
models: str | None = typer.Option(
|
|
176
|
+
None,
|
|
177
|
+
"--models",
|
|
178
|
+
help=(
|
|
179
|
+
"Comma-separated model names. "
|
|
180
|
+
"Overrides models from --config."
|
|
181
|
+
),
|
|
182
|
+
),
|
|
183
|
+
config: str | None = typer.Option(
|
|
184
|
+
None,
|
|
185
|
+
"--config",
|
|
186
|
+
"-c",
|
|
187
|
+
help="Path to a ModelForge YAML configuration file.",
|
|
188
|
+
),
|
|
189
|
+
overwrite: bool = typer.Option(
|
|
190
|
+
False,
|
|
191
|
+
"--overwrite",
|
|
192
|
+
help="Overwrite an existing model file.",
|
|
193
|
+
),
|
|
194
|
+
):
|
|
195
|
+
"""
|
|
196
|
+
Train ModelForge AutoML and save the best pipeline.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
try:
|
|
200
|
+
if not Path(data).exists():
|
|
201
|
+
raise FileNotFoundError(
|
|
202
|
+
f"Dataset not found: {data}"
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if config:
|
|
206
|
+
configuration = ModelForgeConfig.from_file(config)
|
|
207
|
+
config_values = configuration.to_dict()
|
|
208
|
+
else:
|
|
209
|
+
config_values = {}
|
|
210
|
+
|
|
211
|
+
resolved_target = (
|
|
212
|
+
target
|
|
213
|
+
if target is not None
|
|
214
|
+
else config_values.get("target")
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
resolved_task_type = (
|
|
218
|
+
task_type
|
|
219
|
+
if task_type is not None
|
|
220
|
+
else config_values.get("task_type")
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
resolved_objective = (
|
|
224
|
+
objective
|
|
225
|
+
if objective is not None
|
|
226
|
+
else config_values.get(
|
|
227
|
+
"objective",
|
|
228
|
+
"balanced",
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
resolved_cv = (
|
|
233
|
+
cv
|
|
234
|
+
if cv is not None
|
|
235
|
+
else config_values.get(
|
|
236
|
+
"cv",
|
|
237
|
+
5,
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
resolved_test_size = (
|
|
242
|
+
test_size
|
|
243
|
+
if test_size is not None
|
|
244
|
+
else config_values.get(
|
|
245
|
+
"test_size",
|
|
246
|
+
0.2,
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
resolved_experiment_directory = (
|
|
251
|
+
experiment_directory
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
if (
|
|
255
|
+
experiment_directory
|
|
256
|
+
== ".modelforge/experiments"
|
|
257
|
+
and config_values.get(
|
|
258
|
+
"experiment_directory"
|
|
259
|
+
)
|
|
260
|
+
):
|
|
261
|
+
resolved_experiment_directory = (
|
|
262
|
+
config_values["experiment_directory"]
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
if not resolved_target:
|
|
266
|
+
raise ValueError(
|
|
267
|
+
"Target is required. "
|
|
268
|
+
"Provide --target or define "
|
|
269
|
+
"'target' in the configuration file."
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
if models:
|
|
273
|
+
selected_models = [
|
|
274
|
+
model.strip()
|
|
275
|
+
for model in models.split(",")
|
|
276
|
+
if model.strip()
|
|
277
|
+
]
|
|
278
|
+
else:
|
|
279
|
+
selected_models = config_values.get(
|
|
280
|
+
"models"
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
excluded_columns = config_values.get(
|
|
284
|
+
"excluded_columns",
|
|
285
|
+
[],
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
feature_selection = config_values.get(
|
|
289
|
+
"feature_selection",
|
|
290
|
+
{},
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
variance_threshold = feature_selection.get(
|
|
294
|
+
"variance_threshold"
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
correlation_threshold = feature_selection.get(
|
|
298
|
+
"correlation_threshold"
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
console.print()
|
|
302
|
+
|
|
303
|
+
console.print(
|
|
304
|
+
"[bold cyan]ModelForge[/bold cyan]"
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
console.print(
|
|
308
|
+
"[dim]Automated ML Pipeline Discovery[/dim]"
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
console.print()
|
|
312
|
+
|
|
313
|
+
console.print(
|
|
314
|
+
f"[cyan]Dataset:[/cyan] {data}"
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
console.print(
|
|
318
|
+
f"[cyan]Target:[/cyan] "
|
|
319
|
+
f"{resolved_target}"
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
console.print(
|
|
323
|
+
f"[cyan]Objective:[/cyan] "
|
|
324
|
+
f"{resolved_objective}"
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
console.print(
|
|
328
|
+
f"[cyan]CV folds:[/cyan] "
|
|
329
|
+
f"{resolved_cv}"
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
console.print(
|
|
333
|
+
f"[cyan]Test size:[/cyan] "
|
|
334
|
+
f"{resolved_test_size}"
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
console.print(
|
|
338
|
+
f"[cyan]Variance threshold:[/cyan] "
|
|
339
|
+
f"{variance_threshold}"
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
console.print(
|
|
343
|
+
f"[cyan]Correlation threshold:[/cyan] "
|
|
344
|
+
f"{correlation_threshold}"
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
console.print(
|
|
348
|
+
f"[cyan]Experiment directory:[/cyan] "
|
|
349
|
+
f"{resolved_experiment_directory}"
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
if config:
|
|
353
|
+
console.print(
|
|
354
|
+
f"[cyan]Config:[/cyan] "
|
|
355
|
+
f"{config}"
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
console.print()
|
|
359
|
+
|
|
360
|
+
automl = AutoML(
|
|
361
|
+
test_size=resolved_test_size,
|
|
362
|
+
cv=resolved_cv,
|
|
363
|
+
objective=resolved_objective,
|
|
364
|
+
variance_threshold=variance_threshold,
|
|
365
|
+
correlation_threshold=correlation_threshold,
|
|
366
|
+
experiment_directory=(
|
|
367
|
+
resolved_experiment_directory
|
|
368
|
+
),
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
with console.status(
|
|
372
|
+
"[bold green]Running AutoML...[/bold green]"
|
|
373
|
+
):
|
|
374
|
+
result = automl.fit(
|
|
375
|
+
data=data,
|
|
376
|
+
target=resolved_target,
|
|
377
|
+
task_type=resolved_task_type,
|
|
378
|
+
model_names=selected_models,
|
|
379
|
+
excluded_columns=excluded_columns,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
model_path = automl.save(
|
|
383
|
+
output,
|
|
384
|
+
overwrite=overwrite,
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
console.print(
|
|
388
|
+
"[bold green]✓ Training completed[/bold green]"
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
console.print()
|
|
392
|
+
|
|
393
|
+
summary_table = Table(
|
|
394
|
+
title="ModelForge Training Summary"
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
summary_table.add_column(
|
|
398
|
+
"Property",
|
|
399
|
+
style="cyan",
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
summary_table.add_column(
|
|
403
|
+
"Value",
|
|
404
|
+
style="green",
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
summary_table.add_row(
|
|
408
|
+
"Target",
|
|
409
|
+
result["target"]["target"],
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
summary_table.add_row(
|
|
413
|
+
"Task Type",
|
|
414
|
+
result["target"]["task_type"],
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
summary_table.add_row(
|
|
418
|
+
"Models Evaluated",
|
|
419
|
+
str(result["models_evaluated"]),
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
summary_table.add_row(
|
|
423
|
+
"Best Model",
|
|
424
|
+
result["best_model"],
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
summary_table.add_row(
|
|
428
|
+
"Ranking Objective",
|
|
429
|
+
resolved_objective,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
summary_table.add_row(
|
|
433
|
+
"Variance Threshold",
|
|
434
|
+
str(variance_threshold),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
summary_table.add_row(
|
|
438
|
+
"Correlation Threshold",
|
|
439
|
+
str(correlation_threshold),
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
summary_table.add_row(
|
|
443
|
+
"Run ID",
|
|
444
|
+
_safe_value(
|
|
445
|
+
result.get("run_id"),
|
|
446
|
+
"N/A",
|
|
447
|
+
),
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
summary_table.add_row(
|
|
451
|
+
"Experiment ID",
|
|
452
|
+
_safe_value(
|
|
453
|
+
result.get("experiment_id"),
|
|
454
|
+
"N/A",
|
|
455
|
+
),
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
summary_table.add_row(
|
|
459
|
+
"Saved Model",
|
|
460
|
+
model_path,
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
console.print(summary_table)
|
|
464
|
+
|
|
465
|
+
console.print()
|
|
466
|
+
|
|
467
|
+
ranking = result["ranking"]
|
|
468
|
+
|
|
469
|
+
display_columns = [
|
|
470
|
+
column
|
|
471
|
+
for column in [
|
|
472
|
+
"rank",
|
|
473
|
+
"model",
|
|
474
|
+
"overall_score",
|
|
475
|
+
"status",
|
|
476
|
+
]
|
|
477
|
+
if column in ranking.columns
|
|
478
|
+
]
|
|
479
|
+
|
|
480
|
+
ranking_table = Table(
|
|
481
|
+
title="Model Ranking"
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
for column in display_columns:
|
|
485
|
+
ranking_table.add_column(
|
|
486
|
+
column.replace(
|
|
487
|
+
"_",
|
|
488
|
+
" ",
|
|
489
|
+
).title()
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
for _, row in ranking[
|
|
493
|
+
display_columns
|
|
494
|
+
].head(10).iterrows():
|
|
495
|
+
|
|
496
|
+
values = []
|
|
497
|
+
|
|
498
|
+
for column in display_columns:
|
|
499
|
+
value = row[column]
|
|
500
|
+
|
|
501
|
+
if isinstance(
|
|
502
|
+
value,
|
|
503
|
+
float,
|
|
504
|
+
):
|
|
505
|
+
values.append(
|
|
506
|
+
f"{value:.4f}"
|
|
507
|
+
)
|
|
508
|
+
else:
|
|
509
|
+
values.append(
|
|
510
|
+
str(value)
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
ranking_table.add_row(
|
|
514
|
+
*values
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
console.print(
|
|
518
|
+
ranking_table
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
except Exception as exc:
|
|
522
|
+
console.print(
|
|
523
|
+
f"[bold red]✗ Training failed:[/bold red] "
|
|
524
|
+
f"{exc}"
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
raise typer.Exit(
|
|
528
|
+
code=1
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
@app.command()
|
|
533
|
+
def predict(
|
|
534
|
+
model: str = typer.Option(
|
|
535
|
+
...,
|
|
536
|
+
"--model",
|
|
537
|
+
"-m",
|
|
538
|
+
help="Path to a saved ModelForge model.",
|
|
539
|
+
),
|
|
540
|
+
data: str = typer.Option(
|
|
541
|
+
...,
|
|
542
|
+
"--data",
|
|
543
|
+
"-d",
|
|
544
|
+
help="Path to the prediction CSV.",
|
|
545
|
+
),
|
|
546
|
+
output: str | None = typer.Option(
|
|
547
|
+
None,
|
|
548
|
+
"--output",
|
|
549
|
+
"-o",
|
|
550
|
+
help="Optional CSV output path.",
|
|
551
|
+
),
|
|
552
|
+
proba: bool = typer.Option(
|
|
553
|
+
False,
|
|
554
|
+
"--proba",
|
|
555
|
+
help=(
|
|
556
|
+
"Generate class probabilities instead of "
|
|
557
|
+
"class predictions. Only supported for "
|
|
558
|
+
"classification models."
|
|
559
|
+
),
|
|
560
|
+
),
|
|
561
|
+
):
|
|
562
|
+
"""
|
|
563
|
+
Generate predictions using a saved ModelForge model.
|
|
564
|
+
"""
|
|
565
|
+
|
|
566
|
+
try:
|
|
567
|
+
if not Path(model).exists():
|
|
568
|
+
raise FileNotFoundError(
|
|
569
|
+
f"Model not found: {model}"
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
if not Path(data).exists():
|
|
573
|
+
raise FileNotFoundError(
|
|
574
|
+
f"Dataset not found: {data}"
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
persistence = ModelPersistence()
|
|
578
|
+
|
|
579
|
+
pipeline = persistence.load(
|
|
580
|
+
model
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
if proba:
|
|
584
|
+
probabilities = (
|
|
585
|
+
persistence.predict_proba_from_file(
|
|
586
|
+
pipeline,
|
|
587
|
+
data,
|
|
588
|
+
)
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
if output:
|
|
592
|
+
probabilities.to_csv(
|
|
593
|
+
output,
|
|
594
|
+
index=False,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
console.print(
|
|
598
|
+
"[bold green]✓ Prediction "
|
|
599
|
+
"probabilities saved:[/bold green] "
|
|
600
|
+
f"{output}"
|
|
601
|
+
)
|
|
602
|
+
else:
|
|
603
|
+
console.print()
|
|
604
|
+
|
|
605
|
+
console.print(
|
|
606
|
+
"[bold cyan]Prediction "
|
|
607
|
+
"Probabilities[/bold cyan]"
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
console.print()
|
|
611
|
+
|
|
612
|
+
for index, row in probabilities.iterrows():
|
|
613
|
+
values = ", ".join(
|
|
614
|
+
f"{column}={row[column]:.6f}"
|
|
615
|
+
for column in probabilities.columns
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
console.print(
|
|
619
|
+
f"{index}: {values}"
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
console.print()
|
|
623
|
+
|
|
624
|
+
console.print(
|
|
625
|
+
f"[green]✓ Generated probabilities "
|
|
626
|
+
f"for {len(probabilities)} samples.[/green]"
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
else:
|
|
630
|
+
predictions = (
|
|
631
|
+
persistence.predict_from_file(
|
|
632
|
+
pipeline,
|
|
633
|
+
data,
|
|
634
|
+
)
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
if output:
|
|
638
|
+
predictions.to_frame().to_csv(
|
|
639
|
+
output,
|
|
640
|
+
index=False,
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
console.print(
|
|
644
|
+
"[bold green]✓ Predictions saved:[/bold green] "
|
|
645
|
+
f"{output}"
|
|
646
|
+
)
|
|
647
|
+
else:
|
|
648
|
+
console.print()
|
|
649
|
+
|
|
650
|
+
console.print(
|
|
651
|
+
"[bold cyan]Predictions[/bold cyan]"
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
console.print()
|
|
655
|
+
|
|
656
|
+
for (
|
|
657
|
+
index,
|
|
658
|
+
prediction,
|
|
659
|
+
) in enumerate(
|
|
660
|
+
predictions.tolist()
|
|
661
|
+
):
|
|
662
|
+
console.print(
|
|
663
|
+
f"{index}: {prediction}"
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
console.print()
|
|
667
|
+
|
|
668
|
+
console.print(
|
|
669
|
+
f"[green]✓ Generated "
|
|
670
|
+
f"{len(predictions)} predictions.[/green]"
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
except Exception as exc:
|
|
674
|
+
console.print(
|
|
675
|
+
f"[bold red]✗ Prediction failed:[/bold red] "
|
|
676
|
+
f"{exc}"
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
raise typer.Exit(
|
|
680
|
+
code=1
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
@app.command()
|
|
685
|
+
def explain(
|
|
686
|
+
model: str = typer.Option(
|
|
687
|
+
...,
|
|
688
|
+
"--model",
|
|
689
|
+
"-m",
|
|
690
|
+
help="Path to a saved ModelForge model.",
|
|
691
|
+
),
|
|
692
|
+
top: int = typer.Option(
|
|
693
|
+
10,
|
|
694
|
+
"--top",
|
|
695
|
+
"-n",
|
|
696
|
+
help="Number of important features to display.",
|
|
697
|
+
),
|
|
698
|
+
):
|
|
699
|
+
"""
|
|
700
|
+
Display feature importance for a saved model.
|
|
701
|
+
"""
|
|
702
|
+
|
|
703
|
+
try:
|
|
704
|
+
if not Path(model).exists():
|
|
705
|
+
raise FileNotFoundError(
|
|
706
|
+
f"Model not found: {model}"
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
persistence = ModelPersistence()
|
|
710
|
+
|
|
711
|
+
pipeline = persistence.load(
|
|
712
|
+
model
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
from modelforge.explainability import (
|
|
716
|
+
ExplainabilityEngine,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
engine = ExplainabilityEngine()
|
|
720
|
+
|
|
721
|
+
importance = (
|
|
722
|
+
engine.feature_importance(
|
|
723
|
+
pipeline
|
|
724
|
+
)
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
top_features = (
|
|
728
|
+
engine.top_features(
|
|
729
|
+
importance,
|
|
730
|
+
n=top,
|
|
731
|
+
)
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
table = Table(
|
|
735
|
+
title="Model Explainability"
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
table.add_column(
|
|
739
|
+
"Rank",
|
|
740
|
+
style="cyan",
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
table.add_column(
|
|
744
|
+
"Feature",
|
|
745
|
+
style="green",
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
table.add_column(
|
|
749
|
+
"Importance",
|
|
750
|
+
style="yellow",
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
table.add_column(
|
|
754
|
+
"Source",
|
|
755
|
+
style="magenta",
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
for (
|
|
759
|
+
rank,
|
|
760
|
+
(_, row),
|
|
761
|
+
) in enumerate(
|
|
762
|
+
top_features.iterrows(),
|
|
763
|
+
start=1,
|
|
764
|
+
):
|
|
765
|
+
table.add_row(
|
|
766
|
+
str(rank),
|
|
767
|
+
str(row["feature"]),
|
|
768
|
+
f"{row['importance']:.6f}",
|
|
769
|
+
str(row["source"]),
|
|
770
|
+
)
|
|
771
|
+
|
|
772
|
+
console.print()
|
|
773
|
+
|
|
774
|
+
console.print(table)
|
|
775
|
+
|
|
776
|
+
except Exception as exc:
|
|
777
|
+
console.print(
|
|
778
|
+
f"[bold red]✗ Explanation failed:[/bold red] "
|
|
779
|
+
f"{exc}"
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
raise typer.Exit(
|
|
783
|
+
code=1
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
@app.command(name="models")
|
|
788
|
+
def list_models(
|
|
789
|
+
task_type: str | None = typer.Option(
|
|
790
|
+
None,
|
|
791
|
+
"--task-type",
|
|
792
|
+
help="Optional: regression or classification.",
|
|
793
|
+
),
|
|
794
|
+
):
|
|
795
|
+
"""
|
|
796
|
+
List models available in ModelForge.
|
|
797
|
+
"""
|
|
798
|
+
|
|
799
|
+
try:
|
|
800
|
+
registry = ModelRegistry()
|
|
801
|
+
|
|
802
|
+
if task_type is not None:
|
|
803
|
+
task_type = (
|
|
804
|
+
task_type
|
|
805
|
+
.lower()
|
|
806
|
+
.strip()
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
if task_type not in {
|
|
810
|
+
"regression",
|
|
811
|
+
"classification",
|
|
812
|
+
}:
|
|
813
|
+
raise ValueError(
|
|
814
|
+
"task_type must be "
|
|
815
|
+
"'regression' or "
|
|
816
|
+
"'classification'."
|
|
817
|
+
)
|
|
818
|
+
|
|
819
|
+
model_names = registry.list_models(
|
|
820
|
+
task_type=task_type
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
models = [
|
|
824
|
+
registry.get(name)
|
|
825
|
+
for name in model_names
|
|
826
|
+
]
|
|
827
|
+
|
|
828
|
+
table = Table(
|
|
829
|
+
title="ModelForge Model Registry",
|
|
830
|
+
expand=True,
|
|
831
|
+
)
|
|
832
|
+
|
|
833
|
+
table.add_column(
|
|
834
|
+
"Name",
|
|
835
|
+
style="cyan",
|
|
836
|
+
no_wrap=True,
|
|
837
|
+
)
|
|
838
|
+
|
|
839
|
+
table.add_column(
|
|
840
|
+
"Task",
|
|
841
|
+
style="green",
|
|
842
|
+
no_wrap=True,
|
|
843
|
+
)
|
|
844
|
+
|
|
845
|
+
table.add_column(
|
|
846
|
+
"Category",
|
|
847
|
+
style="yellow",
|
|
848
|
+
no_wrap=True,
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
table.add_column(
|
|
852
|
+
"Scaling",
|
|
853
|
+
style="magenta",
|
|
854
|
+
no_wrap=True,
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
table.add_column(
|
|
858
|
+
"Probability",
|
|
859
|
+
style="blue",
|
|
860
|
+
no_wrap=True,
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
for spec in models:
|
|
864
|
+
table.add_row(
|
|
865
|
+
spec.name,
|
|
866
|
+
spec.task_type,
|
|
867
|
+
spec.category,
|
|
868
|
+
(
|
|
869
|
+
"Yes"
|
|
870
|
+
if spec.requires_scaling
|
|
871
|
+
else "No"
|
|
872
|
+
),
|
|
873
|
+
(
|
|
874
|
+
"Yes"
|
|
875
|
+
if spec.supports_probability
|
|
876
|
+
else "No"
|
|
877
|
+
),
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
console.print()
|
|
881
|
+
|
|
882
|
+
console.print(table)
|
|
883
|
+
|
|
884
|
+
console.print()
|
|
885
|
+
|
|
886
|
+
console.print(
|
|
887
|
+
"[bold cyan]Available models:[/bold cyan]"
|
|
888
|
+
)
|
|
889
|
+
|
|
890
|
+
for name in model_names:
|
|
891
|
+
console.print(
|
|
892
|
+
f" {name}"
|
|
893
|
+
)
|
|
894
|
+
|
|
895
|
+
console.print()
|
|
896
|
+
|
|
897
|
+
console.print(
|
|
898
|
+
f"[green]Total models: "
|
|
899
|
+
f"{len(models)}[/green]"
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
except Exception as exc:
|
|
903
|
+
console.print(
|
|
904
|
+
f"[bold red]✗ Failed to list models:[/bold red] "
|
|
905
|
+
f"{exc}"
|
|
906
|
+
)
|
|
907
|
+
|
|
908
|
+
raise typer.Exit(
|
|
909
|
+
code=1
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
@experiments_app.command(name="list")
|
|
914
|
+
def list_experiments(
|
|
915
|
+
directory: str = typer.Option(
|
|
916
|
+
".modelforge/experiments",
|
|
917
|
+
"--directory",
|
|
918
|
+
"-d",
|
|
919
|
+
help="Experiment storage directory.",
|
|
920
|
+
),
|
|
921
|
+
):
|
|
922
|
+
"""
|
|
923
|
+
List recorded ModelForge experiments.
|
|
924
|
+
"""
|
|
925
|
+
|
|
926
|
+
try:
|
|
927
|
+
automl = AutoML(
|
|
928
|
+
experiment_directory=directory
|
|
929
|
+
)
|
|
930
|
+
|
|
931
|
+
experiments = (
|
|
932
|
+
automl.list_experiments()
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
console.print()
|
|
936
|
+
console.print(
|
|
937
|
+
"[bold cyan]ModelForge Experiments[/bold cyan]"
|
|
938
|
+
)
|
|
939
|
+
console.print()
|
|
940
|
+
|
|
941
|
+
if not experiments:
|
|
942
|
+
console.print(
|
|
943
|
+
"[dim]No experiments found.[/dim]"
|
|
944
|
+
)
|
|
945
|
+
console.print()
|
|
946
|
+
|
|
947
|
+
for experiment in experiments:
|
|
948
|
+
experiment_id = (
|
|
949
|
+
experiment.get(
|
|
950
|
+
"experiment_id",
|
|
951
|
+
"",
|
|
952
|
+
)
|
|
953
|
+
)
|
|
954
|
+
|
|
955
|
+
status = _experiment_status(
|
|
956
|
+
experiment
|
|
957
|
+
)
|
|
958
|
+
|
|
959
|
+
target = _experiment_target(
|
|
960
|
+
experiment
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
task_type = _experiment_task_type(
|
|
964
|
+
experiment
|
|
965
|
+
)
|
|
966
|
+
|
|
967
|
+
best_model = _experiment_best_model(
|
|
968
|
+
experiment
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
run_id = _experiment_run_id(
|
|
972
|
+
experiment
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
console.print(
|
|
976
|
+
f"Experiment ID: "
|
|
977
|
+
f"{_safe_value(experiment_id, 'N/A')}"
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
console.print(
|
|
981
|
+
f" Status: "
|
|
982
|
+
f"{_safe_value(status, 'N/A')}"
|
|
983
|
+
)
|
|
984
|
+
|
|
985
|
+
console.print(
|
|
986
|
+
f" Target: "
|
|
987
|
+
f"{_safe_value(target, 'N/A')}"
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
console.print(
|
|
991
|
+
f" Task: "
|
|
992
|
+
f"{_safe_value(task_type, 'N/A')}"
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
console.print(
|
|
996
|
+
f" Best Model: "
|
|
997
|
+
f"{_safe_value(best_model, 'N/A')}"
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
console.print(
|
|
1001
|
+
f" Run ID: "
|
|
1002
|
+
f"{_safe_value(run_id, 'N/A')}"
|
|
1003
|
+
)
|
|
1004
|
+
|
|
1005
|
+
console.print()
|
|
1006
|
+
|
|
1007
|
+
console.print(
|
|
1008
|
+
f"[green]Total experiments: "
|
|
1009
|
+
f"{len(experiments)}[/green]"
|
|
1010
|
+
)
|
|
1011
|
+
|
|
1012
|
+
except Exception as exc:
|
|
1013
|
+
console.print(
|
|
1014
|
+
f"[bold red]✗ Failed to list "
|
|
1015
|
+
f"experiments:[/bold red] "
|
|
1016
|
+
f"{exc}"
|
|
1017
|
+
)
|
|
1018
|
+
|
|
1019
|
+
raise typer.Exit(
|
|
1020
|
+
code=1
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
@experiments_app.command(name="get")
|
|
1025
|
+
def get_experiment(
|
|
1026
|
+
experiment_id: str = typer.Argument(
|
|
1027
|
+
...,
|
|
1028
|
+
help="Experiment ID to inspect.",
|
|
1029
|
+
),
|
|
1030
|
+
directory: str = typer.Option(
|
|
1031
|
+
".modelforge/experiments",
|
|
1032
|
+
"--directory",
|
|
1033
|
+
"-d",
|
|
1034
|
+
help="Experiment storage directory.",
|
|
1035
|
+
),
|
|
1036
|
+
):
|
|
1037
|
+
"""
|
|
1038
|
+
Display details for a ModelForge experiment.
|
|
1039
|
+
"""
|
|
1040
|
+
|
|
1041
|
+
try:
|
|
1042
|
+
automl = AutoML(
|
|
1043
|
+
experiment_directory=directory
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
experiment = automl.get_experiment(
|
|
1047
|
+
experiment_id
|
|
1048
|
+
)
|
|
1049
|
+
|
|
1050
|
+
console.print()
|
|
1051
|
+
|
|
1052
|
+
console.print(
|
|
1053
|
+
f"[bold cyan]Experiment:[/bold cyan] "
|
|
1054
|
+
f"{experiment_id}"
|
|
1055
|
+
)
|
|
1056
|
+
|
|
1057
|
+
console.print()
|
|
1058
|
+
|
|
1059
|
+
summary_table = Table(
|
|
1060
|
+
title="Experiment Summary"
|
|
1061
|
+
)
|
|
1062
|
+
|
|
1063
|
+
summary_table.add_column(
|
|
1064
|
+
"Property",
|
|
1065
|
+
style="cyan",
|
|
1066
|
+
)
|
|
1067
|
+
|
|
1068
|
+
summary_table.add_column(
|
|
1069
|
+
"Value",
|
|
1070
|
+
style="green",
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
summary_table.add_row(
|
|
1074
|
+
"Experiment ID",
|
|
1075
|
+
_safe_value(
|
|
1076
|
+
experiment.get(
|
|
1077
|
+
"experiment_id"
|
|
1078
|
+
),
|
|
1079
|
+
experiment_id,
|
|
1080
|
+
),
|
|
1081
|
+
)
|
|
1082
|
+
|
|
1083
|
+
summary_table.add_row(
|
|
1084
|
+
"Status",
|
|
1085
|
+
_safe_value(
|
|
1086
|
+
_experiment_status(
|
|
1087
|
+
experiment
|
|
1088
|
+
),
|
|
1089
|
+
"N/A",
|
|
1090
|
+
),
|
|
1091
|
+
)
|
|
1092
|
+
|
|
1093
|
+
summary_table.add_row(
|
|
1094
|
+
"Target",
|
|
1095
|
+
_safe_value(
|
|
1096
|
+
_experiment_target(
|
|
1097
|
+
experiment
|
|
1098
|
+
),
|
|
1099
|
+
"N/A",
|
|
1100
|
+
),
|
|
1101
|
+
)
|
|
1102
|
+
|
|
1103
|
+
summary_table.add_row(
|
|
1104
|
+
"Task Type",
|
|
1105
|
+
_safe_value(
|
|
1106
|
+
_experiment_task_type(
|
|
1107
|
+
experiment
|
|
1108
|
+
),
|
|
1109
|
+
"N/A",
|
|
1110
|
+
),
|
|
1111
|
+
)
|
|
1112
|
+
|
|
1113
|
+
summary_table.add_row(
|
|
1114
|
+
"Best Model",
|
|
1115
|
+
_safe_value(
|
|
1116
|
+
_experiment_best_model(
|
|
1117
|
+
experiment
|
|
1118
|
+
),
|
|
1119
|
+
"N/A",
|
|
1120
|
+
),
|
|
1121
|
+
)
|
|
1122
|
+
|
|
1123
|
+
summary_table.add_row(
|
|
1124
|
+
"Run ID",
|
|
1125
|
+
_safe_value(
|
|
1126
|
+
_experiment_run_id(
|
|
1127
|
+
experiment
|
|
1128
|
+
),
|
|
1129
|
+
"N/A",
|
|
1130
|
+
),
|
|
1131
|
+
)
|
|
1132
|
+
|
|
1133
|
+
if "configuration" in experiment:
|
|
1134
|
+
summary_table.add_row(
|
|
1135
|
+
"Configuration",
|
|
1136
|
+
str(
|
|
1137
|
+
experiment["configuration"]
|
|
1138
|
+
),
|
|
1139
|
+
)
|
|
1140
|
+
|
|
1141
|
+
console.print(
|
|
1142
|
+
summary_table
|
|
1143
|
+
)
|
|
1144
|
+
|
|
1145
|
+
console.print()
|
|
1146
|
+
|
|
1147
|
+
console.print(
|
|
1148
|
+
"[bold cyan]Experiment Data[/bold cyan]"
|
|
1149
|
+
)
|
|
1150
|
+
|
|
1151
|
+
console.print()
|
|
1152
|
+
|
|
1153
|
+
details = Table(
|
|
1154
|
+
show_header=True,
|
|
1155
|
+
header_style="bold cyan",
|
|
1156
|
+
)
|
|
1157
|
+
|
|
1158
|
+
details.add_column(
|
|
1159
|
+
"Key"
|
|
1160
|
+
)
|
|
1161
|
+
|
|
1162
|
+
details.add_column(
|
|
1163
|
+
"Value"
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
for key, value in experiment.items():
|
|
1167
|
+
if key in {
|
|
1168
|
+
"experiment_id",
|
|
1169
|
+
"status",
|
|
1170
|
+
"target",
|
|
1171
|
+
"task_type",
|
|
1172
|
+
"best_model",
|
|
1173
|
+
"run_id",
|
|
1174
|
+
"configuration",
|
|
1175
|
+
}:
|
|
1176
|
+
continue
|
|
1177
|
+
|
|
1178
|
+
details.add_row(
|
|
1179
|
+
str(key),
|
|
1180
|
+
str(value),
|
|
1181
|
+
)
|
|
1182
|
+
|
|
1183
|
+
if details.row_count > 0:
|
|
1184
|
+
console.print(details)
|
|
1185
|
+
|
|
1186
|
+
except Exception as exc:
|
|
1187
|
+
console.print(
|
|
1188
|
+
f"[bold red]✗ Failed to get "
|
|
1189
|
+
f"experiment:[/bold red] "
|
|
1190
|
+
f"{exc}"
|
|
1191
|
+
)
|
|
1192
|
+
|
|
1193
|
+
raise typer.Exit(
|
|
1194
|
+
code=1
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
@experiments_app.command(name="compare")
|
|
1199
|
+
def compare_experiments(
|
|
1200
|
+
experiment_ids: list[str] = typer.Argument(
|
|
1201
|
+
...,
|
|
1202
|
+
help="Two or more experiment IDs to compare.",
|
|
1203
|
+
),
|
|
1204
|
+
directory: str = typer.Option(
|
|
1205
|
+
".modelforge/experiments",
|
|
1206
|
+
"--directory",
|
|
1207
|
+
"-d",
|
|
1208
|
+
help="Experiment storage directory.",
|
|
1209
|
+
),
|
|
1210
|
+
):
|
|
1211
|
+
"""Compare the key outcomes and configuration of experiments."""
|
|
1212
|
+
|
|
1213
|
+
if len(experiment_ids) < 2:
|
|
1214
|
+
console.print(
|
|
1215
|
+
"[bold red]✗ Provide at least two experiment IDs.[/bold red]"
|
|
1216
|
+
)
|
|
1217
|
+
raise typer.Exit(code=1)
|
|
1218
|
+
|
|
1219
|
+
try:
|
|
1220
|
+
automl = AutoML(experiment_directory=directory)
|
|
1221
|
+
table = Table(title="ModelForge Experiment Comparison")
|
|
1222
|
+
table.add_column("Experiment", style="cyan")
|
|
1223
|
+
table.add_column("Status", style="green")
|
|
1224
|
+
table.add_column("Task", style="yellow")
|
|
1225
|
+
table.add_column("Target", style="magenta")
|
|
1226
|
+
table.add_column("Best Model", style="blue")
|
|
1227
|
+
table.add_column("Run ID", style="dim")
|
|
1228
|
+
|
|
1229
|
+
for experiment_id in experiment_ids:
|
|
1230
|
+
experiment = automl.get_experiment(experiment_id)
|
|
1231
|
+
table.add_row(
|
|
1232
|
+
experiment_id,
|
|
1233
|
+
_safe_value(_experiment_status(experiment), "N/A"),
|
|
1234
|
+
_safe_value(_experiment_task_type(experiment), "N/A"),
|
|
1235
|
+
_safe_value(_experiment_target(experiment), "N/A"),
|
|
1236
|
+
_safe_value(_experiment_best_model(experiment), "N/A"),
|
|
1237
|
+
_safe_value(_experiment_run_id(experiment), "N/A"),
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
console.print(table)
|
|
1241
|
+
|
|
1242
|
+
except Exception as exc:
|
|
1243
|
+
console.print(
|
|
1244
|
+
f"[bold red]✗ Failed to compare experiments:[/bold red] {exc}"
|
|
1245
|
+
)
|
|
1246
|
+
raise typer.Exit(code=1)
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
@app.callback()
|
|
1250
|
+
def main():
|
|
1251
|
+
"""
|
|
1252
|
+
ModelForge CLI entry point.
|
|
1253
|
+
"""
|
|
1254
|
+
pass
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
if __name__ == "__main__":
|
|
1258
|
+
app()
|