autoforge-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelforge/cli.py ADDED
@@ -0,0 +1,1258 @@
1
+ from pathlib import Path
2
+
3
+ import typer
4
+ from rich.console import Console
5
+ from rich.table import Table
6
+
7
+ from modelforge import AutoML
8
+ from modelforge.config import ModelForgeConfig
9
+ from modelforge.model_registry import ModelRegistry
10
+ from modelforge.persistence import ModelPersistence
11
+
12
+
13
+ app = typer.Typer(
14
+ name="modelforge",
15
+ help=(
16
+ "ModelForge - Transparent, "
17
+ "local-first AutoML framework."
18
+ ),
19
+ no_args_is_help=True,
20
+ )
21
+
22
+ experiments_app = typer.Typer(
23
+ name="experiments",
24
+ help="Manage ModelForge experiments.",
25
+ no_args_is_help=True,
26
+ )
27
+
28
+ app.add_typer(
29
+ experiments_app,
30
+ name="experiments",
31
+ )
32
+
33
+ console = Console()
34
+
35
+
36
+ def _safe_value(value, default=""):
37
+ """Convert experiment values into clean CLI strings."""
38
+ if value is None:
39
+ return default
40
+
41
+ if isinstance(value, (dict, list, tuple)):
42
+ return default
43
+
44
+ return str(value)
45
+
46
+
47
+ def _experiment_target(experiment):
48
+ """Extract the target name from an experiment record."""
49
+ target = experiment.get("target")
50
+
51
+ if isinstance(target, dict):
52
+ return target.get("target", "")
53
+
54
+ return target or ""
55
+
56
+
57
+ def _experiment_task_type(experiment):
58
+ """Extract task type from an experiment record."""
59
+ task_type = experiment.get("task_type")
60
+
61
+ if task_type:
62
+ return task_type
63
+
64
+ target = experiment.get("target")
65
+
66
+ if isinstance(target, dict):
67
+ return target.get("task_type", "")
68
+
69
+ return ""
70
+
71
+
72
+ def _experiment_best_model(experiment):
73
+ """Extract best model from an experiment record."""
74
+ return experiment.get("best_model", "")
75
+
76
+
77
+ def _experiment_run_id(experiment):
78
+ """
79
+ Extract run ID from an experiment record.
80
+
81
+ New records store run_id directly.
82
+ Older records may store it inside run_summary.
83
+ """
84
+ run_id = experiment.get("run_id")
85
+
86
+ if run_id:
87
+ return run_id
88
+
89
+ run_summary = experiment.get("run_summary")
90
+
91
+ if isinstance(run_summary, dict):
92
+ return run_summary.get("run_id", "")
93
+
94
+ return ""
95
+
96
+
97
+ def _experiment_status(experiment):
98
+ """
99
+ Extract experiment/run status.
100
+
101
+ New records may contain status directly.
102
+ Failed records can also expose status through run_summary.
103
+ """
104
+ status = experiment.get("status")
105
+
106
+ if status:
107
+ return status
108
+
109
+ run_summary = experiment.get("run_summary")
110
+
111
+ if isinstance(run_summary, dict):
112
+ return run_summary.get("status", "")
113
+
114
+ return ""
115
+
116
+
117
+ @app.command()
118
+ def train(
119
+ data: str = typer.Option(
120
+ ...,
121
+ "--data",
122
+ "-d",
123
+ help="Path to the training dataset.",
124
+ ),
125
+ target: str | None = typer.Option(
126
+ None,
127
+ "--target",
128
+ "-t",
129
+ help=(
130
+ "Target column. "
131
+ "Can also be supplied through --config."
132
+ ),
133
+ ),
134
+ output: str = typer.Option(
135
+ "model.joblib",
136
+ "--output",
137
+ "--save",
138
+ "-o",
139
+ help=(
140
+ "Path where the best model will be saved. "
141
+ "--save is an alias for --output."
142
+ ),
143
+ ),
144
+ experiment_directory: str = typer.Option(
145
+ ".modelforge/experiments",
146
+ "--experiment-directory",
147
+ help="Directory where experiment artifacts are stored.",
148
+ ),
149
+ task_type: str | None = typer.Option(
150
+ None,
151
+ "--task-type",
152
+ help=(
153
+ "Optional: regression or classification. "
154
+ "Can also be supplied through --config."
155
+ ),
156
+ ),
157
+ objective: str | None = typer.Option(
158
+ None,
159
+ "--objective",
160
+ help=(
161
+ "Ranking objective: balanced, "
162
+ "performance, error, or speed."
163
+ ),
164
+ ),
165
+ cv: int | None = typer.Option(
166
+ None,
167
+ "--cv",
168
+ help="Number of cross-validation folds.",
169
+ ),
170
+ test_size: float | None = typer.Option(
171
+ None,
172
+ "--test-size",
173
+ help="Holdout test-set proportion.",
174
+ ),
175
+ models: str | None = typer.Option(
176
+ None,
177
+ "--models",
178
+ help=(
179
+ "Comma-separated model names. "
180
+ "Overrides models from --config."
181
+ ),
182
+ ),
183
+ config: str | None = typer.Option(
184
+ None,
185
+ "--config",
186
+ "-c",
187
+ help="Path to a ModelForge YAML configuration file.",
188
+ ),
189
+ overwrite: bool = typer.Option(
190
+ False,
191
+ "--overwrite",
192
+ help="Overwrite an existing model file.",
193
+ ),
194
+ ):
195
+ """
196
+ Train ModelForge AutoML and save the best pipeline.
197
+ """
198
+
199
+ try:
200
+ if not Path(data).exists():
201
+ raise FileNotFoundError(
202
+ f"Dataset not found: {data}"
203
+ )
204
+
205
+ if config:
206
+ configuration = ModelForgeConfig.from_file(config)
207
+ config_values = configuration.to_dict()
208
+ else:
209
+ config_values = {}
210
+
211
+ resolved_target = (
212
+ target
213
+ if target is not None
214
+ else config_values.get("target")
215
+ )
216
+
217
+ resolved_task_type = (
218
+ task_type
219
+ if task_type is not None
220
+ else config_values.get("task_type")
221
+ )
222
+
223
+ resolved_objective = (
224
+ objective
225
+ if objective is not None
226
+ else config_values.get(
227
+ "objective",
228
+ "balanced",
229
+ )
230
+ )
231
+
232
+ resolved_cv = (
233
+ cv
234
+ if cv is not None
235
+ else config_values.get(
236
+ "cv",
237
+ 5,
238
+ )
239
+ )
240
+
241
+ resolved_test_size = (
242
+ test_size
243
+ if test_size is not None
244
+ else config_values.get(
245
+ "test_size",
246
+ 0.2,
247
+ )
248
+ )
249
+
250
+ resolved_experiment_directory = (
251
+ experiment_directory
252
+ )
253
+
254
+ if (
255
+ experiment_directory
256
+ == ".modelforge/experiments"
257
+ and config_values.get(
258
+ "experiment_directory"
259
+ )
260
+ ):
261
+ resolved_experiment_directory = (
262
+ config_values["experiment_directory"]
263
+ )
264
+
265
+ if not resolved_target:
266
+ raise ValueError(
267
+ "Target is required. "
268
+ "Provide --target or define "
269
+ "'target' in the configuration file."
270
+ )
271
+
272
+ if models:
273
+ selected_models = [
274
+ model.strip()
275
+ for model in models.split(",")
276
+ if model.strip()
277
+ ]
278
+ else:
279
+ selected_models = config_values.get(
280
+ "models"
281
+ )
282
+
283
+ excluded_columns = config_values.get(
284
+ "excluded_columns",
285
+ [],
286
+ )
287
+
288
+ feature_selection = config_values.get(
289
+ "feature_selection",
290
+ {},
291
+ )
292
+
293
+ variance_threshold = feature_selection.get(
294
+ "variance_threshold"
295
+ )
296
+
297
+ correlation_threshold = feature_selection.get(
298
+ "correlation_threshold"
299
+ )
300
+
301
+ console.print()
302
+
303
+ console.print(
304
+ "[bold cyan]ModelForge[/bold cyan]"
305
+ )
306
+
307
+ console.print(
308
+ "[dim]Automated ML Pipeline Discovery[/dim]"
309
+ )
310
+
311
+ console.print()
312
+
313
+ console.print(
314
+ f"[cyan]Dataset:[/cyan] {data}"
315
+ )
316
+
317
+ console.print(
318
+ f"[cyan]Target:[/cyan] "
319
+ f"{resolved_target}"
320
+ )
321
+
322
+ console.print(
323
+ f"[cyan]Objective:[/cyan] "
324
+ f"{resolved_objective}"
325
+ )
326
+
327
+ console.print(
328
+ f"[cyan]CV folds:[/cyan] "
329
+ f"{resolved_cv}"
330
+ )
331
+
332
+ console.print(
333
+ f"[cyan]Test size:[/cyan] "
334
+ f"{resolved_test_size}"
335
+ )
336
+
337
+ console.print(
338
+ f"[cyan]Variance threshold:[/cyan] "
339
+ f"{variance_threshold}"
340
+ )
341
+
342
+ console.print(
343
+ f"[cyan]Correlation threshold:[/cyan] "
344
+ f"{correlation_threshold}"
345
+ )
346
+
347
+ console.print(
348
+ f"[cyan]Experiment directory:[/cyan] "
349
+ f"{resolved_experiment_directory}"
350
+ )
351
+
352
+ if config:
353
+ console.print(
354
+ f"[cyan]Config:[/cyan] "
355
+ f"{config}"
356
+ )
357
+
358
+ console.print()
359
+
360
+ automl = AutoML(
361
+ test_size=resolved_test_size,
362
+ cv=resolved_cv,
363
+ objective=resolved_objective,
364
+ variance_threshold=variance_threshold,
365
+ correlation_threshold=correlation_threshold,
366
+ experiment_directory=(
367
+ resolved_experiment_directory
368
+ ),
369
+ )
370
+
371
+ with console.status(
372
+ "[bold green]Running AutoML...[/bold green]"
373
+ ):
374
+ result = automl.fit(
375
+ data=data,
376
+ target=resolved_target,
377
+ task_type=resolved_task_type,
378
+ model_names=selected_models,
379
+ excluded_columns=excluded_columns,
380
+ )
381
+
382
+ model_path = automl.save(
383
+ output,
384
+ overwrite=overwrite,
385
+ )
386
+
387
+ console.print(
388
+ "[bold green]✓ Training completed[/bold green]"
389
+ )
390
+
391
+ console.print()
392
+
393
+ summary_table = Table(
394
+ title="ModelForge Training Summary"
395
+ )
396
+
397
+ summary_table.add_column(
398
+ "Property",
399
+ style="cyan",
400
+ )
401
+
402
+ summary_table.add_column(
403
+ "Value",
404
+ style="green",
405
+ )
406
+
407
+ summary_table.add_row(
408
+ "Target",
409
+ result["target"]["target"],
410
+ )
411
+
412
+ summary_table.add_row(
413
+ "Task Type",
414
+ result["target"]["task_type"],
415
+ )
416
+
417
+ summary_table.add_row(
418
+ "Models Evaluated",
419
+ str(result["models_evaluated"]),
420
+ )
421
+
422
+ summary_table.add_row(
423
+ "Best Model",
424
+ result["best_model"],
425
+ )
426
+
427
+ summary_table.add_row(
428
+ "Ranking Objective",
429
+ resolved_objective,
430
+ )
431
+
432
+ summary_table.add_row(
433
+ "Variance Threshold",
434
+ str(variance_threshold),
435
+ )
436
+
437
+ summary_table.add_row(
438
+ "Correlation Threshold",
439
+ str(correlation_threshold),
440
+ )
441
+
442
+ summary_table.add_row(
443
+ "Run ID",
444
+ _safe_value(
445
+ result.get("run_id"),
446
+ "N/A",
447
+ ),
448
+ )
449
+
450
+ summary_table.add_row(
451
+ "Experiment ID",
452
+ _safe_value(
453
+ result.get("experiment_id"),
454
+ "N/A",
455
+ ),
456
+ )
457
+
458
+ summary_table.add_row(
459
+ "Saved Model",
460
+ model_path,
461
+ )
462
+
463
+ console.print(summary_table)
464
+
465
+ console.print()
466
+
467
+ ranking = result["ranking"]
468
+
469
+ display_columns = [
470
+ column
471
+ for column in [
472
+ "rank",
473
+ "model",
474
+ "overall_score",
475
+ "status",
476
+ ]
477
+ if column in ranking.columns
478
+ ]
479
+
480
+ ranking_table = Table(
481
+ title="Model Ranking"
482
+ )
483
+
484
+ for column in display_columns:
485
+ ranking_table.add_column(
486
+ column.replace(
487
+ "_",
488
+ " ",
489
+ ).title()
490
+ )
491
+
492
+ for _, row in ranking[
493
+ display_columns
494
+ ].head(10).iterrows():
495
+
496
+ values = []
497
+
498
+ for column in display_columns:
499
+ value = row[column]
500
+
501
+ if isinstance(
502
+ value,
503
+ float,
504
+ ):
505
+ values.append(
506
+ f"{value:.4f}"
507
+ )
508
+ else:
509
+ values.append(
510
+ str(value)
511
+ )
512
+
513
+ ranking_table.add_row(
514
+ *values
515
+ )
516
+
517
+ console.print(
518
+ ranking_table
519
+ )
520
+
521
+ except Exception as exc:
522
+ console.print(
523
+ f"[bold red]✗ Training failed:[/bold red] "
524
+ f"{exc}"
525
+ )
526
+
527
+ raise typer.Exit(
528
+ code=1
529
+ )
530
+
531
+
532
+ @app.command()
533
+ def predict(
534
+ model: str = typer.Option(
535
+ ...,
536
+ "--model",
537
+ "-m",
538
+ help="Path to a saved ModelForge model.",
539
+ ),
540
+ data: str = typer.Option(
541
+ ...,
542
+ "--data",
543
+ "-d",
544
+ help="Path to the prediction CSV.",
545
+ ),
546
+ output: str | None = typer.Option(
547
+ None,
548
+ "--output",
549
+ "-o",
550
+ help="Optional CSV output path.",
551
+ ),
552
+ proba: bool = typer.Option(
553
+ False,
554
+ "--proba",
555
+ help=(
556
+ "Generate class probabilities instead of "
557
+ "class predictions. Only supported for "
558
+ "classification models."
559
+ ),
560
+ ),
561
+ ):
562
+ """
563
+ Generate predictions using a saved ModelForge model.
564
+ """
565
+
566
+ try:
567
+ if not Path(model).exists():
568
+ raise FileNotFoundError(
569
+ f"Model not found: {model}"
570
+ )
571
+
572
+ if not Path(data).exists():
573
+ raise FileNotFoundError(
574
+ f"Dataset not found: {data}"
575
+ )
576
+
577
+ persistence = ModelPersistence()
578
+
579
+ pipeline = persistence.load(
580
+ model
581
+ )
582
+
583
+ if proba:
584
+ probabilities = (
585
+ persistence.predict_proba_from_file(
586
+ pipeline,
587
+ data,
588
+ )
589
+ )
590
+
591
+ if output:
592
+ probabilities.to_csv(
593
+ output,
594
+ index=False,
595
+ )
596
+
597
+ console.print(
598
+ "[bold green]✓ Prediction "
599
+ "probabilities saved:[/bold green] "
600
+ f"{output}"
601
+ )
602
+ else:
603
+ console.print()
604
+
605
+ console.print(
606
+ "[bold cyan]Prediction "
607
+ "Probabilities[/bold cyan]"
608
+ )
609
+
610
+ console.print()
611
+
612
+ for index, row in probabilities.iterrows():
613
+ values = ", ".join(
614
+ f"{column}={row[column]:.6f}"
615
+ for column in probabilities.columns
616
+ )
617
+
618
+ console.print(
619
+ f"{index}: {values}"
620
+ )
621
+
622
+ console.print()
623
+
624
+ console.print(
625
+ f"[green]✓ Generated probabilities "
626
+ f"for {len(probabilities)} samples.[/green]"
627
+ )
628
+
629
+ else:
630
+ predictions = (
631
+ persistence.predict_from_file(
632
+ pipeline,
633
+ data,
634
+ )
635
+ )
636
+
637
+ if output:
638
+ predictions.to_frame().to_csv(
639
+ output,
640
+ index=False,
641
+ )
642
+
643
+ console.print(
644
+ "[bold green]✓ Predictions saved:[/bold green] "
645
+ f"{output}"
646
+ )
647
+ else:
648
+ console.print()
649
+
650
+ console.print(
651
+ "[bold cyan]Predictions[/bold cyan]"
652
+ )
653
+
654
+ console.print()
655
+
656
+ for (
657
+ index,
658
+ prediction,
659
+ ) in enumerate(
660
+ predictions.tolist()
661
+ ):
662
+ console.print(
663
+ f"{index}: {prediction}"
664
+ )
665
+
666
+ console.print()
667
+
668
+ console.print(
669
+ f"[green]✓ Generated "
670
+ f"{len(predictions)} predictions.[/green]"
671
+ )
672
+
673
+ except Exception as exc:
674
+ console.print(
675
+ f"[bold red]✗ Prediction failed:[/bold red] "
676
+ f"{exc}"
677
+ )
678
+
679
+ raise typer.Exit(
680
+ code=1
681
+ )
682
+
683
+
684
+ @app.command()
685
+ def explain(
686
+ model: str = typer.Option(
687
+ ...,
688
+ "--model",
689
+ "-m",
690
+ help="Path to a saved ModelForge model.",
691
+ ),
692
+ top: int = typer.Option(
693
+ 10,
694
+ "--top",
695
+ "-n",
696
+ help="Number of important features to display.",
697
+ ),
698
+ ):
699
+ """
700
+ Display feature importance for a saved model.
701
+ """
702
+
703
+ try:
704
+ if not Path(model).exists():
705
+ raise FileNotFoundError(
706
+ f"Model not found: {model}"
707
+ )
708
+
709
+ persistence = ModelPersistence()
710
+
711
+ pipeline = persistence.load(
712
+ model
713
+ )
714
+
715
+ from modelforge.explainability import (
716
+ ExplainabilityEngine,
717
+ )
718
+
719
+ engine = ExplainabilityEngine()
720
+
721
+ importance = (
722
+ engine.feature_importance(
723
+ pipeline
724
+ )
725
+ )
726
+
727
+ top_features = (
728
+ engine.top_features(
729
+ importance,
730
+ n=top,
731
+ )
732
+ )
733
+
734
+ table = Table(
735
+ title="Model Explainability"
736
+ )
737
+
738
+ table.add_column(
739
+ "Rank",
740
+ style="cyan",
741
+ )
742
+
743
+ table.add_column(
744
+ "Feature",
745
+ style="green",
746
+ )
747
+
748
+ table.add_column(
749
+ "Importance",
750
+ style="yellow",
751
+ )
752
+
753
+ table.add_column(
754
+ "Source",
755
+ style="magenta",
756
+ )
757
+
758
+ for (
759
+ rank,
760
+ (_, row),
761
+ ) in enumerate(
762
+ top_features.iterrows(),
763
+ start=1,
764
+ ):
765
+ table.add_row(
766
+ str(rank),
767
+ str(row["feature"]),
768
+ f"{row['importance']:.6f}",
769
+ str(row["source"]),
770
+ )
771
+
772
+ console.print()
773
+
774
+ console.print(table)
775
+
776
+ except Exception as exc:
777
+ console.print(
778
+ f"[bold red]✗ Explanation failed:[/bold red] "
779
+ f"{exc}"
780
+ )
781
+
782
+ raise typer.Exit(
783
+ code=1
784
+ )
785
+
786
+
787
+ @app.command(name="models")
788
+ def list_models(
789
+ task_type: str | None = typer.Option(
790
+ None,
791
+ "--task-type",
792
+ help="Optional: regression or classification.",
793
+ ),
794
+ ):
795
+ """
796
+ List models available in ModelForge.
797
+ """
798
+
799
+ try:
800
+ registry = ModelRegistry()
801
+
802
+ if task_type is not None:
803
+ task_type = (
804
+ task_type
805
+ .lower()
806
+ .strip()
807
+ )
808
+
809
+ if task_type not in {
810
+ "regression",
811
+ "classification",
812
+ }:
813
+ raise ValueError(
814
+ "task_type must be "
815
+ "'regression' or "
816
+ "'classification'."
817
+ )
818
+
819
+ model_names = registry.list_models(
820
+ task_type=task_type
821
+ )
822
+
823
+ models = [
824
+ registry.get(name)
825
+ for name in model_names
826
+ ]
827
+
828
+ table = Table(
829
+ title="ModelForge Model Registry",
830
+ expand=True,
831
+ )
832
+
833
+ table.add_column(
834
+ "Name",
835
+ style="cyan",
836
+ no_wrap=True,
837
+ )
838
+
839
+ table.add_column(
840
+ "Task",
841
+ style="green",
842
+ no_wrap=True,
843
+ )
844
+
845
+ table.add_column(
846
+ "Category",
847
+ style="yellow",
848
+ no_wrap=True,
849
+ )
850
+
851
+ table.add_column(
852
+ "Scaling",
853
+ style="magenta",
854
+ no_wrap=True,
855
+ )
856
+
857
+ table.add_column(
858
+ "Probability",
859
+ style="blue",
860
+ no_wrap=True,
861
+ )
862
+
863
+ for spec in models:
864
+ table.add_row(
865
+ spec.name,
866
+ spec.task_type,
867
+ spec.category,
868
+ (
869
+ "Yes"
870
+ if spec.requires_scaling
871
+ else "No"
872
+ ),
873
+ (
874
+ "Yes"
875
+ if spec.supports_probability
876
+ else "No"
877
+ ),
878
+ )
879
+
880
+ console.print()
881
+
882
+ console.print(table)
883
+
884
+ console.print()
885
+
886
+ console.print(
887
+ "[bold cyan]Available models:[/bold cyan]"
888
+ )
889
+
890
+ for name in model_names:
891
+ console.print(
892
+ f" {name}"
893
+ )
894
+
895
+ console.print()
896
+
897
+ console.print(
898
+ f"[green]Total models: "
899
+ f"{len(models)}[/green]"
900
+ )
901
+
902
+ except Exception as exc:
903
+ console.print(
904
+ f"[bold red]✗ Failed to list models:[/bold red] "
905
+ f"{exc}"
906
+ )
907
+
908
+ raise typer.Exit(
909
+ code=1
910
+ )
911
+
912
+
913
+ @experiments_app.command(name="list")
914
+ def list_experiments(
915
+ directory: str = typer.Option(
916
+ ".modelforge/experiments",
917
+ "--directory",
918
+ "-d",
919
+ help="Experiment storage directory.",
920
+ ),
921
+ ):
922
+ """
923
+ List recorded ModelForge experiments.
924
+ """
925
+
926
+ try:
927
+ automl = AutoML(
928
+ experiment_directory=directory
929
+ )
930
+
931
+ experiments = (
932
+ automl.list_experiments()
933
+ )
934
+
935
+ console.print()
936
+ console.print(
937
+ "[bold cyan]ModelForge Experiments[/bold cyan]"
938
+ )
939
+ console.print()
940
+
941
+ if not experiments:
942
+ console.print(
943
+ "[dim]No experiments found.[/dim]"
944
+ )
945
+ console.print()
946
+
947
+ for experiment in experiments:
948
+ experiment_id = (
949
+ experiment.get(
950
+ "experiment_id",
951
+ "",
952
+ )
953
+ )
954
+
955
+ status = _experiment_status(
956
+ experiment
957
+ )
958
+
959
+ target = _experiment_target(
960
+ experiment
961
+ )
962
+
963
+ task_type = _experiment_task_type(
964
+ experiment
965
+ )
966
+
967
+ best_model = _experiment_best_model(
968
+ experiment
969
+ )
970
+
971
+ run_id = _experiment_run_id(
972
+ experiment
973
+ )
974
+
975
+ console.print(
976
+ f"Experiment ID: "
977
+ f"{_safe_value(experiment_id, 'N/A')}"
978
+ )
979
+
980
+ console.print(
981
+ f" Status: "
982
+ f"{_safe_value(status, 'N/A')}"
983
+ )
984
+
985
+ console.print(
986
+ f" Target: "
987
+ f"{_safe_value(target, 'N/A')}"
988
+ )
989
+
990
+ console.print(
991
+ f" Task: "
992
+ f"{_safe_value(task_type, 'N/A')}"
993
+ )
994
+
995
+ console.print(
996
+ f" Best Model: "
997
+ f"{_safe_value(best_model, 'N/A')}"
998
+ )
999
+
1000
+ console.print(
1001
+ f" Run ID: "
1002
+ f"{_safe_value(run_id, 'N/A')}"
1003
+ )
1004
+
1005
+ console.print()
1006
+
1007
+ console.print(
1008
+ f"[green]Total experiments: "
1009
+ f"{len(experiments)}[/green]"
1010
+ )
1011
+
1012
+ except Exception as exc:
1013
+ console.print(
1014
+ f"[bold red]✗ Failed to list "
1015
+ f"experiments:[/bold red] "
1016
+ f"{exc}"
1017
+ )
1018
+
1019
+ raise typer.Exit(
1020
+ code=1
1021
+ )
1022
+
1023
+
1024
+ @experiments_app.command(name="get")
1025
+ def get_experiment(
1026
+ experiment_id: str = typer.Argument(
1027
+ ...,
1028
+ help="Experiment ID to inspect.",
1029
+ ),
1030
+ directory: str = typer.Option(
1031
+ ".modelforge/experiments",
1032
+ "--directory",
1033
+ "-d",
1034
+ help="Experiment storage directory.",
1035
+ ),
1036
+ ):
1037
+ """
1038
+ Display details for a ModelForge experiment.
1039
+ """
1040
+
1041
+ try:
1042
+ automl = AutoML(
1043
+ experiment_directory=directory
1044
+ )
1045
+
1046
+ experiment = automl.get_experiment(
1047
+ experiment_id
1048
+ )
1049
+
1050
+ console.print()
1051
+
1052
+ console.print(
1053
+ f"[bold cyan]Experiment:[/bold cyan] "
1054
+ f"{experiment_id}"
1055
+ )
1056
+
1057
+ console.print()
1058
+
1059
+ summary_table = Table(
1060
+ title="Experiment Summary"
1061
+ )
1062
+
1063
+ summary_table.add_column(
1064
+ "Property",
1065
+ style="cyan",
1066
+ )
1067
+
1068
+ summary_table.add_column(
1069
+ "Value",
1070
+ style="green",
1071
+ )
1072
+
1073
+ summary_table.add_row(
1074
+ "Experiment ID",
1075
+ _safe_value(
1076
+ experiment.get(
1077
+ "experiment_id"
1078
+ ),
1079
+ experiment_id,
1080
+ ),
1081
+ )
1082
+
1083
+ summary_table.add_row(
1084
+ "Status",
1085
+ _safe_value(
1086
+ _experiment_status(
1087
+ experiment
1088
+ ),
1089
+ "N/A",
1090
+ ),
1091
+ )
1092
+
1093
+ summary_table.add_row(
1094
+ "Target",
1095
+ _safe_value(
1096
+ _experiment_target(
1097
+ experiment
1098
+ ),
1099
+ "N/A",
1100
+ ),
1101
+ )
1102
+
1103
+ summary_table.add_row(
1104
+ "Task Type",
1105
+ _safe_value(
1106
+ _experiment_task_type(
1107
+ experiment
1108
+ ),
1109
+ "N/A",
1110
+ ),
1111
+ )
1112
+
1113
+ summary_table.add_row(
1114
+ "Best Model",
1115
+ _safe_value(
1116
+ _experiment_best_model(
1117
+ experiment
1118
+ ),
1119
+ "N/A",
1120
+ ),
1121
+ )
1122
+
1123
+ summary_table.add_row(
1124
+ "Run ID",
1125
+ _safe_value(
1126
+ _experiment_run_id(
1127
+ experiment
1128
+ ),
1129
+ "N/A",
1130
+ ),
1131
+ )
1132
+
1133
+ if "configuration" in experiment:
1134
+ summary_table.add_row(
1135
+ "Configuration",
1136
+ str(
1137
+ experiment["configuration"]
1138
+ ),
1139
+ )
1140
+
1141
+ console.print(
1142
+ summary_table
1143
+ )
1144
+
1145
+ console.print()
1146
+
1147
+ console.print(
1148
+ "[bold cyan]Experiment Data[/bold cyan]"
1149
+ )
1150
+
1151
+ console.print()
1152
+
1153
+ details = Table(
1154
+ show_header=True,
1155
+ header_style="bold cyan",
1156
+ )
1157
+
1158
+ details.add_column(
1159
+ "Key"
1160
+ )
1161
+
1162
+ details.add_column(
1163
+ "Value"
1164
+ )
1165
+
1166
+ for key, value in experiment.items():
1167
+ if key in {
1168
+ "experiment_id",
1169
+ "status",
1170
+ "target",
1171
+ "task_type",
1172
+ "best_model",
1173
+ "run_id",
1174
+ "configuration",
1175
+ }:
1176
+ continue
1177
+
1178
+ details.add_row(
1179
+ str(key),
1180
+ str(value),
1181
+ )
1182
+
1183
+ if details.row_count > 0:
1184
+ console.print(details)
1185
+
1186
+ except Exception as exc:
1187
+ console.print(
1188
+ f"[bold red]✗ Failed to get "
1189
+ f"experiment:[/bold red] "
1190
+ f"{exc}"
1191
+ )
1192
+
1193
+ raise typer.Exit(
1194
+ code=1
1195
+ )
1196
+
1197
+
1198
+ @experiments_app.command(name="compare")
1199
+ def compare_experiments(
1200
+ experiment_ids: list[str] = typer.Argument(
1201
+ ...,
1202
+ help="Two or more experiment IDs to compare.",
1203
+ ),
1204
+ directory: str = typer.Option(
1205
+ ".modelforge/experiments",
1206
+ "--directory",
1207
+ "-d",
1208
+ help="Experiment storage directory.",
1209
+ ),
1210
+ ):
1211
+ """Compare the key outcomes and configuration of experiments."""
1212
+
1213
+ if len(experiment_ids) < 2:
1214
+ console.print(
1215
+ "[bold red]✗ Provide at least two experiment IDs.[/bold red]"
1216
+ )
1217
+ raise typer.Exit(code=1)
1218
+
1219
+ try:
1220
+ automl = AutoML(experiment_directory=directory)
1221
+ table = Table(title="ModelForge Experiment Comparison")
1222
+ table.add_column("Experiment", style="cyan")
1223
+ table.add_column("Status", style="green")
1224
+ table.add_column("Task", style="yellow")
1225
+ table.add_column("Target", style="magenta")
1226
+ table.add_column("Best Model", style="blue")
1227
+ table.add_column("Run ID", style="dim")
1228
+
1229
+ for experiment_id in experiment_ids:
1230
+ experiment = automl.get_experiment(experiment_id)
1231
+ table.add_row(
1232
+ experiment_id,
1233
+ _safe_value(_experiment_status(experiment), "N/A"),
1234
+ _safe_value(_experiment_task_type(experiment), "N/A"),
1235
+ _safe_value(_experiment_target(experiment), "N/A"),
1236
+ _safe_value(_experiment_best_model(experiment), "N/A"),
1237
+ _safe_value(_experiment_run_id(experiment), "N/A"),
1238
+ )
1239
+
1240
+ console.print(table)
1241
+
1242
+ except Exception as exc:
1243
+ console.print(
1244
+ f"[bold red]✗ Failed to compare experiments:[/bold red] {exc}"
1245
+ )
1246
+ raise typer.Exit(code=1)
1247
+
1248
+
1249
+ @app.callback()
1250
+ def main():
1251
+ """
1252
+ ModelForge CLI entry point.
1253
+ """
1254
+ pass
1255
+
1256
+
1257
+ if __name__ == "__main__":
1258
+ app()