algorithm-genome 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- algorithm_genome/__init__.py +13 -0
- algorithm_genome/__main__.py +3 -0
- algorithm_genome/algorithm_library.py +60 -0
- algorithm_genome/assets/default_recommender.joblib +0 -0
- algorithm_genome/assets/model_info.json +28 -0
- algorithm_genome/benchmark_datasets.py +130 -0
- algorithm_genome/bundled_model.py +27 -0
- algorithm_genome/command_line.py +116 -0
- algorithm_genome/create_example_datasets.py +35 -0
- algorithm_genome/dataset_genome.py +110 -0
- algorithm_genome/download_public_datasets.py +251 -0
- algorithm_genome/framework.py +101 -0
- algorithm_genome/improve_meta_model.py +186 -0
- algorithm_genome/model_pipeline.py +68 -0
- algorithm_genome/prepare_additional_datasets.py +105 -0
- algorithm_genome/recommend_algorithms.py +177 -0
- algorithm_genome/terminal_output.py +270 -0
- algorithm_genome/train_meta_model.py +142 -0
- algorithm_genome/train_neural_recommender.py +402 -0
- algorithm_genome-0.3.0.dist-info/METADATA +223 -0
- algorithm_genome-0.3.0.dist-info/RECORD +24 -0
- algorithm_genome-0.3.0.dist-info/WHEEL +5 -0
- algorithm_genome-0.3.0.dist-info/entry_points.txt +2 -0
- algorithm_genome-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Reserve new source groups for final testing before benchmarking them."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import joblib
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
from .benchmark_datasets import dataset_fingerprint, write_json
|
|
14
|
+
from .dataset_genome import prepare_classification_data
|
|
15
|
+
from .download_public_datasets import SOURCE_GROUP_OVERRIDES, UNSUPPORTED_SOURCES
|
|
16
|
+
from .train_meta_model import check_compatibility
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def prepare_additional_datasets(manifest_path, historical_benchmark_path, output_directory,
|
|
20
|
+
previous_model_path=None):
|
|
21
|
+
manifest_path = Path(manifest_path).resolve()
|
|
22
|
+
history = json.loads(Path(historical_benchmark_path).read_text())
|
|
23
|
+
check_compatibility(history)
|
|
24
|
+
historical = {d["id"]: d for d in history["datasets"]}
|
|
25
|
+
known_groups = {d["group"] for d in historical.values()}
|
|
26
|
+
if previous_model_path:
|
|
27
|
+
previous = joblib.load(previous_model_path)
|
|
28
|
+
check_compatibility(previous)
|
|
29
|
+
known_groups.update(previous["training_groups"])
|
|
30
|
+
entries = json.loads(manifest_path.read_text())
|
|
31
|
+
if len({e["id"] for e in entries}) != len(entries):
|
|
32
|
+
raise ValueError("Duplicate dataset IDs in manifest")
|
|
33
|
+
if not set(historical) <= {e["id"] for e in entries}:
|
|
34
|
+
raise ValueError("Expanded manifest must retain the historical datasets")
|
|
35
|
+
prepared, skipped = [], []
|
|
36
|
+
# Historical IDs go first, so an additional copy cannot replace a baseline.
|
|
37
|
+
entries.sort(key=lambda e: e["id"] not in historical)
|
|
38
|
+
seen = {}
|
|
39
|
+
for entry in entries:
|
|
40
|
+
source_id = entry.get("source", {}).get("openml_id")
|
|
41
|
+
if source_id in UNSUPPORTED_SOURCES:
|
|
42
|
+
skipped.append({"dataset": entry["id"], "reason": "unsupported_validation",
|
|
43
|
+
"details": UNSUPPORTED_SOURCES[source_id]})
|
|
44
|
+
continue
|
|
45
|
+
path = (manifest_path.parent / entry["path"]).resolve()
|
|
46
|
+
checksum = entry.get("source", {}).get("sha256")
|
|
47
|
+
if checksum and hashlib.sha256(path.read_bytes()).hexdigest() != checksum:
|
|
48
|
+
raise ValueError(f"Checksum mismatch: {entry['id']}")
|
|
49
|
+
X, y = prepare_classification_data(pd.read_csv(path), entry["target"])
|
|
50
|
+
fingerprint = dataset_fingerprint(X, y)
|
|
51
|
+
group = SOURCE_GROUP_OVERRIDES.get(source_id, entry.get("group", entry["id"]))
|
|
52
|
+
if entry["id"] in historical:
|
|
53
|
+
previous = historical[entry["id"]]
|
|
54
|
+
if previous["fingerprint"] != fingerprint or previous["group"] != group:
|
|
55
|
+
raise ValueError(f"Historical content or source group changed: {entry['id']}")
|
|
56
|
+
if fingerprint in seen:
|
|
57
|
+
skipped.append({"dataset": entry["id"], "reason": "duplicate_content", "same_as": seen[fingerprint]})
|
|
58
|
+
continue
|
|
59
|
+
seen[fingerprint] = entry["id"]
|
|
60
|
+
prepared.append({**entry, "group": group, "path": str(path)})
|
|
61
|
+
new_groups = sorted({e["group"] for e in prepared} - known_groups)
|
|
62
|
+
if len(new_groups) < 4:
|
|
63
|
+
raise ValueError("Need at least four new source groups to reserve fresh tests; download more datasets first")
|
|
64
|
+
output = Path(output_directory)
|
|
65
|
+
prior_split_path = output / "dataset_split.json"
|
|
66
|
+
if prior_split_path.exists():
|
|
67
|
+
# An existing final cohort must not be redrawn when more data arrives.
|
|
68
|
+
prior = json.loads(prior_split_path.read_text())
|
|
69
|
+
fresh_groups = set(prior["fresh_groups"])
|
|
70
|
+
if not fresh_groups or not fresh_groups <= set(new_groups):
|
|
71
|
+
raise ValueError("Previously reserved fresh groups are missing or no longer independent")
|
|
72
|
+
else:
|
|
73
|
+
shuffled = np.random.default_rng(20261115).permutation(new_groups)
|
|
74
|
+
fresh_groups = set(shuffled[:math.ceil(len(new_groups) * 0.25)])
|
|
75
|
+
training = [e for e in prepared if e["group"] not in fresh_groups]
|
|
76
|
+
fresh = [e for e in prepared if e["group"] in fresh_groups]
|
|
77
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
write_json(output / "training_manifest.json", training)
|
|
79
|
+
write_json(output / "fresh_manifest.json", fresh)
|
|
80
|
+
write_json(output / "dataset_split.json", {
|
|
81
|
+
"seed": 20261115, "known_groups": sorted(known_groups), "new_groups": new_groups,
|
|
82
|
+
"fresh_groups": sorted(fresh_groups), "training_datasets": len(training),
|
|
83
|
+
"fresh_datasets": len(fresh), "skipped": skipped,
|
|
84
|
+
"notes": ["Fresh groups reserved before benchmarking or inspecting algorithm scores.",
|
|
85
|
+
"All related variants stay on the same side of the split.",
|
|
86
|
+
"Source-group labels need semantic review; exact duplicate detection is not sufficient."]})
|
|
87
|
+
print(f"Prepared {len(training)} training datasets and {len(fresh)} fresh test datasets.")
|
|
88
|
+
return training, fresh
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main():
|
|
92
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
93
|
+
parser.add_argument("manifest")
|
|
94
|
+
parser.add_argument("--historical-benchmark", default="results/public_benchmark_results.json")
|
|
95
|
+
parser.add_argument("--previous-model", default="results/meta_model/trained_meta_model.joblib")
|
|
96
|
+
parser.add_argument("--output", default="results/additional_dataset_split")
|
|
97
|
+
args = parser.parse_args()
|
|
98
|
+
try:
|
|
99
|
+
prepare_additional_datasets(args.manifest, args.historical_benchmark, args.output, args.previous_model)
|
|
100
|
+
except (ValueError, OSError, KeyError) as exc:
|
|
101
|
+
parser.exit(1, f"Error: {exc}\n")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
main()
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Rank candidates without fitting them; optionally verify and save a predictor."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import time
|
|
5
|
+
import warnings
|
|
6
|
+
from os import PathLike
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import joblib
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import sklearn
|
|
13
|
+
from sklearn.metrics import (accuracy_score, balanced_accuracy_score, confusion_matrix,
|
|
14
|
+
f1_score, precision_recall_fscore_support)
|
|
15
|
+
from sklearn.model_selection import train_test_split
|
|
16
|
+
|
|
17
|
+
from .algorithm_library import algorithm_library, meta_features
|
|
18
|
+
from .dataset_genome import (extract_dataset_genome, normalize_features, prepare_classification_data,
|
|
19
|
+
summarize_classification_data)
|
|
20
|
+
from .model_pipeline import build_model_pipeline, evaluate_candidate
|
|
21
|
+
from .train_meta_model import check_compatibility
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _recommend_from_training(artifact, X, y):
|
|
25
|
+
check_compatibility(artifact)
|
|
26
|
+
if artifact["sklearn_version"] != sklearn.__version__:
|
|
27
|
+
raise ValueError("Use the same scikit-learn version as meta-model training")
|
|
28
|
+
genome = extract_dataset_genome(X, y)
|
|
29
|
+
if sorted(genome) != artifact["genome_schema"]:
|
|
30
|
+
raise ValueError("Dataset genome schema differs from the trained model")
|
|
31
|
+
names = list(algorithm_library())
|
|
32
|
+
features = [meta_features(genome, name) for name in names]
|
|
33
|
+
predictions = artifact["performance_model"].predict(features)
|
|
34
|
+
runtimes = np.expm1(artifact["runtime_model"].predict(features))
|
|
35
|
+
ranking = sorted([
|
|
36
|
+
{"algorithm": name, "predicted_relative_macro_f1": float(score),
|
|
37
|
+
"estimated_cv_seconds": float(runtime)}
|
|
38
|
+
for name, score, runtime in zip(names, predictions, runtimes)
|
|
39
|
+
], key=lambda row: (-row["predicted_relative_macro_f1"], row["algorithm"]))
|
|
40
|
+
outside = [key for key, (low, high) in artifact["feature_ranges"].items() if not low <= genome[key] <= high]
|
|
41
|
+
training_warnings = artifact.get("evaluation", {}).get("production_convergence_warnings", [])
|
|
42
|
+
return {
|
|
43
|
+
"dataset_genome": genome, "ranking": ranking, "features_outside_training_range": outside,
|
|
44
|
+
"meta_model_training_warnings": training_warnings,
|
|
45
|
+
"notes": ["Predictions are relative macro-F1 advantages, not accuracy or confidence probabilities.",
|
|
46
|
+
"Runtime estimates describe three-fold CV on benchmark hardware; they are not time guarantees.",
|
|
47
|
+
"Feature-range checks are a simple extrapolation warning, not calibrated uncertainty.",
|
|
48
|
+
"Only labeled independent-row classification tables are supported."] +
|
|
49
|
+
(["The recommendation model's optimizer reported training warnings; treat its ranking as experimental."]
|
|
50
|
+
if training_warnings else []),
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def recommend_algorithms(artifact, data, target):
|
|
55
|
+
X, y = prepare_classification_data(data, target)
|
|
56
|
+
return {**_recommend_from_training(artifact, X, y),
|
|
57
|
+
"data_summary": summarize_classification_data(data, target, X, y)}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class TrainedClassifier:
|
|
61
|
+
"""Saved final pipeline with original feature names and normalization."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, pipeline, features):
|
|
64
|
+
self.pipeline = pipeline
|
|
65
|
+
self.features = list(features)
|
|
66
|
+
|
|
67
|
+
def save(self, path: str | PathLike) -> Path:
|
|
68
|
+
"""Save this predictor to a joblib file, creating parent directories."""
|
|
69
|
+
path = Path(path)
|
|
70
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
joblib.dump(self, path)
|
|
72
|
+
return path
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def load(cls, path: str | PathLike) -> "TrainedClassifier":
|
|
76
|
+
"""Load a trusted joblib predictor, including existing project artifacts."""
|
|
77
|
+
model = joblib.load(path)
|
|
78
|
+
if not isinstance(model, cls):
|
|
79
|
+
raise ValueError("Expected a saved TrainedClassifier")
|
|
80
|
+
return model
|
|
81
|
+
|
|
82
|
+
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
|
83
|
+
"""Predict string labels; extra columns are ignored and missing ones fail."""
|
|
84
|
+
if not isinstance(data, pd.DataFrame) or not data.columns.is_unique:
|
|
85
|
+
raise ValueError("Provide a DataFrame with unique column names")
|
|
86
|
+
missing = set(self.features) - set(data.columns)
|
|
87
|
+
if missing:
|
|
88
|
+
raise ValueError(f"Missing prediction columns: {sorted(missing)}")
|
|
89
|
+
return self.pipeline.predict(normalize_features(data[self.features]))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def verify_recommendations(artifact, data, target, top_k=3, budget_seconds=300, progress=None):
|
|
93
|
+
if not 1 <= top_k <= len(algorithm_library()):
|
|
94
|
+
raise ValueError("top_k must be between 1 and 10")
|
|
95
|
+
if not math.isfinite(budget_seconds) or budget_seconds <= 0:
|
|
96
|
+
raise ValueError("budget_seconds must be positive and finite")
|
|
97
|
+
started = time.monotonic()
|
|
98
|
+
def notify(event, **details):
|
|
99
|
+
if progress is not None:
|
|
100
|
+
progress({"event": event, **details})
|
|
101
|
+
|
|
102
|
+
X, y = prepare_classification_data(data, target)
|
|
103
|
+
summary = summarize_classification_data(data, target, X, y)
|
|
104
|
+
test_size = max(math.ceil(len(X) * 0.2), y.nunique())
|
|
105
|
+
train_X, test_X, train_y, test_y = train_test_split(
|
|
106
|
+
X, y, test_size=test_size, stratify=y, random_state=42)
|
|
107
|
+
notify("dataset_ready", summary=summary, train_rows=len(train_X), holdout_rows=len(test_X))
|
|
108
|
+
# Rank using only training rows, including all target-dependent genome features.
|
|
109
|
+
notify("ranking_start")
|
|
110
|
+
ranking_started = time.monotonic()
|
|
111
|
+
report = _recommend_from_training(artifact, train_X, train_y)
|
|
112
|
+
ranking_seconds = time.monotonic() - ranking_started
|
|
113
|
+
report["data_summary"] = summary
|
|
114
|
+
notify("ranking_ready", report=report)
|
|
115
|
+
ranking = report["ranking"]
|
|
116
|
+
selected = [row["algorithm"] for row in ranking[:top_k]]
|
|
117
|
+
# Prioritize one alternative family so a bad prior shortlist is not a hard wall.
|
|
118
|
+
library = algorithm_library()
|
|
119
|
+
represented = {library[name]["genome"]["structure"] for name in selected}
|
|
120
|
+
remaining = [row["algorithm"] for row in ranking[top_k:]]
|
|
121
|
+
alternatives = [n for n in remaining if library[n]["genome"]["structure"] not in represented]
|
|
122
|
+
challenger = (alternatives or remaining or [None])[0]
|
|
123
|
+
order = selected + ([challenger] if challenger else [])
|
|
124
|
+
order += [n for n in remaining if n != challenger]
|
|
125
|
+
results = []
|
|
126
|
+
notify("verification_start", budget_seconds=budget_seconds)
|
|
127
|
+
for index, name in enumerate(order, 1):
|
|
128
|
+
if results and time.monotonic() - started >= budget_seconds:
|
|
129
|
+
results.append({"algorithm": name, "status": "skipped_budget"})
|
|
130
|
+
notify("candidate_skipped", algorithm=name, index=index, total=len(order))
|
|
131
|
+
continue
|
|
132
|
+
notify("candidate_start", algorithm=name, index=index, total=len(order),
|
|
133
|
+
role="shortlist" if name in selected else "challenger" if name == challenger else "remaining")
|
|
134
|
+
candidate_started = time.monotonic()
|
|
135
|
+
captured = []
|
|
136
|
+
try:
|
|
137
|
+
with warnings.catch_warnings(record=True) as captured:
|
|
138
|
+
warnings.simplefilter("always")
|
|
139
|
+
result = {"algorithm": name, "status": "ok", **evaluate_candidate(name, train_X, train_y)}
|
|
140
|
+
except Exception as exc:
|
|
141
|
+
result = {"algorithm": name, "status": "failed", "error": str(exc)}
|
|
142
|
+
result["cv_seconds"] = time.monotonic() - candidate_started
|
|
143
|
+
result["warnings"] = list(dict.fromkeys(f"{warning.category.__name__}: {warning.message}" for warning in captured))
|
|
144
|
+
results.append(result)
|
|
145
|
+
notify("candidate_complete", result=result, index=index, total=len(order))
|
|
146
|
+
successful = [r for r in results if r["status"] == "ok"]
|
|
147
|
+
if not successful:
|
|
148
|
+
raise RuntimeError(f"No candidate completed verification successfully: {results}")
|
|
149
|
+
winner = max(successful, key=lambda r: r["cv_macro_f1"])
|
|
150
|
+
notify("holdout_start", algorithm=winner["algorithm"])
|
|
151
|
+
holdout_started = time.monotonic()
|
|
152
|
+
fitted = build_model_pipeline(winner["algorithm"]).fit(train_X, train_y)
|
|
153
|
+
predicted = fitted.predict(test_X)
|
|
154
|
+
labels = sorted(train_y.unique())
|
|
155
|
+
precision, recall, f1, support = precision_recall_fscore_support(test_y, predicted, labels=labels, zero_division=0)
|
|
156
|
+
report.update({"selected_algorithm": winner["algorithm"], "verification_results": results,
|
|
157
|
+
"holdout_metrics": {"macro_f1": float(f1_score(test_y, predicted, average="macro", zero_division=0)),
|
|
158
|
+
"accuracy": float(accuracy_score(test_y, predicted)),
|
|
159
|
+
"balanced_accuracy": float(balanced_accuracy_score(test_y, predicted))},
|
|
160
|
+
"train_rows": len(train_X), "holdout_rows": len(test_X), "soft_budget_seconds": budget_seconds})
|
|
161
|
+
report["holdout_details"] = {"labels": labels,
|
|
162
|
+
"confusion_matrix": confusion_matrix(test_y, predicted, labels=labels).tolist(),
|
|
163
|
+
"per_class": [{"label": label, "precision": float(p), "recall": float(r), "f1": float(f), "support": int(n)}
|
|
164
|
+
for label, p, r, f, n in zip(labels, precision, recall, f1, support)]}
|
|
165
|
+
holdout_seconds = time.monotonic() - holdout_started
|
|
166
|
+
notify("refit_start", algorithm=winner["algorithm"], rows=len(X))
|
|
167
|
+
refit_started = time.monotonic()
|
|
168
|
+
final = TrainedClassifier(build_model_pipeline(winner["algorithm"]).fit(X, y), X.columns)
|
|
169
|
+
report["timings"] = {"initial_ranking_seconds": ranking_seconds, "holdout_evaluation_seconds": holdout_seconds,
|
|
170
|
+
"final_refit_seconds": time.monotonic() - refit_started}
|
|
171
|
+
report["selected_configuration"] = library[winner["algorithm"]]["estimator"].get_params()
|
|
172
|
+
report["total_seconds"] = time.monotonic() - started
|
|
173
|
+
report["notes"].extend([
|
|
174
|
+
"Verification starts with the shortlist, then a different-family challenger, then remaining candidates if time permits.",
|
|
175
|
+
"Budget is checked between candidates; the first CV run, active CV, final evaluation and refit always finish.",
|
|
176
|
+
"Holdout metrics describe the pre-refit winner; the saved predictor is refitted on all usable rows."])
|
|
177
|
+
return report, final
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Readable terminal tables and live progress without extra dependencies."""
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import platform
|
|
7
|
+
import re
|
|
8
|
+
import shutil
|
|
9
|
+
import sys
|
|
10
|
+
import textwrap
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def clean_text(value):
|
|
14
|
+
text = re.sub(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1b\\))", "", str(value))
|
|
15
|
+
return "".join(char if char.isprintable() else " " for char in text)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def algorithm_name(name):
|
|
19
|
+
return name.replace("_", " ").title()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TerminalReporter:
|
|
23
|
+
def __init__(self, verbose=False, stream=None, color=None):
|
|
24
|
+
self.verbose = verbose
|
|
25
|
+
self.stream = stream if stream is not None else sys.stdout
|
|
26
|
+
self.width = max(70, min(115, shutil.get_terminal_size((100, 24)).columns))
|
|
27
|
+
self.color = (bool(getattr(self.stream, "isatty", lambda: False)())
|
|
28
|
+
and "NO_COLOR" not in os.environ and os.environ.get("TERM") != "dumb") if color is None else color
|
|
29
|
+
self.lines = []
|
|
30
|
+
self.ranking_printed = False
|
|
31
|
+
self.warnings_printed = set()
|
|
32
|
+
|
|
33
|
+
def line(self, text="", style=None):
|
|
34
|
+
text = clean_text(text)
|
|
35
|
+
self.lines.append(text)
|
|
36
|
+
colors = {"title": "1;36", "ok": "1;32", "warning": "1;33", "error": "1;31"}
|
|
37
|
+
visible = f"\033[{colors[style]}m{text}\033[0m" if self.color and style in colors else text
|
|
38
|
+
print(visible, file=self.stream, flush=True)
|
|
39
|
+
|
|
40
|
+
def paragraph(self, text, indent=""):
|
|
41
|
+
for line in textwrap.wrap(clean_text(text), width=self.width, initial_indent=indent, subsequent_indent=indent) or [""]:
|
|
42
|
+
self.line(line)
|
|
43
|
+
|
|
44
|
+
def heading(self, text):
|
|
45
|
+
self.line()
|
|
46
|
+
self.line(text, "title")
|
|
47
|
+
self.line("-" * min(self.width, len(text)))
|
|
48
|
+
|
|
49
|
+
def values(self, pairs):
|
|
50
|
+
for label, value in pairs:
|
|
51
|
+
self.paragraph(f"{label}: {value}", " ")
|
|
52
|
+
|
|
53
|
+
def table(self, headers, rows):
|
|
54
|
+
rows = [[clean_text(cell) for cell in row] for row in rows]
|
|
55
|
+
headers = list(map(clean_text, headers))
|
|
56
|
+
widths = [max(len(headers[i]), *(len(row[i]) for row in rows)) if rows else len(headers[i])
|
|
57
|
+
for i in range(len(headers))]
|
|
58
|
+
minimum = [min(width, max(5, min(len(header), 10))) for width, header in zip(widths, headers)]
|
|
59
|
+
while sum(widths) + 3 * len(widths) + 1 > self.width:
|
|
60
|
+
candidates = [i for i in range(len(widths)) if widths[i] > minimum[i]]
|
|
61
|
+
if not candidates:
|
|
62
|
+
break
|
|
63
|
+
index = max(candidates, key=lambda i: widths[i] - minimum[i])
|
|
64
|
+
widths[index] -= 1
|
|
65
|
+
|
|
66
|
+
def render(row):
|
|
67
|
+
return "| " + " | ".join((cell if len(cell) <= width else cell[:width - 3] + "...").ljust(width)
|
|
68
|
+
for cell, width in zip(row, widths)) + " |"
|
|
69
|
+
|
|
70
|
+
border = "+-" + "-+-".join("-" * width for width in widths) + "-+"
|
|
71
|
+
self.line(border)
|
|
72
|
+
self.line(render(headers))
|
|
73
|
+
self.line(border)
|
|
74
|
+
for row in rows:
|
|
75
|
+
self.line(render(row))
|
|
76
|
+
self.line(border)
|
|
77
|
+
|
|
78
|
+
def warning(self, message):
|
|
79
|
+
if message not in self.warnings_printed:
|
|
80
|
+
self.warnings_printed.add(message)
|
|
81
|
+
self.line("WARNING", "warning")
|
|
82
|
+
self.paragraph(message, " ")
|
|
83
|
+
|
|
84
|
+
def model(self, artifact, path):
|
|
85
|
+
self.heading("SAVED RECOMMENDER")
|
|
86
|
+
model = artifact.get("performance_model")
|
|
87
|
+
self.values([("File", path), ("Model", type(model).__name__),
|
|
88
|
+
("Training datasets", len(artifact.get("training_datasets", []))),
|
|
89
|
+
("Training source groups", len(artifact.get("training_groups", [])))])
|
|
90
|
+
self.paragraph("The saved recommender ranks algorithms. Candidate classifiers are fitted only when verification is requested.", " ")
|
|
91
|
+
if self.verbose:
|
|
92
|
+
self.values([("Python", platform.python_version()), ("scikit-learn used for training", artifact.get("sklearn_version", "unknown")),
|
|
93
|
+
("Pipeline version", artifact.get("pipeline_version", "unknown")),
|
|
94
|
+
("Genome inputs", len(artifact.get("genome_schema", []))),
|
|
95
|
+
("Library signature", artifact.get("library_signature", "unknown"))])
|
|
96
|
+
if hasattr(model, "alpha"):
|
|
97
|
+
self.values([("Neural regularization alpha", model.alpha)])
|
|
98
|
+
pipeline = getattr(model, "model", None)
|
|
99
|
+
if hasattr(pipeline, "named_steps"):
|
|
100
|
+
for step in pipeline.named_steps.values():
|
|
101
|
+
if hasattr(step, "hidden_layer_sizes"):
|
|
102
|
+
self.values([("Hidden layer neurons", step.hidden_layer_sizes), ("Activation", step.activation), ("Optimizer", step.solver)])
|
|
103
|
+
for message in artifact.get("evaluation", {}).get("production_convergence_warnings", []):
|
|
104
|
+
self.warning("Recommender training reported an optimizer warning; this model remains experimental.")
|
|
105
|
+
if self.verbose and message not in self.warnings_printed:
|
|
106
|
+
self.paragraph(message, " ")
|
|
107
|
+
self.warnings_printed.add(message)
|
|
108
|
+
|
|
109
|
+
def dataset(self, summary, train_rows=None, holdout_rows=None):
|
|
110
|
+
self.heading("DATASET CHECK")
|
|
111
|
+
self.values([("Target column", summary["target"]), ("Input rows", summary["input_rows"]),
|
|
112
|
+
("Rows without a target removed", summary["missing_target_rows"]),
|
|
113
|
+
("Repeated labeled rows removed", summary["duplicate_rows"]),
|
|
114
|
+
("Usable labeled rows", summary["usable_rows"]),
|
|
115
|
+
("Input columns", f"{summary['n_features']} ({summary['n_numeric']} numeric, {summary['n_categorical']} categorical)"),
|
|
116
|
+
("Missing predictor values", f"{summary['missing_values']} ({100 * summary['missing_fraction']:.2f}%)"),
|
|
117
|
+
("Classes", len(summary["class_counts"]))])
|
|
118
|
+
if train_rows is not None:
|
|
119
|
+
self.values([("Training rows for profiling and CV", train_rows), ("Reserved final holdout rows", holdout_rows)])
|
|
120
|
+
counts = summary["class_counts"]
|
|
121
|
+
items = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
|
|
122
|
+
shown = items if self.verbose else items[:8]
|
|
123
|
+
self.table(["Class", "Rows", "Share"], [[repr(label), count, f"{100 * count / summary['usable_rows']:.1f}%"] for label, count in shown])
|
|
124
|
+
if len(shown) < len(items):
|
|
125
|
+
self.paragraph(f"Showing {len(shown)} of {len(items)} classes. Use --verbose for all classes.")
|
|
126
|
+
|
|
127
|
+
def ranking(self, report):
|
|
128
|
+
self.heading("PREDICTED ALGORITHM RANKING")
|
|
129
|
+
self.table(["Rank", "Algorithm", "Advantage (F1 pts)", "Est. CV (s)"], [
|
|
130
|
+
[index, algorithm_name(row["algorithm"]), f"{100 * row['predicted_relative_macro_f1']:+.2f}", f"{row['estimated_cv_seconds']:.2f}"]
|
|
131
|
+
for index, row in enumerate(report["ranking"], 1)])
|
|
132
|
+
self.paragraph("Advantage is predicted relative macro-F1 above the candidate average, not accuracy or a confidence probability. Runtime estimates are approximate.")
|
|
133
|
+
outside = report.get("features_outside_training_range", [])
|
|
134
|
+
if outside:
|
|
135
|
+
self.warning("Dataset properties outside the recommender's training range: " + ", ".join(outside))
|
|
136
|
+
if self.verbose:
|
|
137
|
+
self.heading("DATASET GENOME USED FOR RANKING")
|
|
138
|
+
self.table(["Property", "Value"], [[name, f"{value:.6g}"] for name, value in report["dataset_genome"].items()])
|
|
139
|
+
self.ranking_printed = True
|
|
140
|
+
|
|
141
|
+
def progress(self, event):
|
|
142
|
+
kind = event["event"]
|
|
143
|
+
if kind == "dataset_ready":
|
|
144
|
+
self.dataset(event["summary"], event["train_rows"], event["holdout_rows"])
|
|
145
|
+
elif kind == "ranking_start":
|
|
146
|
+
self.line("Extracting training-data genome and predicting candidate scores...", "title")
|
|
147
|
+
elif kind == "ranking_ready":
|
|
148
|
+
self.ranking(event["report"])
|
|
149
|
+
elif kind == "verification_start":
|
|
150
|
+
self.heading("LIVE VERIFICATION")
|
|
151
|
+
self.paragraph(f"Test shortlist, then a different-family challenger, then remaining candidates. Soft budget: {event['budget_seconds']:g}s. Active tests finish before stopping.")
|
|
152
|
+
elif kind == "candidate_start":
|
|
153
|
+
self.line(f"[{event['index']:02d}/{event['total']:02d}] {algorithm_name(event['algorithm'])} ({event['role']}) ...")
|
|
154
|
+
elif kind == "candidate_complete":
|
|
155
|
+
result = event["result"]
|
|
156
|
+
if result["status"] == "ok":
|
|
157
|
+
self.line(f" OK macro-F1 {100 * result['cv_macro_f1']:.2f}/100 | std {100 * result['cv_std']:.2f} pts | {result['folds']} folds | {result['cv_seconds']:.2f}s", "ok")
|
|
158
|
+
if self.verbose:
|
|
159
|
+
self.paragraph("Fold scores /100: " + ", ".join(f"{100 * value:.2f}" for value in result["fold_scores"]), " ")
|
|
160
|
+
else:
|
|
161
|
+
self.line(" FAILED", "error")
|
|
162
|
+
self.paragraph(result["error"], " ")
|
|
163
|
+
for message in result.get("warnings", []):
|
|
164
|
+
self.warning(algorithm_name(result["algorithm"]) + ": " + message)
|
|
165
|
+
elif kind == "candidate_skipped":
|
|
166
|
+
self.line(f"[{event['index']:02d}/{event['total']:02d}] {algorithm_name(event['algorithm'])}: skipped (soft budget reached)", "warning")
|
|
167
|
+
elif kind == "holdout_start":
|
|
168
|
+
self.line(f"Checking the measured CV winner, {algorithm_name(event['algorithm'])}, on reserved holdout rows...", "title")
|
|
169
|
+
elif kind == "refit_start":
|
|
170
|
+
self.line(f"Refitting {algorithm_name(event['algorithm'])} on all {event['rows']} usable rows for saving...", "title")
|
|
171
|
+
|
|
172
|
+
def result(self, report):
|
|
173
|
+
if not self.ranking_printed:
|
|
174
|
+
if "data_summary" in report:
|
|
175
|
+
self.dataset(report["data_summary"], report.get("train_rows"), report.get("holdout_rows"))
|
|
176
|
+
self.ranking(report)
|
|
177
|
+
for message in report.get("meta_model_training_warnings", []):
|
|
178
|
+
self.warning("Recommender training reported an optimizer warning; this model remains experimental.")
|
|
179
|
+
if self.verbose and message not in self.warnings_printed:
|
|
180
|
+
self.paragraph(message, " ")
|
|
181
|
+
self.warnings_printed.add(message)
|
|
182
|
+
if "verification_results" not in report:
|
|
183
|
+
self.paragraph("Ranking only: no candidate classifiers were fitted. Add --verify to measure their performance and save a classifier.")
|
|
184
|
+
return
|
|
185
|
+
results = report["verification_results"]
|
|
186
|
+
good = sorted([row for row in results if row["status"] == "ok"], key=lambda row: -row["cv_macro_f1"])
|
|
187
|
+
self.heading("MEASURED CLASSIFIER LEADERBOARD")
|
|
188
|
+
suggested = {row["algorithm"]: index for index, row in enumerate(report["ranking"], 1)}
|
|
189
|
+
self.table(["Algorithm", "Suggested #", "CV F1 /100", "Std (pts)", "CV (s)"], [
|
|
190
|
+
[algorithm_name(row["algorithm"]) + (" *" if row["algorithm"] == report["selected_algorithm"] else ""),
|
|
191
|
+
suggested[row["algorithm"]], f"{100 * row['cv_macro_f1']:.2f}", f"{100 * row['cv_std']:.2f}",
|
|
192
|
+
f"{row['cv_seconds']:.2f}" if "cv_seconds" in row else "-"] for row in good])
|
|
193
|
+
counts = Counter(row["status"] for row in results)
|
|
194
|
+
self.paragraph(f"* Selected winner. Completed: {counts['ok']}; failed: {counts['failed']}; skipped: {counts['skipped_budget']}. CV standard deviation describes folds, not a confidence interval.")
|
|
195
|
+
for row in results:
|
|
196
|
+
if row["status"] != "ok":
|
|
197
|
+
self.paragraph(algorithm_name(row["algorithm"]) + ": " + row.get("error", "skipped because the soft budget was reached"), " ")
|
|
198
|
+
self.heading("FINAL HOLDOUT CHECK - SELECTED CLASSIFIER")
|
|
199
|
+
self.values([("First suggestion", algorithm_name(report["ranking"][0]["algorithm"])),
|
|
200
|
+
("Measured CV winner", algorithm_name(report["selected_algorithm"])),
|
|
201
|
+
("Reserved rows", report["holdout_rows"]),
|
|
202
|
+
("Accuracy", f"{100 * report['holdout_metrics']['accuracy']:.2f}%"),
|
|
203
|
+
("Macro-F1", f"{100 * report['holdout_metrics']['macro_f1']:.2f}/100"),
|
|
204
|
+
("Balanced accuracy", f"{100 * report['holdout_metrics']['balanced_accuracy']:.2f}%")])
|
|
205
|
+
self.paragraph("These are the selected classifier's results on this dataset, measured before the final refit. They are not the recommender's accuracy across datasets.")
|
|
206
|
+
if self.verbose:
|
|
207
|
+
self.holdout_details(report.get("holdout_details"))
|
|
208
|
+
self.heading("TECHNICAL DETAILS")
|
|
209
|
+
self.values([("Random seed", 42), ("CV", "Stratified folds; preprocessing fitted inside each fold"),
|
|
210
|
+
("Preprocessing", "Median numeric imputation + scaling; categorical imputation + one-hot encoding (32-category cap, unknown categories ignored)"),
|
|
211
|
+
("Labels", "Encoded inside each training fold; original labels restored for predictions"),
|
|
212
|
+
("Soft budget", f"{report['soft_budget_seconds']:g}s"),
|
|
213
|
+
("Measured verification wall time", f"{report['total_seconds']:.2f}s")])
|
|
214
|
+
for name, value in report.get("timings", {}).items():
|
|
215
|
+
self.values([(name.replace("_", " "), f"{value:.3f}s")])
|
|
216
|
+
if "selected_configuration" in report:
|
|
217
|
+
self.heading("SELECTED CLASSIFIER SETTINGS")
|
|
218
|
+
self.table(["Setting", "Value"], report["selected_configuration"].items())
|
|
219
|
+
|
|
220
|
+
def holdout_details(self, details):
|
|
221
|
+
if not details:
|
|
222
|
+
return
|
|
223
|
+
self.heading("PER-CLASS HOLDOUT RESULTS")
|
|
224
|
+
self.table(["Class", "Precision %", "Recall %", "F1 /100", "Rows"], [
|
|
225
|
+
[repr(row["label"]), f"{100 * row['precision']:.2f}", f"{100 * row['recall']:.2f}",
|
|
226
|
+
f"{100 * row['f1']:.2f}", row["support"]] for row in details["per_class"]])
|
|
227
|
+
self.heading("CONFUSION MATRIX")
|
|
228
|
+
self.paragraph("Rows = actual class; columns = predicted class.")
|
|
229
|
+
labels, matrix = details["labels"], details["confusion_matrix"]
|
|
230
|
+
self.table(["ID", "Class"], [[f"C{i + 1}", repr(label)] for i, label in enumerate(labels)])
|
|
231
|
+
for start in range(0, len(labels), 8):
|
|
232
|
+
stop = min(start + 8, len(labels))
|
|
233
|
+
self.table(["Actual / predicted", *[f"C{i + 1}" for i in range(start, stop)]],
|
|
234
|
+
[[f"C{i + 1}", *row[start:stop]] for i, row in enumerate(matrix)])
|
|
235
|
+
|
|
236
|
+
def predictions(self, predictions):
|
|
237
|
+
values = [str(value) for value in predictions]
|
|
238
|
+
self.heading("PREDICTIONS FOR NEW ROWS")
|
|
239
|
+
self.paragraph(f"Produced {len(values)} predictions using the saved, reloaded classifier.")
|
|
240
|
+
self.table(["Row", "Predicted class"], [[index, repr(value)] for index, value in enumerate(values[:10], 1)])
|
|
241
|
+
if len(values) > 10:
|
|
242
|
+
self.paragraph("Showing the first ten; the CSV contains every prediction.")
|
|
243
|
+
if self.verbose:
|
|
244
|
+
self.table(["Predicted class", "Rows"], [[repr(label), count] for label, count in Counter(values).items()])
|
|
245
|
+
|
|
246
|
+
def research(self, report):
|
|
247
|
+
fresh = report.get("fresh_evaluation", {})
|
|
248
|
+
if fresh.get("status") != "complete":
|
|
249
|
+
return
|
|
250
|
+
for message in report.get("production_convergence_warnings", []):
|
|
251
|
+
self.warning("Recommender training reported an optimizer warning; this model remains experimental.")
|
|
252
|
+
if self.verbose and message not in self.warnings_printed:
|
|
253
|
+
self.paragraph(message, " ")
|
|
254
|
+
self.warnings_printed.add(message)
|
|
255
|
+
self.heading("SAVED RESEARCH COMPARISON - SEPARATE DATASETS")
|
|
256
|
+
self.paragraph(f"Historical training: {report['dataset_count']} datasets. Final evaluation: {len(fresh['groups'])} separate source groups. These are recorded results, not a new evaluation run.")
|
|
257
|
+
self.table(["Method", "Top-1 %", "Top-3 %", "First loss", "Top-3 loss"], [
|
|
258
|
+
[name.replace("_", " "), f"{100 * row['mean']['top_1']:.1f}", f"{100 * row['mean']['top_3']:.1f}",
|
|
259
|
+
f"{100 * row['mean']['regret']:.2f}", f"{100 * row['mean']['top_3_regret']:.2f}"]
|
|
260
|
+
for name, row in fresh["comparisons"].items()])
|
|
261
|
+
self.paragraph("Loss is macro-F1 points below the best measured candidate; lower is better. Top-3 loss assumes the three suggestions are tested. A small final cohort does not establish broad reliability.")
|
|
262
|
+
|
|
263
|
+
def files(self, pairs):
|
|
264
|
+
self.heading("SAVED OUTPUTS")
|
|
265
|
+
self.values(pairs)
|
|
266
|
+
|
|
267
|
+
def save(self, path):
|
|
268
|
+
path = Path(path)
|
|
269
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
270
|
+
path.write_text("\n".join(self.lines) + "\n")
|