algorithm-genome 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- algorithm_genome/__init__.py +13 -0
- algorithm_genome/__main__.py +3 -0
- algorithm_genome/algorithm_library.py +60 -0
- algorithm_genome/assets/default_recommender.joblib +0 -0
- algorithm_genome/assets/model_info.json +28 -0
- algorithm_genome/benchmark_datasets.py +130 -0
- algorithm_genome/bundled_model.py +27 -0
- algorithm_genome/command_line.py +116 -0
- algorithm_genome/create_example_datasets.py +35 -0
- algorithm_genome/dataset_genome.py +110 -0
- algorithm_genome/download_public_datasets.py +251 -0
- algorithm_genome/framework.py +101 -0
- algorithm_genome/improve_meta_model.py +186 -0
- algorithm_genome/model_pipeline.py +68 -0
- algorithm_genome/prepare_additional_datasets.py +105 -0
- algorithm_genome/recommend_algorithms.py +177 -0
- algorithm_genome/terminal_output.py +270 -0
- algorithm_genome/train_meta_model.py +142 -0
- algorithm_genome/train_neural_recommender.py +402 -0
- algorithm_genome-0.3.0.dist-info/METADATA +223 -0
- algorithm_genome-0.3.0.dist-info/RECORD +24 -0
- algorithm_genome-0.3.0.dist-info/WHEEL +5 -0
- algorithm_genome-0.3.0.dist-info/entry_points.txt +2 -0
- algorithm_genome-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Learn performance rankings; evaluate on held-out dataset families before refit."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import joblib
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import sklearn
|
|
11
|
+
from sklearn.ensemble import RandomForestRegressor
|
|
12
|
+
from sklearn.feature_extraction import DictVectorizer
|
|
13
|
+
from sklearn.model_selection import GroupShuffleSplit
|
|
14
|
+
from sklearn.pipeline import Pipeline
|
|
15
|
+
|
|
16
|
+
from .algorithm_library import algorithm_library, library_signature, meta_features
|
|
17
|
+
from .benchmark_datasets import write_json
|
|
18
|
+
from .dataset_genome import GENOME_VERSION
|
|
19
|
+
from .model_pipeline import PIPELINE_VERSION
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _new_regressor():
|
|
23
|
+
return Pipeline([("encode", DictVectorizer(sparse=False)),
|
|
24
|
+
("regress", RandomForestRegressor(n_estimators=150, min_samples_leaf=2,
|
|
25
|
+
max_features=0.8, random_state=42, n_jobs=1))])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def check_compatibility(metadata):
|
|
29
|
+
if (metadata.get("format_version") != 1 or metadata.get("genome_version") != GENOME_VERSION
|
|
30
|
+
or metadata.get("pipeline_version") != PIPELINE_VERSION
|
|
31
|
+
or metadata.get("library_signature") != library_signature()):
|
|
32
|
+
raise ValueError("Artifact configuration differs from this code; rebuild benchmarks and meta-model")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _ranking_metrics(datasets, prediction):
|
|
36
|
+
per_dataset = []
|
|
37
|
+
for dataset in datasets:
|
|
38
|
+
names = list(algorithm_library())
|
|
39
|
+
observed = {r["algorithm"]: r["cv_macro_f1"] for r in dataset["results"]}
|
|
40
|
+
scores = prediction(dataset, names)
|
|
41
|
+
ranked = sorted(names, key=lambda n: (-scores[n], n))
|
|
42
|
+
best = max(observed.values())
|
|
43
|
+
winners = {n for n in names if np.isclose(observed[n], best, atol=1e-8, rtol=0)}
|
|
44
|
+
actual_ranks = pd.Series([observed[n] for n in names]).rank()
|
|
45
|
+
predicted_ranks = pd.Series([scores[n] for n in names]).rank()
|
|
46
|
+
correlation = actual_ranks.corr(predicted_ranks) if actual_ranks.nunique() > 1 and predicted_ranks.nunique() > 1 else None
|
|
47
|
+
per_dataset.append({"dataset": dataset["id"], "group": dataset["group"],
|
|
48
|
+
"recommended": ranked[0], "top_1": ranked[0] in winners,
|
|
49
|
+
"top_3": bool(set(ranked[:3]) & winners),
|
|
50
|
+
"regret": float(best - observed[ranked[0]]),
|
|
51
|
+
"spearman": float(correlation) if correlation is not None else None})
|
|
52
|
+
return {"top_1_rate": float(np.mean([r["top_1"] for r in per_dataset])),
|
|
53
|
+
"top_3_rate": float(np.mean([r["top_3"] for r in per_dataset])),
|
|
54
|
+
"mean_regret": float(np.mean([r["regret"] for r in per_dataset])),
|
|
55
|
+
"datasets": per_dataset}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def train_meta_model(benchmark_path, output_directory=None):
|
|
59
|
+
"""Train from a benchmark dictionary or JSON path, optionally saving artifacts."""
|
|
60
|
+
benchmark = benchmark_path if isinstance(benchmark_path, Mapping) else json.loads(Path(benchmark_path).read_text())
|
|
61
|
+
check_compatibility(benchmark)
|
|
62
|
+
expected_names = set(algorithm_library())
|
|
63
|
+
datasets, excluded = [], []
|
|
64
|
+
for dataset in benchmark["datasets"]:
|
|
65
|
+
results = dataset["results"]
|
|
66
|
+
complete = (len(results) == len(expected_names)
|
|
67
|
+
and {r["algorithm"] for r in results} == expected_names
|
|
68
|
+
and all(r["status"] == "ok" for r in results))
|
|
69
|
+
if not complete:
|
|
70
|
+
excluded.append(dataset["id"])
|
|
71
|
+
continue
|
|
72
|
+
if not all(np.isfinite(r["cv_macro_f1"]) and 0 <= r["cv_macro_f1"] <= 1
|
|
73
|
+
and np.isfinite(r["cv_seconds"]) and r["cv_seconds"] > 0 for r in results):
|
|
74
|
+
raise ValueError("Benchmark scores or durations are invalid")
|
|
75
|
+
datasets.append(dataset)
|
|
76
|
+
if len({d["id"] for d in datasets}) != len(datasets):
|
|
77
|
+
raise ValueError("Duplicate dataset IDs in benchmark")
|
|
78
|
+
if len({d["fingerprint"] for d in datasets}) != len(datasets):
|
|
79
|
+
raise ValueError("Duplicate dataset content in benchmark")
|
|
80
|
+
groups = [d["group"] for d in datasets]
|
|
81
|
+
if len(set(groups)) < 4:
|
|
82
|
+
raise ValueError("Need at least four independent dataset groups with successful results for every algorithm")
|
|
83
|
+
schema = sorted(datasets[0]["genome"])
|
|
84
|
+
if any(sorted(d["genome"]) != schema or not all(np.isfinite(v) for v in d["genome"].values()) for d in datasets):
|
|
85
|
+
raise ValueError("Dataset genome schemas or values are inconsistent")
|
|
86
|
+
split = GroupShuffleSplit(n_splits=1, test_size=0.25, random_state=42)
|
|
87
|
+
train_indices, test_indices = next(split.split(datasets, groups=groups))
|
|
88
|
+
training = [datasets[i] for i in train_indices]
|
|
89
|
+
testing = [datasets[i] for i in test_indices]
|
|
90
|
+
|
|
91
|
+
def records(selected, genomes=True):
|
|
92
|
+
features, targets, runtimes, weights = [], [], [], []
|
|
93
|
+
group_sizes = pd.Series([d["group"] for d in selected]).value_counts()
|
|
94
|
+
for dataset in selected:
|
|
95
|
+
average = np.mean([r["cv_macro_f1"] for r in dataset["results"]])
|
|
96
|
+
for result in dataset["results"]:
|
|
97
|
+
features.append(meta_features(dataset["genome"], result["algorithm"], genomes))
|
|
98
|
+
# Relative performance removes much of each dataset's difficulty.
|
|
99
|
+
targets.append(result["cv_macro_f1"] - average)
|
|
100
|
+
runtimes.append(np.log1p(result["cv_seconds"]))
|
|
101
|
+
weights.append(1 / group_sizes[dataset["group"]])
|
|
102
|
+
return features, targets, runtimes, weights
|
|
103
|
+
|
|
104
|
+
features, targets, _, weights = records(training)
|
|
105
|
+
evaluated_model = _new_regressor().fit(features, targets, regress__sample_weight=weights)
|
|
106
|
+
identity_features, identity_targets, _, _ = records(training, genomes=False)
|
|
107
|
+
identity_model = _new_regressor().fit(identity_features, identity_targets, regress__sample_weight=weights)
|
|
108
|
+
fixed = {name: float(np.average([
|
|
109
|
+
next(r["cv_macro_f1"] for r in d["results"] if r["algorithm"] == name)
|
|
110
|
+
for d in training], weights=[1 / groups.count(d["group"]) for d in training])) for name in expected_names}
|
|
111
|
+
report = {
|
|
112
|
+
"metric": "macro_f1", "train_groups": sorted({d["group"] for d in training}),
|
|
113
|
+
"evaluation_groups": sorted({d["group"] for d in testing}),
|
|
114
|
+
"excluded_incomplete_datasets": excluded,
|
|
115
|
+
"genome_model": _ranking_metrics(testing, lambda d, names: dict(zip(names,
|
|
116
|
+
evaluated_model.predict([meta_features(d["genome"], n) for n in names])))),
|
|
117
|
+
"algorithm_identity_control": _ranking_metrics(testing, lambda d, names: dict(zip(names,
|
|
118
|
+
identity_model.predict([meta_features(d["genome"], n, False) for n in names])))),
|
|
119
|
+
"fixed_ranking_baseline": _ranking_metrics(testing, lambda d, names: fixed),
|
|
120
|
+
"notes": ["Evaluation holds out dataset groups, not individual experiment rows.",
|
|
121
|
+
"Scores compare recommendations against matched benchmark CV outcomes, not independent final test accuracy.",
|
|
122
|
+
"One group split is preliminary; use more independent datasets and repeated group splits for research claims.",
|
|
123
|
+
"Saved production models are refitted on all complete datasets after evaluation.",
|
|
124
|
+
"Four groups is an execution minimum, not evidence of reliable generalization."],
|
|
125
|
+
}
|
|
126
|
+
features, targets, runtimes, weights = records(datasets)
|
|
127
|
+
artifact = {
|
|
128
|
+
"format_version": 1, "genome_version": GENOME_VERSION, "pipeline_version": PIPELINE_VERSION,
|
|
129
|
+
"library_signature": library_signature(), "sklearn_version": sklearn.__version__,
|
|
130
|
+
"performance_model": _new_regressor().fit(features, targets, regress__sample_weight=weights),
|
|
131
|
+
"runtime_model": _new_regressor().fit(features, runtimes, regress__sample_weight=weights),
|
|
132
|
+
"genome_schema": schema,
|
|
133
|
+
"feature_ranges": {key: [min(d["genome"][key] for d in datasets), max(d["genome"][key] for d in datasets)] for key in schema},
|
|
134
|
+
"training_groups": sorted(set(groups)), "training_datasets": [d["id"] for d in datasets],
|
|
135
|
+
"evaluation": report,
|
|
136
|
+
}
|
|
137
|
+
if output_directory is not None:
|
|
138
|
+
output = Path(output_directory)
|
|
139
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
joblib.dump(artifact, output / "trained_meta_model.joblib")
|
|
141
|
+
write_json(output / "meta_model_evaluation.json", report)
|
|
142
|
+
return artifact
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
"""Export a dataset-of-datasets and compare a small neural recommender fairly."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from collections import Counter
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
import joblib
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import sklearn
|
|
14
|
+
from sklearn.exceptions import ConvergenceWarning
|
|
15
|
+
from sklearn.feature_extraction import DictVectorizer
|
|
16
|
+
from sklearn.model_selection import GroupKFold
|
|
17
|
+
from sklearn.neural_network import MLPRegressor
|
|
18
|
+
from sklearn.pipeline import make_pipeline
|
|
19
|
+
from sklearn.preprocessing import StandardScaler
|
|
20
|
+
|
|
21
|
+
from .algorithm_library import algorithm_library, library_signature, meta_features
|
|
22
|
+
from .benchmark_datasets import write_json
|
|
23
|
+
from .dataset_genome import GENOME_VERSION
|
|
24
|
+
from .improve_meta_model import DatasetRankingModel, _fit, _records
|
|
25
|
+
from .model_pipeline import PIPELINE_VERSION
|
|
26
|
+
from .train_meta_model import _new_regressor, check_compatibility
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# Fixed before evaluation. Larger alpha gives stronger weight regularization.
|
|
30
|
+
NEURAL_ALPHAS = (10.0, 1.0, 0.1)
|
|
31
|
+
METRICS = ("top_1", "top_3", "regret", "top_3_regret", "top_3_within_one_f1_point",
|
|
32
|
+
"top_3_recorded_cv_time_fraction")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def load_benchmark(path):
|
|
36
|
+
benchmark = json.loads(Path(path).read_text())
|
|
37
|
+
check_compatibility(benchmark)
|
|
38
|
+
if benchmark.get("metric") != "macro_f1":
|
|
39
|
+
raise ValueError("Benchmark must use macro_f1")
|
|
40
|
+
names = set(algorithm_library())
|
|
41
|
+
complete, excluded, ids, fingerprints = [], [], set(), set()
|
|
42
|
+
schema = None
|
|
43
|
+
for dataset in benchmark["datasets"]:
|
|
44
|
+
if not dataset.get("group") or not dataset.get("fingerprint"):
|
|
45
|
+
raise ValueError("Every dataset needs a source group and content fingerprint")
|
|
46
|
+
if dataset["id"] in ids or dataset["fingerprint"] in fingerprints:
|
|
47
|
+
raise ValueError("Duplicate dataset IDs or content in benchmark")
|
|
48
|
+
ids.add(dataset["id"])
|
|
49
|
+
fingerprints.add(dataset["fingerprint"])
|
|
50
|
+
results = dataset["results"]
|
|
51
|
+
if (len(results) != len(names) or {r["algorithm"] for r in results} != names
|
|
52
|
+
or any(r["status"] != "ok" for r in results)):
|
|
53
|
+
excluded.append(dataset["id"])
|
|
54
|
+
continue
|
|
55
|
+
if any(not np.isfinite(r["cv_macro_f1"]) or not 0 <= r["cv_macro_f1"] <= 1
|
|
56
|
+
or not np.isfinite(r["cv_seconds"]) or r["cv_seconds"] <= 0 for r in results):
|
|
57
|
+
raise ValueError("Invalid benchmark score or runtime")
|
|
58
|
+
genome = dataset["genome"]
|
|
59
|
+
if schema is None:
|
|
60
|
+
schema = sorted(genome)
|
|
61
|
+
if sorted(genome) != schema or not all(np.isfinite(v) for v in genome.values()):
|
|
62
|
+
raise ValueError("Inconsistent genome schema or non-finite values")
|
|
63
|
+
complete.append(dataset)
|
|
64
|
+
if not complete:
|
|
65
|
+
raise ValueError("No complete datasets in benchmark")
|
|
66
|
+
return complete, excluded
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def export_training_table(datasets, directory):
|
|
70
|
+
"""Metadata and targets are explicit; only dataset_* columns are model inputs."""
|
|
71
|
+
directory = Path(directory)
|
|
72
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
names = list(algorithm_library())
|
|
74
|
+
rows = []
|
|
75
|
+
for dataset in datasets:
|
|
76
|
+
scores = {r["algorithm"]: r["cv_macro_f1"] for r in dataset["results"]}
|
|
77
|
+
best = max(scores.values())
|
|
78
|
+
winners = [n for n in names if np.isclose(scores[n], best, atol=1e-8, rtol=0)]
|
|
79
|
+
row = {"dataset_id": dataset["id"], "source_group": dataset["group"],
|
|
80
|
+
"content_fingerprint": dataset["fingerprint"],
|
|
81
|
+
**{f"dataset_{k}": v for k, v in dataset["genome"].items()},
|
|
82
|
+
**{f"score_{n}": scores[n] for n in names},
|
|
83
|
+
**{f"cv_seconds_{r['algorithm']}": r["cv_seconds"] for r in dataset["results"]},
|
|
84
|
+
"best_algorithms": json.dumps(winners), "best_macro_f1": best}
|
|
85
|
+
rows.append(row)
|
|
86
|
+
frame = pd.DataFrame(rows)
|
|
87
|
+
frame.to_csv(directory / "dataset_training_table.csv", index=False)
|
|
88
|
+
schema = {"rows": len(frame), "independent_source_groups": len({d["group"] for d in datasets}),
|
|
89
|
+
"input_columns": [f"dataset_{k}" for k in sorted(datasets[0]["genome"])],
|
|
90
|
+
"target_columns": [f"score_{n}" for n in names],
|
|
91
|
+
"notes": ["One row per benchmark dataset, not per original observation.",
|
|
92
|
+
"IDs, source groups, best_algorithms, scores, and times are not input features.",
|
|
93
|
+
"Training targets are score columns centered by each row's candidate mean.",
|
|
94
|
+
"best_algorithms preserves all tied measured winners for inspection."]}
|
|
95
|
+
write_json(directory / "training_table_schema.json", schema)
|
|
96
|
+
return frame
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class NeuralRankingModel:
|
|
100
|
+
"""One eight-neuron hidden layer predicts all ten relative candidate scores."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, alpha=1.0):
|
|
103
|
+
self.alpha = alpha
|
|
104
|
+
|
|
105
|
+
def fit(self, datasets):
|
|
106
|
+
self.names = list(algorithm_library())
|
|
107
|
+
features, targets = [], []
|
|
108
|
+
counts = Counter(d["group"] for d in datasets)
|
|
109
|
+
weights = np.array([1 / counts[d["group"]] for d in datasets])
|
|
110
|
+
# Normalize weights to mean one so alpha has the same interpretation
|
|
111
|
+
# when the number of variants per source group changes.
|
|
112
|
+
weights *= len(weights) / weights.sum()
|
|
113
|
+
for dataset in datasets:
|
|
114
|
+
features.append(DatasetRankingModel._features(dataset["genome"]))
|
|
115
|
+
scores = {r["algorithm"]: r["cv_macro_f1"] for r in dataset["results"]}
|
|
116
|
+
row = np.array([scores[n] for n in self.names])
|
|
117
|
+
targets.append(row - row.mean())
|
|
118
|
+
self.model = make_pipeline(DictVectorizer(sparse=False), StandardScaler(),
|
|
119
|
+
MLPRegressor(hidden_layer_sizes=(8,), activation="tanh",
|
|
120
|
+
solver="lbfgs", alpha=self.alpha, max_iter=2000,
|
|
121
|
+
max_fun=20000, random_state=42))
|
|
122
|
+
with warnings.catch_warnings(record=True) as recorded:
|
|
123
|
+
warnings.simplefilter("always", ConvergenceWarning)
|
|
124
|
+
self.model.fit(features, targets, standardscaler__sample_weight=weights,
|
|
125
|
+
mlpregressor__sample_weight=weights)
|
|
126
|
+
self.convergence_warnings = [str(w.message) for w in recorded
|
|
127
|
+
if issubclass(w.category, ConvergenceWarning)]
|
|
128
|
+
return self
|
|
129
|
+
|
|
130
|
+
def predict(self, records):
|
|
131
|
+
if not records:
|
|
132
|
+
return np.array([])
|
|
133
|
+
features = [DatasetRankingModel._features({k.removeprefix("dataset_"): v
|
|
134
|
+
for k, v in record.items() if k.startswith("dataset_")}) for record in records]
|
|
135
|
+
scores = self.model.predict(features)
|
|
136
|
+
# Ranking is unchanged by row centering; maintain the artifact's relative-score meaning.
|
|
137
|
+
scores -= scores.mean(axis=1, keepdims=True)
|
|
138
|
+
return np.array([scores[i, self.names.index(record["algorithm_name"])]
|
|
139
|
+
for i, record in enumerate(records)])
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class AlwaysSVMModel:
|
|
143
|
+
"""SVM first; the rest follow a portfolio learned only from training groups."""
|
|
144
|
+
|
|
145
|
+
def fit(self, datasets):
|
|
146
|
+
self.portfolio = DatasetRankingModel("fixed_portfolio").fit(datasets)
|
|
147
|
+
return self
|
|
148
|
+
|
|
149
|
+
def predict(self, records):
|
|
150
|
+
scores = self.portfolio.predict(records)
|
|
151
|
+
return np.array([2.0 if record["algorithm_name"] == "support_vector_machine" else score
|
|
152
|
+
for record, score in zip(records, scores)])
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def evaluate_ranking(model, datasets):
|
|
156
|
+
names = list(algorithm_library())
|
|
157
|
+
rows = []
|
|
158
|
+
for dataset in datasets:
|
|
159
|
+
predicted = model.predict([meta_features(dataset["genome"], n) for n in names])
|
|
160
|
+
if len(predicted) != len(names) or not np.isfinite(predicted).all():
|
|
161
|
+
raise ValueError("Invalid model predictions")
|
|
162
|
+
ranked = sorted(names, key=lambda n: (-predicted[names.index(n)], n))
|
|
163
|
+
measured = {r["algorithm"]: r for r in dataset["results"]}
|
|
164
|
+
best = max(r["cv_macro_f1"] for r in measured.values())
|
|
165
|
+
first = measured[ranked[0]]["cv_macro_f1"]
|
|
166
|
+
top3 = max(measured[n]["cv_macro_f1"] for n in ranked[:3])
|
|
167
|
+
rows.append({"dataset": dataset["id"], "group": dataset["group"],
|
|
168
|
+
"ranking": ranked, "top_1": bool(np.isclose(first, best, atol=1e-8, rtol=0)),
|
|
169
|
+
"top_3": bool(np.isclose(top3, best, atol=1e-8, rtol=0)),
|
|
170
|
+
"regret": float(best - first), "top_3_regret": float(best - top3),
|
|
171
|
+
"top_3_within_one_f1_point": bool(best - top3 <= 0.01 + 1e-8),
|
|
172
|
+
"top_3_recorded_cv_time_fraction": float(
|
|
173
|
+
sum(measured[n]["cv_seconds"] for n in ranked[:3]) /
|
|
174
|
+
sum(r["cv_seconds"] for r in measured.values()))})
|
|
175
|
+
return rows
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def summarize(rows):
|
|
179
|
+
groups = sorted({r["group"] for r in rows})
|
|
180
|
+
return {metric: float(np.mean([np.mean([r[metric] for r in rows if r["group"] == group])
|
|
181
|
+
for group in groups])) for metric in METRICS}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def write_comparison_summary(report, directory, filename="comparison_summary.csv"):
|
|
185
|
+
rows = []
|
|
186
|
+
for method, comparison in report["comparisons"].items():
|
|
187
|
+
mean = comparison["mean"]
|
|
188
|
+
rows.append({"method": method, "top_1_percent": 100 * mean["top_1"],
|
|
189
|
+
"top_3_percent": 100 * mean["top_3"],
|
|
190
|
+
"first_choice_f1_points_lost": 100 * mean["regret"],
|
|
191
|
+
"best_of_three_f1_points_lost": 100 * mean["top_3_regret"],
|
|
192
|
+
"top_3_within_one_f1_point_percent": 100 * mean["top_3_within_one_f1_point"],
|
|
193
|
+
"top_3_recorded_cv_time_percent": 100 * mean["top_3_recorded_cv_time_fraction"]})
|
|
194
|
+
pd.DataFrame(rows).to_csv(Path(directory) / filename, index=False)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def group_splits(datasets, seed, folds=3):
|
|
198
|
+
groups = [d["group"] for d in datasets]
|
|
199
|
+
if len(set(groups)) < folds:
|
|
200
|
+
raise ValueError("Not enough source groups for cross-validation")
|
|
201
|
+
return list(GroupKFold(n_splits=folds, shuffle=True, random_state=seed).split(datasets, groups=groups))
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def choose_neural_alpha(datasets, seed):
|
|
205
|
+
splits = group_splits(datasets, seed)
|
|
206
|
+
comparisons = []
|
|
207
|
+
audit = []
|
|
208
|
+
for train, valid in splits:
|
|
209
|
+
audit.append({"train_groups": sorted({datasets[i]["group"] for i in train}),
|
|
210
|
+
"validation_groups": sorted({datasets[i]["group"] for i in valid})})
|
|
211
|
+
for alpha in NEURAL_ALPHAS:
|
|
212
|
+
rows, convergence = [], []
|
|
213
|
+
for train, valid in splits:
|
|
214
|
+
model = NeuralRankingModel(alpha).fit([datasets[i] for i in train])
|
|
215
|
+
rows.extend(evaluate_ranking(model, [datasets[i] for i in valid]))
|
|
216
|
+
convergence.extend(model.convergence_warnings)
|
|
217
|
+
comparisons.append({"alpha": alpha, **summarize(rows), "convergence_warnings": convergence})
|
|
218
|
+
winner = min(comparisons, key=lambda c: (c["top_3_regret"], c["regret"], -c["alpha"]))
|
|
219
|
+
return winner["alpha"], comparisons, audit
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def run_neural_experiment(benchmark_path, output_directory, historical_evaluation_path,
|
|
223
|
+
previous_model_path=None, fresh_benchmark_path=None, repeats=3,
|
|
224
|
+
fresh_reference_models=None):
|
|
225
|
+
if repeats < 1:
|
|
226
|
+
raise ValueError("repeats must be positive")
|
|
227
|
+
output = Path(output_directory).resolve()
|
|
228
|
+
project = Path(__file__).resolve().parents[1]
|
|
229
|
+
protected = {project / "results/meta_model", project / "results/improved_meta_model"}
|
|
230
|
+
if previous_model_path:
|
|
231
|
+
protected.add(Path(previous_model_path).resolve().parent)
|
|
232
|
+
if output in protected:
|
|
233
|
+
raise ValueError("Use a separate output directory; existing models must be preserved")
|
|
234
|
+
datasets, excluded = load_benchmark(benchmark_path)
|
|
235
|
+
historical = json.loads(Path(historical_evaluation_path).read_text())
|
|
236
|
+
historical_groups = set(historical["final_evaluation_groups"])
|
|
237
|
+
development = [d for d in datasets if d["group"] not in historical_groups]
|
|
238
|
+
if len({d["group"] for d in development}) < 8:
|
|
239
|
+
raise ValueError("Need at least eight development groups after excluding earlier test groups")
|
|
240
|
+
fresh, fresh_excluded = [], []
|
|
241
|
+
if fresh_benchmark_path:
|
|
242
|
+
fresh, fresh_excluded = load_benchmark(fresh_benchmark_path)
|
|
243
|
+
if ({d["group"] for d in fresh} & ({d["group"] for d in datasets} | historical_groups)
|
|
244
|
+
or {d["fingerprint"] for d in fresh} & {d["fingerprint"] for d in datasets}
|
|
245
|
+
or {d["id"] for d in fresh} & {d["id"] for d in datasets}):
|
|
246
|
+
raise ValueError("Fresh evaluation must use disjoint dataset groups, IDs, and content")
|
|
247
|
+
if any(sorted(d["genome"]) != sorted(datasets[0]["genome"]) for d in fresh):
|
|
248
|
+
raise ValueError("Fresh evaluation genome schema differs")
|
|
249
|
+
previous = None
|
|
250
|
+
if previous_model_path:
|
|
251
|
+
previous = joblib.load(previous_model_path)
|
|
252
|
+
check_compatibility(previous)
|
|
253
|
+
if previous["sklearn_version"] != sklearn.__version__:
|
|
254
|
+
raise ValueError("Previous model requires the same sklearn version")
|
|
255
|
+
if set(previous["training_groups"]) & {d["group"] for d in datasets + fresh}:
|
|
256
|
+
raise ValueError("Frozen previous model has trained on comparison dataset groups")
|
|
257
|
+
references = {}
|
|
258
|
+
for label, path in (fresh_reference_models or {}).items():
|
|
259
|
+
if not fresh:
|
|
260
|
+
raise ValueError("Fresh reference models require a fresh benchmark")
|
|
261
|
+
if not label.startswith("saved_"):
|
|
262
|
+
raise ValueError("Fresh reference labels must start with saved_ to distinguish frozen models")
|
|
263
|
+
reference = joblib.load(path)
|
|
264
|
+
check_compatibility(reference)
|
|
265
|
+
if reference["sklearn_version"] != sklearn.__version__:
|
|
266
|
+
raise ValueError("Fresh reference requires the same sklearn version")
|
|
267
|
+
if (set(reference["training_groups"]) & {d["group"] for d in fresh}
|
|
268
|
+
or set(reference["training_datasets"]) & {d["id"] for d in fresh}):
|
|
269
|
+
raise ValueError("Fresh reference model has trained on fresh evaluation datasets")
|
|
270
|
+
if reference["genome_schema"] != sorted(datasets[0]["genome"]):
|
|
271
|
+
raise ValueError("Fresh reference genome schema differs")
|
|
272
|
+
references[label] = reference
|
|
273
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
274
|
+
export_training_table(datasets, output)
|
|
275
|
+
methods = ["neural_network", "original_pairwise_forest", "joint_forest", "fixed_portfolio", "always_svm"]
|
|
276
|
+
if previous:
|
|
277
|
+
methods.append("previous_saved_model")
|
|
278
|
+
outcomes = {method: [] for method in methods}
|
|
279
|
+
audits, warnings_seen = [], []
|
|
280
|
+
for repeat in range(repeats):
|
|
281
|
+
for fold, (train, valid) in enumerate(group_splits(development, 20261115 + repeat)):
|
|
282
|
+
training = [development[i] for i in train]
|
|
283
|
+
validation = [development[i] for i in valid]
|
|
284
|
+
alpha, comparison, inner_audit = choose_neural_alpha(training, 4200 + repeat * 10 + fold)
|
|
285
|
+
audits.append({"repeat": repeat, "fold": fold,
|
|
286
|
+
"train_groups": sorted({d["group"] for d in training}),
|
|
287
|
+
"validation_groups": sorted({d["group"] for d in validation}),
|
|
288
|
+
"selected_alpha": alpha, "inner_comparison": comparison,
|
|
289
|
+
"inner_splits": inner_audit})
|
|
290
|
+
for method in methods:
|
|
291
|
+
if method == "neural_network":
|
|
292
|
+
model = NeuralRankingModel(alpha).fit(training)
|
|
293
|
+
warnings_seen.extend(model.convergence_warnings)
|
|
294
|
+
elif method == "always_svm":
|
|
295
|
+
model = AlwaysSVMModel().fit(training)
|
|
296
|
+
elif method == "previous_saved_model":
|
|
297
|
+
model = previous["performance_model"]
|
|
298
|
+
else:
|
|
299
|
+
model = _fit(method, training)
|
|
300
|
+
outcomes[method].extend({**row, "repeat": repeat, "fold": fold}
|
|
301
|
+
for row in evaluate_ranking(model, validation))
|
|
302
|
+
print(f"Repeat {repeat + 1}/{repeats}, fold {fold + 1}/3 complete", flush=True)
|
|
303
|
+
alpha, final_comparison, final_inner = choose_neural_alpha(development, 2026)
|
|
304
|
+
report = {"protocol": "exploratory_repeated_nested_group_cv", "repeats": repeats,
|
|
305
|
+
"dataset_count": len(datasets), "development_dataset_count": len(development),
|
|
306
|
+
"development_groups": sorted({d["group"] for d in development}),
|
|
307
|
+
"excluded_previously_examined_groups": sorted(historical_groups),
|
|
308
|
+
"excluded_incomplete_datasets": excluded,
|
|
309
|
+
"neural_configuration": {"hidden_layer_sizes": [8], "activation": "tanh", "solver": "lbfgs",
|
|
310
|
+
"candidate_alphas": list(NEURAL_ALPHAS), "selected_alpha": alpha,
|
|
311
|
+
"selection": "inner group CV: top-3 regret, then top-1 regret"},
|
|
312
|
+
"comparisons": {method: {"mean": summarize(rows),
|
|
313
|
+
"per_repeat": [summarize([r for r in rows if r["repeat"] == i]) for i in range(repeats)],
|
|
314
|
+
"out_of_fold_datasets": rows} for method, rows in outcomes.items()},
|
|
315
|
+
"split_audit": audits, "final_alpha_selection": final_comparison,
|
|
316
|
+
"final_alpha_inner_splits": final_inner, "convergence_warnings": warnings_seen,
|
|
317
|
+
"fresh_evaluation": {"status": "pending", "reason": "No fresh benchmark supplied"},
|
|
318
|
+
"notes": ["Earlier final test groups are excluded from neural model selection and CV.",
|
|
319
|
+
"Internal comparisons are exploratory, not a new untouched final evaluation.",
|
|
320
|
+
"Each development dataset is tested once per repeat; repeats are not independent datasets.",
|
|
321
|
+
"Scores and preprocessing are fitted only on each training split.",
|
|
322
|
+
"Top-3 regret measures the best observed candidate score among the first three suggestions.",
|
|
323
|
+
"Recorded CV time fractions are benchmark estimates, not measured end-to-end speedups.",
|
|
324
|
+
"Production neural model is refitted on historical corpus after internal evaluation."]}
|
|
325
|
+
# Freeze choices before ever inspecting scores on the optional fresh benchmark.
|
|
326
|
+
write_json(output / "experiment_plan.json", {"configuration": report["neural_configuration"],
|
|
327
|
+
"development_groups": report["development_groups"],
|
|
328
|
+
"earlier_test_groups": sorted(historical_groups), "methods": methods,
|
|
329
|
+
"fresh_reference_models": {label: str(Path(path).resolve())
|
|
330
|
+
for label, path in (fresh_reference_models or {}).items()},
|
|
331
|
+
"benchmark_sha256": hashlib.sha256(Path(benchmark_path).read_bytes()).hexdigest()})
|
|
332
|
+
production = NeuralRankingModel(alpha).fit(datasets)
|
|
333
|
+
if fresh:
|
|
334
|
+
evaluations = {}
|
|
335
|
+
for method in methods:
|
|
336
|
+
model = (production if method == "neural_network" else AlwaysSVMModel().fit(datasets)
|
|
337
|
+
if method == "always_svm" else previous["performance_model"]
|
|
338
|
+
if method == "previous_saved_model" else _fit(method, datasets))
|
|
339
|
+
rows = evaluate_ranking(model, fresh)
|
|
340
|
+
evaluations[method] = {"mean": summarize(rows), "datasets": rows}
|
|
341
|
+
for label, reference in references.items():
|
|
342
|
+
rows = evaluate_ranking(reference["performance_model"], fresh)
|
|
343
|
+
evaluations[label] = {"mean": summarize(rows), "datasets": rows,
|
|
344
|
+
"frozen_reference": True,
|
|
345
|
+
"training_dataset_count": len(reference["training_datasets"]),
|
|
346
|
+
"notes": ["Uses its original training corpus; not retrained or used for selection."]}
|
|
347
|
+
report["fresh_evaluation"] = {"status": "complete", "groups": sorted({d["group"] for d in fresh}),
|
|
348
|
+
"excluded_incomplete_datasets": fresh_excluded,
|
|
349
|
+
"comparisons": evaluations,
|
|
350
|
+
"notes": ["Fresh groups were not used for model selection or production refit.",
|
|
351
|
+
"User must ensure these sources have not informed earlier design choices."]}
|
|
352
|
+
features, _, runtimes, weights = _records(datasets)
|
|
353
|
+
schema = sorted(datasets[0]["genome"])
|
|
354
|
+
artifact = {"format_version": 1, "genome_version": GENOME_VERSION, "pipeline_version": PIPELINE_VERSION,
|
|
355
|
+
"library_signature": library_signature(), "sklearn_version": sklearn.__version__,
|
|
356
|
+
"performance_model": production,
|
|
357
|
+
"runtime_model": _new_regressor().fit(features, runtimes, regress__sample_weight=weights),
|
|
358
|
+
"genome_schema": schema, "training_groups": sorted({d["group"] for d in datasets}),
|
|
359
|
+
"training_datasets": [d["id"] for d in datasets],
|
|
360
|
+
"feature_ranges": {k: [min(d["genome"][k] for d in datasets),
|
|
361
|
+
max(d["genome"][k] for d in datasets)] for k in schema},
|
|
362
|
+
"evaluation": report}
|
|
363
|
+
report["production_convergence_warnings"] = production.convergence_warnings
|
|
364
|
+
joblib.dump(artifact, output / "trained_meta_model.joblib")
|
|
365
|
+
write_json(output / "neural_evaluation.json", report)
|
|
366
|
+
write_comparison_summary(report, output)
|
|
367
|
+
if fresh:
|
|
368
|
+
write_comparison_summary(report["fresh_evaluation"], output, "fresh_comparison_summary.csv")
|
|
369
|
+
print(json.dumps({"selected_alpha": alpha, "comparisons": {m: c["mean"]
|
|
370
|
+
for m, c in report["comparisons"].items()},
|
|
371
|
+
"fresh_evaluation_status": report["fresh_evaluation"]["status"]}, indent=2))
|
|
372
|
+
return artifact
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def main(argv=None):
|
|
376
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
377
|
+
parser.add_argument("benchmark")
|
|
378
|
+
parser.add_argument("--output", default="results/neural_meta_model")
|
|
379
|
+
parser.add_argument("--historical-evaluation", default="results/improved_meta_model/improvement_evaluation.json")
|
|
380
|
+
parser.add_argument("--previous-model")
|
|
381
|
+
parser.add_argument("--fresh-benchmark", help="Optional complete benchmark of genuinely new source groups")
|
|
382
|
+
parser.add_argument("--fresh-reference-model", action="append", default=[], metavar="saved_LABEL=PATH",
|
|
383
|
+
help="Frozen model scored only on fresh data; may repeat for historical models")
|
|
384
|
+
parser.add_argument("--repeats", type=int, default=3)
|
|
385
|
+
args = parser.parse_args(argv)
|
|
386
|
+
references = {}
|
|
387
|
+
for item in args.fresh_reference_model:
|
|
388
|
+
label, separator, path = item.partition("=")
|
|
389
|
+
if not separator or not label or not path or label in references:
|
|
390
|
+
parser.error("Each fresh reference must be a unique saved_LABEL=PATH")
|
|
391
|
+
references[label] = path
|
|
392
|
+
try:
|
|
393
|
+
run_neural_experiment(args.benchmark, args.output, args.historical_evaluation,
|
|
394
|
+
args.previous_model, args.fresh_benchmark, args.repeats, references)
|
|
395
|
+
except (ValueError, OSError, KeyError) as exc:
|
|
396
|
+
parser.exit(1, f"Error: {exc}\n")
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
if __name__ == "__main__":
|
|
400
|
+
# Keep custom model classes importable when joblib is loaded elsewhere.
|
|
401
|
+
from .train_neural_recommender import main as imported_main
|
|
402
|
+
imported_main()
|