algorithm-genome 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- algorithm_genome/__init__.py +13 -0
- algorithm_genome/__main__.py +3 -0
- algorithm_genome/algorithm_library.py +60 -0
- algorithm_genome/assets/default_recommender.joblib +0 -0
- algorithm_genome/assets/model_info.json +28 -0
- algorithm_genome/benchmark_datasets.py +130 -0
- algorithm_genome/bundled_model.py +27 -0
- algorithm_genome/command_line.py +116 -0
- algorithm_genome/create_example_datasets.py +35 -0
- algorithm_genome/dataset_genome.py +110 -0
- algorithm_genome/download_public_datasets.py +251 -0
- algorithm_genome/framework.py +101 -0
- algorithm_genome/improve_meta_model.py +186 -0
- algorithm_genome/model_pipeline.py +68 -0
- algorithm_genome/prepare_additional_datasets.py +105 -0
- algorithm_genome/recommend_algorithms.py +177 -0
- algorithm_genome/terminal_output.py +270 -0
- algorithm_genome/train_meta_model.py +142 -0
- algorithm_genome/train_neural_recommender.py +402 -0
- algorithm_genome-0.3.0.dist-info/METADATA +223 -0
- algorithm_genome-0.3.0.dist-info/RECORD +24 -0
- algorithm_genome-0.3.0.dist-info/WHEEL +5 -0
- algorithm_genome-0.3.0.dist-info/entry_points.txt +2 -0
- algorithm_genome-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Download a reproducible OpenML classification corpus with source metadata."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import socket
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
from sklearn.datasets import fetch_openml
|
|
12
|
+
from sklearn.model_selection import train_test_split
|
|
13
|
+
|
|
14
|
+
from .benchmark_datasets import write_json
|
|
15
|
+
from .dataset_genome import prepare_classification_data
|
|
16
|
+
|
|
17
|
+
# Dataset IDs identify specific OpenML versions. Related datasets share a group.
|
|
18
|
+
# These are a selected corpus, not a claim to reproduce the complete CC18 suite.
|
|
19
|
+
PUBLIC_DATASETS = [
|
|
20
|
+
(3, "chess_king_rook", "chess"),
|
|
21
|
+
(6, "letter_recognition", "letter_recognition"),
|
|
22
|
+
(11, "balance_scale", "balance_scale"),
|
|
23
|
+
(12, "multiple_features", "multiple_features"),
|
|
24
|
+
(21, "car_evaluation", "car_evaluation"),
|
|
25
|
+
(23, "contraceptive_method", "contraceptive_method"),
|
|
26
|
+
(24, "mushroom", "mushroom"),
|
|
27
|
+
(29, "credit_approval", "credit_approval"),
|
|
28
|
+
(31, "german_credit", "german_credit"),
|
|
29
|
+
(32, "pen_digits", "pen_digits"),
|
|
30
|
+
(37, "diabetes", "diabetes"),
|
|
31
|
+
(40, "sonar", "sonar"),
|
|
32
|
+
(41, "glass", "glass"),
|
|
33
|
+
(43, "haberman", "haberman"),
|
|
34
|
+
(44, "spambase", "spambase"),
|
|
35
|
+
(46, "splice", "splice"),
|
|
36
|
+
(50, "tic_tac_toe", "tic_tac_toe"),
|
|
37
|
+
(54, "vehicle", "vehicle"),
|
|
38
|
+
(59, "ionosphere", "ionosphere"),
|
|
39
|
+
(62, "zoo", "zoo"),
|
|
40
|
+
(1462, "banknote", "banknote"),
|
|
41
|
+
(1480, "liver_patients", "liver_patients"),
|
|
42
|
+
(1494, "biodegradation", "biodegradation"),
|
|
43
|
+
(1504, "steel_faults", "steel_faults"),
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
# Snapshot of the official OpenML-CC18 study's data IDs, retrieved 2026-11-15:
|
|
47
|
+
# https://www.openml.org/api/v1/json/study/99
|
|
48
|
+
CC18_DATASET_IDS = (
|
|
49
|
+
3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 44, 46, 50,
|
|
50
|
+
54, 151, 182, 188, 38, 307, 300, 458, 469, 554, 1049, 1050, 1053, 1063,
|
|
51
|
+
1067, 1068, 1590, 4134, 1510, 1489, 1494, 1497, 1501, 1480, 1485, 1486,
|
|
52
|
+
1487, 1468, 1475, 1462, 1464, 4534, 6332, 1461, 4538, 1478, 23381, 40499,
|
|
53
|
+
40668, 40966, 40982, 40994, 40983, 40975, 40984, 40979, 40996, 41027,
|
|
54
|
+
23517, 40923, 40927, 40978, 40670, 40701,
|
|
55
|
+
)
|
|
56
|
+
UNSUPPORTED_SOURCES = {
|
|
57
|
+
151: "Electricity is time-ordered; this project does not implement time-series validation",
|
|
58
|
+
40966: "MiceProtein has repeated measurements per mouse; grouped-observation CV is not implemented",
|
|
59
|
+
182: "Satimage has overlapping spatial pixel neighborhoods; spatial validation is not implemented",
|
|
60
|
+
307: "Vowel has repeated utterances per speaker; speaker-grouped validation is not implemented",
|
|
61
|
+
1497: "Robot navigation contains sequential sensor readings; time-series validation is not implemented",
|
|
62
|
+
1487: "Ozone contains daily weather observations; time-series validation is not implemented",
|
|
63
|
+
4538: "Gesture data contains frames from seven videos; video-grouped validation is not implemented",
|
|
64
|
+
40499: "Texture contains samples from shared source textures; image-grouped validation is not implemented",
|
|
65
|
+
40984: "Segment contains regions from seven source images; image-grouped validation is not implemented",
|
|
66
|
+
23517: "Numerai contains encrypted stock-market data without time/entity split metadata; suitable validation is unavailable",
|
|
67
|
+
}
|
|
68
|
+
SOURCE_GROUP_OVERRIDES = {
|
|
69
|
+
14: "multiple_features", 16: "multiple_features", 18: "multiple_features",
|
|
70
|
+
22: "multiple_features", 40979: "multiple_features", 28: "digits",
|
|
71
|
+
15: "breast_cancer", 1510: "breast_cancer", 40670: "splice",
|
|
72
|
+
40975: "car_evaluation", 40982: "steel_faults",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def dataset_catalog(suite="selected"):
|
|
77
|
+
if suite not in {"selected", "cc18"}:
|
|
78
|
+
raise ValueError("suite must be selected or cc18")
|
|
79
|
+
catalog = list(PUBLIC_DATASETS)
|
|
80
|
+
if suite == "cc18":
|
|
81
|
+
existing = {identifier for identifier, _, _ in catalog}
|
|
82
|
+
# Different representations of the same source must stay together.
|
|
83
|
+
catalog.extend((i, f"cc18_dataset_{i}", SOURCE_GROUP_OVERRIDES.get(i, f"openml_source_{i}"))
|
|
84
|
+
for i in CC18_DATASET_IDS if i not in existing)
|
|
85
|
+
return catalog
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _prepare_download(bunch, max_rows):
|
|
89
|
+
if not isinstance(bunch.target, pd.Series):
|
|
90
|
+
raise ValueError("Only datasets with one target are supported")
|
|
91
|
+
X = bunch.data.copy()
|
|
92
|
+
X.columns = X.columns.map(str)
|
|
93
|
+
if X.shape[1] > 250:
|
|
94
|
+
raise ValueError("More than 250 predictors; outside this bounded benchmark")
|
|
95
|
+
categorical = []
|
|
96
|
+
for column in X:
|
|
97
|
+
if not pd.api.types.is_numeric_dtype(X[column]):
|
|
98
|
+
categorical.append(column)
|
|
99
|
+
# Prevent CSV readers from reinterpreting nominal numeric codes as numbers.
|
|
100
|
+
X[column] = X[column].astype(object).map(lambda v: f"category:{v}" if pd.notna(v) else None)
|
|
101
|
+
target = "__classification_target__"
|
|
102
|
+
if target in X:
|
|
103
|
+
raise ValueError("Reserved target column already exists")
|
|
104
|
+
y = bunch.target.map(lambda v: f"class:{v}" if pd.notna(v) else None)
|
|
105
|
+
frame = X.assign(**{target: y}).dropna(subset=[target]).drop_duplicates()
|
|
106
|
+
original_rows = len(bunch.data)
|
|
107
|
+
clean_rows = len(frame)
|
|
108
|
+
if len(frame) > max_rows:
|
|
109
|
+
frame, _ = train_test_split(frame, train_size=max_rows, stratify=frame[target], random_state=42)
|
|
110
|
+
frame = frame.reset_index(drop=True)
|
|
111
|
+
prepare_classification_data(frame, target) # Fail explicitly on unusable classes.
|
|
112
|
+
return frame, {"target": target, "original_rows": original_rows,
|
|
113
|
+
"distinct_labeled_rows": clean_rows, "benchmark_rows": len(frame),
|
|
114
|
+
"categorical_columns": categorical, "sample_seed": 42,
|
|
115
|
+
"sampling": "stratified" if clean_rows > max_rows else "all_distinct_labeled_rows"}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _save_download_state(directory, max_rows, records, ready, catalog):
|
|
119
|
+
"""Replace each JSON file atomically, keeping earlier successes on short runs."""
|
|
120
|
+
report = {"max_rows": max_rows, "downloads": [records[i] for i, _, _ in catalog if i in records]}
|
|
121
|
+
entries = []
|
|
122
|
+
source_groups = {ready[i].get("source_name"): group for i, _, group in PUBLIC_DATASETS if i in ready}
|
|
123
|
+
for identifier, label, group in catalog:
|
|
124
|
+
if identifier in ready:
|
|
125
|
+
name = ready[identifier].get("source_name") or ""
|
|
126
|
+
if name.startswith("mfeat-"):
|
|
127
|
+
group = "multiple_features"
|
|
128
|
+
else:
|
|
129
|
+
group = source_groups.get(name, group)
|
|
130
|
+
entries.append({"id": f"openml_{identifier}_{label}", "group": group,
|
|
131
|
+
"path": f"openml_{identifier}_{label}.csv",
|
|
132
|
+
"target": ready[identifier]["target"], "source": ready[identifier]})
|
|
133
|
+
for name, value in [("download_report.json", report), ("dataset_manifest.json", entries)]:
|
|
134
|
+
destination = directory / name
|
|
135
|
+
temporary = destination.with_suffix(".json.tmp")
|
|
136
|
+
write_json(temporary, value)
|
|
137
|
+
temporary.replace(destination)
|
|
138
|
+
return entries
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def download_public_datasets(directory="data/public_datasets", max_rows=2000, limit=None, retries=3,
|
|
142
|
+
suite="selected"):
|
|
143
|
+
if max_rows < 30:
|
|
144
|
+
raise ValueError("max_rows must be at least 30")
|
|
145
|
+
if limit is not None and limit < 1:
|
|
146
|
+
raise ValueError("limit must be positive")
|
|
147
|
+
if retries < 1:
|
|
148
|
+
raise ValueError("retries must be at least 1")
|
|
149
|
+
catalog = dataset_catalog(suite)
|
|
150
|
+
directory = Path(directory).resolve()
|
|
151
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
152
|
+
report_path = directory / "download_report.json"
|
|
153
|
+
previous = json.loads(report_path.read_text()) if report_path.exists() else {"downloads": []}
|
|
154
|
+
records = {d["openml_id"]: d for d in previous["downloads"]}
|
|
155
|
+
ready = {}
|
|
156
|
+
# Preserve the expanded cache/manifest even when the next command uses the
|
|
157
|
+
# smaller default catalog. --suite controls attempts, not deletion of data.
|
|
158
|
+
state_catalog = dataset_catalog("cc18")
|
|
159
|
+
for identifier, label, _ in state_catalog:
|
|
160
|
+
old = records.get(identifier)
|
|
161
|
+
destination = directory / f"openml_{identifier}_{label}.csv"
|
|
162
|
+
if identifier in UNSUPPORTED_SOURCES:
|
|
163
|
+
if old:
|
|
164
|
+
records[identifier] = {"openml_id": identifier, "status": "failed",
|
|
165
|
+
"error": UNSUPPORTED_SOURCES[identifier]}
|
|
166
|
+
continue
|
|
167
|
+
if old and old.get("status") == "ok":
|
|
168
|
+
if (old.get("max_rows") == max_rows and destination.is_file()
|
|
169
|
+
and hashlib.sha256(destination.read_bytes()).hexdigest() == old.get("sha256")):
|
|
170
|
+
ready[identifier] = old
|
|
171
|
+
else:
|
|
172
|
+
# Do not advertise a missing, changed, or differently sampled file.
|
|
173
|
+
records.pop(identifier)
|
|
174
|
+
selected = catalog[:limit]
|
|
175
|
+
entries = _save_download_state(directory, max_rows, records, ready, state_catalog)
|
|
176
|
+
for index, (identifier, label, group) in enumerate(selected, start=1):
|
|
177
|
+
filename = f"openml_{identifier}_{label}.csv"
|
|
178
|
+
destination = directory / filename
|
|
179
|
+
try:
|
|
180
|
+
if identifier in UNSUPPORTED_SOURCES:
|
|
181
|
+
raise ValueError(UNSUPPORTED_SOURCES[identifier])
|
|
182
|
+
if identifier in ready:
|
|
183
|
+
metadata = ready[identifier]
|
|
184
|
+
print(f"[{index}/{len(selected)}] Using cached dataset {identifier}: {label}", flush=True)
|
|
185
|
+
else:
|
|
186
|
+
print(f"[{index}/{len(selected)}] Downloading OpenML {identifier}: {label}", flush=True)
|
|
187
|
+
# pandas can infer integer/bool category values; sklearn's pandas
|
|
188
|
+
# ARFF parser then applies a string regex to them and fails.
|
|
189
|
+
# The ARFF parser preserves nominal codes as labels instead.
|
|
190
|
+
bunch = fetch_openml(data_id=identifier, as_frame=True, parser="liac-arff",
|
|
191
|
+
data_home=str(directory / "openml_cache"), n_retries=retries, delay=1.0)
|
|
192
|
+
frame, preparation = _prepare_download(bunch, max_rows)
|
|
193
|
+
temporary = destination.with_suffix(".csv.tmp")
|
|
194
|
+
frame.to_csv(temporary, index=False)
|
|
195
|
+
temporary.replace(destination)
|
|
196
|
+
metadata = {"openml_id": identifier, "status": "ok", "max_rows": max_rows,
|
|
197
|
+
"source_url": f"https://www.openml.org/d/{identifier}",
|
|
198
|
+
"source_name": bunch.details.get("name"),
|
|
199
|
+
"source_version": bunch.details.get("version"),
|
|
200
|
+
"license": bunch.details.get("licence", "not provided by source metadata"),
|
|
201
|
+
"description": bunch.DESCR, **preparation,
|
|
202
|
+
"sha256": hashlib.sha256(destination.read_bytes()).hexdigest()}
|
|
203
|
+
records[identifier] = metadata
|
|
204
|
+
ready[identifier] = metadata
|
|
205
|
+
except Exception as exc:
|
|
206
|
+
ready.pop(identifier, None)
|
|
207
|
+
records[identifier] = {"openml_id": identifier, "status": "failed", "error": str(exc)}
|
|
208
|
+
print(f"Failed {identifier}: {exc}", file=sys.stderr, flush=True)
|
|
209
|
+
entries = _save_download_state(directory, max_rows, records, ready, state_catalog)
|
|
210
|
+
if not entries:
|
|
211
|
+
raise RuntimeError("No datasets downloaded. See download_report.json; check network access to OpenML.")
|
|
212
|
+
return entries
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def main(argv=None, default_output="data/public_datasets"):
|
|
216
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
217
|
+
parser.add_argument("--output", default=str(default_output), help="Folder for CSVs, manifest, and source cache")
|
|
218
|
+
parser.add_argument("--max-rows", type=int, default=2000, help="Maximum prepared rows per dataset (default: 2000)")
|
|
219
|
+
parser.add_argument("--limit", type=int, help="Attempt only the first N datasets; keep earlier downloads")
|
|
220
|
+
parser.add_argument("--retries", type=int, default=3, help="OpenML client retries (at least 1; default: 3)")
|
|
221
|
+
parser.add_argument("--suite", choices=["selected", "cc18"], default="selected",
|
|
222
|
+
help="Planned 24 datasets, or those plus additional OpenML-CC18 sources")
|
|
223
|
+
parser.add_argument("--timeout", type=float, default=30, help="Socket timeout in seconds (default: 30)")
|
|
224
|
+
args = parser.parse_args(argv)
|
|
225
|
+
if not 0 < args.timeout < float("inf"):
|
|
226
|
+
parser.error("timeout must be a positive finite number")
|
|
227
|
+
previous_timeout = socket.getdefaulttimeout()
|
|
228
|
+
socket.setdefaulttimeout(args.timeout)
|
|
229
|
+
try:
|
|
230
|
+
entries = download_public_datasets(args.output, args.max_rows, args.limit, args.retries, args.suite)
|
|
231
|
+
successful = {entry["source"]["openml_id"] for entry in entries}
|
|
232
|
+
failed = [i for i, _, _ in dataset_catalog(args.suite)[:args.limit] if i not in successful]
|
|
233
|
+
print(f"\nReady: {len(entries)} datasets. Failed in this run: {len(failed)}.")
|
|
234
|
+
print(f"Training manifest: {Path(args.output).resolve() / 'dataset_manifest.json'}")
|
|
235
|
+
if failed:
|
|
236
|
+
print("Rerun the same command to retry failures; completed downloads will be reused.")
|
|
237
|
+
return 1
|
|
238
|
+
print("Downloads complete. Next, benchmark the datasets before training the recommender.")
|
|
239
|
+
return 0
|
|
240
|
+
except KeyboardInterrupt:
|
|
241
|
+
print("\nStopped. Completed datasets are saved; rerun the same command to continue.", file=sys.stderr)
|
|
242
|
+
return 130
|
|
243
|
+
except (ValueError, RuntimeError) as exc:
|
|
244
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
245
|
+
return 1
|
|
246
|
+
finally:
|
|
247
|
+
socket.setdefaulttimeout(previous_timeout)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
if __name__ == "__main__":
|
|
251
|
+
sys.exit(main())
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Public Python interface. All computation stays in memory until explicitly saved."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Mapping
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from os import PathLike
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import joblib
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import sklearn
|
|
13
|
+
|
|
14
|
+
from .benchmark_datasets import write_json
|
|
15
|
+
from .bundled_model import load_bundled_model
|
|
16
|
+
from .recommend_algorithms import TrainedClassifier, recommend_algorithms, verify_recommendations
|
|
17
|
+
from .train_meta_model import check_compatibility, train_meta_model
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class VerificationResult:
|
|
22
|
+
"""Measured candidate results and a classifier refitted on all usable rows."""
|
|
23
|
+
|
|
24
|
+
report: dict[str, Any]
|
|
25
|
+
classifier: TrainedClassifier
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def selected_algorithm(self) -> str:
|
|
29
|
+
return self.report["selected_algorithm"]
|
|
30
|
+
|
|
31
|
+
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
|
32
|
+
return self.classifier.predict(data)
|
|
33
|
+
|
|
34
|
+
def save(self, directory: str | PathLike) -> Path:
|
|
35
|
+
"""Save the report, ranking CSV, and reloadable selected classifier."""
|
|
36
|
+
directory = Path(directory)
|
|
37
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
write_json(directory / "recommendation_report.json", self.report)
|
|
39
|
+
pd.DataFrame(self.report["ranking"]).to_csv(directory / "predicted_algorithm_ranking.csv", index=False)
|
|
40
|
+
self.classifier.save(directory / "selected_classifier.joblib")
|
|
41
|
+
return directory
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Recommender:
|
|
45
|
+
"""Learn from historical benchmarks and rank algorithms for labeled tables.
|
|
46
|
+
|
|
47
|
+
Use ``Recommender()`` for the bundled pretrained model, ``fit(benchmark)``
|
|
48
|
+
to train your own, or ``load(path)`` to open a custom model.
|
|
49
|
+
Recommendations do not fit candidate classifiers. ``verify`` measures them
|
|
50
|
+
using fold-local preprocessing and a reserved holdout before the final refit.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(self, artifact: Mapping[str, Any] | None = None):
|
|
54
|
+
if artifact is None:
|
|
55
|
+
artifact = load_bundled_model()
|
|
56
|
+
if not isinstance(artifact, Mapping):
|
|
57
|
+
raise ValueError("Expected a trained recommender artifact")
|
|
58
|
+
check_compatibility(artifact)
|
|
59
|
+
if artifact.get("sklearn_version") != sklearn.__version__:
|
|
60
|
+
raise ValueError("Use the same scikit-learn version as meta-model training")
|
|
61
|
+
self.artifact = dict(artifact)
|
|
62
|
+
|
|
63
|
+
@classmethod
|
|
64
|
+
def fit(cls, benchmark: Mapping[str, Any] | str | PathLike) -> "Recommender":
|
|
65
|
+
"""Train and evaluate on separate source groups, then refit on all groups.
|
|
66
|
+
|
|
67
|
+
At least four independent source groups with complete candidate results
|
|
68
|
+
are required. This method writes no files and returns a new recommender.
|
|
69
|
+
"""
|
|
70
|
+
return cls(train_meta_model(benchmark))
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def load(cls, path: str | PathLike) -> "Recommender":
|
|
74
|
+
"""Load a trusted joblib artifact from this or earlier project versions."""
|
|
75
|
+
return cls(joblib.load(path))
|
|
76
|
+
|
|
77
|
+
def save(self, path: str | PathLike) -> Path:
|
|
78
|
+
"""Save a compatible artifact without changing its underlying models."""
|
|
79
|
+
path = Path(path)
|
|
80
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
81
|
+
joblib.dump(self.artifact, path)
|
|
82
|
+
return path
|
|
83
|
+
|
|
84
|
+
def recommend(self, data: pd.DataFrame, target: str) -> dict[str, Any]:
|
|
85
|
+
"""Return the predicted ranking, genome, data summary, and model warnings."""
|
|
86
|
+
return recommend_algorithms(self.artifact, data, target)
|
|
87
|
+
|
|
88
|
+
def verify(
|
|
89
|
+
self, data: pd.DataFrame, target: str, *, top_k: int = 3,
|
|
90
|
+
budget_seconds: float = 300,
|
|
91
|
+
progress: Callable[[dict[str, Any]], None] | None = None,
|
|
92
|
+
) -> VerificationResult:
|
|
93
|
+
"""Measure candidates in ranked order within a soft runtime budget.
|
|
94
|
+
|
|
95
|
+
``top_k`` controls the initial shortlist, followed by a different-family
|
|
96
|
+
challenger and remaining candidates if time permits. Progress callbacks
|
|
97
|
+
receive event dictionaries; library calls otherwise print nothing.
|
|
98
|
+
"""
|
|
99
|
+
report, classifier = verify_recommendations(
|
|
100
|
+
self.artifact, data, target, top_k, budget_seconds, progress)
|
|
101
|
+
return VerificationResult(report, classifier)
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Choose a recommender on development datasets, then evaluate once on held-out groups."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import joblib
|
|
8
|
+
import numpy as np
|
|
9
|
+
import sklearn
|
|
10
|
+
from sklearn.ensemble import ExtraTreesRegressor
|
|
11
|
+
from sklearn.feature_extraction import DictVectorizer
|
|
12
|
+
from sklearn.model_selection import GroupKFold, GroupShuffleSplit
|
|
13
|
+
from sklearn.neighbors import KNeighborsRegressor
|
|
14
|
+
from sklearn.pipeline import make_pipeline
|
|
15
|
+
from sklearn.preprocessing import StandardScaler
|
|
16
|
+
|
|
17
|
+
from .algorithm_library import algorithm_library, library_signature, meta_features
|
|
18
|
+
from .benchmark_datasets import write_json
|
|
19
|
+
from .dataset_genome import GENOME_VERSION
|
|
20
|
+
from .model_pipeline import PIPELINE_VERSION
|
|
21
|
+
from .train_meta_model import _new_regressor, _ranking_metrics, check_compatibility
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DatasetRankingModel:
|
|
25
|
+
"""Predict every algorithm jointly from a dataset, with optional prior shrinkage."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, method="joint_forest"):
|
|
28
|
+
self.method = method
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def _features(genome):
|
|
32
|
+
return {key: np.log1p(value) if key in {
|
|
33
|
+
"n_rows", "n_features", "n_numeric", "n_categorical", "n_classes", "imbalance_ratio"
|
|
34
|
+
} else value for key, value in genome.items()}
|
|
35
|
+
|
|
36
|
+
def fit(self, datasets):
|
|
37
|
+
self.names = list(algorithm_library())
|
|
38
|
+
features, scores, weights = [], [], []
|
|
39
|
+
for dataset in datasets:
|
|
40
|
+
features.append(self._features(dataset["genome"]))
|
|
41
|
+
results = {r["algorithm"]: r["cv_macro_f1"] for r in dataset["results"]}
|
|
42
|
+
row = np.array([results[n] for n in self.names])
|
|
43
|
+
scores.append(row - row.mean())
|
|
44
|
+
weights.append(1 / sum(d["group"] == dataset["group"] for d in datasets))
|
|
45
|
+
self.prior = np.average(scores, axis=0, weights=weights)
|
|
46
|
+
if self.method == "fixed_portfolio":
|
|
47
|
+
return self
|
|
48
|
+
if self.method == "nearest_datasets":
|
|
49
|
+
self.model = make_pipeline(DictVectorizer(sparse=False), StandardScaler(),
|
|
50
|
+
KNeighborsRegressor(n_neighbors=min(3, len(datasets)), weights="distance"))
|
|
51
|
+
self.model.fit(features, scores)
|
|
52
|
+
else:
|
|
53
|
+
self.model = make_pipeline(DictVectorizer(sparse=False), ExtraTreesRegressor(
|
|
54
|
+
n_estimators=150, min_samples_leaf=2, max_features=1.0, random_state=42, n_jobs=1))
|
|
55
|
+
self.model.fit(features, scores, extratreesregressor__sample_weight=weights)
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
def predict(self, records):
|
|
59
|
+
features = [self._features({k.removeprefix("dataset_"): v for k, v in r.items()
|
|
60
|
+
if k.startswith("dataset_")}) for r in records]
|
|
61
|
+
scores = (np.tile(self.prior, (len(records), 1)) if self.method == "fixed_portfolio"
|
|
62
|
+
else self.model.predict(features))
|
|
63
|
+
if self.method == "shrunk_joint_forest":
|
|
64
|
+
scores = 0.5 * scores + 0.5 * self.prior
|
|
65
|
+
return np.array([scores[i, self.names.index(r["algorithm_name"])] for i, r in enumerate(records)])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _records(datasets):
|
|
69
|
+
features, targets, runtimes, weights = [], [], [], []
|
|
70
|
+
for d in datasets:
|
|
71
|
+
mean = np.mean([r["cv_macro_f1"] for r in d["results"]])
|
|
72
|
+
for r in d["results"]:
|
|
73
|
+
features.append(meta_features(d["genome"], r["algorithm"]))
|
|
74
|
+
targets.append(r["cv_macro_f1"] - mean)
|
|
75
|
+
runtimes.append(np.log1p(r["cv_seconds"]))
|
|
76
|
+
weights.append(1 / sum(other["group"] == d["group"] for other in datasets))
|
|
77
|
+
return features, targets, runtimes, weights
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _fit(method, datasets):
|
|
81
|
+
if method == "original_pairwise_forest":
|
|
82
|
+
features, targets, _, weights = _records(datasets)
|
|
83
|
+
return _new_regressor().fit(features, targets, regress__sample_weight=weights)
|
|
84
|
+
return DatasetRankingModel(method).fit(datasets)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _evaluate(model, datasets):
|
|
88
|
+
return _ranking_metrics(datasets, lambda d, names: dict(zip(names,
|
|
89
|
+
model.predict([meta_features(d["genome"], n) for n in names]))))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def improve_meta_model(benchmark_path, output_directory, previous_model_path=None):
|
|
93
|
+
benchmark = json.loads(Path(benchmark_path).read_text())
|
|
94
|
+
check_compatibility(benchmark)
|
|
95
|
+
names = set(algorithm_library())
|
|
96
|
+
datasets, excluded = [], []
|
|
97
|
+
for dataset in benchmark["datasets"]:
|
|
98
|
+
results = dataset["results"]
|
|
99
|
+
if (len(results) != len(names) or {r["algorithm"] for r in results} != names
|
|
100
|
+
or any(r["status"] != "ok" for r in results)):
|
|
101
|
+
excluded.append(dataset["id"])
|
|
102
|
+
continue
|
|
103
|
+
if not all(np.isfinite(r["cv_macro_f1"]) and 0 <= r["cv_macro_f1"] <= 1
|
|
104
|
+
and np.isfinite(r["cv_seconds"]) and r["cv_seconds"] > 0 for r in results):
|
|
105
|
+
raise ValueError("Invalid benchmark score or duration")
|
|
106
|
+
datasets.append(dataset)
|
|
107
|
+
if len({d["id"] for d in datasets}) != len(datasets) or len({d["fingerprint"] for d in datasets}) != len(datasets):
|
|
108
|
+
raise ValueError("Duplicate datasets in benchmark")
|
|
109
|
+
groups = [d["group"] for d in datasets]
|
|
110
|
+
if len(set(groups)) < 8:
|
|
111
|
+
raise ValueError("Improvement experiments require at least eight complete independent dataset groups")
|
|
112
|
+
schema = sorted(datasets[0]["genome"])
|
|
113
|
+
if any(sorted(d["genome"]) != schema or not all(np.isfinite(v) for v in d["genome"].values()) for d in datasets):
|
|
114
|
+
raise ValueError("Inconsistent genome schema or non-finite values")
|
|
115
|
+
development_indices, final_indices = next(GroupShuffleSplit(
|
|
116
|
+
n_splits=1, test_size=0.25, random_state=2026).split(datasets, groups=groups))
|
|
117
|
+
development = [datasets[i] for i in development_indices]
|
|
118
|
+
final = [datasets[i] for i in final_indices]
|
|
119
|
+
output = Path(output_directory)
|
|
120
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
121
|
+
split = {"development_groups": sorted({d["group"] for d in development}),
|
|
122
|
+
"final_evaluation_groups": sorted({d["group"] for d in final})}
|
|
123
|
+
write_json(output / "evaluation_split.json", split)
|
|
124
|
+
methods = ["fixed_portfolio", "original_pairwise_forest", "joint_forest",
|
|
125
|
+
"shrunk_joint_forest", "nearest_datasets"]
|
|
126
|
+
development_groups = [d["group"] for d in development]
|
|
127
|
+
inner_splits = list(GroupKFold(n_splits=3).split(development, groups=development_groups))
|
|
128
|
+
selection = []
|
|
129
|
+
for method in methods:
|
|
130
|
+
outcomes = []
|
|
131
|
+
for train, valid in inner_splits:
|
|
132
|
+
model = _fit(method, [development[i] for i in train])
|
|
133
|
+
outcomes.extend(_evaluate(model, [development[i] for i in valid])["datasets"])
|
|
134
|
+
# Average within source group first so variants do not dominate selection.
|
|
135
|
+
group_regrets = [np.mean([r["regret"] for r in outcomes if r["group"] == group])
|
|
136
|
+
for group in sorted(set(development_groups))]
|
|
137
|
+
selection.append({"method": method, "mean_group_regret": float(np.mean(group_regrets)),
|
|
138
|
+
"top_1_rate": float(np.mean([r["top_1"] for r in outcomes])),
|
|
139
|
+
"out_of_fold_datasets": outcomes})
|
|
140
|
+
selected = min(selection, key=lambda r: r["mean_group_regret"])["method"]
|
|
141
|
+
evaluated = _fit(selected, development)
|
|
142
|
+
report = {**split, "selected_method": selected, "development_comparison": selection,
|
|
143
|
+
"final_evaluation": _evaluate(evaluated, final),
|
|
144
|
+
"retrained_original_baseline": _evaluate(_fit("original_pairwise_forest", development), final),
|
|
145
|
+
"fixed_portfolio_baseline": _evaluate(_fit("fixed_portfolio", development), final),
|
|
146
|
+
"excluded_incomplete_datasets": excluded,
|
|
147
|
+
"notes": ["Method selected using only development-group cross-validation regret.",
|
|
148
|
+
"Final evaluation groups were not used to select the method.",
|
|
149
|
+
"Regret uses matched candidate CV macro-F1, not independent classifier test accuracy.",
|
|
150
|
+
"The saved recommender is refitted on the complete corpus after evaluation.",
|
|
151
|
+
"A fixed portfolio may win: additional complexity is not presumed to help.",
|
|
152
|
+
"This is a selected, sampled public corpus, not the full OpenML-CC18 protocol."]}
|
|
153
|
+
if previous_model_path:
|
|
154
|
+
old = joblib.load(previous_model_path)
|
|
155
|
+
check_compatibility(old)
|
|
156
|
+
if set(old["training_groups"]) & set(split["final_evaluation_groups"]):
|
|
157
|
+
raise ValueError("Previous model has trained on final evaluation groups")
|
|
158
|
+
report["previous_saved_model"] = _evaluate(old["performance_model"], final)
|
|
159
|
+
features, _, runtimes, weights = _records(datasets)
|
|
160
|
+
artifact = {"format_version": 1, "genome_version": GENOME_VERSION, "pipeline_version": PIPELINE_VERSION,
|
|
161
|
+
"library_signature": library_signature(), "sklearn_version": sklearn.__version__,
|
|
162
|
+
"performance_model": _fit(selected, datasets),
|
|
163
|
+
"runtime_model": _new_regressor().fit(features, runtimes, regress__sample_weight=weights),
|
|
164
|
+
"genome_schema": schema, "training_groups": sorted(set(groups)),
|
|
165
|
+
"training_datasets": [d["id"] for d in datasets],
|
|
166
|
+
"feature_ranges": {k: [min(d["genome"][k] for d in datasets), max(d["genome"][k] for d in datasets)] for k in schema},
|
|
167
|
+
"evaluation": report}
|
|
168
|
+
joblib.dump(artifact, output / "trained_meta_model.joblib")
|
|
169
|
+
write_json(output / "improvement_evaluation.json", report)
|
|
170
|
+
print(json.dumps({k: v for k, v in report.items() if k != "development_comparison"}, indent=2))
|
|
171
|
+
return artifact
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
176
|
+
parser.add_argument("benchmark")
|
|
177
|
+
parser.add_argument("--output", default="results/improved_meta_model")
|
|
178
|
+
parser.add_argument("--previous-model")
|
|
179
|
+
args = parser.parse_args()
|
|
180
|
+
try:
|
|
181
|
+
# Persist model classes under their importable module, not __main__,
|
|
182
|
+
# so a separate recommendation command can load the saved artifact.
|
|
183
|
+
from .improve_meta_model import improve_meta_model as run_experiment
|
|
184
|
+
run_experiment(args.benchmark, args.output, args.previous_model)
|
|
185
|
+
except (ValueError, OSError, KeyError) as exc:
|
|
186
|
+
parser.exit(1, f"Error: {exc}\n")
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Consistent preprocessing and model validation for benchmarks and verification."""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from sklearn.base import BaseEstimator, ClassifierMixin, clone
|
|
5
|
+
from sklearn.compose import ColumnTransformer, make_column_selector
|
|
6
|
+
from sklearn.impute import SimpleImputer
|
|
7
|
+
from sklearn.model_selection import StratifiedKFold, cross_val_score
|
|
8
|
+
from sklearn.pipeline import Pipeline
|
|
9
|
+
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler
|
|
10
|
+
from sklearn.utils.validation import check_is_fitted
|
|
11
|
+
|
|
12
|
+
from .algorithm_library import algorithm_library
|
|
13
|
+
|
|
14
|
+
PIPELINE_VERSION = 2
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LabelEncodedClassifier(ClassifierMixin, BaseEstimator):
|
|
18
|
+
"""Fit on integer labels and preserve original labels at the API boundary.
|
|
19
|
+
|
|
20
|
+
This avoids numeric-looking string labels being inconsistently coerced by
|
|
21
|
+
class-weight computation, while preserving distinctions such as '01'/'1'.
|
|
22
|
+
The encoder is fitted separately inside each cross-validation training fold.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, estimator):
|
|
26
|
+
self.estimator = estimator
|
|
27
|
+
|
|
28
|
+
def fit(self, X, y):
|
|
29
|
+
self.label_encoder_ = LabelEncoder()
|
|
30
|
+
encoded = self.label_encoder_.fit_transform(y)
|
|
31
|
+
self.estimator_ = clone(self.estimator).fit(X, encoded)
|
|
32
|
+
self.classes_ = self.label_encoder_.classes_
|
|
33
|
+
self.n_features_in_ = self.estimator_.n_features_in_
|
|
34
|
+
return self
|
|
35
|
+
|
|
36
|
+
def predict(self, X):
|
|
37
|
+
check_is_fitted(self, "estimator_")
|
|
38
|
+
return self.label_encoder_.inverse_transform(self.estimator_.predict(X))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def build_model_pipeline(algorithm_name):
|
|
42
|
+
numeric = Pipeline([
|
|
43
|
+
("impute", SimpleImputer(strategy="median", keep_empty_features=True)),
|
|
44
|
+
("scale", StandardScaler()),
|
|
45
|
+
])
|
|
46
|
+
categorical = Pipeline([
|
|
47
|
+
("impute", SimpleImputer(strategy="most_frequent", keep_empty_features=True)),
|
|
48
|
+
("encode", OneHotEncoder(handle_unknown="ignore", max_categories=32, sparse_output=False)),
|
|
49
|
+
])
|
|
50
|
+
preprocess = ColumnTransformer([
|
|
51
|
+
("numeric", numeric, make_column_selector(dtype_include=np.number)),
|
|
52
|
+
("categorical", categorical, make_column_selector(dtype_exclude=np.number)),
|
|
53
|
+
])
|
|
54
|
+
return Pipeline([("preprocess", preprocess),
|
|
55
|
+
("model", LabelEncodedClassifier(algorithm_library()[algorithm_name]["estimator"]))])
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def evaluate_candidate(name, X, y, folds=3):
|
|
59
|
+
folds = min(folds, int(y.value_counts().min()))
|
|
60
|
+
if folds < 2:
|
|
61
|
+
raise ValueError("Need at least two examples per class for validation")
|
|
62
|
+
splitter = StratifiedKFold(n_splits=folds, shuffle=True, random_state=42)
|
|
63
|
+
scores = cross_val_score(build_model_pipeline(name), X, y, cv=splitter,
|
|
64
|
+
scoring="f1_macro", n_jobs=1, error_score="raise")
|
|
65
|
+
if not np.isfinite(scores).all():
|
|
66
|
+
raise ValueError("Non-finite validation scores")
|
|
67
|
+
return {"cv_macro_f1": float(scores.mean()), "cv_std": float(scores.std()),
|
|
68
|
+
"fold_scores": scores.tolist(), "folds": folds}
|