algorithm-genome 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,251 @@
1
+ """Download a reproducible OpenML classification corpus with source metadata."""
2
+
3
+ import argparse
4
+ import hashlib
5
+ import json
6
+ import socket
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ import pandas as pd
11
+ from sklearn.datasets import fetch_openml
12
+ from sklearn.model_selection import train_test_split
13
+
14
+ from .benchmark_datasets import write_json
15
+ from .dataset_genome import prepare_classification_data
16
+
17
+ # Dataset IDs identify specific OpenML versions. Related datasets share a group.
18
+ # These are a selected corpus, not a claim to reproduce the complete CC18 suite.
19
+ PUBLIC_DATASETS = [
20
+ (3, "chess_king_rook", "chess"),
21
+ (6, "letter_recognition", "letter_recognition"),
22
+ (11, "balance_scale", "balance_scale"),
23
+ (12, "multiple_features", "multiple_features"),
24
+ (21, "car_evaluation", "car_evaluation"),
25
+ (23, "contraceptive_method", "contraceptive_method"),
26
+ (24, "mushroom", "mushroom"),
27
+ (29, "credit_approval", "credit_approval"),
28
+ (31, "german_credit", "german_credit"),
29
+ (32, "pen_digits", "pen_digits"),
30
+ (37, "diabetes", "diabetes"),
31
+ (40, "sonar", "sonar"),
32
+ (41, "glass", "glass"),
33
+ (43, "haberman", "haberman"),
34
+ (44, "spambase", "spambase"),
35
+ (46, "splice", "splice"),
36
+ (50, "tic_tac_toe", "tic_tac_toe"),
37
+ (54, "vehicle", "vehicle"),
38
+ (59, "ionosphere", "ionosphere"),
39
+ (62, "zoo", "zoo"),
40
+ (1462, "banknote", "banknote"),
41
+ (1480, "liver_patients", "liver_patients"),
42
+ (1494, "biodegradation", "biodegradation"),
43
+ (1504, "steel_faults", "steel_faults"),
44
+ ]
45
+
46
+ # Snapshot of the official OpenML-CC18 study's data IDs, retrieved 2026-11-15:
47
+ # https://www.openml.org/api/v1/json/study/99
48
+ CC18_DATASET_IDS = (
49
+ 3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 44, 46, 50,
50
+ 54, 151, 182, 188, 38, 307, 300, 458, 469, 554, 1049, 1050, 1053, 1063,
51
+ 1067, 1068, 1590, 4134, 1510, 1489, 1494, 1497, 1501, 1480, 1485, 1486,
52
+ 1487, 1468, 1475, 1462, 1464, 4534, 6332, 1461, 4538, 1478, 23381, 40499,
53
+ 40668, 40966, 40982, 40994, 40983, 40975, 40984, 40979, 40996, 41027,
54
+ 23517, 40923, 40927, 40978, 40670, 40701,
55
+ )
56
+ UNSUPPORTED_SOURCES = {
57
+ 151: "Electricity is time-ordered; this project does not implement time-series validation",
58
+ 40966: "MiceProtein has repeated measurements per mouse; grouped-observation CV is not implemented",
59
+ 182: "Satimage has overlapping spatial pixel neighborhoods; spatial validation is not implemented",
60
+ 307: "Vowel has repeated utterances per speaker; speaker-grouped validation is not implemented",
61
+ 1497: "Robot navigation contains sequential sensor readings; time-series validation is not implemented",
62
+ 1487: "Ozone contains daily weather observations; time-series validation is not implemented",
63
+ 4538: "Gesture data contains frames from seven videos; video-grouped validation is not implemented",
64
+ 40499: "Texture contains samples from shared source textures; image-grouped validation is not implemented",
65
+ 40984: "Segment contains regions from seven source images; image-grouped validation is not implemented",
66
+ 23517: "Numerai contains encrypted stock-market data without time/entity split metadata; suitable validation is unavailable",
67
+ }
68
+ SOURCE_GROUP_OVERRIDES = {
69
+ 14: "multiple_features", 16: "multiple_features", 18: "multiple_features",
70
+ 22: "multiple_features", 40979: "multiple_features", 28: "digits",
71
+ 15: "breast_cancer", 1510: "breast_cancer", 40670: "splice",
72
+ 40975: "car_evaluation", 40982: "steel_faults",
73
+ }
74
+
75
+
76
+ def dataset_catalog(suite="selected"):
77
+ if suite not in {"selected", "cc18"}:
78
+ raise ValueError("suite must be selected or cc18")
79
+ catalog = list(PUBLIC_DATASETS)
80
+ if suite == "cc18":
81
+ existing = {identifier for identifier, _, _ in catalog}
82
+ # Different representations of the same source must stay together.
83
+ catalog.extend((i, f"cc18_dataset_{i}", SOURCE_GROUP_OVERRIDES.get(i, f"openml_source_{i}"))
84
+ for i in CC18_DATASET_IDS if i not in existing)
85
+ return catalog
86
+
87
+
88
+ def _prepare_download(bunch, max_rows):
89
+ if not isinstance(bunch.target, pd.Series):
90
+ raise ValueError("Only datasets with one target are supported")
91
+ X = bunch.data.copy()
92
+ X.columns = X.columns.map(str)
93
+ if X.shape[1] > 250:
94
+ raise ValueError("More than 250 predictors; outside this bounded benchmark")
95
+ categorical = []
96
+ for column in X:
97
+ if not pd.api.types.is_numeric_dtype(X[column]):
98
+ categorical.append(column)
99
+ # Prevent CSV readers from reinterpreting nominal numeric codes as numbers.
100
+ X[column] = X[column].astype(object).map(lambda v: f"category:{v}" if pd.notna(v) else None)
101
+ target = "__classification_target__"
102
+ if target in X:
103
+ raise ValueError("Reserved target column already exists")
104
+ y = bunch.target.map(lambda v: f"class:{v}" if pd.notna(v) else None)
105
+ frame = X.assign(**{target: y}).dropna(subset=[target]).drop_duplicates()
106
+ original_rows = len(bunch.data)
107
+ clean_rows = len(frame)
108
+ if len(frame) > max_rows:
109
+ frame, _ = train_test_split(frame, train_size=max_rows, stratify=frame[target], random_state=42)
110
+ frame = frame.reset_index(drop=True)
111
+ prepare_classification_data(frame, target) # Fail explicitly on unusable classes.
112
+ return frame, {"target": target, "original_rows": original_rows,
113
+ "distinct_labeled_rows": clean_rows, "benchmark_rows": len(frame),
114
+ "categorical_columns": categorical, "sample_seed": 42,
115
+ "sampling": "stratified" if clean_rows > max_rows else "all_distinct_labeled_rows"}
116
+
117
+
118
+ def _save_download_state(directory, max_rows, records, ready, catalog):
119
+ """Replace each JSON file atomically, keeping earlier successes on short runs."""
120
+ report = {"max_rows": max_rows, "downloads": [records[i] for i, _, _ in catalog if i in records]}
121
+ entries = []
122
+ source_groups = {ready[i].get("source_name"): group for i, _, group in PUBLIC_DATASETS if i in ready}
123
+ for identifier, label, group in catalog:
124
+ if identifier in ready:
125
+ name = ready[identifier].get("source_name") or ""
126
+ if name.startswith("mfeat-"):
127
+ group = "multiple_features"
128
+ else:
129
+ group = source_groups.get(name, group)
130
+ entries.append({"id": f"openml_{identifier}_{label}", "group": group,
131
+ "path": f"openml_{identifier}_{label}.csv",
132
+ "target": ready[identifier]["target"], "source": ready[identifier]})
133
+ for name, value in [("download_report.json", report), ("dataset_manifest.json", entries)]:
134
+ destination = directory / name
135
+ temporary = destination.with_suffix(".json.tmp")
136
+ write_json(temporary, value)
137
+ temporary.replace(destination)
138
+ return entries
139
+
140
+
141
+ def download_public_datasets(directory="data/public_datasets", max_rows=2000, limit=None, retries=3,
142
+ suite="selected"):
143
+ if max_rows < 30:
144
+ raise ValueError("max_rows must be at least 30")
145
+ if limit is not None and limit < 1:
146
+ raise ValueError("limit must be positive")
147
+ if retries < 1:
148
+ raise ValueError("retries must be at least 1")
149
+ catalog = dataset_catalog(suite)
150
+ directory = Path(directory).resolve()
151
+ directory.mkdir(parents=True, exist_ok=True)
152
+ report_path = directory / "download_report.json"
153
+ previous = json.loads(report_path.read_text()) if report_path.exists() else {"downloads": []}
154
+ records = {d["openml_id"]: d for d in previous["downloads"]}
155
+ ready = {}
156
+ # Preserve the expanded cache/manifest even when the next command uses the
157
+ # smaller default catalog. --suite controls attempts, not deletion of data.
158
+ state_catalog = dataset_catalog("cc18")
159
+ for identifier, label, _ in state_catalog:
160
+ old = records.get(identifier)
161
+ destination = directory / f"openml_{identifier}_{label}.csv"
162
+ if identifier in UNSUPPORTED_SOURCES:
163
+ if old:
164
+ records[identifier] = {"openml_id": identifier, "status": "failed",
165
+ "error": UNSUPPORTED_SOURCES[identifier]}
166
+ continue
167
+ if old and old.get("status") == "ok":
168
+ if (old.get("max_rows") == max_rows and destination.is_file()
169
+ and hashlib.sha256(destination.read_bytes()).hexdigest() == old.get("sha256")):
170
+ ready[identifier] = old
171
+ else:
172
+ # Do not advertise a missing, changed, or differently sampled file.
173
+ records.pop(identifier)
174
+ selected = catalog[:limit]
175
+ entries = _save_download_state(directory, max_rows, records, ready, state_catalog)
176
+ for index, (identifier, label, group) in enumerate(selected, start=1):
177
+ filename = f"openml_{identifier}_{label}.csv"
178
+ destination = directory / filename
179
+ try:
180
+ if identifier in UNSUPPORTED_SOURCES:
181
+ raise ValueError(UNSUPPORTED_SOURCES[identifier])
182
+ if identifier in ready:
183
+ metadata = ready[identifier]
184
+ print(f"[{index}/{len(selected)}] Using cached dataset {identifier}: {label}", flush=True)
185
+ else:
186
+ print(f"[{index}/{len(selected)}] Downloading OpenML {identifier}: {label}", flush=True)
187
+ # pandas can infer integer/bool category values; sklearn's pandas
188
+ # ARFF parser then applies a string regex to them and fails.
189
+ # The ARFF parser preserves nominal codes as labels instead.
190
+ bunch = fetch_openml(data_id=identifier, as_frame=True, parser="liac-arff",
191
+ data_home=str(directory / "openml_cache"), n_retries=retries, delay=1.0)
192
+ frame, preparation = _prepare_download(bunch, max_rows)
193
+ temporary = destination.with_suffix(".csv.tmp")
194
+ frame.to_csv(temporary, index=False)
195
+ temporary.replace(destination)
196
+ metadata = {"openml_id": identifier, "status": "ok", "max_rows": max_rows,
197
+ "source_url": f"https://www.openml.org/d/{identifier}",
198
+ "source_name": bunch.details.get("name"),
199
+ "source_version": bunch.details.get("version"),
200
+ "license": bunch.details.get("licence", "not provided by source metadata"),
201
+ "description": bunch.DESCR, **preparation,
202
+ "sha256": hashlib.sha256(destination.read_bytes()).hexdigest()}
203
+ records[identifier] = metadata
204
+ ready[identifier] = metadata
205
+ except Exception as exc:
206
+ ready.pop(identifier, None)
207
+ records[identifier] = {"openml_id": identifier, "status": "failed", "error": str(exc)}
208
+ print(f"Failed {identifier}: {exc}", file=sys.stderr, flush=True)
209
+ entries = _save_download_state(directory, max_rows, records, ready, state_catalog)
210
+ if not entries:
211
+ raise RuntimeError("No datasets downloaded. See download_report.json; check network access to OpenML.")
212
+ return entries
213
+
214
+
215
+ def main(argv=None, default_output="data/public_datasets"):
216
+ parser = argparse.ArgumentParser(description=__doc__)
217
+ parser.add_argument("--output", default=str(default_output), help="Folder for CSVs, manifest, and source cache")
218
+ parser.add_argument("--max-rows", type=int, default=2000, help="Maximum prepared rows per dataset (default: 2000)")
219
+ parser.add_argument("--limit", type=int, help="Attempt only the first N datasets; keep earlier downloads")
220
+ parser.add_argument("--retries", type=int, default=3, help="OpenML client retries (at least 1; default: 3)")
221
+ parser.add_argument("--suite", choices=["selected", "cc18"], default="selected",
222
+ help="Planned 24 datasets, or those plus additional OpenML-CC18 sources")
223
+ parser.add_argument("--timeout", type=float, default=30, help="Socket timeout in seconds (default: 30)")
224
+ args = parser.parse_args(argv)
225
+ if not 0 < args.timeout < float("inf"):
226
+ parser.error("timeout must be a positive finite number")
227
+ previous_timeout = socket.getdefaulttimeout()
228
+ socket.setdefaulttimeout(args.timeout)
229
+ try:
230
+ entries = download_public_datasets(args.output, args.max_rows, args.limit, args.retries, args.suite)
231
+ successful = {entry["source"]["openml_id"] for entry in entries}
232
+ failed = [i for i, _, _ in dataset_catalog(args.suite)[:args.limit] if i not in successful]
233
+ print(f"\nReady: {len(entries)} datasets. Failed in this run: {len(failed)}.")
234
+ print(f"Training manifest: {Path(args.output).resolve() / 'dataset_manifest.json'}")
235
+ if failed:
236
+ print("Rerun the same command to retry failures; completed downloads will be reused.")
237
+ return 1
238
+ print("Downloads complete. Next, benchmark the datasets before training the recommender.")
239
+ return 0
240
+ except KeyboardInterrupt:
241
+ print("\nStopped. Completed datasets are saved; rerun the same command to continue.", file=sys.stderr)
242
+ return 130
243
+ except (ValueError, RuntimeError) as exc:
244
+ print(f"Error: {exc}", file=sys.stderr)
245
+ return 1
246
+ finally:
247
+ socket.setdefaulttimeout(previous_timeout)
248
+
249
+
250
+ if __name__ == "__main__":
251
+ sys.exit(main())
@@ -0,0 +1,101 @@
1
+ """Public Python interface. All computation stays in memory until explicitly saved."""
2
+
3
+ from collections.abc import Callable, Mapping
4
+ from dataclasses import dataclass
5
+ from os import PathLike
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import joblib
10
+ import numpy as np
11
+ import pandas as pd
12
+ import sklearn
13
+
14
+ from .benchmark_datasets import write_json
15
+ from .bundled_model import load_bundled_model
16
+ from .recommend_algorithms import TrainedClassifier, recommend_algorithms, verify_recommendations
17
+ from .train_meta_model import check_compatibility, train_meta_model
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class VerificationResult:
22
+ """Measured candidate results and a classifier refitted on all usable rows."""
23
+
24
+ report: dict[str, Any]
25
+ classifier: TrainedClassifier
26
+
27
+ @property
28
+ def selected_algorithm(self) -> str:
29
+ return self.report["selected_algorithm"]
30
+
31
+ def predict(self, data: pd.DataFrame) -> np.ndarray:
32
+ return self.classifier.predict(data)
33
+
34
+ def save(self, directory: str | PathLike) -> Path:
35
+ """Save the report, ranking CSV, and reloadable selected classifier."""
36
+ directory = Path(directory)
37
+ directory.mkdir(parents=True, exist_ok=True)
38
+ write_json(directory / "recommendation_report.json", self.report)
39
+ pd.DataFrame(self.report["ranking"]).to_csv(directory / "predicted_algorithm_ranking.csv", index=False)
40
+ self.classifier.save(directory / "selected_classifier.joblib")
41
+ return directory
42
+
43
+
44
+ class Recommender:
45
+ """Learn from historical benchmarks and rank algorithms for labeled tables.
46
+
47
+ Use ``Recommender()`` for the bundled pretrained model, ``fit(benchmark)``
48
+ to train your own, or ``load(path)`` to open a custom model.
49
+ Recommendations do not fit candidate classifiers. ``verify`` measures them
50
+ using fold-local preprocessing and a reserved holdout before the final refit.
51
+ """
52
+
53
+ def __init__(self, artifact: Mapping[str, Any] | None = None):
54
+ if artifact is None:
55
+ artifact = load_bundled_model()
56
+ if not isinstance(artifact, Mapping):
57
+ raise ValueError("Expected a trained recommender artifact")
58
+ check_compatibility(artifact)
59
+ if artifact.get("sklearn_version") != sklearn.__version__:
60
+ raise ValueError("Use the same scikit-learn version as meta-model training")
61
+ self.artifact = dict(artifact)
62
+
63
+ @classmethod
64
+ def fit(cls, benchmark: Mapping[str, Any] | str | PathLike) -> "Recommender":
65
+ """Train and evaluate on separate source groups, then refit on all groups.
66
+
67
+ At least four independent source groups with complete candidate results
68
+ are required. This method writes no files and returns a new recommender.
69
+ """
70
+ return cls(train_meta_model(benchmark))
71
+
72
+ @classmethod
73
+ def load(cls, path: str | PathLike) -> "Recommender":
74
+ """Load a trusted joblib artifact from this or earlier project versions."""
75
+ return cls(joblib.load(path))
76
+
77
+ def save(self, path: str | PathLike) -> Path:
78
+ """Save a compatible artifact without changing its underlying models."""
79
+ path = Path(path)
80
+ path.parent.mkdir(parents=True, exist_ok=True)
81
+ joblib.dump(self.artifact, path)
82
+ return path
83
+
84
+ def recommend(self, data: pd.DataFrame, target: str) -> dict[str, Any]:
85
+ """Return the predicted ranking, genome, data summary, and model warnings."""
86
+ return recommend_algorithms(self.artifact, data, target)
87
+
88
+ def verify(
89
+ self, data: pd.DataFrame, target: str, *, top_k: int = 3,
90
+ budget_seconds: float = 300,
91
+ progress: Callable[[dict[str, Any]], None] | None = None,
92
+ ) -> VerificationResult:
93
+ """Measure candidates in ranked order within a soft runtime budget.
94
+
95
+ ``top_k`` controls the initial shortlist, followed by a different-family
96
+ challenger and remaining candidates if time permits. Progress callbacks
97
+ receive event dictionaries; library calls otherwise print nothing.
98
+ """
99
+ report, classifier = verify_recommendations(
100
+ self.artifact, data, target, top_k, budget_seconds, progress)
101
+ return VerificationResult(report, classifier)
@@ -0,0 +1,186 @@
1
+ """Choose a recommender on development datasets, then evaluate once on held-out groups."""
2
+
3
+ import argparse
4
+ import json
5
+ from pathlib import Path
6
+
7
+ import joblib
8
+ import numpy as np
9
+ import sklearn
10
+ from sklearn.ensemble import ExtraTreesRegressor
11
+ from sklearn.feature_extraction import DictVectorizer
12
+ from sklearn.model_selection import GroupKFold, GroupShuffleSplit
13
+ from sklearn.neighbors import KNeighborsRegressor
14
+ from sklearn.pipeline import make_pipeline
15
+ from sklearn.preprocessing import StandardScaler
16
+
17
+ from .algorithm_library import algorithm_library, library_signature, meta_features
18
+ from .benchmark_datasets import write_json
19
+ from .dataset_genome import GENOME_VERSION
20
+ from .model_pipeline import PIPELINE_VERSION
21
+ from .train_meta_model import _new_regressor, _ranking_metrics, check_compatibility
22
+
23
+
24
+ class DatasetRankingModel:
25
+ """Predict every algorithm jointly from a dataset, with optional prior shrinkage."""
26
+
27
+ def __init__(self, method="joint_forest"):
28
+ self.method = method
29
+
30
+ @staticmethod
31
+ def _features(genome):
32
+ return {key: np.log1p(value) if key in {
33
+ "n_rows", "n_features", "n_numeric", "n_categorical", "n_classes", "imbalance_ratio"
34
+ } else value for key, value in genome.items()}
35
+
36
+ def fit(self, datasets):
37
+ self.names = list(algorithm_library())
38
+ features, scores, weights = [], [], []
39
+ for dataset in datasets:
40
+ features.append(self._features(dataset["genome"]))
41
+ results = {r["algorithm"]: r["cv_macro_f1"] for r in dataset["results"]}
42
+ row = np.array([results[n] for n in self.names])
43
+ scores.append(row - row.mean())
44
+ weights.append(1 / sum(d["group"] == dataset["group"] for d in datasets))
45
+ self.prior = np.average(scores, axis=0, weights=weights)
46
+ if self.method == "fixed_portfolio":
47
+ return self
48
+ if self.method == "nearest_datasets":
49
+ self.model = make_pipeline(DictVectorizer(sparse=False), StandardScaler(),
50
+ KNeighborsRegressor(n_neighbors=min(3, len(datasets)), weights="distance"))
51
+ self.model.fit(features, scores)
52
+ else:
53
+ self.model = make_pipeline(DictVectorizer(sparse=False), ExtraTreesRegressor(
54
+ n_estimators=150, min_samples_leaf=2, max_features=1.0, random_state=42, n_jobs=1))
55
+ self.model.fit(features, scores, extratreesregressor__sample_weight=weights)
56
+ return self
57
+
58
+ def predict(self, records):
59
+ features = [self._features({k.removeprefix("dataset_"): v for k, v in r.items()
60
+ if k.startswith("dataset_")}) for r in records]
61
+ scores = (np.tile(self.prior, (len(records), 1)) if self.method == "fixed_portfolio"
62
+ else self.model.predict(features))
63
+ if self.method == "shrunk_joint_forest":
64
+ scores = 0.5 * scores + 0.5 * self.prior
65
+ return np.array([scores[i, self.names.index(r["algorithm_name"])] for i, r in enumerate(records)])
66
+
67
+
68
+ def _records(datasets):
69
+ features, targets, runtimes, weights = [], [], [], []
70
+ for d in datasets:
71
+ mean = np.mean([r["cv_macro_f1"] for r in d["results"]])
72
+ for r in d["results"]:
73
+ features.append(meta_features(d["genome"], r["algorithm"]))
74
+ targets.append(r["cv_macro_f1"] - mean)
75
+ runtimes.append(np.log1p(r["cv_seconds"]))
76
+ weights.append(1 / sum(other["group"] == d["group"] for other in datasets))
77
+ return features, targets, runtimes, weights
78
+
79
+
80
+ def _fit(method, datasets):
81
+ if method == "original_pairwise_forest":
82
+ features, targets, _, weights = _records(datasets)
83
+ return _new_regressor().fit(features, targets, regress__sample_weight=weights)
84
+ return DatasetRankingModel(method).fit(datasets)
85
+
86
+
87
+ def _evaluate(model, datasets):
88
+ return _ranking_metrics(datasets, lambda d, names: dict(zip(names,
89
+ model.predict([meta_features(d["genome"], n) for n in names]))))
90
+
91
+
92
+ def improve_meta_model(benchmark_path, output_directory, previous_model_path=None):
93
+ benchmark = json.loads(Path(benchmark_path).read_text())
94
+ check_compatibility(benchmark)
95
+ names = set(algorithm_library())
96
+ datasets, excluded = [], []
97
+ for dataset in benchmark["datasets"]:
98
+ results = dataset["results"]
99
+ if (len(results) != len(names) or {r["algorithm"] for r in results} != names
100
+ or any(r["status"] != "ok" for r in results)):
101
+ excluded.append(dataset["id"])
102
+ continue
103
+ if not all(np.isfinite(r["cv_macro_f1"]) and 0 <= r["cv_macro_f1"] <= 1
104
+ and np.isfinite(r["cv_seconds"]) and r["cv_seconds"] > 0 for r in results):
105
+ raise ValueError("Invalid benchmark score or duration")
106
+ datasets.append(dataset)
107
+ if len({d["id"] for d in datasets}) != len(datasets) or len({d["fingerprint"] for d in datasets}) != len(datasets):
108
+ raise ValueError("Duplicate datasets in benchmark")
109
+ groups = [d["group"] for d in datasets]
110
+ if len(set(groups)) < 8:
111
+ raise ValueError("Improvement experiments require at least eight complete independent dataset groups")
112
+ schema = sorted(datasets[0]["genome"])
113
+ if any(sorted(d["genome"]) != schema or not all(np.isfinite(v) for v in d["genome"].values()) for d in datasets):
114
+ raise ValueError("Inconsistent genome schema or non-finite values")
115
+ development_indices, final_indices = next(GroupShuffleSplit(
116
+ n_splits=1, test_size=0.25, random_state=2026).split(datasets, groups=groups))
117
+ development = [datasets[i] for i in development_indices]
118
+ final = [datasets[i] for i in final_indices]
119
+ output = Path(output_directory)
120
+ output.mkdir(parents=True, exist_ok=True)
121
+ split = {"development_groups": sorted({d["group"] for d in development}),
122
+ "final_evaluation_groups": sorted({d["group"] for d in final})}
123
+ write_json(output / "evaluation_split.json", split)
124
+ methods = ["fixed_portfolio", "original_pairwise_forest", "joint_forest",
125
+ "shrunk_joint_forest", "nearest_datasets"]
126
+ development_groups = [d["group"] for d in development]
127
+ inner_splits = list(GroupKFold(n_splits=3).split(development, groups=development_groups))
128
+ selection = []
129
+ for method in methods:
130
+ outcomes = []
131
+ for train, valid in inner_splits:
132
+ model = _fit(method, [development[i] for i in train])
133
+ outcomes.extend(_evaluate(model, [development[i] for i in valid])["datasets"])
134
+ # Average within source group first so variants do not dominate selection.
135
+ group_regrets = [np.mean([r["regret"] for r in outcomes if r["group"] == group])
136
+ for group in sorted(set(development_groups))]
137
+ selection.append({"method": method, "mean_group_regret": float(np.mean(group_regrets)),
138
+ "top_1_rate": float(np.mean([r["top_1"] for r in outcomes])),
139
+ "out_of_fold_datasets": outcomes})
140
+ selected = min(selection, key=lambda r: r["mean_group_regret"])["method"]
141
+ evaluated = _fit(selected, development)
142
+ report = {**split, "selected_method": selected, "development_comparison": selection,
143
+ "final_evaluation": _evaluate(evaluated, final),
144
+ "retrained_original_baseline": _evaluate(_fit("original_pairwise_forest", development), final),
145
+ "fixed_portfolio_baseline": _evaluate(_fit("fixed_portfolio", development), final),
146
+ "excluded_incomplete_datasets": excluded,
147
+ "notes": ["Method selected using only development-group cross-validation regret.",
148
+ "Final evaluation groups were not used to select the method.",
149
+ "Regret uses matched candidate CV macro-F1, not independent classifier test accuracy.",
150
+ "The saved recommender is refitted on the complete corpus after evaluation.",
151
+ "A fixed portfolio may win: additional complexity is not presumed to help.",
152
+ "This is a selected, sampled public corpus, not the full OpenML-CC18 protocol."]}
153
+ if previous_model_path:
154
+ old = joblib.load(previous_model_path)
155
+ check_compatibility(old)
156
+ if set(old["training_groups"]) & set(split["final_evaluation_groups"]):
157
+ raise ValueError("Previous model has trained on final evaluation groups")
158
+ report["previous_saved_model"] = _evaluate(old["performance_model"], final)
159
+ features, _, runtimes, weights = _records(datasets)
160
+ artifact = {"format_version": 1, "genome_version": GENOME_VERSION, "pipeline_version": PIPELINE_VERSION,
161
+ "library_signature": library_signature(), "sklearn_version": sklearn.__version__,
162
+ "performance_model": _fit(selected, datasets),
163
+ "runtime_model": _new_regressor().fit(features, runtimes, regress__sample_weight=weights),
164
+ "genome_schema": schema, "training_groups": sorted(set(groups)),
165
+ "training_datasets": [d["id"] for d in datasets],
166
+ "feature_ranges": {k: [min(d["genome"][k] for d in datasets), max(d["genome"][k] for d in datasets)] for k in schema},
167
+ "evaluation": report}
168
+ joblib.dump(artifact, output / "trained_meta_model.joblib")
169
+ write_json(output / "improvement_evaluation.json", report)
170
+ print(json.dumps({k: v for k, v in report.items() if k != "development_comparison"}, indent=2))
171
+ return artifact
172
+
173
+
174
+ if __name__ == "__main__":
175
+ parser = argparse.ArgumentParser(description=__doc__)
176
+ parser.add_argument("benchmark")
177
+ parser.add_argument("--output", default="results/improved_meta_model")
178
+ parser.add_argument("--previous-model")
179
+ args = parser.parse_args()
180
+ try:
181
+ # Persist model classes under their importable module, not __main__,
182
+ # so a separate recommendation command can load the saved artifact.
183
+ from .improve_meta_model import improve_meta_model as run_experiment
184
+ run_experiment(args.benchmark, args.output, args.previous_model)
185
+ except (ValueError, OSError, KeyError) as exc:
186
+ parser.exit(1, f"Error: {exc}\n")
@@ -0,0 +1,68 @@
1
+ """Consistent preprocessing and model validation for benchmarks and verification."""
2
+
3
+ import numpy as np
4
+ from sklearn.base import BaseEstimator, ClassifierMixin, clone
5
+ from sklearn.compose import ColumnTransformer, make_column_selector
6
+ from sklearn.impute import SimpleImputer
7
+ from sklearn.model_selection import StratifiedKFold, cross_val_score
8
+ from sklearn.pipeline import Pipeline
9
+ from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler
10
+ from sklearn.utils.validation import check_is_fitted
11
+
12
+ from .algorithm_library import algorithm_library
13
+
14
+ PIPELINE_VERSION = 2
15
+
16
+
17
+ class LabelEncodedClassifier(ClassifierMixin, BaseEstimator):
18
+ """Fit on integer labels and preserve original labels at the API boundary.
19
+
20
+ This avoids numeric-looking string labels being inconsistently coerced by
21
+ class-weight computation, while preserving distinctions such as '01'/'1'.
22
+ The encoder is fitted separately inside each cross-validation training fold.
23
+ """
24
+
25
+ def __init__(self, estimator):
26
+ self.estimator = estimator
27
+
28
+ def fit(self, X, y):
29
+ self.label_encoder_ = LabelEncoder()
30
+ encoded = self.label_encoder_.fit_transform(y)
31
+ self.estimator_ = clone(self.estimator).fit(X, encoded)
32
+ self.classes_ = self.label_encoder_.classes_
33
+ self.n_features_in_ = self.estimator_.n_features_in_
34
+ return self
35
+
36
+ def predict(self, X):
37
+ check_is_fitted(self, "estimator_")
38
+ return self.label_encoder_.inverse_transform(self.estimator_.predict(X))
39
+
40
+
41
+ def build_model_pipeline(algorithm_name):
42
+ numeric = Pipeline([
43
+ ("impute", SimpleImputer(strategy="median", keep_empty_features=True)),
44
+ ("scale", StandardScaler()),
45
+ ])
46
+ categorical = Pipeline([
47
+ ("impute", SimpleImputer(strategy="most_frequent", keep_empty_features=True)),
48
+ ("encode", OneHotEncoder(handle_unknown="ignore", max_categories=32, sparse_output=False)),
49
+ ])
50
+ preprocess = ColumnTransformer([
51
+ ("numeric", numeric, make_column_selector(dtype_include=np.number)),
52
+ ("categorical", categorical, make_column_selector(dtype_exclude=np.number)),
53
+ ])
54
+ return Pipeline([("preprocess", preprocess),
55
+ ("model", LabelEncodedClassifier(algorithm_library()[algorithm_name]["estimator"]))])
56
+
57
+
58
+ def evaluate_candidate(name, X, y, folds=3):
59
+ folds = min(folds, int(y.value_counts().min()))
60
+ if folds < 2:
61
+ raise ValueError("Need at least two examples per class for validation")
62
+ splitter = StratifiedKFold(n_splits=folds, shuffle=True, random_state=42)
63
+ scores = cross_val_score(build_model_pipeline(name), X, y, cv=splitter,
64
+ scoring="f1_macro", n_jobs=1, error_score="raise")
65
+ if not np.isfinite(scores).all():
66
+ raise ValueError("Non-finite validation scores")
67
+ return {"cv_macro_f1": float(scores.mean()), "cv_std": float(scores.std()),
68
+ "fold_scores": scores.tolist(), "folds": folds}