algorithm-genome 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- algorithm_genome/__init__.py +13 -0
- algorithm_genome/__main__.py +3 -0
- algorithm_genome/algorithm_library.py +60 -0
- algorithm_genome/assets/default_recommender.joblib +0 -0
- algorithm_genome/assets/model_info.json +28 -0
- algorithm_genome/benchmark_datasets.py +130 -0
- algorithm_genome/bundled_model.py +27 -0
- algorithm_genome/command_line.py +116 -0
- algorithm_genome/create_example_datasets.py +35 -0
- algorithm_genome/dataset_genome.py +110 -0
- algorithm_genome/download_public_datasets.py +251 -0
- algorithm_genome/framework.py +101 -0
- algorithm_genome/improve_meta_model.py +186 -0
- algorithm_genome/model_pipeline.py +68 -0
- algorithm_genome/prepare_additional_datasets.py +105 -0
- algorithm_genome/recommend_algorithms.py +177 -0
- algorithm_genome/terminal_output.py +270 -0
- algorithm_genome/train_meta_model.py +142 -0
- algorithm_genome/train_neural_recommender.py +402 -0
- algorithm_genome-0.3.0.dist-info/METADATA +223 -0
- algorithm_genome-0.3.0.dist-info/RECORD +24 -0
- algorithm_genome-0.3.0.dist-info/WHEEL +5 -0
- algorithm_genome-0.3.0.dist-info/entry_points.txt +2 -0
- algorithm_genome-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Learn algorithm recommendations from dataset and algorithm genomes."""
|
|
2
|
+
|
|
3
|
+
from .dataset_genome import extract_dataset_genome
|
|
4
|
+
from .benchmark_datasets import Dataset, benchmark_datasets
|
|
5
|
+
from .framework import Recommender, VerificationResult
|
|
6
|
+
from .recommend_algorithms import TrainedClassifier, recommend_algorithms, verify_recommendations
|
|
7
|
+
from .train_meta_model import train_meta_model
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Dataset", "Recommender", "TrainedClassifier", "VerificationResult",
|
|
11
|
+
"benchmark_datasets", "extract_dataset_genome", "recommend_algorithms",
|
|
12
|
+
"verify_recommendations", "train_meta_model",
|
|
13
|
+
]
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Candidate configurations and explicit algorithm genome attributes."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
|
|
7
|
+
from sklearn.ensemble import ExtraTreesClassifier, GradientBoostingClassifier, RandomForestClassifier
|
|
8
|
+
from sklearn.linear_model import LogisticRegression, RidgeClassifier
|
|
9
|
+
from sklearn.naive_bayes import GaussianNB
|
|
10
|
+
from sklearn.neighbors import KNeighborsClassifier
|
|
11
|
+
from sklearn.svm import SVC
|
|
12
|
+
from sklearn.tree import DecisionTreeClassifier
|
|
13
|
+
|
|
14
|
+
LIBRARY_VERSION = 1
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def algorithm_library():
|
|
18
|
+
# Qualitative properties are hypotheses for the meta-model, not suitability scores.
|
|
19
|
+
definitions = [
|
|
20
|
+
("logistic_regression", LogisticRegression(max_iter=1500, class_weight="balanced", random_state=42),
|
|
21
|
+
"linear", "convex", "low", "high", "medium", "linear", "low", True),
|
|
22
|
+
("decision_tree", DecisionTreeClassifier(class_weight="balanced", random_state=42),
|
|
23
|
+
"tree", "greedy", "low", "high", "medium", "nonlinear", "high", False),
|
|
24
|
+
("random_forest", RandomForestClassifier(n_estimators=80, class_weight="balanced", random_state=42, n_jobs=1),
|
|
25
|
+
"ensemble", "bagging", "medium", "low", "high", "nonlinear", "high", False),
|
|
26
|
+
("extra_trees", ExtraTreesClassifier(n_estimators=80, class_weight="balanced", random_state=42, n_jobs=1),
|
|
27
|
+
"ensemble", "randomized_bagging", "medium", "low", "high", "nonlinear", "high", False),
|
|
28
|
+
("gradient_boosting", GradientBoostingClassifier(n_estimators=70, random_state=42),
|
|
29
|
+
"ensemble", "boosting", "medium", "low", "medium", "nonlinear", "high", False),
|
|
30
|
+
("nearest_neighbors", KNeighborsClassifier(n_neighbors=5),
|
|
31
|
+
"instance", "lazy", "low", "medium", "low", "nonlinear", "high", True),
|
|
32
|
+
("support_vector_machine", SVC(class_weight="balanced", cache_size=200),
|
|
33
|
+
"kernel", "convex", "high", "low", "medium", "nonlinear", "high", True),
|
|
34
|
+
("gaussian_naive_bayes", GaussianNB(),
|
|
35
|
+
"probabilistic", "closed_form", "low", "high", "low", "nonlinear", "low", False),
|
|
36
|
+
("linear_discriminant", LinearDiscriminantAnalysis(solver="lsqr", shrinkage="auto"),
|
|
37
|
+
"linear", "closed_form", "medium", "high", "medium", "linear", "low", True),
|
|
38
|
+
("ridge_classifier", RidgeClassifier(class_weight="balanced", solver="lsqr"),
|
|
39
|
+
"linear", "closed_form", "low", "high", "medium", "linear", "low", True),
|
|
40
|
+
]
|
|
41
|
+
fields = ["structure", "optimization", "training_cost", "interpretability", "robustness",
|
|
42
|
+
"boundary", "capacity", "scaling_sensitive"]
|
|
43
|
+
return {name: {"estimator": estimator, "genome": {
|
|
44
|
+
"paradigm": "supervised", "task": "classification", **dict(zip(fields, attributes)),
|
|
45
|
+
}} for name, estimator, *attributes in definitions}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def library_signature():
|
|
49
|
+
configurations = {name: {"genome": item["genome"], "parameters": item["estimator"].get_params()}
|
|
50
|
+
for name, item in algorithm_library().items()}
|
|
51
|
+
return hashlib.sha256(json.dumps(configurations, sort_keys=True).encode()).hexdigest()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def meta_features(dataset_genome, algorithm_name, include_algorithm_genome=True):
|
|
55
|
+
item = algorithm_library()[algorithm_name]
|
|
56
|
+
features = {f"dataset_{key}": value for key, value in dataset_genome.items()}
|
|
57
|
+
features["algorithm_name"] = algorithm_name
|
|
58
|
+
if include_algorithm_genome:
|
|
59
|
+
features.update({f"algorithm_{key}": value for key, value in item["genome"].items()})
|
|
60
|
+
return features
|
|
Binary file
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Algorithm Genome pretrained neural recommender",
|
|
3
|
+
"package_version": "0.3.0",
|
|
4
|
+
"sklearn_version": "1.9.1",
|
|
5
|
+
"sha256": "c7ea17bdcd38941b6b50b3708d549f0624487b32e2db73303cc319d29a513df7",
|
|
6
|
+
"source_artifact_sha256": "698a9d7c340b5d7e9aca71be09a6828cb184a9c6c61b0125a48e57051797d0e6",
|
|
7
|
+
"training_dataset_count": 51,
|
|
8
|
+
"training_source_group_count": 43,
|
|
9
|
+
"genome_input_count": 17,
|
|
10
|
+
"candidate_count": 10,
|
|
11
|
+
"architecture": "17 inputs, 8 tanh hidden units, 10 relative macro-F1 outputs",
|
|
12
|
+
"status": "experimental",
|
|
13
|
+
"training_warnings": [
|
|
14
|
+
"lbfgs failed to converge after 29 iteration(s) (status=2):\nABNORMAL: \n\nYou might also want to scale the data as shown in:\n https://scikit-learn.org/stable/modules/preprocessing.html"
|
|
15
|
+
],
|
|
16
|
+
"evaluation": {
|
|
17
|
+
"reserved_source_groups": 7,
|
|
18
|
+
"neural_first_choice_f1_points_lost": 2.086,
|
|
19
|
+
"same_corpus_fixed_portfolio_first_choice_f1_points_lost": 1.204,
|
|
20
|
+
"note": "The simple portfolio was stronger on this small examined cohort. This is not a guarantee for new datasets."
|
|
21
|
+
},
|
|
22
|
+
"notes": [
|
|
23
|
+
"This is the existing 51-dataset/43-source-group artifact; no retraining or final-cohort selection was performed for packaging.",
|
|
24
|
+
"Source tables, user data, project-local paths, and selected classifiers are not distributed with the model.",
|
|
25
|
+
"Ranking scores are predicted relative macro-F1 advantages, not accuracy or confidence probabilities.",
|
|
26
|
+
"The production optimizer warning is retained in recommendation reports."
|
|
27
|
+
]
|
|
28
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Build historical experiment records from DataFrames or a local CSV manifest."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import platform
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from collections.abc import Iterable
|
|
10
|
+
from os import PathLike
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import sklearn
|
|
17
|
+
|
|
18
|
+
from .algorithm_library import algorithm_library, library_signature
|
|
19
|
+
from .dataset_genome import GENOME_VERSION, extract_dataset_genome, prepare_classification_data
|
|
20
|
+
from .model_pipeline import PIPELINE_VERSION, evaluate_candidate
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Dataset:
|
|
25
|
+
"""One labeled table; related variants must share a source ``group``."""
|
|
26
|
+
|
|
27
|
+
id: str
|
|
28
|
+
data: pd.DataFrame
|
|
29
|
+
target: str
|
|
30
|
+
group: str | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def write_json(path, value):
|
|
34
|
+
path = Path(path)
|
|
35
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
path.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def dataset_fingerprint(X, y):
|
|
40
|
+
"""Identify exact dataset content regardless of row order or column names."""
|
|
41
|
+
hashed = pd.util.hash_pandas_object(pd.concat([X, y.rename("__label__")], axis=1), index=False)
|
|
42
|
+
return hashlib.sha256(np.sort(hashed.to_numpy()).tobytes()).hexdigest()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def benchmark_datasets(
|
|
46
|
+
manifest_path: str | PathLike | Iterable[Dataset],
|
|
47
|
+
output_path: str | PathLike | None = None,
|
|
48
|
+
progress: bool = False, resume: bool = False,
|
|
49
|
+
) -> dict[str, Any]:
|
|
50
|
+
"""Benchmark a CSV manifest or iterable of :class:`Dataset` objects.
|
|
51
|
+
|
|
52
|
+
Return a training-ready dictionary. Nothing is written unless ``output_path``
|
|
53
|
+
is supplied. Set ``progress=True`` for stderr messages and ``resume=True``
|
|
54
|
+
to reuse complete, compatible results from that output file.
|
|
55
|
+
"""
|
|
56
|
+
if resume and output_path is None:
|
|
57
|
+
raise ValueError("Resuming requires an output_path")
|
|
58
|
+
if isinstance(manifest_path, (str, PathLike)):
|
|
59
|
+
manifest = Path(manifest_path).resolve()
|
|
60
|
+
entries = json.loads(manifest.read_text())
|
|
61
|
+
if not isinstance(entries, list) or not entries:
|
|
62
|
+
raise ValueError("Manifest must be a nonempty list of dataset entries")
|
|
63
|
+
inputs = ((entry, entry.get("group", entry["id"]),
|
|
64
|
+
pd.read_csv((manifest.parent / entry["path"]).resolve())) for entry in entries)
|
|
65
|
+
else:
|
|
66
|
+
datasets = list(manifest_path)
|
|
67
|
+
if not datasets or not all(isinstance(item, Dataset) for item in datasets):
|
|
68
|
+
raise ValueError("Provide a nonempty iterable of Dataset objects")
|
|
69
|
+
inputs = (({"id": item.id, "target": item.target},
|
|
70
|
+
item.id if item.group is None else item.group, item.data) for item in datasets)
|
|
71
|
+
ids, fingerprints, prepared = set(), set(), []
|
|
72
|
+
# Validate the complete manifest before spending time fitting models.
|
|
73
|
+
for entry, group, data in inputs:
|
|
74
|
+
identifier = entry["id"]
|
|
75
|
+
if not isinstance(identifier, str) or not identifier or identifier in ids:
|
|
76
|
+
raise ValueError("Dataset IDs must be nonempty unique strings")
|
|
77
|
+
if not isinstance(group, str) or not group:
|
|
78
|
+
raise ValueError("Dataset groups must be nonempty strings")
|
|
79
|
+
ids.add(identifier)
|
|
80
|
+
X, y = prepare_classification_data(data, entry["target"])
|
|
81
|
+
# Detect identical tables even when row order, filename, or column names differ.
|
|
82
|
+
fingerprint = dataset_fingerprint(X, y)
|
|
83
|
+
if fingerprint in fingerprints:
|
|
84
|
+
raise ValueError(f"Duplicate dataset content detected: {identifier}")
|
|
85
|
+
fingerprints.add(fingerprint)
|
|
86
|
+
prepared.append((entry, group, X, y, fingerprint, len(data)))
|
|
87
|
+
report = {
|
|
88
|
+
"format_version": 1, "genome_version": GENOME_VERSION,
|
|
89
|
+
"pipeline_version": PIPELINE_VERSION, "library_signature": library_signature(),
|
|
90
|
+
"metric": "macro_f1", "cv_folds_requested": 3, "random_state": 42,
|
|
91
|
+
"versions": {"python": platform.python_version(), "sklearn": sklearn.__version__, "pandas": pd.__version__},
|
|
92
|
+
"datasets": [],
|
|
93
|
+
}
|
|
94
|
+
cached = {}
|
|
95
|
+
if resume and Path(output_path).exists():
|
|
96
|
+
prior = json.loads(Path(output_path).read_text())
|
|
97
|
+
for key in ("format_version", "genome_version", "pipeline_version", "library_signature", "metric", "cv_folds_requested", "random_state", "versions"):
|
|
98
|
+
if prior.get(key) != report[key]:
|
|
99
|
+
raise ValueError(f"Cannot resume benchmark: {key} changed")
|
|
100
|
+
cached = {d["id"]: d for d in prior["datasets"]}
|
|
101
|
+
for entry, group, X, y, fingerprint, original_rows in prepared:
|
|
102
|
+
old = cached.get(entry["id"])
|
|
103
|
+
if (old and old["fingerprint"] == fingerprint and old["group"] == group
|
|
104
|
+
and len(old["results"]) == len(algorithm_library())
|
|
105
|
+
and {r["algorithm"] for r in old["results"]} == set(algorithm_library())
|
|
106
|
+
and all(r["status"] == "ok" for r in old["results"])):
|
|
107
|
+
report["datasets"].append(old)
|
|
108
|
+
if progress:
|
|
109
|
+
print(f"Using completed benchmark: {entry['id']}", file=sys.stderr, flush=True)
|
|
110
|
+
if output_path is not None:
|
|
111
|
+
write_json(output_path, report)
|
|
112
|
+
continue
|
|
113
|
+
item = {"id": entry["id"], "group": group, "fingerprint": fingerprint,
|
|
114
|
+
"rows_dropped": original_rows - len(X), "genome": extract_dataset_genome(X, y), "results": []}
|
|
115
|
+
if "source" in entry:
|
|
116
|
+
item["source"] = entry["source"]
|
|
117
|
+
for name in algorithm_library():
|
|
118
|
+
if progress:
|
|
119
|
+
print(f"Benchmarking {entry['id']}: {name}", file=sys.stderr, flush=True)
|
|
120
|
+
started = time.monotonic()
|
|
121
|
+
try:
|
|
122
|
+
result = {"algorithm": name, "status": "ok", **evaluate_candidate(name, X, y)}
|
|
123
|
+
except Exception as exc:
|
|
124
|
+
result = {"algorithm": name, "status": "failed", "error": f"{type(exc).__name__}: {exc}"}
|
|
125
|
+
result["cv_seconds"] = time.monotonic() - started
|
|
126
|
+
item["results"].append(result)
|
|
127
|
+
report["datasets"].append(item)
|
|
128
|
+
if output_path is not None:
|
|
129
|
+
write_json(output_path, report)
|
|
130
|
+
return report
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Load the pretrained recommender distributed inside the installed package."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from importlib.resources import as_file, files
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
import joblib
|
|
8
|
+
import sklearn
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_bundled_model():
|
|
12
|
+
"""Load package resources without a checkout, network access, or model path."""
|
|
13
|
+
assets = files("algorithm_genome").joinpath("assets")
|
|
14
|
+
metadata = json.loads(assets.joinpath("model_info.json").read_text(encoding="utf-8"))
|
|
15
|
+
expected = metadata["sklearn_version"]
|
|
16
|
+
# Check before unpickling, so an incompatible environment fails clearly.
|
|
17
|
+
if sklearn.__version__ != expected:
|
|
18
|
+
raise ValueError(
|
|
19
|
+
f"The bundled model requires scikit-learn=={expected}; "
|
|
20
|
+
f"found {sklearn.__version__}. Install this package's declared dependencies."
|
|
21
|
+
)
|
|
22
|
+
resource = assets.joinpath("default_recommender.joblib")
|
|
23
|
+
# as_file also supports package resources that are not ordinary filesystem files.
|
|
24
|
+
with as_file(resource) as path:
|
|
25
|
+
if hashlib.sha256(path.read_bytes()).hexdigest() != metadata["sha256"]:
|
|
26
|
+
raise ValueError("Bundled model checksum differs; reinstall algorithm-genome")
|
|
27
|
+
return joblib.load(path)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""User commands for benchmarking, training the recommender, and prediction."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from .benchmark_datasets import benchmark_datasets, write_json
|
|
11
|
+
from .framework import Recommender
|
|
12
|
+
from .recommend_algorithms import TrainedClassifier
|
|
13
|
+
from .train_meta_model import train_meta_model
|
|
14
|
+
from .terminal_output import TerminalReporter
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main(argv=None):
|
|
18
|
+
parser = argparse.ArgumentParser(description="Dataset and algorithm genome meta-learning")
|
|
19
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
20
|
+
benchmark = commands.add_parser("benchmark", help="Measure every algorithm on the datasets in a manifest")
|
|
21
|
+
benchmark.add_argument("manifest")
|
|
22
|
+
benchmark.add_argument("--output", default="results/benchmark_results.json")
|
|
23
|
+
benchmark.add_argument("--resume", action="store_true", help="Reuse matching successful dataset benchmarks")
|
|
24
|
+
train = commands.add_parser("train-meta", help="Learn and evaluate the algorithm recommender")
|
|
25
|
+
train.add_argument("benchmark")
|
|
26
|
+
train.add_argument("--output", default="results/meta_model")
|
|
27
|
+
recommend = commands.add_parser("recommend", help="Predict ranking, optionally verify candidates")
|
|
28
|
+
recommend.add_argument("csv")
|
|
29
|
+
recommend.add_argument("--target", required=True)
|
|
30
|
+
recommend.add_argument("--meta-model", help="Custom trusted joblib file; defaults to the bundled pretrained recommender")
|
|
31
|
+
recommend.add_argument("--verify", action="store_true")
|
|
32
|
+
recommend.add_argument("--top-k", type=int, default=3)
|
|
33
|
+
recommend.add_argument("--budget", type=float, default=300)
|
|
34
|
+
recommend.add_argument("--output", default="results/recommendation")
|
|
35
|
+
predict = commands.add_parser("predict", help="Predict labels using a verified saved classifier")
|
|
36
|
+
predict.add_argument("model", help="Trusted selected_classifier.joblib file")
|
|
37
|
+
predict.add_argument("csv")
|
|
38
|
+
predict.add_argument("--output", default="results/predictions.csv")
|
|
39
|
+
for command in [benchmark, train, recommend, predict]:
|
|
40
|
+
command.add_argument("--verbose", action="store_true", help="Show full technical details")
|
|
41
|
+
command.add_argument("--no-color", action="store_true", help="Disable terminal colors")
|
|
42
|
+
command.add_argument("--json", action="store_true", help="Write machine-readable JSON to stdout; verbose diagnostics go to stderr")
|
|
43
|
+
args = parser.parse_args(argv)
|
|
44
|
+
reporter = TerminalReporter(args.verbose, stream=sys.stderr if args.json else sys.stdout,
|
|
45
|
+
color=False if args.no_color else None)
|
|
46
|
+
human = not args.json or args.verbose
|
|
47
|
+
if args.command in {"recommend", "train-meta"}:
|
|
48
|
+
log_path = Path(args.output) / "terminal_output.txt"
|
|
49
|
+
else:
|
|
50
|
+
log_path = Path(args.output).with_suffix(".terminal.txt")
|
|
51
|
+
try:
|
|
52
|
+
if human:
|
|
53
|
+
reporter.heading("ALGORITHM GENOME | " + args.command.upper())
|
|
54
|
+
if args.command == "benchmark":
|
|
55
|
+
result = benchmark_datasets(args.manifest, args.output, resume=args.resume, progress=human)
|
|
56
|
+
if human:
|
|
57
|
+
runs = [row for dataset in result["datasets"] for row in dataset["results"]]
|
|
58
|
+
reporter.values([("Benchmarked datasets", len(result["datasets"])),
|
|
59
|
+
("Successful candidate runs", sum(row["status"] == "ok" for row in runs)),
|
|
60
|
+
("Failed candidate runs", sum(row["status"] == "failed" for row in runs))])
|
|
61
|
+
elif args.command == "train-meta":
|
|
62
|
+
artifact = train_meta_model(args.benchmark, args.output)
|
|
63
|
+
result = artifact["evaluation"]
|
|
64
|
+
if human:
|
|
65
|
+
reporter.values([("Training datasets in final refit", len(artifact["training_datasets"])),
|
|
66
|
+
("Training source groups", len(artifact["training_groups"])),
|
|
67
|
+
("Evaluation source groups", len(result["evaluation_groups"]))])
|
|
68
|
+
reporter.table(["Method", "Top-1 %", "Top-3 %", "F1 points lost"], [
|
|
69
|
+
[name.replace("_", " "), f"{100 * result[name]['top_1_rate']:.1f}",
|
|
70
|
+
f"{100 * result[name]['top_3_rate']:.1f}", f"{100 * result[name]['mean_regret']:.2f}"]
|
|
71
|
+
for name in ["genome_model", "algorithm_identity_control", "fixed_ranking_baseline"]])
|
|
72
|
+
reporter.paragraph("Evaluation uses held-out dataset groups. The saved model is then refitted on all complete historical datasets.")
|
|
73
|
+
elif args.command == "recommend":
|
|
74
|
+
recommender = Recommender.load(args.meta_model) if args.meta_model else Recommender()
|
|
75
|
+
artifact = recommender.artifact
|
|
76
|
+
data = pd.read_csv(args.csv)
|
|
77
|
+
output = Path(args.output)
|
|
78
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
79
|
+
if human:
|
|
80
|
+
reporter.values([("Dataset", args.csv), ("Target", args.target)])
|
|
81
|
+
reporter.model(artifact, args.meta_model or "Bundled pretrained recommender")
|
|
82
|
+
if args.verify:
|
|
83
|
+
verified = recommender.verify(data, args.target, top_k=args.top_k, budget_seconds=args.budget,
|
|
84
|
+
progress=reporter.progress if human else None)
|
|
85
|
+
result = verified.report
|
|
86
|
+
verified.save(output)
|
|
87
|
+
else:
|
|
88
|
+
result = recommender.recommend(data, args.target)
|
|
89
|
+
write_json(output / "recommendation_report.json", result)
|
|
90
|
+
pd.DataFrame(result["ranking"]).to_csv(output / "predicted_algorithm_ranking.csv", index=False)
|
|
91
|
+
if human:
|
|
92
|
+
reporter.result(result)
|
|
93
|
+
else:
|
|
94
|
+
model = TrainedClassifier.load(args.model)
|
|
95
|
+
predictions = model.predict(pd.read_csv(args.csv))
|
|
96
|
+
output = Path(args.output)
|
|
97
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
pd.DataFrame({"prediction": predictions}).to_csv(output, index=False)
|
|
99
|
+
result = {"prediction_rows": len(predictions), "output": str(output)}
|
|
100
|
+
if human:
|
|
101
|
+
reporter.values([("Saved classifier", args.model), ("Input rows", args.csv)])
|
|
102
|
+
reporter.predictions(predictions)
|
|
103
|
+
if args.json:
|
|
104
|
+
print(json.dumps(result, indent=2))
|
|
105
|
+
if human:
|
|
106
|
+
pairs = [("Results", args.output), ("Terminal transcript", log_path)]
|
|
107
|
+
if args.command == "recommend":
|
|
108
|
+
pairs = [("Recommendation report", output / "recommendation_report.json"),
|
|
109
|
+
("Predicted ranking CSV", output / "predicted_algorithm_ranking.csv")]
|
|
110
|
+
if args.verify:
|
|
111
|
+
pairs.append(("Selected classifier", output / "selected_classifier.joblib"))
|
|
112
|
+
pairs.append(("Terminal transcript", log_path))
|
|
113
|
+
reporter.files(pairs)
|
|
114
|
+
reporter.save(log_path)
|
|
115
|
+
except (ValueError, TypeError, OSError, RuntimeError, KeyError) as exc:
|
|
116
|
+
parser.exit(1, f"Error: {exc}\n")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Export four bundled sklearn datasets for a small end-to-end demonstration."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from sklearn.datasets import load_breast_cancer, load_digits, load_iris, load_wine, make_classification
|
|
8
|
+
|
|
9
|
+
from .benchmark_datasets import write_json
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def create_example_datasets(directory):
|
|
13
|
+
directory = Path(directory)
|
|
14
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
15
|
+
entries = []
|
|
16
|
+
for name, loader in [("iris", load_iris), ("wine", load_wine),
|
|
17
|
+
("breast_cancer", load_breast_cancer), ("digits", load_digits)]:
|
|
18
|
+
data = loader()
|
|
19
|
+
frame = pd.DataFrame(data.data, columns=[f"feature_{i}" for i in range(data.data.shape[1])])
|
|
20
|
+
frame["target"] = data.target
|
|
21
|
+
frame.to_csv(directory / f"{name}.csv", index=False)
|
|
22
|
+
entries.append({"id": name, "group": name, "path": f"{name}.csv", "target": "target"})
|
|
23
|
+
write_json(directory / "dataset_manifest.json", entries)
|
|
24
|
+
X, y = make_classification(n_samples=250, n_features=12, n_informative=6, random_state=2026)
|
|
25
|
+
new = pd.DataFrame(X, columns=[f"feature_{i}" for i in range(X.shape[1])])
|
|
26
|
+
new["target"] = y
|
|
27
|
+
new.to_csv(directory / "new_synthetic_dataset.csv", index=False)
|
|
28
|
+
new.drop(columns="target").head(10).to_csv(directory / "unlabeled_prediction_rows.csv", index=False)
|
|
29
|
+
print(f"Created examples in {directory}. Four datasets demonstrate execution, not recommendation reliability.")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
if __name__ == "__main__":
|
|
33
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
34
|
+
parser.add_argument("--output", default="examples/generated_data")
|
|
35
|
+
create_example_datasets(parser.parse_args().output)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Validate labeled classification tables and measure their dataset genome."""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
GENOME_VERSION = 1
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def prepare_classification_data(data, target):
|
|
10
|
+
if not isinstance(data, pd.DataFrame) or not data.columns.is_unique:
|
|
11
|
+
raise ValueError("Provide a DataFrame with unique column names")
|
|
12
|
+
if not all(isinstance(c, str) for c in data.columns):
|
|
13
|
+
raise ValueError("Column names must be strings")
|
|
14
|
+
if target not in data or len(data.columns) < 2:
|
|
15
|
+
raise ValueError("Provide an existing target and at least one predictor")
|
|
16
|
+
clean = data.dropna(subset=[target]).drop_duplicates().reset_index(drop=True)
|
|
17
|
+
if len(clean) < 30:
|
|
18
|
+
raise ValueError("Need at least 30 distinct labeled rows")
|
|
19
|
+
y = clean[target].astype(str)
|
|
20
|
+
if y.nunique() < 2 or y.value_counts().min() < 3:
|
|
21
|
+
raise ValueError("Need at least two classes and three distinct rows per class")
|
|
22
|
+
X = normalize_features(clean.drop(columns=target))
|
|
23
|
+
return X, y
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def normalize_features(data):
|
|
27
|
+
data = data.copy()
|
|
28
|
+
for column in data:
|
|
29
|
+
if pd.api.types.is_numeric_dtype(data[column]) and not pd.api.types.is_bool_dtype(data[column]):
|
|
30
|
+
data[column] = data[column].astype(float).replace([np.inf, -np.inf], np.nan)
|
|
31
|
+
else:
|
|
32
|
+
data[column] = data[column].map(lambda v: str(v) if pd.notna(v) else np.nan).astype(object)
|
|
33
|
+
return data
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def summarize_classification_data(data, target, X, y):
|
|
37
|
+
"""Describe cleaning and class counts for display; never used as model inputs."""
|
|
38
|
+
labeled_rows = len(data.dropna(subset=[target]))
|
|
39
|
+
numeric = X.select_dtypes(include=np.number).shape[1]
|
|
40
|
+
return {"target": target, "input_rows": len(data), "missing_target_rows": len(data) - labeled_rows,
|
|
41
|
+
"duplicate_rows": labeled_rows - len(X), "usable_rows": len(X), "n_features": X.shape[1],
|
|
42
|
+
"n_numeric": numeric, "n_categorical": X.shape[1] - numeric,
|
|
43
|
+
"missing_values": int(X.isna().to_numpy().sum()), "missing_fraction": float(X.isna().to_numpy().mean()),
|
|
44
|
+
"class_counts": {str(label): int(count) for label, count in y.value_counts().items()}}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def extract_dataset_genome(X, y):
|
|
48
|
+
"""Use training rows only. Includes all ten paper features plus target signals.
|
|
49
|
+
|
|
50
|
+
y is positionally aligned with X. No model fits or landmarkers are needed.
|
|
51
|
+
Correlation is quadratic in the number of numeric columns.
|
|
52
|
+
"""
|
|
53
|
+
X = normalize_features(X).reset_index(drop=True)
|
|
54
|
+
y = pd.Series(np.asarray(y)).reset_index(drop=True)
|
|
55
|
+
if len(X) != len(y) or len(X) == 0 or X.shape[1] == 0 or y.isna().any():
|
|
56
|
+
raise ValueError("Genome extraction needs aligned, nonempty X and complete labels")
|
|
57
|
+
numeric = X.select_dtypes(include=np.number)
|
|
58
|
+
categorical = X.select_dtypes(exclude=np.number)
|
|
59
|
+
counts = y.value_counts()
|
|
60
|
+
probabilities = counts / len(y)
|
|
61
|
+
outliers, associations, skewness = [], [], []
|
|
62
|
+
for column in numeric:
|
|
63
|
+
values = numeric[column].dropna()
|
|
64
|
+
if len(values) >= 5:
|
|
65
|
+
q1, q3 = values.quantile([0.25, 0.75])
|
|
66
|
+
iqr = q3 - q1
|
|
67
|
+
if iqr > 0:
|
|
68
|
+
outliers.append(float(((values < q1 - 1.5 * iqr) | (values > q3 + 1.5 * iqr)).mean()))
|
|
69
|
+
skew = values.skew()
|
|
70
|
+
if np.isfinite(skew):
|
|
71
|
+
skewness.append(abs(float(skew)))
|
|
72
|
+
# Correlation ratio eta-squared: variance explained by class means.
|
|
73
|
+
observed = numeric[column].notna()
|
|
74
|
+
v = numeric.loc[observed, column]
|
|
75
|
+
total = float(((v - v.mean()) ** 2).sum())
|
|
76
|
+
if total > 0:
|
|
77
|
+
grouped = v.groupby(y[observed]).agg(["mean", "count"])
|
|
78
|
+
between = ((grouped["mean"] - v.mean()) ** 2 * grouped["count"]).sum()
|
|
79
|
+
associations.append(float(np.clip(between / total, 0, 1)))
|
|
80
|
+
correlation = numeric.corr().to_numpy()
|
|
81
|
+
pairs = correlation[np.triu_indices(len(numeric.columns), k=1)]
|
|
82
|
+
pairs = pairs[np.isfinite(pairs)]
|
|
83
|
+
cardinalities = [categorical[c].nunique() / len(X) for c in categorical]
|
|
84
|
+
# Categorical normalized mutual information, computed from observed counts.
|
|
85
|
+
# High cardinality can inflate this statistic; cardinality is also recorded.
|
|
86
|
+
categorical_associations = []
|
|
87
|
+
from sklearn.metrics import normalized_mutual_info_score
|
|
88
|
+
for column in categorical:
|
|
89
|
+
encoded = pd.factorize(categorical[column], sort=False)[0]
|
|
90
|
+
categorical_associations.append(float(normalized_mutual_info_score(y, encoded)))
|
|
91
|
+
mean = lambda values: float(np.mean(values)) if len(values) else 0.0
|
|
92
|
+
genome = {
|
|
93
|
+
"n_rows": float(len(X)), "n_features": float(X.shape[1]),
|
|
94
|
+
"n_numeric": float(numeric.shape[1]), "n_categorical": float(categorical.shape[1]),
|
|
95
|
+
"missing_fraction": float(X.isna().to_numpy().mean()),
|
|
96
|
+
"n_classes": float(len(counts)), "imbalance_ratio": float(counts.max() / counts.min()),
|
|
97
|
+
"outlier_fraction": mean(outliers),
|
|
98
|
+
"correlated_pair_fraction": mean(np.abs(pairs) > 0.8),
|
|
99
|
+
"feature_sample_ratio": X.shape[1] / len(X),
|
|
100
|
+
"mean_absolute_skewness": mean(skewness),
|
|
101
|
+
"constant_feature_fraction": float((X.nunique(dropna=True) <= 1).mean()),
|
|
102
|
+
"mean_categorical_cardinality_ratio": mean(cardinalities),
|
|
103
|
+
"class_entropy": float(-(probabilities * np.log2(probabilities)).sum()),
|
|
104
|
+
"mean_numeric_target_association": mean(associations),
|
|
105
|
+
"max_numeric_target_association": max(associations, default=0.0),
|
|
106
|
+
"mean_categorical_target_association": mean(categorical_associations),
|
|
107
|
+
}
|
|
108
|
+
if not all(np.isfinite(v) for v in genome.values()):
|
|
109
|
+
raise ValueError("Dataset genome contains non-finite values; check extreme feature magnitudes")
|
|
110
|
+
return genome
|