proteingym-base 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proteingym/base/__about__.py +9 -0
- proteingym/base/__init__.py +61 -0
- proteingym/base/__main__.py +486 -0
- proteingym/base/assay.py +1041 -0
- proteingym/base/data_generators.py +168 -0
- proteingym/base/dataset.py +1275 -0
- proteingym/base/evaluate.py +232 -0
- proteingym/base/lookup_field.py +52 -0
- proteingym/base/manifest.py +225 -0
- proteingym/base/metrics.py +478 -0
- proteingym/base/model.py +139 -0
- proteingym/base/msa.py +361 -0
- proteingym/base/publication.py +95 -0
- proteingym/base/py.typed +0 -0
- proteingym/base/sequence.py +331 -0
- proteingym/base/splits.py +919 -0
- proteingym/base/structure.py +231 -0
- proteingym/base/transformers.py +392 -0
- proteingym_base-0.1.0.dist-info/METADATA +270 -0
- proteingym_base-0.1.0.dist-info/RECORD +23 -0
- proteingym_base-0.1.0.dist-info/WHEEL +4 -0
- proteingym_base-0.1.0.dist-info/entry_points.txt +2 -0
- proteingym_base-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""The protein gym 2 dataset.
|
|
2
|
+
|
|
3
|
+
A standardized format for storing and sharing datasets in the Protein Gym 2
|
|
4
|
+
ecosystem.
|
|
5
|
+
|
|
6
|
+
Attributes :
|
|
7
|
+
Dataset :
|
|
8
|
+
The main class for managing datasets.
|
|
9
|
+
Manifest :
|
|
10
|
+
Represents the metadata and resources of a dataset. Entrypoint for
|
|
11
|
+
loading and validating dataset from metadata.
|
|
12
|
+
evaluate_subset :
|
|
13
|
+
Calculate metrics for cross-validation splits (.splits.pgdata).
|
|
14
|
+
evaluate_dataset :
|
|
15
|
+
Calculate metrics for a plain dataset (.pgdata).
|
|
16
|
+
evaluate_zero_shot :
|
|
17
|
+
Calculate zero-shot metrics against the full dataset of a Subsets
|
|
18
|
+
archive (.splits.pgdata), scoring once and ignoring folds.
|
|
19
|
+
calculate_selected_metrics :
|
|
20
|
+
Calculate selected metrics comparing ground truth and predictions.
|
|
21
|
+
calculate_metrics_by_mode :
|
|
22
|
+
Calculate metrics across scoring modes (test, train_available, per_fold).
|
|
23
|
+
DatasetScoringContext :
|
|
24
|
+
A validated request bundling a Dataset ground truth, predictions, and target.
|
|
25
|
+
SubsetScoringContext :
|
|
26
|
+
A validated request bundling a Subsets ground truth, predictions, target,
|
|
27
|
+
split, and fold(s).
|
|
28
|
+
MetricsResult :
|
|
29
|
+
Typed metrics computed across one or more scoring modes.
|
|
30
|
+
ScoreMode :
|
|
31
|
+
A scoring mode for evaluating cross-validation splits.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from .dataset import Dataset, Subsets
|
|
35
|
+
from .evaluate import evaluate_dataset, evaluate_subset, evaluate_zero_shot
|
|
36
|
+
from .manifest import Manifest
|
|
37
|
+
from .metrics import (
|
|
38
|
+
DatasetScoringContext,
|
|
39
|
+
MetricsProvenance,
|
|
40
|
+
MetricsResult,
|
|
41
|
+
ScoreMode,
|
|
42
|
+
SubsetScoringContext,
|
|
43
|
+
calculate_metrics_by_mode,
|
|
44
|
+
calculate_selected_metrics,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"Dataset",
|
|
49
|
+
"DatasetScoringContext",
|
|
50
|
+
"Manifest",
|
|
51
|
+
"MetricsProvenance",
|
|
52
|
+
"MetricsResult",
|
|
53
|
+
"ScoreMode",
|
|
54
|
+
"Subsets",
|
|
55
|
+
"SubsetScoringContext",
|
|
56
|
+
"calculate_metrics_by_mode",
|
|
57
|
+
"calculate_selected_metrics",
|
|
58
|
+
"evaluate_dataset",
|
|
59
|
+
"evaluate_subset",
|
|
60
|
+
"evaluate_zero_shot",
|
|
61
|
+
]
|
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
import typer
|
|
8
|
+
from pydantic import ValidationError
|
|
9
|
+
|
|
10
|
+
from .__about__ import __version__
|
|
11
|
+
from .data_generators import (
|
|
12
|
+
adjust_target_with_two_dummy_features,
|
|
13
|
+
charge_ladder_dataset,
|
|
14
|
+
)
|
|
15
|
+
from .dataset import Dataset
|
|
16
|
+
from .evaluate import evaluate_dataset as evaluate_dataset_fn
|
|
17
|
+
from .evaluate import evaluate_subset as evaluate_subset_fn
|
|
18
|
+
from .evaluate import evaluate_zero_shot as evaluate_zero_shot_fn
|
|
19
|
+
from .manifest import Manifest
|
|
20
|
+
from .metrics import ScoreMode
|
|
21
|
+
from .model import ModelCard, ModelProject
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger("proteingym.base")
|
|
24
|
+
|
|
25
|
+
app = typer.Typer(
|
|
26
|
+
name="proteingym-base",
|
|
27
|
+
help="CLI for handling ProteinGym resources",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def setup_logger(*, level: int = logging.CRITICAL) -> None:
|
|
32
|
+
"""Set up the logger for the application.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
level (int): The logging level to set. Defaults to
|
|
36
|
+
`logging.CRITICAL`.
|
|
37
|
+
"""
|
|
38
|
+
logger = logging.getLogger("proteingym.base")
|
|
39
|
+
logger.setLevel(level)
|
|
40
|
+
|
|
41
|
+
stream_handler = logging.StreamHandler()
|
|
42
|
+
stream_handler.setLevel(level)
|
|
43
|
+
|
|
44
|
+
formatter = logging.Formatter(
|
|
45
|
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
46
|
+
)
|
|
47
|
+
stream_handler.setFormatter(formatter)
|
|
48
|
+
logger.addHandler(stream_handler)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.callback(invoke_without_command=True)
|
|
52
|
+
def main(
|
|
53
|
+
ctx: typer.Context,
|
|
54
|
+
verbose: Annotated[int, typer.Option("--verbose", "-v", count=True)] = 0,
|
|
55
|
+
version: Annotated[
|
|
56
|
+
bool, typer.Option("--version", help="Show version and exit")
|
|
57
|
+
] = False,
|
|
58
|
+
) -> None:
|
|
59
|
+
"""Main entry point for the CLI.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
ctx (typer.Context): The context for the CLI.
|
|
63
|
+
verbose (int): The verbosity level. Use `-v` or `--verbose` to increase
|
|
64
|
+
verbosity. Each `-v` increases the verbosity level:
|
|
65
|
+
0: CRITICAL, 1: ERROR, 2: WARNING, 3: INFO, 4: DEBUG.
|
|
66
|
+
Defaults to 0 (CRITICAL).
|
|
67
|
+
version (bool): If `True`, show the package version. Defaults to `False`.
|
|
68
|
+
|
|
69
|
+
Raises:
|
|
70
|
+
typer.Exit: If version is `True`, exits after showing the version.
|
|
71
|
+
"""
|
|
72
|
+
setup_logger(level=logging.CRITICAL - verbose * 10)
|
|
73
|
+
|
|
74
|
+
if version:
|
|
75
|
+
typer.echo(f"v{__version__}")
|
|
76
|
+
raise typer.Exit()
|
|
77
|
+
|
|
78
|
+
if not ctx.invoked_subcommand:
|
|
79
|
+
typer.echo("Welcome to the PG2 Dataset CLI!")
|
|
80
|
+
typer.echo("Use --help to see available commands.")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@app.command("build")
|
|
84
|
+
def build(
|
|
85
|
+
manifest_path: Annotated[Path, typer.Argument(help="Path to the manifest file")],
|
|
86
|
+
output_path: Annotated[
|
|
87
|
+
Path | None, typer.Option("--output-path", help="Path to the output directory")
|
|
88
|
+
] = None,
|
|
89
|
+
):
|
|
90
|
+
"""Build a ProteinGym2 Dataset from manifest.
|
|
91
|
+
|
|
92
|
+
Creates a Dataset instance from a manifest TOML file and dumps it as zip to a
|
|
93
|
+
specified directory path.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
manifest_path (Path): The path to the manifest TOML file.
|
|
97
|
+
output_path (Path | None): The directory path to dump the dataset archive. If
|
|
98
|
+
`None`, the current working directory is used. Defaults to `None`.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
Outputs:
|
|
102
|
+
A zip file containing the dataset, saved in the specified path.
|
|
103
|
+
"""
|
|
104
|
+
typer.echo("Loading manifest...")
|
|
105
|
+
dataset_manifest = Manifest.from_path(manifest_path)
|
|
106
|
+
|
|
107
|
+
typer.echo("Building dataset...")
|
|
108
|
+
dataset = Dataset.from_manifest(dataset_manifest)
|
|
109
|
+
|
|
110
|
+
typer.echo("Building dataset archive...")
|
|
111
|
+
archive_path = dataset.dump(path=output_path)
|
|
112
|
+
typer.echo(f"Dataset {dataset.name} archived to: {archive_path}")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@app.command("validate")
|
|
116
|
+
def validate_model(
|
|
117
|
+
project_path: Annotated[
|
|
118
|
+
Path,
|
|
119
|
+
typer.Argument(
|
|
120
|
+
help="Root path to the model project",
|
|
121
|
+
exists=True,
|
|
122
|
+
resolve_path=True,
|
|
123
|
+
),
|
|
124
|
+
],
|
|
125
|
+
):
|
|
126
|
+
"""Validate a model from path."""
|
|
127
|
+
logger = logging.getLogger("proteingym.base")
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
model_project = ModelProject.from_path(project_path)
|
|
131
|
+
typer.echo(
|
|
132
|
+
f"✅ Model {model_project.project['name']} loaded successfully "
|
|
133
|
+
f"with entry points: {model_project.entry_points}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
model_card = ModelCard.from_path(project_path / "README.md")
|
|
137
|
+
typer.echo(
|
|
138
|
+
f"✅ Loaded {model_card.name} with hyper parameters: "
|
|
139
|
+
f"{model_card.hyper_parameters}."
|
|
140
|
+
)
|
|
141
|
+
except ValueError as e:
|
|
142
|
+
typer.echo(f"❌ Validation failed: {str(e)}", err=True)
|
|
143
|
+
raise typer.Exit(1) from e
|
|
144
|
+
except Exception as e:
|
|
145
|
+
logger.error("❌ Error running validation", exc_info=e)
|
|
146
|
+
raise typer.Exit(1) from e
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@app.command("list-datasets")
|
|
150
|
+
def list_datasets(
|
|
151
|
+
path: Annotated[
|
|
152
|
+
Path,
|
|
153
|
+
typer.Argument(
|
|
154
|
+
help="Directory path containing .pgdata dataset archives",
|
|
155
|
+
exists=True,
|
|
156
|
+
file_okay=True,
|
|
157
|
+
dir_okay=True,
|
|
158
|
+
),
|
|
159
|
+
],
|
|
160
|
+
):
|
|
161
|
+
"""List available datasets with optional query filtering."""
|
|
162
|
+
|
|
163
|
+
def find_datasets_with_paths(root_path: Path) -> list[dict]:
|
|
164
|
+
"""Find all .pgdata datasets in the given directory."""
|
|
165
|
+
datasets_with_paths = []
|
|
166
|
+
|
|
167
|
+
if root_path.is_file():
|
|
168
|
+
paths = [path]
|
|
169
|
+
else:
|
|
170
|
+
paths = root_path.rglob("*.pgdata")
|
|
171
|
+
|
|
172
|
+
for model_path in paths:
|
|
173
|
+
dataset = Dataset.from_path(model_path)
|
|
174
|
+
dataset_data = json.loads(dataset.model_dump_json())
|
|
175
|
+
|
|
176
|
+
dataset_entry = {
|
|
177
|
+
**dataset_data,
|
|
178
|
+
"input_filename": model_path.resolve().as_posix(),
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
datasets_with_paths.append(dataset_entry)
|
|
182
|
+
|
|
183
|
+
return datasets_with_paths
|
|
184
|
+
|
|
185
|
+
datasets_with_paths = find_datasets_with_paths(root_path=path)
|
|
186
|
+
|
|
187
|
+
output = json.dumps(datasets_with_paths, indent=2)
|
|
188
|
+
typer.echo(output, nl=False)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@app.command("list-models")
|
|
192
|
+
def list_models(
|
|
193
|
+
path: Annotated[
|
|
194
|
+
Path,
|
|
195
|
+
typer.Argument(
|
|
196
|
+
help="Directory path containing model cards",
|
|
197
|
+
exists=True,
|
|
198
|
+
file_okay=True,
|
|
199
|
+
dir_okay=True,
|
|
200
|
+
),
|
|
201
|
+
],
|
|
202
|
+
):
|
|
203
|
+
"""List available models with optional query filtering.
|
|
204
|
+
|
|
205
|
+
Note: This custom CLI is used instead of yq because:
|
|
206
|
+
- Parses Markdown files with model card frontmatter, not just YAML/JSON
|
|
207
|
+
- Provides sophisticated error handling with ValidationError logging
|
|
208
|
+
- Creates custom JSON output with both model data and file metadata
|
|
209
|
+
- Includes recursive file discovery for .md files
|
|
210
|
+
- Leverages Pydantic validation for type safety and schema compliance
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
def find_models_with_paths(root_path: Path) -> list[dict]:
|
|
214
|
+
"""Find all model cards in the given directory."""
|
|
215
|
+
models_with_paths = []
|
|
216
|
+
|
|
217
|
+
if root_path.is_file():
|
|
218
|
+
paths = [root_path]
|
|
219
|
+
else:
|
|
220
|
+
paths = root_path.rglob("*.md")
|
|
221
|
+
|
|
222
|
+
for model_path in paths:
|
|
223
|
+
try:
|
|
224
|
+
model = ModelCard.from_path(model_path)
|
|
225
|
+
except ValidationError as e:
|
|
226
|
+
logger.error(f"Skipping {model_path}", exc_info=e)
|
|
227
|
+
else:
|
|
228
|
+
model_entry = {
|
|
229
|
+
**model.model_dump(),
|
|
230
|
+
"input_filename": model_path.resolve().as_posix(),
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
models_with_paths.append(model_entry)
|
|
234
|
+
|
|
235
|
+
return models_with_paths
|
|
236
|
+
|
|
237
|
+
models_with_paths = find_models_with_paths(root_path=path)
|
|
238
|
+
|
|
239
|
+
output = json.dumps(models_with_paths, indent=2)
|
|
240
|
+
typer.echo(output, nl=False)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@app.command("generate-data")
|
|
244
|
+
def generate_data(
|
|
245
|
+
feature1: Annotated[
|
|
246
|
+
str,
|
|
247
|
+
typer.Argument(help="Name of the first dummy feature"),
|
|
248
|
+
] = "foo",
|
|
249
|
+
feature2: Annotated[
|
|
250
|
+
str,
|
|
251
|
+
typer.Argument(help="Name of the second dummy feature"),
|
|
252
|
+
] = "bar",
|
|
253
|
+
*,
|
|
254
|
+
n_rows: Annotated[
|
|
255
|
+
int, typer.Option(help="Number of rows to generate in a data frame")
|
|
256
|
+
] = 500,
|
|
257
|
+
sequence_length: Annotated[
|
|
258
|
+
int, typer.Option(help="Length of sequence for the sequence column")
|
|
259
|
+
] = 100,
|
|
260
|
+
fmt: Annotated[
|
|
261
|
+
str,
|
|
262
|
+
typer.Option(
|
|
263
|
+
help="Output format",
|
|
264
|
+
click_type=click.Choice(choices=["csv"], case_sensitive=False),
|
|
265
|
+
),
|
|
266
|
+
] = "csv",
|
|
267
|
+
) -> None:
|
|
268
|
+
"""Generate dummy data with charge ladder and two dummy features.
|
|
269
|
+
|
|
270
|
+
Creates a charge ladder dataset with protein sequences and adds two dummy
|
|
271
|
+
features to the target charge column. The output is formatted as CSV.
|
|
272
|
+
Users can update feature names, but it is currently limited to only two features.
|
|
273
|
+
|
|
274
|
+
Args:
|
|
275
|
+
feature1: Name of the first dummy feature. Defaults to "foo".
|
|
276
|
+
feature2: Name of the second dummy feature. Defaults to "bar".
|
|
277
|
+
n_rows: Number of rows to generate in the data frame. Defaults to 500.
|
|
278
|
+
sequence_length: Length of sequence for the sequence column. Defaults to 100.
|
|
279
|
+
fmt: Output format. Currently only supports "csv". Defaults to "csv".
|
|
280
|
+
|
|
281
|
+
Outputs:
|
|
282
|
+
Prints the generated CSV data to stdout.
|
|
283
|
+
"""
|
|
284
|
+
ladder = charge_ladder_dataset(n_rows, sequence_length)
|
|
285
|
+
|
|
286
|
+
output = ladder.pipe(
|
|
287
|
+
adjust_target_with_two_dummy_features,
|
|
288
|
+
target="charge",
|
|
289
|
+
feature_names=[feature1, feature2],
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
if fmt.lower() == "csv":
|
|
293
|
+
typer.echo(output.write_csv(), nl=False)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@app.command("evaluate-subset")
|
|
297
|
+
def evaluate_subset_command(
|
|
298
|
+
prediction_path: Annotated[
|
|
299
|
+
Path,
|
|
300
|
+
typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
|
|
301
|
+
],
|
|
302
|
+
metric_path: Annotated[
|
|
303
|
+
Path,
|
|
304
|
+
typer.Option(help="Path where the calculated metrics JSON will be saved."),
|
|
305
|
+
],
|
|
306
|
+
dataset_path: Annotated[
|
|
307
|
+
Path,
|
|
308
|
+
typer.Option(help="Path to the ground truth Subsets archive (.splits.pgdata)."),
|
|
309
|
+
],
|
|
310
|
+
split: Annotated[
|
|
311
|
+
str,
|
|
312
|
+
typer.Option(help="Name of the splitting strategy to evaluate."),
|
|
313
|
+
],
|
|
314
|
+
target: Annotated[
|
|
315
|
+
str,
|
|
316
|
+
typer.Option(help="Name of the target variable to score."),
|
|
317
|
+
],
|
|
318
|
+
fold: Annotated[
|
|
319
|
+
str,
|
|
320
|
+
typer.Option(help="Fold index designated as the test fold."),
|
|
321
|
+
],
|
|
322
|
+
selected_metrics: Annotated[
|
|
323
|
+
list[str] | None,
|
|
324
|
+
typer.Option(
|
|
325
|
+
help="Metric name to calculate. Repeat the flag for multiple metrics."
|
|
326
|
+
),
|
|
327
|
+
] = None,
|
|
328
|
+
model_name: Annotated[
|
|
329
|
+
str | None,
|
|
330
|
+
typer.Option(help="Name of the model that generated predictions."),
|
|
331
|
+
] = None,
|
|
332
|
+
score_modes: Annotated[
|
|
333
|
+
list[ScoreMode] | None,
|
|
334
|
+
typer.Option(
|
|
335
|
+
help="Scoring mode to compute. Repeat the flag for multiple modes."
|
|
336
|
+
),
|
|
337
|
+
] = None,
|
|
338
|
+
) -> None:
|
|
339
|
+
"""Calculate metrics for cross-validation splits and save results to JSON.
|
|
340
|
+
|
|
341
|
+
Args:
|
|
342
|
+
prediction_path: Path to the prediction dataset archive (.pgdata file)
|
|
343
|
+
containing model predictions.
|
|
344
|
+
metric_path: Path where the calculated metrics JSON will be saved.
|
|
345
|
+
dataset_path: Path to the ground truth Subsets archive (.splits.pgdata).
|
|
346
|
+
split: Name of the splitting strategy to evaluate.
|
|
347
|
+
target: Name of the target variable to score.
|
|
348
|
+
fold: Fold index (as string) designated as the test fold.
|
|
349
|
+
selected_metrics: Optional list of metric names to calculate. If not
|
|
350
|
+
provided, all discovered metrics are included.
|
|
351
|
+
model_name: Name of the model that generated predictions (stored in
|
|
352
|
+
metadata).
|
|
353
|
+
score_modes: Optional list of scoring modes to compute.
|
|
354
|
+
"""
|
|
355
|
+
result_path = evaluate_subset_fn(
|
|
356
|
+
prediction_path=prediction_path,
|
|
357
|
+
metric_path=metric_path,
|
|
358
|
+
dataset_path=dataset_path,
|
|
359
|
+
split=split,
|
|
360
|
+
target=target,
|
|
361
|
+
fold=fold,
|
|
362
|
+
selected_metrics=selected_metrics,
|
|
363
|
+
model_name=model_name,
|
|
364
|
+
score_modes=score_modes,
|
|
365
|
+
)
|
|
366
|
+
typer.echo(f"Metrics saved to: {result_path}")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@app.command("evaluate-dataset")
|
|
370
|
+
def evaluate_dataset_command(
|
|
371
|
+
prediction_path: Annotated[
|
|
372
|
+
Path,
|
|
373
|
+
typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
|
|
374
|
+
],
|
|
375
|
+
metric_path: Annotated[
|
|
376
|
+
Path,
|
|
377
|
+
typer.Option(help="Path where the calculated metrics JSON will be saved."),
|
|
378
|
+
],
|
|
379
|
+
dataset_path: Annotated[
|
|
380
|
+
Path,
|
|
381
|
+
typer.Option(help="Path to the ground truth Dataset archive (.pgdata)."),
|
|
382
|
+
],
|
|
383
|
+
target: Annotated[
|
|
384
|
+
str,
|
|
385
|
+
typer.Option(help="Name of the target variable to score."),
|
|
386
|
+
],
|
|
387
|
+
selected_metrics: Annotated[
|
|
388
|
+
list[str] | None,
|
|
389
|
+
typer.Option(
|
|
390
|
+
help="Metric name to calculate. Repeat the flag for multiple metrics."
|
|
391
|
+
),
|
|
392
|
+
] = None,
|
|
393
|
+
model_name: Annotated[
|
|
394
|
+
str | None,
|
|
395
|
+
typer.Option(help="Name of the model that generated predictions."),
|
|
396
|
+
] = None,
|
|
397
|
+
) -> None:
|
|
398
|
+
"""Calculate metrics for a plain dataset and save results to JSON.
|
|
399
|
+
|
|
400
|
+
Args:
|
|
401
|
+
prediction_path: Path to the prediction dataset archive (.pgdata file)
|
|
402
|
+
containing model predictions.
|
|
403
|
+
metric_path: Path where the calculated metrics JSON will be saved.
|
|
404
|
+
dataset_path: Path to the ground truth Dataset archive (.pgdata).
|
|
405
|
+
target: Name of the target variable to score.
|
|
406
|
+
selected_metrics: Optional list of metric names to calculate. If not
|
|
407
|
+
provided, all discovered metrics are included.
|
|
408
|
+
model_name: Name of the model that generated predictions (stored in
|
|
409
|
+
metadata).
|
|
410
|
+
"""
|
|
411
|
+
result_path = evaluate_dataset_fn(
|
|
412
|
+
prediction_path=prediction_path,
|
|
413
|
+
metric_path=metric_path,
|
|
414
|
+
dataset_path=dataset_path,
|
|
415
|
+
target=target,
|
|
416
|
+
selected_metrics=selected_metrics,
|
|
417
|
+
model_name=model_name,
|
|
418
|
+
)
|
|
419
|
+
typer.echo(f"Metrics saved to: {result_path}")
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
@app.command("evaluate-zero-shot")
|
|
423
|
+
def evaluate_zero_shot_command(
|
|
424
|
+
prediction_path: Annotated[
|
|
425
|
+
Path,
|
|
426
|
+
typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
|
|
427
|
+
],
|
|
428
|
+
metric_path: Annotated[
|
|
429
|
+
Path,
|
|
430
|
+
typer.Option(help="Path where the calculated metrics JSON will be saved."),
|
|
431
|
+
],
|
|
432
|
+
dataset_path: Annotated[
|
|
433
|
+
Path,
|
|
434
|
+
typer.Option(help="Path to the ground truth Subsets archive (.splits.pgdata)."),
|
|
435
|
+
],
|
|
436
|
+
split: Annotated[
|
|
437
|
+
str,
|
|
438
|
+
typer.Option(help="Name of the splitting strategy (stored in metadata)."),
|
|
439
|
+
],
|
|
440
|
+
target: Annotated[
|
|
441
|
+
str,
|
|
442
|
+
typer.Option(help="Name of the target variable to score."),
|
|
443
|
+
],
|
|
444
|
+
selected_metrics: Annotated[
|
|
445
|
+
list[str] | None,
|
|
446
|
+
typer.Option(
|
|
447
|
+
help="Metric name to calculate. Repeat the flag for multiple metrics."
|
|
448
|
+
),
|
|
449
|
+
] = None,
|
|
450
|
+
model_name: Annotated[
|
|
451
|
+
str | None,
|
|
452
|
+
typer.Option(help="Name of the model that generated predictions."),
|
|
453
|
+
] = None,
|
|
454
|
+
) -> None:
|
|
455
|
+
"""Calculate zero-shot metrics against the full dataset and save to JSON.
|
|
456
|
+
|
|
457
|
+
Scores the requested metrics once against the complete underlying dataset of a
|
|
458
|
+
Subsets archive, ignoring cross-validation folds. Appropriate for zero-shot
|
|
459
|
+
models whose predictions do not depend on the training fold.
|
|
460
|
+
|
|
461
|
+
Args:
|
|
462
|
+
prediction_path: Path to the prediction dataset archive (.pgdata file)
|
|
463
|
+
containing model predictions.
|
|
464
|
+
metric_path: Path where the calculated metrics JSON will be saved.
|
|
465
|
+
dataset_path: Path to the ground truth Subsets archive (.splits.pgdata).
|
|
466
|
+
split: Name of the splitting strategy (stored in metadata).
|
|
467
|
+
target: Name of the target variable to score.
|
|
468
|
+
selected_metrics: Optional list of metric names to calculate. If not
|
|
469
|
+
provided, all discovered metrics are included.
|
|
470
|
+
model_name: Name of the model that generated predictions (stored in
|
|
471
|
+
metadata).
|
|
472
|
+
"""
|
|
473
|
+
result_path = evaluate_zero_shot_fn(
|
|
474
|
+
prediction_path=prediction_path,
|
|
475
|
+
metric_path=metric_path,
|
|
476
|
+
dataset_path=dataset_path,
|
|
477
|
+
split=split,
|
|
478
|
+
target=target,
|
|
479
|
+
selected_metrics=selected_metrics,
|
|
480
|
+
model_name=model_name,
|
|
481
|
+
)
|
|
482
|
+
typer.echo(f"Metrics saved to: {result_path}")
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
if __name__ == "__main__":
|
|
486
|
+
app()
|