proteingym-base 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,9 @@
1
+ """Metadata file to store version used by hatch.
2
+
3
+ > BE CAREFUL WITH MANUALLY CHANGING THIS FILE!
4
+
5
+ References:
6
+ https://hatch.pypa.io/latest/version/
7
+ """
8
+
9
+ __version__ = "0.1.0"
@@ -0,0 +1,61 @@
1
+ """The protein gym 2 dataset.
2
+
3
+ A standardized format for storing and sharing datasets in the Protein Gym 2
4
+ ecosystem.
5
+
6
+ Attributes :
7
+ Dataset :
8
+ The main class for managing datasets.
9
+ Manifest :
10
+ Represents the metadata and resources of a dataset. Entrypoint for
11
+ loading and validating dataset from metadata.
12
+ evaluate_subset :
13
+ Calculate metrics for cross-validation splits (.splits.pgdata).
14
+ evaluate_dataset :
15
+ Calculate metrics for a plain dataset (.pgdata).
16
+ evaluate_zero_shot :
17
+ Calculate zero-shot metrics against the full dataset of a Subsets
18
+ archive (.splits.pgdata), scoring once and ignoring folds.
19
+ calculate_selected_metrics :
20
+ Calculate selected metrics comparing ground truth and predictions.
21
+ calculate_metrics_by_mode :
22
+ Calculate metrics across scoring modes (test, train_available, per_fold).
23
+ DatasetScoringContext :
24
+ A validated request bundling a Dataset ground truth, predictions, and target.
25
+ SubsetScoringContext :
26
+ A validated request bundling a Subsets ground truth, predictions, target,
27
+ split, and fold(s).
28
+ MetricsResult :
29
+ Typed metrics computed across one or more scoring modes.
30
+ ScoreMode :
31
+ A scoring mode for evaluating cross-validation splits.
32
+ """
33
+
34
+ from .dataset import Dataset, Subsets
35
+ from .evaluate import evaluate_dataset, evaluate_subset, evaluate_zero_shot
36
+ from .manifest import Manifest
37
+ from .metrics import (
38
+ DatasetScoringContext,
39
+ MetricsProvenance,
40
+ MetricsResult,
41
+ ScoreMode,
42
+ SubsetScoringContext,
43
+ calculate_metrics_by_mode,
44
+ calculate_selected_metrics,
45
+ )
46
+
47
+ __all__ = [
48
+ "Dataset",
49
+ "DatasetScoringContext",
50
+ "Manifest",
51
+ "MetricsProvenance",
52
+ "MetricsResult",
53
+ "ScoreMode",
54
+ "Subsets",
55
+ "SubsetScoringContext",
56
+ "calculate_metrics_by_mode",
57
+ "calculate_selected_metrics",
58
+ "evaluate_dataset",
59
+ "evaluate_subset",
60
+ "evaluate_zero_shot",
61
+ ]
@@ -0,0 +1,486 @@
1
+ import json
2
+ import logging
3
+ from pathlib import Path
4
+ from typing import Annotated
5
+
6
+ import click
7
+ import typer
8
+ from pydantic import ValidationError
9
+
10
+ from .__about__ import __version__
11
+ from .data_generators import (
12
+ adjust_target_with_two_dummy_features,
13
+ charge_ladder_dataset,
14
+ )
15
+ from .dataset import Dataset
16
+ from .evaluate import evaluate_dataset as evaluate_dataset_fn
17
+ from .evaluate import evaluate_subset as evaluate_subset_fn
18
+ from .evaluate import evaluate_zero_shot as evaluate_zero_shot_fn
19
+ from .manifest import Manifest
20
+ from .metrics import ScoreMode
21
+ from .model import ModelCard, ModelProject
22
+
23
+ logger = logging.getLogger("proteingym.base")
24
+
25
+ app = typer.Typer(
26
+ name="proteingym-base",
27
+ help="CLI for handling ProteinGym resources",
28
+ )
29
+
30
+
31
+ def setup_logger(*, level: int = logging.CRITICAL) -> None:
32
+ """Set up the logger for the application.
33
+
34
+ Args:
35
+ level (int): The logging level to set. Defaults to
36
+ `logging.CRITICAL`.
37
+ """
38
+ logger = logging.getLogger("proteingym.base")
39
+ logger.setLevel(level)
40
+
41
+ stream_handler = logging.StreamHandler()
42
+ stream_handler.setLevel(level)
43
+
44
+ formatter = logging.Formatter(
45
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
46
+ )
47
+ stream_handler.setFormatter(formatter)
48
+ logger.addHandler(stream_handler)
49
+
50
+
51
+ @app.callback(invoke_without_command=True)
52
+ def main(
53
+ ctx: typer.Context,
54
+ verbose: Annotated[int, typer.Option("--verbose", "-v", count=True)] = 0,
55
+ version: Annotated[
56
+ bool, typer.Option("--version", help="Show version and exit")
57
+ ] = False,
58
+ ) -> None:
59
+ """Main entry point for the CLI.
60
+
61
+ Args:
62
+ ctx (typer.Context): The context for the CLI.
63
+ verbose (int): The verbosity level. Use `-v` or `--verbose` to increase
64
+ verbosity. Each `-v` increases the verbosity level:
65
+ 0: CRITICAL, 1: ERROR, 2: WARNING, 3: INFO, 4: DEBUG.
66
+ Defaults to 0 (CRITICAL).
67
+ version (bool): If `True`, show the package version. Defaults to `False`.
68
+
69
+ Raises:
70
+ typer.Exit: If version is `True`, exits after showing the version.
71
+ """
72
+ setup_logger(level=logging.CRITICAL - verbose * 10)
73
+
74
+ if version:
75
+ typer.echo(f"v{__version__}")
76
+ raise typer.Exit()
77
+
78
+ if not ctx.invoked_subcommand:
79
+ typer.echo("Welcome to the PG2 Dataset CLI!")
80
+ typer.echo("Use --help to see available commands.")
81
+
82
+
83
+ @app.command("build")
84
+ def build(
85
+ manifest_path: Annotated[Path, typer.Argument(help="Path to the manifest file")],
86
+ output_path: Annotated[
87
+ Path | None, typer.Option("--output-path", help="Path to the output directory")
88
+ ] = None,
89
+ ):
90
+ """Build a ProteinGym2 Dataset from manifest.
91
+
92
+ Creates a Dataset instance from a manifest TOML file and dumps it as zip to a
93
+ specified directory path.
94
+
95
+ Args:
96
+ manifest_path (Path): The path to the manifest TOML file.
97
+ output_path (Path | None): The directory path to dump the dataset archive. If
98
+ `None`, the current working directory is used. Defaults to `None`.
99
+
100
+
101
+ Outputs:
102
+ A zip file containing the dataset, saved in the specified path.
103
+ """
104
+ typer.echo("Loading manifest...")
105
+ dataset_manifest = Manifest.from_path(manifest_path)
106
+
107
+ typer.echo("Building dataset...")
108
+ dataset = Dataset.from_manifest(dataset_manifest)
109
+
110
+ typer.echo("Building dataset archive...")
111
+ archive_path = dataset.dump(path=output_path)
112
+ typer.echo(f"Dataset {dataset.name} archived to: {archive_path}")
113
+
114
+
115
+ @app.command("validate")
116
+ def validate_model(
117
+ project_path: Annotated[
118
+ Path,
119
+ typer.Argument(
120
+ help="Root path to the model project",
121
+ exists=True,
122
+ resolve_path=True,
123
+ ),
124
+ ],
125
+ ):
126
+ """Validate a model from path."""
127
+ logger = logging.getLogger("proteingym.base")
128
+
129
+ try:
130
+ model_project = ModelProject.from_path(project_path)
131
+ typer.echo(
132
+ f"✅ Model {model_project.project['name']} loaded successfully "
133
+ f"with entry points: {model_project.entry_points}"
134
+ )
135
+
136
+ model_card = ModelCard.from_path(project_path / "README.md")
137
+ typer.echo(
138
+ f"✅ Loaded {model_card.name} with hyper parameters: "
139
+ f"{model_card.hyper_parameters}."
140
+ )
141
+ except ValueError as e:
142
+ typer.echo(f"❌ Validation failed: {str(e)}", err=True)
143
+ raise typer.Exit(1) from e
144
+ except Exception as e:
145
+ logger.error("❌ Error running validation", exc_info=e)
146
+ raise typer.Exit(1) from e
147
+
148
+
149
+ @app.command("list-datasets")
150
+ def list_datasets(
151
+ path: Annotated[
152
+ Path,
153
+ typer.Argument(
154
+ help="Directory path containing .pgdata dataset archives",
155
+ exists=True,
156
+ file_okay=True,
157
+ dir_okay=True,
158
+ ),
159
+ ],
160
+ ):
161
+ """List available datasets with optional query filtering."""
162
+
163
+ def find_datasets_with_paths(root_path: Path) -> list[dict]:
164
+ """Find all .pgdata datasets in the given directory."""
165
+ datasets_with_paths = []
166
+
167
+ if root_path.is_file():
168
+ paths = [path]
169
+ else:
170
+ paths = root_path.rglob("*.pgdata")
171
+
172
+ for model_path in paths:
173
+ dataset = Dataset.from_path(model_path)
174
+ dataset_data = json.loads(dataset.model_dump_json())
175
+
176
+ dataset_entry = {
177
+ **dataset_data,
178
+ "input_filename": model_path.resolve().as_posix(),
179
+ }
180
+
181
+ datasets_with_paths.append(dataset_entry)
182
+
183
+ return datasets_with_paths
184
+
185
+ datasets_with_paths = find_datasets_with_paths(root_path=path)
186
+
187
+ output = json.dumps(datasets_with_paths, indent=2)
188
+ typer.echo(output, nl=False)
189
+
190
+
191
+ @app.command("list-models")
192
+ def list_models(
193
+ path: Annotated[
194
+ Path,
195
+ typer.Argument(
196
+ help="Directory path containing model cards",
197
+ exists=True,
198
+ file_okay=True,
199
+ dir_okay=True,
200
+ ),
201
+ ],
202
+ ):
203
+ """List available models with optional query filtering.
204
+
205
+ Note: This custom CLI is used instead of yq because:
206
+ - Parses Markdown files with model card frontmatter, not just YAML/JSON
207
+ - Provides sophisticated error handling with ValidationError logging
208
+ - Creates custom JSON output with both model data and file metadata
209
+ - Includes recursive file discovery for .md files
210
+ - Leverages Pydantic validation for type safety and schema compliance
211
+ """
212
+
213
+ def find_models_with_paths(root_path: Path) -> list[dict]:
214
+ """Find all model cards in the given directory."""
215
+ models_with_paths = []
216
+
217
+ if root_path.is_file():
218
+ paths = [root_path]
219
+ else:
220
+ paths = root_path.rglob("*.md")
221
+
222
+ for model_path in paths:
223
+ try:
224
+ model = ModelCard.from_path(model_path)
225
+ except ValidationError as e:
226
+ logger.error(f"Skipping {model_path}", exc_info=e)
227
+ else:
228
+ model_entry = {
229
+ **model.model_dump(),
230
+ "input_filename": model_path.resolve().as_posix(),
231
+ }
232
+
233
+ models_with_paths.append(model_entry)
234
+
235
+ return models_with_paths
236
+
237
+ models_with_paths = find_models_with_paths(root_path=path)
238
+
239
+ output = json.dumps(models_with_paths, indent=2)
240
+ typer.echo(output, nl=False)
241
+
242
+
243
+ @app.command("generate-data")
244
+ def generate_data(
245
+ feature1: Annotated[
246
+ str,
247
+ typer.Argument(help="Name of the first dummy feature"),
248
+ ] = "foo",
249
+ feature2: Annotated[
250
+ str,
251
+ typer.Argument(help="Name of the second dummy feature"),
252
+ ] = "bar",
253
+ *,
254
+ n_rows: Annotated[
255
+ int, typer.Option(help="Number of rows to generate in a data frame")
256
+ ] = 500,
257
+ sequence_length: Annotated[
258
+ int, typer.Option(help="Length of sequence for the sequence column")
259
+ ] = 100,
260
+ fmt: Annotated[
261
+ str,
262
+ typer.Option(
263
+ help="Output format",
264
+ click_type=click.Choice(choices=["csv"], case_sensitive=False),
265
+ ),
266
+ ] = "csv",
267
+ ) -> None:
268
+ """Generate dummy data with charge ladder and two dummy features.
269
+
270
+ Creates a charge ladder dataset with protein sequences and adds two dummy
271
+ features to the target charge column. The output is formatted as CSV.
272
+ Users can update feature names, but it is currently limited to only two features.
273
+
274
+ Args:
275
+ feature1: Name of the first dummy feature. Defaults to "foo".
276
+ feature2: Name of the second dummy feature. Defaults to "bar".
277
+ n_rows: Number of rows to generate in the data frame. Defaults to 500.
278
+ sequence_length: Length of sequence for the sequence column. Defaults to 100.
279
+ fmt: Output format. Currently only supports "csv". Defaults to "csv".
280
+
281
+ Outputs:
282
+ Prints the generated CSV data to stdout.
283
+ """
284
+ ladder = charge_ladder_dataset(n_rows, sequence_length)
285
+
286
+ output = ladder.pipe(
287
+ adjust_target_with_two_dummy_features,
288
+ target="charge",
289
+ feature_names=[feature1, feature2],
290
+ )
291
+
292
+ if fmt.lower() == "csv":
293
+ typer.echo(output.write_csv(), nl=False)
294
+
295
+
296
+ @app.command("evaluate-subset")
297
+ def evaluate_subset_command(
298
+ prediction_path: Annotated[
299
+ Path,
300
+ typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
301
+ ],
302
+ metric_path: Annotated[
303
+ Path,
304
+ typer.Option(help="Path where the calculated metrics JSON will be saved."),
305
+ ],
306
+ dataset_path: Annotated[
307
+ Path,
308
+ typer.Option(help="Path to the ground truth Subsets archive (.splits.pgdata)."),
309
+ ],
310
+ split: Annotated[
311
+ str,
312
+ typer.Option(help="Name of the splitting strategy to evaluate."),
313
+ ],
314
+ target: Annotated[
315
+ str,
316
+ typer.Option(help="Name of the target variable to score."),
317
+ ],
318
+ fold: Annotated[
319
+ str,
320
+ typer.Option(help="Fold index designated as the test fold."),
321
+ ],
322
+ selected_metrics: Annotated[
323
+ list[str] | None,
324
+ typer.Option(
325
+ help="Metric name to calculate. Repeat the flag for multiple metrics."
326
+ ),
327
+ ] = None,
328
+ model_name: Annotated[
329
+ str | None,
330
+ typer.Option(help="Name of the model that generated predictions."),
331
+ ] = None,
332
+ score_modes: Annotated[
333
+ list[ScoreMode] | None,
334
+ typer.Option(
335
+ help="Scoring mode to compute. Repeat the flag for multiple modes."
336
+ ),
337
+ ] = None,
338
+ ) -> None:
339
+ """Calculate metrics for cross-validation splits and save results to JSON.
340
+
341
+ Args:
342
+ prediction_path: Path to the prediction dataset archive (.pgdata file)
343
+ containing model predictions.
344
+ metric_path: Path where the calculated metrics JSON will be saved.
345
+ dataset_path: Path to the ground truth Subsets archive (.splits.pgdata).
346
+ split: Name of the splitting strategy to evaluate.
347
+ target: Name of the target variable to score.
348
+ fold: Fold index (as string) designated as the test fold.
349
+ selected_metrics: Optional list of metric names to calculate. If not
350
+ provided, all discovered metrics are included.
351
+ model_name: Name of the model that generated predictions (stored in
352
+ metadata).
353
+ score_modes: Optional list of scoring modes to compute.
354
+ """
355
+ result_path = evaluate_subset_fn(
356
+ prediction_path=prediction_path,
357
+ metric_path=metric_path,
358
+ dataset_path=dataset_path,
359
+ split=split,
360
+ target=target,
361
+ fold=fold,
362
+ selected_metrics=selected_metrics,
363
+ model_name=model_name,
364
+ score_modes=score_modes,
365
+ )
366
+ typer.echo(f"Metrics saved to: {result_path}")
367
+
368
+
369
+ @app.command("evaluate-dataset")
370
+ def evaluate_dataset_command(
371
+ prediction_path: Annotated[
372
+ Path,
373
+ typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
374
+ ],
375
+ metric_path: Annotated[
376
+ Path,
377
+ typer.Option(help="Path where the calculated metrics JSON will be saved."),
378
+ ],
379
+ dataset_path: Annotated[
380
+ Path,
381
+ typer.Option(help="Path to the ground truth Dataset archive (.pgdata)."),
382
+ ],
383
+ target: Annotated[
384
+ str,
385
+ typer.Option(help="Name of the target variable to score."),
386
+ ],
387
+ selected_metrics: Annotated[
388
+ list[str] | None,
389
+ typer.Option(
390
+ help="Metric name to calculate. Repeat the flag for multiple metrics."
391
+ ),
392
+ ] = None,
393
+ model_name: Annotated[
394
+ str | None,
395
+ typer.Option(help="Name of the model that generated predictions."),
396
+ ] = None,
397
+ ) -> None:
398
+ """Calculate metrics for a plain dataset and save results to JSON.
399
+
400
+ Args:
401
+ prediction_path: Path to the prediction dataset archive (.pgdata file)
402
+ containing model predictions.
403
+ metric_path: Path where the calculated metrics JSON will be saved.
404
+ dataset_path: Path to the ground truth Dataset archive (.pgdata).
405
+ target: Name of the target variable to score.
406
+ selected_metrics: Optional list of metric names to calculate. If not
407
+ provided, all discovered metrics are included.
408
+ model_name: Name of the model that generated predictions (stored in
409
+ metadata).
410
+ """
411
+ result_path = evaluate_dataset_fn(
412
+ prediction_path=prediction_path,
413
+ metric_path=metric_path,
414
+ dataset_path=dataset_path,
415
+ target=target,
416
+ selected_metrics=selected_metrics,
417
+ model_name=model_name,
418
+ )
419
+ typer.echo(f"Metrics saved to: {result_path}")
420
+
421
+
422
+ @app.command("evaluate-zero-shot")
423
+ def evaluate_zero_shot_command(
424
+ prediction_path: Annotated[
425
+ Path,
426
+ typer.Option(help="Path to the prediction dataset archive (.pgdata file)."),
427
+ ],
428
+ metric_path: Annotated[
429
+ Path,
430
+ typer.Option(help="Path where the calculated metrics JSON will be saved."),
431
+ ],
432
+ dataset_path: Annotated[
433
+ Path,
434
+ typer.Option(help="Path to the ground truth Subsets archive (.splits.pgdata)."),
435
+ ],
436
+ split: Annotated[
437
+ str,
438
+ typer.Option(help="Name of the splitting strategy (stored in metadata)."),
439
+ ],
440
+ target: Annotated[
441
+ str,
442
+ typer.Option(help="Name of the target variable to score."),
443
+ ],
444
+ selected_metrics: Annotated[
445
+ list[str] | None,
446
+ typer.Option(
447
+ help="Metric name to calculate. Repeat the flag for multiple metrics."
448
+ ),
449
+ ] = None,
450
+ model_name: Annotated[
451
+ str | None,
452
+ typer.Option(help="Name of the model that generated predictions."),
453
+ ] = None,
454
+ ) -> None:
455
+ """Calculate zero-shot metrics against the full dataset and save to JSON.
456
+
457
+ Scores the requested metrics once against the complete underlying dataset of a
458
+ Subsets archive, ignoring cross-validation folds. Appropriate for zero-shot
459
+ models whose predictions do not depend on the training fold.
460
+
461
+ Args:
462
+ prediction_path: Path to the prediction dataset archive (.pgdata file)
463
+ containing model predictions.
464
+ metric_path: Path where the calculated metrics JSON will be saved.
465
+ dataset_path: Path to the ground truth Subsets archive (.splits.pgdata).
466
+ split: Name of the splitting strategy (stored in metadata).
467
+ target: Name of the target variable to score.
468
+ selected_metrics: Optional list of metric names to calculate. If not
469
+ provided, all discovered metrics are included.
470
+ model_name: Name of the model that generated predictions (stored in
471
+ metadata).
472
+ """
473
+ result_path = evaluate_zero_shot_fn(
474
+ prediction_path=prediction_path,
475
+ metric_path=metric_path,
476
+ dataset_path=dataset_path,
477
+ split=split,
478
+ target=target,
479
+ selected_metrics=selected_metrics,
480
+ model_name=model_name,
481
+ )
482
+ typer.echo(f"Metrics saved to: {result_path}")
483
+
484
+
485
+ if __name__ == "__main__":
486
+ app()