scorebook 0.0.10__py3-none-any.whl → 0.0.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scorebook/__init__.py +3 -3
- scorebook/eval_datasets/__init__.py +5 -0
- scorebook/eval_datasets/eval_dataset.py +719 -0
- scorebook/evaluate/_async/evaluate_async.py +58 -28
- scorebook/evaluate/_sync/evaluate.py +58 -28
- scorebook/evaluate/evaluate_helpers.py +31 -8
- scorebook/exceptions.py +48 -0
- scorebook/inference/clients/bedrock.py +1 -1
- scorebook/inference/clients/portkey.py +1 -1
- scorebook/inference/clients/vertex.py +1 -1
- scorebook/settings.py +3 -0
- scorebook/types.py +8 -5
- scorebook/utils/__init__.py +4 -4
- scorebook/utils/io_helpers.py +18 -5
- scorebook/utils/progress_bars.py +752 -70
- scorebook/utils/{build_prompt.py → render_template.py} +13 -12
- {scorebook-0.0.10.dist-info → scorebook-0.0.11.dist-info}/METADATA +2 -1
- {scorebook-0.0.10.dist-info → scorebook-0.0.11.dist-info}/RECORD +21 -20
- scorebook/eval_dataset.py +0 -404
- {scorebook-0.0.10.dist-info → scorebook-0.0.11.dist-info}/WHEEL +0 -0
- {scorebook-0.0.10.dist-info → scorebook-0.0.11.dist-info}/entry_points.txt +0 -0
- {scorebook-0.0.10.dist-info → scorebook-0.0.11.dist-info}/licenses/LICENSE +0 -0
scorebook/utils/io_helpers.py
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
"""Input/output helper functions for Scorebook."""
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import Optional
|
|
4
|
+
from typing import Optional, Tuple, Union
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
def validate_path(
|
|
7
|
+
def validate_path(
|
|
8
|
+
file_path: Union[str, Path], expected_suffix: Optional[Union[str, Tuple[str, ...]]] = None
|
|
9
|
+
) -> Path:
|
|
8
10
|
"""Validate that a file path exists and optionally check its suffix.
|
|
9
11
|
|
|
10
12
|
Args:
|
|
11
13
|
file_path: Path to the file as string or Path object
|
|
12
|
-
expected_suffix: Optional file extension to validate
|
|
14
|
+
expected_suffix: Optional file extension(s) to validate.
|
|
15
|
+
Can be a single string (e.g. ".json") or tuple of strings (e.g. (".yaml", ".yml"))
|
|
13
16
|
|
|
14
17
|
Returns:
|
|
15
18
|
Path object for the validated file path
|
|
@@ -22,7 +25,17 @@ def validate_path(file_path: str, expected_suffix: Optional[str] = None) -> Path
|
|
|
22
25
|
if not path.exists():
|
|
23
26
|
raise FileNotFoundError(f"File not found: {file_path}")
|
|
24
27
|
|
|
25
|
-
if expected_suffix
|
|
26
|
-
|
|
28
|
+
if expected_suffix:
|
|
29
|
+
# Convert single suffix to tuple for uniform handling
|
|
30
|
+
allowed_suffixes = (
|
|
31
|
+
(expected_suffix,) if isinstance(expected_suffix, str) else expected_suffix
|
|
32
|
+
)
|
|
33
|
+
allowed_suffixes_lower = tuple(s.lower() for s in allowed_suffixes)
|
|
34
|
+
|
|
35
|
+
if path.suffix.lower() not in allowed_suffixes_lower:
|
|
36
|
+
suffix_list = ", ".join(f"'{s}'" for s in allowed_suffixes)
|
|
37
|
+
raise ValueError(
|
|
38
|
+
f"File must have one of ({suffix_list}) extensions, got: '{path.suffix}'"
|
|
39
|
+
)
|
|
27
40
|
|
|
28
41
|
return path
|