python-yama 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_yama-0.1.0.dist-info/METADATA +98 -0
- python_yama-0.1.0.dist-info/RECORD +27 -0
- python_yama-0.1.0.dist-info/WHEEL +4 -0
- python_yama-0.1.0.dist-info/entry_points.txt +2 -0
- python_yama-0.1.0.dist-info/licenses/LICENSE +21 -0
- yama/__init__.py +31 -0
- yama/__main__.py +3 -0
- yama/assertions.py +577 -0
- yama/cli.py +318 -0
- yama/llm.py +365 -0
- yama/loaders/__init__.py +6 -0
- yama/loaders/case.py +378 -0
- yama/loaders/common.py +29 -0
- yama/loaders/tool_loader.py +173 -0
- yama/mocks.py +383 -0
- yama/models.py +159 -0
- yama/report.py +854 -0
- yama/runner.py +531 -0
- yama/tools/__init__.py +0 -0
- yama/tools/base.py +19 -0
- yama/tools/bash/__init__.py +0 -0
- yama/tools/bash/_cli_shim.py +71 -0
- yama/tools/bash/bash.py +285 -0
- yama/tools/bash/fs.py +530 -0
- yama/tools/skill/__init__.py +0 -0
- yama/tools/skill/skill.py +57 -0
- yama/workspace.py +115 -0
yama/workspace.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import tomllib
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
DEFAULT_CASE_PATTERNS = [
|
|
10
|
+
"__evals__/cases/**/*.yaml",
|
|
11
|
+
"__evals__/cases/**/*.yml",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ConfigurationError(ValueError):
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class WorkspaceConfig:
|
|
21
|
+
root: Path
|
|
22
|
+
config_path: Path | None = None
|
|
23
|
+
case_patterns: list[str] = field(
|
|
24
|
+
default_factory=lambda: list(DEFAULT_CASE_PATTERNS)
|
|
25
|
+
)
|
|
26
|
+
result_dir: str = ".yama/runs"
|
|
27
|
+
defaults: dict[str, Any] = field(
|
|
28
|
+
default_factory=lambda: {
|
|
29
|
+
"model": {"name": "gpt-5", "temperature": 0},
|
|
30
|
+
"execution": {
|
|
31
|
+
"timeout_seconds_per_step": 120,
|
|
32
|
+
"max_tool_rounds_per_step": 10,
|
|
33
|
+
"repeat": 1,
|
|
34
|
+
},
|
|
35
|
+
}
|
|
36
|
+
)
|
|
37
|
+
plugins: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def find_workspace_config(start: Path) -> Path | None:
|
|
41
|
+
current = start.resolve()
|
|
42
|
+
for directory in (current, *current.parents):
|
|
43
|
+
candidate = directory / "yama.toml"
|
|
44
|
+
if candidate.is_file():
|
|
45
|
+
return candidate
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def load_workspace_config(start: Path) -> WorkspaceConfig:
|
|
50
|
+
path = find_workspace_config(start)
|
|
51
|
+
if path is None:
|
|
52
|
+
return WorkspaceConfig(root=start.resolve())
|
|
53
|
+
with path.open("rb") as handle:
|
|
54
|
+
raw = tomllib.load(handle)
|
|
55
|
+
workspace = raw.get("workspace", {})
|
|
56
|
+
defaults = raw.get("defaults", {})
|
|
57
|
+
merged_defaults = WorkspaceConfig(root=path.parent).defaults
|
|
58
|
+
for section in ("model", "execution"):
|
|
59
|
+
merged_defaults[section].update(defaults.get(section, {}))
|
|
60
|
+
return WorkspaceConfig(
|
|
61
|
+
root=path.parent.resolve(),
|
|
62
|
+
config_path=path,
|
|
63
|
+
case_patterns=list(workspace.get("case_patterns", DEFAULT_CASE_PATTERNS)),
|
|
64
|
+
result_dir=workspace.get("result_dir", ".yama/runs"),
|
|
65
|
+
defaults=merged_defaults,
|
|
66
|
+
plugins=dict(raw.get("plugins", {})),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def resolve_plugin_root(
|
|
71
|
+
config: WorkspaceConfig,
|
|
72
|
+
*,
|
|
73
|
+
explicit_root: Path | None = None,
|
|
74
|
+
plugin_name: str | None = None,
|
|
75
|
+
) -> tuple[Path, dict[str, Any]]:
|
|
76
|
+
if explicit_root is not None:
|
|
77
|
+
root = explicit_root.resolve()
|
|
78
|
+
plugin = {}
|
|
79
|
+
elif plugin_name is not None:
|
|
80
|
+
if plugin_name not in config.plugins:
|
|
81
|
+
raise ConfigurationError(f"unknown plugin: {plugin_name}")
|
|
82
|
+
plugin = config.plugins[plugin_name]
|
|
83
|
+
root = (config.root / plugin["path"]).resolve()
|
|
84
|
+
else:
|
|
85
|
+
root = config.root
|
|
86
|
+
plugin = {}
|
|
87
|
+
if not root.is_dir():
|
|
88
|
+
raise ConfigurationError(f"plugin root does not exist: {root}")
|
|
89
|
+
system_path = root / plugin.get("system_prompt", "SYSTEM.md")
|
|
90
|
+
if not system_path.is_file():
|
|
91
|
+
raise ConfigurationError(f"system prompt does not exist: {system_path}")
|
|
92
|
+
return root, plugin
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def collect_case_paths(
|
|
96
|
+
plugin_root: Path,
|
|
97
|
+
workspace: WorkspaceConfig,
|
|
98
|
+
plugin_config: dict[str, Any],
|
|
99
|
+
explicit_paths: list[Path] | None = None,
|
|
100
|
+
) -> list[Path]:
|
|
101
|
+
if explicit_paths:
|
|
102
|
+
paths = [
|
|
103
|
+
(path if path.is_absolute() else Path.cwd() / path).resolve()
|
|
104
|
+
for path in explicit_paths
|
|
105
|
+
]
|
|
106
|
+
else:
|
|
107
|
+
patterns = plugin_config.get("case_patterns", workspace.case_patterns)
|
|
108
|
+
paths = []
|
|
109
|
+
for pattern in patterns:
|
|
110
|
+
paths.extend(plugin_root.glob(pattern))
|
|
111
|
+
paths = sorted(set(path.resolve() for path in paths))
|
|
112
|
+
missing = [path for path in paths if not path.is_file()]
|
|
113
|
+
if missing:
|
|
114
|
+
raise ConfigurationError(f"case file does not exist: {missing[0]}")
|
|
115
|
+
return paths
|