mindcode 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mindcode/__init__.py +5 -0
- mindcode/__main__.py +4 -0
- mindcode/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/__pycache__/__main__.cpython-313.pyc +0 -0
- mindcode/__pycache__/_version.cpython-313.pyc +0 -0
- mindcode/__pycache__/config.cpython-313.pyc +0 -0
- mindcode/__pycache__/policy.cpython-313.pyc +0 -0
- mindcode/__pycache__/remote.cpython-313.pyc +0 -0
- mindcode/__pycache__/render.cpython-313.pyc +0 -0
- mindcode/__pycache__/runtime.cpython-313.pyc +0 -0
- mindcode/__pycache__/tasking.cpython-313.pyc +0 -0
- mindcode/__pycache__/terminal_core.cpython-313.pyc +0 -0
- mindcode/_version.py +11 -0
- mindcode/approval.py +105 -0
- mindcode/bench/__init__.py +19 -0
- mindcode/bench/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/fake_tools.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/jsonutil.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/paths.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/predictions.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/report.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/runner.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/schema.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/trace.cpython-313.pyc +0 -0
- mindcode/bench/__pycache__/workspace.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__init__.py +22 -0
- mindcode/bench/adapters/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/base.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/mini_bfcl.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/mini_gaia.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/mini_terminal.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/swe_bench.cpython-313.pyc +0 -0
- mindcode/bench/adapters/__pycache__/terminal_bench.cpython-313.pyc +0 -0
- mindcode/bench/adapters/base.py +45 -0
- mindcode/bench/adapters/mini_bfcl.py +18 -0
- mindcode/bench/adapters/mini_gaia.py +18 -0
- mindcode/bench/adapters/mini_terminal.py +18 -0
- mindcode/bench/adapters/swe_bench.py +86 -0
- mindcode/bench/adapters/terminal_bench.py +107 -0
- mindcode/bench/fake_tools.py +73 -0
- mindcode/bench/jsonutil.py +20 -0
- mindcode/bench/paths.py +10 -0
- mindcode/bench/predictions.py +65 -0
- mindcode/bench/report.py +48 -0
- mindcode/bench/runner.py +250 -0
- mindcode/bench/schema.py +78 -0
- mindcode/bench/scorers/__pycache__/base.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/composite.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/exact.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/json_call.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/swe.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/terminal.cpython-313.pyc +0 -0
- mindcode/bench/scorers/__pycache__/terminal_bench.cpython-313.pyc +0 -0
- mindcode/bench/scorers/base.py +22 -0
- mindcode/bench/scorers/composite.py +57 -0
- mindcode/bench/scorers/exact.py +30 -0
- mindcode/bench/scorers/json_call.py +44 -0
- mindcode/bench/scorers/swe.py +29 -0
- mindcode/bench/scorers/terminal.py +43 -0
- mindcode/bench/scorers/terminal_bench.py +25 -0
- mindcode/bench/trace.py +34 -0
- mindcode/bench/workspace.py +127 -0
- mindcode/cli/__init__.py +112 -0
- mindcode/cli/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/cli/__pycache__/_shared.cpython-313.pyc +0 -0
- mindcode/cli/_shared.py +43 -0
- mindcode/cli/commands/__init__.py +1 -0
- mindcode/cli/commands/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/bench.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/chat.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/config_cmd.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/remote.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/shell.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/status.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/task.cpython-313.pyc +0 -0
- mindcode/cli/commands/__pycache__/terminal.cpython-313.pyc +0 -0
- mindcode/cli/commands/bench.py +365 -0
- mindcode/cli/commands/chat.py +86 -0
- mindcode/cli/commands/config_cmd.py +197 -0
- mindcode/cli/commands/remote.py +146 -0
- mindcode/cli/commands/shell.py +108 -0
- mindcode/cli/commands/status.py +37 -0
- mindcode/cli/commands/task.py +417 -0
- mindcode/cli/commands/terminal.py +124 -0
- mindcode/cli/shell/__init__.py +5 -0
- mindcode/cli/shell/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/cli/shell/__pycache__/completion.cpython-313.pyc +0 -0
- mindcode/cli/shell/__pycache__/menu.cpython-313.pyc +0 -0
- mindcode/cli/shell/__pycache__/repl.cpython-313.pyc +0 -0
- mindcode/cli/shell/__pycache__/slash.cpython-313.pyc +0 -0
- mindcode/cli/shell/__pycache__/startup.cpython-313.pyc +0 -0
- mindcode/cli/shell/completion.py +72 -0
- mindcode/cli/shell/menu.py +88 -0
- mindcode/cli/shell/repl.py +597 -0
- mindcode/cli/shell/slash.py +480 -0
- mindcode/cli/shell/startup.py +55 -0
- mindcode/cli/shell/tui.py +897 -0
- mindcode/config.py +173 -0
- mindcode/mcp.py +205 -0
- mindcode/policy.py +173 -0
- mindcode/remote.py +216 -0
- mindcode/render.py +458 -0
- mindcode/runtime.py +541 -0
- mindcode/skills.py +154 -0
- mindcode/subagents.py +43 -0
- mindcode/tasking.py +281 -0
- mindcode/terminal/__init__.py +1 -0
- mindcode/terminal/__main__.py +17 -0
- mindcode/terminal/__pycache__/__init__.cpython-313.pyc +0 -0
- mindcode/terminal/__pycache__/__main__.cpython-313.pyc +0 -0
- mindcode/terminal_core.py +264 -0
- mindcode-0.2.0.dist-info/METADATA +244 -0
- mindcode-0.2.0.dist-info/RECORD +116 -0
- mindcode-0.2.0.dist-info/WHEEL +5 -0
- mindcode-0.2.0.dist-info/entry_points.txt +2 -0
- mindcode-0.2.0.dist-info/top_level.txt +1 -0
mindcode/__init__.py
ADDED
mindcode/__main__.py
ADDED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
mindcode/_version.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""mindcode 版本与品牌信息。"""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.2.0"
|
|
4
|
+
|
|
5
|
+
__logo__ = r"""
|
|
6
|
+
[bold cyan] _ _[/bold cyan]
|
|
7
|
+
[bold cyan] _ __ ___ ___ _ __ ___ (_) ___ ___| |_[/bold cyan]
|
|
8
|
+
[bold cyan] | '_ ` _ \ / _ \ '_ ` _ \| |/ _ \/ __| __|[/bold cyan]
|
|
9
|
+
[bold cyan] | | | | | | __/ | | | | | | __/ (__| |_[/bold cyan]
|
|
10
|
+
[bold cyan] |_| |_| |_|\___|_| |_| |_|_|\___|\___|\__|[/bold cyan]
|
|
11
|
+
"""
|
mindcode/approval.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""审批文案与审批回调装配(TUI 优先,裸终端回退)。
|
|
2
|
+
|
|
3
|
+
``describe_action`` 只依赖 ``mindagent.core`` 的数据结构,不碰
|
|
4
|
+
prompt_toolkit;``make_shell_approval`` 在运行时动态读取 chooser,
|
|
5
|
+
TUI 全屏运行时走 ``MindcodeTui.choose`` 的 RadioList 浮层,
|
|
6
|
+
one-shot / TUI 未启动时回退到 ``menu.prompt_yes_no``。
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from collections.abc import Awaitable, Callable
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from mindagent.core import ActionRequest, AgentContext
|
|
16
|
+
|
|
17
|
+
from .policy import ApprovalCallback
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def risk_label(risk: Any) -> str:
|
|
21
|
+
"""ActionRisk 中文化;未知值回退到其 value/str。"""
|
|
22
|
+
from mindagent.core import ActionRisk
|
|
23
|
+
|
|
24
|
+
mapping = {
|
|
25
|
+
ActionRisk.READ_ONLY: "只读",
|
|
26
|
+
ActionRisk.WRITE: "写操作",
|
|
27
|
+
ActionRisk.DANGEROUS: "危险操作",
|
|
28
|
+
}
|
|
29
|
+
if risk in mapping:
|
|
30
|
+
return mapping[risk]
|
|
31
|
+
return str(getattr(risk, "value", risk))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _one_line(text: Any) -> str:
|
|
35
|
+
return str(text).replace("\r", " ").replace("\n", " ")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def describe_action(action: ActionRequest) -> str:
|
|
39
|
+
"""按工具类型生成一行友好摘要(保证单行,供 TUI 单行 title 用)。"""
|
|
40
|
+
name = action.name
|
|
41
|
+
arguments = (
|
|
42
|
+
action.arguments if isinstance(action.arguments, dict) else {}
|
|
43
|
+
)
|
|
44
|
+
if name in ("file_edit", "file_write"):
|
|
45
|
+
verb = "Edit" if name == "file_edit" else "Write"
|
|
46
|
+
path = _one_line(arguments.get("path", "?"))
|
|
47
|
+
return f"{verb} {path} [{risk_label(action.risk)}]"
|
|
48
|
+
if name == "exec_command":
|
|
49
|
+
command = arguments.get("command", "")
|
|
50
|
+
if isinstance(command, list):
|
|
51
|
+
preview = " ".join(_one_line(item) for item in command)
|
|
52
|
+
else:
|
|
53
|
+
preview = _one_line(command)
|
|
54
|
+
if len(preview) > 120:
|
|
55
|
+
preview = preview[:117] + "..."
|
|
56
|
+
return f"Command {preview} [{risk_label(action.risk)}]"
|
|
57
|
+
if name in ("file_read", "memory_read"):
|
|
58
|
+
path = _one_line(arguments.get("path", "?"))
|
|
59
|
+
return f"Read {path} [{risk_label(action.risk)}]"
|
|
60
|
+
payload = json.dumps(arguments, ensure_ascii=False, sort_keys=True)
|
|
61
|
+
if len(payload) > 240:
|
|
62
|
+
payload = payload[:237] + "..."
|
|
63
|
+
return f"{name} {payload}"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
Chooser = Callable[
|
|
67
|
+
[str, list[tuple[str, str]], str | None],
|
|
68
|
+
Awaitable[str | None],
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def make_shell_approval(
|
|
73
|
+
get_chooser: Callable[[], Chooser | None],
|
|
74
|
+
) -> ApprovalCallback:
|
|
75
|
+
"""装配 `(context, action) -> bool` 审批回调。
|
|
76
|
+
|
|
77
|
+
TUI 运行时 ``get_chooser()`` 返回 ``MindcodeTui.choose``(与 slash
|
|
78
|
+
命令的 ``_choose_option`` 同一条路),用 RadioList 浮层二选一;
|
|
79
|
+
返回 None(取消/Esc/TUI 未运行)一律视为拒绝。chooser 不可用时
|
|
80
|
+
回退到裸 ``menu.prompt_yes_no``(one-shot / 非 TUI 路径)。
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
async def _approve(
|
|
84
|
+
context: AgentContext,
|
|
85
|
+
action: ActionRequest,
|
|
86
|
+
) -> bool:
|
|
87
|
+
summary = describe_action(action)
|
|
88
|
+
chooser = get_chooser()
|
|
89
|
+
if chooser is not None:
|
|
90
|
+
# choose 的 dialog title 是 height=1 的单行 Window,
|
|
91
|
+
# 多行会被裁掉,故 summary 必须压成单行。
|
|
92
|
+
result = await chooser(
|
|
93
|
+
f"允许执行? {summary}",
|
|
94
|
+
[("allow", "允许"), ("deny", "拒绝")],
|
|
95
|
+
"deny",
|
|
96
|
+
)
|
|
97
|
+
return result == "allow"
|
|
98
|
+
# Lazy import:非交互进程构造 policy 时不引入 prompt_toolkit。
|
|
99
|
+
from .cli.shell.menu import prompt_yes_no
|
|
100
|
+
|
|
101
|
+
return await prompt_yes_no(
|
|
102
|
+
f"允许执行 {summary}?", default_yes=False
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
return _approve
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Benchmark facility for mindcode."""
|
|
2
|
+
|
|
3
|
+
from .schema import (
|
|
4
|
+
BenchmarkCase,
|
|
5
|
+
BenchmarkFile,
|
|
6
|
+
BenchmarkResult,
|
|
7
|
+
BenchmarkRunConfig,
|
|
8
|
+
CaseTrace,
|
|
9
|
+
ScoreResult,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"BenchmarkCase",
|
|
14
|
+
"BenchmarkFile",
|
|
15
|
+
"BenchmarkResult",
|
|
16
|
+
"BenchmarkRunConfig",
|
|
17
|
+
"CaseTrace",
|
|
18
|
+
"ScoreResult",
|
|
19
|
+
]
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from .base import BenchmarkAdapter
|
|
4
|
+
from .mini_bfcl import MiniBfclAdapter
|
|
5
|
+
from .mini_gaia import MiniGaiaAdapter
|
|
6
|
+
from .mini_terminal import MiniTerminalAdapter
|
|
7
|
+
from .swe_bench import SWEBenchAdapter
|
|
8
|
+
from .terminal_bench import TerminalBenchAdapter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def built_in_adapters() -> dict[str, BenchmarkAdapter]:
|
|
12
|
+
adapters: list[BenchmarkAdapter] = [
|
|
13
|
+
MiniBfclAdapter(),
|
|
14
|
+
MiniTerminalAdapter(),
|
|
15
|
+
MiniGaiaAdapter(),
|
|
16
|
+
SWEBenchAdapter(),
|
|
17
|
+
TerminalBenchAdapter(),
|
|
18
|
+
]
|
|
19
|
+
return {adapter.suite: adapter for adapter in adapters}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
__all__ = ["BenchmarkAdapter", "built_in_adapters"]
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Protocol
|
|
6
|
+
|
|
7
|
+
from ..schema import BenchmarkCase, BenchmarkFile
|
|
8
|
+
from ..scorers.base import Scorer
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class BenchmarkAdapter(Protocol):
|
|
12
|
+
suite: str
|
|
13
|
+
|
|
14
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]: ...
|
|
15
|
+
|
|
16
|
+
def build_scorer(self) -> Scorer: ...
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load_jsonl_cases(path: Path, suite: str) -> list[BenchmarkCase]:
|
|
20
|
+
cases: list[BenchmarkCase] = []
|
|
21
|
+
with path.open(encoding="utf-8") as file:
|
|
22
|
+
for line_number, line in enumerate(file, start=1):
|
|
23
|
+
if not line.strip():
|
|
24
|
+
continue
|
|
25
|
+
raw = json.loads(line)
|
|
26
|
+
files = tuple(
|
|
27
|
+
BenchmarkFile(path=item["path"], content=item["content"])
|
|
28
|
+
for item in raw.get("files", [])
|
|
29
|
+
)
|
|
30
|
+
cases.append(
|
|
31
|
+
BenchmarkCase(
|
|
32
|
+
case_id=str(raw["case_id"]),
|
|
33
|
+
suite=str(raw.get("suite", suite)),
|
|
34
|
+
category=str(raw.get("category", "uncategorized")),
|
|
35
|
+
prompt=str(raw["prompt"]),
|
|
36
|
+
expected=raw.get("expected"),
|
|
37
|
+
files=files,
|
|
38
|
+
tools=tuple(raw.get("tools", []) or ()),
|
|
39
|
+
metadata={
|
|
40
|
+
**dict(raw.get("metadata", {}) or {}),
|
|
41
|
+
"source_line": line_number,
|
|
42
|
+
},
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
return cases
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ..schema import BenchmarkCase
|
|
6
|
+
from ..scorers.base import Scorer
|
|
7
|
+
from ..scorers.json_call import JsonToolCallScorer
|
|
8
|
+
from .base import load_jsonl_cases
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MiniBfclAdapter:
|
|
12
|
+
suite = "mini_bfcl"
|
|
13
|
+
|
|
14
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]:
|
|
15
|
+
return load_jsonl_cases(root / self.suite / "cases.jsonl", self.suite)
|
|
16
|
+
|
|
17
|
+
def build_scorer(self) -> Scorer:
|
|
18
|
+
return JsonToolCallScorer()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ..schema import BenchmarkCase
|
|
6
|
+
from ..scorers.base import Scorer
|
|
7
|
+
from ..scorers.exact import ExactAnswerScorer
|
|
8
|
+
from .base import load_jsonl_cases
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MiniGaiaAdapter:
|
|
12
|
+
suite = "mini_gaia"
|
|
13
|
+
|
|
14
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]:
|
|
15
|
+
return load_jsonl_cases(root / self.suite / "cases.jsonl", self.suite)
|
|
16
|
+
|
|
17
|
+
def build_scorer(self) -> Scorer:
|
|
18
|
+
return ExactAnswerScorer()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ..schema import BenchmarkCase
|
|
6
|
+
from ..scorers.base import Scorer
|
|
7
|
+
from ..scorers.terminal import FileStateScorer
|
|
8
|
+
from .base import load_jsonl_cases
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MiniTerminalAdapter:
|
|
12
|
+
suite = "mini_terminal"
|
|
13
|
+
|
|
14
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]:
|
|
15
|
+
return load_jsonl_cases(root / self.suite / "cases.jsonl", self.suite)
|
|
16
|
+
|
|
17
|
+
def build_scorer(self) -> Scorer:
|
|
18
|
+
return FileStateScorer()
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..schema import BenchmarkCase, BenchmarkFile
|
|
8
|
+
from ..scorers.base import Scorer
|
|
9
|
+
from ..scorers.swe import PatchGeneratedScorer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SWEBenchAdapter:
|
|
13
|
+
suite = "swe_bench"
|
|
14
|
+
|
|
15
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]:
|
|
16
|
+
path = root if root.is_file() else root / self.suite / "cases.jsonl"
|
|
17
|
+
return [_case_from_raw(raw) for raw in _load_rows(path)]
|
|
18
|
+
|
|
19
|
+
def build_scorer(self) -> Scorer:
|
|
20
|
+
return PatchGeneratedScorer()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _load_rows(path: Path) -> list[dict[str, Any]]:
|
|
24
|
+
if path.suffix == ".json":
|
|
25
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
26
|
+
if isinstance(data, dict):
|
|
27
|
+
data = list(data.values())
|
|
28
|
+
if not isinstance(data, list):
|
|
29
|
+
raise ValueError("SWE-bench JSON dataset must be a list or dict")
|
|
30
|
+
return [dict(item) for item in data]
|
|
31
|
+
with path.open(encoding="utf-8") as file:
|
|
32
|
+
return [json.loads(line) for line in file if line.strip()]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _case_from_raw(raw: dict[str, Any]) -> BenchmarkCase:
|
|
36
|
+
instance_id = str(raw["instance_id"])
|
|
37
|
+
problem = str(raw.get("problem_statement") or raw.get("text") or "")
|
|
38
|
+
prompt = _build_prompt(raw, problem)
|
|
39
|
+
files = tuple(
|
|
40
|
+
BenchmarkFile(path=str(item["path"]), content=str(item["content"]))
|
|
41
|
+
for item in raw.get("files", []) or []
|
|
42
|
+
)
|
|
43
|
+
metadata = {
|
|
44
|
+
key: raw.get(key)
|
|
45
|
+
for key in (
|
|
46
|
+
"repo",
|
|
47
|
+
"base_commit",
|
|
48
|
+
"version",
|
|
49
|
+
"patch",
|
|
50
|
+
"test_patch",
|
|
51
|
+
"FAIL_TO_PASS",
|
|
52
|
+
"PASS_TO_PASS",
|
|
53
|
+
)
|
|
54
|
+
if key in raw
|
|
55
|
+
}
|
|
56
|
+
return BenchmarkCase(
|
|
57
|
+
case_id=instance_id,
|
|
58
|
+
suite="swe_bench",
|
|
59
|
+
category=str(raw.get("category", "swe_issue")),
|
|
60
|
+
prompt=prompt,
|
|
61
|
+
expected={"gold_patch": raw.get("patch")},
|
|
62
|
+
files=files,
|
|
63
|
+
metadata=metadata,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _build_prompt(raw: dict[str, Any], problem: str) -> str:
|
|
68
|
+
repo = raw.get("repo", "")
|
|
69
|
+
base_commit = raw.get("base_commit", "")
|
|
70
|
+
hints = raw.get("hints_text", "")
|
|
71
|
+
parts = [
|
|
72
|
+
"You are solving a SWE-bench task.",
|
|
73
|
+
"Inspect and edit the repository files in the workspace.",
|
|
74
|
+
"When done, ensure the workspace diff is the patch for the fix.",
|
|
75
|
+
"Do not include unrelated changes.",
|
|
76
|
+
"",
|
|
77
|
+
f"instance_id: {raw['instance_id']}",
|
|
78
|
+
]
|
|
79
|
+
if repo:
|
|
80
|
+
parts.append(f"repo: {repo}")
|
|
81
|
+
if base_commit:
|
|
82
|
+
parts.append(f"base_commit: {base_commit}")
|
|
83
|
+
parts.extend(["", "Problem statement:", problem])
|
|
84
|
+
if hints:
|
|
85
|
+
parts.extend(["", "Hints:", str(hints)])
|
|
86
|
+
return "\n".join(parts).strip()
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from ..paths import DEFAULT_TERMINAL_BENCH_ROOT
|
|
10
|
+
from ..schema import BenchmarkCase, BenchmarkFile
|
|
11
|
+
from ..scorers.base import Scorer
|
|
12
|
+
from ..scorers.terminal_bench import TerminalBenchRunScorer
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TerminalBenchAdapter:
|
|
16
|
+
suite = "terminal_bench"
|
|
17
|
+
|
|
18
|
+
def load_cases(self, root: Path) -> list[BenchmarkCase]:
|
|
19
|
+
jsonl_path = root / self.suite / "cases.jsonl"
|
|
20
|
+
if jsonl_path.exists():
|
|
21
|
+
return [_case_from_json(raw) for raw in _load_jsonl(jsonl_path)]
|
|
22
|
+
tasks_root = DEFAULT_TERMINAL_BENCH_ROOT / "original-tasks"
|
|
23
|
+
return [
|
|
24
|
+
_case_from_task_dir(task_dir)
|
|
25
|
+
for task_dir in sorted(tasks_root.iterdir())
|
|
26
|
+
if (task_dir / "task.yaml").exists()
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
def build_scorer(self) -> Scorer:
|
|
30
|
+
return TerminalBenchRunScorer()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _load_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
34
|
+
with path.open(encoding="utf-8") as file:
|
|
35
|
+
return [json.loads(line) for line in file if line.strip()]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _case_from_json(raw: dict[str, Any]) -> BenchmarkCase:
|
|
39
|
+
files = tuple(
|
|
40
|
+
BenchmarkFile(path=str(item["path"]), content=str(item["content"]))
|
|
41
|
+
for item in raw.get("files", []) or []
|
|
42
|
+
)
|
|
43
|
+
return BenchmarkCase(
|
|
44
|
+
case_id=str(raw["case_id"]),
|
|
45
|
+
suite="terminal_bench",
|
|
46
|
+
category=str(raw.get("category", "terminal_task")),
|
|
47
|
+
prompt=str(raw["prompt"]),
|
|
48
|
+
expected=raw.get("expected"),
|
|
49
|
+
files=files,
|
|
50
|
+
metadata=dict(raw.get("metadata", {}) or {}),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _case_from_task_dir(task_dir: Path) -> BenchmarkCase:
|
|
55
|
+
task = yaml.safe_load((task_dir / "task.yaml").read_text(encoding="utf-8"))
|
|
56
|
+
files = tuple(_iter_visible_files(task_dir))
|
|
57
|
+
instruction = str(task["instruction"])
|
|
58
|
+
prompt = (
|
|
59
|
+
"You are solving a Terminal-Bench task in this workspace.\n"
|
|
60
|
+
"Use shell and file tools as needed. Do not look for solution files.\n"
|
|
61
|
+
"When finished, leave the workspace in the solved state and summarize "
|
|
62
|
+
"what you changed.\n\n"
|
|
63
|
+
f"task_id: {task_dir.name}\n\n"
|
|
64
|
+
f"Instruction:\n{instruction}"
|
|
65
|
+
)
|
|
66
|
+
return BenchmarkCase(
|
|
67
|
+
case_id=task_dir.name,
|
|
68
|
+
suite="terminal_bench",
|
|
69
|
+
category=str(task.get("category", "terminal_task")),
|
|
70
|
+
prompt=prompt,
|
|
71
|
+
expected=None,
|
|
72
|
+
files=files,
|
|
73
|
+
metadata={
|
|
74
|
+
"task_path": str(task_dir),
|
|
75
|
+
"difficulty": task.get("difficulty"),
|
|
76
|
+
"tags": task.get("tags", []),
|
|
77
|
+
"parser_name": task.get("parser_name"),
|
|
78
|
+
"max_agent_timeout_sec": task.get("max_agent_timeout_sec"),
|
|
79
|
+
"max_test_timeout_sec": task.get("max_test_timeout_sec"),
|
|
80
|
+
},
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _iter_visible_files(task_dir: Path) -> list[BenchmarkFile]:
|
|
85
|
+
files: list[BenchmarkFile] = []
|
|
86
|
+
excluded_names = {
|
|
87
|
+
"solution.sh",
|
|
88
|
+
"solution.yaml",
|
|
89
|
+
"task.yaml",
|
|
90
|
+
"run-tests.sh",
|
|
91
|
+
"docker-compose.yaml",
|
|
92
|
+
"Dockerfile",
|
|
93
|
+
}
|
|
94
|
+
for path in sorted(task_dir.rglob("*")):
|
|
95
|
+
if not path.is_file():
|
|
96
|
+
continue
|
|
97
|
+
rel = path.relative_to(task_dir)
|
|
98
|
+
if rel.parts[0] == "tests" or path.name in excluded_names:
|
|
99
|
+
continue
|
|
100
|
+
try:
|
|
101
|
+
if path.stat().st_size > 1_000_000:
|
|
102
|
+
continue
|
|
103
|
+
content = path.read_text(encoding="utf-8")
|
|
104
|
+
except UnicodeDecodeError:
|
|
105
|
+
continue
|
|
106
|
+
files.append(BenchmarkFile(path=rel.as_posix(), content=content))
|
|
107
|
+
return files
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from mindagent.core import ActionRisk
|
|
6
|
+
from mindagent.tools.base import BaseTool, ToolContext, ToolDefinition
|
|
7
|
+
|
|
8
|
+
from .schema import BenchmarkCase
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FakeBenchmarkTool(BaseTool):
|
|
12
|
+
def __init__(self, name: str, parameters: dict[str, Any]):
|
|
13
|
+
self.definition = ToolDefinition(
|
|
14
|
+
name=name,
|
|
15
|
+
description=f"Benchmark fake tool: {name}",
|
|
16
|
+
parameters=parameters,
|
|
17
|
+
risk=ActionRisk.READ_ONLY,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
async def execute(
|
|
21
|
+
self,
|
|
22
|
+
arguments: dict[str, Any],
|
|
23
|
+
context: ToolContext,
|
|
24
|
+
) -> dict[str, Any]:
|
|
25
|
+
return {"ok": True, "tool": self.definition.name, "arguments": arguments}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_fake_tools(case: BenchmarkCase) -> list[FakeBenchmarkTool]:
|
|
29
|
+
names = list(case.tools)
|
|
30
|
+
expected = case.expected if isinstance(case.expected, dict) else {}
|
|
31
|
+
for call in expected.get("tool_calls", []):
|
|
32
|
+
name = call.get("name")
|
|
33
|
+
if name and name not in names:
|
|
34
|
+
names.append(name)
|
|
35
|
+
|
|
36
|
+
expected_args_by_name = {
|
|
37
|
+
call.get("name"): call.get("arguments", {})
|
|
38
|
+
for call in expected.get("tool_calls", [])
|
|
39
|
+
if isinstance(call, dict)
|
|
40
|
+
}
|
|
41
|
+
return [
|
|
42
|
+
FakeBenchmarkTool(
|
|
43
|
+
name,
|
|
44
|
+
_parameters_from_arguments(expected_args_by_name.get(name, {})),
|
|
45
|
+
)
|
|
46
|
+
for name in names
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _parameters_from_arguments(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
51
|
+
properties = {
|
|
52
|
+
key: _schema_for_value(value) for key, value in arguments.items()
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
"type": "object",
|
|
56
|
+
"properties": properties,
|
|
57
|
+
"required": list(properties),
|
|
58
|
+
"additionalProperties": False,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _schema_for_value(value: Any) -> dict[str, Any]:
|
|
63
|
+
if isinstance(value, bool):
|
|
64
|
+
return {"type": "boolean"}
|
|
65
|
+
if isinstance(value, int):
|
|
66
|
+
return {"type": "integer"}
|
|
67
|
+
if isinstance(value, float):
|
|
68
|
+
return {"type": "number"}
|
|
69
|
+
if isinstance(value, list):
|
|
70
|
+
return {"type": "array"}
|
|
71
|
+
if isinstance(value, dict):
|
|
72
|
+
return {"type": "object"}
|
|
73
|
+
return {"type": "string"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict, is_dataclass
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def to_jsonable(value: Any) -> Any:
|
|
10
|
+
if isinstance(value, Enum):
|
|
11
|
+
return value.value
|
|
12
|
+
if isinstance(value, Path):
|
|
13
|
+
return str(value)
|
|
14
|
+
if is_dataclass(value):
|
|
15
|
+
return {key: to_jsonable(item) for key, item in asdict(value).items()}
|
|
16
|
+
if isinstance(value, dict):
|
|
17
|
+
return {str(key): to_jsonable(item) for key, item in value.items()}
|
|
18
|
+
if isinstance(value, (list, tuple)):
|
|
19
|
+
return [to_jsonable(item) for item in value]
|
|
20
|
+
return value
|
mindcode/bench/paths.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
|
7
|
+
APP_ROOT = PACKAGE_ROOT.parent
|
|
8
|
+
DEFAULT_BENCHMARK_ROOT = APP_ROOT / "benchmarks"
|
|
9
|
+
DEFAULT_RUNS_ROOT = APP_ROOT / "bench_runs"
|
|
10
|
+
DEFAULT_TERMINAL_BENCH_ROOT = APP_ROOT.parent / "lib" / "terminal-bench"
|