dsh-evokit 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dsh-evokit might be problematic. Click here for more details.
- dsh_evokit-0.1.5.dist-info/METADATA +78 -0
- dsh_evokit-0.1.5.dist-info/RECORD +28 -0
- dsh_evokit-0.1.5.dist-info/WHEEL +4 -0
- dsh_evokit-0.1.5.dist-info/entry_points.txt +2 -0
- dsh_evokit-0.1.5.dist-info/licenses/LICENSE +21 -0
- evokit/__init__.py +18 -0
- evokit/__main__.py +3 -0
- evokit/analysis.py +237 -0
- evokit/anchors.py +106 -0
- evokit/cli.py +104 -0
- evokit/config.py +145 -0
- evokit/coordinator.py +497 -0
- evokit/data/evolve.md +101 -0
- evokit/data/meta.md +194 -0
- evokit/data/onboard.md +56 -0
- evokit/guards.py +42 -0
- evokit/hb_records.py +390 -0
- evokit/mcp.py +77 -0
- evokit/onboarding.py +381 -0
- evokit/portability.py +173 -0
- evokit/privacy.py +19 -0
- evokit/proposal.schema.json +18 -0
- evokit/records.py +270 -0
- evokit/scenario.py +340 -0
- evokit/secondary.py +139 -0
- evokit/storage.py +168 -0
- evokit/tools.py +319 -0
- evokit/workspace.py +1532 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dsh-evokit
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Domain-neutral, pluggable self-evolution experiments
|
|
5
|
+
Project-URL: Homepage, https://github.com/edsml-jz2321/dsh-evokit
|
|
6
|
+
Project-URL: Repository, https://github.com/edsml-jz2321/dsh-evokit
|
|
7
|
+
Project-URL: Documentation, https://github.com/edsml-jz2321/dsh-evokit#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/edsml-jz2321/dsh-evokit/issues
|
|
9
|
+
Author: Jiahao
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: jsonschema<5,>=4.23
|
|
20
|
+
Requires-Dist: mcp<3,>=1.20
|
|
21
|
+
Requires-Dist: pyyaml<7,>=6.0.2
|
|
22
|
+
Provides-Extra: hb
|
|
23
|
+
Requires-Dist: fastmcp==4.0.0b1; (python_version < '3.14') and extra == 'hb'
|
|
24
|
+
Requires-Dist: heavenbase==0.1.2.2; (python_version < '3.14') and extra == 'hb'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
description: Measured self-improvement with paired experiments and a DSH plugin.
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
# evokit
|
|
32
|
+
|
|
33
|
+
evokit gives AI systems a measured self-improvement loop. It versions editable artifacts, records paired experiments, and gates changes against measured noise, cost and latency. Use it as a DeepSeek Harness (DSH) plugin or through its Python SDK, CLI and MCP server.
|
|
34
|
+
|
|
35
|
+
Your system supplies three pieces: **editable file globs**, **stable tasks**, and **a runner that executes and scores each task**. Prompts, skills, code and configuration can all be editable; the task source and scorer stay frozen.
|
|
36
|
+
|
|
37
|
+
The loop is **diagnose → propose → paired experiment → noise-aware gate → holdout confirmation with automatic revert in the operator workflow**. The improver sees diagnosis and selection evidence. A separate operator adopts a candidate, checks sealed confirmation aggregates, and calls `ledger_revert` when confirmation fails; `confirm_adoption` alone does not move the pointer. An optional final split opens once, with human approval, and never selects changes.
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
Requires Python 3.11+, uv, DSH `0.2.2-rc.1`, and its pnpm prerequisite. Windows is the verified local platform; Linux/macOS runtime integration remains unverified. Download the wheel, bundle and `SHA256SUMS` from [Releases](https://github.com/edsml-jz2321/dsh-evokit/releases) after publication and compare their SHA-256 hashes before installing.
|
|
42
|
+
|
|
43
|
+
```console
|
|
44
|
+
uv tool install ./dsh_evokit-0.1.5-py3-none-any.whl
|
|
45
|
+
dsh plugin --profile evokit add ./dsh-evokit-0.1.5.tgz
|
|
46
|
+
dsh plugin --profile evokit list
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Set `EVOKIT_MCP_COMMAND` to the absolute installed `evokit` executable, `EVOKIT_WORKSPACE` to your scenario directory, and `EVOKIT_SKILL_ROOT` to the installed bundle's `skills` directory. See the [DSH guide](integrations/dsh/README.md) for an isolated profile and exact PowerShell commands.
|
|
50
|
+
|
|
51
|
+
Configure the improver model in your DSH profile using **your own provider key** and your provider's supported model name. Keep the key in DSH's local credential settings or a process environment variable, never in this repository. The optional [relay](integrations/relay/README.md) supports file-based keys and accounting; adapt its example endpoint, model identity and illustrative prices before use. No key is included or required for the offline example. Installing or listing the plugin makes no model requests; starting evolution may incur provider charges.
|
|
52
|
+
|
|
53
|
+
## Minimal example
|
|
54
|
+
|
|
55
|
+
From a source checkout, the Node.js command example needs no model or key:
|
|
56
|
+
|
|
57
|
+
```console
|
|
58
|
+
uv sync --frozen --group dev
|
|
59
|
+
uv run python examples/scripted_improver.py command-echo
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Its `scenario.yaml` supplies the three pieces:
|
|
63
|
+
|
|
64
|
+
```yaml
|
|
65
|
+
editable: [prefix.txt]
|
|
66
|
+
tasks: tasks.json
|
|
67
|
+
command: [node, runner.mjs]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The runner reads `EVOKIT_TASK_FILE`, executes against `EVOKIT_VERSION_DIR`, and writes an outcome and cost to `EVOKIT_RESULT_FILE`. The scripted example works in a temporary copy. To connect your own system, follow [onboarding](docs/onboarding.md), then use the bundled `onboard`, `evolve` and `meta` skills.
|
|
71
|
+
|
|
72
|
+
## Safety and limits
|
|
73
|
+
|
|
74
|
+
Set budgets, anchors, noise policy and a stopping rule before experiments. Repeated mean gates use threshold/√R; majority gates retain the single-repeat threshold. Confirmation has a limited wear budget and is not an untouched final benchmark. One public BFCL pilot improved held-out accuracy from **41 to 47 of 50 after confirmation**; this small example does not establish general gains.
|
|
75
|
+
|
|
76
|
+
Scenarios run trusted executable code. Per-task directories are not a security sandbox. Protect sealed tasks and records with OS permissions; MCP filtering cannot hide files an improver can read directly. Review proposed changes for task literals and leakage. The relay accounts reported usage conservatively; it cannot guarantee provider billing or prevent every external cost.
|
|
77
|
+
|
|
78
|
+
See [concepts](docs/concepts.md), [tools](docs/tools.md), [storage](docs/storage.md) and [contributing](CONTRIBUTING.md). This project uses the [MIT license](LICENSE), Copyright (c) 2026 Jiahao.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
evokit/__init__.py,sha256=NgRyVRP2O8U5LtTyFPRRM2HCV29Q6FPp57Kvb3oF48c,482
|
|
2
|
+
evokit/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
|
|
3
|
+
evokit/analysis.py,sha256=_uIjEQWFsqjRD9A17L3W2c6QnTG30m40T-JwG8E10gY,10333
|
|
4
|
+
evokit/anchors.py,sha256=XK2kuCTxvY5RilsgYYyeilNzMGoUt-wCIoUQYy1UDtc,4098
|
|
5
|
+
evokit/cli.py,sha256=6qkie1YR0dozi4THx1vZx4WgG6U5CLbBd8PlYPTZ80o,4278
|
|
6
|
+
evokit/config.py,sha256=3veZ2HEXxpVtH2nbmnce-2QN9NmFp3oVM_LFZsoC6r8,5798
|
|
7
|
+
evokit/coordinator.py,sha256=eazZOirNahiMEPFMyUTIy5EGY91NlmSFtsuK8oS9XkI,21166
|
|
8
|
+
evokit/guards.py,sha256=ObyyrurLDa5Ut0lsprdFgBrjCQreLg9k2n0h4TUJ28s,1668
|
|
9
|
+
evokit/hb_records.py,sha256=69-lLvgKcdxp8PNbmd2_fd6up3WgOHeXlmSKBskNrHs,16469
|
|
10
|
+
evokit/mcp.py,sha256=9wlfnqk3yMQN82Xbe7CxKVYZsRdri8vruYSHMue2ueU,2874
|
|
11
|
+
evokit/onboarding.py,sha256=pYX3fxsL7_mFyfnjduGwZHQwe75NCXkV09NI8wBjUUY,16198
|
|
12
|
+
evokit/portability.py,sha256=qQ3iHPc8Y6QB3LUhSpoUvU7V1ZC7ys2Hzgqk4JZzM0o,7192
|
|
13
|
+
evokit/privacy.py,sha256=gkmxdHjorOakYhXjs_znTrYI3PCGS3NxAm35FkzBbac,585
|
|
14
|
+
evokit/proposal.schema.json,sha256=L2_WeYjzR8ebqtGgYFKvHMHp9VQKoZaayQNp8E3lTDw,823
|
|
15
|
+
evokit/records.py,sha256=FZN13ULdIeFeM2bMPvba2ut1XeLD97McKJareUbgJgY,11830
|
|
16
|
+
evokit/scenario.py,sha256=MpJgUQ-FtfQs3R2LVMe68eDJyT5B8_TOe78czrrXHmI,12646
|
|
17
|
+
evokit/secondary.py,sha256=bZocVOKDaAk-ILs54Tcj-S0ACMDK_KG29mo7muZuwWs,5966
|
|
18
|
+
evokit/storage.py,sha256=5tiH4zuH6bYO2Nb6wyCLzjLM-BWHU4PDetndb-F3Ejs,5632
|
|
19
|
+
evokit/tools.py,sha256=wnL7qrz1x1GQ2zes5Dl4UXRPL8ZIvOo7ADNd0n6PUf0,10671
|
|
20
|
+
evokit/workspace.py,sha256=bqaAgpPU-ZkZ8EqxspgrZVZGmhPGdcI2UwYEtTq7OaA,67194
|
|
21
|
+
evokit/data/evolve.md,sha256=NeXZWUgBuGp9olE30r6fOZvXCYMoOY_p0lD9BNu2OoY,6823
|
|
22
|
+
evokit/data/meta.md,sha256=mharUObf3U8AXrrstXDP-XivfCYQlh1fWc2fiL3v_SQ,11843
|
|
23
|
+
evokit/data/onboard.md,sha256=HQ6bKqE-Es3FCt1LPa7XdUVLpF-ey-gh01LYBE9mvXo,3338
|
|
24
|
+
dsh_evokit-0.1.5.dist-info/METADATA,sha256=p6I4WUKuPYcEJIi-DZ09rxSuLpvokfRNgoB8tTrljNI,5166
|
|
25
|
+
dsh_evokit-0.1.5.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
26
|
+
dsh_evokit-0.1.5.dist-info/entry_points.txt,sha256=-Xokrp6pCNdrVJd2essb-XSTJbb8TVsoleEHhVrEbG4,43
|
|
27
|
+
dsh_evokit-0.1.5.dist-info/licenses/LICENSE,sha256=t3ZR0dfLuKIgBE65zhORaH_EbgOQWbriyK6WCwmcCeY,1063
|
|
28
|
+
dsh_evokit-0.1.5.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jiahao
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
evokit/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""evokit public Python facade: three-piece adapters and the workspace lifecycle."""
|
|
2
|
+
|
|
3
|
+
from .records import RecordStore
|
|
4
|
+
from .scenario import CommandScenario, Result, Scenario, StageCache, Task
|
|
5
|
+
from .secondary import paired_defined_guard, paired_quality_noise
|
|
6
|
+
from .workspace import Workspace
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"RecordStore",
|
|
10
|
+
"CommandScenario",
|
|
11
|
+
"Result",
|
|
12
|
+
"Scenario",
|
|
13
|
+
"StageCache",
|
|
14
|
+
"Task",
|
|
15
|
+
"Workspace",
|
|
16
|
+
"paired_defined_guard",
|
|
17
|
+
"paired_quality_noise",
|
|
18
|
+
]
|
evokit/__main__.py
ADDED
evokit/analysis.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Deterministic splitting and paired analysis, independent of transports and storage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import random
|
|
7
|
+
import statistics
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .scenario import Task
|
|
12
|
+
from .storage import digest
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def split_tasks(
|
|
16
|
+
tasks: list[Task],
|
|
17
|
+
seed: int = 0,
|
|
18
|
+
*,
|
|
19
|
+
stratified: bool = True,
|
|
20
|
+
fractions: tuple[float, ...] = (0.4, 0.4, 0.2, 0.0),
|
|
21
|
+
legacy: bool = False,
|
|
22
|
+
assignments: dict[str, list[str]] | None = None,
|
|
23
|
+
) -> dict[str, list[str]]:
|
|
24
|
+
"""Assign stable ids to seeded or exact evolution partitions.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
tasks (list[Task]): Unique tasks; input order does not affect assignment.
|
|
28
|
+
seed (int): Epoch seed.
|
|
29
|
+
stratified (bool): Preserve proportions within each optional group.
|
|
30
|
+
fractions (tuple[float, ...]): Diagnosis, selection, confirmation and final
|
|
31
|
+
shares. Three values retain the legacy zero-final interpretation.
|
|
32
|
+
legacy (bool): Use the old holdout name when checking retained epochs.
|
|
33
|
+
assignments (dict[str, list[str]] | None): Optional exact disjoint partition
|
|
34
|
+
with diagnosis, selection, confirmation and final keys. Overrides
|
|
35
|
+
seeded allocation; incompatible with legacy holdout naming.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
dict[str, list[str]]: Disjoint sorted ids. Tiny strata may yield empty splits.
|
|
39
|
+
|
|
40
|
+
Raises:
|
|
41
|
+
ValueError: For duplicate ids, invalid proportions or non-partition assignments.
|
|
42
|
+
"""
|
|
43
|
+
if len({task.id for task in tasks}) != len(tasks):
|
|
44
|
+
raise ValueError("Task ids must be unique")
|
|
45
|
+
if assignments is not None:
|
|
46
|
+
names = {"diagnosis", "selection", "confirmation", "final"}
|
|
47
|
+
if not isinstance(assignments, dict) or set(assignments) != names or legacy:
|
|
48
|
+
raise ValueError("Explicit splits require diagnosis, selection, confirmation and final")
|
|
49
|
+
if any(
|
|
50
|
+
not isinstance(ids, list) or any(not isinstance(i, str) for i in ids)
|
|
51
|
+
for ids in assignments.values()
|
|
52
|
+
):
|
|
53
|
+
raise ValueError("Explicit splits must contain lists of task ids")
|
|
54
|
+
flattened = [i for ids in assignments.values() for i in ids]
|
|
55
|
+
if (
|
|
56
|
+
len(flattened) != len(set(flattened))
|
|
57
|
+
or set(flattened) != {t.id for t in tasks}
|
|
58
|
+
or any(not assignments[k] for k in names - {"final"})
|
|
59
|
+
):
|
|
60
|
+
raise ValueError("Explicit splits must partition all tasks without overlap")
|
|
61
|
+
return {
|
|
62
|
+
k: sorted(assignments[k]) for k in ("diagnosis", "selection", "confirmation", "final")
|
|
63
|
+
}
|
|
64
|
+
if len(fractions) == 3:
|
|
65
|
+
fractions = (*fractions, 0.0)
|
|
66
|
+
if (
|
|
67
|
+
len(fractions) != 4
|
|
68
|
+
or any(not math.isfinite(n) or n <= 0 for n in fractions[:3])
|
|
69
|
+
or not math.isfinite(fractions[3])
|
|
70
|
+
or fractions[3] < 0
|
|
71
|
+
):
|
|
72
|
+
raise ValueError("Three positive fractions and an optional nonnegative final are required")
|
|
73
|
+
if not math.isclose(sum(fractions), 1):
|
|
74
|
+
raise ValueError("Split fractions must sum to 1")
|
|
75
|
+
groups: dict[str, list[str]] = defaultdict(list)
|
|
76
|
+
for task in tasks:
|
|
77
|
+
groups[task.group if stratified else "all"].append(task.id)
|
|
78
|
+
names = ("diagnosis", "selection", "holdout" if legacy else "confirmation", "final")
|
|
79
|
+
splits: dict[str, list[str]] = {name: [] for name in names}
|
|
80
|
+
for group, ids in sorted(groups.items()):
|
|
81
|
+
ids.sort()
|
|
82
|
+
random.Random(digest([seed, group])).shuffle(ids)
|
|
83
|
+
# Largest remainder rounding avoids systematically starving holdout in small strata.
|
|
84
|
+
raw = [len(ids) * fraction for fraction in fractions]
|
|
85
|
+
sizes = [int(value) for value in raw]
|
|
86
|
+
for index in sorted(range(4), key=lambda i: (raw[i] - sizes[i], -i), reverse=True)[
|
|
87
|
+
: len(ids) - sum(sizes)
|
|
88
|
+
]:
|
|
89
|
+
sizes[index] += 1
|
|
90
|
+
offset = 0
|
|
91
|
+
for name, size in zip(splits, sizes, strict=True):
|
|
92
|
+
splits[name].extend(ids[offset : offset + size])
|
|
93
|
+
offset += size
|
|
94
|
+
return {name: sorted(ids) for name, ids in splits.items() if not (legacy and name == "final")}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def aggregate(rows: list[dict[str, Any]], method: str) -> dict[str, dict[str, Any]]:
|
|
98
|
+
if method not in {"mean", "majority"}:
|
|
99
|
+
raise ValueError("aggregation must be explicit: mean or majority")
|
|
100
|
+
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
101
|
+
for row in rows:
|
|
102
|
+
grouped[row["task_id"]].append(row)
|
|
103
|
+
result = {}
|
|
104
|
+
for task_id, items in grouped.items():
|
|
105
|
+
values = [float(item["outcome"]) for item in items]
|
|
106
|
+
if method == "majority" and any(value not in (0, 1) for value in values):
|
|
107
|
+
raise ValueError("Majority requires binary outcomes; use mean for scores")
|
|
108
|
+
score = statistics.mean(values)
|
|
109
|
+
if method == "majority":
|
|
110
|
+
score = float(score > 0.5) # Ties fail, deliberately conservative.
|
|
111
|
+
costs = {
|
|
112
|
+
key: statistics.mean(item.get("cost", {}).get(key, 0) for item in items)
|
|
113
|
+
for key in {key for item in items for key in item.get("cost", {})}
|
|
114
|
+
}
|
|
115
|
+
result[task_id] = {
|
|
116
|
+
"score": score,
|
|
117
|
+
"group": items[0]["group"],
|
|
118
|
+
"cost": costs,
|
|
119
|
+
"seconds": statistics.mean(item["seconds"] for item in items),
|
|
120
|
+
"repeats": len(items),
|
|
121
|
+
}
|
|
122
|
+
return result
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _ratio(parent: float, child: float) -> float | None:
|
|
126
|
+
return child / parent if parent else (1.0 if not child else None)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def compare(
|
|
130
|
+
parent: list[dict[str, Any]], child: list[dict[str, Any]], aggregation: str
|
|
131
|
+
) -> dict[str, Any]:
|
|
132
|
+
"""Compare arms on exactly the same tasks and repeats.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
parent (list[dict[str, Any]]): Completed incumbent rows.
|
|
136
|
+
child (list[dict[str, Any]]): Completed candidate rows.
|
|
137
|
+
aggregation (str): `mean` for scores; `majority` for binary votes (ties fail).
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
dict[str, Any]: Fixes, breaks, net score sum, groups, costs and latency ratios.
|
|
141
|
+
|
|
142
|
+
Raises:
|
|
143
|
+
ValueError: If pairing, groups or repeat counts differ.
|
|
144
|
+
"""
|
|
145
|
+
left, right = aggregate(parent, aggregation), aggregate(child, aggregation)
|
|
146
|
+
if not left or left.keys() != right.keys():
|
|
147
|
+
raise ValueError("Paired comparison requires identical nonempty task sets")
|
|
148
|
+
if any(
|
|
149
|
+
left[key]["repeats"] != right[key]["repeats"] or left[key]["group"] != right[key]["group"]
|
|
150
|
+
for key in left
|
|
151
|
+
):
|
|
152
|
+
raise ValueError("Paired comparison requires identical groups and repeat counts")
|
|
153
|
+
deltas = {key: right[key]["score"] - left[key]["score"] for key in sorted(left)}
|
|
154
|
+
groups = {}
|
|
155
|
+
for group in sorted({item["group"] for item in left.values()}):
|
|
156
|
+
values = [delta for key, delta in deltas.items() if left[key]["group"] == group]
|
|
157
|
+
groups[group] = {
|
|
158
|
+
"tasks": len(values),
|
|
159
|
+
"fixes": sum(n > 0 for n in values),
|
|
160
|
+
"breaks": sum(n < 0 for n in values),
|
|
161
|
+
"net": sum(values),
|
|
162
|
+
}
|
|
163
|
+
units = {key for item in [*left.values(), *right.values()] for key in item["cost"]}
|
|
164
|
+
costs = {}
|
|
165
|
+
for unit in sorted(units):
|
|
166
|
+
before = sum(item["cost"].get(unit, 0) for item in left.values())
|
|
167
|
+
after = sum(item["cost"].get(unit, 0) for item in right.values())
|
|
168
|
+
costs[unit] = {"parent": before, "child": after, "ratio": _ratio(before, after)}
|
|
169
|
+
before = sum(item["seconds"] for item in left.values())
|
|
170
|
+
after = sum(item["seconds"] for item in right.values())
|
|
171
|
+
return {
|
|
172
|
+
"aggregation": aggregation,
|
|
173
|
+
"tasks": len(left),
|
|
174
|
+
"fixes": [key for key, value in deltas.items() if value > 0],
|
|
175
|
+
"breaks": [key for key, value in deltas.items() if value < 0],
|
|
176
|
+
"net": sum(deltas.values()),
|
|
177
|
+
"per_task": deltas,
|
|
178
|
+
"groups": groups,
|
|
179
|
+
"cost": costs,
|
|
180
|
+
"latency": {"parent": before, "child": after, "ratio": _ratio(before, after)},
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def estimate_noise(
|
|
185
|
+
rows: list[dict[str, Any]], *, multiplier: float = 1.96, observed_floor: bool = True
|
|
186
|
+
) -> dict[str, Any]:
|
|
187
|
+
"""Estimate paired net spread from repeated incumbent task outcomes.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
rows (list[dict[str, Any]]): At least two complete repeats per task.
|
|
191
|
+
multiplier (float): Standard-deviation multiplier fixed by the epoch.
|
|
192
|
+
observed_floor (bool): Retain the historical absolute-repeat-net floor.
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
dict[str, Any]: Pairwise flip rates, spread, observed nets and threshold method.
|
|
196
|
+
|
|
197
|
+
Raises:
|
|
198
|
+
ValueError: If repeats are incomplete or fewer than two.
|
|
199
|
+
"""
|
|
200
|
+
if (
|
|
201
|
+
type(multiplier) not in (float, int)
|
|
202
|
+
or not math.isfinite(multiplier)
|
|
203
|
+
or multiplier < 0
|
|
204
|
+
or type(observed_floor) is not bool
|
|
205
|
+
):
|
|
206
|
+
raise ValueError("Invalid noise policy")
|
|
207
|
+
tasks: dict[str, dict[int, float]] = defaultdict(dict)
|
|
208
|
+
for row in rows:
|
|
209
|
+
if row["repeat"] in tasks[row["task_id"]]:
|
|
210
|
+
raise ValueError("Duplicate noise observation")
|
|
211
|
+
tasks[row["task_id"]][row["repeat"]] = float(row["outcome"])
|
|
212
|
+
if not tasks:
|
|
213
|
+
raise ValueError("Noise requires observations")
|
|
214
|
+
repeats = sorted(next(iter(tasks.values())))
|
|
215
|
+
if len(repeats) < 2 or any(sorted(values) != repeats for values in tasks.values()):
|
|
216
|
+
raise ValueError("Noise requires at least two complete repeats")
|
|
217
|
+
pairs = [(a, b) for a in repeats for b in repeats if a < b]
|
|
218
|
+
flips = {
|
|
219
|
+
key: sum(values[a] != values[b] for a, b in pairs) / len(pairs)
|
|
220
|
+
for key, values in tasks.items()
|
|
221
|
+
}
|
|
222
|
+
nets = [sum(values[b] - values[a] for values in tasks.values()) for a, b in pairs]
|
|
223
|
+
spread = math.sqrt(sum(2 * statistics.variance(values.values()) for values in tasks.values()))
|
|
224
|
+
threshold = max(max(abs(net) for net in nets) if observed_floor else 0, multiplier * spread)
|
|
225
|
+
return {
|
|
226
|
+
"repeats": len(repeats),
|
|
227
|
+
"tasks": len(tasks),
|
|
228
|
+
"flip_rate_per_task": flips,
|
|
229
|
+
"mean_flip_rate": statistics.mean(flips.values()),
|
|
230
|
+
"net_change_spread": spread,
|
|
231
|
+
"observed_pair_nets": nets,
|
|
232
|
+
"threshold": threshold,
|
|
233
|
+
"multiplier": multiplier,
|
|
234
|
+
"observed_floor": observed_floor,
|
|
235
|
+
"method": "max(enabled observed absolute repeat net floor, multiplier * sqrt(sum(2 * sample variance))); "
|
|
236
|
+
"net score-sum units; conservative single-repeat reference, not a confidence guarantee",
|
|
237
|
+
}
|
evokit/anchors.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Stable semantic code fingerprints independent of interpreter serialization state."""
|
|
2
|
+
|
|
3
|
+
from pathlib import PurePosixPath, PureWindowsPath
|
|
4
|
+
from types import CodeType
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .storage import canonical, digest
|
|
8
|
+
|
|
9
|
+
# Explicit compatibility for the reviewed, released library module only. User
|
|
10
|
+
# sources and callable fingerprints still compare exactly; epochs are not edited.
|
|
11
|
+
V012_SCENARIO_SHA256 = "e32777d2bbfda29198816d2a2556769e15fed79da6372298b148fd1fe831f6a6"
|
|
12
|
+
V013_SCENARIO_SHA256 = "54a0a837bf5e6131cc1b5b79d8fb68dc05c7bf0cc83379206f5c2541d9005d1f"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
V015_SCENARIO_SHA256 = "329260510f85b5f42cdd1d8b54c7baf1e0c9c93e41f3f4ce7bbf1ccebad71e62"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def compatible_v012_sources(expected: dict[str, str]) -> dict[str, str]:
|
|
19
|
+
"""Translate the released default module anchor without changing epoch data.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
expected (dict[str, str]): Persisted source and callable anchors.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
dict[str, str]: Detached expected anchors; only the known v0.1.2 library
|
|
26
|
+
source digest can advance. v0.1.2 requires its original default callback.
|
|
27
|
+
"""
|
|
28
|
+
known = {V013_SCENARIO_SHA256}
|
|
29
|
+
if "callable:secondary_guards" not in expected:
|
|
30
|
+
known.add(V012_SCENARIO_SHA256)
|
|
31
|
+
return {
|
|
32
|
+
k: V015_SCENARIO_SHA256
|
|
33
|
+
if v in known
|
|
34
|
+
and not k.startswith(("source:", "callable:"))
|
|
35
|
+
and (PureWindowsPath(k).name if "\\" in k else PurePosixPath(k).name) == "scenario.py"
|
|
36
|
+
else v
|
|
37
|
+
for k, v in expected.items()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def code_fingerprint(code: CodeType) -> str:
|
|
42
|
+
def constant(value: Any) -> Any:
|
|
43
|
+
if isinstance(value, CodeType):
|
|
44
|
+
return {"code": code_fingerprint(value)}
|
|
45
|
+
if isinstance(value, bytes):
|
|
46
|
+
return {"bytes": value.hex()}
|
|
47
|
+
if isinstance(value, complex):
|
|
48
|
+
return {"complex": [value.real.hex(), value.imag.hex()]}
|
|
49
|
+
if isinstance(value, float):
|
|
50
|
+
return {"float": value.hex()}
|
|
51
|
+
if isinstance(value, tuple):
|
|
52
|
+
return {"tuple": [constant(item) for item in value]}
|
|
53
|
+
if isinstance(value, frozenset):
|
|
54
|
+
return {"frozenset": sorted((constant(item) for item in value), key=canonical)}
|
|
55
|
+
if value is Ellipsis:
|
|
56
|
+
return {"ellipsis": True}
|
|
57
|
+
return value
|
|
58
|
+
|
|
59
|
+
return digest(
|
|
60
|
+
{
|
|
61
|
+
"bytecode": code.co_code.hex(),
|
|
62
|
+
"constants": [constant(v) for v in code.co_consts],
|
|
63
|
+
"names": code.co_names,
|
|
64
|
+
"locals": code.co_varnames,
|
|
65
|
+
"free": code.co_freevars,
|
|
66
|
+
"cells": code.co_cellvars,
|
|
67
|
+
"args": code.co_argcount,
|
|
68
|
+
"positional_only": code.co_posonlyargcount,
|
|
69
|
+
"keyword_only": code.co_kwonlyargcount,
|
|
70
|
+
"flags": code.co_flags,
|
|
71
|
+
"exceptions": code.co_exceptiontable.hex(),
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def scorer_sources_match(
|
|
77
|
+
expected: dict[str, str], actual: dict[str, str], *, owned_source: str | None = None
|
|
78
|
+
) -> bool:
|
|
79
|
+
"""Accept relocation of evokit's own scenario module with identical bytes.
|
|
80
|
+
|
|
81
|
+
All user adapter paths, scenario-relative files and callable fingerprints
|
|
82
|
+
still match exactly. Only the explicitly named installed evokit module may
|
|
83
|
+
match a legacy absolute location. Arbitrary adapter relocation is unsafe:
|
|
84
|
+
byte-identical code can read different support relative to __file__.
|
|
85
|
+
"""
|
|
86
|
+
if expected == actual:
|
|
87
|
+
return True
|
|
88
|
+
if owned_source is None or owned_source not in actual or owned_source in expected:
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
def basename(path: str) -> str:
|
|
92
|
+
return PureWindowsPath(path).name if "\\" in path else PurePosixPath(path).name
|
|
93
|
+
|
|
94
|
+
candidates = [
|
|
95
|
+
key
|
|
96
|
+
for key, checksum in expected.items()
|
|
97
|
+
if key not in actual
|
|
98
|
+
and not key.startswith(("source:", "callable:"))
|
|
99
|
+
and basename(key) == basename(owned_source)
|
|
100
|
+
and checksum == actual[owned_source]
|
|
101
|
+
]
|
|
102
|
+
if len(candidates) != 1:
|
|
103
|
+
return False
|
|
104
|
+
relocated = {key: value for key, value in actual.items() if key != owned_source}
|
|
105
|
+
relocated[candidates[0]] = actual[owned_source]
|
|
106
|
+
return expected == relocated
|
evokit/cli.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""CLI parity with the stable MCP tools, plus an internal detached run entrypoint."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import asyncio
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .tools import available_tools, call_tool
|
|
12
|
+
from .workspace import Workspace
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main() -> None:
|
|
16
|
+
"""Invoke a tool with JSON arguments or start the stdio MCP server.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
None: Reads command-line arguments from sys.argv.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
None: Writes JSON; domain/input errors exit 2 with a JSON error on stderr.
|
|
23
|
+
"""
|
|
24
|
+
if len(sys.argv) > 1 and sys.argv[1] in {"init", "doctor"}:
|
|
25
|
+
from .onboarding import run_cli
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
result = run_cli(sys.argv[1:])
|
|
29
|
+
print(json.dumps(result, indent=2, allow_nan=False))
|
|
30
|
+
if any(c["status"] == "fail" for c in result.get("checks", {}).values()):
|
|
31
|
+
raise SystemExit(2)
|
|
32
|
+
except Exception as exc:
|
|
33
|
+
print(json.dumps({"error": str(exc), "type": type(exc).__name__}), file=sys.stderr)
|
|
34
|
+
raise SystemExit(2) from exc
|
|
35
|
+
return
|
|
36
|
+
parser = argparse.ArgumentParser(prog="evokit")
|
|
37
|
+
parser.add_argument("command", help="Tool name, tools, mcp, export or import")
|
|
38
|
+
parser.add_argument("internal", nargs="*")
|
|
39
|
+
parser.add_argument("--config")
|
|
40
|
+
parser.add_argument("--profile", choices=["improver", "operator"], default="improver")
|
|
41
|
+
parser.add_argument("--state", type=Path, help="Operator import/export state path")
|
|
42
|
+
parser.add_argument("--backend", choices=["files", "hb"], help="Operator import target backend")
|
|
43
|
+
parser.add_argument("--output", type=Path, help="New operator JSONL export")
|
|
44
|
+
parser.add_argument("--input", type=Path, help="Operator JSONL import")
|
|
45
|
+
inputs = parser.add_mutually_exclusive_group()
|
|
46
|
+
inputs.add_argument("--args", default="{}", help="Tool arguments as a JSON object")
|
|
47
|
+
inputs.add_argument(
|
|
48
|
+
"--args-file", type=Path, help="UTF-8 JSON arguments, avoiding shell quoting"
|
|
49
|
+
)
|
|
50
|
+
options = parser.parse_args()
|
|
51
|
+
try:
|
|
52
|
+
if options.command == "mcp":
|
|
53
|
+
from .mcp import serve
|
|
54
|
+
|
|
55
|
+
asyncio.run(serve(options.config, profile=options.profile))
|
|
56
|
+
return
|
|
57
|
+
if options.command == "_run":
|
|
58
|
+
config, run_id = options.internal
|
|
59
|
+
Workspace.open(config).execute(run_id)
|
|
60
|
+
return
|
|
61
|
+
if options.command == "tools":
|
|
62
|
+
result = [
|
|
63
|
+
{
|
|
64
|
+
"name": tool.name,
|
|
65
|
+
"group": tool.group,
|
|
66
|
+
"description": tool.description,
|
|
67
|
+
"inputSchema": tool.schema,
|
|
68
|
+
}
|
|
69
|
+
for tool in available_tools(options.config, profile=options.profile)
|
|
70
|
+
]
|
|
71
|
+
elif options.command in {"export", "import"}:
|
|
72
|
+
from .portability import export_records, import_records
|
|
73
|
+
|
|
74
|
+
if not options.config:
|
|
75
|
+
raise ValueError("Import/export requires --config")
|
|
76
|
+
importing = options.command == "import"
|
|
77
|
+
target = options.input if importing else options.output
|
|
78
|
+
if target is None:
|
|
79
|
+
raise ValueError("Use --input for import or --output for export")
|
|
80
|
+
workspace = Workspace.open(
|
|
81
|
+
options.config,
|
|
82
|
+
initialize=False,
|
|
83
|
+
state=options.state,
|
|
84
|
+
storage={"backend": options.backend} if options.backend else None,
|
|
85
|
+
)
|
|
86
|
+
count = (
|
|
87
|
+
import_records(workspace.store, target)
|
|
88
|
+
if importing
|
|
89
|
+
else export_records(workspace.store, target)
|
|
90
|
+
)
|
|
91
|
+
result = {"records": count, "backend": workspace.store.backend}
|
|
92
|
+
else:
|
|
93
|
+
arguments = json.loads(
|
|
94
|
+
options.args_file.read_text(encoding="utf-8-sig")
|
|
95
|
+
if options.args_file
|
|
96
|
+
else options.args
|
|
97
|
+
)
|
|
98
|
+
if options.config:
|
|
99
|
+
arguments["config"] = options.config
|
|
100
|
+
result = call_tool(options.command, arguments, profile=options.profile)
|
|
101
|
+
print(json.dumps(result, indent=2, allow_nan=False))
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
print(json.dumps({"error": str(exc), "type": type(exc).__name__}), file=sys.stderr)
|
|
104
|
+
raise SystemExit(2) from exc
|