dsh-evokit 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dsh-evokit might be problematic. Click here for more details.
- dsh_evokit-0.1.5/.gitignore +14 -0
- dsh_evokit-0.1.5/CHANGELOG.md +21 -0
- dsh_evokit-0.1.5/CONTRIBUTING.md +25 -0
- dsh_evokit-0.1.5/LICENSE +21 -0
- dsh_evokit-0.1.5/PKG-INFO +78 -0
- dsh_evokit-0.1.5/README.md +52 -0
- dsh_evokit-0.1.5/README.zh.md +52 -0
- dsh_evokit-0.1.5/docs/caching.md +22 -0
- dsh_evokit-0.1.5/docs/concepts.md +155 -0
- dsh_evokit-0.1.5/docs/onboarding.md +252 -0
- dsh_evokit-0.1.5/docs/storage.md +182 -0
- dsh_evokit-0.1.5/docs/tools.md +332 -0
- dsh_evokit-0.1.5/examples/command-echo/prefix.txt +1 -0
- dsh_evokit-0.1.5/examples/command-echo/runner.mjs +16 -0
- dsh_evokit-0.1.5/examples/command-echo/scenario.yaml +10 -0
- dsh_evokit-0.1.5/examples/command-echo/tasks.json +14 -0
- dsh_evokit-0.1.5/examples/scripted_improver.py +78 -0
- dsh_evokit-0.1.5/examples/toy-rules/adapter.py +59 -0
- dsh_evokit-0.1.5/examples/toy-rules/rules.json +1 -0
- dsh_evokit-0.1.5/examples/toy-rules/scenario.yaml +5 -0
- dsh_evokit-0.1.5/examples/toy-rules/stage1.json +1 -0
- dsh_evokit-0.1.5/integrations/dsh/LICENSE +21 -0
- dsh_evokit-0.1.5/integrations/dsh/README.md +68 -0
- dsh_evokit-0.1.5/integrations/dsh/check_package.py +43 -0
- dsh_evokit-0.1.5/integrations/dsh/cordis.patch.yml +21 -0
- dsh_evokit-0.1.5/integrations/dsh/improver.patch.yml +26 -0
- dsh_evokit-0.1.5/integrations/dsh/list-tools.mjs +27 -0
- dsh_evokit-0.1.5/integrations/dsh/list-tools.patch.yml +35 -0
- dsh_evokit-0.1.5/integrations/dsh/mcp.json +10 -0
- dsh_evokit-0.1.5/integrations/dsh/package.json +35 -0
- dsh_evokit-0.1.5/integrations/dsh/plugin.json +11 -0
- dsh_evokit-0.1.5/integrations/dsh/skills/evolve/SKILL.md +101 -0
- dsh_evokit-0.1.5/integrations/dsh/skills/meta/SKILL.md +194 -0
- dsh_evokit-0.1.5/integrations/dsh/skills/onboard/SKILL.md +56 -0
- dsh_evokit-0.1.5/integrations/relay/README.md +143 -0
- dsh_evokit-0.1.5/integrations/relay/improver.example.json +19 -0
- dsh_evokit-0.1.5/integrations/relay/relay.py +764 -0
- dsh_evokit-0.1.5/market/README.md +17 -0
- dsh_evokit-0.1.5/market/icon.svg +1 -0
- dsh_evokit-0.1.5/market/index.json +25 -0
- dsh_evokit-0.1.5/mcp.json +10 -0
- dsh_evokit-0.1.5/plugin.json +11 -0
- dsh_evokit-0.1.5/pyproject.toml +60 -0
- dsh_evokit-0.1.5/scripts/build_release.ps1 +14 -0
- dsh_evokit-0.1.5/scripts/check_package.py +50 -0
- dsh_evokit-0.1.5/scripts/fill_market.py +31 -0
- dsh_evokit-0.1.5/skills/evolve/SKILL.md +101 -0
- dsh_evokit-0.1.5/skills/meta/SKILL.md +194 -0
- dsh_evokit-0.1.5/skills/onboard/SKILL.md +56 -0
- dsh_evokit-0.1.5/src/evokit/__init__.py +18 -0
- dsh_evokit-0.1.5/src/evokit/__main__.py +3 -0
- dsh_evokit-0.1.5/src/evokit/analysis.py +237 -0
- dsh_evokit-0.1.5/src/evokit/anchors.py +106 -0
- dsh_evokit-0.1.5/src/evokit/cli.py +104 -0
- dsh_evokit-0.1.5/src/evokit/config.py +145 -0
- dsh_evokit-0.1.5/src/evokit/coordinator.py +497 -0
- dsh_evokit-0.1.5/src/evokit/guards.py +42 -0
- dsh_evokit-0.1.5/src/evokit/hb_records.py +390 -0
- dsh_evokit-0.1.5/src/evokit/mcp.py +77 -0
- dsh_evokit-0.1.5/src/evokit/onboarding.py +381 -0
- dsh_evokit-0.1.5/src/evokit/portability.py +173 -0
- dsh_evokit-0.1.5/src/evokit/privacy.py +19 -0
- dsh_evokit-0.1.5/src/evokit/proposal.schema.json +18 -0
- dsh_evokit-0.1.5/src/evokit/records.py +270 -0
- dsh_evokit-0.1.5/src/evokit/scenario.py +340 -0
- dsh_evokit-0.1.5/src/evokit/secondary.py +139 -0
- dsh_evokit-0.1.5/src/evokit/storage.py +168 -0
- dsh_evokit-0.1.5/src/evokit/tools.py +319 -0
- dsh_evokit-0.1.5/src/evokit/workspace.py +1532 -0
- dsh_evokit-0.1.5/tests/conftest.py +128 -0
- dsh_evokit-0.1.5/tests/test_accounting_relay.py +585 -0
- dsh_evokit-0.1.5/tests/test_analysis.py +62 -0
- dsh_evokit-0.1.5/tests/test_bundle.py +19 -0
- dsh_evokit-0.1.5/tests/test_config_validation.py +63 -0
- dsh_evokit-0.1.5/tests/test_dsh_inspector.py +39 -0
- dsh_evokit-0.1.5/tests/test_examples_mcp.py +159 -0
- dsh_evokit-0.1.5/tests/test_explicit_splits.py +56 -0
- dsh_evokit-0.1.5/tests/test_interleaved.py +53 -0
- dsh_evokit-0.1.5/tests/test_legacy_storage.py +71 -0
- dsh_evokit-0.1.5/tests/test_lifecycle.py +226 -0
- dsh_evokit-0.1.5/tests/test_meta.py +448 -0
- dsh_evokit-0.1.5/tests/test_onboarding.py +140 -0
- dsh_evokit-0.1.5/tests/test_paired_quality.py +129 -0
- dsh_evokit-0.1.5/tests/test_record_stores.py +483 -0
- dsh_evokit-0.1.5/tests/test_review_regressions.py +247 -0
- dsh_evokit-0.1.5/tests/test_secondary_guards.py +67 -0
- dsh_evokit-0.1.5/tests/test_v013_compatibility.py +101 -0
- dsh_evokit-0.1.5/tests/test_versions.py +59 -0
- dsh_evokit-0.1.5/uv.lock +2431 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## v0.1.5
|
|
4
|
+
|
|
5
|
+
- Add generic paired-defined secondary comparisons, explicit both-undefined
|
|
6
|
+
applicability, count guards, and complete-repeat quality noise estimates.
|
|
7
|
+
- Anchor configurable primary noise multipliers and optional observed-net floors;
|
|
8
|
+
scale the reference for pooled repeat means. Gates over the mean of R>1 repeats
|
|
9
|
+
now use threshold/sqrt(R), which is more permissive than v0.1.4. Single-repeat
|
|
10
|
+
gates are unchanged; majority aggregation keeps the single-repeat threshold.
|
|
11
|
+
- Preserve reviewed v0.1.2/v0.1.3 default-noise epochs without rewriting their
|
|
12
|
+
stored anchors. Repeat quality guards aggregate task means in task units.
|
|
13
|
+
|
|
14
|
+
## v0.1.4
|
|
15
|
+
|
|
16
|
+
- Select `deepseek-v4.1-flash` for proposals through a dedicated improver route,
|
|
17
|
+
require served build `deepseek-v4-1-flash-261017`, and enable high thinking with
|
|
18
|
+
a 32,768-token output budget. Preserve target and generator route settings.
|
|
19
|
+
Account reasoning as a subset of output and remove DeepSeek cache controls.
|
|
20
|
+
- Strengthen the evolve playbook with measured detectability, full-diff literal review,
|
|
21
|
+
record-cited history, predicted task flips and verified artifact load scope.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Use Python 3.11+ and uv. Keep changes focused and include a regression test for
|
|
4
|
+
behavioral fixes. The public contract is documented in README.md and docs/;
|
|
5
|
+
docs/tools.md owns CLI/MCP inputs and results. Edit skills/ first, then copy each
|
|
6
|
+
skill to integrations/dsh/skills/ so the wheel and DSH bundle stay identical.
|
|
7
|
+
|
|
8
|
+
```console
|
|
9
|
+
uv sync --frozen --group dev
|
|
10
|
+
uv run --frozen ruff check src tests examples scripts integrations
|
|
11
|
+
uv run --frozen ruff format --check src tests examples scripts integrations
|
|
12
|
+
uv run --frozen pytest -q tests --basetemp=.test-tmp
|
|
13
|
+
uv build
|
|
14
|
+
uv run --frozen python scripts/check_package.py
|
|
15
|
+
npm pack ./integrations/dsh --ignore-scripts --pack-destination dist
|
|
16
|
+
uv run --frozen python integrations/dsh/check_package.py
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
For the complete optional backend suite, run `uv sync --frozen --group dev --extra hb`
|
|
20
|
+
and `uv run --frozen --extra hb pytest -q tests --basetemp=.test-tmp`.
|
|
21
|
+
Use a private temporary home for backend state; the test fixtures configure it.
|
|
22
|
+
All tests use synthetic data and local fake services. Never add real credentials,
|
|
23
|
+
private tasks, runtime state or paid model calls. Scenarios execute trusted code.
|
|
24
|
+
Keep local experiments in ignored directories; retain only reproducible evidence
|
|
25
|
+
that is safe to share. Record user-visible changes in CHANGELOG.md.
|
dsh_evokit-0.1.5/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jiahao
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dsh-evokit
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Domain-neutral, pluggable self-evolution experiments
|
|
5
|
+
Project-URL: Homepage, https://github.com/edsml-jz2321/dsh-evokit
|
|
6
|
+
Project-URL: Repository, https://github.com/edsml-jz2321/dsh-evokit
|
|
7
|
+
Project-URL: Documentation, https://github.com/edsml-jz2321/dsh-evokit#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/edsml-jz2321/dsh-evokit/issues
|
|
9
|
+
Author: Jiahao
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: jsonschema<5,>=4.23
|
|
20
|
+
Requires-Dist: mcp<3,>=1.20
|
|
21
|
+
Requires-Dist: pyyaml<7,>=6.0.2
|
|
22
|
+
Provides-Extra: hb
|
|
23
|
+
Requires-Dist: fastmcp==4.0.0b1; (python_version < '3.14') and extra == 'hb'
|
|
24
|
+
Requires-Dist: heavenbase==0.1.2.2; (python_version < '3.14') and extra == 'hb'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
description: Measured self-improvement with paired experiments and a DSH plugin.
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
# evokit
|
|
32
|
+
|
|
33
|
+
evokit gives AI systems a measured self-improvement loop. It versions editable artifacts, records paired experiments, and gates changes against measured noise, cost and latency. Use it as a DeepSeek Harness (DSH) plugin or through its Python SDK, CLI and MCP server.
|
|
34
|
+
|
|
35
|
+
Your system supplies three pieces: **editable file globs**, **stable tasks**, and **a runner that executes and scores each task**. Prompts, skills, code and configuration can all be editable; the task source and scorer stay frozen.
|
|
36
|
+
|
|
37
|
+
The loop is **diagnose → propose → paired experiment → noise-aware gate → holdout confirmation with automatic revert in the operator workflow**. The improver sees diagnosis and selection evidence. A separate operator adopts a candidate, checks sealed confirmation aggregates, and calls `ledger_revert` when confirmation fails; `confirm_adoption` alone does not move the pointer. An optional final split opens once, with human approval, and never selects changes.
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
Requires Python 3.11+, uv, DSH `0.2.2-rc.1`, and its pnpm prerequisite. Windows is the verified local platform; Linux/macOS runtime integration remains unverified. Download the wheel, bundle and `SHA256SUMS` from [Releases](https://github.com/edsml-jz2321/dsh-evokit/releases) after publication and compare their SHA-256 hashes before installing.
|
|
42
|
+
|
|
43
|
+
```console
|
|
44
|
+
uv tool install ./dsh_evokit-0.1.5-py3-none-any.whl
|
|
45
|
+
dsh plugin --profile evokit add ./dsh-evokit-0.1.5.tgz
|
|
46
|
+
dsh plugin --profile evokit list
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Set `EVOKIT_MCP_COMMAND` to the absolute installed `evokit` executable, `EVOKIT_WORKSPACE` to your scenario directory, and `EVOKIT_SKILL_ROOT` to the installed bundle's `skills` directory. See the [DSH guide](integrations/dsh/README.md) for an isolated profile and exact PowerShell commands.
|
|
50
|
+
|
|
51
|
+
Configure the improver model in your DSH profile using **your own provider key** and your provider's supported model name. Keep the key in DSH's local credential settings or a process environment variable, never in this repository. The optional [relay](integrations/relay/README.md) supports file-based keys and accounting; adapt its example endpoint, model identity and illustrative prices before use. No key is included or required for the offline example. Installing or listing the plugin makes no model requests; starting evolution may incur provider charges.
|
|
52
|
+
|
|
53
|
+
## Minimal example
|
|
54
|
+
|
|
55
|
+
From a source checkout, the Node.js command example needs no model or key:
|
|
56
|
+
|
|
57
|
+
```console
|
|
58
|
+
uv sync --frozen --group dev
|
|
59
|
+
uv run python examples/scripted_improver.py command-echo
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Its `scenario.yaml` supplies the three pieces:
|
|
63
|
+
|
|
64
|
+
```yaml
|
|
65
|
+
editable: [prefix.txt]
|
|
66
|
+
tasks: tasks.json
|
|
67
|
+
command: [node, runner.mjs]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The runner reads `EVOKIT_TASK_FILE`, executes against `EVOKIT_VERSION_DIR`, and writes an outcome and cost to `EVOKIT_RESULT_FILE`. The scripted example works in a temporary copy. To connect your own system, follow [onboarding](docs/onboarding.md), then use the bundled `onboard`, `evolve` and `meta` skills.
|
|
71
|
+
|
|
72
|
+
## Safety and limits
|
|
73
|
+
|
|
74
|
+
Set budgets, anchors, noise policy and a stopping rule before experiments. Repeated mean gates use threshold/√R; majority gates retain the single-repeat threshold. Confirmation has a limited wear budget and is not an untouched final benchmark. One public BFCL pilot improved held-out accuracy from **41 to 47 of 50 after confirmation**; this small example does not establish general gains.
|
|
75
|
+
|
|
76
|
+
Scenarios run trusted executable code. Per-task directories are not a security sandbox. Protect sealed tasks and records with OS permissions; MCP filtering cannot hide files an improver can read directly. Review proposed changes for task literals and leakage. The relay accounts reported usage conservatively; it cannot guarantee provider billing or prevent every external cost.
|
|
77
|
+
|
|
78
|
+
See [concepts](docs/concepts.md), [tools](docs/tools.md), [storage](docs/storage.md) and [contributing](CONTRIBUTING.md). This project uses the [MIT license](LICENSE), Copyright (c) 2026 Jiahao.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Measured self-improvement with paired experiments and a DSH plugin.
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# evokit
|
|
6
|
+
|
|
7
|
+
evokit gives AI systems a measured self-improvement loop. It versions editable artifacts, records paired experiments, and gates changes against measured noise, cost and latency. Use it as a DeepSeek Harness (DSH) plugin or through its Python SDK, CLI and MCP server.
|
|
8
|
+
|
|
9
|
+
Your system supplies three pieces: **editable file globs**, **stable tasks**, and **a runner that executes and scores each task**. Prompts, skills, code and configuration can all be editable; the task source and scorer stay frozen.
|
|
10
|
+
|
|
11
|
+
The loop is **diagnose → propose → paired experiment → noise-aware gate → holdout confirmation with automatic revert in the operator workflow**. The improver sees diagnosis and selection evidence. A separate operator adopts a candidate, checks sealed confirmation aggregates, and calls `ledger_revert` when confirmation fails; `confirm_adoption` alone does not move the pointer. An optional final split opens once, with human approval, and never selects changes.
|
|
12
|
+
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
Requires Python 3.11+, uv, DSH `0.2.2-rc.1`, and its pnpm prerequisite. Windows is the verified local platform; Linux/macOS runtime integration remains unverified. Download the wheel, bundle and `SHA256SUMS` from [Releases](https://github.com/edsml-jz2321/dsh-evokit/releases) after publication and compare their SHA-256 hashes before installing.
|
|
16
|
+
|
|
17
|
+
```console
|
|
18
|
+
uv tool install ./dsh_evokit-0.1.5-py3-none-any.whl
|
|
19
|
+
dsh plugin --profile evokit add ./dsh-evokit-0.1.5.tgz
|
|
20
|
+
dsh plugin --profile evokit list
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Set `EVOKIT_MCP_COMMAND` to the absolute installed `evokit` executable, `EVOKIT_WORKSPACE` to your scenario directory, and `EVOKIT_SKILL_ROOT` to the installed bundle's `skills` directory. See the [DSH guide](integrations/dsh/README.md) for an isolated profile and exact PowerShell commands.
|
|
24
|
+
|
|
25
|
+
Configure the improver model in your DSH profile using **your own provider key** and your provider's supported model name. Keep the key in DSH's local credential settings or a process environment variable, never in this repository. The optional [relay](integrations/relay/README.md) supports file-based keys and accounting; adapt its example endpoint, model identity and illustrative prices before use. No key is included or required for the offline example. Installing or listing the plugin makes no model requests; starting evolution may incur provider charges.
|
|
26
|
+
|
|
27
|
+
## Minimal example
|
|
28
|
+
|
|
29
|
+
From a source checkout, the Node.js command example needs no model or key:
|
|
30
|
+
|
|
31
|
+
```console
|
|
32
|
+
uv sync --frozen --group dev
|
|
33
|
+
uv run python examples/scripted_improver.py command-echo
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Its `scenario.yaml` supplies the three pieces:
|
|
37
|
+
|
|
38
|
+
```yaml
|
|
39
|
+
editable: [prefix.txt]
|
|
40
|
+
tasks: tasks.json
|
|
41
|
+
command: [node, runner.mjs]
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The runner reads `EVOKIT_TASK_FILE`, executes against `EVOKIT_VERSION_DIR`, and writes an outcome and cost to `EVOKIT_RESULT_FILE`. The scripted example works in a temporary copy. To connect your own system, follow [onboarding](docs/onboarding.md), then use the bundled `onboard`, `evolve` and `meta` skills.
|
|
45
|
+
|
|
46
|
+
## Safety and limits
|
|
47
|
+
|
|
48
|
+
Set budgets, anchors, noise policy and a stopping rule before experiments. Repeated mean gates use threshold/√R; majority gates retain the single-repeat threshold. Confirmation has a limited wear budget and is not an untouched final benchmark. One public BFCL pilot improved held-out accuracy from **41 to 47 of 50 after confirmation**; this small example does not establish general gains.
|
|
49
|
+
|
|
50
|
+
Scenarios run trusted executable code. Per-task directories are not a security sandbox. Protect sealed tasks and records with OS permissions; MCP filtering cannot hide files an improver can read directly. Review proposed changes for task literals and leakage. The relay accounts reported usage conservatively; it cannot guarantee provider billing or prevent every external cost.
|
|
51
|
+
|
|
52
|
+
See [concepts](docs/concepts.md), [tools](docs/tools.md), [storage](docs/storage.md) and [contributing](CONTRIBUTING.md). This project uses the [MIT license](LICENSE), Copyright (c) 2026 Jiahao.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: 通过配对实验和 DSH 插件开展可测量的自进化。
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# evokit
|
|
6
|
+
|
|
7
|
+
evokit 为 AI 系统提供可测量的自进化循环:对可编辑产物进行版本管理,记录配对实验,并依据实测噪声、成本和延迟决定是否接受变更。可作为 DeepSeek Harness(DSH)插件使用,也可通过 Python SDK、CLI 和 MCP 服务接入。
|
|
8
|
+
|
|
9
|
+
你的系统只需提供三部分:**可编辑文件的 glob 模式**、**稳定的任务集**、**执行任务并评分的运行器**。提示词、技能、代码和配置都可以编辑;任务来源和评分器保持冻结。
|
|
10
|
+
|
|
11
|
+
循环为 **诊断 → 提案 → 配对实验 → 噪声感知门禁 → 留出集确认,操作工作流自动回退**。改进者只能读取诊断和选择证据。独立操作方接受候选后检查密封确认集的汇总结果;确认失败时调用 `ledger_revert`。单独调用 `confirm_adoption` 不会移动当前版本指针。可选最终集仅在人类批准后开启一次,不用于选择变更。
|
|
12
|
+
|
|
13
|
+
## 快速开始
|
|
14
|
+
|
|
15
|
+
需要 Python 3.11+、uv、DSH `0.2.2-rc.1` 及其 pnpm 依赖。已验证的本地平台为 Windows;Linux/macOS 运行时集成尚未验证。发布后,从 [Releases](https://github.com/edsml-jz2321/dsh-evokit/releases) 下载 wheel、插件包和 `SHA256SUMS`,核对 SHA-256 后再安装。
|
|
16
|
+
|
|
17
|
+
```console
|
|
18
|
+
uv tool install ./dsh_evokit-0.1.5-py3-none-any.whl
|
|
19
|
+
dsh plugin --profile evokit add ./dsh-evokit-0.1.5.tgz
|
|
20
|
+
dsh plugin --profile evokit list
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
将 `EVOKIT_MCP_COMMAND` 设为已安装的 `evokit` 可执行文件绝对路径,`EVOKIT_WORKSPACE` 设为场景目录,`EVOKIT_SKILL_ROOT` 设为已安装插件的 `skills` 目录。隔离配置与完整 PowerShell 命令见 [DSH 指南](integrations/dsh/README.md)。
|
|
24
|
+
|
|
25
|
+
在自己的 DSH 配置中选择改进者模型,使用**你自己的服务商密钥**和服务商支持的模型名。密钥应保存在 DSH 本地凭据设置或进程环境变量中,禁止提交到仓库。可选的[中继](integrations/relay/README.md)支持文件密钥和记账;使用前须替换示例端点、模型身份和示意价格。离线示例不需要密钥,也不附带密钥。安装和列出插件不会请求模型;启动进化可能产生服务商费用。
|
|
26
|
+
|
|
27
|
+
## 最小示例
|
|
28
|
+
|
|
29
|
+
在源码目录执行,Node.js 命令示例不需要模型或密钥:
|
|
30
|
+
|
|
31
|
+
```console
|
|
32
|
+
uv sync --frozen --group dev
|
|
33
|
+
uv run python examples/scripted_improver.py command-echo
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
其 `scenario.yaml` 提供上述三部分:
|
|
37
|
+
|
|
38
|
+
```yaml
|
|
39
|
+
editable: [prefix.txt]
|
|
40
|
+
tasks: tasks.json
|
|
41
|
+
command: [node, runner.mjs]
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
运行器读取 `EVOKIT_TASK_FILE`,在 `EVOKIT_VERSION_DIR` 上执行,并向 `EVOKIT_RESULT_FILE` 写入结果和成本。脚本示例使用临时副本。接入自己的系统时,先阅读[接入指南](docs/onboarding.md),再使用附带的 `onboard`、`evolve` 和 `meta` 技能。
|
|
45
|
+
|
|
46
|
+
## 安全与限制
|
|
47
|
+
|
|
48
|
+
实验前须设定预算、锚点、噪声策略和停止规则。重复均值门禁使用 threshold/√R;多数投票保留单次阈值。确认集有使用次数限制,不能视为未接触的最终基准。一个公开 BFCL 试验的留出集准确数经确认从 **41 提升到 47,共 50 题**;这一小规模示例不代表普遍收益。
|
|
49
|
+
|
|
50
|
+
场景会执行可信代码。逐任务目录并非安全沙箱。请用操作系统权限保护密封任务和记录;若改进者能直接读取文件,MCP 过滤无法保密。审查提案中的任务字面量和信息泄漏。中继保守记录服务商报告的用量,不能保证账单完全准确或阻止所有外部费用。
|
|
51
|
+
|
|
52
|
+
另见[概念](docs/concepts.md)、[工具](docs/tools.md)、[存储](docs/storage.md)和[贡献指南](CONTRIBUTING.md)。项目采用 [MIT 许可证](LICENSE),Copyright (c) 2026 Jiahao。
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Reuse stages and provider prompt prefixes
|
|
2
|
+
|
|
3
|
+
Evokit has two independent forms of reuse. A scenario's `StageCache` replays
|
|
4
|
+
JSON results for a stage whose full dependencies are unchanged. Its key must
|
|
5
|
+
include code, configuration and task inputs. Keep public and private measurement
|
|
6
|
+
cache roots separate; cache only valid reusable computation. A fresh stage-2
|
|
7
|
+
sample can share a frozen stage-1 result without reusing its own model response.
|
|
8
|
+
|
|
9
|
+
Provider prompt caching reuses input prefixes at the provider. It still makes
|
|
10
|
+
a model request and needs normal admission/accounting. The optional
|
|
11
|
+
[relay](../integrations/relay/README.md) applies explicit Qwen hints at the wire
|
|
12
|
+
boundary on a deep copy, targeting system and newest text with at most four
|
|
13
|
+
markers. DeepSeek receives no added hints. Scoring prompts, caller data and
|
|
14
|
+
response-cache identities remain unchanged.
|
|
15
|
+
|
|
16
|
+
Price cache creation at 125% of the input rate, hits at 10%, and remaining input
|
|
17
|
+
at 100%. Both counters are subsets of prompt tokens. Reserve at uncached upper
|
|
18
|
+
bounds and settle reported usage; a missing usage report never implies zero.
|
|
19
|
+
Report hit/input tokens per model or host role, including zero/unknown states.
|
|
20
|
+
Fake-upstream tests verify hint shape and arithmetic. A real repeated-prefix
|
|
21
|
+
cache-hit measurement needs separately authorized provider execution and is not
|
|
22
|
+
claimed by the keyless v0.1.3 release.
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# Concepts
|
|
2
|
+
|
|
3
|
+
## System and scenario
|
|
4
|
+
|
|
5
|
+
A system is a fixed model/environment plus editable artifacts: prompts, skills,
|
|
6
|
+
knowledge, tool code, control flow or configuration. A scenario supplies editable
|
|
7
|
+
globs, tasks and a scorer. The framework owns experiments and lineage. The host
|
|
8
|
+
owns proposing and reviewing mechanisms. META coordinates and decides; analysts
|
|
9
|
+
and improvers propose, and executors run arms. The `meta` and `evolve` skills
|
|
10
|
+
define those responsibilities. There is no fixed candidate/judge/critic
|
|
11
|
+
topology and no assumption that a task has a reference answer.
|
|
12
|
+
|
|
13
|
+
## Versions and epochs
|
|
14
|
+
|
|
15
|
+
Each version is a manifest mapping editable relative paths to SHA-256 file bytes.
|
|
16
|
+
The SHA-256 of its canonical manifest is its id. Blobs are verified before use.
|
|
17
|
+
Unchanged supporting files are snapshotted at epoch creation; a version materializes
|
|
18
|
+
those files plus its editable manifest. A candidate directory must retain exactly
|
|
19
|
+
the frozen support files. New, removed and changed editable files are allowed.
|
|
20
|
+
Symlinks, junctions and case collisions are rejected for portability. Snapshots
|
|
21
|
+
track file content; executable permission bits and external dependencies are not
|
|
22
|
+
versioned. Invoke scripts through their runtime and pin dependencies separately.
|
|
23
|
+
|
|
24
|
+
An epoch freezes the task source, seeded split, final scorer version, located
|
|
25
|
+
adapter/config source digests and Python scorer bytecode, noneditable support,
|
|
26
|
+
editable scope, literal guards and budget policy. Diagnosis explains
|
|
27
|
+
failures; selection decides adoption; confirmation decides only keep-or-revert
|
|
28
|
+
after adoption; final is opened once and decides nothing. `holdout` remains a
|
|
29
|
+
configuration/data alias for confirmation, including existing epoch assignments.
|
|
30
|
+
Largest-remainder allocation within each optional group preserves proportions.
|
|
31
|
+
Group stratification distributes each group across splits; it does not keep entire
|
|
32
|
+
groups in one split. Tiny groups may have empty partitions; readiness counts reveal
|
|
33
|
+
that limitation. The installed evokit scenario module may relocate only with identical bytes.
|
|
34
|
+
User adapter paths, callable fingerprints and scenario-relative anchors still
|
|
35
|
+
match exactly. A new state directory starts a new epoch. Optional YAML `splits`
|
|
36
|
+
sets diagnosis, selection, confirmation (or holdout) and final fractions summing to
|
|
37
|
+
1. Defaults are 0.4/0.4/0.2/0. Final defaults to empty; adding it needs a new epoch.
|
|
38
|
+
Neither tools nor public queries reveal confirmation/final identities, per-task
|
|
39
|
+
scores, outputs or traces. Only counts are visible before opening final.
|
|
40
|
+
|
|
41
|
+
Before the first arm (including a keyless smoke), META records an epoch contract:
|
|
42
|
+
objective, cost/latency ratio guards, split/scorer anchors, confirmation `min_net`
|
|
43
|
+
rule, repeated-incumbent noise plan, total/per-generation budget and unit,
|
|
44
|
+
stagnation window and confirmation wear budget (default 4). The min_net form
|
|
45
|
+
represents either zero or a pre-chosen negative noise threshold. The tool refuses
|
|
46
|
+
changes once work has started; changing anchors then requires a new epoch. META
|
|
47
|
+
owns enforcing the campaign's budget and stopping plan. Existing scenario gate
|
|
48
|
+
budgets remain enforced by the gate.
|
|
49
|
+
|
|
50
|
+
`confirm_adoption` pairs the adopted current child against its parent on
|
|
51
|
+
confirmation, reusing complete retained measurements. It returns aggregate fixes,
|
|
52
|
+
breaks, net, cost/latency ratios and the contract's verdict. Rule or ratio-guard
|
|
53
|
+
failure marks the proposal not confirmed; META uses `ledger_revert` with a reason.
|
|
54
|
+
Confirmation never selects among candidates. Each opening consumes a wear slot,
|
|
55
|
+
even on execution failure; resuming the same adoption reuses its slot. Completed
|
|
56
|
+
decisions are idempotent. Exhaustion requires a new epoch with fresh tasks. These
|
|
57
|
+
reused confirmation results become optimistic estimates; report them separately
|
|
58
|
+
from final measurements. Improvers receive only the `not_confirmed` proposal bit.
|
|
59
|
+
|
|
60
|
+
At campaign end, with human approval, `final_measure` opens a separate final split
|
|
61
|
+
once and pairs current against the epoch's initial version. It returns aggregates
|
|
62
|
+
and changes no pointer. The opening is durable before execution, so a failed
|
|
63
|
+
measurement still consumes it. Opening final closes evolution in that epoch.
|
|
64
|
+
Without final tasks, use a new epoch with fresh tasks; confirmation must never
|
|
65
|
+
serve as the final measurement. A final result cannot justify adoption or tuning.
|
|
66
|
+
|
|
67
|
+
## Runs and replay
|
|
68
|
+
|
|
69
|
+
An arm is version × split × repeat count. `run_launch` returns immediately by
|
|
70
|
+
default; a detached process works through task slots with a bounded thread pool.
|
|
71
|
+
`wait=true` executes synchronously. Distinct arm ids can run concurrently. A run
|
|
72
|
+
has one executor at a time, and completed task/repeat records are never rerun.
|
|
73
|
+
An interrupted slot without a committed newline is retried. Scenarios with external
|
|
74
|
+
side effects must make those operations idempotent; this is not an exactly-once
|
|
75
|
+
provider transaction. Resume with the same run id and the same parameters.
|
|
76
|
+
|
|
77
|
+
Every task/repeat has its own materialized directory under its arm. Results record
|
|
78
|
+
outcome, reason, numeric costs, seconds, output, trace path, version, epoch, repeat
|
|
79
|
+
and scorer version. Traces are archived before the result commits. Execution errors
|
|
80
|
+
are explicit failed rows and block a gate. Fix execution defects and use a new run
|
|
81
|
+
to retry scored errors; resume preserves them as evidence.
|
|
82
|
+
|
|
83
|
+
Optional `run_replay` uses `StageCache.replay(stage, key, compute)`. The scenario
|
|
84
|
+
owns cache-key completeness: include stage implementation/version, relevant config,
|
|
85
|
+
input, model/environment identities and randomness when applicable. The cache
|
|
86
|
+
serializes one producer per key and persists JSON outputs. The toy's stage 2 edit
|
|
87
|
+
leaves the stage-1 key unchanged, so stage-1 cost becomes zero. Confirmation and final caches use
|
|
88
|
+
separate private namespaces. Cache speed does not establish behavior improvement.
|
|
89
|
+
|
|
90
|
+
## Paired decisions
|
|
91
|
+
|
|
92
|
+
Both arms evaluate identical tasks, groups and repeat counts. Mean aggregation
|
|
93
|
+
preserves scores in [0,1]. Majority aggregation accepts binary outcomes and treats
|
|
94
|
+
a tied vote as failure. Net is the sum of child-minus-parent per-task aggregated
|
|
95
|
+
scores, not a percentage. Fixes and breaks name tasks with positive/negative deltas.
|
|
96
|
+
Cost and latency ratios normalize each task over repeats, then sum tasks. Absolute
|
|
97
|
+
cost and task-seconds budgets count the complete candidate arm, including all repeats.
|
|
98
|
+
A positive child cost over a zero parent cost has an unbounded ratio, represented
|
|
99
|
+
as JSON null, and fails a ratio budget. Missing cost units count as zero.
|
|
100
|
+
|
|
101
|
+
Repeated incumbent selection runs estimate each task's pairwise flip rate and
|
|
102
|
+
paired net spread. The recorded threshold is the maximum of the largest observed
|
|
103
|
+
absolute repeat net and 1.96 times the square root of twice the summed per-task
|
|
104
|
+
sample variances. It is a conservative single-repeat reference, not a confidence
|
|
105
|
+
guarantee, and does not model correlated provider drift. Use same-window controls
|
|
106
|
+
and more repeats when the margin is small. Gates using mean aggregation divide this
|
|
107
|
+
single-repeat reference by sqrt(R); majority aggregation retains it unchanged.
|
|
108
|
+
|
|
109
|
+
The public gate verifies editable scope, scans introduced content for public task
|
|
110
|
+
ids, declared public literals and public gold strings, checks epoch/scorer/private split isolation and
|
|
111
|
+
budgets, and requires generic-mechanism attestation. Literal scans cannot prove
|
|
112
|
+
the absence of semantic or encoded leaks. Default cost and latency ratio ceilings
|
|
113
|
+
are 1.2; scenarios may declare alternatives. Gain strictly above the measured
|
|
114
|
+
threshold promotes. Nonnegative gain can adopt-no-harm with a substantive rationale.
|
|
115
|
+
All hard checks apply to both verdicts. Adoption rechecks evidence against current
|
|
116
|
+
before appending an event. Promising runner-ups remain queued for a new proposal
|
|
117
|
+
and fresh testing on the next base.
|
|
118
|
+
|
|
119
|
+
Private confirmation/final ids and `private_literals` are checked only in the operator's
|
|
120
|
+
post-adoption audit, with no match status returned to the improver. A public
|
|
121
|
+
pass/fail response that depends on a secret literal is a membership oracle even
|
|
122
|
+
when it hides the matched text. This privacy boundary takes precedence over using
|
|
123
|
+
secret membership as an adoption signal. Public literals/gold_strings must never
|
|
124
|
+
contain private confirmation/final labels. Guard matching uses word boundaries and newly
|
|
125
|
+
introduced lines; removal of a pre-existing literal is allowed. Numeric or one/two
|
|
126
|
+
character task ids count only when quoted, to avoid treating generic scalar
|
|
127
|
+
changes as instance patches. Use descriptive task ids and explicitly declared
|
|
128
|
+
public literals when a stricter policy is appropriate. This heuristic remains a
|
|
129
|
+
review backstop, not proof against arbitrary task-specific behavior.
|
|
130
|
+
|
|
131
|
+
## Lineage and recursion
|
|
132
|
+
|
|
133
|
+
The append-only event ledger records proposals, runs, verdicts, adoptions, noise
|
|
134
|
+
estimates, queue entries and rationale. Events form a hash chain. The ledger is
|
|
135
|
+
authoritative for the current base and base chain. A revert appends a new
|
|
136
|
+
pointer event back to the current adoption's parent; failed proposals, adoptions
|
|
137
|
+
and their evidence remain in history. The stored current pointer is
|
|
138
|
+
a convenience cache. Files uses JSONL; HB stores the events natively. See
|
|
139
|
+
[storage](storage.md) for backend selection and holdout placement. Never edit or truncate history to hide a failed experiment. A damaged
|
|
140
|
+
ledger fails loudly and needs explicit recovery preserving the original bytes.
|
|
141
|
+
|
|
142
|
+
Level 0 evolves system artifacts. Level 1 evolves the improver playbook and proposal
|
|
143
|
+
practice. Each proposal snapshots the exact playbook bytes and records its digest.
|
|
144
|
+
Metrics report proposal acceptance rate, absolute prediction error and gain/cost.
|
|
145
|
+
Gain/cost includes every committed public experiment result, including controls,
|
|
146
|
+
smokes and rejected candidates, so failed experiments remain part of the expense.
|
|
147
|
+
META uses the operator tool profile to record contracts, adopt, confirm, revert and
|
|
148
|
+
open final. The default improver profile omits those tools and refuses guessed
|
|
149
|
+
operator tool names at dispatch. An operator reads confirmation rate through
|
|
150
|
+
`meta_metrics`; the improver profile removes that rate. `Workspace.measure_holdout`
|
|
151
|
+
is a legacy SDK alias: contracted epochs use the confirmation rule and wear;
|
|
152
|
+
uncontracted legacy clients retain count-only measurement behavior. This is a
|
|
153
|
+
capability boundary, not a sandbox for an improver with arbitrary shell access.
|
|
154
|
+
Use separate host permissions for private state. Level 2 can change scoring or
|
|
155
|
+
gate policy only at an epoch boundary, with calibration and fresh noise.
|