zanii-evolve 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zanii_evolve-0.1.0/LICENSE +202 -0
- zanii_evolve-0.1.0/NOTICE +17 -0
- zanii_evolve-0.1.0/PKG-INFO +407 -0
- zanii_evolve-0.1.0/README.md +359 -0
- zanii_evolve-0.1.0/pyproject.toml +90 -0
- zanii_evolve-0.1.0/setup.cfg +4 -0
- zanii_evolve-0.1.0/tests/test_brief.py +196 -0
- zanii_evolve-0.1.0/tests/test_core.py +194 -0
- zanii_evolve-0.1.0/tests/test_growth.py +489 -0
- zanii_evolve-0.1.0/tests/test_memory.py +136 -0
- zanii_evolve-0.1.0/tests/test_server.py +152 -0
- zanii_evolve-0.1.0/zanii_evolve/__init__.py +44 -0
- zanii_evolve-0.1.0/zanii_evolve/analyst.py +204 -0
- zanii_evolve-0.1.0/zanii_evolve/brief.py +407 -0
- zanii_evolve-0.1.0/zanii_evolve/calibrate.py +119 -0
- zanii_evolve-0.1.0/zanii_evolve/components.py +108 -0
- zanii_evolve-0.1.0/zanii_evolve/config.py +103 -0
- zanii_evolve-0.1.0/zanii_evolve/critic.py +147 -0
- zanii_evolve-0.1.0/zanii_evolve/dashboard/index.html +448 -0
- zanii_evolve-0.1.0/zanii_evolve/digester.py +221 -0
- zanii_evolve-0.1.0/zanii_evolve/domain.py +125 -0
- zanii_evolve-0.1.0/zanii_evolve/driver.py +99 -0
- zanii_evolve-0.1.0/zanii_evolve/evaluate.py +135 -0
- zanii_evolve-0.1.0/zanii_evolve/gitops.py +127 -0
- zanii_evolve-0.1.0/zanii_evolve/history.py +211 -0
- zanii_evolve-0.1.0/zanii_evolve/judgecal.py +80 -0
- zanii_evolve-0.1.0/zanii_evolve/lessons.py +91 -0
- zanii_evolve-0.1.0/zanii_evolve/llm.py +113 -0
- zanii_evolve-0.1.0/zanii_evolve/loop.py +606 -0
- zanii_evolve-0.1.0/zanii_evolve/pareto.py +48 -0
- zanii_evolve-0.1.0/zanii_evolve/propose.py +410 -0
- zanii_evolve-0.1.0/zanii_evolve/providers.py +249 -0
- zanii_evolve-0.1.0/zanii_evolve/registry.py +76 -0
- zanii_evolve-0.1.0/zanii_evolve/report.py +265 -0
- zanii_evolve-0.1.0/zanii_evolve/schedule.py +54 -0
- zanii_evolve-0.1.0/zanii_evolve/selection.py +143 -0
- zanii_evolve-0.1.0/zanii_evolve/server.py +404 -0
- zanii_evolve-0.1.0/zanii_evolve/tasksets.py +249 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/PATTERNS.md +28 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/SKILL.md +54 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/adapter.py +23 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/briefs.py +45 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/harness/agent.py +62 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/harness/mechanisms.py +312 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/harness/prompts.py +37 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/harness/run.py +90 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/tasks.jsonl +3 -0
- zanii_evolve-0.1.0/zanii_evolve/templates/brief/zanii_evolve.json +14 -0
- zanii_evolve-0.1.0/zanii_evolve.egg-info/PKG-INFO +407 -0
- zanii_evolve-0.1.0/zanii_evolve.egg-info/SOURCES.txt +51 -0
- zanii_evolve-0.1.0/zanii_evolve.egg-info/dependency_links.txt +1 -0
- zanii_evolve-0.1.0/zanii_evolve.egg-info/requires.txt +33 -0
- zanii_evolve-0.1.0/zanii_evolve.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Zanii Evolve
|
|
2
|
+
Copyright 2026 zaniiai agency.
|
|
3
|
+
|
|
4
|
+
This product includes software derived from the RRSI reference implementation:
|
|
5
|
+
Copyright 2026 Google LLC
|
|
6
|
+
Copyright 2026 The rrsi Authors
|
|
7
|
+
Licensed under the Apache License, Version 2.0. Files carrying these notices
|
|
8
|
+
were modified by zaniiai agency; modified files say so in their headers.
|
|
9
|
+
|
|
10
|
+
third_party/harbor_terminus2
|
|
11
|
+
The Terminus-2 agent from harbor (https://github.com/laude-institute/harbor),
|
|
12
|
+
Apache License 2.0; see third_party/harbor_terminus2/LICENSE.
|
|
13
|
+
|
|
14
|
+
third_party/archipelago
|
|
15
|
+
The react_toolbelt agent and runner from archipelago
|
|
16
|
+
(https://github.com/Mercor-Intelligence/archipelago),
|
|
17
|
+
Apache License 2.0; see third_party/archipelago/LICENSE.
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: zanii-evolve
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Regularized self-improvement of LLM agent harnesses: gains that hold up out of distribution, with verifiable change history
|
|
5
|
+
Author: zaniiai agency
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/vigilancetrent/zanii-evolve
|
|
8
|
+
Project-URL: Source, https://github.com/vigilancetrent/zanii-evolve
|
|
9
|
+
Project-URL: Issues, https://github.com/vigilancetrent/zanii-evolve/issues
|
|
10
|
+
Keywords: llm,agents,agent-harness,self-improvement,evaluation,prompt-optimization,auditability,proof-of-action
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
License-File: NOTICE
|
|
25
|
+
Requires-Dist: anthropic[vertex]>=0.40
|
|
26
|
+
Provides-Extra: agentic
|
|
27
|
+
Requires-Dist: fastmcp>=2.0; extra == "agentic"
|
|
28
|
+
Requires-Dist: litellm>=1.50; extra == "agentic"
|
|
29
|
+
Requires-Dist: loguru>=0.7; extra == "agentic"
|
|
30
|
+
Requires-Dist: mcp>=1.0; extra == "agentic"
|
|
31
|
+
Requires-Dist: openai>=1.0; extra == "agentic"
|
|
32
|
+
Requires-Dist: pydantic>=2.0; extra == "agentic"
|
|
33
|
+
Requires-Dist: pydantic-settings>=2.0; extra == "agentic"
|
|
34
|
+
Requires-Dist: httpx>=0.27; extra == "agentic"
|
|
35
|
+
Requires-Dist: tenacity>=8.0; extra == "agentic"
|
|
36
|
+
Requires-Dist: pymupdf>=1.24; extra == "agentic"
|
|
37
|
+
Requires-Dist: zaniidb-agent-memory>=0.11; python_version >= "3.11" and extra == "agentic"
|
|
38
|
+
Provides-Extra: coding
|
|
39
|
+
Requires-Dist: harbor>=0.18; extra == "coding"
|
|
40
|
+
Requires-Dist: zaniidb-agent-memory>=0.11; python_version >= "3.11" and extra == "coding"
|
|
41
|
+
Provides-Extra: provable
|
|
42
|
+
Requires-Dist: zanii>=0.25; extra == "provable"
|
|
43
|
+
Provides-Extra: memory
|
|
44
|
+
Requires-Dist: zaniidb-agent-memory>=0.11; python_version >= "3.11" and extra == "memory"
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
|
|
49
|
+
<p align="center">
|
|
50
|
+
<img src="https://raw.githubusercontent.com/vigilancetrent/zanii-evolve/main/assets/banner.svg" width="100%" alt="Zanii Evolve">
|
|
51
|
+
</p>
|
|
52
|
+
|
|
53
|
+
<p align="center">
|
|
54
|
+
<a href="https://pypi.org/project/zanii-evolve/"><img src="https://img.shields.io/pypi/v/zanii-evolve?color=0f766e" alt="PyPI"></a>
|
|
55
|
+
<a href="https://pypi.org/project/zanii-evolve/"><img src="https://img.shields.io/pypi/pyversions/zanii-evolve" alt="Python versions"></a>
|
|
56
|
+
<a href="https://github.com/vigilancetrent/zanii-evolve/actions/workflows/tests.yml"><img src="https://github.com/vigilancetrent/zanii-evolve/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
|
|
57
|
+
<a href="https://github.com/vigilancetrent/zanii-evolve/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue" alt="License"></a>
|
|
58
|
+
</p>
|
|
59
|
+
|
|
60
|
+
Zanii Evolve improves an LLM agent by editing its **harness** (the prompts,
|
|
61
|
+
control flow, tools, skills and memory around a frozen model), and only keeps
|
|
62
|
+
the edits that survive a regularized search. A gain has to clear the
|
|
63
|
+
measurement noise, pay for any extra tokens it costs, and pass a leakage screen
|
|
64
|
+
before it becomes the new baseline. The result is an agent that gets better on
|
|
65
|
+
work it has not seen, not one that memorized its test set.
|
|
66
|
+
|
|
67
|
+
Every accepted change is a git commit with a stated hypothesis and a measured
|
|
68
|
+
effect. A run ends with a certificate you can hand to a reviewer, optionally
|
|
69
|
+
signed so that anyone can verify who issued it without trusting you.
|
|
70
|
+
|
|
71
|
+
Maintained by [zaniiai agency](https://github.com/zaniiai).
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Contents
|
|
76
|
+
|
|
77
|
+
- [Why](#why)
|
|
78
|
+
- [Install](#install)
|
|
79
|
+
- [Quickstart](#quickstart)
|
|
80
|
+
- [Bring your own tasks](#bring-your-own-tasks)
|
|
81
|
+
- [Certificates, signing and the registry](#certificates-signing-and-the-registry)
|
|
82
|
+
- [Memory](#memory)
|
|
83
|
+
- [Dashboard and HTTP API](#dashboard-and-http-api)
|
|
84
|
+
- [Configuration](#configuration)
|
|
85
|
+
- [Command reference](#command-reference)
|
|
86
|
+
- [Project layout](#project-layout)
|
|
87
|
+
- [Acknowledgements](#acknowledgements)
|
|
88
|
+
- [License](#license)
|
|
89
|
+
|
|
90
|
+
## Why
|
|
91
|
+
|
|
92
|
+
Searching over harness edits against a fixed task set works, and it overfits:
|
|
93
|
+
the harness learns the tasks, and the in-distribution gain shrinks or vanishes
|
|
94
|
+
on anything new. Zanii Evolve keeps the edit space open and constrains *how the
|
|
95
|
+
search moves* instead.
|
|
96
|
+
|
|
97
|
+
<p align="center">
|
|
98
|
+
<img src="https://raw.githubusercontent.com/vigilancetrent/zanii-evolve/main/assets/pipeline.svg" width="100%" alt="One round: analyze, propose, screen, evaluate, select; evidence out">
|
|
99
|
+
</p>
|
|
100
|
+
|
|
101
|
+
**On the proposal side**
|
|
102
|
+
|
|
103
|
+
- An annealed edit budget limits how many independent changes one candidate may bundle, so late gains are attributable.
|
|
104
|
+
- The proposer sees the full edit history, so a hypothesis that already failed is not tried again.
|
|
105
|
+
- When progress stalls, slots are reserved for components the run has never touched.
|
|
106
|
+
|
|
107
|
+
**On the selection side**
|
|
108
|
+
|
|
109
|
+
- A critic screens every diff for task-specific logic before any evaluation is spent on it.
|
|
110
|
+
- A noise-adjusted floor rejects gains that sit inside evaluation variance.
|
|
111
|
+
- A cost rule requires added inference tokens to be paid for by measured gain.
|
|
112
|
+
- Components that stop contributing are flagged for pruning.
|
|
113
|
+
|
|
114
|
+
Each rule maps to one place in the code:
|
|
115
|
+
|
|
116
|
+
| Rule | Code |
|
|
117
|
+
|---|---|
|
|
118
|
+
| Score and cost estimate | `zanii_evolve/evaluate.py` (`aggregate`; a missing trial scores 0 with the full denominator) |
|
|
119
|
+
| Annealed edit budget | `zanii_evolve/schedule.py` (`edit_budget`), enforced in `propose.py` |
|
|
120
|
+
| Edit history, tried set, recent yield, stall flag | `zanii_evolve/history.py` |
|
|
121
|
+
| Failure analysis | `zanii_evolve/analyst.py`, `digester.py` |
|
|
122
|
+
| Leakage critic | `zanii_evolve/critic.py` (domain denylist plus LLM review, bounded repair) |
|
|
123
|
+
| Noise floor, cost rule, score or cost objective | `zanii_evolve/selection.py` |
|
|
124
|
+
| Noise band calibration | `zanii_evolve/calibrate.py` (bootstrap over base trials, or repeated base runs) |
|
|
125
|
+
| Non-compensatory domain guards | `Domain.guards` (for example a valid-rate drop in engineering design) |
|
|
126
|
+
|
|
127
|
+
## Install
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
pip install zanii-evolve # search core, providers, certificates, brief engine
|
|
131
|
+
pip install "zanii-evolve[provable]" # + signed certificate receipts (zanii)
|
|
132
|
+
pip install "zanii-evolve[memory]" # + ZaniiDB memory backend (Python 3.11+)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The command line, the bundled domains and the starting harnesses live in the
|
|
136
|
+
repository. To run evolutions, clone it:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
git clone https://github.com/vigilancetrent/zanii-evolve.git
|
|
140
|
+
cd zanii-evolve
|
|
141
|
+
pip install -e ".[dev]"
|
|
142
|
+
python -m pytest tests
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Requires Python 3.10 or newer. Benchmark runners keep their own environments:
|
|
146
|
+
harbor for the coding domain, and a Python 3.11 environment with
|
|
147
|
+
`pip install -e ".[agentic]"` for the workspace and engineering domains, pointed
|
|
148
|
+
to by `ZANII_EVOLVE_AGENT_PYTHON`.
|
|
149
|
+
|
|
150
|
+
## Quickstart
|
|
151
|
+
|
|
152
|
+
Pick a model provider. The model string names it:
|
|
153
|
+
|
|
154
|
+
| Model string | Backend | Credentials |
|
|
155
|
+
|---|---|---|
|
|
156
|
+
| `anthropic/<model>` | Anthropic API | `ANTHROPIC_API_KEY` |
|
|
157
|
+
| `vertex/<model>` | Vertex AI, round-robin over projects | `gcloud auth application-default login`, `ZANII_EVOLVE_VERTEX_PROJECTS` |
|
|
158
|
+
| `openai/<model>` | Any OpenAI-compatible endpoint (OpenAI, LiteLLM, vLLM) | `ZANII_EVOLVE_OPENAI_BASE_URL`, `ZANII_EVOLVE_OPENAI_API_KEY` |
|
|
159
|
+
| `ollama/<model>` | A local Ollama server, for on-prem and air-gapped use | `ZANII_EVOLVE_OLLAMA_BASE_URL` (no key is ever sent) |
|
|
160
|
+
|
|
161
|
+
Custom backends register with `ZANII_EVOLVE_PROVIDERS="name=module:Class"`.
|
|
162
|
+
|
|
163
|
+
Every domain then follows the same lifecycle:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python zanii_evolve.py --domain marketing smoke # liveness check on a couple of tasks
|
|
167
|
+
python zanii_evolve.py --domain marketing baseline # measure the starting harness H0
|
|
168
|
+
python zanii_evolve.py --domain marketing run # evolve; resumable, `touch runs/marketing/STOP` to stop
|
|
169
|
+
python zanii_evolve.py --domain marketing status # incumbent against H0
|
|
170
|
+
python zanii_evolve.py --domain marketing heldout --label final # held-out split, never used for selection
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Each round drafts candidates in separate git worktrees on branches off
|
|
174
|
+
`evolve/<domain>`, screens and evaluates them, and fast-forwards the branch to
|
|
175
|
+
the winner, so the current best harness is always a commit.
|
|
176
|
+
|
|
177
|
+
### Bundled domains
|
|
178
|
+
|
|
179
|
+
| Domain | Evolve set | Out-of-distribution checks | Notes |
|
|
180
|
+
|---|---|---|---|
|
|
181
|
+
| [`coding`](https://github.com/vigilancetrent/zanii-evolve/tree/main/domains/coding) | Terminal-Bench 2.1 | SWE-bench Verified | Docker and harbor |
|
|
182
|
+
| [`workspace`](https://github.com/vigilancetrent/zanii-evolve/tree/main/domains/workspace) | Harvey LAB | JobBench, GDPval, APEX-Agents | Harvey LAB checkout at a pinned commit |
|
|
183
|
+
| [`eng`](https://github.com/vigilancetrent/zanii-evolve/tree/main/domains/eng) | EngDesign | EngDesign v1, Frontier-Eng | grading venv, jailed tool gateway |
|
|
184
|
+
| [`marketing`](https://github.com/vigilancetrent/zanii-evolve/tree/main/domains/marketing) | 48 briefs, 8 fictional clients | held-out split | brief domain, no adapter code |
|
|
185
|
+
| [`memory`](https://github.com/vigilancetrent/zanii-evolve/tree/main/domains/memory) | 40 multi-session histories | held-out split | exercises ZaniiDB recall and supersede |
|
|
186
|
+
|
|
187
|
+
Results are reported per domain for the evolve, held-out and out-of-distribution
|
|
188
|
+
splits, always against H0 measured in the same window with the same frozen
|
|
189
|
+
model. Published numbers will ship with signed certificates.
|
|
190
|
+
|
|
191
|
+
## Bring your own tasks
|
|
192
|
+
|
|
193
|
+
Any work that is "a brief in, a text deliverable out, graded by checks and a
|
|
194
|
+
rubric" needs no adapter code.
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
python zanii_evolve.py --domain acme init-domain # scaffold domains/acme
|
|
198
|
+
python zanii_evolve.py --domain acme import-tasks briefs.csv --replace
|
|
199
|
+
python zanii_evolve.py --domain acme redteam --n 10 # prompt-injection split
|
|
200
|
+
python zanii_evolve.py --domain acme ingest-traces prod.jsonl --rubric "Answers the question|No made-up facts"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
A spreadsheet needs one column, `brief`. Optional columns are `id`, `split`,
|
|
204
|
+
`kind`, `must_include` and `must_not_include` (`|`-separated), `max_words`,
|
|
205
|
+
`min_words`, `max_chars`, and `rubric` (`|`-separated pass/fail criteria). Any
|
|
206
|
+
other column (`client`, `audience`, ...) reaches the harness as context. Rows
|
|
207
|
+
without a split get a seeded evolve/held-out draw, and the whole file is
|
|
208
|
+
validated before anything is written.
|
|
209
|
+
|
|
210
|
+
The underlying task format is one JSON object per line:
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
{"id": "acme-001", "split": "evolve", "kind": "promotional email", "brief": "...",
|
|
214
|
+
"checks": [{"type": "max_words", "value": 150}, {"type": "must_include", "values": ["Book now"]}],
|
|
215
|
+
"rubric": [{"criterion": "Matches the brand voice.", "weight": 1}]}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Checks: `max_words`, `min_words`, `max_chars`, `must_include`, `must_include_any`,
|
|
219
|
+
`must_not_include`, `regex`, `regex_count`, `field_max_chars`. Rubric criteria are
|
|
220
|
+
judged pass/fail by `judge_model`. Checks, rubric and expected answers never
|
|
221
|
+
reach the harness, and grading runs outside the candidate's editable tree, so a
|
|
222
|
+
candidate cannot edit its own judge.
|
|
223
|
+
|
|
224
|
+
**Production traces.** `ingest-traces` accepts records with `input`, `prompt`
|
|
225
|
+
or an OpenTelemetry `gen_ai.prompt` attribute. Records with negative feedback
|
|
226
|
+
come first, duplicates are dropped, emails, phone numbers and card-like numbers
|
|
227
|
+
are redacted, and re-running on the same file adds nothing. Production outputs
|
|
228
|
+
are never copied into a task.
|
|
229
|
+
|
|
230
|
+
**Red-team split.** Each variant embeds an instruction carrying a unique canary
|
|
231
|
+
string that the deliverable must not contain, keeps the original checks, and
|
|
232
|
+
lives in its own `redteam` split: `heldout --label rt --set redteam`.
|
|
233
|
+
|
|
234
|
+
**Judge calibration.** Before trusting a rubric judge with a run, grade a
|
|
235
|
+
handful of deliverables by hand and compare:
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
python zanii_evolve.py --domain acme judge-agreement --labels human.jsonl
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
It reports raw agreement, Cohen's kappa and per-criterion agreement, weakest first.
|
|
242
|
+
|
|
243
|
+
**Custom domains.** Anything else is one module, `domains/<name>/adapter.py`,
|
|
244
|
+
exporting a `zanii_evolve.domain.Domain` that runs and scores a harness, renders
|
|
245
|
+
traces for the analyst, and supplies the domain's leakage denylist and guards.
|
|
246
|
+
The core never reads a benchmark format itself.
|
|
247
|
+
|
|
248
|
+
## Certificates, signing and the registry
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
python zanii_evolve.py --domain acme report --out cert # cert.json + cert.md
|
|
252
|
+
python zanii_evolve.py verify-report cert.json
|
|
253
|
+
python zanii_evolve.py --domain acme pareto # score/cost trade-off of every measured harness
|
|
254
|
+
python zanii_evolve.py --domain acme publish # harness archive + certificate -> registry/
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
A certificate records the incumbent against H0, the noise band, every accepted
|
|
258
|
+
edit with its hypothesis and measured score and cost change, the outcome of
|
|
259
|
+
every candidate, and every held-out evaluation. It carries a sha256 digest of
|
|
260
|
+
its body, so any edit after issue is detected.
|
|
261
|
+
|
|
262
|
+
Signing comes in two strengths:
|
|
263
|
+
|
|
264
|
+
- **HMAC** (`ZANII_EVOLVE_SIGNING_KEY`) proves the issuer to anyone who holds the same key.
|
|
265
|
+
- **Proof-of-action receipt** (`pip install "zanii-evolve[provable]"`) is an Ed25519
|
|
266
|
+
signature by a `did:key` identity that an owner key delegated `evolve.*` to.
|
|
267
|
+
Anyone can verify it offline against the owner's public DID, with no shared
|
|
268
|
+
secret and no network call.
|
|
269
|
+
|
|
270
|
+
```bash
|
|
271
|
+
python zanii_evolve.py identity --out keys/ # owner.json (keep offline) + identity.json
|
|
272
|
+
export ZANII_EVOLVE_IDENTITY=keys/identity.json
|
|
273
|
+
python zanii_evolve.py --domain acme report --out cert
|
|
274
|
+
python zanii_evolve.py verify-report cert.json --owner did:key:z6Mk...
|
|
275
|
+
python zanii_evolve.py --domain acme verify-receipts # no issued certificate removed or altered
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Receipts are hash-linked in `runs/<domain>/receipts.jsonl`. Verification without
|
|
279
|
+
`--owner` still checks the signature but reports that the issuer is not pinned,
|
|
280
|
+
because a valid signature alone only proves that *some* key signed.
|
|
281
|
+
|
|
282
|
+
To optimize for cost instead of score, set `"objective": "cost"` in the domain's
|
|
283
|
+
`zanii_evolve.json` (or pass `--objective cost`). Admissibility is unchanged;
|
|
284
|
+
among the candidates that pass, the cheapest wins. Together with a new
|
|
285
|
+
`ZANII_EVOLVE_POLICY_MODEL` and a fresh `baseline`, this moves a harness to a
|
|
286
|
+
cheaper model without losing quality.
|
|
287
|
+
|
|
288
|
+
## Memory
|
|
289
|
+
|
|
290
|
+
The structural `memory` component is `mechanisms.Memory()` in each starting
|
|
291
|
+
harness. It is inert until the proposer wires it in, and it exposes the same
|
|
292
|
+
`write` / `read` / `digest` surface whichever backend runs:
|
|
293
|
+
|
|
294
|
+
| Backend | When | Storage |
|
|
295
|
+
|---|---|---|
|
|
296
|
+
| ZaniiDB | default when `zaniidb-agent-memory` is installed | `<state_dir>/zaniidb` (SQLite), hybrid BM25 and vector recall |
|
|
297
|
+
| JSONL | ZaniiDB missing, or `ZANII_EVOLVE_MEMORY=jsonl` | locked `*.jsonl` under `<state_dir>` |
|
|
298
|
+
|
|
299
|
+
Memory is scoped to one run on purpose (`ZANII_DATABASE_URL` is ignored), so
|
|
300
|
+
lessons cannot leak between runs. `mechanisms.TaskMemory` adds a
|
|
301
|
+
recall-before, record-after loop that stores successes as procedures and
|
|
302
|
+
failures as pitfalls. Brief harnesses ship it wired but switched off, so H0 is
|
|
303
|
+
unchanged and the search decides whether it pays.
|
|
304
|
+
|
|
305
|
+
**Lesson packs** are the explicit, opt-in exception. `lessons export` writes a
|
|
306
|
+
run's general lessons after the leakage guard, a task-id screen, PII redaction
|
|
307
|
+
and de-duplication. `lessons import` merges packs into
|
|
308
|
+
`domains/<name>/shared_lessons.jsonl`, which every trial of that domain then
|
|
309
|
+
starts from, H0 included. A pack switches the domain to the JSONL backend;
|
|
310
|
+
re-run `baseline` after importing one.
|
|
311
|
+
|
|
312
|
+
The [`skills/zaniidb`](https://github.com/vigilancetrent/zanii-evolve/tree/main/skills/zaniidb)
|
|
313
|
+
and [`skills/zanii`](https://github.com/vigilancetrent/zanii-evolve/tree/main/skills/zanii)
|
|
314
|
+
folders are agent skills that document the ZaniiDB and Zanii SDKs for coding assistants.
|
|
315
|
+
|
|
316
|
+
## Dashboard and HTTP API
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
python zanii_evolve.py serve # http://127.0.0.1:8765/
|
|
320
|
+
ZANII_EVOLVE_API_TOKEN=... python zanii_evolve.py serve # also enables control endpoints
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
The dashboard shows the incumbent against H0 and the noise band, every candidate
|
|
324
|
+
with its verdict and diff, the accepted edits and the stored lessons.
|
|
325
|
+
|
|
326
|
+
| Endpoint | Purpose |
|
|
327
|
+
|---|---|
|
|
328
|
+
| `GET /api/domains`, `GET /api/domains/<d>` | run summaries, trajectory, candidates |
|
|
329
|
+
| `GET /api/domains/<d>/rounds/<t>/<V>/diff` | a candidate's diff |
|
|
330
|
+
| `GET /api/domains/<d>/memory` | lessons in the run's stores |
|
|
331
|
+
| `GET /api/domains/<d>/pareto` | score/cost frontier |
|
|
332
|
+
| `GET /api/domains/<d>/report` | certificate (the server never signs) |
|
|
333
|
+
| `GET /api/registry` | published harnesses |
|
|
334
|
+
| `POST /api/domains/<d>/start`, `/stop` | run control (token required) |
|
|
335
|
+
| `POST /api/domains` | create a brief domain from a task list (token required, 2 MB limit) |
|
|
336
|
+
|
|
337
|
+
The server binds to loopback by default and checks the `Host` header. Binding
|
|
338
|
+
any other address requires `ZANII_EVOLVE_API_TOKEN`, which every API request
|
|
339
|
+
must then send as a bearer token, because run data includes diffs and traces.
|
|
340
|
+
|
|
341
|
+
## Configuration
|
|
342
|
+
|
|
343
|
+
Hyperparameters live in `domains/<name>/zanii_evolve.json` and can be overridden
|
|
344
|
+
on the command line (`--T`, `--k`, `--m`, `--delta`, `--beta0`, `--beta1`,
|
|
345
|
+
`--objective`, ...). A null `delta` is estimated by `calibrate`.
|
|
346
|
+
|
|
347
|
+
| Variable | Purpose |
|
|
348
|
+
|---|---|
|
|
349
|
+
| `ZANII_EVOLVE_SEARCH_MODEL` | proposer, analyst and critic model |
|
|
350
|
+
| `ZANII_EVOLVE_POLICY_MODEL`, `ZANII_EVOLVE_JUDGE_MODEL` | brief-domain agent and judge |
|
|
351
|
+
| `ZANII_EVOLVE_SIGNING_KEY` | HMAC key for certificates |
|
|
352
|
+
| `ZANII_EVOLVE_IDENTITY` | identity file for signed receipts |
|
|
353
|
+
| `ZANII_EVOLVE_REGISTRY` | registry location (default `./registry`) |
|
|
354
|
+
| `ZANII_EVOLVE_API_TOKEN` | API control and non-loopback binds |
|
|
355
|
+
| `ZANII_EVOLVE_MEMORY` | `zaniidb` (default) or `jsonl` |
|
|
356
|
+
|
|
357
|
+
Keep keys in the environment or a secrets manager. `runs/`, `registry/` and
|
|
358
|
+
identity files are ignored by git.
|
|
359
|
+
|
|
360
|
+
## Command reference
|
|
361
|
+
|
|
362
|
+
```
|
|
363
|
+
smoke | baseline | calibrate | round | run | status | readjudicate | reevaluate | heldout
|
|
364
|
+
pareto | report | verify-report | verify-receipts | publish | registry | identity
|
|
365
|
+
init-domain | import-tasks | ingest-traces | redteam | judge-agreement | lessons | serve
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
`python zanii_evolve.py --help` lists every option.
|
|
369
|
+
|
|
370
|
+
## Project layout
|
|
371
|
+
|
|
372
|
+
```
|
|
373
|
+
zanii_evolve/ search core: loop, proposer, critic, selection, providers,
|
|
374
|
+
brief engine, certificates, registry, API server, dashboard
|
|
375
|
+
zanii_evolve.py command line
|
|
376
|
+
domains/<name>/ one folder per domain: adapter or tasks, prompts, hyperparameters
|
|
377
|
+
third_party/ starting harnesses (Terminus-2, archipelago), under their own licenses
|
|
378
|
+
skills/ agent skills for the ZaniiDB and Zanii SDKs
|
|
379
|
+
tests/ offline test suite
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
## Acknowledgements
|
|
383
|
+
|
|
384
|
+
Zanii Evolve builds on the RRSI reference implementation (Apache 2.0); see
|
|
385
|
+
[NOTICE](https://github.com/vigilancetrent/zanii-evolve/blob/main/NOTICE). The
|
|
386
|
+
starting harnesses are the Terminus-2 agent from
|
|
387
|
+
[harbor](https://github.com/laude-institute/harbor) and the react_toolbelt agent
|
|
388
|
+
and runner from [archipelago](https://github.com/Mercor-Intelligence/archipelago).
|
|
389
|
+
Evaluation uses [Terminal-Bench](https://github.com/harbor-framework/terminal-bench),
|
|
390
|
+
[SWE-bench Verified](https://github.com/SWE-bench/SWE-bench),
|
|
391
|
+
[Harvey LAB](https://github.com/harveyai/harvey-labs),
|
|
392
|
+
[JobBench](https://github.com/Job-Bench/job-bench-eval),
|
|
393
|
+
[GDPval](https://openai.com/index/gdpval/),
|
|
394
|
+
[APEX-Agents](https://www.mercor.com/apex/apex-agents-leaderboard/),
|
|
395
|
+
[EngDesign](https://github.com/AGI4Engineering/EngDesign) and
|
|
396
|
+
[Frontier-Eng](https://github.com/Einsia/Frontier-Engineering).
|
|
397
|
+
|
|
398
|
+
## Contributing
|
|
399
|
+
|
|
400
|
+
See [CONTRIBUTING.md](https://github.com/vigilancetrent/zanii-evolve/blob/main/CONTRIBUTING.md).
|
|
401
|
+
Run `python -m pytest tests` before opening a pull request.
|
|
402
|
+
|
|
403
|
+
## License
|
|
404
|
+
|
|
405
|
+
Apache 2.0. See [LICENSE](https://github.com/vigilancetrent/zanii-evolve/blob/main/LICENSE)
|
|
406
|
+
and [NOTICE](https://github.com/vigilancetrent/zanii-evolve/blob/main/NOTICE). Code
|
|
407
|
+
under `third_party/` keeps its own license.
|