chatcortex 0.6.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chatcortex-0.6.0 → chatcortex-0.8.0}/PKG-INFO +48 -4
- {chatcortex-0.6.0 → chatcortex-0.8.0}/README.md +46 -1
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/__init__.py +8 -1
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/execution/executor.py +36 -3
- chatcortex-0.8.0/chatcortex/loop/__init__.py +10 -0
- chatcortex-0.8.0/chatcortex/loop/closed_loop.py +228 -0
- chatcortex-0.8.0/chatcortex/registry/cli.py +48 -0
- chatcortex-0.8.0/chatcortex/registry/package.py +167 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/telemetry/calibration.py +7 -3
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/PKG-INFO +48 -4
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/SOURCES.txt +7 -0
- chatcortex-0.8.0/chatcortex.egg-info/entry_points.txt +2 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/pyproject.toml +6 -4
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_calibration.py +10 -0
- chatcortex-0.8.0/tests/test_closed_loop.py +283 -0
- chatcortex-0.8.0/tests/test_execution.py +98 -0
- chatcortex-0.8.0/tests/test_registry_pack.py +138 -0
- chatcortex-0.6.0/tests/test_execution.py +0 -30
- {chatcortex-0.6.0 → chatcortex-0.8.0}/LICENSE +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/adapters/__init__.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/adapters/openai_backend.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/evaluation/approximation_metrics.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/evaluation/harness.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/graph/agent_graph.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/graph/node_ids.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/optimization/architecture_candidate.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/optimization/pareto.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/capability_registry.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/demo_components.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/loader.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/metadata.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/base.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/budget.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/engine.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/exhaustive_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/heuristic_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/random_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/task_specification.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/telemetry/logger.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/requires.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/top_level.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/setup.cfg +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_agent_graph.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_executor_backend.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_node_ids.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_openai_backend_install.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_pareto.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_registry.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_registry_loader.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_regression_gates.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_synthesizers.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_task_specification.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chatcortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
|
|
5
5
|
Author: Siddharth Saraswat
|
|
6
6
|
License: MIT
|
|
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
|
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
20
|
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
-
Requires-Python: >=3.
|
|
21
|
+
Requires-Python: >=3.10
|
|
23
22
|
Description-Content-Type: text/markdown
|
|
24
23
|
License-File: LICENSE
|
|
25
24
|
Requires-Dist: networkx>=2.8
|
|
@@ -37,7 +36,7 @@ Dynamic: license-file
|
|
|
37
36
|
|
|
38
37
|
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
39
38
|
[](https://pypi.org/project/chatcortex/)
|
|
40
|
-
[](https://www.python.org/)
|
|
41
40
|
[](LICENSE)
|
|
42
41
|
|
|
43
42
|
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
@@ -48,6 +47,17 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
48
47
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
49
48
|
```
|
|
50
49
|
|
|
50
|
+
**ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
|
|
51
|
+
|
|
52
|
+
| Component | Field | Declared | Calibrated | Truth |
|
|
53
|
+
|---|---|---|---|---|
|
|
54
|
+
| `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
|
|
55
|
+
| `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
|
|
56
|
+
| `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
|
|
57
|
+
| `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
|
|
58
|
+
|
|
59
|
+
> **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
|
|
60
|
+
|
|
51
61
|
## Installation
|
|
52
62
|
|
|
53
63
|
```bash
|
|
@@ -106,10 +116,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
|
106
116
|
python examples/registry_from_json.py
|
|
107
117
|
```
|
|
108
118
|
|
|
119
|
+
Run the flagship closed loop — synthesize → execute → calibrate →
|
|
120
|
+
re-synthesize, four iterations of measured behavior flowing back into the
|
|
121
|
+
registry:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
python examples/closed_loop.py
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
With an OpenAI key you can run the same loop against real backends:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
pip install "chatcortex[real]"
|
|
131
|
+
export OPENAI_API_KEY=... # every reply spends your credits
|
|
132
|
+
python examples/closed_loop_real.py
|
|
133
|
+
```
|
|
134
|
+
|
|
109
135
|
To search over your own components, build a `CapabilityRegistry` and register
|
|
110
136
|
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
111
137
|
`examples/` for complete walkthroughs.
|
|
112
138
|
|
|
139
|
+
## Registry packs
|
|
140
|
+
|
|
141
|
+
A registry ships as a JSON pack in the loader's document shape. Saving it
|
|
142
|
+
proves the round-trip, and one command validates any pack offline:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from chatcortex import build_demo_registry, save_registry, validate_registry_file
|
|
146
|
+
|
|
147
|
+
save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
|
|
148
|
+
print(validate_registry_file("registry.json").valid) # True
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
chatcortex-registry validate registry.json
|
|
153
|
+
# components validated: 12
|
|
154
|
+
# result: VALID (exit 0; issues and duplicates are named when invalid)
|
|
155
|
+
```
|
|
156
|
+
|
|
113
157
|
## How It Works
|
|
114
158
|
|
|
115
159
|
ChatCortex is organized as a layered architecture synthesis system:
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
4
4
|
[](https://pypi.org/project/chatcortex/)
|
|
5
|
-
[](https://www.python.org/)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
|
|
8
8
|
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
@@ -13,6 +13,17 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
13
13
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
+
**ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
|
|
17
|
+
|
|
18
|
+
| Component | Field | Declared | Calibrated | Truth |
|
|
19
|
+
|---|---|---|---|---|
|
|
20
|
+
| `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
|
|
21
|
+
| `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
|
|
22
|
+
| `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
|
|
23
|
+
| `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
|
|
24
|
+
|
|
25
|
+
> **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
|
|
26
|
+
|
|
16
27
|
## Installation
|
|
17
28
|
|
|
18
29
|
```bash
|
|
@@ -71,10 +82,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
|
71
82
|
python examples/registry_from_json.py
|
|
72
83
|
```
|
|
73
84
|
|
|
85
|
+
Run the flagship closed loop — synthesize → execute → calibrate →
|
|
86
|
+
re-synthesize, four iterations of measured behavior flowing back into the
|
|
87
|
+
registry:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
python examples/closed_loop.py
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
With an OpenAI key you can run the same loop against real backends:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
pip install "chatcortex[real]"
|
|
97
|
+
export OPENAI_API_KEY=... # every reply spends your credits
|
|
98
|
+
python examples/closed_loop_real.py
|
|
99
|
+
```
|
|
100
|
+
|
|
74
101
|
To search over your own components, build a `CapabilityRegistry` and register
|
|
75
102
|
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
76
103
|
`examples/` for complete walkthroughs.
|
|
77
104
|
|
|
105
|
+
## Registry packs
|
|
106
|
+
|
|
107
|
+
A registry ships as a JSON pack in the loader's document shape. Saving it
|
|
108
|
+
proves the round-trip, and one command validates any pack offline:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from chatcortex import build_demo_registry, save_registry, validate_registry_file
|
|
112
|
+
|
|
113
|
+
save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
|
|
114
|
+
print(validate_registry_file("registry.json").valid) # True
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
chatcortex-registry validate registry.json
|
|
119
|
+
# components validated: 12
|
|
120
|
+
# result: VALID (exit 0; issues and duplicates are named when invalid)
|
|
121
|
+
```
|
|
122
|
+
|
|
78
123
|
## How It Works
|
|
79
124
|
|
|
80
125
|
ChatCortex is organized as a layered architecture synthesis system:
|
|
@@ -13,10 +13,12 @@ Core capabilities include:
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
from .graph.agent_graph import AgentGraph
|
|
16
|
+
from .loop import LoopIteration, run_closed_loop
|
|
16
17
|
from .registry.capability_registry import CapabilityRegistry
|
|
17
18
|
from .registry.demo_components import build_demo_registry
|
|
18
19
|
from .registry.loader import load_registry
|
|
19
20
|
from .registry.metadata import ComponentMetadata
|
|
21
|
+
from .registry.package import RegistryReport, save_registry, validate_registry_file
|
|
20
22
|
from .synthesis.beam_synthesizer import BeamSynthesizer
|
|
21
23
|
from .synthesis.budget import SynthesisBudget
|
|
22
24
|
from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
@@ -39,9 +41,14 @@ __all__ = [
|
|
|
39
41
|
"SynthesisBudget",
|
|
40
42
|
"AgentGraph",
|
|
41
43
|
"CapabilityRegistry",
|
|
44
|
+
"LoopIteration",
|
|
42
45
|
"ComponentMetadata",
|
|
43
46
|
"build_demo_registry",
|
|
44
47
|
"load_registry",
|
|
48
|
+
"run_closed_loop",
|
|
49
|
+
"save_registry",
|
|
50
|
+
"validate_registry_file",
|
|
51
|
+
"RegistryReport",
|
|
45
52
|
]
|
|
46
53
|
|
|
47
|
-
__version__ = "0.
|
|
54
|
+
__version__ = "0.8.0"
|
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from typing import Any, Literal, Optional, Protocol
|
|
4
4
|
|
|
5
5
|
from chatcortex.graph.agent_graph import AgentGraph
|
|
6
|
+
from chatcortex.registry.capability_registry import CapabilityRegistry
|
|
6
7
|
from chatcortex.registry.metadata import ComponentMetadata
|
|
7
8
|
from chatcortex.telemetry.logger import TelemetryLogger
|
|
8
9
|
|
|
@@ -68,15 +69,34 @@ class AgentExecutor:
|
|
|
68
69
|
The chain stops on the first failure, exactly like the simulator.
|
|
69
70
|
A backend exception is recorded as a failed probe (zero cost) and
|
|
70
71
|
also stops the chain, so partial telemetry survives the error.
|
|
72
|
+
|
|
73
|
+
Hidden ground truth: by default the simulator draws outcomes from the
|
|
74
|
+
metadata carried by the graph, i.e. from the *declared* values, so a
|
|
75
|
+
calibration loop fed by it can only rediscover its own priors plus
|
|
76
|
+
sampling noise. Pass ``ground_truth`` (a registry holding each
|
|
77
|
+
component's *true* cost, latency and reliability) and the simulator
|
|
78
|
+
draws from those instead, looked up by component name. The synthesizer
|
|
79
|
+
and calibration never see that registry, which makes the simulator a
|
|
80
|
+
real test of whether a loop converges on values it was not told.
|
|
81
|
+
Components absent from ``ground_truth`` behave as declared.
|
|
82
|
+
``ground_truth`` cannot be combined with ``backend``: a backend
|
|
83
|
+
already is the source of truth.
|
|
71
84
|
"""
|
|
72
85
|
def __init__(
|
|
73
86
|
self,
|
|
74
87
|
mode: ExecutionMode = "deterministic",
|
|
75
88
|
seed: Optional[int] = None,
|
|
76
89
|
backend: Optional[ExecutionProtocol] = None,
|
|
90
|
+
ground_truth: Optional[CapabilityRegistry] = None,
|
|
77
91
|
):
|
|
92
|
+
if backend is not None and ground_truth is not None:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
"ground_truth only applies to the simulator; a backend "
|
|
95
|
+
"is its own source of truth, so pass one or the other"
|
|
96
|
+
)
|
|
78
97
|
self.mode = mode
|
|
79
98
|
self.backend = backend
|
|
99
|
+
self.ground_truth = ground_truth
|
|
80
100
|
# local random generator - NO RNG pollution, no side-effects, clean isolation
|
|
81
101
|
self.random = random.Random(seed)
|
|
82
102
|
|
|
@@ -115,8 +135,12 @@ class AgentExecutor:
|
|
|
115
135
|
probe = self._invoke_backend(metadata, node_input)
|
|
116
136
|
return probe.latency_ms, probe.cost, probe.success, probe.output
|
|
117
137
|
|
|
138
|
+
# What the component really does: the hidden truth when one is
|
|
139
|
+
# supplied, otherwise the declared metadata itself.
|
|
140
|
+
actual = self._true_metadata(metadata)
|
|
141
|
+
|
|
118
142
|
# Simulated latency
|
|
119
|
-
simulated_latency =
|
|
143
|
+
simulated_latency = actual.avg_latency_ms
|
|
120
144
|
|
|
121
145
|
# Optional: actually sleep (disable if too slow)
|
|
122
146
|
# time.sleep(simulated_latency / 1000)
|
|
@@ -125,11 +149,20 @@ class AgentExecutor:
|
|
|
125
149
|
if self.mode == "deterministic":
|
|
126
150
|
success = True
|
|
127
151
|
elif self.mode == "probabilistic":
|
|
128
|
-
success = self.random.random() <=
|
|
152
|
+
success = self.random.random() <= actual.reliability_score
|
|
129
153
|
else:
|
|
130
154
|
raise ValueError("Invalid execution mode")
|
|
131
155
|
|
|
132
|
-
return simulated_latency,
|
|
156
|
+
return simulated_latency, actual.cost_per_call, success, None
|
|
157
|
+
|
|
158
|
+
def _true_metadata(self, declared: ComponentMetadata) -> ComponentMetadata:
|
|
159
|
+
"""The metadata the simulator samples from for one component."""
|
|
160
|
+
if self.ground_truth is None:
|
|
161
|
+
return declared
|
|
162
|
+
try:
|
|
163
|
+
return self.ground_truth.get(declared.name)
|
|
164
|
+
except KeyError:
|
|
165
|
+
return declared
|
|
133
166
|
|
|
134
167
|
def _invoke_backend(
|
|
135
168
|
self, metadata: ComponentMetadata, node_input: Any
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""The tradeoff-driven closed loop: synthesize -> execute -> calibrate.
|
|
2
|
+
|
|
3
|
+
Re-exports the loop entry points so users can start from the package::
|
|
4
|
+
|
|
5
|
+
from chatcortex.loop import run_closed_loop
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .closed_loop import LoopIteration, run_closed_loop
|
|
9
|
+
|
|
10
|
+
__all__ = ["LoopIteration", "run_closed_loop"]
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The closed loop: synthesize -> execute -> calibrate -> re-synthesize.
|
|
2
|
+
|
|
3
|
+
:func:`run_closed_loop` glues three already-shipped mechanisms into one
|
|
4
|
+
iterative drive: a synthesizer maps the *current* registry to a Pareto
|
|
5
|
+
frontier, an executor runs the frontier's architectures and produces
|
|
6
|
+
telemetry records, and :func:`chatcortex.telemetry.calibration.calibrate`
|
|
7
|
+
blends the measured success/latency/cost back into the registry — so the
|
|
8
|
+
next iteration synthesizes against a registry that remembers what actually
|
|
9
|
+
happened, not only what was declared.
|
|
10
|
+
|
|
11
|
+
Determinism: given a deterministic-or-seeded synthesizer, a seeded (or
|
|
12
|
+
deterministic) executor, and seeded synthesis budgets, two calls with the
|
|
13
|
+
same inputs produce iteration-for-iteration identical records. The seeded
|
|
14
|
+
executor's RNG advances across iterations, which is still deterministic —
|
|
15
|
+
it is state, not scheduling.
|
|
16
|
+
|
|
17
|
+
Component mapping: telemetry records carry graph *node ids*;
|
|
18
|
+
``calibrate`` resolves them through
|
|
19
|
+
:func:`chatcortex.graph.node_ids.component_name_from_node_id` and silently
|
|
20
|
+
ignores names it cannot match, so traces from graphs built from the
|
|
21
|
+
previous registry still calibrate the components they were produced from.
|
|
22
|
+
|
|
23
|
+
Calibration feedback is *iterative*: each iteration blends only the
|
|
24
|
+
latest traces into the previous iteration's calibration, so priors
|
|
25
|
+
compound toward empirical values across iterations at roughly
|
|
26
|
+
``1 - (1 - alpha) ** k`` after ``k`` iterations.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import math
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
from typing import Any, Callable, Dict, List, Optional, Union
|
|
32
|
+
|
|
33
|
+
from chatcortex.execution.executor import AgentExecutor
|
|
34
|
+
from chatcortex.registry.capability_registry import CapabilityRegistry
|
|
35
|
+
from chatcortex.synthesis.base import Synthesizer
|
|
36
|
+
from chatcortex.synthesis.budget import SynthesisBudget
|
|
37
|
+
from chatcortex.synthesis.task_specification import TaskSpecification
|
|
38
|
+
from chatcortex.telemetry.calibration import calibrate
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class LoopIteration:
|
|
43
|
+
"""Everything one loop iteration observed, kept and serializable.
|
|
44
|
+
|
|
45
|
+
``registry`` holds the *calibrated* registry this iteration produced
|
|
46
|
+
(the next iteration's synthesis input and the value users typically
|
|
47
|
+
keep); ``to_dict()`` embeds it as the loader-shaped
|
|
48
|
+
``registry_snapshot`` so a full loop can be checkpointed to JSON.
|
|
49
|
+
``traces`` are the raw telemetry records behind the calibration.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
iteration: int
|
|
53
|
+
frontier_size: int
|
|
54
|
+
frontier_objectives: List[Dict[str, float]]
|
|
55
|
+
executed_candidates: int
|
|
56
|
+
trace_count: int
|
|
57
|
+
updated_components: List[str]
|
|
58
|
+
traces: List[Dict[str, Any]]
|
|
59
|
+
registry: CapabilityRegistry
|
|
60
|
+
prior_registry: CapabilityRegistry
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict:
|
|
63
|
+
"""JSON-safe record including the calibrated registry snapshot."""
|
|
64
|
+
return {
|
|
65
|
+
"iteration": self.iteration,
|
|
66
|
+
"frontier_size": self.frontier_size,
|
|
67
|
+
"frontier_objectives": self.frontier_objectives,
|
|
68
|
+
"executed_candidates": self.executed_candidates,
|
|
69
|
+
"trace_count": self.trace_count,
|
|
70
|
+
"updated_components": list(self.updated_components),
|
|
71
|
+
"traces": [dict(trace) for trace in self.traces],
|
|
72
|
+
"registry_snapshot": self.registry.to_dict(),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _make_synthesizer(
|
|
77
|
+
synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
|
|
78
|
+
registry: CapabilityRegistry,
|
|
79
|
+
) -> Synthesizer:
|
|
80
|
+
"""Instantiate the synthesis strategy against the *current* registry.
|
|
81
|
+
|
|
82
|
+
Accepts a synthesizer *class* (constructed with its defaults) or a
|
|
83
|
+
one-argument factory such as
|
|
84
|
+
``lambda reg: BeamSynthesizer(reg, beam_width=5)``. A pre-built
|
|
85
|
+
instance is rejected: its constructor bound it to the previous
|
|
86
|
+
registry, so reusing it would silently synthesize against stale
|
|
87
|
+
metadata and the loop would learn nothing.
|
|
88
|
+
"""
|
|
89
|
+
if isinstance(synthesizer, type) and issubclass(synthesizer, Synthesizer):
|
|
90
|
+
return synthesizer(registry)
|
|
91
|
+
if callable(synthesizer):
|
|
92
|
+
return synthesizer(registry)
|
|
93
|
+
raise TypeError(
|
|
94
|
+
"synthesizer must be a Synthesizer subclass or a "
|
|
95
|
+
f"factory(registry) callable, {synthesizer!r} given; passing a "
|
|
96
|
+
"pre-built instance would bind the loop to stale metadata"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def run_closed_loop(
|
|
101
|
+
registry: CapabilityRegistry,
|
|
102
|
+
task: TaskSpecification,
|
|
103
|
+
synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
|
|
104
|
+
executor: AgentExecutor,
|
|
105
|
+
iterations: int = 3,
|
|
106
|
+
alpha: float = 0.3,
|
|
107
|
+
budget: Optional[SynthesisBudget] = None,
|
|
108
|
+
candidates_per_iteration: Optional[int] = None,
|
|
109
|
+
) -> List[LoopIteration]:
|
|
110
|
+
"""Run the synthesize -> execute -> calibrate cycle ``iterations`` times.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
registry: The declared (prior) registry; iteration 0 synthesizes
|
|
114
|
+
against it unchanged.
|
|
115
|
+
task: The synthesis problem, as for any synthesizer.
|
|
116
|
+
synthesizer: A :class:`~chatcortex.synthesis.base.Synthesizer`
|
|
117
|
+
subclass or ``factory(registry) -> Synthesizer`` callable. It
|
|
118
|
+
is instantiated fresh against the *current* registry every
|
|
119
|
+
iteration, because calibrate returns a new registry.
|
|
120
|
+
executor: A shared :class:`AgentExecutor` instance (simulated or
|
|
121
|
+
backend-backed); its seeded RNG advances across iterations,
|
|
122
|
+
which keeps the whole run deterministic.
|
|
123
|
+
iterations: How many loop rounds to run (>= 1).
|
|
124
|
+
alpha: Blend weight passed to :func:`calibrate`; validated, so
|
|
125
|
+
every round only executes under a valid blend. Non-finite
|
|
126
|
+
values are rejected like any out-of-range value.
|
|
127
|
+
candidates_per_iteration: Execute only the first ``N`` frontier
|
|
128
|
+
members (objective-sorted, deterministic) instead of all of
|
|
129
|
+
them; useful when executions are expensive. Must be ``>= 1``
|
|
130
|
+
when given, and is validated before anything executes.
|
|
131
|
+
budget: Optional per-iteration :class:`SynthesisBudget`; it is a
|
|
132
|
+
frozen dataclass, so the same constraints apply every round
|
|
133
|
+
without state leaking between iterations.
|
|
134
|
+
candidates_per_iteration: Execute only the first ``N`` frontier
|
|
135
|
+
members (objective-sorted, deterministic) instead of all of
|
|
136
|
+
them; useful when executions are expensive.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
One :class:`LoopIteration` per round; iteration ``k``'s
|
|
140
|
+
``frontier_size``/``frontier_objectives`` describe the frontier
|
|
141
|
+
synthesized from iteration ``k-1``'s calibrated registry, and
|
|
142
|
+
``registry`` is the registry calibrated *during* round ``k``.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
ValueError: If ``iterations < 1``, ``alpha`` is outside ``[0, 1]``,
|
|
146
|
+
or ``candidates_per_iteration`` is given below 1 — all before
|
|
147
|
+
any round runs — or if a round synthesizes an empty frontier
|
|
148
|
+
(nothing to execute would stall the loop).
|
|
149
|
+
TypeError: If ``synthesizer`` is neither a Synthesizer subclass
|
|
150
|
+
nor a factory callable.
|
|
151
|
+
"""
|
|
152
|
+
if iterations < 1:
|
|
153
|
+
raise ValueError(f"iterations must be >= 1, {iterations} given")
|
|
154
|
+
if candidates_per_iteration is not None and candidates_per_iteration < 1:
|
|
155
|
+
raise ValueError(
|
|
156
|
+
"candidates_per_iteration must be >= 1 when given, "
|
|
157
|
+
f"{candidates_per_iteration} given"
|
|
158
|
+
)
|
|
159
|
+
if not math.isfinite(alpha) or alpha < 0.0 or alpha > 1.0:
|
|
160
|
+
raise ValueError(f"alpha must be in [0.0, 1.0], {alpha!r} given")
|
|
161
|
+
|
|
162
|
+
records: List[LoopIteration] = []
|
|
163
|
+
current = registry
|
|
164
|
+
for iteration in range(iterations):
|
|
165
|
+
engine = _make_synthesizer(synthesizer, current)
|
|
166
|
+
frontier = engine.synthesize(task, budget)
|
|
167
|
+
if not frontier:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"iteration {iteration}: the synthesizer returned an "
|
|
170
|
+
"empty frontier; the task may be infeasible under the "
|
|
171
|
+
"current registry and constraints"
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
selected = frontier
|
|
175
|
+
if candidates_per_iteration is not None:
|
|
176
|
+
selected = sorted(
|
|
177
|
+
frontier,
|
|
178
|
+
key=lambda c: (c.total_cost, c.total_latency, -c.total_reliability),
|
|
179
|
+
)[:candidates_per_iteration]
|
|
180
|
+
|
|
181
|
+
traces: List[Dict[str, Any]] = []
|
|
182
|
+
for candidate in selected:
|
|
183
|
+
traces.extend(
|
|
184
|
+
executor.execute(candidate.graph).telemetry.records
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
calibrated = calibrate(current, traces, alpha=alpha)
|
|
188
|
+
updated = [
|
|
189
|
+
prior.name
|
|
190
|
+
for prior in current.list_all()
|
|
191
|
+
if _differs(calibrated.get(prior.name), prior)
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
records.append(
|
|
195
|
+
LoopIteration(
|
|
196
|
+
iteration=iteration,
|
|
197
|
+
frontier_size=len(frontier),
|
|
198
|
+
frontier_objectives=[
|
|
199
|
+
{
|
|
200
|
+
"cost": c.total_cost,
|
|
201
|
+
"latency": c.total_latency,
|
|
202
|
+
"reliability": c.total_reliability,
|
|
203
|
+
}
|
|
204
|
+
for c in frontier
|
|
205
|
+
],
|
|
206
|
+
executed_candidates=len(selected),
|
|
207
|
+
trace_count=len(traces),
|
|
208
|
+
updated_components=updated,
|
|
209
|
+
traces=[dict(trace) for trace in traces],
|
|
210
|
+
registry=calibrated,
|
|
211
|
+
prior_registry=current,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
current = calibrated
|
|
215
|
+
return records
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _differs(calibrated, prior) -> bool:
|
|
219
|
+
"""True when calibration changed any performance field.
|
|
220
|
+
|
|
221
|
+
Exact comparison is safe: ``blend == prior`` exactly when the
|
|
222
|
+
empirical value equals the prior, with no rounding involved.
|
|
223
|
+
"""
|
|
224
|
+
return (
|
|
225
|
+
calibrated.reliability_score != prior.reliability_score
|
|
226
|
+
or calibrated.avg_latency_ms != prior.avg_latency_ms
|
|
227
|
+
or calibrated.cost_per_call != prior.cost_per_call
|
|
228
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Console script for the registry pack: ``chatcortex-registry validate``.
|
|
2
|
+
|
|
3
|
+
``main`` is the ``[project.scripts]`` entry point. It never raises —
|
|
4
|
+
validation results come back as a :class:`RegistryReport` — and exits 0
|
|
5
|
+
only when the file is a fully valid pack, so shell checks can gate on it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
from .package import RegistryReport, validate_registry_file
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _render_report(report: RegistryReport) -> List[str]:
|
|
15
|
+
"""Human-readable lines; issue/duplicate order follows the report."""
|
|
16
|
+
lines = [f"components validated: {report.component_count}"]
|
|
17
|
+
for duplicate in report.duplicate_names:
|
|
18
|
+
lines.append(f"duplicate component: '{duplicate}'")
|
|
19
|
+
lines.extend(f"error: {issue}" for issue in report.issues)
|
|
20
|
+
lines.append(f"result: {'VALID' if report.valid else 'INVALID'}")
|
|
21
|
+
return lines
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
25
|
+
"""Entry point returning the process exit code (0 valid, 1 invalid)."""
|
|
26
|
+
parser = argparse.ArgumentParser(
|
|
27
|
+
prog="chatcortex-registry",
|
|
28
|
+
description="Registry pack tooling for ChatCortex.",
|
|
29
|
+
)
|
|
30
|
+
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
31
|
+
|
|
32
|
+
validate = subcommands.add_parser(
|
|
33
|
+
"validate",
|
|
34
|
+
help="Validate a registry JSON pack and print a structured report.",
|
|
35
|
+
)
|
|
36
|
+
validate.add_argument("file", help="Path to the registry JSON file.")
|
|
37
|
+
|
|
38
|
+
args = parser.parse_args(argv)
|
|
39
|
+
report = validate_registry_file(args.file)
|
|
40
|
+
|
|
41
|
+
print(f"chatcortex-registry validate {args.file}")
|
|
42
|
+
for line in _render_report(report):
|
|
43
|
+
print(f" {line}")
|
|
44
|
+
return 0 if report.valid else 1
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__":
|
|
48
|
+
raise SystemExit(main())
|