chatcortex 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chatcortex-0.6.0 → chatcortex-0.7.0}/PKG-INFO +37 -1
- {chatcortex-0.6.0 → chatcortex-0.7.0}/README.md +36 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/__init__.py +8 -1
- chatcortex-0.7.0/chatcortex/loop/__init__.py +10 -0
- chatcortex-0.7.0/chatcortex/loop/closed_loop.py +228 -0
- chatcortex-0.7.0/chatcortex/registry/cli.py +48 -0
- chatcortex-0.7.0/chatcortex/registry/package.py +167 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/telemetry/calibration.py +7 -3
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex.egg-info/PKG-INFO +37 -1
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex.egg-info/SOURCES.txt +7 -0
- chatcortex-0.7.0/chatcortex.egg-info/entry_points.txt +2 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/pyproject.toml +4 -1
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_calibration.py +10 -0
- chatcortex-0.7.0/tests/test_closed_loop.py +283 -0
- chatcortex-0.7.0/tests/test_registry_pack.py +138 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/LICENSE +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/adapters/__init__.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/adapters/openai_backend.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/evaluation/approximation_metrics.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/evaluation/harness.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/execution/executor.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/graph/agent_graph.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/graph/node_ids.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/optimization/architecture_candidate.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/optimization/pareto.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/registry/capability_registry.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/registry/demo_components.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/registry/loader.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/registry/metadata.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/base.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/budget.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/engine.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/exhaustive_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/heuristic_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/random_synthesizer.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/synthesis/task_specification.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex/telemetry/logger.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex.egg-info/requires.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/chatcortex.egg-info/top_level.txt +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/setup.cfg +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_agent_graph.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_execution.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_executor_backend.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_node_ids.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_openai_backend_install.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_pareto.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_registry.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_registry_loader.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_regression_gates.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_synthesizers.py +0 -0
- {chatcortex-0.6.0 → chatcortex-0.7.0}/tests/test_task_specification.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chatcortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
|
|
5
5
|
Author: Siddharth Saraswat
|
|
6
6
|
License: MIT
|
|
@@ -48,6 +48,8 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
48
48
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
+
**ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
|
|
52
|
+
|
|
51
53
|
## Installation
|
|
52
54
|
|
|
53
55
|
```bash
|
|
@@ -106,10 +108,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
|
106
108
|
python examples/registry_from_json.py
|
|
107
109
|
```
|
|
108
110
|
|
|
111
|
+
Run the flagship closed loop — synthesize → execute → calibrate →
|
|
112
|
+
re-synthesize, four iterations of measured behavior flowing back into the
|
|
113
|
+
registry:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
python examples/closed_loop.py
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
With an OpenAI key you can run the same loop against real backends:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install "chatcortex[real]"
|
|
123
|
+
export OPENAI_API_KEY=... # every reply spends your credits
|
|
124
|
+
python examples/closed_loop_real.py
|
|
125
|
+
```
|
|
126
|
+
|
|
109
127
|
To search over your own components, build a `CapabilityRegistry` and register
|
|
110
128
|
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
111
129
|
`examples/` for complete walkthroughs.
|
|
112
130
|
|
|
131
|
+
## Registry packs
|
|
132
|
+
|
|
133
|
+
A registry ships as a JSON pack in the loader's document shape. Saving it
|
|
134
|
+
proves the round-trip, and one command validates any pack offline:
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from chatcortex import build_demo_registry, save_registry, validate_registry_file
|
|
138
|
+
|
|
139
|
+
save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
|
|
140
|
+
print(validate_registry_file("registry.json").valid) # True
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
chatcortex-registry validate registry.json
|
|
145
|
+
# components validated: 12
|
|
146
|
+
# result: VALID (exit 0; issues and duplicates are named when invalid)
|
|
147
|
+
```
|
|
148
|
+
|
|
113
149
|
## How It Works
|
|
114
150
|
|
|
115
151
|
ChatCortex is organized as a layered architecture synthesis system:
|
|
@@ -13,6 +13,8 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
13
13
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
+
**ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
|
|
17
|
+
|
|
16
18
|
## Installation
|
|
17
19
|
|
|
18
20
|
```bash
|
|
@@ -71,10 +73,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
|
71
73
|
python examples/registry_from_json.py
|
|
72
74
|
```
|
|
73
75
|
|
|
76
|
+
Run the flagship closed loop — synthesize → execute → calibrate →
|
|
77
|
+
re-synthesize, four iterations of measured behavior flowing back into the
|
|
78
|
+
registry:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python examples/closed_loop.py
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
With an OpenAI key you can run the same loop against real backends:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install "chatcortex[real]"
|
|
88
|
+
export OPENAI_API_KEY=... # every reply spends your credits
|
|
89
|
+
python examples/closed_loop_real.py
|
|
90
|
+
```
|
|
91
|
+
|
|
74
92
|
To search over your own components, build a `CapabilityRegistry` and register
|
|
75
93
|
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
76
94
|
`examples/` for complete walkthroughs.
|
|
77
95
|
|
|
96
|
+
## Registry packs
|
|
97
|
+
|
|
98
|
+
A registry ships as a JSON pack in the loader's document shape. Saving it
|
|
99
|
+
proves the round-trip, and one command validates any pack offline:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from chatcortex import build_demo_registry, save_registry, validate_registry_file
|
|
103
|
+
|
|
104
|
+
save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
|
|
105
|
+
print(validate_registry_file("registry.json").valid) # True
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
chatcortex-registry validate registry.json
|
|
110
|
+
# components validated: 12
|
|
111
|
+
# result: VALID (exit 0; issues and duplicates are named when invalid)
|
|
112
|
+
```
|
|
113
|
+
|
|
78
114
|
## How It Works
|
|
79
115
|
|
|
80
116
|
ChatCortex is organized as a layered architecture synthesis system:
|
|
@@ -13,10 +13,12 @@ Core capabilities include:
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
from .graph.agent_graph import AgentGraph
|
|
16
|
+
from .loop import LoopIteration, run_closed_loop
|
|
16
17
|
from .registry.capability_registry import CapabilityRegistry
|
|
17
18
|
from .registry.demo_components import build_demo_registry
|
|
18
19
|
from .registry.loader import load_registry
|
|
19
20
|
from .registry.metadata import ComponentMetadata
|
|
21
|
+
from .registry.package import RegistryReport, save_registry, validate_registry_file
|
|
20
22
|
from .synthesis.beam_synthesizer import BeamSynthesizer
|
|
21
23
|
from .synthesis.budget import SynthesisBudget
|
|
22
24
|
from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
@@ -39,9 +41,14 @@ __all__ = [
|
|
|
39
41
|
"SynthesisBudget",
|
|
40
42
|
"AgentGraph",
|
|
41
43
|
"CapabilityRegistry",
|
|
44
|
+
"LoopIteration",
|
|
42
45
|
"ComponentMetadata",
|
|
43
46
|
"build_demo_registry",
|
|
44
47
|
"load_registry",
|
|
48
|
+
"run_closed_loop",
|
|
49
|
+
"save_registry",
|
|
50
|
+
"validate_registry_file",
|
|
51
|
+
"RegistryReport",
|
|
45
52
|
]
|
|
46
53
|
|
|
47
|
-
__version__ = "0.
|
|
54
|
+
__version__ = "0.7.0"
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""The tradeoff-driven closed loop: synthesize -> execute -> calibrate.
|
|
2
|
+
|
|
3
|
+
Re-exports the loop entry points so users can start from the package::
|
|
4
|
+
|
|
5
|
+
from chatcortex.loop import run_closed_loop
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .closed_loop import LoopIteration, run_closed_loop
|
|
9
|
+
|
|
10
|
+
__all__ = ["LoopIteration", "run_closed_loop"]
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The closed loop: synthesize -> execute -> calibrate -> re-synthesize.
|
|
2
|
+
|
|
3
|
+
:func:`run_closed_loop` glues three already-shipped mechanisms into one
|
|
4
|
+
iterative drive: a synthesizer maps the *current* registry to a Pareto
|
|
5
|
+
frontier, an executor runs the frontier's architectures and produces
|
|
6
|
+
telemetry records, and :func:`chatcortex.telemetry.calibration.calibrate`
|
|
7
|
+
blends the measured success/latency/cost back into the registry — so the
|
|
8
|
+
next iteration synthesizes against a registry that remembers what actually
|
|
9
|
+
happened, not only what was declared.
|
|
10
|
+
|
|
11
|
+
Determinism: given a deterministic-or-seeded synthesizer, a seeded (or
|
|
12
|
+
deterministic) executor, and seeded synthesis budgets, two calls with the
|
|
13
|
+
same inputs produce iteration-for-iteration identical records. The seeded
|
|
14
|
+
executor's RNG advances across iterations, which is still deterministic —
|
|
15
|
+
it is state, not scheduling.
|
|
16
|
+
|
|
17
|
+
Component mapping: telemetry records carry graph *node ids*;
|
|
18
|
+
``calibrate`` resolves them through
|
|
19
|
+
:func:`chatcortex.graph.node_ids.component_name_from_node_id` and silently
|
|
20
|
+
ignores names it cannot match, so traces from graphs built from the
|
|
21
|
+
previous registry still calibrate the components they were produced from.
|
|
22
|
+
|
|
23
|
+
Calibration feedback is *iterative*: each iteration blends only the
|
|
24
|
+
latest traces into the previous iteration's calibration, so priors
|
|
25
|
+
compound toward empirical values across iterations at roughly
|
|
26
|
+
``1 - (1 - alpha) ** k`` after ``k`` iterations.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import math
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
from typing import Any, Callable, Dict, List, Optional, Union
|
|
32
|
+
|
|
33
|
+
from chatcortex.execution.executor import AgentExecutor
|
|
34
|
+
from chatcortex.registry.capability_registry import CapabilityRegistry
|
|
35
|
+
from chatcortex.synthesis.base import Synthesizer
|
|
36
|
+
from chatcortex.synthesis.budget import SynthesisBudget
|
|
37
|
+
from chatcortex.synthesis.task_specification import TaskSpecification
|
|
38
|
+
from chatcortex.telemetry.calibration import calibrate
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class LoopIteration:
|
|
43
|
+
"""Everything one loop iteration observed, kept and serializable.
|
|
44
|
+
|
|
45
|
+
``registry`` holds the *calibrated* registry this iteration produced
|
|
46
|
+
(the next iteration's synthesis input and the value users typically
|
|
47
|
+
keep); ``to_dict()`` embeds it as the loader-shaped
|
|
48
|
+
``registry_snapshot`` so a full loop can be checkpointed to JSON.
|
|
49
|
+
``traces`` are the raw telemetry records behind the calibration.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
iteration: int
|
|
53
|
+
frontier_size: int
|
|
54
|
+
frontier_objectives: List[Dict[str, float]]
|
|
55
|
+
executed_candidates: int
|
|
56
|
+
trace_count: int
|
|
57
|
+
updated_components: List[str]
|
|
58
|
+
traces: List[Dict[str, Any]]
|
|
59
|
+
registry: CapabilityRegistry
|
|
60
|
+
prior_registry: CapabilityRegistry
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict:
|
|
63
|
+
"""JSON-safe record including the calibrated registry snapshot."""
|
|
64
|
+
return {
|
|
65
|
+
"iteration": self.iteration,
|
|
66
|
+
"frontier_size": self.frontier_size,
|
|
67
|
+
"frontier_objectives": self.frontier_objectives,
|
|
68
|
+
"executed_candidates": self.executed_candidates,
|
|
69
|
+
"trace_count": self.trace_count,
|
|
70
|
+
"updated_components": list(self.updated_components),
|
|
71
|
+
"traces": [dict(trace) for trace in self.traces],
|
|
72
|
+
"registry_snapshot": self.registry.to_dict(),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _make_synthesizer(
|
|
77
|
+
synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
|
|
78
|
+
registry: CapabilityRegistry,
|
|
79
|
+
) -> Synthesizer:
|
|
80
|
+
"""Instantiate the synthesis strategy against the *current* registry.
|
|
81
|
+
|
|
82
|
+
Accepts a synthesizer *class* (constructed with its defaults) or a
|
|
83
|
+
one-argument factory such as
|
|
84
|
+
``lambda reg: BeamSynthesizer(reg, beam_width=5)``. A pre-built
|
|
85
|
+
instance is rejected: its constructor bound it to the previous
|
|
86
|
+
registry, so reusing it would silently synthesize against stale
|
|
87
|
+
metadata and the loop would learn nothing.
|
|
88
|
+
"""
|
|
89
|
+
if isinstance(synthesizer, type) and issubclass(synthesizer, Synthesizer):
|
|
90
|
+
return synthesizer(registry)
|
|
91
|
+
if callable(synthesizer):
|
|
92
|
+
return synthesizer(registry)
|
|
93
|
+
raise TypeError(
|
|
94
|
+
"synthesizer must be a Synthesizer subclass or a "
|
|
95
|
+
f"factory(registry) callable, {synthesizer!r} given; passing a "
|
|
96
|
+
"pre-built instance would bind the loop to stale metadata"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def run_closed_loop(
|
|
101
|
+
registry: CapabilityRegistry,
|
|
102
|
+
task: TaskSpecification,
|
|
103
|
+
synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
|
|
104
|
+
executor: AgentExecutor,
|
|
105
|
+
iterations: int = 3,
|
|
106
|
+
alpha: float = 0.3,
|
|
107
|
+
budget: Optional[SynthesisBudget] = None,
|
|
108
|
+
candidates_per_iteration: Optional[int] = None,
|
|
109
|
+
) -> List[LoopIteration]:
|
|
110
|
+
"""Run the synthesize -> execute -> calibrate cycle ``iterations`` times.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
registry: The declared (prior) registry; iteration 0 synthesizes
|
|
114
|
+
against it unchanged.
|
|
115
|
+
task: The synthesis problem, as for any synthesizer.
|
|
116
|
+
synthesizer: A :class:`~chatcortex.synthesis.base.Synthesizer`
|
|
117
|
+
subclass or ``factory(registry) -> Synthesizer`` callable. It
|
|
118
|
+
is instantiated fresh against the *current* registry every
|
|
119
|
+
iteration, because calibrate returns a new registry.
|
|
120
|
+
executor: A shared :class:`AgentExecutor` instance (simulated or
|
|
121
|
+
backend-backed); its seeded RNG advances across iterations,
|
|
122
|
+
which keeps the whole run deterministic.
|
|
123
|
+
iterations: How many loop rounds to run (>= 1).
|
|
124
|
+
alpha: Blend weight passed to :func:`calibrate`; validated, so
|
|
125
|
+
every round only executes under a valid blend. Non-finite
|
|
126
|
+
values are rejected like any out-of-range value.
|
|
127
|
+
candidates_per_iteration: Execute only the first ``N`` frontier
|
|
128
|
+
members (objective-sorted, deterministic) instead of all of
|
|
129
|
+
them; useful when executions are expensive. Must be ``>= 1``
|
|
130
|
+
when given, and is validated before anything executes.
|
|
131
|
+
budget: Optional per-iteration :class:`SynthesisBudget`; it is a
|
|
132
|
+
frozen dataclass, so the same constraints apply every round
|
|
133
|
+
without state leaking between iterations.
|
|
134
|
+
candidates_per_iteration: Execute only the first ``N`` frontier
|
|
135
|
+
members (objective-sorted, deterministic) instead of all of
|
|
136
|
+
them; useful when executions are expensive.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
One :class:`LoopIteration` per round; iteration ``k``'s
|
|
140
|
+
``frontier_size``/``frontier_objectives`` describe the frontier
|
|
141
|
+
synthesized from iteration ``k-1``'s calibrated registry, and
|
|
142
|
+
``registry`` is the registry calibrated *during* round ``k``.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
ValueError: If ``iterations < 1``, ``alpha`` is outside ``[0, 1]``,
|
|
146
|
+
or ``candidates_per_iteration`` is given below 1 — all before
|
|
147
|
+
any round runs — or if a round synthesizes an empty frontier
|
|
148
|
+
(nothing to execute would stall the loop).
|
|
149
|
+
TypeError: If ``synthesizer`` is neither a Synthesizer subclass
|
|
150
|
+
nor a factory callable.
|
|
151
|
+
"""
|
|
152
|
+
if iterations < 1:
|
|
153
|
+
raise ValueError(f"iterations must be >= 1, {iterations} given")
|
|
154
|
+
if candidates_per_iteration is not None and candidates_per_iteration < 1:
|
|
155
|
+
raise ValueError(
|
|
156
|
+
"candidates_per_iteration must be >= 1 when given, "
|
|
157
|
+
f"{candidates_per_iteration} given"
|
|
158
|
+
)
|
|
159
|
+
if not math.isfinite(alpha) or alpha < 0.0 or alpha > 1.0:
|
|
160
|
+
raise ValueError(f"alpha must be in [0.0, 1.0], {alpha!r} given")
|
|
161
|
+
|
|
162
|
+
records: List[LoopIteration] = []
|
|
163
|
+
current = registry
|
|
164
|
+
for iteration in range(iterations):
|
|
165
|
+
engine = _make_synthesizer(synthesizer, current)
|
|
166
|
+
frontier = engine.synthesize(task, budget)
|
|
167
|
+
if not frontier:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"iteration {iteration}: the synthesizer returned an "
|
|
170
|
+
"empty frontier; the task may be infeasible under the "
|
|
171
|
+
"current registry and constraints"
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
selected = frontier
|
|
175
|
+
if candidates_per_iteration is not None:
|
|
176
|
+
selected = sorted(
|
|
177
|
+
frontier,
|
|
178
|
+
key=lambda c: (c.total_cost, c.total_latency, -c.total_reliability),
|
|
179
|
+
)[:candidates_per_iteration]
|
|
180
|
+
|
|
181
|
+
traces: List[Dict[str, Any]] = []
|
|
182
|
+
for candidate in selected:
|
|
183
|
+
traces.extend(
|
|
184
|
+
executor.execute(candidate.graph).telemetry.records
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
calibrated = calibrate(current, traces, alpha=alpha)
|
|
188
|
+
updated = [
|
|
189
|
+
prior.name
|
|
190
|
+
for prior in current.list_all()
|
|
191
|
+
if _differs(calibrated.get(prior.name), prior)
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
records.append(
|
|
195
|
+
LoopIteration(
|
|
196
|
+
iteration=iteration,
|
|
197
|
+
frontier_size=len(frontier),
|
|
198
|
+
frontier_objectives=[
|
|
199
|
+
{
|
|
200
|
+
"cost": c.total_cost,
|
|
201
|
+
"latency": c.total_latency,
|
|
202
|
+
"reliability": c.total_reliability,
|
|
203
|
+
}
|
|
204
|
+
for c in frontier
|
|
205
|
+
],
|
|
206
|
+
executed_candidates=len(selected),
|
|
207
|
+
trace_count=len(traces),
|
|
208
|
+
updated_components=updated,
|
|
209
|
+
traces=[dict(trace) for trace in traces],
|
|
210
|
+
registry=calibrated,
|
|
211
|
+
prior_registry=current,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
current = calibrated
|
|
215
|
+
return records
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _differs(calibrated, prior) -> bool:
|
|
219
|
+
"""True when calibration changed any performance field.
|
|
220
|
+
|
|
221
|
+
Exact comparison is safe: ``blend == prior`` exactly when the
|
|
222
|
+
empirical value equals the prior, with no rounding involved.
|
|
223
|
+
"""
|
|
224
|
+
return (
|
|
225
|
+
calibrated.reliability_score != prior.reliability_score
|
|
226
|
+
or calibrated.avg_latency_ms != prior.avg_latency_ms
|
|
227
|
+
or calibrated.cost_per_call != prior.cost_per_call
|
|
228
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Console script for the registry pack: ``chatcortex-registry validate``.
|
|
2
|
+
|
|
3
|
+
``main`` is the ``[project.scripts]`` entry point. It never raises —
|
|
4
|
+
validation results come back as a :class:`RegistryReport` — and exits 0
|
|
5
|
+
only when the file is a fully valid pack, so shell checks can gate on it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
from .package import RegistryReport, validate_registry_file
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _render_report(report: RegistryReport) -> List[str]:
|
|
15
|
+
"""Human-readable lines; issue/duplicate order follows the report."""
|
|
16
|
+
lines = [f"components validated: {report.component_count}"]
|
|
17
|
+
for duplicate in report.duplicate_names:
|
|
18
|
+
lines.append(f"duplicate component: '{duplicate}'")
|
|
19
|
+
lines.extend(f"error: {issue}" for issue in report.issues)
|
|
20
|
+
lines.append(f"result: {'VALID' if report.valid else 'INVALID'}")
|
|
21
|
+
return lines
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
25
|
+
"""Entry point returning the process exit code (0 valid, 1 invalid)."""
|
|
26
|
+
parser = argparse.ArgumentParser(
|
|
27
|
+
prog="chatcortex-registry",
|
|
28
|
+
description="Registry pack tooling for ChatCortex.",
|
|
29
|
+
)
|
|
30
|
+
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
31
|
+
|
|
32
|
+
validate = subcommands.add_parser(
|
|
33
|
+
"validate",
|
|
34
|
+
help="Validate a registry JSON pack and print a structured report.",
|
|
35
|
+
)
|
|
36
|
+
validate.add_argument("file", help="Path to the registry JSON file.")
|
|
37
|
+
|
|
38
|
+
args = parser.parse_args(argv)
|
|
39
|
+
report = validate_registry_file(args.file)
|
|
40
|
+
|
|
41
|
+
print(f"chatcortex-registry validate {args.file}")
|
|
42
|
+
for line in _render_report(report):
|
|
43
|
+
print(f" {line}")
|
|
44
|
+
return 0 if report.valid else 1
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__":
|
|
48
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Registry packs: save, validate, and round-trip registry JSON files.
|
|
2
|
+
|
|
3
|
+
The document shape is the loader's shipped one — ``{"components": [...]}``
|
|
4
|
+
with exactly the required per-component fields — plus nothing. A pack
|
|
5
|
+
written by :func:`save_registry` is guaranteed reloadable:
|
|
6
|
+
:meth:`CapabilityRegistry.to_dict` emits the loader's shape and the save
|
|
7
|
+
*proves* the round-trip by loading the file back and comparing registries.
|
|
8
|
+
|
|
9
|
+
:func:`validate_registry_file` never raises: it returns a
|
|
10
|
+
:class:`RegistryReport` with a component count, one actionable issue per
|
|
11
|
+
failing row (message names row and field, loader style), and every
|
|
12
|
+
duplicate name — so a broken pack can be inspected offline, e.g. through
|
|
13
|
+
the ``chatcortex-registry validate`` console command.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, List, Union
|
|
20
|
+
|
|
21
|
+
from .capability_registry import CapabilityRegistry
|
|
22
|
+
from .loader import load_registry
|
|
23
|
+
from .metadata import validate_json_payload
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def save_registry(
|
|
27
|
+
registry: CapabilityRegistry,
|
|
28
|
+
path: Union[str, Path],
|
|
29
|
+
) -> Path:
|
|
30
|
+
"""Write ``registry`` to ``path`` in the loader's JSON document shape.
|
|
31
|
+
|
|
32
|
+
The file gets the exact 0.x shipped shape — ``{"components": [...]}``
|
|
33
|
+
from :meth:`CapabilityRegistry.to_dict`, no additional fields — and
|
|
34
|
+
the save validates its own round-trip: the freshly written file is
|
|
35
|
+
loaded back and must reproduce the same component sequence, or a
|
|
36
|
+
``RuntimeError`` is raised. Every detail a ``load_registry`` call
|
|
37
|
+
would consume is verifiably present in the file.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
registry: Populated registry to package.
|
|
41
|
+
path: Destination file path (parent directory must exist).
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
The ``Path`` written, for chaining.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
RuntimeError: If the written file does not round-trip to an
|
|
48
|
+
equivalent registry.
|
|
49
|
+
"""
|
|
50
|
+
file_path = Path(path)
|
|
51
|
+
document = registry.to_dict()
|
|
52
|
+
file_path.write_text(
|
|
53
|
+
json.dumps(document, indent=2) + "\n",
|
|
54
|
+
encoding="utf-8",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
reloaded = load_registry(file_path)
|
|
58
|
+
if reloaded.list_all() != registry.list_all(): # frozen dataclass equality
|
|
59
|
+
raise RuntimeError(
|
|
60
|
+
f"save_registry round-trip check failed for {file_path}: "
|
|
61
|
+
"the written file does not reload to the same registry"
|
|
62
|
+
)
|
|
63
|
+
return file_path
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class RegistryReport:
|
|
68
|
+
"""Structured result of validating a registry pack file.
|
|
69
|
+
|
|
70
|
+
Attributes:
|
|
71
|
+
component_count: Rows that pass full validation (equals the
|
|
72
|
+
loaded component count when the pack is valid).
|
|
73
|
+
issues: One message per failing row or document-level problem,
|
|
74
|
+
in loader style so row and field are named, e.g.
|
|
75
|
+
``components[1].reliability_score out of range``.
|
|
76
|
+
duplicate_names: Component names appearing in more than one row,
|
|
77
|
+
each reported once (first-occurrence order).
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
component_count: int = 0
|
|
81
|
+
issues: List[str] = field(default_factory=list)
|
|
82
|
+
duplicate_names: List[str] = field(default_factory=list)
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def valid(self) -> bool:
|
|
86
|
+
"""True when the file reloads into a complete, duplicate-free registry."""
|
|
87
|
+
return not self.issues and not self.duplicate_names
|
|
88
|
+
|
|
89
|
+
def to_dict(self) -> dict:
|
|
90
|
+
return {
|
|
91
|
+
"component_count": self.component_count,
|
|
92
|
+
"issues": list(self.issues),
|
|
93
|
+
"duplicate_names": list(self.duplicate_names),
|
|
94
|
+
"valid": self.valid,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def validate_registry_file(path: Union[str, Path]) -> RegistryReport:
|
|
99
|
+
"""Validate a registry pack and report what is wrong, without raising.
|
|
100
|
+
|
|
101
|
+
Unlike :func:`load_registry` — which fails fast on the first row —
|
|
102
|
+
this walks every row, collecting one issue per failing row and *all*
|
|
103
|
+
duplicate names, so a corrupt pack can be repaired in one pass.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
path: Path to the JSON document described in
|
|
107
|
+
:mod:`chatcortex.registry.loader`.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
A :class:`RegistryReport`; ``report.valid`` is True exactly when
|
|
111
|
+
:func:`load_registry` would succeed on the same file.
|
|
112
|
+
"""
|
|
113
|
+
file_path = Path(path)
|
|
114
|
+
report = RegistryReport()
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
text = file_path.read_text(encoding="utf-8")
|
|
118
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
119
|
+
report.issues.append(f"Cannot read registry file {file_path}: {exc}")
|
|
120
|
+
return report
|
|
121
|
+
|
|
122
|
+
try:
|
|
123
|
+
document: Any = json.loads(text)
|
|
124
|
+
except json.JSONDecodeError as exc:
|
|
125
|
+
report.issues.append(f"Registry file {file_path} is not valid JSON: {exc}")
|
|
126
|
+
return report
|
|
127
|
+
|
|
128
|
+
if not isinstance(document, dict) or not isinstance(document.get("components"), list):
|
|
129
|
+
report.issues.append(
|
|
130
|
+
f"Registry file {file_path} must contain a top-level object "
|
|
131
|
+
"with a 'components' list"
|
|
132
|
+
)
|
|
133
|
+
return report
|
|
134
|
+
|
|
135
|
+
rows = document["components"]
|
|
136
|
+
name_occurrences: dict = {}
|
|
137
|
+
loadable = True
|
|
138
|
+
|
|
139
|
+
for row, payload in enumerate(rows):
|
|
140
|
+
if not isinstance(payload, dict):
|
|
141
|
+
report.issues.append(
|
|
142
|
+
f"components[{row}] must be an object, {type(payload).__name__} given"
|
|
143
|
+
)
|
|
144
|
+
loadable = False
|
|
145
|
+
continue
|
|
146
|
+
try:
|
|
147
|
+
metadata_kwargs = validate_json_payload(payload)
|
|
148
|
+
except (TypeError, ValueError) as exc:
|
|
149
|
+
report.issues.append(f"components[{row}].{exc}")
|
|
150
|
+
loadable = False
|
|
151
|
+
continue
|
|
152
|
+
|
|
153
|
+
name = metadata_kwargs["name"]
|
|
154
|
+
if name in name_occurrences:
|
|
155
|
+
report.duplicate_names.append(name)
|
|
156
|
+
else:
|
|
157
|
+
name_occurrences[name] = row
|
|
158
|
+
report.component_count += 1
|
|
159
|
+
|
|
160
|
+
# load_registry also rejects duplicates; mirror that in `valid`.
|
|
161
|
+
if report.duplicate_names:
|
|
162
|
+
loadable = False
|
|
163
|
+
if not loadable:
|
|
164
|
+
report.issues.append(
|
|
165
|
+
f"Registry file {file_path} is not loadable; see issues above"
|
|
166
|
+
)
|
|
167
|
+
return report
|
|
@@ -24,6 +24,7 @@ are ignored rather than raising.
|
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
26
|
import dataclasses
|
|
27
|
+
import math
|
|
27
28
|
from collections import defaultdict
|
|
28
29
|
from typing import Dict, List
|
|
29
30
|
|
|
@@ -60,7 +61,8 @@ def calibrate(
|
|
|
60
61
|
``cost``, and ``success`` keys.
|
|
61
62
|
alpha: Blend weight on the empirical values, in ``[0, 1]``; 0
|
|
62
63
|
keeps the priors, 1 replaces them, the default 0.3 leans
|
|
63
|
-
toward the declared values.
|
|
64
|
+
toward the declared values. Non-finite values are rejected
|
|
65
|
+
like any out-of-range value.
|
|
64
66
|
|
|
65
67
|
Returns:
|
|
66
68
|
A NEW registry; the input registry and its frozen metadata are
|
|
@@ -72,9 +74,11 @@ def calibrate(
|
|
|
72
74
|
shared unchanged.
|
|
73
75
|
|
|
74
76
|
Raises:
|
|
75
|
-
ValueError: If ``alpha`` is outside ``[0.0, 1.0]
|
|
77
|
+
ValueError: If ``alpha`` is outside ``[0.0, 1.0]`` or not finite.
|
|
76
78
|
"""
|
|
77
|
-
|
|
79
|
+
# NaN passes naive range comparisons, so a missing finite check would
|
|
80
|
+
# let NaN flow into metadata and propagate silently through blends.
|
|
81
|
+
if not math.isfinite(alpha) or alpha < 0.0 or alpha > 1.0:
|
|
78
82
|
raise ValueError(f"alpha must be in [0.0, 1.0], {alpha!r} given")
|
|
79
83
|
|
|
80
84
|
all_metadata = registry.list_all()
|