chatcortex 0.6.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {chatcortex-0.6.0 → chatcortex-0.8.0}/PKG-INFO +48 -4
  2. {chatcortex-0.6.0 → chatcortex-0.8.0}/README.md +46 -1
  3. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/__init__.py +8 -1
  4. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/execution/executor.py +36 -3
  5. chatcortex-0.8.0/chatcortex/loop/__init__.py +10 -0
  6. chatcortex-0.8.0/chatcortex/loop/closed_loop.py +228 -0
  7. chatcortex-0.8.0/chatcortex/registry/cli.py +48 -0
  8. chatcortex-0.8.0/chatcortex/registry/package.py +167 -0
  9. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/telemetry/calibration.py +7 -3
  10. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/PKG-INFO +48 -4
  11. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/SOURCES.txt +7 -0
  12. chatcortex-0.8.0/chatcortex.egg-info/entry_points.txt +2 -0
  13. {chatcortex-0.6.0 → chatcortex-0.8.0}/pyproject.toml +6 -4
  14. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_calibration.py +10 -0
  15. chatcortex-0.8.0/tests/test_closed_loop.py +283 -0
  16. chatcortex-0.8.0/tests/test_execution.py +98 -0
  17. chatcortex-0.8.0/tests/test_registry_pack.py +138 -0
  18. chatcortex-0.6.0/tests/test_execution.py +0 -30
  19. {chatcortex-0.6.0 → chatcortex-0.8.0}/LICENSE +0 -0
  20. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/adapters/__init__.py +0 -0
  21. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/adapters/openai_backend.py +0 -0
  22. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/evaluation/approximation_metrics.py +0 -0
  23. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/evaluation/harness.py +0 -0
  24. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/graph/agent_graph.py +0 -0
  25. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/graph/node_ids.py +0 -0
  26. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/optimization/architecture_candidate.py +0 -0
  27. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/optimization/pareto.py +0 -0
  28. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/capability_registry.py +0 -0
  29. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/demo_components.py +0 -0
  30. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/loader.py +0 -0
  31. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/registry/metadata.py +0 -0
  32. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/base.py +0 -0
  33. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/beam_synthesizer.py +0 -0
  34. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/budget.py +0 -0
  35. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/engine.py +0 -0
  36. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/exhaustive_synthesizer.py +0 -0
  37. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/heuristic_synthesizer.py +0 -0
  38. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -0
  39. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -0
  40. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/random_synthesizer.py +0 -0
  41. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/synthesis/task_specification.py +0 -0
  42. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex/telemetry/logger.py +0 -0
  43. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/dependency_links.txt +0 -0
  44. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/requires.txt +0 -0
  45. {chatcortex-0.6.0 → chatcortex-0.8.0}/chatcortex.egg-info/top_level.txt +0 -0
  46. {chatcortex-0.6.0 → chatcortex-0.8.0}/setup.cfg +0 -0
  47. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_agent_graph.py +0 -0
  48. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_executor_backend.py +0 -0
  49. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_node_ids.py +0 -0
  50. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_openai_backend_install.py +0 -0
  51. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_pareto.py +0 -0
  52. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_registry.py +0 -0
  53. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_registry_loader.py +0 -0
  54. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_regression_gates.py +0 -0
  55. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_synthesizers.py +0 -0
  56. {chatcortex-0.6.0 → chatcortex-0.8.0}/tests/test_task_specification.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chatcortex
3
- Version: 0.6.0
3
+ Version: 0.8.0
4
4
  Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
5
5
  Author: Siddharth Saraswat
6
6
  License: MIT
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
13
13
  Classifier: Intended Audience :: Developers
14
14
  Classifier: License :: OSI Approved :: MIT License
15
15
  Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
16
  Classifier: Programming Language :: Python :: 3.10
18
17
  Classifier: Programming Language :: Python :: 3.11
19
18
  Classifier: Programming Language :: Python :: 3.12
20
19
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
20
  Classifier: Topic :: Software Development :: Libraries
22
- Requires-Python: >=3.9
21
+ Requires-Python: >=3.10
23
22
  Description-Content-Type: text/markdown
24
23
  License-File: LICENSE
25
24
  Requires-Dist: networkx>=2.8
@@ -37,7 +36,7 @@ Dynamic: license-file
37
36
 
38
37
  [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
39
38
  [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
40
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
39
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
41
40
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
42
41
 
43
42
  **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
@@ -48,6 +47,17 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
48
47
  Task Specification → Architecture Search → Pareto-Optimal Architectures
49
48
  ```
50
49
 
50
+ **ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
51
+
52
+ | Component | Field | Declared | Calibrated | Truth |
53
+ |---|---|---|---|---|
54
+ | `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
55
+ | `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
56
+ | `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
57
+ | `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
58
+
59
+ > **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
60
+
51
61
  ## Installation
52
62
 
53
63
  ```bash
@@ -106,10 +116,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
106
116
  python examples/registry_from_json.py
107
117
  ```
108
118
 
119
+ Run the flagship closed loop — synthesize → execute → calibrate →
120
+ re-synthesize, four iterations of measured behavior flowing back into the
121
+ registry:
122
+
123
+ ```bash
124
+ python examples/closed_loop.py
125
+ ```
126
+
127
+ With an OpenAI key you can run the same loop against real backends:
128
+
129
+ ```bash
130
+ pip install "chatcortex[real]"
131
+ export OPENAI_API_KEY=... # every reply spends your credits
132
+ python examples/closed_loop_real.py
133
+ ```
134
+
109
135
  To search over your own components, build a `CapabilityRegistry` and register
110
136
  `ComponentMetadata` describing each model, tool, or verifier — see
111
137
  `examples/` for complete walkthroughs.
112
138
 
139
+ ## Registry packs
140
+
141
+ A registry ships as a JSON pack in the loader's document shape. Saving it
142
+ proves the round-trip, and one command validates any pack offline:
143
+
144
+ ```python
145
+ from chatcortex import build_demo_registry, save_registry, validate_registry_file
146
+
147
+ save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
148
+ print(validate_registry_file("registry.json").valid) # True
149
+ ```
150
+
151
+ ```bash
152
+ chatcortex-registry validate registry.json
153
+ # components validated: 12
154
+ # result: VALID (exit 0; issues and duplicates are named when invalid)
155
+ ```
156
+
113
157
  ## How It Works
114
158
 
115
159
  ChatCortex is organized as a layered architecture synthesis system:
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
4
4
  [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
5
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
5
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
6
6
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
7
 
8
8
  **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
@@ -13,6 +13,17 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
13
13
  Task Specification → Architecture Search → Pareto-Optimal Architectures
14
14
  ```
15
15
 
16
+ **ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
17
+
18
+ | Component | Field | Declared | Calibrated | Truth |
19
+ |---|---|---|---|---|
20
+ | `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
21
+ | `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
22
+ | `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
23
+ | `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
24
+
25
+ > **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
26
+
16
27
  ## Installation
17
28
 
18
29
  ```bash
@@ -71,10 +82,44 @@ architecture, and calibrate the registry's metadata from real telemetry — run:
71
82
  python examples/registry_from_json.py
72
83
  ```
73
84
 
85
+ Run the flagship closed loop — synthesize → execute → calibrate →
86
+ re-synthesize, four iterations of measured behavior flowing back into the
87
+ registry:
88
+
89
+ ```bash
90
+ python examples/closed_loop.py
91
+ ```
92
+
93
+ With an OpenAI key you can run the same loop against real backends:
94
+
95
+ ```bash
96
+ pip install "chatcortex[real]"
97
+ export OPENAI_API_KEY=... # every reply spends your credits
98
+ python examples/closed_loop_real.py
99
+ ```
100
+
74
101
  To search over your own components, build a `CapabilityRegistry` and register
75
102
  `ComponentMetadata` describing each model, tool, or verifier — see
76
103
  `examples/` for complete walkthroughs.
77
104
 
105
+ ## Registry packs
106
+
107
+ A registry ships as a JSON pack in the loader's document shape. Saving it
108
+ proves the round-trip, and one command validates any pack offline:
109
+
110
+ ```python
111
+ from chatcortex import build_demo_registry, save_registry, validate_registry_file
112
+
113
+ save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
114
+ print(validate_registry_file("registry.json").valid) # True
115
+ ```
116
+
117
+ ```bash
118
+ chatcortex-registry validate registry.json
119
+ # components validated: 12
120
+ # result: VALID (exit 0; issues and duplicates are named when invalid)
121
+ ```
122
+
78
123
  ## How It Works
79
124
 
80
125
  ChatCortex is organized as a layered architecture synthesis system:
@@ -13,10 +13,12 @@ Core capabilities include:
13
13
  """
14
14
 
15
15
  from .graph.agent_graph import AgentGraph
16
+ from .loop import LoopIteration, run_closed_loop
16
17
  from .registry.capability_registry import CapabilityRegistry
17
18
  from .registry.demo_components import build_demo_registry
18
19
  from .registry.loader import load_registry
19
20
  from .registry.metadata import ComponentMetadata
21
+ from .registry.package import RegistryReport, save_registry, validate_registry_file
20
22
  from .synthesis.beam_synthesizer import BeamSynthesizer
21
23
  from .synthesis.budget import SynthesisBudget
22
24
  from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
@@ -39,9 +41,14 @@ __all__ = [
39
41
  "SynthesisBudget",
40
42
  "AgentGraph",
41
43
  "CapabilityRegistry",
44
+ "LoopIteration",
42
45
  "ComponentMetadata",
43
46
  "build_demo_registry",
44
47
  "load_registry",
48
+ "run_closed_loop",
49
+ "save_registry",
50
+ "validate_registry_file",
51
+ "RegistryReport",
45
52
  ]
46
53
 
47
- __version__ = "0.6.0"
54
+ __version__ = "0.8.0"
@@ -3,6 +3,7 @@ from dataclasses import dataclass
3
3
  from typing import Any, Literal, Optional, Protocol
4
4
 
5
5
  from chatcortex.graph.agent_graph import AgentGraph
6
+ from chatcortex.registry.capability_registry import CapabilityRegistry
6
7
  from chatcortex.registry.metadata import ComponentMetadata
7
8
  from chatcortex.telemetry.logger import TelemetryLogger
8
9
 
@@ -68,15 +69,34 @@ class AgentExecutor:
68
69
  The chain stops on the first failure, exactly like the simulator.
69
70
  A backend exception is recorded as a failed probe (zero cost) and
70
71
  also stops the chain, so partial telemetry survives the error.
72
+
73
+ Hidden ground truth: by default the simulator draws outcomes from the
74
+ metadata carried by the graph, i.e. from the *declared* values, so a
75
+ calibration loop fed by it can only rediscover its own priors plus
76
+ sampling noise. Pass ``ground_truth`` (a registry holding each
77
+ component's *true* cost, latency and reliability) and the simulator
78
+ draws from those instead, looked up by component name. The synthesizer
79
+ and calibration never see that registry, which makes the simulator a
80
+ real test of whether a loop converges on values it was not told.
81
+ Components absent from ``ground_truth`` behave as declared.
82
+ ``ground_truth`` cannot be combined with ``backend``: a backend
83
+ already is the source of truth.
71
84
  """
72
85
  def __init__(
73
86
  self,
74
87
  mode: ExecutionMode = "deterministic",
75
88
  seed: Optional[int] = None,
76
89
  backend: Optional[ExecutionProtocol] = None,
90
+ ground_truth: Optional[CapabilityRegistry] = None,
77
91
  ):
92
+ if backend is not None and ground_truth is not None:
93
+ raise ValueError(
94
+ "ground_truth only applies to the simulator; a backend "
95
+ "is its own source of truth, so pass one or the other"
96
+ )
78
97
  self.mode = mode
79
98
  self.backend = backend
99
+ self.ground_truth = ground_truth
80
100
  # local random generator - NO RNG pollution, no side-effects, clean isolation
81
101
  self.random = random.Random(seed)
82
102
 
@@ -115,8 +135,12 @@ class AgentExecutor:
115
135
  probe = self._invoke_backend(metadata, node_input)
116
136
  return probe.latency_ms, probe.cost, probe.success, probe.output
117
137
 
138
+ # What the component really does: the hidden truth when one is
139
+ # supplied, otherwise the declared metadata itself.
140
+ actual = self._true_metadata(metadata)
141
+
118
142
  # Simulated latency
119
- simulated_latency = metadata.avg_latency_ms
143
+ simulated_latency = actual.avg_latency_ms
120
144
 
121
145
  # Optional: actually sleep (disable if too slow)
122
146
  # time.sleep(simulated_latency / 1000)
@@ -125,11 +149,20 @@ class AgentExecutor:
125
149
  if self.mode == "deterministic":
126
150
  success = True
127
151
  elif self.mode == "probabilistic":
128
- success = self.random.random() <= metadata.reliability_score
152
+ success = self.random.random() <= actual.reliability_score
129
153
  else:
130
154
  raise ValueError("Invalid execution mode")
131
155
 
132
- return simulated_latency, metadata.cost_per_call, success, None
156
+ return simulated_latency, actual.cost_per_call, success, None
157
+
158
+ def _true_metadata(self, declared: ComponentMetadata) -> ComponentMetadata:
159
+ """The metadata the simulator samples from for one component."""
160
+ if self.ground_truth is None:
161
+ return declared
162
+ try:
163
+ return self.ground_truth.get(declared.name)
164
+ except KeyError:
165
+ return declared
133
166
 
134
167
  def _invoke_backend(
135
168
  self, metadata: ComponentMetadata, node_input: Any
@@ -0,0 +1,10 @@
1
+ """The tradeoff-driven closed loop: synthesize -> execute -> calibrate.
2
+
3
+ Re-exports the loop entry points so users can start from the package::
4
+
5
+ from chatcortex.loop import run_closed_loop
6
+ """
7
+
8
+ from .closed_loop import LoopIteration, run_closed_loop
9
+
10
+ __all__ = ["LoopIteration", "run_closed_loop"]
@@ -0,0 +1,228 @@
1
+ """The closed loop: synthesize -> execute -> calibrate -> re-synthesize.
2
+
3
+ :func:`run_closed_loop` glues three already-shipped mechanisms into one
4
+ iterative drive: a synthesizer maps the *current* registry to a Pareto
5
+ frontier, an executor runs the frontier's architectures and produces
6
+ telemetry records, and :func:`chatcortex.telemetry.calibration.calibrate`
7
+ blends the measured success/latency/cost back into the registry — so the
8
+ next iteration synthesizes against a registry that remembers what actually
9
+ happened, not only what was declared.
10
+
11
+ Determinism: given a deterministic-or-seeded synthesizer, a seeded (or
12
+ deterministic) executor, and seeded synthesis budgets, two calls with the
13
+ same inputs produce iteration-for-iteration identical records. The seeded
14
+ executor's RNG advances across iterations, which is still deterministic —
15
+ it is state, not scheduling.
16
+
17
+ Component mapping: telemetry records carry graph *node ids*;
18
+ ``calibrate`` resolves them through
19
+ :func:`chatcortex.graph.node_ids.component_name_from_node_id` and silently
20
+ ignores names it cannot match, so traces from graphs built from the
21
+ previous registry still calibrate the components they were produced from.
22
+
23
+ Calibration feedback is *iterative*: each iteration blends only the
24
+ latest traces into the previous iteration's calibration, so priors
25
+ compound toward empirical values across iterations at roughly
26
+ ``1 - (1 - alpha) ** k`` after ``k`` iterations.
27
+ """
28
+
29
+ import math
30
+ from dataclasses import dataclass
31
+ from typing import Any, Callable, Dict, List, Optional, Union
32
+
33
+ from chatcortex.execution.executor import AgentExecutor
34
+ from chatcortex.registry.capability_registry import CapabilityRegistry
35
+ from chatcortex.synthesis.base import Synthesizer
36
+ from chatcortex.synthesis.budget import SynthesisBudget
37
+ from chatcortex.synthesis.task_specification import TaskSpecification
38
+ from chatcortex.telemetry.calibration import calibrate
39
+
40
+
41
+ @dataclass
42
+ class LoopIteration:
43
+ """Everything one loop iteration observed, kept and serializable.
44
+
45
+ ``registry`` holds the *calibrated* registry this iteration produced
46
+ (the next iteration's synthesis input and the value users typically
47
+ keep); ``to_dict()`` embeds it as the loader-shaped
48
+ ``registry_snapshot`` so a full loop can be checkpointed to JSON.
49
+ ``traces`` are the raw telemetry records behind the calibration.
50
+ """
51
+
52
+ iteration: int
53
+ frontier_size: int
54
+ frontier_objectives: List[Dict[str, float]]
55
+ executed_candidates: int
56
+ trace_count: int
57
+ updated_components: List[str]
58
+ traces: List[Dict[str, Any]]
59
+ registry: CapabilityRegistry
60
+ prior_registry: CapabilityRegistry
61
+
62
+ def to_dict(self) -> dict:
63
+ """JSON-safe record including the calibrated registry snapshot."""
64
+ return {
65
+ "iteration": self.iteration,
66
+ "frontier_size": self.frontier_size,
67
+ "frontier_objectives": self.frontier_objectives,
68
+ "executed_candidates": self.executed_candidates,
69
+ "trace_count": self.trace_count,
70
+ "updated_components": list(self.updated_components),
71
+ "traces": [dict(trace) for trace in self.traces],
72
+ "registry_snapshot": self.registry.to_dict(),
73
+ }
74
+
75
+
76
+ def _make_synthesizer(
77
+ synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
78
+ registry: CapabilityRegistry,
79
+ ) -> Synthesizer:
80
+ """Instantiate the synthesis strategy against the *current* registry.
81
+
82
+ Accepts a synthesizer *class* (constructed with its defaults) or a
83
+ one-argument factory such as
84
+ ``lambda reg: BeamSynthesizer(reg, beam_width=5)``. A pre-built
85
+ instance is rejected: its constructor bound it to the previous
86
+ registry, so reusing it would silently synthesize against stale
87
+ metadata and the loop would learn nothing.
88
+ """
89
+ if isinstance(synthesizer, type) and issubclass(synthesizer, Synthesizer):
90
+ return synthesizer(registry)
91
+ if callable(synthesizer):
92
+ return synthesizer(registry)
93
+ raise TypeError(
94
+ "synthesizer must be a Synthesizer subclass or a "
95
+ f"factory(registry) callable, {synthesizer!r} given; passing a "
96
+ "pre-built instance would bind the loop to stale metadata"
97
+ )
98
+
99
+
100
+ def run_closed_loop(
101
+ registry: CapabilityRegistry,
102
+ task: TaskSpecification,
103
+ synthesizer: Union[type, Callable[[CapabilityRegistry], Synthesizer]],
104
+ executor: AgentExecutor,
105
+ iterations: int = 3,
106
+ alpha: float = 0.3,
107
+ budget: Optional[SynthesisBudget] = None,
108
+ candidates_per_iteration: Optional[int] = None,
109
+ ) -> List[LoopIteration]:
110
+ """Run the synthesize -> execute -> calibrate cycle ``iterations`` times.
111
+
112
+ Args:
113
+ registry: The declared (prior) registry; iteration 0 synthesizes
114
+ against it unchanged.
115
+ task: The synthesis problem, as for any synthesizer.
116
+ synthesizer: A :class:`~chatcortex.synthesis.base.Synthesizer`
117
+ subclass or ``factory(registry) -> Synthesizer`` callable. It
118
+ is instantiated fresh against the *current* registry every
119
+ iteration, because calibrate returns a new registry.
120
+ executor: A shared :class:`AgentExecutor` instance (simulated or
121
+ backend-backed); its seeded RNG advances across iterations,
122
+ which keeps the whole run deterministic.
123
+ iterations: How many loop rounds to run (>= 1).
124
+ alpha: Blend weight passed to :func:`calibrate`; validated, so
125
+ every round only executes under a valid blend. Non-finite
126
+ values are rejected like any out-of-range value.
127
+ candidates_per_iteration: Execute only the first ``N`` frontier
128
+ members (objective-sorted, deterministic) instead of all of
129
+ them; useful when executions are expensive. Must be ``>= 1``
130
+ when given, and is validated before anything executes.
131
+ budget: Optional per-iteration :class:`SynthesisBudget`; it is a
132
+ frozen dataclass, so the same constraints apply every round
133
+ without state leaking between iterations.
134
+ candidates_per_iteration: Execute only the first ``N`` frontier
135
+ members (objective-sorted, deterministic) instead of all of
136
+ them; useful when executions are expensive.
137
+
138
+ Returns:
139
+ One :class:`LoopIteration` per round; iteration ``k``'s
140
+ ``frontier_size``/``frontier_objectives`` describe the frontier
141
+ synthesized from iteration ``k-1``'s calibrated registry, and
142
+ ``registry`` is the registry calibrated *during* round ``k``.
143
+
144
+ Raises:
145
+ ValueError: If ``iterations < 1``, ``alpha`` is outside ``[0, 1]``,
146
+ or ``candidates_per_iteration`` is given below 1 — all before
147
+ any round runs — or if a round synthesizes an empty frontier
148
+ (nothing to execute would stall the loop).
149
+ TypeError: If ``synthesizer`` is neither a Synthesizer subclass
150
+ nor a factory callable.
151
+ """
152
+ if iterations < 1:
153
+ raise ValueError(f"iterations must be >= 1, {iterations} given")
154
+ if candidates_per_iteration is not None and candidates_per_iteration < 1:
155
+ raise ValueError(
156
+ "candidates_per_iteration must be >= 1 when given, "
157
+ f"{candidates_per_iteration} given"
158
+ )
159
+ if not math.isfinite(alpha) or alpha < 0.0 or alpha > 1.0:
160
+ raise ValueError(f"alpha must be in [0.0, 1.0], {alpha!r} given")
161
+
162
+ records: List[LoopIteration] = []
163
+ current = registry
164
+ for iteration in range(iterations):
165
+ engine = _make_synthesizer(synthesizer, current)
166
+ frontier = engine.synthesize(task, budget)
167
+ if not frontier:
168
+ raise ValueError(
169
+ f"iteration {iteration}: the synthesizer returned an "
170
+ "empty frontier; the task may be infeasible under the "
171
+ "current registry and constraints"
172
+ )
173
+
174
+ selected = frontier
175
+ if candidates_per_iteration is not None:
176
+ selected = sorted(
177
+ frontier,
178
+ key=lambda c: (c.total_cost, c.total_latency, -c.total_reliability),
179
+ )[:candidates_per_iteration]
180
+
181
+ traces: List[Dict[str, Any]] = []
182
+ for candidate in selected:
183
+ traces.extend(
184
+ executor.execute(candidate.graph).telemetry.records
185
+ )
186
+
187
+ calibrated = calibrate(current, traces, alpha=alpha)
188
+ updated = [
189
+ prior.name
190
+ for prior in current.list_all()
191
+ if _differs(calibrated.get(prior.name), prior)
192
+ ]
193
+
194
+ records.append(
195
+ LoopIteration(
196
+ iteration=iteration,
197
+ frontier_size=len(frontier),
198
+ frontier_objectives=[
199
+ {
200
+ "cost": c.total_cost,
201
+ "latency": c.total_latency,
202
+ "reliability": c.total_reliability,
203
+ }
204
+ for c in frontier
205
+ ],
206
+ executed_candidates=len(selected),
207
+ trace_count=len(traces),
208
+ updated_components=updated,
209
+ traces=[dict(trace) for trace in traces],
210
+ registry=calibrated,
211
+ prior_registry=current,
212
+ )
213
+ )
214
+ current = calibrated
215
+ return records
216
+
217
+
218
+ def _differs(calibrated, prior) -> bool:
219
+ """True when calibration changed any performance field.
220
+
221
+ Exact comparison is safe: ``blend == prior`` exactly when the
222
+ empirical value equals the prior, with no rounding involved.
223
+ """
224
+ return (
225
+ calibrated.reliability_score != prior.reliability_score
226
+ or calibrated.avg_latency_ms != prior.avg_latency_ms
227
+ or calibrated.cost_per_call != prior.cost_per_call
228
+ )
@@ -0,0 +1,48 @@
1
+ """Console script for the registry pack: ``chatcortex-registry validate``.
2
+
3
+ ``main`` is the ``[project.scripts]`` entry point. It never raises —
4
+ validation results come back as a :class:`RegistryReport` — and exits 0
5
+ only when the file is a fully valid pack, so shell checks can gate on it.
6
+ """
7
+
8
+ import argparse
9
+ from typing import List, Optional
10
+
11
+ from .package import RegistryReport, validate_registry_file
12
+
13
+
14
+ def _render_report(report: RegistryReport) -> List[str]:
15
+ """Human-readable lines; issue/duplicate order follows the report."""
16
+ lines = [f"components validated: {report.component_count}"]
17
+ for duplicate in report.duplicate_names:
18
+ lines.append(f"duplicate component: '{duplicate}'")
19
+ lines.extend(f"error: {issue}" for issue in report.issues)
20
+ lines.append(f"result: {'VALID' if report.valid else 'INVALID'}")
21
+ return lines
22
+
23
+
24
+ def main(argv: Optional[List[str]] = None) -> int:
25
+ """Entry point returning the process exit code (0 valid, 1 invalid)."""
26
+ parser = argparse.ArgumentParser(
27
+ prog="chatcortex-registry",
28
+ description="Registry pack tooling for ChatCortex.",
29
+ )
30
+ subcommands = parser.add_subparsers(dest="command", required=True)
31
+
32
+ validate = subcommands.add_parser(
33
+ "validate",
34
+ help="Validate a registry JSON pack and print a structured report.",
35
+ )
36
+ validate.add_argument("file", help="Path to the registry JSON file.")
37
+
38
+ args = parser.parse_args(argv)
39
+ report = validate_registry_file(args.file)
40
+
41
+ print(f"chatcortex-registry validate {args.file}")
42
+ for line in _render_report(report):
43
+ print(f" {line}")
44
+ return 0 if report.valid else 1
45
+
46
+
47
+ if __name__ == "__main__":
48
+ raise SystemExit(main())