chatcortex 0.4.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. chatcortex-0.7.0/PKG-INFO +208 -0
  2. chatcortex-0.7.0/README.md +173 -0
  3. chatcortex-0.7.0/chatcortex/__init__.py +54 -0
  4. chatcortex-0.7.0/chatcortex/adapters/__init__.py +7 -0
  5. chatcortex-0.7.0/chatcortex/adapters/openai_backend.py +106 -0
  6. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/evaluation/approximation_metrics.py +12 -47
  7. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/evaluation/harness.py +95 -90
  8. chatcortex-0.7.0/chatcortex/execution/executor.py +146 -0
  9. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/graph/agent_graph.py +30 -12
  10. chatcortex-0.7.0/chatcortex/graph/node_ids.py +27 -0
  11. chatcortex-0.7.0/chatcortex/loop/__init__.py +10 -0
  12. chatcortex-0.7.0/chatcortex/loop/closed_loop.py +228 -0
  13. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/optimization/architecture_candidate.py +1 -1
  14. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/optimization/pareto.py +42 -32
  15. chatcortex-0.7.0/chatcortex/registry/capability_registry.py +93 -0
  16. chatcortex-0.7.0/chatcortex/registry/cli.py +48 -0
  17. chatcortex-0.7.0/chatcortex/registry/demo_components.py +61 -0
  18. chatcortex-0.7.0/chatcortex/registry/loader.py +93 -0
  19. chatcortex-0.7.0/chatcortex/registry/metadata.py +139 -0
  20. chatcortex-0.7.0/chatcortex/registry/package.py +167 -0
  21. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/synthesis/base.py +4 -4
  22. chatcortex-0.7.0/chatcortex/synthesis/beam_synthesizer.py +34 -0
  23. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/synthesis/budget.py +7 -7
  24. chatcortex-0.7.0/chatcortex/synthesis/engine.py +246 -0
  25. chatcortex-0.7.0/chatcortex/synthesis/exhaustive_synthesizer.py +24 -0
  26. chatcortex-0.7.0/chatcortex/synthesis/heuristic_synthesizer.py +55 -0
  27. chatcortex-0.7.0/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +30 -0
  28. chatcortex-0.7.0/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +42 -0
  29. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/synthesis/random_synthesizer.py +104 -89
  30. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/synthesis/task_specification.py +4 -5
  31. chatcortex-0.7.0/chatcortex/telemetry/calibration.py +114 -0
  32. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex/telemetry/logger.py +2 -4
  33. chatcortex-0.7.0/chatcortex.egg-info/PKG-INFO +208 -0
  34. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex.egg-info/SOURCES.txt +27 -8
  35. chatcortex-0.7.0/chatcortex.egg-info/entry_points.txt +2 -0
  36. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex.egg-info/requires.txt +3 -1
  37. {chatcortex-0.4.0 → chatcortex-0.7.0}/pyproject.toml +90 -72
  38. {chatcortex-0.4.0 → chatcortex-0.7.0}/setup.cfg +4 -4
  39. chatcortex-0.7.0/tests/test_agent_graph.py +63 -0
  40. chatcortex-0.7.0/tests/test_calibration.py +183 -0
  41. chatcortex-0.7.0/tests/test_closed_loop.py +283 -0
  42. chatcortex-0.7.0/tests/test_execution.py +30 -0
  43. chatcortex-0.7.0/tests/test_executor_backend.py +208 -0
  44. chatcortex-0.7.0/tests/test_node_ids.py +74 -0
  45. chatcortex-0.7.0/tests/test_openai_backend_install.py +151 -0
  46. chatcortex-0.7.0/tests/test_pareto.py +45 -0
  47. chatcortex-0.7.0/tests/test_registry.py +115 -0
  48. chatcortex-0.7.0/tests/test_registry_loader.py +126 -0
  49. chatcortex-0.7.0/tests/test_registry_pack.py +138 -0
  50. chatcortex-0.7.0/tests/test_regression_gates.py +474 -0
  51. chatcortex-0.7.0/tests/test_synthesizers.py +52 -0
  52. chatcortex-0.7.0/tests/test_task_specification.py +21 -0
  53. chatcortex-0.4.0/PKG-INFO +0 -354
  54. chatcortex-0.4.0/README.md +0 -321
  55. chatcortex-0.4.0/chatcortex/__init__.py +0 -33
  56. chatcortex-0.4.0/chatcortex/budget_sweep_experiment.py +0 -311
  57. chatcortex-0.4.0/chatcortex/demo_phase1.py +0 -142
  58. chatcortex-0.4.0/chatcortex/demo_phase2.py +0 -122
  59. chatcortex-0.4.0/chatcortex/demo_phase3.py +0 -442
  60. chatcortex-0.4.0/chatcortex/execution/executor.py +0 -64
  61. chatcortex-0.4.0/chatcortex/registry/capability_registry.py +0 -62
  62. chatcortex-0.4.0/chatcortex/registry/metadata.py +0 -44
  63. chatcortex-0.4.0/chatcortex/synthesis/beam_synthesizer.py +0 -113
  64. chatcortex-0.4.0/chatcortex/synthesis/exhaustive_synthesizer.py +0 -91
  65. chatcortex-0.4.0/chatcortex/synthesis/heuristic_synthesizer.py +0 -111
  66. chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -142
  67. chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer_v2.py +0 -195
  68. chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer_v3.py +0 -199
  69. chatcortex-0.4.0/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -141
  70. chatcortex-0.4.0/chatcortex/synthesis/system_score_beam_synthesizer.py +0 -118
  71. chatcortex-0.4.0/chatcortex.egg-info/PKG-INFO +0 -354
  72. {chatcortex-0.4.0 → chatcortex-0.7.0}/LICENSE +0 -0
  73. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex.egg-info/dependency_links.txt +0 -0
  74. {chatcortex-0.4.0 → chatcortex-0.7.0}/chatcortex.egg-info/top_level.txt +0 -0
@@ -0,0 +1,208 @@
1
+ Metadata-Version: 2.4
2
+ Name: chatcortex
3
+ Version: 0.7.0
4
+ Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
5
+ Author: Siddharth Saraswat
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/siddharth1012/chatCortex
8
+ Project-URL: Repository, https://github.com/siddharth1012/chatCortex
9
+ Project-URL: Issues, https://github.com/siddharth1012/chatCortex/issues
10
+ Keywords: automl,multi-objective-optimization,agent-architecture,llm-agents,pareto-optimization,architecture-search,beam-search,ai-systems
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Classifier: Topic :: Software Development :: Libraries
22
+ Requires-Python: >=3.9
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: networkx>=2.8
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=7.0; extra == "dev"
28
+ Requires-Dist: black>=23.0; extra == "dev"
29
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
30
+ Provides-Extra: viz
31
+ Requires-Dist: matplotlib>=3.7; extra == "viz"
32
+ Provides-Extra: real
33
+ Requires-Dist: openai>=2.48; extra == "real"
34
+ Dynamic: license-file
35
+
36
+ # ChatCortex
37
+
38
+ [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
39
+ [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
40
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
41
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
42
+
43
+ **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
44
+
45
+ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
46
+
47
+ ```
48
+ Task Specification → Architecture Search → Pareto-Optimal Architectures
49
+ ```
50
+
51
+ **ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
52
+
53
+ ## Installation
54
+
55
+ ```bash
56
+ pip install chatcortex
57
+ ```
58
+
59
+ Or from source:
60
+
61
+ ```bash
62
+ git clone https://github.com/siddharth1012/chatCortex
63
+ cd chatCortex
64
+ pip install -e .
65
+ ```
66
+
67
+ ## Quickstart
68
+
69
+ ```python
70
+ from chatcortex import (
71
+ TaskSpecification,
72
+ BeamSynthesizer,
73
+ build_demo_registry,
74
+ )
75
+
76
+ # A registry of components with cost/latency/reliability tradeoffs
77
+ registry = build_demo_registry(
78
+ capabilities=("retrieval", "generation", "verification")
79
+ )
80
+
81
+ # The task: an ordered capability chain with hard constraints
82
+ task = TaskSpecification(
83
+ required_capabilities=["retrieval", "generation", "verification"],
84
+ max_cost=0.02,
85
+ max_latency=1500,
86
+ )
87
+
88
+ # Synthesize Pareto-optimal architectures
89
+ synth = BeamSynthesizer(registry, beam_width=5)
90
+ for arch in synth.synthesize(task):
91
+ print(
92
+ f"cost=${arch.total_cost:.4f} "
93
+ f"latency={arch.total_latency:.0f}ms "
94
+ f"reliability={arch.total_reliability:.3f}"
95
+ )
96
+ ```
97
+
98
+ Or run the full demo comparing three synthesis strategies:
99
+
100
+ ```bash
101
+ python examples/quickstart.py
102
+ ```
103
+
104
+ To see the full loop — load a JSON registry, synthesize a frontier, execute one
105
+ architecture, and calibrate the registry's metadata from real telemetry — run:
106
+
107
+ ```bash
108
+ python examples/registry_from_json.py
109
+ ```
110
+
111
+ Run the flagship closed loop — synthesize → execute → calibrate →
112
+ re-synthesize, four iterations of measured behavior flowing back into the
113
+ registry:
114
+
115
+ ```bash
116
+ python examples/closed_loop.py
117
+ ```
118
+
119
+ With an OpenAI key you can run the same loop against real backends:
120
+
121
+ ```bash
122
+ pip install "chatcortex[real]"
123
+ export OPENAI_API_KEY=... # every reply spends your credits
124
+ python examples/closed_loop_real.py
125
+ ```
126
+
127
+ To search over your own components, build a `CapabilityRegistry` and register
128
+ `ComponentMetadata` describing each model, tool, or verifier — see
129
+ `examples/` for complete walkthroughs.
130
+
131
+ ## Registry packs
132
+
133
+ A registry ships as a JSON pack in the loader's document shape. Saving it
134
+ proves the round-trip, and one command validates any pack offline:
135
+
136
+ ```python
137
+ from chatcortex import build_demo_registry, save_registry, validate_registry_file
138
+
139
+ save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
140
+ print(validate_registry_file("registry.json").valid) # True
141
+ ```
142
+
143
+ ```bash
144
+ chatcortex-registry validate registry.json
145
+ # components validated: 12
146
+ # result: VALID (exit 0; issues and duplicates are named when invalid)
147
+ ```
148
+
149
+ ## How It Works
150
+
151
+ ChatCortex is organized as a layered architecture synthesis system:
152
+
153
+ ```
154
+ TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
155
+ → Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
156
+ ```
157
+
158
+ **Core concepts:**
159
+
160
+ - **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
161
+ - **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
162
+ - **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
163
+ - **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
164
+ - **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
165
+
166
+ **Available synthesizers:**
167
+
168
+ | Synthesizer | Strategy |
169
+ |---|---|
170
+ | `HeuristicSynthesizer` | Greedy deterministic construction |
171
+ | `RandomSynthesizer` | Random baseline |
172
+ | `BeamSynthesizer` | Budget-aware beam search |
173
+ | `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
174
+ | `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
175
+ | `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
176
+
177
+ Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
178
+
179
+ ## Documentation
180
+
181
+ - [Architecture](docs/ARCHITECTURE.md) — system design and component layers
182
+ - [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
183
+ - [Examples](examples/) — runnable demos and budget-sweep experiments
184
+ - [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
185
+ - [Changelog](CHANGELOG.md)
186
+
187
+ ## Research Context
188
+
189
+ ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
190
+
191
+ If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
192
+
193
+ ## Roadmap
194
+
195
+ - [x] Component modeling, capability registry, task specification, DAG representation
196
+ - [x] Exact Pareto frontier computation (exhaustive search)
197
+ - [x] Budget-aware beam search with Pareto pruning and progressive widening
198
+ - [ ] Graph-structured (non-chain) agent synthesis
199
+ - [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
200
+ - [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
201
+
202
+ ## Status
203
+
204
+ Research framework under active development (alpha). APIs may change between minor versions.
205
+
206
+ ## License
207
+
208
+ [MIT](LICENSE) — developed by **Siddharth Saraswat**
@@ -0,0 +1,173 @@
1
+ # ChatCortex
2
+
3
+ [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
4
+ [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
5
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+
8
+ **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
9
+
10
+ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
11
+
12
+ ```
13
+ Task Specification → Architecture Search → Pareto-Optimal Architectures
14
+ ```
15
+
16
+ **ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
17
+
18
+ ## Installation
19
+
20
+ ```bash
21
+ pip install chatcortex
22
+ ```
23
+
24
+ Or from source:
25
+
26
+ ```bash
27
+ git clone https://github.com/siddharth1012/chatCortex
28
+ cd chatCortex
29
+ pip install -e .
30
+ ```
31
+
32
+ ## Quickstart
33
+
34
+ ```python
35
+ from chatcortex import (
36
+ TaskSpecification,
37
+ BeamSynthesizer,
38
+ build_demo_registry,
39
+ )
40
+
41
+ # A registry of components with cost/latency/reliability tradeoffs
42
+ registry = build_demo_registry(
43
+ capabilities=("retrieval", "generation", "verification")
44
+ )
45
+
46
+ # The task: an ordered capability chain with hard constraints
47
+ task = TaskSpecification(
48
+ required_capabilities=["retrieval", "generation", "verification"],
49
+ max_cost=0.02,
50
+ max_latency=1500,
51
+ )
52
+
53
+ # Synthesize Pareto-optimal architectures
54
+ synth = BeamSynthesizer(registry, beam_width=5)
55
+ for arch in synth.synthesize(task):
56
+ print(
57
+ f"cost=${arch.total_cost:.4f} "
58
+ f"latency={arch.total_latency:.0f}ms "
59
+ f"reliability={arch.total_reliability:.3f}"
60
+ )
61
+ ```
62
+
63
+ Or run the full demo comparing three synthesis strategies:
64
+
65
+ ```bash
66
+ python examples/quickstart.py
67
+ ```
68
+
69
+ To see the full loop — load a JSON registry, synthesize a frontier, execute one
70
+ architecture, and calibrate the registry's metadata from real telemetry — run:
71
+
72
+ ```bash
73
+ python examples/registry_from_json.py
74
+ ```
75
+
76
+ Run the flagship closed loop — synthesize → execute → calibrate →
77
+ re-synthesize, four iterations of measured behavior flowing back into the
78
+ registry:
79
+
80
+ ```bash
81
+ python examples/closed_loop.py
82
+ ```
83
+
84
+ With an OpenAI key you can run the same loop against real backends:
85
+
86
+ ```bash
87
+ pip install "chatcortex[real]"
88
+ export OPENAI_API_KEY=... # every reply spends your credits
89
+ python examples/closed_loop_real.py
90
+ ```
91
+
92
+ To search over your own components, build a `CapabilityRegistry` and register
93
+ `ComponentMetadata` describing each model, tool, or verifier — see
94
+ `examples/` for complete walkthroughs.
95
+
96
+ ## Registry packs
97
+
98
+ A registry ships as a JSON pack in the loader's document shape. Saving it
99
+ proves the round-trip, and one command validates any pack offline:
100
+
101
+ ```python
102
+ from chatcortex import build_demo_registry, save_registry, validate_registry_file
103
+
104
+ save_registry(build_demo_registry(), "registry.json") # writes, then reloads to prove it
105
+ print(validate_registry_file("registry.json").valid) # True
106
+ ```
107
+
108
+ ```bash
109
+ chatcortex-registry validate registry.json
110
+ # components validated: 12
111
+ # result: VALID (exit 0; issues and duplicates are named when invalid)
112
+ ```
113
+
114
+ ## How It Works
115
+
116
+ ChatCortex is organized as a layered architecture synthesis system:
117
+
118
+ ```
119
+ TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
120
+ → Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
121
+ ```
122
+
123
+ **Core concepts:**
124
+
125
+ - **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
126
+ - **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
127
+ - **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
128
+ - **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
129
+ - **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
130
+
131
+ **Available synthesizers:**
132
+
133
+ | Synthesizer | Strategy |
134
+ |---|---|
135
+ | `HeuristicSynthesizer` | Greedy deterministic construction |
136
+ | `RandomSynthesizer` | Random baseline |
137
+ | `BeamSynthesizer` | Budget-aware beam search |
138
+ | `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
139
+ | `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
140
+ | `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
141
+
142
+ Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
143
+
144
+ ## Documentation
145
+
146
+ - [Architecture](docs/ARCHITECTURE.md) — system design and component layers
147
+ - [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
148
+ - [Examples](examples/) — runnable demos and budget-sweep experiments
149
+ - [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
150
+ - [Changelog](CHANGELOG.md)
151
+
152
+ ## Research Context
153
+
154
+ ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
155
+
156
+ If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
157
+
158
+ ## Roadmap
159
+
160
+ - [x] Component modeling, capability registry, task specification, DAG representation
161
+ - [x] Exact Pareto frontier computation (exhaustive search)
162
+ - [x] Budget-aware beam search with Pareto pruning and progressive widening
163
+ - [ ] Graph-structured (non-chain) agent synthesis
164
+ - [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
165
+ - [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
166
+
167
+ ## Status
168
+
169
+ Research framework under active development (alpha). APIs may change between minor versions.
170
+
171
+ ## License
172
+
173
+ [MIT](LICENSE) — developed by **Siddharth Saraswat**
@@ -0,0 +1,54 @@
1
+ """
2
+ ChatCortex
3
+
4
+ Framework for automated synthesis and optimization of AI agent
5
+ architectures under multi-objective constraints.
6
+
7
+ Core capabilities include:
8
+
9
+ - Task specification
10
+ - Agent architecture synthesis
11
+ - Multi-objective optimization
12
+ - Pareto frontier analysis
13
+ """
14
+
15
+ from .graph.agent_graph import AgentGraph
16
+ from .loop import LoopIteration, run_closed_loop
17
+ from .registry.capability_registry import CapabilityRegistry
18
+ from .registry.demo_components import build_demo_registry
19
+ from .registry.loader import load_registry
20
+ from .registry.metadata import ComponentMetadata
21
+ from .registry.package import RegistryReport, save_registry, validate_registry_file
22
+ from .synthesis.beam_synthesizer import BeamSynthesizer
23
+ from .synthesis.budget import SynthesisBudget
24
+ from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
25
+ from .synthesis.heuristic_synthesizer import HeuristicSynthesizer
26
+ from .synthesis.pareto_partial_beam_synthesizer import ParetoPartialBeamSynthesizer
27
+ from .synthesis.progressive_pareto_beam_synthesizer import (
28
+ ProgressiveParetoBeamSynthesizer,
29
+ )
30
+ from .synthesis.random_synthesizer import RandomSynthesizer
31
+ from .synthesis.task_specification import TaskSpecification
32
+
33
+ __all__ = [
34
+ "TaskSpecification",
35
+ "HeuristicSynthesizer",
36
+ "RandomSynthesizer",
37
+ "BeamSynthesizer",
38
+ "ExhaustiveSynthesizer",
39
+ "ParetoPartialBeamSynthesizer",
40
+ "ProgressiveParetoBeamSynthesizer",
41
+ "SynthesisBudget",
42
+ "AgentGraph",
43
+ "CapabilityRegistry",
44
+ "LoopIteration",
45
+ "ComponentMetadata",
46
+ "build_demo_registry",
47
+ "load_registry",
48
+ "run_closed_loop",
49
+ "save_registry",
50
+ "validate_registry_file",
51
+ "RegistryReport",
52
+ ]
53
+
54
+ __version__ = "0.7.0"
@@ -0,0 +1,7 @@
1
+ """Optional real-execution backends.
2
+
3
+ These require extra dependencies (see the ``real`` extra in
4
+ ``pyproject.toml``); importing this package never pulls them in. The
5
+ heavy imports happen inside :class:`OpenAIBackend.__init__`, so the
6
+ zero-dependency core stays untouched.
7
+ """
@@ -0,0 +1,106 @@
1
+ """OpenAI chat-completions execution backend (optional ``real`` extra).
2
+
3
+ Installation: ``pip install chatcortex[real]`` — this module imports
4
+ ``openai`` lazily inside :func:`OpenAIBackend.__init__`, so the package
5
+ never imports it unless a backend is actually built. The API key comes
6
+ from ``OPENAI_API_KEY`` (the SDK's standard env resolution); tests can
7
+ inject a stub ``client`` instead and need no key or network.
8
+
9
+ What is measured vs. declared:
10
+
11
+ - **Latency** and **success** are measured from the real SDK call.
12
+ - **Cost** is not derivable from a completion alone (token pricing is
13
+ model-specific and changes over time), so the component's declared
14
+ ``cost_per_call`` is recorded. Component metadata stays pure: nothing
15
+ here mutates it, calibration is the component that turns telemetry
16
+ into updated metadata.
17
+ """
18
+
19
+ import time
20
+ from typing import Any, Callable, Optional
21
+
22
+ from ..execution.executor import ProbeResult
23
+ from ..registry.metadata import ComponentMetadata
24
+
25
+ DEFAULT_MODEL = "gpt-5.5-mini"
26
+
27
+
28
+ class OpenAIBackend:
29
+ """Runs one graph node as one chat-completions API call.
30
+
31
+ Args:
32
+ model: Model name used for every component; ignored when
33
+ ``model_for`` is supplied.
34
+ model_for: Optional ``Callable[[ComponentMetadata], str]`` that
35
+ maps a component to the model it should run on (e.g. cheap
36
+ models for retrieval tooling, stronger models for
37
+ generation); takes precedence over ``model``.
38
+ client: Optional pre-built OpenAI client (or any object exposing
39
+ ``chat.completions.create``). Meant for tests; when given,
40
+ the ``openai`` package is never imported.
41
+ """
42
+
43
+ def __init__(
44
+ self,
45
+ model: str = DEFAULT_MODEL,
46
+ model_for: Optional[Callable[[ComponentMetadata], str]] = None,
47
+ client: Optional[Any] = None,
48
+ ):
49
+ self._model = model
50
+ self.model_for = model_for
51
+ if client is not None:
52
+ self._client = client
53
+ else:
54
+ try:
55
+ import openai
56
+ except ImportError as exc: # pragma: no cover - exercised only without extra
57
+ raise ImportError(
58
+ "OpenAIBackend requires the optional 'openai' dependency; "
59
+ "install it with: pip install chatcortex[real]"
60
+ ) from exc
61
+ self._client = openai.OpenAI()
62
+
63
+ def _model_for_component(self, metadata: ComponentMetadata) -> str:
64
+ if self.model_for is not None:
65
+ return self.model_for(metadata)
66
+ return self._model
67
+
68
+ def __call__(
69
+ self,
70
+ metadata: ComponentMetadata,
71
+ node_input: Any = None,
72
+ ) -> ProbeResult:
73
+ model = self._model_for_component(metadata)
74
+ prompt = str(node_input) if node_input is not None else (
75
+ f"Perform the '{metadata.capabilities[0]}' task as component "
76
+ f"'{metadata.name}'."
77
+ )
78
+
79
+ started = time.perf_counter()
80
+ try:
81
+ response = self._client.chat.completions.create(
82
+ model=model,
83
+ messages=[
84
+ {
85
+ "role": "user",
86
+ "content": prompt,
87
+ }
88
+ ],
89
+ )
90
+ except Exception as exc: # network/HTTP/key errors are probe failures
91
+ latency_ms = (time.perf_counter() - started) * 1000.0
92
+ return ProbeResult(
93
+ success=False,
94
+ latency_ms=latency_ms,
95
+ cost=0.0,
96
+ output=f"openai call failed: {type(exc).__name__}: {exc}",
97
+ )
98
+
99
+ latency_ms = (time.perf_counter() - started) * 1000.0
100
+ output = response.choices[0].message.content
101
+ return ProbeResult(
102
+ success=bool(output),
103
+ latency_ms=latency_ms,
104
+ cost=metadata.cost_per_call,
105
+ output=output,
106
+ )
@@ -1,4 +1,3 @@
1
- import random
2
1
  from typing import Iterable, List, Tuple
3
2
 
4
3
  from chatcortex.optimization.architecture_candidate import ArchitectureCandidate
@@ -11,13 +10,9 @@ def compute_coverage(
11
10
  ) -> float:
12
11
  return frontier_coverage(approx_frontier, true_frontier)
13
12
 
14
- def compute_hypervolume(
15
- frontier: Iterable[ArchitectureCandidate],
16
- reference_point: Tuple[float, float, float],
17
- ) -> float:
18
- return hypervolume_monte_carlo(frontier, reference_point)
19
-
20
13
 
14
+ # Note: one seed per estimate, not a shared stream — the two estimates below
15
+ # are independent Monte Carlo runs of the same algorithm.
21
16
  def hypervolume_loss(
22
17
  approx_frontier,
23
18
  true_frontier,
@@ -25,42 +20,12 @@ def hypervolume_loss(
25
20
  num_samples=200000,
26
21
  seed=42,
27
22
  ) -> float:
28
- ref_cost, ref_latency, ref_reliability = reference_point
29
-
30
- random_number_generator = random.Random(seed)
31
-
32
- dominated_true = 0
33
- dominated_approx = 0
34
-
35
- for _ in range(num_samples):
36
-
37
- sample_cost = random_number_generator.uniform(0, ref_cost)
38
- sample_latency = random_number_generator.uniform(0, ref_latency)
39
- sample_reliability = random_number_generator.uniform(ref_reliability, 1.0)
40
-
41
- for candidate in true_frontier:
42
- if (
43
- candidate.total_cost <= sample_cost
44
- and candidate.total_latency <= sample_latency
45
- and candidate.total_reliability >= sample_reliability
46
- ):
47
- dominated_true += 1
48
- break
49
-
50
- for candidate in approx_frontier:
51
- if (
52
- candidate.total_cost <= sample_cost
53
- and candidate.total_latency <= sample_latency
54
- and candidate.total_reliability >= sample_reliability
55
- ):
56
- dominated_approx += 1
57
- break
58
-
59
-
60
- box_volume = ref_cost * ref_latency * (1.0 - ref_reliability)
61
-
62
- hv_true = (dominated_true / num_samples) * box_volume
63
- hv_approx = (dominated_approx / num_samples) * box_volume
23
+ hv_true = hypervolume_monte_carlo(
24
+ true_frontier, reference_point, num_samples=num_samples, seed=seed
25
+ )
26
+ hv_approx = hypervolume_monte_carlo(
27
+ approx_frontier, reference_point, num_samples=num_samples, seed=seed
28
+ )
64
29
 
65
30
  return max(0.0, hv_true - hv_approx)
66
31
 
@@ -79,7 +44,7 @@ def additive_regret(
79
44
 
80
45
  if not dominating:
81
46
  return (0.0, 0.0, 0.0)
82
-
47
+
83
48
  min_cost_diff = min(candidate.total_cost - p.total_cost for p in dominating)
84
49
  min_latency_diff = min(candidate.total_latency - p.total_latency for p in dominating)
85
50
  min_reliability_diff = min(p.total_reliability - candidate.total_reliability for p in dominating)
@@ -112,7 +77,7 @@ def average_regret(
112
77
  total_cost_regret += cr
113
78
  total_latency_regret += lr
114
79
  total_reliability_regret += rr
115
-
80
+
116
81
  n = len(approx_list)
117
82
 
118
83
  return (
@@ -131,7 +96,7 @@ def evaluate_approximation(
131
96
  """
132
97
 
133
98
  coverage = compute_coverage(approx_frontier, true_frontier)
134
-
99
+
135
100
  hv_loss = hypervolume_loss(approx_frontier, true_frontier, reference_point)
136
101
  avg_regret = average_regret(approx_frontier, true_frontier)
137
102
 
@@ -141,4 +106,4 @@ def evaluate_approximation(
141
106
  "avg_cost_regret": avg_regret[0],
142
107
  "avg_latency_regret": avg_regret[1],
143
108
  "avg_reliability_regret": avg_regret[2],
144
- }
109
+ }