chatcortex 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chatcortex-0.6.0/PKG-INFO +172 -0
- chatcortex-0.6.0/README.md +137 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/__init__.py +47 -33
- chatcortex-0.6.0/chatcortex/adapters/__init__.py +7 -0
- chatcortex-0.6.0/chatcortex/adapters/openai_backend.py +106 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/evaluation/approximation_metrics.py +12 -47
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/evaluation/harness.py +95 -90
- chatcortex-0.6.0/chatcortex/execution/executor.py +146 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/graph/agent_graph.py +30 -12
- chatcortex-0.6.0/chatcortex/graph/node_ids.py +27 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/optimization/architecture_candidate.py +1 -1
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/optimization/pareto.py +42 -32
- chatcortex-0.6.0/chatcortex/registry/capability_registry.py +93 -0
- chatcortex-0.6.0/chatcortex/registry/demo_components.py +61 -0
- chatcortex-0.6.0/chatcortex/registry/loader.py +93 -0
- chatcortex-0.6.0/chatcortex/registry/metadata.py +139 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/synthesis/base.py +4 -4
- chatcortex-0.6.0/chatcortex/synthesis/beam_synthesizer.py +34 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/synthesis/budget.py +7 -7
- chatcortex-0.6.0/chatcortex/synthesis/engine.py +246 -0
- chatcortex-0.6.0/chatcortex/synthesis/exhaustive_synthesizer.py +24 -0
- chatcortex-0.6.0/chatcortex/synthesis/heuristic_synthesizer.py +55 -0
- chatcortex-0.6.0/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +30 -0
- chatcortex-0.6.0/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +42 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/synthesis/random_synthesizer.py +104 -89
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/synthesis/task_specification.py +4 -5
- chatcortex-0.6.0/chatcortex/telemetry/calibration.py +110 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex/telemetry/logger.py +2 -4
- chatcortex-0.6.0/chatcortex.egg-info/PKG-INFO +172 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex.egg-info/SOURCES.txt +20 -8
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex.egg-info/requires.txt +3 -1
- {chatcortex-0.4.0 → chatcortex-0.6.0}/pyproject.toml +87 -72
- {chatcortex-0.4.0 → chatcortex-0.6.0}/setup.cfg +4 -4
- chatcortex-0.6.0/tests/test_agent_graph.py +63 -0
- chatcortex-0.6.0/tests/test_calibration.py +173 -0
- chatcortex-0.6.0/tests/test_execution.py +30 -0
- chatcortex-0.6.0/tests/test_executor_backend.py +208 -0
- chatcortex-0.6.0/tests/test_node_ids.py +74 -0
- chatcortex-0.6.0/tests/test_openai_backend_install.py +151 -0
- chatcortex-0.6.0/tests/test_pareto.py +45 -0
- chatcortex-0.6.0/tests/test_registry.py +115 -0
- chatcortex-0.6.0/tests/test_registry_loader.py +126 -0
- chatcortex-0.6.0/tests/test_regression_gates.py +474 -0
- chatcortex-0.6.0/tests/test_synthesizers.py +52 -0
- chatcortex-0.6.0/tests/test_task_specification.py +21 -0
- chatcortex-0.4.0/PKG-INFO +0 -354
- chatcortex-0.4.0/README.md +0 -321
- chatcortex-0.4.0/chatcortex/budget_sweep_experiment.py +0 -311
- chatcortex-0.4.0/chatcortex/demo_phase1.py +0 -142
- chatcortex-0.4.0/chatcortex/demo_phase2.py +0 -122
- chatcortex-0.4.0/chatcortex/demo_phase3.py +0 -442
- chatcortex-0.4.0/chatcortex/execution/executor.py +0 -64
- chatcortex-0.4.0/chatcortex/registry/capability_registry.py +0 -62
- chatcortex-0.4.0/chatcortex/registry/metadata.py +0 -44
- chatcortex-0.4.0/chatcortex/synthesis/beam_synthesizer.py +0 -113
- chatcortex-0.4.0/chatcortex/synthesis/exhaustive_synthesizer.py +0 -91
- chatcortex-0.4.0/chatcortex/synthesis/heuristic_synthesizer.py +0 -111
- chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -142
- chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer_v2.py +0 -195
- chatcortex-0.4.0/chatcortex/synthesis/pareto_partial_beam_synthesizer_v3.py +0 -199
- chatcortex-0.4.0/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -141
- chatcortex-0.4.0/chatcortex/synthesis/system_score_beam_synthesizer.py +0 -118
- chatcortex-0.4.0/chatcortex.egg-info/PKG-INFO +0 -354
- {chatcortex-0.4.0 → chatcortex-0.6.0}/LICENSE +0 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.4.0 → chatcortex-0.6.0}/chatcortex.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: chatcortex
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
|
|
5
|
+
Author: Siddharth Saraswat
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/siddharth1012/chatCortex
|
|
8
|
+
Project-URL: Repository, https://github.com/siddharth1012/chatCortex
|
|
9
|
+
Project-URL: Issues, https://github.com/siddharth1012/chatCortex/issues
|
|
10
|
+
Keywords: automl,multi-objective-optimization,agent-architecture,llm-agents,pareto-optimization,architecture-search,beam-search,ai-systems
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: networkx>=2.8
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
28
|
+
Requires-Dist: black>=23.0; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
30
|
+
Provides-Extra: viz
|
|
31
|
+
Requires-Dist: matplotlib>=3.7; extra == "viz"
|
|
32
|
+
Provides-Extra: real
|
|
33
|
+
Requires-Dist: openai>=2.48; extra == "real"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# ChatCortex
|
|
37
|
+
|
|
38
|
+
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
39
|
+
[](https://pypi.org/project/chatcortex/)
|
|
40
|
+
[](https://www.python.org/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
44
|
+
|
|
45
|
+
Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
|
|
46
|
+
|
|
47
|
+
```
|
|
48
|
+
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Installation
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install chatcortex
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Or from source:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
git clone https://github.com/siddharth1012/chatCortex
|
|
61
|
+
cd chatCortex
|
|
62
|
+
pip install -e .
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Quickstart
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from chatcortex import (
|
|
69
|
+
TaskSpecification,
|
|
70
|
+
BeamSynthesizer,
|
|
71
|
+
build_demo_registry,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# A registry of components with cost/latency/reliability tradeoffs
|
|
75
|
+
registry = build_demo_registry(
|
|
76
|
+
capabilities=("retrieval", "generation", "verification")
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# The task: an ordered capability chain with hard constraints
|
|
80
|
+
task = TaskSpecification(
|
|
81
|
+
required_capabilities=["retrieval", "generation", "verification"],
|
|
82
|
+
max_cost=0.02,
|
|
83
|
+
max_latency=1500,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# Synthesize Pareto-optimal architectures
|
|
87
|
+
synth = BeamSynthesizer(registry, beam_width=5)
|
|
88
|
+
for arch in synth.synthesize(task):
|
|
89
|
+
print(
|
|
90
|
+
f"cost=${arch.total_cost:.4f} "
|
|
91
|
+
f"latency={arch.total_latency:.0f}ms "
|
|
92
|
+
f"reliability={arch.total_reliability:.3f}"
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Or run the full demo comparing three synthesis strategies:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
python examples/quickstart.py
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
To see the full loop — load a JSON registry, synthesize a frontier, execute one
|
|
103
|
+
architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
python examples/registry_from_json.py
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
To search over your own components, build a `CapabilityRegistry` and register
|
|
110
|
+
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
111
|
+
`examples/` for complete walkthroughs.
|
|
112
|
+
|
|
113
|
+
## How It Works
|
|
114
|
+
|
|
115
|
+
ChatCortex is organized as a layered architecture synthesis system:
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
|
|
119
|
+
→ Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Core concepts:**
|
|
123
|
+
|
|
124
|
+
- **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
|
|
125
|
+
- **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
|
|
126
|
+
- **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
|
|
127
|
+
- **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
|
|
128
|
+
- **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
|
|
129
|
+
|
|
130
|
+
**Available synthesizers:**
|
|
131
|
+
|
|
132
|
+
| Synthesizer | Strategy |
|
|
133
|
+
|---|---|
|
|
134
|
+
| `HeuristicSynthesizer` | Greedy deterministic construction |
|
|
135
|
+
| `RandomSynthesizer` | Random baseline |
|
|
136
|
+
| `BeamSynthesizer` | Budget-aware beam search |
|
|
137
|
+
| `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
|
|
138
|
+
| `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
|
|
139
|
+
| `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
|
|
140
|
+
|
|
141
|
+
Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
|
|
142
|
+
|
|
143
|
+
## Documentation
|
|
144
|
+
|
|
145
|
+
- [Architecture](docs/ARCHITECTURE.md) — system design and component layers
|
|
146
|
+
- [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
|
|
147
|
+
- [Examples](examples/) — runnable demos and budget-sweep experiments
|
|
148
|
+
- [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
|
|
149
|
+
- [Changelog](CHANGELOG.md)
|
|
150
|
+
|
|
151
|
+
## Research Context
|
|
152
|
+
|
|
153
|
+
ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
|
|
154
|
+
|
|
155
|
+
If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
|
|
156
|
+
|
|
157
|
+
## Roadmap
|
|
158
|
+
|
|
159
|
+
- [x] Component modeling, capability registry, task specification, DAG representation
|
|
160
|
+
- [x] Exact Pareto frontier computation (exhaustive search)
|
|
161
|
+
- [x] Budget-aware beam search with Pareto pruning and progressive widening
|
|
162
|
+
- [ ] Graph-structured (non-chain) agent synthesis
|
|
163
|
+
- [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
|
|
164
|
+
- [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
|
|
165
|
+
|
|
166
|
+
## Status
|
|
167
|
+
|
|
168
|
+
Research framework under active development (alpha). APIs may change between minor versions.
|
|
169
|
+
|
|
170
|
+
## License
|
|
171
|
+
|
|
172
|
+
[MIT](LICENSE) — developed by **Siddharth Saraswat**
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# ChatCortex
|
|
2
|
+
|
|
3
|
+
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/chatcortex/)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
9
|
+
|
|
10
|
+
Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install chatcortex
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Or from source:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
git clone https://github.com/siddharth1012/chatCortex
|
|
26
|
+
cd chatCortex
|
|
27
|
+
pip install -e .
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Quickstart
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from chatcortex import (
|
|
34
|
+
TaskSpecification,
|
|
35
|
+
BeamSynthesizer,
|
|
36
|
+
build_demo_registry,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
# A registry of components with cost/latency/reliability tradeoffs
|
|
40
|
+
registry = build_demo_registry(
|
|
41
|
+
capabilities=("retrieval", "generation", "verification")
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
# The task: an ordered capability chain with hard constraints
|
|
45
|
+
task = TaskSpecification(
|
|
46
|
+
required_capabilities=["retrieval", "generation", "verification"],
|
|
47
|
+
max_cost=0.02,
|
|
48
|
+
max_latency=1500,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# Synthesize Pareto-optimal architectures
|
|
52
|
+
synth = BeamSynthesizer(registry, beam_width=5)
|
|
53
|
+
for arch in synth.synthesize(task):
|
|
54
|
+
print(
|
|
55
|
+
f"cost=${arch.total_cost:.4f} "
|
|
56
|
+
f"latency={arch.total_latency:.0f}ms "
|
|
57
|
+
f"reliability={arch.total_reliability:.3f}"
|
|
58
|
+
)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Or run the full demo comparing three synthesis strategies:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
python examples/quickstart.py
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
To see the full loop — load a JSON registry, synthesize a frontier, execute one
|
|
68
|
+
architecture, and calibrate the registry's metadata from real telemetry — run:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
python examples/registry_from_json.py
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
To search over your own components, build a `CapabilityRegistry` and register
|
|
75
|
+
`ComponentMetadata` describing each model, tool, or verifier — see
|
|
76
|
+
`examples/` for complete walkthroughs.
|
|
77
|
+
|
|
78
|
+
## How It Works
|
|
79
|
+
|
|
80
|
+
ChatCortex is organized as a layered architecture synthesis system:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
|
|
84
|
+
→ Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
**Core concepts:**
|
|
88
|
+
|
|
89
|
+
- **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
|
|
90
|
+
- **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
|
|
91
|
+
- **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
|
|
92
|
+
- **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
|
|
93
|
+
- **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
|
|
94
|
+
|
|
95
|
+
**Available synthesizers:**
|
|
96
|
+
|
|
97
|
+
| Synthesizer | Strategy |
|
|
98
|
+
|---|---|
|
|
99
|
+
| `HeuristicSynthesizer` | Greedy deterministic construction |
|
|
100
|
+
| `RandomSynthesizer` | Random baseline |
|
|
101
|
+
| `BeamSynthesizer` | Budget-aware beam search |
|
|
102
|
+
| `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
|
|
103
|
+
| `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
|
|
104
|
+
| `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
|
|
105
|
+
|
|
106
|
+
Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
|
|
107
|
+
|
|
108
|
+
## Documentation
|
|
109
|
+
|
|
110
|
+
- [Architecture](docs/ARCHITECTURE.md) — system design and component layers
|
|
111
|
+
- [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
|
|
112
|
+
- [Examples](examples/) — runnable demos and budget-sweep experiments
|
|
113
|
+
- [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
|
|
114
|
+
- [Changelog](CHANGELOG.md)
|
|
115
|
+
|
|
116
|
+
## Research Context
|
|
117
|
+
|
|
118
|
+
ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
|
|
119
|
+
|
|
120
|
+
If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
|
|
121
|
+
|
|
122
|
+
## Roadmap
|
|
123
|
+
|
|
124
|
+
- [x] Component modeling, capability registry, task specification, DAG representation
|
|
125
|
+
- [x] Exact Pareto frontier computation (exhaustive search)
|
|
126
|
+
- [x] Budget-aware beam search with Pareto pruning and progressive widening
|
|
127
|
+
- [ ] Graph-structured (non-chain) agent synthesis
|
|
128
|
+
- [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
|
|
129
|
+
- [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
|
|
130
|
+
|
|
131
|
+
## Status
|
|
132
|
+
|
|
133
|
+
Research framework under active development (alpha). APIs may change between minor versions.
|
|
134
|
+
|
|
135
|
+
## License
|
|
136
|
+
|
|
137
|
+
[MIT](LICENSE) — developed by **Siddharth Saraswat**
|
|
@@ -1,33 +1,47 @@
|
|
|
1
|
-
"""
|
|
2
|
-
ChatCortex
|
|
3
|
-
|
|
4
|
-
Framework for automated synthesis and optimization of AI agent
|
|
5
|
-
architectures under multi-objective constraints.
|
|
6
|
-
|
|
7
|
-
Core capabilities include:
|
|
8
|
-
|
|
9
|
-
- Task specification
|
|
10
|
-
- Agent architecture synthesis
|
|
11
|
-
- Multi-objective optimization
|
|
12
|
-
- Pareto frontier analysis
|
|
13
|
-
"""
|
|
14
|
-
|
|
15
|
-
from .
|
|
16
|
-
|
|
17
|
-
from .
|
|
18
|
-
from .
|
|
19
|
-
from .
|
|
20
|
-
from .synthesis.
|
|
21
|
-
|
|
22
|
-
from .
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
1
|
+
"""
|
|
2
|
+
ChatCortex
|
|
3
|
+
|
|
4
|
+
Framework for automated synthesis and optimization of AI agent
|
|
5
|
+
architectures under multi-objective constraints.
|
|
6
|
+
|
|
7
|
+
Core capabilities include:
|
|
8
|
+
|
|
9
|
+
- Task specification
|
|
10
|
+
- Agent architecture synthesis
|
|
11
|
+
- Multi-objective optimization
|
|
12
|
+
- Pareto frontier analysis
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from .graph.agent_graph import AgentGraph
|
|
16
|
+
from .registry.capability_registry import CapabilityRegistry
|
|
17
|
+
from .registry.demo_components import build_demo_registry
|
|
18
|
+
from .registry.loader import load_registry
|
|
19
|
+
from .registry.metadata import ComponentMetadata
|
|
20
|
+
from .synthesis.beam_synthesizer import BeamSynthesizer
|
|
21
|
+
from .synthesis.budget import SynthesisBudget
|
|
22
|
+
from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
23
|
+
from .synthesis.heuristic_synthesizer import HeuristicSynthesizer
|
|
24
|
+
from .synthesis.pareto_partial_beam_synthesizer import ParetoPartialBeamSynthesizer
|
|
25
|
+
from .synthesis.progressive_pareto_beam_synthesizer import (
|
|
26
|
+
ProgressiveParetoBeamSynthesizer,
|
|
27
|
+
)
|
|
28
|
+
from .synthesis.random_synthesizer import RandomSynthesizer
|
|
29
|
+
from .synthesis.task_specification import TaskSpecification
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"TaskSpecification",
|
|
33
|
+
"HeuristicSynthesizer",
|
|
34
|
+
"RandomSynthesizer",
|
|
35
|
+
"BeamSynthesizer",
|
|
36
|
+
"ExhaustiveSynthesizer",
|
|
37
|
+
"ParetoPartialBeamSynthesizer",
|
|
38
|
+
"ProgressiveParetoBeamSynthesizer",
|
|
39
|
+
"SynthesisBudget",
|
|
40
|
+
"AgentGraph",
|
|
41
|
+
"CapabilityRegistry",
|
|
42
|
+
"ComponentMetadata",
|
|
43
|
+
"build_demo_registry",
|
|
44
|
+
"load_registry",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
__version__ = "0.6.0"
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Optional real-execution backends.
|
|
2
|
+
|
|
3
|
+
These require extra dependencies (see the ``real`` extra in
|
|
4
|
+
``pyproject.toml``); importing this package never pulls them in. The
|
|
5
|
+
heavy imports happen inside :class:`OpenAIBackend.__init__`, so the
|
|
6
|
+
zero-dependency core stays untouched.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""OpenAI chat-completions execution backend (optional ``real`` extra).
|
|
2
|
+
|
|
3
|
+
Installation: ``pip install chatcortex[real]`` — this module imports
|
|
4
|
+
``openai`` lazily inside :func:`OpenAIBackend.__init__`, so the package
|
|
5
|
+
never imports it unless a backend is actually built. The API key comes
|
|
6
|
+
from ``OPENAI_API_KEY`` (the SDK's standard env resolution); tests can
|
|
7
|
+
inject a stub ``client`` instead and need no key or network.
|
|
8
|
+
|
|
9
|
+
What is measured vs. declared:
|
|
10
|
+
|
|
11
|
+
- **Latency** and **success** are measured from the real SDK call.
|
|
12
|
+
- **Cost** is not derivable from a completion alone (token pricing is
|
|
13
|
+
model-specific and changes over time), so the component's declared
|
|
14
|
+
``cost_per_call`` is recorded. Component metadata stays pure: nothing
|
|
15
|
+
here mutates it, calibration is the component that turns telemetry
|
|
16
|
+
into updated metadata.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any, Callable, Optional
|
|
21
|
+
|
|
22
|
+
from ..execution.executor import ProbeResult
|
|
23
|
+
from ..registry.metadata import ComponentMetadata
|
|
24
|
+
|
|
25
|
+
DEFAULT_MODEL = "gpt-5.5-mini"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class OpenAIBackend:
|
|
29
|
+
"""Runs one graph node as one chat-completions API call.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
model: Model name used for every component; ignored when
|
|
33
|
+
``model_for`` is supplied.
|
|
34
|
+
model_for: Optional ``Callable[[ComponentMetadata], str]`` that
|
|
35
|
+
maps a component to the model it should run on (e.g. cheap
|
|
36
|
+
models for retrieval tooling, stronger models for
|
|
37
|
+
generation); takes precedence over ``model``.
|
|
38
|
+
client: Optional pre-built OpenAI client (or any object exposing
|
|
39
|
+
``chat.completions.create``). Meant for tests; when given,
|
|
40
|
+
the ``openai`` package is never imported.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
model: str = DEFAULT_MODEL,
|
|
46
|
+
model_for: Optional[Callable[[ComponentMetadata], str]] = None,
|
|
47
|
+
client: Optional[Any] = None,
|
|
48
|
+
):
|
|
49
|
+
self._model = model
|
|
50
|
+
self.model_for = model_for
|
|
51
|
+
if client is not None:
|
|
52
|
+
self._client = client
|
|
53
|
+
else:
|
|
54
|
+
try:
|
|
55
|
+
import openai
|
|
56
|
+
except ImportError as exc: # pragma: no cover - exercised only without extra
|
|
57
|
+
raise ImportError(
|
|
58
|
+
"OpenAIBackend requires the optional 'openai' dependency; "
|
|
59
|
+
"install it with: pip install chatcortex[real]"
|
|
60
|
+
) from exc
|
|
61
|
+
self._client = openai.OpenAI()
|
|
62
|
+
|
|
63
|
+
def _model_for_component(self, metadata: ComponentMetadata) -> str:
|
|
64
|
+
if self.model_for is not None:
|
|
65
|
+
return self.model_for(metadata)
|
|
66
|
+
return self._model
|
|
67
|
+
|
|
68
|
+
def __call__(
|
|
69
|
+
self,
|
|
70
|
+
metadata: ComponentMetadata,
|
|
71
|
+
node_input: Any = None,
|
|
72
|
+
) -> ProbeResult:
|
|
73
|
+
model = self._model_for_component(metadata)
|
|
74
|
+
prompt = str(node_input) if node_input is not None else (
|
|
75
|
+
f"Perform the '{metadata.capabilities[0]}' task as component "
|
|
76
|
+
f"'{metadata.name}'."
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
started = time.perf_counter()
|
|
80
|
+
try:
|
|
81
|
+
response = self._client.chat.completions.create(
|
|
82
|
+
model=model,
|
|
83
|
+
messages=[
|
|
84
|
+
{
|
|
85
|
+
"role": "user",
|
|
86
|
+
"content": prompt,
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
)
|
|
90
|
+
except Exception as exc: # network/HTTP/key errors are probe failures
|
|
91
|
+
latency_ms = (time.perf_counter() - started) * 1000.0
|
|
92
|
+
return ProbeResult(
|
|
93
|
+
success=False,
|
|
94
|
+
latency_ms=latency_ms,
|
|
95
|
+
cost=0.0,
|
|
96
|
+
output=f"openai call failed: {type(exc).__name__}: {exc}",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
latency_ms = (time.perf_counter() - started) * 1000.0
|
|
100
|
+
output = response.choices[0].message.content
|
|
101
|
+
return ProbeResult(
|
|
102
|
+
success=bool(output),
|
|
103
|
+
latency_ms=latency_ms,
|
|
104
|
+
cost=metadata.cost_per_call,
|
|
105
|
+
output=output,
|
|
106
|
+
)
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import random
|
|
2
1
|
from typing import Iterable, List, Tuple
|
|
3
2
|
|
|
4
3
|
from chatcortex.optimization.architecture_candidate import ArchitectureCandidate
|
|
@@ -11,13 +10,9 @@ def compute_coverage(
|
|
|
11
10
|
) -> float:
|
|
12
11
|
return frontier_coverage(approx_frontier, true_frontier)
|
|
13
12
|
|
|
14
|
-
def compute_hypervolume(
|
|
15
|
-
frontier: Iterable[ArchitectureCandidate],
|
|
16
|
-
reference_point: Tuple[float, float, float],
|
|
17
|
-
) -> float:
|
|
18
|
-
return hypervolume_monte_carlo(frontier, reference_point)
|
|
19
|
-
|
|
20
13
|
|
|
14
|
+
# Note: one seed per estimate, not a shared stream — the two estimates below
|
|
15
|
+
# are independent Monte Carlo runs of the same algorithm.
|
|
21
16
|
def hypervolume_loss(
|
|
22
17
|
approx_frontier,
|
|
23
18
|
true_frontier,
|
|
@@ -25,42 +20,12 @@ def hypervolume_loss(
|
|
|
25
20
|
num_samples=200000,
|
|
26
21
|
seed=42,
|
|
27
22
|
) -> float:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
for _ in range(num_samples):
|
|
36
|
-
|
|
37
|
-
sample_cost = random_number_generator.uniform(0, ref_cost)
|
|
38
|
-
sample_latency = random_number_generator.uniform(0, ref_latency)
|
|
39
|
-
sample_reliability = random_number_generator.uniform(ref_reliability, 1.0)
|
|
40
|
-
|
|
41
|
-
for candidate in true_frontier:
|
|
42
|
-
if (
|
|
43
|
-
candidate.total_cost <= sample_cost
|
|
44
|
-
and candidate.total_latency <= sample_latency
|
|
45
|
-
and candidate.total_reliability >= sample_reliability
|
|
46
|
-
):
|
|
47
|
-
dominated_true += 1
|
|
48
|
-
break
|
|
49
|
-
|
|
50
|
-
for candidate in approx_frontier:
|
|
51
|
-
if (
|
|
52
|
-
candidate.total_cost <= sample_cost
|
|
53
|
-
and candidate.total_latency <= sample_latency
|
|
54
|
-
and candidate.total_reliability >= sample_reliability
|
|
55
|
-
):
|
|
56
|
-
dominated_approx += 1
|
|
57
|
-
break
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
box_volume = ref_cost * ref_latency * (1.0 - ref_reliability)
|
|
61
|
-
|
|
62
|
-
hv_true = (dominated_true / num_samples) * box_volume
|
|
63
|
-
hv_approx = (dominated_approx / num_samples) * box_volume
|
|
23
|
+
hv_true = hypervolume_monte_carlo(
|
|
24
|
+
true_frontier, reference_point, num_samples=num_samples, seed=seed
|
|
25
|
+
)
|
|
26
|
+
hv_approx = hypervolume_monte_carlo(
|
|
27
|
+
approx_frontier, reference_point, num_samples=num_samples, seed=seed
|
|
28
|
+
)
|
|
64
29
|
|
|
65
30
|
return max(0.0, hv_true - hv_approx)
|
|
66
31
|
|
|
@@ -79,7 +44,7 @@ def additive_regret(
|
|
|
79
44
|
|
|
80
45
|
if not dominating:
|
|
81
46
|
return (0.0, 0.0, 0.0)
|
|
82
|
-
|
|
47
|
+
|
|
83
48
|
min_cost_diff = min(candidate.total_cost - p.total_cost for p in dominating)
|
|
84
49
|
min_latency_diff = min(candidate.total_latency - p.total_latency for p in dominating)
|
|
85
50
|
min_reliability_diff = min(p.total_reliability - candidate.total_reliability for p in dominating)
|
|
@@ -112,7 +77,7 @@ def average_regret(
|
|
|
112
77
|
total_cost_regret += cr
|
|
113
78
|
total_latency_regret += lr
|
|
114
79
|
total_reliability_regret += rr
|
|
115
|
-
|
|
80
|
+
|
|
116
81
|
n = len(approx_list)
|
|
117
82
|
|
|
118
83
|
return (
|
|
@@ -131,7 +96,7 @@ def evaluate_approximation(
|
|
|
131
96
|
"""
|
|
132
97
|
|
|
133
98
|
coverage = compute_coverage(approx_frontier, true_frontier)
|
|
134
|
-
|
|
99
|
+
|
|
135
100
|
hv_loss = hypervolume_loss(approx_frontier, true_frontier, reference_point)
|
|
136
101
|
avg_regret = average_regret(approx_frontier, true_frontier)
|
|
137
102
|
|
|
@@ -141,4 +106,4 @@ def evaluate_approximation(
|
|
|
141
106
|
"avg_cost_regret": avg_regret[0],
|
|
142
107
|
"avg_latency_regret": avg_regret[1],
|
|
143
108
|
"avg_reliability_regret": avg_regret[2],
|
|
144
|
-
}
|
|
109
|
+
}
|