chatcortex 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chatcortex-0.3.0/PKG-INFO +230 -0
- chatcortex-0.3.0/README.md +199 -0
- chatcortex-0.3.0/chatcortex/__init__.py +7 -0
- chatcortex-0.3.0/chatcortex/budget_sweep_experiment.py +142 -0
- chatcortex-0.3.0/chatcortex/demo_phase1.py +142 -0
- chatcortex-0.3.0/chatcortex/demo_phase2.py +122 -0
- chatcortex-0.3.0/chatcortex/demo_phase3.py +215 -0
- chatcortex-0.3.0/chatcortex/evaluation/approximation_metrics.py +144 -0
- chatcortex-0.3.0/chatcortex/evaluation/harness.py +90 -0
- chatcortex-0.3.0/chatcortex/execution/executor.py +64 -0
- chatcortex-0.3.0/chatcortex/graph/agent_graph.py +99 -0
- chatcortex-0.3.0/chatcortex/optimization/architecture_candidate.py +23 -0
- chatcortex-0.3.0/chatcortex/optimization/pareto.py +203 -0
- chatcortex-0.3.0/chatcortex/registry/capability_registry.py +62 -0
- chatcortex-0.3.0/chatcortex/registry/metadata.py +44 -0
- chatcortex-0.3.0/chatcortex/synthesis/base.py +39 -0
- chatcortex-0.3.0/chatcortex/synthesis/beam_synthesizer.py +113 -0
- chatcortex-0.3.0/chatcortex/synthesis/budget.py +68 -0
- chatcortex-0.3.0/chatcortex/synthesis/exhaustive_synthesizer.py +91 -0
- chatcortex-0.3.0/chatcortex/synthesis/heuristic_synthesizer.py +111 -0
- chatcortex-0.3.0/chatcortex/synthesis/random_synthesizer.py +89 -0
- chatcortex-0.3.0/chatcortex/synthesis/task_specification.py +51 -0
- chatcortex-0.3.0/chatcortex/telemetry/logger.py +40 -0
- chatcortex-0.3.0/chatcortex.egg-info/PKG-INFO +230 -0
- chatcortex-0.3.0/chatcortex.egg-info/SOURCES.txt +29 -0
- chatcortex-0.3.0/chatcortex.egg-info/requires.txt +10 -0
- chatcortex-0.3.0/pyproject.toml +55 -0
- chatcortex-0.1.0/PKG-INFO +0 -155
- chatcortex-0.1.0/README.md +0 -143
- chatcortex-0.1.0/chatcortex/__init__.py +0 -5
- chatcortex-0.1.0/chatcortex/config.py +0 -13
- chatcortex-0.1.0/chatcortex/engine.py +0 -48
- chatcortex-0.1.0/chatcortex/exceptions.py +0 -17
- chatcortex-0.1.0/chatcortex/ingestion.py +0 -26
- chatcortex-0.1.0/chatcortex/retriever.py +0 -15
- chatcortex-0.1.0/chatcortex/session.py +0 -19
- chatcortex-0.1.0/chatcortex.egg-info/PKG-INFO +0 -155
- chatcortex-0.1.0/chatcortex.egg-info/SOURCES.txt +0 -15
- chatcortex-0.1.0/pyproject.toml +0 -17
- chatcortex-0.1.0/tests/test_engine.py +0 -0
- {chatcortex-0.1.0 → chatcortex-0.3.0}/LICENSE +0 -0
- {chatcortex-0.1.0 → chatcortex-0.3.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.1.0 → chatcortex-0.3.0}/chatcortex.egg-info/top_level.txt +0 -0
- {chatcortex-0.1.0 → chatcortex-0.3.0}/setup.cfg +0 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: chatcortex
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Research framework for multi-objective synthesis and evaluation of AI agent architectures under bounded budgets
|
|
5
|
+
Author: Siddharth Saraswat
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/siddharth1012/chatCortex
|
|
8
|
+
Project-URL: Repository, https://github.com/siddharth1012/chatCortex
|
|
9
|
+
Project-URL: Issues, https://github.com/siddharth1012/chatCortex/issues
|
|
10
|
+
Keywords: multi-objective-optimization,agent-architecture,llm-agents,pareto-optimization,architecture-search,beam-search,ai-systems-research
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: networkx>=2.8
|
|
23
|
+
Requires-Dist: numpy>=1.23
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
26
|
+
Requires-Dist: black>=23.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
28
|
+
Provides-Extra: viz
|
|
29
|
+
Requires-Dist: matplotlib>=3.7; extra == "viz"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# ChatCortex
|
|
33
|
+
|
|
34
|
+
**ChatCortex** is a research-oriented framework for automated synthesis
|
|
35
|
+
and multi-objective optimization of LLM-based agent architectures.
|
|
36
|
+
|
|
37
|
+
It is designed as a controlled experimental platform for modeling,
|
|
38
|
+
generating, evaluating, and optimizing tool-augmented AI agent systems.
|
|
39
|
+
|
|
40
|
+
------------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
## Vision
|
|
43
|
+
|
|
44
|
+
ChatCortex aims to eliminate manual agent wiring and ad-hoc prompt
|
|
45
|
+
engineering by introducing:
|
|
46
|
+
|
|
47
|
+
- Formal task modeling
|
|
48
|
+
- Constraint-aware synthesis
|
|
49
|
+
- Multi-objective optimization
|
|
50
|
+
- Exact Pareto frontier computation
|
|
51
|
+
- Deterministic and stochastic execution simulation
|
|
52
|
+
|
|
53
|
+
Long-term goal:
|
|
54
|
+
|
|
55
|
+
Automated architecture synthesis of reliable AI agents from high-level
|
|
56
|
+
intent.
|
|
57
|
+
|
|
58
|
+
------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
## Architecture Overview
|
|
61
|
+
|
|
62
|
+
ChatCortex is organized into layered research components:
|
|
63
|
+
|
|
64
|
+
1. **User / Intent Layer** (future)
|
|
65
|
+
2. **TaskSpecification** (formal model)
|
|
66
|
+
3. **Synthesis Engine** (greedy / exhaustive)
|
|
67
|
+
4. **AgentGraph** (DAG representation)
|
|
68
|
+
5. **Execution Engine** (deterministic / probabilistic)
|
|
69
|
+
6. **Telemetry**
|
|
70
|
+
7. **Evaluation Harness**
|
|
71
|
+
8. **Pareto Optimization**
|
|
72
|
+
|
|
73
|
+
------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
# Core Components
|
|
76
|
+
|
|
77
|
+
## 1️. Component Metadata
|
|
78
|
+
|
|
79
|
+
Formal, immutable representation of:
|
|
80
|
+
|
|
81
|
+
- Models
|
|
82
|
+
- Tools
|
|
83
|
+
- Memory modules
|
|
84
|
+
- Verification modules
|
|
85
|
+
|
|
86
|
+
Each component defines:
|
|
87
|
+
|
|
88
|
+
- Capabilities
|
|
89
|
+
- Cost
|
|
90
|
+
- Latency
|
|
91
|
+
- Reliability
|
|
92
|
+
- Privacy level
|
|
93
|
+
|
|
94
|
+
------------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
## 2️. TaskSpecification
|
|
97
|
+
|
|
98
|
+
Defines a task formally:
|
|
99
|
+
|
|
100
|
+
- Ordered required capabilities
|
|
101
|
+
- Hard constraints (max cost, latency, privacy)
|
|
102
|
+
- Multi-objective weights
|
|
103
|
+
|
|
104
|
+
Separates feasibility from optimization preference.
|
|
105
|
+
|
|
106
|
+
------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
## 3️. Synthesizers
|
|
109
|
+
|
|
110
|
+
### HeuristicSynthesizer
|
|
111
|
+
|
|
112
|
+
Greedy deterministic builder selecting best component per stage.
|
|
113
|
+
|
|
114
|
+
### ExhaustiveSynthesizer
|
|
115
|
+
|
|
116
|
+
Explores full Cartesian architecture space and enables exact Pareto
|
|
117
|
+
frontier computation.
|
|
118
|
+
|
|
119
|
+
------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
## 4️. AgentGraph
|
|
122
|
+
|
|
123
|
+
Directed Acyclic Graph (DAG) representation of agent architecture.
|
|
124
|
+
|
|
125
|
+
Aggregates:
|
|
126
|
+
|
|
127
|
+
- Total cost (additive)
|
|
128
|
+
- Total latency (sequential assumption)
|
|
129
|
+
- Aggregate reliability (multiplicative)
|
|
130
|
+
|
|
131
|
+
------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
## 5️. Execution Engine
|
|
134
|
+
|
|
135
|
+
Supports:
|
|
136
|
+
|
|
137
|
+
- Deterministic mode (structural validation)
|
|
138
|
+
- Probabilistic mode (reliability simulation)
|
|
139
|
+
- Fixed random seed for reproducibility
|
|
140
|
+
|
|
141
|
+
------------------------------------------------------------------------
|
|
142
|
+
|
|
143
|
+
## 6️. Evaluation Harness
|
|
144
|
+
|
|
145
|
+
Runs experiments across:
|
|
146
|
+
|
|
147
|
+
- Multiple tasks
|
|
148
|
+
- Multiple synthesizers
|
|
149
|
+
- Multiple stochastic trials
|
|
150
|
+
|
|
151
|
+
Produces:
|
|
152
|
+
|
|
153
|
+
- Average cost
|
|
154
|
+
- Average latency
|
|
155
|
+
- Success rate
|
|
156
|
+
|
|
157
|
+
------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
## 7️. Pareto Optimization
|
|
160
|
+
|
|
161
|
+
Exact multi-objective Pareto frontier computation across:
|
|
162
|
+
|
|
163
|
+
- Cost (minimize)
|
|
164
|
+
- Latency (minimize)
|
|
165
|
+
- Reliability (maximize)
|
|
166
|
+
|
|
167
|
+
Provides ground-truth optimal architecture trade-offs.
|
|
168
|
+
|
|
169
|
+
------------------------------------------------------------------------
|
|
170
|
+
|
|
171
|
+
# Research Positioning
|
|
172
|
+
|
|
173
|
+
ChatCortex is intended as:
|
|
174
|
+
|
|
175
|
+
- A systems-AI research framework
|
|
176
|
+
- A controlled environment for architecture optimization experiments
|
|
177
|
+
|
|
178
|
+
It emphasizes:
|
|
179
|
+
|
|
180
|
+
- Reproducibility
|
|
181
|
+
- Formal modeling
|
|
182
|
+
- Separation of concerns
|
|
183
|
+
- Experimental rigor
|
|
184
|
+
|
|
185
|
+
------------------------------------------------------------------------
|
|
186
|
+
|
|
187
|
+
# Roadmap
|
|
188
|
+
|
|
189
|
+
### Phase 1 (Complete)
|
|
190
|
+
|
|
191
|
+
- Formal modeling
|
|
192
|
+
- Heuristic synthesis
|
|
193
|
+
- Execution simulation
|
|
194
|
+
- Evaluation harness
|
|
195
|
+
|
|
196
|
+
### Phase 2 (Complete)
|
|
197
|
+
|
|
198
|
+
- Exhaustive architecture search
|
|
199
|
+
- Exact 3-objective Pareto optimization
|
|
200
|
+
|
|
201
|
+
### Phase 3 (Planned)
|
|
202
|
+
|
|
203
|
+
- Heuristic search (beam search, evolutionary refinement)
|
|
204
|
+
- Statistical robustness analysis
|
|
205
|
+
- Intent-to-task automation layer
|
|
206
|
+
- Real model/tool integration
|
|
207
|
+
|
|
208
|
+
------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
# Installation
|
|
211
|
+
|
|
212
|
+
pip install chatcortex
|
|
213
|
+
|
|
214
|
+
------------------------------------------------------------------------
|
|
215
|
+
|
|
216
|
+
# Status
|
|
217
|
+
|
|
218
|
+
ChatCortex is currently a research framework under active development.
|
|
219
|
+
|
|
220
|
+
It is not yet a production agent orchestration library.
|
|
221
|
+
|
|
222
|
+
------------------------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
# License
|
|
225
|
+
|
|
226
|
+
MIT License
|
|
227
|
+
|
|
228
|
+
------------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
Developed by Siddharth Saraswat
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# ChatCortex
|
|
2
|
+
|
|
3
|
+
**ChatCortex** is a research-oriented framework for automated synthesis
|
|
4
|
+
and multi-objective optimization of LLM-based agent architectures.
|
|
5
|
+
|
|
6
|
+
It is designed as a controlled experimental platform for modeling,
|
|
7
|
+
generating, evaluating, and optimizing tool-augmented AI agent systems.
|
|
8
|
+
|
|
9
|
+
------------------------------------------------------------------------
|
|
10
|
+
|
|
11
|
+
## Vision
|
|
12
|
+
|
|
13
|
+
ChatCortex aims to eliminate manual agent wiring and ad-hoc prompt
|
|
14
|
+
engineering by introducing:
|
|
15
|
+
|
|
16
|
+
- Formal task modeling
|
|
17
|
+
- Constraint-aware synthesis
|
|
18
|
+
- Multi-objective optimization
|
|
19
|
+
- Exact Pareto frontier computation
|
|
20
|
+
- Deterministic and stochastic execution simulation
|
|
21
|
+
|
|
22
|
+
Long-term goal:
|
|
23
|
+
|
|
24
|
+
Automated architecture synthesis of reliable AI agents from high-level
|
|
25
|
+
intent.
|
|
26
|
+
|
|
27
|
+
------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
## Architecture Overview
|
|
30
|
+
|
|
31
|
+
ChatCortex is organized into layered research components:
|
|
32
|
+
|
|
33
|
+
1. **User / Intent Layer** (future)
|
|
34
|
+
2. **TaskSpecification** (formal model)
|
|
35
|
+
3. **Synthesis Engine** (greedy / exhaustive)
|
|
36
|
+
4. **AgentGraph** (DAG representation)
|
|
37
|
+
5. **Execution Engine** (deterministic / probabilistic)
|
|
38
|
+
6. **Telemetry**
|
|
39
|
+
7. **Evaluation Harness**
|
|
40
|
+
8. **Pareto Optimization**
|
|
41
|
+
|
|
42
|
+
------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
# Core Components
|
|
45
|
+
|
|
46
|
+
## 1️. Component Metadata
|
|
47
|
+
|
|
48
|
+
Formal, immutable representation of:
|
|
49
|
+
|
|
50
|
+
- Models
|
|
51
|
+
- Tools
|
|
52
|
+
- Memory modules
|
|
53
|
+
- Verification modules
|
|
54
|
+
|
|
55
|
+
Each component defines:
|
|
56
|
+
|
|
57
|
+
- Capabilities
|
|
58
|
+
- Cost
|
|
59
|
+
- Latency
|
|
60
|
+
- Reliability
|
|
61
|
+
- Privacy level
|
|
62
|
+
|
|
63
|
+
------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
## 2️. TaskSpecification
|
|
66
|
+
|
|
67
|
+
Defines a task formally:
|
|
68
|
+
|
|
69
|
+
- Ordered required capabilities
|
|
70
|
+
- Hard constraints (max cost, latency, privacy)
|
|
71
|
+
- Multi-objective weights
|
|
72
|
+
|
|
73
|
+
Separates feasibility from optimization preference.
|
|
74
|
+
|
|
75
|
+
------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
## 3️. Synthesizers
|
|
78
|
+
|
|
79
|
+
### HeuristicSynthesizer
|
|
80
|
+
|
|
81
|
+
Greedy deterministic builder selecting best component per stage.
|
|
82
|
+
|
|
83
|
+
### ExhaustiveSynthesizer
|
|
84
|
+
|
|
85
|
+
Explores full Cartesian architecture space and enables exact Pareto
|
|
86
|
+
frontier computation.
|
|
87
|
+
|
|
88
|
+
------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
## 4️. AgentGraph
|
|
91
|
+
|
|
92
|
+
Directed Acyclic Graph (DAG) representation of agent architecture.
|
|
93
|
+
|
|
94
|
+
Aggregates:
|
|
95
|
+
|
|
96
|
+
- Total cost (additive)
|
|
97
|
+
- Total latency (sequential assumption)
|
|
98
|
+
- Aggregate reliability (multiplicative)
|
|
99
|
+
|
|
100
|
+
------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
## 5️. Execution Engine
|
|
103
|
+
|
|
104
|
+
Supports:
|
|
105
|
+
|
|
106
|
+
- Deterministic mode (structural validation)
|
|
107
|
+
- Probabilistic mode (reliability simulation)
|
|
108
|
+
- Fixed random seed for reproducibility
|
|
109
|
+
|
|
110
|
+
------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
## 6️. Evaluation Harness
|
|
113
|
+
|
|
114
|
+
Runs experiments across:
|
|
115
|
+
|
|
116
|
+
- Multiple tasks
|
|
117
|
+
- Multiple synthesizers
|
|
118
|
+
- Multiple stochastic trials
|
|
119
|
+
|
|
120
|
+
Produces:
|
|
121
|
+
|
|
122
|
+
- Average cost
|
|
123
|
+
- Average latency
|
|
124
|
+
- Success rate
|
|
125
|
+
|
|
126
|
+
------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
## 7️. Pareto Optimization
|
|
129
|
+
|
|
130
|
+
Exact multi-objective Pareto frontier computation across:
|
|
131
|
+
|
|
132
|
+
- Cost (minimize)
|
|
133
|
+
- Latency (minimize)
|
|
134
|
+
- Reliability (maximize)
|
|
135
|
+
|
|
136
|
+
Provides ground-truth optimal architecture trade-offs.
|
|
137
|
+
|
|
138
|
+
------------------------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
# Research Positioning
|
|
141
|
+
|
|
142
|
+
ChatCortex is intended as:
|
|
143
|
+
|
|
144
|
+
- A systems-AI research framework
|
|
145
|
+
- A controlled environment for architecture optimization experiments
|
|
146
|
+
|
|
147
|
+
It emphasizes:
|
|
148
|
+
|
|
149
|
+
- Reproducibility
|
|
150
|
+
- Formal modeling
|
|
151
|
+
- Separation of concerns
|
|
152
|
+
- Experimental rigor
|
|
153
|
+
|
|
154
|
+
------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
# Roadmap
|
|
157
|
+
|
|
158
|
+
### Phase 1 (Complete)
|
|
159
|
+
|
|
160
|
+
- Formal modeling
|
|
161
|
+
- Heuristic synthesis
|
|
162
|
+
- Execution simulation
|
|
163
|
+
- Evaluation harness
|
|
164
|
+
|
|
165
|
+
### Phase 2 (Complete)
|
|
166
|
+
|
|
167
|
+
- Exhaustive architecture search
|
|
168
|
+
- Exact 3-objective Pareto optimization
|
|
169
|
+
|
|
170
|
+
### Phase 3 (Planned)
|
|
171
|
+
|
|
172
|
+
- Heuristic search (beam search, evolutionary refinement)
|
|
173
|
+
- Statistical robustness analysis
|
|
174
|
+
- Intent-to-task automation layer
|
|
175
|
+
- Real model/tool integration
|
|
176
|
+
|
|
177
|
+
------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
# Installation
|
|
180
|
+
|
|
181
|
+
pip install chatcortex
|
|
182
|
+
|
|
183
|
+
------------------------------------------------------------------------
|
|
184
|
+
|
|
185
|
+
# Status
|
|
186
|
+
|
|
187
|
+
ChatCortex is currently a research framework under active development.
|
|
188
|
+
|
|
189
|
+
It is not yet a production agent orchestration library.
|
|
190
|
+
|
|
191
|
+
------------------------------------------------------------------------
|
|
192
|
+
|
|
193
|
+
# License
|
|
194
|
+
|
|
195
|
+
MIT License
|
|
196
|
+
|
|
197
|
+
------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
Developed by Siddharth Saraswat
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from atexit import register
|
|
2
|
+
from random import Random
|
|
3
|
+
from statistics import mean, stdev
|
|
4
|
+
|
|
5
|
+
from chatcortex.evaluation.approximation_metrics import evaluate_approximation
|
|
6
|
+
from chatcortex.synthesis.base import Synthesizer
|
|
7
|
+
from chatcortex.synthesis.beam_synthesizer import BeamSynthesizer
|
|
8
|
+
from chatcortex.synthesis.budget import SynthesisBudget
|
|
9
|
+
from chatcortex.synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
10
|
+
from chatcortex.synthesis.heuristic_synthesizer import HeuristicSynthesizer, SynthesisError
|
|
11
|
+
from chatcortex.synthesis.random_synthesizer import RandomSynthesizer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def run_budget_sweep(registry, task):
|
|
15
|
+
|
|
16
|
+
exhaustive = ExhaustiveSynthesizer(registry)
|
|
17
|
+
true_frontier = exhaustive.synthesize(task)
|
|
18
|
+
|
|
19
|
+
print(f"True Pareto size: {len(true_frontier)}")
|
|
20
|
+
|
|
21
|
+
reference_point = (0.1, 3000, 0.0)
|
|
22
|
+
|
|
23
|
+
budgets = [10, 20, 30, 40, 60]
|
|
24
|
+
beam_widths = [3, 5, 7, 10]
|
|
25
|
+
random_seeds = [42, 43, 44, 45, 46] # Multi-run random baseline
|
|
26
|
+
|
|
27
|
+
results = []
|
|
28
|
+
|
|
29
|
+
for b in budgets:
|
|
30
|
+
print(f"--- Budget {b} ---")
|
|
31
|
+
|
|
32
|
+
greedy_budget = SynthesisBudget(max_evaluations=b, random_seed=42)
|
|
33
|
+
greedy = HeuristicSynthesizer(registry)
|
|
34
|
+
greedy_frontier = greedy.synthesize(task, greedy_budget)
|
|
35
|
+
|
|
36
|
+
greedy_report = evaluate_approximation(
|
|
37
|
+
approx_frontier=greedy_frontier,
|
|
38
|
+
true_frontier=true_frontier,
|
|
39
|
+
reference_point=reference_point,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
results.append({
|
|
43
|
+
"algorithm": "Greedy",
|
|
44
|
+
"budget": b,
|
|
45
|
+
"beam_width": None,
|
|
46
|
+
"coverage_mean": greedy_report["coverage"],
|
|
47
|
+
"coverage_std": 0.0,
|
|
48
|
+
"hypervolume_loss_mean": greedy_report["hypervolume_loss"],
|
|
49
|
+
"hypervolume_loss_std": 0.0,
|
|
50
|
+
"avg_cost_regret": greedy_report["avg_cost_regret"],
|
|
51
|
+
"avg_latency_regret": greedy_report["avg_latency_regret"],
|
|
52
|
+
"avg_reliability_regret": greedy_report["avg_reliability_regret"],
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
print(
|
|
56
|
+
f"Greedy: "
|
|
57
|
+
f"coverage={greedy_report['coverage']:.2f}, "
|
|
58
|
+
f"hv_loss={greedy_report['hypervolume_loss']:.2f}"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# --------------------------------------------------------------
|
|
62
|
+
# Random (Multi-run baseline)
|
|
63
|
+
# --------------------------------------------------------------
|
|
64
|
+
random_coverages = []
|
|
65
|
+
random_hv_losses = []
|
|
66
|
+
random_cost_regrets = []
|
|
67
|
+
random_latency_regrets = []
|
|
68
|
+
random_reliability_regrets = []
|
|
69
|
+
|
|
70
|
+
for seed in random_seeds:
|
|
71
|
+
random_budget = SynthesisBudget(max_evaluations=b, random_seed=seed)
|
|
72
|
+
random_syn = RandomSynthesizer(registry)
|
|
73
|
+
|
|
74
|
+
random_frontier = random_syn.synthesize(task, random_budget)
|
|
75
|
+
|
|
76
|
+
report = evaluate_approximation(
|
|
77
|
+
approx_frontier=random_frontier,
|
|
78
|
+
true_frontier=true_frontier,
|
|
79
|
+
reference_point=reference_point,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
random_coverages.append(report["coverage"])
|
|
83
|
+
random_hv_losses.append(report["hypervolume_loss"])
|
|
84
|
+
random_cost_regrets.append(report["avg_cost_regret"])
|
|
85
|
+
random_latency_regrets.append(report["avg_latency_regret"])
|
|
86
|
+
random_reliability_regrets.append(report["avg_reliability_regret"])
|
|
87
|
+
|
|
88
|
+
results.append({
|
|
89
|
+
"algorithm": "Random",
|
|
90
|
+
"budget": b,
|
|
91
|
+
"beam_width": None,
|
|
92
|
+
"coverage_mean": mean(random_coverages),
|
|
93
|
+
"coverage_std": stdev(random_coverages),
|
|
94
|
+
"hypervolume_loss_mean": mean(random_hv_losses),
|
|
95
|
+
"hypervolume_loss_std": stdev(random_hv_losses),
|
|
96
|
+
"avg_cost_regret": mean(random_cost_regrets),
|
|
97
|
+
"avg_latency_regret": mean(random_latency_regrets),
|
|
98
|
+
"avg_reliability_regret": mean(random_reliability_regrets),
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
print(
|
|
102
|
+
f"Random: "
|
|
103
|
+
f"coverage={mean(random_coverages):.2f} ± {stdev(random_coverages):.2f}, "
|
|
104
|
+
f"hv_loss={mean(random_hv_losses):.2f}"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# --------------------------------------------------------------
|
|
108
|
+
# Beam Sweep
|
|
109
|
+
# --------------------------------------------------------------
|
|
110
|
+
for w in beam_widths:
|
|
111
|
+
|
|
112
|
+
beam_budget = SynthesisBudget(max_evaluations=b, random_seed=42)
|
|
113
|
+
beam = BeamSynthesizer(registry, beam_width=w)
|
|
114
|
+
|
|
115
|
+
beam_frontier = beam.synthesize(task, beam_budget)
|
|
116
|
+
|
|
117
|
+
beam_report = evaluate_approximation(
|
|
118
|
+
approx_frontier=beam_frontier,
|
|
119
|
+
true_frontier=true_frontier,
|
|
120
|
+
reference_point=reference_point,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
results.append({
|
|
124
|
+
"algorithm": "Beam",
|
|
125
|
+
"budget": b,
|
|
126
|
+
"beam_width": w,
|
|
127
|
+
"coverage_mean": beam_report["coverage"],
|
|
128
|
+
"coverage_std": 0.0,
|
|
129
|
+
"hypervolume_loss_mean": beam_report["hypervolume_loss"],
|
|
130
|
+
"hypervolume_loss_std": 0.0,
|
|
131
|
+
"avg_cost_regret": beam_report["avg_cost_regret"],
|
|
132
|
+
"avg_latency_regret": beam_report["avg_latency_regret"],
|
|
133
|
+
"avg_reliability_regret": beam_report["avg_reliability_regret"],
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
print(
|
|
137
|
+
f"Beam (w={w}): "
|
|
138
|
+
f"coverage={beam_report['coverage']:.2f}, "
|
|
139
|
+
f"hv_loss={beam_report['hypervolume_loss']:.2f}"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return results
|