chatcortex 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chatcortex-0.2.0 → chatcortex-0.3.0}/PKG-INFO +21 -2
- chatcortex-0.3.0/chatcortex/__init__.py +7 -0
- chatcortex-0.3.0/chatcortex/budget_sweep_experiment.py +142 -0
- chatcortex-0.3.0/chatcortex/demo_phase3.py +215 -0
- chatcortex-0.3.0/chatcortex/evaluation/approximation_metrics.py +144 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/graph/agent_graph.py +25 -0
- chatcortex-0.3.0/chatcortex/optimization/pareto.py +203 -0
- chatcortex-0.3.0/chatcortex/synthesis/base.py +39 -0
- chatcortex-0.3.0/chatcortex/synthesis/beam_synthesizer.py +113 -0
- chatcortex-0.3.0/chatcortex/synthesis/budget.py +68 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/synthesis/exhaustive_synthesizer.py +36 -20
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/synthesis/heuristic_synthesizer.py +41 -14
- chatcortex-0.3.0/chatcortex/synthesis/random_synthesizer.py +89 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex.egg-info/PKG-INFO +21 -2
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex.egg-info/SOURCES.txt +8 -0
- chatcortex-0.3.0/chatcortex.egg-info/requires.txt +10 -0
- chatcortex-0.3.0/pyproject.toml +55 -0
- chatcortex-0.2.0/chatcortex/__init__.py +0 -1
- chatcortex-0.2.0/chatcortex/optimization/pareto.py +0 -43
- chatcortex-0.2.0/pyproject.toml +0 -17
- {chatcortex-0.2.0 → chatcortex-0.3.0}/LICENSE +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/README.md +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/demo_phase1.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/demo_phase2.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/evaluation/harness.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/execution/executor.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/optimization/architecture_candidate.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/registry/capability_registry.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/registry/metadata.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/synthesis/task_specification.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex/telemetry/logger.py +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/chatcortex.egg-info/top_level.txt +0 -0
- {chatcortex-0.2.0 → chatcortex-0.3.0}/setup.cfg +0 -0
|
@@ -1,13 +1,32 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chatcortex
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Research framework for multi-objective synthesis and evaluation of AI agent architectures under bounded budgets
|
|
5
5
|
Author: Siddharth Saraswat
|
|
6
6
|
License: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/siddharth1012/chatCortex
|
|
8
|
+
Project-URL: Repository, https://github.com/siddharth1012/chatCortex
|
|
9
|
+
Project-URL: Issues, https://github.com/siddharth1012/chatCortex/issues
|
|
10
|
+
Keywords: multi-objective-optimization,agent-architecture,llm-agents,pareto-optimization,architecture-search,beam-search,ai-systems-research
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
8
19
|
Requires-Python: >=3.9
|
|
9
20
|
Description-Content-Type: text/markdown
|
|
10
21
|
License-File: LICENSE
|
|
22
|
+
Requires-Dist: networkx>=2.8
|
|
23
|
+
Requires-Dist: numpy>=1.23
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
26
|
+
Requires-Dist: black>=23.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
28
|
+
Provides-Extra: viz
|
|
29
|
+
Requires-Dist: matplotlib>=3.7; extra == "viz"
|
|
11
30
|
Dynamic: license-file
|
|
12
31
|
|
|
13
32
|
# ChatCortex
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from atexit import register
|
|
2
|
+
from random import Random
|
|
3
|
+
from statistics import mean, stdev
|
|
4
|
+
|
|
5
|
+
from chatcortex.evaluation.approximation_metrics import evaluate_approximation
|
|
6
|
+
from chatcortex.synthesis.base import Synthesizer
|
|
7
|
+
from chatcortex.synthesis.beam_synthesizer import BeamSynthesizer
|
|
8
|
+
from chatcortex.synthesis.budget import SynthesisBudget
|
|
9
|
+
from chatcortex.synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
10
|
+
from chatcortex.synthesis.heuristic_synthesizer import HeuristicSynthesizer, SynthesisError
|
|
11
|
+
from chatcortex.synthesis.random_synthesizer import RandomSynthesizer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def run_budget_sweep(registry, task):
|
|
15
|
+
|
|
16
|
+
exhaustive = ExhaustiveSynthesizer(registry)
|
|
17
|
+
true_frontier = exhaustive.synthesize(task)
|
|
18
|
+
|
|
19
|
+
print(f"True Pareto size: {len(true_frontier)}")
|
|
20
|
+
|
|
21
|
+
reference_point = (0.1, 3000, 0.0)
|
|
22
|
+
|
|
23
|
+
budgets = [10, 20, 30, 40, 60]
|
|
24
|
+
beam_widths = [3, 5, 7, 10]
|
|
25
|
+
random_seeds = [42, 43, 44, 45, 46] # Multi-run random baseline
|
|
26
|
+
|
|
27
|
+
results = []
|
|
28
|
+
|
|
29
|
+
for b in budgets:
|
|
30
|
+
print(f"--- Budget {b} ---")
|
|
31
|
+
|
|
32
|
+
greedy_budget = SynthesisBudget(max_evaluations=b, random_seed=42)
|
|
33
|
+
greedy = HeuristicSynthesizer(registry)
|
|
34
|
+
greedy_frontier = greedy.synthesize(task, greedy_budget)
|
|
35
|
+
|
|
36
|
+
greedy_report = evaluate_approximation(
|
|
37
|
+
approx_frontier=greedy_frontier,
|
|
38
|
+
true_frontier=true_frontier,
|
|
39
|
+
reference_point=reference_point,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
results.append({
|
|
43
|
+
"algorithm": "Greedy",
|
|
44
|
+
"budget": b,
|
|
45
|
+
"beam_width": None,
|
|
46
|
+
"coverage_mean": greedy_report["coverage"],
|
|
47
|
+
"coverage_std": 0.0,
|
|
48
|
+
"hypervolume_loss_mean": greedy_report["hypervolume_loss"],
|
|
49
|
+
"hypervolume_loss_std": 0.0,
|
|
50
|
+
"avg_cost_regret": greedy_report["avg_cost_regret"],
|
|
51
|
+
"avg_latency_regret": greedy_report["avg_latency_regret"],
|
|
52
|
+
"avg_reliability_regret": greedy_report["avg_reliability_regret"],
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
print(
|
|
56
|
+
f"Greedy: "
|
|
57
|
+
f"coverage={greedy_report['coverage']:.2f}, "
|
|
58
|
+
f"hv_loss={greedy_report['hypervolume_loss']:.2f}"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# --------------------------------------------------------------
|
|
62
|
+
# Random (Multi-run baseline)
|
|
63
|
+
# --------------------------------------------------------------
|
|
64
|
+
random_coverages = []
|
|
65
|
+
random_hv_losses = []
|
|
66
|
+
random_cost_regrets = []
|
|
67
|
+
random_latency_regrets = []
|
|
68
|
+
random_reliability_regrets = []
|
|
69
|
+
|
|
70
|
+
for seed in random_seeds:
|
|
71
|
+
random_budget = SynthesisBudget(max_evaluations=b, random_seed=seed)
|
|
72
|
+
random_syn = RandomSynthesizer(registry)
|
|
73
|
+
|
|
74
|
+
random_frontier = random_syn.synthesize(task, random_budget)
|
|
75
|
+
|
|
76
|
+
report = evaluate_approximation(
|
|
77
|
+
approx_frontier=random_frontier,
|
|
78
|
+
true_frontier=true_frontier,
|
|
79
|
+
reference_point=reference_point,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
random_coverages.append(report["coverage"])
|
|
83
|
+
random_hv_losses.append(report["hypervolume_loss"])
|
|
84
|
+
random_cost_regrets.append(report["avg_cost_regret"])
|
|
85
|
+
random_latency_regrets.append(report["avg_latency_regret"])
|
|
86
|
+
random_reliability_regrets.append(report["avg_reliability_regret"])
|
|
87
|
+
|
|
88
|
+
results.append({
|
|
89
|
+
"algorithm": "Random",
|
|
90
|
+
"budget": b,
|
|
91
|
+
"beam_width": None,
|
|
92
|
+
"coverage_mean": mean(random_coverages),
|
|
93
|
+
"coverage_std": stdev(random_coverages),
|
|
94
|
+
"hypervolume_loss_mean": mean(random_hv_losses),
|
|
95
|
+
"hypervolume_loss_std": stdev(random_hv_losses),
|
|
96
|
+
"avg_cost_regret": mean(random_cost_regrets),
|
|
97
|
+
"avg_latency_regret": mean(random_latency_regrets),
|
|
98
|
+
"avg_reliability_regret": mean(random_reliability_regrets),
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
print(
|
|
102
|
+
f"Random: "
|
|
103
|
+
f"coverage={mean(random_coverages):.2f} ± {stdev(random_coverages):.2f}, "
|
|
104
|
+
f"hv_loss={mean(random_hv_losses):.2f}"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# --------------------------------------------------------------
|
|
108
|
+
# Beam Sweep
|
|
109
|
+
# --------------------------------------------------------------
|
|
110
|
+
for w in beam_widths:
|
|
111
|
+
|
|
112
|
+
beam_budget = SynthesisBudget(max_evaluations=b, random_seed=42)
|
|
113
|
+
beam = BeamSynthesizer(registry, beam_width=w)
|
|
114
|
+
|
|
115
|
+
beam_frontier = beam.synthesize(task, beam_budget)
|
|
116
|
+
|
|
117
|
+
beam_report = evaluate_approximation(
|
|
118
|
+
approx_frontier=beam_frontier,
|
|
119
|
+
true_frontier=true_frontier,
|
|
120
|
+
reference_point=reference_point,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
results.append({
|
|
124
|
+
"algorithm": "Beam",
|
|
125
|
+
"budget": b,
|
|
126
|
+
"beam_width": w,
|
|
127
|
+
"coverage_mean": beam_report["coverage"],
|
|
128
|
+
"coverage_std": 0.0,
|
|
129
|
+
"hypervolume_loss_mean": beam_report["hypervolume_loss"],
|
|
130
|
+
"hypervolume_loss_std": 0.0,
|
|
131
|
+
"avg_cost_regret": beam_report["avg_cost_regret"],
|
|
132
|
+
"avg_latency_regret": beam_report["avg_latency_regret"],
|
|
133
|
+
"avg_reliability_regret": beam_report["avg_reliability_regret"],
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
print(
|
|
137
|
+
f"Beam (w={w}): "
|
|
138
|
+
f"coverage={beam_report['coverage']:.2f}, "
|
|
139
|
+
f"hv_loss={beam_report['hypervolume_loss']:.2f}"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return results
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from chatcortex.budget_sweep_experiment import run_budget_sweep
|
|
2
|
+
from chatcortex.registry.capability_registry import CapabilityRegistry
|
|
3
|
+
from chatcortex.registry.metadata import ComponentMetadata
|
|
4
|
+
|
|
5
|
+
from chatcortex.synthesis.task_specification import TaskSpecification
|
|
6
|
+
from chatcortex.synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
|
|
7
|
+
from chatcortex.synthesis.heuristic_synthesizer import HeuristicSynthesizer
|
|
8
|
+
from chatcortex.synthesis.beam_synthesizer import BeamSynthesizer
|
|
9
|
+
from chatcortex.synthesis.random_synthesizer import RandomSynthesizer
|
|
10
|
+
from chatcortex.synthesis.budget import SynthesisBudget
|
|
11
|
+
|
|
12
|
+
from chatcortex.evaluation.approximation_metrics import evaluate_approximation
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
|
|
17
|
+
registry = CapabilityRegistry()
|
|
18
|
+
|
|
19
|
+
# -------------------------------------------------------
|
|
20
|
+
# Structured Retrieval Models
|
|
21
|
+
# -------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
registry.register(ComponentMetadata(
|
|
24
|
+
name="cheap_retrieval",
|
|
25
|
+
component_type="tool",
|
|
26
|
+
capabilities=["retrieval"],
|
|
27
|
+
cost_per_call=0.001,
|
|
28
|
+
avg_latency_ms=100,
|
|
29
|
+
reliability_score=0.85,
|
|
30
|
+
privacy_level="internal",
|
|
31
|
+
))
|
|
32
|
+
|
|
33
|
+
registry.register(ComponentMetadata(
|
|
34
|
+
name="balanced_retrieval",
|
|
35
|
+
component_type="tool",
|
|
36
|
+
capabilities=["retrieval"],
|
|
37
|
+
cost_per_call=0.003,
|
|
38
|
+
avg_latency_ms=200,
|
|
39
|
+
reliability_score=0.93,
|
|
40
|
+
privacy_level="internal",
|
|
41
|
+
))
|
|
42
|
+
|
|
43
|
+
registry.register(ComponentMetadata(
|
|
44
|
+
name="strong_retrieval",
|
|
45
|
+
component_type="tool",
|
|
46
|
+
capabilities=["retrieval"],
|
|
47
|
+
cost_per_call=0.006,
|
|
48
|
+
avg_latency_ms=400,
|
|
49
|
+
reliability_score=0.98,
|
|
50
|
+
privacy_level="internal",
|
|
51
|
+
))
|
|
52
|
+
|
|
53
|
+
# -------------------------------------------------------
|
|
54
|
+
# Structured Generators
|
|
55
|
+
# -------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
registry.register(ComponentMetadata(
|
|
58
|
+
name="tiny_generator",
|
|
59
|
+
component_type="model",
|
|
60
|
+
capabilities=["generation"],
|
|
61
|
+
cost_per_call=0.005,
|
|
62
|
+
avg_latency_ms=200,
|
|
63
|
+
reliability_score=0.80,
|
|
64
|
+
privacy_level="external",
|
|
65
|
+
))
|
|
66
|
+
|
|
67
|
+
registry.register(ComponentMetadata(
|
|
68
|
+
name="small_generator",
|
|
69
|
+
component_type="model",
|
|
70
|
+
capabilities=["generation"],
|
|
71
|
+
cost_per_call=0.01,
|
|
72
|
+
avg_latency_ms=400,
|
|
73
|
+
reliability_score=0.88,
|
|
74
|
+
privacy_level="external",
|
|
75
|
+
))
|
|
76
|
+
|
|
77
|
+
registry.register(ComponentMetadata(
|
|
78
|
+
name="large_generator",
|
|
79
|
+
component_type="model",
|
|
80
|
+
capabilities=["generation"],
|
|
81
|
+
cost_per_call=0.02,
|
|
82
|
+
avg_latency_ms=700,
|
|
83
|
+
reliability_score=0.93,
|
|
84
|
+
privacy_level="external",
|
|
85
|
+
))
|
|
86
|
+
|
|
87
|
+
registry.register(ComponentMetadata(
|
|
88
|
+
name="ultra_generator",
|
|
89
|
+
component_type="model",
|
|
90
|
+
capabilities=["generation"],
|
|
91
|
+
cost_per_call=0.04,
|
|
92
|
+
avg_latency_ms=1200,
|
|
93
|
+
reliability_score=0.97,
|
|
94
|
+
privacy_level="external",
|
|
95
|
+
))
|
|
96
|
+
|
|
97
|
+
# -------------------------------------------------------
|
|
98
|
+
# Structured Verifiers
|
|
99
|
+
# -------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
registry.register(ComponentMetadata(
|
|
102
|
+
name="fast_verifier",
|
|
103
|
+
component_type="verification",
|
|
104
|
+
capabilities=["verification"],
|
|
105
|
+
cost_per_call=0.003,
|
|
106
|
+
avg_latency_ms=200,
|
|
107
|
+
reliability_score=0.90,
|
|
108
|
+
privacy_level="internal",
|
|
109
|
+
))
|
|
110
|
+
|
|
111
|
+
registry.register(ComponentMetadata(
|
|
112
|
+
name="strict_verifier",
|
|
113
|
+
component_type="verification",
|
|
114
|
+
capabilities=["verification"],
|
|
115
|
+
cost_per_call=0.008,
|
|
116
|
+
avg_latency_ms=500,
|
|
117
|
+
reliability_score=0.96,
|
|
118
|
+
privacy_level="internal",
|
|
119
|
+
))
|
|
120
|
+
|
|
121
|
+
# -------------------------------------------------------
|
|
122
|
+
# Refinement Stage (New – Enables 72 Architectures)
|
|
123
|
+
# -------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
registry.register(ComponentMetadata(
|
|
126
|
+
name="no_refinement",
|
|
127
|
+
component_type="refinement",
|
|
128
|
+
capabilities=["refinement"],
|
|
129
|
+
cost_per_call=0.0,
|
|
130
|
+
avg_latency_ms=0,
|
|
131
|
+
reliability_score=1.00,
|
|
132
|
+
privacy_level="internal",
|
|
133
|
+
))
|
|
134
|
+
|
|
135
|
+
registry.register(ComponentMetadata(
|
|
136
|
+
name="fast_refine",
|
|
137
|
+
component_type="refinement",
|
|
138
|
+
capabilities=["refinement"],
|
|
139
|
+
cost_per_call=0.004, # asymmetric moderate cost
|
|
140
|
+
avg_latency_ms=150,
|
|
141
|
+
reliability_score=1.12, # multiplicative boost
|
|
142
|
+
privacy_level="internal",
|
|
143
|
+
))
|
|
144
|
+
|
|
145
|
+
registry.register(ComponentMetadata(
|
|
146
|
+
name="robust_refine",
|
|
147
|
+
component_type="refinement",
|
|
148
|
+
capabilities=["refinement"],
|
|
149
|
+
cost_per_call=0.012,
|
|
150
|
+
avg_latency_ms=350,
|
|
151
|
+
reliability_score=1.22,
|
|
152
|
+
privacy_level="internal",
|
|
153
|
+
))
|
|
154
|
+
|
|
155
|
+
task = TaskSpecification(
|
|
156
|
+
required_capabilities=[
|
|
157
|
+
"retrieval",
|
|
158
|
+
"generation",
|
|
159
|
+
"verification",
|
|
160
|
+
"refinement",
|
|
161
|
+
],
|
|
162
|
+
max_cost=0.05,
|
|
163
|
+
max_latency=2000,
|
|
164
|
+
privacy_constraint=None,
|
|
165
|
+
objective_weights={
|
|
166
|
+
"cost": 1.0,
|
|
167
|
+
"latency": 0.001,
|
|
168
|
+
"error": 1.0,
|
|
169
|
+
},
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# budget = SynthesisBudget(
|
|
173
|
+
# max_evaluations=10,
|
|
174
|
+
# random_seed=42,
|
|
175
|
+
# )
|
|
176
|
+
|
|
177
|
+
# print("\n=== GREEDY ===")
|
|
178
|
+
# greedy = HeuristicSynthesizer(registry)
|
|
179
|
+
# greedy_frontier = greedy.synthesize(task, budget)
|
|
180
|
+
|
|
181
|
+
# print("\n=== BEAM (width=2) ===")
|
|
182
|
+
# beam = BeamSynthesizer(registry, beam_width=2)
|
|
183
|
+
# beam_frontier = beam.synthesize(task, budget)
|
|
184
|
+
|
|
185
|
+
# print("\n=== RANDOM ===")
|
|
186
|
+
# random_syn = RandomSynthesizer(registry)
|
|
187
|
+
# random_frontier = random_syn.synthesize(task, budget)
|
|
188
|
+
|
|
189
|
+
# reference_point = (1.0, 5000.0, 0.0)
|
|
190
|
+
|
|
191
|
+
# print("\n=== EVALUATION REPORTS ===")
|
|
192
|
+
|
|
193
|
+
# for name, frontier in [
|
|
194
|
+
# ("Greedy", greedy_frontier),
|
|
195
|
+
# ("Beam", beam_frontier),
|
|
196
|
+
# ("Random", random_frontier),
|
|
197
|
+
# ]:
|
|
198
|
+
|
|
199
|
+
# report = evaluate_approximation(
|
|
200
|
+
# approx_frontier=frontier,
|
|
201
|
+
# true_frontier=true_frontier,
|
|
202
|
+
# reference_point=reference_point,
|
|
203
|
+
# )
|
|
204
|
+
|
|
205
|
+
# print(f"\n{name} Results:")
|
|
206
|
+
# for k, v in report.items():
|
|
207
|
+
# print(f" {k}: {v}")
|
|
208
|
+
|
|
209
|
+
print("\n=== DONE ===\n")
|
|
210
|
+
|
|
211
|
+
results = run_budget_sweep(registry, task)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
if __name__ == "__main__":
|
|
215
|
+
main()
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import random
|
|
2
|
+
from typing import Iterable, List, Tuple
|
|
3
|
+
|
|
4
|
+
from chatcortex.optimization.architecture_candidate import ArchitectureCandidate
|
|
5
|
+
from chatcortex.optimization.pareto import dominates, frontier_coverage, hypervolume_monte_carlo
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def compute_coverage(
|
|
9
|
+
approx_frontier: Iterable[ArchitectureCandidate],
|
|
10
|
+
true_frontier: Iterable[ArchitectureCandidate],
|
|
11
|
+
) -> float:
|
|
12
|
+
return frontier_coverage(approx_frontier, true_frontier)
|
|
13
|
+
|
|
14
|
+
def compute_hypervolume(
|
|
15
|
+
frontier: Iterable[ArchitectureCandidate],
|
|
16
|
+
reference_point: Tuple[float, float, float],
|
|
17
|
+
) -> float:
|
|
18
|
+
return hypervolume_monte_carlo(frontier, reference_point)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def hypervolume_loss(
|
|
22
|
+
approx_frontier,
|
|
23
|
+
true_frontier,
|
|
24
|
+
reference_point,
|
|
25
|
+
num_samples=200000,
|
|
26
|
+
seed=42,
|
|
27
|
+
) -> float:
|
|
28
|
+
ref_cost, ref_latency, ref_reliability = reference_point
|
|
29
|
+
|
|
30
|
+
random_number_generator = random.Random(seed)
|
|
31
|
+
|
|
32
|
+
dominated_true = 0
|
|
33
|
+
dominated_approx = 0
|
|
34
|
+
|
|
35
|
+
for _ in range(num_samples):
|
|
36
|
+
|
|
37
|
+
sample_cost = random_number_generator.uniform(0, ref_cost)
|
|
38
|
+
sample_latency = random_number_generator.uniform(0, ref_latency)
|
|
39
|
+
sample_reliability = random_number_generator.uniform(ref_reliability, 1.0)
|
|
40
|
+
|
|
41
|
+
for candidate in true_frontier:
|
|
42
|
+
if (
|
|
43
|
+
candidate.total_cost <= sample_cost
|
|
44
|
+
and candidate.total_latency <= sample_latency
|
|
45
|
+
and candidate.total_reliability >= sample_reliability
|
|
46
|
+
):
|
|
47
|
+
dominated_true += 1
|
|
48
|
+
break
|
|
49
|
+
|
|
50
|
+
for candidate in approx_frontier:
|
|
51
|
+
if (
|
|
52
|
+
candidate.total_cost <= sample_cost
|
|
53
|
+
and candidate.total_latency <= sample_latency
|
|
54
|
+
and candidate.total_reliability >= sample_reliability
|
|
55
|
+
):
|
|
56
|
+
dominated_approx += 1
|
|
57
|
+
break
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
box_volume = ref_cost * ref_latency * (1.0 - ref_reliability)
|
|
61
|
+
|
|
62
|
+
hv_true = (dominated_true / num_samples) * box_volume
|
|
63
|
+
hv_approx = (dominated_approx / num_samples) * box_volume
|
|
64
|
+
|
|
65
|
+
return max(0.0, hv_true - hv_approx)
|
|
66
|
+
|
|
67
|
+
def additive_regret(
|
|
68
|
+
candidate: ArchitectureCandidate,
|
|
69
|
+
true_frontier: Iterable[ArchitectureCandidate],
|
|
70
|
+
) -> Tuple[float, float, float]:
|
|
71
|
+
"""
|
|
72
|
+
returns
|
|
73
|
+
- (cost_regret, latency_regret, reliability_regret)
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
dominating = [
|
|
77
|
+
p for p in true_frontier if dominates(p, candidate)
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
if not dominating:
|
|
81
|
+
return (0.0, 0.0, 0.0)
|
|
82
|
+
|
|
83
|
+
min_cost_diff = min(candidate.total_cost - p.total_cost for p in dominating)
|
|
84
|
+
min_latency_diff = min(candidate.total_latency - p.total_latency for p in dominating)
|
|
85
|
+
min_reliability_diff = min(p.total_reliability - candidate.total_reliability for p in dominating)
|
|
86
|
+
|
|
87
|
+
return (
|
|
88
|
+
max(0.0, min_cost_diff),
|
|
89
|
+
max(0.0, min_latency_diff),
|
|
90
|
+
max(0.0, min_reliability_diff),
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def average_regret(
|
|
94
|
+
approx_frontier: Iterable[ArchitectureCandidate],
|
|
95
|
+
true_frontier: Iterable[ArchitectureCandidate]
|
|
96
|
+
) -> Tuple[float, float, float]:
|
|
97
|
+
"""
|
|
98
|
+
Average additive regret across approx frontier
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
approx_list = list(approx_frontier)
|
|
102
|
+
|
|
103
|
+
if not approx_list:
|
|
104
|
+
return (0.0, 0.0, 0.0)
|
|
105
|
+
|
|
106
|
+
total_cost_regret = 0.0
|
|
107
|
+
total_latency_regret = 0.0
|
|
108
|
+
total_reliability_regret = 0.0
|
|
109
|
+
|
|
110
|
+
for candidate in approx_list:
|
|
111
|
+
cr, lr, rr = additive_regret(candidate, true_frontier)
|
|
112
|
+
total_cost_regret += cr
|
|
113
|
+
total_latency_regret += lr
|
|
114
|
+
total_reliability_regret += rr
|
|
115
|
+
|
|
116
|
+
n = len(approx_list)
|
|
117
|
+
|
|
118
|
+
return (
|
|
119
|
+
total_cost_regret / n,
|
|
120
|
+
total_latency_regret / n,
|
|
121
|
+
total_reliability_regret / n,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def evaluate_approximation(
|
|
125
|
+
approx_frontier: List[ArchitectureCandidate],
|
|
126
|
+
true_frontier: List[ArchitectureCandidate],
|
|
127
|
+
reference_point: Tuple[float, float, float],
|
|
128
|
+
) -> dict:
|
|
129
|
+
"""
|
|
130
|
+
Returns a full approximate report
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
coverage = compute_coverage(approx_frontier, true_frontier)
|
|
134
|
+
|
|
135
|
+
hv_loss = hypervolume_loss(approx_frontier, true_frontier, reference_point)
|
|
136
|
+
avg_regret = average_regret(approx_frontier, true_frontier)
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
"coverage": coverage,
|
|
140
|
+
"hypervolume_loss": hv_loss,
|
|
141
|
+
"avg_cost_regret": avg_regret[0],
|
|
142
|
+
"avg_latency_regret": avg_regret[1],
|
|
143
|
+
"avg_reliability_regret": avg_regret[2],
|
|
144
|
+
}
|
|
@@ -31,6 +31,31 @@ class AgentGraph:
|
|
|
31
31
|
self._graph.remove_edge(from_node, to_node)
|
|
32
32
|
raise ValueError("Edge creates cucle. AgentGraph must remain acyclic")
|
|
33
33
|
|
|
34
|
+
def copy(self) -> "AgentGraph":
|
|
35
|
+
"""
|
|
36
|
+
Deep copy of the AgentGraph
|
|
37
|
+
|
|
38
|
+
Copies:
|
|
39
|
+
- All nodes
|
|
40
|
+
- associated metadata
|
|
41
|
+
- all edges
|
|
42
|
+
|
|
43
|
+
Preserves DAG structure
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
new_graph = AgentGraph()
|
|
47
|
+
|
|
48
|
+
# Copy nodes with metadata
|
|
49
|
+
for node_id, data in self._graph.nodes(data=True):
|
|
50
|
+
metadata = data["metadata"]
|
|
51
|
+
new_graph.add_component(node_id, metadata)
|
|
52
|
+
|
|
53
|
+
# Copy edges
|
|
54
|
+
for source, target in self._graph.edges():
|
|
55
|
+
new_graph.add_edge(source, target)
|
|
56
|
+
|
|
57
|
+
return new_graph
|
|
58
|
+
|
|
34
59
|
# Validation
|
|
35
60
|
|
|
36
61
|
def validate(self) -> bool:
|