chatcortex 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chatcortex-0.7.0 → chatcortex-0.8.0}/PKG-INFO +13 -5
- {chatcortex-0.7.0 → chatcortex-0.8.0}/README.md +11 -2
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/__init__.py +1 -1
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/execution/executor.py +36 -3
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/PKG-INFO +13 -5
- {chatcortex-0.7.0 → chatcortex-0.8.0}/pyproject.toml +3 -4
- chatcortex-0.8.0/tests/test_execution.py +98 -0
- chatcortex-0.7.0/tests/test_execution.py +0 -30
- {chatcortex-0.7.0 → chatcortex-0.8.0}/LICENSE +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/adapters/__init__.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/adapters/openai_backend.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/evaluation/approximation_metrics.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/evaluation/harness.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/graph/agent_graph.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/graph/node_ids.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/loop/__init__.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/loop/closed_loop.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/optimization/architecture_candidate.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/optimization/pareto.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/capability_registry.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/cli.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/demo_components.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/loader.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/metadata.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/package.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/base.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/beam_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/budget.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/engine.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/exhaustive_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/heuristic_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/random_synthesizer.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/task_specification.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/telemetry/calibration.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/telemetry/logger.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/SOURCES.txt +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/dependency_links.txt +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/entry_points.txt +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/requires.txt +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/top_level.txt +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/setup.cfg +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_agent_graph.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_calibration.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_closed_loop.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_executor_backend.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_node_ids.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_openai_backend_install.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_pareto.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry_loader.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry_pack.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_regression_gates.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_synthesizers.py +0 -0
- {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_task_specification.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chatcortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
|
|
5
5
|
Author: Siddharth Saraswat
|
|
6
6
|
License: MIT
|
|
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
|
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
20
|
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
-
Requires-Python: >=3.
|
|
21
|
+
Requires-Python: >=3.10
|
|
23
22
|
Description-Content-Type: text/markdown
|
|
24
23
|
License-File: LICENSE
|
|
25
24
|
Requires-Dist: networkx>=2.8
|
|
@@ -37,7 +36,7 @@ Dynamic: license-file
|
|
|
37
36
|
|
|
38
37
|
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
39
38
|
[](https://pypi.org/project/chatcortex/)
|
|
40
|
-
[](https://www.python.org/)
|
|
41
40
|
[](LICENSE)
|
|
42
41
|
|
|
43
42
|
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
@@ -48,7 +47,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
48
47
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
49
48
|
```
|
|
50
49
|
|
|
51
|
-
**ChatCortex
|
|
50
|
+
**ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
|
|
51
|
+
|
|
52
|
+
| Component | Field | Declared | Calibrated | Truth |
|
|
53
|
+
|---|---|---|---|---|
|
|
54
|
+
| `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
|
|
55
|
+
| `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
|
|
56
|
+
| `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
|
|
57
|
+
| `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
|
|
58
|
+
|
|
59
|
+
> **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
|
|
52
60
|
|
|
53
61
|
## Installation
|
|
54
62
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
4
4
|
[](https://pypi.org/project/chatcortex/)
|
|
5
|
-
[](https://www.python.org/)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
|
|
8
8
|
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
@@ -13,7 +13,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
13
13
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
**ChatCortex
|
|
16
|
+
**ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
|
|
17
|
+
|
|
18
|
+
| Component | Field | Declared | Calibrated | Truth |
|
|
19
|
+
|---|---|---|---|---|
|
|
20
|
+
| `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
|
|
21
|
+
| `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
|
|
22
|
+
| `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
|
|
23
|
+
| `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
|
|
24
|
+
|
|
25
|
+
> **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
|
|
17
26
|
|
|
18
27
|
## Installation
|
|
19
28
|
|
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from typing import Any, Literal, Optional, Protocol
|
|
4
4
|
|
|
5
5
|
from chatcortex.graph.agent_graph import AgentGraph
|
|
6
|
+
from chatcortex.registry.capability_registry import CapabilityRegistry
|
|
6
7
|
from chatcortex.registry.metadata import ComponentMetadata
|
|
7
8
|
from chatcortex.telemetry.logger import TelemetryLogger
|
|
8
9
|
|
|
@@ -68,15 +69,34 @@ class AgentExecutor:
|
|
|
68
69
|
The chain stops on the first failure, exactly like the simulator.
|
|
69
70
|
A backend exception is recorded as a failed probe (zero cost) and
|
|
70
71
|
also stops the chain, so partial telemetry survives the error.
|
|
72
|
+
|
|
73
|
+
Hidden ground truth: by default the simulator draws outcomes from the
|
|
74
|
+
metadata carried by the graph, i.e. from the *declared* values, so a
|
|
75
|
+
calibration loop fed by it can only rediscover its own priors plus
|
|
76
|
+
sampling noise. Pass ``ground_truth`` (a registry holding each
|
|
77
|
+
component's *true* cost, latency and reliability) and the simulator
|
|
78
|
+
draws from those instead, looked up by component name. The synthesizer
|
|
79
|
+
and calibration never see that registry, which makes the simulator a
|
|
80
|
+
real test of whether a loop converges on values it was not told.
|
|
81
|
+
Components absent from ``ground_truth`` behave as declared.
|
|
82
|
+
``ground_truth`` cannot be combined with ``backend``: a backend
|
|
83
|
+
already is the source of truth.
|
|
71
84
|
"""
|
|
72
85
|
def __init__(
|
|
73
86
|
self,
|
|
74
87
|
mode: ExecutionMode = "deterministic",
|
|
75
88
|
seed: Optional[int] = None,
|
|
76
89
|
backend: Optional[ExecutionProtocol] = None,
|
|
90
|
+
ground_truth: Optional[CapabilityRegistry] = None,
|
|
77
91
|
):
|
|
92
|
+
if backend is not None and ground_truth is not None:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
"ground_truth only applies to the simulator; a backend "
|
|
95
|
+
"is its own source of truth, so pass one or the other"
|
|
96
|
+
)
|
|
78
97
|
self.mode = mode
|
|
79
98
|
self.backend = backend
|
|
99
|
+
self.ground_truth = ground_truth
|
|
80
100
|
# local random generator - NO RNG pollution, no side-effects, clean isolation
|
|
81
101
|
self.random = random.Random(seed)
|
|
82
102
|
|
|
@@ -115,8 +135,12 @@ class AgentExecutor:
|
|
|
115
135
|
probe = self._invoke_backend(metadata, node_input)
|
|
116
136
|
return probe.latency_ms, probe.cost, probe.success, probe.output
|
|
117
137
|
|
|
138
|
+
# What the component really does: the hidden truth when one is
|
|
139
|
+
# supplied, otherwise the declared metadata itself.
|
|
140
|
+
actual = self._true_metadata(metadata)
|
|
141
|
+
|
|
118
142
|
# Simulated latency
|
|
119
|
-
simulated_latency =
|
|
143
|
+
simulated_latency = actual.avg_latency_ms
|
|
120
144
|
|
|
121
145
|
# Optional: actually sleep (disable if too slow)
|
|
122
146
|
# time.sleep(simulated_latency / 1000)
|
|
@@ -125,11 +149,20 @@ class AgentExecutor:
|
|
|
125
149
|
if self.mode == "deterministic":
|
|
126
150
|
success = True
|
|
127
151
|
elif self.mode == "probabilistic":
|
|
128
|
-
success = self.random.random() <=
|
|
152
|
+
success = self.random.random() <= actual.reliability_score
|
|
129
153
|
else:
|
|
130
154
|
raise ValueError("Invalid execution mode")
|
|
131
155
|
|
|
132
|
-
return simulated_latency,
|
|
156
|
+
return simulated_latency, actual.cost_per_call, success, None
|
|
157
|
+
|
|
158
|
+
def _true_metadata(self, declared: ComponentMetadata) -> ComponentMetadata:
|
|
159
|
+
"""The metadata the simulator samples from for one component."""
|
|
160
|
+
if self.ground_truth is None:
|
|
161
|
+
return declared
|
|
162
|
+
try:
|
|
163
|
+
return self.ground_truth.get(declared.name)
|
|
164
|
+
except KeyError:
|
|
165
|
+
return declared
|
|
133
166
|
|
|
134
167
|
def _invoke_backend(
|
|
135
168
|
self, metadata: ComponentMetadata, node_input: Any
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chatcortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
|
|
5
5
|
Author: Siddharth Saraswat
|
|
6
6
|
License: MIT
|
|
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
|
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
20
|
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
-
Requires-Python: >=3.
|
|
21
|
+
Requires-Python: >=3.10
|
|
23
22
|
Description-Content-Type: text/markdown
|
|
24
23
|
License-File: LICENSE
|
|
25
24
|
Requires-Dist: networkx>=2.8
|
|
@@ -37,7 +36,7 @@ Dynamic: license-file
|
|
|
37
36
|
|
|
38
37
|
[](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
|
|
39
38
|
[](https://pypi.org/project/chatcortex/)
|
|
40
|
-
[](https://www.python.org/)
|
|
41
40
|
[](LICENSE)
|
|
42
41
|
|
|
43
42
|
**ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
|
|
@@ -48,7 +47,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
|
|
|
48
47
|
Task Specification → Architecture Search → Pareto-Optimal Architectures
|
|
49
48
|
```
|
|
50
49
|
|
|
51
|
-
**ChatCortex
|
|
50
|
+
**ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
|
|
51
|
+
|
|
52
|
+
| Component | Field | Declared | Calibrated | Truth |
|
|
53
|
+
|---|---|---|---|---|
|
|
54
|
+
| `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
|
|
55
|
+
| `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
|
|
56
|
+
| `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
|
|
57
|
+
| `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
|
|
58
|
+
|
|
59
|
+
> **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
|
|
52
60
|
|
|
53
61
|
## Installation
|
|
54
62
|
|
|
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "chatcortex"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.8.0"
|
|
8
8
|
description = "Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = { text = "MIT" }
|
|
12
12
|
|
|
13
13
|
authors = [
|
|
@@ -31,7 +31,6 @@ classifiers = [
|
|
|
31
31
|
"Intended Audience :: Developers",
|
|
32
32
|
"License :: OSI Approved :: MIT License",
|
|
33
33
|
"Programming Language :: Python :: 3",
|
|
34
|
-
"Programming Language :: Python :: 3.9",
|
|
35
34
|
"Programming Language :: Python :: 3.10",
|
|
36
35
|
"Programming Language :: Python :: 3.11",
|
|
37
36
|
"Programming Language :: Python :: 3.12",
|
|
@@ -83,7 +82,7 @@ pythonpath = ["."]
|
|
|
83
82
|
|
|
84
83
|
[tool.ruff]
|
|
85
84
|
line-length = 100
|
|
86
|
-
target-version = "
|
|
85
|
+
target-version = "py310"
|
|
87
86
|
|
|
88
87
|
[tool.ruff.lint]
|
|
89
88
|
select = ["E", "F", "W", "I"]
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from chatcortex import AgentGraph, CapabilityRegistry, ComponentMetadata
|
|
4
|
+
from chatcortex.execution.executor import AgentExecutor
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def build_graph():
|
|
8
|
+
g = AgentGraph()
|
|
9
|
+
g.add_component(
|
|
10
|
+
"a",
|
|
11
|
+
ComponentMetadata(
|
|
12
|
+
name="a",
|
|
13
|
+
component_type="tool",
|
|
14
|
+
capabilities=["x"],
|
|
15
|
+
cost_per_call=0.001,
|
|
16
|
+
avg_latency_ms=1,
|
|
17
|
+
reliability_score=0.5,
|
|
18
|
+
privacy_level="internal",
|
|
19
|
+
),
|
|
20
|
+
)
|
|
21
|
+
return g
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_deterministic_execution():
|
|
25
|
+
result = AgentExecutor(mode="deterministic").execute(build_graph())
|
|
26
|
+
assert result.summary() is not None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _truth(reliability=0.5, latency=1, cost=0.001, name="a"):
|
|
30
|
+
registry = CapabilityRegistry()
|
|
31
|
+
registry.register(
|
|
32
|
+
ComponentMetadata(
|
|
33
|
+
name=name,
|
|
34
|
+
component_type="tool",
|
|
35
|
+
capabilities=["x"],
|
|
36
|
+
cost_per_call=cost,
|
|
37
|
+
avg_latency_ms=latency,
|
|
38
|
+
reliability_score=reliability,
|
|
39
|
+
privacy_level="internal",
|
|
40
|
+
)
|
|
41
|
+
)
|
|
42
|
+
return registry
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _records(executor, runs=1):
|
|
46
|
+
records = []
|
|
47
|
+
for _ in range(runs):
|
|
48
|
+
records.extend(executor.execute(build_graph()).telemetry.records)
|
|
49
|
+
return records
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_ground_truth_latency_and_cost_replace_declared_values():
|
|
53
|
+
executor = AgentExecutor(ground_truth=_truth(latency=250, cost=0.02))
|
|
54
|
+
(record,) = _records(executor)
|
|
55
|
+
assert record["latency_ms"] == 250
|
|
56
|
+
assert record["cost"] == 0.02
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_ground_truth_reliability_drives_success_not_the_declaration():
|
|
60
|
+
# Declared reliability is 0.5; the hidden truth says the component
|
|
61
|
+
# never works, then that it always works.
|
|
62
|
+
never = AgentExecutor(
|
|
63
|
+
mode="probabilistic", seed=7, ground_truth=_truth(reliability=0.0)
|
|
64
|
+
)
|
|
65
|
+
# random() <= 0.0 is possible only at exactly 0.0, so allow none.
|
|
66
|
+
assert not any(r["success"] for r in _records(never, runs=200))
|
|
67
|
+
|
|
68
|
+
always = AgentExecutor(
|
|
69
|
+
mode="probabilistic", seed=7, ground_truth=_truth(reliability=1.0)
|
|
70
|
+
)
|
|
71
|
+
assert all(r["success"] for r in _records(always, runs=200))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_component_missing_from_ground_truth_behaves_as_declared():
|
|
75
|
+
executor = AgentExecutor(ground_truth=_truth(latency=999, name="other"))
|
|
76
|
+
(record,) = _records(executor)
|
|
77
|
+
assert record["latency_ms"] == 1
|
|
78
|
+
assert record["cost"] == 0.001
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_no_ground_truth_keeps_the_declared_simulation():
|
|
82
|
+
with_none = _records(AgentExecutor(mode="probabilistic", seed=3), runs=50)
|
|
83
|
+
with_same = _records(
|
|
84
|
+
AgentExecutor(mode="probabilistic", seed=3, ground_truth=_truth()),
|
|
85
|
+
runs=50,
|
|
86
|
+
)
|
|
87
|
+
assert with_none == with_same
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_ground_truth_with_backend_is_rejected():
|
|
91
|
+
with pytest.raises(ValueError, match="ground_truth"):
|
|
92
|
+
AgentExecutor(backend=lambda metadata, node_input: None, ground_truth=_truth())
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_probabilistic_execution_is_seeded():
|
|
96
|
+
r1 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
|
|
97
|
+
r2 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
|
|
98
|
+
assert r1.summary() == r2.summary()
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
from chatcortex import AgentGraph, ComponentMetadata
|
|
2
|
-
from chatcortex.execution.executor import AgentExecutor
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
def build_graph():
|
|
6
|
-
g = AgentGraph()
|
|
7
|
-
g.add_component(
|
|
8
|
-
"a",
|
|
9
|
-
ComponentMetadata(
|
|
10
|
-
name="a",
|
|
11
|
-
component_type="tool",
|
|
12
|
-
capabilities=["x"],
|
|
13
|
-
cost_per_call=0.001,
|
|
14
|
-
avg_latency_ms=1,
|
|
15
|
-
reliability_score=0.5,
|
|
16
|
-
privacy_level="internal",
|
|
17
|
-
),
|
|
18
|
-
)
|
|
19
|
-
return g
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def test_deterministic_execution():
|
|
23
|
-
result = AgentExecutor(mode="deterministic").execute(build_graph())
|
|
24
|
-
assert result.summary() is not None
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def test_probabilistic_execution_is_seeded():
|
|
28
|
-
r1 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
|
|
29
|
-
r2 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
|
|
30
|
-
assert r1.summary() == r2.summary()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py
RENAMED
|
File without changes
|
{chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|