chatcortex 0.7.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {chatcortex-0.7.0 → chatcortex-0.8.0}/PKG-INFO +13 -5
  2. {chatcortex-0.7.0 → chatcortex-0.8.0}/README.md +11 -2
  3. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/__init__.py +1 -1
  4. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/execution/executor.py +36 -3
  5. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/PKG-INFO +13 -5
  6. {chatcortex-0.7.0 → chatcortex-0.8.0}/pyproject.toml +3 -4
  7. chatcortex-0.8.0/tests/test_execution.py +98 -0
  8. chatcortex-0.7.0/tests/test_execution.py +0 -30
  9. {chatcortex-0.7.0 → chatcortex-0.8.0}/LICENSE +0 -0
  10. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/adapters/__init__.py +0 -0
  11. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/adapters/openai_backend.py +0 -0
  12. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/evaluation/approximation_metrics.py +0 -0
  13. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/evaluation/harness.py +0 -0
  14. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/graph/agent_graph.py +0 -0
  15. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/graph/node_ids.py +0 -0
  16. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/loop/__init__.py +0 -0
  17. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/loop/closed_loop.py +0 -0
  18. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/optimization/architecture_candidate.py +0 -0
  19. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/optimization/pareto.py +0 -0
  20. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/capability_registry.py +0 -0
  21. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/cli.py +0 -0
  22. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/demo_components.py +0 -0
  23. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/loader.py +0 -0
  24. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/metadata.py +0 -0
  25. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/registry/package.py +0 -0
  26. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/base.py +0 -0
  27. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/beam_synthesizer.py +0 -0
  28. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/budget.py +0 -0
  29. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/engine.py +0 -0
  30. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/exhaustive_synthesizer.py +0 -0
  31. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/heuristic_synthesizer.py +0 -0
  32. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +0 -0
  33. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +0 -0
  34. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/random_synthesizer.py +0 -0
  35. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/synthesis/task_specification.py +0 -0
  36. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/telemetry/calibration.py +0 -0
  37. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex/telemetry/logger.py +0 -0
  38. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/SOURCES.txt +0 -0
  39. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/dependency_links.txt +0 -0
  40. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/entry_points.txt +0 -0
  41. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/requires.txt +0 -0
  42. {chatcortex-0.7.0 → chatcortex-0.8.0}/chatcortex.egg-info/top_level.txt +0 -0
  43. {chatcortex-0.7.0 → chatcortex-0.8.0}/setup.cfg +0 -0
  44. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_agent_graph.py +0 -0
  45. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_calibration.py +0 -0
  46. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_closed_loop.py +0 -0
  47. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_executor_backend.py +0 -0
  48. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_node_ids.py +0 -0
  49. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_openai_backend_install.py +0 -0
  50. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_pareto.py +0 -0
  51. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry.py +0 -0
  52. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry_loader.py +0 -0
  53. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_registry_pack.py +0 -0
  54. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_regression_gates.py +0 -0
  55. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_synthesizers.py +0 -0
  56. {chatcortex-0.7.0 → chatcortex-0.8.0}/tests/test_task_specification.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chatcortex
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
5
5
  Author: Siddharth Saraswat
6
6
  License: MIT
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
13
13
  Classifier: Intended Audience :: Developers
14
14
  Classifier: License :: OSI Approved :: MIT License
15
15
  Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
16
  Classifier: Programming Language :: Python :: 3.10
18
17
  Classifier: Programming Language :: Python :: 3.11
19
18
  Classifier: Programming Language :: Python :: 3.12
20
19
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
20
  Classifier: Topic :: Software Development :: Libraries
22
- Requires-Python: >=3.9
21
+ Requires-Python: >=3.10
23
22
  Description-Content-Type: text/markdown
24
23
  License-File: LICENSE
25
24
  Requires-Dist: networkx>=2.8
@@ -37,7 +36,7 @@ Dynamic: license-file
37
36
 
38
37
  [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
39
38
  [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
40
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
39
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
41
40
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
42
41
 
43
42
  **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
@@ -48,7 +47,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
48
47
  Task Specification → Architecture Search → Pareto-Optimal Architectures
49
48
  ```
50
49
 
51
- **ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
50
+ **ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
51
+
52
+ | Component | Field | Declared | Calibrated | Truth |
53
+ |---|---|---|---|---|
54
+ | `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
55
+ | `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
56
+ | `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
57
+ | `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
58
+
59
+ > **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
52
60
 
53
61
  ## Installation
54
62
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
4
4
  [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
5
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
5
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
6
6
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
7
 
8
8
  **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
@@ -13,7 +13,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
13
13
  Task Specification → Architecture Search → Pareto-Optimal Architectures
14
14
  ```
15
15
 
16
- **ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
16
+ **ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
17
+
18
+ | Component | Field | Declared | Calibrated | Truth |
19
+ |---|---|---|---|---|
20
+ | `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
21
+ | `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
22
+ | `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
23
+ | `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
24
+
25
+ > **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
17
26
 
18
27
  ## Installation
19
28
 
@@ -51,4 +51,4 @@ __all__ = [
51
51
  "RegistryReport",
52
52
  ]
53
53
 
54
- __version__ = "0.7.0"
54
+ __version__ = "0.8.0"
@@ -3,6 +3,7 @@ from dataclasses import dataclass
3
3
  from typing import Any, Literal, Optional, Protocol
4
4
 
5
5
  from chatcortex.graph.agent_graph import AgentGraph
6
+ from chatcortex.registry.capability_registry import CapabilityRegistry
6
7
  from chatcortex.registry.metadata import ComponentMetadata
7
8
  from chatcortex.telemetry.logger import TelemetryLogger
8
9
 
@@ -68,15 +69,34 @@ class AgentExecutor:
68
69
  The chain stops on the first failure, exactly like the simulator.
69
70
  A backend exception is recorded as a failed probe (zero cost) and
70
71
  also stops the chain, so partial telemetry survives the error.
72
+
73
+ Hidden ground truth: by default the simulator draws outcomes from the
74
+ metadata carried by the graph, i.e. from the *declared* values, so a
75
+ calibration loop fed by it can only rediscover its own priors plus
76
+ sampling noise. Pass ``ground_truth`` (a registry holding each
77
+ component's *true* cost, latency and reliability) and the simulator
78
+ draws from those instead, looked up by component name. The synthesizer
79
+ and calibration never see that registry, which makes the simulator a
80
+ real test of whether a loop converges on values it was not told.
81
+ Components absent from ``ground_truth`` behave as declared.
82
+ ``ground_truth`` cannot be combined with ``backend``: a backend
83
+ already is the source of truth.
71
84
  """
72
85
  def __init__(
73
86
  self,
74
87
  mode: ExecutionMode = "deterministic",
75
88
  seed: Optional[int] = None,
76
89
  backend: Optional[ExecutionProtocol] = None,
90
+ ground_truth: Optional[CapabilityRegistry] = None,
77
91
  ):
92
+ if backend is not None and ground_truth is not None:
93
+ raise ValueError(
94
+ "ground_truth only applies to the simulator; a backend "
95
+ "is its own source of truth, so pass one or the other"
96
+ )
78
97
  self.mode = mode
79
98
  self.backend = backend
99
+ self.ground_truth = ground_truth
80
100
  # local random generator - NO RNG pollution, no side-effects, clean isolation
81
101
  self.random = random.Random(seed)
82
102
 
@@ -115,8 +135,12 @@ class AgentExecutor:
115
135
  probe = self._invoke_backend(metadata, node_input)
116
136
  return probe.latency_ms, probe.cost, probe.success, probe.output
117
137
 
138
+ # What the component really does: the hidden truth when one is
139
+ # supplied, otherwise the declared metadata itself.
140
+ actual = self._true_metadata(metadata)
141
+
118
142
  # Simulated latency
119
- simulated_latency = metadata.avg_latency_ms
143
+ simulated_latency = actual.avg_latency_ms
120
144
 
121
145
  # Optional: actually sleep (disable if too slow)
122
146
  # time.sleep(simulated_latency / 1000)
@@ -125,11 +149,20 @@ class AgentExecutor:
125
149
  if self.mode == "deterministic":
126
150
  success = True
127
151
  elif self.mode == "probabilistic":
128
- success = self.random.random() <= metadata.reliability_score
152
+ success = self.random.random() <= actual.reliability_score
129
153
  else:
130
154
  raise ValueError("Invalid execution mode")
131
155
 
132
- return simulated_latency, metadata.cost_per_call, success, None
156
+ return simulated_latency, actual.cost_per_call, success, None
157
+
158
+ def _true_metadata(self, declared: ComponentMetadata) -> ComponentMetadata:
159
+ """The metadata the simulator samples from for one component."""
160
+ if self.ground_truth is None:
161
+ return declared
162
+ try:
163
+ return self.ground_truth.get(declared.name)
164
+ except KeyError:
165
+ return declared
133
166
 
134
167
  def _invoke_backend(
135
168
  self, metadata: ComponentMetadata, node_input: Any
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chatcortex
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
5
5
  Author: Siddharth Saraswat
6
6
  License: MIT
@@ -13,13 +13,12 @@ Classifier: Intended Audience :: Science/Research
13
13
  Classifier: Intended Audience :: Developers
14
14
  Classifier: License :: OSI Approved :: MIT License
15
15
  Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
16
  Classifier: Programming Language :: Python :: 3.10
18
17
  Classifier: Programming Language :: Python :: 3.11
19
18
  Classifier: Programming Language :: Python :: 3.12
20
19
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
20
  Classifier: Topic :: Software Development :: Libraries
22
- Requires-Python: >=3.9
21
+ Requires-Python: >=3.10
23
22
  Description-Content-Type: text/markdown
24
23
  License-File: LICENSE
25
24
  Requires-Dist: networkx>=2.8
@@ -37,7 +36,7 @@ Dynamic: license-file
37
36
 
38
37
  [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
39
38
  [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
40
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
39
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
41
40
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
42
41
 
43
42
  **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
@@ -48,7 +47,16 @@ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → to
48
47
  Task Specification → Architecture Search → Pareto-Optimal Architectures
49
48
  ```
50
49
 
51
- **ChatCortex is the only agent framework that re-synthesizes your architecture from what actually happened.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against *measured* behavior instead of declared guesses. The shipped demo makes this concrete: in four closed-loop iterations, `examples/closed_loop.py` moves a component's declared reliability from 0.78 to a corrected 0.53, lifts a budget retriever from 0.82 to 0.93, and reshapes the search frontier around the values the system actually measured.
50
+ **ChatCortex re-synthesizes your architecture from what was measured.** Execution telemetry is calibrated back into the component registry, so every new synthesis round runs against observed behavior instead of declared guesses. `examples/closed_loop.py` tests this offline against a **hidden ground truth**: the starting registry is wrong about four components, and the simulator draws outcomes from true values the loop never sees. After four iterations (alpha 0.4, seed 42):
51
+
52
+ | Component | Field | Declared | Calibrated | Truth |
53
+ |---|---|---|---|---|
54
+ | `retrieval_speedster` | reliability | 0.78 | 0.542 | 0.55 |
55
+ | `retrieval_budget` | reliability | 0.82 | 0.908 | 0.95 |
56
+ | `generation_balanced` | latency (ms) | 180 | 301.9 | 320 |
57
+ | `verification_reliable` | cost per call | 0.006 | 0.0086 | 0.009 |
58
+
59
+ > **What this does and does not show.** The mis-declared values move toward the truth, and mean error across all twelve components falls for latency (11.67 → 1.51 ms) and cost. Reliability is the exception: its mean error *rises* (0.030 → 0.041), because a handful of pass/fail probes is a noisy measurement. Treat calibrated reliability as approximate until you have many runs per component.
52
60
 
53
61
  ## Installation
54
62
 
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "chatcortex"
7
- version = "0.7.0"
7
+ version = "0.8.0"
8
8
  description = "Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets"
9
9
  readme = "README.md"
10
- requires-python = ">=3.9"
10
+ requires-python = ">=3.10"
11
11
  license = { text = "MIT" }
12
12
 
13
13
  authors = [
@@ -31,7 +31,6 @@ classifiers = [
31
31
  "Intended Audience :: Developers",
32
32
  "License :: OSI Approved :: MIT License",
33
33
  "Programming Language :: Python :: 3",
34
- "Programming Language :: Python :: 3.9",
35
34
  "Programming Language :: Python :: 3.10",
36
35
  "Programming Language :: Python :: 3.11",
37
36
  "Programming Language :: Python :: 3.12",
@@ -83,7 +82,7 @@ pythonpath = ["."]
83
82
 
84
83
  [tool.ruff]
85
84
  line-length = 100
86
- target-version = "py39"
85
+ target-version = "py310"
87
86
 
88
87
  [tool.ruff.lint]
89
88
  select = ["E", "F", "W", "I"]
@@ -0,0 +1,98 @@
1
+ import pytest
2
+
3
+ from chatcortex import AgentGraph, CapabilityRegistry, ComponentMetadata
4
+ from chatcortex.execution.executor import AgentExecutor
5
+
6
+
7
+ def build_graph():
8
+ g = AgentGraph()
9
+ g.add_component(
10
+ "a",
11
+ ComponentMetadata(
12
+ name="a",
13
+ component_type="tool",
14
+ capabilities=["x"],
15
+ cost_per_call=0.001,
16
+ avg_latency_ms=1,
17
+ reliability_score=0.5,
18
+ privacy_level="internal",
19
+ ),
20
+ )
21
+ return g
22
+
23
+
24
+ def test_deterministic_execution():
25
+ result = AgentExecutor(mode="deterministic").execute(build_graph())
26
+ assert result.summary() is not None
27
+
28
+
29
+ def _truth(reliability=0.5, latency=1, cost=0.001, name="a"):
30
+ registry = CapabilityRegistry()
31
+ registry.register(
32
+ ComponentMetadata(
33
+ name=name,
34
+ component_type="tool",
35
+ capabilities=["x"],
36
+ cost_per_call=cost,
37
+ avg_latency_ms=latency,
38
+ reliability_score=reliability,
39
+ privacy_level="internal",
40
+ )
41
+ )
42
+ return registry
43
+
44
+
45
+ def _records(executor, runs=1):
46
+ records = []
47
+ for _ in range(runs):
48
+ records.extend(executor.execute(build_graph()).telemetry.records)
49
+ return records
50
+
51
+
52
+ def test_ground_truth_latency_and_cost_replace_declared_values():
53
+ executor = AgentExecutor(ground_truth=_truth(latency=250, cost=0.02))
54
+ (record,) = _records(executor)
55
+ assert record["latency_ms"] == 250
56
+ assert record["cost"] == 0.02
57
+
58
+
59
+ def test_ground_truth_reliability_drives_success_not_the_declaration():
60
+ # Declared reliability is 0.5; the hidden truth says the component
61
+ # never works, then that it always works.
62
+ never = AgentExecutor(
63
+ mode="probabilistic", seed=7, ground_truth=_truth(reliability=0.0)
64
+ )
65
+ # random() <= 0.0 is possible only at exactly 0.0, so allow none.
66
+ assert not any(r["success"] for r in _records(never, runs=200))
67
+
68
+ always = AgentExecutor(
69
+ mode="probabilistic", seed=7, ground_truth=_truth(reliability=1.0)
70
+ )
71
+ assert all(r["success"] for r in _records(always, runs=200))
72
+
73
+
74
+ def test_component_missing_from_ground_truth_behaves_as_declared():
75
+ executor = AgentExecutor(ground_truth=_truth(latency=999, name="other"))
76
+ (record,) = _records(executor)
77
+ assert record["latency_ms"] == 1
78
+ assert record["cost"] == 0.001
79
+
80
+
81
+ def test_no_ground_truth_keeps_the_declared_simulation():
82
+ with_none = _records(AgentExecutor(mode="probabilistic", seed=3), runs=50)
83
+ with_same = _records(
84
+ AgentExecutor(mode="probabilistic", seed=3, ground_truth=_truth()),
85
+ runs=50,
86
+ )
87
+ assert with_none == with_same
88
+
89
+
90
+ def test_ground_truth_with_backend_is_rejected():
91
+ with pytest.raises(ValueError, match="ground_truth"):
92
+ AgentExecutor(backend=lambda metadata, node_input: None, ground_truth=_truth())
93
+
94
+
95
+ def test_probabilistic_execution_is_seeded():
96
+ r1 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
97
+ r2 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
98
+ assert r1.summary() == r2.summary()
@@ -1,30 +0,0 @@
1
- from chatcortex import AgentGraph, ComponentMetadata
2
- from chatcortex.execution.executor import AgentExecutor
3
-
4
-
5
- def build_graph():
6
- g = AgentGraph()
7
- g.add_component(
8
- "a",
9
- ComponentMetadata(
10
- name="a",
11
- component_type="tool",
12
- capabilities=["x"],
13
- cost_per_call=0.001,
14
- avg_latency_ms=1,
15
- reliability_score=0.5,
16
- privacy_level="internal",
17
- ),
18
- )
19
- return g
20
-
21
-
22
- def test_deterministic_execution():
23
- result = AgentExecutor(mode="deterministic").execute(build_graph())
24
- assert result.summary() is not None
25
-
26
-
27
- def test_probabilistic_execution_is_seeded():
28
- r1 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
29
- r2 = AgentExecutor(mode="probabilistic", seed=42).execute(build_graph())
30
- assert r1.summary() == r2.summary()
File without changes
File without changes