chatcortex 0.3.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. chatcortex-0.6.0/PKG-INFO +172 -0
  2. chatcortex-0.6.0/README.md +137 -0
  3. chatcortex-0.6.0/chatcortex/__init__.py +47 -0
  4. chatcortex-0.6.0/chatcortex/adapters/__init__.py +7 -0
  5. chatcortex-0.6.0/chatcortex/adapters/openai_backend.py +106 -0
  6. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/evaluation/approximation_metrics.py +12 -47
  7. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/evaluation/harness.py +95 -90
  8. chatcortex-0.6.0/chatcortex/execution/executor.py +146 -0
  9. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/graph/agent_graph.py +30 -12
  10. chatcortex-0.6.0/chatcortex/graph/node_ids.py +27 -0
  11. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/optimization/architecture_candidate.py +1 -1
  12. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/optimization/pareto.py +42 -32
  13. chatcortex-0.6.0/chatcortex/registry/capability_registry.py +93 -0
  14. chatcortex-0.6.0/chatcortex/registry/demo_components.py +61 -0
  15. chatcortex-0.6.0/chatcortex/registry/loader.py +93 -0
  16. chatcortex-0.6.0/chatcortex/registry/metadata.py +139 -0
  17. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/synthesis/base.py +4 -4
  18. chatcortex-0.6.0/chatcortex/synthesis/beam_synthesizer.py +34 -0
  19. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/synthesis/budget.py +7 -7
  20. chatcortex-0.6.0/chatcortex/synthesis/engine.py +246 -0
  21. chatcortex-0.6.0/chatcortex/synthesis/exhaustive_synthesizer.py +24 -0
  22. chatcortex-0.6.0/chatcortex/synthesis/heuristic_synthesizer.py +55 -0
  23. chatcortex-0.6.0/chatcortex/synthesis/pareto_partial_beam_synthesizer.py +30 -0
  24. chatcortex-0.6.0/chatcortex/synthesis/progressive_pareto_beam_synthesizer.py +42 -0
  25. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/synthesis/random_synthesizer.py +104 -89
  26. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/synthesis/task_specification.py +4 -5
  27. chatcortex-0.6.0/chatcortex/telemetry/calibration.py +110 -0
  28. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex/telemetry/logger.py +2 -4
  29. chatcortex-0.6.0/chatcortex.egg-info/PKG-INFO +172 -0
  30. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex.egg-info/SOURCES.txt +22 -5
  31. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex.egg-info/requires.txt +3 -1
  32. {chatcortex-0.3.0 → chatcortex-0.6.0}/pyproject.toml +87 -55
  33. {chatcortex-0.3.0 → chatcortex-0.6.0}/setup.cfg +4 -4
  34. chatcortex-0.6.0/tests/test_agent_graph.py +63 -0
  35. chatcortex-0.6.0/tests/test_calibration.py +173 -0
  36. chatcortex-0.6.0/tests/test_execution.py +30 -0
  37. chatcortex-0.6.0/tests/test_executor_backend.py +208 -0
  38. chatcortex-0.6.0/tests/test_node_ids.py +74 -0
  39. chatcortex-0.6.0/tests/test_openai_backend_install.py +151 -0
  40. chatcortex-0.6.0/tests/test_pareto.py +45 -0
  41. chatcortex-0.6.0/tests/test_registry.py +115 -0
  42. chatcortex-0.6.0/tests/test_registry_loader.py +126 -0
  43. chatcortex-0.6.0/tests/test_regression_gates.py +474 -0
  44. chatcortex-0.6.0/tests/test_synthesizers.py +52 -0
  45. chatcortex-0.6.0/tests/test_task_specification.py +21 -0
  46. chatcortex-0.3.0/PKG-INFO +0 -230
  47. chatcortex-0.3.0/README.md +0 -199
  48. chatcortex-0.3.0/chatcortex/__init__.py +0 -7
  49. chatcortex-0.3.0/chatcortex/budget_sweep_experiment.py +0 -142
  50. chatcortex-0.3.0/chatcortex/demo_phase1.py +0 -142
  51. chatcortex-0.3.0/chatcortex/demo_phase2.py +0 -122
  52. chatcortex-0.3.0/chatcortex/demo_phase3.py +0 -215
  53. chatcortex-0.3.0/chatcortex/execution/executor.py +0 -64
  54. chatcortex-0.3.0/chatcortex/registry/capability_registry.py +0 -62
  55. chatcortex-0.3.0/chatcortex/registry/metadata.py +0 -44
  56. chatcortex-0.3.0/chatcortex/synthesis/beam_synthesizer.py +0 -113
  57. chatcortex-0.3.0/chatcortex/synthesis/exhaustive_synthesizer.py +0 -91
  58. chatcortex-0.3.0/chatcortex/synthesis/heuristic_synthesizer.py +0 -111
  59. chatcortex-0.3.0/chatcortex.egg-info/PKG-INFO +0 -230
  60. {chatcortex-0.3.0 → chatcortex-0.6.0}/LICENSE +0 -0
  61. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex.egg-info/dependency_links.txt +0 -0
  62. {chatcortex-0.3.0 → chatcortex-0.6.0}/chatcortex.egg-info/top_level.txt +0 -0
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.4
2
+ Name: chatcortex
3
+ Version: 0.6.0
4
+ Summary: Framework for multi-objective synthesis and optimization of AI agent architectures under bounded evaluation budgets
5
+ Author: Siddharth Saraswat
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/siddharth1012/chatCortex
8
+ Project-URL: Repository, https://github.com/siddharth1012/chatCortex
9
+ Project-URL: Issues, https://github.com/siddharth1012/chatCortex/issues
10
+ Keywords: automl,multi-objective-optimization,agent-architecture,llm-agents,pareto-optimization,architecture-search,beam-search,ai-systems
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Classifier: Topic :: Software Development :: Libraries
22
+ Requires-Python: >=3.9
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: networkx>=2.8
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=7.0; extra == "dev"
28
+ Requires-Dist: black>=23.0; extra == "dev"
29
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
30
+ Provides-Extra: viz
31
+ Requires-Dist: matplotlib>=3.7; extra == "viz"
32
+ Provides-Extra: real
33
+ Requires-Dist: openai>=2.48; extra == "real"
34
+ Dynamic: license-file
35
+
36
+ # ChatCortex
37
+
38
+ [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
39
+ [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
40
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
41
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
42
+
43
+ **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
44
+
45
+ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
46
+
47
+ ```
48
+ Task Specification → Architecture Search → Pareto-Optimal Architectures
49
+ ```
50
+
51
+ ## Installation
52
+
53
+ ```bash
54
+ pip install chatcortex
55
+ ```
56
+
57
+ Or from source:
58
+
59
+ ```bash
60
+ git clone https://github.com/siddharth1012/chatCortex
61
+ cd chatCortex
62
+ pip install -e .
63
+ ```
64
+
65
+ ## Quickstart
66
+
67
+ ```python
68
+ from chatcortex import (
69
+ TaskSpecification,
70
+ BeamSynthesizer,
71
+ build_demo_registry,
72
+ )
73
+
74
+ # A registry of components with cost/latency/reliability tradeoffs
75
+ registry = build_demo_registry(
76
+ capabilities=("retrieval", "generation", "verification")
77
+ )
78
+
79
+ # The task: an ordered capability chain with hard constraints
80
+ task = TaskSpecification(
81
+ required_capabilities=["retrieval", "generation", "verification"],
82
+ max_cost=0.02,
83
+ max_latency=1500,
84
+ )
85
+
86
+ # Synthesize Pareto-optimal architectures
87
+ synth = BeamSynthesizer(registry, beam_width=5)
88
+ for arch in synth.synthesize(task):
89
+ print(
90
+ f"cost=${arch.total_cost:.4f} "
91
+ f"latency={arch.total_latency:.0f}ms "
92
+ f"reliability={arch.total_reliability:.3f}"
93
+ )
94
+ ```
95
+
96
+ Or run the full demo comparing three synthesis strategies:
97
+
98
+ ```bash
99
+ python examples/quickstart.py
100
+ ```
101
+
102
+ To see the full loop — load a JSON registry, synthesize a frontier, execute one
103
+ architecture, and calibrate the registry's metadata from real telemetry — run:
104
+
105
+ ```bash
106
+ python examples/registry_from_json.py
107
+ ```
108
+
109
+ To search over your own components, build a `CapabilityRegistry` and register
110
+ `ComponentMetadata` describing each model, tool, or verifier — see
111
+ `examples/` for complete walkthroughs.
112
+
113
+ ## How It Works
114
+
115
+ ChatCortex is organized as a layered architecture synthesis system:
116
+
117
+ ```
118
+ TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
119
+ → Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
120
+ ```
121
+
122
+ **Core concepts:**
123
+
124
+ - **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
125
+ - **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
126
+ - **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
127
+ - **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
128
+ - **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
129
+
130
+ **Available synthesizers:**
131
+
132
+ | Synthesizer | Strategy |
133
+ |---|---|
134
+ | `HeuristicSynthesizer` | Greedy deterministic construction |
135
+ | `RandomSynthesizer` | Random baseline |
136
+ | `BeamSynthesizer` | Budget-aware beam search |
137
+ | `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
138
+ | `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
139
+ | `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
140
+
141
+ Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
142
+
143
+ ## Documentation
144
+
145
+ - [Architecture](docs/ARCHITECTURE.md) — system design and component layers
146
+ - [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
147
+ - [Examples](examples/) — runnable demos and budget-sweep experiments
148
+ - [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
149
+ - [Changelog](CHANGELOG.md)
150
+
151
+ ## Research Context
152
+
153
+ ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
154
+
155
+ If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
156
+
157
+ ## Roadmap
158
+
159
+ - [x] Component modeling, capability registry, task specification, DAG representation
160
+ - [x] Exact Pareto frontier computation (exhaustive search)
161
+ - [x] Budget-aware beam search with Pareto pruning and progressive widening
162
+ - [ ] Graph-structured (non-chain) agent synthesis
163
+ - [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
164
+ - [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
165
+
166
+ ## Status
167
+
168
+ Research framework under active development (alpha). APIs may change between minor versions.
169
+
170
+ ## License
171
+
172
+ [MIT](LICENSE) — developed by **Siddharth Saraswat**
@@ -0,0 +1,137 @@
1
+ # ChatCortex
2
+
3
+ [![CI](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml/badge.svg)](https://github.com/siddharth1012/chatCortex/actions/workflows/ci.yml)
4
+ [![PyPI version](https://img.shields.io/pypi/v/chatcortex)](https://pypi.org/project/chatcortex/)
5
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+
8
+ **ChatCortex** is a framework for **automated synthesis and optimization of AI agent architectures** — think *AutoML for AI agents*.
9
+
10
+ Instead of manually wiring LLM pipelines (`retrieval → LLM → verifier → tool`), ChatCortex treats agent design as a **multi-objective architecture search problem** and automatically discovers architectures that optimize **cost**, **latency**, and **reliability** simultaneously.
11
+
12
+ ```
13
+ Task Specification → Architecture Search → Pareto-Optimal Architectures
14
+ ```
15
+
16
+ ## Installation
17
+
18
+ ```bash
19
+ pip install chatcortex
20
+ ```
21
+
22
+ Or from source:
23
+
24
+ ```bash
25
+ git clone https://github.com/siddharth1012/chatCortex
26
+ cd chatCortex
27
+ pip install -e .
28
+ ```
29
+
30
+ ## Quickstart
31
+
32
+ ```python
33
+ from chatcortex import (
34
+ TaskSpecification,
35
+ BeamSynthesizer,
36
+ build_demo_registry,
37
+ )
38
+
39
+ # A registry of components with cost/latency/reliability tradeoffs
40
+ registry = build_demo_registry(
41
+ capabilities=("retrieval", "generation", "verification")
42
+ )
43
+
44
+ # The task: an ordered capability chain with hard constraints
45
+ task = TaskSpecification(
46
+ required_capabilities=["retrieval", "generation", "verification"],
47
+ max_cost=0.02,
48
+ max_latency=1500,
49
+ )
50
+
51
+ # Synthesize Pareto-optimal architectures
52
+ synth = BeamSynthesizer(registry, beam_width=5)
53
+ for arch in synth.synthesize(task):
54
+ print(
55
+ f"cost=${arch.total_cost:.4f} "
56
+ f"latency={arch.total_latency:.0f}ms "
57
+ f"reliability={arch.total_reliability:.3f}"
58
+ )
59
+ ```
60
+
61
+ Or run the full demo comparing three synthesis strategies:
62
+
63
+ ```bash
64
+ python examples/quickstart.py
65
+ ```
66
+
67
+ To see the full loop — load a JSON registry, synthesize a frontier, execute one
68
+ architecture, and calibrate the registry's metadata from real telemetry — run:
69
+
70
+ ```bash
71
+ python examples/registry_from_json.py
72
+ ```
73
+
74
+ To search over your own components, build a `CapabilityRegistry` and register
75
+ `ComponentMetadata` describing each model, tool, or verifier — see
76
+ `examples/` for complete walkthroughs.
77
+
78
+ ## How It Works
79
+
80
+ ChatCortex is organized as a layered architecture synthesis system:
81
+
82
+ ```
83
+ TaskSpecification → CapabilityRegistry → Synthesis Engine → AgentGraph (DAG)
84
+ → Execution Engine → Telemetry → Evaluation Harness → Pareto Optimization
85
+ ```
86
+
87
+ **Core concepts:**
88
+
89
+ - **ComponentMetadata** — declarative description of an agent component (model, retriever, tool, verifier, memory) with cost per call, latency, reliability score, and privacy level.
90
+ - **CapabilityRegistry** — stores components and filters candidates by capability and privacy constraints.
91
+ - **TaskSpecification** — the synthesis problem: an ordered capability chain plus hard constraints (max cost, max latency, privacy) and objective weights.
92
+ - **AgentGraph** — architectures represented as DAGs; cost is additive, latency sequential, reliability multiplicative.
93
+ - **Synthesizers** — search strategies that map a task to an (approximate) Pareto frontier.
94
+
95
+ **Available synthesizers:**
96
+
97
+ | Synthesizer | Strategy |
98
+ |---|---|
99
+ | `HeuristicSynthesizer` | Greedy deterministic construction |
100
+ | `RandomSynthesizer` | Random baseline |
101
+ | `BeamSynthesizer` | Budget-aware beam search |
102
+ | `ParetoPartialBeamSynthesizer` | Beam search with per-stage Pareto pruning |
103
+ | `ProgressiveParetoBeamSynthesizer` | Depth-aware beam widening (best Pareto recovery) |
104
+ | `ExhaustiveSynthesizer` | Exact Pareto frontier (small spaces) |
105
+
106
+ Approximation quality is measured with **Pareto coverage**, **hypervolume loss**, and **average regret** — see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
107
+
108
+ ## Documentation
109
+
110
+ - [Architecture](docs/ARCHITECTURE.md) — system design and component layers
111
+ - [Benchmarks](docs/BENCHMARKS.md) — experimental evaluation of synthesis strategies
112
+ - [Examples](examples/) — runnable demos and budget-sweep experiments
113
+ - [Contributing](CONTRIBUTING.md) — development setup and PR guidelines
114
+ - [Changelog](CHANGELOG.md)
115
+
116
+ ## Research Context
117
+
118
+ ChatCortex is a controlled experimental platform for studying automated agent architecture synthesis: multi-objective optimization, architecture search, reliability–cost tradeoffs, and AutoML-style agent design. Experiments cover 5-stage pipelines with Pareto frontiers up to 95 architectures and evaluation budgets from 20 to 180 evaluations; progressive beam widening shows improved Pareto recovery over static beam strategies.
119
+
120
+ If you use ChatCortex in your research, please cite it (see [CITATION.cff](CITATION.cff)).
121
+
122
+ ## Roadmap
123
+
124
+ - [x] Component modeling, capability registry, task specification, DAG representation
125
+ - [x] Exact Pareto frontier computation (exhaustive search)
126
+ - [x] Budget-aware beam search with Pareto pruning and progressive widening
127
+ - [ ] Graph-structured (non-chain) agent synthesis
128
+ - [ ] Real model / tool integrations (OpenAI chat-completions backend already available behind the `chatcortex[real]` extra; more backends planned)
129
+ - [x] Registry loading from JSON (`load_registry`, with error messages naming the failing field); YAML planned
130
+
131
+ ## Status
132
+
133
+ Research framework under active development (alpha). APIs may change between minor versions.
134
+
135
+ ## License
136
+
137
+ [MIT](LICENSE) — developed by **Siddharth Saraswat**
@@ -0,0 +1,47 @@
1
+ """
2
+ ChatCortex
3
+
4
+ Framework for automated synthesis and optimization of AI agent
5
+ architectures under multi-objective constraints.
6
+
7
+ Core capabilities include:
8
+
9
+ - Task specification
10
+ - Agent architecture synthesis
11
+ - Multi-objective optimization
12
+ - Pareto frontier analysis
13
+ """
14
+
15
+ from .graph.agent_graph import AgentGraph
16
+ from .registry.capability_registry import CapabilityRegistry
17
+ from .registry.demo_components import build_demo_registry
18
+ from .registry.loader import load_registry
19
+ from .registry.metadata import ComponentMetadata
20
+ from .synthesis.beam_synthesizer import BeamSynthesizer
21
+ from .synthesis.budget import SynthesisBudget
22
+ from .synthesis.exhaustive_synthesizer import ExhaustiveSynthesizer
23
+ from .synthesis.heuristic_synthesizer import HeuristicSynthesizer
24
+ from .synthesis.pareto_partial_beam_synthesizer import ParetoPartialBeamSynthesizer
25
+ from .synthesis.progressive_pareto_beam_synthesizer import (
26
+ ProgressiveParetoBeamSynthesizer,
27
+ )
28
+ from .synthesis.random_synthesizer import RandomSynthesizer
29
+ from .synthesis.task_specification import TaskSpecification
30
+
31
+ __all__ = [
32
+ "TaskSpecification",
33
+ "HeuristicSynthesizer",
34
+ "RandomSynthesizer",
35
+ "BeamSynthesizer",
36
+ "ExhaustiveSynthesizer",
37
+ "ParetoPartialBeamSynthesizer",
38
+ "ProgressiveParetoBeamSynthesizer",
39
+ "SynthesisBudget",
40
+ "AgentGraph",
41
+ "CapabilityRegistry",
42
+ "ComponentMetadata",
43
+ "build_demo_registry",
44
+ "load_registry",
45
+ ]
46
+
47
+ __version__ = "0.6.0"
@@ -0,0 +1,7 @@
1
+ """Optional real-execution backends.
2
+
3
+ These require extra dependencies (see the ``real`` extra in
4
+ ``pyproject.toml``); importing this package never pulls them in. The
5
+ heavy imports happen inside :class:`OpenAIBackend.__init__`, so the
6
+ zero-dependency core stays untouched.
7
+ """
@@ -0,0 +1,106 @@
1
+ """OpenAI chat-completions execution backend (optional ``real`` extra).
2
+
3
+ Installation: ``pip install chatcortex[real]`` — this module imports
4
+ ``openai`` lazily inside :func:`OpenAIBackend.__init__`, so the package
5
+ never imports it unless a backend is actually built. The API key comes
6
+ from ``OPENAI_API_KEY`` (the SDK's standard env resolution); tests can
7
+ inject a stub ``client`` instead and need no key or network.
8
+
9
+ What is measured vs. declared:
10
+
11
+ - **Latency** and **success** are measured from the real SDK call.
12
+ - **Cost** is not derivable from a completion alone (token pricing is
13
+ model-specific and changes over time), so the component's declared
14
+ ``cost_per_call`` is recorded. Component metadata stays pure: nothing
15
+ here mutates it, calibration is the component that turns telemetry
16
+ into updated metadata.
17
+ """
18
+
19
+ import time
20
+ from typing import Any, Callable, Optional
21
+
22
+ from ..execution.executor import ProbeResult
23
+ from ..registry.metadata import ComponentMetadata
24
+
25
+ DEFAULT_MODEL = "gpt-5.5-mini"
26
+
27
+
28
+ class OpenAIBackend:
29
+ """Runs one graph node as one chat-completions API call.
30
+
31
+ Args:
32
+ model: Model name used for every component; ignored when
33
+ ``model_for`` is supplied.
34
+ model_for: Optional ``Callable[[ComponentMetadata], str]`` that
35
+ maps a component to the model it should run on (e.g. cheap
36
+ models for retrieval tooling, stronger models for
37
+ generation); takes precedence over ``model``.
38
+ client: Optional pre-built OpenAI client (or any object exposing
39
+ ``chat.completions.create``). Meant for tests; when given,
40
+ the ``openai`` package is never imported.
41
+ """
42
+
43
+ def __init__(
44
+ self,
45
+ model: str = DEFAULT_MODEL,
46
+ model_for: Optional[Callable[[ComponentMetadata], str]] = None,
47
+ client: Optional[Any] = None,
48
+ ):
49
+ self._model = model
50
+ self.model_for = model_for
51
+ if client is not None:
52
+ self._client = client
53
+ else:
54
+ try:
55
+ import openai
56
+ except ImportError as exc: # pragma: no cover - exercised only without extra
57
+ raise ImportError(
58
+ "OpenAIBackend requires the optional 'openai' dependency; "
59
+ "install it with: pip install chatcortex[real]"
60
+ ) from exc
61
+ self._client = openai.OpenAI()
62
+
63
+ def _model_for_component(self, metadata: ComponentMetadata) -> str:
64
+ if self.model_for is not None:
65
+ return self.model_for(metadata)
66
+ return self._model
67
+
68
+ def __call__(
69
+ self,
70
+ metadata: ComponentMetadata,
71
+ node_input: Any = None,
72
+ ) -> ProbeResult:
73
+ model = self._model_for_component(metadata)
74
+ prompt = str(node_input) if node_input is not None else (
75
+ f"Perform the '{metadata.capabilities[0]}' task as component "
76
+ f"'{metadata.name}'."
77
+ )
78
+
79
+ started = time.perf_counter()
80
+ try:
81
+ response = self._client.chat.completions.create(
82
+ model=model,
83
+ messages=[
84
+ {
85
+ "role": "user",
86
+ "content": prompt,
87
+ }
88
+ ],
89
+ )
90
+ except Exception as exc: # network/HTTP/key errors are probe failures
91
+ latency_ms = (time.perf_counter() - started) * 1000.0
92
+ return ProbeResult(
93
+ success=False,
94
+ latency_ms=latency_ms,
95
+ cost=0.0,
96
+ output=f"openai call failed: {type(exc).__name__}: {exc}",
97
+ )
98
+
99
+ latency_ms = (time.perf_counter() - started) * 1000.0
100
+ output = response.choices[0].message.content
101
+ return ProbeResult(
102
+ success=bool(output),
103
+ latency_ms=latency_ms,
104
+ cost=metadata.cost_per_call,
105
+ output=output,
106
+ )
@@ -1,4 +1,3 @@
1
- import random
2
1
  from typing import Iterable, List, Tuple
3
2
 
4
3
  from chatcortex.optimization.architecture_candidate import ArchitectureCandidate
@@ -11,13 +10,9 @@ def compute_coverage(
11
10
  ) -> float:
12
11
  return frontier_coverage(approx_frontier, true_frontier)
13
12
 
14
- def compute_hypervolume(
15
- frontier: Iterable[ArchitectureCandidate],
16
- reference_point: Tuple[float, float, float],
17
- ) -> float:
18
- return hypervolume_monte_carlo(frontier, reference_point)
19
-
20
13
 
14
+ # Note: one seed per estimate, not a shared stream — the two estimates below
15
+ # are independent Monte Carlo runs of the same algorithm.
21
16
  def hypervolume_loss(
22
17
  approx_frontier,
23
18
  true_frontier,
@@ -25,42 +20,12 @@ def hypervolume_loss(
25
20
  num_samples=200000,
26
21
  seed=42,
27
22
  ) -> float:
28
- ref_cost, ref_latency, ref_reliability = reference_point
29
-
30
- random_number_generator = random.Random(seed)
31
-
32
- dominated_true = 0
33
- dominated_approx = 0
34
-
35
- for _ in range(num_samples):
36
-
37
- sample_cost = random_number_generator.uniform(0, ref_cost)
38
- sample_latency = random_number_generator.uniform(0, ref_latency)
39
- sample_reliability = random_number_generator.uniform(ref_reliability, 1.0)
40
-
41
- for candidate in true_frontier:
42
- if (
43
- candidate.total_cost <= sample_cost
44
- and candidate.total_latency <= sample_latency
45
- and candidate.total_reliability >= sample_reliability
46
- ):
47
- dominated_true += 1
48
- break
49
-
50
- for candidate in approx_frontier:
51
- if (
52
- candidate.total_cost <= sample_cost
53
- and candidate.total_latency <= sample_latency
54
- and candidate.total_reliability >= sample_reliability
55
- ):
56
- dominated_approx += 1
57
- break
58
-
59
-
60
- box_volume = ref_cost * ref_latency * (1.0 - ref_reliability)
61
-
62
- hv_true = (dominated_true / num_samples) * box_volume
63
- hv_approx = (dominated_approx / num_samples) * box_volume
23
+ hv_true = hypervolume_monte_carlo(
24
+ true_frontier, reference_point, num_samples=num_samples, seed=seed
25
+ )
26
+ hv_approx = hypervolume_monte_carlo(
27
+ approx_frontier, reference_point, num_samples=num_samples, seed=seed
28
+ )
64
29
 
65
30
  return max(0.0, hv_true - hv_approx)
66
31
 
@@ -79,7 +44,7 @@ def additive_regret(
79
44
 
80
45
  if not dominating:
81
46
  return (0.0, 0.0, 0.0)
82
-
47
+
83
48
  min_cost_diff = min(candidate.total_cost - p.total_cost for p in dominating)
84
49
  min_latency_diff = min(candidate.total_latency - p.total_latency for p in dominating)
85
50
  min_reliability_diff = min(p.total_reliability - candidate.total_reliability for p in dominating)
@@ -112,7 +77,7 @@ def average_regret(
112
77
  total_cost_regret += cr
113
78
  total_latency_regret += lr
114
79
  total_reliability_regret += rr
115
-
80
+
116
81
  n = len(approx_list)
117
82
 
118
83
  return (
@@ -131,7 +96,7 @@ def evaluate_approximation(
131
96
  """
132
97
 
133
98
  coverage = compute_coverage(approx_frontier, true_frontier)
134
-
99
+
135
100
  hv_loss = hypervolume_loss(approx_frontier, true_frontier, reference_point)
136
101
  avg_regret = average_regret(approx_frontier, true_frontier)
137
102
 
@@ -141,4 +106,4 @@ def evaluate_approximation(
141
106
  "avg_cost_regret": avg_regret[0],
142
107
  "avg_latency_regret": avg_regret[1],
143
108
  "avg_reliability_regret": avg_regret[2],
144
- }
109
+ }