aid-runtime-python 0.1.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. aid_runtime/__init__.py +500 -0
  2. aid_runtime/agents/__init__.py +94 -0
  3. aid_runtime/agents/chat_model.py +351 -0
  4. aid_runtime/agents/deep_agents.py +600 -0
  5. aid_runtime/agents/engine.py +102 -0
  6. aid_runtime/agents/guardrails.py +519 -0
  7. aid_runtime/agents/ir.py +622 -0
  8. aid_runtime/agents/loop.py +1308 -0
  9. aid_runtime/agents/memory.py +241 -0
  10. aid_runtime/agents/policy.py +163 -0
  11. aid_runtime/agents/state.py +182 -0
  12. aid_runtime/cache.py +185 -0
  13. aid_runtime/concurrency.py +70 -0
  14. aid_runtime/control.py +329 -0
  15. aid_runtime/cost.py +168 -0
  16. aid_runtime/errors.py +337 -0
  17. aid_runtime/evals/__init__.py +72 -0
  18. aid_runtime/evals/dataset.py +77 -0
  19. aid_runtime/evals/gate.py +430 -0
  20. aid_runtime/evals/metrics.py +335 -0
  21. aid_runtime/evals/predicate.py +211 -0
  22. aid_runtime/evals/runner.py +224 -0
  23. aid_runtime/gateway.py +473 -0
  24. aid_runtime/pipeline.py +168 -0
  25. aid_runtime/prompts.py +185 -0
  26. aid_runtime/providers/__init__.py +20 -0
  27. aid_runtime/providers/base.py +57 -0
  28. aid_runtime/providers/fake.py +256 -0
  29. aid_runtime/providers/http.py +344 -0
  30. aid_runtime/py.typed +0 -0
  31. aid_runtime/retrieval/__init__.py +178 -0
  32. aid_runtime/retrieval/chunking.py +222 -0
  33. aid_runtime/retrieval/citations.py +131 -0
  34. aid_runtime/retrieval/connectors.py +131 -0
  35. aid_runtime/retrieval/embedding.py +293 -0
  36. aid_runtime/retrieval/gate.py +120 -0
  37. aid_runtime/retrieval/hybrid.py +164 -0
  38. aid_runtime/retrieval/metrics.py +213 -0
  39. aid_runtime/retrieval/pgvector.py +258 -0
  40. aid_runtime/retrieval/rerank.py +133 -0
  41. aid_runtime/retrieval/retriever.py +335 -0
  42. aid_runtime/retrieval/stores.py +161 -0
  43. aid_runtime/retrieval/types.py +433 -0
  44. aid_runtime/steps.py +284 -0
  45. aid_runtime/structured.py +150 -0
  46. aid_runtime/tools.py +271 -0
  47. aid_runtime/tracing.py +241 -0
  48. aid_runtime/types.py +332 -0
  49. aid_runtime_python-0.1.39.dist-info/METADATA +268 -0
  50. aid_runtime_python-0.1.39.dist-info/RECORD +52 -0
  51. aid_runtime_python-0.1.39.dist-info/WHEEL +5 -0
  52. aid_runtime_python-0.1.39.dist-info/top_level.txt +1 -0
aid_runtime/types.py ADDED
@@ -0,0 +1,332 @@
1
+ """Data shapes shared across the runtime.
2
+
3
+ These are deliberately plain validated records: everything that crosses a process
4
+ boundary (a provider response, a tool call, an eval baseline) is a Pydantic model
5
+ so it round-trips through JSON without a bespoke encoder.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ from collections.abc import Sequence
13
+ from enum import StrEnum
14
+ from typing import Any, Literal
15
+
16
+ from pydantic import BaseModel, ConfigDict, Field, field_validator
17
+
18
+ __all__ = [
19
+ "Capability",
20
+ "CompletionRequest",
21
+ "CompletionResponse",
22
+ "CostProfile",
23
+ "FinishReason",
24
+ "GenerationResult",
25
+ "Message",
26
+ "ModelCacheConfig",
27
+ "ModelSpec",
28
+ "PinnedModel",
29
+ "Role",
30
+ "SemanticCacheConfig",
31
+ "StreamChunk",
32
+ "TokenUsage",
33
+ "ToolCall",
34
+ "ToolDefinition",
35
+ "ToolResult",
36
+ ]
37
+
38
+ Role = Literal["system", "user", "assistant", "tool"]
39
+ FinishReason = Literal["stop", "length", "tool-calls", "content-filter", "error"]
40
+
41
+
42
+ class Capability(StrEnum):
43
+ """Capabilities a *model* can be negotiated against.
44
+
45
+ These are the model-side vocabulary. The IR's capability set
46
+ (``http-trigger``, ``json-schema``, ``otel``, ``streaming``, ``tool-calling``,
47
+ ``cursor-memory``) is a target-side vocabulary; the generator maps one to the
48
+ other. ``STREAMING`` is served: an adapter that advertises it also implements
49
+ :meth:`~aid_runtime.providers.Provider.stream`. ``EMBEDDINGS`` and
50
+ ``RERANK`` are likewise model-side: the IR's ``MODEL_CAPABILITIES`` already
51
+ admits both for embedding and rerank models, so declaring them here is what
52
+ lets a model be negotiated against a retrieval binding at all.
53
+ """
54
+
55
+ TOOL_CALLING = "tool-calling"
56
+ STRUCTURED_OUTPUT = "structured-output"
57
+ STREAMING = "streaming"
58
+ LONG_CONTEXT = "long-context"
59
+ EMBEDDINGS = "embeddings"
60
+ RERANK = "rerank"
61
+
62
+
63
+ class ToolCall(BaseModel):
64
+ """A model's request to invoke a tool."""
65
+
66
+ model_config = ConfigDict(frozen=True, extra="forbid")
67
+
68
+ id: str = Field(min_length=1)
69
+ name: str = Field(min_length=1)
70
+ arguments: dict[str, Any] = Field(default_factory=dict)
71
+
72
+
73
+ class Message(BaseModel):
74
+ """One turn of a chat transcript.
75
+
76
+ ``tool_calls`` on an assistant turn and ``tool_call_id`` on a tool turn are what
77
+ make a multi-step tool loop replayable: without them the next request would carry
78
+ an observation the provider cannot attribute to the call that produced it.
79
+ """
80
+
81
+ model_config = ConfigDict(frozen=True, extra="forbid")
82
+
83
+ role: Role
84
+ content: str
85
+ tool_calls: tuple[ToolCall, ...] = ()
86
+ tool_call_id: str | None = None
87
+ name: str | None = None
88
+
89
+ @classmethod
90
+ def system(cls, content: str) -> Message:
91
+ return cls(role="system", content=content)
92
+
93
+ @classmethod
94
+ def user(cls, content: str) -> Message:
95
+ return cls(role="user", content=content)
96
+
97
+ @classmethod
98
+ def assistant(cls, content: str) -> Message:
99
+ return cls(role="assistant", content=content)
100
+
101
+ @classmethod
102
+ def assistant_tool_calls(cls, content: str, tool_calls: Sequence[ToolCall]) -> Message:
103
+ return cls(role="assistant", content=content, tool_calls=tuple(tool_calls))
104
+
105
+ @classmethod
106
+ def tool(cls, call_id: str, content: str, *, name: str | None = None) -> Message:
107
+ return cls(role="tool", content=content, tool_call_id=call_id, name=name)
108
+
109
+
110
+ class CostProfile(BaseModel):
111
+ """Published price for a pinned model, in USD per 1,000 tokens."""
112
+
113
+ model_config = ConfigDict(frozen=True, extra="forbid")
114
+
115
+ input_per_1k_usd: float = Field(ge=0.0)
116
+ output_per_1k_usd: float = Field(ge=0.0)
117
+
118
+ def cost_usd(self, usage: TokenUsage) -> float:
119
+ raw = (
120
+ usage.input_tokens * self.input_per_1k_usd
121
+ + usage.output_tokens * self.output_per_1k_usd
122
+ ) / 1000.0
123
+ return round(raw, 6)
124
+
125
+
126
+ class SemanticCacheConfig(BaseModel):
127
+ """Opt-in semantic reuse: an explicit threshold and TTL, with no defaults."""
128
+
129
+ model_config = ConfigDict(frozen=True, extra="forbid")
130
+
131
+ threshold: float = Field(ge=0.0, le=1.0)
132
+ ttl_seconds: int = Field(gt=0)
133
+
134
+
135
+ class ModelCacheConfig(BaseModel):
136
+ """Per-model cache policy carried by ``models[].cache`` (spec v0.3)."""
137
+
138
+ model_config = ConfigDict(frozen=True, extra="forbid")
139
+
140
+ prompt: bool = False
141
+ semantic: SemanticCacheConfig | None = None
142
+
143
+
144
+ class ModelSpec(BaseModel):
145
+ """A *pinned* model: an exact provider model id, never a floating alias.
146
+
147
+ Pinning is the whole point of the alias table. ``name`` must be the
148
+ provider's immutable snapshot id (``gpt-4o-mini-2024-07-18``), because a
149
+ floating ``gpt-4o-mini`` silently changes behaviour under a stored eval
150
+ baseline and turns a real regression into a mystery.
151
+ """
152
+
153
+ model_config = ConfigDict(frozen=True, extra="forbid")
154
+
155
+ name: str = Field(min_length=1)
156
+ provider: str = Field(min_length=1)
157
+ cost: CostProfile
158
+ context_window: int = Field(gt=0)
159
+ max_output_tokens: int = Field(gt=0)
160
+ capabilities: frozenset[Capability] = frozenset()
161
+ cache: ModelCacheConfig | None = None
162
+
163
+ @field_validator("capabilities", mode="before")
164
+ @classmethod
165
+ def _coerce_capabilities(cls, value: Any) -> Any:
166
+ if isinstance(value, (set, frozenset)):
167
+ return frozenset(Capability(v) for v in value)
168
+ return value
169
+
170
+
171
+ class PinnedModel(BaseModel):
172
+ """An alias resolved against the pinned table, with its adapter attached."""
173
+
174
+ model_config = ConfigDict(frozen=True, extra="forbid", arbitrary_types_allowed=True)
175
+
176
+ alias: str = Field(min_length=1)
177
+ spec: ModelSpec
178
+
179
+ @property
180
+ def provider(self) -> str:
181
+ return self.spec.provider
182
+
183
+ @property
184
+ def capabilities(self) -> frozenset[Capability]:
185
+ return self.spec.capabilities
186
+
187
+ def supports(self, capabilities: frozenset[Capability]) -> bool:
188
+ return capabilities <= self.spec.capabilities
189
+
190
+
191
+ class TokenUsage(BaseModel):
192
+ """Token accounting for a single model call."""
193
+
194
+ model_config = ConfigDict(frozen=True, extra="forbid")
195
+
196
+ input_tokens: int = Field(ge=0)
197
+ output_tokens: int = Field(ge=0)
198
+
199
+ @property
200
+ def total_tokens(self) -> int:
201
+ return self.input_tokens + self.output_tokens
202
+
203
+ def __add__(self, other: TokenUsage) -> TokenUsage:
204
+ return TokenUsage(
205
+ input_tokens=self.input_tokens + other.input_tokens,
206
+ output_tokens=self.output_tokens + other.output_tokens,
207
+ )
208
+
209
+
210
+ class ToolDefinition(BaseModel):
211
+ """A tool as the model sees it: name, description, JSON Schema parameters."""
212
+
213
+ model_config = ConfigDict(frozen=True, extra="forbid")
214
+
215
+ name: str = Field(min_length=1)
216
+ description: str = ""
217
+ parameters: dict[str, Any] = Field(default_factory=dict)
218
+
219
+ def as_openai_tool(self) -> dict[str, Any]:
220
+ return {
221
+ "type": "function",
222
+ "function": {
223
+ "name": self.name,
224
+ "description": self.description,
225
+ "parameters": self.parameters or {"type": "object", "properties": {}},
226
+ },
227
+ }
228
+
229
+
230
+ class ToolResult(BaseModel):
231
+ """The outcome of executing a :class:`ToolCall`."""
232
+
233
+ model_config = ConfigDict(frozen=True, extra="forbid")
234
+
235
+ call_id: str
236
+ name: str
237
+ content: str
238
+ is_error: bool = False
239
+
240
+ @classmethod
241
+ def from_value(cls, call: ToolCall, value: Any) -> ToolResult:
242
+ content = (
243
+ value if isinstance(value, str) else json.dumps(value, sort_keys=True, default=str)
244
+ )
245
+ return cls(call_id=call.id, name=call.name, content=content)
246
+
247
+ @classmethod
248
+ def from_error(cls, call: ToolCall, message: str) -> ToolResult:
249
+ return cls(call_id=call.id, name=call.name, content=message, is_error=True)
250
+
251
+
252
+ class CompletionRequest(BaseModel):
253
+ """A single model call.
254
+
255
+ It may be served request/response (:meth:`ModelGateway.generate`) or as an SSE
256
+ byte stream (:meth:`ModelGateway.stream`). The same request shape feeds both.
257
+ """
258
+
259
+ model_config = ConfigDict(frozen=True, extra="forbid")
260
+
261
+ messages: tuple[Message, ...] = Field(min_length=1)
262
+ temperature: float | None = Field(default=None, ge=0.0, le=2.0)
263
+ max_output_tokens: int | None = Field(default=None, gt=0)
264
+ tools: tuple[ToolDefinition, ...] = ()
265
+ stop: tuple[str, ...] = ()
266
+ metadata: dict[str, str] = Field(default_factory=dict)
267
+
268
+ @classmethod
269
+ def prompt(cls, text: str, *, system: str | None = None) -> CompletionRequest:
270
+ messages = []
271
+ if system is not None:
272
+ messages.append(Message.system(system))
273
+ messages.append(Message.user(text))
274
+ return cls(messages=tuple(messages))
275
+
276
+ def fingerprint(self) -> str:
277
+ """Stable hash of everything that can change the model's output."""
278
+ payload = self.model_dump(mode="json", exclude_none=True)
279
+ blob = json.dumps(payload, sort_keys=True, separators=(",", ":"))
280
+ return hashlib.sha256(blob.encode("utf-8")).hexdigest()
281
+
282
+
283
+ class CompletionResponse(BaseModel):
284
+ """A provider response plus the accounting needed to charge for it."""
285
+
286
+ model_config = ConfigDict(frozen=True, extra="forbid")
287
+
288
+ text: str
289
+ model: str
290
+ provider: str
291
+ usage: TokenUsage
292
+ tool_calls: tuple[ToolCall, ...] = ()
293
+ finish_reason: FinishReason = "stop"
294
+
295
+
296
+ class StreamChunk(BaseModel):
297
+ """One increment of a streamed completion.
298
+
299
+ ``delta`` is the new text since the previous chunk. The terminal chunk carries
300
+ ``finish_reason`` and ``usage`` so the gateway can account for the call exactly
301
+ as :meth:`ModelGateway.generate` does.
302
+ """
303
+
304
+ model_config = ConfigDict(frozen=True, extra="forbid")
305
+
306
+ delta: str = ""
307
+ finish_reason: FinishReason | None = None
308
+ usage: TokenUsage | None = None
309
+
310
+
311
+ class GenerationResult(BaseModel):
312
+ """What the gateway returns: the response, what it cost, and how it got there."""
313
+
314
+ model_config = ConfigDict(frozen=True, extra="forbid")
315
+
316
+ response: CompletionResponse
317
+ alias: str
318
+ cost_usd: float = Field(ge=0.0)
319
+ attempts: tuple[str, ...] = ()
320
+ cache_hit: bool = False
321
+
322
+ @property
323
+ def text(self) -> str:
324
+ return self.response.text
325
+
326
+ @property
327
+ def usage(self) -> TokenUsage:
328
+ return self.response.usage
329
+
330
+ @property
331
+ def fell_back(self) -> bool:
332
+ return len(self.attempts) > 1
@@ -0,0 +1,268 @@
1
+ Metadata-Version: 2.4
2
+ Name: aid-runtime-python
3
+ Version: 0.1.39
4
+ Summary: The AID Framework Python runtime: model gateway with pinning and cost budgets, prompt registry, tool executor, OTel GenAI tracing, plain-async pipeline, and an eval harness that gates on statistical regression against a stored baseline.
5
+ Author: The AID Framework authors
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/aid-framework/aid-framework
8
+ Project-URL: Documentation, https://github.com/aid-framework/aid-framework/blob/main/docs/design.md
9
+ Project-URL: Issues, https://github.com/aid-framework/aid-framework/issues
10
+ Keywords: ai,codegen,llm,evals,opentelemetry,pydantic
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Typing :: Typed
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ Requires-Dist: httpx<1,>=0.27
21
+ Requires-Dist: opentelemetry-api<2,>=1.24
22
+ Requires-Dist: pydantic<3,>=2.7
23
+ Requires-Dist: pydantic-ai-slim<3,>=2
24
+ Provides-Extra: pgvector
25
+ Requires-Dist: psycopg[binary]<4,>=3.1; extra == "pgvector"
26
+ Provides-Extra: deepagents
27
+ Requires-Dist: deepagents==0.7.32; extra == "deepagents"
28
+ Provides-Extra: dev
29
+ Requires-Dist: mypy<3,>=1.11; extra == "dev"
30
+ Requires-Dist: opentelemetry-sdk<2,>=1.24; extra == "dev"
31
+ Requires-Dist: pytest<10,>=8; extra == "dev"
32
+ Requires-Dist: pytest-asyncio<2,>=0.24; extra == "dev"
33
+ Requires-Dist: ruff<1,>=0.6; extra == "dev"
34
+
35
+ # AID runtime — Python
36
+
37
+ The Python runtime library: the behavior layer imported by apps generated by the
38
+ [`py-fastapi`](../../packages/generators/py-fastapi/) generator. Generated code is thin glue; this is where
39
+ the behavior lives ([`docs/design.md` §4](../../docs/design.md), principle 1).
40
+
41
+ Distribution name **`aid-runtime-python`**, import package **`aid_runtime`**. The two differ deliberately: a
42
+ distribution name is conventionally hyphenated, and a Python import package cannot be. That asymmetry is
43
+ asserted in `tests/test_package.py` so it cannot drift.
44
+
45
+ Requires Python 3.11+.
46
+
47
+ ```sh
48
+ pip install -e ".[dev]" # the runtime plus its test/lint/type tooling
49
+ pytest -q
50
+ pip install -e ".[pgvector]" # only if you want the pgvector store adapter
51
+ ```
52
+
53
+ ## What it provides
54
+
55
+ | Module | Provides |
56
+ | --- | --- |
57
+ | `gateway.py` | `ModelGateway` — alias resolution, capability negotiation, ordered fallback, cost accounting, GenAI spans |
58
+ | `providers/` | The `Provider` protocol, one HTTP adapter (`OpenAICompatibleProvider`), and one deterministic fake (`FakeProvider`) |
59
+ | `prompts.py` | `PromptTemplate` / `PromptRegistry` — versioned prompts, typed rendering, few-shot assembly |
60
+ | `structured.py` | `build_agent` — structured output via Pydantic AI only |
61
+ | `tools.py` | `Tool` / `ToolExecutor` — declared definitions, argument validation, structured results |
62
+ | `pipeline.py` | `Pipeline` / `PipelineContext` — plain-async named steps over a frozen context |
63
+ | `agents/` | The native agent runtime: `AgentRuntime` — the loop, transfer vs. delegate, step/token/cost budgets with bounded degrade, memory, and a checkpointed human-in-the-loop pause |
64
+ | `retrieval/` | Retrieval: connectors, chunkers, the `Embedder` / `Reranker` / `VectorStore` protocols, hybrid fusion, citations, and retrieval metrics |
65
+ | `evals/` | Datasets, metrics, and the statistical baseline gate |
66
+ | `tracing.py` | OTel GenAI spans: `llm_span`, `agent_span`, `record_usage`, `record_cost`, `record_attributes`, `record_hitl`, `record_exception` |
67
+ | `cost.py` | `Budget`, `CostLedger`, `estimate_cost_usd`, `estimate_tokens` |
68
+ | `types.py` | The request/response/usage/spec models everything else is typed against |
69
+
70
+ 194 names are exported from `aid_runtime`. The layering is acyclic and one-directional: `errors` and `types` import
71
+ nothing else in the package, `gateway` is imported only by the package `__init__` and the eval runner, and no
72
+ core module imports a higher layer back.
73
+
74
+ ## Model pinning
75
+
76
+ An **alias resolves to an exact provider model id**, including a pinned version. That is not cosmetic: a
77
+ baseline recorded against `gpt-4o` means nothing if the alias floats to `gpt-4o-2026-…` on the next run, so an
78
+ **unknown alias is an error** (`UnknownModelAliasError`) rather than a silent fall-through to a provider
79
+ default.
80
+
81
+ ```python
82
+ from aid_runtime import (
83
+ Budget,
84
+ CompletionRequest,
85
+ CostProfile,
86
+ ModelGateway,
87
+ ModelSpec,
88
+ OpenAICompatibleProvider,
89
+ PinnedModel,
90
+ )
91
+
92
+ spec = ModelSpec(
93
+ name="gpt-4o-2024-08-06",
94
+ provider="openai-compatible",
95
+ cost=CostProfile(input_per_1k_usd=0.0025, output_per_1k_usd=0.01),
96
+ context_window=128_000,
97
+ max_output_tokens=4096,
98
+ )
99
+
100
+ gateway = ModelGateway(
101
+ providers={"openai-compatible": OpenAICompatibleProvider(api_key=...)},
102
+ aliases={"small": PinnedModel(alias="small", spec=spec)},
103
+ budget=Budget(max_usd=5.0, scope="nightly"),
104
+ )
105
+
106
+ result = await gateway.generate("small", CompletionRequest.prompt("hello"))
107
+ result.text, result.usage, result.cost_usd, result.alias, result.attempts, result.fell_back
108
+ ```
109
+
110
+ - `resolve(alias)` returns the `PinnedModel` (alias + `spec`); `provider_for(alias)` its provider. `require(alias,
111
+ capabilities)` asserts the capabilities *and* returns the pinned model, checking **both** sides — a model that
112
+ supports tool calling behind an adapter that does not serialise tool definitions cannot tool-call — and raises
113
+ `CapabilityMismatchError` naming what is missing.
114
+ - `generate(alias, request, *, fallbacks=(), budget=None)` tries the alias then each fallback **in the order
115
+ given**, and reports the aliases actually attempted on the result. Fallbacks are ordered, not conditional.
116
+ - Every charged call opens a span and updates the ledger. `CostLedger` tracks per-call `LedgerEntry` records
117
+ and yields a `SpendSummary`; `Budget` (`max_usd`, `max_total_tokens`, `scope`) is enforced before the call,
118
+ and `estimate_usage` / `estimate_cost_usd` price a request from a model's `CostProfile` when a caller wants
119
+ to check before spending.
120
+
121
+ ## The eval harness gates on statistics, not equality
122
+
123
+ Phase 0 ratifies sampling-based gating: a model is **expected** to vary between runs, so a metric that
124
+ compares two runs for equality is either always red or measuring nothing. Non-determinism here is not a bug to
125
+ be fixed; it is the property the gate has to be robust against.
126
+
127
+ - `Dataset` — versioned cases with inputs, expectations, tags and metadata; `from_records` / `from_json` /
128
+ `from_path`, `to_json`, and lookups by id (`case`) and by tag (`by_tag`).
129
+ - Metrics — `ExactMatch`, `NormalizedContains`, `JsonSchemaValid`, and `LlmJudge`. `metric_names` and
130
+ `default_metrics` cover the common set.
131
+ - `run_evaluation(dataset, runner, metrics, runs=5)` samples the runner **N times** and produces an `EvalRun`
132
+ of per-run metric means, so a single unlucky run does not decide the verdict. `evaluate_once` is the
133
+ single-pass primitive; a `DatasetRun` gives per-case outcomes and a rendered `summary()`.
134
+ - `EvalBaseline.from_run(run, model=..., note=...)` stores a baseline as JSON: per metric the sample count,
135
+ mean and spread. `runtimes/python/tests/fixtures/capitals-baseline.json` is a committed example.
136
+ - `BaselineGate(thresholds).evaluate(run, baseline)` compares the current sample against the baseline using a
137
+ one-sided normal test, with each `Threshold` carrying `metric`, an optional absolute `min_mean`, a
138
+ `max_regression` allowance, and an `alpha` significance level. A metric passes by clearing `min_mean`, by
139
+ being within `max_regression` of the baseline, or by regressing only within sampling noise.
140
+
141
+ Two deliberate limits:
142
+
143
+ - **`LlmJudge` is implemented and never CI-required.** A judge is itself a model call, and a gate that
144
+ depends on one non-deterministic call inside the gate you are trying to trust is the problem this design
145
+ avoids. `gateway_judge(gateway, alias)` builds the judge call when a caller wants it.
146
+ - **`GateConfigurationError` refuses a threshold on a model-derived metric**, because an N-run sample of a
147
+ probabilistic judge cannot carry a fixed threshold honestly.
148
+
149
+ `cost` is the enforced budget; a latency p95 budget belongs with Phase 2 dashboards, not here.
150
+
151
+ ## Retrieval
152
+
153
+ `retrieval/` is an IR-driven layer: `binding_from_spec(retriever, embedding=..., store=...)` folds the IR's
154
+ `retrievers`, `embeddings` and `stores` entries into one `RetrieverBinding`, which a `Retriever` then ingests
155
+ and searches through. The runtime consumes the names the IR emits, field for field.
156
+
157
+ - Three protocols, in the same style as `providers.Provider`: `Embedder`, `Reranker`, `VectorStore`. Phase 1
158
+ ships the implementations listed below and no others.
159
+ - `FakeDeterministicEmbedder` maps text to a stable unit vector with a hash — keyless and offline, which is
160
+ what lets a retrieval gate run in CI. `OpenAICompatibleEmbedder` is the one hosted adapter: a single POST to
161
+ `/embeddings` over `httpx`, with the retry, timeout and error-chaining behaviour of the HTTP provider.
162
+ - `LexicalReranker` is the only reranker. It is deterministic and lexical; there is no hosted rerank adapter.
163
+ - Chunk ids are **content hashes**, so re-ingesting the same input produces the same ids and a no-op diff.
164
+ Ingest is therefore held to idempotence rather than to the byte-identical codegen guarantee.
165
+ - **Pinning.** `embedModel` names an IR `embeddings` entry and resolves at codegen time to an exact `modelId`
166
+ plus its recorded width; the resolved id, width and store `kind` travel into the generated code. An alias is
167
+ never resolved at runtime, because a baseline recorded against a floating model means nothing.
168
+ - **Cost.** Embedding calls charge the existing `CostLedger` through the existing `Budget`, using the catalog's
169
+ `CostProfile` for the pinned model. There is no retrieval-specific budgeting machinery.
170
+
171
+ `RetrievalMetric` (`RecallAtK`, `MeanReciprocalRank`, `CitationGrounding`) is deliberately a **separate
172
+ protocol** from the shipped `Metric`, which scores a single string prediction. Retrieval scores a *retrieved
173
+ set*: recall@k and MRR need the ranked ids, and citation-grounding needs the passages behind the citations.
174
+ Widening `Metric` would have changed an interface the existing gate already threshold-checks, for every
175
+ existing implementation — so retrieval is sampled through `RetrievalGate.sample(...)` instead, producing a
176
+ standard `EvalRun`, and then judged by the **unchanged** `BaselineGate`. Retrieval metrics are deterministic,
177
+ so they pass the gate's own eligibility check.
178
+
179
+ ### pgvector
180
+
181
+ `PgVectorStore` is a thin adapter over `psycopg` and the `vector` extension. Two consequences worth stating:
182
+
183
+ - It is **locally provisioned and not covered by any gate.** No CI job starts a database, and no required
184
+ check touches one; the in-memory adapter carries the store gates, and the two adapters share the same
185
+ `VectorStore` protocol and the same `similarity` semantics. `compose.pgvector.yml` beside this README brings
186
+ up a database for local use.
187
+ - The `[pgvector]` extra is **import-safe when absent**: importing `aid_runtime` or `aid_runtime.retrieval`
188
+ never imports the driver, and the adapter loading it raises a `ConfigurationError` naming the extra.
189
+
190
+ ## Control flow is plain async
191
+
192
+ `Pipeline` is a list of named async (or plain) steps over a `PipelineContext` that is **frozen**, so a step
193
+ cannot mutate what an earlier step observed — it returns a new context, which keeps a failed run's state
194
+ inspectable. A step may return a context, a mapping, or `None`. Each step runs in its own span, a step raising
195
+ anything aborts the run with the original exception chained (`PipelineError.__cause__`), and `StopPipeline` is
196
+ the supported early exit.
197
+
198
+ **Pydantic AI is used for structured output and nothing else** (`structured.build_agent`). The orchestration
199
+ library stays out of the control-flow path deliberately, so it remains swappable, and **no graph/orchestration
200
+ dependency is taken at all**: the agent loop in `agents/` is this package's own, and escalation routing and
201
+ third-party orchestration frameworks remain out of scope ([`docs/design.md` §14](../../docs/design.md)).
202
+
203
+ Tracing follows the OTel GenAI semantic conventions: `gen_ai.*` attributes for the operation, model, provider,
204
+ request parameters and usage, with AID-specific values namespaced `aid.*` so the two never collide. A failing
205
+ call records its exception exactly once.
206
+
207
+ ## The agent runtime
208
+
209
+ `AgentRuntime.from_ir(document, providers=..., ...)` reads the emitted IR and runs an agent to a stop
210
+ condition: model call, tool calls, observations, until a terminal answer, `maxSteps`, or a budget breach.
211
+ Model calls go through `ModelGateway`, so fallback, cost accounting and tracing apply without a second
212
+ implementation.
213
+
214
+ **Transfer and delegation are different, and the API says which.** `_transfer` (`handoffs`) hands the
215
+ conversation to the target and never resumes the caller, so `RunResult.agent_id` is the target and `path`
216
+ grows. `_delegate` (`delegates`) runs the target in an isolated `[system, user]` context and returns its answer
217
+ as a single tool observation, so `RunResult.agent_id` is the caller and `path` is unchanged. Both refuse to
218
+ re-enter an agent already on the path.
219
+
220
+ **A budget breach fails hard.** `BudgetExceededError` stops the run; the gateway preflight and the ledger both
221
+ enforce the ceiling. An agent declaring `onBudgetExhausted: 'degrade'` (v0.2) instead steps exactly one rung
222
+ down its declared alias ladder — no retry loop, and the build-time gate rejects the declaration when no
223
+ strictly cheaper alias exists. Each degraded step is recorded in `StepRecord.degraded_from`, and a checkpoint
224
+ carries its current alias so a resumed run re-routes from where it stopped.
225
+
226
+ **Memory** is in-process plus one store-backed adapter. `thread` scope is the checkpointer unit: trimmed to its
227
+ `tokenBudget`, never written to a store. `user` and `global` scopes read and write the store and render as one
228
+ extra system message, so injection is visible in the request rather than hidden.
229
+
230
+ **Human-in-the-loop** pauses by returning a checkpoint (completed steps, pending call, current alias), and
231
+ `resume(checkpoint, decision)` continues it. The pause is recorded as an `aid.hitl` span event, not an
232
+ attribute, because a run can pause more than once.
233
+
234
+ ## Development
235
+
236
+ Four gates, all run by the `python` job in [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) over
237
+ real content:
238
+
239
+ ```sh
240
+ pytest
241
+ ruff check .
242
+ ruff format --check .
243
+ mypy
244
+ ```
245
+
246
+ `mypy` runs strict over `src` **and** `tests`, so the tests are type-checked too. The `python` job is a real
247
+ gate but is **advisory**: the `main` ruleset requires the `node`, `workflows` and `licenses` contexts only.
248
+
249
+ `tests/fixtures/capitals-baseline.json` is regenerated from `FakeProvider(accuracy=0.6, seed=0)` over 10 runs
250
+ — the fake is seeded and deterministic per call, so the committed baseline reproduces exactly while a fresh
251
+ sample of it still varies, which is exactly the property the gate has to tolerate.
252
+
253
+ **Non-responsibility:** no app-specific prompts, tools, or rubrics. Those belong to the generated app's
254
+ `business/` tier ([`docs/design.md` §8.4](../../docs/design.md), the seam).
255
+
256
+ **Status:** Phase 0 scope implemented, plus the Phase 1 retrieval runtime. One thing deferred and
257
+ undeclared, on purpose: `ModelGateway.stream` raises `FeatureDeferredError`, and no provider advertises
258
+ `Capability.STREAMING` or target declares the capability, because the transport is request/response-only.
259
+ Retrieval likewise fails loudly rather than
260
+ approximating: non-file sources, `semantic` chunking, `incremental` refresh, retriever `filters`, a hosted
261
+ reranker, and the `qdrant` / `pinecone` / `weaviate` / `custom` store kinds all raise `FeatureDeferredError`.
262
+ Also deferred (§14): escalation routing, memory, guardrails, prompt/semantic caching, rate limiting, token
263
+ buckets, and adapters beyond the one HTTP provider and the fake.
264
+
265
+ Emitting retrieval into a generated app and indexing it from the CLI are later layers, so this package adds no
266
+ entry point for either.
267
+
268
+ Reference: [`docs/design.md` §9, §12, §14, §17 #2](../../docs/design.md).
@@ -0,0 +1,52 @@
1
+ aid_runtime/__init__.py,sha256=E8pW04ODgOND66tyve5B8CYcSjCjZeMuX93Mey_o0I0,11009
2
+ aid_runtime/cache.py,sha256=jhw_EBd9dfWEUzYeibOvIkVBqqiLUjqF5WKEPz9GkhE,6836
3
+ aid_runtime/concurrency.py,sha256=z0tFQHw35ttFLKfazGBjCMPOLQOnUotE_OPpvR1piH4,2829
4
+ aid_runtime/control.py,sha256=u4U97QFdYSSjOPY3D6f1-GZhkGhygEdR-apdgw_jJok,13797
5
+ aid_runtime/cost.py,sha256=Hc7X6mtkeanL9eJ1lEZZCQ9LxZurhL1it0gm3s9LpQE,5465
6
+ aid_runtime/errors.py,sha256=TKkZyZZcFTl5yxNKlA8jUF4ORpYHLGFEvc6r9FK4X70,12572
7
+ aid_runtime/gateway.py,sha256=p0qsaQUAYrwSzlWwpUi1wY6pdOeU98QjsWLzq8045Fo,18267
8
+ aid_runtime/pipeline.py,sha256=Y2jiiYISLLxgBKGBj5J2rseEDzE8RXP8v5pJ_edjHfM,6334
9
+ aid_runtime/prompts.py,sha256=eN3p_Yc-J6pyKaL2w4I-EmwvcPKErnXBvvI_Syq13ZM,7083
10
+ aid_runtime/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
+ aid_runtime/steps.py,sha256=44v8-ebN77VGZePmfhBYi0KORFBrH2GWFSzlfczaJhs,12047
12
+ aid_runtime/structured.py,sha256=Uga-VoGJN56lUtr-M3nmUjAKGF32cAAk-p7WceUgPzg,5061
13
+ aid_runtime/tools.py,sha256=7LeaPI5aJioNe3vexUv94tvIRpOwxj1Ujt723qvO3UI,9906
14
+ aid_runtime/tracing.py,sha256=Ii2ypuHxHLZ4lmE6oYhcCJiA9dFE5KhXKRTsdQnDDLE,8524
15
+ aid_runtime/types.py,sha256=cVYgmacDIhStQT5dX_X67-q57sPBBaWC5nybtGVqVsw,10389
16
+ aid_runtime/agents/__init__.py,sha256=XOOecYSBtWzD0Ylaurbz3w2Tuffpzc_5nNeMIznCucU,2398
17
+ aid_runtime/agents/chat_model.py,sha256=fZdCmSpoECuBVGZE1s8NmT2erT33RhDuLh3yJM6kAJE,14808
18
+ aid_runtime/agents/deep_agents.py,sha256=ZM6ZSc92arCvtsYGsIGdF-NryqN_tvXtRqbdlHaGpeE,26948
19
+ aid_runtime/agents/engine.py,sha256=RQUV49u7MLDmirnFv8YsRGdIutzUGsGaLA6Y2JT5wj0,4129
20
+ aid_runtime/agents/guardrails.py,sha256=RM_iR_h7LQiulN8KA6scKeCyoTxECqywqJO0eLHPpKQ,19363
21
+ aid_runtime/agents/ir.py,sha256=BaIGEzwHSQnP2q7s1GjNwRTICju9LsPGdAcsboA4KSw,24151
22
+ aid_runtime/agents/loop.py,sha256=T-QnC9BQ9K3RyZfHdAuzWiwMXBpQjDLGdHrLgf2fnFM,52608
23
+ aid_runtime/agents/memory.py,sha256=6_D6k2bzAiR2euNvzr6HB28wsUUq-R-WTQc12gkB8ao,9156
24
+ aid_runtime/agents/policy.py,sha256=ael0EW_hWfLu95rbqFMrR4WyqAPtgiSNvNw97xnthdw,6122
25
+ aid_runtime/agents/state.py,sha256=4itT7BWv2JKSeWpRrDyxLguOOVnz9gGH_qgRdCbWCaw,6086
26
+ aid_runtime/evals/__init__.py,sha256=2eSGrGxdQHmnVQb5RJS146ZHc-5VARBJTUrTxcEdCv8,1379
27
+ aid_runtime/evals/dataset.py,sha256=waG4tYSJsgsZ9DLxkhUvF_X-aQB8w2OexYxc7Wwdiyg,2494
28
+ aid_runtime/evals/gate.py,sha256=_nLWXvcuHwq8nE3HpiONg_wBkrzZyGrGT49ZNeKMyTY,15524
29
+ aid_runtime/evals/metrics.py,sha256=Z-rLJjG0dBTdTJTVGTt8WzE0avyYkIG_Ot8sLDaD99Y,10631
30
+ aid_runtime/evals/predicate.py,sha256=JaeN1LJFpliI4JHYdFPtXfjH5_Aad-2-U3ZC6_hManE,8726
31
+ aid_runtime/evals/runner.py,sha256=Lkl-kAVxegPO40pCBUcprRBKvfL6bjYzVf__xe6Z-Mc,7951
32
+ aid_runtime/providers/__init__.py,sha256=KkRkwVRZwZ5H_Rqke8EsipxyMgYuwODB9CGXiHL_gDE,577
33
+ aid_runtime/providers/base.py,sha256=1VYTZ_V9FR6f6pS0MB3LFeu-Zzv4toQvm3CwAQNbeT0,1851
34
+ aid_runtime/providers/fake.py,sha256=oAcpaDOUIp7lmKGEEWG5ktagstg2dvoHR-9xaUeCDLo,9526
35
+ aid_runtime/providers/http.py,sha256=c5DLmX1FqtAUO01MqDf0m82x8zHD480LyL-QtTrxnn4,13168
36
+ aid_runtime/retrieval/__init__.py,sha256=LR6qyiE7pY9QsB1h4g9j1W7acpSINFbgxA9U1otLt90,4580
37
+ aid_runtime/retrieval/chunking.py,sha256=GMJvY1GoIua-FkpCH1S5qNmUrWSaSbl6sBJPCrSDAg4,8296
38
+ aid_runtime/retrieval/citations.py,sha256=UL9fsxLTzzO0n8tEW3X2O5erJ0hrk08NYMkZ9Pk8Jdg,4287
39
+ aid_runtime/retrieval/connectors.py,sha256=JNEwq6S8zKX1GdlWe4Gn2Rdon87DWXlG_WNQfZTlN34,4744
40
+ aid_runtime/retrieval/embedding.py,sha256=ZPjALe_nEzYpa8N5COExPGrojjZFX8awTz6vzJYfx_4,12374
41
+ aid_runtime/retrieval/gate.py,sha256=fxrI7Og9d3J-NKUcn3QI-2N4K3o4-LQehXxCK19qzCM,4842
42
+ aid_runtime/retrieval/hybrid.py,sha256=-GIBeI3njT5uaEvSeq2t151Zlv2yOPHOPje4TJLNVyU,5804
43
+ aid_runtime/retrieval/metrics.py,sha256=LN36BIXzpbhkG9grzE0q5wncgXt_Y85-b5jGJ038-3Y,7904
44
+ aid_runtime/retrieval/pgvector.py,sha256=kdE6Gdk9JhR6-QxN353CmuNYMLyr5eBvnQl4krLzLew,10040
45
+ aid_runtime/retrieval/rerank.py,sha256=L4ICBi9YHDzhnKVtImKh5Wzre1HDzS8at-AKDkXBJMc,5114
46
+ aid_runtime/retrieval/retriever.py,sha256=hVLP0xOLKioYsa-aS1akyJl5p-WbpVjm8OfCw26m4CE,13941
47
+ aid_runtime/retrieval/stores.py,sha256=VEi68AaMHcQjdAbWOHflp3jCl1_1db71rIzZjTouLlo,5857
48
+ aid_runtime/retrieval/types.py,sha256=8vKP21rmE9LglnB5gtv0dLBIkXzaPqdG3-GS98K0gpc,13498
49
+ aid_runtime_python-0.1.39.dist-info/METADATA,sha256=NqIxXTImMqSvUWC2zPniEyZoopmh3ByGXVD2ICZVOGI,16364
50
+ aid_runtime_python-0.1.39.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
51
+ aid_runtime_python-0.1.39.dist-info/top_level.txt,sha256=QvODgNVRDDMmLcTnoIQ4jYnhqf-N4b9sHWEmaUUxp44,12
52
+ aid_runtime_python-0.1.39.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ aid_runtime