castia 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- castia-0.1.0/.gitignore +31 -0
- castia-0.1.0/CHANGELOG.md +8 -0
- castia-0.1.0/LICENSE +21 -0
- castia-0.1.0/PKG-INFO +198 -0
- castia-0.1.0/README.md +162 -0
- castia-0.1.0/RELEASING.md +65 -0
- castia-0.1.0/pyproject.toml +72 -0
- castia-0.1.0/src/castia/TRACING.md +115 -0
- castia-0.1.0/src/castia/__init__.py +72 -0
- castia-0.1.0/src/castia/__main__.py +319 -0
- castia-0.1.0/src/castia/activity.py +172 -0
- castia-0.1.0/src/castia/activity_routing.py +55 -0
- castia-0.1.0/src/castia/application.py +220 -0
- castia-0.1.0/src/castia/auth.py +26 -0
- castia-0.1.0/src/castia/cards.py +181 -0
- castia-0.1.0/src/castia/connector.py +312 -0
- castia-0.1.0/src/castia/context.py +179 -0
- castia-0.1.0/src/castia/credentials.py +231 -0
- castia-0.1.0/src/castia/dependencies.py +51 -0
- castia-0.1.0/src/castia/deploy.py +211 -0
- castia-0.1.0/src/castia/dispatch.py +160 -0
- castia-0.1.0/src/castia/drive.py +154 -0
- castia-0.1.0/src/castia/entities.py +147 -0
- castia-0.1.0/src/castia/evalsuite.py +406 -0
- castia-0.1.0/src/castia/identity.py +63 -0
- castia-0.1.0/src/castia/invokes.py +113 -0
- castia-0.1.0/src/castia/mail.py +132 -0
- castia-0.1.0/src/castia/mailbox.py +81 -0
- castia-0.1.0/src/castia/messages.py +164 -0
- castia-0.1.0/src/castia/model.py +181 -0
- castia-0.1.0/src/castia/observability.py +316 -0
- castia-0.1.0/src/castia/py.typed +0 -0
- castia-0.1.0/src/castia/server.py +267 -0
- castia-0.1.0/src/castia/streaming.py +172 -0
- castia-0.1.0/src/castia/surfaces.py +27 -0
- castia-0.1.0/src/castia/tools.py +294 -0
- castia-0.1.0/src/castia/tracing.py +115 -0
- castia-0.1.0/tests/test_evalsuite.py +182 -0
- castia-0.1.0/tests/test_identity.py +130 -0
- castia-0.1.0/tests/test_observability.py +109 -0
- castia-0.1.0/tests/test_rich.py +898 -0
castia-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Byte-compiled / optimized / cache
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
.eggs/
|
|
12
|
+
|
|
13
|
+
# Virtual environments
|
|
14
|
+
.venv/
|
|
15
|
+
venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Tooling caches
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.ruff_cache/
|
|
21
|
+
.mypy_cache/
|
|
22
|
+
.pytype/
|
|
23
|
+
|
|
24
|
+
# Environment / secrets
|
|
25
|
+
.env
|
|
26
|
+
.env.*
|
|
27
|
+
|
|
28
|
+
# Editors / OS
|
|
29
|
+
.vscode/
|
|
30
|
+
.idea/
|
|
31
|
+
.DS_Store
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-09-09)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Continuous Integration
|
|
7
|
+
|
|
8
|
+
* automate per-language releases with release-please + PyPI Trusted Publishing ([#3](https://github.com/sethjuarez/castia/issues/3)) ([cb9db23](https://github.com/sethjuarez/castia/commit/cb9db23c1dda6ad270d51339f5adde829925c045))
|
castia-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Seth Juarez
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
castia-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: castia
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Idiomatic, FastAPI-style Python SDK for Microsoft Foundry hosted agents.
|
|
5
|
+
Project-URL: Homepage, https://github.com/sethjuarez/castia
|
|
6
|
+
Project-URL: Repository, https://github.com/sethjuarez/castia
|
|
7
|
+
Project-URL: Documentation, https://castia.dev
|
|
8
|
+
Author: Seth Juarez
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: activity,adaptive-cards,agents,fastapi,foundry,microsoft-foundry,sdk,teams
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: azure-ai-projects>=2.3.0
|
|
24
|
+
Requires-Dist: azure-identity>=1.19
|
|
25
|
+
Requires-Dist: fastapi>=0.115
|
|
26
|
+
Requires-Dist: httpx<1,>=0.28.1
|
|
27
|
+
Requires-Dist: microsoft-opentelemetry>=1.3.8
|
|
28
|
+
Requires-Dist: pyjwt>=2.9
|
|
29
|
+
Requires-Dist: uvicorn[standard]>=0.30
|
|
30
|
+
Provides-Extra: deploy
|
|
31
|
+
Requires-Dist: ruamel-yaml>=0.18; extra == 'deploy'
|
|
32
|
+
Provides-Extra: test
|
|
33
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
34
|
+
Requires-Dist: ruamel-yaml>=0.18; extra == 'test'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# castia
|
|
38
|
+
|
|
39
|
+
Idiomatic, FastAPI-style Python SDK for [Microsoft Foundry](https://ai.azure.com)
|
|
40
|
+
hosted agents.
|
|
41
|
+
|
|
42
|
+
`castia` lets a hosted agent speak Foundry's three wire protocols — Activity
|
|
43
|
+
(Teams/Bot Framework), OpenAI `responses`, and `invocations` — through
|
|
44
|
+
protocol-named decorators, dependency injection (`Depends`), and typed builders
|
|
45
|
+
for messages, Adaptive Cards, entities, and invoke envelopes. You decorate a
|
|
46
|
+
handler, return a value, and the framework does the rest — the auth chains,
|
|
47
|
+
hosting, Activity routing, and telemetry stay out of your file.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install castia
|
|
53
|
+
# or, with uv:
|
|
54
|
+
uv add castia
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Requires Python 3.11+.
|
|
58
|
+
|
|
59
|
+
## Quickstart
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from castia import Agent, Depends, Model, Teams
|
|
63
|
+
|
|
64
|
+
app = Agent(name="my-agent")
|
|
65
|
+
|
|
66
|
+
def gpt4o() -> Model:
|
|
67
|
+
return Model("gpt-4o")
|
|
68
|
+
|
|
69
|
+
@app.message(Teams.direct, Teams.group, Teams.channel_mention)
|
|
70
|
+
async def reply(text: str, model: Model = Depends(gpt4o)) -> str:
|
|
71
|
+
return await model.respond(text)
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
app.run()
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Decorate a handler with the surfaces it answers on, return a `str`, and the
|
|
78
|
+
framework sends it as the Teams reply. The model is built **once** for the
|
|
79
|
+
process and injected via `Depends` — the model choice stays visible in your file
|
|
80
|
+
instead of being buried in the framework. (`Model()` with no argument falls back
|
|
81
|
+
to `AZURE_AI_MODEL_DEPLOYMENT_NAME`; `get_model` / `use_model("gpt-4o")` are
|
|
82
|
+
zero-config conveniences.)
|
|
83
|
+
|
|
84
|
+
### Composing protocols with routers
|
|
85
|
+
|
|
86
|
+
Like FastAPI's `include_router`, an `Agent` composes `Router`s so each protocol
|
|
87
|
+
can live in its own module:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from castia import Agent
|
|
91
|
+
from handlers import activity, responses, invocations
|
|
92
|
+
|
|
93
|
+
app = Agent(name="my-agent")
|
|
94
|
+
app.include(activity.router, responses.router, invocations.router)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Richer replies
|
|
98
|
+
|
|
99
|
+
Handlers can take a `Message` and reach for typed builders — Adaptive Cards,
|
|
100
|
+
suggested actions, citations, mentions, sensitivity labels, live-typing
|
|
101
|
+
streamers, and reactions:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from castia import Depends, Message, Model, Reaction, Router, Teams
|
|
105
|
+
|
|
106
|
+
router = Router()
|
|
107
|
+
|
|
108
|
+
@router.activity(Teams.direct)
|
|
109
|
+
async def reply(text: str, msg: Message, model: Model = Depends(get_model)) -> None:
|
|
110
|
+
await msg.react(Reaction.eyes)
|
|
111
|
+
answer = await model.respond(text)
|
|
112
|
+
await msg.say(answer)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Protocols
|
|
116
|
+
|
|
117
|
+
`castia` publishes handlers for the protocols in `PUBLISHABLE_PROTOCOLS`:
|
|
118
|
+
|
|
119
|
+
- **Activity** — Teams / Bot Framework message and invoke turns.
|
|
120
|
+
- **`responses`** — the OpenAI `responses` wire shape.
|
|
121
|
+
- **`invocations`** — Foundry invoke envelopes (tool execution, agent-to-agent).
|
|
122
|
+
|
|
123
|
+
## Observability & evaluation
|
|
124
|
+
|
|
125
|
+
`castia` configures Foundry/Agent 365 telemetry for you when the agent starts.
|
|
126
|
+
By default it emits GenAI spans (the `chat {model}` spans the Foundry Traces UI
|
|
127
|
+
keys off) but does **not** record the prompt/response **content** onto them.
|
|
128
|
+
|
|
129
|
+
Recording content is what makes an agent's traces *evaluable* — trace-based
|
|
130
|
+
evaluators read the input/output text from the GenAI spans, which is only present
|
|
131
|
+
when content recording is enabled. Turn it on deliberately via
|
|
132
|
+
`configure_observability`:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from castia.observability import configure_observability
|
|
136
|
+
|
|
137
|
+
# Records prompt/response text onto GenAI spans so traces can be evaluated.
|
|
138
|
+
configure_observability(enable_content_recording=True)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Resolution order for each flag is **explicit argument > environment variable >
|
|
142
|
+
default**:
|
|
143
|
+
|
|
144
|
+
| Flag | Argument | Environment variable | Default |
|
|
145
|
+
| --- | --- | --- | --- |
|
|
146
|
+
| Content recording | `enable_content_recording` | `AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED` | off |
|
|
147
|
+
| GenAI tracing | `enable_genai_tracing` | `AZURE_EXPERIMENTAL_ENABLE_GENAI_TRACING` | on |
|
|
148
|
+
|
|
149
|
+
Passing nothing preserves the default behavior. Telemetry setup is best-effort:
|
|
150
|
+
a failure is logged, never raised, so it can't break startup or a turn.
|
|
151
|
+
|
|
152
|
+
> **Security caveat:** enabling content recording writes prompt and response
|
|
153
|
+
> **text** to Application Insights. Only enable it where storing that content is
|
|
154
|
+
> acceptable for your data-handling and privacy requirements.
|
|
155
|
+
|
|
156
|
+
### Building a scored eval suite
|
|
157
|
+
|
|
158
|
+
Once your traces are evaluable, `python -m castia eval` wraps the
|
|
159
|
+
`azd ai agent eval` extension to synthesize and run a **scored** eval suite — a
|
|
160
|
+
generated JSONL dataset plus an auto-generated, weighted **rubric** (a custom
|
|
161
|
+
multi-dimension evaluator):
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
# Offline gate — validate eval.yaml + rubric files, no Azure, free in CI:
|
|
165
|
+
python -m castia eval check
|
|
166
|
+
|
|
167
|
+
# Synthesize a rubric + dataset from the agent instruction (billable):
|
|
168
|
+
python -m castia eval generate --agent my-agent --max-samples 25
|
|
169
|
+
|
|
170
|
+
# Re-upload locally edited rubric/dataset files as a new version:
|
|
171
|
+
python -m castia eval update --evaluator-only
|
|
172
|
+
|
|
173
|
+
# Submit a scored run against the deployed agent (billable):
|
|
174
|
+
python -m castia eval run
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
`check` is a pure, offline referential-integrity gate: it resolves every
|
|
178
|
+
evaluator/dataset `local_uri` and validates each rubric dimensions file.
|
|
179
|
+
`generate` and `run` submit **billable** Foundry jobs, so both accept
|
|
180
|
+
`--dry-run` to print the resolved `azd` command line without submitting
|
|
181
|
+
anything. The `azd` wrappers need the build-time extra: `pip install
|
|
182
|
+
'castia[deploy]'`.
|
|
183
|
+
|
|
184
|
+
The rubric dimensions file is a **bare JSON list** where each entry is keyed by
|
|
185
|
+
`id` (a stable slug like `correct_outcome`), with an optional
|
|
186
|
+
`always_applicable: true` on the catch-all dimension. That cross-SDK shape is
|
|
187
|
+
pinned in the monorepo at [`spec/conformance/rubric/`](../../spec/conformance/rubric).
|
|
188
|
+
|
|
189
|
+
## Design
|
|
190
|
+
|
|
191
|
+
`castia` is deliberately import-cheap: `import castia` never pulls in the
|
|
192
|
+
instrumented Azure/OpenAI/httpx stacks, so telemetry can be configured before
|
|
193
|
+
those libraries load. The heavy imports are deferred into the methods that need
|
|
194
|
+
them.
|
|
195
|
+
|
|
196
|
+
## License
|
|
197
|
+
|
|
198
|
+
MIT © 2026 Seth Juarez
|
castia-0.1.0/README.md
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# castia
|
|
2
|
+
|
|
3
|
+
Idiomatic, FastAPI-style Python SDK for [Microsoft Foundry](https://ai.azure.com)
|
|
4
|
+
hosted agents.
|
|
5
|
+
|
|
6
|
+
`castia` lets a hosted agent speak Foundry's three wire protocols — Activity
|
|
7
|
+
(Teams/Bot Framework), OpenAI `responses`, and `invocations` — through
|
|
8
|
+
protocol-named decorators, dependency injection (`Depends`), and typed builders
|
|
9
|
+
for messages, Adaptive Cards, entities, and invoke envelopes. You decorate a
|
|
10
|
+
handler, return a value, and the framework does the rest — the auth chains,
|
|
11
|
+
hosting, Activity routing, and telemetry stay out of your file.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install castia
|
|
17
|
+
# or, with uv:
|
|
18
|
+
uv add castia
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Requires Python 3.11+.
|
|
22
|
+
|
|
23
|
+
## Quickstart
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from castia import Agent, Depends, Model, Teams
|
|
27
|
+
|
|
28
|
+
app = Agent(name="my-agent")
|
|
29
|
+
|
|
30
|
+
def gpt4o() -> Model:
|
|
31
|
+
return Model("gpt-4o")
|
|
32
|
+
|
|
33
|
+
@app.message(Teams.direct, Teams.group, Teams.channel_mention)
|
|
34
|
+
async def reply(text: str, model: Model = Depends(gpt4o)) -> str:
|
|
35
|
+
return await model.respond(text)
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
app.run()
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Decorate a handler with the surfaces it answers on, return a `str`, and the
|
|
42
|
+
framework sends it as the Teams reply. The model is built **once** for the
|
|
43
|
+
process and injected via `Depends` — the model choice stays visible in your file
|
|
44
|
+
instead of being buried in the framework. (`Model()` with no argument falls back
|
|
45
|
+
to `AZURE_AI_MODEL_DEPLOYMENT_NAME`; `get_model` / `use_model("gpt-4o")` are
|
|
46
|
+
zero-config conveniences.)
|
|
47
|
+
|
|
48
|
+
### Composing protocols with routers
|
|
49
|
+
|
|
50
|
+
Like FastAPI's `include_router`, an `Agent` composes `Router`s so each protocol
|
|
51
|
+
can live in its own module:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from castia import Agent
|
|
55
|
+
from handlers import activity, responses, invocations
|
|
56
|
+
|
|
57
|
+
app = Agent(name="my-agent")
|
|
58
|
+
app.include(activity.router, responses.router, invocations.router)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Richer replies
|
|
62
|
+
|
|
63
|
+
Handlers can take a `Message` and reach for typed builders — Adaptive Cards,
|
|
64
|
+
suggested actions, citations, mentions, sensitivity labels, live-typing
|
|
65
|
+
streamers, and reactions:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from castia import Depends, Message, Model, Reaction, Router, Teams
|
|
69
|
+
|
|
70
|
+
router = Router()
|
|
71
|
+
|
|
72
|
+
@router.activity(Teams.direct)
|
|
73
|
+
async def reply(text: str, msg: Message, model: Model = Depends(get_model)) -> None:
|
|
74
|
+
await msg.react(Reaction.eyes)
|
|
75
|
+
answer = await model.respond(text)
|
|
76
|
+
await msg.say(answer)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Protocols
|
|
80
|
+
|
|
81
|
+
`castia` publishes handlers for the protocols in `PUBLISHABLE_PROTOCOLS`:
|
|
82
|
+
|
|
83
|
+
- **Activity** — Teams / Bot Framework message and invoke turns.
|
|
84
|
+
- **`responses`** — the OpenAI `responses` wire shape.
|
|
85
|
+
- **`invocations`** — Foundry invoke envelopes (tool execution, agent-to-agent).
|
|
86
|
+
|
|
87
|
+
## Observability & evaluation
|
|
88
|
+
|
|
89
|
+
`castia` configures Foundry/Agent 365 telemetry for you when the agent starts.
|
|
90
|
+
By default it emits GenAI spans (the `chat {model}` spans the Foundry Traces UI
|
|
91
|
+
keys off) but does **not** record the prompt/response **content** onto them.
|
|
92
|
+
|
|
93
|
+
Recording content is what makes an agent's traces *evaluable* — trace-based
|
|
94
|
+
evaluators read the input/output text from the GenAI spans, which is only present
|
|
95
|
+
when content recording is enabled. Turn it on deliberately via
|
|
96
|
+
`configure_observability`:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from castia.observability import configure_observability
|
|
100
|
+
|
|
101
|
+
# Records prompt/response text onto GenAI spans so traces can be evaluated.
|
|
102
|
+
configure_observability(enable_content_recording=True)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Resolution order for each flag is **explicit argument > environment variable >
|
|
106
|
+
default**:
|
|
107
|
+
|
|
108
|
+
| Flag | Argument | Environment variable | Default |
|
|
109
|
+
| --- | --- | --- | --- |
|
|
110
|
+
| Content recording | `enable_content_recording` | `AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED` | off |
|
|
111
|
+
| GenAI tracing | `enable_genai_tracing` | `AZURE_EXPERIMENTAL_ENABLE_GENAI_TRACING` | on |
|
|
112
|
+
|
|
113
|
+
Passing nothing preserves the default behavior. Telemetry setup is best-effort:
|
|
114
|
+
a failure is logged, never raised, so it can't break startup or a turn.
|
|
115
|
+
|
|
116
|
+
> **Security caveat:** enabling content recording writes prompt and response
|
|
117
|
+
> **text** to Application Insights. Only enable it where storing that content is
|
|
118
|
+
> acceptable for your data-handling and privacy requirements.
|
|
119
|
+
|
|
120
|
+
### Building a scored eval suite
|
|
121
|
+
|
|
122
|
+
Once your traces are evaluable, `python -m castia eval` wraps the
|
|
123
|
+
`azd ai agent eval` extension to synthesize and run a **scored** eval suite — a
|
|
124
|
+
generated JSONL dataset plus an auto-generated, weighted **rubric** (a custom
|
|
125
|
+
multi-dimension evaluator):
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
# Offline gate — validate eval.yaml + rubric files, no Azure, free in CI:
|
|
129
|
+
python -m castia eval check
|
|
130
|
+
|
|
131
|
+
# Synthesize a rubric + dataset from the agent instruction (billable):
|
|
132
|
+
python -m castia eval generate --agent my-agent --max-samples 25
|
|
133
|
+
|
|
134
|
+
# Re-upload locally edited rubric/dataset files as a new version:
|
|
135
|
+
python -m castia eval update --evaluator-only
|
|
136
|
+
|
|
137
|
+
# Submit a scored run against the deployed agent (billable):
|
|
138
|
+
python -m castia eval run
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`check` is a pure, offline referential-integrity gate: it resolves every
|
|
142
|
+
evaluator/dataset `local_uri` and validates each rubric dimensions file.
|
|
143
|
+
`generate` and `run` submit **billable** Foundry jobs, so both accept
|
|
144
|
+
`--dry-run` to print the resolved `azd` command line without submitting
|
|
145
|
+
anything. The `azd` wrappers need the build-time extra: `pip install
|
|
146
|
+
'castia[deploy]'`.
|
|
147
|
+
|
|
148
|
+
The rubric dimensions file is a **bare JSON list** where each entry is keyed by
|
|
149
|
+
`id` (a stable slug like `correct_outcome`), with an optional
|
|
150
|
+
`always_applicable: true` on the catch-all dimension. That cross-SDK shape is
|
|
151
|
+
pinned in the monorepo at [`spec/conformance/rubric/`](../../spec/conformance/rubric).
|
|
152
|
+
|
|
153
|
+
## Design
|
|
154
|
+
|
|
155
|
+
`castia` is deliberately import-cheap: `import castia` never pulls in the
|
|
156
|
+
instrumented Azure/OpenAI/httpx stacks, so telemetry can be configured before
|
|
157
|
+
those libraries load. The heavy imports are deferred into the methods that need
|
|
158
|
+
them.
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
|
|
162
|
+
MIT © 2026 Seth Juarez
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Releasing `castia` (Python)
|
|
2
|
+
|
|
3
|
+
Releases are automated by [release-please](https://github.com/googleapis/release-please)
|
|
4
|
+
and published to PyPI with **Trusted Publishing** (OIDC) — no API token is stored
|
|
5
|
+
in the repository. You should never build or upload from a laptop.
|
|
6
|
+
|
|
7
|
+
## One-time PyPI setup (before the first publish)
|
|
8
|
+
|
|
9
|
+
`castia` is not on PyPI yet, so register a **pending** trusted publisher:
|
|
10
|
+
|
|
11
|
+
1. Go to <https://pypi.org/manage/account/publishing/>.
|
|
12
|
+
2. Add a new pending publisher with exactly:
|
|
13
|
+
- **PyPI project name:** `castia`
|
|
14
|
+
- **Owner:** `sethjuarez`
|
|
15
|
+
- **Repository name:** `castia`
|
|
16
|
+
- **Workflow name:** `release-please.yml`
|
|
17
|
+
- **Environment name:** `pypi`
|
|
18
|
+
3. (Recommended) In the GitHub repo, create a **`pypi` environment**
|
|
19
|
+
(Settings → Environments) and add required reviewers so a human approves each
|
|
20
|
+
publish.
|
|
21
|
+
|
|
22
|
+
After the first successful publish, the pending publisher becomes a normal
|
|
23
|
+
trusted publisher automatically.
|
|
24
|
+
|
|
25
|
+
## The steady-state flow
|
|
26
|
+
|
|
27
|
+
1. Merge Conventional-Commit PRs into `main` (see [`CONTRIBUTING.md`](../../CONTRIBUTING.md)).
|
|
28
|
+
2. release-please maintains an open **release PR** titled like
|
|
29
|
+
`chore(main): release python 0.2.0`. It bumps `pyproject.toml`'s version and
|
|
30
|
+
updates `CHANGELOG.md`.
|
|
31
|
+
3. **Merging that release PR** creates the tag `python-v<version>` and a GitHub
|
|
32
|
+
release, then the `release-please` workflow's `publish-python` job builds the
|
|
33
|
+
wheel + sdist and uploads them to PyPI.
|
|
34
|
+
|
|
35
|
+
You do not push tags by hand; merging the release PR is the release action.
|
|
36
|
+
|
|
37
|
+
## First release (0.1.0) bootstrap
|
|
38
|
+
|
|
39
|
+
`.release-please-manifest.json` starts the `packages/python` component at
|
|
40
|
+
`0.0.0`. The commit that introduces release-please carries a
|
|
41
|
+
`Release-As: 0.1.0` footer, which makes release-please open its first release PR
|
|
42
|
+
at **0.1.0** (matching the version already in `pyproject.toml`) regardless of the
|
|
43
|
+
commit types before it. Merging that first release PR publishes `castia 0.1.0`.
|
|
44
|
+
|
|
45
|
+
No sticky configuration is left behind: after 0.1.0 ships, the manifest advances
|
|
46
|
+
to `0.1.0` and subsequent versions are derived normally from Conventional
|
|
47
|
+
Commits.
|
|
48
|
+
|
|
49
|
+
## Versioning scheme
|
|
50
|
+
|
|
51
|
+
- Per-language tags: the Python SDK releases as `python-v<version>` (e.g.
|
|
52
|
+
`python-v0.1.0`). A future Rust SDK would release independently as
|
|
53
|
+
`rust-v<version>`.
|
|
54
|
+
- Pre-1.0: `feat` → minor, `fix` → patch, breaking changes are **not**
|
|
55
|
+
auto-promoted to a major (`bump-minor-pre-major`).
|
|
56
|
+
|
|
57
|
+
## Verifying a release build locally (optional)
|
|
58
|
+
|
|
59
|
+
You never publish locally, but you can reproduce what CI builds:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
cd packages/python
|
|
63
|
+
uv build
|
|
64
|
+
uvx twine check dist/*
|
|
65
|
+
```
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "castia"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Idiomatic, FastAPI-style Python SDK for Microsoft Foundry hosted agents."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
authors = [{ name = "Seth Juarez" }]
|
|
9
|
+
keywords = [
|
|
10
|
+
"foundry",
|
|
11
|
+
"microsoft-foundry",
|
|
12
|
+
"agents",
|
|
13
|
+
"teams",
|
|
14
|
+
"fastapi",
|
|
15
|
+
"sdk",
|
|
16
|
+
"activity",
|
|
17
|
+
"adaptive-cards",
|
|
18
|
+
]
|
|
19
|
+
classifiers = [
|
|
20
|
+
"Development Status :: 3 - Alpha",
|
|
21
|
+
"Intended Audience :: Developers",
|
|
22
|
+
"License :: OSI Approved :: MIT License",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
dependencies = [
|
|
32
|
+
"fastapi>=0.115",
|
|
33
|
+
"uvicorn[standard]>=0.30",
|
|
34
|
+
"httpx>=0.28.1,<1",
|
|
35
|
+
"pyjwt>=2.9",
|
|
36
|
+
"azure-ai-projects>=2.3.0",
|
|
37
|
+
"azure-identity>=1.19",
|
|
38
|
+
"microsoft-opentelemetry>=1.3.8",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/sethjuarez/castia"
|
|
43
|
+
Repository = "https://github.com/sethjuarez/castia"
|
|
44
|
+
Documentation = "https://castia.dev"
|
|
45
|
+
|
|
46
|
+
[project.optional-dependencies]
|
|
47
|
+
# Build-time only: used by `python -m castia deploy` to rewrite azure.yaml from
|
|
48
|
+
# the decorators. Deliberately not a runtime dependency so it never ships in the
|
|
49
|
+
# hosted container.
|
|
50
|
+
deploy = ["ruamel.yaml>=0.18"]
|
|
51
|
+
# Test-time only. Includes ruamel.yaml because the eval-suite tests parse real
|
|
52
|
+
# eval.yaml / rubric fixtures.
|
|
53
|
+
test = ["pytest>=8", "ruamel.yaml>=0.18"]
|
|
54
|
+
|
|
55
|
+
[tool.uv]
|
|
56
|
+
prerelease = "allow"
|
|
57
|
+
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
|
|
61
|
+
[build-system]
|
|
62
|
+
requires = ["hatchling"]
|
|
63
|
+
build-backend = "hatchling.build"
|
|
64
|
+
|
|
65
|
+
[tool.hatch.build.targets.wheel]
|
|
66
|
+
packages = ["src/castia"]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint.flake8-bugbear]
|
|
69
|
+
# `Depends(...)` is an inert marker -- like FastAPI's, it never calls the
|
|
70
|
+
# provider, it just records it -- so calling it in an argument default is safe.
|
|
71
|
+
# This silences B008 without a `# noqa` on the handler.
|
|
72
|
+
extend-immutable-calls = ["castia.Depends"]
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Trace labels in the Foundry Traces UI
|
|
2
|
+
|
|
3
|
+
A working reference for the span badges in the trace tree — **In Process**,
|
|
4
|
+
**HTTP**, **Invoke Agent**, **Chat** — and a warning about one badge that lies.
|
|
5
|
+
Written after a bug bash where we watched the framework/HTTP badges flip on their
|
|
6
|
+
own and spent a long time blaming our own code before finding the real cause: a
|
|
7
|
+
non-deterministic query in the Foundry Traces portal.
|
|
8
|
+
|
|
9
|
+
## Where the labels come from
|
|
10
|
+
|
|
11
|
+
Two different things feed the badges, and only one of them is under our control.
|
|
12
|
+
|
|
13
|
+
**1. Our own operation spans — stable, ours to set.** The spans we create in
|
|
14
|
+
`tracing.py` (`invoke_agent`, `execute_tool`) carry `gen_ai.operation.name`, a
|
|
15
|
+
fixed vocabulary the portal recognises: `invoke_agent` → Invoke Agent, `chat` →
|
|
16
|
+
Chat, `execute_tool` → Execute Tool. We set these explicitly, at span creation,
|
|
17
|
+
on spans we own. They render correctly and do not flicker.
|
|
18
|
+
|
|
19
|
+
**2. Framework and client span "kind" — computed at export.** For every other
|
|
20
|
+
span (the SDK's `agents.adapter.process`, the auth/MSI HTTP calls, …) the badge
|
|
21
|
+
derives from the Azure-Monitor *dependency type* the exporter writes into App
|
|
22
|
+
Insights. That logic lives in
|
|
23
|
+
`azure/monitor/opentelemetry/exporter/export/trace/_exporter.py`:
|
|
24
|
+
|
|
25
|
+
| Span shape | Exported type | Badge |
|
|
26
|
+
| --- | --- | --- |
|
|
27
|
+
| `SpanKind.INTERNAL`, no mapping attributes | `InProc` | **In Process** |
|
|
28
|
+
| `SpanKind.INTERNAL` **with** `gen_ai.system` | `gen_ai.*` | (GenAI type) |
|
|
29
|
+
| `SpanKind.CLIENT` + `http.method` / `http.request.method` | `HTTP` | **HTTP** (renders the verb) |
|
|
30
|
+
| `SpanKind.CLIENT` + `db.system` / messaging / rpc | that system | (its own type) |
|
|
31
|
+
| `SpanKind.SERVER` / `CONSUMER` | — | a Request, not a dependency |
|
|
32
|
+
| `SpanKind.CLIENT` with none of the recognised attributes | *(blank)* | **Other** |
|
|
33
|
+
|
|
34
|
+
This exported `type` is written once, is single-valued, and is immutable in App
|
|
35
|
+
Insights. `az monitor app-insights query` returns the correct `HTTP` / `InProc`
|
|
36
|
+
for these spans every time, hours or days later.
|
|
37
|
+
|
|
38
|
+
## The one rule: stamp identity, restyle nothing
|
|
39
|
+
|
|
40
|
+
`_AgentIdentitySpanProcessor.on_start` (in `observability.py`) runs on **every**
|
|
41
|
+
span on the provider. Its only job is to stamp the agent-identity attributes the
|
|
42
|
+
Foundry portal needs to associate the run with the agent:
|
|
43
|
+
|
|
44
|
+
- `gen_ai.agent.name`, `gen_ai.agent.version`, `gen_ai.agent.id`
|
|
45
|
+
- `microsoft.foundry.project.id`, `gen_ai.azure_ai_project.id`
|
|
46
|
+
|
|
47
|
+
Those keys are neutral — none is a mapping attribute the exporter keys off, so
|
|
48
|
+
they do not change any span's exported `type`. Keep it that way. **Do not** stamp
|
|
49
|
+
classification-affecting attributes (`gen_ai.operation.name`, `gen_ai.system`,
|
|
50
|
+
`http.*`) onto framework or client spans from a blanket processor; set operation
|
|
51
|
+
names only on spans we own, at creation, in `tracing.py`.
|
|
52
|
+
|
|
53
|
+
This rule is about keeping the underlying App Insights `type` clean. It is good
|
|
54
|
+
hygiene, but note it is **not** what makes the framework badge flicker — see next.
|
|
55
|
+
|
|
56
|
+
## The badge that lies: portal query is non-deterministic
|
|
57
|
+
|
|
58
|
+
**Symptom.** In the Traces UI, the **kind** badge on framework and HTTP spans
|
|
59
|
+
(e.g. `agents.adapter.process`, `GET /msi/token`) randomly flips to
|
|
60
|
+
**default / Other** on runs that are already complete and stored. No redeploy, no
|
|
61
|
+
new run, no data change — just refreshing or reopening the same trace is enough to
|
|
62
|
+
change it. It flips back on a later read. Seen across agent versions (confirmed on
|
|
63
|
+
v13 op `6efde7c97049e6214d5aee83d1fc4305` and a v18 run `ad28e31a…`).
|
|
64
|
+
|
|
65
|
+
**Root cause — portal-side, not our telemetry.** The Traces detail query
|
|
66
|
+
`spanAppInsightsLLMCallBaseQueryOpt` (fired when a run is opened) builds two
|
|
67
|
+
projections of the *same* spans and unions them:
|
|
68
|
+
|
|
69
|
+
- `normal_spans`: `span_type = type` — the real exported type (`HTTP` / `InProc`)
|
|
70
|
+
- `gen_ai_spans`: `span_type = "default"` — a hardcoded literal, applied to
|
|
71
|
+
**every** span with **no filter**
|
|
72
|
+
|
|
73
|
+
Then it collapses per span id with a non-deterministic aggregate:
|
|
74
|
+
|
|
75
|
+
```kusto
|
|
76
|
+
gen_ai_spans
|
|
77
|
+
| ...
|
|
78
|
+
| union normal_spans
|
|
79
|
+
| summarize spanType = any(span_type), ...
|
|
80
|
+
by id, operationId = operation_Id, operationParentId = operation_ParentId
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Each span id now has two rows with conflicting `span_type` — its real type and
|
|
84
|
+
`"default"` — and `any()` in KQL is explicitly non-deterministic. Every execution
|
|
85
|
+
arbitrarily returns one or the other, so the badge is a coin-flip on each read.
|
|
86
|
+
|
|
87
|
+
**Why it looked like "decay" or a per-version difference.** Nothing decayed and
|
|
88
|
+
nothing was version-specific. A span that "stayed correct for hours" was just
|
|
89
|
+
winning the `any()` toss on the reads we happened to look at; a span that "flipped
|
|
90
|
+
on its own" lost one. The captured detail query is byte-for-byte identical across
|
|
91
|
+
v13 and v18 except the `operation_Id` and version filter.
|
|
92
|
+
|
|
93
|
+
**None of our code is involved.** Our agent emits correct spans; App Insights
|
|
94
|
+
stores the correct `type` immutably. The flip is entirely in the portal's derived
|
|
95
|
+
query. Reported to the Foundry team; suggested fix is to filter `gen_ai_spans` to
|
|
96
|
+
spans with `gen_ai.operation.name` set (so a span contributes one `span_type`), or
|
|
97
|
+
to replace `any(span_type)` with a deterministic pick (`arg_max` / `coalesce` /
|
|
98
|
+
`max`) that prefers the real type.
|
|
99
|
+
|
|
100
|
+
## How to verify (ground truth)
|
|
101
|
+
|
|
102
|
+
Do **not** trust the UI badge for framework/HTTP spans — query App Insights
|
|
103
|
+
directly, where the type is immutable and correct:
|
|
104
|
+
|
|
105
|
+
```kusto
|
|
106
|
+
dependencies
|
|
107
|
+
| where operation_Id == "<your-operation-id>"
|
|
108
|
+
| project id, name, type, timestamp
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
- A framework span (e.g. `agents.adapter.process`) shows `type == "InProc"`.
|
|
112
|
+
- An auth/MSI call (e.g. `GET /msi/token`) shows `type == "HTTP"`.
|
|
113
|
+
|
|
114
|
+
If those rows are correct but the UI badge reads **Other**, it is the portal
|
|
115
|
+
`any()` query, not the runtime and not our processor.
|