costzera 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- costzera-0.1.0/.gitignore +6 -0
- costzera-0.1.0/LICENSE +21 -0
- costzera-0.1.0/PKG-INFO +122 -0
- costzera-0.1.0/README.md +82 -0
- costzera-0.1.0/pyproject.toml +47 -0
- costzera-0.1.0/scripts/verify_cache_tokens.py +26 -0
- costzera-0.1.0/scripts/verify_instrumentors.py +174 -0
- costzera-0.1.0/src/costzera/__init__.py +202 -0
- costzera-0.1.0/src/costzera/_instrument.py +47 -0
- costzera-0.1.0/src/costzera/_pipeline.py +339 -0
- costzera-0.1.0/src/costzera/_wire.py +241 -0
- costzera-0.1.0/tests/fixtures/real_anthropic.json +91 -0
- costzera-0.1.0/tests/fixtures/real_langchain.json +91 -0
- costzera-0.1.0/tests/fixtures/sample_batch.json +90 -0
- costzera-0.1.0/tests/test_sdk.py +359 -0
costzera-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 costzera-ai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
costzera-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: costzera
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find and fix what makes AI agents slow and expensive. Prompts and completions never leave your process.
|
|
5
|
+
Project-URL: Homepage, https://costzera.com
|
|
6
|
+
Project-URL: Documentation, https://costzera.com/docs#sdk
|
|
7
|
+
Project-URL: Repository, https://github.com/costzera-ai/costzera-sdk
|
|
8
|
+
Project-URL: Issues, https://github.com/costzera-ai/costzera-sdk/issues
|
|
9
|
+
Author: Costzera
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,cost,llm,observability,openinference,opentelemetry
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: System :: Monitoring
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: opentelemetry-api>=1.24
|
|
18
|
+
Requires-Dist: opentelemetry-sdk>=1.24
|
|
19
|
+
Provides-Extra: anthropic
|
|
20
|
+
Requires-Dist: openinference-instrumentation-anthropic; extra == 'anthropic'
|
|
21
|
+
Provides-Extra: bedrock
|
|
22
|
+
Requires-Dist: openinference-instrumentation-bedrock; extra == 'bedrock'
|
|
23
|
+
Provides-Extra: crewai
|
|
24
|
+
Requires-Dist: openinference-instrumentation-crewai; extra == 'crewai'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
27
|
+
Provides-Extra: dspy
|
|
28
|
+
Requires-Dist: openinference-instrumentation-dspy; extra == 'dspy'
|
|
29
|
+
Provides-Extra: langchain
|
|
30
|
+
Requires-Dist: openinference-instrumentation-langchain; extra == 'langchain'
|
|
31
|
+
Provides-Extra: litellm
|
|
32
|
+
Requires-Dist: openinference-instrumentation-litellm; extra == 'litellm'
|
|
33
|
+
Provides-Extra: llama-index
|
|
34
|
+
Requires-Dist: openinference-instrumentation-llama-index; extra == 'llama-index'
|
|
35
|
+
Provides-Extra: openai
|
|
36
|
+
Requires-Dist: openinference-instrumentation-openai; extra == 'openai'
|
|
37
|
+
Provides-Extra: openai-agents
|
|
38
|
+
Requires-Dist: openinference-instrumentation-openai-agents; extra == 'openai-agents'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# Costzera for Python
|
|
42
|
+
|
|
43
|
+
Find and fix what makes your AI agents slow and expensive, without a tracing
|
|
44
|
+
tool and without sending us your prompts.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install "costzera[openai]" # or [anthropic], [langchain], [llama-index], ...
|
|
48
|
+
export COSTZERA_API_KEY=czk_... # Settings → API keys → Send traces
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import costzera
|
|
53
|
+
costzera.init(agent="support-agent")
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
That is the whole setup for agents built on OpenAI, Anthropic, LangChain,
|
|
57
|
+
LlamaIndex, LiteLLM, CrewAI, Bedrock, the OpenAI Agents SDK or DSPy: their model,
|
|
58
|
+
tool and retrieval calls are traced automatically.
|
|
59
|
+
|
|
60
|
+
## Hand-written agents
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
@costzera.tool
|
|
64
|
+
def fetch_order(order_id: str) -> dict: ...
|
|
65
|
+
|
|
66
|
+
with costzera.run("handle_ticket"): # one trace per run of the agent
|
|
67
|
+
with costzera.step("plan"): # a named stage within it
|
|
68
|
+
...
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Group each run of your agent in `costzera.run()`. Without it, every model call
|
|
72
|
+
is its own trace, and Costzera cannot see loops, retries or repeated tool calls
|
|
73
|
+
across a run.
|
|
74
|
+
|
|
75
|
+
## What leaves your process
|
|
76
|
+
|
|
77
|
+
Costzera receives the model, token counts, timings, errors, tool names, and the
|
|
78
|
+
*sizes* and *fingerprints* of inputs and outputs. It never receives prompts,
|
|
79
|
+
completions, tool arguments, tool results or retrieved documents. User and
|
|
80
|
+
session ids are sent as fingerprints.
|
|
81
|
+
|
|
82
|
+
This is enforced by allowlist in [`costzera/_wire.py`](https://github.com/costzera-ai/costzera-sdk/blob/main/python/src/costzera/_wire.py):
|
|
83
|
+
a span is converted into what is sent, and only the attributes named there are
|
|
84
|
+
copied. Anything else, including anything a framework adds in a future
|
|
85
|
+
release, is never read into the payload.
|
|
86
|
+
|
|
87
|
+
Exception messages are not sent by default, because they often quote the
|
|
88
|
+
input that caused them. Only the exception's type is. Pass
|
|
89
|
+
`error_messages=True` to send them, cut to 500 characters.
|
|
90
|
+
|
|
91
|
+
## Already using OpenTelemetry?
|
|
92
|
+
|
|
93
|
+
`init()` adds Costzera to your existing tracer provider instead of replacing
|
|
94
|
+
it, so your current exporters keep working. To send the same traces to
|
|
95
|
+
Langfuse, Phoenix or anything else, add their exporter as you normally would.
|
|
96
|
+
|
|
97
|
+
## Options
|
|
98
|
+
|
|
99
|
+
| `init(...)` | Default | |
|
|
100
|
+
|---|---|---|
|
|
101
|
+
| `agent` | `COSTZERA_AGENT` | The name the agent appears under in Costzera |
|
|
102
|
+
| `api_key` | `COSTZERA_API_KEY` | A key with the Send traces scope |
|
|
103
|
+
| `sample_rate` | `1.0` | Share of successful traces kept. Traces with an error are always kept |
|
|
104
|
+
| `instrument` | `True` | Trace installed frameworks automatically |
|
|
105
|
+
| `error_messages` | `False` | Also send exception messages |
|
|
106
|
+
|
|
107
|
+
Without a key, `init()` logs once and does nothing.
|
|
108
|
+
|
|
109
|
+
## Behaviour you can rely on
|
|
110
|
+
|
|
111
|
+
- Only whole traces are sent, once their root span finishes.
|
|
112
|
+
- Tracing never raises into your code. Export runs on a background thread with
|
|
113
|
+
bounded memory; if Costzera is unreachable, traces are dropped, not queued
|
|
114
|
+
forever.
|
|
115
|
+
- Traces are batched, at most one request every 15 seconds per process under
|
|
116
|
+
steady load.
|
|
117
|
+
- `costzera.flush()` sends what has finished, for short scripts and serverless
|
|
118
|
+
functions. It also runs at exit.
|
|
119
|
+
|
|
120
|
+
## Licence
|
|
121
|
+
|
|
122
|
+
MIT
|
costzera-0.1.0/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Costzera for Python
|
|
2
|
+
|
|
3
|
+
Find and fix what makes your AI agents slow and expensive, without a tracing
|
|
4
|
+
tool and without sending us your prompts.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pip install "costzera[openai]" # or [anthropic], [langchain], [llama-index], ...
|
|
8
|
+
export COSTZERA_API_KEY=czk_... # Settings → API keys → Send traces
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
import costzera
|
|
13
|
+
costzera.init(agent="support-agent")
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
That is the whole setup for agents built on OpenAI, Anthropic, LangChain,
|
|
17
|
+
LlamaIndex, LiteLLM, CrewAI, Bedrock, the OpenAI Agents SDK or DSPy: their model,
|
|
18
|
+
tool and retrieval calls are traced automatically.
|
|
19
|
+
|
|
20
|
+
## Hand-written agents
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
@costzera.tool
|
|
24
|
+
def fetch_order(order_id: str) -> dict: ...
|
|
25
|
+
|
|
26
|
+
with costzera.run("handle_ticket"): # one trace per run of the agent
|
|
27
|
+
with costzera.step("plan"): # a named stage within it
|
|
28
|
+
...
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Group each run of your agent in `costzera.run()`. Without it, every model call
|
|
32
|
+
is its own trace, and Costzera cannot see loops, retries or repeated tool calls
|
|
33
|
+
across a run.
|
|
34
|
+
|
|
35
|
+
## What leaves your process
|
|
36
|
+
|
|
37
|
+
Costzera receives the model, token counts, timings, errors, tool names, and the
|
|
38
|
+
*sizes* and *fingerprints* of inputs and outputs. It never receives prompts,
|
|
39
|
+
completions, tool arguments, tool results or retrieved documents. User and
|
|
40
|
+
session ids are sent as fingerprints.
|
|
41
|
+
|
|
42
|
+
This is enforced by allowlist in [`costzera/_wire.py`](https://github.com/costzera-ai/costzera-sdk/blob/main/python/src/costzera/_wire.py):
|
|
43
|
+
a span is converted into what is sent, and only the attributes named there are
|
|
44
|
+
copied. Anything else, including anything a framework adds in a future
|
|
45
|
+
release, is never read into the payload.
|
|
46
|
+
|
|
47
|
+
Exception messages are not sent by default, because they often quote the
|
|
48
|
+
input that caused them. Only the exception's type is. Pass
|
|
49
|
+
`error_messages=True` to send them, cut to 500 characters.
|
|
50
|
+
|
|
51
|
+
## Already using OpenTelemetry?
|
|
52
|
+
|
|
53
|
+
`init()` adds Costzera to your existing tracer provider instead of replacing
|
|
54
|
+
it, so your current exporters keep working. To send the same traces to
|
|
55
|
+
Langfuse, Phoenix or anything else, add their exporter as you normally would.
|
|
56
|
+
|
|
57
|
+
## Options
|
|
58
|
+
|
|
59
|
+
| `init(...)` | Default | |
|
|
60
|
+
|---|---|---|
|
|
61
|
+
| `agent` | `COSTZERA_AGENT` | The name the agent appears under in Costzera |
|
|
62
|
+
| `api_key` | `COSTZERA_API_KEY` | A key with the Send traces scope |
|
|
63
|
+
| `sample_rate` | `1.0` | Share of successful traces kept. Traces with an error are always kept |
|
|
64
|
+
| `instrument` | `True` | Trace installed frameworks automatically |
|
|
65
|
+
| `error_messages` | `False` | Also send exception messages |
|
|
66
|
+
|
|
67
|
+
Without a key, `init()` logs once and does nothing.
|
|
68
|
+
|
|
69
|
+
## Behaviour you can rely on
|
|
70
|
+
|
|
71
|
+
- Only whole traces are sent, once their root span finishes.
|
|
72
|
+
- Tracing never raises into your code. Export runs on a background thread with
|
|
73
|
+
bounded memory; if Costzera is unreachable, traces are dropped, not queued
|
|
74
|
+
forever.
|
|
75
|
+
- Traces are batched, at most one request every 15 seconds per process under
|
|
76
|
+
steady load.
|
|
77
|
+
- `costzera.flush()` sends what has finished, for short scripts and serverless
|
|
78
|
+
functions. It also runs at exit.
|
|
79
|
+
|
|
80
|
+
## Licence
|
|
81
|
+
|
|
82
|
+
MIT
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.21"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "costzera"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Find and fix what makes AI agents slow and expensive. Prompts and completions never leave your process."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Costzera" }]
|
|
13
|
+
keywords = ["llm", "agents", "observability", "cost", "opentelemetry", "openinference"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: System :: Monitoring",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"opentelemetry-api>=1.24",
|
|
21
|
+
"opentelemetry-sdk>=1.24",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
openai = ["openinference-instrumentation-openai"]
|
|
26
|
+
anthropic = ["openinference-instrumentation-anthropic"]
|
|
27
|
+
langchain = ["openinference-instrumentation-langchain"]
|
|
28
|
+
llama-index = ["openinference-instrumentation-llama-index"]
|
|
29
|
+
litellm = ["openinference-instrumentation-litellm"]
|
|
30
|
+
crewai = ["openinference-instrumentation-crewai"]
|
|
31
|
+
bedrock = ["openinference-instrumentation-bedrock"]
|
|
32
|
+
openai-agents = ["openinference-instrumentation-openai-agents"]
|
|
33
|
+
dspy = ["openinference-instrumentation-dspy"]
|
|
34
|
+
dev = ["pytest>=7"]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://costzera.com"
|
|
38
|
+
Documentation = "https://costzera.com/docs#sdk"
|
|
39
|
+
Repository = "https://github.com/costzera-ai/costzera-sdk"
|
|
40
|
+
Issues = "https://github.com/costzera-ai/costzera-sdk/issues"
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.wheel]
|
|
43
|
+
packages = ["src/costzera"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Does the Anthropic instrumentor report cached tokens? Two calls sharing a
|
|
2
|
+
cached ~6k-token system prompt; the second should read from the cache."""
|
|
3
|
+
import gzip, json, os, sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
for line in Path(sys.argv[1]).read_text(encoding="utf-8").splitlines():
|
|
6
|
+
if line.startswith("ANTHROPIC_API_KEY="):
|
|
7
|
+
os.environ["ANTHROPIC_API_KEY"] = line.split("=", 1)[1].strip().strip('"')
|
|
8
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
|
9
|
+
import costzera, anthropic
|
|
10
|
+
sent = []
|
|
11
|
+
costzera.init(agent="cache-check", api_key="czk_local_only",
|
|
12
|
+
_transport=lambda b: (sent.append(json.loads(gzip.decompress(b))), (202, {}))[1])
|
|
13
|
+
system = [{"type": "text", "text": "Reference notes. " + " ".join(f"Fact {i}: item {i} ships in {i % 7 + 1} days." for i in range(900)),
|
|
14
|
+
"cache_control": {"type": "ephemeral"}}]
|
|
15
|
+
client = anthropic.Anthropic()
|
|
16
|
+
usage = []
|
|
17
|
+
with costzera.run("cache_run"):
|
|
18
|
+
for q in ("How long does item 5 take?", "How long does item 12 take?"):
|
|
19
|
+
r = client.messages.create(model="claude-opus-5-5", max_tokens=1024, output_config={"effort": "low"},
|
|
20
|
+
system=system, messages=[{"role": "user", "content": q}])
|
|
21
|
+
usage.append({k: getattr(r.usage, k, None) for k in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens")})
|
|
22
|
+
costzera.flush(10)
|
|
23
|
+
print("API usage:", usage)
|
|
24
|
+
for s in sent[0]["traces"][0]["spans"]:
|
|
25
|
+
if s["span_kind"] == "LLM":
|
|
26
|
+
print("span tokens:", {k: v for k, v in s["attributes"].items() if k.startswith("llm.token_count")})
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Check what the real OpenInference instrumentors emit, through the SDK.
|
|
2
|
+
|
|
3
|
+
Makes a few live Claude calls (Anthropic SDK, then LangChain), each a small
|
|
4
|
+
tool-use exchange, with the Costzera SDK capturing what it *would* send --
|
|
5
|
+
nothing goes to Costzera. Then checks:
|
|
6
|
+
|
|
7
|
+
- the secret marker in the prompt is in no payload;
|
|
8
|
+
- every LLM span carries model and token counts;
|
|
9
|
+
- tool spans exist and are named;
|
|
10
|
+
- each run is one complete trace.
|
|
11
|
+
|
|
12
|
+
Writes the payloads to tests/fixtures/real_*.json for the server contract test,
|
|
13
|
+
and prints the attribute names each instrumentor emitted (names only).
|
|
14
|
+
|
|
15
|
+
python scripts/verify_instrumentors.py path/to/.env # needs ANTHROPIC_API_KEY
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import gzip
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from opentelemetry.sdk.trace import SpanProcessor
|
|
27
|
+
|
|
28
|
+
MODEL = "claude-opus-5-5"
|
|
29
|
+
SECRET = "ZEBRA-7741-account-code"
|
|
30
|
+
HERE = Path(__file__).resolve().parent
|
|
31
|
+
FIXTURES = HERE.parent / "tests" / "fixtures"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_key(env_path: str) -> None:
|
|
35
|
+
for line in Path(env_path).read_text(encoding="utf-8").splitlines():
|
|
36
|
+
if line.startswith("ANTHROPIC_API_KEY="):
|
|
37
|
+
os.environ["ANTHROPIC_API_KEY"] = line.split("=", 1)[1].strip().strip('"')
|
|
38
|
+
return
|
|
39
|
+
sys.exit("ANTHROPIC_API_KEY not found in the env file")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class RawKeys(SpanProcessor):
|
|
43
|
+
"""Attribute names per span, as the instrumentor emitted them."""
|
|
44
|
+
|
|
45
|
+
def __init__(self):
|
|
46
|
+
self.spans = []
|
|
47
|
+
|
|
48
|
+
def on_end(self, span):
|
|
49
|
+
self.spans.append((span.name, (span.attributes or {}).get("openinference.span.kind"),
|
|
50
|
+
sorted((span.attributes or {}).keys())))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def main(env_path: str) -> int:
|
|
54
|
+
load_key(env_path)
|
|
55
|
+
sys.path.insert(0, str(HERE.parent / "src"))
|
|
56
|
+
import costzera
|
|
57
|
+
|
|
58
|
+
sent: list = []
|
|
59
|
+
|
|
60
|
+
def transport(body: bytes):
|
|
61
|
+
sent.append(json.loads(gzip.decompress(body)))
|
|
62
|
+
return 202, {"data": {"rejected": {"over_cap": 0}}}
|
|
63
|
+
|
|
64
|
+
assert costzera.init(agent="verify-agent", api_key="czk_local_only", _transport=transport)
|
|
65
|
+
raw = RawKeys()
|
|
66
|
+
costzera._provider.add_span_processor(raw)
|
|
67
|
+
|
|
68
|
+
question = f"My account code is {SECRET}. What is the status of order A-1001? Use the tool."
|
|
69
|
+
tool_schema = {
|
|
70
|
+
"name": "get_order_status",
|
|
71
|
+
"description": "Look up an order's shipping status by id.",
|
|
72
|
+
"input_schema": {
|
|
73
|
+
"type": "object",
|
|
74
|
+
"properties": {"order_id": {"type": "string"}},
|
|
75
|
+
"required": ["order_id"],
|
|
76
|
+
},
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
# ── 1. Anthropic SDK, a manual tool loop ─────────────────────────────
|
|
80
|
+
import anthropic
|
|
81
|
+
|
|
82
|
+
client = anthropic.Anthropic()
|
|
83
|
+
|
|
84
|
+
@costzera.tool
|
|
85
|
+
def get_order_status(order_id: str) -> dict:
|
|
86
|
+
return {"order_id": order_id, "status": "shipped", "note": SECRET}
|
|
87
|
+
|
|
88
|
+
with costzera.run("anthropic_sdk_run"):
|
|
89
|
+
messages = [{"role": "user", "content": question}]
|
|
90
|
+
first = client.messages.create(
|
|
91
|
+
model=MODEL, max_tokens=2048, output_config={"effort": "low"},
|
|
92
|
+
tools=[tool_schema], messages=messages,
|
|
93
|
+
)
|
|
94
|
+
messages.append({"role": "assistant", "content": first.content})
|
|
95
|
+
results = []
|
|
96
|
+
for block in first.content:
|
|
97
|
+
if block.type == "tool_use":
|
|
98
|
+
out = get_order_status(**block.input)
|
|
99
|
+
results.append({"type": "tool_result", "tool_use_id": block.id, "content": json.dumps(out)})
|
|
100
|
+
if results:
|
|
101
|
+
messages.append({"role": "user", "content": results})
|
|
102
|
+
client.messages.create(
|
|
103
|
+
model=MODEL, max_tokens=2048, output_config={"effort": "low"},
|
|
104
|
+
tools=[tool_schema], messages=messages,
|
|
105
|
+
)
|
|
106
|
+
costzera.flush(10)
|
|
107
|
+
anthropic_batches = list(sent)
|
|
108
|
+
sent.clear()
|
|
109
|
+
|
|
110
|
+
# ── 2. LangChain ─────────────────────────────────────────────────────
|
|
111
|
+
from langchain_anthropic import ChatAnthropic
|
|
112
|
+
from langchain_core.messages import HumanMessage, ToolMessage
|
|
113
|
+
from langchain_core.tools import tool as lc_tool
|
|
114
|
+
|
|
115
|
+
@lc_tool
|
|
116
|
+
def lookup_order(order_id: str) -> str:
|
|
117
|
+
"""Look up an order's shipping status by id."""
|
|
118
|
+
return json.dumps({"order_id": order_id, "status": "shipped", "note": SECRET})
|
|
119
|
+
|
|
120
|
+
llm = ChatAnthropic(model=MODEL, max_tokens=2048).bind_tools([lookup_order])
|
|
121
|
+
with costzera.run("langchain_run"):
|
|
122
|
+
msgs = [HumanMessage(question)]
|
|
123
|
+
ai = llm.invoke(msgs)
|
|
124
|
+
msgs.append(ai)
|
|
125
|
+
for call in ai.tool_calls:
|
|
126
|
+
msgs.append(ToolMessage(lookup_order.invoke(call["args"]), tool_call_id=call["id"]))
|
|
127
|
+
if ai.tool_calls:
|
|
128
|
+
llm.invoke(msgs)
|
|
129
|
+
costzera.flush(10)
|
|
130
|
+
langchain_batches = list(sent)
|
|
131
|
+
|
|
132
|
+
# ── checks ───────────────────────────────────────────────────────────
|
|
133
|
+
FIXTURES.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
ok = True
|
|
135
|
+
for label, batches in (("anthropic", anthropic_batches), ("langchain", langchain_batches)):
|
|
136
|
+
print(f"\n=== {label}: {len(batches)} request(s)")
|
|
137
|
+
if not batches:
|
|
138
|
+
print(" NOTHING SENT"); ok = False; continue
|
|
139
|
+
payload = batches[0] if len(batches) == 1 else {**batches[0], "traces": [t for b in batches for t in b["traces"]]}
|
|
140
|
+
(FIXTURES / f"real_{label}.json").write_text(json.dumps(payload, indent=2))
|
|
141
|
+
text = json.dumps(batches)
|
|
142
|
+
leaked = SECRET in text
|
|
143
|
+
print(f" traces: {len(payload['traces'])} secret in payload: {leaked}")
|
|
144
|
+
ok &= not leaked and len(payload["traces"]) == 1
|
|
145
|
+
for tr in payload["traces"]:
|
|
146
|
+
for s in tr["spans"]:
|
|
147
|
+
a, m = s["attributes"], s.get("measured", {})
|
|
148
|
+
if s["span_kind"] == "LLM":
|
|
149
|
+
has = all(k in a for k in ("llm.token_count.prompt", "llm.token_count.completion")) and (
|
|
150
|
+
"llm.model_name" in a)
|
|
151
|
+
ok &= has
|
|
152
|
+
print(f" LLM {s['name']:<28} model={a.get('llm.model_name')} in={a.get('llm.token_count.prompt')} "
|
|
153
|
+
f"out={a.get('llm.token_count.completion')} cache_read={a.get('llm.token_count.prompt_details.cache_read')} "
|
|
154
|
+
f"finish={m.get('finish_reason')} in_chars={m.get('input_chars')} {'OK' if has else 'MISSING FIELDS'}")
|
|
155
|
+
else:
|
|
156
|
+
print(f" {s['span_kind']:<4} {s['name']:<28} parent={'root' if not s['parent_id'] else 'child'} "
|
|
157
|
+
f"tool={a.get('tool.name')} hash={str(m.get('input_hash'))[:8]} items={m.get('output_items')}")
|
|
158
|
+
|
|
159
|
+
print("\n=== attribute names emitted (raw, before conversion)")
|
|
160
|
+
seen = {}
|
|
161
|
+
for name, kind, keys in raw.spans:
|
|
162
|
+
for k in keys:
|
|
163
|
+
# Collapse indexed keys so the list stays readable.
|
|
164
|
+
import re
|
|
165
|
+
seen.setdefault(kind or "-", set()).add(re.sub(r"\.\d+\.", ".N.", k))
|
|
166
|
+
for kind, keys in sorted(seen.items()):
|
|
167
|
+
print(f" {kind}: {', '.join(sorted(keys))}")
|
|
168
|
+
|
|
169
|
+
print("\nRESULT:", "PASS" if ok else "FAIL")
|
|
170
|
+
return 0 if ok else 1
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else ".env"))
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Costzera: find and fix what makes AI agents slow and expensive.
|
|
2
|
+
|
|
3
|
+
import costzera
|
|
4
|
+
costzera.init(agent="support-agent") # reads COSTZERA_API_KEY
|
|
5
|
+
|
|
6
|
+
That is the whole setup for agents built on a supported framework: their model,
|
|
7
|
+
tool and retrieval calls are traced automatically. For hand-written agents:
|
|
8
|
+
|
|
9
|
+
@costzera.tool
|
|
10
|
+
def fetch_order(order_id): ...
|
|
11
|
+
|
|
12
|
+
with costzera.run("handle_ticket"): # one trace per run of the agent
|
|
13
|
+
with costzera.step("plan"): # a named stage within it
|
|
14
|
+
...
|
|
15
|
+
|
|
16
|
+
Prompts, completions, tool arguments and results never leave your process.
|
|
17
|
+
Costzera receives their sizes and fingerprints, the model, token counts,
|
|
18
|
+
timings and errors -- what it needs to find waste -- and nothing else. See
|
|
19
|
+
`costzera._wire` for exactly what is sent.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import atexit
|
|
25
|
+
import functools
|
|
26
|
+
import inspect
|
|
27
|
+
import json
|
|
28
|
+
import logging
|
|
29
|
+
import os
|
|
30
|
+
import threading
|
|
31
|
+
from contextlib import contextmanager
|
|
32
|
+
from typing import Any, Callable, Iterator, Optional, TypeVar
|
|
33
|
+
|
|
34
|
+
from opentelemetry import trace as otel_trace
|
|
35
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
36
|
+
from opentelemetry.trace import Status, StatusCode
|
|
37
|
+
|
|
38
|
+
from ._instrument import instrument_installed
|
|
39
|
+
from ._pipeline import Settings, TraceExporter, Transport
|
|
40
|
+
|
|
41
|
+
__version__ = "0.1.0"
|
|
42
|
+
__all__ = ["init", "run", "step", "tool", "flush", "shutdown", "__version__"]
|
|
43
|
+
|
|
44
|
+
DEFAULT_ENDPOINT = "https://costzera.com/api/v1/traces"
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger("costzera")
|
|
47
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
48
|
+
|
|
49
|
+
_lock = threading.Lock()
|
|
50
|
+
_exporter: Optional[TraceExporter] = None
|
|
51
|
+
_provider: Optional[TracerProvider] = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def init(
|
|
55
|
+
agent: Optional[str] = None,
|
|
56
|
+
*,
|
|
57
|
+
api_key: Optional[str] = None,
|
|
58
|
+
endpoint: Optional[str] = None,
|
|
59
|
+
sample_rate: float = 1.0,
|
|
60
|
+
instrument: bool = True,
|
|
61
|
+
error_messages: bool = False,
|
|
62
|
+
_transport: Optional[Transport] = None,
|
|
63
|
+
) -> bool:
|
|
64
|
+
"""Start sending traces. Safe to call more than once; later calls do nothing.
|
|
65
|
+
|
|
66
|
+
agent: the name this agent appears under in Costzera. Defaults to
|
|
67
|
+
COSTZERA_AGENT.
|
|
68
|
+
api_key: a Costzera key with the Send traces scope. Defaults to
|
|
69
|
+
COSTZERA_API_KEY.
|
|
70
|
+
sample_rate: the share of successful traces to keep, 0-1. Traces with an
|
|
71
|
+
error are always kept.
|
|
72
|
+
instrument: trace installed frameworks automatically.
|
|
73
|
+
error_messages: also send exception messages, cut to 500 characters. Off by
|
|
74
|
+
default, because a message often quotes the input that
|
|
75
|
+
caused it.
|
|
76
|
+
|
|
77
|
+
Returns True when tracing started. Without a key it logs once and does
|
|
78
|
+
nothing, so an unconfigured environment behaves exactly as before.
|
|
79
|
+
"""
|
|
80
|
+
global _exporter, _provider
|
|
81
|
+
with _lock:
|
|
82
|
+
if _exporter is not None:
|
|
83
|
+
return True
|
|
84
|
+
key = api_key or os.environ.get("COSTZERA_API_KEY")
|
|
85
|
+
name = (agent or os.environ.get("COSTZERA_AGENT") or "").strip()
|
|
86
|
+
if not key:
|
|
87
|
+
logger.warning("Costzera is not tracing: set COSTZERA_API_KEY or pass api_key.")
|
|
88
|
+
return False
|
|
89
|
+
if not name:
|
|
90
|
+
logger.warning("Costzera is not tracing: pass agent= or set COSTZERA_AGENT.")
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
settings = Settings(
|
|
94
|
+
api_key=key,
|
|
95
|
+
agent=name[:100],
|
|
96
|
+
endpoint=endpoint or os.environ.get("COSTZERA_ENDPOINT") or DEFAULT_ENDPOINT,
|
|
97
|
+
sdk_version=__version__,
|
|
98
|
+
sample_rate=min(1.0, max(0.0, float(sample_rate))),
|
|
99
|
+
error_messages=error_messages,
|
|
100
|
+
)
|
|
101
|
+
_exporter = TraceExporter(settings, transport=_transport)
|
|
102
|
+
|
|
103
|
+
# Join an OpenTelemetry setup the application already has rather than
|
|
104
|
+
# replacing it: its own exporters keep working, and ours is added.
|
|
105
|
+
current = otel_trace.get_tracer_provider()
|
|
106
|
+
if isinstance(current, TracerProvider):
|
|
107
|
+
_provider = current
|
|
108
|
+
else:
|
|
109
|
+
_provider = TracerProvider()
|
|
110
|
+
otel_trace.set_tracer_provider(_provider)
|
|
111
|
+
_provider.add_span_processor(_exporter)
|
|
112
|
+
|
|
113
|
+
if instrument:
|
|
114
|
+
instrument_installed(_provider)
|
|
115
|
+
atexit.register(shutdown)
|
|
116
|
+
return True
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _tracer():
|
|
120
|
+
return (_provider or otel_trace.get_tracer_provider()).get_tracer("costzera", __version__)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _safe_json(value: Any) -> str:
|
|
124
|
+
try:
|
|
125
|
+
return json.dumps(value, default=str, sort_keys=True)
|
|
126
|
+
except Exception:
|
|
127
|
+
return repr(value)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@contextmanager
|
|
131
|
+
def _span(name: str, kind: str, attributes: Optional[dict] = None) -> Iterator[Any]:
|
|
132
|
+
attrs = {"openinference.span.kind": kind, **(attributes or {})}
|
|
133
|
+
with _tracer().start_as_current_span(name, attributes=attrs, record_exception=False) as span:
|
|
134
|
+
try:
|
|
135
|
+
yield span
|
|
136
|
+
except BaseException as exc:
|
|
137
|
+
span.record_exception(exc)
|
|
138
|
+
span.set_status(Status(StatusCode.ERROR))
|
|
139
|
+
raise
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@contextmanager
|
|
143
|
+
def run(name: str = "run") -> Iterator[None]:
|
|
144
|
+
"""One run of the agent: everything inside is one trace."""
|
|
145
|
+
with _span(name, "AGENT"):
|
|
146
|
+
yield
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@contextmanager
|
|
150
|
+
def step(name: str) -> Iterator[None]:
|
|
151
|
+
"""A named stage of a run, so cost and latency can be attributed to it."""
|
|
152
|
+
with _span(name, "CHAIN", {"metadata": json.dumps({"pipeline_step": name})}):
|
|
153
|
+
yield
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def tool(fn: Optional[F] = None, *, name: Optional[str] = None) -> Any:
|
|
157
|
+
"""Trace a function as a tool call. Works on sync and async functions.
|
|
158
|
+
|
|
159
|
+
Arguments and the return value are recorded only so their size and
|
|
160
|
+
fingerprint can be measured; they are not sent.
|
|
161
|
+
"""
|
|
162
|
+
def decorate(func: F) -> F:
|
|
163
|
+
tool_name = name or func.__name__
|
|
164
|
+
|
|
165
|
+
def attrs(args: tuple, kwargs: dict) -> dict:
|
|
166
|
+
return {"tool.name": tool_name, "input.value": _safe_json({"args": args, "kwargs": kwargs})}
|
|
167
|
+
|
|
168
|
+
if inspect.iscoroutinefunction(func):
|
|
169
|
+
@functools.wraps(func)
|
|
170
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
171
|
+
with _span(tool_name, "TOOL", attrs(args, kwargs)) as span:
|
|
172
|
+
result = await func(*args, **kwargs)
|
|
173
|
+
span.set_attribute("output.value", _safe_json(result))
|
|
174
|
+
return result
|
|
175
|
+
return async_wrapper # type: ignore[return-value]
|
|
176
|
+
|
|
177
|
+
@functools.wraps(func)
|
|
178
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
179
|
+
with _span(tool_name, "TOOL", attrs(args, kwargs)) as span:
|
|
180
|
+
result = func(*args, **kwargs)
|
|
181
|
+
span.set_attribute("output.value", _safe_json(result))
|
|
182
|
+
return result
|
|
183
|
+
return wrapper # type: ignore[return-value]
|
|
184
|
+
|
|
185
|
+
return decorate(fn) if fn is not None else decorate
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def flush(timeout_s: float = 3.0) -> bool:
|
|
189
|
+
"""Send everything finished so far. For short scripts and serverless."""
|
|
190
|
+
return _exporter.force_flush(int(timeout_s * 1000)) if _exporter else True
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def shutdown() -> None:
|
|
194
|
+
"""Flush and stop. Registered at exit; call it yourself only to stop early."""
|
|
195
|
+
global _exporter
|
|
196
|
+
with _lock:
|
|
197
|
+
exporter, _exporter = _exporter, None
|
|
198
|
+
if exporter is not None:
|
|
199
|
+
try:
|
|
200
|
+
exporter.shutdown()
|
|
201
|
+
except Exception:
|
|
202
|
+
pass
|