softprobe 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- softprobe-0.1.2/.github/workflows/check-agent-skill-placement.yml +40 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/.gitignore +2 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/PKG-INFO +34 -3
- softprobe-0.1.2/README.md +71 -0
- softprobe-0.1.2/examples/README.md +41 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/examples/basic.py +13 -16
- {softprobe-0.1.0 → softprobe-0.1.2}/examples/langchain_agent.py +3 -5
- softprobe-0.1.2/examples/phase2-py/main.py +161 -0
- softprobe-0.1.2/examples/phase3-live-py/main.py +73 -0
- softprobe-0.1.2/examples/tool-call-py/main.py +153 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/pyproject.toml +1 -1
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/__init__.py +14 -2
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/attributes.py +2 -1
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/client.py +98 -25
- softprobe-0.1.2/src/softprobe/config.py +261 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/identity.py +2 -1
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/langchain.py +114 -55
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/langchain_instrument.py +11 -4
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/normalize.py +1 -3
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/observation.py +4 -3
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/openai.py +5 -4
- softprobe-0.1.2/src/softprobe/propagation.py +308 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/redaction.py +1 -1
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/scores.py +4 -4
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/tools.py +2 -1
- {softprobe-0.1.0 → softprobe-0.1.2}/src/softprobe/types.py +3 -2
- softprobe-0.1.2/tests/test_agent_skill_placement.py +21 -0
- softprobe-0.1.2/tests/test_baggage_propagation.py +123 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/tests/test_client.py +65 -25
- softprobe-0.1.2/tests/test_config.py +304 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/tests/test_langchain.py +238 -8
- {softprobe-0.1.0 → softprobe-0.1.2}/tests/test_openai.py +1 -1
- softprobe-0.1.2/tests/test_scores.py +161 -0
- softprobe-0.1.0/README.md +0 -40
- softprobe-0.1.0/examples/README.md +0 -29
- softprobe-0.1.0/src/softprobe/config.py +0 -70
- softprobe-0.1.0/src/softprobe/propagation.py +0 -52
- softprobe-0.1.0/tests/test_config.py +0 -57
- {softprobe-0.1.0 → softprobe-0.1.2}/.github/workflows/ci.yml +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/.github/workflows/release.yml +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/LICENSE +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/contracts/fixtures/expected-nested-spans.json +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/contracts/fixtures/expected-scores.json +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/contracts/fixtures/privacy-redaction.json +0 -0
- {softprobe-0.1.0 → softprobe-0.1.2}/tests/test_tools.py +0 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Agent skill placement
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request_target:
|
|
5
|
+
|
|
6
|
+
permissions:
|
|
7
|
+
pull-requests: read
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
check:
|
|
11
|
+
name: Agent skill placement
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/github-script@v9
|
|
15
|
+
with:
|
|
16
|
+
script: |
|
|
17
|
+
const { owner, repo } = context.repo;
|
|
18
|
+
const pull_number = context.payload.pull_request.number;
|
|
19
|
+
const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number });
|
|
20
|
+
const files = await github.paginate(github.rest.pulls.listFiles, {
|
|
21
|
+
owner,
|
|
22
|
+
repo,
|
|
23
|
+
pull_number,
|
|
24
|
+
per_page: 100,
|
|
25
|
+
});
|
|
26
|
+
if (pull.changed_files > files.length) {
|
|
27
|
+
core.setFailed(`GitHub returned ${files.length} of ${pull.changed_files} changed files. Cannot verify every path.`);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const violations = files
|
|
32
|
+
.filter(file => file.status !== 'removed')
|
|
33
|
+
.filter(file => /(^|\/)\.(?:agents|cursor|agent)\/skills(?:\/|$)/.test(file.filename))
|
|
34
|
+
.map(file => file.filename);
|
|
35
|
+
if (violations.length > 0) {
|
|
36
|
+
core.error('Keep developer skills in the softprobe/dev workspace root .agents/skills/.');
|
|
37
|
+
core.setFailed(`This repository adds or changes prohibited skill paths:\n${violations.join('\n')}`);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
core.info('The pull request adds no skills under repository-local agent skill directories.');
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: softprobe
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Softprobe LLM observability SDK for Python
|
|
5
5
|
Project-URL: Homepage, https://docs.softprobe.ai/en/agent-qa/
|
|
6
6
|
Project-URL: Documentation, https://docs.softprobe.ai/en/agent-qa/langchain
|
|
@@ -50,11 +50,42 @@ Docs: [Agent QA → LangChain](https://docs.softprobe.ai/en/agent-qa/langchain)
|
|
|
50
50
|
See [`examples/README.md`](examples/README.md).
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
export
|
|
54
|
-
export
|
|
53
|
+
export INGESTION_KEY="<Ingestion Key>"
|
|
54
|
+
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://explorer-staging.softprobe.ai/api/thelake/v1/traces"
|
|
55
|
+
export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer ${INGESTION_KEY}"
|
|
56
|
+
export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf"
|
|
57
|
+
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment.name=staging"
|
|
55
58
|
python examples/basic.py
|
|
56
59
|
```
|
|
57
60
|
|
|
61
|
+
The Ingestion Key is the bearer value in the standard OTLP headers. The full
|
|
62
|
+
trace endpoint selects where traces go. Set `SOFTPROBE_BASE_URL` only when
|
|
63
|
+
explicit score writes should use a non-production API; it does not affect trace
|
|
64
|
+
export. Without an OTLP endpoint, the SDK uses the production trace URL. You can
|
|
65
|
+
pass the trace settings directly:
|
|
66
|
+
|
|
67
|
+
Without a custom endpoint, the SDK exports traces to
|
|
68
|
+
`https://explorer.softprobe.ai/api/thelake/v1/traces`.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
import os
|
|
72
|
+
|
|
73
|
+
from softprobe import SoftprobeClient
|
|
74
|
+
|
|
75
|
+
client = SoftprobeClient(
|
|
76
|
+
ingestion_key=os.environ["INGESTION_KEY"],
|
|
77
|
+
otlp={
|
|
78
|
+
"traces_endpoint": "https://explorer-staging.softprobe.ai/api/thelake/v1/traces",
|
|
79
|
+
"protocol": "http/protobuf",
|
|
80
|
+
},
|
|
81
|
+
resource_attributes={"deployment.environment.name": "staging"},
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`base_url` is optional here; it defaults to the production score API. Set it
|
|
86
|
+
separately if score writes should use staging. It never changes the trace
|
|
87
|
+
destination. Without an OTLP endpoint, traces use the production trace URL.
|
|
88
|
+
|
|
58
89
|
## Develop
|
|
59
90
|
|
|
60
91
|
```bash
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# softprobe (Python)
|
|
2
|
+
|
|
3
|
+
Softprobe **Agent QA** Python SDK — LLM / agent observability via OTLP to Softprobe Explorer.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install softprobe
|
|
7
|
+
# LangChain:
|
|
8
|
+
pip install 'softprobe[langchain]'
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Docs: [Agent QA → LangChain](https://docs.softprobe.ai/en/agent-qa/langchain) · [Quick start](https://docs.softprobe.ai/en/agent-qa/getting-started)
|
|
12
|
+
|
|
13
|
+
## Examples
|
|
14
|
+
|
|
15
|
+
| Script | Path |
|
|
16
|
+
|--------|------|
|
|
17
|
+
| Basic `from_env()` | [`examples/basic.py`](examples/basic.py) |
|
|
18
|
+
| LangChain best path | [`examples/langchain_agent.py`](examples/langchain_agent.py) |
|
|
19
|
+
|
|
20
|
+
See [`examples/README.md`](examples/README.md).
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
export INGESTION_KEY="<Ingestion Key>"
|
|
24
|
+
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://explorer-staging.softprobe.ai/api/thelake/v1/traces"
|
|
25
|
+
export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer ${INGESTION_KEY}"
|
|
26
|
+
export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf"
|
|
27
|
+
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment.name=staging"
|
|
28
|
+
python examples/basic.py
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The Ingestion Key is the bearer value in the standard OTLP headers. The full
|
|
32
|
+
trace endpoint selects where traces go. Set `SOFTPROBE_BASE_URL` only when
|
|
33
|
+
explicit score writes should use a non-production API; it does not affect trace
|
|
34
|
+
export. Without an OTLP endpoint, the SDK uses the production trace URL. You can
|
|
35
|
+
pass the trace settings directly:
|
|
36
|
+
|
|
37
|
+
Without a custom endpoint, the SDK exports traces to
|
|
38
|
+
`https://explorer.softprobe.ai/api/thelake/v1/traces`.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import os
|
|
42
|
+
|
|
43
|
+
from softprobe import SoftprobeClient
|
|
44
|
+
|
|
45
|
+
client = SoftprobeClient(
|
|
46
|
+
ingestion_key=os.environ["INGESTION_KEY"],
|
|
47
|
+
otlp={
|
|
48
|
+
"traces_endpoint": "https://explorer-staging.softprobe.ai/api/thelake/v1/traces",
|
|
49
|
+
"protocol": "http/protobuf",
|
|
50
|
+
},
|
|
51
|
+
resource_attributes={"deployment.environment.name": "staging"},
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`base_url` is optional here; it defaults to the production score API. Set it
|
|
56
|
+
separately if score writes should use staging. It never changes the trace
|
|
57
|
+
destination. Without an OTLP endpoint, traces use the production trace URL.
|
|
58
|
+
|
|
59
|
+
## Develop
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
uv sync --extra langchain --extra openai --extra dev
|
|
63
|
+
uv run pytest -q
|
|
64
|
+
uv run mypy src/softprobe
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Publish
|
|
68
|
+
|
|
69
|
+
Create a GitHub Release with tag `vX.Y.Z`. The [release workflow](.github/workflows/release.yml) publishes to PyPI via Trusted Publishing (OIDC).
|
|
70
|
+
|
|
71
|
+
> **Note:** The older public repo [`softprobe/softprobe-python`](https://github.com/softprobe/softprobe-python) (Hybrid/replay SDK) is **archived**. This repo is the Agent QA / LLM package.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Softprobe Python examples
|
|
2
|
+
|
|
3
|
+
Docs-faithful samples for [`softprobe`](https://pypi.org/project/softprobe/).
|
|
4
|
+
|
|
5
|
+
Install steps match [Agent QA → LangChain](https://docs.softprobe.ai/en/agent-qa/langchain).
|
|
6
|
+
|
|
7
|
+
| File | What it shows |
|
|
8
|
+
|------|----------------|
|
|
9
|
+
| [`basic.py`](./basic.py) | `SoftprobeClient.from_env()` — agent + generation |
|
|
10
|
+
| [`langchain_agent.py`](./langchain_agent.py) | Env auto / `instrument()` + app `thread_id` |
|
|
11
|
+
| [`phase2-py/`](./phase2-py/) | Nested agent/session propagation and score writes |
|
|
12
|
+
| [`phase3-live-py/`](./phase3-live-py/) | Live OpenAI generation capture |
|
|
13
|
+
| [`tool-call-py/`](./tool-call-py/) | Agent → generation → correlated tool calls |
|
|
14
|
+
|
|
15
|
+
## Credentials
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
export INGESTION_KEY="<Ingestion Key>"
|
|
19
|
+
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://explorer-staging.softprobe.ai/api/thelake/v1/traces"
|
|
20
|
+
export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer ${INGESTION_KEY}"
|
|
21
|
+
export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf"
|
|
22
|
+
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment.name=staging"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Set `INGESTION_KEY` to the Ingestion Key from Explorer before running these examples. The exact OTLP traces endpoint selects the trace destination. Set `SOFTPROBE_BASE_URL` only for examples that write scores to a non-production API. For a local thelake started with `SOFTPROBE_LOCAL_ANONYMOUS=1`, set both `SOFTPROBE_BASE_URL=http://127.0.0.1:8090` and `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:8090/v1/traces`; thelake ignores credentials in that explicit mode.
|
|
26
|
+
|
|
27
|
+
## Run
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
uv sync --extra langchain --extra openai
|
|
31
|
+
uv run python examples/basic.py
|
|
32
|
+
|
|
33
|
+
# LangChain agent (needs GEMINI_KEY / GOOGLE_API_KEY or OPENAI_API_KEY):
|
|
34
|
+
uv pip install 'langgraph>=0.2' 'langchain-google-genai>=2' 'langchain-openai>=0.2'
|
|
35
|
+
uv run python examples/langchain_agent.py
|
|
36
|
+
|
|
37
|
+
# Integration samples (start thelake first; optional API keys vary by sample):
|
|
38
|
+
uv run python examples/phase2-py/main.py
|
|
39
|
+
uv run --extra openai python examples/phase3-live-py/main.py
|
|
40
|
+
uv run python examples/tool-call-py/main.py
|
|
41
|
+
```
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"""Softprobe basic example — SoftprobeClient.from_env().
|
|
2
2
|
|
|
3
|
-
Requires
|
|
4
|
-
|
|
5
|
-
export SOFTPROBE_PUBLIC_KEY=…
|
|
6
|
-
export SOFTPROBE_BASE_URL=…
|
|
3
|
+
Requires the standard OTLP endpoint and authorization-header environment
|
|
4
|
+
variables shown in `examples/README.md`.
|
|
7
5
|
"""
|
|
8
6
|
|
|
9
7
|
from __future__ import annotations
|
|
@@ -23,18 +21,17 @@ def main() -> None:
|
|
|
23
21
|
as_type="agent",
|
|
24
22
|
session_id=session_id,
|
|
25
23
|
input={"goal": "say hello"},
|
|
26
|
-
) as agent
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
pass
|
|
24
|
+
) as agent, client.generation(
|
|
25
|
+
name="example.generation",
|
|
26
|
+
parent=agent,
|
|
27
|
+
session_id=session_id,
|
|
28
|
+
model="example-model",
|
|
29
|
+
provider="example",
|
|
30
|
+
input={"messages": [{"role": "user", "content": "hello"}]},
|
|
31
|
+
usage={"input_tokens": 4, "output_tokens": 6},
|
|
32
|
+
output={"content": "hello from Softprobe"},
|
|
33
|
+
):
|
|
34
|
+
pass
|
|
38
35
|
|
|
39
36
|
client.force_flush()
|
|
40
37
|
client.shutdown()
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"""Softprobe LangChain example — docs best path (zero invoke callbacks edits).
|
|
2
2
|
|
|
3
|
-
Requires
|
|
4
|
-
|
|
5
|
-
export SOFTPROBE_PUBLIC_KEY=…
|
|
6
|
-
export SOFTPROBE_BASE_URL=…
|
|
3
|
+
Requires the standard OTLP endpoint and authorization-header environment
|
|
4
|
+
variables shown in `examples/README.md`.
|
|
7
5
|
export GEMINI_KEY=… # or OPENAI_API_KEY
|
|
8
6
|
|
|
9
7
|
Softprobe reads configurable.thread_id — it does not mint session UUIDs.
|
|
@@ -15,7 +13,7 @@ import json
|
|
|
15
13
|
import os
|
|
16
14
|
import sys
|
|
17
15
|
|
|
18
|
-
# Best path: auto-instrument when
|
|
16
|
+
# Best path: auto-instrument when standard OTLP authorization is set.
|
|
19
17
|
import softprobe.langchain # noqa: F401
|
|
20
18
|
from softprobe.langchain import get_handler, instrument
|
|
21
19
|
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Phase 2 nested Softprobe Python sample app."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from softprobe import SoftprobeClient
|
|
11
|
+
|
|
12
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
13
|
+
OUT_PATH = Path(os.environ.get("PHASE2_IDS_PATH", ROOT / "examples" / ".phase2-ids-py.json"))
|
|
14
|
+
SESSION_ID = os.environ.get("PHASE2_SESSION_ID", "sess-phase2-py")
|
|
15
|
+
BASE_URL = os.environ.get("SOFTPROBE_BASE_URL", "http://127.0.0.1:8090")
|
|
16
|
+
ingestion_key = os.environ.get("INGESTION_KEY", "e2e-token")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def main() -> None:
|
|
20
|
+
client = SoftprobeClient.from_env(
|
|
21
|
+
ingestion_key=ingestion_key,
|
|
22
|
+
base_url=BASE_URL,
|
|
23
|
+
service_name="phase2-example-py",
|
|
24
|
+
service_version="0.1.0",
|
|
25
|
+
environment="e2e",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
trace_id = ""
|
|
29
|
+
agent_span_id = ""
|
|
30
|
+
generation_span_id = ""
|
|
31
|
+
|
|
32
|
+
with client.observation(
|
|
33
|
+
name="phase2.agent",
|
|
34
|
+
as_type="agent",
|
|
35
|
+
session_id=SESSION_ID,
|
|
36
|
+
user_id="user-phase2-py",
|
|
37
|
+
tags=["phase2", "python"],
|
|
38
|
+
input={"goal": "phase2"},
|
|
39
|
+
) as agent:
|
|
40
|
+
trace_id = agent.trace_id
|
|
41
|
+
agent_span_id = agent.span_id
|
|
42
|
+
|
|
43
|
+
with client.observation(
|
|
44
|
+
name="phase2.chain",
|
|
45
|
+
as_type="chain",
|
|
46
|
+
session_id=SESSION_ID,
|
|
47
|
+
):
|
|
48
|
+
with client.observation(
|
|
49
|
+
name="phase2.retriever",
|
|
50
|
+
as_type="retriever",
|
|
51
|
+
session_id=SESSION_ID,
|
|
52
|
+
input={"query": "docs"},
|
|
53
|
+
output={"docs": ["a"]},
|
|
54
|
+
):
|
|
55
|
+
client.start_embedding(
|
|
56
|
+
name="phase2.embedding",
|
|
57
|
+
session_id=SESSION_ID,
|
|
58
|
+
attributes={
|
|
59
|
+
"gen_ai.request.model": "text-embedding-3-small",
|
|
60
|
+
"gen_ai.usage.input_tokens": 8,
|
|
61
|
+
"gen_ai.usage.total_tokens": 8,
|
|
62
|
+
},
|
|
63
|
+
).end()
|
|
64
|
+
|
|
65
|
+
with client.observation(
|
|
66
|
+
name="phase2.tool",
|
|
67
|
+
as_type="tool",
|
|
68
|
+
session_id=SESSION_ID,
|
|
69
|
+
input={"name": "lookup"},
|
|
70
|
+
output={"ok": True},
|
|
71
|
+
) as tool:
|
|
72
|
+
tool.add_content_event(
|
|
73
|
+
"gen_ai.tool.message",
|
|
74
|
+
{"role": "tool", "name": "lookup", "content": "ok"},
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
with client.generation(
|
|
78
|
+
name="phase2.generation",
|
|
79
|
+
session_id=SESSION_ID,
|
|
80
|
+
model="gpt-4o-mini",
|
|
81
|
+
provider="openai",
|
|
82
|
+
operation_name="chat",
|
|
83
|
+
usage={"input_tokens": 40, "output_tokens": 10, "total_tokens": 50},
|
|
84
|
+
cost={"input": 0.0001, "output": 0.0002, "total": 0.0003},
|
|
85
|
+
input={"messages": [{"role": "user", "content": "hi"}]},
|
|
86
|
+
output={"content": "hello from py"},
|
|
87
|
+
prompt_event=[{"role": "user", "content": "hi"}],
|
|
88
|
+
completion_event=[{"role": "assistant", "content": "hello from py"}],
|
|
89
|
+
) as generation:
|
|
90
|
+
generation_span_id = generation.span_id
|
|
91
|
+
generation.update(
|
|
92
|
+
response_model="gpt-4o-mini-2024-07-18",
|
|
93
|
+
finish_reasons=["stop"],
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
client.start_evaluator(
|
|
97
|
+
name="phase2.evaluator",
|
|
98
|
+
session_id=SESSION_ID,
|
|
99
|
+
output={"score": 0.95},
|
|
100
|
+
).end()
|
|
101
|
+
client.start_guardrail(
|
|
102
|
+
name="phase2.guardrail",
|
|
103
|
+
session_id=SESSION_ID,
|
|
104
|
+
output={"passed": True},
|
|
105
|
+
).end()
|
|
106
|
+
client.start_observation(
|
|
107
|
+
name="phase2.span",
|
|
108
|
+
as_type="span",
|
|
109
|
+
session_id=SESSION_ID,
|
|
110
|
+
).end()
|
|
111
|
+
|
|
112
|
+
score_ids = {
|
|
113
|
+
"span": f"score-phase2-py-span-{generation_span_id}",
|
|
114
|
+
"trace": f"score-phase2-py-trace-{trace_id[:16]}",
|
|
115
|
+
"session": f"score-phase2-py-session-{SESSION_ID}",
|
|
116
|
+
}
|
|
117
|
+
client.create_score(
|
|
118
|
+
score_id=score_ids["span"],
|
|
119
|
+
name="faithfulness",
|
|
120
|
+
data_type="numeric",
|
|
121
|
+
source="evaluator",
|
|
122
|
+
numeric_value=0.91,
|
|
123
|
+
trace_id=trace_id,
|
|
124
|
+
span_id=generation_span_id,
|
|
125
|
+
)
|
|
126
|
+
client.create_score(
|
|
127
|
+
score_id=score_ids["trace"],
|
|
128
|
+
name="quality",
|
|
129
|
+
data_type="categorical",
|
|
130
|
+
source="api",
|
|
131
|
+
string_value="good",
|
|
132
|
+
trace_id=trace_id,
|
|
133
|
+
)
|
|
134
|
+
client.create_score(
|
|
135
|
+
score_id=score_ids["session"],
|
|
136
|
+
name="thumbs",
|
|
137
|
+
data_type="boolean",
|
|
138
|
+
source="user",
|
|
139
|
+
boolean_value=True,
|
|
140
|
+
session_id=SESSION_ID,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
client.force_flush()
|
|
144
|
+
client.shutdown()
|
|
145
|
+
|
|
146
|
+
payload = {
|
|
147
|
+
"language": "python",
|
|
148
|
+
"sessionId": SESSION_ID,
|
|
149
|
+
"traceId": trace_id,
|
|
150
|
+
"agentSpanId": agent_span_id,
|
|
151
|
+
"generationSpanId": generation_span_id,
|
|
152
|
+
"scoreIds": score_ids,
|
|
153
|
+
"serviceName": "phase2-example-py",
|
|
154
|
+
}
|
|
155
|
+
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
OUT_PATH.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
157
|
+
print(json.dumps(payload))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
if __name__ == "__main__":
|
|
161
|
+
main()
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Phase 3 live OpenAI/Gemini sample using softprobe.openai.observe_openai."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from openai import OpenAI
|
|
11
|
+
|
|
12
|
+
from softprobe import SoftprobeClient
|
|
13
|
+
from softprobe.openai import observe_openai
|
|
14
|
+
|
|
15
|
+
OUT_PATH = Path(os.environ["PHASE3_IDS_PATH"])
|
|
16
|
+
SESSION_ID = os.environ["PHASE3_SESSION_ID"]
|
|
17
|
+
PROVIDER = os.environ.get("PHASE3_PROVIDER", "openai")
|
|
18
|
+
MODEL = os.environ.get("PHASE3_MODEL", "gpt-4o-mini")
|
|
19
|
+
API_KEY = os.environ["PHASE3_API_KEY"]
|
|
20
|
+
BASE_URL = os.environ.get("PHASE3_BASE_URL")
|
|
21
|
+
BASE = os.environ.get("SOFTPROBE_BASE_URL", "http://127.0.0.1:8090")
|
|
22
|
+
ingestion_key = os.environ.get("INGESTION_KEY", "e2e-token")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main() -> None:
|
|
26
|
+
sp = SoftprobeClient.from_env(
|
|
27
|
+
ingestion_key=ingestion_key,
|
|
28
|
+
base_url=BASE,
|
|
29
|
+
service_name=f"phase3-live-py-{PROVIDER}",
|
|
30
|
+
service_version="0.1.0",
|
|
31
|
+
environment="e2e",
|
|
32
|
+
)
|
|
33
|
+
raw = OpenAI(api_key=API_KEY, base_url=BASE_URL) if BASE_URL else OpenAI(api_key=API_KEY)
|
|
34
|
+
client = observe_openai(
|
|
35
|
+
raw,
|
|
36
|
+
softprobe_client=sp,
|
|
37
|
+
session_id=SESSION_ID,
|
|
38
|
+
generation_name=f"phase3.{PROVIDER}.chat",
|
|
39
|
+
)
|
|
40
|
+
response = client.chat.completions.create(
|
|
41
|
+
model=MODEL,
|
|
42
|
+
messages=[
|
|
43
|
+
{"role": "system", "content": "Reply with a single digit only."},
|
|
44
|
+
{"role": "user", "content": "What is 1 + 1?"},
|
|
45
|
+
],
|
|
46
|
+
temperature=0,
|
|
47
|
+
name=f"phase3-{PROVIDER}-math",
|
|
48
|
+
session_id=SESSION_ID,
|
|
49
|
+
)
|
|
50
|
+
content = response.choices[0].message.content
|
|
51
|
+
span_id = client.last_generation_span_id
|
|
52
|
+
trace_id = client.last_generation_trace_id
|
|
53
|
+
sp.force_flush()
|
|
54
|
+
sp.shutdown()
|
|
55
|
+
|
|
56
|
+
payload = {
|
|
57
|
+
"language": "python",
|
|
58
|
+
"provider": PROVIDER,
|
|
59
|
+
"sessionId": SESSION_ID,
|
|
60
|
+
"traceId": trace_id,
|
|
61
|
+
"generationSpanId": span_id,
|
|
62
|
+
"model": MODEL,
|
|
63
|
+
"responseId": getattr(response, "id", None),
|
|
64
|
+
"content": content,
|
|
65
|
+
"serviceName": f"phase3-live-py-{PROVIDER}",
|
|
66
|
+
}
|
|
67
|
+
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
OUT_PATH.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
69
|
+
print(json.dumps(payload))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
if __name__ == "__main__":
|
|
73
|
+
main()
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Issue #7 Part A: agent → generation → tool with tool-call attributes."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from softprobe import SoftprobeClient
|
|
11
|
+
from softprobe.tools import record_tool_calls, record_tool_definitions
|
|
12
|
+
|
|
13
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
14
|
+
OUT_PATH = Path(
|
|
15
|
+
os.environ.get("TOOL_CALL_IDS_PATH", ROOT / "examples" / ".tool-call-ids-py.json")
|
|
16
|
+
)
|
|
17
|
+
SESSION_ID = os.environ.get("TOOL_CALL_SESSION_ID", "sess-tool-call-py")
|
|
18
|
+
BASE_URL = os.environ.get("SOFTPROBE_BASE_URL", "http://127.0.0.1:8090")
|
|
19
|
+
ingestion_key = os.environ.get("INGESTION_KEY", "e2e-token")
|
|
20
|
+
|
|
21
|
+
TOOLS = [
|
|
22
|
+
{
|
|
23
|
+
"type": "function",
|
|
24
|
+
"function": {
|
|
25
|
+
"name": "lookup",
|
|
26
|
+
"description": "Lookup a document",
|
|
27
|
+
"parameters": {
|
|
28
|
+
"type": "object",
|
|
29
|
+
"properties": {"q": {"type": "string"}},
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
]
|
|
34
|
+
TOOL_CALLS = [
|
|
35
|
+
{
|
|
36
|
+
"id": "call_lookup_1",
|
|
37
|
+
"type": "function",
|
|
38
|
+
"function": {"name": "lookup", "arguments": '{"q":"docs"}'},
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main() -> None:
|
|
44
|
+
client = SoftprobeClient.from_env(
|
|
45
|
+
ingestion_key=ingestion_key,
|
|
46
|
+
base_url=BASE_URL,
|
|
47
|
+
service_name="tool-call-example-py",
|
|
48
|
+
service_version="0.1.0",
|
|
49
|
+
environment="e2e",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
trace_id = ""
|
|
53
|
+
agent_span_id = ""
|
|
54
|
+
generation_span_id = ""
|
|
55
|
+
tool_span_id = ""
|
|
56
|
+
|
|
57
|
+
with client.observation(
|
|
58
|
+
name="toolcall.agent",
|
|
59
|
+
as_type="agent",
|
|
60
|
+
session_id=SESSION_ID,
|
|
61
|
+
user_id="user-tool-call-py",
|
|
62
|
+
tags=["tool-call", "python"],
|
|
63
|
+
input={"goal": "answer with tools"},
|
|
64
|
+
) as agent:
|
|
65
|
+
trace_id = agent.trace_id
|
|
66
|
+
agent_span_id = agent.span_id
|
|
67
|
+
|
|
68
|
+
with client.generation(
|
|
69
|
+
name="toolcall.generation",
|
|
70
|
+
session_id=SESSION_ID,
|
|
71
|
+
model="gpt-4o-mini",
|
|
72
|
+
provider="openai",
|
|
73
|
+
operation_name="chat",
|
|
74
|
+
usage={"input_tokens": 40, "output_tokens": 12, "total_tokens": 52},
|
|
75
|
+
input={
|
|
76
|
+
"messages": [{"role": "user", "content": "find docs"}],
|
|
77
|
+
"tools": TOOLS,
|
|
78
|
+
},
|
|
79
|
+
output={
|
|
80
|
+
"content": None,
|
|
81
|
+
"tool_calls": [
|
|
82
|
+
{
|
|
83
|
+
"id": "call_lookup_1",
|
|
84
|
+
"name": "lookup",
|
|
85
|
+
"arguments": '{"q":"docs"}',
|
|
86
|
+
}
|
|
87
|
+
],
|
|
88
|
+
},
|
|
89
|
+
prompt_event=[{"role": "user", "content": "find docs"}],
|
|
90
|
+
completion_event=[
|
|
91
|
+
{
|
|
92
|
+
"role": "assistant",
|
|
93
|
+
"content": None,
|
|
94
|
+
"tool_calls": [
|
|
95
|
+
{
|
|
96
|
+
"id": "call_lookup_1",
|
|
97
|
+
"name": "lookup",
|
|
98
|
+
"arguments": '{"q":"docs"}',
|
|
99
|
+
}
|
|
100
|
+
],
|
|
101
|
+
}
|
|
102
|
+
],
|
|
103
|
+
) as generation:
|
|
104
|
+
generation_span_id = generation.span_id
|
|
105
|
+
record_tool_definitions(generation, TOOLS)
|
|
106
|
+
record_tool_calls(generation, TOOL_CALLS)
|
|
107
|
+
generation.update(
|
|
108
|
+
response_model="gpt-4o-mini-2024-07-18",
|
|
109
|
+
finish_reasons=["tool_calls"],
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
tool = client.start_tool(
|
|
113
|
+
name="toolcall.lookup",
|
|
114
|
+
tool_name="lookup",
|
|
115
|
+
tool_call_id="call_lookup_1",
|
|
116
|
+
kind="function",
|
|
117
|
+
status="ok",
|
|
118
|
+
index=0,
|
|
119
|
+
parent=generation,
|
|
120
|
+
session_id=SESSION_ID,
|
|
121
|
+
input={"q": "docs"},
|
|
122
|
+
output={"ok": True, "hits": ["a"]},
|
|
123
|
+
)
|
|
124
|
+
tool_span_id = tool.span_id
|
|
125
|
+
tool.add_content_event(
|
|
126
|
+
"gen_ai.tool.message",
|
|
127
|
+
{
|
|
128
|
+
"role": "tool",
|
|
129
|
+
"name": "lookup",
|
|
130
|
+
"tool_call_id": "call_lookup_1",
|
|
131
|
+
"content": "ok",
|
|
132
|
+
},
|
|
133
|
+
)
|
|
134
|
+
tool.end()
|
|
135
|
+
|
|
136
|
+
payload = {
|
|
137
|
+
"language": "python",
|
|
138
|
+
"serviceName": "tool-call-example-py",
|
|
139
|
+
"sessionId": SESSION_ID,
|
|
140
|
+
"traceId": trace_id,
|
|
141
|
+
"agentSpanId": agent_span_id,
|
|
142
|
+
"generationSpanId": generation_span_id,
|
|
143
|
+
"toolSpanId": tool_span_id,
|
|
144
|
+
"toolCallId": "call_lookup_1",
|
|
145
|
+
}
|
|
146
|
+
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
OUT_PATH.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
148
|
+
print(json.dumps(payload))
|
|
149
|
+
client.shutdown()
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
if __name__ == "__main__":
|
|
153
|
+
main()
|