ddapm-test-agent 1.38.0__tar.gz → 1.40.0__tar.gz

ddapm_test_agent-1.40.0/CLAUDE.md

@@ -0,0 +1,304 @@
+# dd-apm-test-agent
+
+A test agent that emulates Datadog APM endpoints for testing client libraries. It receives traces, telemetry, metrics, and other data from tracers, providing endpoints for validation, snapshot testing, and local development.
+
+## Quick Reference
+
+```bash
+# Activate virtual environment
+source .venv/bin/activate
+
+# Run the test agent
+ddapm-test-agent --port=8126
+
+# Run with web UI
+ddapm-test-agent --port=8126 --web-ui-port=8080
+
+# Run all tests
+python -m pytest tests/ -v
+
+# Run specific test file
+python -m pytest tests/test_agent.py -v
+
+# Run a single test
+python -m pytest tests/test_agent.py::test_trace -v
+
+# Create a release note
+reno new <feature-name>
+
+# Format snapshots
+ddapm-test-agent-fmt path/to/snapshots
+
+# Lint snapshots (check mode)
+ddapm-test-agent-fmt --check path/to/snapshots
+```
+
+## Project Structure
+
+```
+ddapm_test_agent/           # Main package
+  agent.py                  # Core application, routes, middleware (~95KB - main entry point)
+  trace.py                  # Trace decoding and handling (v0.4, v0.5, v0.7 formats)
+  trace_snapshot.py         # Snapshot comparison logic
+  trace_checks.py           # Trace validation checks
+  checks.py                 # Check framework infrastructure
+  apmtelemetry.py           # APM telemetry event handling
+  tracestats.py             # Trace statistics handling
+  remoteconfig.py           # Remote configuration server
+  logs.py                   # OTLP logs handling
+  metrics.py                # OTLP metrics handling
+  vcr_proxy.py              # VCR cassette recording/playback for 3rd party APIs
+  llmobs_event_platform.py  # LLM Observability API endpoints
+  client.py                 # Test client utilities
+  fmt.py                    # Snapshot formatting CLI
+  cmd.py                    # CLI entry points
+
+tests/
+  conftest.py               # Pytest fixtures (agent, payloads, helpers)
+  test_agent.py             # Core agent endpoint tests
+  test_snapshot.py          # Snapshot functionality tests
+  test_trace.py             # Trace handling tests
+  test_session.py           # Session management tests
+  test_<module>.py          # Module-specific tests
+  trace_utils.py            # Test utilities for trace generation
+
+releasenotes/
+  notes/                    # Reno release notes (YAML)
+```
+
+## Code Style
+
+- **Formatter**: black (line length 120)
+- **Import sorting**: isort (single line, google style)
+- **Linting**: flake8
+- **Type checking**: mypy (strict settings enabled in setup.cfg)
+
+### Type Annotations
+
+All functions require complete type annotations. The mypy config enforces:
+- `disallow_incomplete_defs = true`
+- `disallow_untyped_decorators = true`
+- `warn_return_any = true`
+
+```python
+from typing import Any, Awaitable, Callable, Dict, List, Optional
+
+async def handle_request(self, request: Request) -> web.Response:
+    ...
+
+def decode_payload(data: bytes, content_type: str) -> List[Dict[str, Any]]:
+    ...
+```
+
+### Import Style
+
+Imports are single-line, alphabetically ordered within groups:
+
+```python
+# Standard library
+import asyncio
+import json
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+# Third party
+from aiohttp import web
+from aiohttp.web import Request
+import msgpack
+
+# Local
+from ddapm_test_agent.trace import Span
+from ddapm_test_agent.trace import Trace
+```
+
+## Testing
+
+Tests use pytest with aiohttp's pytest plugin. The `agent` fixture provides a test client.
+
+### Test Style
+
+- No docstrings in simple test functions (docstrings allowed for complex parametrized tests)
+- Concise assertions with helpful error messages
+- Use existing fixtures from `conftest.py`
+- Async tests for HTTP endpoints
+
+```python
+async def test_trace_put_v04(
+    agent,
+    v04_reference_http_trace_payload_headers,
+    v04_reference_http_trace_payload_data,
+):
+    resp = await agent.put(
+        "/v0.4/traces",
+        headers=v04_reference_http_trace_payload_headers,
+        data=v04_reference_http_trace_payload_data,
+    )
+    assert resp.status == 200, await resp.text()
+
+
+async def test_info(agent):
+    resp = await agent.get("/info")
+    assert resp.status == 200
+    data = await resp.json()
+    assert "version" in data
+    assert "endpoints" in data
+```
+
+### Key Fixtures (from conftest.py)
+
+- `agent` - aiohttp test client connected to the test agent
+- `v04_reference_http_trace_payload_data` - Sample v0.4 trace payload (msgpack)
+- `v04_reference_http_trace_payload_headers` - Headers for trace requests
+- `do_reference_v04_http_trace` - Helper function to send trace requests
+- `snapshot_dir` - Temporary directory for snapshot tests
+- `testagent` - Full subprocess test agent for integration tests
+
+### Running Tests
+
+```bash
+# All tests
+python -m pytest tests/ -v
+
+# Specific file
+python -m pytest tests/test_agent.py -v
+
+# Single test
+python -m pytest tests/test_snapshot.py::test_snapshot_single_trace -v
+
+# With coverage
+python -m pytest tests/ --cov=ddapm_test_agent
+```
+
+## Key Concepts
+
+### Session Tokens
+
+Requests can be associated with a test session using tokens:
+- Query param: `?test_session_token=my_test`
+- Header: `X-Datadog-Test-Session-Token: my_test`
+
+### Trace Formats
+
+The agent supports multiple trace formats:
+- v0.4: Standard msgpack format (most common)
+- v0.5: Optimized format with string interning
+- v0.7: Latest format with additional features
+- v1: Legacy format
+
+### Snapshot Testing
+
+Characterization testing for traces:
+1. Send traces to the agent
+2. Call `/test/session/snapshot` to generate/compare snapshots
+3. Snapshots are normalized JSON files
+
+### VCR Proxy
+
+Record and replay 3rd party API calls:
+- Endpoint: `/vcr/{provider}/...`
+- Supports: OpenAI, Azure OpenAI, Anthropic, AWS Bedrock, etc.
+- Cassettes stored in `vcr-cassettes/` directory
+
+## Adding New Endpoints
+
+1. Add handler method to appropriate module (or `agent.py` for core endpoints)
+2. Register route in `make_app()` or module's `get_routes()` method
+3. Add tests following existing patterns
+4. Create release note: `reno new <feature-name>`
+
+### Handler Pattern
+
+```python
+async def handle_my_endpoint(self, request: Request) -> web.Response:
+    token = _session_token(request)
+    body = await request.json()
+
+    # Process request...
+
+    return web.json_response({"status": "ok"})
+```
+
+### Route Registration (in agent.py)
+
+```python
+# In make_app() function
+app.router.add_route("GET", "/my/endpoint", agent.handle_my_endpoint)
+app.router.add_route("POST", "/my/endpoint", agent.handle_my_endpoint)
+```
+
+## Release Notes
+
+Use reno for release notes:
+
+```bash
+reno new my-feature-name
+```
+
+Edit the generated file in `releasenotes/notes/`:
+
+```yaml
+---
+features:
+  - |
+    Add new endpoint for X functionality.
+```
+
+Keep release notes concise (1-3 sentences).
+
+## Common Patterns
+
+### Decoding Payloads
+
+```python
+import msgpack
+
+# msgpack (most trace payloads)
+data = msgpack.unpackb(await request.read(), raw=False)
+
+# JSON
+data = await request.json()
+
+# With content-type detection
+content_type = request.content_type
+if "msgpack" in content_type:
+    data = msgpack.unpackb(await request.read(), raw=False)
+else:
+    data = await request.json()
+```
+
+### Session-Aware Data Access
+
+```python
+def _requests_by_session(self, token: Optional[str]) -> List[Request]:
+    if token is None:
+        return self._requests
+    return [r for r in self._requests if r.get("token") == token]
+```
+
+### Middleware Pattern
+
+```python
+from aiohttp.web import middleware
+
+@middleware
+async def my_middleware(request: Request, handler: _Handler) -> web.Response:
+    # Pre-processing
+    response = await handler(request)
+    # Post-processing
+    return response
+```
+
+## Environment Variables
+
+Key configuration options (see README.md for full list):
+
+- `PORT` - HTTP port (default: 8126)
+- `SNAPSHOT_DIR` - Snapshot storage directory
+- `SNAPSHOT_CI` - Enable CI mode (fail if snapshot missing)
+- `ENABLED_CHECKS` - Comma-separated list of trace checks
+- `DD_AGENT_URL` - Proxy to real Datadog agent
+- `LOG_LEVEL` - Logging level (DEBUG, INFO, WARNING, ERROR)
+- `VCR_CASSETTES_DIRECTORY` - VCR cassette storage path

{ddapm_test_agent-1.38.0 → ddapm_test_agent-1.40.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddapm-test-agent
-Version: 1.38.0
+Version: 1.40.0
 Summary: Test agent for Datadog APM client libraries
 Home-page: https://github.com/Datadog/dd-apm-test-agent
 Author: Kyle Verhoog
@@ -140,6 +140,16 @@ At the trace-level, proxying can also be disabled by including the `X-Datadog-Ag
 is handled, regardless of whether an agent URL is set.
 
 
+### LLM Observability Proxy
+
+The test agent provides a mechanism to dual ship LLM Observability events to Datadog, regardless of whether the Datadog agent is running.
+If using the Datadog agent, set the `DD_AGENT_URL` environment variable or `--agent-url` command-line argument to the URL of the Datadog agent (see [Proxy](#proxy) for more details).
+
+If not running a Datadog agent, set the `DD_SITE` environment variable or `--dd-site` command-line argument to the site of the Datadog instance to forward events to. Additionally, set the `DD_API_KEY` environment variable or `--dd-api-key` command-line argument to the API key to use for the Datadog instance.
+
+To disable LLM Observability event forwarding, set the `DISABLE_LLMOBS_DATA_FORWARDING` environment variable or `--disable-llmobs-data-forwarding` command-line argument to `true`.
+
+
 ### Snapshot testing
 
 The test agent provides a form of [characterization testing](https://en.wikipedia.org/wiki/Characterization_test) which

{ddapm_test_agent-1.38.0 → ddapm_test_agent-1.40.0}/README.md

@@ -93,6 +93,16 @@ At the trace-level, proxying can also be disabled by including the `X-Datadog-Ag
 is handled, regardless of whether an agent URL is set.
 
 
+### LLM Observability Proxy
+
+The test agent provides a mechanism to dual ship LLM Observability events to Datadog, regardless of whether the Datadog agent is running.
+If using the Datadog agent, set the `DD_AGENT_URL` environment variable or `--agent-url` command-line argument to the URL of the Datadog agent (see [Proxy](#proxy) for more details).
+
+If not running a Datadog agent, set the `DD_SITE` environment variable or `--dd-site` command-line argument to the site of the Datadog instance to forward events to. Additionally, set the `DD_API_KEY` environment variable or `--dd-api-key` command-line argument to the API key to use for the Datadog instance.
+
+To disable LLM Observability event forwarding, set the `DISABLE_LLMOBS_DATA_FORWARDING` environment variable or `--disable-llmobs-data-forwarding` command-line argument to `true`.
+
+
 ### Snapshot testing
 
 The test agent provides a form of [characterization testing](https://en.wikipedia.org/wiki/Characterization_test) which

{ddapm_test_agent-1.38.0 → ddapm_test_agent-1.40.0}/ddapm_test_agent/agent.py

@@ -53,6 +53,7 @@ from .checks import CheckTrace
 from .checks import Checks
 from .checks import start_trace
 from .integration import Integration
+from .llmobs_event_platform import LLMObsEventPlatformAPI
 from .logs import LOGS_ENDPOINT
 from .logs import OTLPLogsGRPCServicer
 from .logs import decode_logs_request
@@ -209,7 +210,7 @@ async def _prepare_and_send_request(data: bytes, request: Request, headers: Mapp
     log.info("Forwarding request to agent at %r", full_agent_url)
     log.debug(f"Using headers: {headers}")
 
-    (client_response, body) = await _forward_request(data, headers, full_agent_url)
+    client_response, body = await _forward_request(data, headers, full_agent_url)
     return web.Response(
         status=client_response.status,
         headers=client_response.headers,
@@ -805,6 +806,34 @@ class Agent:
             raise web.HTTPBadRequest(text=msg)
 
     async def handle_evp_proxy_v2_api_v2_llmobs(self, request: Request) -> web.Response:
+        if request.app["disable_llmobs_data_forwarding"]:
+            return web.HTTPOk()
+
+        dd_site = request.app["dd_site"]
+        dd_api_key = request.app["dd_api_key"]
+        agent_url = request.app["agent_url"]
+        headers = request.headers.copy()
+        if agent_url:
+            url = f"{agent_url}/evp_proxy/v2/api/v2/llmobs"  # use configured agent URL if provided
+        elif dd_api_key is None:
+            log.error("No DD_API_KEY set to forward LLM Observability events to Datadog. Skipping forwarding.")
+            return web.HTTPOk()
+        elif not dd_site:
+            log.error("No DD_SITE set to forward LLM Observability events to Datadog. Skipping forwarding.")
+            return web.HTTPOk()
+        else:
+            url = f"https://llmobs-intake.{dd_site}/api/v2/llmobs"
+            headers["DD-API-KEY"] = dd_api_key
+
+        async with ClientSession() as session:
+            async with session.post(url, headers=headers, data=await request.read()) as resp:
+                if not resp.ok:
+                    log.warning(
+                        f"Failed to forward LLM Observability events to Datadog: {resp.status} {await resp.text()}"
+                    )
+                else:
+                    log.info(f"Forwarded LLM Observability events to Datadog: {resp.status} {await resp.text()}")
+
         return web.HTTPOk()
 
     async def handle_evp_proxy_v2_llmobs_eval_metric(self, request: Request) -> web.Response:
@@ -881,9 +910,20 @@ class Agent:
         return web.HTTPAccepted()
 
     async def handle_info(self, request: Request) -> web.Response:
+        # CORS headers for cross-origin requests from Datadog UI
+        headers = {
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Methods": "GET, OPTIONS",
+            "Access-Control-Allow-Headers": "Content-Type",
+        }
+
+        # Handle OPTIONS preflight
+        if request.method == "OPTIONS":
+            return web.Response(status=200, headers=headers)
+
         return web.json_response(
             {
-                "version": "test",
+                "version": os.environ.get("TEST_AGENT_VERSION", "test"),
                 "endpoints": [
                     "/v0.4/traces",
                     "/v0.5/traces",
@@ -902,6 +942,7 @@ class Agent:
                 "peer_tags": ["db.name", "mongodb.db", "messaging.system"],
                 "span_events": True,  # Advertise support for the top-level Span field for Span Events
             },
+            headers=headers,
         )
 
     async def _handle_traces(self, request: Request, version: Literal["v0.4", "v0.5", "v0.7", "v1"]) -> web.Response:
@@ -1601,6 +1642,9 @@ def make_app(
     vcr_ci_mode: bool,
     vcr_provider_map: str,
     vcr_ignore_headers: str,
+    dd_site: str,
+    dd_api_key: str | None,
+    disable_llmobs_data_forwarding: bool,
     enable_web_ui: bool = False,
 ) -> web.Application:
     agent = Agent()
@@ -1650,6 +1694,7 @@ def make_app(
             web.post("/evp_proxy/v2/api/v2/exposures", agent.handle_evp_proxy_v2_api_v2_exposures),
             web.post("/evp_proxy/v4/api/v2/errorsintake", agent.handle_evp_proxy_v4_api_v2_errorsintake),
             web.get("/info", agent.handle_info),
+            web.options("/info", agent.handle_info),
             web.get("/test/session/start", agent.handle_session_start),
             web.get("/test/session/clear", agent.handle_session_clear),
             web.get("/test/session/snapshot", agent.handle_snapshot),
@@ -1682,6 +1727,12 @@ def make_app(
             ),
         ]
     )
+
+    # Add LLM Observability Event Platform API routes
+    # These provide Datadog Event Platform compatible endpoints for local development
+    llmobs_event_platform_api = LLMObsEventPlatformAPI(agent)
+    app.add_routes(llmobs_event_platform_api.get_routes())
+
     checks = Checks(
         checks=[
             CheckMetaTracerVersionHeader,
@@ -1707,6 +1758,9 @@ def make_app(
     app["snapshot_removed_attrs"] = snapshot_removed_attrs
     app["snapshot_regex_placeholders"] = snapshot_regex_placeholders
     app["vcr_cassettes_directory"] = vcr_cassettes_directory
+    app["dd_site"] = dd_site
+    app["dd_api_key"] = dd_api_key
+    app["disable_llmobs_data_forwarding"] = disable_llmobs_data_forwarding
     return app
 
 
@@ -2010,6 +2064,24 @@ def main(args: Optional[List[str]] = None) -> None:
         default=int(os.environ.get("MAX_REQUESTS", 200)),
         help="Maximum number of requests to keep in memory for the UI (default: 200). Older requests are discarded when limit is reached.",
     )
+    parser.add_argument(
+        "--dd-site",
+        type=str,
+        default=os.environ.get("DD_SITE", "datadoghq.com"),
+        help="Datadog site to use for the agent. Example: --dd-site=datadoghq.com",
+    )
+    parser.add_argument(
+        "--dd-api-key",
+        type=str,
+        default=os.environ.get("DD_API_KEY", ""),
+        help="Datadog API key to use for the agent. Example: --dd-api-key=1234567890",
+    )
+    parser.add_argument(
+        "--disable-llmobs-data-forwarding",
+        action="store_true",
+        default=os.environ.get("DISABLE_LLMOBS_DATA_FORWARDING", "").lower() in ("true", "1", "yes"),
+        help="Disable data forwarding to Datadog.",
+    )
     parsed_args = parser.parse_args(args=args)
     logging.basicConfig(level=parsed_args.log_level)
 
@@ -2056,6 +2128,9 @@ def main(args: Optional[List[str]] = None) -> None:
         vcr_ci_mode=parsed_args.vcr_ci_mode,
         vcr_provider_map=parsed_args.vcr_provider_map,
         vcr_ignore_headers=parsed_args.vcr_ignore_headers,
+        dd_site=parsed_args.dd_site,
+        dd_api_key=parsed_args.dd_api_key,
+        disable_llmobs_data_forwarding=parsed_args.disable_llmobs_data_forwarding,
         enable_web_ui=parsed_args.web_ui_port > 0,
     )