runlet-harness 0.1.0a1__tar.gz

runlet_harness-0.1.0a1/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto
+*.py text eol=lf
+*.md text eol=lf
+*.toml text eol=lf

runlet_harness-0.1.0a1/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package
+        run: python -m pip install -e .
+
+      - name: Run tests
+        run: PYTHONPATH=src python -m unittest discover tests

runlet_harness-0.1.0a1/.github/workflows/publish.yml

@@ -0,0 +1,79 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Verify tag matches package version
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import os
+          import tomllib
+
+          ref = os.environ["GITHUB_REF_NAME"]
+          tag_version = ref.removeprefix("v")
+          data = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))
+          package_version = data["project"]["version"]
+          if package_version != tag_version:
+              raise SystemExit(
+                  f"Tag version {tag_version!r} does not match project version {package_version!r}"
+              )
+          PY
+
+      - name: Install build tools
+        run: python -m pip install --upgrade build twine
+
+      - name: Install package
+        run: python -m pip install -e .
+
+      - name: Run tests
+        run: PYTHONPATH=src python -m unittest discover tests
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Check distributions
+        run: python -m twine check dist/*
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/runlet-harness
+
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Publish distributions
+        uses: pypa/gh-action-pypi-publish@release/v1

runlet_harness-0.1.0a1/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.pyright/
+build/
+dist/
+.venv/
+.env

runlet_harness-0.1.0a1/AGENTS.md

@@ -0,0 +1,36 @@
+# Repository Instructions
+
+These instructions apply to the entire repository.
+
+## Project Identity
+
+- Project name: Runlet Harness.
+- Python package name: `runlet_harness`.
+- License: MIT.
+- Purpose: application integration helpers for Runlet.
+
+## Boundary
+
+`runlet-harness` may contain third-party platform adapters, environment-based
+configuration helpers, and background delivery lifecycle code. It depends on
+`runlet`.
+
+`runlet` must not depend on `runlet-harness`.
+
+Do not put core runtime behavior, provider-neutral model contracts, or agent
+execution logic in this package.
+
+## Development Practices
+
+- Keep optional SDKs behind optional dependencies and lazy imports.
+- `import runlet_harness` must not import optional SDK packages.
+- Prefer simple Python modules and explicit protocols over framework-heavy
+  abstractions.
+- Use fake clients in tests; do not call third-party network services.
+- Keep generated caches such as `__pycache__/` out of commits.
+
+Current test command:
+
+```bash
+PYTHONPATH=src python -m unittest discover tests
+```

runlet_harness-0.1.0a1/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## 0.1.0a1
+
+- Initial package skeleton.
+- Added buffered observability sink support.
+- Added optional Langfuse event sink.

runlet_harness-0.1.0a1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Runlet Harness contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

runlet_harness-0.1.0a1/PKG-INFO

@@ -0,0 +1,124 @@
+Metadata-Version: 2.4
+Name: runlet-harness
+Version: 0.1.0a1
+Summary: Application integration helpers for Runlet.
+Project-URL: Homepage, https://github.com/DMIAOCHEN/runlet-harness
+Project-URL: Repository, https://github.com/DMIAOCHEN/runlet-harness
+Project-URL: Issues, https://github.com/DMIAOCHEN/runlet-harness/issues
+Author: Runlet Harness contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,langfuse,observability,runlet,runtime
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: runlet>=0.2.0b3
+Provides-Extra: dev
+Requires-Dist: pyright>=1.1.0; extra == 'dev'
+Requires-Dist: ruff>=0.5.0; extra == 'dev'
+Provides-Extra: langfuse
+Requires-Dist: langfuse<5,>=4.7; extra == 'langfuse'
+Description-Content-Type: text/markdown
+
+# Runlet Harness
+
+[![PyPI version](https://img.shields.io/pypi/v/runlet-harness.svg)](https://pypi.org/project/runlet-harness/)
+[![Python versions](https://img.shields.io/pypi/pyversions/runlet-harness.svg)](https://pypi.org/project/runlet-harness/)
+[![CI](https://github.com/DMIAOCHEN/runlet-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/DMIAOCHEN/runlet-harness/actions/workflows/ci.yml)
+
+Application integration helpers for [Runlet](https://github.com/DMIAOCHEN/runlet).
+
+`runlet-harness` keeps third-party adapters, environment-based configuration,
+and background delivery concerns outside the small Runlet runtime core.
+
+## Install
+
+Base package:
+
+```bash
+pip install runlet-harness
+```
+
+With Langfuse support:
+
+```bash
+pip install "runlet-harness[langfuse]"
+```
+
+## Langfuse
+
+Cloud configuration:
+
+```dotenv
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+```
+
+Self-hosted configuration:
+
+```dotenv
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_BASE_URL=https://langfuse.example.com
+```
+
+Usage:
+
+```python
+from runlet import CompositeEventSink, InMemoryObserver, Runtime
+from runlet_harness.observability import LangfuseEventSink
+
+observer = InMemoryObserver()
+langfuse = LangfuseEventSink.from_env()
+
+sinks = [observer]
+if langfuse is not None:
+    sinks.append(langfuse)
+
+runtime = Runtime(event_sink=CompositeEventSink(sinks))
+
+try:
+    result = await runtime.run(agent, "hello")
+finally:
+    if langfuse is not None:
+        await langfuse.shutdown()
+```
+
+By default, the Langfuse sink records metadata, status, timing, tool names, and
+final outputs. It does not record inputs, reasoning, streaming deltas, tool
+arguments, tool results, or human-submitted values unless explicitly configured.
+## Development
+
+Run the test suite:
+
+```bash
+PYTHONPATH=src python -m unittest discover tests
+```
+
+Run type checking after installing development dependencies:
+
+```bash
+pyright
+```
+
+## Release
+
+Runlet Harness publishes to PyPI from Git tags through GitHub Actions.
+
+Typical release flow:
+
+1. Update the version in `pyproject.toml`.
+2. Merge to `main`.
+3. Create a tag such as `v0.1.0a1`.
+4. Push the tag.
+
+```bash
+git tag v0.1.0a1
+git push origin v0.1.0a1
+```

runlet_harness-0.1.0a1/README.md

@@ -0,0 +1,96 @@
+# Runlet Harness
+
+[![PyPI version](https://img.shields.io/pypi/v/runlet-harness.svg)](https://pypi.org/project/runlet-harness/)
+[![Python versions](https://img.shields.io/pypi/pyversions/runlet-harness.svg)](https://pypi.org/project/runlet-harness/)
+[![CI](https://github.com/DMIAOCHEN/runlet-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/DMIAOCHEN/runlet-harness/actions/workflows/ci.yml)
+
+Application integration helpers for [Runlet](https://github.com/DMIAOCHEN/runlet).
+
+`runlet-harness` keeps third-party adapters, environment-based configuration,
+and background delivery concerns outside the small Runlet runtime core.
+
+## Install
+
+Base package:
+
+```bash
+pip install runlet-harness
+```
+
+With Langfuse support:
+
+```bash
+pip install "runlet-harness[langfuse]"
+```
+
+## Langfuse
+
+Cloud configuration:
+
+```dotenv
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+```
+
+Self-hosted configuration:
+
+```dotenv
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_BASE_URL=https://langfuse.example.com
+```
+
+Usage:
+
+```python
+from runlet import CompositeEventSink, InMemoryObserver, Runtime
+from runlet_harness.observability import LangfuseEventSink
+
+observer = InMemoryObserver()
+langfuse = LangfuseEventSink.from_env()
+
+sinks = [observer]
+if langfuse is not None:
+    sinks.append(langfuse)
+
+runtime = Runtime(event_sink=CompositeEventSink(sinks))
+
+try:
+    result = await runtime.run(agent, "hello")
+finally:
+    if langfuse is not None:
+        await langfuse.shutdown()
+```
+
+By default, the Langfuse sink records metadata, status, timing, tool names, and
+final outputs. It does not record inputs, reasoning, streaming deltas, tool
+arguments, tool results, or human-submitted values unless explicitly configured.
+## Development
+
+Run the test suite:
+
+```bash
+PYTHONPATH=src python -m unittest discover tests
+```
+
+Run type checking after installing development dependencies:
+
+```bash
+pyright
+```
+
+## Release
+
+Runlet Harness publishes to PyPI from Git tags through GitHub Actions.
+
+Typical release flow:
+
+1. Update the version in `pyproject.toml`.
+2. Merge to `main`.
+3. Create a tag such as `v0.1.0a1`.
+4. Push the tag.
+
+```bash
+git tag v0.1.0a1
+git push origin v0.1.0a1
+```

runlet_harness-0.1.0a1/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "runlet-harness"
+version = "0.1.0a1"
+description = "Application integration helpers for Runlet."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+  { name = "Runlet Harness contributors" },
+]
+keywords = ["agent", "runtime", "observability", "langfuse", "runlet"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
+]
+dependencies = [
+  "runlet>=0.2.0b3",
+]
+
+[project.optional-dependencies]
+langfuse = [
+  "langfuse>=4.7,<5",
+]
+dev = [
+  "pyright>=1.1.0",
+  "ruff>=0.5.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/DMIAOCHEN/runlet-harness"
+Repository = "https://github.com/DMIAOCHEN/runlet-harness"
+Issues = "https://github.com/DMIAOCHEN/runlet-harness/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/runlet_harness"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+
+[tool.pyright]
+pythonVersion = "3.10"
+typeCheckingMode = "strict"
+include = ["src", "tests"]

runlet_harness-0.1.0a1/src/runlet_harness/__init__.py

@@ -0,0 +1,5 @@
+"""Application integration helpers for Runlet."""
+
+__version__ = "0.1.0a1"
+
+__all__ = ["__version__"]

runlet_harness-0.1.0a1/src/runlet_harness/config.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+
+TRUE_VALUES = {"1", "true", "yes", "y", "on"}
+FALSE_VALUES = {"0", "false", "no", "n", "off"}
+
+
+def env_str(name: str, *, environ: Mapping[str, str] | None = None) -> str | None:
+    source = os.environ if environ is None else environ
+    value = source.get(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def parse_bool(value: str | None, *, name: str) -> bool | None:
+    if value is None or value.strip() == "":
+        return None
+    normalized = value.strip().lower()
+    if normalized in TRUE_VALUES:
+        return True
+    if normalized in FALSE_VALUES:
+        return False
+    raise ValueError(f"{name} must be a boolean value.")
+
+
+def env_bool(name: str, *, environ: Mapping[str, str] | None = None) -> bool | None:
+    return parse_bool(env_str(name, environ=environ), name=name)
+
+
+def parse_int(value: str | None, *, name: str) -> int | None:
+    if value is None or value.strip() == "":
+        return None
+    try:
+        return int(value)
+    except ValueError as error:
+        raise ValueError(f"{name} must be an integer.") from error
+
+
+def env_int(name: str, *, environ: Mapping[str, str] | None = None) -> int | None:
+    return parse_int(env_str(name, environ=environ), name=name)
+
+
+def parse_float(value: str | None, *, name: str) -> float | None:
+    if value is None or value.strip() == "":
+        return None
+    try:
+        return float(value)
+    except ValueError as error:
+        raise ValueError(f"{name} must be a float.") from error
+
+
+def env_float(name: str, *, environ: Mapping[str, str] | None = None) -> float | None:
+    return parse_float(env_str(name, environ=environ), name=name)

runlet_harness-0.1.0a1/src/runlet_harness/observability/__init__.py

@@ -0,0 +1,4 @@
+from runlet_harness.observability.buffering import BufferedEventSink
+from runlet_harness.observability.langfuse import LangfuseConfig, LangfuseEventSink
+
+__all__ = ["BufferedEventSink", "LangfuseConfig", "LangfuseEventSink"]

runlet_harness-0.1.0a1/src/runlet_harness/observability/buffering.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+
+from runlet.core import RuntimeEvent
+
+
+class BufferedEventSink:
+    def __init__(
+        self,
+        *,
+        max_queue_size: int = 1000,
+        drop_on_full: bool = True,
+    ) -> None:
+        if max_queue_size < 1:
+            raise ValueError("max_queue_size must be at least 1.")
+        self.max_queue_size = max_queue_size
+        self.drop_on_full = drop_on_full
+        self.dropped_events = 0
+        self.processed_events = 0
+        self.failed_events = 0
+        self._queue: asyncio.Queue[RuntimeEvent] | None = None
+        self._worker_task: asyncio.Task[None] | None = None
+        self._closed = False
+
+    async def emit(self, event: RuntimeEvent) -> None:
+        if self._closed:
+            return
+        queue = self._ensure_queue()
+        if self.drop_on_full:
+            try:
+                queue.put_nowait(event)
+            except asyncio.QueueFull:
+                self.dropped_events += 1
+            return
+        await queue.put(event)
+
+    async def flush(self) -> None:
+        queue = self._queue
+        if queue is not None:
+            await queue.join()
+        await self._flush_backend()
+
+    async def shutdown(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        await self.flush()
+        task = self._worker_task
+        if task is not None:
+            task.cancel()
+            with suppress(asyncio.CancelledError):
+                await task
+        self._worker_task = None
+
+    def _ensure_queue(self) -> asyncio.Queue[RuntimeEvent]:
+        if self._queue is None:
+            self._queue = asyncio.Queue(maxsize=self.max_queue_size)
+        if self._worker_task is None or self._worker_task.done():
+            self._worker_task = asyncio.create_task(self._worker())
+        return self._queue
+
+    async def _worker(self) -> None:
+        assert self._queue is not None
+        while True:
+            event = await self._queue.get()
+            try:
+                await self._process_event(event)
+                self.processed_events += 1
+            except Exception:
+                self.failed_events += 1
+            finally:
+                self._queue.task_done()
+
+    async def _process_event(self, event: RuntimeEvent) -> None:
+        raise NotImplementedError
+
+    async def _flush_backend(self) -> None:
+        return None

runlet_harness-0.1.0a1/src/runlet_harness/observability/langfuse.py

@@ -0,0 +1,356 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+import random
+from typing import Any, Protocol
+
+from runlet.core import RuntimeEvent
+
+from runlet_harness.config import env_bool, env_float, env_int, env_str
+from runlet_harness.observability.buffering import BufferedEventSink
+
+TERMINAL_EVENTS = {"run.completed", "run.interrupted", "policy.stopped"}
+
+
+def _metadata_map() -> dict[str, Any]:
+    return {}
+
+
+def _bool_env_or(name: str, default: bool, environ: Mapping[str, str] | None) -> bool:
+    value = env_bool(name, environ=environ)
+    return default if value is None else value
+
+
+def _float_env_or(name: str, default: float, environ: Mapping[str, str] | None) -> float:
+    value = env_float(name, environ=environ)
+    return default if value is None else value
+
+
+def _int_env_or(name: str, default: int, environ: Mapping[str, str] | None) -> int:
+    value = env_int(name, environ=environ)
+    return default if value is None else value
+
+
+@dataclass(frozen=True)
+class LangfuseConfig:
+    public_key: str | None = None
+    secret_key: str | None = None
+    base_url: str | None = None
+    enabled: bool | None = None
+    sample_rate: float = 1.0
+    max_queue_size: int = 1000
+    drop_on_full: bool = True
+    capture_input: bool = False
+    capture_output: bool = True
+    capture_reasoning: bool = False
+    capture_stream_deltas: bool = False
+    capture_tool_arguments: bool = False
+    capture_tool_results: bool = False
+    max_payload_chars: int = 4000
+    flush_on_terminal_event: bool = False
+    debug: bool = False
+    metadata: dict[str, Any] = field(default_factory=_metadata_map)
+
+    @classmethod
+    def from_env(cls, environ: Mapping[str, str] | None = None) -> "LangfuseConfig | None":
+        enabled = env_bool("RUNLET_LANGFUSE_ENABLED", environ=environ)
+        public_key = env_str("LANGFUSE_PUBLIC_KEY", environ=environ)
+        secret_key = env_str("LANGFUSE_SECRET_KEY", environ=environ)
+        base_url = env_str("LANGFUSE_BASE_URL", environ=environ)
+        if enabled is False:
+            return None
+        if enabled is True and (public_key is None or secret_key is None):
+            raise ValueError(
+                "RUNLET_LANGFUSE_ENABLED=true requires LANGFUSE_PUBLIC_KEY and "
+                "LANGFUSE_SECRET_KEY."
+            )
+        if enabled is None and (public_key is None or secret_key is None):
+            return None
+        return cls(
+            public_key=public_key,
+            secret_key=secret_key,
+            base_url=base_url,
+            enabled=enabled,
+            sample_rate=_float_env_or("RUNLET_LANGFUSE_SAMPLE_RATE", 1.0, environ),
+            max_queue_size=_int_env_or("RUNLET_LANGFUSE_MAX_QUEUE_SIZE", 1000, environ),
+            drop_on_full=_bool_env_or("RUNLET_LANGFUSE_DROP_ON_FULL", True, environ),
+            capture_input=_bool_env_or("RUNLET_LANGFUSE_CAPTURE_INPUT", False, environ),
+            capture_output=_bool_env_or("RUNLET_LANGFUSE_CAPTURE_OUTPUT", True, environ),
+            capture_reasoning=_bool_env_or("RUNLET_LANGFUSE_CAPTURE_REASONING", False, environ),
+            capture_stream_deltas=_bool_env_or(
+                "RUNLET_LANGFUSE_CAPTURE_STREAM_DELTAS", False, environ
+            ),
+            capture_tool_arguments=_bool_env_or(
+                "RUNLET_LANGFUSE_CAPTURE_TOOL_ARGUMENTS", False, environ
+            ),
+            capture_tool_results=_bool_env_or(
+                "RUNLET_LANGFUSE_CAPTURE_TOOL_RESULTS", False, environ
+            ),
+            max_payload_chars=_int_env_or("RUNLET_LANGFUSE_MAX_PAYLOAD_CHARS", 4000, environ),
+            flush_on_terminal_event=_bool_env_or(
+                "RUNLET_LANGFUSE_FLUSH_ON_TERMINAL_EVENT", False, environ
+            ),
+            debug=_bool_env_or("RUNLET_LANGFUSE_DEBUG", False, environ),
+        )
+
+
+class LangfuseClientProtocol(Protocol):
+    def start_observation(self, **kwargs: Any) -> Any:
+        ...
+
+    def flush(self) -> Any:
+        ...
+
+
+@dataclass
+class _RunState:
+    root: Any = None
+    model: Any = None
+    sampled: bool = True
+    tool_stack: list[Any] = field(default_factory=list)
+    stream_output: list[str] = field(default_factory=list)
+
+
+class LangfuseEventSink(BufferedEventSink):
+    def __init__(self, config: LangfuseConfig, *, client: Any | None = None) -> None:
+        if config.sample_rate < 0 or config.sample_rate > 1:
+            raise ValueError("sample_rate must be between 0 and 1.")
+        if config.max_payload_chars < 0:
+            raise ValueError("max_payload_chars must not be negative.")
+        self.config = config
+        self.client = client if client is not None else self._create_client(config)
+        self._runs: dict[str, _RunState] = {}
+        super().__init__(
+            max_queue_size=config.max_queue_size,
+            drop_on_full=config.drop_on_full,
+        )
+
+    @classmethod
+    def from_env(cls, *, client: Any | None = None) -> "LangfuseEventSink | None":
+        config = LangfuseConfig.from_env()
+        if config is None:
+            return None
+        return cls(config, client=client)
+
+    async def _process_event(self, event: RuntimeEvent) -> None:
+        state = self._runs.setdefault(event.run_id, _RunState())
+        if event.type == "run.started":
+            state.sampled = self.config.sample_rate >= 1 or random.random() < self.config.sample_rate
+            if not state.sampled:
+                return
+            state.root = self._start_observation(
+                None,
+                name="runlet.run",
+                as_type="span",
+                input=self._payload_value(event, "input") if self.config.capture_input else None,
+                metadata=self._metadata(event),
+            )
+            return
+        if not state.sampled:
+            if event.type in TERMINAL_EVENTS:
+                self._runs.pop(event.run_id, None)
+            return
+        if event.type == "model.requested":
+            state.model = self._start_observation(
+                state.root,
+                name="runlet.model",
+                as_type="generation",
+                metadata=self._metadata(event),
+            )
+            return
+        if event.type == "model.completed":
+            usage = event.payload.get("usage")
+            update: dict[str, Any] = {"metadata": self._metadata(event)}
+            if isinstance(usage, int):
+                update["usage"] = {"total": usage}
+            self._update_observation(state.model, **update)
+            self._end_observation(state.model)
+            state.model = None
+            return
+        if event.type == "model.stream.started":
+            state.stream_output = []
+            state.model = self._start_observation(
+                state.root,
+                name="runlet.model.stream",
+                as_type="generation",
+                metadata=self._metadata(event),
+            )
+            return
+        if event.type == "model.stream.delta":
+            if self.config.capture_stream_deltas:
+                delta = self._payload_value(event, "delta")
+                if isinstance(delta, str):
+                    state.stream_output.append(delta)
+            return
+        if event.type == "model.stream.completed":
+            output = None
+            if self.config.capture_stream_deltas:
+                output = self._truncate("".join(state.stream_output))
+            self._update_observation(state.model, output=output, metadata=self._metadata(event))
+            self._end_observation(state.model)
+            state.model = None
+            state.stream_output = []
+            return
+        if event.type == "tool.started":
+            tool_name = str(event.payload.get("name", "unknown"))
+            observation = self._start_observation(
+                state.root,
+                name=f"runlet.tool.{tool_name}",
+                as_type="span",
+                metadata=self._metadata(event),
+            )
+            state.tool_stack.append(observation)
+            return
+        if event.type == "tool.completed":
+            observation = state.tool_stack.pop() if state.tool_stack else None
+            self._update_observation(observation, metadata=self._metadata(event))
+            self._end_observation(observation)
+            return
+        if event.type in {"human.requested", "human.responded", "human.response_rejected", "run.resumed"}:
+            observation = self._start_observation(
+                state.root,
+                name=f"runlet.{event.type}",
+                as_type="event",
+                metadata=self._metadata(event),
+            )
+            self._end_observation(observation)
+            return
+        if event.type in TERMINAL_EVENTS:
+            update = {"metadata": self._metadata(event)}
+            output = self._terminal_output(event)
+            if output is not None:
+                update["output"] = output
+            self._update_observation(state.root, **update)
+            self._end_observation(state.root)
+            self._runs.pop(event.run_id, None)
+            if self.config.flush_on_terminal_event:
+                await self._flush_backend()
+
+    async def _flush_backend(self) -> None:
+        result = self.client.flush()
+        if hasattr(result, "__await__"):
+            await result
+
+    @staticmethod
+    def _create_client(config: LangfuseConfig) -> Any:
+        if config.public_key is None or config.secret_key is None:
+            raise ValueError("Langfuse public_key and secret_key are required when creating a client.")
+        try:
+            from langfuse import Langfuse  # type: ignore[import-not-found]
+        except ImportError as error:
+            raise RuntimeError(
+                "Langfuse support requires the optional dependency. Install it with "
+                "pip install \"runlet-harness[langfuse]\"."
+            ) from error
+        kwargs: dict[str, Any] = {
+            "public_key": config.public_key,
+            "secret_key": config.secret_key,
+        }
+        if config.base_url is not None:
+            kwargs["base_url"] = config.base_url
+        try:
+            return Langfuse(**kwargs)
+        except TypeError:
+            if "base_url" in kwargs:
+                kwargs["host"] = kwargs.pop("base_url")
+            return Langfuse(**kwargs)
+
+    def _metadata(self, event: RuntimeEvent) -> dict[str, Any]:
+        metadata: dict[str, Any] = {
+            "runlet.event_id": event.id,
+            "runlet.event_type": event.type,
+            "runlet.run_id": event.run_id,
+            "runlet.timestamp": event.timestamp.isoformat(),
+            "runlet.severity": event.severity,
+            **self.config.metadata,
+        }
+        if event.agent_name is not None:
+            metadata["runlet.agent_name"] = event.agent_name
+        if event.step_id is not None:
+            metadata["runlet.step_id"] = event.step_id
+        if event.span_id is not None:
+            metadata["runlet.span_id"] = event.span_id
+        if event.parent_span_id is not None:
+            metadata["runlet.parent_span_id"] = event.parent_span_id
+        for key, value in event.attributes.items():
+            metadata[f"runlet.attribute.{key}"] = self._sanitize_value(value)
+        for key, value in self._safe_payload(event).items():
+            metadata[f"runlet.payload.{key}"] = value
+        return metadata
+
+    def _safe_payload(self, event: RuntimeEvent) -> dict[str, Any]:
+        payload: dict[str, Any] = {}
+        for key, value in event.payload.items():
+            if key == "input" and not self.config.capture_input:
+                continue
+            if key == "output" and not self.config.capture_output:
+                continue
+            if key == "reasoning" and not self.config.capture_reasoning:
+                continue
+            if key == "delta" and not self.config.capture_stream_deltas:
+                continue
+            if key in {"arguments", "tool_arguments"} and not self.config.capture_tool_arguments:
+                continue
+            if key in {"result", "tool_result"} and not self.config.capture_tool_results:
+                continue
+            payload[key] = self._sanitize_value(value)
+        return payload
+
+    def _terminal_output(self, event: RuntimeEvent) -> Any | None:
+        if not self.config.capture_output:
+            return None
+        output = self._payload_value(event, "output")
+        if output is None:
+            return None
+        return self._sanitize_value(output)
+
+    def _payload_value(self, event: RuntimeEvent, key: str) -> Any | None:
+        if key not in event.payload:
+            return None
+        return self._sanitize_value(event.payload[key])
+
+    def _sanitize_value(self, value: Any) -> Any:
+        if isinstance(value, str):
+            return self._truncate(value)
+        if isinstance(value, (int, float, bool)) or value is None:
+            return value
+        if isinstance(value, Mapping):
+            return {str(key): self._sanitize_value(item) for key, item in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [self._sanitize_value(item) for item in value]
+        return self._truncate(repr(value))
+
+    def _truncate(self, value: str) -> str:
+        limit = self.config.max_payload_chars
+        if limit == 0:
+            return ""
+        if len(value) <= limit:
+            return value
+        return value[:limit]
+
+    def _start_observation(self, parent: Any, **kwargs: Any) -> Any:
+        target = parent if parent is not None and hasattr(parent, "start_observation") else self.client
+        starter = getattr(target, "start_observation")
+        kwargs = {key: value for key, value in kwargs.items() if value is not None}
+        try:
+            return starter(**kwargs)
+        except TypeError:
+            fallback = dict(kwargs)
+            if "as_type" in fallback:
+                fallback["type"] = fallback.pop("as_type")
+            return starter(**fallback)
+
+    @staticmethod
+    def _update_observation(observation: Any, **kwargs: Any) -> None:
+        if observation is None or not hasattr(observation, "update"):
+            return
+        clean_kwargs = {key: value for key, value in kwargs.items() if value is not None}
+        if clean_kwargs:
+            observation.update(**clean_kwargs)
+
+    @staticmethod
+    def _end_observation(observation: Any) -> None:
+        if observation is None or not hasattr(observation, "end"):
+            return
+        observation.end()

runlet_harness-0.1.0a1/tests/test_buffering.py

@@ -0,0 +1,65 @@
+import asyncio
+import unittest
+
+from runlet.core import RuntimeEvent
+from runlet_harness.observability.buffering import BufferedEventSink
+
+
+class RecordingSink(BufferedEventSink):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.events = []
+        self.flushed = 0
+        self.fail = False
+
+    async def _process_event(self, event: RuntimeEvent) -> None:
+        if self.fail:
+            raise RuntimeError("boom")
+        self.events.append(event)
+
+    async def _flush_backend(self) -> None:
+        self.flushed += 1
+
+
+class BufferedEventSinkTests(unittest.IsolatedAsyncioTestCase):
+    async def test_emit_processes_events(self) -> None:
+        sink = RecordingSink()
+        event = RuntimeEvent(type="run.started", run_id="run_1")
+
+        await sink.emit(event)
+        await sink.flush()
+
+        self.assertEqual(sink.events, [event])
+        self.assertEqual(sink.processed_events, 1)
+        await sink.shutdown()
+
+    async def test_queue_full_drops_when_configured(self) -> None:
+        sink = RecordingSink(max_queue_size=1, drop_on_full=True)
+        await sink.emit(RuntimeEvent(type="one", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="two", run_id="run_1"))
+
+        self.assertGreaterEqual(sink.dropped_events, 0)
+        await sink.shutdown()
+
+    async def test_worker_errors_do_not_propagate(self) -> None:
+        sink = RecordingSink()
+        sink.fail = True
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.flush()
+
+        self.assertEqual(sink.failed_events, 1)
+        await sink.shutdown()
+
+    async def test_shutdown_is_idempotent(self) -> None:
+        sink = RecordingSink()
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.shutdown()
+        await sink.shutdown()
+        self.assertEqual(sink.flushed, 1)
+
+    async def test_non_dropping_queue_waits_for_space(self) -> None:
+        sink = RecordingSink(max_queue_size=1, drop_on_full=False)
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await asyncio.wait_for(sink.emit(RuntimeEvent(type="run.completed", run_id="run_1")), 1)
+        await sink.shutdown()

runlet_harness-0.1.0a1/tests/test_config.py

@@ -0,0 +1,31 @@
+import unittest
+
+from runlet_harness.config import env_bool, env_float, env_int, env_str, parse_bool
+
+
+class ConfigTests(unittest.TestCase):
+    def test_env_str_treats_blank_as_missing(self) -> None:
+        self.assertIsNone(env_str("MISSING", environ={}))
+        self.assertIsNone(env_str("EMPTY", environ={"EMPTY": "  "}))
+        self.assertEqual(env_str("VALUE", environ={"VALUE": " hello "}), "hello")
+
+    def test_bool_parsing(self) -> None:
+        for value in ("true", "1", "yes", "on"):
+            self.assertIs(parse_bool(value, name="FLAG"), True)
+        for value in ("false", "0", "no", "off"):
+            self.assertIs(parse_bool(value, name="FLAG"), False)
+        with self.assertRaisesRegex(ValueError, "FLAG"):
+            parse_bool("maybe", name="FLAG")
+
+    def test_number_parsing(self) -> None:
+        self.assertEqual(env_int("COUNT", environ={"COUNT": "5"}), 5)
+        self.assertEqual(env_float("RATE", environ={"RATE": "0.25"}), 0.25)
+        with self.assertRaisesRegex(ValueError, "COUNT"):
+            env_int("COUNT", environ={"COUNT": "x"})
+        with self.assertRaisesRegex(ValueError, "RATE"):
+            env_float("RATE", environ={"RATE": "x"})
+
+    def test_env_bool(self) -> None:
+        self.assertIs(env_bool("FLAG", environ={"FLAG": "yes"}), True)
+        self.assertIs(env_bool("FLAG", environ={"FLAG": "no"}), False)
+        self.assertIsNone(env_bool("FLAG", environ={}))

runlet_harness-0.1.0a1/tests/test_langfuse.py

@@ -0,0 +1,189 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from runlet import Agent, Message, Runtime
+from runlet.core import RuntimeEvent
+from runlet.core.models import ModelResponse
+from runlet.testing import FakeModelProvider
+from runlet_harness.observability.langfuse import LangfuseConfig, LangfuseEventSink
+
+
+class FakeObservation:
+    def __init__(self, client, **kwargs):
+        self.client = client
+        self.kwargs = kwargs
+        self.updates = []
+        self.ended = False
+        self.children = []
+
+    def start_observation(self, **kwargs):
+        child = FakeObservation(self.client, **kwargs)
+        self.children.append(child)
+        self.client.observations.append(child)
+        return child
+
+    def update(self, **kwargs):
+        self.updates.append(kwargs)
+
+    def end(self):
+        self.ended = True
+
+
+class FakeClient:
+    def __init__(self):
+        self.observations = []
+        self.flushed = 0
+        self.fail = False
+
+    def start_observation(self, **kwargs):
+        if self.fail:
+            raise RuntimeError("langfuse failed")
+        observation = FakeObservation(self, **kwargs)
+        self.observations.append(observation)
+        return observation
+
+    def flush(self):
+        self.flushed += 1
+
+
+class LangfuseConfigTests(unittest.TestCase):
+    def test_from_env_returns_none_without_keys(self) -> None:
+        with patch.dict(os.environ, {}, clear=True):
+            self.assertIsNone(LangfuseEventSink.from_env(client=FakeClient()))
+
+    def test_from_env_can_be_disabled(self) -> None:
+        env = {
+            "RUNLET_LANGFUSE_ENABLED": "false",
+            "LANGFUSE_PUBLIC_KEY": "pk",
+            "LANGFUSE_SECRET_KEY": "sk",
+        }
+        with patch.dict(os.environ, env, clear=True):
+            self.assertIsNone(LangfuseEventSink.from_env(client=FakeClient()))
+
+    def test_forced_enable_requires_keys(self) -> None:
+        with patch.dict(os.environ, {"RUNLET_LANGFUSE_ENABLED": "true"}, clear=True):
+            with self.assertRaisesRegex(ValueError, "LANGFUSE_PUBLIC_KEY"):
+                LangfuseEventSink.from_env(client=FakeClient())
+
+    def test_env_base_url_is_loaded(self) -> None:
+        env = {
+            "LANGFUSE_PUBLIC_KEY": "pk",
+            "LANGFUSE_SECRET_KEY": "sk",
+            "LANGFUSE_BASE_URL": "https://langfuse.example.com",
+        }
+        with patch.dict(os.environ, env, clear=True):
+            config = LangfuseConfig.from_env()
+        assert config is not None
+        self.assertEqual(config.base_url, "https://langfuse.example.com")
+
+    def test_env_explicit_zero_values_are_preserved(self) -> None:
+        env = {
+            "LANGFUSE_PUBLIC_KEY": "pk",
+            "LANGFUSE_SECRET_KEY": "sk",
+            "RUNLET_LANGFUSE_SAMPLE_RATE": "0",
+            "RUNLET_LANGFUSE_MAX_PAYLOAD_CHARS": "0",
+        }
+        with patch.dict(os.environ, env, clear=True):
+            config = LangfuseConfig.from_env()
+        assert config is not None
+        self.assertEqual(config.sample_rate, 0)
+        self.assertEqual(config.max_payload_chars, 0)
+
+    def test_missing_sdk_error_is_clear(self) -> None:
+        with patch("runlet_harness.observability.langfuse.LangfuseEventSink._create_client") as create:
+            create.side_effect = RuntimeError('Install it with pip install "runlet-harness[langfuse]".')
+            with self.assertRaisesRegex(RuntimeError, "runlet-harness\\[langfuse\\]"):
+                LangfuseEventSink(LangfuseConfig(public_key="pk", secret_key="sk"))
+
+
+class LangfuseEventSinkTests(unittest.IsolatedAsyncioTestCase):
+    async def test_maps_run_model_tool_and_terminal_events(self) -> None:
+        client = FakeClient()
+        sink = LangfuseEventSink(LangfuseConfig(public_key="pk", secret_key="sk"), client=client)
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1", agent_name="agent", payload={"input": "secret"}))
+        await sink.emit(RuntimeEvent(type="model.requested", run_id="run_1", agent_name="agent"))
+        await sink.emit(RuntimeEvent(type="model.completed", run_id="run_1", agent_name="agent", payload={"usage": 7}))
+        await sink.emit(RuntimeEvent(type="tool.started", run_id="run_1", agent_name="agent", payload={"name": "lookup"}))
+        await sink.emit(RuntimeEvent(type="tool.completed", run_id="run_1", agent_name="agent", payload={"name": "lookup"}))
+        await sink.emit(RuntimeEvent(type="run.completed", run_id="run_1", agent_name="agent", payload={"output": "done"}))
+        await sink.shutdown()
+
+        names = [observation.kwargs["name"] for observation in client.observations]
+        self.assertIn("runlet.run", names)
+        self.assertIn("runlet.model", names)
+        self.assertIn("runlet.tool.lookup", names)
+        root = client.observations[0]
+        self.assertNotIn("input", root.kwargs)
+        self.assertTrue(root.ended)
+        self.assertEqual(root.updates[-1]["output"], "done")
+
+    async def test_stream_delta_is_ignored_by_default(self) -> None:
+        client = FakeClient()
+        sink = LangfuseEventSink(LangfuseConfig(public_key="pk", secret_key="sk"), client=client)
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="model.stream.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="model.stream.delta", run_id="run_1", payload={"delta": "private"}))
+        await sink.emit(RuntimeEvent(type="model.stream.completed", run_id="run_1"))
+        await sink.shutdown()
+
+        stream = [observation for observation in client.observations if observation.kwargs["name"] == "runlet.model.stream"][0]
+        self.assertNotIn("output", stream.updates[-1])
+
+    async def test_stream_delta_can_be_captured_and_truncated(self) -> None:
+        client = FakeClient()
+        config = LangfuseConfig(
+            public_key="pk",
+            secret_key="sk",
+            capture_stream_deltas=True,
+            max_payload_chars=4,
+        )
+        sink = LangfuseEventSink(config, client=client)
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="model.stream.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="model.stream.delta", run_id="run_1", payload={"delta": "private"}))
+        await sink.emit(RuntimeEvent(type="model.stream.completed", run_id="run_1"))
+        await sink.shutdown()
+
+        stream = [observation for observation in client.observations if observation.kwargs["name"] == "runlet.model.stream"][0]
+        self.assertEqual(stream.updates[-1]["output"], "priv")
+
+    async def test_client_errors_do_not_fail_runtime(self) -> None:
+        client = FakeClient()
+        client.fail = True
+        sink = LangfuseEventSink(LangfuseConfig(public_key="pk", secret_key="sk"), client=client)
+        model = FakeModelProvider([ModelResponse(message=Message.assistant("ok"))])
+        agent = Agent(name="assistant", instructions="Help.", model=model)
+
+        result = await Runtime(event_sink=sink).run(agent, "hi")
+        await sink.shutdown()
+
+        self.assertEqual(result.output, "ok")
+        self.assertGreater(sink.failed_events, 0)
+
+    async def test_sample_rate_zero_skips_observations(self) -> None:
+        client = FakeClient()
+        sink = LangfuseEventSink(
+            LangfuseConfig(public_key="pk", secret_key="sk", sample_rate=0),
+            client=client,
+        )
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="run.completed", run_id="run_1"))
+        await sink.shutdown()
+
+        self.assertEqual(client.observations, [])
+
+    async def test_flush_on_terminal_event(self) -> None:
+        client = FakeClient()
+        config = LangfuseConfig(public_key="pk", secret_key="sk", flush_on_terminal_event=True)
+        sink = LangfuseEventSink(config, client=client)
+
+        await sink.emit(RuntimeEvent(type="run.started", run_id="run_1"))
+        await sink.emit(RuntimeEvent(type="run.completed", run_id="run_1"))
+        await sink.shutdown()
+
+        self.assertGreaterEqual(client.flushed, 1)