traceAI-hermes-agent 0.1.0__py3-none-any.whl

traceai_hermes_agent/__init__.py

@@ -0,0 +1,232 @@
+"""Future AGI tracing plugin for Hermes Agent.
+
+Hermes loads ``register(ctx)`` from the ``hermes_agent.plugins`` entry point
+``futureagi`` once the plugin is enabled (``hermes plugins enable futureagi``).
+It registers observer callbacks on Hermes' documented plugin hooks and exports
+one OpenTelemetry trace per Hermes turn to Future AGI.
+
+Configuration (``$HERMES_HOME/.env`` works; in a multi-profile gateway each
+profile's own ``.env`` is used):
+
+- ``FI_API_KEY`` / ``FI_SECRET_KEY``: required. Without them the plugin stays a no-op.
+- ``FI_BASE_URL``: collector base URL (default ``https://api.futureagi.com``).
+- ``FI_PROJECT_NAME``: Future AGI project (default ``hermes-agent``).
+- ``FI_HERMES_CAPTURE_CONTENT``: ``true`` to also export prompts, responses,
+  tool arguments, tool results and error text. Off by default.
+- ``FI_HERMES_MAX_CHARS``: per-attribute cap on captured text (default 8000).
+
+The plugin never reads Hermes' session files, config or memory: everything it
+exports comes from the hook payloads. It installs no signal handlers and does
+not touch the global OpenTelemetry provider.
+"""
+
+from __future__ import annotations
+
+import atexit
+import json
+import logging
+import os
+import threading
+import uuid
+from typing import Any, Callable, Dict, Optional
+
+from opentelemetry import trace as trace_api
+
+from traceai_hermes_agent._recorder import HOOKS, HermesSpanRecorder
+from traceai_hermes_agent.version import __version__
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+DEFAULT_PROJECT_NAME = "hermes-agent"
+DEFAULT_BASE_URL = "https://api.futureagi.com"
+_TRUE = {"1", "true", "yes", "on"}
+# Exits and session teardown wait at most this long for the last spans.
+EXPORT_TIMEOUT_SECONDS = 3
+FLUSH_TIMEOUT_MILLIS = 2000
+# Hooks that may create the provider. on_session_finalize only flushes: it can
+# run on the main thread at exit, outside any profile's secret scope.
+_BUILD_HOOKS = frozenset(HOOKS) - {"on_session_finalize", "subagent_stop"}
+
+
+class _NotInScope(Exception):
+    """A multi-profile gateway asked for a secret outside any profile's scope."""
+
+
+def _setting(name: str) -> Optional[str]:
+    """Read one setting the way Hermes reads profile secrets.
+
+    In a multi-profile gateway every profile's ``.env`` is kept out of
+    ``os.environ`` and served through ``agent.secret_scope``; outside Hermes
+    (or in a normal single-profile run) this is ``os.environ``.
+    """
+    try:
+        from agent.secret_scope import UnscopedSecretError, get_secret
+    except Exception:
+        return os.environ.get(name)
+    try:
+        return get_secret(name)
+    except UnscopedSecretError as exc:
+        raise _NotInScope(name) from exc
+
+
+def _flag(value: Optional[str]) -> bool:
+    return (value or "").strip().lower() in _TRUE
+
+
+def _int(value: Optional[str], default: int) -> int:
+    try:
+        return int((value or "").strip() or default)
+    except ValueError:
+        return default
+
+
+def build_tracer_provider(*, api_key: str, secret_key: str, base_url: str,
+                          project_name: str) -> Any:
+    """A private TracerProvider exporting to ``$base_url/tracer/v1/traces``.
+
+    Built directly instead of through ``fi_instrumentation.register()``, which
+    installs SIGINT/SIGTERM handlers and reads keys from ``os.environ``:
+    neither is acceptable inside the Hermes process. Resource attributes match
+    what ``register()`` sends for an observe project.
+    """
+    from opentelemetry.exporter.otlp.proto.http import Compression
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+    resource = Resource(attributes={
+        "project_name": project_name,
+        "project_type": "observe",
+        "project_version_name": "DEFAULT_PROJECT_VERSION_NAME",
+        "project_version_id": str(uuid.uuid4()),
+        "eval_tags": json.dumps([]),
+        "metadata": json.dumps({}),
+        "semantic_convention": "fi",
+        "service.name": "hermes-agent",
+    })
+    exporter = OTLPSpanExporter(
+        endpoint=base_url.rstrip("/") + "/tracer/v1/traces",
+        headers={"X-Api-Key": api_key, "X-Secret-Key": secret_key},
+        timeout=EXPORT_TIMEOUT_SECONDS,
+        # The Future AGI collector does not read Content-Encoding.
+        compression=Compression.NoCompression,
+    )
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(BatchSpanProcessor(exporter, export_timeout_millis=FLUSH_TIMEOUT_MILLIS))
+    return provider
+
+
+class FutureAGIHermesPlugin:
+    """Owns the tracer provider and the recorder; built lazily on the first turn hook.
+
+    Lazy so that ``hermes plugins list`` and other commands that only import
+    plugins never start an exporter, so that a missing key is reported once,
+    and so that in a multi-profile gateway the settings are read inside the
+    profile's own secret scope. Hermes creates one plugin instance per profile.
+    """
+
+    def __init__(self, tracer_provider: Optional[Any] = None) -> None:
+        self._provider = tracer_provider
+        self._owns_provider = tracer_provider is None
+        self._recorder: Optional[HermesSpanRecorder] = None
+        self._disabled = False
+        self._lock = threading.Lock()
+        self.settings: Dict[str, Any] = {}
+
+    def recorder(self, build: bool = True) -> Optional[HermesSpanRecorder]:
+        if self._recorder is not None or self._disabled or not build:
+            return self._recorder
+        with self._lock:
+            if self._recorder is not None or self._disabled:
+                return self._recorder
+            try:
+                self._recorder = self._build()
+            except _NotInScope:
+                return None  # try again on a hook that runs in a profile's scope
+            except Exception:
+                logger.warning("traceAI-hermes-agent: tracing disabled, setup failed", exc_info=True)
+                self._recorder = None
+            if self._recorder is None:
+                self._disabled = True
+            return self._recorder
+
+    def _build(self) -> Optional[HermesSpanRecorder]:
+        capture = _flag(_setting("FI_HERMES_CAPTURE_CONTENT"))
+        max_chars = _int(_setting("FI_HERMES_MAX_CHARS"), 8000)
+        if self._provider is None:
+            api_key, secret_key = _setting("FI_API_KEY"), _setting("FI_SECRET_KEY")
+            if not (api_key and secret_key):
+                logger.warning(
+                    "traceAI-hermes-agent: FI_API_KEY and FI_SECRET_KEY are not set; "
+                    "Hermes turns are not exported"
+                )
+                return None
+            self.settings = {
+                "base_url": _setting("FI_BASE_URL") or DEFAULT_BASE_URL,
+                "project_name": _setting("FI_PROJECT_NAME") or DEFAULT_PROJECT_NAME,
+            }
+            self._provider = build_tracer_provider(api_key=api_key, secret_key=secret_key,
+                                                   **self.settings)
+            atexit.register(self.shutdown)
+        self.settings.update(capture_content=capture, max_chars=max_chars)
+        tracer = trace_api.get_tracer(__name__, __version__, tracer_provider=self._provider)
+        return HermesSpanRecorder(tracer, capture_content=capture, max_chars=max_chars)
+
+    def callback(self, hook: str) -> Callable[..., None]:
+        method = "on_" + hook if not hook.startswith("on_") else hook
+        build = hook in _BUILD_HOOKS
+        # A long-lived gateway exports a session as soon as Hermes tears it down.
+        flush_after = hook == "on_session_finalize"
+
+        def _callback(**kwargs: Any) -> None:
+            try:
+                recorder = self.recorder(build=build)
+                if recorder is not None:
+                    getattr(recorder, method)(**kwargs)
+                    if flush_after:
+                        self.flush()
+            except Exception:  # never let tracing break Hermes
+                logger.debug("traceAI-hermes-agent: %s callback failed", hook, exc_info=True)
+            return None
+
+        _callback.__name__ = "futureagi_" + hook
+        return _callback
+
+    def flush(self) -> None:
+        provider = self._provider
+        if provider is not None and hasattr(provider, "force_flush"):
+            try:
+                provider.force_flush(timeout_millis=FLUSH_TIMEOUT_MILLIS)
+            except Exception:
+                logger.debug("traceAI-hermes-agent: flush failed", exc_info=True)
+
+    def shutdown(self) -> None:
+        """Close open spans (marked unfinished) and export them; runs at process exit.
+
+        Bounded: an unreachable collector delays exit by at most a few seconds.
+        """
+        if self._recorder is not None:
+            self._recorder.end_all()
+        if self._owns_provider and self._provider is not None and hasattr(self._provider, "shutdown"):
+            try:
+                self._provider.shutdown()
+            except Exception:
+                logger.debug("traceAI-hermes-agent: shutdown failed", exc_info=True)
+        else:
+            self.flush()
+
+
+def register(ctx: Any, tracer_provider: Optional[Any] = None) -> None:
+    """Hermes plugin entry point: register the tracing callbacks on ``ctx``.
+
+    ``tracer_provider`` is for tests and embedders; Hermes calls ``register(ctx)``.
+    """
+    plugin = FutureAGIHermesPlugin(tracer_provider)
+    for hook in HOOKS:
+        ctx.register_hook(hook, plugin.callback(hook))
+
+
+__all__ = ["FutureAGIHermesPlugin", "HermesSpanRecorder", "build_tracer_provider", "register",
+           "__version__"]

traceai_hermes_agent/_recorder.py

@@ -0,0 +1,605 @@
+"""Turn Hermes Agent plugin-hook payloads into OpenTelemetry spans.
+
+One trace per Hermes turn:
+
+    hermes.turn                 AGENT   pre_llm_call ... on_session_end
+      chat <model>              LLM     pre_api_request ... post_api_request / api_request_error
+      <tool name>               TOOL    post_tool_call (start = end - duration_ms)
+
+Every turn carries ``session.id`` (the Hermes session id), so the turns of one
+``hermes chat`` session group under one Future AGI session.
+
+Hooks can fire on Hermes worker threads, so spans are tracked by the ids the
+hooks carry (turn_id, api_request_id, tool_call_id) and parented explicitly,
+never through the ambient OpenTelemetry context.
+
+Prompt, response, tool-argument, tool-result and error text is only read
+when ``capture_content`` is true. With it off, the only things exported are
+ids, model and provider names, token counts, sizes, durations, outcomes and
+Hermes' error categories (never the error messages).
+
+The plugin observes only: it does not register on ``pre_tool_call``, which
+Hermes treats as a fail-closed policy hook.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping, Optional
+from urllib.parse import urlparse
+
+from opentelemetry import context as context_api
+from opentelemetry import trace as trace_api
+from opentelemetry.trace import Span, Status, StatusCode
+
+logger = logging.getLogger(__name__)
+
+SPAN_KIND = "gen_ai.span.kind"
+OPERATION = "gen_ai.operation.name"
+SESSION_ID = "session.id"
+CONVERSATION_ID = "gen_ai.conversation.id"
+INPUT_VALUE = "input.value"
+INPUT_MIME = "input.mime_type"
+OUTPUT_VALUE = "output.value"
+OUTPUT_MIME = "output.mime_type"
+JSON_MIME = "application/json"
+TEXT_MIME = "text/plain"
+
+TURN_SPAN_NAME = "hermes.turn"
+# Tool statuses Hermes reports on post_tool_call that are not a success.
+_FAILED_TOOL_STATUSES = frozenset({"error", "blocked", "timeout", "cancelled"})
+_TRUNCATED = "...[truncated {0} chars]"
+
+
+@dataclass
+class _Turn:
+    span: Span
+    session_id: str
+    children: List[str] = field(default_factory=list)
+
+
+def _ns(epoch_seconds: Any) -> Optional[int]:
+    """Epoch seconds (Hermes' started_at / ended_at) to OTel nanoseconds."""
+    if isinstance(epoch_seconds, bool) or not isinstance(epoch_seconds, (int, float)):
+        return None
+    if epoch_seconds <= 0:
+        return None
+    return int(epoch_seconds * 1_000_000_000)
+
+
+def _str(value: Any) -> str:
+    return value if isinstance(value, str) else ("" if value is None else str(value))
+
+
+def _int(value: Any) -> Optional[int]:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float) and value.is_integer():
+        return int(value)
+    return None
+
+
+def _host(url: Any) -> Optional[str]:
+    """Only the host of a provider base URL; a URL can carry credentials."""
+    if not isinstance(url, str) or not url:
+        return None
+    try:
+        return urlparse(url).hostname
+    except ValueError:
+        return None
+
+
+def _set(span: Span, attributes: Mapping[str, Any]) -> None:
+    for key, value in attributes.items():
+        if value is None or value == "":
+            continue
+        span.set_attribute(key, value)
+
+
+class HermesSpanRecorder:
+    """Maps the documented Hermes plugin hooks to spans on one tracer.
+
+    Each ``on_*`` method takes the hook's keyword arguments. Methods never raise
+    and always return None (never a directive), so a recorder bug cannot change
+    what Hermes does.
+    """
+
+    def __init__(self, tracer: trace_api.Tracer, *, capture_content: bool = False,
+                 max_chars: int = 8000) -> None:
+        self._tracer = tracer
+        self._capture = bool(capture_content)
+        self._max_chars = max(256, int(max_chars))
+        self._lock = threading.RLock()
+        self._turns: Dict[str, _Turn] = {}
+        # open LLM/tool spans: key -> (span, turn key)
+        self._children: Dict[str, tuple] = {}
+        # session id -> most recent open turn key (subagent parenting, session end)
+        self._session_turn: Dict[str, str] = {}
+        # child session id -> parent turn key (from subagent_start)
+        self._subagent_parent: Dict[str, str] = {}
+        # api_request_ids whose earlier attempt already ended (retries)
+        self._attempted: Dict[str, int] = {}
+        # LLM span key -> start time in epoch seconds (time to first chunk)
+        self._llm_start: Dict[str, float] = {}
+
+    @property
+    def capture_content(self) -> bool:
+        return self._capture
+
+    # ------------------------------------------------------------------ helpers
+
+    def _text(self, value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            text = value
+        else:
+            try:
+                text = json.dumps(value, ensure_ascii=False, default=str)
+            except (TypeError, ValueError):
+                text = str(value)
+        if len(text) > self._max_chars:
+            text = text[: self._max_chars] + _TRUNCATED.format(len(text) - self._max_chars)
+        return text
+
+    def _content(self, span: Span, key: str, mime_key: str, value: Any) -> None:
+        if not self._capture or value is None:
+            return
+        text = self._text(value)
+        if text is None:
+            return
+        span.set_attribute(key, text)
+        span.set_attribute(mime_key, TEXT_MIME if isinstance(value, str) else JSON_MIME)
+
+    @staticmethod
+    def _turn_key(kw: Mapping[str, Any]) -> str:
+        turn_id = _str(kw.get("turn_id"))
+        if turn_id:
+            return turn_id
+        return "{0}:{1}".format(_str(kw.get("session_id")), _str(kw.get("task_id")))
+
+    def _ensure_turn(self, kw: Mapping[str, Any]) -> _Turn:
+        key = self._turn_key(kw)
+        session_id = _str(kw.get("session_id"))
+        with self._lock:
+            turn = self._turns.get(key)
+            if turn is not None:
+                return turn
+            parent_key = self._subagent_parent.get(session_id)
+            parent_session = _str(kw.get("parent_session_id"))
+            if parent_key is None and parent_session:
+                parent_key = self._session_turn.get(parent_session)
+            parent = self._turns.get(parent_key) if parent_key else None
+            # An empty Context makes this a root span even if something else
+            # in the Hermes process left a span current on this thread.
+            ctx = (trace_api.set_span_in_context(parent.span) if parent is not None
+                   else context_api.Context())
+            span = self._tracer.start_span(TURN_SPAN_NAME, context=ctx)
+            _set(span, {
+                SPAN_KIND: "AGENT",
+                OPERATION: "invoke_agent",
+                "gen_ai.agent.name": "hermes",
+                SESSION_ID: session_id,
+                CONVERSATION_ID: session_id,
+                "hermes.session_id": session_id,
+                "hermes.turn_id": _str(kw.get("turn_id")),
+                "hermes.task_id": _str(kw.get("task_id")),
+                "hermes.platform": _str(kw.get("platform")),
+                "hermes.model": _str(kw.get("model")),
+                "hermes.parent_session_id": parent_session,
+            })
+            if isinstance(kw.get("is_first_turn"), bool):
+                span.set_attribute("hermes.is_first_turn", kw["is_first_turn"])
+            turn = _Turn(span=span, session_id=session_id)
+            self._turns[key] = turn
+            if session_id:
+                self._session_turn[session_id] = key
+            return turn
+
+    def _start_child(self, key: str, name: str, kw: Mapping[str, Any],
+                     start_time: Optional[int] = None) -> Span:
+        turn = self._ensure_turn(kw)
+        span = self._tracer.start_span(
+            name, context=trace_api.set_span_in_context(turn.span), start_time=start_time,
+        )
+        if turn.session_id:
+            span.set_attribute(SESSION_ID, turn.session_id)
+        with self._lock:
+            stale = self._children.pop(key, None)
+            self._children[key] = (span, self._turn_key(kw))
+            turn.children.append(key)
+        if stale is not None:
+            # Same id started twice without an end hook: close the first one.
+            stale[0].set_attribute("hermes.unfinished", True)
+            stale[0].end()
+        return span
+
+    def _pop_child(self, key: str) -> Optional[Span]:
+        with self._lock:
+            entry = self._children.pop(key, None)
+        return entry[0] if entry else None
+
+    def _end_turn(self, key: str, *, attributes: Mapping[str, Any],
+                  status: Optional[Status] = None) -> None:
+        with self._lock:
+            turn = self._turns.pop(key, None)
+            if turn is None:
+                return
+            if self._session_turn.get(turn.session_id) == key:
+                del self._session_turn[turn.session_id]
+            orphans = [self._children.pop(k)[0] for k in turn.children if k in self._children]
+            for child in turn.children:
+                if child.startswith("llm:"):
+                    self._attempted.pop(child[4:], None)
+                    self._llm_start.pop(child, None)
+        for span in orphans:
+            span.set_attribute("hermes.unfinished", True)
+            span.end()
+        _set(turn.span, attributes)
+        if status is not None:
+            turn.span.set_status(status)
+        turn.span.end()
+
+    def _guard(self, hook: str, fn, kw: Mapping[str, Any]) -> None:
+        try:
+            fn(kw)
+        except Exception:  # never let tracing break the agent loop
+            logger.debug("traceAI-hermes-agent: %s handler failed", hook, exc_info=True)
+        return None
+
+    # -------------------------------------------------------------------- turns
+
+    def on_pre_llm_call(self, **kw: Any) -> None:
+        return self._guard("pre_llm_call", self._pre_llm_call, kw)
+
+    def _pre_llm_call(self, kw: Mapping[str, Any]) -> None:
+        turn = self._ensure_turn(kw)
+        self._content(turn.span, INPUT_VALUE, INPUT_MIME, kw.get("user_message"))
+
+    def on_post_llm_call(self, **kw: Any) -> None:
+        return self._guard("post_llm_call", self._post_llm_call, kw)
+
+    def _post_llm_call(self, kw: Mapping[str, Any]) -> None:
+        with self._lock:
+            turn = self._turns.get(self._turn_key(kw))
+        if turn is not None:
+            self._content(turn.span, OUTPUT_VALUE, OUTPUT_MIME, kw.get("assistant_response"))
+
+    def on_session_end(self, **kw: Any) -> None:
+        return self._guard("on_session_end", self._session_end, kw)
+
+    def _exit_reason(self, raw: Any) -> str:
+        """Hermes' exit reason, or only its category unless content capture is on.
+
+        Hermes embeds exception text in some reasons, e.g.
+        ``repeated_outer_errors(Error during API call #3: <str(e)>)``.
+        """
+        reason = _str(raw).strip()
+        if self._capture or not reason:
+            return self._text(reason) or ""
+        return reason.split("(", 1)[0].strip()
+
+    def _session_end(self, kw: Mapping[str, Any]) -> None:
+        failed = kw.get("failed") is True
+        interrupted = kw.get("interrupted") is True
+        reason = self._exit_reason(kw.get("turn_exit_reason") or kw.get("reason"))
+        attributes = {
+            "hermes.completed": kw.get("completed") if isinstance(kw.get("completed"), bool) else None,
+            "hermes.failed": failed,
+            "hermes.interrupted": interrupted,
+            "hermes.turn_exit_reason": reason,
+        }
+        # Turns closed without this hook (session teardown, exit) stay UNSET.
+        status = Status(StatusCode.ERROR, reason or "turn failed") if failed else Status(StatusCode.OK)
+        if _str(kw.get("turn_id")):
+            self._end_turn(self._turn_key(kw), attributes=attributes, status=status)
+            return
+        # CLI/TUI exit shapes carry no turn_id: close whatever is open for the session.
+        self._end_session(_str(kw.get("session_id")), attributes=attributes, status=status)
+
+    def on_session_finalize(self, **kw: Any) -> None:
+        return self._guard("on_session_finalize", self._session_finalize, kw)
+
+    def _session_finalize(self, kw: Mapping[str, Any]) -> None:
+        for session_id in (kw.get("session_id"), kw.get("old_session_id")):
+            if session_id:
+                self._end_session(_str(session_id), attributes={"hermes.unfinished": True})
+
+    def _end_session(self, session_id: str, *, attributes: Mapping[str, Any],
+                     status: Optional[Status] = None) -> None:
+        with self._lock:
+            keys = [k for k, t in self._turns.items() if t.session_id == session_id]
+        for key in keys:
+            self._end_turn(key, attributes=attributes, status=status)
+
+    def on_subagent_start(self, **kw: Any) -> None:
+        return self._guard("subagent_start", self._subagent_start, kw)
+
+    def _subagent_start(self, kw: Mapping[str, Any]) -> None:
+        child = _str(kw.get("child_session_id"))
+        parent_turn = _str(kw.get("parent_turn_id"))
+        if not child:
+            return
+        with self._lock:
+            if parent_turn and parent_turn in self._turns:
+                self._subagent_parent[child] = parent_turn
+            else:
+                parent_key = self._session_turn.get(_str(kw.get("parent_session_id")))
+                if parent_key:
+                    self._subagent_parent[child] = parent_key
+
+    def on_subagent_stop(self, **kw: Any) -> None:
+        return self._guard("subagent_stop", self._subagent_stop, kw)
+
+    def _subagent_stop(self, kw: Mapping[str, Any]) -> None:
+        with self._lock:
+            self._subagent_parent.pop(_str(kw.get("child_session_id")), None)
+
+    # ---------------------------------------------------------------- LLM calls
+
+    @staticmethod
+    def _llm_key(kw: Mapping[str, Any]) -> str:
+        return "llm:" + _str(kw.get("api_request_id"))
+
+    def on_pre_api_request(self, **kw: Any) -> None:
+        return self._guard("pre_api_request", self._pre_api_request, kw)
+
+    def _attempt_start(self, kw: Mapping[str, Any]) -> float:
+        """Start of this provider attempt, in epoch seconds.
+
+        Hermes sets ``started_at`` once per logical call, before its retry
+        loop, so every retry carries the first attempt's start. Only the first
+        attempt uses it; a retry starts when its own hook fires.
+        """
+        started = kw.get("started_at")
+        retry = _int(kw.get("retry_count")) or 0
+        with self._lock:
+            earlier = self._attempted.get(_str(kw.get("api_request_id")), 0)
+        if retry == 0 and not earlier and isinstance(started, (int, float)) and started > 0:
+            return float(started)
+        return time.time()
+
+    def _pre_api_request(self, kw: Mapping[str, Any]) -> None:
+        model = _str(kw.get("model"))
+        key = self._llm_key(kw)
+        start = self._attempt_start(kw)
+        span = self._start_child(key, "chat {0}".format(model) if model else "chat",
+                                 kw, start_time=_ns(start))
+        with self._lock:
+            self._llm_start[key] = start
+        _set(span, {
+            SPAN_KIND: "LLM",
+            OPERATION: "chat",
+            "gen_ai.request.model": model,
+            "gen_ai.provider.name": _str(kw.get("provider")),
+            "server.address": _host(kw.get("base_url")),
+            "hermes.api_mode": _str(kw.get("api_mode")),
+            "hermes.api_request_id": _str(kw.get("api_request_id")),
+            "hermes.api_call_count": _int(kw.get("api_call_count")),
+            "hermes.retry_count": _int(kw.get("retry_count")),
+            "hermes.message_count": _int(kw.get("message_count")),
+            "hermes.tool_count": _int(kw.get("tool_count")),
+            "hermes.approx_input_tokens": _int(kw.get("approx_input_tokens")),
+            "hermes.request_char_count": _int(kw.get("request_char_count")),
+            "gen_ai.request.max_tokens": _int(kw.get("max_tokens")),
+        })
+        if self._capture:
+            self._content(span, INPUT_VALUE, INPUT_MIME, self._llm_input(kw))
+
+    def _llm_input(self, kw: Mapping[str, Any]) -> Any:
+        """The latest messages of the request, newest kept when over the cap.
+
+        The head of the request body is the system prompt, which alone fills
+        the cap; it is reported by size only.
+        """
+        request = kw.get("request")
+        body = request.get("body") if isinstance(request, Mapping) else None
+        messages = None
+        if isinstance(body, Mapping):
+            messages = body.get("messages") or body.get("input")
+        elif isinstance(request, Mapping):
+            messages = request.get("messages")
+        if not isinstance(messages, list):
+            messages = kw.get("request_messages")
+        if not isinstance(messages, list):
+            return None
+        system_chars = 0
+        rest = []
+        for message in messages:
+            if isinstance(message, Mapping) and message.get("role") in ("system", "developer"):
+                system_chars += len(_str(message.get("content")))
+            else:
+                rest.append(message)
+        if isinstance(body, Mapping) and isinstance(body.get("system"), (str, list)):
+            system_chars += len(_str(body.get("system")))
+        tail: List[Any] = []
+        budget = self._max_chars - 200
+        for message in reversed(rest):
+            size = len(json.dumps(message, ensure_ascii=False, default=str))
+            if tail and size > budget:
+                break
+            tail.insert(0, message)
+            budget -= size
+        return {"system_prompt_chars": system_chars, "message_count": len(messages),
+                "messages": tail}
+
+    def on_post_api_request(self, **kw: Any) -> None:
+        return self._guard("post_api_request", self._post_api_request, kw)
+
+    def _llm_span_to_end(self, kw: Mapping[str, Any]) -> Span:
+        """The open LLM span for this request, or one started late if its pre hook never fired."""
+        key = self._llm_key(kw)
+        span = self._pop_child(key)
+        if span is not None:
+            return span
+        model = _str(kw.get("model"))
+        start = self._attempt_start(kw)
+        span = self._start_child(key, "chat {0}".format(model) if model else "chat", kw,
+                                 start_time=_ns(start))
+        self._pop_child(key)
+        with self._lock:
+            self._llm_start[key] = start
+        _set(span, {SPAN_KIND: "LLM", OPERATION: "chat", "gen_ai.request.model": model,
+                    "gen_ai.provider.name": _str(kw.get("provider"))})
+        return span
+
+    def _post_api_request(self, kw: Mapping[str, Any]) -> None:
+        span = self._llm_span_to_end(kw)
+        usage = kw.get("usage") if isinstance(kw.get("usage"), Mapping) else {}
+        finish = _str(kw.get("finish_reason"))
+        _set(span, {
+            "gen_ai.response.model": _str(kw.get("response_model")),
+            "gen_ai.response.finish_reasons": [finish] if finish else None,
+            # Hermes' prompt_tokens already includes cache reads and writes.
+            "gen_ai.usage.input_tokens": _int(usage.get("prompt_tokens")),
+            "gen_ai.usage.output_tokens": _int(usage.get("output_tokens")),
+            "gen_ai.usage.total_tokens": _int(usage.get("total_tokens")),
+            "gen_ai.usage.cache_read_tokens": _int(usage.get("cache_read_tokens")),
+            "gen_ai.usage.cache_write_tokens": _int(usage.get("cache_write_tokens")),
+            "gen_ai.usage.output_tokens.reasoning": _int(usage.get("reasoning_tokens")),
+            "hermes.assistant_tool_call_count": _int(kw.get("assistant_tool_call_count")),
+            "hermes.assistant_content_chars": _int(kw.get("assistant_content_chars")),
+        })
+        start = self._finish_attempt(kw)
+        first_chunk = kw.get("first_chunk_at")
+        if isinstance(first_chunk, (int, float)) and start is not None:
+            span.set_attribute("hermes.time_to_first_chunk_ms",
+                               round(max(0.0, first_chunk - start) * 1000))
+        if self._capture:
+            self._content(span, OUTPUT_VALUE, OUTPUT_MIME, _assistant_output(kw.get("assistant_message")))
+        span.set_status(Status(StatusCode.OK))
+        span.end(end_time=self._end_time(kw, start))
+
+    def _finish_attempt(self, kw: Mapping[str, Any]) -> Optional[float]:
+        """Record that this request id had an attempt; return that attempt's start."""
+        key = self._llm_key(kw)
+        with self._lock:
+            request_id = _str(kw.get("api_request_id"))
+            self._attempted[request_id] = self._attempted.get(request_id, 0) + 1
+            return self._llm_start.pop(key, None)
+
+    @staticmethod
+    def _end_time(kw: Mapping[str, Any], start: Optional[float]) -> Optional[int]:
+        ended = kw.get("ended_at")
+        if isinstance(ended, (int, float)) and start is not None and ended >= start:
+            return _ns(ended)
+        # ended_at is measured from the logical call's start; for a retry that
+        # can be earlier than the attempt's own start. Use "now".
+        return None if not isinstance(ended, (int, float)) else time.time_ns()
+
+    def on_api_request_error(self, **kw: Any) -> None:
+        return self._guard("api_request_error", self._api_request_error, kw)
+
+    def _api_request_error(self, kw: Mapping[str, Any]) -> None:
+        span = self._llm_span_to_end(kw)
+        reason = _str(kw.get("reason"))
+        error = kw.get("error")
+        # Hermes sends error={"type": <class or category>, "message": <text>}.
+        error_type = _str(error.get("type")) if isinstance(error, Mapping) else (
+            type(error).__name__ if isinstance(error, BaseException) else "")
+        error_message = error.get("message") if isinstance(error, Mapping) else error
+        _set(span, {
+            "error.type": reason or error_type,
+            "hermes.error_class": error_type if reason else None,
+            "http.response.status_code": _int(kw.get("status_code")),
+            "hermes.retry_count": _int(kw.get("retry_count")),
+            "hermes.max_retries": _int(kw.get("max_retries")),
+            "hermes.retryable": kw.get("retryable") if isinstance(kw.get("retryable"), bool) else None,
+        })
+        # Provider error text can echo the prompt or local paths: it is content.
+        description = reason or error_type or "provider request failed"
+        if self._capture and error_message:
+            description = self._text(str(error_message)) or description
+        span.set_status(Status(StatusCode.ERROR, description))
+        start = self._finish_attempt(kw)
+        span.end(end_time=self._end_time(kw, start))
+
+    # --------------------------------------------------------------- tool calls
+
+    def _tool_key(self, kw: Mapping[str, Any]) -> str:
+        call_id = _str(kw.get("tool_call_id"))
+        if call_id:
+            return "tool:" + call_id
+        return "tool:{0}:{1}".format(self._turn_key(kw), _str(kw.get("tool_name")))
+
+    def on_post_tool_call(self, **kw: Any) -> None:
+        return self._guard("post_tool_call", self._post_tool_call, kw)
+
+    def _post_tool_call(self, kw: Mapping[str, Any]) -> None:
+        # Built from post_tool_call alone, timed by Hermes' duration_ms.
+        key = self._tool_key(kw)
+        duration = max(0, _int(kw.get("duration_ms")) or 0)
+        name = _str(kw.get("tool_name")) or "tool"
+        end = time.time_ns()
+        span = self._start_child(key, name, kw, start_time=end - duration * 1_000_000)
+        self._pop_child(key)
+        _set(span, {
+            SPAN_KIND: "TOOL",
+            OPERATION: "execute_tool",
+            "gen_ai.tool.name": name,
+            "gen_ai.tool.call.id": _str(kw.get("tool_call_id")),
+            "hermes.api_request_id": _str(kw.get("api_request_id")),
+        })
+        self._content(span, INPUT_VALUE, INPUT_MIME, kw.get("args"))
+        status = _str(kw.get("status"))
+        error_type = _str(kw.get("error_type"))
+        _set(span, {
+            "hermes.tool.status": status,
+            "hermes.tool.duration_ms": _int(kw.get("duration_ms")),
+            "error.type": error_type,
+        })
+        self._content(span, OUTPUT_VALUE, OUTPUT_MIME, kw.get("result"))
+        if status in _FAILED_TOOL_STATUSES or error_type:
+            description = error_type or status
+            if self._capture and kw.get("error_message"):
+                description = self._text(kw.get("error_message")) or description
+            span.set_status(Status(StatusCode.ERROR, description))
+        else:
+            span.set_status(Status(StatusCode.OK))
+        span.end(end_time=end)
+
+    # ----------------------------------------------------------------- shutdown
+
+    def end_all(self) -> None:
+        """Close every open span (process exit); they are marked unfinished."""
+        with self._lock:
+            keys = list(self._turns)
+        for key in keys:
+            self._end_turn(key, attributes={"hermes.unfinished": True})
+        with self._lock:
+            leftovers = [entry[0] for entry in self._children.values()]
+            self._children.clear()
+        for span in leftovers:
+            span.set_attribute("hermes.unfinished", True)
+            span.end()
+
+
+def _assistant_output(message: Any) -> Any:
+    """Text, or the requested tool calls, from Hermes' normalized assistant message."""
+    if message is None:
+        return None
+    content = message.get("content") if isinstance(message, Mapping) else getattr(message, "content", None)
+    if isinstance(content, str) and content:
+        return content
+    calls = (message.get("tool_calls") if isinstance(message, Mapping)
+             else getattr(message, "tool_calls", None)) or []
+    out = []
+    for call in calls:
+        fn = call.get("function") if isinstance(call, Mapping) else getattr(call, "function", None)
+        name = fn.get("name") if isinstance(fn, Mapping) else getattr(fn, "name", None)
+        args = fn.get("arguments") if isinstance(fn, Mapping) else getattr(fn, "arguments", None)
+        out.append({"name": name, "arguments": args})
+    return out or content
+
+
+HOOKS: Iterable[str] = (
+    "pre_llm_call", "post_llm_call", "pre_api_request", "post_api_request",
+    "api_request_error", "post_tool_call", "on_session_end",
+    "on_session_finalize", "subagent_start", "subagent_stop",
+)

traceai_hermes_agent/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"  # x-release-please-version

traceai_hermes_agent-0.1.0.dist-info/METADATA

@@ -0,0 +1,203 @@
+Metadata-Version: 2.4
+Name: traceAI-hermes-agent
+Version: 0.1.0
+Summary: Future AGI tracing plugin for Hermes Agent: turns, model calls and tool calls as OpenTelemetry spans
+License: Apache-2.0
+Keywords: opentelemetry,hermes-agent,nous-research,agent,tracing,observability,traceai,futureagi,llm
+Author: Future AGI
+Author-email: no-reply@futureagi.com
+Requires-Python: >=3.11,<3.15
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: OpenTelemetry
+Classifier: Framework :: OpenTelemetry :: Instrumentations
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Monitoring
+Requires-Dist: opentelemetry-api (>=1.35.0)
+Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.35.0)
+Requires-Dist: opentelemetry-sdk (>=1.35.0)
+Project-URL: Changelog, https://github.com/future-agi/traceAI/blob/dev/python/frameworks/hermes-agent/CHANGELOG.md
+Project-URL: Documentation, https://docs.futureagi.com/docs/tracing
+Project-URL: Homepage, https://futureagi.com
+Project-URL: Issues, https://github.com/future-agi/traceAI/issues
+Project-URL: Repository, https://github.com/future-agi/traceAI/tree/dev/python/frameworks/hermes-agent
+Description-Content-Type: text/markdown
+
+# traceAI-hermes-agent
+
+A [Hermes Agent](https://hermes-agent.nousresearch.com/) plugin that exports
+Hermes turns, model calls and tool calls to Future AGI as OpenTelemetry spans.
+It registers observer callbacks on Hermes' documented plugin hooks; it is not
+part of Hermes and does not change what Hermes does.
+
+```
+hermes.turn            AGENT  one user message to final answer (session, platform, outcome)
+├── chat gpt-6-luna    LLM    one provider attempt: model, provider, tokens, finish reason
+├── read_file          TOOL   one tool call: name, call id, duration, status
+└── chat gpt-6-luna    LLM
+```
+
+One trace per top-level Hermes turn. Every span carries `session.id` (the Hermes
+session id), so the turns of a `hermes chat` session group into one Future AGI
+session. A subagent's turn sits inside its parent turn's trace and carries the
+subagent's own session id.
+
+Alpha. Tested with Hermes Agent v2026.9.24 (0.21.5) on Python 3.12, in CLI
+sessions against an OpenAI-compatible provider. Supports Python 3.11 to 3.14;
+the test suite also passes on Python 3.14 against Hermes v0.22.0, which runs on 3.14.
+
+## Install
+
+The plugin is enabled per Hermes home and stays off until you enable it.
+
+**Managed Hermes installs** (the `install.sh` installer, Hermes Desktop): install
+the directory plugin in `hermes-plugin/futureagi/`. `hermes plugins install`
+installs `traceAI-hermes-agent` from its `pyproject.toml` into Hermes'
+environment (and `enable` retries it if that failed).
+
+```bash
+hermes plugins install future-agi/traceAI/python/frameworks/hermes-agent/hermes-plugin/futureagi
+hermes plugins enable futureagi
+```
+
+**Hermes in a Python environment you manage** (a source checkout, a container
+image): install the package into the environment Hermes runs in. It declares the
+`hermes_agent.plugins` entry point `futureagi`.
+
+```bash
+pip install traceAI-hermes-agent
+hermes plugins enable futureagi
+```
+
+To try an unreleased change, install from a traceAI checkout instead:
+`pip install /path/to/traceAI/python/frameworks/hermes-agent`.
+
+## Configure
+
+Put the keys in `$HERMES_HOME/.env` (normally `~/.hermes/.env`) or in the
+environment Hermes runs in:
+
+```bash
+FI_API_KEY=your-futureagi-api-key
+FI_SECRET_KEY=your-futureagi-secret-key
+FI_PROJECT_NAME=hermes-agent
+```
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `FI_API_KEY`, `FI_SECRET_KEY` | none | Required. Without them the plugin logs one warning and does nothing. |
+| `FI_BASE_URL` | `https://api.futureagi.com` | Spans go to `$FI_BASE_URL/tracer/v1/traces`. |
+| `FI_PROJECT_NAME` | `hermes-agent` | Future AGI project. |
+| `FI_HERMES_CAPTURE_CONTENT` | off | `true` also exports prompts, responses, tool arguments and results, and error text. |
+| `FI_HERMES_MAX_CHARS` | `8000` | Per-attribute cap on captured text. |
+
+In a multi-profile gateway (`gateway.multiplex_profiles`), each profile's turns
+use that profile's own `.env` values, read through Hermes' profile secret scope.
+A profile without keys is not traced; it never borrows another profile's keys.
+
+Restart Hermes (and its gateway, if you run one) after enabling the plugin.
+
+## What is captured
+
+| Hermes hook | Span | Fields |
+| --- | --- | --- |
+| `pre_llm_call` | `hermes.turn` start | `session.id`, `hermes.turn_id`, `hermes.task_id`, `hermes.platform`, `hermes.model`, first-turn flag, parent session |
+| `pre_api_request` | `chat <model>` start | `gen_ai.request.model`, `gen_ai.provider.name`, `server.address` (host only), API mode, attempt and retry counts, message and tool counts, approximate input tokens |
+| `post_api_request` | `chat <model>` end | `gen_ai.response.model`, finish reason, `gen_ai.usage.input_tokens` (includes cache), `output_tokens`, `total_tokens`, cache read/write tokens, reasoning tokens, time to first chunk |
+| `api_request_error` | `chat <model>`, status `ERROR` | Hermes' error category (`rate_limit`, `overloaded`, `timeout`, ...) as `error.type` and status, the error class as `hermes.error_class`, HTTP status, retryable flag |
+| `post_tool_call` | span named after the tool | `gen_ai.tool.name`, `gen_ai.tool.call.id`, status, duration (the span is timed by Hermes' `duration_ms`); `ERROR` with Hermes' error type when the tool reports `error`, `blocked`, `timeout` or `cancelled` |
+| `on_session_end` | `hermes.turn` end | completed / failed / interrupted, exit reason category; `ERROR` when the turn failed, otherwise `OK` |
+| `subagent_start` | | parents the child session's turn under the parent turn |
+| `on_session_finalize`, process exit | | closes spans that are still open (marked `hermes.unfinished`) and flushes |
+
+Each provider attempt is its own span. The first attempt starts at the
+`started_at` Hermes reports; a retry starts when its own `pre_api_request`
+fires, so a failed attempt and its retry do not overlap.
+
+The plugin does not register on `pre_tool_call`: Hermes treats that hook as a
+fail-closed policy hook (a slow or failing callback blocks the tool), and its
+callbacks never return a directive. A tool still running when Hermes exits
+therefore has no span; a tool cancelled with Ctrl+C is reported, with status
+`cancelled`.
+
+## What is not captured
+
+- Cost. The hooks carry token counts, not Hermes' cost estimate.
+- Auxiliary model calls (titling, compression, vision, approvals) and streaming deltas.
+- Gateway delivery, cron and kanban events, which are not agent turns.
+- Gateway (messaging platform) sessions were not run in testing; they use the same hooks.
+
+## Privacy
+
+Always exported (metadata): session, turn and task ids, platform, model id,
+provider name, the provider's host (from the base URL, without path or
+credentials), tool names and call ids, token and message counts, durations,
+statuses and Hermes' error categories. For a self-hosted provider the host can
+be an internal hostname and the model id a local file name.
+
+Not exported unless `FI_HERMES_CAPTURE_CONTENT=true`: the user message, the
+assistant's answer, conversation history, the request sent to the model, tool
+arguments, tool results, and all error text. Hermes puts exception text into
+some turn exit reasons (`repeated_outer_errors(Error during API call #3: ...)`);
+by default only the part before the parenthesis is exported.
+
+With `FI_HERMES_CAPTURE_CONTENT=true`:
+
+- the turn gets the user message as `input.value` and the final answer as `output.value`;
+- each model call gets the latest messages of the request as input (newest kept
+  when over the cap; the system prompt is reported by size only, as
+  `system_prompt_chars`) and the assistant text or requested tool calls as output;
+- each tool call gets its arguments and result; arguments can contain local paths;
+- exit reasons and error statuses carry the full error text.
+
+Each value is truncated to `FI_HERMES_MAX_CHARS`.
+
+The plugin never reads Hermes' files: no session transcripts, `config.yaml`,
+memory or skills. It installs no signal handlers, does not set the global
+OpenTelemetry provider, and sends uncompressed OTLP/HTTP (the Future AGI
+collector does not read `Content-Encoding`). When Future AGI is unreachable,
+Hermes' exit and session teardown wait a few seconds for the export, then drop
+those spans: about 3 seconds with current OpenTelemetry releases, at most about
+6 with the oldest supported one (1.35).
+
+## Not this plugin
+
+- **Hermes' Langfuse plugin** sends to Langfuse's API, which Future AGI does not ingest.
+- **Gateway health export** (`monitoring.export.otlp`) reports gateway health and delivery, not turns.
+- **NeMo Relay file sinks** (ATOF / ATIF) write local files.
+- **NeMo Relay's OpenTelemetry exporter**, which Hermes core drives, can also
+  send OTLP to Future AGI (a `plugins.toml` with `header_env` for the two keys and
+  `resource_attributes.project_name`). Tested with Hermes v2026.9.24 and
+  nemo-relay 0.8.4 (`gen_ai` projection): one trace per session with turn, model
+  and tool spans, but the model spans carry the full prompt and response
+  (`gen_ai.input.messages` / `gen_ai.output.messages`) even with
+  `enable_full_payloads = false`, no `session.id` is set, and the configuration is
+  process-wide, so every profile of a gateway exports to one project. This plugin
+  exists for a content-off default, Future AGI sessions and per-profile keys.
+
+## Troubleshooting
+
+- `hermes plugins list` must show `futureagi` as enabled in the Hermes home you use.
+- The missing-keys warning and setup failures are logged by `traceai_hermes_agent`
+  to `$HERMES_HOME/logs/agent.log` and `errors.log`.
+- A gateway loads plugins and `.env` at start: restart it after changes.
+
+## Tests
+
+```bash
+pip install -e . pytest
+pytest tests
+```
+
+With `hermes-agent` installed in the same environment, the contract tests also
+check the hook names against `hermes_cli.plugins.VALID_HOOKS` and that Hermes
+discovers the entry point.
+

traceai_hermes_agent-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+traceai_hermes_agent/__init__.py,sha256=RDcr2GibZn1BYUditGYPDNzOzTyPogQ_NTSw5El_YJQ,9705
+traceai_hermes_agent/_recorder.py,sha256=lrvMYGiUjX34HFQVWqp2F8dchno6yTpeuqRn1enLBPc,26534
+traceai_hermes_agent/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+traceai_hermes_agent/version.py,sha256=IOEzhCbX916JIkvZr03GFY4bl9YqSkfjdqLO4qJ2FOs,50
+traceai_hermes_agent-0.1.0.dist-info/METADATA,sha256=Q0mfDmyWknjSYgV4CNhzNoDJvQZcPSh-l8_mMVT9-9A,10521
+traceai_hermes_agent-0.1.0.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
+traceai_hermes_agent-0.1.0.dist-info/entry_points.txt,sha256=gjQ_GgkrpVOvhAxZy2oDsPuhFKGFQTfDbxZkBNIJwOc,64
+traceai_hermes_agent-0.1.0.dist-info/RECORD,,

traceai_hermes_agent-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.5.0
+Root-Is-Purelib: true
+Tag: py3-none-any

traceai_hermes_agent-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[hermes_agent.plugins]
+futureagi=traceai_hermes_agent:register
+