world-model-optimizer 0.2.0__py3-none-any.whl

wmo/connect/store.py

@@ -0,0 +1,199 @@
+"""Context bundle persistence and rendering under `<project>/.wmo/context/`.
+
+A bundle is one pull's replayable artifact: `manifest.json` (what was pulled, when, from where)
+plus `items.jsonl` (one normalized `ContextItem` per line). "Filesystem as DB", like the model
+store: loading a bundle is just reading its folder. `ContextStore.save`/`load` persist and read
+bundles, and `render_markdown` turns a bundle into a deterministic markdown document callers can
+write into a model's knowledge dir.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from wmo.config.config import ARTIFACT_DIR
+from wmo.config.store import validate_name
+from wmo.connect.types import ContextItem, PullQuery
+
+_MANIFEST_FILENAME = "manifest.json"
+_ITEMS_FILENAME = "items.jsonl"
+
+
+class BundleManifest(BaseModel):
+    """Provenance for one saved bundle: what was pulled, when, by which connector.
+
+    Attributes:
+        name: Bundle name (the directory name under `.wmo/context/`).
+        connector: The connector that produced the bundle.
+        query: The exact `PullQuery` used, kept for replayable re-pulls.
+        pulled_at: ISO-8601 timestamp of the pull.
+        item_count: Number of items in `items.jsonl`.
+        account: Human-readable identity the pull ran as, when known.
+    """
+
+    name: str
+    connector: str
+    query: PullQuery
+    pulled_at: str
+    item_count: int
+    account: str | None = None
+
+
+class ContextStore:
+    """Named context bundles on disk under `<root>/.wmo/context/<name>/`.
+
+    `root` is the PROJECT directory (the parent of `.wmo/`), defaulting to the current working
+    directory; tests pass a tmp path. Note this differs from `WorldModelStore`, whose root is
+    the `.wmo` artifact dir itself.
+    """
+
+    def __init__(self, root: str | Path | None = None) -> None:
+        self.root = Path(root) if root is not None else Path.cwd()
+        self.context_dir = self.root / ARTIFACT_DIR / "context"
+
+    def bundle_dir(self, name: str) -> Path:
+        """The directory a bundle named `name` lives in (may not exist)."""
+        return self.context_dir / _validated_bundle_name(name)
+
+    def save(
+        self, manifest: BundleManifest, items: list[ContextItem], *, overwrite: bool = False
+    ) -> Path:
+        """Write one bundle (`manifest.json` + `items.jsonl`); returns its directory.
+
+        Raises:
+            FileExistsError: When the bundle already exists and `overwrite` is False.
+            ValueError: When the bundle name is not a safe single path segment.
+        """
+        directory = self.bundle_dir(manifest.name)
+        if directory.exists():
+            if not overwrite:
+                raise FileExistsError(
+                    f"context bundle {manifest.name!r} already exists at {directory}; "
+                    "pass overwrite=True to replace it"
+                )
+            shutil.rmtree(directory)
+        directory.mkdir(parents=True)
+        manifest_text = manifest.model_dump_json(indent=2) + "\n"
+        (directory / _MANIFEST_FILENAME).write_text(manifest_text, encoding="utf-8")
+        lines = "".join(item.model_dump_json() + "\n" for item in items)
+        (directory / _ITEMS_FILENAME).write_text(lines, encoding="utf-8")
+        return directory
+
+    def load(self, name: str) -> tuple[BundleManifest, list[ContextItem]]:
+        """Read one bundle back as (manifest, items).
+
+        Raises:
+            FileNotFoundError: When no bundle named `name` exists.
+        """
+        directory = self.bundle_dir(name)
+        manifest_path = directory / _MANIFEST_FILENAME
+        if not manifest_path.exists():
+            raise FileNotFoundError(
+                f"no context bundle named {name!r} under {self.context_dir}; "
+                "pull a bundle and persist it with ContextStore.save first"
+            )
+        manifest = BundleManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
+        items_text = (directory / _ITEMS_FILENAME).read_text(encoding="utf-8")
+        items = [
+            ContextItem.model_validate_json(line)
+            for line in items_text.splitlines()
+            if line.strip()
+        ]
+        return manifest, items
+
+    def list_bundles(self) -> list[BundleManifest]:
+        """Manifests of every saved bundle, sorted by directory name."""
+        if not self.context_dir.exists():
+            return []
+        manifests: list[BundleManifest] = []
+        for child in sorted(self.context_dir.iterdir()):
+            manifest_path = child / _MANIFEST_FILENAME
+            if child.is_dir() and manifest_path.exists():
+                manifests.append(
+                    BundleManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
+                )
+        return manifests
+
+    def delete(self, name: str) -> bool:
+        """Remove one bundle directory; returns whether it existed."""
+        directory = self.bundle_dir(name)
+        if not directory.exists():
+            return False
+        shutil.rmtree(directory)
+        return True
+
+
+def render_markdown(
+    manifest: BundleManifest, items: list[ContextItem], *, max_chars: int | None = None
+) -> str:
+    """Render a bundle as one deterministic markdown document.
+
+    A provenance header (connector, pulled_at, query) is followed by one `## title` section per
+    item (a kind/date/url fact line, then the body). When the result would exceed `max_chars`,
+    whole items are dropped from the tail and a final "... n items omitted" line makes the
+    truncation visible, never silent.
+    """
+    header = _render_header(manifest)
+    sections = [_render_item(item) for item in items]
+    full = "\n\n".join([header, *sections]) + "\n"
+    if max_chars is None or len(full) <= max_chars:
+        return full
+    candidate = full
+    for kept in range(len(items) - 1, -1, -1):
+        omitted = len(items) - kept
+        tail = f"... {omitted} items omitted"
+        candidate = "\n\n".join([header, *sections[:kept], tail]) + "\n"
+        if len(candidate) <= max_chars:
+            return candidate
+    # Even the header alone is over budget; return the loud minimal form anyway.
+    return candidate
+
+
+def _validated_bundle_name(name: str) -> str:
+    """Reject unsafe bundle names with bundle-specific wording (same rules as model names)."""
+    try:
+        return validate_name(name)
+    except ValueError as exc:
+        raise ValueError(
+            f"invalid context bundle name {name!r}: use letters, digits, '.', '_', '-' "
+            "(must start with a letter or digit, no path separators)"
+        ) from exc
+
+
+def _render_header(manifest: BundleManifest) -> str:
+    """The provenance block: bundle name, connector, pull time, identity, query, item count."""
+    query_parts = [
+        f"{field}={value}"
+        for field, value in manifest.query.model_dump(mode="json").items()
+        if value is not None
+    ]
+    lines = [
+        f"# Context bundle: {manifest.name}",
+        "",
+        f"- connector: {manifest.connector}",
+        f"- pulled_at: {manifest.pulled_at}",
+    ]
+    if manifest.account:
+        lines.append(f"- account: {manifest.account}")
+    lines.append(f"- query: {', '.join(query_parts)}")
+    lines.append(f"- items: {manifest.item_count}")
+    return "\n".join(lines)
+
+
+def _render_item(item: ContextItem) -> str:
+    """One `## title` section: a kind/date/url fact line, then the body."""
+    facts = [item.kind.value]
+    if item.created_at:
+        facts.append(f"created {item.created_at}")
+    if item.updated_at:
+        facts.append(f"updated {item.updated_at}")
+    if item.url:
+        facts.append(item.url)
+    section = f"## {item.title}\n\n{' | '.join(facts)}"
+    body = item.body.strip()
+    if body:
+        section += f"\n\n{body}"
+    return section

wmo/connect/types.py

@@ -0,0 +1,156 @@
+"""Normalized types for context connectors.
+
+Connectors (github, google, slack, notion, ...) authenticate against a service and pull content
+into these vendor-agnostic shapes. Everything downstream (the bundle store, markdown rendering,
+knowledge attachment) operates on `ContextItem`, never on raw vendor payloads. The `opt_str`,
+`capped`, and `strip_html` helpers are the shared coercions for raw vendor JSON and fetched
+content bodies.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+from collections.abc import Iterator
+from contextlib import contextmanager
+from enum import StrEnum
+from typing import Literal
+
+import httpx
+from pydantic import BaseModel, Field
+
+from wmo.core.types import JsonObject, JsonValue
+
+# Fetched content is capped so one huge document or page cannot blow up a bundle.
+CONTENT_CAP_CHARS = 200_000
+TRUNCATION_MARKER = f"\n[content truncated at {CONTENT_CAP_CHARS} characters]"
+
+
+def opt_str(value: JsonValue | None) -> str | None:
+    """`value` when it is a non-empty string, else None (vendor JSON field coercion)."""
+    return value if isinstance(value, str) and value else None
+
+
+def capped(text: str) -> str:
+    """Cap fetched content, appending a loud truncation marker when anything was cut."""
+    if len(text) <= CONTENT_CAP_CHARS:
+        return text
+    return text[:CONTENT_CAP_CHARS] + TRUNCATION_MARKER
+
+
+def strip_html(markup: str) -> str:
+    """A small HTML-to-text fallback: drop script/style, break on block ends, strip tags."""
+    text = re.sub(r"(?is)<(script|style)\b.*?</\1>", " ", markup)
+    text = re.sub(r"(?is)<br\s*/?>|</p>|</div>", "\n", text)
+    text = re.sub(r"(?s)<[^>]*>", " ", text)
+    text = html.unescape(text)
+    lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+class ConnectError(RuntimeError):
+    """A connector operation failed; messages say what went wrong and what to do about it."""
+
+
+@contextmanager
+def transport_errors(host: str) -> Iterator[None]:
+    """Turn httpx transport failures inside the block into actionable ConnectErrors.
+
+    Connectors wrap their HTTP calls with this so network-level failures (DNS errors, refused
+    connections, timeouts) honor the ConnectError contract instead of escaping to callers as
+    raw httpx tracebacks.
+
+    Args:
+        host: The host the block talks to, named in the error message.
+
+    Raises:
+        ConnectError: For any `httpx.HTTPError` raised inside the block.
+    """
+    try:
+        yield
+    except httpx.HTTPError as exc:
+        detail = str(exc) or type(exc).__name__
+        raise ConnectError(
+            f"could not reach {host} ({detail}); check your network connection and retry"
+        ) from exc
+
+
+class ItemKind(StrEnum):
+    """The normalized kind of one pulled content item."""
+
+    DOCUMENT = "document"
+    PAGE = "page"
+    ISSUE = "issue"
+    PULL_REQUEST = "pull_request"
+    MESSAGE = "message"
+    THREAD = "thread"
+    EMAIL = "email"
+    EVENT = "event"
+    FILE = "file"
+
+
+class ContextItem(BaseModel):
+    """One normalized piece of pulled content (an issue, a page, a message, ...).
+
+    Attributes:
+        id: Stable identifier within the source service (issue number, page id, message ts).
+        source: The connector name that produced the item (e.g. "github").
+        kind: What the item is, from the normalized `ItemKind` vocabulary.
+        title: Short human title; shown as the item's markdown section heading.
+        body: The content itself, plain text or markdown.
+        url: Canonical link back to the item, when the service has one.
+        created_at: ISO-8601 creation timestamp, when known.
+        updated_at: ISO-8601 last-modified timestamp, when known.
+        metadata: Connector-specific extras (labels, authors, channel ids) as arbitrary JSON.
+    """
+
+    id: str
+    source: str
+    kind: ItemKind
+    title: str
+    body: str
+    url: str | None = None
+    created_at: str | None = None
+    updated_at: str | None = None
+    metadata: JsonObject = Field(default_factory=dict)
+
+
+class PullQuery(BaseModel):
+    """What to pull: the parameters every connector's `pull` accepts.
+
+    Attributes:
+        target: Service-specific container: a repo "owner/name", a channel name, a calendar id,
+            a drive folder.
+        query: Free-text or service search-syntax filter.
+        since: ISO-8601 date or datetime lower bound on item time.
+        until: ISO-8601 date or datetime upper bound on item time.
+        limit: Maximum number of items a connector may fetch (connectors must cap at this).
+    """
+
+    target: str | None = None
+    query: str | None = None
+    since: str | None = None
+    until: str | None = None
+    limit: int = 100
+
+
+class ConnectorAuth(BaseModel):
+    """A stored credential for one connector.
+
+    Attributes:
+        kind: "oauth" for browser/device OAuth grants, "token" for pasted or env-injected tokens.
+        access_token: The bearer credential API calls send.
+        refresh_token: OAuth refresh token, when the provider issued one.
+        expires_at: ISO-8601 absolute expiry of `access_token`, when known.
+        scopes: The granted OAuth scopes.
+        account: Human-readable identity captured at connect time (e.g. "octocat").
+        extra: Connector-specific extras (e.g. a slack team id) as arbitrary JSON.
+    """
+
+    kind: Literal["oauth", "token"]
+    access_token: str
+    refresh_token: str | None = None
+    expires_at: str | None = None
+    scopes: list[str] = Field(default_factory=list)
+    account: str | None = None
+    extra: JsonObject = Field(default_factory=dict)

wmo/core/__init__.py

@@ -0,0 +1,21 @@
+"""Core data types shared across the harness."""
+
+from wmo.core.types import (
+    Action,
+    ActionKind,
+    EnvState,
+    Observation,
+    Session,
+    Step,
+    Trace,
+)
+
+__all__ = [
+    "Action",
+    "ActionKind",
+    "EnvState",
+    "Observation",
+    "Session",
+    "Step",
+    "Trace",
+]

wmo/core/parsing.py

@@ -0,0 +1,281 @@
+"""Robust parsing of model completions into structured values.
+
+Two concerns live here because both the serving engine and the optimizer need them, and `wmo.core`
+has no dependencies (so neither imports the other):
+
+- `extract_json_object`: pull the first complete JSON object out of a noisy LLM reply.
+- `parse_observation`: turn a world-model completion into a structured `Observation`.
+
+The world-model output contract (see `wmo.core.render.build_env_prompt`) asks the model to reply
+with a JSON object ``{"output": str, "is_error": bool, "state_note": str}``. `parse_observation`
+is lenient: a reply that is not JSON is treated as a plain-text observation, so a model that ignores
+the contract still produces a usable (non-error) observation rather than crashing the step.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+from pydantic import BaseModel, ValidationError, field_validator
+
+from wmo.core.types import JsonObject, JsonValue, Observation
+
+
+def accepted_confidence(value: float | int | str | bool) -> float | None:
+    """The one definition of a usable stated confidence: a finite number in [0, 1], else None.
+
+    Gates and calibration both consume this, so the acceptance rule must not fork: booleans are
+    not confidences (JSON `true` is not 1.0), NaN/inf are garbage, and OUT-OF-RANGE numerics
+    degrade to "not stated" rather than clamping — a model answering 85 (percent) or 7 (out of
+    10) has violated the 0.0-1.0 contract, and clamping such a reply to 1.0 would record maximal
+    certainty on exactly the steps where the model is off the rails. Missing conservatively
+    gates as LOW; malformed must never gate as certain.
+    """
+    if isinstance(value, bool):
+        return None
+    try:
+        parsed = float(value)
+    except ValueError:
+        return None
+    if not (0.0 <= parsed <= 1.0):  # also rejects NaN (all comparisons false) and +/-inf
+        return None
+    return parsed
+
+
+def extract_json_object(text: str) -> str | None:
+    """Return the first complete JSON object substring in `text`, or None if there is none.
+
+    Scans from the first ``{`` to its balanced closing ``}``, tracking string literals and escapes.
+    This tolerates ```json fences, surrounding prose, nested objects, and multiple objects (the
+    first is returned) — cases a greedy/lazy regex gets wrong.
+    """
+    start = text.find("{")
+    if start == -1:
+        return None
+    depth = 0
+    in_string = False
+    escaped = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+class _RawObservation(BaseModel):
+    """Lenient view of the world-model JSON contract before normalization.
+
+    The reasoning-mode fields (`reasoning`, `kb_note`, `ground_query` — see
+    `wmo.core.render.output_contract`) default to empty so base-contract replies parse unchanged.
+    """
+
+    reasoning: str = ""
+    output: str = ""
+    is_error: bool = False
+    state_note: str = ""
+    kb_note: str = ""
+    ground_query: str = ""
+    state_update: str = ""
+    # Verbalized confidence (WS-A6): None when the model didn't state one. Lenient like the rest
+    # of the contract — an off-contract value degrades to "no stated confidence", never a crash.
+    confidence: float | None = None
+    confidence_why: str = ""
+
+    @field_validator("confidence", mode="before")
+    @classmethod
+    def _lenient_confidence(cls, value: JsonValue) -> float | None:
+        """Coerce the raw JSON field through `accepted_confidence` (off-contract -> None)."""
+        if isinstance(value, int | float | str):
+            return accepted_confidence(value)
+        return None
+
+
+# The keys that mark a reply as following the observation contract (any one present is enough).
+# Used to tell a real — possibly empty — contract response apart from arbitrary JSON that happens
+# to validate against `_RawObservation`'s all-defaulted fields. Deliberately ONLY the core keys:
+# every complete contract reply (base or reasoning mode) carries `output`/`is_error`, while a
+# reasoning-mode superset key alone (e.g. off-contract JSON with a "reasoning" field but no
+# "output") must fall through to the plain-text fallback, not become an empty observation.
+# Confidence-mode keys are deliberately excluded too: an arbitrary API payload with its own
+# "confidence" field must not be mistaken for a contract reply.
+_CONTRACT_KEYS = frozenset({"output", "is_error", "state_note"})
+
+
+def parse_observation(text: str) -> Observation:
+    """Parse a world-model completion into a structured Observation.
+
+    Prefers the JSON contract ``{"output", "is_error", "state_note"}`` and its reasoning-mode
+    superset (``reasoning``/``kb_note``/``ground_query``). ``output`` becomes the observation the
+    agent sees; every other populated field is carried in ``metadata`` (``state_note`` feeds the
+    session scratchpad, ``kb_note`` the cross-session knowledge base, ``ground_query`` the
+    grounder, ``reasoning`` is kept for inspection only). Falls back to treating the whole reply
+    as plain observation text when it is not the expected JSON, so an off-contract model still
+    yields a usable observation.
+    """
+    raw = extract_json_object(text)
+    if raw is not None:
+        try:
+            obj: object = json.loads(raw)
+        except json.JSONDecodeError:
+            obj = None
+        # Recognize the contract by the PRESENCE of its keys, not by truthy values.
+        # `_RawObservation` defaults every field, so arbitrary JSON like `{"foo": 1}` would validate
+        # to an all-empty observation; requiring a contract key keeps that falling through to raw
+        # text. But a legitimate silent success `{"output": "", "is_error": false, ...}` (many shell
+        # writes/redirects print nothing) MUST be honored as an empty observation, not re-serialized
+        # as visible JSON text — closed-loop rollouts would otherwise show spurious output.
+        if isinstance(obj, dict) and _CONTRACT_KEYS.intersection(obj):
+            try:
+                parsed = _RawObservation.model_validate(obj)
+            except ValidationError:
+                parsed = None
+            if parsed is not None:
+                metadata: JsonObject = {}
+                for key, value in (
+                    ("state_note", parsed.state_note),
+                    ("reasoning", parsed.reasoning),
+                    ("kb_note", parsed.kb_note),
+                    ("ground_query", parsed.ground_query),
+                    ("state_update", parsed.state_update),
+                    ("confidence_why", parsed.confidence_why),
+                ):
+                    if value:
+                        metadata[key] = value
+                # Separate from the truthiness loop: a stated confidence of 0.0 must survive.
+                if parsed.confidence is not None:
+                    metadata["confidence"] = parsed.confidence
+                return Observation(
+                    content=parsed.output, is_error=parsed.is_error, metadata=metadata
+                )
+    # A contract reply cut off mid-generation is not valid JSON at all: salvage the fields the
+    # text already contains rather than surfacing the raw truncated JSON as the observation.
+    salvaged = _salvage_truncated_contract(text)
+    if salvaged is not None:
+        return salvaged
+    return Observation(content=text.strip())
+
+
+def _salvage_truncated_contract(text: str) -> Observation | None:
+    """Recover a contract reply whose JSON never closed (token-budget truncation).
+
+    Long deliberations plus long escaped observations can blow the completion budget mid-string;
+    without this, the ENTIRE raw contract text (reasoning included) becomes the observation the
+    agent sees — observed live as a catastrophic 0.26-fidelity step. Conservative trigger: the
+    text must look like a contract object (starts with ``{`` and names an ``"output"`` key) and
+    must NOT have parsed as complete JSON (callers try that first). Recovered string fields are
+    unescaped up to the truncation point.
+    """
+    stripped = text.strip()
+    if not stripped.startswith("{") or '"output"' not in stripped:
+        return None
+    output = _string_field_value(stripped, "output")
+    if output is None:
+        return None
+    metadata: JsonObject = {}
+    # Recover every metadata-carried contract field the truncated text still contains —
+    # dropping state_note/state_update here would silently stall the scratchpad and belief
+    # profile for the rest of the session.
+    salvage_keys = (
+        "reasoning",
+        "state_note",
+        "kb_note",
+        "ground_query",
+        "state_update",
+        "confidence_why",
+    )
+    for key in salvage_keys:
+        value = _string_field_value(stripped, key)
+        if value:
+            metadata[key] = value
+    # Salvage a stated confidence too: truncation correlates with HARD steps, so silently
+    # dropping their confidences would bias any calibration analysis toward the easy ones.
+    # Same acceptance rule as the validator — the two paths must not fork.
+    match = _CONFIDENCE_VALUE.search(stripped)
+    if match is not None:
+        confidence = accepted_confidence(match.group(1))
+        if confidence is not None:
+            metadata["confidence"] = confidence
+    is_error = re.search(r'"is_error"\s*:\s*true', stripped) is not None
+    return Observation(content=output, is_error=is_error, metadata=metadata)
+
+
+def _string_field_value(text: str, key: str) -> str | None:
+    """Extract `key`'s JSON string value from possibly-truncated JSON, unescaping as we go."""
+    marker = f'"{key}"'
+    at = text.find(marker)
+    if at == -1:
+        return None
+    i = at + len(marker)
+    while i < len(text) and text[i] in ": \t\n":
+        i += 1
+    if i >= len(text) or text[i] != '"':
+        return None
+    i += 1
+    out: list[str] = []
+    escaped = False
+    while i < len(text):
+        ch = text[i]
+        if escaped:
+            if ch == "u" and i + 4 < len(text):
+                # \uXXXX escape: decode the four hex digits (accents/box-drawing chars are
+                # common in terminal corpora; dropping the backslash rendered 'u00e9' garbage).
+                hex_digits = text[i + 1 : i + 5]
+                try:
+                    out.append(chr(int(hex_digits, 16)))
+                    i += 4
+                except ValueError:
+                    out.append(ch)
+            else:
+                out.append(_UNESCAPE.get(ch, ch))
+            escaped = False
+        elif ch == "\\":
+            escaped = True
+        elif ch == '"':
+            break  # properly terminated string
+        else:
+            out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+_UNESCAPE = {"n": "\n", "t": "\t", "r": "\r", '"': '"', "\\": "\\", "/": "/"}
+
+# The numeric confidence value in possibly-truncated contract text (salvage path only; complete
+# JSON goes through `_RawObservation`).
+_CONFIDENCE_VALUE = re.compile(r'"confidence"\s*:\s*([0-9]+(?:\.[0-9]+)?)')
+
+
+def dumps_observation_contract(observation: Observation) -> str:
+    """Render an Observation back into the JSON output contract (used to seed/demo the format).
+
+    Carries the confidence fields when present (key order mirroring the contract:
+    justification before the number, both after `is_error`) — the verify pass embeds this as
+    the draft, and a draft missing the field the contract demands invites the reviser to drop
+    it too, thinning stated confidence exactly on the verified (low-confidence) population.
+    """
+    payload: JsonObject = {"output": observation.content, "is_error": observation.is_error}
+    why = observation.metadata.get("confidence_why")
+    if isinstance(why, str) and why:
+        payload["confidence_why"] = why
+    confidence = observation.metadata.get("confidence")
+    if isinstance(confidence, int | float) and not isinstance(confidence, bool):
+        payload["confidence"] = float(confidence)
+    note = observation.metadata.get("state_note")
+    if isinstance(note, str) and note:
+        payload["state_note"] = note
+    return json.dumps(payload)