apsimo 1.3.0__py3-none-any.whl

apsimo/mining/__init__.py

@@ -0,0 +1,27 @@
+"""Mining: escalation detection + training-corpus export.
+
+The model-agnostic mining half of a continuous self-improvement loop:
+escalation events (build-agent consultations, heavy-model / cloud-failover
+turns) become skills-memory distillation inputs and golden eval cases, and
+the verbatim turn capture becomes fine-tune-ready JSONL for an external
+training pipeline. Everything stays under COLONY_STATE_DIR.
+"""
+from apsimo.mining.corpus import export_corpus
+from apsimo.mining.escalations import EscalationMiner
+from apsimo.mining.models import (
+    EscalationRecord,
+    MinedTurn,
+    corpus_export_enabled,
+    mining_mode,
+)
+from apsimo.mining.store import MiningStore
+
+__all__ = [
+    "EscalationMiner",
+    "EscalationRecord",
+    "MinedTurn",
+    "MiningStore",
+    "corpus_export_enabled",
+    "export_corpus",
+    "mining_mode",
+]

apsimo/mining/corpus.py

@@ -0,0 +1,239 @@
+"""Training-corpus exporter: turn store -> fine-tune-ready JSONL.
+
+Row shape (the DeepSpec / standard fine-tune contract, one JSON object per
+line): {"id": ..., "conversations": [{"role": "user"|"assistant", "content":
+...}, ...], "meta": {...}}. Conversations always start with a user message
+and alternate user/assistant, so they parse under GeneralParser-style
+tooling without warnings.
+
+PII stance: everything stays local. Source rows never leave the sidecar's
+SQLite; the export file is always written under COLONY_STATE_DIR/exports/
+(path traversal is not accepted). Optional redact=true masks credential /
+token shapes via the sidecar's redact machinery. Consumers (fine-tune
+pipelines) read the file from the state dir; nothing is uploaded by this
+code.
+
+Quality gate: system/skill-origin turns are excluded using the same
+sanitize/origin machinery the reachout policy uses (is_system_origin +
+sanitize_text), machine markers (e.g. reply-context [[rc ...]] suffixes)
+are stripped, cron/self sessions are excluded by default, and exact
+duplicate exchanges are deduplicated.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from apsimo.mining.store import MiningStore
+
+logger = logging.getLogger(__name__)
+
+_RC_MARKER = re.compile(r"\s*\[\[rc [^\]]*\]\]")
+
+
+def _parse_when(value: Optional[str]) -> Optional[float]:
+    """ISO timestamp or relative '<n>d'/'<n>h' -> epoch seconds."""
+    if not value:
+        return None
+    v = value.strip()
+    m = re.fullmatch(r"(\d+)([dh])", v)
+    if m:
+        n, unit = int(m.group(1)), m.group(2)
+        return time.time() - n * (86400 if unit == "d" else 3600)
+    try:
+        return datetime.fromisoformat(v.replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        raise ValueError(f"unparseable time filter: {value!r}")
+
+
+def _clean(text: str, redact: bool) -> str:
+    out = _RC_MARKER.sub("", text or "").strip()
+    try:
+        from apsimo.delivery.reachout_policy import sanitize_text
+
+        out = sanitize_text(out)
+    except Exception:
+        pass
+    if redact:
+        try:
+            from apsimo.redact import redact_sensitive_text
+
+            out = redact_sensitive_text(out)
+        except Exception:
+            pass
+    return out.strip()
+
+
+def _system_origin(text: str) -> bool:
+    try:
+        from apsimo.delivery.reachout_policy import is_system_origin
+
+        return is_system_origin(text or "")
+    except Exception:
+        return False
+
+
+def export_corpus(
+    store: MiningStore,
+    *,
+    state_dir: Path,
+    contact_id: Optional[str] = None,
+    channels: Optional[List[str]] = None,
+    since: Optional[str] = None,
+    until: Optional[str] = None,
+    group: str = "turn",
+    min_chars: int = 2,
+    include_cron: bool = False,
+    include_escalations: bool = False,
+    dedup: bool = True,
+    redact: bool = False,
+    limit: int = 100000,
+    filename: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Export filtered conversations as JSONL; returns stats + path."""
+    import os
+
+    if group not in ("turn", "session"):
+        raise ValueError("group must be 'turn' or 'session'")
+    if contact_id is None:
+        contact_id = os.environ.get("COLONY_OWNER_CONTACT_ID", "owner")
+
+    turns = store.list_turns(
+        contact_id=contact_id,
+        channels=channels,
+        since_ts=_parse_when(since),
+        until_ts=_parse_when(until),
+        limit=limit,
+    )
+
+    skipped = {"quality": 0, "cron": 0, "dedup": 0}
+    seen: set = set()
+    exchanges: List[Dict[str, Any]] = []
+    for t in turns:
+        if not include_cron and t.session_id.startswith("cron_"):
+            skipped["cron"] += 1
+            continue
+        user = _clean(t.user_text, redact)
+        asst = _clean(t.assistant_text, redact)
+        if (
+            len(user) < min_chars
+            or len(asst) < min_chars
+            or _system_origin(t.user_text)
+            or _system_origin(t.assistant_text)
+        ):
+            skipped["quality"] += 1
+            continue
+        if dedup:
+            h = hashlib.sha256(f"{user}\x00{asst}".encode()).hexdigest()
+            if h in seen:
+                skipped["dedup"] += 1
+                continue
+            seen.add(h)
+        exchanges.append(
+            {
+                "session_id": t.session_id,
+                "user": user,
+                "assistant": asst,
+                "meta": {
+                    "channel": t.channel_id,
+                    "contact": t.contact_id,
+                    "session": t.session_id,
+                    "ts": t.ts,
+                    "model": t.model,
+                },
+            }
+        )
+
+    rows: List[Dict[str, Any]] = []
+    if group == "turn":
+        for i, ex in enumerate(exchanges):
+            rows.append(
+                {
+                    "id": f"turn-{i}",
+                    "conversations": [
+                        {"role": "user", "content": ex["user"]},
+                        {"role": "assistant", "content": ex["assistant"]},
+                    ],
+                    "meta": ex["meta"],
+                }
+            )
+    else:
+        by_session: Dict[str, List[Dict[str, Any]]] = {}
+        for ex in exchanges:
+            by_session.setdefault(ex["session_id"], []).append(ex)
+        for i, (sid, exs) in enumerate(by_session.items()):
+            conv: List[Dict[str, str]] = []
+            for ex in exs:
+                conv.append({"role": "user", "content": ex["user"]})
+                conv.append({"role": "assistant", "content": ex["assistant"]})
+            rows.append(
+                {
+                    "id": f"session-{i}",
+                    "conversations": conv,
+                    "meta": {**exs[0]["meta"], "exchanges": len(exs)},
+                }
+            )
+
+    escalations_included = 0
+    if include_escalations:
+        for e in store.list_escalations(limit=1000):
+            user = _clean(e.task_context, redact)
+            asst = _clean(e.escalated_answer, redact)
+            if len(user) < min_chars or len(asst) < min_chars:
+                continue
+            rows.append(
+                {
+                    "id": f"escalation-{e.id[:8]}",
+                    "conversations": [
+                        {"role": "user", "content": user},
+                        {"role": "assistant", "content": asst},
+                    ],
+                    "meta": {
+                        "kind": e.kind,
+                        "channel": e.channel_id,
+                        "matched": e.matched,
+                        "outcome": e.outcome,
+                        "model": e.model,
+                        "ts": e.ts,
+                    },
+                }
+            )
+            escalations_included += 1
+
+    exports_dir = Path(state_dir) / "exports"
+    exports_dir.mkdir(parents=True, exist_ok=True)
+    name = filename or f"corpus-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}.jsonl"
+    name = Path(name).name  # no traversal: basename only, always under exports/
+    out_path = exports_dir / name
+    with open(out_path, "w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    stats = {
+        "path": str(out_path),
+        "rows": len(rows),
+        "exchanges": len(exchanges),
+        "sessions": len({ex["session_id"] for ex in exchanges}),
+        "turns_scanned": len(turns),
+        "skipped": skipped,
+        "escalations_included": escalations_included,
+        "group": group,
+        "contact_id": contact_id,
+        "redacted": redact,
+    }
+    try:
+        from apsimo.events.journal import append_event
+
+        append_event("mining.corpus_export", stats)
+    except Exception:
+        logger.debug("corpus export journal failed", exc_info=True)
+    logger.info(
+        "corpus export: %d rows (%d sessions) -> %s", len(rows), stats["sessions"], out_path
+    )
+    return stats

apsimo/mining/escalations.py

@@ -0,0 +1,289 @@
+"""EscalationMiner: detect + bank escalation events from the live turn stream.
+
+Two detectors run on every synced turn:
+
+1. CONSULTATION: the agent shelled out to a build/coding agent during the
+   turn (a terminal-class tool ran AND the turn text matches
+   COLONY_ESCALATION_CONSULT_REGEX). The situation was hard enough that the
+   agent escalated to a stronger coding agent: exactly the material worth
+   distilling into a skill and worth banking as a golden case.
+2. PROVIDER ESCALATION: the turn's model (optional per-turn metadata from the
+   host) matches COLONY_ESCALATION_HEAVY_RE: a heavy-model or cloud-failover
+   turn.
+
+Each record banks: task/prompt context, the prior local attempt in the same
+session (when present), the escalated answer, channel, model, ts. Outcome is
+tracked lightweight: the NEXT turn in the same session marks the record
+followed_up with the user's reaction excerpt.
+
+Modes (COLONY_ESCALATION_MINING): shadow banks + journals only; live also
+feeds records into skills-memory distillation (domain "escalation"), which
+itself honors COLONY_SKILLS_DISTILL. Everything is best-effort: a miner
+failure must never affect the turn pipeline.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any, Callable, List, Optional
+
+from apsimo.mining.models import (
+    EscalationRecord,
+    MinedTurn,
+    consult_regex,
+    heavy_model_regex,
+    mining_mode,
+    turn_cap,
+)
+from apsimo.mining.store import MiningStore
+
+logger = logging.getLogger(__name__)
+
+_TERMINAL_TOOLS = {"terminal", "shell", "bash", "execute_code", "shell_exec"}
+
+
+def _session_tool_activity(session_id: str, max_lines: int = 400):
+    """Recent host tool-activity for a session: (summaries_text, tool_names).
+
+    Consultations often show only in the tool COMMANDS (e.g. `claude -p ...`
+    run via the terminal tool), not in the reply text. When the host exposes
+    its tool-activity stream (COLONY_TOOL_ACTIVITY_FILE, a JSONL of
+    {ts, session, tool, summary}), include those summaries in the detection
+    text and use the tool names as a terminal-use fallback signal.
+    Best-effort: any failure returns ("", empty set).
+    """
+    import json as _json
+    import os as _os
+
+    path = _os.environ.get("COLONY_TOOL_ACTIVITY_FILE", "")
+    if not path or not session_id:
+        return "", set()
+    try:
+        with open(_os.path.expanduser(path), "rb") as f:
+            try:
+                f.seek(-200_000, 2)
+            except OSError:
+                f.seek(0)
+            lines = f.read().decode("utf-8", "replace").splitlines()[-max_lines:]
+        out = []
+        tools = set()
+        for ln in lines:
+            try:
+                r = _json.loads(ln)
+            except Exception:
+                continue
+            if r.get("session") == session_id:
+                if r.get("tool"):
+                    tools.add(str(r["tool"]))
+                if r.get("summary"):
+                    out.append(str(r["summary"]))
+        return "\n".join(out[-50:]), tools
+    except Exception:
+        return "", set()
+
+
+class EscalationMiner:
+    def __init__(
+        self,
+        store: MiningStore,
+        *,
+        skill_store: Any = None,
+        router_getter: Optional[Callable[[], Any]] = None,
+    ) -> None:
+        self._store = store
+        self._skill_store = skill_store
+        self._router_getter = router_getter
+
+    # -- turn intake -----------------------------------------------------------
+
+    def observe_turn(
+        self,
+        *,
+        session_id: str,
+        contact_id: str,
+        channel_id: str,
+        user_text: str,
+        assistant_text: str,
+        summary: str = "",
+        tools_used: Optional[List[str]] = None,
+        model: str = "",
+    ) -> Optional[EscalationRecord]:
+        """Bank the verbatim turn, close out a pending escalation's outcome,
+        and detect a new escalation. Returns the new record, if any."""
+        mode = mining_mode()
+        if mode == "off":
+            return None
+        try:
+            cap = turn_cap()
+            tools = list(tools_used or [])
+            prior = self._store.last_turn_in_session(session_id)
+
+            # Outcome follow-up: the next turn's user message is the reaction
+            # to a previously banked escalation in this session.
+            try:
+                open_esc = self._store.latest_open_escalation(session_id)
+                if open_esc is not None and user_text:
+                    open_esc.outcome = "followed_up"
+                    open_esc.outcome_note = user_text[:300]
+                    self._store.update_escalation(open_esc)
+            except Exception:
+                logger.debug("escalation outcome follow-up failed", exc_info=True)
+
+            turn = MinedTurn(
+                session_id=session_id,
+                contact_id=contact_id,
+                channel_id=channel_id,
+                user_text=(user_text or "")[:cap],
+                assistant_text=(assistant_text or "")[:cap],
+                summary=(summary or "")[:1000],
+                tools_used=tools[:20],
+                model=model or "",
+            )
+            self._store.add_turn(turn)
+
+            record = self._detect(turn, prior)
+            if record is None:
+                return None
+            self._store.add_escalation(record)
+            self._journal(record, mode)
+            if mode == "live":
+                self._feed_distiller(record)
+            return record
+        except Exception:
+            logger.debug("escalation miner observe_turn failed", exc_info=True)
+            return None
+
+    # -- detection --------------------------------------------------------------
+
+    def _detect(
+        self, turn: MinedTurn, prior: Optional[MinedTurn]
+    ) -> Optional[EscalationRecord]:
+        text = f"{turn.user_text}\n{turn.assistant_text}\n{turn.summary}"
+        tool_text, activity_tools = _session_tool_activity(turn.session_id)
+        if tool_text:
+            text = f"{text}\n{tool_text}"
+
+        # (a) build-agent consultation. Terminal-use comes from the synced
+        # tools_used when the host sends it, with the host tool-activity
+        # stream as the fallback source.
+        all_tools = set(turn.tools_used) | activity_tools
+        ran_terminal = any(t in _TERMINAL_TOOLS for t in all_tools)
+        pattern = consult_regex()
+        if ran_terminal and pattern:
+            try:
+                m = re.search(pattern, text, re.IGNORECASE)
+            except re.error:
+                m = None
+                logger.warning("invalid COLONY_ESCALATION_CONSULT_REGEX; detector idle")
+            if m:
+                return EscalationRecord(
+                    kind="consultation",
+                    session_id=turn.session_id,
+                    contact_id=turn.contact_id,
+                    channel_id=turn.channel_id,
+                    task_context=turn.user_text[:2000],
+                    local_attempt=(prior.assistant_text[:2000] if prior else ""),
+                    escalated_answer=turn.assistant_text[:4000],
+                    model=turn.model,
+                    matched=m.group(0)[:120],
+                )
+
+        # (b) heavy-model / cloud-failover turn
+        heavy = heavy_model_regex()
+        if heavy and turn.model:
+            try:
+                hm = re.search(heavy, turn.model, re.IGNORECASE)
+            except re.error:
+                hm = None
+                logger.warning("invalid COLONY_ESCALATION_HEAVY_RE; detector idle")
+            if hm:
+                return EscalationRecord(
+                    kind="provider_escalation",
+                    session_id=turn.session_id,
+                    contact_id=turn.contact_id,
+                    channel_id=turn.channel_id,
+                    task_context=turn.user_text[:2000],
+                    local_attempt=(prior.assistant_text[:2000] if prior else ""),
+                    escalated_answer=turn.assistant_text[:4000],
+                    model=turn.model,
+                    matched=hm.group(0)[:120],
+                )
+        return None
+
+    # -- consumers ---------------------------------------------------------------
+
+    def _journal(self, record: EscalationRecord, mode: str) -> None:
+        try:
+            from apsimo.events.journal import append_event
+
+            append_event(
+                "mining.escalation",
+                {
+                    "escalation_id": record.id,
+                    "kind": record.kind,
+                    "channel_id": record.channel_id,
+                    "session_id": record.session_id,
+                    "contact_id": record.contact_id,
+                    "model": record.model,
+                    "matched": record.matched,
+                    "mode": mode,
+                    "summary": record.task_context[:200],
+                },
+            )
+        except Exception:
+            logger.debug("mining.escalation journal failed", exc_info=True)
+
+    def _feed_distiller(self, record: EscalationRecord) -> None:
+        """Live mode: escalations are high-value distillation inputs."""
+        try:
+            from apsimo.skills_memory import (
+                distill_from_completion,
+                skills_distill_mode,
+            )
+
+            if skills_distill_mode() == "off" or self._skill_store is None:
+                return
+            router = self._router_getter() if self._router_getter else None
+            if router is None:
+                return
+            result_text = record.escalated_answer
+            if record.local_attempt:
+                result_text = (
+                    f"Local attempt (before escalating):\n{record.local_attempt}\n\n"
+                    f"Escalated resolution:\n{record.escalated_answer}"
+                )
+
+            async def _run() -> None:
+                try:
+                    skill = await distill_from_completion(
+                        router,
+                        self._skill_store,
+                        domain="escalation",
+                        task_text=record.task_context,
+                        result_text=result_text,
+                        source_ref=record.id,
+                    )
+                    if skill is not None:
+                        record.distilled = 1
+                        self._store.update_escalation(record)
+                except Exception:
+                    logger.debug("escalation distillation failed", exc_info=True)
+
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(_run())
+            except RuntimeError:
+                asyncio.run(_run())
+        except Exception:
+            logger.debug("escalation distiller feed failed", exc_info=True)
+
+    # -- reads -------------------------------------------------------------------
+
+    def recent(self, *, kind: Optional[str] = None, limit: int = 50) -> List[dict]:
+        return [e.to_row() for e in self._store.list_escalations(kind=kind, limit=limit)]
+
+    def stats(self) -> dict:
+        s = self._store.escalation_stats()
+        s["mode"] = mining_mode()
+        return s