AbstractRuntime 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

abstractruntime/memory/active_context.py

@@ -357,7 +357,12 @@ class ActiveContextPolicy:
                     lines.append(f"{role}: {content}" if role else content)
             text = "\n".join([l for l in lines if l]).strip()
             if len(text) > 360:
-                return text[:357] + "…"
+                #[WARNING:TRUNCATION] bounded preview for UI/observability
+                marker = "… (truncated)"
+                keep = max(0, 360 - len(marker))
+                if keep <= 0:
+                    return marker[:360].rstrip()
+                return text[:keep].rstrip() + marker
             return text
 
         for artifact_id in resolved_artifacts:

abstractruntime/memory/kg_packets.py

@@ -0,0 +1,164 @@
+"""KG → Active Memory packetization (v0).
+
+This module bridges symbolic KG assertions (subject/predicate/object + metadata)
+into a bounded, LLM-friendly Active Memory block.
+
+Goals:
+- deterministic ordering + formatting
+- token-budgeted packing (uses AbstractCore TokenUtils when available)
+- provenance-preserving packets (span_id, writer ids, retrieval scores)
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+from .token_budget import estimate_tokens
+
+
+KG_MEMORY_PACKETS_VERSION = 0
+
+_WS = re.compile(r"\s+")
+
+
+def _clean_text(value: Any) -> str:
+    if value is None:
+        return ""
+    s = value if isinstance(value, str) else str(value)
+    return _WS.sub(" ", s).strip()
+
+
+def packetize_assertions(items: Iterable[Any]) -> List[Dict[str, Any]]:
+    """Convert assertion dicts into compact, JSON-safe memory packets (v0)."""
+    packets: list[Dict[str, Any]] = []
+    seen: set[tuple[str, str, str, str]] = set()
+
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        subject = _clean_text(item.get("subject"))
+        predicate = _clean_text(item.get("predicate"))
+        obj = _clean_text(item.get("object"))
+        if not (subject and predicate and obj):
+            continue
+
+        observed_at = _clean_text(item.get("observed_at"))
+
+        # Retrieval metadata (semantic queries) is stored in attributes._retrieval.
+        retrieval_score: Optional[float] = None
+        attrs = item.get("attributes") if isinstance(item.get("attributes"), dict) else {}
+        ret = attrs.get("_retrieval") if isinstance(attrs, dict) else None
+        if isinstance(ret, dict):
+            s = ret.get("score")
+            if isinstance(s, (int, float)):
+                retrieval_score = float(s)
+
+        prov = item.get("provenance") if isinstance(item.get("provenance"), dict) else {}
+        span_id = _clean_text(prov.get("span_id"))
+        writer_run_id = _clean_text(prov.get("writer_run_id"))
+        writer_workflow_id = _clean_text(prov.get("writer_workflow_id"))
+
+        scope = _clean_text(item.get("scope"))
+        owner_id = _clean_text(item.get("owner_id"))
+
+        # Stable statement surface for LLM consumption.
+        statement = f"{subject} —{predicate}→ {obj}"
+
+        key = (subject.casefold(), predicate.casefold(), obj.casefold(), observed_at)
+        if key in seen:
+            continue
+        seen.add(key)
+
+        pkt: Dict[str, Any] = {
+            "version": KG_MEMORY_PACKETS_VERSION,
+            "statement": statement,
+            "subject": subject,
+            "predicate": predicate,
+            "object": obj,
+        }
+        if observed_at:
+            pkt["observed_at"] = observed_at
+        if scope:
+            pkt["scope"] = scope
+        if owner_id:
+            pkt["owner_id"] = owner_id
+        if span_id:
+            pkt["span_id"] = span_id
+        if writer_run_id:
+            pkt["writer_run_id"] = writer_run_id
+        if writer_workflow_id:
+            pkt["writer_workflow_id"] = writer_workflow_id
+        if retrieval_score is not None:
+            pkt["retrieval_score"] = retrieval_score
+
+        packets.append(pkt)
+
+    return packets
+
+
+def pack_active_memory_text(
+    packets: Iterable[Dict[str, Any]],
+    *,
+    scope: str,
+    max_input_tokens: int,
+    model: Optional[str] = None,
+    title: str = "KG ACTIVE MEMORY",
+    include_scores: bool = True,
+) -> Tuple[str, List[Dict[str, Any]], int, int]:
+    """Render a token-budgeted Active Memory block.
+
+    Returns:
+      (text, kept_packets, estimated_tokens, dropped_packets)
+    """
+    budget = int(max_input_tokens) if isinstance(max_input_tokens, int) else int(max_input_tokens or 0)
+    if budget <= 0:
+        return "", [], 0, 0
+
+    scope2 = _clean_text(scope) or "session"
+
+    packets_list = [p for p in packets if isinstance(p, dict)]
+
+    order_hint = "similarity-desc" if include_scores else "observed_at-desc"
+    header_lines = [
+        f"## {title}",
+        f"(scope={scope2}; order={order_hint}; budget={budget} max_input_tokens)",
+        "",
+    ]
+    lines = list(header_lines)
+    tokens = estimate_tokens("\n".join(lines), model=model)
+
+    kept: list[Dict[str, Any]] = []
+
+    for pkt in packets_list:
+        stmt = _clean_text(pkt.get("statement"))
+        if not stmt:
+            continue
+
+        ts = _clean_text(pkt.get("observed_at"))
+        prefix = f"[{ts}] " if ts else ""
+
+        parts: list[str] = []
+        span_id = _clean_text(pkt.get("span_id"))
+        if span_id:
+            parts.append(f"span:{span_id}")
+
+        if include_scores:
+            score = pkt.get("retrieval_score")
+            if isinstance(score, (int, float)):
+                parts.append(f"score:{float(score):.3f}")
+
+        suffix = f" ({'; '.join(parts)})" if parts else ""
+        line = f"- {prefix}{stmt}{suffix}"
+
+        add = estimate_tokens("\n" + line, model=model)
+        if tokens + add > budget:
+            break
+
+        tokens += add
+        lines.append(line)
+        kept.append(pkt)
+
+    text = "\n".join(lines).strip() if kept else ""
+    dropped_total = max(0, len(packets_list) - len(kept))
+    return text, kept, int(tokens if kept else 0), int(dropped_total)

abstractruntime/memory/memact_composer.py

@@ -0,0 +1,175 @@
+"""KG/spans → MemAct Active Memory composition helpers (v0).
+
+This module provides deterministic, bounded mapping from durable memory sources
+(starting with the temporal KG) into MemAct's structured Active Memory blocks.
+
+Design goals:
+- deterministic: stable ordering + stable statements
+- bounded: honors packet budgets and optional max_items caps
+- auditable: returns a JSON-safe trace describing inputs + selections
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+
+def _as_dict_list(value: Any) -> List[Dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    out: List[Dict[str, Any]] = []
+    for item in value:
+        if isinstance(item, dict):
+            out.append(dict(item))
+    return out
+
+
+def _normalize_marker(marker: Any) -> str:
+    s = marker if isinstance(marker, str) else str(marker or "")
+    s2 = s.strip()
+    if not s2:
+        return "KG:"
+    return s2
+
+
+def _has_marker(text: Any, marker: str) -> bool:
+    if not isinstance(text, str):
+        return False
+    s = text.strip()
+    if not s:
+        return False
+    # Case-insensitive prefix match.
+    return s.casefold().startswith(marker.casefold())
+
+
+def _packet_statement(pkt: Dict[str, Any]) -> str:
+    stmt = pkt.get("statement")
+    if isinstance(stmt, str) and stmt.strip():
+        return stmt.strip()
+    subj = pkt.get("subject")
+    pred = pkt.get("predicate")
+    obj = pkt.get("object")
+    parts = [subj, pred, obj]
+    if all(isinstance(p, str) and p.strip() for p in parts):
+        return f"{subj.strip()} —{pred.strip()}→ {obj.strip()}"
+    return ""
+
+
+def _render_memact_context_line(pkt: Dict[str, Any], *, marker: str) -> str:
+    stmt = _packet_statement(pkt)
+    if not stmt:
+        return ""
+    marker2 = _normalize_marker(marker)
+
+    # Keep the injected statement stable for dedupe (avoid including retrieval_score).
+    suffix_parts: List[str] = []
+    span_id = pkt.get("span_id")
+    if isinstance(span_id, str) and span_id.strip():
+        suffix_parts.append(f"span:{span_id.strip()}")
+    writer_workflow_id = pkt.get("writer_workflow_id")
+    if isinstance(writer_workflow_id, str) and writer_workflow_id.strip():
+        suffix_parts.append(f"wf:{writer_workflow_id.strip()}")
+
+    suffix = f" ({'; '.join(suffix_parts)})" if suffix_parts else ""
+    return f"{marker2} {stmt}{suffix}".strip()
+
+
+def _packets_from_kg_result(kg_result: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], str]:
+    packets = _as_dict_list(kg_result.get("packets"))
+    if packets:
+        return packets, "kg_result.packets"
+    # Fallback: derive packets from raw items (no packing / max_input_tokens=0).
+    items = _as_dict_list(kg_result.get("items"))
+    if not items:
+        return [], "none"
+    try:
+        from .kg_packets import packetize_assertions
+
+        return packetize_assertions(items), "packetize_assertions(items)"
+    except Exception:
+        return [], "packetize_failed"
+
+
+def compose_memact_current_context_from_kg_result(
+    vars: Dict[str, Any],
+    *,
+    kg_result: Dict[str, Any],
+    stimulus: str,
+    marker: str = "KG:",
+    max_items: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Apply a KG-derived reconstruction of MemAct CURRENT CONTEXT.
+
+    Behavior (v0):
+    - Remove any existing CURRENT CONTEXT entries prefixed with `marker`
+      (treat them as "previous KG reconstruction").
+    - Add new entries derived from the KG packets (or packetized items fallback).
+    - Returns `{ok, delta, trace}` where `delta` matches MemAct envelope fields.
+    """
+    if not isinstance(vars, dict):
+        return {"ok": False, "error": "vars must be a dict"}
+    if not isinstance(kg_result, dict):
+        return {"ok": False, "error": "kg_result must be a dict"}
+
+    try:
+        from .active_memory import ensure_memact_memory, apply_memact_envelope
+    except Exception as e:  # pragma: no cover
+        return {"ok": False, "error": f"memact active_memory unavailable: {e}"}
+
+    mem = ensure_memact_memory(vars)
+    marker2 = _normalize_marker(marker)
+
+    existing_ctx = mem.get("current_context")
+    existing_ctx_list = existing_ctx if isinstance(existing_ctx, list) else []
+    removed: List[str] = []
+    for rec in existing_ctx_list:
+        if not isinstance(rec, dict):
+            continue
+        text = rec.get("text")
+        if _has_marker(text, marker2):
+            removed.append(str(text))
+
+    packets, packet_source = _packets_from_kg_result(kg_result)
+    if isinstance(max_items, int) and max_items > 0 and len(packets) > max_items:
+        packets = packets[: int(max_items)]
+
+    added: List[str] = []
+    selected: List[Dict[str, Any]] = []
+    for pkt in packets:
+        line = _render_memact_context_line(pkt, marker=marker2)
+        if not line:
+            continue
+        added.append(line)
+        if len(selected) < 50:
+            selected.append(
+                {
+                    "statement": _packet_statement(pkt),
+                    "span_id": pkt.get("span_id"),
+                    "observed_at": pkt.get("observed_at"),
+                    "retrieval_score": pkt.get("retrieval_score"),
+                }
+            )
+
+    delta: Dict[str, Any] = {"current_context": {"added": added, "removed": removed}}
+    applied = apply_memact_envelope(vars, envelope=delta)
+
+    trace: Dict[str, Any] = {
+        "stimulus": str(stimulus or "").strip(),
+        "marker": marker2,
+        "packet_source": packet_source,
+        "kg": {
+            "ok": bool(kg_result.get("ok")) if "ok" in kg_result else None,
+            "count": kg_result.get("count"),
+            "packed_count": kg_result.get("packed_count"),
+            "estimated_tokens": kg_result.get("estimated_tokens"),
+            "dropped": kg_result.get("dropped"),
+            "effort": kg_result.get("effort"),
+            "warnings": kg_result.get("warnings"),
+        },
+        "delta_preview": {"removed": len(removed), "added": len(added)},
+        "selected": selected,
+        "applied": applied,
+    }
+    ok = bool(applied.get("ok")) if isinstance(applied, dict) else False
+    return {"ok": ok, "delta": delta, "trace": trace, "active_memory": mem}
+

abstractruntime/memory/recall_levels.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Optional
+
+
+class RecallLevel(str, Enum):
+    """Framework-wide recall effort policy (binding).
+
+    See `docs/adr/memory-recall-levels.md`.
+    """
+
+    URGENT = "urgent"
+    STANDARD = "standard"
+    DEEP = "deep"
+
+
+def parse_recall_level(raw: Any) -> Optional[RecallLevel]:
+    """Parse a recall_level value.
+
+    Returns:
+      - None when the field is absent/blank (caller did not opt into policy).
+      - RecallLevel when valid.
+
+    Raises:
+      - ValueError when a non-blank value is present but invalid (no silent fallback).
+    """
+
+    if raw is None:
+        return None
+    if isinstance(raw, bool):
+        raise ValueError("recall_level must be a string")
+    s = str(raw).strip().lower()
+    if not s:
+        return None
+    if s in ("urgent", "standard", "deep"):
+        return RecallLevel(s)
+    raise ValueError(f"Unknown recall_level: {s}")
+
+
+@dataclass(frozen=True)
+class SpanRecallPolicy:
+    """Budgets for span-index recall (memory_query)."""
+
+    limit_spans_default: int
+    limit_spans_max: int
+
+    deep_allowed: bool
+    deep_limit_spans_max: int
+    deep_limit_messages_per_span_max: int
+
+    connected_allowed: bool
+    neighbor_hops_max: int
+
+    max_messages_default: int
+    max_messages_max: int
+
+
+@dataclass(frozen=True)
+class RehydratePolicy:
+    """Budgets for rehydration into context (memory_rehydrate)."""
+
+    max_messages_default: int
+    max_messages_max: int
+
+
+@dataclass(frozen=True)
+class KgQueryPolicy:
+    """Budgets for KG recall (memory_kg_query)."""
+
+    min_score_default: float
+    min_score_floor: float
+    limit_default: int
+    limit_max: int
+    max_input_tokens_default: int
+    max_input_tokens_max: int
+
+
+@dataclass(frozen=True)
+class RecallPolicy:
+    level: RecallLevel
+    span: SpanRecallPolicy
+    rehydrate: RehydratePolicy
+    kg: KgQueryPolicy
+
+
+_POLICIES: dict[RecallLevel, RecallPolicy] = {
+    RecallLevel.URGENT: RecallPolicy(
+        level=RecallLevel.URGENT,
+        span=SpanRecallPolicy(
+            limit_spans_default=2,
+            limit_spans_max=3,
+            deep_allowed=False,
+            deep_limit_spans_max=0,
+            deep_limit_messages_per_span_max=0,
+            connected_allowed=False,
+            neighbor_hops_max=0,
+            max_messages_default=30,
+            max_messages_max=60,
+        ),
+        rehydrate=RehydratePolicy(max_messages_default=30, max_messages_max=60),
+        kg=KgQueryPolicy(
+            min_score_default=0.55,
+            min_score_floor=0.5,
+            limit_default=20,
+            limit_max=40,
+            max_input_tokens_default=600,
+            max_input_tokens_max=1000,
+        ),
+    ),
+    RecallLevel.STANDARD: RecallPolicy(
+        level=RecallLevel.STANDARD,
+        span=SpanRecallPolicy(
+            limit_spans_default=5,
+            limit_spans_max=8,
+            deep_allowed=True,
+            deep_limit_spans_max=25,
+            deep_limit_messages_per_span_max=200,
+            connected_allowed=True,
+            neighbor_hops_max=1,
+            max_messages_default=80,
+            max_messages_max=150,
+        ),
+        rehydrate=RehydratePolicy(max_messages_default=80, max_messages_max=200),
+        kg=KgQueryPolicy(
+            min_score_default=0.4,
+            min_score_floor=0.25,
+            limit_default=80,
+            limit_max=200,
+            max_input_tokens_default=1200,
+            max_input_tokens_max=3000,
+        ),
+    ),
+    RecallLevel.DEEP: RecallPolicy(
+        level=RecallLevel.DEEP,
+        span=SpanRecallPolicy(
+            limit_spans_default=8,
+            limit_spans_max=20,
+            deep_allowed=True,
+            deep_limit_spans_max=100,
+            deep_limit_messages_per_span_max=400,
+            connected_allowed=True,
+            neighbor_hops_max=2,
+            max_messages_default=200,
+            max_messages_max=600,
+        ),
+        rehydrate=RehydratePolicy(max_messages_default=200, max_messages_max=800),
+        kg=KgQueryPolicy(
+            min_score_default=0.25,
+            min_score_floor=0.0,
+            limit_default=200,
+            limit_max=1000,
+            max_input_tokens_default=2400,
+            max_input_tokens_max=6000,
+        ),
+    ),
+}
+
+
+def policy_for(level: RecallLevel) -> RecallPolicy:
+    return _POLICIES[level]
+

abstractruntime/memory/token_budget.py

@@ -0,0 +1,86 @@
+"""Token-budget helpers for active context selection.
+
+These utilities are intentionally best-effort and dependency-light:
+- Prefer AbstractCore's TokenUtils when available (better model-aware heuristics).
+- Fall back to simple character-based estimation otherwise.
+
+They exist to keep VisualFlow workflows bounded by an explicit `max_input_tokens`
+budget (ADR-0008) even when the underlying model supports much larger contexts.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+
+def estimate_tokens(text: str, *, model: Optional[str] = None) -> int:
+    """Best-effort token estimation for a string."""
+    s = str(text or "")
+    if not s:
+        return 0
+    try:
+        from abstractcore.utils.token_utils import TokenUtils
+
+        return int(TokenUtils.estimate_tokens(s, model))
+    except Exception:
+        # Conservative fallback: ~4 chars per token.
+        return max(1, int(len(s) / 4))
+
+
+def estimate_message_tokens(message: Dict[str, Any], *, model: Optional[str] = None) -> int:
+    """Estimate tokens for a chat message dict (role+content)."""
+    if not isinstance(message, dict):
+        return 0
+    role = str(message.get("role") or "").strip()
+    content = "" if message.get("content") is None else str(message.get("content"))
+    # Include a small role prefix so token estimation reflects chat formatting overhead.
+    text = f"{role}: {content}" if role else content
+    return estimate_tokens(text, model=model)
+
+
+def trim_messages_to_max_input_tokens(
+    messages: Iterable[Any],
+    *,
+    max_input_tokens: int,
+    model: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """Trim oldest non-system messages until the estimated budget fits.
+
+    Rules:
+    - Preserve all system messages.
+    - Preserve the most recent non-system message (typically the user prompt).
+    - Drop oldest non-system messages first.
+    """
+    try:
+        budget = int(max_input_tokens)
+    except Exception:
+        return [dict(m) for m in messages if isinstance(m, dict)]
+
+    if budget <= 0:
+        return [dict(m) for m in messages if isinstance(m, dict)]
+
+    typed: List[Dict[str, Any]] = [dict(m) for m in messages if isinstance(m, dict)]
+    system_messages = [m for m in typed if m.get("role") == "system"]
+    non_system = [m for m in typed if m.get("role") != "system"]
+
+    if not non_system:
+        return system_messages
+
+    # Pre-compute per-message token estimates (cheaper than re-tokenizing whole text repeatedly).
+    sys_tokens = sum(estimate_message_tokens(m, model=model) for m in system_messages)
+    non_tokens = [estimate_message_tokens(m, model=model) for m in non_system]
+
+    # Always keep the final non-system message.
+    kept: List[Dict[str, Any]] = [non_system[-1]]
+    total = sys_tokens + non_tokens[-1]
+
+    # If we're already over budget, we still return system + last message.
+    for msg, tok in zip(reversed(non_system[:-1]), reversed(non_tokens[:-1])):
+        if total + tok > budget:
+            break
+        kept.append(msg)
+        total += tok
+
+    kept.reverse()
+    return system_messages + kept
+

abstractruntime/storage/__init__.py

@@ -5,6 +5,7 @@ from .in_memory import InMemoryRunStore, InMemoryLedgerStore
 from .json_files import JsonFileRunStore, JsonlLedgerStore
 from .ledger_chain import HashChainedLedgerStore, verify_ledger_chain
 from .observable import ObservableLedgerStore, ObservableLedgerStoreProtocol
+from .offloading import OffloadingLedgerStore, OffloadingRunStore, offload_large_values
 from .snapshots import Snapshot, SnapshotStore, InMemorySnapshotStore, JsonSnapshotStore
 
 __all__ = [
@@ -19,9 +20,11 @@ __all__ = [
     "verify_ledger_chain",
     "ObservableLedgerStore",
     "ObservableLedgerStoreProtocol",
+    "OffloadingRunStore",
+    "OffloadingLedgerStore",
+    "offload_large_values",
     "Snapshot",
     "SnapshotStore",
     "InMemorySnapshotStore",
     "JsonSnapshotStore",
 ]
-