loop-memory 0.4.0__py3-none-any.whl

loop_memory/memory/types.py

@@ -0,0 +1,199 @@
+"""Typed memory objects shared across tiers.
+
+A `MemoryItem` is the atomic unit of memory. Each tier (short / long /
+episodic / procedural) wraps a collection of items with its own
+retention, retrieval, and update semantics.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+
+@dataclass
+class MemoryItem:
+    """Atomic memory record.
+
+    Attributes:
+        text: human-readable content.
+        embedding: optional numeric vector (set when an embedder is wired in).
+        importance: 0..1 score influencing retention priority.
+        created_at: unix timestamp.
+        ttl: time-to-live in seconds; ``None`` means it never expires.
+        kind: 'fact' | 'episode' | 'plan' | 'reflection' | free-form.
+        tags: lightweight labels for filtering.
+        source: optional pointer back to the originating event.
+    """
+
+    text: str
+    embedding: list[float] | None = None
+    importance: float = 0.5
+    created_at: float = field(default_factory=time.time)
+    ttl: float | None = None
+    kind: str = "fact"
+    tags: list[str] = field(default_factory=list)
+    source: str | None = None
+    id: str = field(default_factory=lambda: uuid.uuid4().hex)
+
+    def is_expired(self, now: float | None = None) -> bool:
+        if self.ttl is None:
+            return False
+        cur = now if now is not None else time.time()
+        return (cur - self.created_at) > self.ttl
+
+    def score(self, now: float | None = None) -> float:
+        """Recency × importance score in [0, 1]."""
+        cur = now if now is not None else time.time()
+        age = max(0.0, cur - self.created_at)
+        # half-life decay: drop by half every `decay_half_life` seconds
+        decay_half_life = 60 * 60 * 24 * 7  # 1 week
+        recency = 0.5 ** (age / decay_half_life)
+        return max(0.0, min(1.0, self.importance * (0.25 + 0.75 * recency)))
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b, strict=False))
+    na = math.sqrt(sum(x * x for x in a)) or 1e-12
+    nb = math.sqrt(sum(x * x for x in b)) or 1e-12
+    return dot / (na * nb)
+
+
+@dataclass
+class ShortTermMemory:
+    """Ephemeral scratchpad — last N turns, FIFO eviction.
+
+    Mirrors the LLM's working context window. Optional summarization
+    is handled by the engine's reflection step.
+    """
+
+    capacity: int = 16
+    _items: list[MemoryItem] = field(default_factory=list)
+
+    def push(self, item: MemoryItem) -> None:
+        self._items.append(item)
+        if len(self._items) > self.capacity:
+            self._items = self._items[-self.capacity :]
+
+    def extend(self, items: Iterable[MemoryItem]) -> None:
+        for it in items:
+            self.push(it)
+
+    def items(self) -> list[MemoryItem]:
+        return list(self._items)
+
+    def clear(self) -> None:
+        self._items.clear()
+
+
+@dataclass
+class LongTermMemory:
+    """Persistent, semantic facts.
+
+    Vector-similarity retrieval when an embedder is provided; otherwise
+    falls back to importance-weighted lexical scoring.
+    """
+
+    _items: list[MemoryItem] = field(default_factory=list)
+    dedupe_threshold: float = 0.92  # cosine threshold for duplicate suppression
+
+    def add(self, item: MemoryItem) -> bool:
+        """Add an item. Returns False if it was deduplicated."""
+        if item.embedding is not None:
+            for existing in self._items:
+                if existing.embedding is not None:
+                    sim = cosine_similarity(item.embedding, existing.embedding)
+                    if sim >= self.dedupe_threshold:
+                        # boost existing importance instead of duplicating
+                        existing.importance = max(existing.importance, item.importance)
+                        existing.created_at = min(existing.created_at, item.created_at)
+                        return False
+        self._items.append(item)
+        return True
+
+    def extend(self, items: Iterable[MemoryItem]) -> list[MemoryItem]:
+        added: list[MemoryItem] = []
+        for it in items:
+            if self.add(it):
+                added.append(it)
+        return added
+
+    def search(
+        self,
+        query_embedding: list[float] | None = None,
+        top_k: int = 5,
+        now: float | None = None,
+    ) -> list[MemoryItem]:
+        scored: list[tuple[float, MemoryItem]] = []
+        for it in self._items:
+            if it.is_expired(now):
+                continue
+            base = it.score(now)
+            if query_embedding is not None and it.embedding is not None:
+                base = 0.7 * base + 0.3 * cosine_similarity(query_embedding, it.embedding)
+            scored.append((base, it))
+        scored.sort(key=lambda x: x[0], reverse=True)
+        return [it for _, it in scored[:top_k]]
+
+    def forget(self, predicate) -> int:
+        before = len(self._items)
+        self._items = [it for it in self._items if not predicate(it)]
+        return before - len(self._items)
+
+    def gc(self, now: float | None = None) -> int:
+        return self.forget(lambda it: it.is_expired(now))
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+
+@dataclass
+class EpisodicMemory:
+    """Time-ordered event stream: what happened, in what order.
+
+    Used to answer "what did we do recently?" rather than "what do we
+    know?". Think of it as a transactional log that the reflection step
+    periodically compacts into long-term facts.
+    """
+
+    max_events: int = 1000
+    _events: list[MemoryItem] = field(default_factory=list)
+
+    def record(self, event: MemoryItem) -> None:
+        event.kind = event.kind or "episode"
+        self._events.append(event)
+        if len(self._events) > self.max_events:
+            self._events = self._events[-self.max_events :]
+
+    def recent(self, n: int = 5) -> list[MemoryItem]:
+        return list(self._events[-n:])
+
+    def between(self, t0: float, t1: float) -> list[MemoryItem]:
+        return [e for e in self._events if t0 <= e.created_at <= t1]
+
+
+@dataclass
+class ProceduralMemory:
+    """Structured task plan / current-goal stack.
+
+    Lets the engine track what the user is *trying* to do across turns
+    so a single message can be interpreted as the next step of an open
+    plan rather than a fresh request.
+    """
+
+    goals: list[MemoryItem] = field(default_factory=list)
+
+    def push(self, goal: MemoryItem) -> None:
+        goal.kind = "plan"
+        self.goals.append(goal)
+
+    def current(self) -> MemoryItem | None:
+        return self.goals[-1] if self.goals else None
+
+    def complete_top(self) -> MemoryItem | None:
+        return self.goals.pop() if self.goals else None

loop_memory/privacy/__init__.py

@@ -0,0 +1,22 @@
+"""Privacy utilities — secret redaction & private-tag handling.
+
+Goals:
+
+* Never let a recognised secret (API key, JWT, SSH private key, …)
+  reach the long-term store. Redact in-place so the surrounding
+  context is preserved (e.g. "I set OPENAI_API_KEY=sk-…1234"
+  becomes "I set OPENAI_API_KEY=[REDACTED:openai_key]") and the
+  LLM still knows roughly what was being talked about.
+* Honour the user-controlled ``<private>...</private>`` tags to
+  mark regions that should be skipped entirely.
+* Be deterministic so re-running distillation on the same text
+  produces the same redactions (no drift on round-trips).
+"""
+
+from .redact import redact_text, redact_batch, RedactionSummary, REDACT_KINDS
+from .private import strip_private_spans, has_private_blocks
+
+__all__ = [
+    "redact_text", "redact_batch", "RedactionSummary", "REDACT_KINDS",
+    "strip_private_spans", "has_private_blocks",
+]

loop_memory/privacy/private.py

@@ -0,0 +1,46 @@
+"""User-controlled ``<private>...</private>`` span handling.
+
+Users can mark ranges they want kept out of the long-term store
+entirely (``This is my actual birthday: <private>1990-01-01</private>``)
+or just out of distillation / recall. The span delimiters are
+preserved so the agent still sees the marker itself, but the
+content inside is replaced by ``[PRIVATE:redacted]``.
+
+Pattern notes:
+
+* Case-insensitive (``<PRIVATE>...</PRIVATE>`` too).
+* Non-greedy so multiple spans per turn all get caught.
+* Multi-line: spans may span newlines.
+"""
+
+from __future__ import annotations
+
+import re
+
+_PRIVATE_RE = re.compile(
+    r"<private>([\s\S]*?)</private>",
+    re.IGNORECASE,
+)
+
+
+def strip_private_spans(text: str, *, replacement: str | None = None) -> str:
+    """Replace every ``<private>...</private>`` body with a marker.
+
+    Default replacement is ``"[PRIVATE:redacted]"``. Pass a different
+    ``replacement`` if your pipeline needs something else (e.g. an
+    empty string when the entire span is also being dropped).
+    """
+    if not text:
+        return text
+    rep = replacement if replacement is not None else "[PRIVATE:redacted]"
+    return _PRIVATE_RE.sub(rep, text)
+
+
+def has_private_blocks(text: str) -> bool:
+    """``True`` iff ``text`` contains at least one private span
+    (after stripping content). Useful as a cheap pre-filter so
+    callers don't run distillation on text that will be empty.
+    """
+    if not text:
+        return False
+    return _PRIVATE_RE.search(text) is not None

loop_memory/privacy/redact.py

@@ -0,0 +1,188 @@
+"""Pattern-based secret redaction.
+
+The redaction layer runs **before** anything that touches long-term
+storage (memory upserts, distilled wiki body, ask/inject output, export
+markdown) so a leaked key never leaves the ingest path.
+
+Design choices:
+
+* Deterministic — same input always yields the same output. The
+  patterns are ordered by length so the longest match wins on
+  overlaps (an AWS key is preferred over a generic 32-char
+  blob in the same span).
+* Idempotent — re-running redact on already-redacted text is a
+  no-op (placeholders are recognised and skipped).
+* Surgeries only the secret — the surrounding context is kept so
+  the LLM still sees *what* was being talked about
+  (``set OPENAI_API_KEY=sk-…`` becomes
+  ``set OPENAI_API_KEY=[REDACTED:openai_key]``).
+* Cheap — pure regex, no model call, microseconds on a 4 KB
+  transcript fragment.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+
+# ----- Pattern catalogue ------------------------------------------------
+# Each entry is (kind, compiled regex). Order matters: longer / more
+# specific patterns must come first so a ``sk-…`` token isn't matched
+# as a generic 32-char blob. ``redact_text`` walks the list top-down.
+
+REDACT_KINDS: tuple[str, ...] = (
+    "private_key_block",        # -----BEGIN ... PRIVATE KEY-----
+    "openai_key",               # sk-<not ant/proj/svc>...
+    "openai_project_key",       # sk-proj-...
+    "openai_service_key",       # sk-svc-...
+    "anthropic_key",            # sk-ant-...
+    "gemini_key",               # AIza...  (Google AI Studio)
+    "github_pat",               # ghp_ / gho_ / ghs_ / ghu_ / ghr_
+    "slack_token",              # xoxb- / xoxp- / xoxa- / xoxs-
+    "aws_access_key",           # AKIA / ASIA
+    "jwt",                      # eyJ...eyJ...{sig}
+    "bearer_token",             # ``Bearer xxx`` in HTTP headers
+    "generic_high_entropy",     # fallback: 40+ char opaque blob in
+                                # an env-style assignment (``KEY=xxx``,
+                                # ``"api_key": "xxx"``)
+)
+
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    # Markers in long-term constants used to bypass subsequent patterns
+    # (so re-running on already-redacted text doesn't re-match them).
+    ("__redacted_placeholder__", re.compile(r"\[REDACTED:[a-z_]+\]")),
+
+    # Code-fence-style private keys (capture the whole BEGIN..END block).
+    ("private_key_block",
+     re.compile(
+         r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP |)PRIVATE KEY-----"
+         r"[\s\S]*?-----END (?:RSA |DSA |EC |OPENSSH |PGP |)PRIVATE KEY-----",
+         re.MULTILINE,
+     )),
+
+    # Order matters: more specific patterns MUST come first so the
+    # generic OpenAI matcher doesn't scoop up an Anthropic key.
+    ("openai_project_key",
+     re.compile(r"\bsk-proj-[A-Za-z0-9_\-]{20,}")),
+    ("openai_service_key",
+     re.compile(r"\bsk-svc-[A-Za-z0-9_\-]{20,}")),
+    ("anthropic_key",
+     re.compile(r"\bsk-ant-[A-Za-z0-9_\-]{20,}")),
+    # OpenAI user keys: ``sk-...`` where the suffix is NOT a known
+    # sub-brand. Excludes ``sk-ant-``, ``sk-proj-``, ``sk-svc-`` so
+    # those don't fall through to this generic bucket.
+    ("openai_key",
+     re.compile(r"\bsk-(?!ant-|proj-|svc-)[A-Za-z0-9_\-]{20,}")),
+    ("gemini_key",
+     re.compile(r"\bAIza[A-Za-z0-9_\-]{20,}\b")),
+    ("github_pat",
+     re.compile(r"\b(?:ghp|gho|ghs|ghu|ghr)_[A-Za-z0-9]{25,}\b")),
+    ("slack_token",
+     re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b")),
+    ("aws_access_key",
+     re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
+    ("google_api_key",
+     re.compile(r"\bAIzaSy[A-Za-z0-9_-]{33}\b")),
+
+    # JWT: three dot-separated base64url segments, the first being
+    # always ``eyJ...``.
+    ("jwt",
+     re.compile(r"\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b")),
+
+    # Bearer token (used in HTTP headers and tool outputs).
+    ("bearer_token",
+     re.compile(r"(?i)\bBearer\s+[A-Za-z0-9_\-\.~+\/]{20,}=*")),
+]
+
+# Generic high-entropy fallbacks — only when context suggests a key
+# (env-style assignment or quoted JSON value).
+_GENERIC_ENV = re.compile(
+    r"""(?ix)
+    (?: ^ | [\s,;{(] )
+    (?: (?:[A-Z][A-Z0-9_]{2,})            # ALL_CAPS_KEY_NAME
+        | ["']?(?:api[_-]?key|secret|token|password|passwd|access[_-]?key)["']?
+    )
+    \s*[:=]\s*
+    ["']?
+    ([A-Za-z0-9_\-\.~+\/]{40,}=*)      # long opaque blob
+    ["']?
+    """,
+)
+
+
+@dataclass
+class RedactionSummary:
+    """Counts of redactions, by kind. Returned by :func:`redact_text`
+    so callers (telemetry, UI preview) can show "redacted 3 API
+    keys" without re-running the regexes.
+    """
+    counts: dict[str, int] = field(default_factory=dict)
+    total_chars: int = 0        # how many characters were replaced
+
+    @property
+    def total(self) -> int:
+        return sum(self.counts.values())
+
+    def add(self, kind: str, n_chars: int) -> None:
+        self.counts[kind] = self.counts.get(kind, 0) + 1
+        self.total_chars += n_chars
+
+    def merge(self, other: RedactionSummary) -> None:
+        for k, v in other.counts.items():
+            self.counts[k] = self.counts.get(k, 0) + v
+        self.total_chars += other.total_chars
+
+
+def _placeholder(kind: str) -> str:
+    return f"[REDACTED:{kind}]"
+
+
+def redact_text(text: str, *, summary: RedactionSummary | None = None) -> str:
+    """Return ``text`` with every recognised secret replaced by a
+    short placeholder.
+
+    The transformation is identity if no pattern matches. A summary
+    can be supplied to accumulate counts across many calls.
+    """
+    if not text:
+        return text
+    out = text
+    for kind, pat in _PATTERNS:
+        if kind == "__redacted_placeholder__":
+            continue
+        # Replace, tracking match length for the summary.
+        def _sub(m: re.Match[str], redaction_kind: str = kind) -> str:
+            n = len(m.group(0))
+            if summary is not None:
+                summary.add(redaction_kind, n)
+            return _placeholder(redaction_kind)
+        out = pat.sub(_sub, out)
+
+    # Generic fallback: only when an env-style assignment is found.
+    def _gen_sub(m: re.Match[str]) -> str:
+        opaq = m.group(1)
+        if not opaq:
+            return m.group(0)
+        n = len(opaq)
+        if summary is not None:
+            summary.add("generic_high_entropy", n)
+        # Preserve the leading key name + separator.
+        # The capture is the opaque blob; rebuild surrounding context.
+        # m.group(0) has full match like 'API_KEY="abcdef..."'.
+        prefix = m.group(0).rsplit(opaq, 1)[0]
+        return prefix + _placeholder("generic_high_entropy")
+    out = _GENERIC_ENV.sub(_gen_sub, out)
+
+    return out
+
+
+def redact_batch(texts: Iterable[str]) -> list[str]:
+    """Apply :func:`redact_text` to a batch of strings.
+
+    A shared summary is returned separately if needed
+    (``RedactionSummary`` is on ``summary``; this helper only
+    returns the redacted texts).
+    """
+    summary = RedactionSummary()
+    return [redact_text(t, summary=summary) for t in texts]