memoryagent-lib 0.1.1__py3-none-any.whl

memoryagent/utils.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import re
+from typing import Iterable, List, Set
+
+
+_WORD_RE = re.compile(r"[a-zA-Z0-9']+")
+
+
+def tokenize(text: str) -> List[str]:
+    return [t.lower() for t in _WORD_RE.findall(text or "")]
+
+
+def unique_tokens(text: str) -> Set[str]:
+    return set(tokenize(text))
+
+
+def safe_div(numerator: float, denominator: float) -> float:
+    return numerator / denominator if denominator else 0.0
+
+
+def clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
+    return max(low, min(high, value))
+
+
+def hash_embed(text: str, dim: int) -> List[float]:
+    tokens = tokenize(text)
+    if dim <= 0:
+        raise ValueError("dim must be positive")
+    vector = [0.0] * dim
+    for token in tokens:
+        idx = hash(token) % dim
+        vector[idx] += 1.0
+    norm = sum(v * v for v in vector) ** 0.5 or 1.0
+    return [v / norm for v in vector]

memoryagent/workers.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import List, Tuple
+
+from memoryagent.config import MemorySystemConfig
+from memoryagent.indexers import EpisodicIndexer
+from memoryagent.models import MemoryItem, MemoryType, StorageTier, utc_now
+from memoryagent.storage.base import MetadataStore, ObjectStore, VectorIndex
+
+
+class ConsolidationWorker:
+    def __init__(
+        self,
+        metadata_store: MetadataStore,
+        vector_index: VectorIndex,
+        config: MemorySystemConfig,
+    ) -> None:
+        self.metadata_store = metadata_store
+        self.vector_index = vector_index
+        self.config = config
+        self.indexer = EpisodicIndexer(vector_index)
+
+    async def run_once(self, owner: str) -> List[MemoryItem]:
+        items = await self.metadata_store.list_by_owner(owner)
+        working = [item for item in items if item.type == MemoryType.WORKING and item.tier == StorageTier.HOT]
+        perceptual = [item for item in items if item.type == MemoryType.PERCEPTUAL and item.tier == StorageTier.HOT]
+
+        new_items: List[MemoryItem] = []
+
+        if working:
+            summary = " | ".join([item.summary for item in working[:5]])
+            new_items.append(
+                MemoryItem(
+                    type=MemoryType.EPISODIC,
+                    owner=owner,
+                    summary=f"Session summary: {summary}",
+                    tags=["session-summary"],
+                    confidence=0.6,
+                )
+            )
+
+        if perceptual:
+            snippets = [item.summary for item in perceptual[: self.config.consolidation.perceptual_summary_limit]]
+            new_items.append(
+                MemoryItem(
+                    type=MemoryType.EPISODIC,
+                    owner=owner,
+                    summary=f"Perceptual highlights: {' | '.join(snippets)}",
+                    tags=["perceptual-summary"],
+                    confidence=0.55,
+                )
+            )
+
+        tag_counts = Counter()
+        for item in working + perceptual:
+            tag_counts.update(item.tags)
+
+        for tag, count in tag_counts.items():
+            if count >= self.config.consolidation.semantic_min_count:
+                new_items.append(
+                    MemoryItem(
+                        type=MemoryType.SEMANTIC,
+                        owner=owner,
+                        summary=f"Observed recurring tag: {tag}",
+                        tags=[tag, "derived"],
+                        confidence=0.65,
+                        stability=0.6,
+                    )
+                )
+
+        for item in new_items:
+            await self.metadata_store.upsert(item)
+            await self.indexer.index_hot(item)
+
+        return new_items
+
+
+class ArchiverWorker:
+    def __init__(
+        self,
+        metadata_store: MetadataStore,
+        object_store: ObjectStore,
+        vector_index: VectorIndex,
+    ) -> None:
+        self.metadata_store = metadata_store
+        self.object_store = object_store
+        self.vector_index = vector_index
+        self.indexer = EpisodicIndexer(vector_index)
+
+    async def run_once(self, owner: str) -> List[MemoryItem]:
+        items = await self.metadata_store.list_by_owner(owner)
+        to_archive = [item for item in items if item.tier == StorageTier.HOT and item.type != MemoryType.WORKING]
+
+        archived: List[MemoryItem] = []
+        for item in to_archive:
+            date_path = item.created_at.strftime("%Y/%m/%d")
+            key = f"{owner}/{date_path}/daily_notes"
+            payload = {
+                "id": str(item.id),
+                "summary": item.summary,
+                "content": item.content,
+                "tags": item.tags,
+                "type": item.type.value,
+                "owner": item.owner,
+                "created_at": item.created_at.isoformat(),
+            }
+            if hasattr(self.object_store, "append"):
+                object_path = await self.object_store.append(key, payload)
+            else:
+                object_path = await self.object_store.put(key, payload)
+            item.pointer["object_key"] = object_path
+            item.pointer["archive_key"] = key
+            item.tier = StorageTier.COLD
+            item.updated_at = utc_now()
+            await self.metadata_store.upsert(item)
+            await self.indexer.index_archive(item)
+            archived.append(item)
+        return archived
+
+
+class RehydratorWorker:
+    def __init__(
+        self,
+        metadata_store: MetadataStore,
+        vector_index: VectorIndex,
+        access_threshold: int = 3,
+    ) -> None:
+        self.metadata_store = metadata_store
+        self.vector_index = vector_index
+        self.access_threshold = access_threshold
+        self._access_counts = {}
+
+    async def record_access(self, item_id) -> None:
+        item_id = str(item_id)
+        self._access_counts[item_id] = self._access_counts.get(item_id, 0) + 1
+
+    async def run_once(self, owner: str) -> List[MemoryItem]:
+        items = await self.metadata_store.list_by_owner(owner)
+        warmed: List[MemoryItem] = []
+        for item in items:
+            if item.tier != StorageTier.COLD:
+                continue
+            count = self._access_counts.get(str(item.id), 0)
+            if count >= self.access_threshold:
+                item.tier = StorageTier.HOT
+                item.updated_at = utc_now()
+                await self.metadata_store.upsert(item)
+                await self.vector_index.upsert(
+                    item.id,
+                    text=item.text(),
+                    metadata={"owner": item.owner, "tier": StorageTier.HOT.value, "type": item.type.value, "item": item},
+                )
+                warmed.append(item)
+        return warmed
+
+
+class Compactor:
+    def __init__(self, metadata_store: MetadataStore) -> None:
+        self.metadata_store = metadata_store
+
+    async def run_once(self, owner: str) -> List[MemoryItem]:
+        items = await self.metadata_store.list_by_owner(owner)
+        removed: List[MemoryItem] = []
+        for item in items:
+            if item.is_expired():
+                await self.metadata_store.delete(item.id)
+                removed.append(item)
+        return removed

memoryagent_lib-0.1.1.dist-info/METADATA

@@ -0,0 +1,186 @@
+Metadata-Version: 2.4
+Name: memoryagent-lib
+Version: 0.1.1
+Summary: Add your description here
+Author-email: Jiawei Zheng <jw.zhengai@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/jia-wei-zheng/MemoryAgent
+Project-URL: Repository, https://github.com/jia-wei-zheng/MemoryAgent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: python-dotenv>=0.9.9
+Requires-Dist: openai>=2.16.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sqlite-vec>=0.1.6
+
+# 
+
+<div align="center">
+  <img src="https://raw.githubusercontent.com/jia-wei-zheng/MemoryAgent/refs/heads/master/memoryagent_logo.jpg?token=GHSAT0AAAAAADSXDGNDXIWZF42E6S3D2XPG2MDPSDQ" alt="MemoryAgent" width="500">
+  <h1>MemoryAgent: An Open, Modular Memory Framework for Agents (Beta)</h1>
+</div>
+
+MemoryAgent is a reusable memory framework for LLM-based agent systems. It provides tiered memory (working, episodic, semantic, perceptual), hot/cold storage, archive indexing, confidence-based retrieval escalation, and optional local vector search via sqlite-vec.
+
+## Highlights
+- **Tiered memory**: working (TTL), episodic, semantic, perceptual
+- **Storage tiers**: hot metadata (SQLite + sqlite-vec), cold archive (filesystem), archive index (vector index)
+- **Memory retrieval pipeline**: hot -> archive -> cold hydration with rerank + context packaging
+- **Local mode**: SQLite + sqlite-vec (optional) + filesystem
+- **Async-friendly** with sync convenience methods
+
+## Project Layout
+```
+memoryagent/
+  config.py            # Default system settings and retrieval thresholds
+  models.py            # Pydantic data models for memory items, queries, bundles
+  system.py            # MemorySystem entry point and wiring
+  retrieval.py         # Retrieval orchestration and reranking
+  confidence.py        # Confidence scoring components
+  policy.py            # Conversation + routing policies
+  indexers.py          # Episodic/semantic/perceptual indexers
+  workers.py           # Consolidation, archiving, rehydration, compaction
+  storage/
+    base.py            # Storage adapter interfaces
+    in_memory.py       # Simple in-memory vector + graph stores
+    local_disk.py      # SQLite metadata/features + sqlite-vec + file object store
+  examples/
+    minimal.py         # Basic usage example
+    openai_agent.py    # CLI OpenAI agent with memory retrieval
+    memory_api_server.py # Local API for memory + chat
+    memory_viz.html    # Web UI for chat + memory visualization
+```
+
+## Installation
+Python 3.10+ required.
+
+Development (sync deps from `uv.lock`):
+```bash
+uv sync
+```
+
+Use as a dependency:
+```bash
+uv add memoryagent-lib
+# or
+pip install memoryagent-lib
+```
+
+Optional extras:
+```bash
+uv add openai sqlite-vec
+# or
+pip install openai sqlite-vec
+```
+
+## Quick Start
+```python
+from memoryagent import MemoryEvent, MemorySystem
+
+memory = MemorySystem()
+owner = "user-001"
+
+memory.write(
+    MemoryEvent(
+        content="User prefers concise summaries about climate policy.",
+        type="semantic",
+        owner=owner,
+        tags=["preference", "summary"],
+        confidence=0.7,
+        stability=0.8,
+    )
+)
+
+bundle = memory.retrieve("What policy topics did we cover?", owner=owner)
+print(bundle.confidence.total)
+for block in bundle.blocks:
+    print(block.text)
+
+memory.flush(owner)
+```
+
+## Enable sqlite-vec (Local Vector Search)
+```python
+from memoryagent import MemorySystem, MemorySystemConfig
+
+config = MemorySystemConfig(
+    use_sqlite_vec=True,
+    vector_dim=1536,  # match your embedding model
+)
+
+memory = MemorySystem(config=config)
+```
+
+If sqlite-vec cannot be auto-loaded, set an explicit path:
+```python
+from pathlib import Path
+from memoryagent import MemorySystemConfig
+
+config = MemorySystemConfig(
+    use_sqlite_vec=True,
+    vector_dim=1536,
+    sqlite_vec_extension_path=Path("/path/to/sqlite_vec.dylib"),
+)
+```
+
+## Policies
+### Conversation storage policy
+`HeuristicMemoryPolicy` decides whether a turn should be stored and whether it becomes episodic or semantic memory.
+
+### Routing policy
+`MemoryRoutingPolicy` decides where a memory should be written:
+- **Hot** metadata store
+- **Vector index**
+- **Feature store** (perceptual)
+- **Cold** archive (via workers)
+
+## Background Workers
+- `ConsolidationWorker`: working → episodic/semantic
+- `ArchiverWorker`: hot → cold + archive index
+- `RehydratorWorker`: cold → hot (based on access)
+- `Compactor`: cleanup/TTL
+
+## Examples
+### OpenAI Agent (CLI)
+```bash
+python -m memoryagent.examples.openai_agent
+```
+- Uses OpenAI responses + embeddings.
+- Stores session transcript as a single working memory item.
+
+### Memory Visualization + API
+Start the API server:
+```bash
+python -m memoryagent.examples.memory_api_server
+```
+Open in browser:
+```
+http://127.0.0.1:8000/memory_viz.html
+```
+
+An example (System records semantic memory and updating working memory): 
+
+![Screenshot](https://raw.githubusercontent.com/jia-wei-zheng/MemoryAgent/master/Memory%20Agent%20_%20Live%20Console.jpeg?token=GHSAT0AAAAAADSXDGNDWOWFKKV777VMZM6U2MDPSTQ)
+
+
+The page calls:
+- `GET /api/memory?owner=user-001`
+- `POST /api/chat`
+
+## Data Stores
+- **Hot metadata**: `.memoryagent_hot.sqlite`
+- **Vector index**: `.memoryagent_vectors.sqlite` (sqlite-vec)
+- **Features**: `.memoryagent_features.sqlite`
+- **Cold archive**: `.memoryagent_cold/records/<owner>/YYYY/MM/DD/daily_notes.json`
+
+## Configuration
+See `memoryagent/config.py` for defaults:
+- `working_ttl_seconds`
+- `retrieval_plan` thresholds and budgets
+- `use_sqlite_vec`, `vector_dim`, `sqlite_vec_extension_path`
+
+## Notes
+- Working memory is stored as a single session transcript (updated each turn).
+- Episodic/semantic memories are candidates for cold archive.
+
+## License

memoryagent_lib-0.1.1.dist-info/RECORD

@@ -0,0 +1,22 @@
+memoryagent/__init__.py,sha256=WUv2YjbJzU4lpHIVPDLTrv3wglraPJh6Ky7P9f_r8OU,743
+memoryagent/confidence.py,sha256=bEBrQFj7nTt7xIaFc21BqUaBQOy3DoIVpwTsBUysv54,2626
+memoryagent/config.py,sha256=IO_aAkNP3kq0BukIEFsc3-CkcRlYhlDb7QG5e8lf3Tk,1334
+memoryagent/consolidation.py,sha256=jG5AJr9PNxeqdPi3kD4bbj51YZkrxE4-ZSSTNCnN6Yc,123
+memoryagent/indexers.py,sha256=I2Sv-jyWqs5K83BIXPvSDp9_Z18xClX8QedJyt3ZJzA,2030
+memoryagent/models.py,sha256=mOt-rX8wlzT_n4JsvTDd6iKeJW-bdPp8VGFrb61OgdQ,4185
+memoryagent/policy.py,sha256=doZbUIBAwQVr4BhPjcAtDs_GAM35Z04RYDnc3zf2AJo,6032
+memoryagent/retrieval.py,sha256=PAvaHXAHxk6yXn5iZDR6t39UODwOciWNQimPredbtU4,6124
+memoryagent/system.py,sha256=Dw-t8BXPs1TVNcKDvwavJlNwR95QErulX4c_fy-IyQc,7758
+memoryagent/utils.py,sha256=PZ5AaZwUZDNxj64g70UTXK9ynLtGLeSiHpp8GCmbTIs,883
+memoryagent/workers.py,sha256=2TrwovaAx_NZhcPCwsl-FpLwWCmFC0HfPzHQkJsKO9U,6235
+memoryagent/examples/export_memory.py,sha256=_MEIeGNhXOe5gz9WYJZIl6Kzvdo-L1uuYx_CNJVY-L0,3262
+memoryagent/examples/memory_api_server.py,sha256=AhZ0vynfBzYlUC1-kVkL5u8OHIolGp0Im_fhUQGb2zY,7882
+memoryagent/examples/minimal.py,sha256=KjduPgOIOCy0B27LPxLlMwsEko19-piSlrZOPgv6pcc,1154
+memoryagent/examples/openai_agent.py,sha256=u31uMM7SM-RYSotail49hs5CpRy01DTyrKOLADkNY8M,4577
+memoryagent/storage/base.py,sha256=JYT0HcGIwUz-uydpy9huQ5ABK3G8XshDE6aM3PLpG2s,2453
+memoryagent/storage/in_memory.py,sha256=-AfeYc9TO1EtD59vd2_JTZdJMmTlQYAYQK7fcpKhYAk,3373
+memoryagent/storage/local_disk.py,sha256=EejHxueMmjL7PPPVVl6p8nNmrvJEdxyXQNORly9Nzow,15768
+memoryagent_lib-0.1.1.dist-info/METADATA,sha256=awFooSFKAoQEPg8ZogIRpQ_uAxYh3U9E1mCwvAmnDho,5710
+memoryagent_lib-0.1.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+memoryagent_lib-0.1.1.dist-info/top_level.txt,sha256=0svscYmHfWNY9RQMcUEEmE5I6ksCYfy4wR7P-CLrcIU,12
+memoryagent_lib-0.1.1.dist-info/RECORD,,

memoryagent_lib-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

memoryagent_lib-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+memoryagent