agent-palm 1.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agent_palm/__init__.py ADDED
@@ -0,0 +1,424 @@
1
+ """
2
+ agent_palm - AI PalmPilot
3
+ Agent-native memory layer (serverless, in-process)
4
+ """
5
+
6
+ from .types import (
7
+ MemoryPool,
8
+ MemorySource,
9
+ ReviewVerdict,
10
+ MemoryLevel,
11
+ MemoryRef,
12
+ MemoryNode,
13
+ RecallNode,
14
+ )
15
+ from .store import PalmStore, PalmStorageBusyError, PalmModelMismatchError
16
+ from .embed import OnnxEmbedder, HashEmbedder
17
+ from .extract import heuristic_extract
18
+ from .retrieval import pack_budget, verify_citations, format_context, build_grounded_context, extract_temporal_constraint, TemporalConstraint, apply_temporal_filter
19
+
20
+ __all__ = [
21
+ "MemoryPool",
22
+ "MemorySource",
23
+ "ReviewVerdict",
24
+ "MemoryLevel",
25
+ "MemoryRef",
26
+ "MemoryNode",
27
+ "RecallNode",
28
+ "PalmStore",
29
+ "PalmStorageBusyError",
30
+ "PalmModelMismatchError",
31
+ "OnnxEmbedder",
32
+ "HashEmbedder",
33
+ "heuristic_extract",
34
+ "pack_budget",
35
+ "verify_citations",
36
+ "format_context",
37
+ "build_grounded_context",
38
+ "PalmPilot",
39
+ ]
40
+
41
+ __version__ = "1.3.1"
42
+
43
+
44
+ class PalmPilot:
45
+ """
46
+ High-level agent-facing API for agent_palm.
47
+
48
+ Usage:
49
+ palm = PalmPilot(agent_id="sebastian")
50
+
51
+ # Store a memory
52
+ uri = palm.remember(
53
+ "Decided to use ONNX Runtime for embeddings - no more llama.cpp crashes!",
54
+ namespace="decisions",
55
+ tags=["architecture", "embeddings"],
56
+ )
57
+
58
+ # Search with token budget
59
+ results = palm.search(
60
+ "embedding crashes",
61
+ max_tokens=2000,
62
+ pool="both",
63
+ )
64
+
65
+ # Explicit full-detail recall
66
+ detail = palm.recall(uri, level="l2", max_tokens=4000)
67
+
68
+ # Self-improving review
69
+ palm.annotate(uri, "This decision was superseded by newer benchmarks",
70
+ confidence=0.9, verdict="superseded")
71
+ """
72
+
73
+ def __init__(
74
+ self,
75
+ agent_id: str = None,
76
+ root: str = None,
77
+ use_hash_embed: bool = False,
78
+ ):
79
+ import os
80
+
81
+ self.agent_id = agent_id or os.environ.get("USER", "agent")
82
+ self.root = root or os.environ.get("MEMPALM_PATH", os.path.expanduser("~/.mempalm"))
83
+ self.use_hash_embed = use_hash_embed
84
+
85
+ # Initialize stores
86
+ self._init_stores()
87
+ self._init_embedder()
88
+
89
+ # Review manager - given both stores so shared-pool memories can be
90
+ # annotated, superseded, queued for review and marked as used, plus the
91
+ # embedder so supersede() vectors the replacement with the real model.
92
+ from .review import ReviewManager
93
+ self._review = ReviewManager(
94
+ self.store,
95
+ shared_store=self.shared_store,
96
+ embedder=self.embedder,
97
+ )
98
+
99
+ def _init_stores(self):
100
+ """Initialize personal and shared stores"""
101
+ import os
102
+ from .store import PalmStore
103
+
104
+ os.makedirs(os.path.join(self.root, "agents"), exist_ok=True)
105
+ os.makedirs(os.path.join(self.root, "models"), exist_ok=True)
106
+
107
+ # Personal store
108
+ personal_path = os.path.join(self.root, "agents", f"{self.agent_id}.db")
109
+ self.store = PalmStore(db_path=personal_path)
110
+
111
+ # Shared store - always create
112
+ shared_path = os.path.join(self.root, "hermes.db")
113
+ self.shared_store = PalmStore(db_path=shared_path)
114
+
115
+ def _init_embedder(self):
116
+ """Initialize embedder"""
117
+ from .embed import get_embedder
118
+ self.embedder = get_embedder(use_hash=self.use_hash_embed)
119
+
120
+ def remember(
121
+ self,
122
+ content: str,
123
+ namespace: str = "default",
124
+ tags: list = None,
125
+ type_: str = "note",
126
+ source: str = "inferred",
127
+ ttl_days: int = None,
128
+ pool: str = "personal",
129
+ ) -> str:
130
+ """
131
+ Store a memory with automatic L0/L1 extraction and embedding.
132
+
133
+ Returns:
134
+ URI of the stored memory
135
+ """
136
+ tags = tags or []
137
+
138
+ # Auto-extract L0/L1
139
+ l0, l1 = heuristic_extract(content)
140
+
141
+ # Select store
142
+ store = self.shared_store if pool == "shared" else self.store
143
+ owner = "hermes" if pool == "shared" else self.agent_id
144
+
145
+ # ── Embed BEFORE the row is committed ──────────────────────────────
146
+ #
147
+ # The row must not exist without its vector: an un-vectorized memory
148
+ # is invisible to semantic search (FTS still finds it), so a partially
149
+ # written row is a silent data-quality bug. If the embedder raises
150
+ # (e.g. the ONNX model is not installed yet), nothing is written and
151
+ # the caller gets the original error.
152
+ #
153
+ # The duplicate case is detected up-front so duplicates never pay the
154
+ # embed cost. When an existing row already carries a vector we reuse
155
+ # it: the content hash matched, so the content (and therefore the
156
+ # embedding) is unchanged.
157
+ #
158
+ # If the row exists but its vector is MISSING (e.g. it was written by
159
+ # an older version that could fail between the row insert and the
160
+ # vector insert), we embed and re-store the vector so the memory is
161
+ # repaired rather than left permanently un-searchable.
162
+ import hashlib
163
+ content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
164
+ existing_id = store.get_node_id_by_hash(pool, namespace, content_hash)
165
+ existing_has_vector = existing_id is not None and store.has_vector(existing_id)
166
+
167
+ embedding = None
168
+ if not existing_has_vector:
169
+ embedding = self.embedder.embed_single(l0 + " " + l1)
170
+
171
+ # Store
172
+ uri, is_dup, near_dup = store.remember(
173
+ content=content,
174
+ pool=pool,
175
+ owner=owner,
176
+ author=self.agent_id,
177
+ namespace=namespace,
178
+ type_=type_,
179
+ source=source,
180
+ tags=tags,
181
+ fields=None,
182
+ ttl_days=ttl_days,
183
+ )
184
+
185
+ # Store the vector whenever we computed one:
186
+ # * new row -> always needs its vector
187
+ # * dedup, no vector -> repairing a missing vector
188
+ # * dedup, has vector -> skipped above (embedding is None)
189
+ if embedding is not None:
190
+ node_id = store.get_node_id(uri)
191
+ if node_id:
192
+ store.store_vector(node_id, embedding)
193
+
194
+ return uri
195
+
196
+ def search(
197
+ self,
198
+ query: str,
199
+ namespace: str = None,
200
+ pool: str = "both",
201
+ level: str = "l1",
202
+ max_tokens: int = 2000,
203
+ limit: int = 10,
204
+ include_superseded: bool = False,
205
+ ) -> list:
206
+ """
207
+ Search memories with hybrid retrieval and temporal filtering.
208
+
209
+ Args:
210
+ query: Search query
211
+ namespace: Namespace to search (None = all)
212
+ pool: "personal", "shared", or "both"
213
+ level: "l0" or "l1" (L2 requires explicit recall)
214
+ max_tokens: Token budget for results
215
+ limit: Maximum results
216
+ include_superseded: Include superseded memories
217
+
218
+ Returns:
219
+ List of MemoryNode-like dicts
220
+ """
221
+ # Embed query
222
+ query_embedding = self.embedder.embed_single(query)
223
+
224
+ # Extract temporal constraint from query
225
+ temporal_constraint = extract_temporal_constraint(query)
226
+
227
+ all_results = []
228
+ pools = ["personal", "shared"] if pool == "both" else [pool]
229
+
230
+ for p in pools:
231
+ if p == "shared":
232
+ if not self.shared_store:
233
+ continue
234
+ store = self.shared_store
235
+ else:
236
+ store = self.store
237
+
238
+ try:
239
+ # Vector search
240
+ vec_results = store.search_vec(query_embedding, pool=p, namespace=namespace, k=50)
241
+
242
+ # FTS search
243
+ fts_results = store.search_fts(query, pool=p, namespace=namespace, k=50)
244
+
245
+ # Fuse and fetch with temporal filtering
246
+ results = store.fuse_and_fetch(
247
+ vec_results, fts_results,
248
+ k=limit, max_tokens=max_tokens,
249
+ include_superseded=include_superseded,
250
+ query_text=query,
251
+ temporal_constraint=temporal_constraint
252
+ )
253
+
254
+ all_results.extend(results)
255
+
256
+ except Exception as e:
257
+ # Log but continue
258
+ pass
259
+
260
+ # Sort by score
261
+ all_results.sort(key=lambda x: x.get("score", 0), reverse=True)
262
+
263
+ return all_results[:limit]
264
+
265
+ def recall(
266
+ self,
267
+ uri: str,
268
+ level: str = "l1",
269
+ max_tokens: int = 4000,
270
+ offset: int = 0
271
+ ) -> dict:
272
+ """
273
+ Recall full memory content at specified level.
274
+
275
+ Args:
276
+ uri: Memory URI
277
+ level: "l0", "l1", or "l2"
278
+ max_tokens: Max tokens for L2 pagination
279
+ offset: Pagination offset for L2
280
+
281
+ Returns:
282
+ Dict with text, truncated, total_tokens, offset
283
+ """
284
+ # Try personal first
285
+ try:
286
+ result = self.store.recall(uri, level=level, max_tokens=max_tokens, offset=offset)
287
+ if result["text"]:
288
+ return result
289
+ except Exception:
290
+ pass
291
+
292
+ # Try shared
293
+ if self.shared_store:
294
+ try:
295
+ result = self.shared_store.recall(uri, level=level, max_tokens=max_tokens, offset=offset)
296
+ if result["text"]:
297
+ return result
298
+ except Exception:
299
+ pass
300
+
301
+ return {"error": f"Memory not found: {uri}", "text": "", "truncated": False, "total_tokens": 0, "offset": 0}
302
+
303
+ def forget(self, uri: str) -> bool:
304
+ """Permanently delete a memory"""
305
+ try:
306
+ self.store.forget(uri)
307
+ return True
308
+ except Exception:
309
+ if self.shared_store:
310
+ try:
311
+ self.shared_store.forget(uri)
312
+ return True
313
+ except Exception:
314
+ pass
315
+ return False
316
+
317
+ def annotate(
318
+ self,
319
+ uri: str,
320
+ review_text: str,
321
+ confidence: float,
322
+ verdict: str,
323
+ evidence_uri: str = None,
324
+ tags: list = None,
325
+ ) -> dict:
326
+ """
327
+ Add review annotation (self-improving review loop).
328
+
329
+ Args:
330
+ uri: Memory URI to annotate
331
+ review_text: Review content
332
+ confidence: 0.0-1.0
333
+ verdict: "confirmed", "uncertain", "superseded", "disputed"
334
+ evidence_uri: Optional URI of supporting evidence
335
+ tags: Optional tags
336
+
337
+ Returns:
338
+ Dict with new trust score and annotation info
339
+ """
340
+ return self._review.annotate(
341
+ uri=uri,
342
+ reviewer=self.agent_id,
343
+ verdict=verdict,
344
+ confidence=confidence,
345
+ review_text=review_text,
346
+ evidence_uri=evidence_uri,
347
+ tags=tags or [],
348
+ )
349
+
350
+ def supersede(
351
+ self,
352
+ old_uri: str,
353
+ new_content: str,
354
+ **remember_kwargs
355
+ ) -> str:
356
+ """Create new memory that supersedes old one"""
357
+ return self._review.supersede(old_uri, new_content, **remember_kwargs)
358
+
359
+ def review_queue(
360
+ self,
361
+ namespace: str = None,
362
+ limit: int = 10,
363
+ pool: str = "personal"
364
+ ) -> list:
365
+ """Get memories needing review"""
366
+ return self._review.review_queue(namespace, limit, pool)
367
+
368
+ def mark_used(self, uris: list):
369
+ """Mark memories as used (for implicit review signal)"""
370
+ self._review.mark_used(uris)
371
+
372
+ def verify_citations(self, response: str) -> list:
373
+ """Verify citations in model response"""
374
+ from agent_palm.retrieval import verify_citations
375
+ # Get recently cited URIs from recent searches
376
+ # This would need to track recently retrieved URIs
377
+ return verify_citations(response, [])
378
+
379
+ def stats(self, namespace: str = None, pool: str = "both") -> dict:
380
+ """Get namespace statistics"""
381
+ results = {}
382
+
383
+ if pool in ["personal", "both"]:
384
+ results["personal"] = self.store.namespace_stats("personal", namespace)
385
+
386
+ if pool in ["shared", "both"] and self.shared_store:
387
+ results["shared"] = self.shared_store.namespace_stats("shared", namespace)
388
+
389
+ return results
390
+
391
+ def backup(self, destination: str = None, pool: str = "both") -> dict:
392
+ """Backup database using SQLite Backup API"""
393
+ import os
394
+ import time
395
+
396
+ if not destination:
397
+ timestamp = time.strftime("%Y%m%d_%H%M%S")
398
+ destination = os.path.join(self.root, "backups", f"mempalm_backup_{timestamp}")
399
+
400
+ results = {}
401
+
402
+ if pool in ["personal", "both"]:
403
+ dest = f"{destination}_{self.agent_id}.db"
404
+ self.store.backup(dest)
405
+ results["personal"] = dest
406
+
407
+ if pool in ["shared", "both"] and self.shared_store:
408
+ dest = f"{destination}_hermes.db"
409
+ self.shared_store.backup(dest)
410
+ results["shared"] = dest
411
+
412
+ return results
413
+
414
+ def close(self):
415
+ """Close all connections"""
416
+ self.store.close()
417
+ if self.shared_store:
418
+ self.shared_store.close()
419
+
420
+ def __enter__(self):
421
+ return self
422
+
423
+ def __exit__(self, exc_type, exc_val, exc_tb):
424
+ self.close()
agent_palm/embed.py ADDED
@@ -0,0 +1,176 @@
1
+ """
2
+ ONNX Runtime embeddings for agent_palm
3
+ Lazy loading, HashEmbedder for tests, all-MiniLM-L6-v2 int8
4
+ """
5
+
6
+ import os
7
+ import hashlib
8
+ import numpy as np
9
+ from typing import List, Optional, Sequence
10
+ from pathlib import Path
11
+
12
+
13
+ class OnnxEmbedder:
14
+ """
15
+ ONNX Runtime embedder for all-MiniLM-L6-v2
16
+ Lazy loads model on first use to keep import fast.
17
+ """
18
+
19
+ def __init__(self, model_dir: Optional[str] = None):
20
+ self._session = None
21
+ self._tokenizer = None
22
+ self._model_dir = model_dir
23
+ self._model_loaded = False
24
+
25
+ @property
26
+ def model_dir(self) -> str:
27
+ if self._model_dir:
28
+ return self._model_dir
29
+
30
+ # Default to MEMPALM_PATH/models or local models dir
31
+ mempalm_path = os.environ.get("MEMPALM_PATH")
32
+ if mempalm_path:
33
+ return os.path.join(mempalm_path, "models")
34
+
35
+ # Fallback to local models dir
36
+ return os.path.join(os.path.dirname(__file__), "..", "models")
37
+
38
+ def _ensure_loaded(self):
39
+ """Lazy load model and tokenizer on first use"""
40
+ if self._model_loaded:
41
+ return
42
+
43
+ model_dir = self.model_dir
44
+ model_path = os.path.join(model_dir, "all-MiniLM-L6-v2_int8.onnx")
45
+ tokenizer_path = os.path.join(model_dir, "tokenizer.json")
46
+
47
+ # Check if model exists
48
+ if not os.path.exists(model_path):
49
+ raise FileNotFoundError(
50
+ f"Embedding model not found at {model_path}. "
51
+ f"Download from: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2 "
52
+ f"and convert to ONNX int8, or run: python -m agent_palm.export_model"
53
+ )
54
+
55
+ # Load tokenizer
56
+ try:
57
+ from tokenizers import Tokenizer
58
+ self._tokenizer = Tokenizer.from_file(
59
+ os.path.join(model_dir, "tokenizer.json")
60
+ )
61
+ except Exception as e:
62
+ raise RuntimeError(f"Failed to load tokenizer: {e}")
63
+
64
+ # Load ONNX session
65
+ try:
66
+ import onnxruntime as ort
67
+ self._session = ort.InferenceSession(
68
+ os.path.join(model_dir, "all-MiniLM-L6-v2_int8.onnx"),
69
+ providers=["CPUExecutionProvider"],
70
+ sess_options=ort.SessionOptions()
71
+ )
72
+ except Exception as e:
73
+ raise RuntimeError(f"Failed to load ONNX model: {e}")
74
+
75
+ self._model_loaded = True
76
+
77
+ def embed(self, texts: Sequence[str]) -> List[np.ndarray]:
78
+ """
79
+ Embed a batch of texts.
80
+ Returns list of 384-dim float32 arrays (L2 normalized).
81
+ """
82
+ self._ensure_loaded()
83
+
84
+ # Tokenize
85
+ encodings = self._tokenizer.encode_batch(list(texts))
86
+
87
+ # Pad to max length in batch
88
+ max_len = max(len(e.ids) for e in encodings)
89
+ input_ids = np.array([
90
+ e.ids + [0] * (max_len - len(e.ids)) for e in encodings
91
+ ], dtype=np.int64)
92
+
93
+ attention_mask = np.array([
94
+ e.attention_mask + [0] * (max_len - len(e.attention_mask)) for e in encodings
95
+ ], dtype=np.int64)
96
+
97
+ # token_type_ids (all zeros for single-sequence inputs)
98
+ token_type_ids = np.zeros_like(input_ids)
99
+
100
+ # Run inference
101
+ outputs = self._session.run(
102
+ None,
103
+ {
104
+ "input_ids": input_ids,
105
+ "attention_mask": attention_mask,
106
+ "token_type_ids": token_type_ids,
107
+ }
108
+ )
109
+
110
+ # outputs[0] shape: (batch, seq_len, hidden_dim)
111
+ embeddings = outputs[0]
112
+
113
+ # Mean pooling with attention mask
114
+ pooled = []
115
+ for i, emb in enumerate(embeddings):
116
+ mask = attention_mask[i]
117
+ # Mean pool over non-padded tokens
118
+ valid = mask.astype(bool)
119
+ if valid.any():
120
+ vec = emb[valid].mean(axis=0)
121
+ else:
122
+ vec = np.zeros(emb.shape[1], dtype=np.float32)
123
+
124
+ # L2 normalize
125
+ norm = np.linalg.norm(vec)
126
+ if norm > 0:
127
+ vec = vec / norm
128
+
129
+ pooled.append(vec.astype(np.float32))
130
+
131
+ return pooled
132
+
133
+ def embed_single(self, text: str) -> np.ndarray:
134
+ """Embed a single text, return 384-dim float32 array"""
135
+ return self.embed([text])[0]
136
+
137
+
138
+ class HashEmbedder:
139
+ """
140
+ Deterministic hash-based embedder for tests.
141
+ No model required, completely offline, deterministic.
142
+ """
143
+
144
+ def __init__(self, dim: int = 384, seed: int = 42):
145
+ self.dim = dim
146
+ self.seed = seed
147
+
148
+ def embed(self, texts: Sequence[str]) -> List[np.ndarray]:
149
+ """Deterministic pseudo-random embeddings based on text hash"""
150
+ results = []
151
+ for text in texts:
152
+ # Use hash of text as seed for deterministic output
153
+ h = hashlib.md5(text.encode()).hexdigest()
154
+ seed = int(h[:8], 16) ^ self.seed
155
+ rng = np.random.default_rng(seed)
156
+ vec = rng.standard_normal(self.dim).astype(np.float32)
157
+ # L2 normalize
158
+ norm = np.linalg.norm(vec)
159
+ if norm > 0:
160
+ vec = vec / norm
161
+ results.append(vec)
162
+ return results
163
+
164
+ def embed_single(self, text: str) -> np.ndarray:
165
+ return self.embed([text])[0]
166
+
167
+
168
+ def get_embedder(
169
+ use_hash: bool = False,
170
+ model_dir: Optional[str] = None,
171
+ **kwargs
172
+ ):
173
+ """Factory function to get appropriate embedder"""
174
+ if use_hash:
175
+ return HashEmbedder(**kwargs)
176
+ return OnnxEmbedder(model_dir=model_dir)
@@ -0,0 +1,42 @@
1
+ """
2
+ Custom exceptions for agent_palm
3
+ """
4
+
5
+ class PalmError(Exception):
6
+ """Base exception for agent_palm"""
7
+ pass
8
+
9
+
10
+ class PalmStorageBusyError(Exception):
11
+ """Raised when SQLite returns SQLITE_BUSY/LOCKED after retries"""
12
+ def __init__(self, message: str, retryable: bool = True):
13
+ super().__init__(message)
14
+ self.retryable = retryable
15
+
16
+
17
+ class PalmModelMismatchError(Exception):
18
+ """Raised when database embedding model doesn't match runtime config"""
19
+ def __init__(self, message: str, db_model: str = None, expected_model: str = None):
20
+ super().__init__(message)
21
+ self.db_model = db_model
22
+ self.expected_model = expected_model
23
+
24
+
25
+ class PalmStorageFullError(Exception):
26
+ """Raised when storage is full (SQLITE_FULL)"""
27
+ pass
28
+
29
+
30
+ class PalmValidationError(Exception):
31
+ """Raised when input validation fails"""
32
+ pass
33
+
34
+
35
+ class PalmNotFoundError(Exception):
36
+ """Raised when a memory URI is not found"""
37
+ pass
38
+
39
+
40
+ class PalmPermissionError(Exception):
41
+ """Raised when an operation is not permitted (e.g., forget shared memory)"""
42
+ pass