AbstractMemory 0.1.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/AbstractMemory.egg-info/PKG-INFO +162 -33
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/AbstractMemory.egg-info/SOURCES.txt +1 -0
- abstractmemory-0.2.2/AbstractMemory.egg-info/requires.txt +20 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/PKG-INFO +162 -33
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/README.md +147 -24
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/__init__.py +3 -3
- abstractmemory-0.2.2/abstractmemory/embeddings/__init__.py +317 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/storage/dual_manager.py +2 -11
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/storage/lancedb_storage.py +152 -33
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/pyproject.toml +11 -9
- abstractmemory-0.1.0/AbstractMemory.egg-info/requires.txt +0 -12
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/AbstractMemory.egg-info/dependency_links.txt +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/AbstractMemory.egg-info/top_level.txt +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/LICENSE +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/cognitive/__init__.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/components/__init__.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/components/core.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/components/episodic.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/components/semantic.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/components/working.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/core/__init__.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/core/interfaces.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/core/temporal.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/graph/__init__.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/graph/knowledge_graph.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/simple.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/storage/__init__.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/abstractmemory/storage/markdown_storage.py +0 -0
- {abstractmemory-0.1.0 → abstractmemory-0.2.2}/setup.cfg +0 -0
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: AbstractMemory
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Production-ready memory system for LLM agents - NO MOCKS, real semantic search, clear LLM vs embedding provider separation
|
|
5
5
|
Author-email: AbstractMemory Team <lpalbou@gmail.com>
|
|
6
6
|
Maintainer-email: AbstractMemory Team <palbou@gmail.com>
|
|
7
7
|
License-Expression: MIT
|
|
8
|
-
Project-URL: Homepage, https://github.com/lpalbou/
|
|
9
|
-
Project-URL: Documentation, https://github.com/lpalbou/
|
|
10
|
-
Project-URL: Repository, https://github.com/lpalbou/
|
|
11
|
-
Project-URL: Bug Reports, https://github.com/lpalbou/
|
|
12
|
-
Keywords: llm,memory,
|
|
8
|
+
Project-URL: Homepage, https://github.com/lpalbou/AbstractMemory
|
|
9
|
+
Project-URL: Documentation, https://github.com/lpalbou/AbstractMemory#readme
|
|
10
|
+
Project-URL: Repository, https://github.com/lpalbou/AbstractMemory
|
|
11
|
+
Project-URL: Bug Reports, https://github.com/lpalbou/AbstractMemory/issues
|
|
12
|
+
Keywords: llm,memory,semantic-search,embeddings,ai,agents,knowledge-graph,temporal,grounded-memory,vector-search
|
|
13
13
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -30,9 +30,15 @@ Requires-Dist: pytest; extra == "dev"
|
|
|
30
30
|
Requires-Dist: black; extra == "dev"
|
|
31
31
|
Requires-Dist: mypy; extra == "dev"
|
|
32
32
|
Provides-Extra: llm
|
|
33
|
-
Requires-Dist:
|
|
33
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "llm"
|
|
34
|
+
Provides-Extra: embeddings
|
|
35
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "embeddings"
|
|
36
|
+
Requires-Dist: lancedb>=0.6.0; extra == "embeddings"
|
|
34
37
|
Provides-Extra: storage
|
|
35
|
-
Requires-Dist: lancedb; extra == "storage"
|
|
38
|
+
Requires-Dist: lancedb>=0.6.0; extra == "storage"
|
|
39
|
+
Provides-Extra: all
|
|
40
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "all"
|
|
41
|
+
Requires-Dist: lancedb>=0.6.0; extra == "all"
|
|
36
42
|
Dynamic: license-file
|
|
37
43
|
|
|
38
44
|
# AbstractMemory
|
|
@@ -199,20 +205,27 @@ memory = create_memory(
|
|
|
199
205
|
|
|
200
206
|
## 📚 Documentation
|
|
201
207
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
- **[
|
|
206
|
-
- **[
|
|
207
|
-
- **[
|
|
208
|
-
- **[
|
|
208
|
+
**👉 [START HERE: Complete Documentation Guide](docs/README.md)**
|
|
209
|
+
|
|
210
|
+
### Core Guides
|
|
211
|
+
- **[🚀 Quick Start](docs/README.md#-start-here)** - Get running in 5 minutes
|
|
212
|
+
- **[🔍 Semantic Search](docs/semantic-search.md)** - Vector embeddings and similarity search
|
|
213
|
+
- **[🧠 Memory Types](docs/memory-types.md)** - ScratchpadMemory, BufferMemory, GroundedMemory
|
|
214
|
+
- **[📊 Performance Guide](docs/semantic-search.md#performance-characteristics)** - Embedding timing and optimization
|
|
215
|
+
|
|
216
|
+
### Advanced Topics
|
|
217
|
+
- **[🏗️ Architecture](docs/architecture.md)** - System design and two-tier strategy
|
|
218
|
+
- **[💾 Storage Systems](docs/storage-systems.md)** - Markdown + LanceDB dual storage
|
|
219
|
+
- **[🎯 Usage Patterns](docs/usage-patterns.md)** - Real-world examples and best practices
|
|
220
|
+
- **[🔗 Integration Guide](docs/integration.md)** - AbstractLLM ecosystem integration
|
|
221
|
+
- **[📖 API Reference](docs/api-reference.md)** - Complete method documentation
|
|
209
222
|
|
|
210
223
|
## 🔬 Key Features
|
|
211
224
|
|
|
212
225
|
### ✅ Purpose-Built Memory Types
|
|
213
|
-
- **ScratchpadMemory**: ReAct thought-action-observation cycles
|
|
214
|
-
- **BufferMemory**: Simple conversation history
|
|
215
|
-
- **GroundedMemory**:
|
|
226
|
+
- **ScratchpadMemory**: ReAct thought-action-observation cycles for task agents
|
|
227
|
+
- **BufferMemory**: Simple conversation history with capacity limits
|
|
228
|
+
- **GroundedMemory**: Four-tier architecture with semantic search and temporal context
|
|
216
229
|
|
|
217
230
|
### ✅ State-of-the-Art Research Integration
|
|
218
231
|
- **MemGPT/Letta Pattern**: Self-editing core memory
|
|
@@ -238,12 +251,19 @@ Core Memory ──→ Semantic Memory ──→ Working Memory ──→ Episodi
|
|
|
238
251
|
- **🔗 Bidirectional Links**: Connect interactions to AI insights
|
|
239
252
|
- **📊 Search Capabilities**: Text-based and semantic similarity search
|
|
240
253
|
|
|
254
|
+
### ✅ Semantic Search with AbstractCore
|
|
255
|
+
- **🎯 Real Embeddings**: Uses AbstractCore's EmbeddingManager with Google's EmbeddingGemma (768D)
|
|
256
|
+
- **⚡ Immediate Indexing**: Embeddings generated instantly during `add_interaction()` (~36ms)
|
|
257
|
+
- **🔍 Vector Similarity**: True semantic search finds contextually relevant content
|
|
258
|
+
- **🗄️ Dual Storage**: Observable markdown files + searchable LanceDB vectors
|
|
259
|
+
- **🎯 Production Ready**: Sub-second search, proven with 200+ real implementation tests
|
|
260
|
+
|
|
241
261
|
## 🧪 Testing & Validation
|
|
242
262
|
|
|
243
|
-
AbstractMemory includes **200+ comprehensive tests**
|
|
263
|
+
AbstractMemory includes **200+ comprehensive tests** using ONLY real implementations:
|
|
244
264
|
|
|
245
265
|
```bash
|
|
246
|
-
# Run all tests
|
|
266
|
+
# Run all tests (NO MOCKS - only real implementations)
|
|
247
267
|
python -m pytest tests/ -v
|
|
248
268
|
|
|
249
269
|
# Run specific test suites
|
|
@@ -255,41 +275,150 @@ python -m pytest tests/integration/ -v # Full system integration
|
|
|
255
275
|
# Test with real LLM providers (requires AbstractCore)
|
|
256
276
|
python -m pytest tests/integration/test_llm_real_usage.py -v
|
|
257
277
|
|
|
258
|
-
# Test comprehensive dual storage
|
|
278
|
+
# Test comprehensive dual storage with real embeddings
|
|
259
279
|
python -m pytest tests/storage/test_dual_storage_comprehensive.py -v
|
|
260
280
|
```
|
|
261
281
|
|
|
282
|
+
**IMPORTANT**: All tests use real implementations:
|
|
283
|
+
- Real embedding providers (AbstractCore EmbeddingManager)
|
|
284
|
+
- Real LLM providers (Anthropic, OpenAI, Ollama via AbstractCore)
|
|
285
|
+
- Real memory components and storage systems
|
|
286
|
+
- NO MOCKS anywhere in the codebase
|
|
287
|
+
|
|
288
|
+
## 🚀 Quick Start
|
|
289
|
+
|
|
290
|
+
### Installation
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
# Install with semantic search capabilities (recommended)
|
|
294
|
+
pip install abstractmemory[embeddings]
|
|
295
|
+
|
|
296
|
+
# Or install everything
|
|
297
|
+
pip install abstractmemory[all]
|
|
298
|
+
|
|
299
|
+
# Basic memory only (no semantic search)
|
|
300
|
+
pip install abstractmemory
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### 📋 Upgrading from v0.1.0?
|
|
304
|
+
|
|
305
|
+
**Version 0.2.0 adds semantic search!** See [Migration Guide](CHANGELOG.md#-migration-guide) for:
|
|
306
|
+
- New AbstractCore dependency (`pip install abstractcore>=2.1.0`)
|
|
307
|
+
- LanceDB schema changes (recreate `.db` files)
|
|
308
|
+
- New `embedding_provider` parameter
|
|
309
|
+
|
|
310
|
+
### ⚠️ Critical: LLM vs Embedding Provider Separation
|
|
311
|
+
|
|
312
|
+
**Understanding the difference between LLM and Embedding providers:**
|
|
313
|
+
|
|
314
|
+
- 🔄 **LLM Providers** (text generation): Change freely between Anthropic, OpenAI, Ollama, etc.
|
|
315
|
+
- 🔒 **Embedding Providers** (semantic search): Must remain consistent within a storage space
|
|
316
|
+
|
|
317
|
+
**For semantic search consistency:**
|
|
318
|
+
- ✅ **Choose ONE embedding model and stick with it per storage space**
|
|
319
|
+
- ✅ **You can customize which embedding model to use (AbstractCore, OpenAI, Ollama, etc.)**
|
|
320
|
+
- ❌ **Don't change embedding models mid-project - it breaks vector search**
|
|
321
|
+
- 🚨 **AbstractMemory automatically warns when embedding model changes detected**
|
|
322
|
+
|
|
323
|
+
**Example of correct separation:**
|
|
324
|
+
```python
|
|
325
|
+
# LLM for text generation (can change anytime)
|
|
326
|
+
llm = create_llm("anthropic") # or "openai", "ollama", etc.
|
|
327
|
+
|
|
328
|
+
# Dedicated embedding provider (must stay consistent)
|
|
329
|
+
embedder = EmbeddingManager() # AbstractCore embeddings
|
|
330
|
+
|
|
331
|
+
memory = create_memory("grounded", embedding_provider=embedder) # NOT llm!
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
### Basic Usage
|
|
335
|
+
|
|
336
|
+
```python
|
|
337
|
+
from abstractllm.embeddings import EmbeddingManager
|
|
338
|
+
from abstractmemory import create_memory
|
|
339
|
+
|
|
340
|
+
# 1. Create embedding manager for semantic search
|
|
341
|
+
em = EmbeddingManager() # Uses EmbeddingGemma (768D vectors)
|
|
342
|
+
|
|
343
|
+
# 2. Create memory with dual storage
|
|
344
|
+
memory = create_memory(
|
|
345
|
+
"grounded",
|
|
346
|
+
storage_backend="dual", # Markdown + LanceDB
|
|
347
|
+
storage_path="./memory_files", # Observable files
|
|
348
|
+
storage_uri="./memory.db", # Vector search
|
|
349
|
+
embedding_provider=em # Real embeddings
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
# 3. Add interactions (embeddings generated immediately!)
|
|
353
|
+
memory.set_current_user("alice")
|
|
354
|
+
memory.add_interaction(
|
|
355
|
+
"I'm working on machine learning projects",
|
|
356
|
+
"Great! ML has amazing applications in many fields."
|
|
357
|
+
)
|
|
358
|
+
# ↳ Takes ~36ms: embedding generated and stored instantly
|
|
359
|
+
|
|
360
|
+
# 4. Semantic search finds contextually relevant content
|
|
361
|
+
results = memory.search_stored_interactions("artificial intelligence research")
|
|
362
|
+
# ↳ Finds ML interaction via semantic similarity (not keywords!)
|
|
363
|
+
print(f"Found {len(results)} relevant conversations")
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
### 📋 What Happens When You Add Interactions
|
|
367
|
+
|
|
368
|
+
```python
|
|
369
|
+
memory.add_interaction("I love Python", "Great choice!")
|
|
370
|
+
# ↓ IMMEDIATE PROCESSING:
|
|
371
|
+
# 1. Text combined: "I love Python Great choice!"
|
|
372
|
+
# 2. EmbeddingManager.embed() called (36ms)
|
|
373
|
+
# 3. 768D vector generated with EmbeddingGemma
|
|
374
|
+
# 4. Saved to markdown file: ./memory_files/verbatim/alice/...
|
|
375
|
+
# 5. Stored in LanceDB: vector + text + metadata
|
|
376
|
+
# 6. Interaction immediately searchable via semantic similarity
|
|
377
|
+
```
|
|
378
|
+
|
|
262
379
|
## 🔗 AbstractLLM Ecosystem Integration
|
|
263
380
|
|
|
264
|
-
AbstractMemory seamlessly integrates with
|
|
381
|
+
AbstractMemory seamlessly integrates with AbstractCore, maintaining clear separation between LLM and embedding providers:
|
|
265
382
|
|
|
266
|
-
###
|
|
383
|
+
### Critical Architecture: LLM vs Embedding Separation
|
|
267
384
|
```python
|
|
268
385
|
from abstractllm import create_llm
|
|
386
|
+
from abstractllm.embeddings import EmbeddingManager
|
|
269
387
|
from abstractmemory import create_memory
|
|
270
388
|
|
|
271
|
-
#
|
|
272
|
-
|
|
389
|
+
# SEPARATE PROVIDERS for different purposes:
|
|
390
|
+
|
|
391
|
+
# 1. LLM Provider - for TEXT GENERATION (can change freely)
|
|
392
|
+
llm_provider = create_llm("anthropic", model="claude-3-5-haiku-latest")
|
|
393
|
+
|
|
394
|
+
# 2. Embedding Provider - for SEMANTIC SEARCH (must stay consistent)
|
|
395
|
+
embedding_provider = EmbeddingManager()
|
|
273
396
|
|
|
274
|
-
# Create memory with embedding
|
|
397
|
+
# Create memory with DEDICATED embedding provider
|
|
275
398
|
memory = create_memory(
|
|
276
399
|
"grounded",
|
|
277
400
|
enable_kg=True,
|
|
278
401
|
storage_backend="dual",
|
|
279
402
|
storage_path="./memory",
|
|
280
403
|
storage_uri="./memory.db",
|
|
281
|
-
embedding_provider=
|
|
404
|
+
embedding_provider=embedding_provider # DEDICATED for embeddings
|
|
282
405
|
)
|
|
283
406
|
|
|
284
|
-
# Use
|
|
407
|
+
# Use in agent reasoning with CLEAR separation
|
|
285
408
|
context = memory.get_full_context(query)
|
|
286
|
-
response =
|
|
287
|
-
memory.add_interaction(query, response.content)
|
|
409
|
+
response = llm_provider.generate(prompt, system_prompt=context) # LLM for text
|
|
410
|
+
memory.add_interaction(query, response.content) # Embeddings handled internally
|
|
288
411
|
|
|
289
|
-
# Search
|
|
412
|
+
# Search uses embedding provider for semantic similarity
|
|
290
413
|
similar_memories = memory.search_stored_interactions("related concepts")
|
|
291
414
|
```
|
|
292
415
|
|
|
416
|
+
### Key Points:
|
|
417
|
+
- **LLM Provider**: Change freely between Anthropic ↔ OpenAI ↔ Ollama
|
|
418
|
+
- **Embedding Provider**: Must remain consistent within storage space
|
|
419
|
+
- **Never** pass LLM provider as embedding provider
|
|
420
|
+
- **Always** use dedicated embedding provider for semantic search
|
|
421
|
+
|
|
293
422
|
### With AbstractAgent (Future)
|
|
294
423
|
```python
|
|
295
424
|
from abstractagent import create_agent
|
|
@@ -306,7 +435,7 @@ response = agent.execute(task, user_id="alice")
|
|
|
306
435
|
## 🏛️ Architecture Principles
|
|
307
436
|
|
|
308
437
|
1. **No Over-Engineering**: Memory complexity matches agent requirements
|
|
309
|
-
2. **Real Implementation Testing**:
|
|
438
|
+
2. **Real Implementation Testing**: NO MOCKS anywhere - all tests use real implementations
|
|
310
439
|
3. **SOTA Research Foundation**: Built on proven patterns (MemGPT, Zep, Graphiti)
|
|
311
440
|
4. **Clean Abstractions**: Simple interfaces, powerful implementations
|
|
312
441
|
5. **Performance Optimized**: Fast operations for simple agents, scalable for complex ones
|
|
@@ -17,6 +17,7 @@ abstractmemory/components/working.py
|
|
|
17
17
|
abstractmemory/core/__init__.py
|
|
18
18
|
abstractmemory/core/interfaces.py
|
|
19
19
|
abstractmemory/core/temporal.py
|
|
20
|
+
abstractmemory/embeddings/__init__.py
|
|
20
21
|
abstractmemory/graph/__init__.py
|
|
21
22
|
abstractmemory/graph/knowledge_graph.py
|
|
22
23
|
abstractmemory/storage/__init__.py
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: AbstractMemory
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Production-ready memory system for LLM agents - NO MOCKS, real semantic search, clear LLM vs embedding provider separation
|
|
5
5
|
Author-email: AbstractMemory Team <lpalbou@gmail.com>
|
|
6
6
|
Maintainer-email: AbstractMemory Team <palbou@gmail.com>
|
|
7
7
|
License-Expression: MIT
|
|
8
|
-
Project-URL: Homepage, https://github.com/lpalbou/
|
|
9
|
-
Project-URL: Documentation, https://github.com/lpalbou/
|
|
10
|
-
Project-URL: Repository, https://github.com/lpalbou/
|
|
11
|
-
Project-URL: Bug Reports, https://github.com/lpalbou/
|
|
12
|
-
Keywords: llm,memory,
|
|
8
|
+
Project-URL: Homepage, https://github.com/lpalbou/AbstractMemory
|
|
9
|
+
Project-URL: Documentation, https://github.com/lpalbou/AbstractMemory#readme
|
|
10
|
+
Project-URL: Repository, https://github.com/lpalbou/AbstractMemory
|
|
11
|
+
Project-URL: Bug Reports, https://github.com/lpalbou/AbstractMemory/issues
|
|
12
|
+
Keywords: llm,memory,semantic-search,embeddings,ai,agents,knowledge-graph,temporal,grounded-memory,vector-search
|
|
13
13
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -30,9 +30,15 @@ Requires-Dist: pytest; extra == "dev"
|
|
|
30
30
|
Requires-Dist: black; extra == "dev"
|
|
31
31
|
Requires-Dist: mypy; extra == "dev"
|
|
32
32
|
Provides-Extra: llm
|
|
33
|
-
Requires-Dist:
|
|
33
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "llm"
|
|
34
|
+
Provides-Extra: embeddings
|
|
35
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "embeddings"
|
|
36
|
+
Requires-Dist: lancedb>=0.6.0; extra == "embeddings"
|
|
34
37
|
Provides-Extra: storage
|
|
35
|
-
Requires-Dist: lancedb; extra == "storage"
|
|
38
|
+
Requires-Dist: lancedb>=0.6.0; extra == "storage"
|
|
39
|
+
Provides-Extra: all
|
|
40
|
+
Requires-Dist: abstractcore>=2.1.0; extra == "all"
|
|
41
|
+
Requires-Dist: lancedb>=0.6.0; extra == "all"
|
|
36
42
|
Dynamic: license-file
|
|
37
43
|
|
|
38
44
|
# AbstractMemory
|
|
@@ -199,20 +205,27 @@ memory = create_memory(
|
|
|
199
205
|
|
|
200
206
|
## 📚 Documentation
|
|
201
207
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
- **[
|
|
206
|
-
- **[
|
|
207
|
-
- **[
|
|
208
|
-
- **[
|
|
208
|
+
**👉 [START HERE: Complete Documentation Guide](docs/README.md)**
|
|
209
|
+
|
|
210
|
+
### Core Guides
|
|
211
|
+
- **[🚀 Quick Start](docs/README.md#-start-here)** - Get running in 5 minutes
|
|
212
|
+
- **[🔍 Semantic Search](docs/semantic-search.md)** - Vector embeddings and similarity search
|
|
213
|
+
- **[🧠 Memory Types](docs/memory-types.md)** - ScratchpadMemory, BufferMemory, GroundedMemory
|
|
214
|
+
- **[📊 Performance Guide](docs/semantic-search.md#performance-characteristics)** - Embedding timing and optimization
|
|
215
|
+
|
|
216
|
+
### Advanced Topics
|
|
217
|
+
- **[🏗️ Architecture](docs/architecture.md)** - System design and two-tier strategy
|
|
218
|
+
- **[💾 Storage Systems](docs/storage-systems.md)** - Markdown + LanceDB dual storage
|
|
219
|
+
- **[🎯 Usage Patterns](docs/usage-patterns.md)** - Real-world examples and best practices
|
|
220
|
+
- **[🔗 Integration Guide](docs/integration.md)** - AbstractLLM ecosystem integration
|
|
221
|
+
- **[📖 API Reference](docs/api-reference.md)** - Complete method documentation
|
|
209
222
|
|
|
210
223
|
## 🔬 Key Features
|
|
211
224
|
|
|
212
225
|
### ✅ Purpose-Built Memory Types
|
|
213
|
-
- **ScratchpadMemory**: ReAct thought-action-observation cycles
|
|
214
|
-
- **BufferMemory**: Simple conversation history
|
|
215
|
-
- **GroundedMemory**:
|
|
226
|
+
- **ScratchpadMemory**: ReAct thought-action-observation cycles for task agents
|
|
227
|
+
- **BufferMemory**: Simple conversation history with capacity limits
|
|
228
|
+
- **GroundedMemory**: Four-tier architecture with semantic search and temporal context
|
|
216
229
|
|
|
217
230
|
### ✅ State-of-the-Art Research Integration
|
|
218
231
|
- **MemGPT/Letta Pattern**: Self-editing core memory
|
|
@@ -238,12 +251,19 @@ Core Memory ──→ Semantic Memory ──→ Working Memory ──→ Episodi
|
|
|
238
251
|
- **🔗 Bidirectional Links**: Connect interactions to AI insights
|
|
239
252
|
- **📊 Search Capabilities**: Text-based and semantic similarity search
|
|
240
253
|
|
|
254
|
+
### ✅ Semantic Search with AbstractCore
|
|
255
|
+
- **🎯 Real Embeddings**: Uses AbstractCore's EmbeddingManager with Google's EmbeddingGemma (768D)
|
|
256
|
+
- **⚡ Immediate Indexing**: Embeddings generated instantly during `add_interaction()` (~36ms)
|
|
257
|
+
- **🔍 Vector Similarity**: True semantic search finds contextually relevant content
|
|
258
|
+
- **🗄️ Dual Storage**: Observable markdown files + searchable LanceDB vectors
|
|
259
|
+
- **🎯 Production Ready**: Sub-second search, proven with 200+ real implementation tests
|
|
260
|
+
|
|
241
261
|
## 🧪 Testing & Validation
|
|
242
262
|
|
|
243
|
-
AbstractMemory includes **200+ comprehensive tests**
|
|
263
|
+
AbstractMemory includes **200+ comprehensive tests** using ONLY real implementations:
|
|
244
264
|
|
|
245
265
|
```bash
|
|
246
|
-
# Run all tests
|
|
266
|
+
# Run all tests (NO MOCKS - only real implementations)
|
|
247
267
|
python -m pytest tests/ -v
|
|
248
268
|
|
|
249
269
|
# Run specific test suites
|
|
@@ -255,41 +275,150 @@ python -m pytest tests/integration/ -v # Full system integration
|
|
|
255
275
|
# Test with real LLM providers (requires AbstractCore)
|
|
256
276
|
python -m pytest tests/integration/test_llm_real_usage.py -v
|
|
257
277
|
|
|
258
|
-
# Test comprehensive dual storage
|
|
278
|
+
# Test comprehensive dual storage with real embeddings
|
|
259
279
|
python -m pytest tests/storage/test_dual_storage_comprehensive.py -v
|
|
260
280
|
```
|
|
261
281
|
|
|
282
|
+
**IMPORTANT**: All tests use real implementations:
|
|
283
|
+
- Real embedding providers (AbstractCore EmbeddingManager)
|
|
284
|
+
- Real LLM providers (Anthropic, OpenAI, Ollama via AbstractCore)
|
|
285
|
+
- Real memory components and storage systems
|
|
286
|
+
- NO MOCKS anywhere in the codebase
|
|
287
|
+
|
|
288
|
+
## 🚀 Quick Start
|
|
289
|
+
|
|
290
|
+
### Installation
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
# Install with semantic search capabilities (recommended)
|
|
294
|
+
pip install abstractmemory[embeddings]
|
|
295
|
+
|
|
296
|
+
# Or install everything
|
|
297
|
+
pip install abstractmemory[all]
|
|
298
|
+
|
|
299
|
+
# Basic memory only (no semantic search)
|
|
300
|
+
pip install abstractmemory
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### 📋 Upgrading from v0.1.0?
|
|
304
|
+
|
|
305
|
+
**Version 0.2.0 adds semantic search!** See [Migration Guide](CHANGELOG.md#-migration-guide) for:
|
|
306
|
+
- New AbstractCore dependency (`pip install abstractcore>=2.1.0`)
|
|
307
|
+
- LanceDB schema changes (recreate `.db` files)
|
|
308
|
+
- New `embedding_provider` parameter
|
|
309
|
+
|
|
310
|
+
### ⚠️ Critical: LLM vs Embedding Provider Separation
|
|
311
|
+
|
|
312
|
+
**Understanding the difference between LLM and Embedding providers:**
|
|
313
|
+
|
|
314
|
+
- 🔄 **LLM Providers** (text generation): Change freely between Anthropic, OpenAI, Ollama, etc.
|
|
315
|
+
- 🔒 **Embedding Providers** (semantic search): Must remain consistent within a storage space
|
|
316
|
+
|
|
317
|
+
**For semantic search consistency:**
|
|
318
|
+
- ✅ **Choose ONE embedding model and stick with it per storage space**
|
|
319
|
+
- ✅ **You can customize which embedding model to use (AbstractCore, OpenAI, Ollama, etc.)**
|
|
320
|
+
- ❌ **Don't change embedding models mid-project - it breaks vector search**
|
|
321
|
+
- 🚨 **AbstractMemory automatically warns when embedding model changes detected**
|
|
322
|
+
|
|
323
|
+
**Example of correct separation:**
|
|
324
|
+
```python
|
|
325
|
+
# LLM for text generation (can change anytime)
|
|
326
|
+
llm = create_llm("anthropic") # or "openai", "ollama", etc.
|
|
327
|
+
|
|
328
|
+
# Dedicated embedding provider (must stay consistent)
|
|
329
|
+
embedder = EmbeddingManager() # AbstractCore embeddings
|
|
330
|
+
|
|
331
|
+
memory = create_memory("grounded", embedding_provider=embedder) # NOT llm!
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
### Basic Usage
|
|
335
|
+
|
|
336
|
+
```python
|
|
337
|
+
from abstractllm.embeddings import EmbeddingManager
|
|
338
|
+
from abstractmemory import create_memory
|
|
339
|
+
|
|
340
|
+
# 1. Create embedding manager for semantic search
|
|
341
|
+
em = EmbeddingManager() # Uses EmbeddingGemma (768D vectors)
|
|
342
|
+
|
|
343
|
+
# 2. Create memory with dual storage
|
|
344
|
+
memory = create_memory(
|
|
345
|
+
"grounded",
|
|
346
|
+
storage_backend="dual", # Markdown + LanceDB
|
|
347
|
+
storage_path="./memory_files", # Observable files
|
|
348
|
+
storage_uri="./memory.db", # Vector search
|
|
349
|
+
embedding_provider=em # Real embeddings
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
# 3. Add interactions (embeddings generated immediately!)
|
|
353
|
+
memory.set_current_user("alice")
|
|
354
|
+
memory.add_interaction(
|
|
355
|
+
"I'm working on machine learning projects",
|
|
356
|
+
"Great! ML has amazing applications in many fields."
|
|
357
|
+
)
|
|
358
|
+
# ↳ Takes ~36ms: embedding generated and stored instantly
|
|
359
|
+
|
|
360
|
+
# 4. Semantic search finds contextually relevant content
|
|
361
|
+
results = memory.search_stored_interactions("artificial intelligence research")
|
|
362
|
+
# ↳ Finds ML interaction via semantic similarity (not keywords!)
|
|
363
|
+
print(f"Found {len(results)} relevant conversations")
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
### 📋 What Happens When You Add Interactions
|
|
367
|
+
|
|
368
|
+
```python
|
|
369
|
+
memory.add_interaction("I love Python", "Great choice!")
|
|
370
|
+
# ↓ IMMEDIATE PROCESSING:
|
|
371
|
+
# 1. Text combined: "I love Python Great choice!"
|
|
372
|
+
# 2. EmbeddingManager.embed() called (36ms)
|
|
373
|
+
# 3. 768D vector generated with EmbeddingGemma
|
|
374
|
+
# 4. Saved to markdown file: ./memory_files/verbatim/alice/...
|
|
375
|
+
# 5. Stored in LanceDB: vector + text + metadata
|
|
376
|
+
# 6. Interaction immediately searchable via semantic similarity
|
|
377
|
+
```
|
|
378
|
+
|
|
262
379
|
## 🔗 AbstractLLM Ecosystem Integration
|
|
263
380
|
|
|
264
|
-
AbstractMemory seamlessly integrates with
|
|
381
|
+
AbstractMemory seamlessly integrates with AbstractCore, maintaining clear separation between LLM and embedding providers:
|
|
265
382
|
|
|
266
|
-
###
|
|
383
|
+
### Critical Architecture: LLM vs Embedding Separation
|
|
267
384
|
```python
|
|
268
385
|
from abstractllm import create_llm
|
|
386
|
+
from abstractllm.embeddings import EmbeddingManager
|
|
269
387
|
from abstractmemory import create_memory
|
|
270
388
|
|
|
271
|
-
#
|
|
272
|
-
|
|
389
|
+
# SEPARATE PROVIDERS for different purposes:
|
|
390
|
+
|
|
391
|
+
# 1. LLM Provider - for TEXT GENERATION (can change freely)
|
|
392
|
+
llm_provider = create_llm("anthropic", model="claude-3-5-haiku-latest")
|
|
393
|
+
|
|
394
|
+
# 2. Embedding Provider - for SEMANTIC SEARCH (must stay consistent)
|
|
395
|
+
embedding_provider = EmbeddingManager()
|
|
273
396
|
|
|
274
|
-
# Create memory with embedding
|
|
397
|
+
# Create memory with DEDICATED embedding provider
|
|
275
398
|
memory = create_memory(
|
|
276
399
|
"grounded",
|
|
277
400
|
enable_kg=True,
|
|
278
401
|
storage_backend="dual",
|
|
279
402
|
storage_path="./memory",
|
|
280
403
|
storage_uri="./memory.db",
|
|
281
|
-
embedding_provider=
|
|
404
|
+
embedding_provider=embedding_provider # DEDICATED for embeddings
|
|
282
405
|
)
|
|
283
406
|
|
|
284
|
-
# Use
|
|
407
|
+
# Use in agent reasoning with CLEAR separation
|
|
285
408
|
context = memory.get_full_context(query)
|
|
286
|
-
response =
|
|
287
|
-
memory.add_interaction(query, response.content)
|
|
409
|
+
response = llm_provider.generate(prompt, system_prompt=context) # LLM for text
|
|
410
|
+
memory.add_interaction(query, response.content) # Embeddings handled internally
|
|
288
411
|
|
|
289
|
-
# Search
|
|
412
|
+
# Search uses embedding provider for semantic similarity
|
|
290
413
|
similar_memories = memory.search_stored_interactions("related concepts")
|
|
291
414
|
```
|
|
292
415
|
|
|
416
|
+
### Key Points:
|
|
417
|
+
- **LLM Provider**: Change freely between Anthropic ↔ OpenAI ↔ Ollama
|
|
418
|
+
- **Embedding Provider**: Must remain consistent within storage space
|
|
419
|
+
- **Never** pass LLM provider as embedding provider
|
|
420
|
+
- **Always** use dedicated embedding provider for semantic search
|
|
421
|
+
|
|
293
422
|
### With AbstractAgent (Future)
|
|
294
423
|
```python
|
|
295
424
|
from abstractagent import create_agent
|
|
@@ -306,7 +435,7 @@ response = agent.execute(task, user_id="alice")
|
|
|
306
435
|
## 🏛️ Architecture Principles
|
|
307
436
|
|
|
308
437
|
1. **No Over-Engineering**: Memory complexity matches agent requirements
|
|
309
|
-
2. **Real Implementation Testing**:
|
|
438
|
+
2. **Real Implementation Testing**: NO MOCKS anywhere - all tests use real implementations
|
|
310
439
|
3. **SOTA Research Foundation**: Built on proven patterns (MemGPT, Zep, Graphiti)
|
|
311
440
|
4. **Clean Abstractions**: Simple interfaces, powerful implementations
|
|
312
441
|
5. **Performance Optimized**: Fast operations for simple agents, scalable for complex ones
|