shama 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shama-0.1.0/.gitignore +37 -0
- shama-0.1.0/LICENSE +21 -0
- shama-0.1.0/MANIFEST.in +5 -0
- shama-0.1.0/PKG-INFO +597 -0
- shama-0.1.0/README.md +535 -0
- shama-0.1.0/pyproject.toml +99 -0
- shama-0.1.0/shama/__init__.py +33 -0
- shama-0.1.0/shama/audit/__init__.py +0 -0
- shama-0.1.0/shama/audit/logger.py +285 -0
- shama-0.1.0/shama/client.py +542 -0
- shama-0.1.0/shama/core/__init__.py +0 -0
- shama-0.1.0/shama/core/exceptions.py +51 -0
- shama-0.1.0/shama/core/interfaces.py +367 -0
- shama-0.1.0/shama/core/models.py +233 -0
- shama-0.1.0/shama/healing/__init__.py +0 -0
- shama-0.1.0/shama/healing/contradiction.py +179 -0
- shama-0.1.0/shama/healing/corrector.py +307 -0
- shama-0.1.0/shama/healing/decay.py +164 -0
- shama-0.1.0/shama/memory/__init__.py +0 -0
- shama-0.1.0/shama/memory/promoter.py +287 -0
- shama-0.1.0/shama/memory/retriever.py +207 -0
- shama-0.1.0/shama/memory/writer.py +248 -0
- shama-0.1.0/shama/providers/__init__.py +0 -0
- shama-0.1.0/shama/providers/embeddings.py +163 -0
- shama-0.1.0/shama/providers/huggingface.py +543 -0
- shama-0.1.0/shama/providers/llm.py +368 -0
- shama-0.1.0/shama/py.typed +0 -0
- shama-0.1.0/shama/scheduler/__init__.py +0 -0
- shama-0.1.0/shama/scheduler/tasks.py +257 -0
- shama-0.1.0/shama/stores/__init__.py +0 -0
- shama-0.1.0/shama/stores/cache/__init__.py +0 -0
- shama-0.1.0/shama/stores/cache/redis.py +94 -0
- shama-0.1.0/shama/stores/graph/__init__.py +0 -0
- shama-0.1.0/shama/stores/graph/neo4j.py +305 -0
- shama-0.1.0/shama/stores/vector/__init__.py +0 -0
- shama-0.1.0/shama/stores/vector/qdrant.py +434 -0
- shama-0.1.0/tests/__init__.py +0 -0
- shama-0.1.0/tests/conftest.py +19 -0
- shama-0.1.0/tests/smoke_test.py +93 -0
- shama-0.1.0/tests/test_integration.py +150 -0
- shama-0.1.0/tests/test_shama.py +734 -0
shama-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.pyo
|
|
5
|
+
*.pyd
|
|
6
|
+
*.egg
|
|
7
|
+
*.egg-info/
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
.eggs/
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# SHAMA runtime
|
|
16
|
+
shama_audit.db
|
|
17
|
+
*.db
|
|
18
|
+
|
|
19
|
+
# Environment
|
|
20
|
+
.env
|
|
21
|
+
|
|
22
|
+
# Testing
|
|
23
|
+
.pytest_cache/
|
|
24
|
+
.coverage
|
|
25
|
+
htmlcov/
|
|
26
|
+
|
|
27
|
+
# Type checking
|
|
28
|
+
.mypy_cache/
|
|
29
|
+
|
|
30
|
+
# IDE
|
|
31
|
+
.vscode/
|
|
32
|
+
.idea/
|
|
33
|
+
*.swp
|
|
34
|
+
|
|
35
|
+
# OS
|
|
36
|
+
.DS_Store
|
|
37
|
+
Thumbs.db
|
shama-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SHAMA - Self-Healing Agent Memory Architecture, an immune system for AI agent memory Architectured and developed by Gowtham Sai Yadlapalli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
shama-0.1.0/MANIFEST.in
ADDED
shama-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: shama
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Self-Healing Agent Memory Architecture — an immune system for AI agent memory
|
|
5
|
+
Project-URL: Homepage, https://github.com/gowthamsai09/shama
|
|
6
|
+
Project-URL: Documentation, https://github.com/gowthamsai09/shama#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/gowthamsai09/shama
|
|
8
|
+
Project-URL: Issues, https://github.com/gowthamsai09/shama/issues
|
|
9
|
+
Author-email: Gowtham Sai Yadlapalli <gowthamsaiyadlapalli1234@gmail.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,ai,knowledge-graph,llm,memory,rag,self-healing,vector-database
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: anyio>=4.3.0
|
|
22
|
+
Requires-Dist: celery>=5.3.0
|
|
23
|
+
Requires-Dist: neo4j>=5.20.0
|
|
24
|
+
Requires-Dist: pydantic>=2.6.0
|
|
25
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
26
|
+
Requires-Dist: qdrant-client>=1.9.0
|
|
27
|
+
Requires-Dist: redis>=5.0.0
|
|
28
|
+
Provides-Extra: all
|
|
29
|
+
Requires-Dist: anthropic>=0.28.0; extra == 'all'
|
|
30
|
+
Requires-Dist: cohere>=5.5.0; extra == 'all'
|
|
31
|
+
Requires-Dist: huggingface-hub>=0.23.0; extra == 'all'
|
|
32
|
+
Requires-Dist: openai>=1.30.0; extra == 'all'
|
|
33
|
+
Provides-Extra: anthropic
|
|
34
|
+
Requires-Dist: anthropic>=0.28.0; extra == 'anthropic'
|
|
35
|
+
Requires-Dist: openai>=1.30.0; extra == 'anthropic'
|
|
36
|
+
Provides-Extra: azure
|
|
37
|
+
Requires-Dist: openai>=1.30.0; extra == 'azure'
|
|
38
|
+
Provides-Extra: cohere
|
|
39
|
+
Requires-Dist: cohere>=5.5.0; extra == 'cohere'
|
|
40
|
+
Provides-Extra: deepseek
|
|
41
|
+
Requires-Dist: openai>=1.30.0; extra == 'deepseek'
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: httpx>=0.27.0; extra == 'dev'
|
|
44
|
+
Requires-Dist: mypy>=1.10.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: openai>=1.30.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
50
|
+
Provides-Extra: huggingface
|
|
51
|
+
Requires-Dist: huggingface-hub>=0.23.0; extra == 'huggingface'
|
|
52
|
+
Provides-Extra: huggingface-local
|
|
53
|
+
Requires-Dist: accelerate>=0.30.0; extra == 'huggingface-local'
|
|
54
|
+
Requires-Dist: huggingface-hub>=0.23.0; extra == 'huggingface-local'
|
|
55
|
+
Requires-Dist: sentence-transformers>=3.0.0; extra == 'huggingface-local'
|
|
56
|
+
Requires-Dist: sentencepiece>=0.2.0; extra == 'huggingface-local'
|
|
57
|
+
Requires-Dist: torch>=2.2.0; extra == 'huggingface-local'
|
|
58
|
+
Requires-Dist: transformers>=4.40.0; extra == 'huggingface-local'
|
|
59
|
+
Provides-Extra: openai
|
|
60
|
+
Requires-Dist: openai>=1.30.0; extra == 'openai'
|
|
61
|
+
Description-Content-Type: text/markdown
|
|
62
|
+
|
|
63
|
+
# SHAMA - Self-Healing Agent Memory Architecture
|
|
64
|
+
|
|
65
|
+
> An immune system for AI agent memory. Memories that know what they've forgotten - and fix it.
|
|
66
|
+
|
|
67
|
+
[](https://badge.fury.io/py/shama)
|
|
68
|
+
[](https://opensource.org/licenses/MIT)
|
|
69
|
+
[](https://www.python.org/downloads/)
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## The Problem
|
|
74
|
+
|
|
75
|
+
Every AI agent today loses context, hallucinates past events, or gets poisoned memory over long sessions. Existing solutions (conversation buffers, naive RAG) have no mechanism to detect stale facts, resolve contradictions, or autonomously correct errors.
|
|
76
|
+
|
|
77
|
+
## The Solution
|
|
78
|
+
|
|
79
|
+
SHAMA is a **drop-in memory layer** that gives your agent:
|
|
80
|
+
|
|
81
|
+
- **Dual memory store** - episodic (what happened) + semantic (what is true)
|
|
82
|
+
- **Confidence half-life decay** - `C(t) = C₀ × 2^(−t/τ)` - memories decay probabilistically over time
|
|
83
|
+
- **Autonomous contradiction detection** - scans for conflicting facts on every write, resolved by LLM judge
|
|
84
|
+
- **Self-correction loop** - re-verifies and deprecates stale/wrong memories automatically
|
|
85
|
+
- **Full audit trail** - every memory lifecycle event logged for complete data ownership
|
|
86
|
+
- **Swappable backends** - Qdrant, Neo4j, Redis by default; swap any component with one config change
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Table of Contents
|
|
91
|
+
|
|
92
|
+
- [SHAMA - Self-Healing Agent Memory Architecture](#shama--self-healing-agent-memory-architecture)
|
|
93
|
+
- [The Problem](#the-problem)
|
|
94
|
+
- [The Solution](#the-solution)
|
|
95
|
+
- [Table of Contents](#table-of-contents)
|
|
96
|
+
- [Architecture](#architecture)
|
|
97
|
+
- [Confidence Half-Life](#confidence-half-life)
|
|
98
|
+
- [Prerequisites](#prerequisites)
|
|
99
|
+
- [Step 1 - Get API Keys](#step-1--get-api-keys)
|
|
100
|
+
- [Embedding Key - OpenAI (required)](#embedding-key--openai-required)
|
|
101
|
+
- [LLM Key - DeepSeek (for reasoning, contradiction judging, promotion)](#llm-key--deepseek-for-reasoning-contradiction-judging-promotion)
|
|
102
|
+
- [Step 2 - Clone \& Install](#step-2--clone--install)
|
|
103
|
+
- [HuggingFace - Fully Local (no API keys, full privacy)](#huggingface--fully-local-no-api-keys-full-privacy)
|
|
104
|
+
- [Step 3 - Configure Environment](#step-3--configure-environment)
|
|
105
|
+
- [Step 4 - Start Infrastructure (Docker)](#step-4--start-infrastructure-docker)
|
|
106
|
+
- [Step 5 - Verify Infrastructure](#step-5--verify-infrastructure)
|
|
107
|
+
- [Qdrant](#qdrant)
|
|
108
|
+
- [Neo4j](#neo4j)
|
|
109
|
+
- [Redis](#redis)
|
|
110
|
+
- [All three via Python](#all-three-via-python)
|
|
111
|
+
- [Public API Reference](#public-api-reference)
|
|
112
|
+
- [Swappable Backends](#swappable-backends)
|
|
113
|
+
- [Provider Combinations](#provider-combinations)
|
|
114
|
+
- [Quick usage reference](#quick-usage-reference)
|
|
115
|
+
- [Background Scheduler](#background-scheduler)
|
|
116
|
+
- [License](#license)
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## Architecture
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
INPUT (text)
|
|
124
|
+
└► LLM.score_importance() ← how important is this memory?
|
|
125
|
+
└► Embedding.embed() ← convert to vector
|
|
126
|
+
└► EpisodicNode written ← append-only event log (Qdrant)
|
|
127
|
+
└► Redis working memory updated ← last 20 turns cached per session
|
|
128
|
+
└► Audit event logged ← immutable SQLite trail
|
|
129
|
+
|
|
130
|
+
PROMOTION JOB (every 60 min)
|
|
131
|
+
└► Fetch unpromoted episodic nodes
|
|
132
|
+
└► Cluster by cosine similarity (threshold 0.80)
|
|
133
|
+
└► LLM distills cluster → entity-relation-value triples
|
|
134
|
+
└► SemanticNode written ← knowledge graph (Qdrant + Neo4j)
|
|
135
|
+
└► Episodic nodes marked promoted
|
|
136
|
+
|
|
137
|
+
CONTRADICTION SCAN (every semantic write)
|
|
138
|
+
└► Find nodes with same entity + relation, different value
|
|
139
|
+
└► LLM judge: is_contradiction? winner?
|
|
140
|
+
└► CONFLICTS_WITH edge added in Neo4j
|
|
141
|
+
└► Both nodes → status = CONTESTED
|
|
142
|
+
└► SelfCorrector: winner → ACTIVE, loser → DEPRECATED
|
|
143
|
+
|
|
144
|
+
DECAY SCHEDULER (every 15 min)
|
|
145
|
+
└► Scan nodes below confidence threshold 0.30
|
|
146
|
+
└► C(t) = C₀ × 2^(−t/τ)
|
|
147
|
+
└► confidence < 0.10 → auto-deprecate
|
|
148
|
+
└► 0.10 < confidence < 0.30 → re-verify via LLM
|
|
149
|
+
└► confirmed → confidence restored, status ACTIVE
|
|
150
|
+
└► refuted → status DEPRECATED
|
|
151
|
+
└► uncertain → status CONTESTED, escalated
|
|
152
|
+
|
|
153
|
+
RECALL (query string)
|
|
154
|
+
└► Embed query
|
|
155
|
+
└► ANN search: top-10 episodic + top-10 semantic (Qdrant)
|
|
156
|
+
└► Graph hop: neighbors of top-3 semantic hits (Neo4j, 1-2 hops)
|
|
157
|
+
└► Merge + deduplicate
|
|
158
|
+
└► Re-rank: score = relevance×0.5 + confidence×0.3 + recency×0.2
|
|
159
|
+
└► Filter: confidence >= 0.15
|
|
160
|
+
└► Trim to 4000 token budget
|
|
161
|
+
└► Return RetrievedContext with confidence-annotated memories
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Confidence Half-Life
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
C(t) = C₀ × 2^(−t/τ)
|
|
168
|
+
|
|
169
|
+
C₀ = original confidence at write time (1.0)
|
|
170
|
+
t = hours elapsed since creation
|
|
171
|
+
τ = half-life in hours (per memory type)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
| Memory type | Half-life (τ) | After 1 half-life | After 2 half-lives |
|
|
175
|
+
|--------------------------|--------------------|-------------------|--------------------|
|
|
176
|
+
| Conversational event | 24 hrs | 0.50 | 0.25 |
|
|
177
|
+
| Tool output / API result | 48 hrs | 0.50 | 0.25 |
|
|
178
|
+
| Distilled semantic fact | 720 hrs (30 days) | 0.50 | 0.25 |
|
|
179
|
+
| User preference | 2160 hrs (90 days) | 0.50 | 0.25 |
|
|
180
|
+
|
|
181
|
+
`C(t) < 0.30` → re-verify job fires
|
|
182
|
+
`C(t) < 0.10` → auto-deprecate
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Prerequisites
|
|
187
|
+
|
|
188
|
+
Before starting, make sure you have:
|
|
189
|
+
|
|
190
|
+
| Tool | Version| Install |
|
|
191
|
+
|----------------|--------|--------------------------------------------|
|
|
192
|
+
| Python | 3.11+ | https://python.org |
|
|
193
|
+
| Docker Desktop | Latest | https://docker.com/products/docker-desktop |
|
|
194
|
+
| Git | Any | https://git-scm.com |
|
|
195
|
+
| pip | 23+ | comes with Python |
|
|
196
|
+
|
|
197
|
+
Check your versions:
|
|
198
|
+
```bash
|
|
199
|
+
python --version # must be 3.11+
|
|
200
|
+
docker --version # must be installed
|
|
201
|
+
docker compose version
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## Step 1 - Get API Keys
|
|
207
|
+
|
|
208
|
+
SHAMA uses **two separate API keys** - one for embeddings, one for LLM reasoning.
|
|
209
|
+
|
|
210
|
+
### Embedding Key - OpenAI (required)
|
|
211
|
+
|
|
212
|
+
SHAMA uses OpenAI for converting text to vectors. DeepSeek does not provide an embedding API, so OpenAI is required for embeddings even when using DeepSeek as the LLM.
|
|
213
|
+
|
|
214
|
+
1. Go to https://platform.openai.com/api-keys
|
|
215
|
+
2. Click **"Create new secret key"**
|
|
216
|
+
3. Name it `shama-embeddings`
|
|
217
|
+
4. Copy the key - it starts with `sk-...`
|
|
218
|
+
5. Make sure your account has billing enabled (embeddings are very cheap - ~$0.001 per 1000 chunks)
|
|
219
|
+
|
|
220
|
+
### LLM Key - DeepSeek (for reasoning, contradiction judging, promotion)
|
|
221
|
+
|
|
222
|
+
DeepSeek is the recommended LLM provider - significantly cheaper than GPT-4o with comparable reasoning quality.
|
|
223
|
+
|
|
224
|
+
1. Go to https://platform.deepseek.com
|
|
225
|
+
2. Sign up / log in
|
|
226
|
+
3. Go to **API Keys** → **Create API Key**
|
|
227
|
+
4. Name it `shama-llm`
|
|
228
|
+
5. Copy the key
|
|
229
|
+
6. Add credits (minimum $5 recommended for testing)
|
|
230
|
+
|
|
231
|
+
### HuggingFace - Fully Local (no API keys, full privacy) or use Hugging face free API
|
|
232
|
+
|
|
233
|
+
1. Get your token at https://huggingface.co/settings/tokens
|
|
234
|
+
2. sign up/ login
|
|
235
|
+
3. Go to **profile** → **API Keys** → **Create API Key**
|
|
236
|
+
4. Name it `shama-llm`
|
|
237
|
+
5. Copy the key
|
|
238
|
+
|
|
239
|
+
## For local usage
|
|
240
|
+
```python
|
|
241
|
+
# Runs entirely on your machine - zero API calls, zero cost after download
|
|
242
|
+
client = ShamaClient.from_config(
|
|
243
|
+
huggingface_local_llm_model="microsoft/Phi-3-mini-4k-instruct", # ~3.8GB
|
|
244
|
+
huggingface_local_embedding_model="BAAI/bge-base-en-v1.5", # ~440MB
|
|
245
|
+
huggingface_local_device="cpu", # or "cuda" / "mps" (Apple Silicon)
|
|
246
|
+
)
|
|
247
|
+
# First run downloads models. Subsequent runs use cache.
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
> **Using OpenAI for both?** You can use one OpenAI key for both embedding and LLM - just set `openai_api_key` and leave `deepseek_api_key` empty.
|
|
251
|
+
>
|
|
252
|
+
> **Using Anthropic?** Set `anthropic_api_key` + `embedding_api_key` (OpenAI key for embeddings).
|
|
253
|
+
>
|
|
254
|
+
> **Using HuggingFace API?** You can use one HuggingFace key for both embedding and LLM - just set `HUGGINGFACE_API_KEY` `HF_JUDGE_MODEL` `HF_FAST_MODEL`and `HF_EMBEDDING_MODEL`.
|
|
255
|
+
|
|
256
|
+
---
|
|
257
|
+
|
|
258
|
+
## Step 2 - Clone & Install
|
|
259
|
+
|
|
260
|
+
```bash
|
|
261
|
+
# Clone the repo (or unzip the package you received)
|
|
262
|
+
git clone https://github.com/gowthamsai09/shama
|
|
263
|
+
cd shama
|
|
264
|
+
|
|
265
|
+
# Create a virtual environment (strongly recommended)
|
|
266
|
+
python -m venv .venv
|
|
267
|
+
|
|
268
|
+
# Activate it
|
|
269
|
+
# macOS / Linux:
|
|
270
|
+
source .venv/bin/activate
|
|
271
|
+
# Windows:
|
|
272
|
+
.venv\Scripts\activate
|
|
273
|
+
|
|
274
|
+
# Install SHAMA with all dependencies for testing
|
|
275
|
+
pip install -e ".[dev,openai]"
|
|
276
|
+
```
|
|
277
|
+
```bash
|
|
278
|
+
pip install shama[huggingface-local]
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Verify installation:
|
|
282
|
+
```bash
|
|
283
|
+
python -c "import shama; print(shama.__version__)"
|
|
284
|
+
# Expected: 0.1.0
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Step 3 - Configure Environment
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
# Copy the example env file
|
|
293
|
+
cp .env
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
Open `.env` and fill in your values:
|
|
297
|
+
|
|
298
|
+
```env
|
|
299
|
+
# Infrastructure (Docker will handle these - leave as default)
|
|
300
|
+
QDRANT_URL=http://localhost:6333
|
|
301
|
+
NEO4J_URI=bolt://localhost:7687
|
|
302
|
+
NEO4J_USER=neo4j
|
|
303
|
+
NEO4J_PASSWORD=your password
|
|
304
|
+
REDIS_URL=redis://localhost:6379
|
|
305
|
+
SHAMA_AUDIT_DB_PATH=./shama_audit.db
|
|
306
|
+
|
|
307
|
+
# LLM Provider
|
|
308
|
+
# Option A: DeepSeek for LLM + OpenAI for embeddings (recommended - cheapest)
|
|
309
|
+
DEEPSEEK_API_KEY=your_deepseek_key_here
|
|
310
|
+
EMBEDDING_API_KEY=your_openai_key_here
|
|
311
|
+
|
|
312
|
+
# Option B: OpenAI for everything (simplest)
|
|
313
|
+
# OPENAI_API_KEY=sk-...
|
|
314
|
+
|
|
315
|
+
# Option C: Anthropic for LLM + OpenAI for embeddings
|
|
316
|
+
# ANTHROPIC_API_KEY=sk-ant-...
|
|
317
|
+
# EMBEDDING_API_KEY=sk-...
|
|
318
|
+
|
|
319
|
+
# Option D - HuggingFace Inference API (LLM + embeddings both from HF)
|
|
320
|
+
# HUGGINGFACE_API_KEY=hf_...
|
|
321
|
+
# HF_JUDGE_MODEL=mistralai/Mistral-7B-Instruct-v0.3
|
|
322
|
+
# HF_FAST_MODEL=mistralai/Mistral-7B-Instruct-v0.3
|
|
323
|
+
# HF_EMBEDDING_MODEL=BAAI/bge-large-en-v1.5
|
|
324
|
+
|
|
325
|
+
# Option E - Fully local (no API keys needed)
|
|
326
|
+
# HF_LOCAL_LLM_MODEL=microsoft/Phi-3-mini-4k-instruct
|
|
327
|
+
# HF_LOCAL_EMBEDDING_MODEL=BAAI/bge-base-en-v1.5
|
|
328
|
+
# HF_LOCAL_DEVICE=cpu
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
# Embedding Config
|
|
332
|
+
EMBEDDING_MODEL=text-embedding-3-small
|
|
333
|
+
EMBEDDING_DIMENSIONS=1536
|
|
334
|
+
|
|
335
|
+
# SHAMA Tuning (defaults are fine for testing)
|
|
336
|
+
SHAMA_REVERIFY_THRESHOLD=0.30
|
|
337
|
+
SHAMA_DEPRECATE_THRESHOLD=0.10
|
|
338
|
+
SHAMA_EPISODIC_HALF_LIFE=24.0
|
|
339
|
+
SHAMA_SEMANTIC_HALF_LIFE=720.0
|
|
340
|
+
SHAMA_MAX_CONTEXT_TOKENS=4000
|
|
341
|
+
SHAMA_DECAY_INTERVAL_MINUTES=15
|
|
342
|
+
SHAMA_PROMOTION_INTERVAL_MINUTES=60
|
|
343
|
+
|
|
344
|
+
# Recommended HuggingFace models
|
|
345
|
+
HUGGINGFACE_API_KEY=hf_your_token
|
|
346
|
+
HF_JUDGE_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
|
347
|
+
HF_FAST_MODEL=meta-llama/Meta-Llama-3-8B-Instruct
|
|
348
|
+
HF_EMBEDDING_MODEL=BAAI/bge-large-en-v1.5
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
> **Important:** Also update `NEO4J_PASSWORD` in `docker-compose.yml` to match your `.env`:
|
|
352
|
+
> ```yaml
|
|
353
|
+
> NEO4J_AUTH: neo4j/your password
|
|
354
|
+
> ```
|
|
355
|
+
|
|
356
|
+
---
|
|
357
|
+
|
|
358
|
+
## Step 4 - Start Infrastructure (Docker)
|
|
359
|
+
|
|
360
|
+
SHAMA needs three services running: Qdrant (vector DB), Neo4j (graph DB), Redis (cache).
|
|
361
|
+
Docker Compose starts all three with one command.
|
|
362
|
+
|
|
363
|
+
```bash
|
|
364
|
+
# Start all services in background
|
|
365
|
+
docker compose up -d
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
Expected output:
|
|
369
|
+
```
|
|
370
|
+
Container shama-qdrant Started
|
|
371
|
+
Container shama-neo4j Started
|
|
372
|
+
Container shama-redis Started
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
This downloads ~800MB of images on first run. Subsequent starts are instant.
|
|
376
|
+
|
|
377
|
+
---
|
|
378
|
+
|
|
379
|
+
## Step 5 - Verify Infrastructure
|
|
380
|
+
|
|
381
|
+
Run each check before proceeding:
|
|
382
|
+
|
|
383
|
+
### Qdrant
|
|
384
|
+
```bash
|
|
385
|
+
curl http://localhost:6333/health
|
|
386
|
+
# Expected: {"title":"qdrant - vector search engine","version":"..."}
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
### Neo4j
|
|
390
|
+
Open http://localhost:7474 in your browser.
|
|
391
|
+
- Username: `neo4j`
|
|
392
|
+
- Password: whatever you set in `.env` (e.g. `shama_2026`)
|
|
393
|
+
- You should see the Neo4j Browser UI.
|
|
394
|
+
|
|
395
|
+
### Redis
|
|
396
|
+
```bash
|
|
397
|
+
docker exec shama-redis redis-cli ping
|
|
398
|
+
# Expected: PONG
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
### All three via Python
|
|
402
|
+
```bash
|
|
403
|
+
python -c "
|
|
404
|
+
import asyncio
|
|
405
|
+
import os
|
|
406
|
+
from dotenv import load_dotenv
|
|
407
|
+
load_dotenv()
|
|
408
|
+
|
|
409
|
+
async def check():
|
|
410
|
+
from shama.stores.vector.qdrant import QdrantVectorStore
|
|
411
|
+
from shama.stores.cache.redis import RedisCacheStore
|
|
412
|
+
|
|
413
|
+
v = QdrantVectorStore(url=os.getenv('QDRANT_URL', 'http://localhost:6333'))
|
|
414
|
+
await v.initialize()
|
|
415
|
+
print('Qdrant:', await v.health_check())
|
|
416
|
+
|
|
417
|
+
r = RedisCacheStore(url=os.getenv('REDIS_URL', 'redis://localhost:6379'))
|
|
418
|
+
await r.initialize()
|
|
419
|
+
print('Redis: ', await r.health_check())
|
|
420
|
+
|
|
421
|
+
asyncio.run(check())
|
|
422
|
+
"
|
|
423
|
+
# Expected:
|
|
424
|
+
# Qdrant: True
|
|
425
|
+
# Redis: True
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
## Public API Reference
|
|
430
|
+
|
|
431
|
+
```python
|
|
432
|
+
from shama import ShamaClient
|
|
433
|
+
|
|
434
|
+
client = ShamaClient.from_config(...)
|
|
435
|
+
await client.initialize()
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
| Method | Parameters | Returns | Description |
|
|
439
|
+
|------------------------|-------------------------------------------------------------|-------------------|-------------------------------------------------|
|
|
440
|
+
| `remember()` | `content, agent_id, session_id, source, turn_index` | `EpisodicNode` | Write raw observation to episodic memory |
|
|
441
|
+
| `remember_fact()` | `entity, relation, value, agent_id, session_id, confidence` | `SemanticNode` | Write structured fact + auto contradiction scan |
|
|
442
|
+
| `recall()` | `query, agent_id, session_id, min_confidence, max_tokens` | `RetrievedContext`| Retrieve ranked memory context |
|
|
443
|
+
| `export_agent_data()` | `agent_id` | `dict` | Export all data as JSON (data portability) |
|
|
444
|
+
| `delete_agent_data()` | `agent_id` | `dict` | Hard delete all agent data (GDPR) |
|
|
445
|
+
| `get_audit_trail()` | `agent_id, event_types, since, limit` | `list[dict]` | Full audit history |
|
|
446
|
+
| `run_decay_pass()` | `agent_id` | `dict` | Manual decay trigger |
|
|
447
|
+
| `run_promotion_pass()` | `agent_id` | `dict` | Manual promotion trigger |
|
|
448
|
+
| `health_check()` | - | `dict[str, bool]` | All backend health status |
|
|
449
|
+
|
|
450
|
+
---
|
|
451
|
+
|
|
452
|
+
## Swappable Backends
|
|
453
|
+
|
|
454
|
+
Implement any interface from `shama.core.interfaces` and pass it to `from_components()`:
|
|
455
|
+
|
|
456
|
+
| Layer | Interface | Default | Swap to |
|
|
457
|
+
|------------|---------------------|-------------------|------------------------------|
|
|
458
|
+
| Vector DB | `VectorStore` | Qdrant | Pinecone, Weaviate, pgvector |
|
|
459
|
+
| Graph DB | `GraphStore` | Neo4j | Amazon Neptune, FalkorDB |
|
|
460
|
+
| Cache | `CacheStore` | Redis | DragonflyDB, Memcached |
|
|
461
|
+
| Embeddings | `EmbeddingProvider` | OpenAI | Cohere, local models |
|
|
462
|
+
| LLM | `LLMProvider` | DeepSeek / OpenAI | Any LLM |
|
|
463
|
+
| Audit | `AuditStore` | SQLite | PostgreSQL, ClickHouse |
|
|
464
|
+
|
|
465
|
+
```python
|
|
466
|
+
from shama import ShamaClient
|
|
467
|
+
from my_company.stores import MyPineconeStore
|
|
468
|
+
|
|
469
|
+
client = ShamaClient.from_components(
|
|
470
|
+
vector_store=MyPineconeStore(),
|
|
471
|
+
graph_store=...,
|
|
472
|
+
cache_store=...,
|
|
473
|
+
embedding_provider=...,
|
|
474
|
+
llm_provider=...,
|
|
475
|
+
audit_store=...,
|
|
476
|
+
)
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
---
|
|
480
|
+
|
|
481
|
+
## Provider Combinations
|
|
482
|
+
|
|
483
|
+
| Use case | LLM | Embeddings | Install |
|
|
484
|
+
|--------------------------|-----------------------|----------------------|----------------------------------------|
|
|
485
|
+
| HF cloud (cheapest) | Mistral-7B via HF API | BGE-large via HF API | `pip install shama[huggingface]` |
|
|
486
|
+
| Fully local / air-gapped | Phi-3 local | BGE-base local | `pip install shama[huggingface-local]` |
|
|
487
|
+
| Best local quality | Llama-3-8B local | BGE-large local | `pip install shama[huggingface-local]` |
|
|
488
|
+
|
|
489
|
+
```python
|
|
490
|
+
# DeepSeek LLM + OpenAI embeddings (recommended for cost)
|
|
491
|
+
client = ShamaClient.from_config(
|
|
492
|
+
deepseek_api_key="your_deepseek_key",
|
|
493
|
+
embedding_api_key="your_openai_key", # OpenAI used only for embeddings
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
# OpenAI for everything (simplest)
|
|
497
|
+
client = ShamaClient.from_config(
|
|
498
|
+
openai_api_key="sk-...",
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
# Anthropic LLM + OpenAI embeddings
|
|
502
|
+
client = ShamaClient.from_config(
|
|
503
|
+
anthropic_api_key="sk-ant-...",
|
|
504
|
+
embedding_api_key="sk-...", # OpenAI key for embeddings
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# Azure OpenAI (full Azure stack)
|
|
508
|
+
client = ShamaClient.from_config(
|
|
509
|
+
azure_api_key="...",
|
|
510
|
+
azure_endpoint="https://my-resource.openai.azure.com/",
|
|
511
|
+
azure_judge_deployment="gpt-4o",
|
|
512
|
+
azure_fast_deployment="gpt-4o-mini",
|
|
513
|
+
azure_embedding_deployment="text-embedding-3-small",
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
Get your token at https://huggingface.co/settings/tokens (Read scope is enough).
|
|
517
|
+
# HuggingFace LLM + HuggingFace embeddings (cloud, cheapest after DeepSeek)
|
|
518
|
+
client = ShamaClient.from_config(
|
|
519
|
+
huggingface_api_key="hf_...",
|
|
520
|
+
huggingface_judge_model="mistralai/Mistral-7B-Instruct-v0.3",
|
|
521
|
+
huggingface_fast_model="mistralai/Mistral-7B-Instruct-v0.3",
|
|
522
|
+
huggingface_embedding_model="BAAI/bge-large-en-v1.5", # 1024 dims
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
# HuggingFace LLM + OpenAI embeddings (best quality embeddings)
|
|
526
|
+
client = ShamaClient.from_config(
|
|
527
|
+
huggingface_api_key="hf_...",
|
|
528
|
+
embedding_api_key="sk-...", # OpenAI key for embeddings only
|
|
529
|
+
)
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
## Quick usage reference
|
|
533
|
+
|
|
534
|
+
```python
|
|
535
|
+
# Option 1: HF Inference API - both LLM and embeddings
|
|
536
|
+
client = ShamaClient.from_config(
|
|
537
|
+
huggingface_api_key="hf_...",
|
|
538
|
+
huggingface_embedding_model="BAAI/bge-large-en-v1.5",
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
# Option 2: HF for LLM + OpenAI for embeddings
|
|
542
|
+
client = ShamaClient.from_config(
|
|
543
|
+
huggingface_api_key="hf_...",
|
|
544
|
+
embedding_api_key="sk-...",
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
# Option 3: Fully local - zero API cost, full privacy
|
|
548
|
+
client = ShamaClient.from_config(
|
|
549
|
+
huggingface_local_llm_model="microsoft/Phi-3-mini-4k-instruct",
|
|
550
|
+
huggingface_local_embedding_model="BAAI/bge-base-en-v1.5",
|
|
551
|
+
huggingface_local_device="cpu",
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
# Option 4: from_components - maximum flexibility
|
|
555
|
+
from shama import ShamaClient, HuggingFaceLLMProvider, HuggingFaceLocalEmbeddingProvider
|
|
556
|
+
|
|
557
|
+
client = ShamaClient.from_components(
|
|
558
|
+
llm_provider=HuggingFaceLLMProvider(api_key="hf_...", judge_model="Qwen/Qwen2.5-72B-Instruct"),
|
|
559
|
+
embedding_provider=HuggingFaceLocalEmbeddingProvider(model_name="BAAI/bge-large-en-v1.5"),
|
|
560
|
+
# ... other components
|
|
561
|
+
)
|
|
562
|
+
```
|
|
563
|
+
|
|
564
|
+
---
|
|
565
|
+
|
|
566
|
+
## Background Scheduler
|
|
567
|
+
|
|
568
|
+
SHAMA's self-healing runs automatically via Celery. Start it alongside your application:
|
|
569
|
+
|
|
570
|
+
```python
|
|
571
|
+
# In your app startup
|
|
572
|
+
from shama.scheduler.tasks import register_shama_context
|
|
573
|
+
|
|
574
|
+
register_shama_context({
|
|
575
|
+
**client.get_scheduler_context(),
|
|
576
|
+
"agent_registry": ["agent-001", "agent-002"], # agents to process
|
|
577
|
+
})
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
```bash
|
|
581
|
+
# Terminal 1 - Celery worker
|
|
582
|
+
celery -A shama.scheduler.tasks worker --loglevel=info
|
|
583
|
+
|
|
584
|
+
# Terminal 2 - Celery beat (scheduler)
|
|
585
|
+
celery -A shama.scheduler.tasks beat --loglevel=info
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
Default schedule:
|
|
589
|
+
- Decay pass: every **15 minutes**
|
|
590
|
+
- Promotion pass: every **60 minutes**
|
|
591
|
+
- Re-verify and contradiction resolution: **on-demand** (triggered by decay engine)
|
|
592
|
+
|
|
593
|
+
---
|
|
594
|
+
|
|
595
|
+
## License
|
|
596
|
+
|
|
597
|
+
MIT - use freely, including commercially.
|