trw-memory 0.3.0__tar.gz

trw_memory-0.3.0/.env.example

@@ -0,0 +1,43 @@
+# TRW Memory Engine — Environment Variables
+# All settings use MEMORY_ prefix (case-insensitive via pydantic-settings).
+# Copy to .env and uncomment/fill values as needed.
+
+# ── API Authentication (optional) ─────────────────────────────────────────
+# Disabled if not set — enables X-API-Key header validation
+# MEMORY_API_KEY=your-api-key-here
+
+# ── Field-Level Encryption (optional) ─────────────────────────────────────
+# Disabled if not set — hex-encoded 32-byte key
+# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
+# MEMORY_MASTER_KEY=
+
+# ── Storage Backend ────────────────────────────────────────────────────────
+# MEMORY_STORAGE_BACKEND=sqlite   # sqlite (default) or yaml
+# MEMORY_STORAGE_PATH=.memory
+# MEMORY_SQLITE_DB_NAME=memory.db
+
+# ── Encryption Settings ───────────────────────────────────────────────────
+# MEMORY_ENCRYPTION_ENABLED=false
+# MEMORY_KEY_SOURCE=env           # env, keyring, or file
+# MEMORY_KEY_FILE_PATH=~/.trw-memory/master.key
+
+# ── Retrieval Tuning ──────────────────────────────────────────────────────
+# MEMORY_BM25_CANDIDATES=50
+# MEMORY_VECTOR_CANDIDATES=50
+# MEMORY_RRF_K=60
+
+# ── Deduplication ─────────────────────────────────────────────────────────
+# MEMORY_DEDUP_ENABLED=true
+# MEMORY_DEDUP_SKIP_THRESHOLD=0.95
+# MEMORY_DEDUP_MERGE_THRESHOLD=0.85
+
+# ── Tier Lifecycle ────────────────────────────────────────────────────────
+# MEMORY_HOT_MAX_ENTRIES=50
+# MEMORY_HOT_TTL_DAYS=7
+# MEMORY_COLD_THRESHOLD_DAYS=90
+# MEMORY_RETENTION_DAYS=365
+
+# ── Scoring Weights (must sum to 1.0) ────────────────────────────────────
+# MEMORY_SCORE_RELEVANCE_WEIGHT=0.4
+# MEMORY_SCORE_RECENCY_WEIGHT=0.3
+# MEMORY_SCORE_IMPORTANCE_WEIGHT=0.3

trw_memory-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python-version }}-
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,vectors,bm25,mcp]"
+          pip install cryptography keyring sentence-transformers
+      - name: Lint
+        run: ruff check src/
+      - name: Type check
+        run: mypy --strict src/trw_memory/
+      - name: Test
+        run: pytest tests/ -x --timeout=120 -q

trw_memory-0.3.0/.github/workflows/release.yml

@@ -0,0 +1,39 @@
+name: Release to PyPI
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  id-token: write  # Required for trusted publishing
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install build tools
+        run: pip install build
+      - name: Build package
+        run: python -m build
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

trw_memory-0.3.0/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+dist/
+build/
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+
+# Testing
+.pytest_cache/
+.hypothesis/
+.coverage
+htmlcov/
+coverage.xml
+.mypy_cache/
+
+# IDE
+.vscode/
+.idea/
+
+# TRW runtime state
+.trw/
+
+# Internal Claude Code settings — not for public consumption
+.claude/
+
+# Internal PRD/sprint docs — development artefacts, not part of the public package
+docs/
+
+# Scratch workspace
+scratch/

trw_memory-0.3.0/CHANGELOG.md

@@ -0,0 +1,124 @@
+# Changelog
+
+All notable changes to the TRW Memory package.
+
+## [0.3.0] — 2026-03-19
+
+### Added
+
+- **Structured logging** — new `trw_memory/_logging.py` module with `configure_logging()`, `-v/-q/--log-level` CLI flags, `NullHandler` in `__init__.py` (library best practice), and `exc_info=True` added to exception handlers in `embeddings/local.py` and `migration/from_trw.py`. All 28 source files normalized to `structlog.get_logger(__name__)`. 10 sentence-style log messages converted to snake_case event names.
+- **BSL 1.1 license** — `pyproject.toml` updated from CC BY-NC-SA to `BUSL-1.1` (Change Date 2030-03-21, Change License Apache 2.0) in preparation for open-source publication.
+- **CI hardening** — ruff format checks now blocking in the memory CI workflow; pre-existing ruff lint issues resolved. mypy `--strict` fully clean across all 77 files.
+
+### Changed
+
+- **Version aligned to monorepo v24.4_TRW release** — package version bumped from 0.2.0 to 0.3.0 at the v24.4 framework release (`98608d2a`).
+
+---
+
+## [0.2.0] — 2026-03-19
+
+### Added
+
+- **Structured logging foundation** — `_logging.py` introduced as part of the monorepo-wide logging overhaul. Library stays silent by default (`NullHandler`); CLI configures logging at startup only.
+- **CLI verbosity flags** — `-v/--verbose`, `-q/--quiet`, `--log-level` added to all CLI subcommands.
+
+### Changed
+
+- **Logger normalization** — all 28 source files migrated from bare `structlog.get_logger()` to `structlog.get_logger(__name__)` for module-level filtering.
+- **Log event naming** — 10 sentence-style event strings converted to snake_case.
+
+---
+
+## [0.1.3] — 2026-03-17
+
+### Fixed
+
+- **WAL mode reliability** — enabled `PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;` on `SQLiteBackend` with a warning log if WAL activation fails. Allows concurrent reads without write serialization.
+- **Decay loop `datetime.now()` per-iteration** — captured once before the loop instead of calling it on every iteration.
+- **`graph.py` thread-safe commits** — `conn.commit()` calls now accept an optional `threading.Lock` parameter; wired in from all callers.
+
+---
+
+## [0.1.2] — 2026-03-15
+
+### Changed
+
+- **mypy `--strict` clean** — resolved all 13 pre-existing type errors: fixed `type: ignore` codes in `sqlite_backend.py`, `local.py`, and `client.py`; widened formatter params to `Sequence[Mapping]` for TypedDict covariance in `cli_formatters.py`; added `_backend_or_raise` property for None safety in `llamaindex.py`. Zero errors across 77 files.
+- **Ruff expanded** — rule set expanded from 14 to 26 categories (matching monorepo standard). 244 violations fixed, 0 remaining.
+- **`delete_by_namespace` cursor fix** — `cursor.rowcount` moved inside the write lock to prevent a race condition with concurrent deletes.
+
+---
+
+## [0.1.1] — 2026-03-10
+
+### Changed
+
+- **Standalone REST API removed** — the orphaned `trw-memory/api/` package (8 files, ~1,428 LOC) and its 5 matching test files were deleted. The API was superseded by the backend package and was unreachable without a running platform instance. The removal cleans up dead code and reduces confusion for consumers.
+- **`_parsing.py` shared helpers** — `storage/_parsing.py` introduced with `parse_dt`, `parse_json_list`, `parse_json_dict_str`, `parse_json_dict_int` to replace duplicated parsing in `sqlite_backend.py` and `yaml_backend.py`. Fixes a subtle UTC normalization bug in `yaml_backend`.
+- **Sprint 39/40 quality review** — CLI import merge check fixed (keyword recall replaced with `backend.get(eid)` for reliable ID-based dedup during `--merge` imports); `BackendOwnerMixin` extracted to eliminate identical `close/__enter__/__exit__` boilerplate across LangChain, CrewAI, and VSCode integration adapters; `corpus.py` docstrings corrected (said "YAML" but writes JSON).
+
+---
+
+## [0.1.0] — 2026-02-27
+
+### Added
+
+- **Initial release** — `trw-memory` introduced as a standalone persistent memory engine extracted from `trw-mcp`. Shipped as part of the monorepo alongside trw-mcp's knowledge topology and hybrid retrieval work (commit `b28c2490`).
+
+#### Storage backends
+
+- **SQLite primary backend** — `storage/sqlite_backend.py` with FTS5 full-text search, WAL journal mode, and `check_same_thread=False` for concurrent HTTP client access.
+- **YAML secondary backend** — `storage/yaml_backend.py` for backward compatibility with pre-migration entries.
+
+#### Retrieval
+
+- **BM25 sparse retrieval** — `retrieval/bm25.py` via `rank-bm25` with hyphenated-tag expansion and zero-IDF fallback.
+- **Dense vector retrieval** — `retrieval/dense.py` via `sqlite-vec` (384-dim all-MiniLM-L6-v2).
+- **Hybrid search with RRF** — `retrieval/hybrid.py` combines BM25 and dense rankings via Reciprocal Rank Fusion (k=60). Gracefully degrades to BM25-only when vectors are unavailable.
+
+#### Lifecycle
+
+- **Tiered storage** — `lifecycle/tiers.py`: hot tier (in-memory LRU), warm tier (sqlite-vec + JSONL sidecar), cold tier (YAML archive). Four automatic transitions: Hot→Warm (TTL/overflow), Warm→Cold (idle+low-impact), Cold→Warm (on access), Cold→Purge (365d+low-impact). Purge audit trail at `.trw/memory/purge_audit.jsonl`.
+- **Scoring engine** — `lifecycle/scoring.py`: Q-learning with EMA updates, Ebbinghaus forgetting curve, Bayesian MACLA calibration. Stanford Generative Agents importance formula: `w1*relevance + w2*recency + w3*importance`.
+- **LLM consolidation** — `lifecycle/consolidation.py`: embedding-based cluster detection (single-linkage agglomerative, pairwise cosine threshold), LLM-powered summarization via `anthropic` SDK (claude-haiku), original entry archival to cold tier with atomic rollback.
+- **Semantic dedup** — `lifecycle/dedup.py`: three-tier write-time dedup (skip ≥0.95, merge ≥0.85, store <0.85) via cosine similarity. `merge_entries()` with union tags/evidence, max impact, recurrence increment, and `merged_from` audit trail.
+
+#### Knowledge graph
+
+- **`graph.py`** — similarity, tag co-occurrence, and consolidation edges; BFS traversal; importance boost/decay with outcome history.
+
+#### Remote sync
+
+- **`sync/remote.py`** — publish/fetch with anonymization and fail-open design.
+- **`sync/conflict.py`** — vector clock comparison, merge, and conflict resolution.
+- **`sync/retry_queue.py`** — thread-safe JSONL queue with 500-entry depth cap.
+- **`sync/subscriber.py`** — SSE real-time stream with automatic reconnection.
+
+#### Security
+
+- **`security/`** — AES-256-GCM field-level encryption, PII detection and redaction, memory poisoning detection, RBAC, and full audit trail.
+
+#### Integrations
+
+- **LangChain** — `TRWChatMessageHistory` and `TRWVectorStore` adapters.
+- **LlamaIndex** — reader and writer components.
+- **CrewAI** — `TRWCrewStorage` component.
+- **OpenAI-compatible** — `LocalMemoryAdapter` with 4 function-call operations (store, recall, search, forget).
+
+#### CLI
+
+- **`trw-memory` CLI** — 8 subcommands (`store`, `recall`, `search`, `forget`, `consolidate`, `export`, `import`, `status`) with JSON, table, and YAML output formats. `cli_parser.py` extracted for line-count compliance.
+
+#### MCP tools
+
+- **6 MCP tools** (`tools/`) — `store`, `recall`, `search`, `consolidate`, `forget`, `status` for direct integration into Claude Code sessions.
+
+#### Client SDK
+
+- **`MemoryClient`** (`client.py`) — high-level async Python client with `store()`, `recall()`, `forget()`, and `search()` methods.
+- **TypeScript SDK** — `@trw/memory-sdk` with `MemoryClient` class and full TypeScript types.
+
+#### Benchmarks
+
+- **`benchmarks/`** — 4 benchmark modules (latency, throughput, memory, quality) with configurable thresholds and `python -m benchmarks` entry point.

trw_memory-0.3.0/CLAUDE.md

@@ -0,0 +1,42 @@
+# trw-memory
+
+Standalone persistent memory engine for AI agents — hybrid retrieval (BM25 + dense vectors), tiered storage (SQLite primary, YAML secondary), semantic dedup, and knowledge-graph traversal. Used by trw-mcp as its memory backend via `memory_adapter.py`.
+
+## Build & Test
+
+```bash
+cd trw-memory
+.venv/bin/python -m pytest tests/test_specific_file.py -v   # Single file (preferred)
+.venv/bin/python -m pytest tests/ -v                         # Full suite (delivery only)
+.venv/bin/python -m mypy --strict src/trw_memory/            # Type check
+pip install -e ".[dev]"                                       # Dev install
+pip install -e ".[dev,vectors,bm25]"                         # With optional deps
+```
+
+## Architecture
+
+| Module | Purpose |
+|--------|---------|
+| `storage/` | SQLite + YAML dual backends (`sqlite_backend.py`, `yaml_backend.py`) |
+| `retrieval/` | BM25 keyword + dense vector hybrid search |
+| `lifecycle/` | Tier promotion/demotion, utility scoring, dedup, consolidation |
+| `graph.py` | Knowledge graph with BFS traversal |
+| `sync/` | Remote publish/fetch with vector clocks |
+| `security/` | Field-level encryption, PII detection, RBAC |
+| `tools/` | MCP tool wrappers (optional `[mcp]` dep) |
+| `client.py` | Public `MemoryClient` — primary consumer API |
+
+## Testing Patterns
+
+- **In-memory SQLite**: `SQLiteBackend(":memory:")` — use for unit tests, no `tmp_path` needed
+- **Disk SQLite**: `SQLiteBackend(tmp_path / "test.db")` — use for integration tests
+- **Shared fixtures**: `tests/conftest.py` provides `sqlite_backend`, `sqlite_memory_backend`, `memory_client`, `make_entry()`, `make_entry_dict()`
+- `asyncio_mode = "auto"` — write `async def test_foo():` without `@pytest.mark.asyncio`
+
+## Key Gotchas
+
+- `sqlite-vec` is an optional dep (`[vectors]`) — skip dense vector tests with `pytest.importorskip("sqlite_vec")`
+- `sentence-transformers` is optional (`[embeddings]`) — embeddings fail gracefully when unavailable
+- Pydantic v2: `use_enum_values=True` required for YAML round-trip; `populate_by_name=True` when using `Field(alias=...)`
+- structlog: never use `event=` as a kwarg — it's reserved; use `action=` or `operation=`
+- Coverage threshold: 85% (fail_under in pyproject.toml)

trw_memory-0.3.0/LICENSE

@@ -0,0 +1,41 @@
+Business Source License 1.1
+
+Licensor:  Tyler Wall
+Licensed Work:  TRW Framework
+                The Licensed Work is (c) 2026 Tyler Wall.
+
+Additional Use Grant:
+  You may use the Licensed Work for any purpose other than offering
+  a commercial product or service that competes with the Licensed Work.
+
+  "Compete" means providing a substantially similar product or service
+  that replaces or substitutes for the Licensed Work's core functionality:
+  persistent memory with scoring and decay, sprint orchestration,
+  requirements lifecycle management, phase gates, or agent team
+  coordination for AI coding agents.
+
+  For clarity, the following uses are always permitted:
+  - Using TRW in your own software development projects
+  - Integrating TRW into your development workflow
+  - Building applications, tools, or services that use TRW internally
+  - Modifying TRW for your own non-competing use
+  - Evaluating, testing, and benchmarking TRW
+
+Change Date:  2030-03-21
+
+Change License:  Apache License, Version 2.0
+
+Notice:
+
+  The Business Source License (this document, or the "License") is not
+  an Open Source license. However, the Licensed Work will eventually be
+  made available under the Change License, as stated in this License.
+
+  License text copyright (c) 2017 MariaDB Corporation Ab, All Rights
+  Reserved. "Business Source License" is a trademark of MariaDB
+  Corporation Ab.
+
+  For the full BSL 1.1 license text, see:
+  https://mariadb.com/bsl11/
+
+SPDX-License-Identifier: BUSL-1.1