dejasearch 0.3.0__tar.gz

dejasearch-0.3.0/.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+
+[*.{yml,yaml,toml,json}]
+indent_size = 2
+
+[Makefile]
+indent_style = tab

dejasearch-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.10", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache embedding model
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/fastembed_cache
+            ~/AppData/Local/Temp/fastembed_cache
+          key: fastembed-multilingual-e5-small
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest -v
+        env:
+          PYTHONIOENCODING: utf-8

dejasearch-0.3.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+.eggs/
+.pytest_cache/
+*.db
+*.lock
+tests/golden_pairs.json

dejasearch-0.3.0/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+## 0.3.0 (2026-04-02)
+
+### Features
+- **`get_context` MCP tool** — retrieve a chunk with surrounding turns (±window) from the same session (#4)
+- **Secret filtering** — passwords, API keys, tokens, and private keys are redacted during indexing (#5)
+- **`deja redact`** — update secrets in existing index without re-embedding (seconds vs full reindex)
+- **`platformdirs`** — index stored in OS-standard location; auto-detects legacy `~/.claude/deja/` (#7)
+
+### Tests
+- Tests for get_context, secret redaction, redact command, stats health check (41 total)
+
+### CI
+- Upgraded to Node24 actions (checkout@v5, setup-python@v6, cache@v5)
+
+## 0.2.0 (2026-04-02)
+
+### Breaking Changes
+- `get_session` MCP tool renamed to `get_session_chunks` (honest about returning chunks, not raw messages)
+- Requires FastMCP >= 3.0.0 (upgraded from 2.x)
+
+### Bug Fixes
+- **Incremental indexing correctness (P0)** — message_index no longer collides on incremental runs; dangling user at offset boundary no longer lost; stable upsert preserves rowid (#16)
+- **Streaming indexer** — process turns in batches (TURNS_PER_BATCH=50) instead of loading entire file into memory (#9). Note: fastembed + ONNX runtime still uses ~3GB RAM for the model itself
+- **FTS query** — token-wise AND instead of exact phrase match; `nginx proxy` now finds results with both words in any order (#18)
+- **Search filters** — overfetch candidates (k=100) when project/date filters are active (#19)
+- **FastMCP upgrade** — use public `ctx.lifespan_context` API instead of private `_lifespan_result` (#10)
+- **SQLite threading** — `check_same_thread=False` for async MCP server
+- **Windows UTF-8** — force UTF-8 stdout/stderr on Windows to prevent cp1252 crashes on Cyrillic
+- **Hardcoded paths** — index location changed to `~/.claude/deja/` (no user-specific paths in code)
+
+### Features
+- `deja eval` command with MRR@5 scoring for search quality benchmarking
+- Time decay scoring (disabled by default, `time_decay=True` to enable)
+- Auto-indexing docs in README (Claude Code Stop hook)
+- SQLite indexes on `session_id` and `(project_path, timestamp)`
+
+### Repo
+- GitHub Actions CI (ubuntu + windows, Python 3.10 + 3.13)
+- README with badges, architecture diagram, usage docs
+- CONTRIBUTING.md, .editorconfig, LICENSE (MIT)
+- Social preview image
+
+## 0.1.0 (2026-03-31)
+
+Initial MVP release.
+
+- JSONL parser with offset support and tool_result separation
+- Chunker with sentence-boundary splitting (1500 chars, 200 overlap)
+- Indexer with incremental/safe reindex, GC, batch embedding
+- Hybrid search: vector KNN + FTS5 + Reciprocal Rank Fusion
+- MCP server (FastMCP, stdio transport) with `search` and `get_session` tools
+- CLI: `deja index`, `deja serve`
+- fastembed (multilingual-e5-small, 384-dim ONNX) + sqlite-vec + SQLite FTS5

dejasearch-0.3.0/CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Contributing
+
+## Setup
+
+```bash
+git clone https://github.com/CynepMyx/deja.git
+cd deja
+python -m venv .venv
+.venv/Scripts/activate      # Windows
+# source .venv/bin/activate  # Linux/macOS
+pip install -e ".[dev]"
+```
+
+First test run downloads the embedding model (~117 MB).
+
+## Tests
+
+```bash
+pytest -v
+```
+
+Tests use temporary directories and in-memory databases — no external services required.
+
+## Code style
+
+- Python 3.10+ (no walrus operator abuse, keep it readable)
+- No docstrings on obvious functions
+- `print(..., file=sys.stderr)` for logging — stdout is reserved for MCP JSON-RPC
+- Follow existing patterns in the codebase
+
+## Pull requests
+
+1. Create a branch from `main`
+2. Make your changes
+3. Ensure all tests pass
+4. Open a PR with a clear description of what and why

dejasearch-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Oleg Usoltsev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dejasearch-0.3.0/PKG-INFO

@@ -0,0 +1,172 @@
+Metadata-Version: 2.4
+Name: dejasearch
+Version: 0.3.0
+Summary: Semantic search MCP server for Claude Code sessions
+Project-URL: Repository, https://github.com/CynepMyx/deja
+Project-URL: Issues, https://github.com/CynepMyx/deja/issues
+Author: Oleg Usoltsev
+License-Expression: MIT
+License-File: LICENSE
+Keywords: claude-code,embeddings,mcp,semantic-search,sqlite
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: fastembed<1.0,>=0.6.0
+Requires-Dist: fastmcp<4.0,>=3.0.0
+Requires-Dist: platformdirs>=4.0
+Requires-Dist: sqlite-vec==0.1.8
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# deja
+
+[![CI](https://github.com/CynepMyx/deja/actions/workflows/ci.yml/badge.svg)](https://github.com/CynepMyx/deja/actions)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue?logo=python&logoColor=white)](https://python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+[![MCP](https://img.shields.io/badge/MCP-compatible-8A2BE2?logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiIgZmlsbD0id2hpdGUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PGNpcmNsZSBjeD0iOCIgY3k9IjgiIHI9IjYiIGZpbGw9Im5vbmUiIHN0cm9rZT0id2hpdGUiIHN0cm9rZS13aWR0aD0iMS41Ii8+PGNpcmNsZSBjeD0iOCIgY3k9IjgiIHI9IjIiLz48L3N2Zz4=)](https://modelcontextprotocol.io/)
+
+> Semantic search over your Claude Code session history. Ask questions about past conversations by meaning, not just keywords.
+
+**deja** is an [MCP server](https://modelcontextprotocol.io/) that indexes Claude Code JSONL sessions and provides hybrid search (vector + full-text) directly from Claude Code.
+
+## How it works
+
+```
+~/.claude/projects/*/        deja index         index.db
+     *.jsonl           ──────────────►    (SQLite + vec + FTS5)
+                        embeddings
+                                           │
+                                           │  deja serve (MCP stdio)
+                                           ▼
+                                      Claude Code
+                                    "search past sessions"
+```
+
+1. **Index** — parses JSONL session files, extracts conversation turns, embeds with `multilingual-e5-small`, stores in SQLite
+2. **Serve** — MCP server opens the index and answers search queries via stdio transport
+
+Search combines vector KNN (semantic similarity) and FTS5 (keyword matching) via Reciprocal Rank Fusion.
+
+## Install
+
+```bash
+git clone https://github.com/CynepMyx/deja.git
+cd deja
+python -m venv .venv
+.venv/Scripts/pip install -e .        # Windows
+# .venv/bin/pip install -e .          # Linux/macOS
+```
+
+First run downloads the embedding model (~117 MB).
+
+## Usage
+
+### Build the index
+
+```bash
+deja index              # incremental — only new/changed files
+deja index --reindex    # full rebuild
+```
+
+Scans all `~/.claude/projects/*/*.jsonl` files.
+
+### Add to Claude Code
+
+Add to `~/.claude.json` under `mcpServers`:
+
+```json
+"deja": {
+    "type": "stdio",
+    "command": "/path/to/deja/.venv/Scripts/deja.exe",
+    "args": ["serve"],
+    "env": {
+        "PYTHONUNBUFFERED": "1"
+    }
+}
+```
+
+Restart Claude Code — deja will appear as a connected MCP server.
+
+### MCP Tools
+
+| Tool | Description |
+|------|-------------|
+| `search` | Hybrid semantic + keyword search across all sessions |
+| `get_session_chunks` | Get indexed chunks for a session (not raw messages) |
+
+**search** parameters:
+- `query` (string) — what to search for
+- `limit` (int, default 10) — max results
+- `project` (string, optional) — filter by project
+- `date_from` / `date_to` (string, optional) — ISO date range
+
+### Auto-indexing (optional)
+
+Index automatically when a Claude Code session ends. Add a Stop hook to `~/.claude/settings.json`:
+
+```json
+"hooks": {
+    "Stop": [
+        {
+            "matcher": "",
+            "hooks": [
+                {
+                    "type": "command",
+                    "command": "/path/to/deja/.venv/bin/deja index"
+                }
+            ]
+        }
+    ]
+}
+```
+
+On Windows with Git Bash, wrap in a shell script:
+
+```bash
+#!/bin/bash
+DEJA="/path/to/deja/.venv/Scripts/deja.exe"
+[ -f "$DEJA" ] && "$DEJA" index >/dev/null 2>&1 &
+```
+
+```json
+"command": "bash /path/to/deja-index.sh"
+```
+
+PID lock prevents concurrent indexers — safe with multiple sessions.
+
+## Stack
+
+- **[fastembed](https://github.com/qdrant/fastembed)** — ONNX embeddings (`intfloat/multilingual-e5-small`, 384-dim)
+- **[sqlite-vec](https://github.com/asg017/sqlite-vec)** — vector KNN search in SQLite
+- **SQLite FTS5** — full-text keyword search
+- **[FastMCP](https://github.com/jlowin/fastmcp)** — MCP server framework
+
+## Performance
+
+| Metric | Value |
+|--------|-------|
+| Incremental index | < 30 sec |
+| Search latency (warm) | < 500 ms |
+| First search (cold start) | < 5 sec |
+| RAM (search) | ~150 MB |
+| RAM (indexing) | ~300 MB |
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+[MIT](LICENSE)

dejasearch-0.3.0/README.md

@@ -0,0 +1,143 @@
+# deja
+
+[![CI](https://github.com/CynepMyx/deja/actions/workflows/ci.yml/badge.svg)](https://github.com/CynepMyx/deja/actions)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue?logo=python&logoColor=white)](https://python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+[![MCP](https://img.shields.io/badge/MCP-compatible-8A2BE2?logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiIgZmlsbD0id2hpdGUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PGNpcmNsZSBjeD0iOCIgY3k9IjgiIHI9IjYiIGZpbGw9Im5vbmUiIHN0cm9rZT0id2hpdGUiIHN0cm9rZS13aWR0aD0iMS41Ii8+PGNpcmNsZSBjeD0iOCIgY3k9IjgiIHI9IjIiLz48L3N2Zz4=)](https://modelcontextprotocol.io/)
+
+> Semantic search over your Claude Code session history. Ask questions about past conversations by meaning, not just keywords.
+
+**deja** is an [MCP server](https://modelcontextprotocol.io/) that indexes Claude Code JSONL sessions and provides hybrid search (vector + full-text) directly from Claude Code.
+
+## How it works
+
+```
+~/.claude/projects/*/        deja index         index.db
+     *.jsonl           ──────────────►    (SQLite + vec + FTS5)
+                        embeddings
+                                           │
+                                           │  deja serve (MCP stdio)
+                                           ▼
+                                      Claude Code
+                                    "search past sessions"
+```
+
+1. **Index** — parses JSONL session files, extracts conversation turns, embeds with `multilingual-e5-small`, stores in SQLite
+2. **Serve** — MCP server opens the index and answers search queries via stdio transport
+
+Search combines vector KNN (semantic similarity) and FTS5 (keyword matching) via Reciprocal Rank Fusion.
+
+## Install
+
+```bash
+git clone https://github.com/CynepMyx/deja.git
+cd deja
+python -m venv .venv
+.venv/Scripts/pip install -e .        # Windows
+# .venv/bin/pip install -e .          # Linux/macOS
+```
+
+First run downloads the embedding model (~117 MB).
+
+## Usage
+
+### Build the index
+
+```bash
+deja index              # incremental — only new/changed files
+deja index --reindex    # full rebuild
+```
+
+Scans all `~/.claude/projects/*/*.jsonl` files.
+
+### Add to Claude Code
+
+Add to `~/.claude.json` under `mcpServers`:
+
+```json
+"deja": {
+    "type": "stdio",
+    "command": "/path/to/deja/.venv/Scripts/deja.exe",
+    "args": ["serve"],
+    "env": {
+        "PYTHONUNBUFFERED": "1"
+    }
+}
+```
+
+Restart Claude Code — deja will appear as a connected MCP server.
+
+### MCP Tools
+
+| Tool | Description |
+|------|-------------|
+| `search` | Hybrid semantic + keyword search across all sessions |
+| `get_session_chunks` | Get indexed chunks for a session (not raw messages) |
+
+**search** parameters:
+- `query` (string) — what to search for
+- `limit` (int, default 10) — max results
+- `project` (string, optional) — filter by project
+- `date_from` / `date_to` (string, optional) — ISO date range
+
+### Auto-indexing (optional)
+
+Index automatically when a Claude Code session ends. Add a Stop hook to `~/.claude/settings.json`:
+
+```json
+"hooks": {
+    "Stop": [
+        {
+            "matcher": "",
+            "hooks": [
+                {
+                    "type": "command",
+                    "command": "/path/to/deja/.venv/bin/deja index"
+                }
+            ]
+        }
+    ]
+}
+```
+
+On Windows with Git Bash, wrap in a shell script:
+
+```bash
+#!/bin/bash
+DEJA="/path/to/deja/.venv/Scripts/deja.exe"
+[ -f "$DEJA" ] && "$DEJA" index >/dev/null 2>&1 &
+```
+
+```json
+"command": "bash /path/to/deja-index.sh"
+```
+
+PID lock prevents concurrent indexers — safe with multiple sessions.
+
+## Stack
+
+- **[fastembed](https://github.com/qdrant/fastembed)** — ONNX embeddings (`intfloat/multilingual-e5-small`, 384-dim)
+- **[sqlite-vec](https://github.com/asg017/sqlite-vec)** — vector KNN search in SQLite
+- **SQLite FTS5** — full-text keyword search
+- **[FastMCP](https://github.com/jlowin/fastmcp)** — MCP server framework
+
+## Performance
+
+| Metric | Value |
+|--------|-------|
+| Incremental index | < 30 sec |
+| Search latency (warm) | < 500 ms |
+| First search (cold start) | < 5 sec |
+| RAM (search) | ~150 MB |
+| RAM (indexing) | ~300 MB |
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+[MIT](LICENSE)

dejasearch-0.3.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dejasearch"
+version = "0.3.0"
+description = "Semantic search MCP server for Claude Code sessions"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+    { name = "Oleg Usoltsev" },
+]
+keywords = ["claude-code", "mcp", "semantic-search", "embeddings", "sqlite"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "fastembed>=0.6.0,<1.0",
+    "sqlite-vec==0.1.8",
+    "fastmcp>=3.0.0,<4.0",
+    "platformdirs>=4.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+]
+
+[project.urls]
+Repository = "https://github.com/CynepMyx/deja"
+Issues = "https://github.com/CynepMyx/deja/issues"
+
+[project.scripts]
+deja = "deja.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/deja"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

dejasearch-0.3.0/src/deja/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

dejasearch-0.3.0/src/deja/chunker.py

@@ -0,0 +1,47 @@
+MAX_CHUNK_SIZE = 1500
+OVERLAP = 200
+
+def _split_text(text: str) -> list[str]:
+    if len(text) <= MAX_CHUNK_SIZE:
+        return [text]
+
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + MAX_CHUNK_SIZE
+        if end >= len(text):
+            chunks.append(text[start:])
+            break
+
+        search_region = text[end - OVERLAP:end]
+        for sep in ["\n\n", ". ", ".\n", "\n"]:
+            pos = search_region.rfind(sep)
+            if pos != -1:
+                end = end - OVERLAP + pos + len(sep)
+                break
+
+        chunks.append(text[start:end])
+        start = end - OVERLAP
+        if start < 0:
+            start = 0
+
+    return chunks
+
+def make_chunks(
+    turn: dict, session_id: str, project_path: str
+) -> list[dict]:
+    embed_text = f"{turn['user_text']}\n\n{turn['assistant_text']}"
+    parts = _split_text(embed_text)
+
+    return [
+        {
+            "chunk_text": part,
+            "tool_result_text": turn.get("tool_result_text", "") if i == 0 else "",
+            "session_id": session_id,
+            "message_index": turn["message_index"],
+            "split_index": i,
+            "timestamp": turn.get("timestamp", ""),
+            "project_path": project_path,
+        }
+        for i, part in enumerate(parts)
+    ]