siftd 0.1.1__tar.gz

siftd-0.1.1/.claude-plugin/marketplace.json

@@ -0,0 +1,28 @@
+{
+  "name": "siftd",
+  "owner": {
+    "name": "kaygee",
+    "url": "https://github.com/kgruel"
+  },
+  "metadata": {
+    "description": "Siftd plugin marketplace — conversation research for Claude Code agents",
+    "version": "1.0.0"
+  },
+  "plugins": [
+    {
+      "name": "siftd",
+      "source": "./plugin",
+      "description": "Search and research past conversations from CLI coding sessions",
+      "version": "1.0.0",
+      "author": {
+        "name": "kaygee",
+        "url": "https://github.com/kgruel"
+      },
+      "repository": "https://github.com/kgruel/siftd",
+      "license": "MIT",
+      "keywords": ["conversations", "search", "research", "history", "analytics"],
+      "category": "productivity",
+      "tags": ["research", "history", "memory"]
+    }
+  ]
+}

siftd-0.1.1/.githooks/pre-commit

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exec "$(dirname "$0")/../scripts/lint.sh"

siftd-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Run lint
+        run: uv run ruff check src/
+
+      - name: Run tests
+        run: uv run pytest tests/ -v --tb=short
+
+  test-with-embeddings:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies with embeddings
+        run: uv sync --extra dev --extra embed
+
+      - name: Run tests with embeddings
+        run: uv run pytest tests/ -v --tb=short

siftd-0.1.1/.github/workflows/publish.yml

@@ -0,0 +1,30 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # Required for trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

siftd-0.1.1/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.pyc
+.subtask/
+bench/runs/
+.pytest_cache/
+.ruff_cache/
+.obsidian/
+.venv/
+dist/
+.claude/
+HANDOFF.md
+ROADMAP.md

siftd-0.1.1/CHANGELOG.md

@@ -0,0 +1,96 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.1] - 2026-01-29
+
+### Added
+
+- `siftd install embed` — Convenience command to auto-detect installation method and install embedding dependencies
+- `:var` parameterized syntax for query files — safe quoting via sqlite3, alongside existing `$var` text substitution
+- `ADAPTER_INTERFACE_VERSION = 1` — Required attribute for all adapters, enables future interface migrations
+- `ON DELETE CASCADE` on schema foreign keys — Child records now cascade on parent delete
+
+### Changed
+
+- Adapter `discover()` function now requires `locations` keyword argument (fallback removed)
+- Error messages for missing `[embed]` extra now reference `siftd install embed` and suggest FTS5 alternative
+
+### Removed
+
+- `Conversation.default_model` field — Was defined but never populated or used
+
+### Fixed
+
+- Type checker (`ty`) configuration for optional dependencies — No longer blocks commits
+- `bench/corpus_analysis.py` type annotation bug
+
+## [0.1.0] - 2026-01-28
+
+Initial public release.
+
+### Added
+
+#### Core Features
+- **Ingestion** — Aggregate conversation logs from multiple CLI coding tools
+- **FTS5 Search** — Full-text search across all conversations via `siftd query -s`
+- **Semantic Search** — Vector similarity search via `siftd ask` (requires `[embed]` extra)
+- **Tagging** — Apply tags to conversations, workspaces, and tool calls for organization
+
+#### Adapters
+- Claude Code (Anthropic) — `~/.claude/projects`
+- Aider — `~/.aider`
+- Gemini CLI (Google) — `~/.gemini/tmp`
+- Codex CLI (OpenAI) — `~/.codex/sessions`
+- Drop-in adapter support via `~/.config/siftd/adapters/`
+- Entry-point adapter registration for pip-installable adapters
+
+#### CLI Commands
+- `siftd ingest` — Ingest logs from all discovered sources
+- `siftd status` — Show database statistics
+- `siftd query` — List/filter conversations with flexible filters
+- `siftd ask` — Semantic search over conversations (optional `[embed]` extra)
+- `siftd tag` — Apply or remove tags on entities
+- `siftd tags` — List, rename, or delete tags
+- `siftd tools` — Summarize tool usage by category
+- `siftd export` — Export conversations for PR review workflows
+- `siftd doctor` — Run health checks and maintenance
+- `siftd peek` — Inspect live sessions from disk (bypasses SQLite)
+- `siftd path` — Show XDG paths
+- `siftd config` — View or modify configuration
+- `siftd adapters` — List discovered adapters
+- `siftd copy` — Copy built-in resources for customization
+- `siftd backfill` — Backfill derived data from existing records
+
+#### Query System
+- User-defined SQL queries via `~/.config/siftd/queries/*.sql`
+- `$var` syntax for text substitution
+- Built-in queries: `cost.sql`, `shell-analysis.sql`
+
+#### Python API
+- `siftd.api.list_conversations()` — Query conversations with filters
+- `siftd.api.get_conversation()` — Get full conversation detail
+- `siftd.api.export_conversations()` — Export for external tools
+- `siftd.api.hybrid_search()` — Combined FTS5 + semantic search
+
+#### Storage
+- SQLite with FTS5 for full-text search
+- ULID primary keys throughout
+- Normalized schema with proper foreign key constraints
+- Extensible `*_attributes` tables for variable metadata
+
+#### Developer Experience
+- XDG Base Directory compliance for paths
+- `--db PATH` override for all commands
+- JSON output mode for scripting (`--json`)
+
+---
+
+[Unreleased]: https://github.com/anthropics/siftd/compare/v0.1.1...HEAD
+[0.1.1]: https://github.com/anthropics/siftd/compare/v0.1.0...v0.1.1
+[0.1.0]: https://github.com/anthropics/siftd/releases/tag/v0.1.0

siftd-0.1.1/CLAUDE.md

@@ -0,0 +1,69 @@
+Personal LLM usage analytics. Ingests conversation logs from CLI coding tools, stores in SQLite, queries via FTS5 and user-defined SQL files.
+
+## Install
+
+```bash
+uv pip install .           # core (FTS5 search, tags, queries)
+uv pip install .[embed]    # with semantic search (siftd ask)
+```
+
+## Architecture
+
+Core loop: **Ingest → Store → Query**
+
+- **Adapters** own parsing and raw format knowledge. Storage is adapter-agnostic.
+- **Storage** is normalized SQLite. Schema is fixed for core entities, extensible via `*_attributes` tables.
+- **Queries** are user-defined `.sql` files with `$var` substitution. The system is a data platform, not a reporting tool.
+
+## Design Principles
+
+1. **Manual first, automate when patterns emerge** — labels are user-applied, enrichment is deferred, cost is approximate. Don't build automation until real usage reveals what's worth automating.
+2. **Query-time computation over stored redundancy** — cost is derived via JOIN, not pre-computed. Avoids stale data and schema coupling.
+3. **Attributes for variable metadata** — when the field set varies by provider or adapter, use key/value `*_attributes` tables instead of adding nullable columns.
+4. **Adapters are the parsing boundary** — each adapter knows its raw format, dedup strategy, and provider source. Everything downstream is normalized.
+5. **Approximate is fine when labeled** — approximate cost is useful. Don't over-engineer precision until billing context demands it.
+
+## Branching
+
+- Work in a `wip/<topic>` branch during sessions. Handoff updates, test scaffolding, and iterative commits go here.
+- Subtask merges target `main` independently (they're isolated worktrees).
+- Merge/rebase WIP to main when the work is ready to ship.
+
+## Conventions
+
+- `commit=False` default on storage functions; caller controls transaction boundaries
+- ULIDs for all primary keys
+- XDG paths: data `~/.local/share/siftd`, config `~/.config/siftd`
+- New CLI commands follow existing argparse patterns in `src/cli.py`
+- New adapters implement `can_handle(source)`, `parse(source)`, `discover()`, set `HARNESS_SOURCE`
+- Queries go in `~/.config/siftd/queries/*.sql`, use `$var` for parameters
+
+## Agent Memory (siftd)
+
+Search past conversations:
+```
+siftd ask "your query"              # semantic search
+siftd ask -w projectname "query"    # filter by workspace
+siftd query <id>                    # drill down into conversation
+```
+
+Tag useful findings:
+```
+siftd tag <id> research:<topic>   # bookmark for later
+siftd query -l research:<topic>   # retrieve tagged
+```
+
+Tag conventions:
+- `research:*` — Investigation findings worth preserving
+- `useful:*` — General bookmarks (useful:pattern, useful:example)
+- `decision:*` — Key architectural/design decisions
+- `handoff:update` — Sessions that modified HANDOFF.md
+- `rationale:*` — Why we chose X over Y
+- `genesis:*` — First mention of a concept
+
+When you find something useful via `siftd ask`, tag it before moving on.
+
+Before ending a session that updates HANDOFF.md:
+```
+siftd tag --last handoff:update
+```

siftd-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 siftd contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

siftd-0.1.1/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: siftd
+Version: 0.1.1
+Summary: Personal LLM usage analytics. Ingest conversation logs from CLI coding tools, query via FTS5 and semantic search.
+License-Expression: MIT
+License-File: LICENSE
+Keywords: analytics,claude,conversation,llm,search
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Documentation
+Requires-Python: >=3.12
+Requires-Dist: httpx
+Requires-Dist: loguru
+Requires-Dist: pyyaml
+Requires-Dist: tomlkit
+Requires-Dist: tqdm
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Requires-Dist: ty; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: fastembed; extra == 'embed'
+Requires-Dist: huggingface-hub; extra == 'embed'
+Requires-Dist: numpy; extra == 'embed'
+Requires-Dist: onnxruntime; extra == 'embed'
+Requires-Dist: tokenizers; extra == 'embed'
+Description-Content-Type: text/markdown
+
+# siftd
+
+Ingest and query conversation logs from LLM coding tools. Stores in SQLite, searches via FTS5 and embeddings.
+
+Warning: This project is under active development and breaking changes may occur.
+
+## Install
+
+```bash
+pip install siftd              # core (query, tags, ingest)
+pip install siftd[embed]       # with semantic search
+```
+
+## Usage
+
+```bash
+# Ingest logs from Claude Code, Gemini CLI, Codex, Aider
+siftd ingest
+
+# List recent conversations
+siftd query -w .               # current workspace
+siftd query --since 7d         # last week
+
+# Semantic search (requires [embed])
+siftd ask "how did I handle auth"
+siftd ask -w myproject "error handling"
+
+# Tag and filter
+siftd tag 01JGK3 decision:auth
+siftd query -l decision:
+```
+
+## Supported Tools
+
+- Claude Code
+- Gemini CLI
+- Codex CLI
+- Aider
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `ingest` | Import conversation logs |
+| `query` | List/filter conversations |
+| `ask` | Semantic search |
+| `tag` | Apply tags to conversations |
+| `peek` | View conversation contents |
+| `doctor` | Check configuration |
+
+## Documentation
+
+- [CLI Reference](docs/cli.md)
+
+## License
+
+MIT

siftd-0.1.1/README.md

@@ -0,0 +1,57 @@
+# siftd
+
+Ingest and query conversation logs from LLM coding tools. Stores in SQLite, searches via FTS5 and embeddings.
+
+Warning: This project is under active development and breaking changes may occur.
+
+## Install
+
+```bash
+pip install siftd              # core (query, tags, ingest)
+pip install siftd[embed]       # with semantic search
+```
+
+## Usage
+
+```bash
+# Ingest logs from Claude Code, Gemini CLI, Codex, Aider
+siftd ingest
+
+# List recent conversations
+siftd query -w .               # current workspace
+siftd query --since 7d         # last week
+
+# Semantic search (requires [embed])
+siftd ask "how did I handle auth"
+siftd ask -w myproject "error handling"
+
+# Tag and filter
+siftd tag 01JGK3 decision:auth
+siftd query -l decision:
+```
+
+## Supported Tools
+
+- Claude Code
+- Gemini CLI
+- Codex CLI
+- Aider
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `ingest` | Import conversation logs |
+| `query` | List/filter conversations |
+| `ask` | Semantic search |
+| `tag` | Apply tags to conversations |
+| `peek` | View conversation contents |
+| `doctor` | Check configuration |
+
+## Documentation
+
+- [CLI Reference](docs/cli.md)
+
+## License
+
+MIT

siftd-0.1.1/bench/build.py

@@ -0,0 +1,122 @@
+"""Build an embeddings database from a strategy file.
+
+Usage:
+    python bench/build.py --strategy bench/strategies/exchange-window.json
+    python bench/build.py --strategy bench/strategies/exchange-window.json --output /tmp/test.db
+"""
+
+import argparse
+import json
+import sqlite3
+import sys
+from datetime import datetime
+from pathlib import Path
+
+# bench/ is not a package — add src/ to path so siftd imports work
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from siftd.embeddings.fastembed_backend import FastEmbedBackend
+from siftd.paths import data_dir
+from siftd.storage.embeddings import open_embeddings_db, store_chunk, set_meta
+
+
+def extract_chunks(main_conn: sqlite3.Connection, params: dict) -> list[dict]:
+    """Extract chunks from main DB using the exchange-window chunker."""
+    from fastembed import TextEmbedding
+    from siftd.embeddings.chunker import extract_exchange_window_chunks
+
+    target_tokens = params.get("target_tokens", 256)
+    max_tokens = params.get("max_tokens", 512)
+    overlap_tokens = params.get("overlap_tokens", 25)
+
+    emb = TextEmbedding("BAAI/bge-small-en-v1.5")
+    tokenizer = emb.model.tokenizer
+
+    return extract_exchange_window_chunks(
+        main_conn,
+        tokenizer,
+        target_tokens=target_tokens,
+        max_tokens=max_tokens,
+        overlap_tokens=overlap_tokens,
+    )
+
+
+def build(strategy_path: Path, output_path: Path, db_path: Path) -> None:
+    """Build embeddings DB from strategy."""
+    strategy = json.loads(strategy_path.read_text())
+    params = strategy["params"]
+
+    # Extract chunks from main DB
+    main_conn = sqlite3.connect(db_path)
+    chunks = extract_chunks(main_conn, params)
+    main_conn.close()
+
+    if not chunks:
+        print("No chunks extracted. Check strategy params and main DB.")
+        return
+
+    print(f"Extracted {len(chunks)} chunks")
+
+    # Embed in batches
+    backend = FastEmbedBackend()
+    batch_size = 64
+    all_embeddings: list[list[float]] = []
+
+    for i in range(0, len(chunks), batch_size):
+        batch_texts = [c["text"] for c in chunks[i : i + batch_size]]
+        batch_embeddings = backend.embed(batch_texts)
+        all_embeddings.extend(batch_embeddings)
+        print(f"  Embedded batch {i // batch_size + 1}/{(len(chunks) + batch_size - 1) // batch_size}")
+
+    # Store in embeddings DB
+    embed_conn = open_embeddings_db(output_path)
+    set_meta(embed_conn, "backend", backend.model)
+    set_meta(embed_conn, "dimension", str(backend.dimension))
+
+    for chunk, embedding in zip(chunks, all_embeddings):
+        store_chunk(
+            embed_conn,
+            chunk["conversation_id"],
+            chunk["chunk_type"],
+            chunk["text"],
+            embedding,
+            token_count=chunk.get("token_count"),
+        )
+
+    embed_conn.commit()
+    embed_conn.close()
+
+    print(f"Built {len(chunks)} chunks → {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build embeddings DB from a strategy file")
+    parser.add_argument("--strategy", type=Path, required=True, help="Path to strategy JSON file")
+    parser.add_argument("--output", type=Path, default=None, help="Output embeddings DB path")
+    parser.add_argument("--db", type=Path, default=None, help="Path to main siftd.db")
+    args = parser.parse_args()
+
+    if not args.strategy.exists():
+        print(f"Strategy file not found: {args.strategy}")
+        sys.exit(1)
+
+    # Resolve main DB path
+    db = args.db or (data_dir() / "siftd.db")
+    if not db.exists():
+        print(f"Main DB not found: {db}")
+        sys.exit(1)
+
+    # Resolve output path
+    if args.output:
+        output = args.output
+    else:
+        strategy = json.loads(args.strategy.read_text())
+        name = strategy.get("name", args.strategy.stem)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        output = data_dir() / f"embeddings_{name}_{timestamp}.db"
+
+    build(args.strategy, output, db)
+
+
+if __name__ == "__main__":
+    main()