docsif 1.0.0__tar.gz → 1.1.2__tar.gz

docsif-1.1.2/.github/workflows/publish.yml

@@ -0,0 +1,89 @@
+name: Publish
+
+on:
+  push:
+    tags: ['v*']
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build distributions
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          python-version: '3.12'
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Verify tag matches src/sif/_version.py
+        run: |
+          VERSION="$(python3 -c "import runpy; print(runpy.run_path('src/sif/_version.py')['__version__'])")"
+          test "v${VERSION}" = "${GITHUB_REF_NAME}" || {
+            echo "tag ${GITHUB_REF_NAME} does not match src/sif/_version.py version ${VERSION}"
+            exit 1
+          }
+
+      - name: Build
+        run: uv build
+
+      - name: Check metadata
+        run: uvx twine check dist/*
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/docsif
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: publish-testpypi
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/docsif
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

docsif-1.1.2/.github/workflows/test-matrix.yml

@@ -0,0 +1,74 @@
+name: Test Matrix
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: py${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+          cache-suffix: ${{ matrix.python-version }}
+
+      - name: Install Python
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: uv sync --frozen --extra dev --extra embed
+
+      - name: Run full test suite
+        run: env -u FORCE_COLOR NO_COLOR=1 uv run --frozen pytest
+
+  lint:
+    name: lint / py3.12 / ubuntu-latest
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          python-version: '3.12'
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Sync dependencies
+        run: uv sync --frozen --extra dev
+
+      - name: Ruff check
+        run: uv run --frozen ruff check src tests
+
+      - name: Ruff format check
+        run: uv run --frozen ruff format --check src tests
+
+      - name: Type check
+        run: uv run --frozen ty check src/sif

docsif-1.1.2/.planning/.continue-here.md

@@ -0,0 +1,69 @@
+---
+context: default
+phase: milestone-v1.1
+task: —
+total_tasks: —
+status: complete
+last_updated: 2026-11-12T10:50:56.449Z
+---
+
+# Pause Handoff — v1.1 shipped, no active phase
+
+## Critical Anti-Patterns
+
+| Pattern | Description | Severity | Prevention Mechanism |
+|---------|-------------|----------|---------------------|
+| Doc edits re-stale verifications | Editing a covered file (benchmarking.md, UAT) after a verifier ran flips the phase to `stale`, forcing another verifier pass | advisory | Batch all docs/UAT edits BEFORE the final per-phase verification, then make no covered-file edits |
+| In-memory→file SQLite backup with vec0 hangs | Backing up a `:memory:` DB containing a sqlite-vec virtual table blocks indefinitely | advisory | Build file-backed DBs directly, or evaluate in-process (mirror test_bench_snapshot) rather than backing up |
+| sentence-transformers MPS/joblib deadlock | Per-doc `model.encode()` in a script deadlocks on macOS (loky semaphore) | advisory | Precompute embeddings in ONE batch call, single-thread (`OMP/MKL=1`, `torch.set_num_threads(1)`), explicit `device="cpu"` |
+
+_No blocking constraints this session._
+
+<current_state>
+No active phase or milestone work. v1.1 "Live Index and Search Quality" is COMPLETE and SHIPPED (2026-11-12), archived to `.planning/milestones/`, tagged `v1.1`, and pushed (`dev` @ a5a89ce + tag). Working tree is clean except 13 pre-existing scratch files (research cache, tmp, tooling JSON).
+</current_state>
+
+<completed_work>
+
+Completed this session (all committed + pushed):
+- Re-verified all 7 phases (11–16.1) at HEAD with fresh fingerprints — all passed
+- Fixed re-review WR-01: `clear_expired()` batched deletion under SQLite's variable limit (`aa520be`), 1000-row regression pins the 999 limit
+- validate-phase 12 + 16.1; secure-phase 16 + 16.1 (SECURITY.md verified, threats_open 0)
+- Real-model held-out bench run (Qwen3-Embedding-0.6B, nDCG@10 0.9233, Hit@10 1.0, p50/p95 94.3/117.6ms), recorded in docs/benchmarking.md; WINDOWS id 10 closed
+- Milestone v1.1 archived (7 phases, 27 plans, 67 tasks), ROADMAP collapsed, REQUIREMENTS removed, PROJECT evolved, retrospective written, tag pushed
+</completed_work>
+
+<remaining_work>
+
+- None in v1.1. Next milestone is undefined — start with `/gsd-new-milestone` (phase numbering continues at 17).
+</remaining_work>
+
+<decisions_made>
+
+- Fixed (not deferred) the WR-01 cache-cleanup regression — milestone-introduced, small safe fix
+- Used in-process batched single-threaded evaluation for the real-model bench (MPS deadlock + backup hang)
+- Corrected 16-UAT status to canonical `complete` (scanner terminal state; evidence-backed)
+- Deferred WR-02 (Python 3.10 `setlimit` test-reliability) as a documented follow-up — production fix is clean
+</decisions_made>
+
+<blockers>
+None.
+</blockers>
+
+## Required Reading (in order)
+1. `.planning/MILESTONES.md` — v1.0 + v1.1 history
+2. `.planning/PROJECT.md` — current state, validated requirements, key decisions
+3. `.planning/RETROSPECTIVE.md` — v1.1 what-worked / lessons
+
+## Infrastructure State
+- Branch `dev`, synced with `origin/dev` @ a5a89ce; tag `v1.1` pushed
+- Real Qwen3-Embedding-0.6B cached at `~/Library/Caches/sif/` and `~/.sif/models/`
+- No running watchers/daemons; no async external jobs
+
+<context>
+The v1.1 milestone close is fully finished. The session's arc: finish the audit's tech_debt items (16.1 phase + retroactive validate/secure runs), re-verify every phase at HEAD, fix a real cleanup bug found during re-review, produce genuine real-model evidence to make the UAT truthful, then archive/tag/push. Nothing is mid-flight.
+</context>
+
+<next_action>
+Start with: `/clear`, then `/gsd-new-milestone` to define the next milestone. Optionally review `.planning/RETROSPECTIVE.md` for v1.1 lessons first.
+</next_action>

docsif-1.1.2/.planning/HANDOFF.json

@@ -0,0 +1,45 @@
+{
+  "version": "1.0",
+  "timestamp": "2026-11-12T10:50:56.449Z",
+  "phase": "milestone-v1.1",
+  "phase_name": "v1.1 Live Index and Search Quality (SHIPPED)",
+  "phase_dir": ".planning/milestones/v1.1-phases",
+  "plan": null,
+  "task": null,
+  "total_tasks": null,
+  "status": "complete",
+  "completed_tasks": [
+    {"id": "v1.1-reverify", "name": "Re-verified all 7 phases (11-16.1) at HEAD with fresh fingerprints", "status": "done", "commit": "dff344b..cf23b4d"},
+    {"id": "wr-01", "name": "Fixed cache clear_expired SQLite variable-limit bug (batched deletion, 1000-row regression)", "status": "done", "commit": "aa520be"},
+    {"id": "coverage", "name": "validate-phase 12 + 16.1; secure-phase 16 + 16.1 (SECURITY.md verified, threats_open 0)", "status": "done", "commit": "5a70a23,ff87985,951ec28,0899d90"},
+    {"id": "realmodel", "name": "Ran real-model held-out bench (nDCG@10 0.9233, Hit@10 1.0); recorded in docs/benchmarking.md; WINDOWS id 10 closed", "status": "done", "commit": "5d6d524"},
+    {"id": "milestone-close", "name": "Archived v1.1 milestone (7 phases, 27 plans, 67 tasks); ROADMAP collapsed; REQUIREMENTS removed; PROJECT evolved; retrospective written; tag v1.1; pushed", "status": "done", "commit": "ed63fca..a5a89ce"}
+  ],
+  "remaining_tasks": [],
+  "blockers": [],
+  "async_jobs": [],
+  "human_actions_pending": [],
+  "decisions": [
+    {"decision": "Fix the re-review WR-01 cache-cleanup bug now rather than defer", "rationale": "Milestone-introduced regression that aborts cleanup at ~1000 expired rows; small safe batched fix", "phase": "11-reverify"},
+    {"decision": "Use in-process batched single-threaded evaluation for the real-model bench instead of the CLI bench", "rationale": "MPS/joblib deadlock + cache-dir mismatch + in-memory-to-file vec0 backup hang; in-process mirrors test_bench_snapshot", "phase": "16"},
+    {"decision": "Corrected 16-UAT status from 'passed' to canonical 'complete'", "rationale": "audit-open scanner treats only 'complete' as terminal; UAT genuinely complete (2/2, evidence-backed)", "phase": "16"},
+    {"decision": "Deferred WR-02 test-reliability (Python 3.10 setlimit) as documented follow-up", "rationale": "Production fix clean; test-only concern; avoids re-stale loop", "phase": "11-reverify"}
+  ],
+  "uncommitted_files": [
+    "?? .planning/agent-history.json",
+    "?? .planning/research/.cache/05bfeb8fe0b053d9433d563826d721818e4c527a92e3db6c0efc155dbba51a95.json",
+    "?? .planning/research/.cache/07ca94279fedeea49c0ac97e8d9f09661f8f3746b72d8cbfb367134b999b2b8e.json",
+    "?? .planning/research/.cache/1cd113f8f9ed602c2ad81ced9a84d243e1db92f290152d8c68bf546c0df035f6.json",
+    "?? .planning/research/.cache/503959ddb0808d08e478038f9779ead6885e902fd899e1474b85e8557c1fdd8d.json",
+    "?? .planning/research/.cache/9248b3705cde6b4b9fd537e3a1e7bbe830fd00b533b7ce3bbb925c9c0b037735.json",
+    "?? .planning/research/.cache/9eb3e31fe811e10f52548dac1e5d45dc6ead1d5bd28be6ca2b24a2a9fe312bab.json",
+    "?? .planning/research/.cache/c1d1cd7e6ef0c3666c66542508d3a8bcb53732e4aeaaaadfaa30ca99b673214b.json",
+    "?? .planning/research/.cache/d0a0228a1c2905610c5a9a20e053796bd36d343c4998ac18321598c2cf46624a.json",
+    "?? .planning/research/.cache/e76ba5ee87c72924e3109896fdb9cf6c7483c3f5ace5fc5a6aa5669acab0eeb6.json",
+    "?? .planning/research/.cache/faa1d584d63bb5ea6191d7b32f019088afb9c839d307e2c10db0778282fa88c9.json",
+    "?? .planning/state.json",
+    "?? .planning/tmp/"
+  ],
+  "next_action": "Run /gsd-new-milestone to define the next milestone (phase numbering continues at 17). v1.1 is complete, archived, tagged, and pushed.",
+  "context_notes": "No active phase or milestone work in progress. v1.1 shipped 2026-11-12 (7 phases incl. inserted 16.1 tech-debt, 27 plans, 21/21 requirements). All retroactive coverage runs done; the audit's tech_debt verdict is fully addressed. The 13 untracked files are pre-existing scratch (research cache, tmp, tooling JSON) — intentionally not committed."
+}

docsif-1.1.2/.planning/MILESTONES.md

@@ -0,0 +1,86 @@
+# Milestones
+
+## v1.1 Live Index and Search Quality (Shipped: 2026-11-12)
+
+**Phases completed:** 7 phases, 27 plans, 67 tasks
+
+**Key accomplishments:**
+- One shared SQLite connection factory (busy_timeout→WAL→synchronous + implicit BEGIN IMMEDIATE) for both CLI and MCP paths, plus kernel-flock per-DB single-writer arbitration proven end-to-end with deterministic two-writer subprocess tests
+- Per-file incremental indexing core (IncrementalIndexer) extracted from update_cmd into one shared presentation-free loop with per-file BEGIN IMMEDIATE transactions (D-04), proven golden-equivalent on index state and CLI output (SC3)
+- Dead third copy of the indexing loop retired (DocumentIndexer module + ConnectionPool + duplicate DatabaseConnection, D-07) and embed_cmd brought under the same per-DB flock as update_cmd (D-01), closing the phase with the full quality gate green (SC4)
+- End-to-end `sif watch` slice: observer -> EventDebouncer -> fold -> worker -> per-burst flock -> update_paths, proven by a real-subprocess e2e test plus the shared-predicate, missing-root-guard, and watchdog-cap landings
+- Complete foreground host around the 12-01 pipeline: tiered --embed depth with embedding computed outside any write transaction, signal-driven shutdown with WAL truncate checkpoint, repeatable -c filtering, and missing-root recovery — all unit-proven with machine-verified TDD gates
+- Real-FS e2e battery proving every Phase 12 SC against the real CLI subprocess (edit/delete, os.replace atomic save, temp-noise exclusion, missing-root recovery, SIGINT/SIGTERM exit 0 with a demonstrated 0-byte WAL), plus generator-only docs regeneration that turns the full quality gate green with nothing deselected
+- FileWatcher grown into the long-lived-host substrate: live rescope of the watched collection set (exact-set diff, reconcile-on-add, documents preserved on removal), idle PRAGMA wal_checkpoint(TRUNCATE) riding the tick with busy/recent-burst skips, and ENOSPC schedule failures that warn once with the inotify remedy — plus the mcp_watch/rescope/checkpoint settings fields.
+- `sif watch --detach` backgrounds the proven Phase 12 host through a stdlib double-fork with a deterministic pipe handshake (PID reported only once armed), `--stop` corroborates identity via ps argv markers before SIGTERM→bounded-wait→SIGKILL escalation, and `sif status` gains a Watch row — with the SC4 SIGKILL/re-detach proof that kernel-released flock leaves no stale lock.
+- The proven FileWatcher now lives inside the MCP server under SIF_MCP_WATCH=1 — one shared watch host (BM25-only depth, logger-only output, serve-first queued reconcile, rescope) wired around the stdio serve loop and into the HTTP transport via a FastAPI lifespan, proven by a real stdio subprocess e2e that answers queries while indexing a live burst with a 100% JSON-clean stdout.
+- `sif watch --install-service/--uninstall-service` generates a launchd LaunchAgent or systemd user unit that runs the FOREGROUND watcher under the OS supervisor with the install-time `-c/--embed` flags baked into an absolute-interpreter argv (SC3, never forks), installs it with explicit-argv manager calls (bootstrap/bootout-retry or daemon-reload/enable --now, verified, friendly missing-binary errors), and the generator-owned docs are regenerated to tell the whole LIVE-09 story — with a real launchd round-trip proven live twice on this machine.
+- Stat-walk drift detection (mtime vs documents) + capped BM25-only refresh in the search trio's command layer — `sif search` now finds post-index edits with no watch running, and a clean index pays zero writes, zero locks, zero notices.
+- Derived vector-staleness marking on every search surface: a corrected "no embeddings" D-08 signal computed in the shared searcher enrichment path, forwarded through MCP metadata, plus a `sif status` Vector lag row — zero schema change, zero new dependencies.
+- The staleness marker renders on all five CLI output surfaces — dim `(stale vectors)` table annotation (no new column), CSV `vectors_stale` column, Markdown annotation, XML boolean attribute, JSON via `to_dict()` — plus the one-line stderr `sif index embed` remedy hint, so no format silently drops the signal (D-12/D-13, SC3).
+- Weighted RRF fusion (weights/k in multiply form, byte-identical at defaults) plumbed end-to-end: `sif query` flags, SIF_ settings, and MCP query params all resolve into SearchOptions and reach RRFFusion.fuse — with golden tests pinning repr-exact default output on a real-DB fixture.
+- FTS5 vocab shadow + SuggestionEngine (prefix completions, per-word DYM rewrites with an SC2 pin, frequency×recency history) + capped aware-UTC query_log storage — the one engine three hosts wire in 15-03.
+- `sif suggest`'s three-section table/JSON surface, zero-result did-you-mean across all three CLI searches and the MCP query tool (stdout-pure, conditional JSON object), and fail-soft query-history writes with opt-out and a single clear path — phase gate green at 94.85% coverage.
+- nDCG@10/Hit@10 with per-category breakdown, warmed p50/p95 latency, and a settings-resolved baseline layered onto `SearchEvaluator`/`sif bench` — legacy flat metrics byte-compatible
+- SIF_RERANK_ENABLED (default false) + --rerank make rerank explicit; SIF_RERANK_CANDIDATE_LIMIT (5-50) and SIF_RERANK_RELATIVE_THRESHOLD ((0,1]) are Pydantic-validated; the shipped -C flag resolves through the shared `resolve_candidate_limit` clamp
+- Deterministic ConceptEmbedder + direct-DB builder, and a committed 150-doc synthetic corpus with 56 graded queries (36 tuning / 20 held-out) regenerated byte-identically by a seeded drift-gated generator — proven retrievable and CLI-joined with non-zero nDCG@10
+- 45-config declared-grid sweep with the rounded-delta 0.01 tie rule (tuning-only, embedding-once, settings baseline with exact-zero deltas) plus bench rerank off/on states strictly additive to the pinned flat --json contract
+- Shipped defaults locked to a committed held-out snapshot (nDCG@10 0.8120 reproducing exactly, sweep winner's +0.0066 delta honestly below the 0.01 bar -> verdict "unchanged") with the full measurement system documented in docs-validated docs/benchmarking.md
+- Class-injected `open_initialized_database` boundary converts foreseeable DB failures to remedy-bearing Click errors end-to-end on index update/embed, plus constructor-only sqlite-vec guards on vsearch/query and suggest opener adoption — 7 of 30 convention sites, tracer-proven before Plan 02's sweep.
+- All 30 CLI DB open/schema sites now route through the Plan 01 shared boundary — 23 converted here (collection/context/get/ls/bench/watch/main), the daemon arm() site via the unmanaged variant, status/cleanup handlers narrowed to D-04 known classes with pure stdout, and an AST inventory gate that structurally forbids future direct init_schema in CLI code.
+- LLM cache get() now compares parsed aware-UTC instants through one shared expiry parser (legacy naive-local rows read correctly east AND west of UTC), and every embedding-cache connection comes from the Phase 11 factory — 14-WR-03 read side and W-1 closed with zero migration and byte-identical fail-soft fallbacks.
+- FileWatcher.enqueue_reconcile(collections: Iterable[Collection]) — narrow public reconcile-intent API replacing the MCP host's private-queue reach-in, with all three reconcile producers routed and serve-first ordering pinned at unit and protocol level
+- Closed 11-WR-03 and 14-WR-02/03/04 with plan-and-test Source evidence, preserved 15-IN-06 as an explicit partial closure, and re-proved the entire phase with 1586 passing tests, 95.17% coverage, clean static gates, and byte-identical generated docs.
+
+---
+
+## v1.0 MVP — qmd 功能对等的本地混合搜索引擎 (Shipped: 2026-11-07)
+
+**Phases completed:** 11 phases, 68 plans, 112 tasks
+
+**Key accomplishments:**
+- Added all missing runtime dependencies to pyproject.toml so `pip install -e .` succeeds and the CLI launches without ModuleNotFoundError.
+- Converted FTS5 virtual tables to external content tables with SQLite triggers, removed stale sqlite_repository.py, and fixed BM5 rowid joins.
+- Fixed inverted checksum logic in index update and removed hardcoded embedding model paths, aligning CLI and embedder defaults with Settings
+- Replaced random/placeholder embedding implementations in the factory with real `sentence-transformers` and `llama-cpp-python` loaders, and unified the `ModelType` enum across modules.
+- Made sqlite-vec a hard requirement for vector search, removed all Python brute-force fallbacks, and rewrote the vsearch CLI to fail fast with clear errors.
+- Implemented `multi-get` command with auto-detection for comma-separated IDs, glob patterns, and single docid/path fallback, plus full unit test coverage.
+- Top-level `docsift ls` command displaying indexed documents as a virtual file tree, with optional collection and subpath filtering
+- Collection update-cmd, include/exclude subcommands, and pre-index shell hook with fail-fast subprocess execution
+- Wired `--all` flag into `search`, `query`, and `vsearch` CLI commands so default searches respect `include_by_default` collection setting.
+- Implemented OpenAIEmbedder with dimension auto-detection, added MODELSCOPE to ModelType, refactored the factory to return Embedder protocol instances, and created unit tests for all backends.
+- Refactored EmbeddingManager to work with the Embedder protocol and added comprehensive unit tests.
+- OpenAIEmbedder(Embedder) with generic OpenAI-compatible endpoint support (api_base → {api_base}/embeddings), per-model dimension auto-detection cached to openai_dim_cache.json with a 7-day TTL and fail-fast SIF_EMBEDDING_DIM mismatch validation, wired through EmbeddingModelFactory — closing the single 03-VERIFICATION VEC-01 gap and delivering both missing Wave-0 test files.
+- Idempotent `sif index embed`: delete-before-insert replaces a re-chunked document's vec0 rows, and a per-document exact-set check skips complete documents unless `--force` — closing UAT gap G-03-3 where 7 runs on a 3-chunk collection left 21 searchable embedding rows returning the document 7 times.
+- Complete
+- QueryExpansion with embedding-based pseudo-relevance feedback returning list[str], and SmartSnippetExtractor scoring sentences by weighted query term frequency.
+- Snippet column in query/search CLI tables fed by transient content fetch and prefix-route snippet coverage.
+- Unified contexts table with SAVEPOINT atomic migration, ContextRepository with context_type filtering, and SearchResult.context_description field preserved through RRF fusion and cross-encoder reranking
+- Attach path context descriptions to all search results via batch query across BM25, vector, hybrid, and pipeline searchers.
+- Shared `_collapse_duplicate_path_rows` helper collapses every same-file spelling into one canonical row on ALL re-add paths (truth 21), and `context list` escapes all Rich table cells so markup-bearing content renders literally (CR-02).
+- escape() on every user-derived string crossing Rich in the search/vsearch/query tables and both explain prints — markup-bearing queries, titles, collection names, and content render literally instead of crashing with rich.errors.MarkupError
+- Bracket-named context targets are now addable — rich.markup.escape at the three confirmation prints (add create, add update, remove) stops the MarkupError that rolled back writes inside the transaction, locked by three machine-validated RED-first regression tests.
+- Auto-generated CLI reference via Click introspection, replacing stale manual documentation with accurate command/argument/option tables for all 32 commands.
+- Auto-generated configuration reference from Pydantic Settings introspection, eliminating 6 phantom fields and documenting 25 real settings with Qwen3 defaults.
+- Complete rewrite of docs/quickstart.md to match current CLI signatures, replacing all broken examples (collection create, add-path, update, search similar, mcp start) with working commands verified against installed package
+- Updated README.md and docs/index.md to reflect current v1.0 features, Qwen3 default models, and correct CLI examples while removing "planned" labels from already-implemented features.
+- Updated four technical docs and created an architecture diagram generator to reflect current implementation after Phases 1-5
+- Renamed Python package directory from `src/docsift/` to `src/sif/` and updated all `pyproject.toml` metadata, scripts, URLs, and tool configs.
+- SIF branding applied to all core constants, package metadata, and CLI entry point with automatic model cache migration from old docsift paths
+- Bulk mechanical rename of all internal imports and string references from `docsift` to `sif` across 54 source files, with zero remaining `from docsift` imports and full compile/import verification.
+- Updated Pydantic Settings env_prefix from "DOCSIFT_" to "SIF_", changed default DB path to "sif.db", and verified all filesystem paths resolve under sif/ directories.
+- All 31 test files migrated from `docsift` to `sif` imports, patch targets, and command references with pytest collecting 388 tests successfully
+- Full quality suite verification of DocSift-to-SIF rename with 377 tests passing, zero rename-related errors, and consistent SIF branding.
+- Deleted the D-01 dead-code cluster and re-baselined coverage from 78% to 81.48% (5320 statements, 985 missed), with 690 tests passing and live CLI imports intact.
+- Raised full-suite line coverage from 81.48% to 85.34% with real-SQLite CLI tests: collection 99.15%, main 86.99%, get 100%, and all D-06 order probes green.
+- Raised full-suite line coverage from 85.34% to 88.57% with deterministic reranker tests and 64 search CLI tests; rerank.py reached 98.60%, search.py 99.44%, and XML now safely escapes machine output.
+- Raised full-suite coverage from 88.57% to 90.89% with real-SQLite database/cache tests; repositories, database, schema, and cache reached 100%, migrations reached 92%, and real SQL exposed and proved fixes for stale FTS rows, lazy stats initialization, and broken migration scripts.
+- Raised full-suite coverage from 90.89% to 93.48% with 78 new behavior tests; all seven target files reached 100%, ModelScope/network/home boundaries remained isolated, and the tests exposed and proved fixes for custom logging formatter and indeterminate Rich progress defects.
+- Raised full-suite coverage from 93.48% to 96.16% with 83 new behavior tests, reached 100% parser/scanner/server/stdio coverage, and fixed chunker non-termination plus MCP session-limit defects exposed by the new edge cases.
+- Rehearsed the 90% gate at TOTAL 96.16% with zero long-tail additions and landed `[tool.coverage.report] fail_under = 90` as the single pyproject coverage config change — the canonical suite now exits 0 only when src/sif line coverage stays at or above 90%.
+- Symlink-alias paths now resolve to indexed documents across sif get/multi-get and MCP get_document via normalize_path-wrapped lookups, get.py owns its Database lifecycle, and the index-vs-disk fallback boundary is documented in the generated CLI reference.
+- Generalized legacy-FTS-trigger detector migrating direct-DELETE triggers to 'delete'-command form with a detection-gated 'rebuild' that purges stale MATCHable entries on the same open
+- W3C XML 1.0 Char-production sanitizer stripping forbidden code points from every `--xml`-bound string, plus context-manager Database ownership for search/query/vsearch with closure regression locks
+- Context-manager Database ownership at all 21 remaining CLI construction sites (6 modules) plus a TrackingDatabase closure harness asserting success- and exception-path closure
+
+---

docsif-1.1.2/.planning/PROJECT.md

@@ -0,0 +1,193 @@
+# DocSift
+
+## What This Is
+
+DocSift 是 [qmd](https://github.com/tobi/qmd) 的 Python 重构版本，一个面向个人知识库的**本地混合搜索引擎**。它索引 Markdown 笔记、文档和会议纪要，支持关键词搜索（BM25）、语义向量搜索（Vector Search），以及基于 LLM 的重排序（Rerank）和查询扩展。同时提供 CLI 工具和 MCP（Model Context Protocol）服务器两种使用方式，完全在本地运行，无需外部 API。
+
+## Core Value
+
+用户可以在自己的笔记和文档库中，用自然语言快速、准确地找到需要的信息——无论关键词是否匹配。
+
+## Current Milestone: v1.2 Search Precision and Agent Experience
+
+**Goal:** 让搜索精准可控（元数据过滤 + 正确性修复）、让 MCP 达到 2026 生态就绪（SDK v2 + agent 包装）、让质量提升有 held-out 证据（个人语料评测 + Harrier 评估），并补齐分发与 CI 缺口。
+
+**Target features:**
+
+*搜索精准化*
+- 元数据过滤 AST：path/mtime/tag/frontmatter 键，RRF 前应用，CLI + MCP 共享（qmd 未发布旗舰 parity）
+- `metadata` 发现工具（键/类型/值统计）
+- 正确性修复：parser 分派接线、向量按文档去重、hybrid offset 切片、标题入 FTS
+- 规模：消除每查询全表扫描（staleness/contexts）、`sif index vacuum` + backup、embed 流式分批
+
+*MCP 现代化与 Agent 体验*
+- 官方 Python SDK v2（MCPServer）升级，双时代（2025/2026）客户端服务
+- 工具注解（readOnlyHint/idempotentHint）+ structuredContent/outputSchema
+- MCP query 补齐 rerank/expand/snippet pipeline（与 CLI 对等）、懒加载 embedder、lex/vec 工具接线
+- Host 头校验（Origin 已有）、`sif mcp daemon` stub 收口
+- Claude Code skill/plugin + MCP Registry 上架
+
+*质量可证*
+- 个人语料 known-item 评测仪器（`sif eval --personal`，免人工标注）
+- Harrier-OSS-v1-0.6b EN+ZH held-out 评估（胜出才换默认；embedder 身份标记防同维陷阱）
+- rapidfuzz 测量替换 difflib、chunk size/overlap 扫描 + 标题面包屑实验
+
+*工程与分发*
+- 测试 CI 矩阵（py3.10–3.13 × ubuntu/macos）+ Windows smoke（hc remote yoga）
+- 发布 v1.1.x 到 PyPI（trusted publishing）+ Makefile/extras/pre-commit 清理
+- `sif models pull/status` + 下载器 bug 修复 + 非 TTY 静默；安装文档重写（uv 优先）
+
+## Milestone v1.1: Live Index and Search Quality（已发布 2026-11-12）
+
+**Goal:** 让索引从手动 `index update` 变为自动常新（Live Index），并让搜索质量可配置、可量化、持续可调（Search Quality）。
+
+**Target features:**
+
+*Live Index（四种形态，依赖递进）*
+- `sif watch` 前台命令：文件变更 → 增量重索引（watchdog 地基已在 `src/sif/indexing/watcher.py`，未接线）
+- MCP server 内嵌刷新：watch 与 MCP 同进程，AI 助手查询时索引常新
+- 后台守护进程：`--detach` / launchd / systemd 常驻
+- 命令前自动检测：未开 watch 时，搜索命令前轻量增量兜底
+
+*更新深度可配置*
+- 默认仅 BM25/FTS 增量（快）；`--embed` 选项开启向量自动更新
+
+*Search Quality*
+- 混合权重可配置（BM25 / 向量 / RRF k 暴露为设置）
+- Query suggestions（索引词项/历史 did-you-mean）
+- Reranker 调优（Qwen3-Reranker 阈值/延迟）
+- bench 基准集扩充（为全部调优提供量化依据）
+
+## Requirements
+
+### Validated
+
+- ✓ 集合（Collection）的增删改查 — 现有
+- ✓ 基于文件扫描的文档索引（含 checksum 去重） — 现有
+- ✓ SQLite FTS5 全文搜索（BM25） — 现有
+- ✓ 基础混合搜索（BM25 + Vector + RRF） — 现有
+- ✓ MCP 服务器骨架（stdio + HTTP 双传输） — 现有
+- ✓ 多格式输出（rich table / JSON / CSV / Markdown / XML） — 现有
+- ✓ 文档片段（chunk）和嵌入存储 — 现有
+- ✓ 修复现有 bug 和占位实现（stub）：硬编码模型路径、随机 embedding fallback、MCP 连接安全等 — Validated in Phase 01
+- ✓ 依赖声明完整性：补全 `pyproject.toml` 中缺失的运行时依赖 — Validated in Phase 01
+- ✓ 补齐缺失的 CLI 命令：multi-get、ls、update-cmd、pull、bench — Validated in Phase 02/04
+- ✓ 嵌入模型灵活可配置：本地 GGUF（llama-cpp-python）、sentence-transformers、ModelScope，及可选 OpenAI 兼容 API（默认 Qwen3 系列） — Validated in Phase 03
+- ✓ LLM 重排序（Qwen3-Reranker）和查询扩展（SearchPipeline），`query` 搜索质量达标 — Validated in Phase 04/05
+- ✓ 统一 MCP 服务器实现（`src/sif/mcp/`，stdio + Streamable HTTP） — Validated in Phase 09
+- ✓ 上下文描述：path/collection/global 三种作用域，context CLI 全命令集，搜索结果自动附加描述 — Validated in Phase 05（2026-11-06 重新验证 24/24 通过，第五次陈旧验证再生）
+- ✓ 测试覆盖率常驻门禁：TOTAL 96.22%（1011 项测试），`[tool.coverage.report] fail_under = 90` 落地且无任何豁免行 — Validated in Phase 10（2026-11-06 重新验证 35/35 通过，fresh fingerprint）
+- ✓ 搜索质量：bench 基准集扩充（nDCG@10/Hit@10/p50-p95、56 条分类查询 36/20 分割、45 配置扫描、held-out 默认值锁定，诚实结论：默认值不变） — Validated in Phase 16
+- ✓ 搜索质量：Reranker 调优（显式 opt-in `SIF_RERANK_ENABLED`/`--rerank`、候选上限 5-50 夹取、相对阈值 α=0.5、over-fetch 让上限真正生效） — Validated in Phase 16
+- ✓ 技术债收口：symlink/别名路径经 `get`/`multi-get`/MCP `get_document` 解析到已索引文档（DEBT-PN）；旧版 FTS 触发器迁移 + 检测门控 rebuild 清除陈旧索引（CR-01）；`--xml` 输出恒为良构 XML（CR-02）；全部 26 处 CLI Database 构造点上下文管理器持有、异常路径确定性关闭（WR-01） — Validated in Phase 10.1（2026-11-06 重新验证 16/16 通过，fresh fingerprint）
+
+### Validated（v1.1 — shipped 2026-11-12）
+
+- ✓ Live Index：`sif watch` 前台命令（增量重索引 + 防抖） — Validated in Phase 12
+- ✓ Live Index：更新深度可配置（默认 BM25/FTS，`--embed` 开启向量自动更新） — Validated in Phase 12（与 MCP 内嵌刷新项合并验证 LIVE-05 深度矩阵）
+- ✓ Live Index：搜索命令前自动增量检测兜底（LIVE-10 stat 漂移检测 + 上限 + 干净零成本） — Validated in Phase 14（2026-11-08，UAT 2/2 含 3000 文档实测 +105ms）
+- ✓ 搜索质量：分层更新深度下向量滞后结果标记 staleness（QUAL-06，五格式 + MCP + status 行） — Validated in Phase 14
+
+- ✓ Live Index：MCP server 内嵌 watch 刷新（SIF_MCP_WATCH，双传输，BM25 强制深度） — Validated in Phase 13
+- ✓ Live Index：后台守护进程（--detach / launchd / systemd，PID/stop/status + 服务安装） — Validated in Phase 13（2026-11-08 重验 40/40，UAT 5/5）
+- ✓ 搜索质量：混合权重（BM25/向量/RRF k）可配置（QUAL-01，SC1 逐字节一致 + CLI/MCP/设置全链路） — Validated in Phase 15
+- ✓ 搜索质量：Query suggestions（did-you-mean，QUAL-02/03/04，`sif suggest` 三段式 + query_log + 100ms 预算实测 ≤10ms） — Validated in Phase 15（2026-11-08 重验 30/30，UAT 5/5）
+
+### Active
+
+<!-- v1.2 高层目标；细化 REQ-ID 见 REQUIREMENTS.md（定义中） -->
+
+- [ ] 元数据过滤（path/mtime/tag/frontmatter，RRF 前）+ metadata 发现 + 分页修复
+- [ ] 搜索正确性与规模（parser 接线、去重、offset、去全表扫、标题 FTS、vacuum）
+- [ ] MCP SDK v2 现代化 + 注解/structuredContent + query pipeline 对等 + agent 包装
+- [ ] 个人语料评测仪器 + Harrier held-out 评估 + rapidfuzz/chunk 调优
+- [ ] 测试 CI + Windows smoke + PyPI 发布 + models UX + 安装文档
+
+### Out of Scope
+
+- 多用户/权限系统 — 本地单用户工具，不需要
+- 实时协作/同步 — 超出 qmd 原有范围
+- Web UI — qmd 也没有，仅 CLI + MCP
+- 非 Markdown 格式的复杂解析（如 PDF、Word、HTML、EPUB）—— 先聚焦文本和代码文件；**2026-11-13 用户再次确认不做**（竞品虽有 PDF，但维持本地文本聚焦；pypdf/trafilatura 选型研究已存档备用）
+- Windows 完整支持 — 推迟 v2（POSIX fork/systemd-everywhere/flock 降级需专门设计）；v1.2 仅 CI smoke（hc remote yoga）
+- spellfix1（FUTR-01）— ASCII-only 距离核心 + sdist-only 打包税 + 零中文收益（2026-11-13 研究结论）
+- 语义分块 / late chunking — NAACL 2025 反面证据；last-token pooling 模型机械不兼容
+- ANN 向量索引 — 100k 向量 brute force 75-150ms 够用；stable sqlite-vec 量化+rescore 是现成逃生门
+- chat-with-notes / ask — MCP 消费者本身就是 chat 层，投资引用级检索而非问答
+
+## Context
+
+- 原始项目 qmd 使用 TypeScript / Bun 开发，依赖 `node-llama-cpp` 加载 GGUF 模型进行本地 embedding 和 rerank。
+- 当前 docsift 代码库已完成大部分核心架构（分层设计、Repository 模式、插件化 Chunker / Searcher），但存在多处 stub、bug 和未实现的 CLI 命令。
+- 目标是**功能对等**的 Python 版本，同时改进架构（更好的模块化、可测试性）。
+- 需要保持与 qmd 的 CLI 命令和 MCP 工具接口兼容，方便现有用户迁移。
+
+## Constraints
+
+- **Tech Stack**: Python 3.10+、SQLite（FTS5 + 可选 sqlite-vec）、Click CLI、Pydantic Settings
+- **Local-First**: 默认所有模型本地运行，不依赖外部 API；API 调用仅作为可选配置
+- **Compatibility**: CLI 接口和 MCP Tool Schema 尽量与 qmd 保持一致
+- **Build**: hatchling 构建后端，pytest 测试（常驻 fail_under=90 覆盖率门禁）
+- **Type checking**: ty 为唯一类型检查器（mypy 已于 2026-11-06 退役，`ty check src/sif` 0 诊断）
+
+## Key Decisions
+
+| Decision | Rationale | Outcome |
+|----------|-----------|---------|
+| 远端维度探测 probe-free，缓存键 `base_url::model_name` | 避免对远端后端产生计费探测请求 | Phase 03 Complete |
+| 上下文规范化：写入时存 `normalize_path` 规范形，读取时双侧重规范化匹配，显式 re-add 时合并旧 verbatim 行；永不批量回填迁移 | realpath 不可逆，SQL 预过滤永远无法匹配 verbatim 旧数据行（05-08/05-09，WR-01/WR-02/CR-02 根因） | Phase 05 Complete |
+| 完全复刻 qmd 功能 | 用户明确要求功能对等 | — Complete (v1.0) |
+| 本地模型 + API 切换双支持 | 提升灵活性和可访问性 | — Complete (v1.0) |
+| 优先修复 bug/stub，再补 CLI，再做 rerank/MCP | 先让现有代码可用，再扩展功能 | Phase 01 Complete |
+| 向量库幂等：re-chunk 事务内 delete-before-insert + 精确 chunk-id 集合跳过，`--force` 才全量重嵌 | G-03-3：重复 embed 不再产生孤儿向量/重复计费 | Phase 03 Complete (UAT 7/7) |
+| embed_cmd 每集合一个事务，失败报告在事务外抛出 | CR-02：单集合失败不回滚已成功集合、退出码如实非零 | Phase 03 Complete (live-verified) |
+| 覆盖率门禁 `fail_under = 90` 最后落地（D-04），禁用 exclude_lines / cov-branch / pragma 豁免 | 门禁不可绕过；覆盖率成为永久回归约束，只增不减 | Phase 10 Complete (96.22%, 1011 tests) |
+| DEBT-PN 查找顺序（OQ2）：精确路径仅规范化形查找；ID/裸文件名保持原始；glob 原始优先、规范化兜底 | realpath 不可逆前提下最小惊讶：别名路径可命中，手迁移相对模式行不破坏 | Phase 10.1 Complete |
+| FTS 迁移检测门控 `VALUES('rebuild')`（OQ1）：仅当检出旧版 direct-DELETE 触发器才重建索引，永不用 needs_rebuild_* 做门 | 修复状态而非仅机制：清除坏触发器时期的孤儿索引项；新库/已迁移库零开销 | Phase 10.1 Complete |
+| CLI Database 生命周期：`with Database(index_path) as db:` 自 init_schema 起持有命令体，嵌套事务块原样保留 | 成功与异常路径都确定性关闭连接（57→13 ResourceWarnings，余量测试侧）；main.py 作用域置于 try/except 内部保证异常先关句柄 | Phase 10.1 Complete |
+| XML 输出净化器 = XML 1.0 Char 产生式补集，先于 `xml_escape`/`quoteattr` 应用 | 转义只管 `<>&'"`，禁用码位归净化器管——两者不可互相替代；`ET.fromstring` 往返测试锁定 | Phase 10.1 Complete |
+| mypy 退役，ty 为唯一类型检查器 | 17 条 ty 诊断清零（断言收窄/Protocol/cast，零行为变化），全部 mypy 残留清除；typing-debt 四批次收官 | v1.0 Complete |
+| Watch 管线：per-burst flock + per-path 定向更新 | 每个防抖批次短锁（watch 空闲不持锁，D-04）；`update_paths` 成本正比于变更规模，stat 为准（D-06），Phase 13/14 宿主复用 | Phase 12 Complete |
+| 单一排除谓词 `FileScanner.matches`（扫描与 watch 同源） | LIVE-04 一致性由构造保证；`DEFAULT_IGNORE_PATTERNS` 增补编辑器噪声（.sb-*/4913/.#*/*.tmp） | Phase 12 Complete |
+| missing-root 防护在共享核心（D-11） | 顺带修复 update_cmd 对消失根目录的潜在全量删除 bug；两条更新路径都有回归测试 | Phase 12 Complete |
+| 文档内容永不作为 Rich markup 渲染 | get/multi-get 内容 `markup=False` 直出、表头 `escape()`——与 05-10/05-11 同类第三例；回归测试锁定（字面渲染、批次不中断、回退路径同等契约） | v1.0 Complete |
+| docs 局部套件 `--no-cov` | 全局 `addopts` 覆盖率 + `fail_under=90` 使 docs-only 运行必失败（CI 自 Phase 10 起恒红）；Makefile 与 docs.yml 双修 | v1.0 Complete |
+| v1.2 范围四决策（2026-11-13）：PDF 不做；MCP 升级 SDK v2；Harrier 纳入但 held-out 门禁；Windows 推迟 + CI smoke | 用户拍板；研究三路交叉验证（代码库差距/qmd 生态/技术选型） | — Pending |
+| Harrier 默认切换门禁：EN+ZH held-out 胜出 + embedder 身份标记（非仅宽度） | 独立证据仅英文（localGPT A/B nDCG 0.915 vs Qwen3-Embedding-4B 0.875；此前转述的 vs-0.6B 0.805 未在该精确值核实）；Harrier 与 Qwen3-0.6B 同 1024 维，宽度守护会静默接受坏切换；query 侧必须 prompt_name | — Pending |
+| Windows smoke 走 hc remote yoga（Win11 26300） | 用户指定；yoga 未装 Python（仅 Store alias），CI phase 需先 uv/Python setup | — Pending |
+| DYM 选型：rapidfuzz 测量替换 > symspellpy 推迟 > spellfix1 跳过 | 个人 KB 规模 difflib 已够用；rapidfuzz 去首字母桶是召回修复；symspellpy 仅 >200k 词表再议 | — Pending |
+
+## Context（v1.1 shipped 2026-11-12）
+
+- 版本 1.1.0；7 phases (11–16.1), 27 plans, 67 tasks；~18,064 LOC src；52 feat commits since v1.0.1
+- 质量门禁：1591 passed, 1 skipped @ 95.18% coverage（fail_under=90 常驻）、ruff/format/ty 全绿
+- Live Index：WAL+flock 并发 DB 核心、`sif watch` 前台/MCP 内嵌/后台守护进程三宿主、命令前漂移刷新 + staleness 标记
+- Search Quality：可配置 RRF 权重、did-you-mean + 历史 + `sif suggest`、bench 扩充 + reranker 调优（held-out 论证下默认值不变）
+- Tech debt (16.1)：全 CLI 错误边界约定、cache TTL instant 比较、embedding cache factory 委派、公开 reconcile-enqueue API
+- 遗留（非阻塞）：WR-02 cleanup OperationalError 抑制（OQ2 有意保留）、14-IN-01 代码已修待 ledger 核销、info-findings 扫描、Phase 16 real-model 证据已记录（WINDOWS id 10 closed）
+
+## Context（v1.0 shipped 2026-11-07）
+
+- 版本 1.0.0；692 commits（2026-04-14 → 2026-11-07）；src 12.5K 行 / tests 23.1K 行
+- 质量门禁：pytest 1015 passed @ 96.20%（fail_under=90 常驻、零豁免行）、ruff/format/ty 全绿
+- 里程碑收尾时全量文档事实核验：512 条声明 0 失败；四个陈旧验证全部再生（05/06/07/10.1）
+
+## Evolution
+
+This document evolves at phase transitions and milestone boundaries.
+
+**After each phase transition** (via `/gsd-transition`):
+1. Requirements invalidated? → Move to Out of Scope with reason
+2. Requirements validated? → Move to Validated with phase reference
+3. New requirements emerged? → Add to Active
+4. Decisions to log? → Add to Key Decisions
+5. "What This Is" still accurate? → Update if drifted
+
+**After each milestone** (via `/gsd-complete-milestone`):
+1. Full review of all sections
+2. Core Value check — still the right priority?
+3. Audit Out of Scope — reasons still valid?
+4. Update Context with current state
+
+---
+*Last updated: 2026-11-13 after v1.2 milestone start（范围确认：元数据过滤 + MCP 现代化 + 质量仪器 + 分发；PDF 维持排除，Windows 推迟 + smoke）*