zra-mcp 0.3.0__tar.gz

zra_mcp-0.3.0/.env.example

@@ -0,0 +1,79 @@
+# ── Zotero ──
+# Local mode: reads from Zotero desktop app (fast, no auth needed)
+ZOTERO_LOCAL=true
+# Web API credentials (required for writes and hybrid mode)
+# Get your key at https://www.zotero.org/settings/keys
+ZOTERO_LIBRARY_ID=0
+ZOTERO_LIBRARY_TYPE=user
+ZOTERO_API_KEY=
+# Hybrid mode: set ZOTERO_LOCAL=true AND provide ZOTERO_API_KEY + ZOTERO_LIBRARY_ID
+# → reads stay local (fast), writes go through web API (writable)
+#
+# Note: this is an MCP *server* — the LLM is supplied by your MCP client
+# (Cursor / Claude / Cherry Studio / …), so no OpenAI/Anthropic key is needed here.
+
+# ── Embedding ──
+# Backend selection (auto / onnx_int8 / sentence_transformers):
+#   auto                    — ONNX INT8 if available, otherwise FP32 (default)
+#   onnx_int8               — ONNX Runtime INT8 quantized (~347MB, 2-3x faster, 4x smaller)
+#   sentence_transformers   — Original FP32 PyTorch (~2.3GB, highest precision)
+# ONNX INT8 is recommended for most users — it is 2-3x faster on CPU,
+# uses 4x less disk space, and has <2% retrieval precision impact.
+# EMBEDDING_BACKEND=auto
+
+# Model name (used by sentence_transformers backend only; ONNX INT8 uses a
+# pre-quantized model automatically).
+EMBEDDING_MODEL=BAAI/bge-m3
+# HuggingFace mirror for users in China (uncomment if download is slow)
+# HF_ENDPOINT=https://hf-mirror.com
+
+# Cap embedding sequence length to bound memory (chunks are short; 1024 is
+# ample and prevents GPU/MPS out-of-memory on pathological long inputs).
+# EMBEDDING_MAX_SEQ_LEN=1024
+
+# ── Tables & figures ──
+# Tables and figures are indexed as lightweight caption-anchored records (where
+# they are + caption + rough content), not structured into cells. No setting is
+# needed. For true table structuring, preprocess your PDFs with a visual parser
+# (docling / open-parse / unstructured) — see the README "Tables & figures" note.
+
+# ── Reranker (optional, improves search precision) ──
+# cross-encoder model for re-scoring retrieved chunks
+# Default: cross-encoder/ms-marco-MiniLM-L-6-v2 (~80MB, auto-downloaded)
+# Set to "none" to disable reranking entirely
+RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+
+# ── ChromaDB ──
+CHROMA_PERSIST_DIR=.chroma_db
+
+# ── Text Cleaning ──
+# Strip journal boilerplate (headers, footers, article-info blocks) before
+# chunking. 52 blacklist rules cover EN+CN journals. Set to "false" to disable.
+ZRA_CLEAN_ENABLED=true
+
+# ── Server Behavior ──
+# Auto-sync index on MCP server startup (set to "false" to disable)
+ZRA_AUTO_SYNC=true
+
+# ── Online literature search (optional) ──
+# Semantic Scholar: higher rate limits — https://www.semanticscholar.org/product/api
+SEMANTIC_SCHOLAR_API_KEY=
+# OpenAlex polite pool — your contact email
+OPENALEX_MAILTO=
+# Unpaywall + OA PDF waterfall contact email
+UNPAYWALL_EMAIL=
+# CORE repository full-text — https://core.ac.uk/services/api
+CORE_API_KEY=
+
+# ── CNKI / 知网 (optional, disabled by default) ──
+# CNKI browser automation via Chrome DevTools Protocol (inspired by cookjohn/cnki-skills).
+# Only needed when you want to search Chinese journal papers.
+# Setup: see README.md "CNKI 中文文献检索" section.
+CNKI_ENABLED=false
+# Start Chrome with: --remote-debugging-port=9222, then log in to CNKI.
+CNKI_CDP_URL=http://127.0.0.1:9222
+# Alternative: Playwright storage state JSON exported after manual login
+# CNKI_STORAGE_STATE=.cnki_storage_state.json
+# CNKI_HEADLESS=false
+# CNKI_USE_CHROME=true
+# CNKI_TIMEOUT_MS=45000

zra_mcp-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: ruff check .
+
+      - name: Unit + core tests
+        # tests/mcp are integration tests that need a live Zotero + index, so
+        # they are intentionally excluded from CI.
+        run: pytest tests/core tests/unit -q

zra_mcp-0.3.0/.gitignore

@@ -0,0 +1,58 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+*.egg
+.venv/
+
+# Environment / credentials (contain API keys & tokens)
+.env
+.pypirc
+*.pypirc
+
+# IDE
+.vscode/
+.idea/
+.cursor/
+.claude/
+*.swp
+*.swo
+
+# ChromaDB / Vector store
+.chroma_db/
+
+# Sync state + retrieval logs (generated per-machine, inside .chroma_db/)
+_sync_state.json
+_retrieval_log.jsonl
+_retrieval_log.idx
+
+# User-defined query synonyms (per-machine)
+query_dict_user.json
+
+# Evaluation baseline (per-machine, depends on local library content)
+tests/eval_baseline.json
+
+# Ruff
+.ruff_cache/
+
+# uv
+.local/
+uv.lock
+
+# pytest
+.pytest_cache/
+htmlcov/
+.coverage
+
+# Docs (internal guides, not for distribution)
+docs/internal/
+
+# Development progress / roadmap (internal)
+DEVELOPMENT.md
+ARCHITECTURE.md
+
+# OS
+.DS_Store
+Thumbs.db

zra_mcp-0.3.0/CHANGELOG.md

@@ -0,0 +1,218 @@
+# Changelog
+
+All notable changes to this project are documented here.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **BM25 sparse keyword index on chunk texts** — `rank_bm25` with CJK-aware
+  tokenizer (character unigrams+bigrams for CN, alpha words for EN). Two-way
+  RRF fusion: BM25 (lexical) + ChromaDB (semantic). Persists to
+  `.chroma_db/_bm25_index.pkl`. Auto-rebuilt on every sync.
+- **Contextual chunk enrichment** — each chunk text is prepended with
+  `[Keywords: ...] [Title: ...] [Section: ...]` before embedding and BM25
+  indexing. Implements Anthropic "Contextual Retrieval" 2024 technique
+  using existing metadata — zero additional cost. Keywords are filtered
+  from Zotero tags (excludes organizational labels).
+- **Dual-format output (JSON + Markdown context_block)** — key retrieval tools now
+  return a `context_block` field containing pre-rendered LLM-optimized Markdown
+  alongside the existing JSON items. Blockquote (>) for evidence, star ratings
+  (★★★) for relevance tiers, sentence-boundary truncation. Covers all 8
+  retrieval/insight tools: `search_papers`, `get_paper_content`,
+  `generate_review_note`, `suggest_citations`, `find_similar_papers`,
+  `find_arguments`, `generate_reading_note`, `suggest_tags`.
+- **Relevance tiers** — each `search_papers` result now carries a `relevance_tier`
+  field ("high"/"medium"/"low") computed from Cross-Encoder score percentiles.
+- **Full-pipeline evaluation mode** — `run_evaluation.py --full-pipeline` tests
+  the complete `search_papers()` pipeline (BM25 + CE rerank + MMR + RRF fusion),
+  not just raw semantic search. `evaluate_full_pipeline()` in evaluation.py.
+- **Retrieval log rotation** — auto-cleans log entries older than 90 days.
+  Triggers on server startup and after each `sync_index`. Uses `f.tell()` for
+  cross-platform byte offset accuracy.
+- **Expanded section detection** — Roman numerals (I., II.), Chapter prefix
+  (CHAPTER 1:), section symbol (§1.), letter subsections (A., B.). ~50 new
+  type classification keywords covering non-standard academic headings.
+  Number-only fallback for papers with bare numbered markers.
+- **ONNX INT8 embedding backend** (`EMBEDDING_BACKEND=auto` / `onnx_int8`) —
+  2-3x faster, 4x smaller (347MB vs 2.3GB) embedding on CPU. Zero-config.
+- **MMR (Maximal Marginal Relevance) diversity reranking** — chunk-level MMR
+  with per-document cap (max 3 chunks per paper) and per-document penalty.
+  Default λ=0.4 (grid-search tuned from 0.6).
+- **Multi-layer bilingual query expansion** — 3-layer system with ~300 built-in
+  methodology term pairs, auto-extracted Zotero tags, and user-defined synonyms.
+- **Neighbor chunk expansion** — `expand_neighbors=True` returns hit chunk ±1
+  adjacent chunk within the same section.
+- **Min chunk size floor (200 chars)** — post-chunking merge pass eliminating
+  fragments below the FloTorch 2026 threshold for e2e accuracy.
+
+### Changed
+- **Two-way RRF fusion** — simplified from three-way (Zotero API + BM25 + Dense)
+  to two-way (BM25 + Dense). Zotero API remains for paper metadata, filters,
+  and empty query mode.
+- **MMR diversity_weight tuned: 0.6 → 0.4** — grid search on 10 queries.
+- **Chunk quality scoring simplified** — dropped unused heuristic fields
+  (coherence_score, information_density, boilerplate_ratio).
+- **MCP tools: 32 → 36** with add_query_synonym, recent_retrievals,
+  retrieval_trace, and retrieval_stats.
+- **CHUNKING_VERSION: v2.9.0 → v3.1.0-contextual-chunks**.
+
+### Fixed
+- **SQLite authors field** — was empty string, now populated from Zotero item metadata.
+- **sync_index dead parameters** — removed unused `chunk_size` and `chunk_overlap`.
+- **Section detection `\s*` newline bug** — `\s*` in heading patterns now replaced
+  with `[ \\t]*` to prevent greedy newline consumption causing false headings.
+- **inspect_index pagination bug** — `len(docs)` was only the last page's count,
+  now uses `total_count` for accurate stats.
+- **diversity_weight docstring** — corrected from 0.6 to 0.4 in both search.py
+  and server.py.
+
+## [0.3.0] - 2026-07-06
+
+A major RAG quality release — production-grade retrieval pipeline with text cleaning,
+chunk quality scoring, SQLite metadata layer, section-parent context expansion,
+embedding diagnostics, retrieval observability, and systematic evaluation.
+
+### Added
+
+**RAG Pipeline — Data Quality**
+- **Text cleaning engine** (`research_core/parsers/text_cleaner.py`): 52 blacklist
+  regex rules removing journal boilerplate — EN article-info blocks, CN volume/issue
+  lines, CLC numbers, funding footers, page numbers, DOI lines. Controlled by
+  `ZRA_CLEAN_ENABLED=true` (default on). Cleaning stats reported in `sync_index`.
+
+**RAG Pipeline — Chunk Quality**
+- **Chunk quality scoring** (`research_core/parsers/chunker.py`, v2.9.0): 7 quality
+  fields per chunk — `coherence_score`, `information_density`, `boilerplate_ratio`,
+  `sentence_count`, `starts_with_conjunction`, `language` (zh/en/mixed), `quality_flag`
+  (good/noisy/incomplete/boilerplate). Lightweight heuristic scoring for quality-aware
+  retrieval filtering.
+
+**RAG Pipeline — Metadata & Context**
+- **SQLite metadata database** (`research_core/rag/database.py`): 7 relational tables
+  (`papers`, `sections`, `chunks_meta`, `figures`, `table_records`, and cross-reference
+  tables) inside `.chroma_db/papers.db`. Zero user setup — auto-created on first sync.
+- **Section-parent context expansion** (`Retriever.expand_to_section()`): hit a chunk
+  → fetch its entire enclosing section from SQLite + ChromaDB, providing LLM with
+  complete paragraph context instead of isolated fragments. Cache-batched.
+- **Result enrichment** (`Retriever.enrich()`): batch-fetches paper + section metadata
+  (abstract, authors, year, DOI, keywords, section heading/type) via SQLite JOIN.
+
+**RAG Pipeline — Diagnostics & Evaluation**
+- **Embedding quality diagnostics** (`research_core/rag/embedding_diagnostics.py`):
+  6-phase analysis — per-paper intra-similarity, cross-paper separation ratio, outlier
+  chunk detection (centroid coherence < 0.3), chunk length-similarity Pearson
+  correlation, section-type embedding separation, automated issues + fix suggestions.
+- **Evaluation framework** (`research_core/rag/evaluation.py`): Recall@5/10/20, MRR,
+  NDCG@10 metrics. 60 golden queries (`tests/eval_queries.json`) across direct hit,
+  cross-document, and no-answer categories. `scripts/run_evaluation.py` with
+  `--save-baseline` / `--compare` for A/B testing.
+- **Index audit script** (`scripts/audit_index.py`): 7-phase full-library quality
+  audit — paginated scan, per-paper scoring, library coverage, noise detection,
+  embedding separation, health scoring, and actionable recommendations.
+
+**RAG Pipeline — Observability**
+- **Retrieval logging** (`research_core/rag/logger.py`): JSONL append-only trace
+  logging with byte-offset index. Captures: query, strategy, candidate counts,
+  reranker details, top-20 results with scores/sources, latency breakdown
+  (keyword/semantic/rerank/total). 3 new MCP tools: `recent_retrievals`,
+  `retrieval_trace` (replay by trace ID), `retrieval_stats` (aggregate analytics).
+
+**MCP Tools**
+- Total tool count: 32 → 35 (3 retrieval log tools: `recent_retrievals`,
+  `retrieval_trace`, `retrieval_stats`).
+
+**Documentation**
+- READMEs (EN/CN) updated with all new RAG features, enhanced client setup sections
+  for Claude Desktop, Cherry Studio, and Codex CLI.
+- `DEVELOPMENT_PLAN.md` updated: Phase 0/1/2 marked complete, decision log synced.
+- `DEVELOPMENT_LOG.md` — developer-facing detailed change log from project inception
+  to v0.3.0.
+
+### Changed
+
+- **Chunking overlap rewritten** (`chunker.py`, v2.8.0): sentence-based (1 sentence)
+  → character-based (100 chars) with sentence-boundary completion. Forward-then-backward
+  search algorithm ensures overlap always captures meaningful context.
+- **Section detection** (`section_detector.py`): IMRaD classification via regex heading
+  patterns (EN numbered "1. Introduction", CN "一、引言"). Quality-aware — skips
+  boilerplate/incomplete chunks for heading detection.
+- **Retriever search** now auto-enriches results with paper + section metadata from
+  SQLite (zero latency cost — single JOIN). `expand_context` parameter enables
+  section-parent context expansion.
+- **`search_papers`** emits JSONL retrieval trace on every call with full latency
+  breakdown.
+- **Architecture**: `research_core/rag/` now includes `database.py`, `evaluation.py`,
+  `logger.py`, `embedding_diagnostics.py`.
+- Removed emoji from all terminal output for Windows GBK compatibility.
+
+### Fixed
+
+- Sentence-boundary overlap algorithm: forward-then-backward search now correctly
+  finds sentence boundaries within the overlap region. Previous backward-only search
+  missed boundaries, producing only 1% overlap for English text.
+- Syntax errors in `search.py` from duplicate `matched_passage`/`matched_page` lines
+  after edit.
+- `.claude/` directory added to `.gitignore`.
+
+## [0.2.0] - 2026-06-11
+
+A large reliability- and productization-focused release. The project is now a
+standalone MCP server (no agent scaffold) with 32 single-intent tools.
+
+### Added
+- **Standalone MCP server** exposing 32 tools across Discover / Read / Write /
+  Cite / Insight / Admin, each mapping to one user intent and composing via `item_key`.
+- **`expand_citation_network`** — forward/backward citation-graph expansion via
+  OpenAlex, with multi-seed DOI support.
+- **`find_related_literature`** — one call runs 5 parallel strategies
+  (Corpus-First, keyword, citation network, S2 recommendations, OpenAlex related
+  works) with three-index verification.
+- **Table & figure cross-referencing** — prose that cites "Table 3 / Figure 2"
+  is linked to caption-anchored table/figure records, resolved together in
+  `get_paper_content` (`referenced_tables` / `referenced_figures`).
+- **CJK-aware chunking** — sentence splitting and soft-wrap repair so Chinese
+  text is no longer cut mid-word.
+- Shared HTTP client with global concurrency cap, per-host rate limiting,
+  retry/backoff, and a short-TTL response cache.
+- Configurable `EMBEDDING_MAX_SEQ_LEN` (bounds GPU/MPS memory) and `HF_ENDPOINT`
+  (HuggingFace mirror, e.g. for users in China).
+- Bilingual README and a Cherry Studio setup guide (English + Chinese) aimed at
+  non-developers.
+
+### Changed
+- **Tables and figures are now caption-anchored records** instead of being parsed
+  into structured cells. Reliable table structuring is a vision problem;
+  geometric/line-based detection produced garbage on borderless academic tables
+  and mis-segmented multi-column prose. Table *values* stay searchable; users who
+  need true structure are pointed to docling / open-parse / unstructured.
+- Hardened MCP tools: write operations are dry-run by default and require explicit
+  confirmation; responses are size-capped to protect the LLM context window;
+  errors return structured bilingual diagnostics.
+- Improved RAG recall and speed; thread-safe lazy initialization of Zotero client,
+  retriever, and indexer.
+- Incremental index sync by default (Zotero version tracking); auto full rebuild
+  when the embedding model or chunking version changes.
+
+### Removed
+- Built-in structured table extraction (PyMuPDF "lite" mode + Table Transformer
+  "ml" mode), the `[tables]` optional extra, and `table_ml.py`.
+- Legacy agent scaffold (the project is MCP-server only).
+
+### Fixed
+- Correct Zotero item filtering and ChromaDB `search_ef` modification.
+- Hard chunk-size cap to prevent embedding out-of-memory on pathological inputs.
+- Unit tests updated for the shared HTTP client refactor.
+
+## [0.1.1] - earlier
+
+- Initial public iteration: literature search, RAG pipeline, reading analysis,
+  citation management, review/reading-note generation, tag suggestions, and the
+  first Cherry Studio setup guide.
+
+[Unreleased]: https://github.com/qiobn/zotero-research-assistant/compare/v0.3.0...main
+[0.3.0]: https://github.com/qiobn/zotero-research-assistant/releases/tag/v0.3.0
+[0.2.0]: https://github.com/qiobn/zotero-research-assistant/releases/tag/v0.2.0
+[0.1.1]: https://github.com/qiobn/zotero-research-assistant/releases/tag/v0.1.1

zra_mcp-0.3.0/CLAUDE.md

@@ -0,0 +1,119 @@
+# CLAUDE.md
+
+## Project Overview
+
+This is **Zotero Research Assistant** — an MCP (Model Context Protocol) server that turns a Zotero reference library into an AI-searchable knowledge base. 36 MCP tools across 6 categories. **Core focus: production-grade RAG pipeline (chunking + retrieval quality) for academic papers.**
+
+- **Author:** qiobn
+- **Language:** Python 3.11+
+- **Package:** `zotero-research-assistant`
+- **Entry:** `project_a_mcp/server.py` → `zra-mcp` CLI command
+- **Key deps:** ChromaDB, onnxruntime (INT8 default), sentence-transformers (FP32 fallback), PyMuPDF, FastMCP, PyZotero
+
+## Architecture
+
+```
+research_core/
+  parsers/     — PDF extraction, text cleaner (52 rules), chunker, section detector
+  rag/         — ChromaDB store, retriever, SQLite metadata DB, evaluation, logger, diagnostics
+  tools/       — 36 MCP tool implementations
+  zotero/      — Zotero local + web API client
+project_a_mcp/ — MCP server entry point (stdio)
+scripts/       — CLI utils (index_library, audit_index, run_evaluation, publish)
+tests/         — pytest suite + 60 golden eval queries
+docs/          — Setup guides (Cherry Studio CN/EN)
+```
+
+## Development Rules
+
+### Commit & Log Protocol
+
+**On every significant change (new feature, bug fix, non-trivial refactor), you MUST:**
+
+1. **Update `DEVELOPMENT_LOG.md`** — Record:
+   - What was changed (with commit hash)
+   - What problem it solved
+   - Technical decisions made and their rationale
+   - Future optimization directions
+   - Known issues introduced
+
+2. **Update `CHANGELOG.md`** — Keep a Changelog format (`Added`/`Changed`/`Fixed`/`Removed` sections under current version). User-facing, less technical than DEVELOPMENT_LOG.
+
+3. **Commit with a conventional commit message — ALWAYS include a body, never an empty commit:**
+   - First line: `<type>: <summary>` (50-80 chars, imperative mood)
+   - Then a BLANK LINE
+   - Then body paragraphs explaining **what** changed, **why**, and any design notes
+   - Types: `feat:` / `fix:` / `docs:` / `refactor:` / `chore:`
+   - Example:
+     ```
+     feat: add query rewrite for bilingual academic search
+
+     Dictionary-based CN<->EN term expansion with three layers:
+     Layer 1: ~300 built-in methodology pairs from query_dict.json
+     Layer 2: auto-extracted from user's Zotero tags during sync
+     Layer 3: user-defined via add_query_synonym MCP tool
+
+     Expansion runs in search_papers() — zero added latency, no LLM
+     dependency. Each expanded term runs independent semantic search
+     with RRF merging and expansion weight scoring.
+     ```
+
+4. **Push** after each logical unit of work (not after every micro-edit).
+
+### Code Style
+
+- Follow surrounding code patterns: comment density, naming, idiom
+- Use `from __future__ import annotations` in all new files
+- Use dataclasses for data containers
+- Return typed dataclass instances, not raw dicts
+- All write operations default to dry-run preview; require explicit confirmation
+- MCP tools: one tool per user intent, compose via `item_key`
+- Windows compatibility: no emoji in terminal output (use ASCII alternatives), use `os.path` or `pathlib`
+
+### Testing
+
+- Unit tests in `tests/` with pytest
+- Run: `pytest tests/ -v`
+- Lint: `ruff check .`
+- Format: `ruff format .`
+- Evaluation: `python scripts/run_evaluation.py`
+
+### Environment
+
+- `.env` is gitignored; `.env.example` is committed (template only, no real secrets)
+- All configurable values have env var overrides
+- Defaults are zero-config: users should get basic functionality without any `.env` edits
+- New env vars must be documented in: `.env.example`, both READMEs' config tables
+
+### Documentation Sync
+
+When adding/removing features or changing behavior, update:
+1. `README.md` (English) + `README_zh.md` (Chinese) — keep in sync
+2. `CHANGELOG.md` — keep-a-changelog format
+3. `DEVELOPMENT_LOG.md` — technical details and decisions
+4. `DEVELOPMENT_PLAN.md` — progress bars and task checkboxes
+5. `.env.example` — new env vars with comments
+
+## Key Files Reference
+
+| File | Purpose |
+|------|---------|
+| `DEVELOPMENT_LOG.md` | Detailed technical changelog (dev-facing) |
+| `CHANGELOG.md` | Release changelog (user-facing) |
+| `DEVELOPMENT_PLAN.md` | Task tracking + progress bars |
+| `README.md` / `README_zh.md` | Bilingual project READMEs |
+| `research_core/parsers/chunker.py` | Chunking with CHUNKING_VERSION for auto-rebuild |
+| `research_core/parsers/text_cleaner.py` | 52 blacklist rules, CLEANER_VERSION |
+| `research_core/rag/database.py` | SQLite metadata DB (auto-created on first sync) |
+| `research_core/rag/logger.py` | JSONL retrieval trace logging |
+| `research_core/rag/evaluation.py` | Recall@K, MRR, NDCG |
+| `research_core/rag/retriever.py` | ChromaDB retriever with section expansion + enrichment |
+
+## Current State (v0.4.0-dev)
+
+- Phase 0/1/2/3 (partial) complete; Phase 4 (v0.4.0) in progress
+- 36 MCP tools (32 always-on + 4 CNKI-conditional), all operational
+- Key features: BM25+Dense hybrid retrieval, ONNX INT8 embedding, MMR diversity,
+  bilingual query expansion, contextual chunk enrichment, dual-format output
+- Known issues: evaluation only run on small test set (8 papers)
+- Next priorities: run full-pipeline evaluation on real library, chunk size tuning

zra_mcp-0.3.0/DEVELOPMENT_LOG.md

@@ -0,0 +1,8 @@
+# Development Log
+
+> Detailed technical development log with problem/solution/decision records.
+
+- **[中文版](./docs/DEVELOPMENT_LOG.md)** — 完整中文版
+- **[English](./docs/DEVELOPMENT_LOG_EN.md)** — Full English version
+
+See also: [CHANGELOG.md](./CHANGELOG.md) for user-facing release notes.

zra_mcp-0.3.0/DEVELOPMENT_PLAN.md

@@ -0,0 +1,148 @@
+# Development Plan — RAG Full-Pipeline Optimization
+
+> Last updated: 2026-07-13 | Current version: v0.4.0-dev
+
+---
+
+## Progress Overview
+
+```
+Phase 0 (Audit)    ████████████████████ 100%  ✅ DONE
+Phase 1 (P0)       ████████████████████ 100%  ✅ DONE
+Phase 2 (P1)       ████████████████████ 100%  ✅ DONE
+Phase 3 (P2)       ████████░░░░░░░░░░░░  40%  (completed: 3.1, 3.3; remaining: 3.2, 3.4, 3.5)
+Phase 4 (v0.4.0)   ░░░░░░░░░░░░░░░░░░░░   0%  Architecture audit findings
+```
+
+---
+
+## Phase 3: P2 — Refinements (Partial)
+
+### 3.1 Query Rewrite (Academic Scene) ✅
+
+> Implemented: `research_core/rag/query_rewriter.py`. Three-layer bilingual expansion (built-in ~310 pairs + Zotero tags + user synonyms). LRU-cached. `research_core/rag/query_dict.json`.
+
+### 3.2 Adaptive Chunk Granularity ⬜
+
+> **RESEARCHED & DEFERRED**: 2025-2026 literature (NAACL, Chroma, PaperQA2) shows fixed-size chunking is the strong baseline. Semantic/adaptive chunking does not consistently beat it. PaperQA2 uses fixed ~9000 chars with downstream LLM reranking. Chunk size is the dominant variable, not the splitter method.
+
+### 3.3 Search Result Post-Processing ✅
+
+> All three originally planned items completed:
+> - MMR diversity (λ=0.4, grid-search tuned, max 3 chunks/paper)
+> - Neighbor chunk expansion (±1 chunk, section-constrained)
+> - Source diversity (MMR cap + per-document penalty)
+
+### 3.4 Comprehensive Diagnostic MCP Tool ⬜
+
+> **DEFERRED**: Normal users don't trace why results ranked a certain way. De-prioritized.
+
+### 3.5 Metadata-Enhanced Re-Ranking ⬜
+
+> **RESEARCHED & DEFERRED**: Marginal improvement for personal libraries (5-10%). The strongest signals (citation count, journal tier) are external and add latency. Not worth the complexity at this scale.
+
+---
+
+## Phase 4: v0.4.0 — Architecture Audit Findings (2026-07-13)
+
+Issues identified during full architecture review:
+
+### 4.1 Evaluation Tests Full Pipeline 🔴 HIGH (in progress)
+
+> **Problem**: `evaluate_retrieval()` only tests `retriever.search()` (pure semantic). BM25, CE reranker, MMR, and RRF fusion have NEVER been evaluated. Every retrieval component you've built lacks quantitative validation.
+>
+> **Fix**: Add `--full-pipeline` mode to `run_evaluation.py` that calls `search_papers()` instead of `retriever.search()`. Compare semantic-only vs full-pipeline metrics. Save baselines for future regression testing.
+
+**Estimate:** 1 day
+
+---
+
+### 4.2 Log Rotation ⬜ 🟡 MEDIUM
+
+> **Problem**: `_retrieval_log.jsonl` grows unboundedly. Large libraries could hit GB-scale log files.
+>
+> **Fix**: Add size-based rotation (e.g., keep last 100MB / 10K entries). Or time-based (keep 30 days).
+
+**Estimate:** 0.5 day
+
+---
+
+### 4.3 Authors Field in SQLite ⬜ 🟡 MEDIUM
+
+> **Problem**: SQLite `papers.authors` is always `""`. Comment says "ZoteroItem doesn't expose authors as JSON" but `Item.authors` is `list[str]` — it's available.
+>
+> **Fix**: `json.dumps(item.authors)` when writing to SQLite in `_index_metadata()`.
+
+**Estimate:** 5 minutes
+
+---
+
+### 4.4 Section Parent Linking ⬜ 🟡 MEDIUM
+
+> **Problem**: `section_detector.py` computes `parent_idx` (subsection hierarchy) but SQLite `sections.parent_id` is always NULL. Subsections of Methods, etc. are flattened.
+>
+> **Fix**: Store parent-child relationships in SQLite during `_index_metadata()`.
+
+**Estimate:** 0.5 day
+
+---
+
+### 4.5 Dead Parameters in sync_index ⬜ 🟢 LOW
+
+> **Problem**: `sync_index(chunk_size=800, chunk_overlap=120)` accepts parameters that the chunker ignores (uses its own `target_chunk_size=600` internally since v2).
+>
+> **Fix**: Remove dead parameters or add deprecation warning.
+
+**Estimate:** 5 minutes
+
+---
+
+### 4.6 Docstring Bug ⬜ 🟢 LOW
+
+> **Problem**: `search_papers()` docstring says `diversity_weight=0.6`, actual default is `0.4` (grid-search tuned).
+>
+> **Fix**: Update docstring.
+
+**Estimate:** 1 minute
+
+---
+
+### 4.7 BM25 Chinese Tokenization ⬜ 🟢 LOW
+
+> **Problem**: Character bigrams work but jieba segmentation would be more accurate for Chinese BM25 queries.
+>
+> **Fix**: Optional jieba dependency, use if available, fall back to bigrams.
+
+**Estimate:** 0.5 day
+
+---
+
+### 4.8 Personalized Re-Ranking ⬜ 🔵 FUTURE
+
+> **Problem**: User engagement signals (annotations, reading depth, saved notes) are never used for ranking.
+>
+> **Fix**: Light boost for papers with user annotations/notes. Available from Zotero local API.
+
+**Estimate:** 1 day
+
+---
+
+## Summary
+
+| Phase | # Tasks | Completed | Remaining |
+|-------|---------|-----------|-----------|
+| Phase 0 (Audit) | 5 | 5 | 0 |
+| Phase 1 (P0) | 3 | 3 | 0 |
+| Phase 2 (P1) | 5 | 4 | 1 (deferred) |
+| Phase 3 (P2) | 5 | 2 | 3 (deferred) |
+| Phase 4 (v0.4.0) | 8 | 0 | 8 |
+| **Total** | **26** | **14** | **12** |
+
+### Immediate Next Steps
+
+```
+→ Phase 4.1: Full-pipeline evaluation (in progress)
+→ Phase 4.3: Authors field fix (trivial, quick win)
+→ Phase 4.6: Docstring fix (trivial)
+→ Phase 4.5: Dead parameters cleanup
+```