citeindex 0.12.0__tar.gz

citeindex-0.12.0/.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(wc -l /home/ajiap/project/citeindex/citeindex/agents/*.py /home/ajiap/project/citeindex/citeindex/ingestion/*.py /home/ajiap/project/citeindex/citeindex/ingestion/pipelines/*.py /home/ajiap/project/citeindex/citeindex/*.py)"
+    ]
+  }
+}

citeindex-0.12.0/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.venv
+
+# Environment
+.env
+
+# Corpus (user data, never committed)
+corpus/
+
+# Test output
+test_output/
+
+# Rust (removed, prevent accidents)
+target/
+Cargo.lock
+*.rs.bk
+
+# IDE / editor
+.vscode/
+.idea/
+*.swp
+*.swo
+*~

citeindex-0.12.0/.omc/project-memory.json

@@ -0,0 +1,293 @@
+{
+  "version": "1.0.0",
+  "lastScanned": 1776178956720,
+  "projectRoot": "/home/ajiap/project/citeindex",
+  "techStack": {
+    "languages": [
+      {
+        "name": "Python",
+        "version": null,
+        "confidence": "high",
+        "markers": [
+          "pyproject.toml"
+        ]
+      }
+    ],
+    "frameworks": [
+      {
+        "name": "pytest",
+        "version": null,
+        "category": "testing"
+      }
+    ],
+    "packageManager": null,
+    "runtime": null
+  },
+  "build": {
+    "buildCommand": null,
+    "testCommand": "pytest",
+    "lintCommand": "ruff check",
+    "devCommand": null,
+    "scripts": {}
+  },
+  "conventions": {
+    "namingStyle": null,
+    "importStyle": null,
+    "testPattern": null,
+    "fileOrganization": null
+  },
+  "structure": {
+    "isMonorepo": false,
+    "workspaces": [],
+    "mainDirectories": [
+      "docs",
+      "tests"
+    ],
+    "gitBranches": {
+      "defaultBranch": "0.10.04",
+      "branchingStrategy": null
+    }
+  },
+  "customNotes": [],
+  "directoryMap": {
+    "__pycache__": {
+      "path": "__pycache__",
+      "purpose": null,
+      "fileCount": 2,
+      "lastAccessed": 1776178956711,
+      "keyFiles": [
+        "test_install.cpython-312-pytest-8.3.5.pyc",
+        "test_page_extraction.cpython-312-pytest-8.3.5.pyc"
+      ]
+    },
+    "citeindex": {
+      "path": "citeindex",
+      "purpose": null,
+      "fileCount": 15,
+      "lastAccessed": 1776178956715,
+      "keyFiles": [
+        "__init__.py",
+        "citation_style.py",
+        "cli.py",
+        "file_converter.py",
+        "llm.py"
+      ]
+    },
+    "citeindex-rs": {
+      "path": "citeindex-rs",
+      "purpose": null,
+      "fileCount": 4,
+      "lastAccessed": 1776178956715,
+      "keyFiles": [
+        "Cargo.lock",
+        "Cargo.toml",
+        "citeindex-tui.log",
+        "config.toml"
+      ]
+    },
+    "corpus": {
+      "path": "corpus",
+      "purpose": null,
+      "fileCount": 2,
+      "lastAccessed": 1776178956717,
+      "keyFiles": [
+        "_url_content_hashes.json",
+        "ingestion_log.jsonl"
+      ]
+    },
+    "dist": {
+      "path": "dist",
+      "purpose": "Distribution/build output",
+      "fileCount": 2,
+      "lastAccessed": 1776178956717,
+      "keyFiles": [
+        "cite_extractor-0.10.8-py3-none-any.whl",
+        "cite_extractor-0.10.8.tar.gz"
+      ]
+    },
+    "docs": {
+      "path": "docs",
+      "purpose": "Documentation",
+      "fileCount": 1,
+      "lastAccessed": 1776178956717,
+      "keyFiles": [
+        "v12-runtime-migration.md"
+      ]
+    },
+    "example": {
+      "path": "example",
+      "purpose": null,
+      "fileCount": 55,
+      "lastAccessed": 1776178956717,
+      "keyFiles": [
+        "2013-tarsākyā:.json",
+        "2013_庄子集释_中华书局.json",
+        "2025_心念之病_光从东方来.md.json",
+        "A_Jiatandongfangjiaohui_2025_为何保守派倾向于认为其他教派是异端_Youtube.md.json",
+        "Bai-Wengu-白文固_2003_唐代僧籍管理制度.json"
+      ]
+    },
+    "instruction": {
+      "path": "instruction",
+      "purpose": null,
+      "fileCount": 3,
+      "lastAccessed": 1776178956718,
+      "keyFiles": [
+        "CiteIndex Summary v11.pdf",
+        "CiteIndex v12 Full.pdf",
+        "Summary.md"
+      ]
+    },
+    "test_output": {
+      "path": "test_output",
+      "purpose": null,
+      "fileCount": 1,
+      "lastAccessed": 1776178956718,
+      "keyFiles": [
+        "Media_content_from_URL.md.json"
+      ]
+    },
+    "tests": {
+      "path": "tests",
+      "purpose": "Test files",
+      "fileCount": 22,
+      "lastAccessed": 1776178956718,
+      "keyFiles": [
+        "Bai-Wengu-白文固_2003_唐代僧籍管理制度.pdf",
+        "Bai-Yudong-白玉冬-2018-丝路景教与汪古渊流.pdf",
+        "Guo-qingfan-庄子集释 (郭庆藩).pdf",
+        "README.md.backup",
+        "main.py.backup"
+      ]
+    }
+  },
+  "hotPaths": [
+    {
+      "path": "README.md",
+      "accessCount": 1,
+      "lastAccessed": 1776179072424,
+      "type": "file"
+    },
+    {
+      "path": "pyproject.toml",
+      "accessCount": 1,
+      "lastAccessed": 1776179072470,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/__init__.py",
+      "accessCount": 1,
+      "lastAccessed": 1776179079362,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/v12_runtime.py",
+      "accessCount": 1,
+      "lastAccessed": 1776179096866,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/cli.py",
+      "accessCount": 1,
+      "lastAccessed": 1776179096911,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/Cargo.toml",
+      "accessCount": 1,
+      "lastAccessed": 1776179104160,
+      "type": "file"
+    },
+    {
+      "path": "docs/v12-runtime-migration.md",
+      "accessCount": 1,
+      "lastAccessed": 1776179104203,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/models.py",
+      "accessCount": 1,
+      "lastAccessed": 1776179104260,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/llm.py",
+      "accessCount": 1,
+      "lastAccessed": 1776180019547,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/chat.py",
+      "accessCount": 1,
+      "lastAccessed": 1776180019587,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/generation.py",
+      "accessCount": 1,
+      "lastAccessed": 1776180019616,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/query_planner.py",
+      "accessCount": 1,
+      "lastAccessed": 1776181697113,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/integrity.py",
+      "accessCount": 1,
+      "lastAccessed": 1776181697156,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/kernel/src/agent_runtime/mod.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182005187,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/kernel/src/tools/mod.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182005231,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/tui/src/app.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182005267,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/core/src/engine.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182014922,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/core/src/config.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182014951,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/crates/core/src/ipc.rs",
+      "accessCount": 1,
+      "lastAccessed": 1776182056484,
+      "type": "file"
+    },
+    {
+      "path": "citeindex-rs/config.toml",
+      "accessCount": 1,
+      "lastAccessed": 1776182061838,
+      "type": "file"
+    },
+    {
+      "path": "citeindex/agents/coordinator.py",
+      "accessCount": 1,
+      "lastAccessed": 1776182061876,
+      "type": "file"
+    }
+  ],
+  "userDirectives": []
+}

citeindex-0.12.0/.omc/sessions/a5782e06-815d-4db8-b4dc-fd90a82b045f.json

@@ -0,0 +1,8 @@
+{
+  "session_id": "a5782e06-815d-4db8-b4dc-fd90a82b045f",
+  "ended_at": "2026-04-14T20:02:26.698Z",
+  "reason": "prompt_input_exit",
+  "agents_spawned": 0,
+  "agents_completed": 0,
+  "modes_used": []
+}

citeindex-0.12.0/.omc/sessions/c68a47d6-0746-405c-84f8-03d8c678c6c0.json

@@ -0,0 +1,8 @@
+{
+  "session_id": "c68a47d6-0746-405c-84f8-03d8c678c6c0",
+  "ended_at": "2026-04-14T15:03:12.144Z",
+  "reason": "other",
+  "agents_spawned": 0,
+  "agents_completed": 0,
+  "modes_used": []
+}

citeindex-0.12.0/.python-version

@@ -0,0 +1 @@
+3.12.8

citeindex-0.12.0/PKG-INFO

@@ -0,0 +1,221 @@
+Metadata-Version: 2.3
+Name: citeindex
+Version: 0.12.0
+Summary: Ingest sources with proper citation — PDF, URL, media, Office, DJVU
+Author-email: ajia <yyjfwoaini@gmail.com>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: citeproc-py>=0.7.0
+Requires-Dist: crawl4ai>=0.7.0
+Requires-Dist: dspy-ai>=2.6.27
+Requires-Dist: fasttext>=0.9.2
+Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: litellm>=1.83.0
+Requires-Dist: lxml>=4.9.0
+Requires-Dist: mineru[all]>=2.6.4
+Requires-Dist: ocrmypdf>=16.10.4
+Requires-Dist: paddleocr>=3.1.0
+Requires-Dist: paddlepaddle>=3.1.0
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: pyannote-audio>=3.1.0
+Requires-Dist: pymediainfo>=7.0.1
+Requires-Dist: pymupdf[mupdf-third]>=1.26.3
+Requires-Dist: pypdf2>=3.0.1
+Requires-Dist: pypinyin>=0.51.0
+Requires-Dist: python-dateutil>=2.8.0
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: readability-lxml>=0.8.1
+Requires-Dist: requests>=2.31.0
+Requires-Dist: setuptools>=80.9.0
+Requires-Dist: trafilatura>=1.6.0
+Requires-Dist: urllib3>=2.0.0
+Requires-Dist: whisperx>=3.1.0
+Requires-Dist: yt-dlp>=2025.7.21
+Description-Content-Type: text/markdown
+
+# CiteIndex
+
+**v0.12.0** — Ingest sources with proper citation. PDF, URL, media, Office, DJVU.
+
+Deterministic citation extraction, Merkle-verified integrity, CJK-first OCR.
+Every claim is traced, verified, and cited — no hallucinations.
+
+## Install
+
+```bash
+# Using rye (recommended)
+rye sync
+
+# Or pip
+pip install -e .
+```
+
+## CLI
+
+```bash
+# Ingest a PDF
+citeindex paper.pdf
+
+# Ingest a URL
+citeindex https://example.com/article
+
+# Crawl and ingest all articles from a site
+citeindex https://example.com/articles --all-url-article --crawl-depth 2
+
+# Crawl and re-ingest only changed pages
+citeindex https://example.com/articles --update-url-article
+
+# Options
+citeindex paper.pdf --llm ollama/qwen3 --type thesis --is-primary
+citeindex paper.pdf --text-direction vertical --vertical-lang ch
+citeindex scanned.pdf --lang auto --page-range "1-10"
+citeindex paper.pdf --no-layout  # disable column/footnote detection
+```
+
+## Python API
+
+```python
+from citeindex import ingest, IngestionConfig
+
+# Simple
+result = ingest("paper.pdf")
+print(result["status"])  # "ok"
+
+# With config
+config = IngestionConfig(
+    llm_model="ollama/qwen3",
+    text_direction="vertical",
+    is_primary=True,
+)
+result = ingest("paper.pdf", corpus_root="my_corpus", config=config)
+```
+
+## Ingestion Pipelines
+
+CiteIndex automatically detects the input type and routes to the correct pipeline:
+
+### Digital PDF
+
+```
+PDF → GROBID (metadata) → MinerU (layout) → DSPy reconciliation
+    → document structure (pages/columns/paragraphs/lines)
+    → Merkle tree → store to corpus/
+```
+
+- **GROBID** extracts metadata and references deterministically
+- **MinerU** performs layout analysis (columns, footnotes, tables)
+- **DSPy** reconciles GROBID output with pattern extraction as fallback
+- Builds section-hierarchical document structure with actual page numbers
+
+### Scanned PDF
+
+```
+PDF → OCRmyPDF (normalize) → PaddleOCR (vertical detect) → MinerU (layout)
+    → Tesseract (text) → GROBID (citations) → document structure
+    → Merkle tree → store to corpus/
+```
+
+- **OCRmyPDF** normalizes and adds text layer to scanned pages
+- **PaddleOCR** detects CJK vertical text layouts
+- **Tesseract** provides OCR with auto-detected language
+- Supports `--text-direction vertical` for traditional Chinese/Japanese
+
+### URL Article
+
+```
+URL → Playwright/requests (fetch) → trafilatura (content)
+    → Zotero (metadata) → CSL JSON → deterministic chunking
+    → hashes → Merkle tree → store to corpus/
+```
+
+- **Playwright** renders JavaScript-heavy pages (fallback to **requests**)
+- **trafilatura** extracts clean text with heading structure
+- **Zotero** extracts citation metadata (title, authors, date, DOI)
+- Discovers in-page citation guidance (若要引用 / Cite this / etc.)
+- Supports batch crawling with `--all-url-article` and `--update-url-article`
+
+### Media
+
+```
+URL/File → yt-dlp (download) → ffmpeg (audio) → WhisperX (transcription)
+        → pyannote (diarization, optional) → CSL JSON
+        → chunking → hashes → Merkle tree → store to corpus/
+```
+
+- **yt-dlp** downloads from YouTube, Vimeo, podcasts, etc.
+- **WhisperX** transcribes with word-level timestamps
+- **pyannote** speaker diarization (optional)
+- Supports audio (`.mp3`, `.wav`, `.m4a`) and video (`.mp4`, `.mkv`, `.webm`)
+
+### Office & DJVU
+
+Office documents (`.docx`, `.doc`, `.rtf`, `.odt`, `.pptx`, `.ppt`, `.odp`) and DJVU (`.djvu`) are converted to PDF via LibreOffice/ddjvu, then routed to the digital or scanned PDF pipeline.
+
+## Configuration Reference
+
+| Option | CLI Flag | Default | Description |
+|--------|----------|---------|-------------|
+| `llm_model` | `--llm` | `ollama/qwen3` | LLM model for citation extraction |
+| `text_direction` | `--text-direction`, `-td` | `horizontal` | `horizontal`, `auto`, or `vertical` |
+| `vertical_lang` | `--vertical-lang` | `ch` | CJK language: `ch` (Chinese) or `japan` |
+| `lang` | `--lang`, `-l` | `auto` | OCR language (auto-detect or Tesseract code) |
+| `page_range` | `--page-range`, `-p` | `1-5, -3` | Pages to extract (e.g. `"1-10"`, `"1-5, -3"`) |
+| `doc_type_override` | `--type`, `-t` | auto | `book`, `thesis`, `journal`, or `bookchapter` |
+| `use_layout_analysis` | `--no-layout` | `True` | Disable column/footnote detection |
+| `is_primary` | `--is-primary` | `False` | Line-level granularity (vs paragraph-level) |
+| `use_pageindex` | `--use-pageindex` | `False` | LLM-driven section hierarchy (requires Ollama) |
+| `pageindex_model` | `--pageindex-model` | `ollama/qwen3.5:cloud` | LLM for PageIndex tree building |
+| `citation_style` | (API only) | `chicago-author-date` | CSL citation style for output |
+| `corpus_root` | `--corpus-root` | `corpus` | Output directory for ingested artifacts |
+| `schema_version` | `--schema-version` | `1.0.0` | Output schema version tag |
+
+## Output
+
+Each ingestion produces a corpus folder (e.g., `corpus/Author_2024_Title/`) containing:
+
+| File | Description |
+|------|-------------|
+| `csl.json` | Citation metadata (CSL-JSON with `ci_*` extensions: `content_hash`, `merkle_root`, `source_type`, `ingestion_timestamp`) |
+| `document.json` | Structured document tree (PageIndex) — sections, pages, paragraphs, lines |
+| `merkle.json` | SHA-256 Merkle tree for integrity verification |
+| `ingestion_output.json` | Full ingestion result with all pipeline outputs |
+| `library.md` | Human-readable citation with extracted text and footnotes |
+
+### Return Value
+
+The `ingest()` function returns a dict:
+
+```python
+{
+    "status": "ok",                    # "ok" or "blocked"
+    "document_path": "corpus/Author_2024_Title",
+    "standardized_csl_json": { ... },  # Full CSL-JSON with ci_ extensions
+    "sub_pipeline_outputs": { ... },   # Raw pipeline results
+    "ingestion_log_entry": { ... },     # Log entry with merkle_root
+}
+
+# On failure:
+{
+    "status": "blocked",
+    "stage": "detect_resource_type",
+    "error_code": "unsupported_input",
+    "error_message": "Unsupported input: ...",
+    "next_action": "Provide PDF, URL, or media file",
+}
+```
+
+## Supported Formats
+
+| Format | Extension / Protocol |
+|--------|----------------------|
+| Digital PDF | `.pdf` (with embedded text) |
+| Scanned PDF | `.pdf` (image-based, OCR applied) |
+| URL Article | `http://` / `https://` |
+| Media | `.mp3`, `.wav`, `.m4a`, `.mp4`, `.mkv`, `.webm` |
+| Office | `.docx`, `.doc`, `.rtf`, `.odt`, `.pptx`, `.ppt`, `.odp` |
+| DJVU | `.djvu` |
+
+## License
+
+MIT