crossref-local 0.3.0__tar.gz → 0.3.1__tar.gz

{crossref_local-0.3.0 → crossref_local-0.3.1}/.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+          pip install -e ".[all]"
 
       - name: Create test database
         run: |

{crossref_local-0.3.0 → crossref_local-0.3.1}/.gitignore

@@ -680,3 +680,4 @@ GITIGNORED/
 # Test fixtures (generated)
 tests/fixtures/*.db
 tests/fixtures/*.json
+_.venv/

{crossref_local-0.3.0 → crossref_local-0.3.1}/CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.1] - 2026-01-14
+
+### Added
+- FastAPI HTTP server with RESTful endpoints (`/works`, `/works/{doi}`, `/works/batch`)
+- MCP server for Claude Desktop integration (`crossref-local serve`)
+- Remote client for accessing API over SSH tunnel
+- Comprehensive test suite (114 tests covering all modules)
+- Shell scripts with proper usage/help options
+
+### Changed
+- Renamed API endpoints to RESTful conventions (`/search` → `/works`)
+- Cleaned up Python API exposure - internal modules no longer in `__all__`
+- Improved module docstrings with full API documentation
+
+### Fixed
+- SQLite threading issue for FastAPI multi-threaded access
+- Batch endpoint path in remote client
+- FastMCP API compatibility (`description` → `instructions`)
+
 ## [0.3.0] - 2026-01-11
 
 ### Added

{crossref_local-0.3.0 → crossref_local-0.3.1}/Makefile

@@ -7,7 +7,7 @@
 #   make status    - Show database status
 
 .PHONY: help install dev test status db-info db-schema db-indices fts-build fts-status \
-        citations-status citations-rebuild check clean
+        citations-status citations-rebuild check clean nfs-setup nfs-status nfs-stop
 
 # Paths
 PROJECT_ROOT := $(shell pwd)
@@ -149,6 +149,9 @@ status: ## Show overall system status (run this first!)
 		echo "    Hint: Check data symlink or run 'make download'"; \
 	fi
 	@echo ""
+	@echo "=== NFS Server ==="
+	@$(SCRIPTS)/nfs/check.sh
+	@echo ""
 	@echo "=== Running Processes ==="
 	@ps aux | grep -E "(rebuild_citations|build_fts|sqlite3)" | grep -v grep | head -5 || echo "  No database processes running"
 	@echo ""
@@ -162,6 +165,7 @@ status: ## Show overall system status (run this first!)
 	fi
 	@echo ""
 	@echo "For detailed database info: make db-info"
+	@echo "For NFS details: make nfs-status"
 
 db-info: ## Show database tables, indices, and row counts
 	@$(SCRIPTS)/database/99_db_info.sh
@@ -252,3 +256,14 @@ create-missing-indices: ## Create any missing indices
 
 maintain-indices: ## Analyze and optimize indices
 	@$(SCRIPTS)/database/99_maintain_indexes.sh
+
+##@ NFS Server
+
+nfs-setup: ## Setup NFS server to share database (requires .env with SUDO_PASSWORD)
+	@$(SCRIPTS)/nfs/setup_nfs.sh
+
+nfs-status: ## Show NFS server status and exports
+	@$(SCRIPTS)/nfs/status.sh
+
+nfs-stop: ## Stop NFS server
+	@$(SCRIPTS)/nfs/stop.sh

{crossref_local-0.3.0 → crossref_local-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crossref-local
-Version: 0.3.0
+Version: 0.3.1
 Summary: Local CrossRef database with 167M+ works and full-text search
 Project-URL: Homepage, https://github.com/ywatanabe1989/crossref_local
 Project-URL: Repository, https://github.com/ywatanabe1989/crossref_local
@@ -18,9 +18,25 @@ Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.10
 Requires-Dist: click>=8.0
+Provides-Extra: all
+Requires-Dist: fastapi>=0.100; extra == 'all'
+Requires-Dist: fastmcp>=0.4; extra == 'all'
+Requires-Dist: matplotlib>=3.7; extra == 'all'
+Requires-Dist: networkx>=3.0; extra == 'all'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'all'
+Requires-Dist: pytest-cov>=4.0; extra == 'all'
+Requires-Dist: pytest>=7.0; extra == 'all'
+Requires-Dist: pyvis>=0.3; extra == 'all'
+Requires-Dist: uvicorn>=0.20; extra == 'all'
+Provides-Extra: api
+Requires-Dist: fastapi>=0.100; extra == 'api'
+Requires-Dist: uvicorn>=0.20; extra == 'api'
 Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=0.4; extra == 'mcp'
 Provides-Extra: viz
 Requires-Dist: matplotlib>=3.7; extra == 'viz'
 Requires-Dist: networkx>=3.0; extra == 'viz'
@@ -121,24 +137,93 @@ async def main():
 crossref-local search "CRISPR genome editing" -n 5
 crossref-local get 10.1038/nature12373
 crossref-local impact-factor Nature -y 2023  # IF: 54.067
+crossref-local info  # Database stats
 ```
 
 With abstracts (`-a` flag):
 ```
-$ crossref-local search "CRISPR" -n 1 -a
+$ crossref-local search "RS-1 enhances CRISPR" -n 1 -a
 
-Found 87,473 matches in 18.2ms
+Found 4 matches in 128.4ms
 
 1. RS-1 enhances CRISPR/Cas9- and TALEN-mediated knock-in efficiency (2016)
    DOI: 10.1038/ncomms10548
    Journal: Nature Communications
    Abstract: Zinc-finger nuclease, transcription activator-like effector nuclease
-   and CRISPR/Cas9 are becoming major tools for genome editing. Importantly,
-   knock-in in several non-rodent species has been finally achieved...
+   and CRISPR/Cas9 are becoming major tools for genome editing...
 ```
 
 </details>
 
+<details>
+<summary><strong>HTTP API</strong></summary>
+
+Start the FastAPI server:
+```bash
+crossref-local api --host 0.0.0.0 --port 3333
+```
+
+Endpoints:
+```bash
+# Search works (FTS5)
+curl "http://localhost:3333/works?q=CRISPR&limit=10"
+
+# Get by DOI
+curl "http://localhost:3333/works/10.1038/nature12373"
+
+# Batch DOI lookup
+curl -X POST "http://localhost:3333/works/batch" \
+  -H "Content-Type: application/json" \
+  -d '{"dois": ["10.1038/nature12373", "10.1126/science.aax0758"]}'
+
+# Database info
+curl "http://localhost:3333/info"
+```
+
+Remote access via SSH tunnel:
+```bash
+# On local machine
+ssh -L 3333:127.0.0.1:3333 nas
+
+# Python client
+from crossref_local import configure_remote
+configure_remote("http://localhost:3333")
+```
+
+</details>
+
+<details>
+<summary><strong>MCP Server (Claude Desktop)</strong></summary>
+
+Run as MCP server for Claude Desktop integration:
+```bash
+crossref-local serve
+```
+
+Add to Claude Desktop config (`~/.config/claude/claude_desktop_config.json`):
+```json
+{
+  "mcpServers": {
+    "crossref-local": {
+      "command": "crossref-local",
+      "args": ["serve"],
+      "env": {
+        "CROSSREF_LOCAL_DB": "/path/to/crossref.db"
+      }
+    }
+  }
+}
+```
+
+Available tools:
+- `search_works` - Full-text search across 167M+ papers
+- `get_work` - Get paper by DOI
+- `count_works` - Count matching papers
+- `database_info` - Database statistics
+- `calculate_impact_factor` - Journal impact factor
+
+</details>
+
 <details>
 <summary><strong>Impact Factor</strong></summary>
 
@@ -188,6 +273,27 @@ Searching 167M records in milliseconds via FTS5.
 
 </details>
 
+<details>
+<summary><strong>Related Projects</strong></summary>
+
+**[openalex-local](https://github.com/ywatanabe1989/openalex-local)** - Sister project with OpenAlex data:
+
+| Feature | crossref-local | openalex-local |
+|---------|----------------|----------------|
+| Works | 167M | 284M |
+| Abstracts | ~21% | ~45-60% |
+| Update frequency | Real-time | Monthly |
+| DOI authority | ✓ (source) | Uses CrossRef |
+| Citations | Raw references | Linked works |
+| Concepts/Topics | ❌ | ✓ |
+| Author IDs | ❌ | ✓ |
+| Best for | DOI lookup, raw refs | Semantic search |
+
+**When to use CrossRef**: Real-time DOI updates, raw reference parsing, authoritative metadata.
+**When to use OpenAlex**: Semantic search, citation analysis, topic discovery.
+
+</details>
+
 
 ---
 

{crossref_local-0.3.0 → crossref_local-0.3.1}/README.md

@@ -92,24 +92,93 @@ async def main():
 crossref-local search "CRISPR genome editing" -n 5
 crossref-local get 10.1038/nature12373
 crossref-local impact-factor Nature -y 2023  # IF: 54.067
+crossref-local info  # Database stats
 ```
 
 With abstracts (`-a` flag):
 ```
-$ crossref-local search "CRISPR" -n 1 -a
+$ crossref-local search "RS-1 enhances CRISPR" -n 1 -a
 
-Found 87,473 matches in 18.2ms
+Found 4 matches in 128.4ms
 
 1. RS-1 enhances CRISPR/Cas9- and TALEN-mediated knock-in efficiency (2016)
    DOI: 10.1038/ncomms10548
    Journal: Nature Communications
    Abstract: Zinc-finger nuclease, transcription activator-like effector nuclease
-   and CRISPR/Cas9 are becoming major tools for genome editing. Importantly,
-   knock-in in several non-rodent species has been finally achieved...
+   and CRISPR/Cas9 are becoming major tools for genome editing...
 ```
 
 </details>
 
+<details>
+<summary><strong>HTTP API</strong></summary>
+
+Start the FastAPI server:
+```bash
+crossref-local api --host 0.0.0.0 --port 3333
+```
+
+Endpoints:
+```bash
+# Search works (FTS5)
+curl "http://localhost:3333/works?q=CRISPR&limit=10"
+
+# Get by DOI
+curl "http://localhost:3333/works/10.1038/nature12373"
+
+# Batch DOI lookup
+curl -X POST "http://localhost:3333/works/batch" \
+  -H "Content-Type: application/json" \
+  -d '{"dois": ["10.1038/nature12373", "10.1126/science.aax0758"]}'
+
+# Database info
+curl "http://localhost:3333/info"
+```
+
+Remote access via SSH tunnel:
+```bash
+# On local machine
+ssh -L 3333:127.0.0.1:3333 nas
+
+# Python client
+from crossref_local import configure_remote
+configure_remote("http://localhost:3333")
+```
+
+</details>
+
+<details>
+<summary><strong>MCP Server (Claude Desktop)</strong></summary>
+
+Run as MCP server for Claude Desktop integration:
+```bash
+crossref-local serve
+```
+
+Add to Claude Desktop config (`~/.config/claude/claude_desktop_config.json`):
+```json
+{
+  "mcpServers": {
+    "crossref-local": {
+      "command": "crossref-local",
+      "args": ["serve"],
+      "env": {
+        "CROSSREF_LOCAL_DB": "/path/to/crossref.db"
+      }
+    }
+  }
+}
+```
+
+Available tools:
+- `search_works` - Full-text search across 167M+ papers
+- `get_work` - Get paper by DOI
+- `count_works` - Count matching papers
+- `database_info` - Database statistics
+- `calculate_impact_factor` - Journal impact factor
+
+</details>
+
 <details>
 <summary><strong>Impact Factor</strong></summary>
 
@@ -159,6 +228,27 @@ Searching 167M records in milliseconds via FTS5.
 
 </details>
 
+<details>
+<summary><strong>Related Projects</strong></summary>
+
+**[openalex-local](https://github.com/ywatanabe1989/openalex-local)** - Sister project with OpenAlex data:
+
+| Feature | crossref-local | openalex-local |
+|---------|----------------|----------------|
+| Works | 167M | 284M |
+| Abstracts | ~21% | ~45-60% |
+| Update frequency | Real-time | Monthly |
+| DOI authority | ✓ (source) | Uses CrossRef |
+| Citations | Raw references | Linked works |
+| Concepts/Topics | ❌ | ✓ |
+| Author IDs | ❌ | ✓ |
+| Best for | DOI lookup, raw refs | Semantic search |
+
+**When to use CrossRef**: Real-time DOI updates, raw reference parsing, authoritative metadata.
+**When to use OpenAlex**: Semantic search, citation analysis, topic discovery.
+
+</details>
+
 
 ---
 

{crossref_local-0.3.0 → crossref_local-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crossref-local"
-version = "0.3.0"
+version = "0.3.1"
 description = "Local CrossRef database with 167M+ works and full-text search"
 readme = "README.md"
 license = "AGPL-3.0"
@@ -41,15 +41,27 @@ dependencies = [
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
+    "pytest-asyncio>=0.21",
 ]
 viz = [
     "pyvis>=0.3",
     "networkx>=3.0",
     "matplotlib>=3.7",
 ]
+mcp = [
+    "fastmcp>=0.4",
+]
+api = [
+    "fastapi>=0.100",
+    "uvicorn>=0.20",
+]
+all = [
+    "crossref-local[dev,viz,mcp,api]",
+]
 
 [project.scripts]
 crossref-local = "crossref_local.cli:main"
+crossref-local-mcp = "crossref_local.mcp_server:main"
 
 [project.urls]
 Homepage = "https://github.com/ywatanabe1989/crossref_local"
@@ -61,3 +73,4 @@ packages = ["src/crossref_local"]
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 python_files = ["test_*.py"]
+asyncio_mode = "auto"

crossref_local-0.3.1/scripts/README.md

@@ -0,0 +1,132 @@
+# Scripts Directory
+
+Helper scripts for CrossRef Local database management and deployment.
+
+## Directory Structure
+
+```
+scripts/
+├── database/         # Database maintenance scripts
+│   ├── 00_rebuild_all.sh
+│   ├── 02_create_missing_indexes.sh
+│   ├── 99_db_info.sh
+│   ├── 99_maintain_indexes.sh
+│   └── 99_switch_to_optimized.sh
+└── deployment/       # Container deployment
+    ├── install_apptainer.sh
+    ├── build_apptainer.sh
+    ├── run_apptainer.sh
+    └── run_docker.sh
+```
+
+## Database Scripts
+
+### `database/00_rebuild_all.sh`
+
+Full database rebuild from Crossref Public Data File.
+
+```bash
+./scripts/database/00_rebuild_all.sh --help
+./scripts/database/00_rebuild_all.sh all       # Full rebuild (~10-14 days)
+./scripts/database/00_rebuild_all.sh fts       # Rebuild FTS index only
+```
+
+### `database/02_create_missing_indexes.sh`
+
+Create missing indexes on citations table.
+
+```bash
+./scripts/database/02_create_missing_indexes.sh --help
+./scripts/database/02_create_missing_indexes.sh           # Default database
+./scripts/database/02_create_missing_indexes.sh --dry-run # Preview changes
+```
+
+### `database/99_db_info.sh`
+
+Display database schema, tables, indices, and row counts.
+
+```bash
+./scripts/database/99_db_info.sh --help
+./scripts/database/99_db_info.sh           # Quick summary
+./scripts/database/99_db_info.sh --full    # Full schema dump
+./scripts/database/99_db_info.sh --tables  # Tables and counts only
+```
+
+### `database/99_maintain_indexes.sh`
+
+Check and create missing database indexes.
+
+```bash
+./scripts/database/99_maintain_indexes.sh --help
+./scripts/database/99_maintain_indexes.sh              # Check/create indexes
+./scripts/database/99_maintain_indexes.sh --check-only # Only check, don't modify
+```
+
+## Deployment Scripts
+
+### `deployment/install_apptainer.sh`
+
+Install Apptainer container runtime.
+
+```bash
+./scripts/deployment/install_apptainer.sh --help
+./scripts/deployment/install_apptainer.sh           # Install default version
+./scripts/deployment/install_apptainer.sh -v 1.2.5  # Specific version
+```
+
+### `deployment/build_apptainer.sh`
+
+Build Apptainer/Singularity container image.
+
+```bash
+./scripts/deployment/build_apptainer.sh --help
+./scripts/deployment/build_apptainer.sh         # Build with defaults
+./scripts/deployment/build_apptainer.sh --force # Force rebuild
+```
+
+### `deployment/run_apptainer.sh`
+
+Run crossref-local with Apptainer container.
+
+```bash
+./scripts/deployment/run_apptainer.sh --help
+./scripts/deployment/run_apptainer.sh              # Start API server
+./scripts/deployment/run_apptainer.sh search CRISPR # Run search
+./scripts/deployment/run_apptainer.sh shell        # Interactive shell
+```
+
+### `deployment/run_docker.sh`
+
+Run crossref-local with Docker container.
+
+```bash
+./scripts/deployment/run_docker.sh --help
+./scripts/deployment/run_docker.sh              # Start API server
+./scripts/deployment/run_docker.sh search CRISPR # Run search
+./scripts/deployment/run_docker.sh shell        # Interactive shell
+```
+
+## Quick Reference
+
+**First-time setup:**
+```bash
+# 1. Check/create indexes (do once, takes hours)
+./scripts/database/99_maintain_indexes.sh
+
+# 2. Check database info
+./scripts/database/99_db_info.sh
+
+# 3. Run tests
+make test
+```
+
+**Long-running operations:**
+```bash
+# Use screen for operations that take hours/days
+screen -S rebuild
+./scripts/database/00_rebuild_all.sh fts
+# Ctrl-A D to detach
+screen -r rebuild  # Reattach
+```
+
+<!-- EOF -->

crossref_local-0.3.1/scripts/database/02_create_missing_indexes.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+# -*- coding: utf-8 -*-
+# File: scripts/database/02_create_missing_indexes.sh
+# Description: Create missing indexes on citations table for impact factor calculations
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
+DB="${CROSSREF_LOCAL_DB:-${PROJECT_ROOT}/data/crossref.db}"
+LOG_DIR="${PROJECT_ROOT}/logs"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+usage() {
+    cat << EOF
+Usage: $(basename "$0") [OPTIONS]
+
+Create missing indexes on citations table for impact factor calculations.
+Run in screen for long-running index creation.
+
+OPTIONS:
+    -d, --db PATH    Database path (default: \$CROSSREF_LOCAL_DB or data/crossref.db)
+    -n, --dry-run    Show what would be done without executing
+    -h, --help       Show this help message
+
+EXAMPLES:
+    $(basename "$0")                    # Create indexes on default database
+    $(basename "$0") --db /path/to.db   # Custom database path
+
+    # Long-running (use screen):
+    screen -S index-rebuild
+    $(basename "$0")
+    # Ctrl-A D to detach
+EOF
+}
+
+check_db() {
+    if [[ ! -f "$DB" ]]; then
+        echo -e "${RED}ERROR: Database not found: ${DB}${NC}" >&2
+        exit 1
+    fi
+}
+
+create_indexes() {
+    local LOG="${LOG_DIR}/index_creation_$(date +%Y%m%d_%H%M%S).log"
+    mkdir -p "$LOG_DIR"
+
+    echo "=== Index Creation Started: $(date) ===" | tee -a "$LOG"
+    echo "Database: $DB" | tee -a "$LOG"
+
+    echo "" | tee -a "$LOG"
+    echo "Checking existing indexes..." | tee -a "$LOG"
+    sqlite3 "$DB" ".indexes citations" | tee -a "$LOG"
+
+    echo "" | tee -a "$LOG"
+    echo "[$(date)] Creating idx_citations_cited_new ON citations(cited_doi, citing_year)..." | tee -a "$LOG"
+    time sqlite3 "$DB" "CREATE INDEX IF NOT EXISTS idx_citations_cited_new ON citations(cited_doi, citing_year);" 2>&1 | tee -a "$LOG"
+    echo -e "${GREEN}[$(date)] idx_citations_cited_new completed${NC}" | tee -a "$LOG"
+
+    echo "" | tee -a "$LOG"
+    echo "[$(date)] Creating idx_citations_year_new ON citations(citing_year)..." | tee -a "$LOG"
+    time sqlite3 "$DB" "CREATE INDEX IF NOT EXISTS idx_citations_year_new ON citations(citing_year);" 2>&1 | tee -a "$LOG"
+    echo -e "${GREEN}[$(date)] idx_citations_year_new completed${NC}" | tee -a "$LOG"
+
+    echo "" | tee -a "$LOG"
+    echo "[$(date)] Verifying indexes..." | tee -a "$LOG"
+    sqlite3 "$DB" ".indexes citations" | tee -a "$LOG"
+
+    echo "" | tee -a "$LOG"
+    echo "=== Index Creation Completed: $(date) ===" | tee -a "$LOG"
+    echo "Log saved to: $LOG"
+}
+
+# Parse arguments
+DRY_RUN=0
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -d|--db) DB="$2"; shift 2 ;;
+        -n|--dry-run) DRY_RUN=1; shift ;;
+        -h|--help) usage; exit 0 ;;
+        *) echo -e "${RED}Unknown option: $1${NC}"; usage; exit 1 ;;
+    esac
+done
+
+check_db
+
+if [[ "$DRY_RUN" == "1" ]]; then
+    echo -e "${YELLOW}DRY RUN - Would create indexes on: $DB${NC}"
+    echo "Indexes to create:"
+    echo "  - idx_citations_cited_new ON citations(cited_doi, citing_year)"
+    echo "  - idx_citations_year_new ON citations(citing_year)"
+    exit 0
+fi
+
+create_indexes