lit-acquisition 0.1.0__tar.gz

lit_acquisition-0.1.0/.gitignore

@@ -0,0 +1,24 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# IDE
+.idea/
+.vscode/
+*.swp

lit_acquisition-0.1.0/PKG-INFO

@@ -0,0 +1,219 @@
+Metadata-Version: 2.4
+Name: lit-acquisition
+Version: 0.1.0
+Summary: Multilingual biomedical literature acquisition toolkit - search, download, and classify academic papers from 15+ providers
+Author: Lingua Seeker Maintainers
+License-Expression: MIT
+Keywords: acquisition,biomedical,crossref,literature,multilingual,openalex,pubmed
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.12
+Requires-Dist: httpx[socks]>=0.27.0
+Requires-Dist: loguru>=0.7.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: pyjstage2>=0.1.2
+Requires-Dist: pymupdf>=1.27.2
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.2.0; extra == 'dev'
+Requires-Dist: ruff>=0.5.0; extra == 'dev'
+Provides-Extra: rust-io
+Requires-Dist: rust-io; extra == 'rust-io'
+Provides-Extra: web-search
+Requires-Dist: firecrawl-py>=4.28.2; extra == 'web-search'
+Requires-Dist: google-search-results>=2.4.2; extra == 'web-search'
+Requires-Dist: tavily-python>=0.5.0; extra == 'web-search'
+Description-Content-Type: text/markdown
+
+# lit-acquisition
+
+Multilingual biomedical literature acquisition toolkit — search, download, and classify academic papers from 15+ providers.
+
+## Features
+
+- **15+ provider integrations**: Crossref, PubMed, OpenAlex, EuropePMC, DOAJ, J-STAGE, arXiv, bioRxiv, medRxiv, SciELO, BASE, CORE, OpenAIRE, CiNii, Unpaywall
+- **Multilingual search**: Query translation into 6 languages (en, zh, ja, de, fr, ru) with language-aware provider routing
+- **PDF download**: DOI → Unpaywall OA resolution, PMCID → EuropePMC render, direct URL with HTML→PDF redirect handling
+- **Relevance gate**: LLM-based classification to filter irrelevant downloads
+- **Literature type classification**: Keyword-based classification (case report, sequencing, functional study) across 10+ languages
+- **Web search fallback**: Firecrawl, Tavily, and SerpApi adapters for discovering papers beyond academic APIs
+- **Provider health tracking**: Automatic health monitoring with sliding-window stats and unhealthy provider deprioritization
+
+## Installation
+
+```bash
+pip install lit-acquisition
+```
+
+With web search support:
+
+```bash
+pip install "lit-acquisition[web-search]"
+```
+
+With Rust native extensions (faster HTTP I/O):
+
+```bash
+pip install "lit-acquisition[rust-io]"
+```
+
+## Quick Start
+
+### Configure
+
+```python
+from lit_acquisition import configure
+
+configure(
+    # LLM for relevance gate and query translation
+    llm_base_url="https://api.openai.com/v1",
+    llm_api_key="sk-...",
+    llm_model="gpt-4o",
+
+    # Optional: dedicated translation model
+    translation_base_url="https://api.openai.com/v1",
+    translation_api_key="sk-...",
+    translation_model="gpt-4o-mini",
+
+    # Optional: web search providers
+    firecrawl_api_key="fc-...",
+    tavily_api_key="tvly-...",
+
+    # Optional: network proxy
+    proxy="http://127.0.0.1:7890",
+
+    # Optional: PubMed API key (higher rate limits)
+    pubmed_api_key="...",
+)
+```
+
+Or via environment variables:
+
+```bash
+export LIT_LLM_BASE_URL=https://api.openai.com/v1
+export LIT_LLM_API_KEY=sk-...
+export LIT_LLM_MODEL=gpt-4o
+```
+
+### Search a Single Provider
+
+```python
+import asyncio
+from lit_acquisition import search_provider
+
+async def main():
+    result = await search_provider(
+        provider="crossref",
+        query="MECP2 Rett syndrome case report",
+        limit=20,
+    )
+    print(f"Found {len(result.items)} items")
+    for item in result.items:
+        print(f"  - {item.get('title', 'untitled')}")
+
+asyncio.run(main())
+```
+
+### Run the Full Multilingual Pipeline
+
+```python
+import asyncio
+from lit_acquisition import multilingual_acquisition_workflow
+
+async def main():
+    result = await multilingual_acquisition_workflow({
+        "query": "MECP2 Rett syndrome case report",
+        "action": "search",          # or "download" to also fetch PDFs
+        "limit": 30,
+        "language": "auto",
+        "relevance_gate": True,       # LLM-based relevance filtering
+        "literature_types": ["case_report"],
+    })
+    print(f"Success: {result['success']}")
+    print(f"Items: {len(result['items'])}")
+    print(f"Downloads: {len(result['downloads'])}")
+
+asyncio.run(main())
+```
+
+### Download PDFs
+
+```python
+import asyncio
+from lit_acquisition import download_file_from_url
+
+async def main():
+    file_path, final_url, warnings = await download_file_from_url(
+        url="https://example.com/paper.pdf",
+        download_path="./downloads",
+        filename_stem="my_paper",
+    )
+    print(f"Downloaded to: {file_path}")
+
+asyncio.run(main())
+```
+
+### Use the PubMed Service
+
+```python
+import asyncio
+from lit_acquisition import get_pubmed_service
+
+async def main():
+    svc = get_pubmed_service()
+    candidates = await svc.search_candidates("BRCA1 breast cancer", candidate_limit=10)
+    for c in candidates:
+        print(f"  PMID: {c.pmid}, Title: {c.title}")
+
+asyncio.run(main())
+```
+
+## Supported Providers
+
+| Provider | Search | Download | Notes |
+|----------|--------|----------|-------|
+| Crossref | ✓ | — | Metadata only |
+| Unpaywall | ✓ | ✓ | OA PDF resolution via DOI |
+| OpenAlex | ✓ | — | Metadata only |
+| EuropePMC | ✓ | ✓ | Full text via PMCID render |
+| PMC | ✓ | ✓ | esearch + esummary |
+| DOAJ | ✓ | — | Metadata only |
+| J-STAGE | ✓ | — | Japanese literature |
+| CiNii | ✓ | — | Japanese research |
+| arXiv | ✓ | ✓ | Preprint server |
+| bioRxiv | ✓ | ✓ | Preprint server |
+| medRxiv | ✓ | ✓ | Preprint server |
+| SciELO | ✓ | — | Latin American literature |
+| BASE | ✓ | — | Multidisciplinary |
+| CORE | ✓ | — | Open access |
+| OpenAIRE | ✓ | — | European research |
+
+## Configuration Reference
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `LIT_LLM_BASE_URL` | LLM API base URL | — |
+| `LIT_LLM_API_KEY` | LLM API key | — |
+| `LIT_LLM_MODEL` | LLM model name | — |
+| `LIT_LLM_API_KEYS` | Comma-separated API key pool | — |
+| `LIT_LLM_MAX_TOKENS` | Max tokens for LLM | `8192` |
+| `LIT_TRANSLATION_BASE_URL` | Translation LLM base URL | Falls back to LLM config |
+| `LIT_TRANSLATION_API_KEY` | Translation LLM API key | Falls back to LLM config |
+| `LIT_TRANSLATION_MODEL` | Translation LLM model | Falls back to LLM config |
+| `LIT_FIRECRAWL_API_KEY` | Firecrawl API key | — |
+| `LIT_TAVILY_API_KEY` | Tavily API key | — |
+| `LIT_SERPAPI_API_KEY` | SerpApi API key | — |
+| `LIT_PROXY` | HTTP/HTTPS/SOCKS proxy URL | — |
+| `LIT_NO_PROXY` | Comma-separated proxy bypass domains | `cn,ncbi.nlm.nih.gov,...` |
+| `LIT_PUBMED_API_KEY` | PubMed eutils API key | — |
+
+## License
+
+MIT

lit_acquisition-0.1.0/README.md

@@ -0,0 +1,187 @@
+# lit-acquisition
+
+Multilingual biomedical literature acquisition toolkit — search, download, and classify academic papers from 15+ providers.
+
+## Features
+
+- **15+ provider integrations**: Crossref, PubMed, OpenAlex, EuropePMC, DOAJ, J-STAGE, arXiv, bioRxiv, medRxiv, SciELO, BASE, CORE, OpenAIRE, CiNii, Unpaywall
+- **Multilingual search**: Query translation into 6 languages (en, zh, ja, de, fr, ru) with language-aware provider routing
+- **PDF download**: DOI → Unpaywall OA resolution, PMCID → EuropePMC render, direct URL with HTML→PDF redirect handling
+- **Relevance gate**: LLM-based classification to filter irrelevant downloads
+- **Literature type classification**: Keyword-based classification (case report, sequencing, functional study) across 10+ languages
+- **Web search fallback**: Firecrawl, Tavily, and SerpApi adapters for discovering papers beyond academic APIs
+- **Provider health tracking**: Automatic health monitoring with sliding-window stats and unhealthy provider deprioritization
+
+## Installation
+
+```bash
+pip install lit-acquisition
+```
+
+With web search support:
+
+```bash
+pip install "lit-acquisition[web-search]"
+```
+
+With Rust native extensions (faster HTTP I/O):
+
+```bash
+pip install "lit-acquisition[rust-io]"
+```
+
+## Quick Start
+
+### Configure
+
+```python
+from lit_acquisition import configure
+
+configure(
+    # LLM for relevance gate and query translation
+    llm_base_url="https://api.openai.com/v1",
+    llm_api_key="sk-...",
+    llm_model="gpt-4o",
+
+    # Optional: dedicated translation model
+    translation_base_url="https://api.openai.com/v1",
+    translation_api_key="sk-...",
+    translation_model="gpt-4o-mini",
+
+    # Optional: web search providers
+    firecrawl_api_key="fc-...",
+    tavily_api_key="tvly-...",
+
+    # Optional: network proxy
+    proxy="http://127.0.0.1:7890",
+
+    # Optional: PubMed API key (higher rate limits)
+    pubmed_api_key="...",
+)
+```
+
+Or via environment variables:
+
+```bash
+export LIT_LLM_BASE_URL=https://api.openai.com/v1
+export LIT_LLM_API_KEY=sk-...
+export LIT_LLM_MODEL=gpt-4o
+```
+
+### Search a Single Provider
+
+```python
+import asyncio
+from lit_acquisition import search_provider
+
+async def main():
+    result = await search_provider(
+        provider="crossref",
+        query="MECP2 Rett syndrome case report",
+        limit=20,
+    )
+    print(f"Found {len(result.items)} items")
+    for item in result.items:
+        print(f"  - {item.get('title', 'untitled')}")
+
+asyncio.run(main())
+```
+
+### Run the Full Multilingual Pipeline
+
+```python
+import asyncio
+from lit_acquisition import multilingual_acquisition_workflow
+
+async def main():
+    result = await multilingual_acquisition_workflow({
+        "query": "MECP2 Rett syndrome case report",
+        "action": "search",          # or "download" to also fetch PDFs
+        "limit": 30,
+        "language": "auto",
+        "relevance_gate": True,       # LLM-based relevance filtering
+        "literature_types": ["case_report"],
+    })
+    print(f"Success: {result['success']}")
+    print(f"Items: {len(result['items'])}")
+    print(f"Downloads: {len(result['downloads'])}")
+
+asyncio.run(main())
+```
+
+### Download PDFs
+
+```python
+import asyncio
+from lit_acquisition import download_file_from_url
+
+async def main():
+    file_path, final_url, warnings = await download_file_from_url(
+        url="https://example.com/paper.pdf",
+        download_path="./downloads",
+        filename_stem="my_paper",
+    )
+    print(f"Downloaded to: {file_path}")
+
+asyncio.run(main())
+```
+
+### Use the PubMed Service
+
+```python
+import asyncio
+from lit_acquisition import get_pubmed_service
+
+async def main():
+    svc = get_pubmed_service()
+    candidates = await svc.search_candidates("BRCA1 breast cancer", candidate_limit=10)
+    for c in candidates:
+        print(f"  PMID: {c.pmid}, Title: {c.title}")
+
+asyncio.run(main())
+```
+
+## Supported Providers
+
+| Provider | Search | Download | Notes |
+|----------|--------|----------|-------|
+| Crossref | ✓ | — | Metadata only |
+| Unpaywall | ✓ | ✓ | OA PDF resolution via DOI |
+| OpenAlex | ✓ | — | Metadata only |
+| EuropePMC | ✓ | ✓ | Full text via PMCID render |
+| PMC | ✓ | ✓ | esearch + esummary |
+| DOAJ | ✓ | — | Metadata only |
+| J-STAGE | ✓ | — | Japanese literature |
+| CiNii | ✓ | — | Japanese research |
+| arXiv | ✓ | ✓ | Preprint server |
+| bioRxiv | ✓ | ✓ | Preprint server |
+| medRxiv | ✓ | ✓ | Preprint server |
+| SciELO | ✓ | — | Latin American literature |
+| BASE | ✓ | — | Multidisciplinary |
+| CORE | ✓ | — | Open access |
+| OpenAIRE | ✓ | — | European research |
+
+## Configuration Reference
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `LIT_LLM_BASE_URL` | LLM API base URL | — |
+| `LIT_LLM_API_KEY` | LLM API key | — |
+| `LIT_LLM_MODEL` | LLM model name | — |
+| `LIT_LLM_API_KEYS` | Comma-separated API key pool | — |
+| `LIT_LLM_MAX_TOKENS` | Max tokens for LLM | `8192` |
+| `LIT_TRANSLATION_BASE_URL` | Translation LLM base URL | Falls back to LLM config |
+| `LIT_TRANSLATION_API_KEY` | Translation LLM API key | Falls back to LLM config |
+| `LIT_TRANSLATION_MODEL` | Translation LLM model | Falls back to LLM config |
+| `LIT_FIRECRAWL_API_KEY` | Firecrawl API key | — |
+| `LIT_TAVILY_API_KEY` | Tavily API key | — |
+| `LIT_SERPAPI_API_KEY` | SerpApi API key | — |
+| `LIT_PROXY` | HTTP/HTTPS/SOCKS proxy URL | — |
+| `LIT_NO_PROXY` | Comma-separated proxy bypass domains | `cn,ncbi.nlm.nih.gov,...` |
+| `LIT_PUBMED_API_KEY` | PubMed eutils API key | — |
+
+## License
+
+MIT

lit_acquisition-0.1.0/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "lit-acquisition"
+version = "0.1.0"
+description = "Multilingual biomedical literature acquisition toolkit - search, download, and classify academic papers from 15+ providers"
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+authors = [
+    { name = "Lingua Seeker Maintainers" },
+]
+keywords = ["literature", "biomedical", "acquisition", "pubmed", "crossref", "openalex", "multilingual"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+]
+dependencies = [
+    "httpx[socks]>=0.27.0",
+    "pydantic>=2.7.0",
+    "loguru>=0.7.0",
+    "pymupdf>=1.27.2",
+    "openai>=1.0.0",
+    "pyjstage2>=0.1.2",
+]
+
+[project.optional-dependencies]
+web-search = [
+    "firecrawl-py>=4.28.2",
+    "tavily-python>=0.5.0",
+    "google-search-results>=2.4.2",
+]
+rust-io = [
+    "rust-io",
+]
+dev = [
+    "pytest>=8.2.0",
+    "pytest-asyncio>=0.23.0",
+    "ruff>=0.5.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lit_acquisition"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.ruff.lint]
+ignore = ["BLE001", "S110", "PLC0206"]

lit_acquisition-0.1.0/src/lit_acquisition/__init__.py

@@ -0,0 +1,150 @@
+"""lit-acquisition: Multilingual biomedical literature acquisition toolkit.
+
+Search, download, and classify academic papers from 15+ providers including
+Crossref, PubMed, OpenAlex, EuropePMC, DOAJ, J-STAGE, arXiv, and more.
+
+Quick start::
+
+    from lit_acquisition import configure, search_provider, download_file_from_url
+
+    # Configure your LLM provider (for query translation and relevance gate)
+    configure(
+        llm_base_url="https://api.openai.com/v1",
+        llm_api_key="sk-...",
+        llm_model="gpt-4o",
+    )
+
+    # Search a single provider
+    result = await search_provider(provider="crossref", query="MECP2 Rett syndrome")
+
+    # Or run the full multilingual pipeline
+    from lit_acquisition import multilingual_acquisition_workflow
+    result = await multilingual_acquisition_workflow({"query": "MECP2 Rett syndrome", "action": "search"})
+"""
+
+from .config import (
+    LitAcquisitionConfig,
+    LLMConfig,
+    NetworkConfig,
+    PubMedConfig,
+    TranslationLLMConfig,
+    WebSearchConfig,
+    configure,
+    get_config,
+    reload_config,
+)
+from .contracts import (
+    DownloadResult,
+    OnlineAcquisitionGatewayRequest,
+    OnlineAcquisitionGatewayResult,
+    OnlineAcquisitionItem,
+    OnlineAcquisitionRequest,
+    OnlineAcquisitionResponse,
+    OnlineAcquisitionRouteInfo,
+    OnlineAcquisitionSourceTraceEntry,
+)
+from .gateway import (
+    call_provider,
+    call_provider_with_retry,
+    download_file_from_url,
+    resolve_oa_url,
+    search_provider,
+)
+from .literature_type_classifier import (
+    LiteratureType,
+    classify_item,
+    classify_items,
+    filter_by_type,
+)
+from .normalizers import normalize_items
+from .provider_health import ProviderHealthTracker, ProviderStats, get_health_tracker
+from .pubmed_service import (
+    OnlineAcquisitionPubMedArticle,
+    OnlineAcquisitionPubMedCandidate,
+    OnlineAcquisitionPubMedService,
+    get_pubmed_service,
+)
+from .query_translator import (
+    TARGET_LANGUAGES,
+    TranslatedQueries,
+    translate_query,
+)
+from .relevance_gate import (
+    RelevanceGateResult,
+    RelevanceJudgment,
+    run_relevance_gate,
+)
+from .search_service import (
+    build_provider_plan,
+    dedupe_candidates,
+    rank_candidates,
+    search_multilingual,
+    search_parallel,
+)
+from .workflow import (
+    multilingual_acquisition_workflow,
+    online_acquisition_workflow,
+)
+
+__all__ = [
+    "TARGET_LANGUAGES",
+    "DownloadResult",
+    "LLMConfig",
+    # Config
+    "LitAcquisitionConfig",
+    # Literature type classifier
+    "LiteratureType",
+    "NetworkConfig",
+    "OnlineAcquisitionGatewayRequest",
+    "OnlineAcquisitionGatewayResult",
+    # Contracts
+    "OnlineAcquisitionItem",
+    "OnlineAcquisitionPubMedArticle",
+    "OnlineAcquisitionPubMedCandidate",
+    # PubMed
+    "OnlineAcquisitionPubMedService",
+    "OnlineAcquisitionRequest",
+    "OnlineAcquisitionResponse",
+    "OnlineAcquisitionRouteInfo",
+    "OnlineAcquisitionSourceTraceEntry",
+    "ProviderHealthTracker",
+    "ProviderStats",
+    "PubMedConfig",
+    # Relevance gate
+    "RelevanceGateResult",
+    "RelevanceJudgment",
+    # Query translator
+    "TranslatedQueries",
+    "TranslationLLMConfig",
+    "WebSearchConfig",
+    # Search
+    "build_provider_plan",
+    # Gateway
+    "call_provider",
+    "call_provider_with_retry",
+    "classify_item",
+    "classify_items",
+    "configure",
+    "dedupe_candidates",
+    "download_file_from_url",
+    "filter_by_type",
+    "get_config",
+    # Provider health
+    "get_health_tracker",
+    "get_pubmed_service",
+    "multilingual_acquisition_workflow",
+    # Normalizers
+    "normalize_items",
+    # Workflow
+    "online_acquisition_workflow",
+    "rank_candidates",
+    "reload_config",
+    "resolve_oa_url",
+    "run_relevance_gate",
+    "search_multilingual",
+    "search_parallel",
+    "search_provider",
+    "translate_query",
+]
+
+__version__ = "0.1.0"

lit_acquisition-0.1.0/src/lit_acquisition/_parse_service.py

@@ -0,0 +1,29 @@
+"""Optional MinerU parse service integration.
+
+The multilingual workflow uses MinerU for early PDF parsing before the
+relevance gate.  When the ``parse_document`` module is not available
+(e.g. when lit-acquisition is used standalone), batch parsing is skipped
+and the relevance gate falls back to PyMuPDF text extraction.
+"""
+
+from __future__ import annotations
+
+
+def create_parse_service():
+    """Create a parse service instance.
+
+    Returns:
+        A parse service object with ``parse_local_files`` method.
+
+    Raises:
+        ImportError: When the parse_document module is not available.
+    """
+    try:
+        from lit_acquisition_parse_document import create_parse_service as _create
+    except ImportError as exc:
+        raise ImportError(
+            "MinerU parse service is not available. "
+            "Install the 'lit-acquisition-parse' extra or use the workflow "
+            "without early batch parsing."
+        ) from exc
+    return _create()

lit_acquisition-0.1.0/src/lit_acquisition/_rust_io.py

@@ -0,0 +1,26 @@
+"""Optional Rust native extension loader.
+
+When the ``rust-io`` package is installed, HTTP I/O is handled by the
+fast Rust extension.  When it is not available, the module falls back
+to ``None``, and the gateway uses ``httpx`` as a pure-Python fallback.
+
+Install the Rust extension for better performance::
+
+    pip install rust-io
+"""
+
+from __future__ import annotations
+
+from loguru import logger
+
+NET_AVAILABLE: bool = False
+net_io = None
+
+_NATIVE_IMPORT_ERRORS = (ImportError, RuntimeError, SystemError, OSError)
+
+try:
+    import rust_io.net as net_io  # noqa: F401
+
+    NET_AVAILABLE = True
+except _NATIVE_IMPORT_ERRORS:
+    logger.debug("rust_io.net not available - using httpx fallback for HTTP I/O")