scitex 2.14.0__py3-none-any.whl

scitex/scholar/pipelines/ScholarPipelineSingle.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-14 (ywatanabe)"
+# File: src/scitex/scholar/pipelines/ScholarPipelineSingle.py
+"""
+Single paper acquisition pipeline orchestrator.
+
+Functionalities:
+  - Orchestrates full paper acquisition pipeline from query to storage
+  - Single command: query (DOI/title) + project -> complete paper in library
+  - Coordinates all workers: metadata, URLs, download, extraction, storage
+
+IO:
+  - output-files:
+    - library/MASTER/{paper_id}/metadata.json
+    - library/MASTER/{paper_id}/main.pdf
+    - library/MASTER/{paper_id}/content.txt
+    - library/MASTER/{paper_id}/tables.json
+    - library/MASTER/{paper_id}/images/
+    - library/{project}/{paper_id} -> ../MASTER/{paper_id}
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from typing import Optional
+
+from scitex import logging
+from scitex.scholar.storage import PaperIO
+
+from ._single_steps import PipelineHelpersMixin, PipelineStepsMixin
+
+logger = logging.getLogger(__name__)
+
+
+class ScholarPipelineSingle(PipelineStepsMixin, PipelineHelpersMixin):
+    """Orchestrates full paper acquisition pipeline."""
+
+    def __init__(
+        self, browser_mode: str = "interactive", chrome_profile: str = "system"
+    ):
+        self.name = self.__class__.__name__
+        self.browser_mode = browser_mode
+        self.chrome_profile = chrome_profile
+
+    async def process_single_paper(
+        self,
+        doi_or_title: str,
+        project: Optional[str] = None,
+        force: bool = False,
+    ):
+        """Process single paper from query (DOI or Title) to complete storage.
+
+        Pipeline:
+        1. Normalize as DOI
+        2. Create Paper object (resolve DOI from title if needed)
+        3. Add paper ID (8-digit hash)
+        4. Resolve metadata (ScholarEngine)
+        5. Setup browser
+        6. Find PDF URLs
+        7. Download PDF
+        8. Extract content (text, tables, images)
+        9. Link to project (if specified)
+        10. Log final status
+
+        Args:
+            doi_or_title: DOI or title string
+            project: Optional project name for symlinking
+            force: If True, ignore existing files and force fresh processing
+
+        Returns
+        -------
+            Tuple of (Complete Paper object, symlink_path)
+        """
+        # Step 1-3: Initialize
+        doi = self._step_01_normalize_as_doi(doi_or_title)
+        paper = await self._step_02_create_paper(doi, doi_or_title)
+        paper = self._step_03_add_paper_id(paper)
+
+        io = PaperIO(paper)
+        logger.info(f"{self.name}: Paper directory: {io.paper_dir}")
+
+        with logger.to(io.paper_dir / "logs" / "pipeline.log"):
+            # Step 4: Metadata
+            paper = await self._step_04_resolve_metadata(paper, io, force)
+
+            # Steps 5-7: Browser and PDF
+            browser_manager, context, auth_gateway = await self._step_05_setup_browser(
+                paper, io
+            )
+            if context:
+                await self._step_06_find_pdf_urls(
+                    paper, io, context, auth_gateway, force
+                )
+                await self._step_07_download_pdf(
+                    paper, io, context, auth_gateway, force
+                )
+            if browser_manager:
+                await browser_manager.close()
+
+            # Step 8: Content extraction
+            self._step_08_extract_content(io, force)
+
+            # Step 9-10: Finalize
+            symlink_path = self._step_09_link_to_project(paper, io, project)
+            self._step_10_log_final_status(io)
+
+            return paper, symlink_path
+
+
+def main(args):
+    """Run single paper pipeline."""
+    pipeline = ScholarPipelineSingle(
+        browser_mode=args.browser_mode, chrome_profile=args.chrome_profile
+    )
+    paper, symlink_path = asyncio.run(
+        pipeline.process_single_paper(
+            doi_or_title=args.doi_or_title,
+            project=args.project,
+            force=args.force,
+        )
+    )
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Orchestrate full paper acquisition pipeline"
+    )
+    parser.add_argument(
+        "--doi-or-title", type=str, required=True, help="DOI or paper title"
+    )
+    parser.add_argument(
+        "--project", type=str, default=None, help="Project name for symlinking"
+    )
+    parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode (default: stealth)",
+    )
+    parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        required=True,
+        help="Chrome profile name (default: system)",
+    )
+    parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        default=False,
+        help="Force fresh processing",
+    )
+    return parser.parse_args()
+
+
+def run_main() -> None:
+    """Initialize scitex framework, run main function, and cleanup."""
+    import sys
+
+    import matplotlib.pyplot as plt
+
+    import scitex as stx
+
+    args = parse_args()
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+        sys, plt, args=args, file=__file__, sdir_suffix=None, verbose=False, agg=True
+    )
+    exit_status = main(args)
+    stx.session.close(
+        CONFIG, verbose=False, notify=False, message="", exit_status=exit_status
+    )
+
+
+if __name__ == "__main__":
+    run_main()
+
+# Usage:
+# python -m scitex.scholar.pipelines.ScholarPipelineSingle \
+#     --doi-or-title "10.1038/nature12373" \
+#     --project test \
+#     --chrome-profile system \
+#     --browser-mode stealth
+
+# EOF

scitex/scholar/pipelines/SearchQueryParser.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File: ./src/scitex/scholar/pipelines/SearchQueryParser.py
+
+"""
+SearchQueryParser - Parse advanced search queries with filters
+
+Supports:
+  - Positive keywords: "hippocampus sharp wave"
+  - Negative keywords: "-seizure -epilepsy"
+  - Year range: "year:2020-2024" or "year:>2020"
+  - Impact factor: "impact_factor:>5"
+  - Open access: "open_access:true"
+  - Citation count: "citations:>100"
+
+Example:
+  query = "hippocampus sharp wave -seizure year:2020-2024 impact_factor:>5 open_access:true"
+  parser = SearchQueryParser(query)
+
+  filters = parser.get_filters()
+  # {
+  #   'positive_keywords': ['hippocampus', 'sharp', 'wave'],
+  #   'negative_keywords': ['seizure'],
+  #   'year_start': 2020,
+  #   'year_end': 2024,
+  #   'min_impact_factor': 5,
+  #   'open_access': True
+  # }
+"""
+
+import re
+from typing import Dict, List, Optional, Any
+from datetime import datetime
+
+
+class SearchQueryParser:
+    """Parse advanced search queries with filters."""
+
+    def __init__(self, query: str):
+        """Initialize parser with query string.
+
+        Args:
+            query: Search query with optional filters
+        """
+        self.original_query = query
+        self.positive_keywords: List[str] = []
+        self.negative_keywords: List[str] = []
+        self.year_start: Optional[int] = None
+        self.year_end: Optional[int] = None
+        self.min_impact_factor: Optional[float] = None
+        self.max_impact_factor: Optional[float] = None
+        self.min_citations: Optional[int] = None
+        self.max_citations: Optional[int] = None
+        self.open_access: Optional[bool] = None
+        self.document_type: Optional[str] = None
+
+        self._parse()
+
+    def _parse(self):
+        """Parse the query string."""
+        remaining_text = self.original_query
+
+        # Extract year filter: year:2020-2024 or year:>2020 or year:<2024
+        year_pattern = r"year:(\d{4})-(\d{4})|year:([><])(\d{4})"
+        year_matches = re.findall(year_pattern, remaining_text)
+        for match in year_matches:
+            if match[0] and match[1]:  # Range format: 2020-2024
+                self.year_start = int(match[0])
+                self.year_end = int(match[1])
+            elif match[2] and match[3]:  # Comparison format: >2020
+                year = int(match[3])
+                if match[2] == ">":
+                    self.year_start = year
+                else:  # <
+                    self.year_end = year
+        remaining_text = re.sub(year_pattern, "", remaining_text)
+
+        # Extract impact factor filter: impact_factor:>5 or if:>5
+        if_pattern = r"(?:impact_factor|if):([><])?([\d.]+)"
+        if_matches = re.findall(if_pattern, remaining_text)
+        for match in if_matches:
+            value = float(match[1])
+            if match[0] == ">":
+                self.min_impact_factor = value
+            elif match[0] == "<":
+                self.max_impact_factor = value
+            else:
+                self.min_impact_factor = value
+        remaining_text = re.sub(if_pattern, "", remaining_text)
+
+        # Extract citation count filter: citations:>100
+        cit_pattern = r"citations?:([><])?([\d]+)"
+        cit_matches = re.findall(cit_pattern, remaining_text)
+        for match in cit_matches:
+            value = int(match[1])
+            if match[0] == ">":
+                self.min_citations = value
+            elif match[0] == "<":
+                self.max_citations = value
+            else:
+                self.min_citations = value
+        remaining_text = re.sub(cit_pattern, "", remaining_text)
+
+        # Extract open access filter: open_access:true or oa:true
+        oa_pattern = r"(?:open_access|oa):(true|false|yes|no|1|0)"
+        oa_match = re.search(oa_pattern, remaining_text, re.IGNORECASE)
+        if oa_match:
+            oa_value = oa_match.group(1).lower()
+            self.open_access = oa_value in ["true", "yes", "1"]
+        remaining_text = re.sub(oa_pattern, "", remaining_text, flags=re.IGNORECASE)
+
+        # Extract document type filter: type:article or type:review
+        type_pattern = r"type:(article|review|conference|book)"
+        type_match = re.search(type_pattern, remaining_text, re.IGNORECASE)
+        if type_match:
+            self.document_type = type_match.group(1).lower()
+        remaining_text = re.sub(type_pattern, "", remaining_text, flags=re.IGNORECASE)
+
+        # Extract negative keywords (words starting with -)
+        neg_pattern = r"-(\w+)"
+        neg_matches = re.findall(neg_pattern, remaining_text)
+        self.negative_keywords = neg_matches
+        remaining_text = re.sub(neg_pattern, "", remaining_text)
+
+        # Extract quoted phrases as single keywords
+        quoted_pattern = r'"([^"]+)"'
+        quoted_matches = re.findall(quoted_pattern, remaining_text)
+        self.positive_keywords.extend(quoted_matches)
+        remaining_text = re.sub(quoted_pattern, "", remaining_text)
+
+        # Remaining words are positive keywords
+        words = remaining_text.split()
+        words = [w.strip() for w in words if w.strip()]
+        self.positive_keywords.extend(words)
+
+    def get_keyword_query(self) -> str:
+        """Get cleaned keyword query (positive keywords only)."""
+        return " ".join(self.positive_keywords)
+
+    def get_filters(self) -> Dict[str, Any]:
+        """Get all filters as dictionary."""
+        filters = {}
+
+        if self.positive_keywords:
+            filters["positive_keywords"] = self.positive_keywords
+        if self.negative_keywords:
+            filters["negative_keywords"] = self.negative_keywords
+        if self.year_start is not None:
+            filters["year_start"] = self.year_start
+        if self.year_end is not None:
+            filters["year_end"] = self.year_end
+        if self.min_impact_factor is not None:
+            filters["min_impact_factor"] = self.min_impact_factor
+        if self.max_impact_factor is not None:
+            filters["max_impact_factor"] = self.max_impact_factor
+        if self.min_citations is not None:
+            filters["min_citations"] = self.min_citations
+        if self.max_citations is not None:
+            filters["max_citations"] = self.max_citations
+        if self.open_access is not None:
+            filters["open_access"] = self.open_access
+        if self.document_type is not None:
+            filters["document_type"] = self.document_type
+
+        return filters
+
+    def get_api_filters(self) -> Dict[str, Any]:
+        """Get filters that can be pushed to API level."""
+        api_filters = {}
+
+        if self.year_start is not None:
+            api_filters["year_start"] = self.year_start
+        if self.year_end is not None:
+            api_filters["year_end"] = self.year_end
+        if self.open_access is not None:
+            api_filters["open_access"] = self.open_access
+        if self.document_type is not None:
+            api_filters["document_type"] = self.document_type
+
+        return api_filters
+
+    def get_post_filters(self) -> Dict[str, Any]:
+        """Get filters that must be applied post-API (client-side)."""
+        post_filters = {}
+
+        if self.negative_keywords:
+            post_filters["negative_keywords"] = self.negative_keywords
+        if self.min_impact_factor is not None:
+            post_filters["min_impact_factor"] = self.min_impact_factor
+        if self.max_impact_factor is not None:
+            post_filters["max_impact_factor"] = self.max_impact_factor
+        if self.min_citations is not None:
+            post_filters["min_citations"] = self.min_citations
+        if self.max_citations is not None:
+            post_filters["max_citations"] = self.max_citations
+
+        return post_filters
+
+    def to_pubmed_query(self) -> str:
+        """Convert to PubMed E-utilities query format."""
+        parts = []
+
+        # Positive keywords
+        if self.positive_keywords:
+            keyword_part = " ".join(self.positive_keywords)
+            parts.append(f"({keyword_part})[Title/Abstract]")
+
+        # Negative keywords
+        for neg in self.negative_keywords:
+            parts.append(f"NOT {neg}[Title/Abstract]")
+
+        # Year range
+        if self.year_start and self.year_end:
+            parts.append(f"{self.year_start}:{self.year_end}[pdat]")
+        elif self.year_start:
+            current_year = datetime.now().year
+            parts.append(f"{self.year_start}:{current_year}[pdat]")
+        elif self.year_end:
+            parts.append(f"1900:{self.year_end}[pdat]")
+
+        return " AND ".join(parts) if parts else ""
+
+    def to_crossref_params(self) -> Dict[str, Any]:
+        """Convert to CrossRef API parameters."""
+        params = {}
+
+        # Query
+        if self.positive_keywords:
+            params["query.bibliographic"] = " ".join(self.positive_keywords)
+
+        # Year filter
+        if self.year_start:
+            params["filter"] = f"from-pub-date:{self.year_start}"
+        if self.year_end:
+            if "filter" in params:
+                params["filter"] += f",until-pub-date:{self.year_end}"
+            else:
+                params["filter"] = f"until-pub-date:{self.year_end}"
+
+        return params
+
+    def to_arxiv_query(self) -> str:
+        """Convert to arXiv API query format."""
+        parts = []
+
+        # Positive keywords
+        for kw in self.positive_keywords:
+            if " " in kw:  # Quoted phrase
+                parts.append(f'all:"{kw}"')
+            else:
+                parts.append(f"all:{kw}")
+
+        # Negative keywords
+        for neg in self.negative_keywords:
+            parts.append(f"ANDNOT all:{neg}")
+
+        return " AND ".join(parts) if parts else ""
+
+    def __repr__(self) -> str:
+        return (
+            f"SearchQueryParser("
+            f"positive={self.positive_keywords}, "
+            f"negative={self.negative_keywords}, "
+            f"year={self.year_start}-{self.year_end}, "
+            f"if>={self.min_impact_factor}, "
+            f"oa={self.open_access})"
+        )
+
+
+# EOF

scitex/scholar/pipelines/__init__.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Scholar pipelines - Primary implementations for paper processing workflows.
+
+This module contains the battle-tested pipeline implementations moved from core/.
+These are the single source of truth for all Scholar workflows.
+
+Pipelines:
+- ScholarPipelineSingle: Process one paper (DOI → metadata → URLs → PDF)
+- ScholarPipelineParallel: Process multiple papers in parallel
+- ScholarPipelineBibTeX: Import and process BibTeX files
+- ScholarPipelineMetadataSingle: Enrich one paper with metadata only (API-only, no browser)
+- ScholarPipelineMetadataParallel: Enrich multiple papers with metadata in parallel (API-only)
+
+Usage:
+    # Single paper processing
+    from scitex.scholar.pipelines import ScholarPipelineSingle
+    pipeline = ScholarPipelineSingle()
+    paper = await pipeline.process_paper_async(doi="10.1038/...")
+
+    # Parallel processing
+    from scitex.scholar.pipelines import ScholarPipelineParallel
+    pipeline = ScholarPipelineParallel(num_workers=8)
+    papers = await pipeline.process_papers_from_collection_async(papers)
+
+    # BibTeX import
+    from scitex.scholar.pipelines import ScholarPipelineBibTeX
+    pipeline = ScholarPipelineBibTeX(num_workers=8)
+    papers = await pipeline.process_bibtex_file_async(
+        bibtex_path="papers.bib",
+        project="my_project"
+    )
+
+    # Via Scholar API (recommended for most users)
+    from scitex.scholar.core import Scholar
+    scholar = Scholar(project="my_project")
+    paper = await scholar.process_paper_async(doi="10.1038/...")
+"""
+
+from .ScholarPipelineSingle import ScholarPipelineSingle
+from .ScholarPipelineParallel import ScholarPipelineParallel
+from .ScholarPipelineBibTeX import ScholarPipelineBibTeX
+from .ScholarPipelineMetadataSingle import ScholarPipelineMetadataSingle
+from .ScholarPipelineMetadataParallel import ScholarPipelineMetadataParallel
+
+__all__ = [
+    "ScholarPipelineSingle",
+    "ScholarPipelineParallel",
+    "ScholarPipelineBibTeX",
+    "ScholarPipelineMetadataSingle",
+    "ScholarPipelineMetadataParallel",
+]
+
+# EOF