scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/scholar/__init__.py

@@ -1,24 +1,314 @@
-#!/usr/bin/env python3
-"""Scitex scholar module."""
+"""
+SciTeX Scholar - Scientific Literature Management Made Simple
 
-from ._local_search import LocalSearchEngine, logger
-from ._paper import Paper
-from ._pdf_downloader import PDFDownloader, logger
-from ._search import build_index, get_scholar_dir, logger, search_sync
-from ._vector_search import VectorSearchEngine, logger
-from ._web_sources import logger
+This module provides a unified interface for:
+- Searching scientific literature across multiple sources
+- Automatic paper enrichment with journal metrics
+- PDF downloads and local library management
+- Bibliography generation in multiple formats
+
+Quick Start:
+    from scitex.scholar import Scholar
+
+    scholar = Scholar()
+    papers = scholar.search("deep learning")
+    papers.save("pac.bib")
+"""
+
+# # Import main class
+# from .core._Scholar import Scholar, search, search_quick, enrich_bibtex
+
+# Import configuration
+from scitex.scholar.config import ScholarConfig
+from scitex.scholar.auth import ScholarAuthManager
+from scitex.scholar.browser import ScholarBrowserManager
+from scitex.scholar.metadata_engines import ScholarEngine
+from scitex.scholar.url_finder import ScholarURLFinder
+from scitex.scholar.pdf_download import ScholarPDFDownloader
+from scitex.scholar.storage import ScholarLibrary
+from scitex.scholar.core import Paper, Papers, Scholar
+from . import utils
 
 __all__ = [
-    "LocalSearchEngine",
-    "PDFDownloader",
+    "ScholarConfig",
+    "ScholarEngine",
+    "ScholarURLFinder",
+    "ScholarAuthManager",
+    "ScholarBrowserManager",
+
     "Paper",
-    "VectorSearchEngine",
-    "build_index",
-    "get_scholar_dir",
-    "logger",
-    "logger",
-    "logger",
-    "logger",
-    "logger",
-    "search_sync",
+    "Papers",
+    "Scholar",
+
+    "utils",
 ]
+
+# # Import core classes for advanced users
+# from scitex.scholar.core import Paper
+# from .core.Papers import Papers
+
+# # DOI resolver is available via: python -m scitex.scholar.resolve_doi_asyncs
+# from . import doi
+
+# # Backward compatibility alias
+# PaperCollection = Papers
+
+# # Import utility functions
+# from .utils._formatters import (
+#     papers_to_bibtex,
+#     papers_to_ris,
+#     papers_to_json,
+#     papers_to_markdown
+# )
+
+# # Import enrichment functionality
+# from .metadata.enrichment._MetadataEnricher import (
+#     MetadataEnricher,
+#     _enrich_papers_with_all,
+#     _enrich_papers_with_impact_factors,
+#     _enrich_papers_with_citations,
+# )
+
+# # PDF download functionality
+# from .download._ScholarPDFDownloader import (
+#     ScholarPDFDownloader,
+#     download_pdf_async,
+#     download_pdf_asyncs_async,
+# )
+# from .download._SmartScholarPDFDownloader import SmartScholarPDFDownloader
+
+# # Browser-based download functionality removed - simplified structure
+
+# # Create module-level convenience function
+# def download_pdf_asyncs(
+#     dois,
+#     download_dir=None,
+#     force=False,
+#     max_worker=4,
+#     show_async_progress=True,
+#     acknowledge_ethical_usage=None,
+#     **kwargs
+# ):
+#     """
+#     Download PDFs for DOIs using default Scholar instance.
+
+#     This is a convenience function that creates a Scholar instance if needed.
+#     For more control, use Scholar().download_pdf_asyncs() directly.
+
+#     Args:
+#         dois: DOI strings (list or single string) or Papers/Paper objects
+#         download_dir: Directory to save PDFs
+#         force: Force re-download
+#         max_worker: Maximum concurrent downloads
+#         show_async_progress: Show download progress
+#         acknowledge_ethical_usage: Acknowledge ethical usage for Sci-Hub
+#         **kwargs: Additional arguments
+
+#     Returns:
+#         Dictionary with download results
+
+#     Examples:
+#         >>> import scitex as stx
+#         >>> stx.scholar.download_pdf_asyncs(["10.1234/doi1", "10.5678/doi2"])
+#         >>> stx.scholar.download_pdf_asyncs("10.1234/single-doi")
+#     """
+#     scholar = Scholar()
+#     return scholar.download_pdf_asyncs(
+#         dois,
+#         download_dir=download_dir,
+#         force=force,
+#         max_worker=max_worker,
+#         show_async_progress=show_async_progress,
+#         acknowledge_ethical_usage=acknowledge_ethical_usage,
+#         **kwargs
+#     )
+
+# # Version
+# __version__ = "0.1.0"
+
+# # What users see with "from scitex.scholar import *"
+# __all__ = [
+#     # Main interface
+#     'Scholar',
+#     'ScholarConfig',
+
+
+#     # Convenience functions
+#     'search',
+#     'search_quick',
+#     'enrich_bibtex',
+#     'download_pdf_asyncs',  # NEW: Module-level convenience function
+
+#     "doi",
+#     "resolve_doi_asyncs",
+
+#     # Core classes
+#     'Paper',
+#     'Papers',
+#     'PaperCollection',  # Backward compatibility alias
+
+#     # Format converters
+#     'papers_to_bibtex',
+#     'papers_to_ris',
+#     'papers_to_json',
+#     'papers_to_markdown',
+
+#     # Enrichment
+#     'MetadataEnricher',
+
+#     # PDF download functionality
+#     'ScholarPDFDownloader',
+#     'download_pdf_async',
+#     'download_pdf_asyncs_async',
+
+#     # Browser-based functionality
+
+#     # Authentication
+#     'ScholarAuthManager',
+#     # 'OpenAthensAuthenticator',
+#     # 'ShibbolethAuthenticator',
+#     # 'EZProxyAuthenticator',
+
+#     # Resolution
+#     'SingleDOIResolver',
+#     'OpenURLResolver',
+#     'ResumableOpenURLResolver',
+#     # 'BatchDOIResolver',
+
+#     # Enrichment
+#     'MetadataEnricher',
+#     'JCR_YEAR',
+
+#     # Validation
+#     'PDFValidator',
+#     'ValidationResult',
+
+#     # # Database
+#     # 'PaperDatabase',
+#     # 'DatabaseEntry',
+#     # 'DatabaseIndex',
+
+#     # Semantic Search
+#     # 'SemanticSearchEngine',
+#     # 'VectorDatabase',
+#     # 'Embedder',
+# ]
+
+# # # For backward compatibility, provide access to old functions with deprecation warnings
+# # def __getattr__(name):
+# #     """Provide backward compatibility with deprecation warnings."""
+# #     import warnings
+
+# #     # Handle special IPython attributes
+# #     if name in ['__custom_documentations__', '__wrapped__']:
+# #         raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+# #     # Map old names to new functionality
+# #     compatibility_map = {
+# #         'search_sync': 'search',
+# #         'build_index': 'Scholar()._index_local_pdfs',
+# #         'get_scholar_dir': 'Scholar().get_workspace_dir()',
+# #         'LocalSearchEngine': 'Scholar',
+# #         'VectorSearchEngine': 'Scholar',
+# #         'ScholarPDFDownloader': 'Scholar',
+# #         'search_papers': 'search',
+# #         'SemanticScholarPaper': 'Paper',
+# #         'PaperMetadata': 'Paper',
+# #         'PaperAcquisition': 'Scholar',
+# #         'SemanticScholarClient': 'Scholar',
+# #         'JournalMetrics': 'Scholar',
+# #         'PaperEnrichmentService': 'Scholar',
+# #         'generate_enriched_bibliography': 'Papers.save'
+# #     }
+
+# #     if name in compatibility_map:
+# #         warnings.warn(
+# #             f"{name} is deprecated. Use {compatibility_map[name]} instead.",
+# #             DeprecationWarning,
+# #             stacklevel=2
+# #         )
+
+# #         # Return the Scholar class for most cases
+# #         if name in ['search_sync', 'search_papers']:
+# #             return search
+# #         elif name == 'build_index':
+# #             def build_index(paths, **kwargs):
+# #                 scholar = Scholar()
+# #                 stats = {}
+# #                 for path in paths:
+# #                     stats.update(scholar._index_local_pdfs(path))
+# #                 return stats
+# #             return build_index
+# #         else:
+# #             return Scholar
+
+# #     from scitex.errors import ScholarError
+# #     raise ScholarError(
+# #         f"Module attribute not found: '{name}'",
+# #         context={"module": __name__, "attribute": name},
+# #         suggestion=f"Available attributes: Scholar, Paper, Papers, search, enrich_bibtex"
+# #     )
+
+
+# # Import new modules
+# from .auth import (
+#     ScholarAuthManager,
+#     # OpenAthensAuthenticator,
+#     # ShibbolethAuthenticator,
+#     # EZProxyAuthenticator,
+# )
+# from .metadata.doi._SingleDOIResovler import SingleDOIResolver
+# from .open_url import OpenURLResolver, ResumableOpenURLResolver
+# from .metadata.enrichment import (
+#     MetadataEnricher,
+#     JCR_YEAR,
+# )
+# # from .cli import resolve_doi_asyncs
+# from .validation import PDFValidator, ValidationResult
+# # from .database import PaperDatabase, DatabaseEntry, DatabaseIndex
+# # from .search import SemanticSearchEngine, VectorDatabase, Embedder
+
+# # Module docstring for help()
+# def _module_docstring():
+#     """
+#     SciTeX Scholar - Scientific Literature Management
+
+#     Main Classes:
+#         Scholar: Main interface for all functionality
+#         Paper: Represents a scientific paper
+#         Papers: Collection of papers with analysis tools
+
+#     Quick Start:
+#         >>> from scitex.scholar import Scholar
+#         >>> scholar = Scholar()
+#         >>> papers = scholar.search("machine learning")
+#         >>> papers.filter(year_min=2020).save("ml_pac.bib")
+
+#     Common Workflows:
+#         # Search and enrich
+#         papers = scholar.search("deep learning", year_min=2022)
+
+#         # Download PDFs
+#         scholar.download_pdf_asyncs(papers)
+
+#         # Filter results
+#         high_impact = papers.filter(impact_factor_min=5.0)
+
+#         # Save bibliography
+#         papers.save("bibliography.bib", format="bibtex")
+
+#         # Search local library
+#         scholar._index_local_pdfs("./my_papers")
+#         local = scholar.search_local("transformer")
+
+#     For more information, see the documentation at:
+#     https://github.com/ywatanabe1989/SciTeX-Code
+#     """
+#     pass
+
+# # Set module docstring
+# __doc__ = _module_docstring.__doc__
+
+# # EOF
+
+# EOF

scitex/scholar/__main__.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/__main__.py
+
+"""Scholar CLI entry point - Subcommand-based interface.
+
+Clean interface routing to battle-tested pipeline implementations:
+- single: Process single paper (DOI or title)
+- parallel: Process multiple papers in parallel
+- bibtex: Process papers from BibTeX file
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+
+from scitex import logging
+
+logger = logging.getLogger(__name__)
+
+
+def create_parser():
+    """Create main argument parser with subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="python -m scitex.scholar",
+        description="""
+SciTeX Scholar - Scientific Literature Management
+═════════════════════════════════════════════════
+
+Clean subcommand interface to battle-tested pipelines:
+  single   - Process a single paper (DOI or title)
+  parallel - Process multiple papers in parallel
+  bibtex   - Process papers from BibTeX file
+
+STORAGE: ~/.scitex/scholar/library/
+  MASTER/{8DIGITID}/  - Centralized storage (no duplicates)
+  {project}/          - Project symlinks to MASTER
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers(
+        dest="command",
+        help="Available commands",
+        required=True,
+    )
+
+    # ========================================
+    # Subcommand: single
+    # ========================================
+    single_parser = subparsers.add_parser(
+        "single",
+        help="Process a single paper",
+        description="Process a single paper from DOI or title through full pipeline",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    single_parser.add_argument(
+        "--doi",
+        type=str,
+        help='DOI of the paper (e.g., "10.1038/nature12373")',
+        metavar="DOI",
+    )
+    single_parser.add_argument(
+        "--title",
+        type=str,
+        help="Paper title (will resolve DOI automatically)",
+        metavar="TITLE",
+    )
+    single_parser.add_argument(
+        "--project",
+        type=str,
+        help="Project name for organizing papers",
+        metavar="NAME",
+    )
+    single_parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode for PDF download (default: stealth)",
+    )
+    single_parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        default="system",
+        help="Chrome profile name (default: system)",
+    )
+    single_parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        help="Force re-download even if files exist",
+    )
+
+    # ========================================
+    # Subcommand: parallel
+    # ========================================
+    parallel_parser = subparsers.add_parser(
+        "parallel",
+        help="Process multiple papers in parallel",
+        description="Process multiple papers using parallel workers with dedicated browser profiles",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parallel_parser.add_argument(
+        "--dois",
+        type=str,
+        nargs="+",
+        help="Space-separated DOIs",
+        metavar="DOI",
+    )
+    parallel_parser.add_argument(
+        "--titles",
+        type=str,
+        nargs="+",
+        help="Space-separated paper titles (use quotes for multi-word titles)",
+        metavar="TITLE",
+    )
+    parallel_parser.add_argument(
+        "--project",
+        type=str,
+        help="Project name for organizing papers",
+        metavar="NAME",
+    )
+    parallel_parser.add_argument(
+        "--num-workers",
+        type=int,
+        default=4,
+        help="Number of parallel workers (default: 4)",
+        metavar="N",
+    )
+    parallel_parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode for all workers (default: stealth)",
+    )
+    parallel_parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        default="system",
+        help="Base Chrome profile to sync from (default: system)",
+    )
+
+    # ========================================
+    # Subcommand: bibtex
+    # ========================================
+    bibtex_parser = subparsers.add_parser(
+        "bibtex",
+        help="Process papers from BibTeX file",
+        description="Process all papers from a BibTeX file in parallel",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    bibtex_parser.add_argument(
+        "--bibtex",
+        type=str,
+        required=True,
+        help="Path to BibTeX file",
+        metavar="FILE",
+    )
+    bibtex_parser.add_argument(
+        "--project",
+        type=str,
+        help="Project name for organizing papers",
+        metavar="NAME",
+    )
+    bibtex_parser.add_argument(
+        "--output",
+        type=str,
+        help="Output path for enriched BibTeX (default: {input}_processed.bib)",
+        metavar="FILE",
+    )
+    bibtex_parser.add_argument(
+        "--num-workers",
+        type=int,
+        default=4,
+        help="Number of parallel workers (default: 4)",
+        metavar="N",
+    )
+    bibtex_parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode for all workers (default: stealth)",
+    )
+    bibtex_parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        default="system",
+        help="Base Chrome profile to sync from (default: system)",
+    )
+
+    return parser
+
+
+async def run_single_pipeline(args):
+    """Run single paper pipeline."""
+    from .pipelines.ScholarPipelineSingle import ScholarPipelineSingle
+
+    # Validate input
+    if not args.doi and not args.title:
+        logger.error("Either --doi or --title is required")
+        return 1
+
+    doi_or_title = args.doi if args.doi else args.title
+
+    logger.info(f"Running single paper pipeline: {doi_or_title}")
+
+    pipeline = ScholarPipelineSingle(
+        browser_mode=args.browser_mode,
+        chrome_profile=args.chrome_profile,
+    )
+
+    paper, symlink_path = await pipeline.process_single_paper(
+        doi_or_title=doi_or_title,
+        project=args.project,
+        force=args.force,
+    )
+
+    logger.success(f"Single paper pipeline completed")
+    return 0
+
+
+async def run_parallel_pipeline(args):
+    """Run parallel papers pipeline."""
+    from .pipelines.ScholarPipelineParallel import ScholarPipelineParallel
+
+    # Validate input
+    if not args.dois and not args.titles:
+        logger.error("Either --dois or --titles is required")
+        return 1
+
+    # Combine DOIs and titles into single list
+    queries = []
+    if args.dois:
+        queries.extend(args.dois)
+    if args.titles:
+        queries.extend(args.titles)
+
+    logger.info(
+        f"Running parallel pipeline: {len(queries)} papers with {args.num_workers} workers"
+    )
+
+    pipeline = ScholarPipelineParallel(
+        num_workers=args.num_workers,
+        browser_mode=args.browser_mode,
+        base_chrome_profile=args.chrome_profile,
+    )
+
+    papers = await pipeline.process_papers_from_list_async(
+        doi_or_title_list=queries,
+        project=args.project,
+    )
+
+    logger.success(
+        f"Parallel pipeline completed: {len(papers)} papers processed"
+    )
+    return 0
+
+
+async def run_bibtex_pipeline(args):
+    """Run BibTeX file pipeline."""
+    from pathlib import Path
+
+    from .pipelines.ScholarPipelineBibTeX import ScholarPipelineBibTeX
+
+    bibtex_path = Path(args.bibtex)
+    if not bibtex_path.exists():
+        logger.error(f"BibTeX file not found: {bibtex_path}")
+        return 1
+
+    logger.info(f"Running BibTeX pipeline: {bibtex_path}")
+
+    pipeline = ScholarPipelineBibTeX(
+        num_workers=args.num_workers,
+        browser_mode=args.browser_mode,
+        base_chrome_profile=args.chrome_profile,
+    )
+
+    papers = await pipeline.process_bibtex_file_async(
+        bibtex_path=bibtex_path,
+        project=args.project,
+        output_bibtex_path=args.output,
+    )
+
+    logger.success(f"BibTeX pipeline completed: {len(papers)} papers processed")
+    return 0
+
+
+async def main_async():
+    """Main async entry point."""
+    parser = create_parser()
+    args = parser.parse_args()
+
+    # Route to appropriate pipeline
+    if args.command == "single":
+        return await run_single_pipeline(args)
+    elif args.command == "parallel":
+        return await run_parallel_pipeline(args)
+    elif args.command == "bibtex":
+        return await run_bibtex_pipeline(args)
+    else:
+        logger.error(f"Unknown command: {args.command}")
+        return 1
+
+
+def main():
+    """Synchronous entry point."""
+    return asyncio.run(main_async())
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+
+# EOF