scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/enrichment/_LibraryEnricher.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-12 19:24:08 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/enrichment/_LibraryEnricher.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import json
-from typing import Dict, Optional
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-
-from .enrichers._SmartEnricher import SmartEnricher
-
-logger = logging.getLogger(__name__)
-
-
-class LibraryEnricher:
-    def __init__(self, config: Optional[ScholarConfig] = None):
-        self.config = config or ScholarConfig()
-        self.smart_enricher = SmartEnricher(config=self.config)
-
-    async def enrich_project_async(self, project: str) -> Dict[str, int]:
-        """Enrich all papers in a project library."""
-        library_dir = self.config.path_manager.library_dir / project
-        master_dir = library_dir.parent / "MASTER"
-        results = {"processed": 0, "enriched": 0, "errors": 0}
-
-        for paper_dir in master_dir.iterdir():
-            if not paper_dir.is_dir():
-                continue
-
-            metadata_path = paper_dir / "metadata.json"
-            if not metadata_path.exists():
-                continue
-
-            try:
-                with open(metadata_path) as f:
-                    metadata = json.load(f)
-
-                original_metadata = metadata.copy()
-
-                # Apply smart enricher
-                metadata = self.smart_enricher.enrich_metadata_json(metadata)
-
-                # Save if changed
-                if metadata != original_metadata:
-                    with open(metadata_path, "w") as f:
-                        json.dump(metadata, f, indent=2)
-                    results["enriched"] += 1
-                    logger.success(f"Enriched {paper_dir.name}")
-
-                results["processed"] += 1
-            except Exception as e:
-                logger.error(f"Error enriching {metadata_path}: {e}")
-                results["errors"] += 1
-
-        return results
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    async def main_async():
-        from pprint import pprint
-
-        from scitex.scholar.metadata.enrichment import (
-            LibraryEnricher,
-            SmartEnricher,
-        )
-
-        # 1. Enrich single metadata file
-        enricher = SmartEnricher()
-        metadata = {
-            "doi": "10.1038/nature12373",
-            "title": None,
-            "journal": None,
-        }
-        enriched = enricher.enrich_metadata_json(metadata)
-        pprint(
-            enriched
-        )  # Now includes abstract, citation_count, keywords, impact_factor
-
-        # 2. Enrich entire project library
-        library_enricher = LibraryEnricher()
-        results = await library_enricher.enrich_project_async("hippocampus")
-        pprint(f"Enriched {results['enriched']} of {results['processed']} papers")
-
-    asyncio.run(main_async())
-
-# python -m scitex.scholar.metadata.enrichment._LibraryEnricher
-
-# EOF

scitex/scholar/.legacy/metadata/enrichment/__init__.py

@@ -1,2 +0,0 @@
-from ._LibraryEnricher import LibraryEnricher
-from .enrichers import SmartEnricher

scitex/scholar/.legacy/metadata/enrichment/enrichers/_ImpactFactorEnricher.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-12 14:18:34 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/enrichment/enrichers/_ImpactFactorEnricher.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import json
-from contextlib import contextmanager
-from datetime import datetime
-from functools import lru_cache
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-
-logger = logging.getLogger(__name__)
-
-
-class ImpactFactorEnricher:
-    def __init__(self, config: Optional[ScholarConfig] = None, cache_size: int = 1000):
-        self.config = config or ScholarConfig()
-        self._factor_instance = None
-        self._init_package()
-        self._get_metrics = lru_cache(maxsize=cache_size)(self._get_metrics_uncached)
-
-    def _init_package(self) -> None:
-        """Initialize impact_factor package."""
-        try:
-            from impact_factor.core import Factor
-
-            self._factor_instance = Factor()
-            logger.info("Impact factor package initialized")
-        except ImportError:
-            logger.warning("impact_factor package not available")
-
-    def enrich_metadata_json(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        """Enrich JSON metadata with impact factors."""
-        if not self._factor_instance:
-            return metadata
-
-        journal = metadata.get("journal")
-        if not journal or metadata.get("impact_factor") is not None:
-            return metadata
-
-        metrics = self._get_metrics(journal)
-        if metrics and "impact_factor" in metrics:
-            metadata["impact_factor"] = metrics["impact_factor"]
-            metadata["impact_factor_source"] = "JCR 2024"
-
-            if "quartile" in metrics:
-                metadata["journal_quartile"] = metrics["quartile"]
-                metadata["quartile_source"] = "JCR 2024"
-
-            if "enriched_at" not in metadata:
-                metadata["enriched_at"] = datetime.now().isoformat()
-
-            logger.info(f"Found impact factor {metrics['impact_factor']} for {journal}")
-
-        return metadata
-
-    def _get_metrics_uncached(self, journal_name: str) -> Optional[Dict[str, Any]]:
-        """Get journal metrics."""
-        try:
-            with suppress_db_logs():
-                results = self._factor_instance.search(journal_name)
-            if results:
-                result = results[0]
-                return {
-                    "impact_factor": float(result.get("factor", None)),
-                    "quartile": result.get("jcr", "Unknown"),
-                }
-        except Exception as exc:
-            logger.debug(f"Impact factor lookup failed: {exc}")
-        return None
-
-    def enrich_metadata_file(self, json_path: Path) -> bool:
-        """Enrich a single metadata.json file in place."""
-        with open(json_path) as f:
-            metadata = json.load(f)
-
-        enriched = self.enrich_metadata_json(metadata)
-
-        if enriched != metadata:
-            with open(json_path, "w") as f:
-                json.dump(enriched, f, indent=2)
-            return True
-        return False
-
-
-@contextmanager
-def suppress_db_logs():
-    old_level = logging.getLogger("sqlalchemy.engine").level
-    logging.getLogger("sqlalchemy.engine").setLevel(logging.ERROR)
-    try:
-        yield
-    finally:
-        logging.getLogger("sqlalchemy.engine").setLevel(old_level)
-
-
-# EOF

scitex/scholar/.legacy/metadata/enrichment/enrichers/_SmartEnricher.py

@@ -1,164 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-12 14:22:25 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/enrichment/enrichers/_SmartEnricher.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-from datetime import datetime
-
-import json
-from pathlib import Path
-from typing import Any
-from typing import Dict, List, Optional
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-
-from ..sources._UnifiedMetadataSource import UnifiedMetadataSource
-from ._ImpactFactorEnricher import ImpactFactorEnricher
-
-logger = logging.getLogger(__name__)
-
-
-class SmartEnricher:
-    """Smart enricher that checks JSON contents before making API calls."""
-
-    def __init__(self, config: Optional[ScholarConfig] = None):
-        self.config = config or ScholarConfig()
-        self.unified_source = UnifiedMetadataSource(config)
-        self.impact_enricher = ImpactFactorEnricher(config)
-
-    # def enrich_metadata_json(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-    #     """Enrich metadata only for missing fields."""
-    #     doi = metadata.get("doi")
-    #     if not doi:
-    #         return metadata
-
-    #     # Check what's missing before making any API calls
-    #     missing_fields = self._check_missing_fields(metadata)
-    #     if not missing_fields:
-    #         logger.debug(
-    #             "All metadata fields already present, skipping API calls"
-    #         )
-    #         return metadata
-
-    #     logger.debug(f"Missing fields for enrichment: {missing_fields}")
-
-    #     # Only fetch if we need API-based fields
-    #     api_fields = ["abstract", "citation_count", "keywords"]
-    #     needs_api = any(field in missing_fields for field in api_fields)
-
-    #     if needs_api:
-    #         comprehensive_data = (
-    #             self.unified_source.get_comprehensive_metadata(doi)
-    #         )
-    #         if comprehensive_data:
-    #             # Only update missing fields
-    #             for field in missing_fields:
-    #                 if field in comprehensive_data:
-    #                     metadata[field] = comprehensive_data[field]
-    #                     source_field = f"{field}_source"
-    #                     if source_field in comprehensive_data:
-    #                         metadata[source_field] = comprehensive_data[
-    #                             source_field
-    #                         ]
-
-    #     # Add impact factor only if journal is present and impact_factor is missing
-    #     if "impact_factor" in missing_fields and metadata.get("journal"):
-    #         metadata = self.impact_enricher.enrich_metadata_json(metadata)
-
-    #     # Set enrichment timestamp only if we actually enriched something
-    #     # if any(
-    #     #     field not in missing_fields or field in metadata
-    #     #     for field in missing_fields
-    #     # ):
-    #     #     if "enriched_at" not in metadata:
-    #     #         metadata["enriched_at"] = datetime.now().isoformat()
-    #     if any(field in metadata for field in missing_fields):
-    #         if "enriched_at" not in metadata:
-    #             metadata["enriched_at"] = datetime.now().isoformat()
-
-    #     return metadata
-
-    def enrich_metadata_json(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        """Enrich metadata only for missing fields."""
-        doi = metadata.get("doi")
-        if not doi:
-            return metadata
-
-        missing_fields = self._check_missing_fields(metadata)
-        if not missing_fields:
-            return metadata
-
-        logger.info(f"Enriching DOI {doi} - missing: {missing_fields}")
-
-        api_fields = ["abstract", "citation_count", "keywords"]
-        needs_api = any(field in missing_fields for field in api_fields)
-
-        if needs_api:
-            comprehensive_data = self.unified_source.get_comprehensive_metadata(doi)
-            if comprehensive_data:
-                logger.info(
-                    f"Got comprehensive data with keys: {list(comprehensive_data.keys())}"
-                )
-
-                # Only update missing fields with non-null values
-                for field in missing_fields:
-                    if field in comprehensive_data and comprehensive_data[field]:
-                        metadata[field] = comprehensive_data[field]
-                        logger.debug(f"Updated {field}: {comprehensive_data[field]}")
-
-        # Add impact factor only if journal is present and impact_factor is missing
-        if "impact_factor" in missing_fields and metadata.get("journal"):
-            metadata = self.impact_enricher.enrich_metadata_json(metadata)
-
-        if any(field in metadata for field in missing_fields):
-            if "enriched_at" not in metadata:
-                metadata["enriched_at"] = datetime.now().isoformat()
-
-        return metadata
-
-        # Continue with impact factor enrichment...
-
-    def _check_missing_fields(self, metadata: Dict[str, Any]) -> List[str]:
-        """Check which metadata fields are missing or empty."""
-        required_fields = [
-            "title",
-            "journal",
-            "abstract",
-            "citation_count",
-            "keywords",
-            "impact_factor",
-            "journal_quartile",
-            "issn",
-            "volume",
-            "issue",
-            "publisher",
-        ]
-        missing = []
-        for field in required_fields:
-            value = metadata.get(field)
-            if value is None or (isinstance(value, (str, list)) and not value):
-                missing.append(field)
-        return missing
-
-    def enrich_metadata_file(self, json_path: Path) -> bool:
-        """Enrich a single metadata.json file in place."""
-        with open(json_path) as f:
-            metadata = json.load(f)
-
-        enriched = self.enrich_metadata_json(metadata)
-
-        if enriched != metadata:
-            with open(json_path, "w") as f:
-                json.dump(enriched, f, indent=2)
-            return True
-        return False
-
-
-# EOF

scitex/scholar/.legacy/metadata/enrichment/enrichers/__init__.py

@@ -1,4 +0,0 @@
-from ._SmartEnricher import SmartEnricher
-from ._ImpactFactorEnricher import ImpactFactorEnricher
-
-__all__ = ["SmartEnricher", "ImpactFactorEnricher"]

scitex/scholar/.legacy/metadata/enrichment/sources/_UnifiedMetadataSource.py

@@ -1,126 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-12 14:11:04 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/enrichment/sources/_UnifiedMetadataSource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-from typing import Any, Dict, Optional
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-from scitex.scholar.metadata.doi.sources import (
-    CrossRefSource,
-    OpenAlexSource,
-    PubMedSource,
-    SemanticScholarSource,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class UnifiedMetadataSource:
-    """Unified source that reuses DOI sources for comprehensive metadata retrieval."""
-
-    # def __init__(self, config: Optional[ScholarConfig] = None):
-    #     self.config = config or ScholarConfig()
-
-    #     self.sources = [
-    #         SemanticScholarSource(
-    #             email=self.config.resolve("semantic_scholar_email", None),
-    #             api_key=self.config.resolve("semantic_scholar_api_key", None),
-    #         ),
-    #         PubMedSource(email=self.config.resolve("pubmed_email", None)),
-    #         CrossRefSource(email=self.config.resolve("crossref_email", None)),
-    #         OpenAlexSource(email=self.config.resolve("openalex_email", None)),
-    #     ]
-    def __init__(self, config: Optional[ScholarConfig] = None):
-        self.config = config or ScholarConfig()
-
-        # Initialize rate limit handler
-        from scitex.scholar.metadata.doi.utils._RateLimitHandler import (
-            RateLimitHandler,
-        )
-
-        self.rate_limit_handler = RateLimitHandler()
-
-        self.sources = [
-            SemanticScholarSource(
-                email=self.config.resolve("semantic_scholar_email", None),
-                api_key=self.config.resolve("semantic_scholar_api_key", None),
-            ),
-            PubMedSource(email=self.config.resolve("pubmed_email", None)),
-            CrossRefSource(email=self.config.resolve("crossref_email", None)),
-            OpenAlexSource(email=self.config.resolve("openalex_email", None)),
-        ]
-
-        # Inject rate limit handler into all sources
-        for source in self.sources:
-            if hasattr(source, "set_rate_limit_handler"):
-                source.set_rate_limit_handler(self.rate_limit_handler)
-
-    def get_comprehensive_metadata(self, doi: str) -> Optional[Dict[str, Any]]:
-        """Get all metadata fields in minimal API calls."""
-        if not doi:
-            return None
-
-        # Try sources in priority order
-        for source in self.sources:
-            try:
-                # Use get_metadata_by_doi if available, otherwise get_metadata + search
-                if hasattr(source, "get_metadata_by_doi"):
-                    metadata = source.get_metadata_by_doi(doi)
-                    if metadata:
-                        return self._standardize_metadata(metadata, source.name)
-
-                # Fallback: use abstract getter for specific data
-                if hasattr(source, "get_abstract"):
-                    abstract = source.get_abstract(doi)
-                    if abstract:
-                        return {
-                            "abstract": abstract,
-                            "abstract_source": source.name.lower(),
-                        }
-
-            except Exception as e:
-                logger.debug(f"Metadata fetch failed via {source.name}: {e}")
-
-        return None
-
-    def _standardize_metadata(
-        self, metadata: Dict[str, Any], source_name: str
-    ) -> Dict[str, Any]:
-        """Standardize metadata format across sources."""
-        result = {}
-        field_map = {
-            "title": metadata.get("title"),
-            "abstract": metadata.get("abstract"),
-            "citation_count": metadata.get("citation_count"),
-            "keywords": metadata.get("keywords") or metadata.get("fieldsOfStudy"),
-            "journal": metadata.get("journal") or metadata.get("venue"),
-            "issn": metadata.get("issn"),
-            "volume": metadata.get("volume"),
-            "issue": metadata.get("issue"),
-            "year": metadata.get("year"),
-            "authors": metadata.get("authors"),
-            "doi": metadata.get("doi"),
-            "publisher": metadata.get("publisher"),
-        }
-
-        for field, value in field_map.items():
-            if value is not None and value != "":
-                if field == "title" and value.endswith("."):
-                    value = value[:-1]
-                result[field] = value
-                if field != "doi":
-                    result[f"{field}_source"] = source_name.lower()
-
-        return result
-
-
-# EOF

scitex/scholar/.legacy/metadata/enrichment/sources/__init__.py

@@ -1,3 +0,0 @@
-from ._UnifiedMetadataSource import UnifiedMetadataSource
-
-__all__ = ["UnifiedMetadataSource"]

scitex/scholar/.legacy/metadata/query_to_full_meta_json.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-14 06:12:27 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/query_to_full_meta_json.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import asyncio
-import json
-from typing import Any, Dict, List, Optional
-
-from scitex.scholar.auth import AuthenticationManager
-from scitex.scholar.browser import BrowserManager
-
-# Import the necessary classes from the scitex scholar library
-from scitex.scholar.metadata.doi.resolvers import SingleDOIResolver
-from scitex.scholar.metadata.enrichment.enrichers import SmartEnricher
-from scitex.scholar.metadata.urls import URLHandler
-
-
-def initialize_complete_metadata_structure(metadata):
-    """Initialize all required fields with null values."""
-    complete_structure = {
-        # Core identification
-        "doi": None,
-        "doi_source": None,
-        "scholar_id": None,
-        # Basic metadata
-        "title": None,
-        "title_source": None,
-        "authors": None,
-        "authors_source": None,
-        "year": None,
-        "year_source": None,
-        # Publication details
-        "journal": None,
-        "journal_source": None,
-        "issn": None,
-        "issn_source": None,
-        "volume": None,
-        "volume_source": None,
-        "issue": None,
-        "issue_source": None,
-        # Content
-        "abstract": None,
-        "abstract_source": None,
-        # Metrics
-        "impact_factor": None,
-        "impact_factor_source": None,
-        "citation_count": None,
-        "citation_source": None,
-        # URLs
-        "url_doi": None,
-        "url_doi_source": None,
-        "url_publisher": None,
-        "url_publisher_source": None,
-        "url_openurl_query": None,
-        "url_openurl_source": None,
-        "url_openurl_resolved": None,
-        "url_openurl_resolved_source": None,
-        "url_pdf": None,
-        "url_pdf_source": None,
-        "url_supplementary": None,
-        "url_supplementary_source": None,
-    }
-
-    # Update with existing metadata
-    complete_structure.update(metadata)
-    return complete_structure
-
-
-async def query_to_full_metadata_json(
-    title: str,
-    authors: Optional[List[str]] = None,
-    year: Optional[int] = None,
-) -> Optional[Dict[str, Any]]:
-    """
-    Takes a paper query, resolves its DOI, enriches its metadata, finds all
-    associated URLs, and returns a comprehensive JSON object.
-
-    Args:
-        title: The title of the paper.
-        authors: A list of author names (optional).
-        year: The publication year (optional).
-
-    Returns:
-        A dictionary containing the complete metadata and URLs, or None if
-        the initial DOI cannot be resolved.
-    """
-    print(f"🚀 Starting process for title: {title[:50]}...")
-
-    # 1. Query -> DOI
-    # =================
-    print("Step 1: Resolving DOI from query...")
-    single_resolver = SingleDOIResolver()
-    metadata = await single_resolver.metadata2doi_async(
-        title=title, authors=authors, year=year
-    )
-
-    if not metadata or not metadata.get("doi"):
-        print(f"❌ Could not resolve DOI for '{title}'. Aborting.")
-        return None
-    print(f"✅ DOI found: {metadata['doi']}")
-
-    # 2. DOI -> Enriched Metadata
-    # ===========================
-    print("Step 2: Enriching metadata...")
-    enricher = SmartEnricher()
-    # The enricher modifies the dictionary in place
-    enriched_metadata = enricher.enrich_metadata_json(metadata)
-    print("✅ Metadata enriched with abstract, keywords, citation counts, etc.")
-
-    # 3. Enriched Metadata -> URLs
-    # ============================
-    print("Step 3: Finding all associated URLs (requires browser)...")
-
-    browser_manager = BrowserManager(
-        chrome_profile_name="system",
-        browser_mode="interactive",
-        auth_manager=AuthenticationManager(),
-    )
-
-    try:
-        # Initialize an authenticated browser context
-        (
-            browser,
-            context,
-        ) = await browser_manager.get_authenticated_browser_and_context_async()
-        url_handler = URLHandler(context)
-
-        # Pass the DOI to get all related URLs
-        urls = await url_handler.get_all_urls(doi=enriched_metadata["doi"])
-
-        # Add the found URLs to the main metadata dictionary
-        enriched_metadata["urls"] = urls
-        print(f"✅ Found {len(urls.get('url_pdf', []))} PDF URL(s) and other links.")
-
-    except Exception as e:
-        print(f"⚠️ Could not find URLs due to a browser error: {e}")
-        enriched_metadata["urls"] = {"error": str(e)}
-    finally:
-        await browser_manager.close()
-
-    # 4. Initialize complete structure with null values
-    complete_metadata = initialize_complete_metadata_structure(enriched_metadata)
-
-    return complete_metadata
-
-    # 5. Return Final JSON
-    # ====================
-    print("🎉 Process complete!")
-    return enriched_metadata
-
-
-async def main():
-    """Demonstration of the function."""
-    # Example query for a real paper
-    paper_title = "Attention is All You Need"
-    paper_authors = ["Vaswani, Ashish"]
-    paper_year = 2017
-
-    # Get the complete metadata
-    final_json = await query_to_full_metadata_json(
-        title=paper_title, authors=paper_authors, year=paper_year
-    )
-
-    if final_json:
-        # Print the final JSON object beautifully
-        print("\n--- Final JSON Output ---")
-        print(json.dumps(final_json, indent=2))
-        print("-------------------------\n")
-
-
-if __name__ == "__main__":
-    # To run this example, ensure you have the necessary scitex library
-    # and its dependencies installed and configured.
-    asyncio.run(main())
-
-# EOF