scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/storage/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 Scholar storage module - Library and paper storage management.
 
@@ -7,6 +6,7 @@ Public API (actively used):
 - LibraryManager: Low-level library operations
 - ScholarLibrary: High-level library operations
 - BibTeXHandler: BibTeX import/export and bibliography management
+- BibTeXValidator: BibTeX file validation for syntax and content integrity
 - PaperIO: Individual paper I/O operations
 
 Internal (not for external use):
@@ -14,15 +14,29 @@ Internal (not for external use):
 - _DeduplicationManager: Used by LibraryManager
 """
 
-from ._LibraryManager import LibraryManager
+from ._BibTeXValidator import (
+    BibTeXValidator,
+    ValidationIssue,
+    ValidationResult,
+    ValidationSeverity,
+    validate_bibtex_content,
+    validate_bibtex_file,
+)
 from ._LibraryCacheManager import LibraryCacheManager
-from .ScholarLibrary import ScholarLibrary
+from ._LibraryManager import LibraryManager
 from .BibTeXHandler import BibTeXHandler
 from .PaperIO import PaperIO
+from .ScholarLibrary import ScholarLibrary
 
 __all__ = [
     "LibraryManager",
     "ScholarLibrary",
     "BibTeXHandler",
+    "BibTeXValidator",
+    "ValidationResult",
+    "ValidationIssue",
+    "ValidationSeverity",
+    "validate_bibtex_file",
+    "validate_bibtex_content",
     "PaperIO",
 ]

scitex/scholar/storage/_mixins/__init__.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/scholar/storage/_mixins/__init__.py
+
+"""
+Mixin classes for LibraryManager.
+
+Each mixin provides a specific set of methods for the manager class.
+"""
+
+from ._bibtex_handlers import BibtexHandlersMixin
+from ._library_operations import LibraryOperationsMixin
+from ._metadata_conversion import MetadataConversionMixin
+from ._paper_saving import PaperSavingMixin
+from ._resolution import ResolutionMixin
+from ._storage_helpers import StorageHelpersMixin
+from ._symlink_handlers import SymlinkHandlersMixin
+
+__all__ = [
+    "StorageHelpersMixin",
+    "MetadataConversionMixin",
+    "PaperSavingMixin",
+    "ResolutionMixin",
+    "SymlinkHandlersMixin",
+    "BibtexHandlersMixin",
+    "LibraryOperationsMixin",
+]
+
+
+# EOF

scitex/scholar/storage/_mixins/_bibtex_handlers.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/scholar/storage/_mixins/_bibtex_handlers.py
+
+"""
+BibTeX handling mixin for LibraryManager.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from scitex import logging
+
+logger = logging.getLogger(__name__)
+
+
+class BibtexHandlersMixin:
+    """Mixin providing BibTeX handling methods."""
+
+    def _create_bibtex_info_structure(
+        self,
+        project: str,
+        paper_info: Dict[str, Any],
+        complete_metadata: Dict[str, Any],
+        bibtex_source_filename: str = "papers",
+    ) -> Optional[Path]:
+        """Create info/papers_bib/pac.bib structure."""
+        try:
+            project_dir = self.config.path_manager.get_library_project_dir(project)
+            info_dir = project_dir / "info" / f"{bibtex_source_filename}_bib"
+            info_dir.mkdir(parents=True, exist_ok=True)
+
+            bibtex_file = info_dir / f"{bibtex_source_filename}.bib"
+            unresolved_dir = info_dir / "unresolved"
+            unresolved_dir.mkdir(parents=True, exist_ok=True)
+
+            first_author = "unknown"
+            if complete_metadata.get("authors"):
+                authors = complete_metadata["authors"]
+                if isinstance(authors, list) and authors:
+                    first_author = str(authors[0]).split()[-1].lower()
+                elif isinstance(authors, str):
+                    first_author = authors.split()[-1].lower()
+
+            year = complete_metadata.get("year", "unknown")
+            entry_key = f"{first_author}{year}"
+
+            bibtex_entry = self._generate_bibtex_entry(complete_metadata, entry_key)
+
+            if bibtex_file.exists():
+                with open(bibtex_file, "a", encoding="utf-8") as file_:
+                    file_.write(f"\n{bibtex_entry}")
+            else:
+                with open(bibtex_file, "w", encoding="utf-8") as file_:
+                    file_.write(bibtex_entry)
+
+            if not complete_metadata.get("doi"):
+                unresolved_file = unresolved_dir / f"{entry_key}.json"
+                unresolved_data = {
+                    "title": complete_metadata.get("title", ""),
+                    "authors": complete_metadata.get("authors", []),
+                    "year": complete_metadata.get("year", ""),
+                    "journal": complete_metadata.get("journal", ""),
+                    "scholar_id": complete_metadata.get("scholar_id", ""),
+                    "resolution_failed": True,
+                    "timestamp": complete_metadata.get("created_at", ""),
+                }
+                with open(unresolved_file, "w", encoding="utf-8") as file_:
+                    json.dump(unresolved_data, file_, indent=2)
+                logger.info(f"Added unresolved entry: {unresolved_file}")
+
+            logger.success(f"Updated BibTeX info structure: {bibtex_file}")
+            return info_dir
+
+        except Exception as exc_:
+            logger.warning(f"Failed to create BibTeX info structure: {exc_}")
+            return None
+
+    def _generate_bibtex_entry(self, metadata: Dict[str, Any], entry_key: str) -> str:
+        """Generate BibTeX entry from metadata."""
+        entry_type = "article"
+        if metadata.get("journal"):
+            entry_type = "article"
+        elif metadata.get("booktitle"):
+            entry_type = "inproceedings"
+        elif metadata.get("publisher") and not metadata.get("journal"):
+            entry_type = "book"
+
+        bibtex = f"@{entry_type}{{{entry_key},\n"
+
+        field_mappings = {
+            "title": "title",
+            "authors": "author",
+            "year": "year",
+            "journal": "journal",
+            "doi": "doi",
+            "volume": "volume",
+            "issue": "number",
+            "pages": "pages",
+            "publisher": "publisher",
+            "booktitle": "booktitle",
+            "abstract": "abstract",
+        }
+
+        for meta_field, bibtex_field in field_mappings.items():
+            value = metadata.get(meta_field)
+            if value:
+                if isinstance(value, list):
+                    value = " and ".join(str(val_) for val_ in value)
+                value_escaped = str(value).replace("{", "\\{").replace("}", "\\}")
+                bibtex += f"  {bibtex_field} = {{{value_escaped}}},\n"
+
+                source_field = f"{meta_field}_source"
+                if source_field in metadata:
+                    bibtex += f"  % {bibtex_field}_source = {metadata[source_field]}\n"
+
+        bibtex += f"  % scholar_id = {metadata.get('scholar_id', 'unknown')},\n"
+        bibtex += f"  % created_at = {metadata.get('created_at', 'unknown')},\n"
+        bibtex += f"  % created_by = {metadata.get('created_by', 'unknown')},\n"
+        bibtex += "}\n"
+
+        return bibtex
+
+
+# EOF

scitex/scholar/storage/_mixins/_library_operations.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/scholar/storage/_mixins/_library_operations.py
+
+"""
+Library operations mixin for LibraryManager.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from scitex import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LibraryOperationsMixin:
+    """Mixin providing library operation methods."""
+
+    def update_library_metadata(
+        self,
+        paper_id: str,
+        project: str,
+        doi: str,
+        metadata: Dict[str, Any],
+        create_structure: bool = True,
+    ) -> bool:
+        """Update Scholar library metadata.json with resolved DOI."""
+        try:
+            library_path = self.config.path_manager.library_dir
+            paper_dir = library_path / project / paper_id
+            metadata_file = paper_dir / "metadata.json"
+
+            if create_structure and not paper_dir.exists():
+                self.config.path_manager._ensure_directory(paper_dir)
+                logger.info(f"Created Scholar library structure: {paper_dir}")
+
+            existing_metadata = {}
+            if metadata_file.exists():
+                try:
+                    with open(metadata_file) as file_:
+                        existing_metadata = json.load(file_)
+                except Exception as exc_:
+                    logger.warning(f"Error loading existing metadata: {exc_}")
+
+            updated_metadata = {
+                **existing_metadata,
+                **metadata,
+                "doi": doi,
+                "doi_resolved_at": datetime.now().isoformat(),
+                "doi_source": "batch_doi_resolver",
+            }
+
+            with open(metadata_file, "w") as file_:
+                json.dump(updated_metadata, file_, indent=2)
+
+            logger.success(f"Updated metadata for {paper_id}: DOI {doi}")
+            return True
+
+        except Exception as exc_:
+            logger.error(f"Error updating library metadata for {paper_id}: {exc_}")
+            return False
+
+    def create_writer_directory_structure(self, paper_id: str, project: str) -> Path:
+        """Create basic paper directory structure."""
+        library_path = self.config.path_manager.library_dir
+        paper_dir = library_path / project / paper_id
+
+        self.config.path_manager._ensure_directory(paper_dir)
+
+        for subdir in ["attachments", "screenshots"]:
+            subdir_path = paper_dir / subdir
+            self.config.path_manager._ensure_directory(subdir_path)
+
+        logger.info(f"Created Scholar library structure: {paper_dir}")
+        return paper_dir
+
+    def validate_library_structure(self, project: str) -> Dict[str, Any]:
+        """Validate existing library structure for a project."""
+        validation = {
+            "valid": True,
+            "warnings": [],
+            "errors": [],
+            "paper_count": 0,
+            "missing_metadata": [],
+        }
+
+        library_path = self.config.path_manager.library_dir
+        project_dir = library_path / project
+
+        if not project_dir.exists():
+            validation["errors"].append(
+                f"Project directory does not exist: {project_dir}"
+            )
+            validation["valid"] = False
+            return validation
+
+        for paper_dir in project_dir.iterdir():
+            if paper_dir.is_dir() and len(paper_dir.name) == 8:
+                validation["paper_count"] += 1
+
+                metadata_file = paper_dir / "metadata.json"
+                if not metadata_file.exists():
+                    validation["missing_metadata"].append(paper_dir.name)
+                    validation["warnings"].append(
+                        f"Missing metadata.json: {paper_dir.name}"
+                    )
+
+        return validation
+
+    def resolve_and_update_library(
+        self,
+        papers_with_ids: List[Dict[str, Any]],
+        project: str,
+        sources: Optional[List[str]] = None,
+    ) -> Dict[str, str]:
+        """Resolve DOIs and update Scholar library metadata.json files."""
+        if not self.single_doi_resolver:
+            raise ValueError("SingleDOIResolver is required for resolving DOIs")
+
+        results = {}
+        for paper in papers_with_ids:
+            paper_id = paper.get("paper_id")
+            if not paper_id:
+                logger.warning(
+                    f"Skipping paper without paper_id: {paper.get('title', 'Unknown')}"
+                )
+                continue
+
+            title = paper.get("title")
+            if not title:
+                logger.warning(f"Skipping paper {paper_id} without title")
+                continue
+
+            logger.info(f"Resolving DOI for {paper_id}: {title[:50]}...")
+
+            try:
+                result = asyncio.run(
+                    self.single_doi_resolver.metadata2doi_async(
+                        title=title,
+                        year=paper.get("year"),
+                        authors=paper.get("authors"),
+                        sources=sources,
+                    )
+                )
+
+                if result and isinstance(result, dict) and result.get("doi"):
+                    doi = result["doi"]
+
+                    success = self.update_library_metadata(
+                        paper_id=paper_id,
+                        project=project,
+                        doi=doi,
+                        metadata={
+                            "title": title,
+                            "title_source": "input",
+                            "year": paper.get("year"),
+                            "year_source": "input" if paper.get("year") else None,
+                            "authors": paper.get("authors"),
+                            "authors_source": "input" if paper.get("authors") else None,
+                            "journal": paper.get("journal"),
+                            "journal_source": "input" if paper.get("journal") else None,
+                            "doi_resolution_source": result.get("source"),
+                        },
+                    )
+
+                    if success:
+                        results[paper_id] = doi
+                        logger.success(f"{paper_id}: {doi}")
+                    else:
+                        logger.error(
+                            f"{paper_id}: DOI resolved but metadata update failed"
+                        )
+                else:
+                    logger.warning(f"{paper_id}: No DOI found")
+
+            except Exception as exc_:
+                logger.error(f"{paper_id}: Error during resolution: {exc_}")
+
+        logger.success(
+            f"Resolved {len(results)}/{len(papers_with_ids)} DOIs and updated library metadata"
+        )
+        return results
+
+    def resolve_and_create_library_structure(
+        self,
+        papers: List[Dict[str, Any]],
+        project: str,
+        sources: Optional[List[str]] = None,
+    ) -> Dict[str, Dict[str, str]]:
+        """Synchronous wrapper for resolve_and_create_library_structure_async."""
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                raise RuntimeError(
+                    "Cannot run synchronous version in async context. "
+                    "Use resolve_and_create_library_structure_async() instead."
+                )
+            else:
+                return loop.run_until_complete(
+                    self.resolve_and_create_library_structure_async(
+                        papers, project, sources
+                    )
+                )
+        except RuntimeError:
+            return asyncio.run(
+                self.resolve_and_create_library_structure_async(
+                    papers, project, sources
+                )
+            )
+
+
+# EOF

scitex/scholar/storage/_mixins/_metadata_conversion.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/scholar/storage/_mixins/_metadata_conversion.py
+
+"""
+Metadata conversion mixin for LibraryManager.
+"""
+
+from __future__ import annotations
+
+import copy
+from collections import OrderedDict
+from typing import Any, Dict
+
+from scitex.scholar.metadata_engines.utils import BASE_STRUCTURE
+
+
+class MetadataConversionMixin:
+    """Mixin providing metadata conversion methods."""
+
+    def _dotdict_to_dict(self, obj):
+        """Recursively convert DotDict to plain dict for JSON serialization."""
+        from scitex.dict import DotDict
+
+        if isinstance(obj, DotDict):
+            return {k: self._dotdict_to_dict(v) for k, v in obj._data.items()}
+        elif isinstance(obj, dict):
+            return {k: self._dotdict_to_dict(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self._dotdict_to_dict(item) for item in obj]
+        else:
+            return obj
+
+    def _add_engine_to_list(self, engines_list: list, source: str) -> None:
+        """Helper to add source to engines list if not already present."""
+        if source and source not in engines_list:
+            engines_list.append(source)
+
+    def _convert_to_standardized_metadata(self, flat_metadata: Dict) -> OrderedDict:
+        """Convert flat metadata dict to standardized nested structure."""
+        standardized = copy.deepcopy(BASE_STRUCTURE)
+
+        # ID section
+        if "doi" in flat_metadata:
+            standardized["id"]["doi"] = flat_metadata["doi"]
+            self._add_engine_to_list(
+                standardized["id"]["doi_engines"],
+                flat_metadata.get("doi_source"),
+            )
+        if "scitex_id" in flat_metadata:
+            standardized["id"]["scholar_id"] = flat_metadata["scitex_id"]
+
+        # Basic section
+        if "title" in flat_metadata:
+            standardized["basic"]["title"] = flat_metadata["title"]
+            self._add_engine_to_list(
+                standardized["basic"]["title_engines"],
+                flat_metadata.get("title_source"),
+            )
+        if "authors" in flat_metadata:
+            standardized["basic"]["authors"] = flat_metadata["authors"]
+            self._add_engine_to_list(
+                standardized["basic"]["authors_engines"],
+                flat_metadata.get("authors_source"),
+            )
+        if "year" in flat_metadata:
+            standardized["basic"]["year"] = flat_metadata["year"]
+            self._add_engine_to_list(
+                standardized["basic"]["year_engines"],
+                flat_metadata.get("year_source"),
+            )
+        if "abstract" in flat_metadata:
+            standardized["basic"]["abstract"] = flat_metadata["abstract"]
+            self._add_engine_to_list(
+                standardized["basic"]["abstract_engines"],
+                flat_metadata.get("abstract_source"),
+            )
+
+        # Citation count section
+        self._convert_citation_count(flat_metadata, standardized)
+
+        # Publication section
+        self._convert_publication_fields(flat_metadata, standardized)
+
+        # URL section
+        self._convert_url_fields(flat_metadata, standardized)
+
+        # Path section
+        if "pdf_path" in flat_metadata:
+            standardized["path"]["pdfs"] = [flat_metadata["pdf_path"]]
+            self._add_engine_to_list(
+                standardized["path"]["pdfs_engines"],
+                "ScholarPDFDownloaderWithScreenshotsParallel",
+            )
+
+        return standardized
+
+    def _convert_citation_count(
+        self, flat_metadata: Dict, standardized: OrderedDict
+    ) -> None:
+        """Convert citation count fields to standardized format."""
+        if "citation_count" not in flat_metadata:
+            return
+
+        cc_value = flat_metadata["citation_count"]
+        if isinstance(cc_value, dict):
+            standardized["citation_count"]["total"] = cc_value.get("total")
+            self._add_engine_to_list(
+                standardized["citation_count"]["total_engines"],
+                cc_value.get("total_source"),
+            )
+            for year in [
+                "2025",
+                "2024",
+                "2023",
+                "2022",
+                "2021",
+                "2020",
+                "2019",
+                "2018",
+                "2017",
+                "2016",
+                "2015",
+            ]:
+                if year in cc_value:
+                    standardized["citation_count"][year] = cc_value[year]
+                    if f"{year}_source" in cc_value:
+                        self._add_engine_to_list(
+                            standardized["citation_count"][f"{year}_engines"],
+                            cc_value.get(f"{year}_source"),
+                        )
+        else:
+            standardized["citation_count"]["total"] = cc_value
+            self._add_engine_to_list(
+                standardized["citation_count"]["total_engines"],
+                flat_metadata.get("citation_count_source"),
+            )
+
+    def _convert_publication_fields(
+        self, flat_metadata: Dict, standardized: OrderedDict
+    ) -> None:
+        """Convert publication fields to standardized format."""
+        if "journal" in flat_metadata:
+            standardized["publication"]["journal"] = flat_metadata["journal"]
+            self._add_engine_to_list(
+                standardized["publication"]["journal_engines"],
+                flat_metadata.get("journal_source"),
+            )
+        if "short_journal" in flat_metadata:
+            standardized["publication"]["short_journal"] = flat_metadata[
+                "short_journal"
+            ]
+        if "impact_factor" in flat_metadata:
+            standardized["publication"]["impact_factor"] = flat_metadata[
+                "impact_factor"
+            ]
+        if "issn" in flat_metadata:
+            standardized["publication"]["issn"] = flat_metadata["issn"]
+        if "volume" in flat_metadata:
+            standardized["publication"]["volume"] = flat_metadata["volume"]
+        if "issue" in flat_metadata:
+            standardized["publication"]["issue"] = flat_metadata["issue"]
+        if "pages" in flat_metadata:
+            pages = flat_metadata["pages"]
+            if pages and "-" in str(pages):
+                first, last = str(pages).split("-", 1)
+                standardized["publication"]["first_page"] = first.strip()
+                standardized["publication"]["last_page"] = last.strip()
+        if "publisher" in flat_metadata:
+            standardized["publication"]["publisher"] = flat_metadata["publisher"]
+
+    def _convert_url_fields(
+        self, flat_metadata: Dict, standardized: OrderedDict
+    ) -> None:
+        """Convert URL fields to standardized format."""
+        if "url_doi" in flat_metadata:
+            standardized["url"]["doi"] = flat_metadata["url_doi"]
+        if "url_publisher" in flat_metadata:
+            standardized["url"]["publisher"] = flat_metadata["url_publisher"]
+            self._add_engine_to_list(
+                standardized["url"]["publisher_engines"], "ScholarURLFinder"
+            )
+        if "url_openurl_query" in flat_metadata:
+            standardized["url"]["openurl_query"] = flat_metadata["url_openurl_query"]
+        if "url_openurl_resolved" in flat_metadata:
+            standardized["url"]["openurl_resolved"] = flat_metadata[
+                "url_openurl_resolved"
+            ]
+            self._add_engine_to_list(
+                standardized["url"]["openurl_resolved_engines"], "ScholarURLFinder"
+            )
+        if "urls_pdf" in flat_metadata:
+            standardized["url"]["pdfs"] = flat_metadata["urls_pdf"]
+            self._add_engine_to_list(
+                standardized["url"]["pdfs_engines"], "ScholarURLFinder"
+            )
+
+    def _call_path_manager_get_storage_paths(
+        self, paper_info: Dict, collection_name: str = "MASTER"
+    ) -> Dict[str, Any]:
+        """Helper to call PathManager's get_paper_storage_paths with proper parameters."""
+        doi = paper_info.get("doi")
+        title = paper_info.get("title")
+        authors = paper_info.get("authors", [])
+        year = paper_info.get("year")
+        journal = paper_info.get("journal")
+
+        storage_path, readable_name, paper_id = (
+            self.config.path_manager.get_paper_storage_paths(
+                doi=doi,
+                title=title,
+                authors=authors,
+                year=year,
+                journal=journal,
+                project=collection_name,
+            )
+        )
+
+        return {
+            "storage_path": storage_path,
+            "readable_name": readable_name,
+            "unique_id": paper_id,
+        }
+
+
+# EOF