scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/batch/_ProgressManagerForBatchDOIResolution.py

@@ -1,304 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-11 05:48:42 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/doi/batch/_ProgressManagerForBatchDOIResolution.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""Progress tracking and persistence for batch DOI resolution."""
-
-import json
-import time
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-from scitex import logging
-
-logger = logging.getLogger(__name__)
-
-
-class ProgressManagerForBatchDOIResolution:
-    """Handles progress tracking, persistence, and ETA calculation for batch DOI resolution.
-
-    Responsibilities:
-    - Load/save progress data from/to files
-    - Track successful and failed resolutions
-    - Calculate processing statistics and ETA
-    - Atomic file operations for progress persistence
-    """
-
-    def __init__(self, doi_resolution_progress_file: Path, title_normalizer=None):
-        """Initialize progress manager.
-
-        Args:
-            doi_resolution_progress_file: Path to progress file
-            title_normalizer: Function to normalize titles (for backward compatibility)
-        """
-        self.doi_resolution_progress_file = Path(doi_resolution_progress_file)
-        self.progress_data = self._load_progress()
-        self._start_time = time.time()
-        self._title_normalizer = title_normalizer or self._default_normalize_title
-
-    def _default_normalize_title(self, title: str) -> str:
-        """Default title normalization for backward compatibility."""
-        if not title:
-            return ""
-        # Remove common variations and normalize whitespace
-        normalized = title.lower().strip()
-        # Remove extra whitespace
-        normalized = " ".join(normalized.split())
-        return normalized
-
-    def _load_progress(self) -> Dict[str, Any]:
-        """Load or create progress data with enhanced fields."""
-        if self.doi_resolution_progress_file.exists():
-            try:
-                with open(self.doi_resolution_progress_file, "r") as f:
-                    data = json.load(f)
-                logger.info(f"Resuming from: {self.doi_resolution_progress_file}")
-
-                # Migrate old format if needed
-                if "source_performance" not in data:
-                    data["source_performance"] = {}
-                if "processing_times" not in data:
-                    data["processing_times"] = []
-
-                return data
-            except Exception as e:
-                logger.warning(f"Could not load progress file: {e}")
-
-        # Create new enhanced progress data
-        return {
-            "version": 2,
-            "started_at": datetime.now().isoformat(),
-            "last_updated": datetime.now().isoformat(),
-            "completed": False,
-            "papers": {},
-            "statistics": {
-                "total": 0,
-                "processed": 0,
-                "resolved": 0,
-                "failed": 0,
-                "skipped": 0,
-                "rate_limited": 0,
-            },
-            "source_performance": {},  # Track which sources work best
-            "processing_times": [],  # Track processing times for ETA
-            "duplicate_groups": {},  # Track potential duplicates
-        }
-
-    def save_progress(self):
-        """Save progress atomically."""
-        self.progress_data["last_updated"] = datetime.now().isoformat()
-
-        temp_file = self.doi_resolution_progress_file.with_suffix(".tmp")
-        try:
-            with open(temp_file, "w") as f:
-                json.dump(self.progress_data, f, indent=2)
-            temp_file.replace(self.doi_resolution_progress_file)
-        except Exception as e:
-            logger.error(f"Failed to save progress: {e}")
-            if temp_file.exists():
-                temp_file.unlink()
-
-    def update_progress_success(self, title: str, doi: str):
-        """Update progress for successful resolution."""
-        paper_key = self._title_normalizer(title)
-        self.progress_data["papers"][paper_key] = {
-            "title": title,
-            "doi": doi,
-            "status": "resolved",
-            "timestamp": datetime.now().isoformat(),
-        }
-        self.progress_data["statistics"]["resolved"] += 1
-        self.progress_data["statistics"]["processed"] += 1
-
-    def update_progress_failure(self, title: str):
-        """Update progress for failed resolution."""
-        paper_key = self._title_normalizer(title)
-        self.progress_data["papers"][paper_key] = {
-            "title": title,
-            "status": "failed",
-            "timestamp": datetime.now().isoformat(),
-        }
-        self.progress_data["statistics"]["failed"] += 1
-        self.progress_data["statistics"]["processed"] += 1
-
-    def update_progress_skipped(self, title: str, reason: str = "already_resolved"):
-        """Update progress for skipped resolution."""
-        paper_key = self._title_normalizer(title)
-        self.progress_data["papers"][paper_key] = {
-            "title": title,
-            "status": "skipped",
-            "reason": reason,
-            "timestamp": datetime.now().isoformat(),
-        }
-        self.progress_data["statistics"]["skipped"] += 1
-        self.progress_data["statistics"]["processed"] += 1
-
-    def update_rate_limited(self):
-        """Update rate limiting statistics."""
-        self.progress_data["statistics"]["rate_limited"] += 1
-
-    def set_total_papers(self, total: int):
-        """Set total number of papers to process."""
-        self.progress_data["statistics"]["total"] = total
-
-    def get_paper_key(self, title: str) -> str:
-        """Generate unique key for paper."""
-        return self._title_normalizer(title)
-
-    def is_paper_processed(self, title: str) -> bool:
-        """Check if paper has already been processed."""
-        paper_key = self.get_paper_key(title)
-        return paper_key in self.progress_data["papers"]
-
-    def get_paper_status(self, title: str) -> Optional[Dict[str, Any]]:
-        """Get processing status for a paper."""
-        paper_key = self.get_paper_key(title)
-        return self.progress_data["papers"].get(paper_key)
-
-    def calculate_eta(self) -> Optional[str]:
-        """Calculate estimated time to completion."""
-        stats = self.progress_data["statistics"]
-        processed = stats["processed"]
-        total = stats["total"]
-
-        if processed == 0 or total == 0:
-            return None
-
-        elapsed = time.time() - self._start_time
-        rate = processed / elapsed  # papers per second
-        remaining = total - processed
-
-        if rate <= 0:
-            return None
-
-        eta_seconds = remaining / rate
-
-        # Format ETA nicely
-        if eta_seconds < 60:
-            return f"{eta_seconds:.0f}s"
-        elif eta_seconds < 3600:
-            return f"{eta_seconds / 60:.1f}m"
-        else:
-            return f"{eta_seconds / 3600:.1f}h"
-
-    def get_progress_summary(self) -> Dict[str, Any]:
-        """Get comprehensive progress summary."""
-        stats = self.progress_data["statistics"]
-        eta = self.calculate_eta()
-
-        return {
-            "total": stats["total"],
-            "processed": stats["processed"],
-            "resolved": stats["resolved"],
-            "failed": stats["failed"],
-            "skipped": stats["skipped"],
-            "rate_limited": stats["rate_limited"],
-            "success_rate": (
-                stats["resolved"] / stats["processed"] if stats["processed"] > 0 else 0
-            ),
-            "eta": eta,
-            "doi_resolution_progress_file": str(self.doi_resolution_progress_file),
-        }
-
-    def mark_completed(self):
-        """Mark the batch processing as completed."""
-        self.progress_data["completed"] = True
-        self.progress_data["completed_at"] = datetime.now().isoformat()
-        self.save_progress()
-
-    def reset_progress(self):
-        """Reset progress data to initial state."""
-        self.progress_data = {
-            "version": 2,
-            "started_at": datetime.now().isoformat(),
-            "last_updated": datetime.now().isoformat(),
-            "completed": False,
-            "papers": {},
-            "statistics": {
-                "total": 0,
-                "processed": 0,
-                "resolved": 0,
-                "failed": 0,
-                "skipped": 0,
-                "rate_limited": 0,
-            },
-            "source_performance": {},
-            "processing_times": [],
-            "duplicate_groups": {},
-        }
-        self._start_time = time.time()
-
-    def add_processing_time(self, duration: float):
-        """Add a processing time measurement for ETA calculation."""
-        self.progress_data["processing_times"].append(
-            {
-                "duration": duration,
-                "timestamp": datetime.now().isoformat(),
-            }
-        )
-
-        # Keep only recent measurements (last 100) for performance
-        if len(self.progress_data["processing_times"]) > 100:
-            self.progress_data["processing_times"] = self.progress_data[
-                "processing_times"
-            ][-100:]
-
-    def update_source_performance(self, source: str, success: bool):
-        """Update source performance statistics."""
-        if source not in self.progress_data["source_performance"]:
-            self.progress_data["source_performance"][source] = {
-                "attempts": 0,
-                "successes": 0,
-                "success_rate": 0.0,
-            }
-
-        perf = self.progress_data["source_performance"][source]
-        perf["attempts"] += 1
-        if success:
-            perf["successes"] += 1
-        perf["success_rate"] = perf["successes"] / perf["attempts"]
-
-
-if __name__ == "__main__":
-
-    def main():
-        # Example usage
-        from pathlib import Path
-
-        doi_resolution_progress_file = Path("/tmp/test_progress.json")
-        manager = ProgressManagerForBatchDOIResolution(doi_resolution_progress_file)
-
-        # Simulate batch processing
-        manager.set_total_papers(5)
-
-        # Process some papers
-        manager.update_progress_success("Test Paper 1", "10.1000/test1")
-        manager.update_progress_failure("Test Paper 2")
-        manager.update_progress_skipped("Test Paper 3", "duplicate")
-
-        manager.save_progress()
-
-        # Show summary
-        summary = manager.get_progress_summary()
-        print("Progress Summary:")
-        for key, value in summary.items():
-            print(f"  {key}: {value}")
-
-        # Clean up
-        if doi_resolution_progress_file.exists():
-            doi_resolution_progress_file.unlink()
-
-    main()
-
-# python -m scitex.scholar.metadata.doi.batch._ProgressManagerForBatchDOIResolution
-
-# EOF

scitex/scholar/.legacy/metadata/doi/batch/_SourceStatsManagerForBatchDOIResolution.py

@@ -1,299 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-11 05:40:38 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/doi/batch/_SourceStatsManagerForBatchDOIResolution.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-from pathlib import Path
-
-from typing import Union
-
-"""Configuration management for batch DOI resolution."""
-
-import json
-from datetime import datetime
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-
-logger = logging.getLogger(__name__)
-
-
-class SourceStatsManagerForBatchDOIResolution:
-    """Handles configuration resolution and validation for batch DOI processing.
-
-    Responsibilities:
-    - Configuration parameter resolution with proper defaults
-    - Source performance statistics management
-    - Workspace directory management
-    - Batch-specific configuration validation
-    """
-
-    def __init__(
-        self,
-        progress_path: Optional[Union[str, Path]] = None,
-        max_worker: Optional[int] = None,
-        config: Optional[ScholarConfig] = None,
-    ):
-        """Initialize configuration manager.
-
-        Args:
-            config: ScholarConfig instance, creates default if None
-        """
-        self.config = config or ScholarConfig()
-        self.max_worker = self.config.resolve("max_worker", max_worker)
-        self.progress_path = self.config.get_doi_resolution_progress_path(progress_path)
-        self._source_success_rates: Dict[str, Dict[str, float]] = {}
-        self._load_source_stats()
-
-    def get_doi_resolution_progress_path(
-        self, provided_path: Optional[Path] = None
-    ) -> Path:
-        """Resolve progress file path with automatic generation if needed."""
-        return self.config.get_doi_resolution_progress_path(provided_path)
-
-    def get_workspace_logs_dir(self) -> Path:
-        """Get workspace logs directory from configuration."""
-        return self.config.path_manager.get_workspace_logs_dir()
-
-    def get_library_dir(self) -> Path:
-        """Get Scholar library directory from configuration."""
-        return self.config.path_manager.library_dir
-
-    def resolve_sources(
-        self, provided_sources: Optional[List[str]] = None
-    ) -> List[str]:
-        """Resolve DOI sources with configuration fallbacks."""
-        if provided_sources:
-            return provided_sources
-
-        # Use configuration or default sources
-        return self.config.resolve(
-            "sources",
-            None,
-            ["pubmed", "crossref", "semantic_scholar"],
-            list,
-        )
-
-    def get_rate_limit_config(self) -> Dict[str, Any]:
-        """Get rate limiting configuration."""
-        return {
-            "default_delay": self.config.resolve(
-                "rate_limit_default_delay", None, 1.0, float
-            ),
-            "adaptive_enabled": self.config.resolve(
-                "rate_limit_adaptive", None, True, bool
-            ),
-            "max_retries": self.config.resolve("rate_limit_max_retries", None, 3, int),
-        }
-
-    def validate_configuration(self) -> Dict[str, Any]:
-        """Validate batch configuration and return validation results.
-
-        Returns:
-            Dictionary with validation results and any issues found
-        """
-        validation = {"valid": True, "warnings": [], "errors": []}
-
-        try:
-            # Check workspace directory access
-            workspace_dir = self.get_workspace_logs_dir()
-            if not workspace_dir.exists():
-                try:
-                    workspace_dir.mkdir(parents=True, exist_ok=True)
-                except Exception as e:
-                    validation["errors"].append(
-                        f"Cannot create workspace directory: {e}"
-                    )
-                    validation["valid"] = False
-            elif not workspace_dir.is_dir():
-                validation["errors"].append(
-                    f"Workspace path is not a directory: {workspace_dir}"
-                )
-                validation["valid"] = False
-
-            # Check library directory access
-            library_dir = self.get_library_dir()
-            if not library_dir.exists():
-                validation["warnings"].append(
-                    f"Library directory does not exist: {library_dir}"
-                )
-            elif not library_dir.is_dir():
-                validation["errors"].append(
-                    f"Library path is not a directory: {library_dir}"
-                )
-                validation["valid"] = False
-
-            # Validate max worker_asyncs
-            if self.max_worker <= 0:
-                validation["errors"].append(
-                    f"Invalid self.max_worker value: {self.max_worker}"
-                )
-                validation["valid"] = False
-            elif self.max_worker > 20:
-                validation["warnings"].append(
-                    f"High self.max_worker value may cause rate limiting: {self.max_worker}"
-                )
-
-            # Validate sources
-            sources = self.resolve_sources()
-            if not sources:
-                validation["errors"].append("No DOI sources configured")
-                validation["valid"] = False
-
-        except Exception as e:
-            validation["errors"].append(f"Configuration validation error: {e}")
-            validation["valid"] = False
-
-        return validation
-
-    def _load_source_stats(self):
-        """Load historical source performance stats from workspace directory."""
-        try:
-            workspace_dir = self.get_workspace_logs_dir()
-            stats_file = workspace_dir / "source_stats.json"
-
-            if stats_file.exists():
-                with open(stats_file, "r") as f:
-                    self._source_success_rates = json.load(f)
-                logger.debug(f"Loaded source statistics from {stats_file}")
-            else:
-                self._source_success_rates = {}
-                logger.debug("No existing source statistics found")
-
-        except Exception as e:
-            logger.warning(f"Could not load source statistics: {e}")
-            self._source_success_rates = {}
-
-    def save_source_stats(self):
-        """Save source performance stats to workspace directory."""
-        try:
-            workspace_dir = self.get_workspace_logs_dir()
-            workspace_dir.mkdir(parents=True, exist_ok=True)
-
-            stats_file = workspace_dir / "source_stats.json"
-            with open(stats_file, "w") as f:
-                json.dump(self._source_success_rates, f, indent=2)
-
-            logger.debug(f"Saved source statistics to {stats_file}")
-
-        except Exception as e:
-            logger.warning(f"Could not save source statistics: {e}")
-
-    def get_source_success_rate(self, source: str) -> float:
-        """Get historical success rate for a source.
-
-        Args:
-            source: Source name
-
-        Returns:
-            Success rate (0.0 to 1.0)
-        """
-        return self._source_success_rates.get(source, {}).get("success_rate", 0.0)
-
-    def update_source_stats(self, source: str, success: bool):
-        """Update source performance statistics.
-
-        Args:
-            source: Source name
-            success: Whether the request was successful
-        """
-        if source not in self._source_success_rates:
-            self._source_success_rates[source] = {
-                "attempts": 0,
-                "successes": 0,
-                "success_rate": 0.0,
-                "last_updated": datetime.now().isoformat(),
-            }
-
-        stats = self._source_success_rates[source]
-        stats["attempts"] += 1
-        if success:
-            stats["successes"] += 1
-
-        stats["success_rate"] = stats["successes"] / stats["attempts"]
-        stats["last_updated"] = datetime.now().isoformat()
-
-    def get_all_source_stats(self) -> Dict[str, Dict[str, Any]]:
-        """Get all source performance statistics.
-
-        Returns:
-            Dictionary mapping source names to their statistics
-        """
-        return self._source_success_rates.copy()
-
-    def reset_source_stats(self):
-        """Reset all source performance statistics."""
-        self._source_success_rates = {}
-        self.save_source_stats()
-
-    def get_configuration_summary(self) -> Dict[str, Any]:
-        """Get a summary of current configuration.
-
-        Returns:
-            Dictionary with configuration summary
-        """
-        return {
-            "max_worker": self.max_worker,
-            "sources": self.resolve_sources(),
-            "workspace_dir": str(self.get_workspace_logs_dir()),
-            "library_dir": str(self.get_library_dir()),
-            "rate_limit_config": self.get_rate_limit_config(),
-            "source_stats_count": len(self._source_success_rates),
-        }
-
-
-if __name__ == "__main__":
-
-    def main():
-        # Example usage
-        from pathlib import Path
-
-        # Initialize configuration manager
-        source_stats_manager = SourceStatsManagerForBatchDOIResolution()
-
-        # Test configuration resolution
-        print("Configuration Summary:")
-        summary = source_stats_manager.get_configuration_summary()
-        for key, value in summary.items():
-            print(f"  {key}: {value}")
-
-        # Test validation
-        print("\nConfiguration Validation:")
-        validation = source_stats_manager.validate_configuration()
-        print(f"  Valid: {validation['valid']}")
-        if validation["warnings"]:
-            print(f"  Warnings: {validation['warnings']}")
-        if validation["errors"]:
-            print(f"  Errors: {validation['errors']}")
-
-        # Test source statistics
-        print("\nTesting source statistics:")
-        source_stats_manager.update_source_stats("pubmed", True)
-        source_stats_manager.update_source_stats("pubmed", False)
-        source_stats_manager.update_source_stats("crossref", True)
-
-        stats = source_stats_manager.get_all_source_stats()
-        for source, stat in stats.items():
-            print(
-                f"  {source}: {stat['successes']}/{stat['attempts']} = {stat['success_rate']:.2%}"
-            )
-
-        # Test progress file path generation
-        progress_path = source_stats_manager.progress_path
-        print(f"\nGenerated progress file path: {progress_path}")
-
-    main()
-
-# python -m scitex.scholar.metadata.doi.batch._SourceStatsManagerForBatchDOIResolution
-
-# EOF

scitex/scholar/.legacy/metadata/doi/batch/__init__.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-04 23:25:00 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/doi/batch/__init__.py
-# ----------------------------------------
-
-"""Batch DOI resolution components.
-
-This module contains focused, single-responsibility classes for batch DOI resolution:
-- ProgressManagerForBatchDOIResolution: Progress tracking and persistence
-- MetadataHandlerForBatchDOIResolution: Paper metadata processing and validation
-- SourceStatsManagerForBatchDOIResolution: Configuration resolution and validation
-- LibraryManager: Scholar library organization and management
-"""
-
-from ._ProgressManagerForBatchDOIResolution import ProgressManagerForBatchDOIResolution
-from ._MetadataHandlerForBatchDOIResolution import MetadataHandlerForBatchDOIResolution
-from ._SourceStatsManagerForBatchDOIResolution import (
-    SourceStatsManagerForBatchDOIResolution,
-)
-
-__all__ = [
-    "ProgressManagerForBatchDOIResolution",
-    "MetadataHandlerForBatchDOIResolution",
-    "SourceStatsManagerForBatchDOIResolution",
-]