scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/sources/_CrossRefLocalSource.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-15 09:57:57 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/doi/sources/_CrossRefLocalSource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import json
-import sqlite3
-from pathlib import Path
-from typing import Dict, List, Optional
-
-
-class LocalCrossRef:
-    def __init__(self, db_path: str):
-        self.db_path = Path(db_path)
-
-    def search_by_doi(self, doi: str) -> Optional[Dict]:
-        """Search by DOI"""
-        doi_clean = doi.lower().strip()
-        if doi_clean.startswith("http"):
-            doi_clean = "/".join(doi_clean.split("/")[-2:])
-
-        with sqlite3.connect(self.db_path) as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "SELECT metadata FROM works WHERE doi = ? LIMIT 1",
-                (doi_clean,),
-            )
-            result = cursor.fetchone()
-
-        return json.loads(result[0]) if result else None
-
-    def search_by_title(self, title: str, limit: int = 10) -> List[Dict]:
-        """Search by title using LIKE"""
-        title_clean = title.strip()
-
-        with sqlite3.connect(self.db_path) as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "SELECT metadata FROM works WHERE json_extract(metadata, '$.title[0]') LIKE ? LIMIT ?",
-                (f"%{title_clean}%", limit),
-            )
-            results = cursor.fetchall()
-
-        return [json.loads(row[0]) for row in results]
-
-    def get_stats(self) -> Dict:
-        """Get database statistics"""
-        with sqlite3.connect(self.db_path) as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT COUNT(*) FROM works")
-            total_works = cursor.fetchone()[0]
-
-            cursor.execute("SELECT COUNT(DISTINCT type) FROM works")
-            unique_types = cursor.fetchone()[0]
-
-        return {"total_works": total_works, "unique_types": unique_types}
-
-
-if __name__ == "__main__":
-    db_path = "/home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/crossref_local/data/crossref.db"
-    cr = LocalCrossRef(db_path)
-
-    print("Database stats:")
-    print(cr.get_stats())
-
-    # Test DOI search
-    doi = "10.1038/nature14539"
-    result = cr.search_by_doi(doi)
-    if result:
-        print(f"\nFound: {result.get('title', ['N/A'])[0]}")
-
-# EOF

scitex/scholar/.legacy/metadata/doi/sources/_CrossRefSource.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-14 21:26:40 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/doi/sources/_CrossRefSource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import json
-import time
-from typing import Dict, List, Optional, Union
-
-from scitex import logging
-
-from ..utils import to_complete_metadata_structure
-from ._BaseDOISource import BaseDOISource
-
-logger = logging.getLogger(__name__)
-
-
-class CrossRefSource(BaseDOISource):
-    """CrossRef DOI source - no API key required, generous rate limits."""
-
-    def __init__(self, email: str = "research@example.com"):
-        super().__init__(email)
-        self.base_url = "https://api.crossref.org/works"
-
-    @property
-    def name(self) -> str:
-        return "CrossRef"
-
-    @property
-    def rate_limit_delay(self) -> float:
-        return 0.1
-
-    def search(
-        self,
-        title: Optional[str] = None,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        doi: Optional[str] = None,
-        max_results=5,
-        return_as: Optional[str] = "dict",
-        **kwargs,
-    ) -> Optional[Dict]:
-        """When doi is provided, all the information other than doi is ignored"""
-        if doi:
-            return self._search_by_doi(doi, return_as)
-        else:
-            return self._search_by_metadata(
-                title, year, authors, max_results, return_as
-            )
-
-    def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
-        """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
-        url = f"{self.base_url}/{doi}"
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(url, timeout=30)
-            response.raise_for_status()
-
-            data = response.json()
-            item = data.get("message", {})
-            return self._extract_metadata_from_item(item, return_as)
-        except Exception as exc:
-            logger.warn(f"CrossRef DOI search error: {exc}")
-            return None
-
-    def _search_by_metadata(
-        self,
-        title: str,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        max_results: int = 5,
-        return_as: str = "dict",
-    ) -> Optional[Dict]:
-        """Search by metadata other than doi"""
-        if not title:
-            return None
-
-        params = {
-            "query": title,
-            "rows": max_results,
-            "select": "DOI,title,published-print,published-online,container-title,short-container-title,publisher,volume,issue,ISSN,abstract,author",
-            "mailto": self.email,
-        }
-
-        if year:
-            params["filter"] = f"from-pub-date:{year},until-pub-date:{year}"
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(self.base_url, params=params, timeout=30)
-            response.raise_for_status()
-            data = response.json()
-
-            items = data.get("message", {}).get("items", [])
-            for item in items:
-                item_title = " ".join(item.get("title", []))
-                if item_title.endswith("."):
-                    item_title = item_title[:-1]
-                if self._is_title_match(title, item_title):
-                    return self._extract_metadata_from_item(item, return_as)
-            return None
-        except Exception as exc:
-            logger.warn(f"CrossRef metadata error: {exc}")
-            return None
-
-    def _extract_metadata_from_item(self, item, return_as: str) -> Optional[Dict]:
-        """Extract metadata from CrossRef item"""
-        item_title = " ".join(item.get("title", []))
-        if item_title.endswith("."):
-            item_title = item_title[:-1]
-
-        pub_year = None
-        published = item.get("published-print") or item.get("published-online")
-        if published and published.get("date-parts"):
-            pub_year = published["date-parts"][0][0]
-
-        extracted_authors = []
-        for author in item.get("author", []):
-            given = author.get("given", "")
-            family = author.get("family", "")
-            if family:
-                if given:
-                    extracted_authors.append(f"{given} {family}")
-                else:
-                    extracted_authors.append(family)
-
-        container_titles = item.get("container-title", [])
-        short_container_titles = item.get("short-container-title", [])
-        journal = container_titles[0] if container_titles else None
-        short_journal = short_container_titles[0] if short_container_titles else None
-        issn_list = item.get("ISSN", [])
-        issn = issn_list[0] if issn_list else None
-
-        metadata = {
-            "id": {
-                "doi": item.get("DOI"),
-                "doi_sources": [self.name] if item.get("DOI") else None,
-            },
-            "basic": {
-                "title": item_title if item_title else None,
-                "title_sources": [self.name] if item_title else None,
-                "year": pub_year if pub_year else None,
-                "year_sources": [self.name] if pub_year else None,
-                "abstract": (item.get("abstract") if item.get("abstract") else None),
-                "abstract_sources": ([self.name] if item.get("abstract") else None),
-                "authors": extracted_authors if extracted_authors else None,
-                "authors_sources": [self.name] if extracted_authors else None,
-            },
-            "publication": {
-                "journal": journal if journal else None,
-                "journal_sources": [self.name] if journal else None,
-                "short_journal": short_journal if short_journal else None,
-                "short_journal_sources": ([self.name] if short_journal else None),
-                "publisher": (item.get("publisher") if item.get("publisher") else None),
-                "publisher_sources": ([self.name] if item.get("publisher") else None),
-                "volume": item.get("volume") if item.get("volume") else None,
-                "volume_sources": [self.name] if item.get("volume") else None,
-                "issue": item.get("issue") if item.get("issue") else None,
-                "issue_sources": [self.name] if item.get("issue") else None,
-                "issn": issn if issn else None,
-                "issn_sources": [self.name] if issn else None,
-            },
-            "url": {
-                "doi": (
-                    "https://doi.org/" + item.get("DOI") if item.get("DOI") else None
-                ),
-                "doi_sources": [self.name] if item.get("DOI") else None,
-            },
-            "system": {
-                f"searched_by_{self.name}": True,
-            },
-        }
-
-        metadata = to_complete_metadata_structure(metadata)
-        if return_as == "dict":
-            return metadata
-        if return_as == "json":
-            return json.dumps(metadata, indent=2)
-
-
-if __name__ == "__main__":
-    from pprint import pprint
-
-    TITLE = "Attention is All You Need"
-    # DOI = "https://doi.org/10.48550/arXiv.1706.03762"
-    DOI = "10.1007/978-3-031-84300-6_13"
-
-    # Example: CrossRef search
-    source = CrossRefSource("test@example.com")
-
-    outputs = {}
-
-    # Search by title
-    outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
-
-    # Search by DOI
-    outputs["metadata_by_doi_dict"] = source.search(doi=DOI)
-    outputs["metadata_by_doi_json"] = source.search(doi=DOI, return_as="json")
-
-    for k, v in outputs.items():
-        print("----------------------------------------")
-        print(k)
-        print("----------------------------------------")
-        pprint(v)
-        time.sleep(1)
-
-# python -m scitex.scholar.metadata.doi.sources._CrossRefSource
-
-# EOF

scitex/scholar/.legacy/metadata/doi/sources/_OpenAlexSource.py

@@ -1,306 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-14 21:26:55 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/doi/sources/_OpenAlexSource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-import json
-import time
-from typing import Dict, List, Optional, Union
-
-from scitex import logging
-
-from ..utils import to_complete_metadata_structure
-from ._BaseDOISource import BaseDOISource
-
-logger = logging.getLogger(__name__)
-
-
-class OpenAlexSource(BaseDOISource):
-    """OpenAlex - free and open alternative to proprietary databases."""
-
-    def __init__(self, email: str = "research@example.com"):
-        super().__init__(email)
-        self.base_url = "https://api.openalex.org/works"
-
-    @property
-    def name(self) -> str:
-        return "OpenAlex"
-
-    @property
-    def rate_limit_delay(self) -> float:
-        return 0.1
-
-    def search(
-        self,
-        title: Optional[str] = None,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        doi: Optional[str] = None,
-        max_results=1,
-        return_as: Optional[str] = "dict",
-        **kwargs,
-    ) -> Optional[Dict]:
-        """When doi is provided, all the information other than doi is ignored"""
-        if doi:
-            return self._search_by_doi(doi, return_as)
-        else:
-            return self._search_by_metadata(
-                title, year, authors, max_results, return_as
-            )
-
-    def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
-        """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
-
-        params = {
-            "filter": f"doi:{doi}",
-            "per_page": 1,
-        }
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(self.base_url, params=params, timeout=30)
-            response.raise_for_status()
-
-            data = response.json()
-            results = data.get("results", [])
-            if results:
-                return self._extract_metadata_from_work(results[0], return_as)
-            return None
-        except Exception as exc:
-            logger.warn(f"OpenAlex DOI search error: {exc}")
-            return None
-
-    def _search_by_metadata(
-        self,
-        title: str,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        max_results: int = 1,
-        return_as: str = "dict",
-    ) -> Optional[Dict]:
-        """Search by metadata other than doi"""
-        if not title:
-            return None
-
-        params = {
-            "search": title,
-            "per_page": 5,
-        }
-
-        if year:
-            params["filter"] = f"publication_year:{year}"
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(self.base_url, params=params, timeout=30)
-            response.raise_for_status()
-
-            data = response.json()
-            results = data.get("results", [])
-
-            for work in results:
-                work_title = work.get("title", "")
-                if work_title and work_title.endswith("."):
-                    work_title = work_title[:-1]
-                if work_title and self._is_title_match(title, work_title):
-                    return self._extract_metadata_from_work(work, return_as)
-            return None
-        except Exception as exc:
-            logger.warn(f"OpenAlex metadata error: {exc}")
-            return None
-
-    def _extract_metadata_from_work(self, work, return_as: str) -> Optional[Dict]:
-        """Extract metadata from OpenAlex work"""
-        work_title = work.get("title", "")
-        if work_title and work_title.endswith("."):
-            work_title = work_title[:-1]
-
-        doi_url = work.get("doi", "")
-        doi = doi_url.replace("https://doi.org/", "") if doi_url else None
-
-        indexed_in = work.get("indexed_in", [])
-        if indexed_in:
-            indexed_in = ", ".join(indexed_in)
-        # source = f"{self.name} ({indexed_in})"
-
-        ids = work.get("ids", {})
-        pmid_url = ids.get("pmid", "")
-        pmid = (
-            pmid_url.replace("https://pubmed.ncbi.nlm.nih.gov/", "")
-            if pmid_url
-            else None
-        )
-
-        extracted_authors = []
-        for authorship in work.get("authorships", []):
-            author = authorship.get("author", {})
-            if author.get("display_name"):
-                extracted_authors.append(author["display_name"])
-
-        journal = None
-        issn = None
-        publisher = None
-        volume = None
-        issue = None
-        first_page = None
-        last_page = None
-
-        primary_location = work.get("primary_location", {})
-        if primary_location and primary_location.get("source"):
-            pdf_url = primary_location.get("pdf_url", None)
-            _source = primary_location["source"]
-            journal = _source.get("display_name")
-            issn = _source.get("issn_l")
-            if _source.get("host_organization_name"):
-                publisher = _source["host_organization_name"]
-
-        if not journal:
-            host_venue = work.get("host_venue", {})
-            if host_venue:
-                journal = host_venue.get("display_name")
-                issn = host_venue.get("issn_l")
-                publisher = host_venue.get("publisher")
-
-        biblio = work.get("biblio", {})
-        if biblio:
-            volume = biblio.get("volume")
-            issue = biblio.get("issue")
-            first_page = biblio.get("first_page")
-            last_page = biblio.get("last_page")
-
-        if doi and doi.startswith("10.48550/arxiv"):
-            if not journal:
-                journal = "arXiv (Cornell University)"
-            if not publisher:
-                publisher = "Cornell University"
-
-        citation_count = work.get("cited_by_count")
-        citation_count_by_year = work.get("counts_by_year")
-        if citation_count_by_year:
-            citation_count_by_year = {
-                str(dd["year"]): dd["cited_by_count"] for dd in citation_count_by_year
-            }
-            citation_counts = {
-                "total": citation_count,
-                **citation_count_by_year,
-            }
-            citation_counts_sources = {
-                f"{k}_sources": [self.name] for k, v in citation_counts.items()
-            }
-            citation_counts.update(citation_counts_sources)
-
-        url_publisher = None
-        if primary_location:
-            url_publisher = primary_location.get("landing_page_url")
-        if doi and doi.startswith("10.48550/arxiv") and not url_publisher:
-            arxiv_id = doi.replace("10.48550/arxiv.", "")
-            url_publisher = f"http://arxiv.org/abs/{arxiv_id}"
-
-        keywords = []
-        for keyword in work.get("keywords", []):
-            if keyword.get("display_name"):
-                keywords.append(keyword["display_name"])
-
-        metadata = {
-            "id": {
-                "doi": doi if doi else None,
-                "doi_sources": [self.name] if doi else None,
-                "pmid": pmid if pmid else None,
-                "pmid_sources": [self.name] if pmid else None,
-            },
-            "basic": {
-                "title": work_title if work_title else None,
-                "title_sources": [self.name] if work_title else None,
-                "year": (
-                    work.get("publication_year")
-                    if work.get("publication_year")
-                    else None
-                ),
-                "year_sources": ([self.name] if work.get("publication_year") else None),
-                "authors": extracted_authors if extracted_authors else None,
-                "authors_sources": [self.name] if extracted_authors else None,
-                "keywords": keywords if keywords else None,
-                "keywords_sources": [self.name] if keywords else None,
-                "type": work.get("type") if work.get("type", None) else None,
-                "type_sources": [self.name] if work.get("type") else None,
-            },
-            "citation_count": {
-                **citation_counts,
-            },
-            "publication": {
-                "journal": journal if journal else None,
-                "journal_sources": [self.name] if journal else None,
-                "issn": issn if issn else None,
-                "issn_sources": [self.name] if issn else None,
-                "publisher": publisher if publisher else None,
-                "publisher_sources": [self.name] if publisher else None,
-                "volume": volume if volume else None,
-                "volume_sources": [self.name] if volume else None,
-                "issue": issue if issue else None,
-                "issue_sources": [self.name] if issue else None,
-                "first_page": first_page if first_page else None,
-                "first_page_sources": [self.name] if first_page else None,
-                "last_page": last_page if last_page else None,
-                "last_page_sources": [self.name] if last_page else None,
-            },
-            "url": {
-                "doi": f"https://doi.org/{doi}" if doi else None,
-                "doi_sources": [self.name] if doi else None,
-                "pdf": pdf_url if pdf_url else None,
-                "pdf_sources": [self.name] if pdf_url else None,
-                "publisher": url_publisher if url_publisher else None,
-                "publisher_sources": [self.name] if url_publisher else None,
-            },
-            "system": {
-                f"searched_by_{self.name}": True,
-            },
-        }
-
-        metadata = to_complete_metadata_structure(metadata)
-        if return_as == "dict":
-            return metadata
-        if return_as == "json":
-            return json.dumps(metadata, indent=2)
-
-
-if __name__ == "__main__":
-    from pprint import pprint
-
-    TITLE = "Attention is All You Need"
-    DOI = "10.1038/nature14539"
-
-    source = OpenAlexSource("test@example.com")
-    outputs = {}
-
-    # Search by title
-    outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
-
-    # Search by DOI
-    outputs["metadata_by_doi_dict"] = source.search(doi=DOI)
-    outputs["metadata_by_doi_json"] = source.search(doi=DOI, return_as="json")
-
-    for k, v in outputs.items():
-        print("----------------------------------------")
-        print(k)
-        print("----------------------------------------")
-        pprint(v)
-        time.sleep(1)
-
-# python -m scitex.scholar.metadata.doi.sources._OpenAlexSource
-
-# EOF