crossref-local 0.3.0__py3-none-any.whl

crossref_local/impact_factor/journal_lookup.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+Journal lookup module for fast name-to-ISSN resolution.
+
+Uses OpenAlex journals table (222k journals with IF proxy) for fast lookups.
+Falls back to direct database query if table doesn't exist.
+"""
+
+import json
+import sqlite3
+from typing import Dict, List, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class JournalLookup:
+    """
+    Fast journal name to ISSN lookup.
+
+    Uses journals_openalex table for O(1) lookups with IF proxy data.
+    Falls back to slow works table scan if OpenAlex table doesn't exist.
+    """
+
+    def __init__(self, db_path: str):
+        """
+        Initialize journal lookup.
+
+        Args:
+            db_path: Path to CrossRef SQLite database
+        """
+        self.db_path = db_path
+        self.conn = sqlite3.connect(db_path)
+        self.conn.row_factory = sqlite3.Row
+        self._openalex_exists = self._check_openalex_table()
+        self._issn_lookup_exists = self._check_issn_lookup_table()
+
+        if self._openalex_exists:
+            logger.info("Using journals_openalex table for fast lookups")
+        else:
+            logger.warning(
+                "journals_openalex table not found. "
+                "Run download_openalex_journals.py for fast lookups. "
+                "Falling back to slow query."
+            )
+
+    def _check_openalex_table(self) -> bool:
+        """Check if OpenAlex journals table exists."""
+        cursor = self.conn.execute("""
+            SELECT name FROM sqlite_master
+            WHERE type='table' AND name='journals_openalex'
+        """)
+        return cursor.fetchone() is not None
+
+    def _check_issn_lookup_table(self) -> bool:
+        """Check if ISSN lookup table exists."""
+        cursor = self.conn.execute("""
+            SELECT name FROM sqlite_master
+            WHERE type='table' AND name='issn_lookup'
+        """)
+        return cursor.fetchone() is not None
+
+    def get_issn(self, journal_name: str, strict: bool = True) -> Optional[str]:
+        """
+        Get ISSN for a journal name.
+
+        Args:
+            journal_name: Journal name (case-insensitive)
+            strict: If True, only exact matches. If False, allow partial matches.
+
+        Returns:
+            ISSN string or None if not found
+        """
+        if self._openalex_exists:
+            return self._get_issn_openalex(journal_name, strict)
+        else:
+            return self._get_issn_slow(journal_name, strict)
+
+    def _get_issn_openalex(self, journal_name: str, strict: bool = True) -> Optional[str]:
+        """Fast lookup using OpenAlex journals table."""
+        # Try exact match first
+        cursor = self.conn.execute("""
+            SELECT issn_l FROM journals_openalex
+            WHERE name_lower = ?
+            LIMIT 1
+        """, (journal_name.lower(),))
+
+        result = cursor.fetchone()
+        if result and result[0]:
+            return result[0]
+
+        # If strict mode, don't try partial match
+        if strict:
+            logger.debug(f"Strict mode: no exact match for '{journal_name}'")
+            return None
+
+        # Try partial match (only if not strict)
+        logger.warning(f"Using partial match for '{journal_name}' - results may be inaccurate")
+        cursor = self.conn.execute("""
+            SELECT issn_l, name FROM journals_openalex
+            WHERE name_lower LIKE ?
+            ORDER BY works_count DESC
+            LIMIT 1
+        """, (f"%{journal_name.lower()}%",))
+
+        result = cursor.fetchone()
+        if result and result[0]:
+            logger.warning(f"  Matched to: '{result[1]}'")
+            return result[0]
+        return None
+
+    def _get_issn_slow(self, journal_name: str, strict: bool = True) -> Optional[str]:
+        """Slow lookup by scanning works table."""
+        if strict:
+            # Exact match
+            cursor = self.conn.execute("""
+                SELECT DISTINCT json_extract(metadata, '$.ISSN[0]') as issn
+                FROM works
+                WHERE json_extract(metadata, '$.container-title[0]') = ?
+                AND json_extract(metadata, '$.ISSN[0]') IS NOT NULL
+                LIMIT 1
+            """, (journal_name,))
+        else:
+            # Partial match
+            cursor = self.conn.execute("""
+                SELECT DISTINCT json_extract(metadata, '$.ISSN[0]') as issn
+                FROM works
+                WHERE json_extract(metadata, '$.container-title[0]') LIKE ?
+                AND json_extract(metadata, '$.ISSN[0]') IS NOT NULL
+                LIMIT 1
+            """, (f"%{journal_name}%",))
+
+        result = cursor.fetchone()
+        return result[0] if result else None
+
+    def search(self, query: str, limit: int = 10) -> List[Dict]:
+        """
+        Search for journals by name.
+
+        Args:
+            query: Search query (partial name match)
+            limit: Maximum results to return
+
+        Returns:
+            List of journal info dictionaries with IF proxy
+        """
+        if not self._openalex_exists:
+            return []
+
+        cursor = self.conn.execute("""
+            SELECT name, issn_l, publisher, works_count,
+                   two_year_mean_citedness, h_index
+            FROM journals_openalex
+            WHERE name_lower LIKE ?
+            ORDER BY works_count DESC
+            LIMIT ?
+        """, (f"%{query.lower()}%", limit))
+
+        return [
+            {
+                "name": row["name"],
+                "issn": row["issn_l"],
+                "publisher": row["publisher"],
+                "works_count": row["works_count"],
+                "if_proxy": row["two_year_mean_citedness"],
+                "h_index": row["h_index"]
+            }
+            for row in cursor.fetchall()
+        ]
+
+    def get_info(self, issn: str) -> Optional[Dict]:
+        """
+        Get journal info by ISSN.
+
+        Args:
+            issn: Journal ISSN
+
+        Returns:
+            Journal info dictionary with IF proxy or None
+        """
+        if not self._openalex_exists:
+            return None
+
+        # Try direct ISSN-L match
+        cursor = self.conn.execute("""
+            SELECT name, issn_l, issns, publisher, works_count,
+                   two_year_mean_citedness, h_index, is_oa
+            FROM journals_openalex
+            WHERE issn_l = ?
+            LIMIT 1
+        """, (issn,))
+
+        row = cursor.fetchone()
+
+        # If not found, try issn_lookup table
+        if not row and self._issn_lookup_exists:
+            cursor = self.conn.execute("""
+                SELECT jo.name, jo.issn_l, jo.issns, jo.publisher, jo.works_count,
+                       jo.two_year_mean_citedness, jo.h_index, jo.is_oa
+                FROM issn_lookup il
+                JOIN journals_openalex jo ON il.journal_id = jo.id
+                WHERE il.issn = ?
+                LIMIT 1
+            """, (issn,))
+            row = cursor.fetchone()
+
+        if row:
+            issns = []
+            if row["issns"]:
+                try:
+                    issns = json.loads(row["issns"])
+                except:
+                    pass
+            return {
+                "name": row["name"],
+                "issn": row["issn_l"],
+                "issns": issns,
+                "publisher": row["publisher"],
+                "works_count": row["works_count"],
+                "if_proxy": row["two_year_mean_citedness"],
+                "h_index": row["h_index"],
+                "is_oa": row["is_oa"]
+            }
+        return None
+
+    def get_if_proxy(self, journal_name: str, strict: bool = True) -> Optional[float]:
+        """
+        Get OpenAlex Impact Factor proxy for a journal.
+
+        Args:
+            journal_name: Journal name
+            strict: If True, only exact matches
+
+        Returns:
+            2-year mean citedness (IF proxy) or None
+        """
+        if not self._openalex_exists:
+            return None
+
+        # Try exact match
+        cursor = self.conn.execute("""
+            SELECT two_year_mean_citedness FROM journals_openalex
+            WHERE name_lower = ?
+            LIMIT 1
+        """, (journal_name.lower(),))
+
+        result = cursor.fetchone()
+        if result and result[0]:
+            return result[0]
+
+        if strict:
+            return None
+
+        # Try partial match (only if not strict)
+        cursor = self.conn.execute("""
+            SELECT two_year_mean_citedness FROM journals_openalex
+            WHERE name_lower LIKE ?
+            ORDER BY works_count DESC
+            LIMIT 1
+        """, (f"%{journal_name.lower()}%",))
+
+        result = cursor.fetchone()
+        return result[0] if result and result[0] else None
+
+    def close(self):
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()

crossref_local/models.py

@@ -0,0 +1,186 @@
+"""Data models for crossref_local."""
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+import json
+
+
+@dataclass
+class Work:
+    """
+    Represents a scholarly work from CrossRef.
+
+    Attributes:
+        doi: Digital Object Identifier
+        title: Work title
+        authors: List of author names
+        year: Publication year
+        journal: Journal/container title
+        issn: Journal ISSN
+        volume: Volume number
+        issue: Issue number
+        page: Page range
+        publisher: Publisher name
+        type: Work type (journal-article, book-chapter, etc.)
+        abstract: Abstract text (if available)
+        url: Resource URL
+        citation_count: Number of citations (if available)
+        references: List of reference DOIs
+    """
+
+    doi: str
+    title: Optional[str] = None
+    authors: List[str] = field(default_factory=list)
+    year: Optional[int] = None
+    journal: Optional[str] = None
+    issn: Optional[str] = None
+    volume: Optional[str] = None
+    issue: Optional[str] = None
+    page: Optional[str] = None
+    publisher: Optional[str] = None
+    type: Optional[str] = None
+    abstract: Optional[str] = None
+    url: Optional[str] = None
+    citation_count: Optional[int] = None
+    references: List[str] = field(default_factory=list)
+
+    @classmethod
+    def from_metadata(cls, doi: str, metadata: dict) -> "Work":
+        """
+        Create Work from CrossRef metadata JSON.
+
+        Args:
+            doi: DOI string
+            metadata: CrossRef metadata dictionary
+
+        Returns:
+            Work instance
+        """
+        # Extract authors
+        authors = []
+        for author in metadata.get("author", []):
+            given = author.get("given", "")
+            family = author.get("family", "")
+            if given and family:
+                authors.append(f"{given} {family}")
+            elif family:
+                authors.append(family)
+            elif author.get("name"):
+                authors.append(author["name"])
+
+        # Extract year from published date
+        year = None
+        published = metadata.get("published", {})
+        date_parts = published.get("date-parts", [[]])
+        if date_parts and date_parts[0]:
+            year = date_parts[0][0]
+
+        # Extract references
+        references = []
+        for ref in metadata.get("reference", []):
+            if ref.get("DOI"):
+                references.append(ref["DOI"])
+
+        # Container title (journal name)
+        container_titles = metadata.get("container-title", [])
+        journal = container_titles[0] if container_titles else None
+
+        # ISSN
+        issns = metadata.get("ISSN", [])
+        issn = issns[0] if issns else None
+
+        return cls(
+            doi=doi,
+            title=metadata.get("title", [None])[0] if metadata.get("title") else None,
+            authors=authors,
+            year=year,
+            journal=journal,
+            issn=issn,
+            volume=metadata.get("volume"),
+            issue=metadata.get("issue"),
+            page=metadata.get("page"),
+            publisher=metadata.get("publisher"),
+            type=metadata.get("type"),
+            abstract=metadata.get("abstract"),
+            url=metadata.get("URL"),
+            citation_count=metadata.get("is-referenced-by-count"),
+            references=references,
+        )
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary."""
+        return {
+            "doi": self.doi,
+            "title": self.title,
+            "authors": self.authors,
+            "year": self.year,
+            "journal": self.journal,
+            "issn": self.issn,
+            "volume": self.volume,
+            "issue": self.issue,
+            "page": self.page,
+            "publisher": self.publisher,
+            "type": self.type,
+            "abstract": self.abstract,
+            "url": self.url,
+            "citation_count": self.citation_count,
+            "references": self.references,
+        }
+
+    def citation(self, style: str = "apa") -> str:
+        """
+        Format as citation string.
+
+        Args:
+            style: Citation style (currently only "apa" supported)
+
+        Returns:
+            Formatted citation string
+        """
+        authors_str = ", ".join(self.authors[:3])
+        if len(self.authors) > 3:
+            authors_str += " et al."
+
+        year_str = f"({self.year})" if self.year else "(n.d.)"
+        title_str = self.title or "Untitled"
+        journal_str = f"*{self.journal}*" if self.journal else ""
+
+        parts = [authors_str, year_str, title_str]
+        if journal_str:
+            parts.append(journal_str)
+        if self.volume:
+            parts.append(f"{self.volume}")
+            if self.issue:
+                parts[-1] += f"({self.issue})"
+        if self.page:
+            parts.append(self.page)
+        parts.append(f"https://doi.org/{self.doi}")
+
+        return ". ".join(filter(None, parts))
+
+
+@dataclass
+class SearchResult:
+    """
+    Container for search results with metadata.
+
+    Attributes:
+        works: List of Work objects
+        total: Total number of matches
+        query: Original search query
+        elapsed_ms: Search time in milliseconds
+    """
+
+    works: List[Work]
+    total: int
+    query: str
+    elapsed_ms: float
+
+    def __len__(self) -> int:
+        return len(self.works)
+
+    def __iter__(self):
+        return iter(self.works)
+
+    def __getitem__(self, idx):
+        return self.works[idx]

crossref_local-0.3.0.dist-info/METADATA

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: crossref-local
+Version: 0.3.0
+Summary: Local CrossRef database with 167M+ works and full-text search
+Project-URL: Homepage, https://github.com/ywatanabe1989/crossref_local
+Project-URL: Repository, https://github.com/ywatanabe1989/crossref_local
+Author: Yusuke Watanabe
+License-Expression: AGPL-3.0
+Keywords: academic,citations,crossref,doi,fts5,full-text-search,impact-factor,scholarly
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: viz
+Requires-Dist: matplotlib>=3.7; extra == 'viz'
+Requires-Dist: networkx>=3.0; extra == 'viz'
+Requires-Dist: pyvis>=0.3; extra == 'viz'
+Description-Content-Type: text/markdown
+
+# CrossRef Local
+
+Local CrossRef database with 167M+ scholarly works, full-text search, and impact factor calculation.
+
+[![Tests](https://github.com/ywatanabe1989/crossref-local/actions/workflows/test.yml/badge.svg)](https://github.com/ywatanabe1989/crossref-local/actions/workflows/test.yml)
+[![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-AGPL--3.0-blue.svg)](LICENSE)
+
+<p align="center">
+  <img src="examples/readme_figure.png" alt="CrossRef Local Demo" width="800"/>
+</p>
+
+<details>
+<summary><strong>Why CrossRef Local?</strong></summary>
+
+**Built for the LLM era** - features that matter for AI research assistants:
+
+| Feature | Benefit |
+|---------|---------|
+| 📝 **Abstracts** | Full text for semantic understanding |
+| 📊 **Impact Factor** | Filter by journal quality |
+| 🔗 **Citations** | Prioritize influential papers |
+| ⚡ **Speed** | 167M records in ms, no rate limits |
+
+Perfect for: RAG systems, research assistants, literature review automation.
+
+</details>
+
+<details>
+<summary><strong>Installation</strong></summary>
+
+```bash
+pip install crossref-local
+```
+
+From source:
+```bash
+git clone https://github.com/ywatanabe1989/crossref-local
+cd crossref-local && make install
+```
+
+Database setup (1.5 TB, ~2 weeks to build):
+```bash
+# 1. Download CrossRef data (~100GB compressed)
+aria2c "https://academictorrents.com/details/..."
+
+# 2. Build SQLite database (~days)
+pip install dois2sqlite
+dois2sqlite build /path/to/crossref-data ./data/crossref.db
+
+# 3. Build FTS5 index (~60 hours) & citations table (~days)
+make fts-build-screen
+make citations-build-screen
+```
+
+</details>
+
+<details>
+<summary><strong>Python API</strong></summary>
+
+```python
+from crossref_local import search, get, count
+
+# Full-text search (22ms for 541 matches across 167M records)
+results = search("hippocampal sharp wave ripples")
+for work in results:
+    print(f"{work.title} ({work.year})")
+
+# Get by DOI
+work = get("10.1126/science.aax0758")
+print(work.citation())
+
+# Count matches
+n = count("machine learning")  # 477,922 matches
+```
+
+Async API:
+```python
+from crossref_local import aio
+
+async def main():
+    counts = await aio.count_many(["CRISPR", "neural network", "climate"])
+    results = await aio.search("machine learning")
+```
+
+</details>
+
+<details>
+<summary><strong>CLI</strong></summary>
+
+```bash
+crossref-local search "CRISPR genome editing" -n 5
+crossref-local get 10.1038/nature12373
+crossref-local impact-factor Nature -y 2023  # IF: 54.067
+```
+
+With abstracts (`-a` flag):
+```
+$ crossref-local search "CRISPR" -n 1 -a
+
+Found 87,473 matches in 18.2ms
+
+1. RS-1 enhances CRISPR/Cas9- and TALEN-mediated knock-in efficiency (2016)
+   DOI: 10.1038/ncomms10548
+   Journal: Nature Communications
+   Abstract: Zinc-finger nuclease, transcription activator-like effector nuclease
+   and CRISPR/Cas9 are becoming major tools for genome editing. Importantly,
+   knock-in in several non-rodent species has been finally achieved...
+```
+
+</details>
+
+<details>
+<summary><strong>Impact Factor</strong></summary>
+
+```python
+from crossref_local.impact_factor import ImpactFactorCalculator
+
+with ImpactFactorCalculator() as calc:
+    result = calc.calculate_impact_factor("Nature", target_year=2023)
+    print(f"IF: {result['impact_factor']:.3f}")  # 54.067
+```
+
+| Journal | IF 2023 |
+|---------|---------|
+| Nature | 54.07 |
+| Science | 46.17 |
+| Cell | 54.01 |
+| PLOS ONE | 3.37 |
+
+</details>
+
+<details>
+<summary><strong>Citation Network</strong></summary>
+
+```python
+from crossref_local import get_citing, get_cited, CitationNetwork
+
+citing = get_citing("10.1038/nature12373")  # 1539 papers
+cited = get_cited("10.1038/nature12373")
+
+# Build visualization (like Connected Papers)
+network = CitationNetwork("10.1038/nature12373", depth=2)
+network.save_html("citation_network.html")  # requires: pip install crossref-local[viz]
+```
+
+</details>
+
+<details>
+<summary><strong>Performance</strong></summary>
+
+| Query | Matches | Time |
+|-------|---------|------|
+| `hippocampal sharp wave ripples` | 541 | 22ms |
+| `machine learning` | 477,922 | 113ms |
+| `CRISPR genome editing` | 12,170 | 257ms |
+
+Searching 167M records in milliseconds via FTS5.
+
+</details>
+
+
+---
+
+<p align="center">
+  <a href="https://scitex.ai"><img src="docs/scitex-icon-navy-inverted.png" alt="SciTeX" width="40"/></a>
+  <br>
+  AGPL-3.0 · ywatanabe@scitex.ai
+</p>
+
+<!-- EOF -->

crossref_local-0.3.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+crossref_local/__init__.py,sha256=vJFOk5somcdkuTT7UhVjAR4kj8p5JYVVHgmgCtTmENs,1544
+crossref_local/aio.py,sha256=En2btSn3euRbEYav1919gsmdC8iQaMbgGUso-IThCwo,5490
+crossref_local/api.py,sha256=FPZBStNLD7hnjjnpUzKfBFKo7gv3JHOLEIFSydH53bw,3370
+crossref_local/citations.py,sha256=QFahv84upNnXP_89A8bHxEbAdz7wHbh5LEniGcAiHas,12402
+crossref_local/cli.py,sha256=9ISMQSvWZHYrEO_gYXJB-Ju0lZCr9ifH8jFnltkgYTU,8639
+crossref_local/config.py,sha256=_tHZOHBbbw5BcmkWdTlqbSffijlQZkPkexl2YxW4GmE,1980
+crossref_local/db.py,sha256=xiTYFQxsIXmV6_QmXMWf6eX7GJv5D4icy67lBvQjUQI,3529
+crossref_local/fts.py,sha256=e5mxAbHWrO9E1GvG1WFDFxoCs_4RQIdAPYLSuMf8JCM,3395
+crossref_local/models.py,sha256=b_yYb91O6RwEPpEqe2Wmdz12WIfE5itjEus4-fCLxLI,5476
+crossref_local/impact_factor/__init__.py,sha256=pcgVCPogBisANYE5Vp2PHVGPgxoMsSXr-6utqVE97-4,559
+crossref_local/impact_factor/calculator.py,sha256=eZ13URAZzPdRyAQpS8zXe_T33e2lm_gQhtoJCXbfIGM,15977
+crossref_local/impact_factor/journal_lookup.py,sha256=Ztx6ZeWxfmPvA3KfcW5h_yz01XPstIdk91j3nu2Q-qw,8846
+crossref_local-0.3.0.dist-info/METADATA,sha256=3z5SlpYph6S6V6fwu5omXXyyNeGubZGvyqic8BU8-n8,5659
+crossref_local-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+crossref_local-0.3.0.dist-info/entry_points.txt,sha256=TaFQ1y-tIym2dqgE6xUUeXTvy2uCHNKoYeRO4w6ndWQ,59
+crossref_local-0.3.0.dist-info/RECORD,,

crossref_local-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

crossref_local-0.3.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+crossref-local = crossref_local.cli:main