citations-collector 0.2.3__py3-none-any.whl

citations_collector/discovery/openalex.py

@@ -0,0 +1,252 @@
+"""OpenAlex citation discovery."""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from datetime import datetime
+from typing import Any
+
+import requests
+
+from citations_collector.discovery.base import AbstractDiscoverer
+from citations_collector.models import CitationRecord, CitationSource, ItemRef
+
+logger = logging.getLogger(__name__)
+
+
+def _sanitize_text(text: str | None) -> str | None:
+    """Sanitize text for TSV output - normalize whitespace, remove control chars."""
+    if text is None:
+        return None
+    # Replace newlines, tabs, carriage returns with spaces
+    text = re.sub(r"[\n\r\t]+", " ", text)
+    # Collapse multiple spaces
+    text = re.sub(r" +", " ", text)
+    # Strip leading/trailing whitespace
+    return text.strip() or None
+
+
+class OpenAlexDiscoverer(AbstractDiscoverer):
+    """Discover citations via OpenAlex API."""
+
+    BASE_URL = "https://api.openalex.org"
+    RATE_LIMIT_DELAY = 0.1  # 10 requests/second = 0.1s between requests
+
+    def __init__(self, email: str | None = None, api_key: str | None = None) -> None:
+        """
+        Initialize OpenAlex discoverer.
+
+        Args:
+            email: Email for polite pool (adds to User-Agent)
+            api_key: Optional API key for higher rate limits
+        """
+        self.email = email
+        self.api_key = api_key
+        self.session = requests.Session()
+
+        # Set User-Agent with mailto for polite pool
+        user_agent = "citations-collector"
+        if email:
+            user_agent += f" (mailto:{email})"
+        self.session.headers["User-Agent"] = user_agent
+
+        self._last_request_time = 0.0
+
+    def discover(self, item_ref: ItemRef, since: datetime | None = None) -> list[CitationRecord]:
+        """
+        Discover citations from OpenAlex.
+
+        Args:
+            item_ref: DOI reference to query
+            since: Optional date for incremental updates (from-publication-date filter)
+
+        Returns:
+            List of citation records
+        """
+        if item_ref.ref_type != "doi":
+            logger.warning(f"OpenAlex only supports DOI refs, got {item_ref.ref_type}")
+            return []
+
+        doi = item_ref.ref_value
+
+        # First resolve DOI to OpenAlex ID (required for cites: filter)
+        openalex_id = self._resolve_doi_to_id(doi)
+        if not openalex_id:
+            logger.warning(f"Could not resolve DOI {doi} to OpenAlex ID")
+            return []
+
+        # Query OpenAlex for works that cite this work
+        # Filter format: cites:{openalex_id} (e.g., cites:W4409283533)
+        citations = []
+        cursor = "*"  # OpenAlex uses cursor-based pagination
+
+        while cursor:
+            self._rate_limit()
+
+            params: dict[str, Any] = {
+                "filter": f"cites:{openalex_id}",
+                "per-page": 200,  # Max per page
+                "cursor": cursor,
+            }
+
+            if self.email:
+                params["mailto"] = self.email
+
+            # Add date filter if provided
+            if since:
+                date_str = since.strftime("%Y-%m-%d")
+                params["filter"] += f",from_publication_date:{date_str}"
+
+            try:
+                response = self.session.get(
+                    f"{self.BASE_URL}/works",
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+            except requests.RequestException as e:
+                logger.warning(f"OpenAlex API error for {doi}: {e}")
+                break
+
+            # Parse results
+            results = data.get("results", [])
+            for work in results:
+                citation = self._parse_work(work)
+                if citation:
+                    citations.append(citation)
+
+            # Check for next page
+            meta = data.get("meta", {})
+            cursor = meta.get("next_cursor")
+
+            # Stop if we've processed all results
+            if not cursor or not results:
+                break
+
+        return citations
+
+    def _resolve_doi_to_id(self, doi: str) -> str | None:
+        """
+        Resolve a DOI to its OpenAlex ID.
+
+        Args:
+            doi: The DOI to resolve (e.g., "10.1038/s41586-025-08790-w")
+
+        Returns:
+            OpenAlex ID (e.g., "W4409283533") or None if not found
+        """
+        self._rate_limit()
+
+        try:
+            response = self.session.get(
+                f"{self.BASE_URL}/works/https://doi.org/{doi}",
+                timeout=30,
+            )
+            response.raise_for_status()
+            work = response.json()
+
+            # Extract ID from URL (e.g., "https://openalex.org/W4409283533" -> "W4409283533")
+            openalex_url = work.get("id")
+            if openalex_url:
+                openalex_id: str = openalex_url.split("/")[-1]
+                logger.debug(f"Resolved DOI {doi} to OpenAlex ID {openalex_id}")
+                return openalex_id
+
+        except requests.RequestException as e:
+            logger.warning(f"Failed to resolve DOI {doi} to OpenAlex ID: {e}")
+
+        return None
+
+    def _rate_limit(self) -> None:
+        """Implement rate limiting to stay under 10 req/sec."""
+        elapsed = time.time() - self._last_request_time
+        if elapsed < self.RATE_LIMIT_DELAY:
+            time.sleep(self.RATE_LIMIT_DELAY - elapsed)
+        self._last_request_time = time.time()
+
+    def _parse_work(self, work: dict[str, Any]) -> CitationRecord | None:
+        """
+        Parse an OpenAlex work into a CitationRecord.
+
+        Args:
+            work: OpenAlex work object
+
+        Returns:
+            CitationRecord or None if missing required fields
+        """
+        # Extract DOI
+        doi = work.get("doi")
+        if not doi:
+            return None
+
+        # Remove https://doi.org/ prefix if present
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
+
+        if not doi.startswith("10."):
+            return None
+
+        # Extract title
+        title = _sanitize_text(work.get("title"))
+
+        # Extract authors
+        authorships = work.get("authorships", [])
+        authors = []
+        for authorship in authorships:
+            author_obj = authorship.get("author", {})
+            display_name = author_obj.get("display_name")
+            if display_name:
+                authors.append(display_name)
+        authors_str = _sanitize_text("; ".join(authors)) if authors else None
+
+        # Extract year
+        pub_year = work.get("publication_year")
+
+        # Extract journal/venue
+        primary_location = work.get("primary_location") or {}
+        source = primary_location.get("source") or {}
+        journal = _sanitize_text(source.get("display_name")) if source else None
+
+        # Determine citation type based on work type
+        work_type = work.get("type")
+        citation_type = self._map_work_type(work_type)
+
+        return CitationRecord(
+            item_id="",  # Will be filled by caller
+            item_flavor="",  # Will be filled by caller
+            citation_doi=doi,
+            citation_title=title,
+            citation_authors=authors_str,
+            citation_year=pub_year,
+            citation_journal=journal,
+            citation_type=citation_type,  # type: ignore[arg-type]
+            citation_relationship="Cites",  # type: ignore[arg-type]
+            citation_source=CitationSource("openalex"),
+            citation_status="active",  # type: ignore[arg-type]
+        )
+
+    def _map_work_type(self, work_type: str | None) -> str | None:
+        """
+        Map OpenAlex work type to CitationType.
+
+        OpenAlex types: article, book, dataset, paratext, preprint, etc.
+        See: https://docs.openalex.org/api-entities/works/work-object#type
+        """
+        if not work_type:
+            return None
+
+        type_mapping = {
+            "article": "Publication",
+            "book-chapter": "Book",
+            "monograph": "Book",
+            "book": "Book",
+            "dataset": "Dataset",
+            "preprint": "Preprint",
+            "posted-content": "Preprint",
+            "dissertation": "Thesis",
+            "other": "Other",
+        }
+
+        return type_mapping.get(work_type.lower(), "Other")

citations_collector/discovery/opencitations.py

@@ -0,0 +1,168 @@
+"""OpenCitations citation discovery."""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime
+from typing import cast
+
+import requests
+
+from citations_collector.discovery.base import AbstractDiscoverer
+from citations_collector.models import CitationRecord, CitationSource, ItemRef
+
+logger = logging.getLogger(__name__)
+
+
+def _sanitize_text(text: str | None) -> str | None:
+    """Sanitize text for TSV output - normalize whitespace, remove control chars."""
+    if text is None:
+        return None
+    # Replace newlines, tabs, carriage returns with spaces
+    text = re.sub(r"[\n\r\t]+", " ", text)
+    # Collapse multiple spaces
+    text = re.sub(r" +", " ", text)
+    # Strip leading/trailing whitespace
+    return text.strip() or None
+
+
+class OpenCitationsDiscoverer(AbstractDiscoverer):
+    """Discover citations via OpenCitations COCI API."""
+
+    BASE_URL = "https://opencitations.net/index/coci/api/v1/citations"
+    DOI_API = "https://doi.org"
+
+    def __init__(self) -> None:
+        """Initialize OpenCitations discoverer."""
+        self.session = requests.Session()
+
+    def discover(self, item_ref: ItemRef, since: datetime | None = None) -> list[CitationRecord]:
+        """
+        Discover citations from OpenCitations COCI.
+
+        Args:
+            item_ref: DOI reference to query
+            since: Optional date for incremental updates (creation date filter)
+
+        Returns:
+            List of citation records
+        """
+        if item_ref.ref_type != "doi":
+            logger.warning(f"OpenCitations only supports DOI refs, got {item_ref.ref_type}")
+            return []
+
+        doi = item_ref.ref_value
+        url = f"{self.BASE_URL}/{doi}"
+
+        try:
+            response = self.session.get(url, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+        except requests.RequestException as e:
+            logger.warning(f"OpenCitations API error for {doi}: {e}")
+            return []
+
+        # Parse citations from response
+        citations = []
+        if not isinstance(data, list):
+            data = [data]
+
+        for item in data:
+            citing_doi = item.get("citing")
+            if not citing_doi:
+                continue
+
+            # Apply date filter if provided
+            if since:
+                creation_date = item.get("creation", "")
+                try:
+                    # Parse creation date (formats: YYYY-MM, YYYY-MM-DD, YYYY)
+                    if creation_date:
+                        # Convert to datetime for comparison
+                        if len(creation_date) == 4:  # YYYY
+                            item_date = datetime.strptime(creation_date, "%Y")
+                        elif len(creation_date) == 7:  # YYYY-MM
+                            item_date = datetime.strptime(creation_date, "%Y-%m")
+                        else:  # YYYY-MM-DD
+                            item_date = datetime.strptime(creation_date, "%Y-%m-%d")
+
+                        if item_date < since:
+                            continue  # Skip older citations
+                except ValueError:
+                    pass  # Include if we can't parse date
+
+            # Fetch metadata for the citing DOI
+            metadata = self._fetch_doi_metadata(citing_doi)
+
+            # Create citation record with metadata
+            citation = CitationRecord(
+                item_id="",  # Will be filled by caller
+                item_flavor="",  # Will be filled by caller
+                citation_doi=citing_doi,
+                citation_title=cast(str | None, metadata.get("title")),
+                citation_authors=cast(str | None, metadata.get("authors")),
+                citation_year=cast(int | None, metadata.get("year")),
+                citation_journal=cast(str | None, metadata.get("journal")),
+                citation_relationship="Cites",  # type: ignore[arg-type]
+                citation_source=CitationSource("opencitations"),
+                citation_status="active",  # type: ignore[arg-type]
+            )
+            citations.append(citation)
+
+        return citations
+
+    def _fetch_doi_metadata(self, doi: str) -> dict[str, str | int | None]:
+        """
+        Fetch metadata for a DOI via content negotiation.
+
+        Args:
+            doi: The DOI to fetch metadata for
+
+        Returns:
+            Dictionary with title, authors, year, journal
+        """
+        metadata: dict[str, str | int | None] = {
+            "title": None,
+            "authors": None,
+            "year": None,
+            "journal": None,
+        }
+
+        try:
+            response = self.session.get(
+                f"{self.DOI_API}/{doi}",
+                headers={"Accept": "application/json"},
+                timeout=30,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            # Extract title (sanitize for TSV)
+            metadata["title"] = _sanitize_text(data.get("title"))
+
+            # Extract authors
+            authors = data.get("author", [])
+            if authors:
+                author_names = [
+                    f"{a.get('given', '')} {a.get('family', '')}".strip() for a in authors
+                ]
+                metadata["authors"] = _sanitize_text("; ".join(author_names))
+
+            # Extract year
+            published = data.get("published", {})
+            date_parts = published.get("date-parts", [[]])
+            if date_parts and len(date_parts[0]) > 0:
+                metadata["year"] = date_parts[0][0]
+
+            # Extract journal (may be string or list, sanitize for TSV)
+            container = data.get("container-title")
+            if isinstance(container, list):
+                metadata["journal"] = _sanitize_text(container[0]) if container else None
+            else:
+                metadata["journal"] = _sanitize_text(container)
+
+        except requests.RequestException as e:
+            logger.debug(f"Failed to fetch metadata for DOI {doi}: {e}")
+
+        return metadata

citations_collector/discovery/utils.py

@@ -0,0 +1,62 @@
+"""Utility functions for citation discovery."""
+
+from __future__ import annotations
+
+from citations_collector.models import CitationRecord
+
+
+def build_doi_url(doi: str) -> str:
+    """
+    Build resolver URL for DOI.
+
+    Args:
+        doi: DOI string (without doi: prefix)
+
+    Returns:
+        Full DOI resolver URL
+    """
+    return f"https://doi.org/{doi}"
+
+
+def deduplicate_citations(citations: list[CitationRecord]) -> list[CitationRecord]:
+    """
+    Deduplicate citations by unique key (item_id, item_flavor, citation_doi).
+
+    When duplicates found across sources, collects all sources in citation_sources field.
+
+    Args:
+        citations: List of citation records
+
+    Returns:
+        Deduplicated list with sources merged
+    """
+    # Group citations by unique key
+    grouped: dict[tuple[str, str, str | None], list[CitationRecord]] = {}
+
+    for citation in citations:
+        key = (citation.item_id, citation.item_flavor, citation.citation_doi)
+        if key not in grouped:
+            grouped[key] = []
+        grouped[key].append(citation)
+
+    # Build unique list, merging sources
+    unique = []
+    for _key, group in grouped.items():
+        # Use first citation as base
+        citation = group[0]
+
+        # Collect all sources that found this citation
+        sources = []
+        for c in group:
+            if c.citation_source and c.citation_source not in sources:
+                sources.append(c.citation_source)
+
+        # Always set citation_sources to the list (even if single source)
+        if sources:
+            citation.citation_sources = sources  # type: ignore[assignment]
+            # Keep citation_source set to first source (required field, backward compat)
+            citation.citation_source = sources[0]  # type: ignore[assignment]
+
+        unique.append(citation)
+
+    return unique

citations_collector/importers/__init__.py

@@ -0,0 +1,17 @@
+"""Importers and reference expanders for citations-collector."""
+
+from __future__ import annotations
+
+from citations_collector.importers.bibtex import BibTeXImporter
+from citations_collector.importers.dandi import DANDIImporter
+from citations_collector.importers.github import GitHubMapper
+from citations_collector.importers.zenodo import ZenodoExpander
+from citations_collector.importers.zotero import ZoteroImporter
+
+__all__ = [
+    "BibTeXImporter",
+    "DANDIImporter",
+    "GitHubMapper",
+    "ZenodoExpander",
+    "ZoteroImporter",
+]

citations_collector/importers/bibtex.py

@@ -0,0 +1,178 @@
+"""Import items from BibTeX files."""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+import bibtexparser
+
+from citations_collector.models.generated import (
+    Collection,
+    Item,
+    ItemFlavor,
+    ItemRef,
+    RefType,
+)
+
+logger = logging.getLogger(__name__)
+
+# Suppress bibtexparser's duplicate key warnings - we handle deduplication ourselves
+logging.getLogger("bibtexparser").setLevel(logging.ERROR)
+
+
+class BibTeXImporter:
+    """Import items from BibTeX files with regex-based parsing."""
+
+    def __init__(
+        self,
+        bibtex_file: Path,
+        bib_field: str,
+        ref_type: RefType,
+        ref_regex: str,
+    ) -> None:
+        """
+        Initialize BibTeX importer.
+
+        Args:
+            bibtex_file: Path to .bib file
+            bib_field: BibTeX field to extract reference from (e.g., 'doi')
+            ref_type: Type of reference (e.g., RefType.doi)
+            ref_regex: Regex with named groups (?P<item_id>...) and (?P<flavor_id>...)
+        """
+        self.bibtex_file = bibtex_file
+        self.bib_field = bib_field
+        self.ref_type = ref_type
+        self.ref_pattern = re.compile(ref_regex)
+
+        # Validate regex has required groups
+        if "item_id" not in self.ref_pattern.groupindex:
+            raise ValueError("ref_regex must contain (?P<item_id>...) named group")
+
+    def import_all(self) -> Collection:
+        """
+        Import all entries from BibTeX file.
+
+        Groups entries by item_id, creating one Item per unique item_id
+        with multiple flavors (versions).
+
+        Returns:
+            Collection with items parsed from BibTeX entries
+        """
+        if not self.bibtex_file.exists():
+            raise FileNotFoundError(f"BibTeX file not found: {self.bibtex_file}")
+
+        # Parse BibTeX file
+        library = bibtexparser.parse_file(str(self.bibtex_file))
+
+        # Group flavors by item_id
+        items_dict: dict[str, dict[str, Any]] = {}
+        skipped = 0
+
+        for entry in library.entries:
+            result = self._entry_to_flavor(entry)
+            if result:
+                item_id, flavor, name = result
+                if item_id not in items_dict:
+                    items_dict[item_id] = {"name": name, "flavors": [], "seen_flavors": set()}
+                # Deduplicate flavors by flavor_id
+                if flavor.flavor_id not in items_dict[item_id]["seen_flavors"]:
+                    items_dict[item_id]["flavors"].append(flavor)
+                    items_dict[item_id]["seen_flavors"].add(flavor.flavor_id)
+            else:
+                skipped += 1
+
+        # Build Item objects with merged flavors
+        items = [
+            Item(
+                item_id=item_id,
+                name=data["name"],
+                flavors=data["flavors"],
+            )
+            for item_id, data in items_dict.items()
+        ]
+
+        total_flavors = sum(len(item.flavors) for item in items)
+        logger.info(
+            f"Imported {len(items)} items ({total_flavors} flavors) "
+            f"from {self.bibtex_file.name}, skipped {skipped}"
+        )
+
+        return Collection(
+            name=f"BibTeX: {self.bibtex_file.stem}",
+            description=f"Items imported from {self.bibtex_file}",
+            items=items,
+        )
+
+    def _entry_to_flavor(self, entry: Any) -> tuple[str, ItemFlavor, str] | None:
+        """
+        Convert BibTeX entry to flavor components.
+
+        Args:
+            entry: BibTeX entry from bibtexparser
+
+        Returns:
+            Tuple of (item_id, flavor, name) if reference can be parsed, None otherwise
+        """
+        # Get reference value from specified field
+        ref_value = entry.fields_dict.get(self.bib_field)
+        if not ref_value:
+            logger.debug(f"Entry {entry.key} missing field '{self.bib_field}', skipping")
+            return None
+
+        ref_value_str = ref_value.value if hasattr(ref_value, "value") else str(ref_value)
+
+        # Parse with regex to extract item_id and flavor_id
+        match = self.ref_pattern.match(ref_value_str)
+        if not match:
+            logger.warning(
+                f"Entry {entry.key}: '{self.bib_field}' value '{ref_value_str}' "
+                f"doesn't match regex pattern, skipping"
+            )
+            return None
+
+        # Normalize to lowercase for consistency (DOIs are case-insensitive)
+        item_id = match.group("item_id").lower()
+        flavor_id = match.group("flavor_id").lower() if "flavor_id" in match.groupdict() else "main"
+
+        # Extract metadata
+        title = self._get_field(entry, "title")
+        year = self._get_field(entry, "year")
+        release_date = self._parse_year(year) if year else None
+
+        # Build ItemRef (normalize DOI to lowercase for consistency)
+        item_ref = ItemRef(
+            ref_type=self.ref_type,
+            ref_value=ref_value_str.lower() if self.ref_type == RefType.doi else ref_value_str,
+        )
+
+        # Build ItemFlavor
+        flavor = ItemFlavor(
+            flavor_id=flavor_id,
+            name=title or f"Version {flavor_id}",
+            release_date=release_date,
+            refs=[item_ref],
+        )
+
+        # Return components for grouping by item_id
+        item_name = title or item_id
+        return (item_id, flavor, item_name)
+
+    def _get_field(self, entry: Any, field_name: str) -> str | None:
+        """Extract field value from BibTeX entry."""
+        field = entry.fields_dict.get(field_name)
+        if not field:
+            return None
+        return field.value if hasattr(field, "value") else str(field)
+
+    def _parse_year(self, year_str: str) -> date | None:
+        """Parse year string to date (Jan 1 of that year)."""
+        try:
+            year = int(year_str)
+            return date(year, 1, 1)
+        except (ValueError, TypeError):
+            logger.warning(f"Could not parse year: {year_str}")
+            return None