citations-collector 0.2.3__py3-none-any.whl

citations_collector/importers/dandi.py

@@ -0,0 +1,314 @@
+"""Import dandisets from DANDI Archive API."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+from collections.abc import Callable, Iterator
+from datetime import date
+from typing import Any
+
+import requests
+
+from citations_collector.models import Collection, Item, ItemFlavor, ItemRef, RefType
+
+logger = logging.getLogger(__name__)
+
+
+class DANDIImporter:
+    """
+    Import dandisets from DANDI Archive API.
+
+    Fetches all dandisets (or a subset) from the DANDI Archive API
+    and creates a Collection with version DOIs for citation tracking.
+
+    Example:
+        importer = DANDIImporter()
+        collection = importer.import_all(limit=10)
+    """
+
+    BASE_URL = "https://api.dandiarchive.org/api"
+    PAGE_SIZE = 100  # DANDI API default
+
+    def __init__(self, api_url: str | None = None) -> None:
+        """
+        Initialize DANDI importer.
+
+        Args:
+            api_url: Optional custom API URL (for testing).
+                    Defaults to DANDI Archive production API.
+        """
+        self.api_url = api_url or self.BASE_URL
+        self.session = requests.Session()
+        # Set a reasonable timeout and user agent
+        self.session.headers["User-Agent"] = "citations-collector/0.1"
+
+    def import_specific(
+        self,
+        dandiset_ids: list[str],
+        include_draft: bool = False,
+        progress_callback: Callable[[int, int | None], None] | None = None,
+    ) -> Collection:
+        """
+        Import specific dandisets by their identifiers.
+
+        Args:
+            dandiset_ids: List of dandiset identifiers (e.g., ["000003", "000402"])
+            include_draft: If True, include draft versions without DOIs.
+            progress_callback: Optional callback(current, total) for progress updates.
+
+        Returns:
+            Collection with the specified dandisets and their versions.
+        """
+        items: list[Item] = []
+        total = len(dandiset_ids)
+
+        for idx, dandiset_id in enumerate(dandiset_ids):
+            # Fetch dandiset metadata
+            dandiset = self._fetch_dandiset(dandiset_id)
+            if dandiset is None:
+                logger.warning(f"Dandiset {dandiset_id} not found, skipping")
+                continue
+
+            item = self._dandiset_to_item(dandiset, include_draft=include_draft)
+            if item is not None and item.flavors:
+                items.append(item)
+                logger.debug(f"Imported dandiset {item.item_id} with {len(item.flavors)} versions")
+
+            if progress_callback:
+                progress_callback(idx + 1, total)
+
+        logger.info(f"Imported {len(items)} specific dandisets from DANDI Archive")
+
+        return Collection(
+            name="DANDI Archive",
+            description="Neural data archive with versioned dandisets",
+            homepage="https://dandiarchive.org",
+            source_type="dandi",
+            items=items,
+        )
+
+    def import_all(
+        self,
+        include_draft: bool = False,
+        limit: int | None = None,
+        progress_callback: Callable[[int, int | None], None] | None = None,
+    ) -> Collection:
+        """
+        Import all dandisets as a Collection.
+
+        Args:
+            include_draft: If True, include draft versions without DOIs.
+                          Default False (only published versions with DOIs).
+            limit: Optional limit on number of dandisets to import.
+            progress_callback: Optional callback(current, total) for progress updates.
+
+        Returns:
+            Collection with:
+            - item_id: "dandi:NNNNNN" (e.g., "dandi:000003")
+            - flavor_id: version string (e.g., "0.230629.1955")
+            - ref: DOI (e.g., "10.48324/dandi.000003/0.230629.1955")
+        """
+        items: list[Item] = []
+        count = 0
+
+        for dandiset in self._iter_dandisets():
+            if limit is not None and count >= limit:
+                break
+
+            item = self._dandiset_to_item(dandiset, include_draft=include_draft)
+            if item is not None and item.flavors:  # Only include if has versions
+                items.append(item)
+                count += 1
+
+                if progress_callback:
+                    progress_callback(count, limit)
+
+                logger.debug(f"Imported dandiset {item.item_id} with {len(item.flavors)} versions")
+
+        logger.info(f"Imported {len(items)} dandisets from DANDI Archive")
+
+        return Collection(
+            name="DANDI Archive",
+            description="Neural data archive with versioned dandisets",
+            homepage="https://dandiarchive.org",
+            source_type="dandi",
+            items=items,
+        )
+
+    def _fetch_dandiset(self, dandiset_id: str) -> dict | None:
+        """
+        Fetch a single dandiset by ID.
+
+        Args:
+            dandiset_id: The dandiset identifier (e.g., "000003", "000402")
+
+        Returns:
+            Dandiset metadata dictionary or None if not found
+        """
+        url = f"{self.api_url}/dandisets/{dandiset_id}/"
+
+        try:
+            response = self.session.get(url, timeout=60)
+            response.raise_for_status()
+            return response.json()  # type: ignore[no-any-return]
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch dandiset {dandiset_id}: {e}")
+            return None
+
+    def _iter_dandisets(self) -> Iterator[dict]:
+        """
+        Iterate over all dandisets from the API (handles pagination).
+
+        Yields:
+            Dandiset metadata dictionaries from API
+        """
+        url: str | None = f"{self.api_url}/dandisets/"
+        params: dict[str, Any] = {"page_size": self.PAGE_SIZE, "ordering": "identifier"}
+
+        while url:
+            try:
+                response = self.session.get(url, params=params, timeout=60)
+                response.raise_for_status()
+                data = response.json()
+
+                yield from data.get("results", [])
+
+                # Follow pagination
+                url = data.get("next")
+                params = {}  # Next URL includes params
+
+            except requests.RequestException as e:
+                logger.error(f"DANDI API error: {e}")
+                break
+
+    def _dandiset_to_item(self, dandiset: dict, include_draft: bool = False) -> Item | None:
+        """
+        Convert a dandiset API response to an Item.
+
+        Args:
+            dandiset: Dandiset metadata from API
+            include_draft: Whether to include draft versions
+
+        Returns:
+            Item with flavors for each published version, or None if no versions
+        """
+        identifier = dandiset.get("identifier", "")
+        if not identifier:
+            return None
+
+        # Extract most recent version info for name
+        draft_version = dandiset.get("draft_version", {})
+        most_recent = dandiset.get("most_recent_published_version") or draft_version
+
+        name = most_recent.get("name", f"Dandiset {identifier}")
+        item_id = f"dandi:{identifier}"
+        homepage = f"https://dandiarchive.org/dandiset/{identifier}"
+
+        # Get all versions
+        flavors = self._get_versions(identifier, include_draft=include_draft)
+
+        if not flavors:
+            logger.debug(f"Dandiset {identifier} has no published versions, skipping")
+            return None
+
+        return Item(
+            item_id=item_id,
+            name=name,
+            homepage=homepage,
+            flavors=flavors,
+        )
+
+    def _get_versions(self, dandiset_id: str, include_draft: bool = False) -> list[ItemFlavor]:
+        """
+        Get all versions for a dandiset.
+
+        Args:
+            dandiset_id: The dandiset identifier (e.g., "000003")
+            include_draft: Whether to include draft version
+
+        Returns:
+            List of ItemFlavor objects for each version with a DOI
+        """
+        url: str | None = f"{self.api_url}/dandisets/{dandiset_id}/versions/"
+        flavors: list[ItemFlavor] = []
+
+        try:
+            # Paginate through versions
+            params: dict[str, Any] = {"page_size": 100, "ordering": "-created"}
+
+            while url:
+                response = self.session.get(url, params=params, timeout=60)
+                response.raise_for_status()
+                data = response.json()
+
+                for version in data.get("results", []):
+                    flavor = self._version_to_flavor(dandiset_id, version, include_draft)
+                    if flavor:
+                        flavors.append(flavor)
+
+                url = data.get("next")
+                params = {}
+
+        except requests.RequestException as e:
+            logger.warning(f"Failed to fetch versions for dandiset {dandiset_id}: {e}")
+
+        return flavors
+
+    def _version_to_flavor(
+        self, dandiset_id: str, version: dict, include_draft: bool = False
+    ) -> ItemFlavor | None:
+        """
+        Convert a version API response to an ItemFlavor.
+
+        Args:
+            dandiset_id: The dandiset identifier
+            version: Version metadata from API
+            include_draft: Whether to include draft versions
+
+        Returns:
+            ItemFlavor with DOI ref, or None if draft and not included
+        """
+        version_str = version.get("version", "")
+        status = version.get("status", "")
+
+        # DANDI API uses "Valid" for published versions with DOIs
+        # "Published" status is confusingly used for draft versions
+        # Draft versions have version_str == "draft"
+        is_draft = version_str == "draft"
+
+        # Skip draft versions unless requested
+        if is_draft and not include_draft:
+            return None
+
+        # Published versions (status="Valid") have DOIs in the format:
+        # 10.48324/dandi.{dandiset_id}/{version}
+        # Draft versions don't have DOIs
+        if status == "Valid" and not is_draft:
+            doi = f"10.48324/dandi.{dandiset_id}/{version_str}"
+            refs = [ItemRef(ref_type=RefType.doi, ref_value=doi)]
+        elif include_draft and is_draft:
+            # For drafts, we can still track them but without DOI
+            refs = [
+                ItemRef(
+                    ref_type=RefType.url,
+                    ref_value=f"https://dandiarchive.org/dandiset/{dandiset_id}/draft",
+                )
+            ]
+        else:
+            return None
+
+        # Parse release date
+        created = version.get("created")
+        release_date = None
+        if created:
+            with contextlib.suppress(ValueError, TypeError):
+                # API returns ISO format datetime
+                release_date = date.fromisoformat(created[:10])
+
+        return ItemFlavor(
+            flavor_id=version_str,
+            name=version.get("name"),
+            release_date=release_date,
+            refs=refs,
+        )

citations_collector/importers/github.py

@@ -0,0 +1,147 @@
+"""GitHub repository to Zenodo DOI mapping."""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+
+import requests
+
+from citations_collector.models import ItemRef, RefType
+
+logger = logging.getLogger(__name__)
+
+
+class GitHubMapper:
+    """
+    Map GitHub repositories to Zenodo DOIs.
+
+    Many open source projects use Zenodo to archive releases and get DOIs.
+    This mapper attempts to extract Zenodo DOIs from GitHub repositories
+    via multiple strategies:
+
+    1. Check repository description for Zenodo badge
+    2. Check README for Zenodo DOI badge
+    3. Look for .zenodo.json file
+
+    Example:
+        "datalad/datalad" → 10.5281/zenodo.808846 (concept DOI)
+    """
+
+    GITHUB_API = "https://api.github.com/repos"
+    ZENODO_DOI_PATTERN = re.compile(r"10\.5281/zenodo\.(\d+)")
+
+    def __init__(self, github_token: str | None = None) -> None:
+        """
+        Initialize GitHub mapper.
+
+        Args:
+            github_token: Optional GitHub personal access token for higher rate limits.
+                         If not provided, reads from GITHUB_TOKEN environment variable.
+        """
+        # Use provided token, or fallback to environment variable
+        if github_token is None:
+            github_token = os.getenv("GITHUB_TOKEN")
+
+        self.session = requests.Session()
+        if github_token:
+            self.session.headers["Authorization"] = f"token {github_token}"
+            logger.debug("Using GitHub token for authentication")
+
+    def map_to_doi(self, repo: str) -> ItemRef | None:
+        """
+        Map GitHub repository to Zenodo DOI.
+
+        Args:
+            repo: GitHub repository in "owner/name" format (e.g., "datalad/datalad")
+
+        Returns:
+            ItemRef with DOI if found, None otherwise
+        """
+        # Query GitHub API for repository info
+        url = f"{self.GITHUB_API}/{repo}"
+
+        try:
+            response = self.session.get(url, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+        except requests.RequestException as e:
+            logger.warning(f"GitHub API error for {repo}: {e}")
+            return None
+
+        # Strategy 1: Check repository description
+        description = data.get("description", "")
+        doi = self._extract_zenodo_doi(description)
+        if doi:
+            logger.info(f"Found Zenodo DOI in description for {repo}: {doi}")
+            return ItemRef(ref_type=RefType("doi"), ref_value=doi)
+
+        # Strategy 2: Check README
+        readme_url = f"{self.GITHUB_API}/{repo}/readme"
+        try:
+            readme_response = self.session.get(readme_url, timeout=30)
+            readme_response.raise_for_status()
+            readme_data = readme_response.json()
+
+            # README content is base64 encoded
+            import base64
+
+            content = base64.b64decode(readme_data.get("content", "")).decode("utf-8")
+            doi = self._extract_zenodo_doi(content)
+            if doi:
+                logger.info(f"Found Zenodo DOI in README for {repo}: {doi}")
+                return ItemRef(ref_type=RefType("doi"), ref_value=doi)
+        except requests.RequestException:
+            # README not found or other error, continue to next strategy
+            pass
+
+        # Strategy 3: Check for .zenodo.json file
+        zenodo_json_url = f"{self.GITHUB_API}/{repo}/contents/.zenodo.json"
+        try:
+            zenodo_response = self.session.get(zenodo_json_url, timeout=30)
+            zenodo_response.raise_for_status()
+            zenodo_data = zenodo_response.json()
+
+            import base64
+            import json
+
+            content = base64.b64decode(zenodo_data.get("content", "")).decode("utf-8")
+            zenodo_config = json.loads(content)
+
+            # .zenodo.json might have related_identifiers with DOI
+            for identifier in zenodo_config.get("related_identifiers", []):
+                if identifier.get("scheme") == "doi":
+                    doi_value = identifier.get("identifier")
+                    if doi_value:
+                        # Clean up DOI (remove https://doi.org/ prefix)
+                        doi_clean = doi_value.replace("https://doi.org/", "")
+                        logger.info(f"Found Zenodo DOI in .zenodo.json for {repo}: {doi_clean}")
+                        return ItemRef(ref_type=RefType("doi"), ref_value=doi_clean)
+        except requests.RequestException:
+            # .zenodo.json not found or other error
+            pass
+
+        logger.info(f"No Zenodo DOI found for GitHub repo {repo}")
+        return None
+
+    def _extract_zenodo_doi(self, text: str) -> str | None:
+        """
+        Extract Zenodo DOI from text.
+
+        Looks for common patterns like:
+        - https://doi.org/10.5281/zenodo.808846
+        - 10.5281/zenodo.808846
+        - [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.808846.svg)]
+
+        Args:
+            text: Text to search
+
+        Returns:
+            DOI string if found, None otherwise
+        """
+        match = self.ZENODO_DOI_PATTERN.search(text)
+        if match:
+            # Return full DOI
+            return f"10.5281/zenodo.{match.group(1)}"
+        return None

citations_collector/importers/zenodo.py

@@ -0,0 +1,110 @@
+"""Zenodo concept expansion to version DOIs."""
+
+from __future__ import annotations
+
+import logging
+import os
+
+import requests
+
+from citations_collector.models import ItemRef, RefType
+
+logger = logging.getLogger(__name__)
+
+
+class ZenodoExpander:
+    """
+    Expand Zenodo concept IDs to all version DOIs.
+
+    Zenodo uses a "concept" DOI that represents all versions of a work,
+    and separate version-specific DOIs. This expander queries the Zenodo
+    API to get all version DOIs for a concept.
+
+    Example:
+        Concept ID 808846 (DataLad) expands to:
+        - 10.5281/zenodo.808846 (concept DOI)
+        - 10.5281/zenodo.808847 (v0.1.0)
+        - 10.5281/zenodo.1234567 (v0.2.0)
+        - ... (all other versions)
+    """
+
+    BASE_URL = "https://zenodo.org/api/records"
+
+    def __init__(self, zenodo_token: str | None = None) -> None:
+        """
+        Initialize Zenodo expander.
+
+        Args:
+            zenodo_token: Optional Zenodo personal access token for authentication.
+                         If not provided, reads from ZENODO_TOKEN environment variable.
+        """
+        # Use provided token, or fallback to environment variable
+        if zenodo_token is None:
+            zenodo_token = os.getenv("ZENODO_TOKEN")
+
+        self.session = requests.Session()
+        if zenodo_token:
+            # Zenodo uses Bearer token authentication
+            self.session.headers["Authorization"] = f"Bearer {zenodo_token}"
+            logger.debug("Using Zenodo token for authentication")
+
+    def expand(self, concept_id: str) -> list[ItemRef]:
+        """
+        Expand Zenodo concept ID to all version DOIs.
+
+        Args:
+            concept_id: Zenodo concept ID (numeric, e.g. "808846")
+
+        Returns:
+            List of ItemRef objects with DOI references for each version
+        """
+        # Query Zenodo API for the concept record (latest version)
+        url = f"{self.BASE_URL}/{concept_id}"
+
+        try:
+            response = self.session.get(url, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+        except requests.RequestException as e:
+            logger.warning(f"Zenodo API error for concept {concept_id}: {e}")
+            return []
+
+        # Extract version DOIs
+        refs = []
+
+        # Strategy 1: Get concept DOI (represents all versions)
+        # This is the most comprehensive as it aggregates citations across versions
+        concept_doi = data.get("conceptdoi")
+        if concept_doi:
+            # Extract just the DOI part (remove https://doi.org/ prefix if present)
+            concept_doi_clean = concept_doi.replace("https://doi.org/", "")
+            refs.append(ItemRef(ref_type=RefType("doi"), ref_value=concept_doi_clean))
+
+        # Strategy 2: Get all individual version DOIs
+        # Check if there's a link to all versions
+        links = data.get("links", {})
+        versions_url = links.get("versions")
+
+        if versions_url:
+            # Query the versions endpoint to get all versions
+            try:
+                versions_response = self.session.get(versions_url, timeout=30)
+                versions_response.raise_for_status()
+                versions_data = versions_response.json()
+
+                # Extract DOIs from each version
+                for hit in versions_data.get("hits", {}).get("hits", []):
+                    version_doi = hit.get("doi")
+                    if version_doi:
+                        version_doi_clean = version_doi.replace("https://doi.org/", "")
+                        # Don't duplicate concept DOI
+                        if version_doi_clean != concept_doi_clean:
+                            refs.append(
+                                ItemRef(ref_type=RefType("doi"), ref_value=version_doi_clean)
+                            )
+            except requests.RequestException as e:
+                logger.warning(f"Failed to fetch versions for {concept_id}: {e}")
+                # Fall back to just the concept DOI
+
+        logger.info(f"Expanded Zenodo concept {concept_id} to {len(refs)} DOI references")
+        return refs