scholarlib 0.3.0__py3-none-any.whl

scholarlib/__init__.py

@@ -0,0 +1,3 @@
+"""Shared bibliographic-API library behind the scholarfocus and litreview skills."""
+
+__version__ = "0.3.0"

scholarlib/adapters.py

@@ -0,0 +1,94 @@
+"""Convert each source's native shape into the canonical Record."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from scholarlib.apis.openalex import reconstruct_abstract
+from scholarlib.records import Record
+
+
+def _authors_from_openalex(work: dict) -> list[str]:
+    out = []
+    for a in work.get("authorships") or []:
+        name = ((a.get("author") or {}).get("display_name"))
+        if name:
+            out.append(name)
+    return out
+
+
+def _topics_from_openalex(work: dict) -> list[str]:
+    names = []
+    for t in (work.get("topics") or [])[:5]:
+        if t.get("display_name"):
+            names.append(t["display_name"])
+    for c in (work.get("concepts") or [])[:5]:
+        if c.get("display_name") and c["display_name"] not in names:
+            names.append(c["display_name"])
+    return names
+
+
+def from_openalex_work(work: dict, *, provenance: Optional[str] = None) -> Record:
+    abstract = reconstruct_abstract(work.get("abstract_inverted_index"))
+    loc = work.get("primary_location") or {}
+    src = loc.get("source") or {}
+    best_oa = work.get("best_oa_location") or {}
+    oa = work.get("open_access") or {}
+    return Record(
+        doi=work.get("doi"),
+        openalex_id=work.get("id"),
+        title=work.get("title") or work.get("display_name"),
+        authors=_authors_from_openalex(work),
+        year=work.get("publication_year"),
+        venue=src.get("display_name"),
+        type=work.get("type"),
+        language=work.get("language"),
+        abstract=abstract,
+        abstract_source="openalex" if abstract else None,
+        cited_by_count=work.get("cited_by_count"),
+        referenced_works=list(work.get("referenced_works") or []),
+        related_works=list(work.get("related_works") or []),
+        topics=_topics_from_openalex(work),
+        oa_status=oa.get("oa_status"),
+        oa_url=best_oa.get("pdf_url") or best_oa.get("landing_page_url"),
+        provenance=[provenance] if provenance else [],
+    )
+
+
+def from_crossref_item(item: dict, *, provenance: str = "crossref") -> Record:
+    titles = item.get("title") or []
+    containers = item.get("container-title") or []
+    authors = []
+    for a in item.get("author") or []:
+        nm = " ".join(x for x in (a.get("given"), a.get("family")) if x) or a.get("name")
+        if nm:
+            authors.append(nm)
+    issued = ((item.get("issued") or {}).get("date-parts") or [[None]])[0]
+    return Record(
+        doi=item.get("DOI"),
+        title=titles[0] if titles else None,
+        authors=authors,
+        year=issued[0] if issued else None,
+        venue=containers[0] if containers else None,
+        type=item.get("type"),
+        cited_by_count=item.get("is-referenced-by-count"),
+        provenance=[provenance],
+    )
+
+
+def from_s2_paper(paper: dict, *, provenance: str = "semantic_scholar") -> Record:
+    ext = paper.get("externalIds") or {}
+    tldr = (paper.get("tldr") or {}).get("text")
+    abstract = paper.get("abstract") or tldr
+    return Record(
+        doi=ext.get("DOI"),
+        title=paper.get("title"),
+        year=paper.get("year"),
+        venue=paper.get("venue"),
+        abstract=abstract,
+        abstract_source=("semantic_scholar" if paper.get("abstract")
+                         else ("s2_tldr" if tldr else None)),
+        cited_by_count=paper.get("citationCount"),
+        topics=list(paper.get("fieldsOfStudy") or []),
+        provenance=[provenance],
+    )

scholarlib/apis/core_api.py

@@ -0,0 +1,41 @@
+"""CORE client. Key is optional but raises the rate limit from 10/min to 150/min."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from scholarlib.http.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.core.ac.uk/v3"
+
+
+class COREClient(BaseClient):
+    name = "core"
+    base_url = BASE_URL
+
+    def __init__(self, api_key: Optional[str] = None, **kw):
+        super().__init__(**kw)
+        self.api_key = api_key
+        if not api_key:
+            logger.debug("CORE: no API key — rate limit is 10/min instead of 150/min")
+
+    def _auth_headers(self) -> dict:
+        return {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+
+    def search_works(self, query: str, limit: int = 10) -> list[dict]:
+        data = self.get("/search/works", {"q": query, "limit": limit})
+        return (data or {}).get("results", [])
+
+    def get_work_by_doi(self, doi: str) -> Optional[dict]:
+        d = str(doi).replace("https://doi.org/", "").strip()
+        results = self.search_works(f'doi:"{d}"', limit=1)
+        return results[0] if results else None
+
+    def get_abstract(self, doi: str) -> Optional[str]:
+        w = self.get_work_by_doi(doi)
+        if not w:
+            return None
+        return w.get("abstract") or w.get("description") or None

scholarlib/apis/crossref.py

@@ -0,0 +1,93 @@
+"""CrossRef client. Free, polite pool via a mailto in the User-Agent."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from scholarlib.http.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.crossref.org"
+
+# Verified against the live API. `reference-count` is NOT a valid select field
+# and returns HTTP 400 select-not-available -- that was the long-standing bug.
+# The valid spellings are `references-count` and `is-referenced-by-count`.
+SAFE_SELECT = (
+    "DOI,title,author,issued,type,container-title,"
+    "references-count,is-referenced-by-count,abstract"
+)
+
+
+class CrossRefClient(BaseClient):
+    name = "crossref"
+    base_url = BASE_URL
+
+    def __init__(self, email: Optional[str] = None, **kw):
+        super().__init__(contact_email=email, **kw)
+        self.email = email
+
+    def _auth_params(self) -> dict:
+        return {"mailto": self.email} if self.email else {}
+
+    def get_work_by_doi(self, doi: str) -> Optional[dict]:
+        d = str(doi).replace("https://doi.org/", "").strip()
+        data = self.get(f"/works/{d}")
+        return (data or {}).get("message")
+
+    def search_works_by_author(self, name: str, rows: int = 50,
+                               *, select: Optional[str] = None) -> list[dict]:
+        params: dict = {"query.author": name, "rows": min(rows, 1000)}
+        if select:
+            params["select"] = select
+        data = self.get("/works", params)
+        return (data or {}).get("message", {}).get("items", [])
+
+    def search_works(self, query: str, rows: int = 50,
+                     from_year: Optional[int] = None) -> list[dict]:
+        params: dict = {"query.bibliographic": query, "rows": min(rows, 1000)}
+        if from_year:
+            params["filter"] = f"from-pub-date:{from_year}-01-01"
+        data = self.get("/works", params)
+        return (data or {}).get("message", {}).get("items", [])
+
+    def get_references(self, doi: str) -> list[dict]:
+        """Only publishers who deposit references expose them."""
+        work = self.get_work_by_doi(doi)
+        return (work or {}).get("reference", []) or []
+
+    def get_abstract(self, doi: str) -> Optional[str]:
+        """CrossRef abstracts are JATS-wrapped; strip the tags."""
+        import re
+
+        work = self.get_work_by_doi(doi)
+        raw = (work or {}).get("abstract")
+        if not raw:
+            return None
+        text = re.sub(r"<[^>]+>", " ", raw)
+        text = re.sub(r"\s+", " ", text).strip()
+        return text or None
+
+    def enrich_doi_metadata(self, doi: str) -> Optional[dict]:
+        """Normalise a CrossRef record into a flat shape."""
+        work = self.get_work_by_doi(doi)
+        if not work:
+            return None
+        authors = []
+        for a in work.get("author", []) or []:
+            given, family = a.get("given"), a.get("family")
+            authors.append(" ".join(x for x in (given, family) if x) or a.get("name", ""))
+        issued = ((work.get("issued") or {}).get("date-parts") or [[None]])[0]
+        titles = work.get("title") or []
+        containers = work.get("container-title") or []
+        return {
+            "doi": work.get("DOI"),
+            "title": titles[0] if titles else None,
+            "authors": [a for a in authors if a],
+            "year": issued[0] if issued else None,
+            "journal": containers[0] if containers else None,
+            "type": work.get("type"),
+            "reference_count": work.get("references-count"),
+            "is_referenced_by_count": work.get("is-referenced-by-count"),
+        }

scholarlib/apis/openaire.py

@@ -0,0 +1,88 @@
+"""OpenAIRE client. Free, no key.
+
+Strong on European, multilingual and repository-held material, which matters
+because a fifth of the JSTOR corpus is non-English. The response JSON is deeply
+nested and inconsistently typed, so everything goes through _first().
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+from scholarlib.http.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.openaire.eu/search"
+
+
+def _first(node: Any, *path: str) -> Any:
+    """Walk a path, tolerating dict-or-list-of-dicts at every level."""
+    cur = node
+    for key in path:
+        if isinstance(cur, list):
+            cur = cur[0] if cur else None
+        if not isinstance(cur, dict):
+            return None
+        cur = cur.get(key)
+    if isinstance(cur, list):
+        cur = cur[0] if cur else None
+    return cur
+
+
+def _text(node: Any) -> Optional[str]:
+    """OpenAIRE wraps scalars as {'$': value}."""
+    if isinstance(node, list):
+        node = node[0] if node else None
+    if isinstance(node, dict):
+        node = node.get("$")
+    if node is None:
+        return None
+    s = str(node).strip()
+    return s or None
+
+
+class OpenAIREClient(BaseClient):
+    name = "openaire"
+    base_url = BASE_URL
+
+    def _results(self, data: Optional[dict]) -> list[dict]:
+        res = _first(data, "response", "results")
+        if not isinstance(res, dict):
+            return []
+        rows = res.get("result")
+        if rows is None:
+            return []
+        return rows if isinstance(rows, list) else [rows]
+
+    def _entity(self, row: dict) -> Optional[dict]:
+        ent = _first(row, "metadata", "oaf:entity", "oaf:result")
+        return ent if isinstance(ent, dict) else None
+
+    def search(self, query: str, *, size: int = 20,
+               from_year: Optional[int] = None,
+               to_year: Optional[int] = None) -> list[dict]:
+        params: dict = {"keywords": query, "size": min(size, 100), "format": "json"}
+        if from_year:
+            params["fromDateAccepted"] = f"{from_year}-01-01"
+        if to_year:
+            params["toDateAccepted"] = f"{to_year}-12-31"
+        return self._results(self.get("/publications", params))
+
+    def get_by_doi(self, doi: str) -> Optional[dict]:
+        d = str(doi).replace("https://doi.org/", "").strip()
+        rows = self._results(self.get("/publications", {"doi": d, "format": "json", "size": 1}))
+        return rows[0] if rows else None
+
+    def get_abstract(self, doi: str) -> Optional[str]:
+        row = self.get_by_doi(doi)
+        if not row:
+            return None
+        ent = self._entity(row)
+        if not ent:
+            return None
+        text = _text(ent.get("description"))
+        if text and len(text) > 20:
+            return text
+        return None

scholarlib/apis/openalex.py

@@ -0,0 +1,199 @@
+"""OpenAlex client.
+
+Credit costs measured 2026-09-20: singleton 0, list/cites/group_by 1,
+batch of 50 ids 1, search 10. A free API key raises the daily budget from
+~1000 to ~10000 credits.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Iterable, Iterator, Optional
+
+from scholarlib.http.base import BaseClient
+from scholarlib.http.budget import COSTS, classify
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.openalex.org"
+
+# Fields for researcher profiling (scholarfocus).
+WORK_FIELDS = (
+    "id,doi,title,publication_year,type,"
+    "authorships,concepts,topics,keywords,"
+    "referenced_works,cited_by_count,"
+    "abstract_inverted_index"
+)
+
+# Minimal fields for bulk reference lookups.
+REF_WORK_FIELDS = "id,doi,title,publication_year,authorships,cited_by_count"
+
+# Literature review needs abstracts and the full citation edges.
+# related_works matters because monographs have referenced_works_count == 0.
+LITREVIEW_WORK_FIELDS = (
+    "id,doi,title,display_name,publication_year,type,language,"
+    "authorships,topics,concepts,keywords,"
+    "referenced_works,referenced_works_count,related_works,"
+    "cited_by_count,abstract_inverted_index,"
+    "primary_location,best_oa_location,open_access"
+)
+
+BATCH_SIZE = 50
+
+
+def reconstruct_abstract(inverted_index: Optional[dict]) -> Optional[str]:
+    """Rebuild plain text from OpenAlex's inverted index.
+
+    ~92% of works carry one, which is why Semantic Scholar is an
+    enhancement rather than a dependency.
+    """
+    if not inverted_index:
+        return None
+    positions: list[tuple[int, str]] = []
+    for word, idxs in inverted_index.items():
+        for i in idxs:
+            positions.append((i, word))
+    if not positions:
+        return None
+    positions.sort()
+    return " ".join(word for _, word in positions)
+
+
+def short_id(work_id: str) -> str:
+    """https://openalex.org/W123 -> W123"""
+    return str(work_id).rstrip("/").rsplit("/", 1)[-1]
+
+
+def chunked(items: Iterable, size: int) -> Iterator[list]:
+    buf: list = []
+    for it in items:
+        buf.append(it)
+        if len(buf) >= size:
+            yield buf
+            buf = []
+    if buf:
+        yield buf
+
+
+class OpenAlexClient(BaseClient):
+    name = "openalex"
+    base_url = BASE_URL
+
+    def __init__(self, email: Optional[str] = None, api_key: Optional[str] = None, **kw):
+        super().__init__(contact_email=email, **kw)
+        self.email = email
+        self.api_key = api_key
+
+    def _auth_params(self) -> dict:
+        p = {}
+        if self.email:
+            p["mailto"] = self.email
+        if self.api_key:
+            p["api_key"] = self.api_key
+        return p
+
+    def _cost(self, path: str, params: dict) -> tuple[str, int]:
+        klass = classify(path, params)
+        return klass, COSTS.get(klass, 1)
+
+    def _ttl(self, path: str, params: dict) -> int:
+        """Cost drives TTL: searches are 10 credits, singletons are free."""
+        klass = classify(path, params)
+        if klass == "search":
+            return 90 * 86400
+        if klass == "singleton":
+            return 14 * 86400
+        return 30 * 86400
+
+    # ---- authors --------------------------------------------------------
+
+    def get_author_by_orcid(self, orcid: str) -> Optional[dict]:
+        o = str(orcid).strip()
+        if not o.startswith("http"):
+            o = f"https://orcid.org/{o}"
+        return self.get(f"/authors/{o}")
+
+    def search_authors(self, name: str, limit: int = 10) -> list[dict]:
+        data = self.get("/authors", {"search": name, "per-page": limit})
+        return (data or {}).get("results", [])
+
+    def get_author_by_id(self, author_id: str) -> Optional[dict]:
+        return self.get(f"/authors/{short_id(author_id)}")
+
+    # ---- works ----------------------------------------------------------
+
+    def get_work(self, work_id: str, select: Optional[str] = None) -> Optional[dict]:
+        """Free: singleton lookups cost 0 credits."""
+        params = {"select": select} if select else None
+        return self.get(f"/works/{short_id(work_id)}", params)
+
+    def get_work_by_doi(self, doi: str, select: Optional[str] = None) -> Optional[dict]:
+        """Free. Always prefer this over searching for a title."""
+        d = str(doi).replace("https://doi.org/", "").strip()
+        params = {"select": select} if select else None
+        return self.get(f"/works/doi:{d}", params)
+
+    def get_author_works(self, author_id: str, max_works: int = 100,
+                         select: str = WORK_FIELDS) -> list[dict]:
+        return list(self.get_paged(
+            "/works",
+            {"filter": f"author.id:{short_id(author_id)}",
+             "select": select, "sort": "cited_by_count:desc"},
+            page_size=min(max_works, 200), max_items=max_works,
+        ))
+
+    def get_works_batch(self, work_ids: Iterable[str],
+                        select: str = REF_WORK_FIELDS) -> list[dict]:
+        """50 works per credit — the cheapest way to hydrate known IDs."""
+        ids = [short_id(w) for w in work_ids if w]
+        out: list[dict] = []
+        for chunk in chunked(ids, BATCH_SIZE):
+            data = self.get("/works", {
+                "filter": f"ids.openalex:{'|'.join(chunk)}",
+                "per-page": BATCH_SIZE,
+                "select": select,
+            })
+            if data:
+                out.extend(data.get("results") or [])
+        return out
+
+    def search_works(self, query: str, *, limit: int = 50, from_year: Optional[int] = None,
+                     to_year: Optional[int] = None, types: Optional[list[str]] = None,
+                     languages: Optional[list[str]] = None,
+                     select: str = LITREVIEW_WORK_FIELDS) -> list[dict]:
+        """10 credits per page — the expensive operation. Use sparingly."""
+        filters = []
+        if from_year:
+            filters.append(f"from_publication_date:{from_year}-01-01")
+        if to_year:
+            filters.append(f"to_publication_date:{to_year}-12-31")
+        if types:
+            filters.append(f"type:{'|'.join(types)}")
+        if languages:
+            filters.append(f"language:{'|'.join(languages)}")
+        params = {"search": query, "per-page": min(limit, 200), "select": select}
+        if filters:
+            params["filter"] = ",".join(filters)
+        data = self.get("/works", params)
+        return (data or {}).get("results", [])
+
+    def cited_by(self, work_ids: Iterable[str], *, max_items: int = 200,
+                 from_year: Optional[int] = None,
+                 select: str = LITREVIEW_WORK_FIELDS) -> list[dict]:
+        """Forward snowballing. The OR-packed filter makes 50 seeds cost 1 credit."""
+        ids = [short_id(w) for w in work_ids if w]
+        out: list[dict] = []
+        for chunk in chunked(ids, BATCH_SIZE):
+            filt = f"cites:{'|'.join(chunk)}"
+            if from_year:
+                filt += f",from_publication_date:{from_year}-01-01"
+            out.extend(self.get_paged(
+                "/works", {"filter": filt, "select": select},
+                page_size=200, max_items=max_items,
+            ))
+        return out
+
+    def group_by(self, filter_expr: str, dimension: str) -> list[dict]:
+        """Aggregate shape of a result set for 1 credit, fetching no works."""
+        data = self.get("/works", {"filter": filter_expr, "group_by": dimension})
+        return (data or {}).get("group_by", [])

scholarlib/apis/orcid.py

@@ -0,0 +1,118 @@
+"""ORCID public API client.
+
+Free, no key. The works list is *author-curated*, which makes it far more
+complete than OpenAlex for researchers whose records OpenAlex has fragmented.
+Tuomas Tammisto, for instance, has 50 works in ORCID and 1 in OpenAlex.
+
+It carries no citation counts or topics, so it pairs with OpenAlex: ORCID
+supplies the authoritative work list, OpenAlex hydrates the DOIs (free, since
+singleton lookups cost 0 credits).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from scholarlib.http.base import BaseClient
+from scholarlib.records import Record
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://pub.orcid.org/v3.0"
+
+_TYPE_MAP = {
+    "journal-article": "article",
+    "book": "book",
+    "book-chapter": "chapter",
+    "book-review": "review",
+    "conference-paper": "conference-paper",
+    "dissertation-thesis": "dissertation",
+    "preprint": "preprint",
+    "report": "report",
+    "edited-book": "book",
+}
+
+
+def _val(node, *path):
+    cur = node
+    for k in path:
+        if not isinstance(cur, dict):
+            return None
+        cur = cur.get(k)
+    return cur
+
+
+class OrcidClient(BaseClient):
+    name = "orcid"
+    base_url = BASE_URL
+    default_ttl = 14 * 86400
+
+    def _auth_headers(self) -> dict:
+        return {"Accept": "application/json"}
+
+    def search_by_name(self, name: str, *, rows: int = 10) -> list[dict]:
+        """Find ORCID iDs for a display name. Free, no key."""
+        parts = str(name).strip().split()
+        if len(parts) >= 2:
+            given, family = " ".join(parts[:-1]), parts[-1]
+            q = f'family-name:{family} AND given-names:{given}'
+        else:
+            q = f'family-name:{parts[0]}' if parts else ""
+        if not q:
+            return []
+        data = self.get("/expanded-search/", {"q": q, "rows": rows})
+        out = []
+        for r in (data or {}).get("expanded-result") or []:
+            out.append({
+                "orcid": r.get("orcid-id"),
+                "name": " ".join(x for x in (r.get("given-names"),
+                                             r.get("family-names")) if x),
+                "institutions": list(r.get("institution-name") or []),
+            })
+        return out
+
+    def get_person(self, orcid: str) -> Optional[dict]:
+        return self.get(f"/{_clean(orcid)}/person")
+
+    def get_employments(self, orcid: str) -> list[str]:
+        data = self.get(f"/{_clean(orcid)}/employments")
+        out = []
+        for grp in (data or {}).get("affiliation-group") or []:
+            for s in grp.get("summaries") or []:
+                nm = _val(s, "employment-summary", "organization", "name")
+                if nm and nm not in out:
+                    out.append(nm)
+        return out
+
+    def get_works(self, orcid: str, *, max_works: int = 500) -> list[Record]:
+        """The author-curated work list, as canonical Records."""
+        data = self.get(f"/{_clean(orcid)}/works")
+        groups = (data or {}).get("group") or []
+        out: list[Record] = []
+        for g in groups[:max_works]:
+            summaries = g.get("work-summary") or []
+            if not summaries:
+                continue
+            s = summaries[0]
+            title = _val(s, "title", "title", "value")
+            if not title:
+                continue
+            year = _val(s, "publication-date", "year", "value")
+            ids = {e.get("external-id-type"): e.get("external-id-value")
+                   for e in (_val(g, "external-ids", "external-id") or [])}
+            out.append(Record(
+                doi=ids.get("doi"),
+                title=title,
+                year=int(year) if year and str(year).isdigit() else None,
+                venue=_val(s, "journal-title", "value"),
+                type=_TYPE_MAP.get(s.get("type"), s.get("type")),
+                provenance=["orcid"],
+            ))
+        logger.info("ORCID %s: %d works (%d with DOIs)",
+                    _clean(orcid), len(out), sum(1 for r in out if r.doi))
+        return out
+
+
+def _clean(orcid: str) -> str:
+    return str(orcid).strip().replace("https://orcid.org/", "")