scholarlib 0.3.0__py3-none-any.whl

scholarlib/apis/semantic_scholar.py

@@ -0,0 +1,99 @@
+"""Semantic Scholar client.
+
+Disabled by default. OpenAlex covers ~92% of abstracts, so S2AG is an
+enhancement rather than a dependency; its remaining edge is the `tldr`
+summaries and its own fields-of-study labels.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from scholarlib.http.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.semanticscholar.org/graph/v1"
+
+AUTHOR_FIELDS = "authorId,name,externalIds,affiliations,paperCount,citationCount,hIndex"
+PAPER_FIELDS = (
+    "paperId,externalIds,title,abstract,year,venue,"
+    "fieldsOfStudy,s2FieldsOfStudy,citationCount,tldr"
+)
+
+
+class SemanticScholarClient(BaseClient):
+    name = "semantic_scholar"
+    base_url = BASE_URL
+
+    def __init__(self, api_key: Optional[str] = None, enabled: bool = False, **kw):
+        super().__init__(**kw)
+        self.api_key = api_key
+        self.enabled = enabled
+        # Unauthenticated S2AG 429s almost immediately.
+        if not api_key:
+            object.__setattr__(self.policy, "min_interval", 1.1)
+
+    def _available(self) -> bool:
+        return self.enabled
+
+    def _unavailable_reason(self) -> str:
+        return "semantic_scholar: disabled (enable with --enable-s2ag)"
+
+    def _auth_headers(self) -> dict:
+        return {"x-api-key": self.api_key} if self.api_key else {}
+
+    def health(self) -> tuple[bool, str]:
+        """Probe whether the key actually works. Returns (ok, message)."""
+        was = self.enabled
+        self.enabled = True
+        try:
+            data = self.get("/paper/search", {"query": "anthropology", "limit": 1,
+                                              "fields": "title"}, use_cache=False)
+        finally:
+            self.enabled = was
+        if data and data.get("data") is not None:
+            return True, "ok"
+        if not self.api_key:
+            return False, "no API key configured"
+        return False, "key rejected or rate-limited (403/429)"
+
+    def get_paper(self, paper_id: str, fields: str = PAPER_FIELDS) -> Optional[dict]:
+        """`paper_id` may be an S2 id or a `DOI:10.xxxx/yyy` string."""
+        return self.get(f"/paper/{paper_id}", {"fields": fields})
+
+    def get_paper_by_doi(self, doi: str, fields: str = PAPER_FIELDS) -> Optional[dict]:
+        d = str(doi).replace("https://doi.org/", "").strip()
+        return self.get_paper(f"DOI:{d}", fields)
+
+    def get_abstract(self, doi: str) -> Optional[str]:
+        p = self.get_paper_by_doi(doi, fields="abstract")
+        return (p or {}).get("abstract")
+
+    def get_tldr(self, doi: str) -> Optional[str]:
+        p = self.get_paper_by_doi(doi, fields="tldr")
+        tldr = (p or {}).get("tldr") or {}
+        return tldr.get("text")
+
+    def search_authors(self, name: str, limit: int = 10) -> list[dict]:
+        data = self.get("/author/search", {"query": name, "limit": limit,
+                                           "fields": AUTHOR_FIELDS})
+        return (data or {}).get("data", [])
+
+    def get_author_papers(self, author_id: str, max_papers: int = 100) -> list[dict]:
+        out: list[dict] = []
+        offset = 0
+        while len(out) < max_papers:
+            data = self.get(f"/author/{author_id}/papers", {
+                "fields": PAPER_FIELDS, "limit": min(100, max_papers - len(out)),
+                "offset": offset,
+            })
+            if not data:
+                break
+            rows = data.get("data") or []
+            out.extend(rows)
+            if not rows or "next" not in data:
+                break
+            offset = data["next"]
+        return out[:max_papers]

scholarlib/apis/unpaywall.py

@@ -0,0 +1,47 @@
+"""Unpaywall client: open-access status and full-text locations per DOI.
+
+Free, no key, but the email parameter is required by the API.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from scholarlib.http.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://api.unpaywall.org/v2"
+
+
+class UnpaywallClient(BaseClient):
+    name = "unpaywall"
+    base_url = BASE_URL
+    default_ttl = 30 * 86400  # OA status genuinely changes over time
+
+    def __init__(self, email: Optional[str] = None, **kw):
+        super().__init__(contact_email=email, **kw)
+        self.email = email
+
+    def _available(self) -> bool:
+        return bool(self.email)
+
+    def _unavailable_reason(self) -> str:
+        return "unpaywall: no email configured (apis.unpaywall.email)"
+
+    def _auth_params(self) -> dict:
+        return {"email": self.email} if self.email else {}
+
+    def get_oa(self, doi: str) -> Optional[dict]:
+        d = str(doi).replace("https://doi.org/", "").strip()
+        return self.get(f"/{d}")
+
+    def oa_for(self, doi: str) -> tuple[Optional[str], Optional[str]]:
+        """Return (oa_status, best full-text url)."""
+        data = self.get_oa(doi)
+        if not data:
+            return None, None
+        best = data.get("best_oa_location") or {}
+        url = best.get("url_for_pdf") or best.get("url")
+        return data.get("oa_status"), url

scholarlib/apis/zotero_local.py

@@ -0,0 +1,156 @@
+"""Zotero local HTTP API bridge.
+
+Read-only, and only answers while the Zotero desktop app is running with
+Settings -> Advanced -> "Allow other applications on this computer to
+communicate with Zotero" enabled.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Iterator, Optional
+
+from scholarlib import dedup
+from scholarlib.http.base import BaseClient
+from scholarlib.records import Record
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE = "http://localhost:23119/api/users/0"
+
+_TYPE_MAP = {
+    "journalArticle": "article",
+    "book": "book",
+    "bookSection": "chapter",
+    "conferencePaper": "conference-paper",
+    "thesis": "dissertation",
+    "preprint": "preprint",
+    "report": "report",
+}
+
+_SKIP_TYPES = {"attachment", "note", "annotation"}
+
+
+class ZoteroLocalClient(BaseClient):
+    name = "zotero"
+
+    def __init__(self, base_url: str = DEFAULT_BASE, enabled: bool = True, **kw):
+        super().__init__(**kw)
+        self.base_url = base_url
+        self.enabled = enabled
+        self._checked: Optional[bool] = None
+
+    def _available(self) -> bool:
+        return self.enabled
+
+    def available(self) -> bool:
+        """Probe once; cache the answer for the run."""
+        if self._checked is None:
+            if not self.enabled:
+                self._checked = False
+            else:
+                self._checked = self.get("/items", {"limit": 1}, use_cache=False) is not None
+                if not self._checked:
+                    logger.warning(
+                        "Zotero is not reachable at %s. Start Zotero and enable "
+                        'Settings -> Advanced -> "Allow other applications on this '
+                        'computer to communicate with Zotero". Continuing without it.',
+                        self.base_url,
+                    )
+        return self._checked
+
+    def collections(self) -> list[dict]:
+        return self.get("/collections", {"limit": 100}, use_cache=False) or []
+
+    def collection_by_name(self, name: str) -> Optional[dict]:
+        target = (name or "").strip().casefold()
+        for c in self.collections():
+            if str((c.get("data") or {}).get("name", "")).strip().casefold() == target:
+                return c
+        return None
+
+    def items(self, *, collection_key: Optional[str] = None,
+              limit: int = 100, max_items: int = 10000) -> Iterator[dict]:
+        path = f"/collections/{collection_key}/items" if collection_key else "/items"
+        start = 0
+        while start < max_items:
+            rows = self.get(path, {"limit": limit, "start": start}, use_cache=False)
+            if not rows:
+                return
+            for r in rows:
+                yield r
+            if len(rows) < limit:
+                return
+            start += len(rows)
+
+    @staticmethod
+    def to_record(item: dict) -> Optional[Record]:
+        d = item.get("data") or {}
+        itype = d.get("itemType")
+        if not itype or itype in _SKIP_TYPES:
+            return None
+        creators = [
+            " ".join(x for x in (c.get("firstName"), c.get("lastName")) if x) or c.get("name", "")
+            for c in (d.get("creators") or [])
+            if c.get("creatorType") == "author"
+        ]
+        year = None
+        date = str(d.get("date") or "")
+        for chunk in date.replace("-", " ").replace("/", " ").split():
+            if len(chunk) == 4 and chunk.isdigit():
+                year = int(chunk)
+                break
+        return Record(
+            doi=d.get("DOI"),
+            title=d.get("title"),
+            authors=[c for c in creators if c],
+            year=year,
+            venue=d.get("publicationTitle") or d.get("bookTitle") or d.get("publisher"),
+            type=_TYPE_MAP.get(itype, itype),
+            language=d.get("language") or None,
+            abstract=d.get("abstractNote") or None,
+            abstract_source="zotero" if d.get("abstractNote") else None,
+            in_zotero=True,
+            zotero_key=d.get("key"),
+            zotero_citekey=(d.get("citationKey") or None),
+            provenance=["zotero"],
+        )
+
+    def load_library(self, collection_key: Optional[str] = None) -> list[Record]:
+        if not self.available():
+            return []
+        out: list[Record] = []
+        for item in self.items(collection_key=collection_key):
+            rec = self.to_record(item)
+            if rec and (rec.doi or rec.title):
+                out.append(rec)
+        logger.info("Zotero: loaded %d items", len(out))
+        return out
+
+
+class ZoteroLookup:
+    """DOI-first, then blocked title match, for 'do I already own this?'."""
+
+    def __init__(self, records: list[Record]):
+        self.by_doi: dict[str, Record] = {}
+        self.by_block: dict[tuple, list[Record]] = {}
+        for r in records:
+            if r.doi:
+                self.by_doi[r.doi] = r
+            self.by_block.setdefault((r.first_surname, r.year), []).append(r)
+
+    def __len__(self) -> int:
+        return len(self.by_doi) + sum(len(v) for v in self.by_block.values())
+
+    def match(self, rec: Record, *, threshold: float = 0.93,
+              year_slack: int = 1) -> Optional[Record]:
+        if rec.doi and rec.doi in self.by_doi:
+            return self.by_doi[rec.doi]
+        if not dedup.titles_comparable(rec.title):
+            return None
+        for dy in range(-year_slack, year_slack + 1):
+            y = (rec.year + dy) if rec.year else None
+            for cand in self.by_block.get((rec.first_surname, y), []):
+                if dedup.best_title_similarity(rec.title, cand.title) >= threshold:
+                    return cand
+        return None

scholarlib/cli/jstor_index.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Build and query the local JSTOR metadata index."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sqlite3
+import sys
+from pathlib import Path
+
+from scholarlib.config import ConfigError, expand_path, load_config
+
+logging.basicConfig(stream=sys.stderr, level=logging.INFO, format="%(levelname)s  %(message)s")
+logger = logging.getLogger("jstor-index")
+
+
+def _db_path(args, cfg) -> Path:
+    p = args.db or (cfg.get("jstor") or {}).get("db_path")
+    if not p:
+        raise ConfigError("No JSTOR db path: pass --db or set jstor.db_path in config.yaml")
+    return Path(str(p)).expanduser()
+
+
+def cmd_build(args, cfg) -> int:
+    from scholarlib.jstor.build import build
+
+    src = args.source or (cfg.get("jstor") or {}).get("source_path")
+    if not src:
+        raise ConfigError("No JSTOR source: pass --source or set jstor.source_path")
+    disciplines = None
+    if args.disciplines and not args.all_disciplines:
+        disciplines = frozenset(d.strip() for d in args.disciplines.split(",") if d.strip())
+    elif not args.all_disciplines:
+        cfg_disc = (cfg.get("jstor") or {}).get("disciplines")
+        if cfg_disc:
+            disciplines = frozenset(cfg_disc)
+    languages = frozenset(x.strip() for x in args.languages.split(",")) if args.languages else None
+
+    stats = build(
+        Path(str(src)).expanduser(),
+        _db_path(args, cfg),
+        disciplines=disciplines,
+        languages=languages,
+        keep_reviews=args.keep_reviews or (cfg.get("jstor") or {}).get("keep_reviews", False),
+        resume=args.resume,
+        force=args.force,
+        progress_every=args.progress_every,
+        limit=args.limit,
+    )
+    print(json.dumps(stats, indent=2))
+    return 0
+
+
+def cmd_stats(args, cfg) -> int:
+    from scholarlib.jstor.query import JstorIndex
+
+    idx = JstorIndex(_db_path(args, cfg))
+    print(json.dumps(idx.stats(), indent=2))
+    return 0
+
+
+def cmd_search(args, cfg) -> int:
+    from scholarlib.jstor.query import JstorIndex
+
+    idx = JstorIndex(_db_path(args, cfg))
+    rows = idx.search(
+        args.text,
+        disciplines=args.disciplines.split(",") if args.disciplines else None,
+        types=args.types.split(",") if args.types else None,
+        from_year=args.from_year,
+        to_year=args.to_year,
+        limit=args.limit or 20,
+    )
+    if args.json:
+        print(json.dumps([r.to_dict() for r in rows], indent=2, ensure_ascii=False))
+    else:
+        for r in rows:
+            who = r.first_surname or "?"
+            print(f"{r.year or '????'}  {who:<18.18}  [{r.type or '?':<9.9}]  {r.title}")
+            if r.jstor:
+                print(f"        {r.jstor.get('url')}")
+    return 0
+
+
+def cmd_match(args, cfg) -> int:
+    from scholarlib.jstor.query import JstorIndex
+    from scholarlib.records import Record
+
+    idx = JstorIndex(_db_path(args, cfg))
+    probe = Record(title=args.title, authors=[args.author] if args.author else [], year=args.year)
+    hit, score, method = idx.find_match(probe)
+    print(json.dumps({
+        "matched": hit is not None,
+        "score": round(score, 3),
+        "method": method,
+        "record": hit.to_dict() if hit else None,
+    }, indent=2, ensure_ascii=False))
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="Build and query the local JSTOR metadata index.")
+    ap.add_argument("--config", help="Path to config.yaml")
+    ap.add_argument("--db", help="Path to the index database")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    b = sub.add_parser("build", help="Build or refresh the index")
+    b.add_argument("--source", help="Path to the .jsonl.gz dump")
+    b.add_argument("--disciplines", help="Comma-separated discipline allowlist")
+    b.add_argument("--all-disciplines", action="store_true", help="Index every discipline")
+    b.add_argument("--languages", help="Comma-separated ISO-639-2/B codes")
+    b.add_argument("--keep-reviews", action="store_true",
+                   help="Also index book reviews (~90%% are untitled)")
+    b.add_argument("--resume", action="store_true", help="Continue an interrupted build")
+    b.add_argument("--force", action="store_true", help="Rebuild even if up to date")
+    b.add_argument("--progress-every", type=int, default=250_000)
+    b.add_argument("--limit", type=int, help="Stop after N kept records (for testing)")
+    b.set_defaults(func=cmd_build)
+
+    s = sub.add_parser("stats", help="Show what is in the index")
+    s.set_defaults(func=cmd_stats)
+
+    q = sub.add_parser("search", help="Full-text search the index")
+    q.add_argument("--text", required=True)
+    q.add_argument("--disciplines")
+    q.add_argument("--types")
+    q.add_argument("--from-year", type=int)
+    q.add_argument("--to-year", type=int)
+    q.add_argument("--limit", type=int)
+    q.add_argument("--json", action="store_true")
+    q.set_defaults(func=cmd_search)
+
+    m = sub.add_parser("match", help="Find the JSTOR record matching a title/author/year")
+    m.add_argument("--title", required=True)
+    m.add_argument("--author")
+    m.add_argument("--year", type=int)
+    m.set_defaults(func=cmd_match)
+
+    args = ap.parse_args()
+    try:
+        cfg = load_config(args.config)
+        sys.exit(args.func(args, cfg))
+    except ConfigError as e:
+        logger.error("Configuration problem: %s", e)
+        sys.exit(3)
+    except FileNotFoundError as e:
+        logger.error("%s", e)
+        sys.exit(5)
+    except sqlite3.OperationalError as e:
+        logger.error("Index not usable (%s). Build it first: "
+                     "python -m scholarlib.cli.jstor_index build", e)
+        sys.exit(5)
+    except KeyboardInterrupt:
+        logger.error("Interrupted — rerun with --resume to continue")
+        sys.exit(1)
+    except Exception as e:
+        logger.error("Failed: %s", e)
+        logger.debug("traceback", exc_info=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()