pyPaperFlow 0.2.0__py3-none-any.whl

pyPaperFlow/preprint/biorxiv_fetcher.py

@@ -0,0 +1,404 @@
+from __future__ import annotations
+
+import time
+import re
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from .source_models import SourcePaper
+from .source_utils import (
+    basic_boolean_text_match,
+    build_source_record_dir,
+    download_binary,
+    ensure_directory,
+    extract_year,
+    normalize_text,
+    safe_filename,
+    save_json,
+)
+
+
+BIO_RXIV_API_BASE = "https://api.biorxiv.org/details/biorxiv"
+BIO_RXIV_CROSSREF_API = "https://api.crossref.org/works"
+BIO_RXIV_CROSSREF_PREFIX = "10.64898"
+BIO_RXIV_LANDING_BASE = "https://www.biorxiv.org/content"
+BIO_RXIV_LAUNCH_DATE = datetime(2013, 1, 1)
+
+
+class BioRxivFetcher:
+    def __init__(
+        self,
+        root_dir: str,
+        window_days: int = 365,
+        max_retries: int = 3,
+        request_timeout: float = 60.0,
+    ):
+        self.root_dir = root_dir
+        self.window_days = max(1, int(window_days))
+        self.max_retries = max(1, int(max_retries))
+        self.request_timeout = float(request_timeout)
+        self.headers = {
+            "User-Agent": "pyPaperFlow/0.1.0 (+https://github.com/MaybeBio/pyPaperFlow)",
+            "Accept": "application/json,text/html;q=0.9,*/*;q=0.8",
+        }
+
+    def search(
+        self,
+        query: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        max_results: Optional[int] = None,
+    ) -> List[SourcePaper]:
+        query_text = normalize_text(query)
+        if not query_text:
+            raise ValueError("query must be non-empty")
+
+        records: List[SourcePaper] = []
+        start_dt, end_dt = self._normalize_date_range(start_date, end_date)
+
+        cursor = "*"
+        while True:
+            page_size = 1000
+            if max_results is not None:
+                remaining = max(1, int(max_results) - len(records))
+                page_size = min(page_size, remaining)
+
+            payload = self._request_crossref_page(
+                query_text=query_text,
+                cursor=cursor,
+                page_size=page_size,
+                start_dt=start_dt,
+                end_dt=end_dt,
+            )
+            message = payload.get("message") or {}
+            items = message.get("items") or []
+            if not items:
+                break
+
+            for raw_record in items:
+                if normalize_text(raw_record.get("publisher", "")).lower() != "openrxiv":
+                    continue
+                if not basic_boolean_text_match(self._search_text_crossref(raw_record), query_text):
+                    continue
+                record = self._normalize_crossref_record(raw_record, query=query_text)
+                records.append(record)
+                if max_results is not None and len(records) >= max_results:
+                    return records
+
+            if len(items) < page_size:
+                break
+
+            next_cursor = normalize_text(message.get("next-cursor", ""))
+            if not next_cursor or next_cursor == cursor:
+                break
+            cursor = next_cursor
+
+        return records
+
+    def fetch_from_query(
+        self,
+        query: str,
+        output_dir: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        max_results: Optional[int] = None,
+        download_pdf: bool = True,
+    ) -> List[SourcePaper]:
+        records = self.search(
+            query=query,
+            start_date=start_date,
+            end_date=end_date,
+            max_results=max_results,
+        )
+
+        for record in records:
+            self._save_record(record, output_dir=output_dir, download_pdf=download_pdf)
+
+        return records
+
+    def _normalize_date_range(
+        self,
+        start_date: Optional[str],
+        end_date: Optional[str],
+    ) -> tuple[datetime, datetime]:
+        if start_date:
+            start_dt = datetime.strptime(start_date, "%Y-%m-%d")
+        else:
+            start_dt = BIO_RXIV_LAUNCH_DATE
+
+        if end_date:
+            end_dt = datetime.strptime(end_date, "%Y-%m-%d")
+        else:
+            end_dt = datetime.utcnow()
+
+        if start_dt < BIO_RXIV_LAUNCH_DATE:
+            start_dt = BIO_RXIV_LAUNCH_DATE
+        if start_dt > end_dt:
+            raise ValueError("start_date cannot be later than end_date")
+        return start_dt, end_dt
+
+    def _request_crossref_page(
+        self,
+        query_text: str,
+        cursor: str,
+        page_size: int,
+        start_dt: datetime,
+        end_dt: datetime,
+    ) -> Dict[str, Any]:
+        params: Dict[str, Any] = {
+            "filter": f"prefix:{BIO_RXIV_CROSSREF_PREFIX},type:posted-content",
+            "query.bibliographic": query_text,
+            "rows": page_size,
+            "cursor": cursor,
+            "sort": "relevance",
+        }
+        if start_dt:
+            params["filter"] += f",from-pub-date:{start_dt.strftime('%Y-%m-%d')}"
+        if end_dt:
+            params["filter"] += f",until-pub-date:{end_dt.strftime('%Y-%m-%d')}"
+
+        last_error: Optional[Exception] = None
+
+        for attempt in range(self.max_retries):
+            try:
+                response = requests.get(
+                    BIO_RXIV_CROSSREF_API,
+                    params=params,
+                    headers=self.headers,
+                    timeout=self.request_timeout,
+                )
+                response.raise_for_status()
+                return response.json()
+            except Exception as exc:
+                last_error = exc
+                if attempt + 1 < self.max_retries:
+                    time.sleep(min(2.0, 0.5 * (attempt + 1)))
+
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("Failed to query bioRxiv Crossref search")
+
+    def _search_text_crossref(self, record: Dict[str, Any]) -> str:
+        pieces = [
+            record.get("title", ""),
+            record.get("doi", ""),
+            record.get("publisher", ""),
+            record.get("container-title", ""),
+            record.get("subject", ""),
+            record.get("abstract", ""),
+        ]
+        normalized_parts: List[str] = []
+        for piece in pieces:
+            if isinstance(piece, list):
+                normalized_piece = " ".join(normalize_text(item) for item in piece if normalize_text(item))
+            else:
+                normalized_piece = normalize_text(piece)
+            if normalized_piece:
+                normalized_parts.append(normalized_piece)
+        return " ".join(normalized_parts)
+
+    def _normalize_crossref_record(self, record: Dict[str, Any], query: str) -> SourcePaper:
+        title = normalize_text((record.get("title") or [""])[0])
+        abstract = self._clean_crossref_abstract(record.get("abstract", ""))
+        published_date = self._extract_crossref_date(record)
+        updated_date = normalize_text(record.get("created", {}).get("date-time", ""))
+
+        authors = self._normalize_crossref_authors(record.get("author", []))
+        doi = self._normalize_crossref_doi(record.get("DOI", ""))
+        landing_url = self._landing_url(doi, "")
+
+        source_id = doi or safe_filename(f"biorxiv_{published_date}_{title}")
+        keywords = self._normalize_crossref_keywords(record.get("subject", []))
+        pdf_url = f"{BIO_RXIV_LANDING_BASE}/{doi}.full.pdf" if doi else ""
+
+        return SourcePaper(
+            source="biorxiv",
+            source_id=source_id,
+            title=title,
+            doi=doi,
+            abstract=abstract,
+            authors=authors,
+            published_date=published_date,
+            updated_date=updated_date,
+            journal="bioRxiv",
+            category=", ".join(keywords),
+            landing_url=landing_url,
+            pdf_url=pdf_url,
+            query=query,
+            version="",
+            keywords=keywords,
+            extra={
+                "publisher": record.get("publisher", ""),
+                "prefix": record.get("prefix", ""),
+                "type": record.get("type", ""),
+                "raw_record": record,
+            },
+        )
+
+    def _normalize_crossref_authors(self, authors_value: Any) -> List[str]:
+        if not isinstance(authors_value, list):
+            return []
+
+        authors: List[str] = []
+        for author in authors_value:
+            if not isinstance(author, dict):
+                continue
+            name = normalize_text(
+                author.get("name")
+                or author.get("given")
+                or " ".join(
+                    part
+                    for part in [author.get("given", ""), author.get("family", "")]
+                    if normalize_text(part)
+                )
+            )
+            if name:
+                authors.append(name)
+        return authors
+
+    def _normalize_crossref_keywords(self, keywords_value: Any) -> List[str]:
+        if isinstance(keywords_value, list):
+            return [normalize_text(item) for item in keywords_value if normalize_text(item)]
+        if isinstance(keywords_value, str) and keywords_value.strip():
+            return [part.strip() for part in keywords_value.split(",") if part.strip()]
+        return []
+
+    def _normalize_crossref_doi(self, doi: Any) -> str:
+        return normalize_text(doi)
+
+    def _extract_crossref_date(self, record: Dict[str, Any]) -> str:
+        for key in ("published-online", "published-print", "issued", "created"):
+            value = record.get(key)
+            if isinstance(value, dict):
+                date_parts = value.get("date-parts") or []
+                if date_parts and date_parts[0]:
+                    parts = date_parts[0]
+                    year = parts[0] if len(parts) > 0 else None
+                    month = parts[1] if len(parts) > 1 else 1
+                    day = parts[2] if len(parts) > 2 else 1
+                    if year:
+                        try:
+                            return datetime(int(year), int(month), int(day)).strftime("%Y-%m-%d")
+                        except Exception:
+                            return normalize_text(year)
+        return ""
+
+    def _clean_crossref_abstract(self, abstract: Any) -> str:
+        text = normalize_text(abstract)
+        if not text:
+            return ""
+        soup = BeautifulSoup(text, "html.parser")
+        cleaned = soup.get_text(separator=" ")
+        cleaned = re.sub(r"\s+", " ", cleaned).strip()
+        return cleaned
+
+    def _normalize_authors(self, authors_value: Any) -> List[str]:
+        if isinstance(authors_value, list):
+            authors: List[str] = []
+            for author in authors_value:
+                if isinstance(author, dict):
+                    name = normalize_text(
+                        author.get("name")
+                        or author.get("author_name")
+                        or author.get("full_name")
+                        or " ".join(
+                            part
+                            for part in [author.get("given", ""), author.get("family", "")]
+                            if normalize_text(part)
+                        )
+                    )
+                else:
+                    name = normalize_text(author)
+                if name:
+                    authors.append(name)
+            return authors
+
+        if isinstance(authors_value, str):
+            parts = [part.strip() for part in authors_value.split(";") if part.strip()]
+            return parts or [authors_value]
+
+        return []
+
+    def _normalize_keywords(self, category_value: Any) -> List[str]:
+        if isinstance(category_value, list):
+            return [normalize_text(item) for item in category_value if normalize_text(item)]
+        if isinstance(category_value, str) and category_value.strip():
+            return [part.strip() for part in category_value.split(",") if part.strip()]
+        return []
+
+    def _normalize_doi(self, doi: Any, version: Any) -> str:
+        doi_text = normalize_text(doi)
+        version_text = normalize_text(version)
+        if doi_text and version_text and not doi_text.lower().endswith(f"v{version_text.lower()}"):
+            return f"{doi_text}v{version_text}"
+        return doi_text
+
+    def _landing_url(self, doi: str, version: str) -> str:
+        if not doi:
+            return ""
+        return f"{BIO_RXIV_LANDING_BASE}/{doi}"
+
+    def _candidate_pdf_urls(self, record: Dict[str, Any], doi: str, version: str) -> List[str]:
+        candidates: List[str] = []
+        for key in ("pdf_url", "full_text_url", "fulltext_url", "pdf"):
+            value = normalize_text(record.get(key, ""))
+            if value:
+                candidates.append(value)
+
+        if doi:
+            candidates.append(f"{BIO_RXIV_LANDING_BASE}/{doi}.full.pdf")
+            if version and not doi.lower().endswith(f"v{version.lower()}"):
+                candidates.insert(0, f"{BIO_RXIV_LANDING_BASE}/{doi}v{version}.full.pdf")
+
+        landing_url = self._landing_url(doi, version)
+        if landing_url:
+            candidates.append(landing_url)
+
+        deduped: List[str] = []
+        seen = set()
+        for candidate in candidates:
+            if candidate not in seen:
+                seen.add(candidate)
+                deduped.append(candidate)
+        return deduped
+
+    def _save_record(self, record: SourcePaper, output_dir: Optional[str], download_pdf: bool) -> None:
+        base_dir = output_dir or self.root_dir
+        year = extract_year(record.published_date)
+        record_dir = build_source_record_dir(base_dir, record.source, year, record.source_id)
+        file_stem = safe_filename(record.source_id)
+
+        if download_pdf and record.pdf_url:
+            pdf_path = record_dir / f"{file_stem}.pdf"
+            if self._download_pdf(record, pdf_path):
+                record.pdf_downloaded = True
+                record.pdf_path = str(pdf_path)
+
+        save_json(record_dir / f"{file_stem}.json", record.to_dict())
+
+    def _download_pdf(self, record: SourcePaper, pdf_path: Path) -> bool:
+        candidates = [candidate for candidate in [record.pdf_url, record.landing_url] if candidate]
+
+        for candidate in candidates:
+            if candidate.lower().endswith(".pdf"):
+                if download_binary(candidate, pdf_path, headers=self.headers, timeout=self.request_timeout):
+                    return True
+                continue
+
+            try:
+                response = requests.get(candidate, headers=self.headers, timeout=self.request_timeout)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                meta = soup.find("meta", {"name": "citation_pdf_url"})
+                if meta and meta.get("content"):
+                    pdf_url = normalize_text(meta.get("content"))
+                    if download_binary(pdf_url, pdf_path, headers=self.headers, timeout=self.request_timeout):
+                        return True
+            except Exception:
+                continue
+
+        return False

pyPaperFlow/preprint/source_models.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+
+@dataclass
+class SourcePaper:
+    source: str
+    source_id: str
+    title: str = ""
+    doi: str = ""
+    abstract: str = ""
+    authors: List[str] = field(default_factory=list)
+    published_date: str = ""
+    updated_date: str = ""
+    journal: str = ""
+    category: str = ""
+    landing_url: str = ""
+    pdf_url: str = ""
+    pdf_path: str = ""
+    pdf_downloaded: bool = False
+    query: str = ""
+    version: str = ""
+    keywords: List[str] = field(default_factory=list)
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)

pyPaperFlow/preprint/source_utils.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+import requests
+
+
+BOOLEAN_OR_SPLIT_RE = re.compile(r"\s+OR\s+", re.IGNORECASE)
+BOOLEAN_AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE)
+TOKEN_RE = re.compile(r'"([^"]+)"|\'([^\']+)\'|(\S+)')
+
+
+def normalize_text(value: Any) -> str:
+    if value is None:
+        return ""
+    normalized = str(value).replace("\n", " ").strip()
+    normalized = re.sub(r"\s+", " ", normalized)
+    return normalized
+
+
+def safe_filename(value: Any, fallback: str = "record") -> str:
+    text = normalize_text(value)
+    if not text:
+        text = fallback
+    text = text.replace("/", "_")
+    text = text.replace("\\", "_")
+    text = re.sub(r"[^A-Za-z0-9._-]+", "_", text)
+    text = re.sub(r"_+", "_", text).strip("._-")
+    return text or fallback
+
+
+def extract_year(date_text: Any) -> str:
+    text = normalize_text(date_text)
+    if not text:
+        return "unknown"
+
+    for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m", "%Y/%m"):
+        try:
+            return datetime.strptime(text, fmt).strftime("%Y")
+        except Exception:
+            continue
+
+    match = re.search(r"(19|20)\d{2}", text)
+    if match:
+        return match.group(0)
+    return "unknown"
+
+
+def ensure_directory(path: Path | str) -> Path:
+    directory = Path(path)
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+def build_source_record_dir(base_dir: str | Path, source: str, year: str, source_id: str) -> Path:
+    directory = Path(base_dir) / safe_filename(source) / safe_filename(year, fallback="unknown") / safe_filename(source_id)
+    return ensure_directory(directory)
+
+
+def save_json(path: Path | str, payload: Dict[str, Any]) -> None:
+    output_path = Path(path)
+    ensure_directory(output_path.parent)
+    with output_path.open("w", encoding="utf-8") as handle:
+        json.dump(payload, handle, ensure_ascii=False, indent=2)
+
+
+def download_binary(url: str, output_path: Path | str, headers: Optional[Dict[str, str]] = None, timeout: float = 60.0) -> bool:
+    try:
+        response = requests.get(url, headers=headers, timeout=timeout)
+        response.raise_for_status()
+        content = response.content or b""
+        if not content.startswith(b"%PDF"):
+            return False
+
+        output_file = Path(output_path)
+        ensure_directory(output_file.parent)
+        with output_file.open("wb") as handle:
+            handle.write(content)
+        return True
+    except Exception:
+        return False
+
+
+def parse_boolean_query(query: str) -> List[List[str]]:
+    text = normalize_text(query)
+    if not text:
+        return []
+
+    clauses: List[List[str]] = []
+    for or_clause in BOOLEAN_OR_SPLIT_RE.split(text):
+        and_parts = BOOLEAN_AND_SPLIT_RE.split(or_clause)
+        terms: List[str] = []
+        for part in and_parts:
+            for token_match in TOKEN_RE.findall(part):
+                token = next((group for group in token_match if group), "")
+                token = normalize_text(token).strip('"\'')
+                if not token:
+                    continue
+                if token.upper() in {"AND", "OR", "NOT"}:
+                    continue
+                terms.append(token.lower())
+        if terms:
+            clauses.append(terms)
+    return clauses
+
+
+def basic_boolean_text_match(text: Any, query: str) -> bool:
+    haystack = normalize_text(text).lower()
+    clauses = parse_boolean_query(query)
+    if not clauses:
+        return True
+
+    for clause in clauses:
+        if all(term in haystack for term in clause):
+            return True
+    return False
+
+
+def iter_date_windows(start_date: datetime, end_date: datetime, window_days: int) -> Iterable[tuple[datetime, datetime]]:
+    if window_days < 1:
+        raise ValueError(f"window_days must be >= 1, got {window_days}")
+
+    current_start = start_date
+    while current_start <= end_date:
+        current_end = min(current_start + timedelta(days=window_days - 1), end_date)
+        yield current_start, current_end
+        current_start = current_end + timedelta(days=1)