bibcheck-ref 0.2.0__py3-none-any.whl

bibcheck/__init__.py

@@ -0,0 +1,7 @@
+"""BibCheck: 参考文献存在性验证工具。
+
+通过学术数据库 API (CrossRef, DBLP, Semantic Scholar) 批量验证
+BibTeX 或 PDF 中的参考文献是否真实存在。
+"""
+
+__version__ = "0.2.0"

bibcheck/__main__.py

@@ -0,0 +1,6 @@
+"""CLI 入口: python -m bibcheck"""
+
+from bibcheck.cli import main
+
+if __name__ == "__main__":
+    main()

bibcheck/apis/__init__.py

@@ -0,0 +1 @@
+"""BibCheck API 客户端。"""

bibcheck/apis/arxiv.py

@@ -0,0 +1,127 @@
+"""arXiv API 客户端。
+
+专门用于预印本论文检索（CS、物理、数学等）。
+
+支持:
+- 标题搜索: GET /api/query?search_query=ti:...
+- 返回 Atom XML 格式
+
+API 文档: https://info.arxiv.org/help/api/
+注意: arXiv 建议每 3 秒不超过 1 次请求。
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import xml.etree.ElementTree as ET
+from typing import List, Optional
+from urllib.parse import quote
+
+from bibcheck.apis.base import BaseAPIClient
+from bibcheck.models import APIResult
+
+logger = logging.getLogger("bibcheck")
+
+# Atom / arXiv 命名空间
+_NS = {
+    "atom": "http://www.w3.org/2005/Atom",
+    "arxiv": "http://arxiv.org/schemas/atom",
+}
+
+
+class ArXivClient(BaseAPIClient):
+    """arXiv 预印本搜索。"""
+
+    name = "arxiv"
+    BASE_URL = "http://export.arxiv.org/api/query"
+
+    def __init__(self, rate_limit: float = 0.33, **kwargs):
+        super().__init__(rate_limit=rate_limit, **kwargs)
+
+    def search(
+        self,
+        title: str,
+        authors: Optional[List[str]] = None,
+        year: Optional[int] = None,
+    ) -> List[APIResult]:
+        # 构建搜索查询
+        query = f"ti:{quote(title)}"
+        if authors:
+            # 取第一作者的姓氏缩小范围
+            first_author = authors[0]
+            last_name = first_author.split()[-1] if first_author else ""
+            if last_name:
+                query += f"+AND+au:{quote(last_name)}"
+
+        params = {
+            "search_query": query,
+            "max_results": 5,
+            "sortBy": "relevance",
+            "sortOrder": "descending",
+        }
+
+        resp = self._request_raw("GET", self.BASE_URL, params=params)
+        if resp is None:
+            return []
+
+        try:
+            root = ET.fromstring(resp.text)
+        except ET.ParseError as e:
+            logger.warning(f"[arxiv] XML parse error: {e}")
+            return []
+
+        results = []
+        for entry in root.findall("atom:entry", _NS):
+            parsed = self._parse_entry(entry)
+            if parsed:
+                results.append(parsed)
+
+        return results
+
+    @staticmethod
+    def _parse_entry(entry: ET.Element) -> Optional[APIResult]:
+        """解析 Atom XML 中的单条 entry。"""
+        # 标题（可能含换行）
+        title_el = entry.find("atom:title", _NS)
+        title = " ".join((title_el.text or "").split()) if title_el is not None else ""
+        if not title:
+            return None
+
+        # 作者
+        authors = []
+        for author_el in entry.findall("atom:author", _NS):
+            name_el = author_el.find("atom:name", _NS)
+            if name_el is not None and name_el.text:
+                authors.append(name_el.text.strip())
+
+        # 发布日期 → 年份
+        published_el = entry.find("atom:published", _NS)
+        year = None
+        if published_el is not None and published_el.text:
+            match = re.match(r"(\d{4})", published_el.text)
+            if match:
+                year = int(match.group(1))
+
+        # URL (arXiv abs 链接)
+        url = ""
+        id_el = entry.find("atom:id", _NS)
+        if id_el is not None and id_el.text:
+            url = id_el.text.strip()
+
+        # DOI（如果有）
+        doi = None
+        doi_el = entry.find("arxiv:doi", _NS)
+        if doi_el is not None and doi_el.text:
+            doi = doi_el.text.strip()
+
+        return APIResult(
+            source="arxiv",
+            title=title,
+            authors=authors,
+            year=year,
+            doi=doi,
+            venue="arXiv",
+            url=url,
+            raw_response={},
+        )

bibcheck/apis/base.py

@@ -0,0 +1,280 @@
+"""API 客户端抽象基类。
+
+改进点:
+- 指数退避重试（自动处理 HTTP 429 / 5xx）
+- 解析 Retry-After header
+- 请求失败计数与熔断
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+import threading
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+import requests
+
+from bibcheck.models import APIResult
+
+logger = logging.getLogger("bibcheck")
+
+
+class RateLimiter:
+    """令牌桶速率限制器（线程安全）。"""
+
+    def __init__(self, rate: float):
+        """rate: 每秒允许的请求数。"""
+        self._min_interval = 1.0 / rate if rate > 0 else 0
+        self._last_call = 0.0
+        self._lock = threading.Lock()
+
+    def wait(self):
+        with self._lock:
+            now = time.time()
+            elapsed = now - self._last_call
+            if elapsed < self._min_interval:
+                time.sleep(self._min_interval - elapsed)
+            self._last_call = time.time()
+
+
+class BaseAPIClient(ABC):
+    """API 客户端抽象基类，含速率限制、指数退避重试和熔断机制。"""
+
+    name: str = "base"
+
+    def __init__(
+        self,
+        rate_limit: float = 1.0,
+        timeout: int = 15,
+        max_retries: int = 2,
+        base_backoff: float = 2.0,
+    ):
+        self._limiter = RateLimiter(rate_limit)
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._base_backoff = base_backoff
+        self._session = requests.Session()
+        self._session.headers.update({
+            "User-Agent": "BibCheck/0.1 (Academic Reference Verification Tool; mailto:bibcheck@example.com)"
+        })
+
+        # 请求统计
+        self.request_count = 0
+        self.error_count = 0
+
+        # 熔断器: 连续 429 次数过多时暂时禁用
+        self._consecutive_429 = 0
+        self._circuit_open_until = 0.0  # 熔断恢复时间戳
+        self._circuit_open_count = 0
+        self._lock = threading.Lock()
+
+    @abstractmethod
+    def search(
+        self,
+        title: str,
+        authors: Optional[List[str]] = None,
+        year: Optional[int] = None,
+    ) -> List[APIResult]:
+        """搜索学术数据库，返回匹配结果。"""
+        ...
+
+    def _get(self, url: str, params: dict = None) -> dict:
+        """带速率限制和指数退避重试的 GET 请求。"""
+        return self._request("GET", url, params=params)
+
+    def _post(self, url: str, json_data: dict = None, params: dict = None) -> dict:
+        """带速率限制和指数退避重试的 POST 请求。"""
+        return self._request("POST", url, params=params, json_data=json_data)
+
+    @property
+    def is_circuit_open(self) -> bool:
+        """熔断器是否打开（API 暂时不可用）。"""
+        if self._circuit_open_until <= 0:
+            return False
+        if time.time() >= self._circuit_open_until:
+            # 熔断期结束，半开状态允许重试
+            with self._lock:
+                self._circuit_open_until = 0
+                self._consecutive_429 = 0
+            return False
+        return True
+
+    def _trip_circuit(self):
+        """触发熔断，暂停该 API 一段时间。"""
+        with self._lock:
+            self._circuit_open_count += 1
+            # 熔断时间递增: 60s, 120s, 300s
+            cooldown = min(60 * (2 ** (self._circuit_open_count - 1)), 300)
+            self._circuit_open_until = time.time() + cooldown
+            logger.warning(
+                f"[{self.name}] Circuit breaker OPEN — pausing for {cooldown}s "
+                f"(trip #{self._circuit_open_count})"
+            )
+
+    def _request(
+        self,
+        method: str,
+        url: str,
+        params: dict = None,
+        json_data: dict = None,
+    ) -> dict:
+        """通用请求方法，含重试逻辑和熔断机制。"""
+        # 熔断器检查
+        if self.is_circuit_open:
+            return {"error": f"[{self.name}] circuit breaker open, skipping"}
+
+        last_error = None
+
+        for attempt in range(self._max_retries + 1):
+            self._limiter.wait()
+            self.request_count += 1
+
+            try:
+                if method == "GET":
+                    resp = self._session.get(url, params=params, timeout=self._timeout)
+                else:
+                    resp = self._session.post(
+                        url, params=params, json=json_data, timeout=self._timeout
+                    )
+
+                # 成功 — 重置 429 计数
+                if resp.status_code == 200:
+                    with self._lock:
+                        self._consecutive_429 = 0
+                    return resp.json()
+
+                # 429 Too Many Requests
+                if resp.status_code == 429:
+                    with self._lock:
+                        self._consecutive_429 += 1
+
+                    # 连续 3 次 429 → 触发熔断
+                    if self._consecutive_429 >= 3:
+                        self._trip_circuit()
+                        return {"error": f"[{self.name}] rate limited, circuit breaker tripped"}
+
+                    retry_after = self._parse_retry_after(resp)
+                    wait_time = retry_after or (self._base_backoff * (2 ** attempt))
+                    logger.info(
+                        f"[{self.name}] 429, waiting {wait_time:.0f}s "
+                        f"(attempt {attempt + 1}/{self._max_retries + 1})"
+                    )
+                    time.sleep(wait_time)
+                    continue
+
+                # 5xx 服务器错误 — 重试
+                if resp.status_code >= 500:
+                    wait_time = self._base_backoff * (2 ** attempt)
+                    time.sleep(wait_time)
+                    continue
+
+                # 其他 4xx — 不重试
+                self.error_count += 1
+                return {"error": f"HTTP {resp.status_code}"}
+
+            except requests.exceptions.Timeout:
+                last_error = "timeout"
+                time.sleep(self._base_backoff * (2 ** attempt))
+                continue
+
+            except requests.exceptions.ConnectionError:
+                last_error = "connection error"
+                time.sleep(self._base_backoff * (2 ** attempt))
+                continue
+
+            except (requests.RequestException, ValueError) as e:
+                self.error_count += 1
+                return {"error": str(e)}
+
+        self.error_count += 1
+        return {"error": f"Max retries: {last_error}"}
+
+    def _request_raw(
+        self,
+        method: str,
+        url: str,
+        params: dict = None,
+    ) -> Optional[requests.Response]:
+        """通用请求方法，返回原始 Response 对象（不解析 JSON）。
+
+        用于 arXiv 等返回 XML 格式的 API。
+        复用与 _request 相同的速率限制、重试和熔断逻辑。
+        """
+        if self.is_circuit_open:
+            return None
+
+        last_error = None
+
+        for attempt in range(self._max_retries + 1):
+            self._limiter.wait()
+            self.request_count += 1
+
+            try:
+                if method == "GET":
+                    resp = self._session.get(url, params=params, timeout=self._timeout)
+                else:
+                    resp = self._session.post(url, params=params, timeout=self._timeout)
+
+                if resp.status_code == 200:
+                    with self._lock:
+                        self._consecutive_429 = 0
+                    return resp
+
+                if resp.status_code == 429:
+                    with self._lock:
+                        self._consecutive_429 += 1
+                    if self._consecutive_429 >= 3:
+                        self._trip_circuit()
+                        return None
+                    retry_after = self._parse_retry_after(resp)
+                    wait_time = retry_after or (self._base_backoff * (2 ** attempt))
+                    logger.info(
+                        f"[{self.name}] 429, waiting {wait_time:.0f}s "
+                        f"(attempt {attempt + 1}/{self._max_retries + 1})"
+                    )
+                    time.sleep(wait_time)
+                    continue
+
+                if resp.status_code >= 500:
+                    time.sleep(self._base_backoff * (2 ** attempt))
+                    continue
+
+                self.error_count += 1
+                return None
+
+            except requests.exceptions.Timeout:
+                last_error = "timeout"
+                time.sleep(self._base_backoff * (2 ** attempt))
+                continue
+            except requests.exceptions.ConnectionError:
+                last_error = "connection error"
+                time.sleep(self._base_backoff * (2 ** attempt))
+                continue
+            except requests.RequestException:
+                self.error_count += 1
+                return None
+
+        self.error_count += 1
+        return None
+
+    @staticmethod
+    def _parse_retry_after(resp: requests.Response) -> Optional[float]:
+        """解析 Retry-After header（秒数或日期）。"""
+        retry_after = resp.headers.get("Retry-After")
+        if retry_after is None:
+            return None
+        try:
+            return float(retry_after)
+        except ValueError:
+            # 可能是 HTTP 日期格式，保守等待 10 秒
+            return 10.0
+
+    @property
+    def stats(self) -> dict:
+        return {
+            "name": self.name,
+            "requests": self.request_count,
+            "errors": self.error_count,
+        }

bibcheck/apis/crossref.py

@@ -0,0 +1,119 @@
+"""CrossRef API 客户端。
+
+支持:
+- 标题搜索: GET /works?query.bibliographic=...
+- DOI 直查: GET /works/{doi}
+- Polite Pool: 通过 mailto 参数获得更稳定的服务
+
+API 文档: https://api.crossref.org/swagger-ui/index.html
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from bibcheck.apis.base import BaseAPIClient
+from bibcheck.models import APIResult
+
+
+class CrossRefClient(BaseAPIClient):
+    """CrossRef 学术文献搜索。"""
+
+    name = "crossref"
+    BASE_URL = "https://api.crossref.org/works"
+
+    def __init__(self, rate_limit: float = 5.0, mailto: Optional[str] = None, **kwargs):
+        super().__init__(rate_limit=rate_limit, **kwargs)
+        self._mailto = mailto
+        # CrossRef 的 polite pool 通过 mailto 激活
+        if mailto:
+            self._session.headers["User-Agent"] = (
+                f"BibCheck/0.1 (https://github.com/bibcheck; mailto:{mailto})"
+            )
+
+    def search(
+        self,
+        title: str,
+        authors: Optional[List[str]] = None,
+        year: Optional[int] = None,
+    ) -> List[APIResult]:
+        # 同时用 query.title（精确标题匹配）和 query.bibliographic（宽泛匹配）
+        # query.title 对期刊论文、统计学论文等非 CS 文献效果更好
+        params = {
+            "query.title": title,
+            "rows": 5,
+            "select": "DOI,title,author,published-print,published-online,container-title,URL",
+        }
+        if self._mailto:
+            params["mailto"] = self._mailto
+
+        data = self._get(self.BASE_URL, params=params)
+        if "error" in data or "message" not in data:
+            return []
+
+        results = [self._parse_item(item) for item in data["message"].get("items", [])]
+
+        # 如果 query.title 没结果，fallback 到 query.bibliographic
+        if not results:
+            params.pop("query.title")
+            params["query.bibliographic"] = title
+            data = self._get(self.BASE_URL, params=params)
+            if "error" not in data and "message" in data:
+                results = [self._parse_item(item) for item in data["message"].get("items", [])]
+
+        return results
+
+    def lookup_by_doi(self, doi: str) -> Optional[APIResult]:
+        """通过 DOI 直接查询（精确匹配，最快最可靠）。"""
+        data = self._get(f"{self.BASE_URL}/{doi}")
+        if "error" in data or "message" not in data:
+            return None
+        return self._parse_item(data["message"])
+
+    @staticmethod
+    def _parse_item(item: dict) -> APIResult:
+        """解析 CrossRef 返回的单条记录。"""
+        # 标题
+        item_title = ""
+        if item.get("title"):
+            item_title = item["title"][0] if isinstance(item["title"], list) else item["title"]
+
+        # 作者
+        item_authors = []
+        for author in item.get("author", []):
+            name_parts = []
+            if author.get("given"):
+                name_parts.append(author["given"])
+            if author.get("family"):
+                name_parts.append(author["family"])
+            if name_parts:
+                item_authors.append(" ".join(name_parts))
+
+        # 年份
+        item_year = None
+        for date_field in ("published-print", "published-online"):
+            if item.get(date_field, {}).get("date-parts"):
+                parts = item[date_field]["date-parts"][0]
+                if parts and parts[0]:
+                    item_year = parts[0]
+                    break
+
+        # 期刊/会议
+        venue = ""
+        if item.get("container-title"):
+            venue = (
+                item["container-title"][0]
+                if isinstance(item["container-title"], list)
+                else item["container-title"]
+            )
+
+        return APIResult(
+            source="crossref",
+            title=item_title,
+            authors=item_authors,
+            year=item_year,
+            doi=item.get("DOI"),
+            venue=venue,
+            url=item.get("URL"),
+            raw_response=item,
+        )

bibcheck/apis/dblp.py

@@ -0,0 +1,96 @@
+"""DBLP API 客户端。
+
+API 文档: https://dblp.org/faq/How+to+use+the+dblp+search+API.html
+"""
+
+from __future__ import annotations
+
+import re
+from typing import List, Optional
+
+from bibcheck.apis.base import BaseAPIClient
+from bibcheck.models import APIResult
+
+
+class DBLPClient(BaseAPIClient):
+    """DBLP 计算机科学文献搜索。"""
+
+    name = "dblp"
+    BASE_URL = "https://dblp.org/search/publ/api"
+
+    def __init__(self, rate_limit: float = 1.0, **kwargs):
+        super().__init__(rate_limit=rate_limit, **kwargs)
+
+    def search(
+        self,
+        title: str,
+        authors: Optional[List[str]] = None,
+        year: Optional[int] = None,
+    ) -> List[APIResult]:
+        params = {
+            "q": title,
+            "format": "json",
+            "h": 5,  # 返回条数
+        }
+
+        data = self._get(self.BASE_URL, params=params)
+        if "error" in data:
+            return []
+
+        hits = data.get("result", {}).get("hits", {}).get("hit", [])
+        if not hits:
+            return []
+
+        results = []
+        for hit in hits:
+            info = hit.get("info", {})
+
+            # 提取标题
+            item_title = info.get("title", "")
+            # DBLP 标题可能以句号结尾
+            item_title = item_title.rstrip(".")
+
+            # 提取作者
+            item_authors = []
+            authors_data = info.get("authors", {}).get("author", [])
+            if isinstance(authors_data, dict):
+                authors_data = [authors_data]
+            for author in authors_data:
+                if isinstance(author, dict):
+                    item_authors.append(author.get("text", ""))
+                elif isinstance(author, str):
+                    item_authors.append(author)
+
+            # 提取年份
+            item_year = None
+            if info.get("year"):
+                try:
+                    item_year = int(info["year"])
+                except (ValueError, TypeError):
+                    pass
+
+            # 提取 DOI
+            doi = None
+            if info.get("doi"):
+                doi = info["doi"]
+                # DBLP 的 doi 可能包含前缀
+                if doi and not doi.startswith("10."):
+                    doi_match = re.search(r"(10\.\d{4,}/\S+)", doi)
+                    if doi_match:
+                        doi = doi_match.group(1)
+
+            # 提取期刊/会议
+            venue = info.get("venue", "")
+
+            results.append(APIResult(
+                source=self.name,
+                title=item_title,
+                authors=item_authors,
+                year=item_year,
+                doi=doi,
+                venue=venue,
+                url=info.get("url", ""),
+                raw_response=info,
+            ))
+
+        return results