pdfget 0.1.0__py3-none-any.whl

pdfget/__init__.py

@@ -0,0 +1,12 @@
+"""
+PDFGet - 智能文献搜索与批量下载工具
+"""
+
+__version__ = "0.1.0"
+__author__ = "gqy"
+__email__ = "qingyu_ge@foxmail.com"
+__description__ = "智能文献搜索与批量下载工具，支持高级检索和并发下载"
+
+from .fetcher import PaperFetcher
+
+__all__ = ["PaperFetcher"]

pdfget/__main__.py

@@ -0,0 +1,6 @@
+"""PDF下载器主程序入口"""
+
+from .main import main
+
+if __name__ == "__main__":
+    main()

pdfget/config.py

@@ -0,0 +1,30 @@
+"""PDF下载器配置"""
+
+from pathlib import Path
+
+# 项目根目录
+ROOT_DIR = Path(__file__).parent.parent.parent
+DATA_DIR = ROOT_DIR / "data"
+OUTPUT_DIR = DATA_DIR / "pdfs"
+CACHE_DIR = DATA_DIR / ".cache"
+
+# 创建目录
+for d in [DATA_DIR, OUTPUT_DIR, CACHE_DIR]:
+    d.mkdir(exist_ok=True, parents=True)
+
+# 下载设置
+TIMEOUT = 30
+MAX_RETRIES = 3
+DELAY = 1.0
+MAX_CONCURRENT = 5
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100MB
+
+# API设置
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (compatible; PDFGet/1.0)",
+    "Accept": "application/pdf,*/*",
+}
+
+# 日志设置
+LOG_LEVEL = "INFO"
+LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"

pdfget/downloader.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""
+并发下载器 - 提升PDF下载效率
+使用线程池实现并发下载，同时保持API调用限制
+"""
+
+import logging
+import random
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Callable, Optional
+
+from .fetcher import PaperFetcher
+
+
+class ConcurrentDownloader:
+    """并发下载管理器"""
+
+    def __init__(
+        self,
+        max_workers: int = 3,
+        base_delay: float = 1.0,
+        random_delay_range: float = 0.5,
+        fetcher: Optional[PaperFetcher] = None,
+    ):
+        """
+        初始化并发下载器
+
+        Args:
+            max_workers: 最大并发线程数（默认3）
+            base_delay: 基础延迟时间（秒）
+            random_delay_range: 随机延迟范围（秒）
+            fetcher: PaperFetcher实例（可选）
+        """
+        self.logger = logging.getLogger("ConcurrentDownloader")
+        self.max_workers = max_workers
+        self.base_delay = base_delay
+        self.random_delay_range = random_delay_range
+
+        # 为每个线程创建独立的fetcher实例（避免session冲突）
+        if fetcher:
+            self.base_fetcher = fetcher
+        else:
+            self.base_fetcher = PaperFetcher()
+
+        # 线程安全的进度跟踪
+        self._lock = threading.Lock()
+        self._completed = 0
+        self._successful = 0
+        self._failed = 0
+        self._pdf_count = 0
+
+    def _get_delay(self) -> float:
+        """获取随机延迟时间，避免同步请求"""
+        random_delay = random.uniform(0, self.random_delay_range)
+        return self.base_delay + random_delay
+
+    def _create_thread_fetcher(self) -> PaperFetcher:
+        """为线程创建独立的fetcher实例"""
+        # 复制基础配置，但创建新的session
+        fetcher = PaperFetcher(
+            cache_dir=str(self.base_fetcher.cache_dir),
+            output_dir=str(self.base_fetcher.output_dir),
+        )
+        return fetcher
+
+    def _update_progress(
+        self, success: bool = False, pdf_downloaded: bool = False
+    ) -> None:
+        """线程安全的进度更新"""
+        with self._lock:
+            self._completed += 1
+            if success:
+                self._successful += 1
+                if pdf_downloaded:
+                    self._pdf_count += 1
+            else:
+                self._failed += 1
+
+            # 简单的进度显示
+            progress = (self._completed / self._total) * 100
+            self.logger.info(
+                f"  进度: {self._completed}/{self._total} ({progress:.1f}%) "
+                f"成功: {self._successful} PDF: {self._pdf_count} 失败: {self._failed}"
+            )
+
+    def _download_single(
+        self, doi: str, fetcher: PaperFetcher, timeout: int = 30
+    ) -> Dict[str, Any]:
+        """单个文献的下载任务"""
+        try:
+            # 添加随机延迟
+            time.sleep(self._get_delay())
+
+            result = fetcher.fetch_by_doi(doi, timeout=timeout)
+
+            # 更新进度
+            success = result.get("success", False)
+            pdf_downloaded = bool(result.get("pdf_path"))
+            self._update_progress(success, pdf_downloaded)
+
+            return result
+
+        except Exception as e:
+            self.logger.debug(f"下载失败 ({doi}): {str(e)}")
+            self._update_progress(False)
+            return {"doi": doi, "success": False, "error": str(e)}
+
+    def download_batch(
+        self, dois: List[str], timeout: int = 30
+    ) -> List[Dict[str, Any]]:
+        """
+        并发批量下载文献
+
+        Args:
+            dois: DOI列表
+            timeout: 单个请求超时时间
+
+        Returns:
+            下载结果列表
+        """
+        if not dois:
+            return []
+
+        self.logger.info(
+            f"🚀 启动并发下载：{len(dois)} 篇文献，{self.max_workers} 个并发线程"
+        )
+
+        # 初始化进度跟踪
+        self._total = len(dois)
+        self._completed = 0
+        self._successful = 0
+        self._failed = 0
+        self._pdf_count = 0
+
+        results = []
+
+        # 使用线程池执行并发下载
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # 提交所有下载任务
+            future_to_doi = {}
+
+            for doi in dois:
+                # 为每个线程创建独立的fetcher
+                thread_fetcher = self._create_thread_fetcher()
+                future = executor.submit(
+                    self._download_single, doi, thread_fetcher, timeout
+                )
+                future_to_doi[future] = doi
+
+            # 收集结果（保持原始顺序）
+            for future in as_completed(future_to_doi):
+                doi = future_to_doi[future]
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    self.logger.error(f"并发下载异常 ({doi}): {str(e)}")
+                    results.append({"doi": doi, "success": False, "error": str(e)})
+
+        # 按原始DOI顺序重新排列结果
+        doi_to_result = {r["doi"]: r for r in results}
+        ordered_results = [
+            doi_to_result.get(doi, {"doi": doi, "success": False, "error": "Not found"})
+            for doi in dois
+        ]
+
+        # 最终统计
+        self.logger.info("\n📊 并发下载完成:")
+        self.logger.info(f"   总计: {len(ordered_results)}")
+        self.logger.info(f"   成功: {self._successful}")
+        self.logger.info(f"   PDF: {self._pdf_count}")
+        self.logger.info(f"   失败: {self._failed}")
+        self.logger.info(
+            f"   成功率: {(self._successful / len(ordered_results)) * 100:.1f}%"
+        )
+
+        return ordered_results
+
+    def download_with_progress_callback(
+        self,
+        dois: List[str],
+        timeout: int = 30,
+        progress_callback: Optional[Callable[[int, int, int, int], None]] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        带进度回调的并发下载
+
+        Args:
+            dois: DOI列表
+            timeout: 超时时间
+            progress_callback: 进度回调函数 (completed, successful, pdf_count, total)
+
+        Returns:
+            下载结果列表
+        """
+        if not dois:
+            return []
+
+        self.logger.info(
+            f"🚀 启动并发下载：{len(dois)} 篇文献，{self.max_workers} 个并发线程"
+        )
+
+        # 初始化进度跟踪
+        self._total = len(dois)
+        self._completed = 0
+        self._successful = 0
+        self._failed = 0
+        self._pdf_count = 0
+
+        results = []
+
+        def update_progress_with_callback(
+            success: bool = False, pdf_downloaded: bool = False
+        ) -> None:
+            """带回调的进度更新"""
+            self._update_progress(success, pdf_downloaded)
+            if progress_callback:
+                progress_callback(
+                    self._completed, self._successful, self._pdf_count, self._total
+                )
+
+        # 使用线程池执行并发下载，避免方法赋值
+        try:
+            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                future_to_doi = {}
+
+                for doi in dois:
+                    thread_fetcher = self._create_thread_fetcher()
+                    # 直接使用线程中的update_with_progress方法
+                    future = executor.submit(
+                        self._download_single_with_callback,
+                        doi,
+                        thread_fetcher,
+                        timeout,
+                        update_progress_with_callback,
+                    )
+                    future_to_doi[future] = doi
+
+                for future in as_completed(future_to_doi):
+                    doi = future_to_doi[future]
+                    try:
+                        result = future.result()
+                        results.append(result)
+                    except Exception as e:
+                        self.logger.error(f"并发下载异常 ({doi}): {str(e)}")
+                        results.append({"doi": doi, "success": False, "error": str(e)})
+
+            # 按原始顺序排列结果
+            doi_to_result = {r["doi"]: r for r in results}
+            ordered_results = [
+                doi_to_result.get(
+                    doi, {"doi": doi, "success": False, "error": "Not found"}
+                )
+                for doi in dois
+            ]
+
+            # 最终统计和最后一次回调
+            self.logger.info("\n📊 并发下载完成:")
+            self.logger.info(f"   总计: {len(ordered_results)}")
+            self.logger.info(f"   成功: {self._successful}")
+            self.logger.info(f"   PDF: {self._pdf_count}")
+            self.logger.info(f"   失败: {self._failed}")
+            self.logger.info(
+                f"   成功率: {(self._successful / len(ordered_results)) * 100:.1f}%"
+            )
+
+            if progress_callback:
+                progress_callback(
+                    self._completed, self._successful, self._pdf_count, self._total
+                )
+
+            return ordered_results
+
+        finally:
+            pass
+
+    def _download_single_with_callback(
+        self,
+        doi: str,
+        thread_fetcher: PaperFetcher,
+        timeout: int,
+        progress_callback: Callable[[], None],
+    ) -> Dict[str, Any]:
+        """带回调的单个文献下载（用于并发下载）"""
+        try:
+            # 添加随机延迟避免API限制
+            delay = self._get_delay()
+            time.sleep(delay)
+
+            # 获取文献信息
+            paper_info = thread_fetcher.fetch_by_doi(doi, timeout)
+            if not paper_info:
+                progress_callback()
+                return {"doi": doi, "success": False, "error": "文献信息获取失败"}
+
+            result = {"doi": doi, "success": True, "paper_info": paper_info}
+
+            # 更新进度
+            progress_callback()
+
+            return result
+
+        except Exception as e:
+            progress_callback()
+            self.logger.error(f"下载异常 ({doi}): {str(e)}")
+            return {"doi": doi, "success": False, "error": str(e)}

pdfget/fetcher.py

@@ -0,0 +1,415 @@
+#!/usr/bin/env python3
+"""
+简化版文献获取器 - Linus风格
+只做一件事：下载开放获取文献
+遵循KISS原则：Keep It Simple, Stupid
+"""
+
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+from urllib.parse import quote
+
+import requests
+
+import logging
+
+
+class PaperFetcher:
+    """简单文献获取器"""
+
+    def __init__(self, cache_dir: str = "data/cache", output_dir: str = "data/pdfs"):
+        """
+        初始化获取器
+
+        Args:
+            cache_dir: 缓存目录
+            output_dir: PDF输出目录
+        """
+        self.logger = logging.getLogger("PaperFetcher")
+        self.cache_dir = Path(cache_dir)
+        self.output_dir = Path(output_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # 简单的HTTP会话
+        self.session = requests.Session()
+        self.session.headers.update(
+            {"User-Agent": "Mozilla/5.0 (compatible; PaperFetcher/1.0)"}
+        )
+
+    def parse_query(self, query: str) -> str:
+        """
+        解析高级检索词为Europe PMC格式
+
+        支持的语法：
+        - 布尔运算符：AND, OR, NOT
+        - 字段检索：title:, author:, journal:
+        - 短语检索："exact phrase"
+
+        Args:
+            query: 用户输入的检索词
+
+        Returns:
+            Europe PMC格式的检索词
+        """
+        # 处理短语检索（引号包围的内容）
+        phrase_pattern = r'"([^"]+)"'
+        phrases = re.findall(phrase_pattern, query)
+
+        # 临时替换短语为占位符
+        for i, phrase in enumerate(phrases):
+            query = query.replace(f'"{phrase}"', f"__PHRASE_{i}__")
+
+        # 处理字段检索
+        field_mappings = {
+            "title:": "TITLE:",
+            "author:": "AUTHOR:",
+            "journal:": "JOURNAL:",
+            "abstract:": "ABSTRACT:",
+        }
+
+        for user_field, pmc_field in field_mappings.items():
+            query = query.replace(user_field, pmc_field)
+
+        # 恢复短语，并添加必要的引号
+        for i, phrase in enumerate(phrases):
+            query = query.replace(f"__PHRASE_{i}__", f'"{phrase}"')
+
+        # 处理布尔运算符（确保大写）
+        query = (
+            query.replace(" and ", " AND ")
+            .replace(" or ", " OR ")
+            .replace(" not ", " NOT ")
+        )
+
+        return query.strip()
+
+    def search_papers(self, query: str, limit: int = 50) -> list[dict]:
+        """
+        通过Europe PMC搜索文献
+
+        Args:
+            query: 检索词（支持高级语法）
+            limit: 返回结果数量限制
+
+        Returns:
+            文献列表，包含DOI、标题、作者等信息
+        """
+        self.logger.info(f"🔍 搜索文献: {query}")
+
+        # 解析检索词
+        parsed_query = self.parse_query(query)
+        self.logger.debug(f"  解析后: {parsed_query}")
+
+        # 构建搜索URL
+        search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
+        params = {
+            "query": parsed_query,
+            "resulttype": "core",
+            "format": "json",
+            "pageSize": min(limit, 1000),  # Europe PMC限制最多1000条
+            "cursorMark": "*",
+        }
+
+        try:
+            response = self.session.get(search_url, params=params, timeout=30)  # type: ignore[arg-type]
+            response.raise_for_status()
+
+            data = response.json()
+
+            if data.get("hitCount", 0) == 0:
+                self.logger.info("  ❌ 未找到匹配的文献")
+                return []
+
+            # 处理结果
+            papers = []
+            results = data.get("resultList", {}).get("result", [])
+
+            for i, record in enumerate(results[:limit]):
+                # 获取期刊信息
+                journal_info = record.get("journalInfo", {})
+
+                paper = {
+                    "title": record.get("title", ""),
+                    "authors": [
+                        a.strip() for a in record.get("authorString", "").split(",")
+                    ]
+                    if record.get("authorString")
+                    else [],
+                    "journal": journal_info.get("journal", {}).get("title", ""),
+                    "year": record.get("pubYear", ""),
+                    "doi": record.get("doi", ""),
+                    "pmcid": record.get("pmcid", ""),
+                    "pmid": record.get("pmid", ""),
+                    "abstract": record.get("abstractText", ""),
+                    "isOpenAccess": bool(
+                        record.get("pmcid")
+                    ),  # 有PMCID通常表示开放获取
+                    "source": "Europe PMC",
+                    # 新增的10个字段
+                    "affiliation": record.get("affiliation", ""),
+                    "volume": journal_info.get("volume", ""),
+                    "issue": journal_info.get("issue", ""),
+                    "pages": record.get("pageInfo", ""),
+                    "license": record.get("license", ""),
+                    "citedBy": record.get("citedByCount", 0),
+                    "keywords": record.get("keywordList", []),
+                    "meshTerms": record.get("meshHeadingList", []),
+                    "grants": record.get("grantsList", []),
+                    "hasData": record.get("hasData") == "Y",
+                    "hasSuppl": record.get("hasSuppl") == "Y",
+                }
+                papers.append(paper)
+
+                self.logger.info(
+                    f"  📄 {i + 1}/{min(len(results), limit)}: {paper['title'][:60]}..."
+                )
+
+            self.logger.info(f"  ✅ 找到 {len(papers)} 篇文献")
+            return papers
+
+        except requests.exceptions.Timeout:
+            self.logger.error("  ❌ 搜索超时")
+            return []
+        except requests.exceptions.ConnectionError:
+            self.logger.error("  ❌ 连接失败")
+            return []
+        except Exception as e:
+            self.logger.error(f"  ❌ 搜索失败: {str(e)}")
+            return []
+
+    def fetch_by_doi(self, doi: str, timeout: int = 30) -> dict:
+        """
+        通过DOI获取文献（简化版）
+
+        策略：
+        1. 只处理开放获取文献（有PMCID）
+        2. 快速失败，不重试
+        3. 简单缓存
+        4. 不搞复杂的网络监控和自适应重试
+
+        Args:
+            doi: 文献DOI
+            timeout: 超时时间
+
+        Returns:
+            获取结果字典
+        """
+        self.logger.info(f"🔍 获取文献: {doi}")
+
+        # 检查缓存
+        cached_result = self._get_cache(doi)
+        if cached_result:
+            self.logger.info("  📦 从缓存加载")
+            return cached_result
+
+        # 只使用Europe PMC（主要的开放获取源）
+        result = self._fetch_from_pmc(doi, timeout)
+
+        # 缓存结果
+        self._save_cache(doi, result)
+
+        if result.get("success"):
+            self.logger.info("  ✅ 获取成功")
+        else:
+            self.logger.info(f"  ❌ 获取失败: {result.get('error', 'Unknown error')}")
+
+        return result
+
+    def _fetch_from_pmc(self, doi: str, timeout: int) -> dict:
+        """从Europe PMC获取文献"""
+        try:
+            # 搜索PMCID
+            search_url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:{quote(doi)}&resulttype=core&format=json"
+            self.logger.debug(f"  🔍 Europe PMC URL: {search_url}")
+
+            response = self.session.get(search_url, timeout=timeout)
+            response.raise_for_status()
+
+            data = response.json()
+            if data.get("hitCount", 0) == 0:
+                return {
+                    "success": False,
+                    "error": "Not found in Europe PMC",
+                    "doi": doi,
+                }
+
+            record = data["resultList"]["result"][0]
+            pmcid = record.get("pmcid")
+
+            if not pmcid:
+                self.logger.info("  ⏭️ 无PMCID，非开放获取文献")
+                return {
+                    "success": False,
+                    "error": "Not open access (no PMCID)",
+                    "doi": doi,
+                }
+
+            self.logger.info(f"  📄 找到PMCID: {pmcid}")
+
+            # 尝试下载PDF
+            pdf_result = self._download_pdf(pmcid, doi)
+
+            if pdf_result["success"]:
+                return {
+                    "success": True,
+                    "doi": doi,
+                    "pmcid": pmcid,
+                    "pdf_path": pdf_result["path"],
+                    "content_type": "pdf",
+                    "title": record.get("title"),
+                    "journal": record.get("journalInfo", {})
+                    .get("journal", {})
+                    .get("title"),
+                    "authors": [
+                        a.strip() for a in record.get("authorString", "").split(",")
+                    ]
+                    if record.get("authorString")
+                    else [],
+                    "year": record.get("pubYear"),
+                    "abstract": record.get("abstractText"),
+                }
+
+            # PDF下载失败，返回全文HTML链接
+            return {
+                "success": True,
+                "doi": doi,
+                "pmcid": pmcid,
+                "full_text_url": f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/",
+                "content_type": "html",
+                "title": record.get("title"),
+                "authors": [
+                    a.strip() for a in record.get("authorString", "").split(",")
+                ]
+                if record.get("authorString")
+                else [],
+                "year": record.get("pubYear"),
+                "abstract": record.get("abstractText"),
+            }
+
+        except requests.exceptions.Timeout:
+            return {"success": False, "error": "Request timeout", "doi": doi}
+        except requests.exceptions.ConnectionError:
+            return {"success": False, "error": "Connection error", "doi": doi}
+        except Exception as e:
+            return {"success": False, "error": str(e), "doi": doi}
+
+    def _download_pdf(self, pmcid: str, doi: str) -> dict:
+        """下载PDF文件"""
+        # 尝试几个常见的PDF URL
+        pdf_urls = [
+            f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/pdf/",
+            f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/pdf/{pmcid}.pdf",
+            f"https://europepmc.org/articles/{pmcid}?pdf=render",
+        ]
+
+        for i, pdf_url in enumerate(pdf_urls):
+            try:
+                self.logger.debug(f"  📥 尝试PDF源 {i + 1}: {pdf_url}")
+                response = self.session.get(pdf_url, timeout=30, stream=True)
+                response.raise_for_status()
+
+                content_type = response.headers.get("content-type", "").lower()
+                if "application/pdf" not in content_type:
+                    continue
+
+                # 保存文件
+                safe_doi = "".join(c for c in doi if c.isalnum() or c in "-._")
+                filename = f"{pmcid}_{safe_doi}.pdf"
+                file_path = self.output_dir / filename
+
+                with open(file_path, "wb") as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+
+                self.logger.info(f"  💾 PDF保存成功: {file_path}")
+                return {"success": True, "path": str(file_path)}
+
+            except Exception as e:
+                self.logger.debug(f"  ⚠️ PDF源 {i + 1} 失败: {str(e)}")
+                continue
+
+        return {"success": False, "error": "All PDF sources failed"}
+
+    def _get_cache(self, doi: str) -> dict | None:
+        """简单缓存检查"""
+        cache_file = (
+            self.cache_dir / f"cache_{hashlib.md5(doi.encode()).hexdigest()}.json"
+        )
+
+        if cache_file.exists():
+            try:
+                with open(cache_file, "r") as f:
+                    data = json.load(f)
+
+                # 检查PDF文件是否还存在
+                if data.get("pdf_path") and not Path(data["pdf_path"]).exists():
+                    self.logger.debug("缓存的PDF文件不存在，清除缓存")
+                    cache_file.unlink()
+                    return None
+
+                # 检查缓存是否过期（24小时）
+                if time.time() - data.get("timestamp", 0) > 86400:
+                    self.logger.debug("缓存已过期")
+                    cache_file.unlink()
+                    return None
+
+                return data  # type: ignore
+
+            except Exception as e:
+                self.logger.debug(f"缓存读取失败: {str(e)}")
+                cache_file.unlink()
+                return None
+
+        return None
+
+    def _save_cache(self, doi: str, result: dict) -> None:
+        """保存缓存"""
+        try:
+            cache_file = (
+                self.cache_dir / f"cache_{hashlib.md5(doi.encode()).hexdigest()}.json"
+            )
+            result["timestamp"] = time.time()
+
+            with open(cache_file, "w") as f:
+                json.dump(result, f, indent=2)
+
+        except Exception as e:
+            self.logger.debug(f"缓存保存失败: {str(e)}")
+
+    def fetch_batch(self, dois: list[str], delay: float = 1.0) -> list[dict]:
+        """
+        批量获取文献（简化版）
+
+        Args:
+            dois: DOI列表
+            delay: 请求间延迟（秒）
+
+        Returns:
+            结果列表
+        """
+        self.logger.info(f"🚀 批量获取 {len(dois)} 篇文献")
+        results = []
+
+        for i, doi in enumerate(dois, 1):
+            self.logger.info(f"\n📄 进度: {i}/{len(dois)}")
+
+            try:
+                result = self.fetch_by_doi(doi)
+                results.append(result)
+            except Exception as e:
+                self.logger.error(f"获取文献失败 ({doi}): {e}")
+                results.append({"doi": doi, "success": False, "error": str(e)})
+
+            # 简单延迟，避免被限制
+            if i < len(dois):
+                time.sleep(delay)
+
+        # 统计结果
+        success_count = sum(1 for r in results if r.get("success"))
+        self.logger.info(f"\n📊 批量获取完成: {success_count}/{len(dois)} 成功")
+
+        return results

pdfget/main.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+"""
+PDF下载器主程序
+独立的文献PDF下载工具
+"""
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+import logging
+
+from .fetcher import PaperFetcher
+from .downloader import ConcurrentDownloader
+from .config import TIMEOUT, DELAY, LOG_LEVEL, LOG_FORMAT
+
+
+def main() -> None:
+    """主函数"""
+    parser = argparse.ArgumentParser(
+        description="PDF文献下载器",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+使用示例:
+  # 搜索文献
+  python -m pdfget -s "machine learning cancer"
+  python -m pdfget -s "deep learning" -l 20 -d
+
+  # 并发下载（多线程）
+  python -m pdfget -s "cancer immunotherapy" -l 20 -d -t 5
+  python -m pdfget -i dois.csv -t 3
+
+  # 下载单个文献
+  python -m pdfget --doi 10.1016/j.cell.2020.01.021
+        """,
+    )
+
+    # 输入选项
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--doi", help="单个DOI")
+    group.add_argument("-i", help="输入文件（CSV或TXT）")
+    group.add_argument("-s", help="搜索文献")
+
+    # 可选参数
+    parser.add_argument("-c", default="doi", help="CSV列名（默认: doi）")
+    parser.add_argument("-o", default="data/pdfs", help="输出目录")
+    parser.add_argument("--delay", type=float, default=DELAY, help="请求延迟秒数")
+    parser.add_argument("-l", type=int, default=50, help="搜索结果数量")
+    parser.add_argument("-d", action="store_true", help="下载PDF")
+    parser.add_argument("-t", type=int, default=3, help="并发线程数（默认3）")
+    parser.add_argument("-v", action="store_true", help="详细输出")
+
+    args = parser.parse_args()
+
+    # 设置日志
+    logging.basicConfig(level=logging.DEBUG if args.v else LOG_LEVEL, format=LOG_FORMAT)
+    logger = logging.getLogger("PDFDownloader")
+
+    # 初始化下载器
+    fetcher = PaperFetcher(cache_dir="data/cache", output_dir="data/pdfs")
+
+    logger.info("🚀 PDF下载器启动")
+    logger.info(f"   输出目录: {args.o}")
+
+    try:
+        if args.doi:
+            # 单个DOI下载
+            logger.info(f"\n📄 下载单个文献: {args.doi}")
+            result = fetcher.fetch_by_doi(args.doi, timeout=TIMEOUT)
+
+            if result.get("success"):
+                logger.info("✅ 下载成功!")
+                if result.get("pdf_path"):
+                    logger.info(f"   PDF路径: {result['pdf_path']}")
+                else:
+                    logger.info(f"   HTML链接: {result.get('full_text_url')}")
+            else:
+                logger.error(f"❌ 下载失败: {result.get('error', 'Unknown error')}")
+
+        elif args.s:
+            # 搜索文献
+            logger.info(f"\n🔍 搜索文献: {args.s}")
+            papers = fetcher.search_papers(args.s, limit=args.l)
+
+            if not papers:
+                logger.error("❌ 未找到匹配的文献")
+                exit(1)
+
+            # 显示搜索结果
+            logger.info(f"\n📊 搜索结果 ({len(papers)} 篇):")
+            for i, paper in enumerate(papers, 1):
+                logger.info(f"\n{i}. {paper['title']}")
+                logger.info(
+                    f"   作者: {', '.join(paper['authors'][:3])}{'...' if len(paper['authors']) > 3 else ''}"
+                )
+                logger.info(f"   期刊: {paper['journal']} ({paper['year']})")
+                if paper["doi"]:
+                    logger.info(f"   DOI: {paper['doi']}")
+                logger.info(f"   开放获取: {'是' if paper['isOpenAccess'] else '否'}")
+
+            # 保存搜索结果
+            search_results_file = (
+                Path(args.o) / f"search_results_{int(time.time())}.json"
+            )
+            search_results_file.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(search_results_file, "w", encoding="utf-8") as f:
+                json.dump(
+                    {
+                        "query": args.s,
+                        "timestamp": time.time(),
+                        "total": len(papers),
+                        "results": papers,
+                    },
+                    f,
+                    indent=2,
+                    ensure_ascii=False,
+                )
+
+            logger.info(f"\n💾 搜索结果已保存到: {search_results_file}")
+
+            # 如果需要下载PDF
+            if args.d:
+                logger.info("\n📥 开始下载PDF...")
+
+                # 只下载有PMCID的开放获取文献
+                oa_papers = [p for p in papers if p["pmcid"]]
+                logger.info(f"   找到 {len(oa_papers)} 篇开放获取文献")
+
+                if oa_papers:
+                    # 构造DOI列表
+                    dois = [p["doi"] for p in oa_papers if p["doi"]]
+
+                    if dois:
+                        # 根据线程数决定是否使用并发下载
+                        if len(dois) > 1 and args.t > 1:
+                            logger.info(
+                                f"\n🚀 使用 {args.t} 个线程并发下载 {len(dois)} 篇文献"
+                            )
+                            concurrent_downloader = ConcurrentDownloader(
+                                max_workers=args.t,
+                                base_delay=args.delay,
+                                fetcher=fetcher,
+                            )
+                            results = concurrent_downloader.download_batch(
+                                dois, timeout=TIMEOUT
+                            )
+                        else:
+                            # 单线程下载（保持原有逻辑）
+                            results = fetcher.fetch_batch(dois, delay=args.delay)
+
+                        # 统计结果
+                        success_count = sum(1 for r in results if r.get("success"))
+                        pdf_count = sum(1 for r in results if r.get("pdf_path"))
+                        html_count = sum(1 for r in results if r.get("full_text_url"))
+
+                        logger.info("\n📊 下载统计:")
+                        logger.info(f"   总计: {len(results)}")
+                        logger.info(f"   成功: {success_count}")
+                        logger.info(f"   PDF: {pdf_count}")
+                        logger.info(f"   HTML: {html_count}")
+                        logger.info(f"   失败: {len(results) - success_count}")
+
+                        # 保存下载结果
+                        if success_count > 0:
+                            download_results_file = (
+                                Path(args.o) / "download_results.json"
+                            )
+                            with open(
+                                download_results_file, "w", encoding="utf-8"
+                            ) as f:
+                                json.dump(
+                                    {
+                                        "timestamp": time.time(),
+                                        "total": len(results),
+                                        "success": success_count,
+                                        "results": results,
+                                    },
+                                    f,
+                                    indent=2,
+                                    ensure_ascii=False,
+                                )
+
+                            logger.info(
+                                f"\n💾 下载结果已保存到: {download_results_file}"
+                            )
+
+        else:
+            # 批量下载
+            logger.info(f"\n📚 批量下载: {args.i}")
+
+            # 读取DOI列表
+            input_path = Path(args.i)
+            if not input_path.exists():
+                logger.error(f"❌ 输入文件不存在: {args.i}")
+                exit(1)
+
+            if input_path.suffix.lower() == ".csv":
+                # 读取CSV文件
+                import pandas as pd
+
+                try:
+                    df = pd.read_csv(input_path)
+                    if args.c not in df.columns:
+                        logger.error(f"❌ CSV文件中找不到列: {args.c}")
+                        exit(1)
+
+                    dois = df[args.c].dropna().unique().tolist()
+                    logger.info(f"   找到 {len(dois)} 个唯一DOI")
+
+                except Exception as e:
+                    logger.error(f"❌ 读取CSV文件失败: {e}")
+                    exit(1)
+
+            else:
+                # 读取文本文件（每行一个DOI）
+                try:
+                    with open(input_path, "r") as f:
+                        dois = [line.strip() for line in f if line.strip()]
+                    logger.info(f"   找到 {len(dois)} 个DOI")
+
+                except Exception as e:
+                    logger.error(f"❌ 读取文件失败: {e}")
+                    exit(1)
+
+            # 根据线程数决定是否使用并发下载
+            if len(dois) > 1 and args.t > 1:
+                logger.info(f"\n🚀 使用 {args.t} 个线程并发下载 {len(dois)} 篇文献")
+                concurrent_downloader = ConcurrentDownloader(
+                    max_workers=args.t, base_delay=args.delay, fetcher=fetcher
+                )
+                results = concurrent_downloader.download_batch(dois, timeout=TIMEOUT)
+            else:
+                # 单线程下载（保持原有逻辑）
+                results = fetcher.fetch_batch(dois, delay=args.delay)
+
+            # 统计结果
+            success_count = sum(1 for r in results if r.get("success"))
+            pdf_count = sum(1 for r in results if r.get("pdf_path"))
+            html_count = sum(1 for r in results if r.get("full_text_url"))
+
+            logger.info("\n📊 下载统计:")
+            logger.info(f"   总计: {len(results)}")
+            logger.info(f"   成功: {success_count}")
+            logger.info(f"   PDF: {pdf_count}")
+            logger.info(f"   HTML: {html_count}")
+            logger.info(f"   失败: {len(results) - success_count}")
+
+            # 保存结果
+            if success_count > 0:
+                output_file = Path(args.o) / "download_results.json"
+                output_file.parent.mkdir(parents=True, exist_ok=True)
+
+                with open(output_file, "w", encoding="utf-8") as f:
+                    json.dump(
+                        {
+                            "timestamp": time.time(),
+                            "total": len(results),
+                            "success": success_count,
+                            "results": results,
+                        },
+                        f,
+                        indent=2,
+                        ensure_ascii=False,
+                    )
+
+                logger.info(f"\n💾 结果已保存到: {output_file}")
+
+    except KeyboardInterrupt:
+        logger.info("\n⏹️ 用户中断下载")
+        exit(1)
+    except Exception as e:
+        logger.error(f"\n💥 发生错误: {e}", exc_info=True)
+        exit(1)
+
+    logger.info("\n✨ 下载完成")
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

pdfget-0.1.0.dist-info/METADATA

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: pdfget
+Version: 0.1.0
+Summary: 智能文献搜索与批量下载工具，支持高级检索和并发下载
+Author-email: gqy <qingyu_ge@foxmail.com>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: black>=24.0.0; extra == 'dev'
+Requires-Dist: isort>=5.13.0; extra == 'dev'
+Requires-Dist: mypy>=1.9.0; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.5.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# PDFGet - 高效文献下载工具
+
+智能文献搜索与批量下载工具，支持高级检索和并发下载。
+
+## 1. 项目概述
+
+PDFGet是一个专为科研工作者设计的智能文献搜索与批量下载工具，集成了Europe PMC等权威学术数据库，提供高效的文献获取和管理功能。
+
+### 1.1 主要特性
+
+- 🔍 **高级搜索**：支持布尔运算符、字段检索、短语检索
+- 🚀 **并发下载**：多线程并行下载，3-5倍速度提升
+- 📊 **丰富元数据**：包含作者、单位、期刊、摘要、引用等完整信息
+- 💾 **智能缓存**：24小时缓存，避免重复下载
+- 📄 **批量处理**：支持CSV/TXT文件批量下载
+
+## 2. 安装与配置
+
+### 2.1 系统要求
+
+详细的系统要求和依赖信息请查看 [pyproject.toml](pyproject.toml) 文件。
+
+### 2.2 安装方法
+
+```bash
+# 使用pip安装
+pip install pdfget
+
+# 使用uv安装
+uv add pdfget
+
+# 或从源码安装
+git clone https://github.com/gqy20/pdfget.git
+cd pdfget
+pip install -e .
+```
+
+### 2.3 快速开始
+
+安装完成后，您可以直接使用 `pdfget` 命令：
+
+```bash
+# 搜索文献
+pdfget -s "machine learning" -l 20
+
+# 搜索并下载
+pdfget -s "cancer immunotherapy" -d
+
+# 并发下载（5线程）
+pdfget -s "deep learning" -l 50 -d -t 5
+
+# 单篇文献下载
+pdfget --doi 10.1016/j.cell.2020.01.021
+
+# 批量下载
+pdfget -i dois.csv -d -t 3
+```
+
+如果您使用 uv 作为包管理器，也可以：
+```bash
+# 使用uv运行
+uv run pdfget -s "machine learning" -l 20
+```
+
+## 3. 高级检索语法
+
+### 3.1 布尔运算符
+```bash
+# AND: 同时包含多个关键词
+pdfget -s "cancer AND immunotherapy" -l 30
+
+# OR: 包含任意关键词
+pdfget -s "machine OR deep learning" -l 20
+
+# NOT: 排除特定词汇
+pdfget -s "cancer AND immunotherapy NOT review" -l 30
+
+# 复杂组合
+pdfget -s "(cancer OR tumor) AND immunotherapy NOT mice" -l 25
+```
+
+### 3.2 字段检索
+```bash
+# 标题检索
+pdfget -s 'title:"deep learning"' -l 15
+
+# 作者检索
+pdfget -s 'author:hinton AND title:"neural networks"' -l 10
+
+# 期刊检索
+pdfget -s 'journal:nature AND cancer' -l 20
+
+# 年份检索
+pdfget -s 'cancer AND year:2023' -l 15
+```
+
+### 3.3 短语和精确匹配
+```bash
+# 短语检索（用双引号）
+pdfget -s '"quantum computing"' -l 10
+
+# 混合使用
+pdfget -s '"gene expression" AND (cancer OR tumor) NOT review' -l 20
+```
+
+### 3.4 实用检索技巧
+- 使用括号分组复杂的布尔逻辑
+- 短语用双引号确保精确匹配
+- 可以组合多个字段进行精确检索
+- 使用 NOT 过滤掉不相关的结果（如综述、评论等）
+
+## 4. 性能优势
+
+### 4.1 并发下载效率对比
+
+| 文献数量 | 单线程耗时 | 并发耗时 | 性能提升 |
+|---------|-----------|----------|----------|
+| 5篇     | ~25秒     | ~8秒     | 3x       |
+| 20篇    | ~100秒    | ~25秒    | 4x       |
+| 50篇    | ~250秒    | ~60秒    | 4x       |
+
+## 5. 命令行参数详解
+
+### 5.1 核心参数
+- `-s QUERY` : 搜索文献
+- `--doi DOI` : 下载单个文献
+- `-i FILE` : 批量输入文件
+- `-d` : 下载PDF
+
+### 5.2 优化参数
+- `-l NUM` : 搜索结果数量（默认50）
+- `-t NUM` : 并发线程数（默认3）
+- `-v` : 详细输出
+
+## 6. 输出格式与文件结构
+
+### 6.1 搜索结果格式
+```json
+{
+  "query": "关键词",
+  "total": 10,
+  "results": [
+    {
+      "title": "文献标题",
+      "authors": ["作者1", "作者2"],
+      "journal": "期刊名称",
+      "year": "2025",
+      "doi": "10.1016/xxx",
+      "affiliation": "作者单位",
+      "citedBy": 0,
+      "keywords": ["关键词1", "关键词2"]
+    }
+  ]
+}
+```
+
+### 6.2 文件目录结构
+```
+data/
+├── pdfs/           # 下载的PDF文件
+├── cache/          # 缓存文件
+└── download_results.json  # 下载结果记录
+```
+
+## 7. 许可证
+
+本项目采用 MIT License，允许自由使用和修改。
+
+## 📚 更新日志
+
+<details>
+<summary><strong>📋 查看版本更新历史</strong></summary>
+
+- 🔗 **完整更新日志**: [CHANGELOG.md](CHANGELOG.md)
+- ✨ **最新版本 (v0.1.0)**: 高级文献搜索 + 并发下载 + 智能缓存
+
+</details>
+
+## 🔗 相关链接
+
+- **项目源码**: [GitHub Repository](https://github.com/gqy20/pdfget)
+- **问题反馈**: [GitHub Issues](https://github.com/gqy20/pdfget/issues)

pdfget-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+pdfget/__init__.py,sha256=-qap676xaNk4jiFkzqU7LjCuzrHFIEcZgVzBqpZ2FmE,294
+pdfget/__main__.py,sha256=SAGyFJO_1WAqCYhJtr2QNl7fVI5Gws_nKHYw7SHTjiM,97
+pdfget/config.py,sha256=jyjJr6PwYC5o96wAWD_6Qx-WhwHJ2pJQ634wOf0fcFo,643
+pdfget/downloader.py,sha256=AuM93j95DmPYz5Dma3o5Cz_bG1oF5AUF8kXrcblgl60,10342
+pdfget/fetcher.py,sha256=iHfg72zjbURvndC49B4WMTs3cFCsLh9c1pa6Wb67hyI,14446
+pdfget/main.py,sha256=kzs5AavcSkpjaCWdWM3CtedKkm78qiDpcp8ylaVxqRs,11104
+pdfget-0.1.0.dist-info/METADATA,sha256=WnlOMj622_A6UI93WBnXnUSIRfEs8qnXHqagTZF2r_A,4920
+pdfget-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+pdfget-0.1.0.dist-info/entry_points.txt,sha256=htkdzIZIePSAe0VCp5_0EnZAqcWxOwm3OJ-BcvhXaag,48
+pdfget-0.1.0.dist-info/RECORD,,

pdfget-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pdfget-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pdfget = pdfget.__main__:main