arxiv-pulse 0.5.0__py3-none-any.whl

arxiv_pulse/.ENV.TEMPLATE

@@ -0,0 +1,72 @@
+# arXiv Pulse 配置文件模板
+# 将此文件复制为 .env 并进行配置
+
+# ========================
+# AI API 配置 (支持 OpenAI 格式)
+# ========================
+# 使用 OpenAI 格式的 API，支持 DeepSeek、Paratera AI 等
+# 示例：Paratera AI: https://llmapi.paratera.com
+# 示例：DeepSeek: https://api.deepseek.com
+
+AI_API_KEY=your_api_key_here
+AI_MODEL=DeepSeek-V3.2-Thinking
+AI_BASE_URL=https://llmapi.paratera.com
+
+
+# ========================
+# 数据库配置
+# ========================
+DATABASE_URL=sqlite:///data/arxiv_papers.db
+
+
+# ========================
+# 爬虫配置
+# ========================
+MAX_RESULTS_INITIAL=100    # init命令每个查询的论文数
+MAX_RESULTS_DAILY=20       # sync命令每个查询的论文数
+
+
+# ========================
+# 搜索查询配置
+# ========================
+# 分号分隔，允许查询中包含逗号
+# 默认监控凝聚态物理、DFT、机器学习、力场等研究领域
+SEARCH_QUERIES=condensed matter physics AND cat:cond-mat.*; (ti:"density functional" OR abs:"density functional") AND (cat:physics.comp-ph OR cat:cond-mat.mtrl-sci OR cat:physics.chem-ph); (ti:"machine learning" OR abs:"machine learning") AND (cat:physics.comp-ph OR cat:cond-mat.mtrl-sci OR cat:physics.chem-ph); (ti:"force field" OR abs:"force field") AND (cat:physics.comp-ph OR cat:cond-mat.soft OR cat:physics.chem-ph); (ti:"first principles" OR abs:"first principles" OR ti:"ab initio" OR abs:"ab initio") AND (cat:physics.comp-ph OR cat:cond-mat.mtrl-sci); (ti:"molecular dynamics" OR abs:"molecular dynamics") AND (cat:physics.comp-ph OR cat:cond-mat.soft OR cat:physics.chem-ph); (ti:"quantum chemistry" OR abs:"quantum chemistry") AND (cat:physics.chem-ph OR cat:physics.comp-ph); cat:cond-mat.mtrl-sci AND (ti:"computational" OR abs:"computational" OR ti:"simulation" OR abs:"simulation")
+
+
+# ========================
+# 报告配置
+# ========================
+REPORT_DIR=reports
+# SUMMARY_MODEL 现在由 AI_MODEL 控制，不再需要单独设置
+SUMMARY_MAX_TOKENS=2000    # 总结和翻译的最大token数
+SUMMARY_SENTENCES_LIMIT=3
+TOKEN_PRICE_PER_MILLION=3.0
+REPORT_MAX_PAPERS=50
+
+
+# ========================
+# 同步配置
+# ========================
+YEARS_BACK=3               # 同步回溯的年数
+IMPORTANT_PAPERS_FILE=important_papers.txt
+
+
+# ========================
+# 可选配置
+# ========================
+# 日志级别: DEBUG, INFO, WARNING, ERROR (默认: INFO)
+LOG_LEVEL=INFO
+
+# 爬虫延迟（秒，避免频繁请求 arXiv API）
+CRAWL_DELAY=1.0
+
+# ========================
+# 使用说明
+# ========================
+# 1. 将此文件复制为 .env
+# 2. 设置您的 AI API 密钥
+# 3. 根据需要调整其他配置
+# 4. 运行 pulse init . 初始化目录
+# 5. 运行 pulse sync . 同步论文
+# 6. 使用 pulse search "查询内容" . 搜索论文

arxiv_pulse/__init__.py

@@ -0,0 +1,26 @@
+"""
+arXiv Pulse: An intelligent arXiv literature crawler and analyzer for physics research.
+"""
+
+from .__version__ import __version__
+
+__author__ = "arXiv Pulse Team"
+
+from .arxiv_crawler import ArXivCrawler
+from .config import Config
+from .models import Database, Paper, TranslationCache
+from .output_manager import OutputManager, output
+from .report_generator import ReportGenerator
+from .summarizer import PaperSummarizer
+
+__all__ = [
+    "ArXivCrawler",
+    "Config",
+    "Database",
+    "Paper",
+    "TranslationCache",
+    "OutputManager",
+    "output",
+    "ReportGenerator",
+    "PaperSummarizer",
+]

arxiv_pulse/__version__.py

@@ -0,0 +1,33 @@
+"""
+arXiv Pulse 版本信息 - 动态从包元数据读取
+"""
+
+import importlib.metadata
+
+# 从包元数据读取版本
+try:
+    __version__ = importlib.metadata.version("arxiv-pulse")
+    # 解析版本信息元组
+    try:
+        __version_info__ = tuple(map(int, __version__.split(".")))
+    except ValueError:
+        # 如果版本号包含字母或其他字符，只取数字部分
+        parts = []
+        for part in __version__.split("."):
+            # 提取数字部分
+            digits = ""
+            for char in part:
+                if char.isdigit():
+                    digits += char
+                else:
+                    break
+            if digits:
+                parts.append(int(digits))
+            else:
+                parts.append(0)
+        __version_info__ = tuple(parts)
+
+except importlib.metadata.PackageNotFoundError:
+    # 包未安装时使用默认版本
+    __version__ = "0.5.0"
+    __version_info__ = (0, 5, 0)

arxiv_pulse/arxiv_crawler.py

@@ -0,0 +1,377 @@
+import arxiv
+import asyncio
+import aiohttp
+import os
+from typing import List, Dict, Any, Optional
+from datetime import datetime, timedelta, timezone
+from tqdm import tqdm
+import time
+import logging
+
+from arxiv_pulse.models import Database, Paper
+from arxiv_pulse.config import Config
+from arxiv_pulse.output_manager import output
+
+# 使用根日志记录器的配置（保留用于向后兼容）
+logger = logging.getLogger(__name__)
+
+
+class ArXivCrawler:
+    def __init__(self):
+        self.db = Database()
+        # 配置arXiv客户端，遵守调用频率限制
+        self.client = arxiv.Client(page_size=100, delay_seconds=3.0, num_retries=3)
+        self.config = Config
+
+        # 抑制第三方库的详细日志
+        logging.getLogger("arxiv").setLevel(logging.WARNING)
+        logging.getLogger("httpx").setLevel(logging.WARNING)
+
+    def search_arxiv(
+        self,
+        query: str,
+        max_results: int = 100,
+        days_back: Optional[int] = None,
+        cutoff_date: Optional[datetime] = None,
+    ) -> List[arxiv.Result]:
+        """Search arXiv for papers matching query
+
+        Args:
+            query: arXiv search query
+            max_results: Maximum number of results to return
+            days_back: Optional number of days to look back (deprecated, use cutoff_date)
+            cutoff_date: Optional UTC datetime cutoff; papers older than this will be skipped
+                         and iteration will stop early due to descending date order.
+        """
+        search = arxiv.Search(
+            query=query,
+            max_results=max_results,
+            sort_by=arxiv.SortCriterion.SubmittedDate,
+            sort_order=arxiv.SortOrder.Descending,
+        )
+
+        results = []
+        for paper in self.client.results(search):
+            # If cutoff date is provided, check paper date
+            if cutoff_date is not None and hasattr(paper, "published") and paper.published:
+                # Convert paper.published to UTC aware datetime
+                if paper.published.tzinfo is None:
+                    # Assume naive datetime is UTC
+                    paper_date = paper.published.replace(tzinfo=timezone.utc)
+                else:
+                    paper_date = paper.published.astimezone(timezone.utc)
+
+                if paper_date < cutoff_date:
+                    output.debug(f"遇到旧论文 ({paper_date.date()})，停止爬取")
+                    break
+
+            results.append(paper)
+
+            # Safety: don't exceed max_results even if cutoff_date not reached
+            if len(results) >= max_results:
+                break
+
+        output.debug(f"Found {len(results)} papers for query: {query}")
+        return results
+
+    def filter_new_papers(self, papers: List[arxiv.Result]) -> List[arxiv.Result]:
+        """Filter out papers already in database"""
+        new_papers = []
+        for paper in papers:
+            arxiv_id = paper.entry_id.split("/")[-1]
+            if not self.db.paper_exists(arxiv_id):
+                new_papers.append(paper)
+            else:
+                output.debug(f"Paper {arxiv_id} already exists in database")
+
+        output.debug(f"Filtered to {len(new_papers)} new papers")
+        return new_papers
+
+    def save_papers(self, papers: List[arxiv.Result], search_query: str) -> List[Paper]:
+        """Save papers to database"""
+        saved_papers = []
+        for paper in tqdm(papers, desc="Saving papers"):
+            try:
+                # Check again to avoid race conditions
+                arxiv_id = paper.entry_id.split("/")[-1]
+                if self.db.paper_exists(arxiv_id):
+                    continue
+
+                paper_obj = Paper.from_arxiv_entry(paper, search_query)
+                self.db.add_paper(paper_obj)
+                saved_papers.append(paper_obj)
+
+            except Exception as e:
+                output.error(
+                    "保存论文失败",
+                    details={"paper_id": paper.entry_id, "exception": str(e)},
+                )
+
+        output.done(f"保存完成: {len(saved_papers)} 篇新论文")
+        return saved_papers
+
+    def initial_crawl(self) -> Dict[str, Any]:
+        """Perform initial crawl with multiple queries"""
+        output.do("开始初始爬取")
+        all_saved = []
+
+        for query in self.config.SEARCH_QUERIES:
+            output.do(f"搜索: {query}")
+            try:
+                papers = self.search_arxiv(query, max_results=self.config.MAX_RESULTS_INITIAL)
+                new_papers = self.filter_new_papers(papers)
+                saved = self.save_papers(new_papers, query)
+                all_saved.extend(saved)
+
+                output.done(f"保存: {len(saved)} 篇论文")
+                time.sleep(1)  # Be nice to arXiv API
+
+            except Exception as e:
+                output.error(f"爬取查询失败: {query}", details={"exception": str(e)})
+
+        output.done(f"初始爬取完成: 共保存 {len(all_saved)} 篇论文")
+        return {
+            "total_saved": len(all_saved),
+            "queries_searched": len(self.config.SEARCH_QUERIES),
+            "saved_papers": all_saved,
+        }
+
+    def daily_update(self) -> Dict[str, Any]:
+        """Perform daily update crawl with early stopping optimization"""
+        output.do("开始每日更新")
+        all_saved = []
+
+        # 使用2天的时间窗口，因为arXiv通常在UTC 00:00-02:00更新
+        cutoff_date = datetime.now(timezone.utc) - timedelta(days=2)
+        output.info(f"查找 {cutoff_date.date()} 之后的新论文")
+
+        for query in self.config.SEARCH_QUERIES:
+            output.do(f"搜索: {query}")
+            try:
+                # 使用cutoff_date参数实现早期终止
+                papers = self.search_arxiv(
+                    query,
+                    max_results=self.config.MAX_RESULTS_DAILY,
+                    cutoff_date=cutoff_date,
+                )
+
+                output.debug(f"找到 {len(papers)} 篇最近论文")
+
+                new_papers = self.filter_new_papers(papers)
+                saved = self.save_papers(new_papers, query)
+                all_saved.extend(saved)
+
+                output.done(f"保存: {len(saved)} 篇新论文")
+                time.sleep(1)
+
+            except Exception as e:
+                output.error(f"每日更新失败: {query}", details={"exception": str(e)})
+
+        output.done(f"每日更新完成: 共保存 {len(all_saved)} 篇新论文")
+        return {
+            "total_saved": len(all_saved),
+            "queries_searched": len(self.config.SEARCH_QUERIES),
+            "date_range": f"Since {cutoff_date.date()}",
+            "saved_papers": all_saved,
+        }
+
+    def crawl_by_categories(self, categories: List[str], max_results: int = 50) -> Dict[str, Any]:
+        """Crawl specific arXiv categories"""
+        all_saved = []
+
+        for category in categories:
+            query = f"cat:{category}"
+            output.do(f"搜索类别: {category}")
+            try:
+                papers = self.search_arxiv(query, max_results=max_results)
+                new_papers = self.filter_new_papers(papers)
+                saved = self.save_papers(new_papers, f"cat:{category}")
+                all_saved.extend(saved)
+
+                output.done(f"保存: {len(saved)} 篇论文")
+                time.sleep(1)
+
+            except Exception as e:
+                output.error(f"爬取类别失败: {category}", details={"exception": str(e)})
+
+        return {
+            "total_saved": len(all_saved),
+            "categories": categories,
+            "saved_papers": all_saved,
+        }
+
+    def get_latest_paper_date_for_query(self, query: str) -> Optional[datetime]:
+        """Get the latest paper date for a specific query in database"""
+        with self.db.get_session() as session:
+            latest_paper = (
+                session.query(Paper).filter(Paper.search_query == query).order_by(Paper.published.desc()).first()
+            )
+            return latest_paper.published if latest_paper else None  # type: ignore
+
+    def sync_query(self, query: str, years_back: int = 3) -> Dict[str, Any]:
+        """Sync papers for a specific query, fetching missing papers from recent years"""
+        output.do(f"同步查询: {query}")
+
+        # Get latest paper date in database for this query
+        latest_date = self.get_latest_paper_date_for_query(query)
+
+        if latest_date:
+            # If we have papers, fetch from latest date onward
+            start_date = latest_date.replace(tzinfo=timezone.utc)
+            # 减去一天以确保获取所有可能的新论文，避免因时间精度问题错过论文
+            start_date = start_date - timedelta(days=1)
+            output.debug(f"获取论文从 {start_date.strftime('%Y-%m-%d')} 到现在")
+        else:
+            # If no papers, fetch from years_back years ago
+            start_date = datetime.now(timezone.utc) - timedelta(days=365 * years_back)
+            output.debug(f"获取最近 {years_back} 年的论文 ({start_date.strftime('%Y-%m-%d')} 到现在)")
+
+        # Search arXiv without date filter (use cutoff_date for early stopping)
+        try:
+            papers = self.search_arxiv(
+                query,
+                max_results=Config.ARXIV_MAX_RESULTS,
+                cutoff_date=start_date,
+            )
+            new_papers = self.filter_new_papers(papers)
+            saved = self.save_papers(new_papers, query)
+
+            output.done(f"同步完成: {len(saved)} 篇新论文")
+            time.sleep(1)  # Rate limiting
+
+            return {
+                "query": query,
+                "start_date": start_date,
+                "total_found": len(papers),
+                "new_papers": len(saved),
+                "saved_papers": saved,
+            }
+
+        except Exception as e:
+            output.error(f"同步查询失败: {query}", details={"exception": str(e)})
+            return {"query": query, "error": str(e), "new_papers": 0}
+
+    def sync_all_queries(self, years_back: int = 3) -> Dict[str, Any]:
+        """Sync all configured search queries"""
+        output.do(f"同步所有查询 (回溯 {years_back} 年)")
+
+        all_results = []
+        total_new = 0
+
+        for query in self.config.SEARCH_QUERIES:
+            result = self.sync_query(query, years_back)
+            all_results.append(result)
+            total_new += result.get("new_papers", 0)
+
+        output.done(f"同步完成: 共 {total_new} 篇新论文")
+        return {
+            "total_new_papers": total_new,
+            "query_results": all_results,
+            "years_back": years_back,
+        }
+
+    def sync_important_papers(self) -> Dict[str, Any]:
+        """Ensure important papers are in database"""
+        important_file = Config.IMPORTANT_PAPERS_FILE
+        if not os.path.exists(important_file):
+            output.warn(f"重要论文文件未找到: {important_file}")
+            return {"total_processed": 0, "added": 0, "errors": []}
+
+        added = 0
+        errors = []
+
+        with open(important_file, "r") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+
+                # Extract arXiv ID (format: 1234.56789v1 or 1234.56789)
+                arxiv_id = line.split()[0] if " " in line else line
+
+                # Remove version suffix if present
+                if "v" in arxiv_id:
+                    arxiv_id = arxiv_id.split("v")[0]
+
+                # Check if paper already exists
+                if self.db.paper_exists(arxiv_id):
+                    output.debug(f"重要论文已在数据库中: {arxiv_id}")
+                    continue
+
+                # Try to fetch paper from arXiv
+                try:
+                    search = arxiv.Search(id_list=[arxiv_id])
+                    results = list(self.client.results(search))
+
+                    if results:
+                        paper = results[0]
+                        paper_obj = Paper.from_arxiv_entry(paper, "important")
+                        self.db.add_paper(paper_obj)
+                        added += 1
+                        output.done(f"添加重要论文: {arxiv_id}")
+                    else:
+                        errors.append(f"Paper not found on arXiv: {arxiv_id}")
+
+                except Exception as e:
+                    errors.append(f"Error fetching paper {arxiv_id}: {e}")
+
+                time.sleep(0.5)  # Rate limiting
+
+        return {
+            "total_processed": added + len(errors),
+            "added": added,
+            "errors": errors,
+        }
+
+    def get_crawler_stats(self) -> Dict[str, Any]:
+        """Get crawler statistics"""
+        with self.db.get_session() as session:
+            total = session.query(Paper).count()
+            today = datetime.now().date()
+            today_start = datetime.combine(today, datetime.min.time())
+            today_count = session.query(Paper).filter(Paper.created_at >= today_start).count()
+
+            by_query = {}
+            papers = session.query(Paper).all()
+            for paper in papers:
+                query = paper.search_query
+                by_query[query] = by_query.get(query, 0) + 1
+
+            return {
+                "total_papers": total,
+                "papers_today": today_count,
+                "papers_by_query": by_query,
+            }
+
+
+def main():
+    """Test the crawler"""
+    crawler = ArXivCrawler()
+
+    print("Testing arXiv crawler...")
+    print(f"Search queries: {Config.SEARCH_QUERIES}")
+
+    # Test with a small crawl
+    test_query = Config.SEARCH_QUERIES[0]
+    print(f"\nTesting search for: {test_query}")
+
+    papers = crawler.search_arxiv(test_query, max_results=5)
+    print(f"Found {len(papers)} papers")
+
+    if papers:
+        paper = papers[0]
+        print(f"\nSample paper:")
+        print(f"Title: {paper.title[:100]}...")
+        print(f"Authors: {[author.name for author in paper.authors[:3]]}")
+        print(f"Published: {paper.published}")
+        print(f"Categories: {paper.categories if hasattr(paper, 'categories') else paper.primary_category}")
+
+    # Get stats
+    stats = crawler.get_crawler_stats()
+    print(f"\nDatabase stats:")
+    print(f"Total papers: {stats['total_papers']}")
+    print(f"Papers today: {stats['papers_today']}")
+
+
+if __name__ == "__main__":
+    main()