arxiv-pulse 0.5.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

arxiv_pulse/config.py

@@ -1,5 +1,4 @@
 import os
-import warnings
 
 
 class Config:
@@ -7,8 +6,8 @@ class Config:
     DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///data/arxiv_papers.db")
 
     # Crawler
-    MAX_RESULTS_INITIAL = int(os.getenv("MAX_RESULTS_INITIAL", 100))
-    MAX_RESULTS_DAILY = int(os.getenv("MAX_RESULTS_DAILY", 20))
+    MAX_RESULTS_INITIAL = int(os.getenv("MAX_RESULTS_INITIAL", 10000))
+    MAX_RESULTS_DAILY = int(os.getenv("MAX_RESULTS_DAILY", 500))
 
     # Search queries - use semicolon as separator to allow commas in queries
     SEARCH_QUERIES_RAW = os.getenv(
@@ -29,7 +28,6 @@ class Config:
     SUMMARY_MAX_TOKENS = int(os.getenv("SUMMARY_MAX_TOKENS", 2000))
 
     # Report generation settings
-    SUMMARY_SENTENCES_LIMIT = int(os.getenv("SUMMARY_SENTENCES_LIMIT", 3))
     TOKEN_PRICE_PER_MILLION = float(os.getenv("TOKEN_PRICE_PER_MILLION", 3.0))
 
     # Paths
@@ -40,13 +38,13 @@ class Config:
     REPORT_MAX_PAPERS = int(os.getenv("REPORT_MAX_PAPERS", "50"))
 
     # ArXiv API
-    ARXIV_MAX_RESULTS = 1000
-    ARXIV_SORT_BY = "submittedDate"
-    ARXIV_SORT_ORDER = "descending"
+    ARXIV_MAX_RESULTS = int(os.getenv("ARXIV_MAX_RESULTS", 30000))
+    ARXIV_SORT_BY = os.getenv("ARXIV_SORT_BY", "submittedDate")
+    ARXIV_SORT_ORDER = os.getenv("ARXIV_SORT_ORDER", "descending")
 
     # Sync configuration
     YEARS_BACK = int(os.getenv("YEARS_BACK", 3))  # Years to look back for initial sync
-    IMPORTANT_PAPERS_FILE = os.getenv("IMPORTANT_PAPERS_FILE", "important_papers.txt")
+    IMPORTANT_PAPERS_FILE = os.getenv("IMPORTANT_PAPERS_FILE", "data/important_papers.txt")
 
     @classmethod
     def validate(cls):

arxiv_pulse/models.py

@@ -11,7 +11,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import json
 from typing import Optional
 
@@ -48,8 +48,12 @@ class Paper(Base):
     summary = Column(Text)
 
     # Metadata
-    created_at = Column(DateTime, default=datetime.utcnow)
-    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+    created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None))
+    updated_at = Column(
+        DateTime,
+        default=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
+        onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
+    )
 
     def to_dict(self):
         """Convert to dictionary"""
@@ -112,8 +116,12 @@ class TranslationCache(Base):
     source_text_hash = Column(String(64), nullable=False, unique=True, index=True)
     translated_text = Column(Text, nullable=False)
     target_language = Column(String(10), default="zh")
-    created_at = Column(DateTime, default=datetime.utcnow)
-    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+    created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None))
+    updated_at = Column(
+        DateTime,
+        default=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
+        onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None),
+    )
 
     def __repr__(self):
         return f"<TranslationCache(id={self.id}, hash={self.source_text_hash[:16]}...)>"
@@ -147,7 +155,7 @@ class Database:
             if paper:
                 for key, value in kwargs.items():
                     setattr(paper, key, value)
-                paper.updated_at = datetime.utcnow()
+                paper.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
                 session.commit()
                 return True
             return False
@@ -155,7 +163,7 @@ class Database:
     def get_recent_papers(self, days=7, limit=100):
         """Get recent papers"""
         with self.get_session() as session:
-            cutoff_date = datetime.utcnow() - timedelta(days=days)
+            cutoff_date = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
             return (
                 session.query(Paper)
                 .filter(Paper.published >= cutoff_date)
@@ -233,7 +241,7 @@ class Database:
             if existing:
                 # 更新现有缓存
                 existing.translated_text = translated_text
-                existing.updated_at = datetime.utcnow()
+                existing.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             else:
                 # 创建新缓存
                 cache_entry = TranslationCache(
@@ -249,7 +257,7 @@ class Database:
     def clear_old_translation_cache(self, days_old: int = 30) -> int:
         """清理旧的翻译缓存"""
         with self.get_session() as session:
-            cutoff_date = datetime.utcnow() - timedelta(days=days_old)
+            cutoff_date = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days_old)
             deleted_count = session.query(TranslationCache).filter(TranslationCache.updated_at < cutoff_date).delete()
             session.commit()
             return deleted_count

arxiv_pulse/output_manager.py

@@ -11,7 +11,7 @@
 [error]   - 错误信息（简洁）
 [debug]   - 调试信息（默认不显示）
 
-所有详细日志同时写入日志文件，控制台只显示简洁信息。
+所有输出仅显示在控制台，不写入日志文件。
 """
 
 import sys
@@ -72,34 +72,30 @@ class OutputManager:
         if not self._initialized:
             self._initialized = True
             self._console_enabled = True
-            self._file_logger = None
-            self._min_level = OutputLevel.DO  # 默认显示DO及以上（包括DONE, TIPS, INFO等）
+            # 从环境变量读取日志级别，默认为INFO
+            log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+            level_map = {
+                "DEBUG": OutputLevel.DEBUG,
+                "INFO": OutputLevel.INFO,
+                "WARNING": OutputLevel.WARN,
+                "WARN": OutputLevel.WARN,
+                "ERROR": OutputLevel.ERROR,
+                "DO": OutputLevel.DO,
+                "DONE": OutputLevel.DONE,
+                "TIPS": OutputLevel.TIPS,
+            }
+            self._min_level = level_map.get(log_level, OutputLevel.INFO)
             self._suppressed_modules = set()
-            self._setup_file_logger()
+            # 创建一个基本的日志记录器（不写入文件）
+            self._file_logger = logging.getLogger("arxiv_pulse")
+            self._file_logger.setLevel(logging.DEBUG)
+            # 添加NullHandler避免"No handlers"警告
+            if not self._file_logger.handlers:
+                self._file_logger.addHandler(logging.NullHandler())
 
             # 抑制第三方库的详细日志
             self._suppress_third_party_logs()
 
-    def _setup_file_logger(self):
-        """设置文件日志记录器"""
-        # 创建日志目录
-        os.makedirs("logs", exist_ok=True)
-
-        # 配置文件日志记录器
-        self._file_logger = logging.getLogger("arxiv_crawler")
-        self._file_logger.setLevel(logging.DEBUG)
-
-        # 移除现有处理器
-        for handler in self._file_logger.handlers[:]:
-            self._file_logger.removeHandler(handler)
-
-        # 添加文件处理器
-        file_handler = logging.FileHandler("logs/arxiv_pulse.log", encoding="utf-8")
-        file_handler.setLevel(logging.DEBUG)
-        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        file_handler.setFormatter(formatter)
-        self._file_logger.addHandler(file_handler)
-
     def _suppress_third_party_logs(self):
         """抑制第三方库的详细日志"""
         # 设置第三方库的日志级别为WARNING或更高
@@ -117,16 +113,18 @@ class OutputManager:
             return False
 
         # 检查级别
+        # 数字越小表示级别越低（越不重要）
         level_order = {
-            OutputLevel.DO: 0,
-            OutputLevel.DONE: 1,
-            OutputLevel.TIPS: 2,
-            OutputLevel.INFO: 3,
-            OutputLevel.WARN: 4,
-            OutputLevel.ERROR: 5,
-            OutputLevel.DEBUG: 6,
+            OutputLevel.DEBUG: 0,  # 最低级别
+            OutputLevel.INFO: 1,
+            OutputLevel.WARN: 2,
+            OutputLevel.ERROR: 3,
+            OutputLevel.DO: 4,  # 操作提示，通常显示
+            OutputLevel.DONE: 5,
+            OutputLevel.TIPS: 6,
         }
 
+        # 只有级别数字 >= 最小级别数字的才显示
         return level_order[level] >= level_order[self._min_level]
 
     def _output(
@@ -137,28 +135,6 @@ class OutputManager:
         details: Optional[Dict[str, Any]] = None,
     ):
         """统一输出方法"""
-        # 记录到文件日志
-        log_level = {
-            OutputLevel.DO: logging.INFO,
-            OutputLevel.DONE: logging.INFO,
-            OutputLevel.TIPS: logging.INFO,
-            OutputLevel.INFO: logging.INFO,
-            OutputLevel.WARN: logging.WARNING,
-            OutputLevel.ERROR: logging.ERROR,
-            OutputLevel.DEBUG: logging.DEBUG,
-        }[level]
-
-        # 构建详细日志消息
-        log_message = message
-        if module:
-            log_message = f"[{module}] {message}"
-        if details:
-            details_str = " ".join(f"{k}={v}" for k, v in details.items())
-            log_message = f"{log_message} | {details_str}"
-
-        # 写入文件日志
-        self._file_logger.log(log_level, log_message)
-
         # 控制台输出
         if self._console_enabled and self._should_output(level, module):
             # 获取标签和颜色
@@ -228,7 +204,15 @@ class OutputManager:
     @classmethod
     def get_file_logger(cls) -> logging.Logger:
         """获取文件日志记录器"""
-        return cls()._file_logger
+        instance = cls()
+        if instance._file_logger is None:
+            # 创建基本的日志记录器作为回退
+            instance._file_logger = logging.getLogger("arxiv_pulse_fallback")
+            instance._file_logger.setLevel(logging.DEBUG)
+            if not instance._file_logger.handlers:
+                instance._file_logger.addHandler(logging.NullHandler())
+        assert instance._file_logger is not None
+        return instance._file_logger
 
 
 # 简化别名

arxiv_pulse/report_generator.py

@@ -1,7 +1,6 @@
 import json
 import pandas as pd
-from datetime import datetime, timedelta
-import markdown
+from datetime import datetime, timedelta, timezone
 from typing import Dict, List, Any, Optional
 import logging
 import os
@@ -21,7 +20,6 @@ class ReportGenerator:
         self.total_tokens_used = 0  # 总token使用量
         self.total_cost = 0.0  # 总费用（元）
         self.token_price_per_million = Config.TOKEN_PRICE_PER_MILLION  # 每百万token价格，可从配置覆盖
-        self.summary_sentences_limit = Config.SUMMARY_SENTENCES_LIMIT  # 摘要句子数限制
 
         # 抑制第三方库的详细日志
         import logging
@@ -211,47 +209,6 @@ class ReportGenerator:
         # 确保分数在1-5之间
         return max(1, min(5, score))
 
-    def _truncate_to_sentences(self, text: str, max_sentences: Optional[int] = None) -> str:
-        """将文本截断为指定数量的句子（支持中英文）"""
-        if not text:
-            return ""
-
-        if max_sentences is None:
-            max_sentences = self.summary_sentences_limit
-
-        import re
-
-        # 支持中英文句子分隔符：句号、问号、感叹号、分号、省略号
-        # 英文: . ? ! ; ... 中文: 。！？；…
-        pattern = r"([。！？；…\.\?!;]+|\.{3,})"
-        parts = re.split(pattern, text)
-
-        sentences = []
-        current = ""
-        for i, part in enumerate(parts):
-            current += part
-            if i % 2 == 1:  # 分隔符部分
-                sentences.append(current)
-                current = ""
-
-        # 如果最后还有未结束的句子
-        if current:
-            sentences.append(current)
-
-        # 如果分割失败，按长度简单截断
-        if len(sentences) == 0:
-            return text[:200] + "..." if len(text) > 200 else text
-
-        # 取前max_sentences句
-        result = "".join(sentences[:max_sentences])
-
-        # 如果截断后比原文本短很多，添加省略号
-        if len(result) < len(text) * 0.8:
-            # 移除末尾的句子分隔符，添加省略号
-            result = result.rstrip("。！？；….?!;") + "…"
-
-        return result
-
     def translate_text(self, text: str, target_lang: str = "zh") -> str:
         """使用DeepSeek或OpenAI API翻译文本，优先使用缓存"""
         if not text or not text.strip():
@@ -364,7 +321,7 @@ class ReportGenerator:
 
         with self.db.get_session() as session:
             # Get papers from last 24 hours
-            cutoff = datetime.utcnow() - timedelta(hours=24)
+            cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=24)
             new_papers = (
                 session.query(Paper)
                 .filter(Paper.created_at >= cutoff)
@@ -406,7 +363,7 @@ class ReportGenerator:
 
         with self.db.get_session() as session:
             # Get papers from last 7 days
-            cutoff = datetime.utcnow() - timedelta(days=7)
+            cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=7)
             recent_papers = (
                 session.query(Paper)
                 .filter(Paper.created_at >= cutoff)

arxiv_pulse/search_engine.py

@@ -2,11 +2,9 @@
 增强搜索引擎 - 提供高级搜索和过滤功能
 """
 
-import json
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Optional, Union
+from datetime import datetime, timedelta, timezone
+from typing import List, Dict, Any, Optional
 from dataclasses import dataclass, field
-from pathlib import Path
 
 from sqlalchemy import and_, or_, not_, func, desc, asc
 from sqlalchemy.orm import Session
@@ -21,7 +19,7 @@ class SearchFilter:
 
     # 文本搜索
     query: Optional[str] = None
-    search_fields: List[str] = field(default_factory=lambda: ["title", "abstract", "categories", "search_query"])
+    search_fields: List[str] = field(default_factory=lambda: ["title", "abstract"])
 
     # 分类过滤
     categories: Optional[List[str]] = None
@@ -64,31 +62,117 @@ class SearchEngine:
         self.session = db_session
 
     def build_text_filter(self, query: str, search_fields: List[str], match_all: bool = False):
-        """构建文本搜索过滤器"""
+        """构建文本搜索过滤器，简单模糊匹配（支持单词拆分）"""
         if not query or not search_fields:
             return None
 
-        filters = []
+        # 将查询转换为小写进行不区分大小写的匹配
+        query_lower = query.lower()
+
+        # 拆分为单词（按非字母数字字符，保留中文）
+        import re
+
+        # 使用正则表达式分割，保留中文字符（支持Unicode）
+        words = re.split(r"[^\w]+", query_lower, flags=re.UNICODE)
+        # 过滤掉空字符串和过短的单词（长度>1）
+        words = [w for w in words if w and len(w) > 1]
+
+        # 如果没有有效的单词，使用整个查询作为单个单词
+        if not words:
+            words = [query_lower]
+
+        # 如果只有一个单词，使用简单的字段间OR逻辑
+        if len(words) == 1:
+            word = words[0]
+            field_filters = []
+            for field in search_fields:
+                if field == "title":
+                    field_filters.append(Paper.title.ilike(f"%{word}%"))
+                elif field == "abstract":
+                    field_filters.append(Paper.abstract.ilike(f"%{word}%"))
+                elif field == "categories":
+                    field_filters.append(Paper.categories.ilike(f"%{word}%"))
+                elif field == "search_query":
+                    field_filters.append(Paper.search_query.ilike(f"%{word}%"))
+                elif field == "authors":
+                    field_filters.append(Paper.authors.ilike(f"%{word}%"))
+
+            if field_filters:
+                return or_(*field_filters)
+            return None
+
+        # 多个单词：首先尝试短语匹配（整个查询字符串）
+        phrase_filters = []
         for field in search_fields:
             if field == "title":
-                filters.append(Paper.title.contains(query))
+                phrase_filters.append(Paper.title.ilike(f"%{query_lower}%"))
             elif field == "abstract":
-                filters.append(Paper.abstract.contains(query))
+                phrase_filters.append(Paper.abstract.ilike(f"%{query_lower}%"))
             elif field == "categories":
-                filters.append(Paper.categories.contains(query))
+                phrase_filters.append(Paper.categories.ilike(f"%{query_lower}%"))
             elif field == "search_query":
-                filters.append(Paper.search_query.contains(query))
+                phrase_filters.append(Paper.search_query.ilike(f"%{query_lower}%"))
             elif field == "authors":
-                # 作者字段是JSON字符串，需要特殊处理
-                filters.append(Paper.authors.contains(query))
-
-        if not filters:
+                phrase_filters.append(Paper.authors.ilike(f"%{query_lower}%"))
+
+        # 尝试顺序匹配（单词按顺序出现，中间可间隔）
+        sequence_filters = []
+        if len(words) > 1:
+            # 构建模式：%word1%word2%word3%
+            sequence_pattern = "%" + "%".join(words) + "%"
+            for field in search_fields:
+                if field == "title":
+                    sequence_filters.append(Paper.title.ilike(sequence_pattern))
+                elif field == "abstract":
+                    sequence_filters.append(Paper.abstract.ilike(sequence_pattern))
+                elif field == "categories":
+                    sequence_filters.append(Paper.categories.ilike(sequence_pattern))
+                elif field == "search_query":
+                    sequence_filters.append(Paper.search_query.ilike(sequence_pattern))
+                elif field == "authors":
+                    sequence_filters.append(Paper.authors.ilike(sequence_pattern))
+
+        # 然后添加单词AND匹配（所有单词必须在同一个字段中出现）
+        word_and_filters = []
+        for field in search_fields:
+            if field == "title":
+                # 标题必须包含所有单词
+                title_filters = [Paper.title.ilike(f"%{word}%") for word in words]
+                if title_filters:
+                    word_and_filters.append(and_(*title_filters))
+            elif field == "abstract":
+                # 摘要必须包含所有单词
+                abstract_filters = [Paper.abstract.ilike(f"%{word}%") for word in words]
+                if abstract_filters:
+                    word_and_filters.append(and_(*abstract_filters))
+            elif field == "categories":
+                # 分类必须包含所有单词（通常分类搜索是单个词）
+                category_filters = [Paper.categories.ilike(f"%{word}%") for word in words]
+                if category_filters:
+                    word_and_filters.append(and_(*category_filters))
+            elif field == "search_query":
+                search_query_filters = [Paper.search_query.ilike(f"%{word}%") for word in words]
+                if search_query_filters:
+                    word_and_filters.append(and_(*search_query_filters))
+            elif field == "authors":
+                author_filters = [Paper.authors.ilike(f"%{word}%") for word in words]
+                if author_filters:
+                    word_and_filters.append(and_(*author_filters))
+
+        # 组合所有过滤器：短语匹配 OR 顺序匹配 OR 单词AND匹配
+        all_filters = []
+        if phrase_filters:
+            all_filters.append(or_(*phrase_filters))
+        if sequence_filters:
+            all_filters.append(or_(*sequence_filters))
+        if word_and_filters:
+            all_filters.append(or_(*word_and_filters))
+
+        if not all_filters:
             return None
 
-        if match_all:
-            return and_(*filters)
-        else:
-            return or_(*filters)
+        # 使用OR逻辑连接所有匹配类型
+        return or_(*all_filters)
 
     def build_category_filter(
         self,
@@ -147,7 +231,7 @@ class SearchEngine:
         filters = []
 
         if days_back:
-            cutoff_date = datetime.utcnow() - timedelta(days=days_back)
+            cutoff_date = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days_back)
             filters.append(Paper.published >= cutoff_date)
 
         if date_from:
@@ -290,38 +374,6 @@ class SearchEngine:
             output.error("相似论文搜索失败", details={"exception": str(e)})
             return []
 
-            # 简化的相似性搜索：基于共同关键词或分类
-            # 在实际应用中，可以使用更复杂的文本相似性算法
-            all_papers = self.session.query(Paper).filter(Paper.arxiv_id != paper_id).all()
-
-            # 计算简单相似度：分类重叠
-            similar_papers = []
-            target_cats = set(target_paper.categories.split()) if target_paper.categories else set()
-
-            for paper in all_papers:
-                if not paper.categories:
-                    continue
-
-                paper_cats = set(paper.categories.split())
-                common_cats = target_cats.intersection(paper_cats)
-
-                if common_cats:
-                    # 简单相似度分数：共同分类数 / 总分类数
-                    similarity = len(common_cats) / max(len(target_cats), len(paper_cats))
-                    if similarity >= threshold:
-                        # 临时存储相似度分数
-                        paper.similarity_score = similarity
-                        similar_papers.append(paper)
-
-            # 按相似度排序
-            similar_papers.sort(key=lambda x: getattr(x, "similarity_score", 0), reverse=True)
-
-            return similar_papers[:limit]
-
-        except Exception as e:
-            output.error("相似论文搜索失败", details={"exception": str(e)})
-            return []
-
     def get_search_history(self, limit: int = 10) -> List[Dict[str, Any]]:
         """获取搜索历史（从数据库中的search_query字段提取）"""
         try:
@@ -360,7 +412,7 @@ class SearchEngine:
             output.error("获取搜索历史失败", details={"exception": str(e)})
             return []
 
-    def save_search_query(self, query: str, description: str = None):
+    def save_search_query(self, query: str, description: Optional[str] = None):
         """保存搜索查询到历史（简单实现）"""
         # 这里可以扩展为保存到单独的搜索历史表
         # 目前依赖于Paper表中的search_query字段

arxiv_pulse/summarizer.py

@@ -1,4 +1,3 @@
-import openai
 import json
 import logging
 from typing import List, Dict, Any, Optional