github-ai-scraper 0.1.2__py3-none-any.whl

ai_scraper/output/translator.py

@@ -0,0 +1,303 @@
+"""Simple translation module for repository descriptions."""
+
+import re
+from typing import Optional
+
+
+# 常见 AI 项目关键词翻译字典（按优先级排序）
+TRANSLATION_DICT = {
+    # 完整短语优先翻译
+    "production-ready platform": "生产就绪的平台",
+    "personal ai assistant": "个人 AI 助手",
+    "workflow automation platform": "工作流自动化平台",
+    "ai coding assistant": "AI 编程助手",
+    "coding assistant": "编程助手",
+    "code generation": "代码生成",
+    "image generation": "图像生成",
+    "text generation": "文本生成",
+    "speech recognition": "语音识别",
+    "speech synthesis": "语音合成",
+    "face recognition": "人脸识别",
+    "object detection": "物体检测",
+    "natural language processing": "自然语言处理",
+    "large language model": "大语言模型",
+    "machine learning": "机器学习",
+    "deep learning": "深度学习",
+    "neural network": "神经网络",
+    "computer vision": "计算机视觉",
+    "reinforcement learning": "强化学习",
+    "generative ai": "生成式 AI",
+    "artificial intelligence": "人工智能",
+    "retrieval-augmented generation": "检索增强生成",
+    "stable diffusion": "Stable Diffusion",
+    "web ui": "网页界面",
+    "webui": "网页界面",
+    "open source": "开源",
+    "open-source": "开源",
+    "self-host": "自托管",
+    "self-hosted": "自托管",
+    "production-ready": "生产就绪",
+    "user-friendly": "用户友好",
+    "cross-platform": "跨平台",
+    "real-time": "实时",
+    "high performance": "高性能",
+    "lightweight": "轻量级",
+    "knowledge base": "知识库",
+    "workflow automation": "工作流自动化",
+    "image upscaling": "图像放大",
+    "face swap": "换脸",
+    "video generation": "视频生成",
+    "text to image": "文本生成图像",
+    "text to speech": "文本转语音",
+    "speech to text": "语音转文本",
+    "language model": "语言模型",
+    "vision model": "视觉模型",
+    "multimodal": "多模态",
+    "getting started": "入门",
+    "get started": "入门",
+    "best practices": "最佳实践",
+    "awesome list": "精选列表",
+    "awesome": "精选",
+    "alternative to": "...的替代方案",
+    "next generation": "下一代",
+    "next-gen": "下一代",
+    "powered by": "基于",
+    "based on": "基于",
+    "built with": "使用...构建",
+    "written in": "使用...编写",
+    "designed for": "专为...设计",
+    "easy to use": "易于使用",
+    "free and open source": "免费开源",
+
+    # AI 相关术语
+    "llm": "大语言模型",
+    "nlp": "自然语言处理",
+    "chatbot": "聊天机器人",
+    "chatgpt": "ChatGPT",
+    "gpt": "GPT",
+    "transformer": "Transformer",
+    "diffusion": "扩散模型",
+    "rag": "检索增强生成",
+    "embedding": "嵌入",
+    "vector": "向量",
+    "prompt": "提示词",
+    "prompts": "提示词",
+    "agent": "代理",
+    "agents": "代理",
+    "assistant": "助手",
+    "conversation": "对话",
+    "conversational": "对话式",
+    "context": "上下文",
+    "memory": "记忆",
+    "reasoning": "推理",
+    "planning": "规划",
+    "autonomous": "自主",
+    "intelligent": "智能",
+    "smart": "智能",
+
+    # 技术术语
+    "framework": "框架",
+    "platform": "平台",
+    "toolkit": "工具包",
+    "library": "库",
+    "api": "API",
+    "sdk": "SDK",
+    "cli": "命令行工具",
+    "gui": "图形界面",
+    "dashboard": "仪表盘",
+    "interface": "界面",
+    "workflow": "工作流",
+    "automation": "自动化",
+    "integration": "集成",
+    "pipeline": "管道",
+    "orchestration": "编排",
+    "deployment": "部署",
+    "training": "训练",
+    "fine-tuning": "微调",
+    "inference": "推理",
+    "model": "模型",
+    "models": "模型",
+    "database": "数据库",
+    "storage": "存储",
+    "search": "搜索",
+    "retrieval": "检索",
+    "document": "文档",
+    "pdf": "PDF",
+    "markdown": "Markdown",
+    "server": "服务器",
+    "client": "客户端",
+    "backend": "后端",
+    "frontend": "前端",
+    "service": "服务",
+    "services": "服务",
+    "endpoint": "端点",
+    "request": "请求",
+    "response": "响应",
+
+    # 动词
+    "build": "构建",
+    "building": "构建",
+    "create": "创建",
+    "develop": "开发",
+    "deploy": "部署",
+    "run": "运行",
+    "manage": "管理",
+    "analyze": "分析",
+    "process": "处理",
+    "generate": "生成",
+    "extract": "提取",
+    "convert": "转换",
+    "scrape": "抓取",
+    "crawl": "爬取",
+    "monitor": "监控",
+    "track": "追踪",
+    "visualize": "可视化",
+    "optimize": "优化",
+    "scale": "扩展",
+    "integrate": "集成",
+    "connect": "连接",
+    "support": "支持",
+    "supports": "支持",
+
+    # 形容词
+    "simple": "简单",
+    "fast": "快速",
+    "efficient": "高效",
+    "powerful": "强大",
+    "flexible": "灵活",
+    "extensible": "可扩展",
+    "modular": "模块化",
+    "customizable": "可定制",
+    "scalable": "可扩展",
+    "robust": "稳健",
+    "secure": "安全",
+    "privacy": "隐私",
+    "free": "免费",
+    "modern": "现代",
+    "latest": "最新",
+
+    # 名词
+    "developer": "开发者",
+    "developers": "开发者",
+    "user": "用户",
+    "users": "用户",
+    "team": "团队",
+    "organization": "组织",
+    "enterprise": "企业",
+    "business": "商业",
+    "application": "应用",
+    "applications": "应用",
+    "project": "项目",
+    "projects": "项目",
+    "repository": "仓库",
+    "code": "代码",
+    "data": "数据",
+    "file": "文件",
+    "files": "文件",
+    "community": "社区",
+    "resources": "资源",
+    "tools": "工具",
+    "utilities": "实用工具",
+    "extensions": "扩展",
+    "plugins": "插件",
+    "tutorial": "教程",
+    "documentation": "文档",
+    "examples": "示例",
+    "demo": "演示",
+    "sample": "示例",
+    "template": "模板",
+    "benchmark": "基准测试",
+    "testing": "测试",
+    "experiment": "实验",
+    "research": "研究",
+    "education": "教育",
+    "learning": "学习",
+    "course": "课程",
+    "lesson": "课程",
+    "roadmap": "路线图",
+    "guide": "指南",
+    "handbook": "手册",
+    "collection": "合集",
+    "list": "列表",
+    "curated": "精选",
+    "version": "版本",
+    "release": "发布",
+    "update": "更新",
+    "feature": "功能",
+    "features": "功能",
+    "capability": "能力",
+    "capabilities": "能力",
+    "functionality": "功能",
+    "component": "组件",
+    "module": "模块",
+    "package": "包",
+}
+
+
+def translate_description(description: Optional[str]) -> str:
+    """Translate repository description to Chinese.
+
+    Uses a dictionary-based approach for common AI terms.
+    For descriptions that are already in Chinese, returns as-is.
+    For mixed content, preserves readability.
+
+    Args:
+        description: Original English description.
+
+    Returns:
+        Translated description with Chinese terms mixed naturally.
+    """
+    if description is None:
+        return "暂无描述"
+
+    # Check if already contains significant Chinese characters
+    chinese_chars = re.findall(r'[\u4e00-\u9fff]', description)
+    if len(chinese_chars) > len(description) * 0.3:  # More than 30% Chinese
+        return description
+
+    # Clean the description
+    cleaned = " ".join(description.split())
+
+    # Don't translate if it's very short (likely a name or brand)
+    if len(cleaned) < 20:
+        return cleaned
+
+    # Sort by length (longest first) to avoid partial matches
+    sorted_terms = sorted(TRANSLATION_DICT.keys(), key=len, reverse=True)
+
+    translated = cleaned
+    for term in sorted_terms:
+        # Case-insensitive replacement, but preserve word boundaries
+        pattern = re.compile(r'\b' + re.escape(term) + r'\b', re.IGNORECASE)
+        if pattern.search(translated):
+            translated = pattern.sub(TRANSLATION_DICT[term], translated)
+
+    # Clean up any leftover English fragments that make it awkward
+    # Remove awkward patterns like "代理ic" -> "代理"
+    translated = re.sub(r'代理ic', '代理', translated)
+    translated = re.sub(r'开发ment', '开发', translated)
+    translated = re.sub(r'集成s', '集成', translated)
+    translated = re.sub(r'构建ing', '构建', translated)
+    translated = re.sub(r'分析er', '分析器', translated)
+    translated = re.sub(r'生成ive', '生成式', translated)
+
+    return translated
+
+
+def get_bilingual_description(description: Optional[str]) -> tuple[str, str]:
+    """Get both original and translated description.
+
+    Args:
+        description: Original description.
+
+    Returns:
+        (original, translated) tuple.
+    """
+    original = description or "暂无描述"
+    translated = translate_description(description)
+
+    # If translation is same as original (no changes), don't duplicate
+    if original == translated:
+        return (original, "")
+
+    return (original, translated)

ai_scraper/plugin_system.py

@@ -0,0 +1,146 @@
+"""Plugin system for ai-scraper."""
+
+import importlib.util
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from ai_scraper.models.repository import Repository
+
+
+@dataclass
+class PluginInfo:
+    """Plugin metadata."""
+    name: str
+    version: str
+    description: str
+    author: Optional[str] = None
+
+
+class BasePlugin(ABC):
+    """Base class for plugins."""
+
+    @property
+    @abstractmethod
+    def info(self) -> PluginInfo:
+        """Get plugin information."""
+        pass
+
+    def on_scrape_start(self, config: dict) -> None:
+        """Called when scraping starts."""
+        pass
+
+    def on_repo_found(self, repo: Repository) -> Optional[Repository]:
+        """Called when a repository is found. Can modify or filter.
+
+        Args:
+            repo: Found repository.
+
+        Returns:
+            Modified repository or None to filter out.
+        """
+        return repo
+
+    def on_scrape_complete(self, repos: list[Repository], stats: dict) -> None:
+        """Called when scraping completes."""
+        pass
+
+    def on_export(self, data: Any, format: str) -> Any:
+        """Called before export. Can modify data.
+
+        Args:
+            data: Data to export.
+            format: Export format.
+
+        Returns:
+            Modified data.
+        """
+        return data
+
+
+class PluginManager:
+    """Manage loaded plugins."""
+
+    def __init__(self):
+        self.plugins: dict[str, BasePlugin] = {}
+
+    def load_plugin(self, plugin_path: Path) -> Optional[str]:
+        """Load a plugin from a Python file.
+
+        Args:
+            plugin_path: Path to plugin file.
+
+        Returns:
+            Plugin name if loaded successfully.
+        """
+        try:
+            spec = importlib.util.spec_from_file_location("custom_plugin", plugin_path)
+            if not spec or not spec.loader:
+                return None
+
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+
+            # Find plugin class
+            for attr_name in dir(module):
+                attr = getattr(module, attr_name)
+                if isinstance(attr, type) and issubclass(attr, BasePlugin) and attr is not BasePlugin:
+                    plugin = attr()
+                    self.plugins[plugin.info.name] = plugin
+                    return plugin.info.name
+
+            return None
+        except Exception as e:
+            print(f"Failed to load plugin: {e}")
+            return None
+
+    def load_plugins_from_dir(self, plugin_dir: Path) -> list[str]:
+        """Load all plugins from a directory.
+
+        Args:
+            plugin_dir: Directory containing plugin files.
+
+        Returns:
+            List of loaded plugin names.
+        """
+        loaded = []
+        for plugin_file in plugin_dir.glob("*.py"):
+            name = self.load_plugin(plugin_file)
+            if name:
+                loaded.append(name)
+        return loaded
+
+    def get_plugin(self, name: str) -> Optional[BasePlugin]:
+        """Get a loaded plugin by name."""
+        return self.plugins.get(name)
+
+    def trigger(self, event: str, *args, **kwargs) -> Any:
+        """Trigger an event on all plugins.
+
+        Args:
+            event: Event name (e.g., "on_repo_found").
+            *args: Positional arguments.
+            **kwargs: Keyword arguments.
+
+        Returns:
+            Modified result from plugin chain.
+        """
+        result = args[0] if args else None
+
+        for plugin in self.plugins.values():
+            handler = getattr(plugin, event, None)
+            if handler:
+                try:
+                    if result is not None:
+                        result = handler(result, **kwargs)
+                    else:
+                        handler(*args, **kwargs)
+                except Exception as e:
+                    print(f"Plugin {plugin.info.name} error in {event}: {e}")
+
+        return result
+
+
+# Global plugin manager
+plugin_manager = PluginManager()

ai_scraper/plugins/__init__.py

@@ -0,0 +1,5 @@
+"""Built-in plugins for ai-scraper."""
+
+from ai_scraper.plugin_system import BasePlugin, PluginInfo, PluginManager
+
+__all__ = ["BasePlugin", "PluginInfo", "PluginManager"]

ai_scraper/retry.py

@@ -0,0 +1,134 @@
+"""Retry logic with exponential backoff."""
+
+import asyncio
+import functools
+import logging
+import random
+from typing import Callable, Optional, Type, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class RetryHandler:
+    """Handle retry logic with exponential backoff."""
+
+    def __init__(
+        self,
+        max_retries: int = 3,
+        base_delay: float = 1.0,
+        exponential_base: float = 2.0,
+        max_delay: float = 60.0,
+        jitter: bool = True,
+        retryable_exceptions: Tuple[Type[Exception], ...] = (
+            ConnectionError,
+            TimeoutError,
+        ),
+    ):
+        """Initialize retry handler.
+
+        Args:
+            max_retries: Maximum number of retry attempts.
+            base_delay: Initial delay in seconds.
+            exponential_base: Base for exponential backoff.
+            max_delay: Maximum delay in seconds.
+            jitter: Add random jitter to delays.
+            retryable_exceptions: Exceptions that trigger retry.
+        """
+        self.max_retries = max_retries
+        self.base_delay = base_delay
+        self.exponential_base = exponential_base
+        self.max_delay = max_delay
+        self.jitter = jitter
+        self.retryable_exceptions = retryable_exceptions
+
+    def _calculate_delay(self, attempt: int) -> float:
+        """Calculate delay for a given attempt."""
+        delay = self.base_delay * (self.exponential_base ** attempt)
+        delay = min(delay, self.max_delay)
+
+        if self.jitter:
+            # Add up to 25% jitter
+            delay = delay * (1 + random.random() * 0.25)
+
+        return delay
+
+    async def execute(
+        self,
+        func: Callable,
+        *args,
+        **kwargs,
+    ):
+        """Execute a function with retry logic.
+
+        Args:
+            func: Async function to execute.
+            *args: Positional arguments for func.
+            **kwargs: Keyword arguments for func.
+
+        Returns:
+            Result of the function.
+
+        Raises:
+            Last exception after all retries exhausted.
+        """
+        last_exception = None
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                return await func(*args, **kwargs)
+            except self.retryable_exceptions as e:
+                last_exception = e
+
+                if attempt < self.max_retries:
+                    delay = self._calculate_delay(attempt)
+                    logger.warning(
+                        f"Retry {attempt + 1}/{self.max_retries} after {delay:.2f}s: {e}"
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    logger.error(f"All {self.max_retries} retries exhausted")
+
+        raise last_exception
+
+
+def with_retry(
+    max_retries: int = 3,
+    base_delay: float = 1.0,
+    exponential_base: float = 2.0,
+    max_delay: float = 60.0,
+    jitter: bool = True,
+    retryable_exceptions: Tuple[Type[Exception], ...] = (
+        ConnectionError,
+        TimeoutError,
+    ),
+):
+    """Decorator for retry logic.
+
+    Args:
+        max_retries: Maximum number of retry attempts.
+        base_delay: Initial delay in seconds.
+        exponential_base: Base for exponential backoff.
+        max_delay: Maximum delay in seconds.
+        jitter: Add random jitter to delays.
+        retryable_exceptions: Exceptions that trigger retry.
+
+    Returns:
+        Decorated function.
+    """
+    handler = RetryHandler(
+        max_retries=max_retries,
+        base_delay=base_delay,
+        exponential_base=exponential_base,
+        max_delay=max_delay,
+        jitter=jitter,
+        retryable_exceptions=retryable_exceptions,
+    )
+
+    def decorator(func):
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            return await handler.execute(func, *args, **kwargs)
+
+        return wrapper
+
+    return decorator

ai_scraper/scheduler.py

@@ -0,0 +1,84 @@
+"""Scheduled task management."""
+
+import asyncio
+import signal
+from datetime import datetime
+from typing import Callable, Optional
+
+from croniter import croniter
+
+
+class TaskScheduler:
+    """Schedule and run periodic tasks."""
+
+    def __init__(self):
+        self.tasks: dict[str, dict] = {}
+        self.running = False
+
+    def add_task(
+        self,
+        name: str,
+        cron_expr: str,
+        callback: Callable,
+        *args,
+        **kwargs,
+    ) -> None:
+        """Add a scheduled task.
+
+        Args:
+            name: Task name.
+            cron_expr: Cron expression (e.g., "0 9 * * *" for daily at 9am).
+            callback: Function to call.
+            *args: Positional arguments for callback.
+            **kwargs: Keyword arguments for callback.
+        """
+        self.tasks[name] = {
+            "cron": croniter(cron_expr, datetime.now()),
+            "callback": callback,
+            "args": args,
+            "kwargs": kwargs,
+            "next_run": None,
+        }
+        self._update_next_run(name)
+
+    def _update_next_run(self, name: str) -> None:
+        """Update next run time for a task."""
+        task = self.tasks[name]
+        task["next_run"] = task["cron"].get_next(datetime)
+
+    async def run(self) -> None:
+        """Start the scheduler loop."""
+        self.running = True
+
+        # Handle shutdown signals
+        loop = asyncio.get_event_loop()
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, self.stop)
+
+        while self.running:
+            now = datetime.now()
+            sleep_time = 60.0  # Check every minute
+
+            for name, task in self.tasks.items():
+                if task["next_run"] and task["next_run"] <= now:
+                    # Run the task
+                    try:
+                        if asyncio.iscoroutinefunction(task["callback"]):
+                            await task["callback"](*task["args"], **task["kwargs"])
+                        else:
+                            task["callback"](*task["args"], **task["kwargs"])
+                    except Exception as e:
+                        print(f"Task {name} failed: {e}")
+
+                    # Schedule next run
+                    self._update_next_run(name)
+
+            await asyncio.sleep(sleep_time)
+
+    def stop(self) -> None:
+        """Stop the scheduler."""
+        self.running = False
+
+
+# Global scheduler instance
+scheduler = TaskScheduler()