github-ai-scraper 0.1.2__py3-none-any.whl

ai_scraper/output/markdown.py

@@ -0,0 +1,338 @@
+"""Markdown exporter for generating beautiful reports."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from ai_scraper.models import Repository
+from ai_scraper.output.translator import translate_description
+
+
+# 分类映射（英文 -> 中文）
+CATEGORY_NAMES = {
+    "LLM": "大语言模型",
+    "Chatbot": "聊天机器人",
+    "Generative AI": "生成式 AI",
+    "Computer Vision": "计算机视觉",
+    "NLP": "自然语言处理",
+    "Machine Learning": "机器学习",
+    "Deep Learning": "深度学习",
+    "MLOps": "机器学习运维",
+    "AI Infrastructure": "AI 基础设施",
+    "AI Ethics": "AI 伦理",
+    "Reinforcement Learning": "强化学习",
+    "Robotics": "机器人",
+    "AutoML": "自动化机器学习",
+    "Data Science": "数据科学",
+    "AI Tools": "AI 工具",
+    "Other": "其他",
+}
+
+# 语言图标
+LANGUAGE_ICONS = {
+    "Python": "🐍",
+    "TypeScript": "📘",
+    "JavaScript": "💛",
+    "Java": "☕",
+    "Go": "🐹",
+    "Rust": "🦀",
+    "C++": "⚡",
+    "C": "⚙️",
+    "Jupyter Notebook": "📊",
+    "HTML": "🌐",
+    "CSS": "🎨",
+    "Ruby": "💎",
+    "PHP": "🐘",
+    "Swift": "🍎",
+    "Kotlin": "🎯",
+    "Lua": "🌙",
+    "Shell": "🖥️",
+    "Dart": "🎯",
+    "Scala": "🔴",
+    "R": "📈",
+    "MATLAB": "📐",
+    "Julia": "💜",
+    "Haskell": "λ",
+    "Elixir": "💧",
+    "Clojure": "🧬",
+    "F#": "🔷",
+    "OCaml": "🐫",
+    "Nim": "👑",
+    "Crystal": "💎",
+    "Elm": "🌳",
+    "V": "⚡",
+    "Zig": "⚡",
+    "Nim": "👑",
+}
+
+# 热门程度图标
+STAR_LEVELS = [
+    (100000, "🔥🔥🔥🔥🔥", "超热门"),
+    (50000, "🔥🔥🔥🔥", "非常热门"),
+    (10000, "🔥🔥🔥", "热门"),
+    (5000, "🔥🔥", "较热门"),
+    (1000, "🔥", "值得关注"),
+    (0, "⭐", "新星"),
+]
+
+
+class MarkdownExporter:
+    """Export repositories to beautiful Markdown format."""
+
+    def __init__(self, output_dir: Path, filename: str = "repositories.md"):
+        """Initialize the exporter.
+
+        Args:
+            output_dir: Directory for output files.
+            filename: Name of the output file.
+        """
+        self.output_dir = Path(output_dir)
+        self.filename = filename
+
+    def export_repositories(self, repos: list[Repository]) -> Path:
+        """Export repositories to a Markdown file.
+
+        Args:
+            repos: List of repositories to export.
+
+        Returns:
+            Path to the created file.
+        """
+        # Create output directory if needed
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Generate content
+        content = self._generate_content(repos)
+
+        # Write file
+        output_path = self.output_dir / self.filename
+        output_path.write_text(content, encoding="utf-8")
+
+        return output_path
+
+    def _generate_content(self, repos: list[Repository]) -> str:
+        """Generate the full Markdown content."""
+        lines = []
+
+        # 标题和介绍
+        lines.append("# 🤖 AI 开源项目精选")
+        lines.append("")
+        lines.append("> 精选 GitHub 上最受欢迎的 AI 相关开源项目，助你发现优质资源！")
+        lines.append("")
+
+        # 统计信息
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        total_stars = sum(repo.stars for repo in repos)
+        languages = self._get_language_stats(repos)
+
+        lines.append("## 📊 统计概览")
+        lines.append("")
+        lines.append(f"| 📅 更新时间 | 📦 项目数量 | ⭐ 总星数 |")
+        lines.append(f"|:---:|:---:|:---:|")
+        lines.append(f"| {now} | {len(repos)} | {self._format_number(total_stars)} |")
+        lines.append("")
+        lines.append("### 🌈 语言分布")
+        lines.append("")
+        lang_chart = self._generate_language_chart(languages)
+        lines.append(lang_chart)
+        lines.append("")
+
+        # 目录
+        lines.append("## 📑 目录")
+        lines.append("")
+        lines.append("- [🔥 超热门项目 (100K+ Stars)](#🔥-超热门项目-100k-stars)")
+        lines.append("- [🌟 热门项目 (50K-100K Stars)](#🌟-热门项目-50k-100k-stars)")
+        lines.append("- [⭐ 优质项目 (10K-50K Stars)](#⭐-优质项目-10k-50k-stars)")
+        lines.append("- [💡 新兴项目 (1K-10K Stars)](#💡-新兴项目-1k-10k-stars)")
+        lines.append("")
+
+        # 按星数分组展示
+        grouped_repos = self._group_by_stars(repos)
+
+        for group_name, group_repos in grouped_repos:
+            if group_repos:
+                lines.append(f"## {group_name}")
+                lines.append("")
+                for repo in group_repos:
+                    lines.append(self._format_repo_card(repo))
+                    lines.append("")
+                lines.append("---")
+                lines.append("")
+
+        # 页脚
+        lines.append("")
+        lines.append("## 📝 说明")
+        lines.append("")
+        lines.append("- 数据来源于 GitHub API，按 Star 数排序")
+        lines.append("- 项目描述包含英文原文和中文翻译")
+        lines.append("- 语言图标用于快速识别项目技术栈")
+        lines.append("- 热门程度图标表示项目的受欢迎程度")
+        lines.append("")
+        lines.append("---")
+        lines.append("")
+        lines.append("*由 [AI Scraper](https://github.com/lwx66615/github-ai-scraper) 自动生成*")
+
+        return "\n".join(lines)
+
+    def _format_repo_card(self, repo: Repository) -> str:
+        """Format a single repository as a beautiful card.
+
+        Args:
+            repo: Repository to format.
+
+        Returns:
+            Markdown formatted card.
+        """
+        lang_icon = LANGUAGE_ICONS.get(repo.language or "", "📁")
+        star_level = self._get_star_level(repo.stars)
+
+        # 项目标题
+        title = f"### [{repo.full_name}]({repo.url})"
+        subtitle = f"{lang_icon} {repo.language or '未知'} | {star_level[0]} {self._format_number(repo.stars)} Stars | {star_level[1]}"
+
+        lines = []
+        lines.append(title)
+        lines.append("")
+        lines.append(f"**{subtitle}**")
+        lines.append("")
+
+        # 描述 - 原文和中文翻译
+        original_desc = self._clean_description(repo.description, 200)
+        translated_desc = translate_description(repo.description)
+
+        # 如果翻译后不同，显示双语
+        if translated_desc != original_desc and translated_desc != (repo.description or "暂无描述"):
+            lines.append(f"> {original_desc}")
+            lines.append(f"> ")
+            lines.append(f"> **中文:** {translated_desc}")
+        else:
+            lines.append(f"> {original_desc}")
+        lines.append("")
+
+        # Topics 标签
+        if repo.topics:
+            topics_display = " ".join(f"`{topic}`" for topic in repo.topics[:8])
+            lines.append(f"**标签:** {topics_display}")
+
+        return "\n".join(lines)
+
+    def _group_by_stars(self, repos: list[Repository]) -> list[tuple[str, list[Repository]]]:
+        """Group repositories by star count.
+
+        Args:
+            repos: List of repositories.
+
+        Returns:
+            List of (group_name, repos) tuples.
+        """
+        groups = [
+            ("🔥 超热门项目 (100K+ Stars)", []),
+            ("🌟 热门项目 (50K-100K Stars)", []),
+            ("⭐ 优质项目 (10K-50K Stars)", []),
+            ("💡 新兴项目 (1K-10K Stars)", []),
+            ("🌱 新星项目 (<1K Stars)", []),
+        ]
+
+        for repo in repos:
+            if repo.stars >= 100000:
+                groups[0][1].append(repo)
+            elif repo.stars >= 50000:
+                groups[1][1].append(repo)
+            elif repo.stars >= 10000:
+                groups[2][1].append(repo)
+            elif repo.stars >= 1000:
+                groups[3][1].append(repo)
+            else:
+                groups[4][1].append(repo)
+
+        return groups
+
+    def _get_star_level(self, stars: int) -> tuple[str, str]:
+        """Get star level icon and description.
+
+        Args:
+            stars: Star count.
+
+        Returns:
+            (icon, description) tuple.
+        """
+        for threshold, icon, desc in STAR_LEVELS:
+            if stars >= threshold:
+                return (icon, desc)
+        return ("⭐", "新星")
+
+    def _format_number(self, num: int) -> str:
+        """Format large numbers with K/M suffix.
+
+        Args:
+            num: Number to format.
+
+        Returns:
+            Formatted string.
+        """
+        if num >= 1000000:
+            return f"{num / 1000000:.1f}M"
+        elif num >= 1000:
+            return f"{num / 1000:.1f}K"
+        return str(num)
+
+    def _get_language_stats(self, repos: list[Repository]) -> dict[str, int]:
+        """Get language distribution statistics.
+
+        Args:
+            repos: List of repositories.
+
+        Returns:
+            Dictionary of language counts.
+        """
+        stats = {}
+        for repo in repos:
+            lang = repo.language or "Unknown"
+            stats[lang] = stats.get(lang, 0) + 1
+        return stats
+
+    def _generate_language_chart(self, languages: dict[str, int]) -> str:
+        """Generate a simple text-based language distribution chart.
+
+        Args:
+            languages: Language statistics.
+
+        Returns:
+            Markdown formatted chart.
+        """
+        # Sort by count
+        sorted_langs = sorted(languages.items(), key=lambda x: x[1], reverse=True)[:10]
+
+        lines = []
+        for lang, count in sorted_langs:
+            icon = LANGUAGE_ICONS.get(lang, "📁")
+            bar_len = min(20, int(count / max(languages.values()) * 20))
+            bar = "█" * bar_len + "░" * (20 - bar_len)
+            lines.append(f"| {icon} {lang or '未知'} | `{bar}` | {count} |")
+
+        return "\n".join(lines)
+
+    def _clean_description(self, description: Optional[str], max_len: int = 200) -> str:
+        """Clean description for display.
+
+        Args:
+            description: Original description.
+            max_len: Maximum length.
+
+        Returns:
+            Cleaned description.
+        """
+        if description is None:
+            return "暂无描述"
+
+        # Remove newlines and collapse spaces
+        cleaned = " ".join(description.split())
+
+        # Escape pipe characters
+        cleaned = cleaned.replace("|", r"\|")
+
+        # Truncate if needed
+        if len(cleaned) > max_len:
+            cleaned = cleaned[:max_len] + "..."
+
+        return cleaned

ai_scraper/output/rss.py

@@ -0,0 +1,82 @@
+"""RSS exporter for generating feed subscriptions."""
+
+from datetime import datetime, timezone
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from ai_scraper.models import Repository
+
+
+class RSSExporter:
+    """Export repositories to RSS feed format."""
+
+    def __init__(self, output_dir: Path, filename: str = "repositories.xml"):
+        """Initialize the exporter.
+
+        Args:
+            output_dir: Directory for output files.
+            filename: Name of the output file.
+        """
+        self.output_dir = Path(output_dir)
+        self.filename = filename
+
+    def export_repositories(
+        self,
+        repos: list[Repository],
+        title: str = "AI Repositories Feed",
+        description: str = "Latest AI repositories from GitHub",
+    ) -> Path:
+        """Export repositories to an RSS feed.
+
+        Args:
+            repos: List of repositories to export.
+            title: Feed title.
+            description: Feed description.
+
+        Returns:
+            Path to the created file.
+        """
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Create RSS structure
+        rss = ET.Element("rss", version="2.0")
+        channel = ET.SubElement(rss, "channel")
+
+        # Channel info
+        ET.SubElement(channel, "title").text = title
+        ET.SubElement(channel, "description").text = description
+        ET.SubElement(channel, "link").text = "https://github.com/topics/ai"
+        ET.SubElement(channel, "language").text = "en-us"
+        ET.SubElement(channel, "lastBuildDate").text = datetime.now(timezone.utc).strftime(
+            "%a, %d %b %Y %H:%M:%S GMT"
+        )
+
+        # Add items
+        for repo in repos:
+            item = ET.SubElement(channel, "item")
+
+            ET.SubElement(item, "title").text = repo.full_name
+            ET.SubElement(item, "link").text = repo.url
+            ET.SubElement(item, "description").text = repo.description or "No description"
+            ET.SubElement(item, "pubDate").text = (
+                repo.updated_at.strftime("%a, %d %b %Y %H:%M:%S GMT")
+                if repo.updated_at
+                else ""
+            )
+
+            # Custom elements
+            ET.SubElement(item, "stars").text = str(repo.stars)
+            if repo.language:
+                ET.SubElement(item, "language").text = repo.language
+
+        # Write to file
+        output_path = self.output_dir / self.filename
+
+        # Pretty print
+        ET.indent(rss, space="  ")
+        tree = ET.ElementTree(rss)
+
+        with open(output_path, "wb") as f:
+            tree.write(f, encoding="utf-8", xml_declaration=True)
+
+        return output_path