github-ai-scraper 0.1.2__py3-none-any.whl

ai_scraper/health.py

@@ -0,0 +1,155 @@
+"""Repository health assessment."""
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Optional
+
+from ai_scraper.models.repository import Repository
+
+
+@dataclass
+class HealthScore:
+    """Repository health score breakdown."""
+
+    overall: float
+    activity: float
+    popularity: float
+    maintenance: float
+    community: float
+    grade: str
+
+
+class HealthAssessor:
+    """Assess repository health based on multiple factors."""
+
+    def assess(self, repo: Repository) -> HealthScore:
+        """Assess repository health.
+
+        Args:
+            repo: Repository to assess.
+
+        Returns:
+            Health score breakdown.
+        """
+        activity = self._score_activity(repo)
+        popularity = self._score_popularity(repo)
+        maintenance = self._score_maintenance(repo)
+        community = self._score_community(repo)
+
+        # Weighted overall score
+        overall = (
+            activity * 0.3 +
+            popularity * 0.25 +
+            maintenance * 0.25 +
+            community * 0.2
+        )
+
+        grade = self.get_grade(overall)
+
+        return HealthScore(
+            overall=overall,
+            activity=activity,
+            popularity=popularity,
+            maintenance=maintenance,
+            community=community,
+            grade=grade,
+        )
+
+    def _score_activity(self, repo: Repository) -> float:
+        """Score repository activity (0-100)."""
+        if not repo.pushed_at:
+            return 0
+
+        days_since_push = (datetime.now() - repo.pushed_at).days
+
+        if days_since_push <= 7:
+            return 100
+        elif days_since_push <= 30:
+            return 80
+        elif days_since_push <= 90:
+            return 60
+        elif days_since_push <= 180:
+            return 40
+        elif days_since_push <= 365:
+            return 20
+        else:
+            return 0
+
+    def _score_popularity(self, repo: Repository) -> float:
+        """Score repository popularity (0-100)."""
+        stars = repo.stars
+
+        if stars >= 10000:
+            return 100
+        elif stars >= 5000:
+            return 85
+        elif stars >= 1000:
+            return 70
+        elif stars >= 500:
+            return 55
+        elif stars >= 100:
+            return 40
+        elif stars >= 50:
+            return 25
+        else:
+            return 10
+
+    def _score_maintenance(self, repo: Repository) -> float:
+        """Score repository maintenance (0-100)."""
+        if not repo.open_issues:
+            return 50  # Unknown
+
+        # Lower open issues ratio is better
+        if repo.stars > 0:
+            issue_ratio = repo.open_issues / repo.stars
+            if issue_ratio < 0.01:
+                return 100
+            elif issue_ratio < 0.05:
+                return 80
+            elif issue_ratio < 0.1:
+                return 60
+            elif issue_ratio < 0.2:
+                return 40
+            else:
+                return 20
+
+        return 50
+
+    def _score_community(self, repo: Repository) -> float:
+        """Score community engagement (0-100)."""
+        forks = repo.forks or 0
+
+        if forks >= 1000:
+            return 100
+        elif forks >= 500:
+            return 85
+        elif forks >= 100:
+            return 70
+        elif forks >= 50:
+            return 55
+        elif forks >= 10:
+            return 40
+        elif forks >= 5:
+            return 25
+        else:
+            return 10
+
+    def get_grade(self, score: float) -> str:
+        """Convert score to letter grade.
+
+        Args:
+            score: Score (0-100).
+
+        Returns:
+            Letter grade (A-F).
+        """
+        if score >= 90:
+            return "A"
+        elif score >= 80:
+            return "B"
+        elif score >= 70:
+            return "C"
+        elif score >= 60:
+            return "D"
+        else:
+            return "F"

ai_scraper/i18n.py

@@ -0,0 +1,141 @@
+"""Internationalization support for multi-language search."""
+
+from typing import Optional
+
+
+# Default keyword translations
+DEFAULT_TRANSLATIONS = {
+    "en": {
+        "ai": "ai",
+        "artificial intelligence": "artificial intelligence",
+        "machine learning": "machine learning",
+        "deep learning": "deep learning",
+        "neural network": "neural network",
+        "llm": "llm",
+        "large language model": "large language model",
+        "gpt": "gpt",
+        "transformer": "transformer",
+        "nlp": "nlp",
+        "natural language processing": "natural language processing",
+        "computer vision": "computer vision",
+        "reinforcement learning": "reinforcement learning",
+        "pytorch": "pytorch",
+        "tensorflow": "tensorflow",
+        "huggingface": "huggingface",
+    },
+    "zh": {
+        "ai": "人工智能",
+        "artificial intelligence": "人工智能",
+        "machine learning": "机器学习",
+        "deep learning": "深度学习",
+        "neural network": "神经网络",
+        "llm": "大语言模型",
+        "large language model": "大语言模型",
+        "gpt": "GPT",
+        "transformer": "Transformer",
+        "nlp": "自然语言处理",
+        "natural language processing": "自然语言处理",
+        "computer vision": "计算机视觉",
+        "reinforcement learning": "强化学习",
+        "pytorch": "PyTorch",
+        "tensorflow": "TensorFlow",
+        "huggingface": "Hugging Face",
+    },
+}
+
+
+class I18nManager:
+    """Manage internationalization for keywords."""
+
+    def __init__(self):
+        """Initialize i18n manager with default translations."""
+        self._translations = dict(DEFAULT_TRANSLATIONS)
+
+    def get_keywords(self, language: str) -> set[str]:
+        """Get all keywords for a language.
+
+        Args:
+            language: Language code (e.g., "en", "zh").
+
+        Returns:
+            Set of keywords for the language.
+        """
+        if language in self._translations:
+            return set(self._translations[language].values())
+        # Fallback to English
+        return set(self._translations.get("en", {}).values())
+
+    def add_translation(
+        self,
+        source_lang: str,
+        source_term: str,
+        target_lang: str,
+        target_term: str,
+    ) -> None:
+        """Add a custom translation.
+
+        Args:
+            source_lang: Source language code.
+            source_term: Term in source language.
+            target_lang: Target language code.
+            target_term: Translation in target language.
+        """
+        if target_lang not in self._translations:
+            self._translations[target_lang] = {}
+
+        self._translations[target_lang][source_term] = target_term
+
+    def translate(
+        self,
+        term: str,
+        source_lang: str = "en",
+        target_lang: str = "zh",
+    ) -> Optional[str]:
+        """Translate a term between languages.
+
+        Args:
+            term: Term to translate.
+            source_lang: Source language code.
+            target_lang: Target language code.
+
+        Returns:
+            Translated term or None if not found.
+        """
+        if target_lang not in self._translations:
+            return None
+
+        return self._translations[target_lang].get(term.lower())
+
+
+def get_translated_keywords(
+    keywords: list[str],
+    languages: Optional[list[str]] = None,
+) -> list[str]:
+    """Get keywords translated to multiple languages.
+
+    Args:
+        keywords: List of keywords to translate.
+        languages: List of target language codes. Defaults to ["en", "zh"].
+
+    Returns:
+        List of keywords in all specified languages.
+    """
+    if languages is None:
+        languages = ["en", "zh"]
+
+    i18n = I18nManager()
+    result = []
+
+    for keyword in keywords:
+        keyword_lower = keyword.lower()
+        for lang in languages:
+            # Add original keyword
+            if lang == "en":
+                result.append(keyword)
+            else:
+                # Add translation if available
+                translated = i18n.translate(keyword_lower, "en", lang)
+                if translated:
+                    result.append(translated)
+
+    return list(set(result))  # Remove duplicates

ai_scraper/interactive.py

@@ -0,0 +1,96 @@
+"""Interactive CLI mode."""
+
+from rich.console import Console
+from rich.prompt import Prompt
+from rich.panel import Panel
+
+console = Console()
+
+
+def show_main_menu() -> str:
+    """Show main menu and get user choice.
+
+    Returns:
+        User's menu choice.
+    """
+    console.print(Panel.fit(
+        "[bold cyan]GitHub AI Scraper[/bold cyan]\n"
+        "AI Repository Discovery Tool",
+        border_style="cyan"
+    ))
+
+    console.print("\n[bold]What would you like to do?[/bold]\n")
+    console.print("  [1] Quick Scrape    - Fetch top AI repos (fast)")
+    console.print("  [2] Deep Scrape     - Comprehensive search (slow)")
+    console.print("  [3] Custom Scrape   - Set your own parameters")
+    console.print("  [4] View Results    - List scraped repositories")
+    console.print("  [5] Trending        - See trending repos")
+    console.print("  [6] Export Data     - Export to CSV/JSON")
+    console.print("  [7] Settings        - Configure options")
+    console.print("  [q] Quit\n")
+
+    return Prompt.ask("Select an option", choices=["1", "2", "3", "4", "5", "6", "7", "q"])
+
+
+def get_scrape_params() -> dict:
+    """Interactively get scrape parameters.
+
+    Returns:
+        Dictionary of scrape parameters.
+    """
+    console.print("\n[bold]Custom Scrape Configuration[/bold]\n")
+
+    # Get min_stars with validation
+    while True:
+        try:
+            min_stars = int(Prompt.ask("Minimum stars", default="100"))
+            if min_stars < 0:
+                console.print("[red]Minimum stars must be non-negative[/red]")
+                continue
+            break
+        except ValueError:
+            console.print("[red]Please enter a valid number[/red]")
+
+    # Get max_results with validation
+    while True:
+        try:
+            max_results = int(Prompt.ask("Maximum results", default="100"))
+            if max_results <= 0:
+                console.print("[red]Maximum results must be greater than 0[/red]")
+                continue
+            break
+        except ValueError:
+            console.print("[red]Please enter a valid number[/red]")
+
+    language = Prompt.ask("Language filter (leave empty for all)", default="")
+
+    return {
+        "min_stars": min_stars,
+        "max_results": max_results,
+        "language": language or None,
+    }
+
+
+def show_scrape_progress(current: int, total: int, repo_name: str):
+    """Show progress during scraping.
+
+    Args:
+        current: Current count.
+        total: Total expected.
+        repo_name: Name of current repo.
+    """
+    percent = (current / total * 100) if total > 0 else 0
+    console.print(f"  [{current}/{total}] {percent:.0f}% - {repo_name[:40]}")
+
+
+def confirm_action(message: str) -> bool:
+    """Ask for confirmation.
+
+    Args:
+        message: Confirmation message.
+
+    Returns:
+        True if confirmed.
+    """
+    from rich.prompt import Confirm
+    return Confirm.ask(message)

ai_scraper/keywords/__init__.py

@@ -0,0 +1,5 @@
+"""Keywords extraction module."""
+
+from ai_scraper.keywords.extractor import KeywordExtractor
+
+__all__ = ["KeywordExtractor"]

ai_scraper/keywords/extractor.py

@@ -0,0 +1,274 @@
+"""Keyword extraction from repository metadata."""
+
+import re
+from pathlib import Path
+
+from ai_scraper.models import Repository
+
+
+# Common English stopwords to filter out
+LOW_QUALITY_KEYWORDS = {
+    "aaif",
+}
+
+STOPWORDS: set[str] = {
+    "a", "an", "the", "and", "or", "but", "is", "are", "was", "were",
+    "be", "been", "being", "have", "has", "had", "do", "does", "did",
+    "will", "would", "could", "should", "may", "might", "must", "shall",
+    "can", "need", "dare", "ought", "used", "to", "of", "in", "for",
+    "on", "with", "at", "by", "from", "as", "into", "through", "during",
+    "before", "after", "above", "below", "between", "under", "again",
+    "further", "then", "once", "here", "there", "when", "where", "why",
+    "how", "all", "each", "few", "more", "most", "other", "some", "such",
+    "no", "nor", "not", "only", "own", "same", "so", "than", "too",
+    "very", "just", "also", "now", "that", "this", "these", "those",
+    "what", "which", "who", "whom", "whose", "if", "else", "because",
+    "while", "although", "though", "since", "until", "unless", "however",
+    "therefore", "thus", "hence", "either", "neither", "both", "not",
+    "only", "also", "even", "still", "already", "yet", "just", "only",
+    "i", "me", "my", "myself", "we", "our", "ours", "ourselves",
+    "you", "your", "yours", "yourself", "yourselves", "he", "him",
+    "his", "himself", "she", "her", "hers", "herself", "it", "its",
+    "itself", "they", "them", "their", "theirs", "themselves",
+}
+
+# 无效关键词模式（需要过滤掉）
+INVALID_PATTERNS = [
+    r'^\d+/\w+$',       # 数字/单词模式，如 "0/zero", "112/ai"
+    r'^[\w-]+/[\w-]+$', # 路径模式，如 "owner/repo"
+    r'^\d+$',           # 纯数字
+]
+
+# 最小关键词长度（AI 相关缩写例外）
+MIN_KEYWORD_LENGTH = 3
+VALID_SHORT_KEYWORDS = {
+    "ai", "ml", "dl", "nlp", "cv", "llm", "gpt", "rag", "mcp",
+    "rnn", "cnn", "gan", "vae", "rl", "cl", "asr", "tts",
+}
+
+
+class KeywordExtractor:
+    """Extract and manage keywords from repository metadata."""
+
+    def __init__(self, keywords_file: Path, max_keywords: int = 100):
+        """Initialize the extractor.
+
+        Args:
+            keywords_file: Path to file for persisting keywords.
+            max_keywords: Maximum number of keywords to keep.
+        """
+        self.keywords_file = keywords_file
+        self.max_keywords = max_keywords
+
+    def load_keywords(self) -> set[str]:
+        """Load keywords from file.
+
+        Returns:
+            Set of keywords, or empty set if file doesn't exist.
+        """
+        if not self.keywords_file.exists():
+            return set()
+
+        keywords: set[str] = set()
+        with open(self.keywords_file, "r", encoding="utf-8") as f:
+            for line in f:
+                keyword = line.strip()
+                if keyword:
+                    keywords.add(keyword)
+        return keywords
+
+    def save_keywords(self, keywords: set[str]) -> None:
+        """Save keywords to file.
+
+        Args:
+            keywords: Set of keywords to save.
+        """
+        # Ensure parent directory exists
+        self.keywords_file.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(self.keywords_file, "w", encoding="utf-8") as f:
+            for keyword in sorted(keywords):
+                f.write(f"{keyword}\n")
+
+    def extract_from_repos(self, repos: list[Repository]) -> set[str]:
+        """Extract keywords from a list of repositories.
+
+        Args:
+            repos: List of repositories to extract from.
+
+        Returns:
+            Set of extracted keywords.
+        """
+        keywords: set[str] = set()
+        for repo in repos:
+            keywords.update(self._extract_from_topics(repo))
+            keywords.update(self._extract_from_description(repo))
+            keywords.update(self._extract_from_name(repo))
+
+        # Apply quality filter
+        return self._filter_keywords(keywords)
+
+    def _filter_keywords(self, keywords: set[str]) -> set[str]:
+        """Filter out low-quality keywords.
+
+        Args:
+            keywords: Set of keywords to filter.
+
+        Returns:
+            Filtered set of high-quality keywords.
+        """
+        filtered: set[str] = set()
+
+        for keyword in keywords:
+            keyword_lower = keyword.strip().lower()
+            if not keyword_lower:
+                continue
+
+            # Skip known low-quality tokens discovered from repository noise
+            if keyword_lower in LOW_QUALITY_KEYWORDS:
+                continue
+
+            # Skip if matches invalid patterns
+            skip = False
+            for pattern in INVALID_PATTERNS:
+                if re.match(pattern, keyword_lower):
+                    skip = True
+                    break
+
+            if skip:
+                continue
+
+            # Check minimum length
+            if len(keyword_lower) < MIN_KEYWORD_LENGTH:
+                # Allow known short AI terms
+                if keyword_lower not in VALID_SHORT_KEYWORDS:
+                    continue
+
+            # Skip if it looks like a file path with extension
+            if '.' in keyword_lower and not keyword_lower.startswith('.'):
+                continue
+
+            # Skip if it's mostly numbers
+            digit_count = sum(1 for c in keyword_lower if c.isdigit())
+            if digit_count >= len(keyword_lower) * 0.5:
+                continue
+
+            filtered.add(keyword_lower)
+
+        return filtered
+
+    def _extract_from_topics(self, repo: Repository) -> set[str]:
+        """Extract keywords from repository topics.
+
+        Args:
+            repo: Repository to extract from.
+
+        Returns:
+            Set of keywords from topics (lowercase).
+        """
+        return {topic.lower() for topic in repo.topics}
+
+    def _extract_from_description(self, repo: Repository) -> set[str]:
+        """Extract keywords from repository description.
+
+        Splits on non-alphanumeric characters, filters stopwords,
+        requires min 2 chars, and excludes pure digits.
+
+        Args:
+            repo: Repository to extract from.
+
+        Returns:
+            Set of keywords from description.
+        """
+        if not repo.description:
+            return set()
+
+        keywords: set[str] = set()
+        # Split on non-alphanumeric characters
+        words = re.split(r"[^a-zA-Z0-9]+", repo.description.lower())
+
+        for word in words:
+            # Skip empty strings
+            if not word:
+                continue
+            # Skip short words (less than 2 chars)
+            if len(word) < 2:
+                continue
+            # Skip stopwords
+            if word in STOPWORDS:
+                continue
+            # Skip pure digits
+            if word.isdigit():
+                continue
+
+            keywords.add(word)
+
+        return keywords
+
+    def _extract_from_name(self, repo: Repository) -> set[str]:
+        """Extract keywords from repository name.
+
+        Splits on hyphens and underscores, filters stopwords,
+        requires min 2 chars, and excludes pure digits.
+
+        Args:
+            repo: Repository to extract from.
+
+        Returns:
+            Set of keywords from name.
+        """
+        keywords: set[str] = set()
+        # Split on hyphens and underscores
+        parts = re.split(r"[-_]+", repo.name.lower())
+
+        for part in parts:
+            # Skip empty strings
+            if not part:
+                continue
+            # Skip short parts (less than 2 chars)
+            if len(part) < 2:
+                continue
+            # Skip stopwords
+            if part in STOPWORDS:
+                continue
+            # Skip pure digits
+            if part.isdigit():
+                continue
+
+            keywords.add(part)
+
+        return keywords
+
+    def merge_keywords(
+        self, existing: set[str], new: set[str]
+    ) -> set[str]:
+        """Merge new keywords with existing, respecting max_keywords limit.
+
+        Args:
+            existing: Existing set of keywords.
+            new: New keywords to merge.
+
+        Returns:
+            Merged set of keywords, limited to max_keywords.
+        """
+        merged = existing | new
+        if len(merged) <= self.max_keywords:
+            return merged
+
+        # Prioritize existing keywords, then add new ones up to limit
+        result: set[str] = set(existing)
+        for keyword in sorted(new):
+            if keyword not in result:
+                result.add(keyword)
+                if len(result) >= self.max_keywords:
+                    break
+        return result
+
+    def get_keywords_for_search(self) -> list[str]:
+        """Get keywords as a sorted list for search queries.
+
+        Returns:
+            Sorted list of keywords.
+        """
+        keywords = self.load_keywords()
+        return sorted(keywords)