github-ai-scraper 0.1.2__py3-none-any.whl

ai_scraper/api_server.py

@@ -0,0 +1,196 @@
+"""REST API server for ai-scraper."""
+
+from contextlib import asynccontextmanager
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from pydantic import BaseModel
+
+from ai_scraper.auth import verify_api_key
+from ai_scraper.config import load_config
+from ai_scraper.storage.async_database import AsyncDatabase
+
+
+# Global database instance
+db: Optional[AsyncDatabase] = None
+
+# Authentication enabled flag
+_auth_enabled = False
+
+
+def set_auth_enabled(enabled: bool) -> None:
+    """Enable or disable API authentication."""
+    global _auth_enabled
+    _auth_enabled = enabled
+
+
+async def verify_auth(x_api_key: Optional[str] = Header(None)):
+    """Verify API key if authentication is enabled."""
+    if _auth_enabled and not verify_api_key(x_api_key):
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or missing API key"
+        )
+    return True
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage application lifecycle."""
+    global db
+    config = load_config()
+    db = AsyncDatabase(Path(config.database.path))
+    if Path(config.database.path).exists():
+        await db.init_db()
+    yield
+    if db:
+        await db.close()
+
+
+app = FastAPI(
+    title="GitHub AI Scraper API",
+    description="REST API for accessing scraped AI repositories",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+
+class RepositoryResponse(BaseModel):
+    """Repository API response model."""
+    id: int
+    name: str
+    full_name: str
+    description: Optional[str]
+    stars: int
+    language: Optional[str]
+    topics: list[str]
+    url: str
+
+
+class StatsResponse(BaseModel):
+    """Statistics API response model."""
+    repository_count: int
+    snapshot_count: int
+    total_stars: int
+
+
+@app.get("/api/repos", response_model=list[RepositoryResponse], dependencies=[Depends(verify_auth)])
+async def list_repositories(
+    limit: int = Query(default=20, ge=1, le=100),
+    sort: str = Query(default="stars", pattern="^(stars|updated|relevance)$"),
+    language: Optional[str] = None,
+    min_stars: Optional[int] = None,
+):
+    """List repositories with optional filters."""
+    if not db:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    repos = await db.get_all_repositories(limit=limit, sort_by=sort)
+
+    if language:
+        repos = [r for r in repos if r.language and r.language.lower() == language.lower()]
+    if min_stars:
+        repos = [r for r in repos if r.stars >= min_stars]
+
+    return [
+        RepositoryResponse(
+            id=r.id,
+            name=r.name,
+            full_name=r.full_name,
+            description=r.description,
+            stars=r.stars,
+            language=r.language,
+            topics=r.topics,
+            url=r.url,
+        )
+        for r in repos
+    ]
+
+
+@app.get("/api/repos/{repo_id}", response_model=RepositoryResponse, dependencies=[Depends(verify_auth)])
+async def get_repository(repo_id: int):
+    """Get a specific repository by ID."""
+    if not db:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    # Use direct query instead of full scan
+    repo = await db.get_repository_by_id(repo_id)
+
+    if not repo:
+        raise HTTPException(status_code=404, detail="Repository not found")
+
+    return RepositoryResponse(
+        id=repo.id,
+        name=repo.name,
+        full_name=repo.full_name,
+        description=repo.description,
+        stars=repo.stars,
+        language=repo.language,
+        topics=repo.topics,
+        url=repo.url,
+    )
+
+
+@app.get("/api/stats", response_model=StatsResponse, dependencies=[Depends(verify_auth)])
+async def get_stats():
+    """Get database statistics."""
+    if not db:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    stats = await db.get_stats()
+    return StatsResponse(**stats)
+
+
+@app.get("/api/trending", dependencies=[Depends(verify_auth)])
+async def get_trending(
+    days: int = Query(default=7, ge=1, le=30),
+    limit: int = Query(default=10, ge=1, le=50),
+):
+    """Get trending repositories."""
+    if not db:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    trends = await db.get_trending(days=days, limit=limit)
+    return [
+        {
+            "repo_id": t.repo_id,
+            "repo_name": t.repo_name,
+            "initial_stars": t.initial_stars,
+            "current_stars": t.current_stars,
+            "growth_rate": round(t.growth_rate * 100, 2),
+        }
+        for t in trends
+    ]
+
+
+@app.get("/api/search", dependencies=[Depends(verify_auth)])
+async def search_repositories(
+    q: str = Query(..., min_length=2),
+    limit: int = Query(default=20, ge=1, le=100),
+):
+    """Search repositories by name or description."""
+    if not db:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    repos = await db.search_local(query=q, limit=limit)
+    return [
+        RepositoryResponse(
+            id=r.id,
+            name=r.name,
+            full_name=r.full_name,
+            description=r.description,
+            stars=r.stars,
+            language=r.language,
+            topics=r.topics,
+            url=r.url,
+        )
+        for r in repos
+    ]
+
+
+def run_server(host: str = "0.0.0.0", port: int = 8080):
+    """Run the API server."""
+    import uvicorn
+    uvicorn.run(app, host=host, port=port)

ai_scraper/auth.py

@@ -0,0 +1,68 @@
+"""API authentication module."""
+
+import hashlib
+import os
+import secrets
+from typing import Optional
+
+
+# In-memory store for API keys (in production, use secure storage)
+_api_keys: set[str] = set()
+
+
+def create_api_key() -> str:
+    """Generate a new API key.
+
+    Returns:
+        API key string starting with 'as_'.
+    """
+    key = secrets.token_hex(16)
+    api_key = f"as_{key}"
+    _api_keys.add(api_key)
+    return api_key
+
+
+def verify_api_key(api_key: Optional[str]) -> bool:
+    """Verify an API key.
+
+    Args:
+        api_key: API key to verify.
+
+    Returns:
+        True if valid, False otherwise.
+    """
+    if not api_key:
+        return False
+    return api_key in _api_keys
+
+
+def hash_token(token: str) -> str:
+    """Hash a token for secure storage.
+
+    Args:
+        token: Token to hash.
+
+    Returns:
+        Hashed token.
+    """
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def load_api_keys_from_env() -> None:
+    """Load API keys from environment variable."""
+    env_keys = os.environ.get("AI_SCRAPER_API_KEYS", "")
+    if env_keys:
+        for key in env_keys.split(","):
+            key = key.strip()
+            if key:
+                _api_keys.add(key)
+
+
+def get_valid_api_keys() -> set[str]:
+    """Get all valid API keys (for testing)."""
+    return _api_keys.copy()
+
+
+def clear_api_keys() -> None:
+    """Clear all API keys (for testing)."""
+    _api_keys.clear()

ai_scraper/backup.py

@@ -0,0 +1,112 @@
+"""Database backup management."""
+
+import gzip
+import logging
+import shutil
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class BackupManager:
+    """Manage database backups."""
+
+    def __init__(
+        self,
+        backup_dir: Path,
+        max_backups: int = 10,
+        compress: bool = True,
+    ):
+        """Initialize backup manager.
+
+        Args:
+            backup_dir: Directory for storing backups.
+            max_backups: Maximum number of backups to keep.
+            compress: Whether to compress backups.
+        """
+        self.backup_dir = Path(backup_dir)
+        self.backup_dir.mkdir(parents=True, exist_ok=True)
+        self.max_backups = max_backups
+        self.compress = compress
+
+    def create_backup(self, db_path: Path, name: Optional[str] = None) -> Path:
+        """Create a backup of the database.
+
+        Args:
+            db_path: Path to database file.
+            name: Optional custom name for backup.
+
+        Returns:
+            Path to created backup file.
+        """
+        if not db_path.exists():
+            raise FileNotFoundError(f"Database not found: {db_path}")
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        backup_name = name or f"backup_{timestamp}"
+        backup_name += ".db.gz" if self.compress else ".db"
+
+        backup_path = self.backup_dir / backup_name
+
+        if self.compress:
+            with open(db_path, "rb") as f_in:
+                with gzip.open(backup_path, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+        else:
+            shutil.copy2(db_path, backup_path)
+
+        logger.info(f"Created backup: {backup_path}")
+
+        # Cleanup old backups
+        self._cleanup_old_backups()
+
+        return backup_path
+
+    def restore_backup(self, backup_path: Path, target_path: Path) -> None:
+        """Restore database from backup.
+
+        Args:
+            backup_path: Path to backup file.
+            target_path: Path to restore database to.
+        """
+        if not backup_path.exists():
+            raise FileNotFoundError(f"Backup not found: {backup_path}")
+
+        if backup_path.suffix == ".gz":
+            with gzip.open(backup_path, "rb") as f_in:
+                with open(target_path, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+        else:
+            shutil.copy2(backup_path, target_path)
+
+        logger.info(f"Restored backup to: {target_path}")
+
+    def list_backups(self) -> list[Path]:
+        """List all available backups.
+
+        Returns:
+            List of backup file paths, sorted by modification time (newest first).
+        """
+        backups = list(self.backup_dir.glob("backup_*.db*"))
+        backups.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+        return backups
+
+    def delete_backup(self, backup_path: Path) -> None:
+        """Delete a backup file.
+
+        Args:
+            backup_path: Path to backup file.
+        """
+        if backup_path.exists():
+            backup_path.unlink()
+            logger.info(f"Deleted backup: {backup_path}")
+
+    def _cleanup_old_backups(self) -> None:
+        """Remove old backups exceeding max_backups limit."""
+        backups = self.list_backups()
+
+        while len(backups) > self.max_backups:
+            old_backup = backups.pop()
+            self.delete_backup(old_backup)

ai_scraper/cache.py

@@ -0,0 +1,95 @@
+"""Request caching for GitHub API."""
+
+import hashlib
+import json
+import time
+from pathlib import Path
+from typing import Optional
+
+
+class RequestCache:
+    """File-based cache for API responses."""
+
+    def __init__(self, cache_dir: Path, ttl: int = 3600):
+        """Initialize cache.
+
+        Args:
+            cache_dir: Directory for cache files.
+            ttl: Time-to-live in seconds.
+        """
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.ttl = ttl
+
+    def _get_cache_key(self, url: str, params: Optional[dict] = None) -> str:
+        """Generate cache key from URL and params."""
+        key_data = url + json.dumps(params or {}, sort_keys=True)
+        return hashlib.md5(key_data.encode()).hexdigest()
+
+    def get(self, url: str, params: Optional[dict] = None) -> Optional[dict]:
+        """Get cached response.
+
+        Args:
+            url: Request URL.
+            params: Request parameters.
+
+        Returns:
+            Cached data or None.
+        """
+        key = self._get_cache_key(url, params)
+        cache_file = self.cache_dir / f"{key}.json"
+
+        if not cache_file.exists():
+            return None
+
+        try:
+            with open(cache_file, "r", encoding="utf-8") as f:
+                cached = json.load(f)
+
+            # Check TTL
+            if time.time() - cached.get("timestamp", 0) > self.ttl:
+                cache_file.unlink()
+                return None
+
+            return cached.get("data")
+        except (json.JSONDecodeError, KeyError):
+            return None
+
+    def set(self, url: str, params: Optional[dict], data: dict) -> None:
+        """Cache response.
+
+        Args:
+            url: Request URL.
+            params: Request parameters.
+            data: Response data to cache.
+        """
+        key = self._get_cache_key(url, params)
+        cache_file = self.cache_dir / f"{key}.json"
+
+        with open(cache_file, "w", encoding="utf-8") as f:
+            json.dump({
+                "timestamp": time.time(),
+                "data": data,
+            }, f)
+
+    def clear(self) -> int:
+        """Clear all cached data.
+
+        Returns:
+            Number of files deleted.
+        """
+        count = 0
+        for cache_file in self.cache_dir.glob("*.json"):
+            cache_file.unlink()
+            count += 1
+        return count
+
+    def get_stats(self) -> dict:
+        """Get cache statistics."""
+        files = list(self.cache_dir.glob("*.json"))
+        total_size = sum(f.stat().st_size for f in files)
+        return {
+            "file_count": len(files),
+            "total_size_bytes": total_size,
+            "total_size_mb": round(total_size / 1024 / 1024, 2),
+        }

ai_scraper/classifier.py

@@ -0,0 +1,135 @@
+"""Repository classification system."""
+
+from dataclasses import dataclass
+from typing import Optional
+
+from ai_scraper.models.repository import Repository
+
+
+@dataclass
+class Classification:
+    """Repository classification result."""
+    primary_category: str
+    secondary_categories: list[str]
+    confidence: float
+    tech_stack: list[str]
+    maturity: str  # experimental, production, enterprise
+
+
+# Category definitions with keywords and topics
+CATEGORIES = {
+    "llm": {
+        "keywords": ["llm", "large language model", "gpt", "claude", "llama", "mistral", "transformer"],
+        "topics": ["llm", "gpt", "language-model", "transformers"],
+    },
+    "computer-vision": {
+        "keywords": ["computer vision", "image recognition", "object detection", "segmentation", "yolo", "opencv"],
+        "topics": ["computer-vision", "object-detection", "image-segmentation", "opencv"],
+    },
+    "nlp": {
+        "keywords": ["nlp", "natural language", "text processing", "sentiment", "ner", "spacy", "nltk"],
+        "topics": ["nlp", "natural-language-processing", "text-analysis", "spacy"],
+    },
+    "ml-framework": {
+        "keywords": ["pytorch", "tensorflow", "jax", "keras", "machine learning framework"],
+        "topics": ["pytorch", "tensorflow", "jax", "keras", "deep-learning"],
+    },
+    "reinforcement-learning": {
+        "keywords": ["reinforcement learning", "rl", "q-learning", "policy gradient", "dqn", "ppo"],
+        "topics": ["reinforcement-learning", "deep-reinforcement-learning", "rl"],
+    },
+    "generative-ai": {
+        "keywords": ["generative", "diffusion", "gan", "stable diffusion", "midjourney", "image generation"],
+        "topics": ["generative-ai", "diffusion-model", "gan", "stable-diffusion"],
+    },
+    "ai-tools": {
+        "keywords": ["langchain", "llamaindex", "autogpt", "agent", "ai tool", "ai framework"],
+        "topics": ["langchain", "llamaindex", "autogpt", "ai-agent"],
+    },
+    "mlops": {
+        "keywords": ["mlops", "ml pipeline", "model deployment", "model serving", "mlflow", "kubeflow", "model registry"],
+        "topics": ["mlops", "ml-pipeline", "model-deployment", "mlflow", "kubeflow"],
+    },
+    "ai-infrastructure": {
+        "keywords": ["gpu", "cuda", "inference", "optimization", "quantization", "tensorrt", "onnx runtime"],
+        "topics": ["gpu-computing", "inference", "model-optimization", "quantization"],
+    },
+    "ai-ethics": {
+        "keywords": ["ai ethics", "bias detection", "fairness", "explainability", "interpretability", "responsible ai"],
+        "topics": ["ai-ethics", "fairness", "explainability", "responsible-ai"],
+    },
+}
+
+TECH_STACK = {
+    "pytorch": ["pytorch", "torch"],
+    "tensorflow": ["tensorflow", "tf"],
+    "jax": ["jax", "flax"],
+    "huggingface": ["huggingface", "transformers", "hugging face"],
+    "onnx": ["onnx"],
+    "openai": ["openai", "gpt-4", "gpt-3.5"],
+    "anthropic": ["anthropic", "claude"],
+}
+
+
+class RepositoryClassifier:
+    """Classify repositories into AI categories."""
+
+    def classify(self, repo: Repository) -> Classification:
+        """Classify a repository.
+
+        Args:
+            repo: Repository to classify.
+
+        Returns:
+            Classification result.
+        """
+        text = f"{repo.name} {repo.description or ''}".lower()
+        topics_lower = [t.lower() for t in repo.topics]
+
+        # Score each category
+        scores = {}
+        for category, rules in CATEGORIES.items():
+            score = 0.0
+
+            # Keyword matches
+            for kw in rules["keywords"]:
+                if kw in text:
+                    score += 0.3
+
+            # Topic matches
+            for topic in rules["topics"]:
+                if topic in topics_lower:
+                    score += 0.5
+
+            scores[category] = min(score, 1.0)
+
+        # Get primary and secondary categories
+        sorted_cats = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+        primary = sorted_cats[0][0] if sorted_cats[0][1] > 0.3 else "other"
+        secondary = [cat for cat, score in sorted_cats[1:4] if score > 0.2]
+
+        # Detect tech stack
+        tech_stack = []
+        for tech, keywords in TECH_STACK.items():
+            if any(kw in text for kw in keywords):
+                tech_stack.append(tech)
+
+        # Determine maturity
+        maturity = self._assess_maturity(repo)
+
+        return Classification(
+            primary_category=primary,
+            secondary_categories=secondary,
+            confidence=sorted_cats[0][1],
+            tech_stack=tech_stack,
+            maturity=maturity,
+        )
+
+    def _assess_maturity(self, repo: Repository) -> str:
+        """Assess repository maturity."""
+        if repo.stars >= 10000 and repo.forks and repo.forks >= 500:
+            return "enterprise"
+        elif repo.stars >= 1000:
+            return "production"
+        else:
+            return "experimental"