github-ai-scraper 0.1.2__py3-none-any.whl

ai_scraper/logging_config.py

@@ -0,0 +1,74 @@
+"""Logging configuration for ai-scraper."""
+
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Optional
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+# Shared console for consistent output
+console = Console()
+
+
+def setup_logging(
+    level: str = "INFO",
+    log_file: Optional[Path] = None,
+    json_format: bool = False,
+) -> None:
+    """Set up logging.
+
+    Args:
+        level: Log level (DEBUG, INFO, WARNING, ERROR).
+        log_file: Optional file path for logs.
+        json_format: Use JSON format for logs.
+    """
+    handlers: list[logging.Handler] = [
+        RichHandler(rich_tracebacks=True, console=console)
+    ]
+
+    if log_file:
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(log_file)
+
+        if json_format:
+
+            class JSONFormatter(logging.Formatter):
+                def format(self, record: logging.LogRecord) -> str:
+                    log_entry = {
+                        "timestamp": self.formatTime(record),
+                        "level": record.levelname,
+                        "message": record.getMessage(),
+                        "module": record.module,
+                        "function": record.funcName,
+                    }
+                    return json.dumps(log_entry)
+
+            file_handler.setFormatter(JSONFormatter())
+        else:
+            file_handler.setFormatter(
+                logging.Formatter(
+                    "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+                )
+            )
+        handlers.append(file_handler)
+
+    logging.basicConfig(
+        level=logging.getLevelName(level),
+        format="%(message)s",
+        handlers=handlers,
+    )
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get a configured logger.
+
+    Args:
+        name: Logger name.
+
+    Returns:
+        Configured logger.
+    """
+    return logging.getLogger(name)

ai_scraper/models/__init__.py

@@ -0,0 +1,5 @@
+"""Data models for ai_scraper."""
+
+from ai_scraper.models.repository import Repository, RepoSnapshot, FilterConfig, ScrapeConfig
+
+__all__ = ["Repository", "RepoSnapshot", "FilterConfig", "ScrapeConfig"]

ai_scraper/models/repository.py

@@ -0,0 +1,72 @@
+"""Repository data models."""
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Repository(BaseModel):
+    """Repository information (GitHub or GitLab)."""
+
+    id: int = Field(gt=0, description="Repository ID")
+    name: str = Field(min_length=1, max_length=200)
+    full_name: str = Field(min_length=1, max_length=200)
+    description: Optional[str] = Field(None, max_length=1000)
+    stars: int = Field(ge=0, description="Star count")
+    language: Optional[str] = Field(None, max_length=50)
+    topics: list[str] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+    pushed_at: datetime
+    url: str = Field(pattern=r"^https?://[\w\.-]+/[\w\-\.]+/[\w\-\.]+")
+    open_issues: Optional[int] = Field(None, ge=0)
+    forks: Optional[int] = Field(None, ge=0)
+    contributors: Optional[int] = Field(None, ge=0)
+
+    @field_validator("name", "full_name")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        """Validate and strip name fields."""
+        if not v.strip():
+            raise ValueError("Name cannot be empty")
+        return v.strip()
+
+    @field_validator("topics")
+    @classmethod
+    def validate_topics(cls, v: list[str]) -> list[str]:
+        """Clean and lowercase topics."""
+        return [t.strip().lower() for t in v if t.strip()]
+
+
+class RepoSnapshot(BaseModel):
+    """Repository snapshot for trend analysis."""
+
+    repo_id: int = Field(gt=0)
+    stars: int = Field(ge=0)
+    snapshot_at: datetime
+
+
+class FilterConfig(BaseModel):
+    """Filter configuration for scraping."""
+
+    keywords: list[str] = Field(default_factory=list)
+    topics: list[str] = Field(default_factory=list)
+    languages: list[str] = Field(default_factory=list)
+    exclude_keywords: list[str] = Field(default_factory=list)
+    min_stars: int = Field(default=100, ge=0)
+
+    @field_validator("keywords", "topics", "languages", "exclude_keywords")
+    @classmethod
+    def clean_string_list(cls, v: list[str]) -> list[str]:
+        """Clean string lists by stripping whitespace and removing empty strings."""
+        return [s.strip() for s in v if s.strip()]
+
+
+class ScrapeConfig(BaseModel):
+    """Scrape configuration."""
+
+    data_fields: list[str] = Field(default_factory=list)
+    max_results: int = Field(default=100, gt=0)
+    concurrency: int = Field(default=5, gt=0, le=100)
+    cache_ttl: int = Field(default=3600, ge=0)

ai_scraper/output/__init__.py

@@ -0,0 +1,6 @@
+"""Output module for generating reports."""
+
+from ai_scraper.output.html import HTMLExporter
+from ai_scraper.output.markdown import MarkdownExporter
+
+__all__ = ["HTMLExporter", "MarkdownExporter"]

ai_scraper/output/excel.py

@@ -0,0 +1,79 @@
+"""Excel exporter for generating spreadsheet reports."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from ai_scraper.models import Repository
+
+
+class ExcelExporter:
+    """Export repositories to Excel format."""
+
+    def __init__(self, output_dir: Path, filename: str = "repositories.xlsx"):
+        """Initialize the exporter.
+
+        Args:
+            output_dir: Directory for output files.
+            filename: Name of the output file.
+        """
+        self.output_dir = Path(output_dir)
+        self.filename = filename
+
+    def export_repositories(self, repos: list[Repository], title: str = "AI Repositories") -> Path:
+        """Export repositories to an Excel file.
+
+        Args:
+            repos: List of repositories to export.
+            title: Sheet title.
+
+        Returns:
+            Path to the created file.
+        """
+        try:
+            import openpyxl
+            from openpyxl.styles import Font, PatternFill, Alignment
+        except ImportError:
+            raise ImportError("openpyxl is required for Excel export. Install with: pip install openpyxl")
+
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws.title = title[:31]  # Excel sheet name limit
+
+        # Header style
+        header_font = Font(bold=True, color="FFFFFF")
+        header_fill = PatternFill(start_color="4472C4", end_color="4472C4", fill_type="solid")
+
+        # Headers
+        headers = ["Name", "Stars", "Language", "Topics", "Description", "URL", "Updated"]
+        for col, header in enumerate(headers, 1):
+            cell = ws.cell(row=1, column=col, value=header)
+            cell.font = header_font
+            cell.fill = header_fill
+            cell.alignment = Alignment(horizontal="center")
+
+        # Data rows
+        for row, repo in enumerate(repos, 2):
+            ws.cell(row=row, column=1, value=repo.full_name)
+            ws.cell(row=row, column=2, value=repo.stars)
+            ws.cell(row=row, column=3, value=repo.language or "-")
+            ws.cell(row=row, column=4, value=", ".join(repo.topics[:5]))
+            ws.cell(row=row, column=5, value=repo.description or "")
+            ws.cell(row=row, column=6, value=repo.url)
+            ws.cell(row=row, column=7, value=repo.updated_at.strftime("%Y-%m-%d") if repo.updated_at else "")
+
+        # Adjust column widths
+        ws.column_dimensions['A'].width = 30
+        ws.column_dimensions['B'].width = 10
+        ws.column_dimensions['C'].width = 12
+        ws.column_dimensions['D'].width = 25
+        ws.column_dimensions['E'].width = 50
+        ws.column_dimensions['F'].width = 40
+        ws.column_dimensions['G'].width = 12
+
+        output_path = self.output_dir / self.filename
+        wb.save(output_path)
+
+        return output_path

ai_scraper/output/html.py

@@ -0,0 +1,152 @@
+"""HTML exporter for generating web reports."""
+
+from datetime import datetime
+from html import escape as html_escape
+from pathlib import Path
+from typing import Optional
+
+from ai_scraper.models import Repository
+
+
+class HTMLExporter:
+    """Export repositories to HTML format."""
+
+    def __init__(self, output_dir: Path, filename: str = "index.html"):
+        """Initialize the exporter.
+
+        Args:
+            output_dir: Directory for output files.
+            filename: Name of the output file.
+        """
+        self.output_dir = Path(output_dir)
+        self.filename = filename
+
+    def export_repositories(self, repos: list[Repository], title: str = "AI Repositories") -> Path:
+        """Export repositories to an HTML file.
+
+        Args:
+            repos: List of repositories to export.
+            title: Page title.
+
+        Returns:
+            Path to the created file.
+        """
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        content = self._generate_html(repos, title)
+
+        output_path = self.output_dir / self.filename
+        output_path.write_text(content, encoding="utf-8")
+
+        return output_path
+
+    def _generate_html(self, repos: list[Repository], title: str) -> str:
+        """Generate full HTML content."""
+        # Escape title to prevent XSS
+        safe_title = html_escape(title)
+
+        return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{safe_title}</title>
+    <style>
+        * {{ margin: 0; padding: 0; box-sizing: border-box; }}
+        body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #f6f8fa; }}
+        .container {{ max-width: 1200px; margin: 0 auto; padding: 20px; }}
+        h1 {{ color: #24292f; margin-bottom: 10px; }}
+        .meta {{ color: #57606a; margin-bottom: 20px; }}
+        .stats {{ display: flex; gap: 20px; margin-bottom: 30px; flex-wrap: wrap; }}
+        .stat-box {{ background: white; padding: 15px 20px; border-radius: 8px; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}
+        .stat-number {{ font-size: 24px; font-weight: bold; color: #0969da; }}
+        .stat-label {{ color: #57606a; font-size: 14px; }}
+        table {{ width: 100%; background: white; border-radius: 8px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}
+        th {{ background: #f6f8fa; padding: 12px 16px; text-align: left; font-weight: 600; color: #24292f; }}
+        td {{ padding: 12px 16px; border-top: 1px solid #d0d7de; }}
+        tr:hover {{ background: #f6f8fa; }}
+        a {{ color: #0969da; text-decoration: none; }}
+        a:hover {{ text-decoration: underline; }}
+        .stars {{ color: #e3b341; }}
+        .language {{ background: #ddf4ff; color: #0969da; padding: 2px 8px; border-radius: 12px; font-size: 12px; }}
+        .description {{ color: #57606a; }}
+        @media (max-width: 768px) {{
+            .container {{ padding: 10px; }}
+            table {{ font-size: 14px; }}
+            th, td {{ padding: 8px; }}
+        }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>{safe_title}</h1>
+        <p class="meta">Updated: {datetime.now().strftime("%Y-%m-%d %H:%M")} | Total: {len(repos)} repositories</p>
+
+        <div class="stats">
+            <div class="stat-box">
+                <div class="stat-number">{len(repos)}</div>
+                <div class="stat-label">Repositories</div>
+            </div>
+            <div class="stat-box">
+                <div class="stat-number">{sum(r.stars for r in repos):,}</div>
+                <div class="stat-label">Total Stars</div>
+            </div>
+        </div>
+
+        <table>
+            <thead>
+                <tr>
+                    <th>Name</th>
+                    <th>Stars</th>
+                    <th>Language</th>
+                    <th>Description</th>
+                </tr>
+            </thead>
+            <tbody>
+{self._generate_rows(repos)}
+            </tbody>
+        </table>
+    </div>
+</body>
+</html>"""
+
+    def _generate_rows(self, repos: list[Repository]) -> str:
+        """Generate table rows."""
+        rows = []
+        for repo in repos:
+            # Escape all user-controlled data to prevent XSS
+            safe_url = self._safe_url(repo.url)
+            safe_name = html_escape(repo.full_name, quote=True)
+            safe_language = html_escape(repo.language or "-", quote=True)
+            safe_description = self._clean_description(repo.description)
+
+            rows.append(f"""                <tr>
+                    <td><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_name}</a></td>
+                    <td><span class="stars">★ {repo.stars:,}</span></td>
+                    <td><span class="language">{safe_language}</span></td>
+                    <td class="description">{safe_description}</td>
+                </tr>""")
+        return "\n".join(rows)
+
+    def _safe_url(self, url: str) -> str:
+        """Validate and escape URL to prevent XSS.
+
+        Only allows http:// and https:// URLs.
+        Returns '#' for invalid URLs.
+        """
+        safe_url = html_escape(url, quote=True)
+        if safe_url.startswith(('http://', 'https://')):
+            return safe_url
+        return '#'
+
+    def _clean_description(self, description: Optional[str]) -> str:
+        """Clean description for HTML.
+
+        Escapes HTML characters and truncates to 100 chars with ellipsis.
+        """
+        if not description:
+            return ""
+        escaped = html_escape(description, quote=True)
+        if len(escaped) > 100:
+            return escaped[:97] + "..."
+        return escaped