github-ai-scraper 0.1.2__tar.gz

github_ai_scraper-0.1.2/.dockerignore

@@ -0,0 +1,17 @@
+.git
+.gitignore
+__pycache__
+*.pyc
+*.pyo
+*.egg-info
+.pytest_cache
+.coverage
+htmlcov
+.venv
+venv
+.env
+*.db
+*.json
+data/
+output/
+.cache/

github_ai_scraper-0.1.2/.github/workflows/ci.yml

@@ -0,0 +1,83 @@
+name: CI
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+    branches: [master, main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest tests/ -v --cov=src/ai_scraper --cov-report=xml
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v4
+        with:
+          files: ./coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install linters
+        run: pip install ruff mypy
+
+      - name: Run Ruff
+        run: ruff check src/ tests/
+
+      - name: Run MyPy
+        run: mypy src/ --ignore-missing-imports
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, lint]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/

github_ai_scraper-0.1.2/.github/workflows/release.yml

@@ -0,0 +1,34 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Check package metadata
+        run: twine check dist/*
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: twine upload --verbose dist/*

github_ai_scraper-0.1.2/.gitignore

@@ -0,0 +1,59 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+ENV/
+env/
+.venv/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# Project specific
+data/*.db
+data/*.sqlite
+output.txt
+.claude/
+
+# Go
+cmd/scheduler/scheduler
+cmd/scheduler/scheduler.exe
+
+# Generated files
+keywords.txt
+output/*.json
+output/*.csv
+output/*.html
+output/*.db
+output/*.sqlite

github_ai_scraper-0.1.2/CLAUDE.md

@@ -0,0 +1,108 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Language Requirement
+
+**任何情况下，都使用中文进行交流和回复。** 无论用户使用何种语言提问，都必须使用中文回答。
+
+## Project Overview
+
+GitHub/GitLab AI Scraper is a CLI tool for scraping AI-related high-star repositories from GitHub and GitLab. It supports multi-platform scraping, dynamic keyword extraction, multiple export formats, and a REST API server.
+
+## Common Commands
+
+### Development Setup
+```bash
+pip install -e ".[dev]"   # Install with dev dependencies
+```
+
+### Testing
+```bash
+pytest tests/ -v          # Run all tests
+pytest tests/test_models.py -v  # Run single test file
+pytest -k "repository" -v      # Run tests matching pattern
+```
+
+### CLI Usage
+```bash
+ai-scraper scrape                    # Scrape from GitHub (default)
+ai-scraper scrape --platform gitlab  # Scrape from GitLab
+ai-scraper scrape --gitlab-url https://your-gitlab.com/api/v4  # Self-hosted GitLab
+ai-scraper scrape --incremental      # Only updated repos
+ai-scraper scrape --since 7d         # Repos updated in last 7 days
+ai-scraper scrape --progress         # Show progress bar
+ai-scraper list                      # List scraped repos
+ai-scraper trending                  # Show trending repos
+ai-scraper serve --port 8080         # Start REST API server
+ai-scraper keywords list             # List keywords
+ai-scraper db export --format markdown --output repositories.md
+ai-scraper db clean --invalid        # Remove invalid repos
+ai-scraper db clean --vacuum         # Optimize database
+ai-scraper config show               # Show current config
+```
+
+### Environment Variables
+```bash
+export GITHUB_TOKEN=your_token  # Increases rate limit to 5000/hour
+export GITLAB_TOKEN=your_token  # For GitLab scraping
+```
+
+## Architecture
+
+### Core Data Flow
+1. **API Clients** (`src/ai_scraper/api/`) - Async HTTP clients for GitHub/GitLab with rate limiting and caching
+2. **Filter** (`src/ai_scraper/filters/ai_filter.py`) - Determines if repos are AI-related and scores relevance
+3. **Classifier** (`src/ai_scraper/classifier.py`) - Categorizes repos into AI subdomains (LLM, CV, NLP, MLOps, etc.)
+4. **Database** (`src/ai_scraper/storage/database.py`) - SQLite storage with snapshot-based trend tracking
+5. **Output** (`src/ai_scraper/output/`) - Exporters for Markdown, HTML, Excel, RSS formats
+
+### Key Components
+
+**Repository Model** (`src/ai_scraper/models/repository.py`)
+- Pydantic models: `Repository`, `RepoSnapshot`, `FilterConfig`, `ScrapeConfig`
+- URL pattern supports both GitHub and GitLab: `^https?://[\w\.-]+/[\w\-\.]+/[\w\-\.]+`
+
+**API Clients** (`src/ai_scraper/api/`)
+- `GitHubClient` and `GitLabClient` share similar async patterns
+- Both use `RateLimiter` (token bucket) and optional `RequestCache`
+- Connection pooling via `aiohttp.TCPConnector`
+
+**Keyword Extraction** (`src/ai_scraper/keywords/extractor.py`)
+- Extracts keywords from repo topics, descriptions, and names
+- Filters stopwords, numeric/path/file-like noise, and low-quality keywords while preserving short AI terms like `ai` and `ml`
+- Merges with existing keywords respecting `max_keywords` limit
+
+**Configuration** (`src/ai_scraper/config.py`)
+- YAML-based config loaded from `ai-scraper.yaml`
+- Environment variable substitution: `${GITHUB_TOKEN}` syntax
+- Separate configs for GitHub, GitLab, filter, scrape, database, keywords, output, webhooks
+
+### CLI Structure (`src/ai_scraper/cli.py`)
+- Uses Click with grouped commands: `scrape`, `list`, `trending`, `serve`, `schedule`, `interactive`
+- Sub-groups: `config`, `db`, `keywords`
+- Progress bar via Rich library
+- Windows UTF-8 handling with `clean_text()` function
+
+## Platform Support
+
+Both GitHub and GitLab are supported with platform-specific clients:
+- GitHub: `GitHubClient` with search query syntax `stars:>100 topic:ai pushed:>YYYY-MM-DD`
+- GitLab: `GitLabClient` with simpler search, configurable `base_url` for self-hosted instances
+
+## Export Formats
+
+Located in `src/ai_scraper/output/`:
+- `markdown.py` - Markdown with Chinese translation, category grouping, language icons
+- `html.py` - HTML with responsive styling
+- `excel.py` - Excel workbook format
+- `rss.py` - RSS feed format
+- `translator.py` - Description translation support
+
+## Database Schema
+
+SQLite tables in `data/ai_scraper.db`:
+- `repositories` - Main repo data with relevance scores, timestamps
+- `snapshots` - Star count snapshots for trend analysis
+
+Indexes on: stars, last_updated_at, language, created_at, relevance_score, snapshot_at

github_ai_scraper-0.1.2/Dockerfile

@@ -0,0 +1,42 @@
+# Build stage for Go scheduler (optional)
+FROM golang:1.21-alpine AS go-builder
+
+WORKDIR /build
+COPY go.mod ./ 2>/dev/null || true
+RUN if [ -f go.mod ]; then go mod download; fi
+
+COPY cmds/scheduler/ ./ 2>/dev/null || true
+RUN if [ -f main.go ]; then CGO_ENABLED=0 go build -o scheduler .; fi
+
+# Main stage
+FROM python:3.11-slim
+
+# Install SQLite
+RUN apt-get update && apt-get install -y \
+    sqlite3 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY pyproject.toml ./
+RUN pip install --no-cache-dir -e .
+
+# Copy application code
+COPY src/ ./src/
+COPY ai-scraper.yaml ./
+
+# Copy Go scheduler binary if built
+COPY --from=go-builder /build/scheduler /usr/local/bin/scheduler 2>/dev/null || true
+
+# Create data and output directories
+RUN mkdir -p /app/data /app/output /app/.cache
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV AI_SCRAPER_DATA_DIR=/app/data
+ENV AI_SCRAPER_OUTPUT_DIR=/app/output
+
+# Entry point
+ENTRYPOINT ["python", "-m", "ai_scraper.cli"]
+CMD ["--help"]

github_ai_scraper-0.1.2/PKG-INFO

@@ -0,0 +1,299 @@
+Metadata-Version: 2.4
+Name: github-ai-scraper
+Version: 0.1.2
+Summary: A CLI tool for discovering and scraping AI-related high-star repositories from GitHub and GitLab
+Project-URL: Homepage, https://github.com/lwx66615/github-ai-scraper
+Project-URL: Repository, https://github.com/lwx66615/github-ai-scraper
+Project-URL: Documentation, https://github.com/lwx66615/github-ai-scraper#readme
+Project-URL: Issues, https://github.com/lwx66615/github-ai-scraper/issues
+Project-URL: Changelog, https://github.com/lwx66615/github-ai-scraper/releases
+Author: lwx66615
+License-Expression: MIT
+Keywords: ai,cli,github,gitlab,machine-learning,repositories,scraper,trending
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: aiosqlite>=0.19.0
+Requires-Dist: click>=8.1.0
+Requires-Dist: croniter>=2.0.0
+Requires-Dist: fastapi>=0.109.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: uvicorn>=0.27.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# GitHub/GitLab AI Scraper
+
+English | [简体中文](README_CN.md)
+
+A CLI tool for scraping AI-related high-star repositories from GitHub and GitLab.
+
+## Features
+
+- **Multi-platform support** - Scrape from GitHub or GitLab (including self-hosted instances)
+- Search and filter AI-related repositories by keywords and topics
+- **Dynamic keyword extraction** - Automatically learns new keywords from scraped repos
+- **Markdown/HTML/Excel/RSS report generation** - Multiple export formats with Chinese translation
+- **Incremental scraping** - Fetch only updated repos with `--since` flag
+- **Resume support** - Continue interrupted scrapes with progress tracking
+- **Progress bar display** - Visual progress during scraping
+- **Interactive CLI mode** - Menu-driven interface for easy use
+- **Concurrent scraping** - Parallel requests for faster results
+- **Multi-language search** - Support for Chinese and English keywords
+- Local SQLite storage with trend analysis
+- Configurable filtering and scraping options
+- Rate limiting with GitHub/GitLab API token support
+- Export to CSV/JSON/HTML/Excel/RSS/Markdown formats
+- **REST API server** - Access data via HTTP endpoints with optional authentication
+- **Scheduled scraping** - Cron-based periodic scraping
+- **Webhook notifications** - Notify external services on events
+- **Plugin system** - Extend functionality with custom plugins
+- **Repository health assessment** - Activity, popularity, maintenance scores
+- **Intelligent classification** - LLM, CV, NLP, MLOps, AI Infrastructure categories
+- **Deduplication** - Fork and mirror detection, content similarity
+- **Secure token storage** - Encrypted storage for sensitive tokens
+- **Database backup** - Automatic backup and restore functionality
+- **Error recovery** - Retry logic with exponential backoff
+
+## Installation
+
+```bash
+# Install from PyPI
+pip install github-ai-scraper
+
+# Or install from source for development
+pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```bash
+# Set your GitHub token (optional, increases rate limit)
+export GITHUB_TOKEN=your_token_here
+
+# Scrape AI repositories from GitHub (default)
+ai-scraper scrape
+
+# Scrape from GitLab
+ai-scraper scrape --platform gitlab
+
+# Scrape from self-hosted GitLab
+ai-scraper scrape --platform gitlab --gitlab-url https://your-gitlab.com/api/v4
+
+# Scrape with progress bar
+ai-scraper scrape --progress
+
+# Concurrent scraping (faster)
+ai-scraper scrape --concurrent
+
+# Incremental scraping (repos updated in last 7 days)
+ai-scraper scrape --incremental
+ai-scraper scrape --since 7d
+
+# Resume interrupted scrape
+ai-scraper scrape --resume
+
+# Interactive mode
+ai-scraper interactive
+
+# List scraped repositories
+ai-scraper list
+
+# Show trending repositories
+ai-scraper trending
+
+# Export data
+ai-scraper db export --format html --output index.html
+ai-scraper db export --format xlsx --output repos.xlsx
+ai-scraper db export --format rss --output feed.xml
+ai-scraper db export --format markdown --output repositories.md
+
+# Start REST API server (with authentication)
+ai-scraper serve --port 8080 --auth
+
+# Schedule periodic scraping (daily at 9am)
+ai-scraper schedule --cron "0 9 * * *"
+
+# Backup database
+ai-scraper db backup
+ai-scraper db restore backup_file.db.gz
+```
+
+## Configuration
+
+Create `ai-scraper.yaml` to customize:
+
+```yaml
+github:
+  token: ${GITHUB_TOKEN}
+  cache_ttl: 3600
+
+gitlab:
+  token: ${GITLAB_TOKEN}  # Optional, for GitLab scraping
+  base_url: https://gitlab.com/api/v4  # Or your self-hosted GitLab URL
+  cache_ttl: 3600
+
+filter:
+  min_stars: 100
+  keywords:
+    - ai
+    - machine-learning
+    - 人工智能  # Chinese keyword support
+  topics:
+    - ai
+    - deep-learning
+
+scrape:
+  max_results: 500
+  concurrency: 5
+  concurrent_requests: 5
+
+database:
+  path: ./data/ai_scraper.db
+  backup_dir: ./backups
+  max_backups: 10
+
+api:
+  auth_enabled: true
+  api_keys:
+    - as_your_api_key_here
+
+webhooks:
+  enabled: false
+  endpoints:
+    - url: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
+      events: [scrape_complete, trending_found]
+```
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `ai-scraper scrape` | Scrape AI repositories from GitHub |
+| `ai-scraper scrape --platform gitlab` | Scrape from GitLab |
+| `ai-scraper scrape --platform gitlab --gitlab-url URL` | Scrape from self-hosted GitLab |
+| `ai-scraper scrape --concurrent` | Concurrent scraping for faster results |
+| `ai-scraper scrape --incremental` | Incremental scraping (only updated repos) |
+| `ai-scraper scrape --since 7d` | Fetch repos updated in last 7 days |
+| `ai-scraper scrape --resume` | Resume interrupted scrape |
+| `ai-scraper scrape --progress` | Show progress bar during scraping |
+| `ai-scraper interactive` | Start interactive menu mode |
+| `ai-scraper list` | List scraped repositories |
+| `ai-scraper trending` | Show trending repositories by star growth |
+| `ai-scraper serve` | Start REST API server |
+| `ai-scraper serve --auth` | Start API server with authentication |
+| `ai-scraper schedule` | Schedule periodic scraping |
+| `ai-scraper keywords list` | List all keywords |
+| `ai-scraper keywords extract` | Extract keywords from database |
+| `ai-scraper keywords clear` | Clear keywords |
+| `ai-scraper config init` | Initialize config file |
+| `ai-scraper config show` | Show current config |
+| `ai-scraper db stats` | Show database statistics |
+| `ai-scraper db export` | Export data to CSV/JSON/HTML/Excel/RSS |
+| `ai-scraper db clean --invalid` | Remove repositories with invalid data |
+| `ai-scraper db clean --vacuum` | Optimize database size |
+| `ai-scraper db backup` | Create database backup |
+| `ai-scraper db restore` | Restore from backup |
+| `ai-scraper db backups` | List available backups |
+
+## REST API Endpoints
+
+When running `ai-scraper serve`:
+
+| Endpoint | Description |
+|----------|-------------|
+| `GET /api/repos` | List repositories with filters |
+| `GET /api/repos/{id}` | Get specific repository |
+| `GET /api/stats` | Get database statistics |
+| `GET /api/trending` | Get trending repositories |
+| `GET /api/search?q=...` | Search repositories |
+
+Authentication: Pass `X-API-Key` header when `--auth` is enabled.
+
+## Project Structure
+
+```
+github-ai-scraper/
+├── src/ai_scraper/
+│   ├── cli.py              # CLI entry point
+│   ├── config.py           # Configuration management
+│   ├── interactive.py      # Interactive menu mode
+│   ├── classifier.py       # Repository classification
+│   ├── dedup.py            # Deduplication utilities
+│   ├── health.py           # Health assessment
+│   ├── scheduler.py        # Task scheduling
+│   ├── webhooks.py         # Webhook notifications
+│   ├── plugins.py          # Plugin system
+│   ├── logging_config.py   # Logging configuration
+│   ├── api_server.py       # REST API server
+│   ├── auth.py             # API authentication
+│   ├── retry.py            # Error recovery
+│   ├── i18n.py             # Multi-language support
+│   ├── scrape_progress.py  # Resume support
+│   ├── backup.py           # Database backup
+│   ├── config_watcher.py   # Config hot reload
+│   ├── secure_storage.py   # Token encryption
+│   ├── api/
+│   │   ├── github.py       # GitHub API client
+│   │   └── rate_limiter.py # Token bucket rate limiter
+│   ├── models/
+│   │   └── repository.py   # Data models (Pydantic)
+│   ├── filters/
+│   │   └── ai_filter.py    # AI relevance filter
+│   ├── output/
+│   │   ├── markdown.py     # Markdown exporter
+│   │   ├── html.py         # HTML exporter
+│   │   ├── excel.py        # Excel exporter
+│   │   └── rss.py          # RSS exporter
+│   └── storage/
+│       ├── database.py     # SQLite storage (sync)
+│       └── async_database.py # SQLite storage (async)
+├── plugins/                # Example plugins
+├── tests/                  # Test suite
+├── Dockerfile              # Docker support
+├── docker-compose.yml      # Docker compose
+├── .github/workflows/      # CI/CD workflows
+└── ai-scraper.yaml         # Default configuration
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest tests/ -v
+
+# Build Docker image
+docker build -t ai-scraper .
+```
+
+## API Rate Limits
+
+- Without token: 60 requests/hour
+- With token: 5000 requests/hour
+
+Set `GITHUB_TOKEN` environment variable for higher limits.
+
+## License
+
+MIT