dexscraper 0.1.1.dev0__tar.gz

dexscraper-0.1.1.dev0/.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: vincentkoc
+buy_me_a_coffee: vincentkoc

dexscraper-0.1.1.dev0/.github/workflows/ci.yml

@@ -0,0 +1,98 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build pytest pytest-cov pytest-asyncio
+        pip install -e .[dev]
+
+    - name: Run pre-commit hooks
+      run: |
+        pre-commit run --all-files
+
+    - name: Test with pytest
+      run: |
+        pytest tests/ -v --cov=dexscraper --cov-report=xml --cov-report=term-missing
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        file: ./coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+        fail_ci_if_error: false
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install bandit safety
+        pip install -e .
+
+    - name: Security check with bandit
+      run: |
+        bandit -r dexscraper/
+
+    - name: Safety check
+      run: |
+        safety check
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, security]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build package
+      run: python -m build
+
+    - name: Check package
+      run: python -m twine check dist/*
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist
+        path: dist/

dexscraper-0.1.1.dev0/.github/workflows/release.yml

@@ -0,0 +1,168 @@
+---
+# yamllint disable rule:line-length
+name: Release
+
+'on':
+  push:
+    tags:
+      - 'v*.*.*'
+
+permissions:
+  contents: write
+  discussions: write
+  id-token: write  # IMPORTANT: Required for PyPI trusted publishing
+
+jobs:
+  release:
+    name: Create Release & Deploy to PyPI
+    runs-on: ubuntu-latest
+    environment: release
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: Verify tag format
+        run: |
+          if [[ ! "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "❌ Invalid tag format. Expected: v1.2.3"
+            exit 1
+          fi
+          echo "✅ Tag format is valid: ${{ github.ref_name }}"
+
+      - name: Verify we're on the exact tag commit
+        run: |
+          echo "Current commit: $(git rev-parse HEAD)"
+          echo "Tag commit: $(git rev-parse ${{ github.ref_name }})"
+          if [[ "$(git rev-parse HEAD)" != "$(git rev-parse ${{ github.ref_name }})" ]]; then
+            echo "❌ Not on tag commit!"
+            exit 1
+          fi
+          echo "✅ On exact tag commit"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Generate changelog
+        id: changelog
+        run: |
+          echo "🔍 Generating changelog..."
+          # Get the previous tag
+          PREV_TAG=$(git describe --tags --abbrev=0 ${{ github.ref_name }}^ 2>/dev/null || echo "")
+
+          if [ -z "$PREV_TAG" ]; then
+            echo "changelog=🎉 Initial release of DexScraper Python package!" >> $GITHUB_OUTPUT
+          else
+            echo "changelog<<EOF" >> $GITHUB_OUTPUT
+            echo "Changes since ${PREV_TAG}:" >> $GITHUB_OUTPUT
+            echo "" >> $GITHUB_OUTPUT
+            git log --pretty=format:"* %s (%h)" ${PREV_TAG}..${{ github.ref_name }} >> $GITHUB_OUTPUT
+            echo "" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build package
+        run: |
+          echo "🔨 Building Python package..."
+          python -m build
+
+          echo "📦 Package contents:"
+          ls -la dist/
+
+          echo "✅ Checking package integrity..."
+          python -m twine check dist/*
+
+      - name: Deploy to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          # Trusted publishing - no password/token needed!
+          verbose: true
+          print-hash: true
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "🚀 DexScraper ${{ github.ref_name }}"
+          body: |
+            ## 👻 DexScraper Release ${{ github.ref_name }}
+
+            > ⚠️ **RESEARCH & EDUCATIONAL PURPOSE ONLY** ⚠️
+            > This project has **NO AFFILIATION** with DexScreener. Use at your own risk.
+
+            ### 🎉 What's New
+
+            ${{ steps.changelog.outputs.changelog }}
+
+            ### 📦 Installation
+
+            #### From PyPI (Recommended)
+            ```bash
+            pip install dexscraper==${{ github.ref_name }}
+            ```
+
+            #### Quick Start
+            ```bash
+            # Interactive mode with Rich UI
+            dexscraper interactive
+
+            # Simple trending pairs
+            dexscraper trending --chain solana --limit 10
+
+            # Export to file
+            dexscraper trending --output data.json --format json
+            ```
+
+            #### Python API
+            ```python
+            import asyncio
+            from dexscraper import DexScraper, Chain
+
+            async def main():
+                scraper = DexScraper()
+                batch = await scraper.extract_token_data()
+                print(f"Found {len(batch.tokens)} tokens!")
+
+            asyncio.run(main())
+            ```
+
+            ### ✨ Key Features
+
+            - 🔗 **Multi-chain support**: Solana, Ethereum, Base, BSC, Polygon, Arbitrum, Optimism
+            - 📊 **Rich CLI**: Interactive terminal interface with live updates
+            - 📈 **Multiple formats**: JSON, CSV, OHLC, MetaTrader-compatible
+            - 🛡️ **Enterprise-ready**: Rate limiting, error recovery, Cloudflare bypass
+            - 🔬 **Research focused**: Perfect for academic and educational use
+
+            ### 🔗 Links
+
+            - 📖 **Documentation**: [README.md](https://github.com/vincentkoc/dexscraper/blob/main/README.md)
+            - 🤝 **Contributing**: [CONTRIBUTING.md](https://github.com/vincentkoc/dexscraper/blob/main/CONTRIBUTING.md)
+            - 🐛 **Issues**: [GitHub Issues](https://github.com/vincentkoc/dexscraper/issues)
+            - ☕ **Support**: [Buy me a coffee](https://buymeacoffee.com/vincentkoc)
+
+            ### 🔬 Research & Educational Use Only
+
+            This software is provided for research and educational purposes only. It has no affiliation with DexScreener and should not be used for trading decisions. Always verify data independently and understand the risks of cryptocurrency markets.
+
+          files: |
+            dist/*
+          draft: false
+          prerelease: ${{ contains(github.ref_name, 'beta') || contains(github.ref_name, 'alpha') || contains(github.ref_name, 'rc') }}
+          generate_release_notes: false
+
+      - name: Post-release verification
+        run: |
+          echo "🎉 Release ${{ github.ref_name }} completed successfully!"
+          echo "📦 PyPI: https://pypi.org/project/dexscraper/${{ github.ref_name }}/"
+          echo "🚀 GitHub: https://github.com/vincentkoc/dexscraper/releases/tag/${{ github.ref_name }}"
+          echo "💡 Install with: pip install dexscraper==${{ github.ref_name }}"

dexscraper-0.1.1.dev0/.gitignore

@@ -0,0 +1,170 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# macOS
+.DS_Store
+
+# Project specific
+*.log
+data/
+output/
+*.json

dexscraper-0.1.1.dev0/.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: debug-statements
+
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        language_version: python3
+        args: [--line-length=88]
+        exclude: 'dexscraper/_version\.py'
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+        args: [--profile, black, --line-length=88]
+        exclude: 'dexscraper/_version\.py'
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.3.0
+    hooks:
+      - id: flake8
+        args: [--config=setup.cfg]
+        exclude: 'dexscraper/_version\.py'
+
+# Temporarily disabled mypy to focus on core functionality
+# - repo: https://github.com/pre-commit/mirrors-mypy
+#   rev: v1.17.1
+#   hooks:
+#     - id: mypy
+#       additional_dependencies: [types-requests]
+#       args: [--ignore-missing-imports, --no-strict-optional]
+#       exclude: '^(tests/|dexscraper/_version\.py)'
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.8.6
+    hooks:
+      - id: bandit
+        args: [-c, pyproject.toml]
+        additional_dependencies: ["bandit[toml]"]
+        exclude: '^(tests/|dexscraper/_version\.py)'

dexscraper-0.1.1.dev0/.python-version

@@ -0,0 +1 @@
+3.9

dexscraper-0.1.1.dev0/CLAUDE.md

@@ -0,0 +1,180 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This is a comprehensive Python package for real-time cryptocurrency trading data from DexScreener's WebSocket API. The project has evolved from a single-file script into a full-featured package with multi-chain support, rich CLI interface, and enterprise-ready architecture supporting multiple blockchain networks including Solana, Ethereum, Base, BSC, and more.
+
+## Commands
+
+### Package Installation
+```bash
+pip install -e .[dev]  # Development install with all dependencies
+pip install dexscraper  # PyPI installation (when published)
+```
+
+### Running the Application
+```bash
+# CLI interface
+dexscraper interactive           # Rich interactive mode
+dexscraper trending --chain solana --limit 10  # Simple trending pairs
+
+# Legacy compatibility
+python dex.py  # Still works, calls new package
+```
+
+### Testing
+```bash
+pytest tests/ -v                # Run all tests
+pytest tests/ -v --cov=dexscraper --cov-report=html  # With coverage
+pytest -m integration           # Integration tests only
+pytest -m "not slow"           # Skip slow tests
+```
+
+### Code Quality
+```bash
+black dexscraper/ tests/        # Format code
+isort dexscraper/ tests/        # Sort imports
+flake8 dexscraper/              # Lint code
+mypy dexscraper/                # Type check
+pre-commit run --all-files      # Run all pre-commit hooks
+```
+
+### Security
+```bash
+bandit -r dexscraper/           # Security scan
+safety check                    # Dependency vulnerability scan
+```
+
+## Architecture
+
+### Package Structure
+```
+dexscraper/
+├── __init__.py          # Package exports and version
+├── scraper.py           # Main DexScraper class
+├── config.py            # Configuration classes and enums
+├── models.py            # Data models (TradingPair, TokenProfile, etc.)
+├── protocol.py          # Binary protocol decoder (legacy)
+├── enhanced_protocol.py # Enhanced protocol with OHLC support
+├── cloudflare_bypass.py # Cloudflare bypass utilities
+├── cli.py               # Rich-based CLI interface
+├── logger.py            # Logging configuration
+├── utils.py             # Utility functions
+└── _version.py          # Auto-generated version (setuptools-scm)
+```
+
+### Core Components
+
+**DexScraper Class (`scraper.py`)**:
+- Multi-chain WebSocket management with automatic reconnection
+- Configurable rate limiting and retry logic
+- Binary protocol decoder for DexScreener's proprietary format
+- Support for streaming and batch processing
+- Optional Cloudflare bypass integration
+
+**Configuration System (`config.py`)**:
+- `ScrapingConfig`: Main configuration class with validation
+- `PresetConfigs`: Ready-to-use configurations for common scenarios
+- Enums for `Chain`, `Timeframe`, `RankBy`, `Order`, `DEX`
+- `Filters` class for advanced filtering options
+
+**Data Models (`models.py`)**:
+- `TradingPair`: Complete trading pair information
+- `TokenProfile`: Enhanced token metadata with social links
+- `OHLCData`: MetaTrader-compatible candlestick data
+- `ExtractedTokenBatch`: Batch processing results
+- Export methods for JSON, CSV, and custom formats
+
+**CLI Interface (`cli.py`)**:
+- Rich-based interactive terminal interface
+- Real-time data visualization with tables and progress bars
+- Export functionality to multiple formats
+- Configuration prompts and validation
+
+**Key Functions**:
+- `DexScraper.get_pairs()`: Batch pair retrieval
+- `DexScraper.stream_pairs()`: Real-time streaming
+- `decode_pair()`: Binary decoder for individual trading pairs
+- `decode_enhanced_ohlc()`: OHLC data extraction
+- `clean_string()`: String sanitization
+- `CloudflareBypass.get_session()`: Bypass utilities
+
+### Data Flow
+1. **Configuration**: ScrapingConfig defines chains, filters, and ranking options
+2. **Connection**: WebSocket connects with configurable parameters for any supported chain
+3. **Protocol Handling**: Binary messages validated and decoded using enhanced protocol
+4. **Data Processing**: Messages parsed in chunks with support for different data types:
+   - Standard trading pairs (512-byte chunks)
+   - OHLC data (candlestick format)
+   - Token profiles (metadata)
+5. **Output**: Clean, validated data in multiple formats (JSON, CSV, MetaTrader)
+6. **Streaming**: Real-time updates via async generators or batch processing
+
+### Binary Protocol Details
+- **Message Types**: Support for multiple message formats (pairs, OHLC, profiles)
+- **String Fields**: Length-prefixed (1-2 bytes length + UTF-8 data)
+- **Numeric Data**: Packed as consecutive doubles with 8-byte alignment
+- **Enhanced Protocol**: Additional support for:
+  - OHLC candlestick data
+  - Token profile metadata
+  - Social media links and descriptions
+- **Validation**: Built-in checks for string lengths, numeric ranges, and data integrity
+- **Version Handling**: Multiple protocol versions with backward compatibility
+
+### Testing Strategy
+Comprehensive test coverage across all components:
+- **Unit Tests**: Individual functions and classes
+  - `test_scraper.py`: Core scraper functionality
+  - `test_config.py`: Configuration validation
+  - `test_models.py`: Data model serialization
+  - `test_cli.py`: CLI interface testing
+- **Integration Tests**: End-to-end scenarios
+  - `test_enhanced_ohlc.py`: OHLC data processing
+  - `test_edge_cases.py`: Protocol edge cases
+- **Protocol Tests**: Binary data handling
+  - `test_decode_pair.py`: Pair decoding edge cases
+  - Malformed binary data scenarios
+  - Invalid length fields and corrupted data
+- **Quality Gates**: Pre-commit hooks ensure code quality
+  - Type checking with mypy
+  - Security scanning with bandit
+  - Code formatting with black and isort
+
+### Configuration
+- **Logging**: Configurable via `logger.py` with multiple levels
+- **Multi-Chain**: Support for 8+ blockchain networks via `Chain` enum
+- **Flexible Ranking**: Multiple ranking options (volume, trending, price change)
+- **Advanced Filtering**: Volume, liquidity, market cap, age filters
+- **Rate Limiting**: Configurable request throttling (default 4 req/sec)
+- **Retry Logic**: Exponential backoff with configurable max retries
+- **Headers**: Browser-compatible headers with optional Cloudflare bypass
+- **Preset Configs**: Ready-to-use configurations:
+  - `PresetConfigs.trending()`: Default trending pairs
+  - `PresetConfigs.high_volume()`: Volume-focused
+  - `PresetConfigs.defi_focus()`: Multi-chain DeFi tokens
+
+### Error Handling Philosophy
+Enterprise-grade defensive programming:
+- **Connection-level**: Auto-reconnect with exponential backoff, max retry limits
+- **Protocol-level**: Handle multiple message formats, version compatibility
+- **Data-level**: Comprehensive validation, NaN/Inf handling, type safety
+- **Configuration-level**: Validate all user inputs, provide helpful error messages
+- **CLI-level**: Graceful error display, recovery suggestions
+- **Security-level**: Input sanitization, dependency scanning, safe defaults
+
+### Package Management
+- **setuptools-scm**: Automatic version management from git tags
+- **pyproject.toml**: Modern Python packaging with full metadata
+- **Entry Points**: CLI command registration for `dexscraper` command
+- **Optional Dependencies**: Rich UI optional, core functionality always available
+- **Development Tools**: Complete development workflow with pre-commit hooks
+
+### CI/CD Pipeline
+- **Multi-OS Testing**: Ubuntu, Windows, macOS
+- **Multi-Python**: Python 3.7-3.12 support
+- **Quality Gates**: Linting, type checking, security scanning
+- **Coverage**: Comprehensive test coverage reporting
+- **PyPI Publishing**: Automated releases via GitHub Actions
+- **Trusted Publishing**: Secure PyPI deployment without API tokens