gapless-crypto-data 2.0.2__tar.gz → 2.2.0__tar.gz

{gapless_crypto_data-2.0.2 → gapless_crypto_data-2.2.0}/.github/workflows/ci-cd.yml

@@ -56,11 +56,16 @@ jobs:
         uv run ruff check .
         uv run ruff format --check .
     
+    - name: Run tests
+      run: |
+        source .venv/bin/activate
+        uv run pytest tests/ -v --tb=short
+
     - name: Test CLI entry point
       run: |
         source .venv/bin/activate
         uv run gapless-crypto-data --help
-    
+
     - name: Test package build
       run: |
         source .venv/bin/activate

gapless_crypto_data-2.2.0/.pre-commit-config.yaml

@@ -0,0 +1,63 @@
+# Pre-commit hooks for gapless-crypto-data
+# Ensures code quality and prevents broken code from being committed
+# Uses local tools only to avoid external repository authentication issues
+
+repos:
+  # All hooks use local project dependencies via uv
+  - repo: local
+    hooks:
+      # Ruff linter
+      - id: ruff-check
+        name: Ruff Linter
+        entry: uv run ruff check --fix --exit-non-zero-on-fix
+        language: system
+        types_or: [python, pyi]
+        require_serial: false
+
+      # Ruff formatter
+      - id: ruff-format
+        name: Ruff Formatter
+        entry: uv run ruff format
+        language: system
+        types_or: [python, pyi]
+        require_serial: false
+
+      # Python AST validation
+      - id: check-ast
+        name: Check Python AST
+        entry: uv run python -m py_compile
+        language: system
+        types: [python]
+
+      # File encoding validation
+      - id: validate-file-encoding
+        name: Validate file encoding (UTF-8/ASCII only)
+        entry: bash
+        language: system
+        args:
+          - -c
+          - |
+            echo "🔍 Checking file encoding..."
+            files=$(find src/ tests/ examples/ -name "*.py" -o -name "*.md" -o -name "*.txt" -o -name "*.toml" -o -name "*.yml" -o -name "*.yaml" 2>/dev/null || true)
+            if [ -n "$files" ]; then
+              non_utf8=$(echo "$files" | xargs file --mime-encoding | grep -v -E "(utf-8|us-ascii)" || true)
+              if [ -n "$non_utf8" ]; then
+                echo "❌ Non-UTF-8/ASCII files found:"
+                echo "$non_utf8"
+                echo "All text files must be UTF-8 or ASCII encoded."
+                exit 1
+              else
+                echo "✅ All project files are properly encoded (UTF-8/ASCII)"
+              fi
+            else
+              echo "✅ No text files found to check"
+            fi
+        files: \.(py|md|txt|toml|yml|yaml)$
+        require_serial: true
+
+# Pre-commit configuration
+default_install_hook_types: [pre-commit]
+default_stages: [pre-commit]
+
+# Performance settings
+minimum_pre_commit_version: "3.0.0"

{gapless_crypto_data-2.0.2 → gapless_crypto_data-2.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: gapless-crypto-data
-Version: 2.0.2
-Summary: Cryptocurrency data collection with complete Binance microstructure format - 11-column output including quote volume, trade counts, and taker buy data
+Version: 2.2.0
+Summary: Ultra-fast cryptocurrency data collection with zero gaps guarantee and full 11-column microstructure format
 Project-URL: Homepage, https://github.com/Eon-Labs/gapless-crypto-data
 Project-URL: Documentation, https://github.com/Eon-Labs/gapless-crypto-data#readme
 Project-URL: Repository, https://github.com/Eon-Labs/gapless-crypto-data.git
@@ -57,7 +57,7 @@ Ultra-fast cryptocurrency data collection with zero gaps guarantee and full 11-c
 - 🔒 **Zero gaps guarantee** through authentic API-first validation
 - ⚡ **UV-first** modern Python tooling
 - 🛡️ **Corruption-proof** atomic file operations
-- 📊 **Multi-timeframe support** (1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h)
+- 📊 **Multi-symbol & multi-timeframe support** (1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h)
 - 🔧 **Gap detection and filling** with authentic data only
 - 📈 **Production-grade** data collection for quantitative trading
 
@@ -82,11 +82,17 @@ pip install gapless-crypto-data
 ### CLI Usage
 
 ```bash
-# Collect data for multiple timeframes
+# Collect data for multiple timeframes (default output location)
 gapless-crypto-data --symbol SOLUSDT --timeframes 1m,3m,5m,15m,30m,1h,2h,4h
 
-# Collect specific date range
-gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2023-01-01 --end 2023-12-31
+# Collect multiple symbols at once (native multi-symbol support)
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT,SOLUSDT --timeframes 1h,4h
+
+# Collect specific date range with custom output directory
+gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2023-01-01 --end 2023-12-31 --output-dir ./crypto_data
+
+# Multi-symbol with custom settings
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT --timeframes 5m,1h --start 2024-01-01 --end 2024-06-30 --output-dir ./crypto_data
 
 # Fill gaps in existing data
 gapless-crypto-data --fill-gaps --directory ./data
@@ -149,12 +155,11 @@ AtomicCSVOperations → Final Gapless Dataset with Order Flow Metrics
 gapless-crypto-data [OPTIONS]
 
 Options:
-  --symbol TEXT          Trading pair symbol (e.g., SOLUSDT, BTCUSDT)
+  --symbol TEXT          Trading pair symbol(s) - single symbol or comma-separated list (e.g., SOLUSDT, BTCUSDT,ETHUSDT)
   --timeframes TEXT      Comma-separated timeframes (1m,3m,5m,15m,30m,1h,2h,4h)
   --start TEXT          Start date (YYYY-MM-DD)
   --end TEXT            End date (YYYY-MM-DD)
-  --directory TEXT      Output directory (default: ./data)
-  --workers INTEGER     Number of parallel workers (default: 4)
+  --output-dir TEXT     Output directory for CSV files (default: src/gapless_crypto_data/sample_data/)
   --help                Show this message and exit
 ```
 
@@ -174,12 +179,25 @@ Options:
 
 ### Batch Processing
 
+#### CLI Multi-Symbol (Recommended)
+
+```bash
+# Native multi-symbol support (fastest approach)
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT,SOLUSDT,ADAUSDT --timeframes 1m,5m,15m,1h,4h --start 2023-01-01 --end 2023-12-31
+
+# Alternative: Multiple separate commands for different settings
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT --timeframes 1m,1h --start 2023-01-01 --end 2023-06-30
+gapless-crypto-data --symbol SOLUSDT,ADAUSDT --timeframes 5m,4h --start 2023-07-01 --end 2023-12-31
+```
+
+#### Python API
+
 ```python
 from gapless_crypto_data import BinancePublicDataCollector
 
 collector = BinancePublicDataCollector()
 
-# Process multiple symbols
+# Process multiple symbols (loop-based for complex logic)
 symbols = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "ADAUSDT"]
 timeframes = ["1m", "5m", "15m", "1h", "4h"]
 
@@ -188,8 +206,7 @@ for symbol in symbols:
         symbol=symbol,
         timeframes=timeframes,
         start_date="2023-01-01",
-        end_date="2023-12-31",
-        workers=8  # Parallel downloads
+        end_date="2023-12-31"
     )
 ```
 
@@ -214,29 +231,103 @@ gap_filler.fill_gaps(
 
 ## 🛠️ Development
 
-### Using UV (Recommended)
+### Prerequisites
+
+- **UV Package Manager** (recommended) - [Install UV](https://docs.astral.sh/uv/getting-started/installation/)
+- **Python 3.9+** - UV will manage Python versions automatically
+- **Git** - For repository cloning and version control
 
+### Development Installation Workflow
+
+**IMPORTANT**: This project uses **mandatory pre-commit hooks** to prevent broken code from being committed. All commits are automatically validated for formatting, linting, and basic quality checks.
+
+#### Step 1: Clone Repository
 ```bash
-# Clone repository
 git clone https://github.com/Eon-Labs/gapless-crypto-data.git
 cd gapless-crypto-data
+```
 
-# Create development environment
+#### Step 2: Development Environment Setup
+```bash
+# Create isolated virtual environment
 uv venv
-source .venv/bin/activate  # On Windows: .venv\Scripts\activate
 
-# Install development dependencies
+# Activate virtual environment
+source .venv/bin/activate  # macOS/Linux
+# .venv\Scripts\activate   # Windows
+
+# Install all dependencies (production + development)
 uv sync --dev
+```
+
+#### Step 3: Verify Installation
+```bash
+# Test CLI functionality
+uv run gapless-crypto-data --help
 
-# Run tests
+# Run test suite
 uv run pytest
 
-# Code formatting and linting
+# Quick data collection test
+uv run gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2024-01-01 --end 2024-01-01 --output-dir ./test_data
+```
+
+#### Step 4: Set Up Pre-Commit Hooks (Mandatory)
+```bash
+# Install pre-commit hooks (prevents broken code from being committed)
+uv run pre-commit install
+
+# Test pre-commit hooks
+uv run pre-commit run --all-files
+```
+
+#### Step 5: Development Tools
+```bash
+# Code formatting
 uv run ruff format .
+
+# Linting and auto-fixes
 uv run ruff check --fix .
 
 # Type checking
 uv run mypy src/
+
+# Run specific tests
+uv run pytest tests/test_binance_collector.py -v
+
+# Manual pre-commit validation
+uv run pre-commit run --all-files
+```
+
+### Development Commands Reference
+
+| Task | Command |
+|------|---------|
+| Install dependencies | `uv sync --dev` |
+| Setup pre-commit hooks | `uv run pre-commit install` |
+| Add new dependency | `uv add package-name` |
+| Add dev dependency | `uv add --dev package-name` |
+| Run CLI | `uv run gapless-crypto-data [args]` |
+| Run tests | `uv run pytest` |
+| Format code | `uv run ruff format .` |
+| Lint code | `uv run ruff check --fix .` |
+| Type check | `uv run mypy src/` |
+| Validate pre-commit | `uv run pre-commit run --all-files` |
+| Build package | `uv build` |
+
+### Project Structure for Development
+```
+gapless-crypto-data/
+├── src/gapless_crypto_data/        # Main package
+│   ├── __init__.py                 # Package exports
+│   ├── cli.py                      # CLI interface
+│   ├── collectors/                 # Data collection modules
+│   └── gap_filling/                # Gap detection/filling
+├── tests/                          # Test suite
+├── docs/                           # Documentation
+├── examples/                       # Usage examples
+├── pyproject.toml                  # Project configuration
+└── uv.lock                        # Dependency lock file
 ```
 
 ### Building and Publishing

{gapless_crypto_data-2.0.2 → gapless_crypto_data-2.2.0}/README.md

@@ -14,7 +14,7 @@ Ultra-fast cryptocurrency data collection with zero gaps guarantee and full 11-c
 - 🔒 **Zero gaps guarantee** through authentic API-first validation
 - ⚡ **UV-first** modern Python tooling
 - 🛡️ **Corruption-proof** atomic file operations
-- 📊 **Multi-timeframe support** (1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h)
+- 📊 **Multi-symbol & multi-timeframe support** (1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h)
 - 🔧 **Gap detection and filling** with authentic data only
 - 📈 **Production-grade** data collection for quantitative trading
 
@@ -39,11 +39,17 @@ pip install gapless-crypto-data
 ### CLI Usage
 
 ```bash
-# Collect data for multiple timeframes
+# Collect data for multiple timeframes (default output location)
 gapless-crypto-data --symbol SOLUSDT --timeframes 1m,3m,5m,15m,30m,1h,2h,4h
 
-# Collect specific date range
-gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2023-01-01 --end 2023-12-31
+# Collect multiple symbols at once (native multi-symbol support)
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT,SOLUSDT --timeframes 1h,4h
+
+# Collect specific date range with custom output directory
+gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2023-01-01 --end 2023-12-31 --output-dir ./crypto_data
+
+# Multi-symbol with custom settings
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT --timeframes 5m,1h --start 2024-01-01 --end 2024-06-30 --output-dir ./crypto_data
 
 # Fill gaps in existing data
 gapless-crypto-data --fill-gaps --directory ./data
@@ -106,12 +112,11 @@ AtomicCSVOperations → Final Gapless Dataset with Order Flow Metrics
 gapless-crypto-data [OPTIONS]
 
 Options:
-  --symbol TEXT          Trading pair symbol (e.g., SOLUSDT, BTCUSDT)
+  --symbol TEXT          Trading pair symbol(s) - single symbol or comma-separated list (e.g., SOLUSDT, BTCUSDT,ETHUSDT)
   --timeframes TEXT      Comma-separated timeframes (1m,3m,5m,15m,30m,1h,2h,4h)
   --start TEXT          Start date (YYYY-MM-DD)
   --end TEXT            End date (YYYY-MM-DD)
-  --directory TEXT      Output directory (default: ./data)
-  --workers INTEGER     Number of parallel workers (default: 4)
+  --output-dir TEXT     Output directory for CSV files (default: src/gapless_crypto_data/sample_data/)
   --help                Show this message and exit
 ```
 
@@ -131,12 +136,25 @@ Options:
 
 ### Batch Processing
 
+#### CLI Multi-Symbol (Recommended)
+
+```bash
+# Native multi-symbol support (fastest approach)
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT,SOLUSDT,ADAUSDT --timeframes 1m,5m,15m,1h,4h --start 2023-01-01 --end 2023-12-31
+
+# Alternative: Multiple separate commands for different settings
+gapless-crypto-data --symbol BTCUSDT,ETHUSDT --timeframes 1m,1h --start 2023-01-01 --end 2023-06-30
+gapless-crypto-data --symbol SOLUSDT,ADAUSDT --timeframes 5m,4h --start 2023-07-01 --end 2023-12-31
+```
+
+#### Python API
+
 ```python
 from gapless_crypto_data import BinancePublicDataCollector
 
 collector = BinancePublicDataCollector()
 
-# Process multiple symbols
+# Process multiple symbols (loop-based for complex logic)
 symbols = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "ADAUSDT"]
 timeframes = ["1m", "5m", "15m", "1h", "4h"]
 
@@ -145,8 +163,7 @@ for symbol in symbols:
         symbol=symbol,
         timeframes=timeframes,
         start_date="2023-01-01",
-        end_date="2023-12-31",
-        workers=8  # Parallel downloads
+        end_date="2023-12-31"
     )
 ```
 
@@ -171,29 +188,103 @@ gap_filler.fill_gaps(
 
 ## 🛠️ Development
 
-### Using UV (Recommended)
+### Prerequisites
+
+- **UV Package Manager** (recommended) - [Install UV](https://docs.astral.sh/uv/getting-started/installation/)
+- **Python 3.9+** - UV will manage Python versions automatically
+- **Git** - For repository cloning and version control
 
+### Development Installation Workflow
+
+**IMPORTANT**: This project uses **mandatory pre-commit hooks** to prevent broken code from being committed. All commits are automatically validated for formatting, linting, and basic quality checks.
+
+#### Step 1: Clone Repository
 ```bash
-# Clone repository
 git clone https://github.com/Eon-Labs/gapless-crypto-data.git
 cd gapless-crypto-data
+```
 
-# Create development environment
+#### Step 2: Development Environment Setup
+```bash
+# Create isolated virtual environment
 uv venv
-source .venv/bin/activate  # On Windows: .venv\Scripts\activate
 
-# Install development dependencies
+# Activate virtual environment
+source .venv/bin/activate  # macOS/Linux
+# .venv\Scripts\activate   # Windows
+
+# Install all dependencies (production + development)
 uv sync --dev
+```
+
+#### Step 3: Verify Installation
+```bash
+# Test CLI functionality
+uv run gapless-crypto-data --help
 
-# Run tests
+# Run test suite
 uv run pytest
 
-# Code formatting and linting
+# Quick data collection test
+uv run gapless-crypto-data --symbol BTCUSDT --timeframes 1h --start 2024-01-01 --end 2024-01-01 --output-dir ./test_data
+```
+
+#### Step 4: Set Up Pre-Commit Hooks (Mandatory)
+```bash
+# Install pre-commit hooks (prevents broken code from being committed)
+uv run pre-commit install
+
+# Test pre-commit hooks
+uv run pre-commit run --all-files
+```
+
+#### Step 5: Development Tools
+```bash
+# Code formatting
 uv run ruff format .
+
+# Linting and auto-fixes
 uv run ruff check --fix .
 
 # Type checking
 uv run mypy src/
+
+# Run specific tests
+uv run pytest tests/test_binance_collector.py -v
+
+# Manual pre-commit validation
+uv run pre-commit run --all-files
+```
+
+### Development Commands Reference
+
+| Task | Command |
+|------|---------|
+| Install dependencies | `uv sync --dev` |
+| Setup pre-commit hooks | `uv run pre-commit install` |
+| Add new dependency | `uv add package-name` |
+| Add dev dependency | `uv add --dev package-name` |
+| Run CLI | `uv run gapless-crypto-data [args]` |
+| Run tests | `uv run pytest` |
+| Format code | `uv run ruff format .` |
+| Lint code | `uv run ruff check --fix .` |
+| Type check | `uv run mypy src/` |
+| Validate pre-commit | `uv run pre-commit run --all-files` |
+| Build package | `uv build` |
+
+### Project Structure for Development
+```
+gapless-crypto-data/
+├── src/gapless_crypto_data/        # Main package
+│   ├── __init__.py                 # Package exports
+│   ├── cli.py                      # CLI interface
+│   ├── collectors/                 # Data collection modules
+│   └── gap_filling/                # Gap detection/filling
+├── tests/                          # Test suite
+├── docs/                           # Documentation
+├── examples/                       # Usage examples
+├── pyproject.toml                  # Project configuration
+└── uv.lock                        # Dependency lock file
 ```
 
 ### Building and Publishing

gapless_crypto_data-2.2.0/adversarial_test_output/binance_spot_BTCUSDT-1h_20240101-20240101_0.0y.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0908f3cfeb1dc30e1248833fc32f1c873365de65ba018e55650959766f54f443
+size 3786

gapless_crypto_data-2.2.0/adversarial_test_output/binance_spot_BTCUSDT-1h_20240101-20240101_0.0y.metadata.json

@@ -0,0 +1,100 @@
+{
+  "version": "4.0.0",
+  "generator": "BinancePublicDataCollector",
+  "generation_timestamp": "2025-09-16T16:37:14.514892Z",
+  "data_source": "Binance Public Data Repository",
+  "data_source_url": "https://data.binance.vision/data/spot/monthly/klines",
+  "market_type": "spot",
+  "symbol": "BTCUSDT",
+  "timeframe": "1h",
+  "collection_method": "direct_download",
+  "target_period": {
+    "start": "2024-01-01T00:00:00",
+    "end": "2024-01-01T23:59:59",
+    "total_days": 0
+  },
+  "actual_bars": 24,
+  "date_range": {
+    "start": "2024-01-01 00:00:00",
+    "end": "2024-01-01 23:00:00"
+  },
+  "statistics": {
+    "price_min": 42180.77,
+    "price_max": 44184.1,
+    "volume_total": 27174.299030000002,
+    "volume_mean": 1132.2624595833333
+  },
+  "collection_performance": {
+    "method": "direct_download",
+    "duration": 0.965773,
+    "bars_per_second": 24.850560121270732
+  },
+  "data_integrity": {
+    "chronological_order": true,
+    "data_hash": "8a57572144bae5e8efd23d39b9c87d285ef9ba4669cc78edfe446a458239adaa",
+    "corruption_detected": false,
+    "corrupted_rows_count": 0,
+    "corruption_details": [],
+    "header_detection": {
+      "header_found": false,
+      "header_content": null,
+      "data_start_row": 0
+    }
+  },
+  "timestamp_format_analysis": {
+    "total_rows_analyzed": 744,
+    "formats_detected": {
+      "milliseconds": 744
+    },
+    "transitions_detected": 0,
+    "transition_details": [],
+    "primary_format": "milliseconds",
+    "format_consistency": true
+  },
+  "enhanced_microstructure_format": {
+    "format_version": "4.0.0",
+    "total_columns": 11,
+    "enhanced_features": [
+      "quote_asset_volume",
+      "number_of_trades",
+      "taker_buy_base_asset_volume",
+      "taker_buy_quote_asset_volume",
+      "close_time"
+    ],
+    "analysis_capabilities": [
+      "order_flow_analysis",
+      "liquidity_metrics",
+      "market_microstructure",
+      "trade_weighted_prices",
+      "institutional_data_patterns"
+    ],
+    "professional_features": true,
+    "api_format_compatibility": true
+  },
+  "gap_analysis": {
+    "analysis_performed": true,
+    "total_gaps_detected": 0,
+    "gaps_filled": 0,
+    "gaps_remaining": 0,
+    "gap_details": [],
+    "total_missing_bars": 0,
+    "gap_filling_method": "authentic_binance_api",
+    "data_completeness_score": 1.0,
+    "analysis_timestamp": "2025-09-16T16:37:14.514878Z",
+    "analysis_parameters": {
+      "timeframe": "1h",
+      "expected_interval_minutes": 60,
+      "tolerance_factor": 1.5
+    }
+  },
+  "compliance": {
+    "zero_magic_numbers": true,
+    "temporal_integrity": true,
+    "authentic_spot_data_only": true,
+    "official_binance_source": true,
+    "binance_format_transition_aware": true,
+    "supports_milliseconds_microseconds": true,
+    "full_binance_microstructure_format": true,
+    "professional_trading_ready": true
+  }
+}

gapless_crypto_data-2.2.0/adversarial_test_output/test_with_gaps.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bca2dec3542bd30316f52bd5a56713bcaab7cff4fcfc1de5edd8c044c4a0afba
+size 3764