gapless-crypto-data 1.0.2__tar.gz → 2.0.0__tar.gz

{gapless_crypto_data-1.0.2 → gapless_crypto_data-2.0.0}/.github/workflows/publish.yml

@@ -111,7 +111,7 @@ jobs:
           path: dist/
 
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v2.1.1
+        uses: sigstore/gh-action-sigstore-python@v3.0.1
         with:
           inputs: >-
             ./dist/*.tar.gz

{gapless_crypto_data-1.0.2 → gapless_crypto_data-2.0.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: gapless-crypto-data
-Version: 1.0.2
-Summary: Ultra-fast cryptocurrency data collection with zero gaps guarantee - 22x faster than API calls via Binance public data repository
+Version: 2.0.0
+Summary: Cryptocurrency data collection with complete Binance microstructure format - 11-column output including quote volume, trade counts, and taker buy data
 Project-URL: Homepage, https://github.com/Eon-Labs/gapless-crypto-data
 Project-URL: Documentation, https://github.com/Eon-Labs/gapless-crypto-data#readme
 Project-URL: Repository, https://github.com/Eon-Labs/gapless-crypto-data.git

{gapless_crypto_data-1.0.2 → gapless_crypto_data-2.0.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "gapless-crypto-data"
-version = "1.0.2"
-description = "Ultra-fast cryptocurrency data collection with zero gaps guarantee - 22x faster than API calls via Binance public data repository"
+version = "2.0.0"
+description = "Cryptocurrency data collection with complete Binance microstructure format - 11-column output including quote volume, trade counts, and taker buy data"
 readme = "README.md"
 authors = [
     { name = "Eon Labs", email = "terry@eonlabs.ai" }

{gapless_crypto_data-1.0.2 → gapless_crypto_data-2.0.0}/src/gapless_crypto_data/collectors/binance_public_data_collector.py

@@ -283,13 +283,22 @@ class BinancePublicDataCollector:
 
                     # Filter by date range
                     if self.start_date <= dt <= self.end_date:
+                        # Enhanced processing: capture all 11 essential Binance columns for complete microstructure analysis
                         processed_row = [
-                            dt.strftime("%Y-%m-%d %H:%M:%S"),  # date
+                            dt.strftime("%Y-%m-%d %H:%M:%S"),  # date (from open_time)
                             float(row[1]),  # open
                             float(row[2]),  # high
                             float(row[3]),  # low
                             float(row[4]),  # close
-                            float(row[5]),  # volume
+                            float(row[5]),  # volume (base asset volume)
+                            # Additional microstructure columns for professional analysis
+                            datetime.utcfromtimestamp(
+                                int(row[6]) / (1000000 if len(str(int(row[6]))) >= 16 else 1000)
+                            ).strftime("%Y-%m-%d %H:%M:%S"),  # close_time
+                            float(row[7]),  # quote_asset_volume
+                            int(row[8]),  # number_of_trades
+                            float(row[9]),  # taker_buy_base_asset_volume
+                            float(row[10]),  # taker_buy_quote_asset_volume
                         ]
                         processed_data.append(processed_row)
 
@@ -472,7 +481,7 @@ class BinancePublicDataCollector:
             volumes.append(row[5])
 
         return {
-            "version": "3.0.0",
+            "version": "4.0.0",
             "generator": "BinancePublicDataCollector",
             "generation_timestamp": datetime.utcnow().isoformat() + "Z",
             "data_source": "Binance Public Data Repository",
@@ -523,6 +532,26 @@ class BinancePublicDataCollector:
                     "analysis_note": "Format analysis not available - may be legacy collection",
                 },
             ),
+            "enhanced_microstructure_format": {
+                "format_version": "4.0.0",
+                "total_columns": len(data[0]) if data else 11,
+                "enhanced_features": [
+                    "quote_asset_volume",
+                    "number_of_trades",
+                    "taker_buy_base_asset_volume",
+                    "taker_buy_quote_asset_volume",
+                    "close_time",
+                ],
+                "analysis_capabilities": [
+                    "order_flow_analysis",
+                    "liquidity_metrics",
+                    "market_microstructure",
+                    "trade_weighted_prices",
+                    "institutional_data_patterns",
+                ],
+                "professional_features": True,
+                "api_format_compatibility": True,
+            },
             "compliance": {
                 "zero_magic_numbers": True,
                 "temporal_integrity": True,
@@ -530,6 +559,8 @@ class BinancePublicDataCollector:
                 "official_binance_source": True,
                 "binance_format_transition_aware": True,
                 "supports_milliseconds_microseconds": True,
+                "full_binance_microstructure_format": True,
+                "professional_trading_ready": True,
             },
         }
 
@@ -581,9 +612,23 @@ class BinancePublicDataCollector:
             )
             f.write("#\n")
 
-            # Write CSV header and data
+            # Write enhanced CSV header and data with all microstructure columns
             writer = csv.writer(f)
-            writer.writerow(["date", "open", "high", "low", "close", "volume"])
+            writer.writerow(
+                [
+                    "date",
+                    "open",
+                    "high",
+                    "low",
+                    "close",
+                    "volume",
+                    "close_time",
+                    "quote_asset_volume",
+                    "number_of_trades",
+                    "taker_buy_base_asset_volume",
+                    "taker_buy_quote_asset_volume",
+                ]
+            )
             writer.writerows(data)
 
         # Save metadata as JSON
@@ -781,14 +826,47 @@ class BinancePublicDataCollector:
 
     def _validate_csv_structure(self, df):
         """Validate CSV has correct structure and columns."""
-        expected_columns = ["date", "open", "high", "low", "close", "volume"]
+        # Enhanced expected columns for complete microstructure data
+        expected_columns = [
+            "date",
+            "open",
+            "high",
+            "low",
+            "close",
+            "volume",
+            "close_time",
+            "quote_asset_volume",
+            "number_of_trades",
+            "taker_buy_base_asset_volume",
+            "taker_buy_quote_asset_volume",
+        ]
+
+        # Legacy format for backward compatibility
+        legacy_columns = ["date", "open", "high", "low", "close", "volume"]
+
         errors = []
         warnings = []
 
-        # Check columns exist
-        missing_columns = [col for col in expected_columns if col not in df.columns]
-        if missing_columns:
-            errors.append(f"Missing required columns: {missing_columns}")
+        # Check if it's enhanced or legacy format
+        has_enhanced_format = all(col in df.columns for col in expected_columns)
+        has_legacy_format = all(col in df.columns for col in legacy_columns)
+
+        if has_enhanced_format:
+            # Validate enhanced format
+            missing_columns = [col for col in expected_columns if col not in df.columns]
+            if missing_columns:
+                errors.append(f"Missing enhanced columns: {missing_columns}")
+        elif has_legacy_format:
+            # Legacy format detected
+            warnings.append(
+                "Legacy format detected - missing microstructure columns for advanced analysis"
+            )
+            missing_enhanced = [col for col in expected_columns if col not in df.columns]
+            warnings.append(f"Enhanced features unavailable: {missing_enhanced}")
+        else:
+            # Neither format complete
+            missing_basic = [col for col in legacy_columns if col not in df.columns]
+            errors.append(f"Missing basic required columns: {missing_basic}")
 
         extra_columns = [col for col in df.columns if col not in expected_columns]
         if extra_columns:
@@ -800,10 +878,16 @@ class BinancePublicDataCollector:
 
         return {
             "status": "VALID" if not errors else "INVALID",
+            "format_type": "enhanced"
+            if has_enhanced_format
+            else "legacy"
+            if has_legacy_format
+            else "incomplete",
             "errors": errors,
             "warnings": warnings,
             "columns_found": list(df.columns),
             "expected_columns": expected_columns,
+            "legacy_columns": legacy_columns,
         }
 
     def _validate_datetime_sequence(self, df, expected_timeframe):

gapless_crypto_data-2.0.0/src/gapless_crypto_data/sample_data/binance_spot_BTCUSDT-1h_20240101-20240101_0.0y.csv

@@ -0,0 +1,35 @@
+# Binance Spot Market Data v4.0.0
+# Generated: 2025-09-15T05:56:30.731811Z
+# Source: Binance Public Data Repository
+# Market: SPOT | Symbol: BTCUSDT | Timeframe: 1h
+# Coverage: 24 bars
+# Period: 2024-01-01 00:00:00 to 2024-01-01 23:00:00
+# Collection: direct_download in 0.4s
+# Data Hash: 8a57572144bae5e8...
+# Compliance: Zero-Magic-Numbers, Temporal-Integrity, Official-Binance-Source
+#
+date,open,high,low,close,volume,close_time,quote_asset_volume,number_of_trades,taker_buy_base_asset_volume,taker_buy_quote_asset_volume
+2024-01-01 00:00:00,42283.58,42554.57,42261.02,42475.23,1271.68108,2024-01-01 00:59:59,53957248.973789,47134,682.57581,28957416.819645
+2024-01-01 01:00:00,42475.23,42775.0,42431.65,42613.56,1196.37856,2024-01-01 01:59:59,50984893.3481416,50396,712.32227,30355645.3482764
+2024-01-01 02:00:00,42613.57,42638.41,42500.0,42581.1,685.2198,2024-01-01 02:59:59,29167377.9732375,29863,288.98864,12301017.299762
+2024-01-01 03:00:00,42581.09,42586.64,42230.08,42330.49,794.80391,2024-01-01 03:59:59,33709050.2741136,38620,356.37209,15112996.3758499
+2024-01-01 04:00:00,42330.5,42399.99,42209.46,42399.99,715.4176,2024-01-01 04:59:59,30271621.000739,36038,371.12012,15703618.9319853
+2024-01-01 05:00:00,42399.98,42406.0,42180.77,42234.01,736.53152,2024-01-01 05:59:59,31159533.4764102,32200,340.69312,14414594.8709419
+2024-01-01 06:00:00,42234.01,42424.82,42208.68,42396.69,601.3725,2024-01-01 06:59:59,25454115.8447492,32653,348.7351,14759685.2542202
+2024-01-01 07:00:00,42396.69,42500.0,42396.68,42492.46,653.67718,2024-01-01 07:59:59,27757929.5826532,27661,325.24438,13809558.5581487
+2024-01-01 08:00:00,42492.46,42556.5,42452.58,42549.99,427.19365,2024-01-01 08:59:59,18158032.0713477,25046,202.6665,8614418.4233219
+2024-01-01 09:00:00,42550.0,42699.32,42537.18,42649.69,879.62491,2024-01-01 09:59:59,37495909.1519946,39685,460.83221,19643796.8180602
+2024-01-01 10:00:00,42649.69,42749.36,42628.0,42691.1,856.81851,2024-01-01 10:59:59,36575404.2277785,32952,458.26507,19562257.2900065
+2024-01-01 11:00:00,42691.1,42762.39,42605.21,42690.2,785.16567,2024-01-01 11:59:59,33516541.8167861,39801,401.49317,17138999.1008268
+2024-01-01 12:00:00,42690.21,42780.0,42611.45,42648.38,785.52851,2024-01-01 12:59:59,33542689.8029634,35778,363.27898,15513372.6294569
+2024-01-01 13:00:00,42648.37,42748.0,42612.76,42715.54,497.2024,2024-01-01 13:59:59,21230655.2580876,29662,252.80525,10794600.800384
+2024-01-01 14:00:00,42715.54,42715.55,42580.0,42612.24,711.01152,2024-01-01 14:59:59,30312550.0545163,35688,369.47765,15750630.3780059
+2024-01-01 15:00:00,42612.24,42847.07,42586.51,42783.05,942.70163,2024-01-01 15:59:59,40272827.0639329,47628,510.78905,21818909.6236608
+2024-01-01 16:00:00,42783.05,42856.0,42664.42,42725.15,918.71056,2024-01-01 16:59:59,39292588.8194273,39979,449.47603,19222594.9405082
+2024-01-01 17:00:00,42725.15,42824.9,42704.42,42812.7,677.33678,2024-01-01 17:59:59,28962179.5203376,29871,329.97146,14108987.8934044
+2024-01-01 18:00:00,42812.71,43208.86,42812.7,43111.1,2143.30897,2024-01-01 18:59:59,92299025.1971326,93970,1193.23559,51377573.0698471
+2024-01-01 19:00:00,43111.1,43550.0,43103.64,43517.99,1947.61533,2024-01-01 19:59:59,84416763.6540515,78415,1043.766,45242850.5273276
+2024-01-01 20:00:00,43517.98,43829.99,43510.71,43687.86,2905.07295,2024-01-01 20:59:59,126914170.3804287,88205,1586.93352,69332665.633418
+2024-01-01 21:00:00,43687.85,43792.59,43480.76,43619.21,1541.59126,2024-01-01 21:59:59,67281632.034903,48994,737.28182,32173665.2965155
+2024-01-01 22:00:00,43619.22,43661.1,43361.6,43529.93,1024.36372,2024-01-01 22:59:59,44554969.9325888,39588,482.2796,20973587.992568
+2024-01-01 23:00:00,43529.94,44184.1,43529.93,44179.55,3475.97051,2024-01-01 23:59:59,152707972.5678915,114796,2063.12837,90668650.6843782

gapless_crypto_data-2.0.0/src/gapless_crypto_data/sample_data/binance_spot_BTCUSDT-1h_20240101-20240101_0.0y.metadata.json

@@ -0,0 +1,84 @@
+{
+  "version": "4.0.0",
+  "generator": "BinancePublicDataCollector",
+  "generation_timestamp": "2025-09-15T05:56:30.731811Z",
+  "data_source": "Binance Public Data Repository",
+  "data_source_url": "https://data.binance.vision/data/spot/monthly/klines",
+  "market_type": "spot",
+  "symbol": "BTCUSDT",
+  "timeframe": "1h",
+  "collection_method": "direct_download",
+  "target_period": {
+    "start": "2024-01-01T00:00:00",
+    "end": "2024-01-01T23:59:59",
+    "total_days": 0
+  },
+  "actual_bars": 24,
+  "date_range": {
+    "start": "2024-01-01 00:00:00",
+    "end": "2024-01-01 23:00:00"
+  },
+  "statistics": {
+    "price_min": 42180.77,
+    "price_max": 44184.1,
+    "volume_total": 27174.299030000002,
+    "volume_mean": 1132.2624595833333
+  },
+  "collection_performance": {
+    "method": "direct_download",
+    "duration": 0.410103,
+    "bars_per_second": 58.52188352682131
+  },
+  "data_integrity": {
+    "chronological_order": true,
+    "data_hash": "8a57572144bae5e8efd23d39b9c87d285ef9ba4669cc78edfe446a458239adaa",
+    "corruption_detected": false,
+    "corrupted_rows_count": 0,
+    "corruption_details": [],
+    "header_detection": {
+      "header_found": false,
+      "header_content": null,
+      "data_start_row": 0
+    }
+  },
+  "timestamp_format_analysis": {
+    "total_rows_analyzed": 744,
+    "formats_detected": {
+      "milliseconds": 744
+    },
+    "transitions_detected": 0,
+    "transition_details": [],
+    "primary_format": "milliseconds",
+    "format_consistency": true
+  },
+  "enhanced_microstructure_format": {
+    "format_version": "4.0.0",
+    "total_columns": 11,
+    "enhanced_features": [
+      "quote_asset_volume",
+      "number_of_trades",
+      "taker_buy_base_asset_volume",
+      "taker_buy_quote_asset_volume",
+      "close_time"
+    ],
+    "analysis_capabilities": [
+      "order_flow_analysis",
+      "liquidity_metrics",
+      "market_microstructure",
+      "trade_weighted_prices",
+      "institutional_data_patterns"
+    ],
+    "professional_features": true,
+    "api_format_compatibility": true
+  },
+  "compliance": {
+    "zero_magic_numbers": true,
+    "temporal_integrity": true,
+    "authentic_spot_data_only": true,
+    "official_binance_source": true,
+    "binance_format_transition_aware": true,
+    "supports_milliseconds_microseconds": true,
+    "full_binance_microstructure_format": true,
+    "professional_trading_ready": true
+  }
+}

{gapless_crypto_data-1.0.2 → gapless_crypto_data-2.0.0}/uv.lock

@@ -286,7 +286,7 @@ wheels = [
 
 [[package]]
 name = "gapless-crypto-data"
-version = "1.0.2"
+version = "2.0.0"
 source = { editable = "." }
 dependencies = [
     { name = "pandas" },