gapless-crypto-clickhouse 7.1.0__py3-none-any.whl

gapless_crypto_clickhouse/__init__.py

@@ -0,0 +1,147 @@
+"""
+Gapless Crypto ClickHouse v5.0.0 - ClickHouse-based cryptocurrency data collection with zero-gap guarantee
+
+Market Compatibility:
+- USDT SPOT PAIRS (BTCUSDT, ETHUSDT, SOLUSDT, etc.)
+- USDT-margined PERPETUAL FUTURES (BTCUSDT perps, ETHUSDT perps, etc.)
+- Instrument type distinction via `instrument_type` column ('spot' or 'futures')
+- NO delivery futures, NO coin-margined futures
+
+Core Features:
+- Data collection via Binance public data repository (22x performance vs API calls)
+- Full 11-column microstructure format with order flow and liquidity metrics
+- Zero gaps guarantee through authentic API-first validation
+- UV-based Python tooling
+- Atomic file operations
+- Complete 13-timeframe support (1s, 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d)
+- Ultra-high frequency (1s) to daily (1d) data collection
+- Intelligent monthly-to-daily fallback for seamless recent data access
+- Gap detection and filling with authentic data only
+- CCXT-compatible dual parameter support (timeframe/interval)
+- Backward compatibility with 5-year deprecation period
+
+Data Source:
+    Binance Spot Market: https://data.binance.vision/data/spot/monthly/klines/
+    Market Type: SPOT only (no futures/derivatives)
+    Supported Pairs: USDT-quoted spot pairs exclusively
+
+Usage:
+    # Function-based API
+    import gapless_crypto_clickhouse as gcd
+
+    # Fetch recent data as standard pandas DataFrame
+    df = gcd.fetch_data("BTCUSDT", timeframe="1h", limit=1000)
+
+    # Standard pandas operations for analysis
+    returns = df['close'].pct_change()                     # Returns calculation
+    rolling_vol = df['close'].rolling(20).std()            # Rolling volatility
+    max_drawdown = (df['close'] / df['close'].cummax() - 1).min()  # Drawdown
+
+    # Resampling with pandas
+    df_resampled = df.set_index('date').resample('4H').agg({
+        'open': 'first', 'high': 'max', 'low': 'min',
+        'close': 'last', 'volume': 'sum'
+    })
+
+    # Backward compatibility (legacy interval parameter)
+    df = gcd.fetch_data("BTCUSDT", interval="1h", limit=1000)  # DeprecationWarning
+
+    # Download with date range
+    df = gcd.download("ETHUSDT", timeframe="4h", start="2024-01-01", end="2024-06-30")
+
+    # Get available symbols and timeframes
+    symbols = gcd.get_supported_symbols()
+    timeframes = gcd.get_supported_timeframes()
+
+    # Fill gaps in existing data
+    results = gcd.fill_gaps("./data")
+
+    # Class-based API (for complex workflows)
+    from gapless_crypto_clickhouse import BinancePublicDataCollector, UniversalGapFiller
+
+    collector = BinancePublicDataCollector()
+    result = collector.collect_timeframe_data("1h")
+    df = result["dataframe"]
+
+Package Relationship:
+    This package is a fork of gapless-crypto-data focused on ClickHouse database workflows.
+
+    For file-based workflows (CSV/Parquet only):
+        See https://pypi.org/project/gapless-crypto-data/
+
+    Migrating from gapless-crypto-data:
+        See docs/development/CLI_MIGRATION_GUIDE.md for migration guide.
+        Note: This package never had a CLI (Python API only).
+
+Supported Symbols (713 perpetual symbols - Spot + Futures Aligned):
+    Both spot and futures-um support 713 validated perpetual symbols
+    sourced from binance-futures-availability package (95%+ SLA, daily S3 Vision probes).
+
+    See get_supported_symbols() for complete list.
+    Major pairs: BTCUSDT, ETHUSDT, BNBUSDT, SOLUSDT, XRPUSDT, and 708 more.
+"""
+
+__version__ = "1.0.0"
+__author__ = "Eon Labs"
+__email__ = "terry@eonlabs.com"
+
+# Core classes (advanced/power-user API)
+# Enhanced DataFrame for domain-specific operations
+# Convenience functions (simple/intuitive API)
+# API-only probe hooks for AI coding agents
+from . import __probe__, probe
+from .api import (
+    InstrumentType,  # ADR-0021: Type alias for instrument type hints
+    download,
+    download_multiple,
+    fetch_data,
+    fill_gaps,
+    get_info,
+    get_supported_intervals,
+    get_supported_symbols,
+    get_supported_timeframes,
+    load_parquet,
+    save_parquet,
+)
+from .collectors.binance_public_data_collector import BinancePublicDataCollector
+from .exceptions import (
+    DataCollectionError,
+    GapFillingError,
+    GaplessCryptoDataError,
+    NetworkError,
+    ValidationError,
+)
+from .gap_filling.safe_file_operations import AtomicCSVOperations, SafeCSVMerger
+from .gap_filling.universal_gap_filler import UniversalGapFiller
+from .query_api import query_ohlcv  # v6.0.0: Unified query API with auto-ingestion (ADR-0023)
+
+__all__ = [
+    # Simple function-based API (recommended for most users)
+    "query_ohlcv",  # v6.0.0: Unified query API with auto-ingestion (ADR-0023)
+    "fetch_data",
+    "download",
+    "download_multiple",
+    "get_supported_symbols",
+    "get_supported_timeframes",
+    "get_supported_intervals",  # Legacy compatibility
+    "fill_gaps",
+    "get_info",
+    "save_parquet",
+    "load_parquet",
+    # Type aliases (v3.2.0 - ADR-0021)
+    "InstrumentType",  # Literal["spot", "futures-um"]
+    # Advanced class-based API (for complex workflows)
+    "BinancePublicDataCollector",
+    "UniversalGapFiller",
+    "AtomicCSVOperations",
+    "SafeCSVMerger",
+    # Structured exception hierarchy (v3.2.0)
+    "GaplessCryptoDataError",
+    "DataCollectionError",
+    "ValidationError",
+    "NetworkError",
+    "GapFillingError",
+    # AI agent probe hooks (v6.0.0)
+    "__probe__",
+    "probe",
+]

gapless_crypto_clickhouse/__probe__.py

@@ -0,0 +1,349 @@
+"""
+__probe__.py - API-only probe hooks for AI coding agents
+
+Provides deterministic JSON output for effortless AI agent discovery:
+- API map discovery
+- Capabilities detection
+- Task graph generation for phased spawning
+- Zero-file terminal probing
+
+Usage:
+    import gapless_crypto_clickhouse
+    probe = gapless_crypto_clickhouse.__probe__
+
+    # Initial discovery
+    api_map = probe.discover_api()
+    capabilities = probe.get_capabilities()
+
+    # Phased spawning
+    task_graph = probe.get_task_graph()
+    sub_tasks = probe.generate_uv_cli_tasks(task_graph)
+"""
+
+import inspect
+from typing import Any, Dict, List, Optional
+
+# Internal imports
+from . import api
+from .collectors.binance_public_data_collector import BinancePublicDataCollector
+from .gap_filling.universal_gap_filler import UniversalGapFiller
+
+
+class ProbeAPI:
+    """API-only probe hooks for deterministic JSON output."""
+
+    def __init__(self):
+        self._cache: Dict[str, Any] = {}
+
+    def discover_api(self) -> Dict[str, Any]:
+        """
+        Generate deterministic API map for AI agents.
+
+        Returns:
+            Dict containing complete API surface with metadata
+        """
+        if "api_map" in self._cache:
+            return self._cache["api_map"]
+
+        api_map = {
+            "metadata": {
+                "package": "gapless-crypto-data",
+                "version": "2.15.0",
+                "probe_version": "1.0.0",
+                "type": "cryptocurrency-data-collection",
+                "compatibility": "uv-native",
+            },
+            "functions": self._discover_functions(),
+            "classes": self._discover_classes(),
+            "cli": self._discover_cli(),
+            "endpoints": self._discover_endpoints(),
+        }
+
+        self._cache["api_map"] = api_map
+        return api_map
+
+    def get_capabilities(self) -> Dict[str, Any]:
+        """
+        Report package capabilities for AI agents.
+
+        Returns:
+            Dict with capability matrix and operational limits
+        """
+        if "capabilities" in self._cache:
+            return self._cache["capabilities"]
+
+        capabilities = {
+            "data_collection": {
+                "source": "binance-public-repository",
+                "performance_multiplier": "22x",
+                "supported_markets": ["USDT-spot-pairs"],
+                "timeframes": [
+                    "1s",
+                    "1m",
+                    "3m",
+                    "5m",
+                    "15m",
+                    "30m",
+                    "1h",
+                    "2h",
+                    "4h",
+                    "6h",
+                    "8h",
+                    "12h",
+                    "1d",
+                ],
+                "data_format": "11-column-microstructure",
+                "gap_guarantee": "zero-gaps",
+            },
+            "processing": {
+                "memory_streaming": True,
+                "atomic_operations": True,
+                "resume_capability": True,
+                "parallel_symbols": True,
+            },
+            "integration": {
+                "pandas_compatible": True,
+                "polars_native": True,
+                "pyarrow_backend": True,
+                "ccxt_compatible": True,
+            },
+            "ai_agent_features": {
+                "stateless_probing": True,
+                "task_graph_generation": True,
+                "uv_cli_spawning": True,
+                "deterministic_output": True,
+                "no_file_operations": True,
+            },
+        }
+
+        self._cache["capabilities"] = capabilities
+        return capabilities
+
+    def get_task_graph(self) -> Dict[str, Any]:
+        """
+        Generate task dependency graph for phased AI agent spawning.
+
+        Returns:
+            Dict with task nodes, dependencies, and execution metadata
+        """
+        if "task_graph" in self._cache:
+            return self._cache["task_graph"]
+
+        task_graph = {
+            "nodes": {
+                "discover": {
+                    "type": "discovery",
+                    "command": 'python -c "import gapless_crypto_clickhouse; print(gapless_crypto_clickhouse.__probe__.discover_api())"',
+                    "dependencies": [],
+                    "outputs": ["api_map"],
+                    "phase": 0,
+                },
+                "capabilities": {
+                    "type": "capability_check",
+                    "command": 'python -c "import gapless_crypto_clickhouse; print(gapless_crypto_clickhouse.__probe__.get_capabilities())"',
+                    "dependencies": [],
+                    "outputs": ["capabilities_matrix"],
+                    "phase": 0,
+                },
+                "validate_symbols": {
+                    "type": "validation",
+                    "command": 'python -c "import gapless_crypto_clickhouse; print(gapless_crypto_clickhouse.get_supported_symbols())"',
+                    "dependencies": ["discover"],
+                    "outputs": ["symbol_list"],
+                    "phase": 1,
+                },
+                "validate_timeframes": {
+                    "type": "validation",
+                    "command": 'python -c "import gapless_crypto_clickhouse; print(gapless_crypto_clickhouse.get_supported_timeframes())"',
+                    "dependencies": ["discover"],
+                    "outputs": ["timeframe_list"],
+                    "phase": 1,
+                },
+                "test_collection": {
+                    "type": "integration_test",
+                    "command": "python -c \"import gapless_crypto_clickhouse; df=gapless_crypto_clickhouse.fetch_data('BTCUSDT', '1h', limit=5); print(f'✓ {len(df)} rows collected')\"",
+                    "dependencies": ["validate_symbols", "validate_timeframes"],
+                    "outputs": ["collection_status"],
+                    "phase": 2,
+                },
+            },
+            "execution_plan": {
+                "phase_0": ["discover", "capabilities"],
+                "phase_1": ["validate_symbols", "validate_timeframes"],
+                "phase_2": ["test_collection"],
+            },
+            "parallel_safe": {"phase_0": True, "phase_1": True, "phase_2": False},
+        }
+
+        self._cache["task_graph"] = task_graph
+        return task_graph
+
+    def generate_uv_cli_tasks(self, task_graph: Optional[Dict[str, Any]] = None) -> List[str]:
+        """
+        Generate uv CLI commands for spawning AI agent sub-tasks.
+
+        Args:
+            task_graph: Optional pre-computed task graph
+
+        Returns:
+            List of uv CLI commands ready for execution
+        """
+        if task_graph is None:
+            task_graph = self.get_task_graph()
+
+        # Phase 0: Parallel discovery
+        phase_0_commands = []
+        for node_id in task_graph["execution_plan"]["phase_0"]:
+            node = task_graph["nodes"][node_id]
+            uv_cmd = f"uv run --active {node['command']}"
+            phase_0_commands.append(uv_cmd)
+
+        # Phase 1: Validation (depends on Phase 0)
+        phase_1_commands = []
+        for node_id in task_graph["execution_plan"]["phase_1"]:
+            node = task_graph["nodes"][node_id]
+            uv_cmd = f"uv run --active {node['command']}"
+            phase_1_commands.append(uv_cmd)
+
+        # Phase 2: Integration test (depends on Phase 1)
+        phase_2_commands = []
+        for node_id in task_graph["execution_plan"]["phase_2"]:
+            node = task_graph["nodes"][node_id]
+            uv_cmd = f"uv run --active {node['command']}"
+            phase_2_commands.append(uv_cmd)
+
+        return {
+            "phase_0_parallel": phase_0_commands,
+            "phase_1_parallel": phase_1_commands,
+            "phase_2_sequential": phase_2_commands,
+            "usage": "Execute phases in order. Within each phase, commands can run in parallel.",
+        }
+
+    def get_probe_info(self) -> Dict[str, Any]:
+        """
+        Get probe system metadata and health.
+
+        Returns:
+            Dict with probe system status and capabilities
+        """
+        return {
+            "probe_system": {
+                "version": "1.0.0",
+                "compatible_agents": ["claude-code", "cursor", "copilot", "codeium"],
+                "output_format": "deterministic-json",
+                "caching": "memory-based",
+                "stateless": True,
+            },
+            "health": {
+                "imports_ok": self._check_imports(),
+                "api_accessible": self._check_api_access(),
+                "cache_status": len(self._cache),
+            },
+        }
+
+    def _discover_functions(self) -> Dict[str, Any]:
+        """Discover public functions from api module."""
+        functions = {}
+
+        for name in dir(api):
+            if not name.startswith("_"):
+                obj = getattr(api, name)
+                if callable(obj) and inspect.isfunction(obj):
+                    sig = inspect.signature(obj)
+                    functions[name] = {
+                        "parameters": [p.name for p in sig.parameters.values()],
+                        "docstring": (obj.__doc__ or "").strip(),
+                        "module": obj.__module__,
+                    }
+
+        return functions
+
+    def _discover_classes(self) -> Dict[str, Any]:
+        """Discover public classes."""
+        classes = {
+            "BinancePublicDataCollector": {
+                "module": "gapless_crypto_clickhouse.collectors.binance_public_data_collector",
+                "purpose": "high-performance-data-collection",
+                "methods": self._get_public_methods(BinancePublicDataCollector),
+            },
+            "UniversalGapFiller": {
+                "module": "gapless_crypto_clickhouse.gap_filling.universal_gap_filler",
+                "purpose": "gap-detection-and-filling",
+                "methods": self._get_public_methods(UniversalGapFiller),
+            },
+        }
+
+        return classes
+
+    def _discover_cli(self) -> Dict[str, Any]:
+        """Discover CLI interface."""
+        return {
+            "entry_point": "gapless-crypto-data",
+            "uv_usage": "uv run gapless-crypto-data",
+            "common_patterns": [
+                "uv run gapless-crypto-data --symbol BTCUSDT --timeframes 1h,4h",
+                "uv run gapless-crypto-data --fill-gaps --directory ./data",
+                "uv run gapless-crypto-data --symbol BTCUSDT,ETHUSDT --timeframes 1s,1d",
+            ],
+        }
+
+    def _discover_endpoints(self) -> Dict[str, Any]:
+        """Discover probe endpoints."""
+        return {
+            "__probe__.discover_api": "Complete API surface discovery",
+            "__probe__.get_capabilities": "Package capability matrix",
+            "__probe__.get_task_graph": "Task dependency graph for phased execution",
+            "__probe__.generate_uv_cli_tasks": "uv CLI commands for agent spawning",
+            "__probe__.get_probe_info": "Probe system metadata and health",
+        }
+
+    def _get_public_methods(self, cls) -> List[str]:
+        """Get public methods from a class."""
+        return [
+            name for name in dir(cls) if not name.startswith("_") and callable(getattr(cls, name))
+        ]
+
+    def _check_imports(self) -> bool:
+        """Check if core imports are working."""
+        try:
+            import importlib.util
+
+            return (
+                importlib.util.find_spec("pandas") is not None
+                and importlib.util.find_spec("httpx") is not None
+            )
+        except ImportError:
+            return False
+
+    def _check_api_access(self) -> bool:
+        """Check if API functions are accessible."""
+        try:
+            from . import api
+
+            return hasattr(api, "fetch_data")
+        except Exception:
+            return False
+
+
+# Global probe instance for easy access
+_probe_instance = ProbeAPI()
+
+# Export functions for direct access
+discover_api = _probe_instance.discover_api
+get_capabilities = _probe_instance.get_capabilities
+get_task_graph = _probe_instance.get_task_graph
+generate_uv_cli_tasks = _probe_instance.generate_uv_cli_tasks
+get_probe_info = _probe_instance.get_probe_info
+
+# For backwards compatibility and explicit access
+probe = _probe_instance
+
+__all__ = [
+    "discover_api",
+    "get_capabilities",
+    "get_task_graph",
+    "generate_uv_cli_tasks",
+    "get_probe_info",
+    "probe",
+    "ProbeAPI",
+]