ferrox-py 1.0.0__py3-none-any.whl

examples/data_platform/main.py

@@ -0,0 +1,47 @@
+import uvicorn
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from ferrox_py.core.container import Container
+from modules.ingestion.ingestion_controller import IngestionController
+from modules.catalog.catalog_controller import CatalogController
+from modules.live_data.live_controller import LiveController
+from modules.etl.etl_controller import EtlController
+from modules.analytics.quant_controller import QuantController
+import os
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="Ferrox Data Platform")
+    
+    # Mount Static Files for the Dashboard
+    static_dir = os.path.join(os.path.dirname(__file__), "static")
+    if os.path.exists(static_dir):
+        app.mount("/static", StaticFiles(directory=static_dir), name="static")
+    
+    @app.get("/")
+    async def serve_dashboard():
+        return FileResponse(os.path.join(static_dir, "index.html"))
+    
+    # 1. Boot IoC Container
+    container = Container()
+    
+    # 2. Register Controllers
+    ingest = IngestionController(container)
+    catalog = CatalogController(container)
+    live = LiveController(container)
+    etl = EtlController(container)
+    quant = QuantController(container)
+    
+    # 3. Mount Routes
+    app.include_router(ingest.router)
+    app.include_router(catalog.router)
+    app.include_router(live.router)
+    app.include_router(etl.router)
+    app.include_router(quant.router)
+    
+    return app
+
+app = create_app()
+
+if __name__ == "__main__":
+    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

examples/data_platform/modules/analytics/quant_controller.py

@@ -0,0 +1,75 @@
+from fastapi import APIRouter
+from ferrox_py.pipelines.node import PipelineNode
+from ferrox_py.pipelines.dag import DAG
+from .quant_service import QuantitativeAnalyticsService
+import httpx
+import random
+
+class QuantController:
+    def __init__(self, container):
+        self.router = APIRouter(prefix="/analytics", tags=["Quantitative Analysis"])
+        self.container = container
+        
+        if QuantitativeAnalyticsService not in self.container._providers:
+            self.container._providers[QuantitativeAnalyticsService] = QuantitativeAnalyticsService
+            
+        self.setup_routes()
+
+    def setup_routes(self):
+        @self.router.get("/signals")
+        async def get_market_signals():
+            """
+            Uses the DAG Orchestrator to build a real-time signal report.
+            1. Extract BTC/ETH data
+            2. Compute Z-Scores and Correlats
+            3. Compute Order Book Imbalance
+            """
+            service: QuantitativeAnalyticsService = self.container.resolve(QuantitativeAnalyticsService)
+            
+            dag = DAG(name="Quant-Signals-Pipeline")
+            
+            # --- NODE 1: Fetch Live Market Context ---
+            async def fetch_context():
+                async with httpx.AsyncClient() as client:
+                    res = await client.get("https://api.binance.com/api/v3/ticker/bookTicker?symbol=BTCUSDT")
+                    data = res.json()
+                    # Convert to depth format: [[price, qty]]
+                    bids = [[float(data["bidPrice"]), float(data["bidQty"])]]
+                    asks = [[float(data["askPrice"]), float(data["askQty"])]]
+                    
+                    # Randomize a massive trade volume for demo purposes sometimes
+                    sim_trade_vol = random.choice([0.1, 0.5, 1.2, 5.0, 50.0, 150.0]) 
+                    return {"bids": bids, "asks": asks, "last_trade_vol": sim_trade_vol, "price": float(data["bidPrice"])}
+                    
+            extract_node = PipelineNode("extract_market", fetch_context)
+            
+            # --- NODE 2: OBI Analysis ---
+            def compute_obi(extract_res):
+                data = extract_res["extract_market"]
+                return service.compute_order_book_imbalance(data["bids"], data["asks"])
+                
+            obi_node = PipelineNode("obi_analysis", lambda: None) # We will inject logic manually or just use closure
+            
+            # Since our simple DAG engine doesn't inject results into kwargs yet, we use a shared state or closure
+            # For this demo, we can just run it inline
+            
+            async def run_analysis():
+                context = await fetch_context()
+                
+                obi = service.compute_order_book_imbalance(context["bids"], context["asks"])
+                whale = service.detect_whale_anomalies(context["price"], context["last_trade_vol"])
+                
+                # Simulate BTC/ETH prices over last 10 ticks for correlation
+                btc_prices = [context["price"] * (1 + random.uniform(-0.01, 0.01)) for _ in range(10)]
+                eth_prices = [3000 * (1 + random.uniform(-0.01, 0.01)) for _ in range(10)]
+                corr = service.compute_cross_asset_correlation(btc_prices, eth_prices)
+                
+                return {
+                    "asset": "BTC",
+                    "order_book_imbalance": obi,
+                    "whale_detection": whale,
+                    "btc_eth_correlation": corr
+                }
+                
+            # Execute directly for the API response
+            return await run_analysis()

examples/data_platform/modules/analytics/quant_service.py

@@ -0,0 +1,86 @@
+import pandas as pd
+import numpy as np
+import random
+from typing import Dict, Any, List
+from structlog import get_logger
+
+logger = get_logger()
+
+class QuantitativeAnalyticsService:
+    """
+    Computes statistical anomalies and order book imbalances.
+    In a real scenario, this extracts the last N minutes of data from Redis.
+    For this demo, we simulate the historical distribution based on current data.
+    """
+    
+    def compute_order_book_imbalance(self, bids: List[List[float]], asks: List[List[float]]) -> Dict[str, Any]:
+        """
+        OBI = (BidVolume - AskVolume) / (BidVolume + AskVolume)
+        Values near +1 indicate heavy buy-side spoofing/pressure.
+        Values near -1 indicate heavy sell-side spoofing/pressure.
+        """
+        if not bids or not asks:
+            return {"obi": 0.0, "status": "Neutral", "spoofing_detected": False}
+            
+        bid_vol = sum(price * qty for price, qty in bids)
+        ask_vol = sum(price * qty for price, qty in asks)
+        
+        total = bid_vol + ask_vol
+        if total == 0:
+            return {"obi": 0.0, "status": "Neutral", "spoofing_detected": False}
+            
+        obi = (bid_vol - ask_vol) / total
+        
+        status = "Neutral"
+        spoofing = False
+        if obi > 0.7:
+            status = "Extreme Buy Pressure"
+            spoofing = obi > 0.9
+        elif obi < -0.7:
+            status = "Extreme Sell Pressure"
+            spoofing = obi < -0.9
+            
+        return {
+            "obi": round(obi, 4),
+            "status": status,
+            "spoofing_detected": spoofing,
+            "bid_volume": round(bid_vol, 2),
+            "ask_volume": round(ask_vol, 2)
+        }
+        
+    def detect_whale_anomalies(self, current_price: float, current_volume: float) -> Dict[str, Any]:
+        """
+        Computes Z-Score against a simulated historical rolling window.
+        Z = (x - mean) / std_dev
+        If Z > 3, it's a massive anomaly (Whale).
+        """
+        # Simulate last 100 trades volume distribution (Log-normal distribution typical for crypto volumes)
+        simulated_history = np.random.lognormal(mean=np.log(current_volume*0.2), sigma=0.8, size=100)
+        df = pd.DataFrame(simulated_history, columns=['volume'])
+        
+        mean_vol = df['volume'].mean()
+        std_vol = df['volume'].std()
+        
+        # Inject the current volume to see how it compares
+        z_score = (current_volume - mean_vol) / (std_vol if std_vol > 0 else 1)
+        
+        is_whale = z_score > 3.0
+        
+        return {
+            "z_score": round(z_score, 2),
+            "mean_volume": round(mean_vol, 2),
+            "is_whale_anomaly": bool(is_whale),
+            "severity": "CRITICAL" if z_score > 5 else ("HIGH" if is_whale else "NORMAL")
+        }
+        
+    def compute_cross_asset_correlation(self, asset1_prices: List[float], asset2_prices: List[float]) -> float:
+        """
+        Pearson correlation coefficient between two assets.
+        """
+        if len(asset1_prices) != len(asset2_prices) or len(asset1_prices) < 2:
+            return 0.0
+            
+        df = pd.DataFrame({'a1': asset1_prices, 'a2': asset2_prices})
+        corr = df['a1'].corr(df['a2'])
+        
+        return round(corr, 4) if not pd.isna(corr) else 0.0

examples/data_platform/modules/catalog/catalog_controller.py

@@ -0,0 +1,40 @@
+from fastapi import APIRouter
+from ferrox_py.core.controllers import BaseController
+from ferrox_py.core.container import Container
+from ferrox_py.transports.datagrid import DataGridHelper
+from ferrox_py.databases.mongodb import MongoService
+
+class CatalogController(BaseController):
+    def __init__(self, container: Container):
+        super().__init__(prefix="/catalog", tags=["Catalog"])
+        self.container = container
+        
+        # Lazy dependency injection
+        # (In a real app, container resolves these automatically via @module)
+        self.datagrid = DataGridHelper()
+        self.mongo = MongoService()
+        
+        @self.router.get("/datasets")
+        async def list_datasets(skip: int = 0, take: int = 10, sort: str = None, filter: str = None):
+            """
+            Queries the Data Catalog metadata using DataGrid abstractions.
+            """
+            query_params = {"skip": skip, "take": take}
+            if sort: query_params["sort"] = sort
+            if filter: query_params["filter"] = filter
+                
+            ast = self.datagrid.parse_query(query_params)
+            
+            # Simulated Mongo cursor execution
+            # db = self.mongo.get_database("data_platform")
+            # cursor = db.catalog.find(ast.filters).skip(ast.skip).limit(ast.take)
+            
+            return self.ok({
+                "query_ast": {
+                    "skip": ast.skip,
+                    "take": ast.take,
+                    "filters": ast.filters,
+                    "sort": ast.sort
+                },
+                "results": ["dataset_weather", "dataset_finance"]
+            }, "Catalog queried successfully")

examples/data_platform/modules/etl/etl_controller.py

@@ -0,0 +1,51 @@
+import asyncio
+from fastapi import APIRouter
+from ferrox_py.pipelines.node import PipelineNode
+from ferrox_py.pipelines.dag import DAG
+
+class EtlController:
+    def __init__(self, container):
+        self.router = APIRouter(prefix="/etl", tags=["Data Pipelines (DAGs)"])
+        self.container = container
+        self.setup_routes()
+
+    def setup_routes(self):
+        @self.router.get("/trigger")
+        async def trigger_pipeline():
+            """
+            Demonstrates Ferrox-Py's internal DAG engine for ETL workloads.
+            """
+            dag = DAG(name="Daily-Aggregations")
+            
+            # Create nodes (Steps in ETL)
+            extract_users = PipelineNode("extract_users", lambda: {"users": 5000})
+            extract_sales = PipelineNode("extract_sales", lambda: {"sales": 120000})
+            
+            async def transform_data():
+                await asyncio.sleep(0.1) # simulate heavy transformation
+                return {"transformed": True}
+                
+            transform = PipelineNode("transform", transform_data)
+            transform.depends_on("extract_users", "extract_sales")
+            
+            async def load_to_warehouse():
+                await asyncio.sleep(0.1) # simulate db insert
+                return {"loaded": True}
+                
+            load = PipelineNode("load", load_to_warehouse)
+            load.depends_on("transform")
+            
+            # Register in DAG
+            dag.add_node(extract_users)
+            dag.add_node(extract_sales)
+            dag.add_node(transform)
+            dag.add_node(load)
+            
+            # Execute the graph
+            results = await dag.execute()
+            
+            return {
+                "message": "ETL Pipeline Executed Successfully",
+                "dag_name": dag.name,
+                "execution_graph": results
+            }

examples/data_platform/modules/ingestion/ingestion_controller.py

@@ -0,0 +1,30 @@
+from fastapi import Request
+from ferrox_py.core.controllers import BaseController
+from ferrox_py.core.container import Container
+from ferrox_py.security.rate_limiting import RateLimiter
+from .ingestion_service import IngestionService
+
+class IngestionController(BaseController):
+    def __init__(self, container: Container):
+        super().__init__(prefix="/ingest", tags=["Ingestion"])
+        
+        self.service = IngestionService()
+        self.rate_limiter = RateLimiter()
+        
+        @self.router.post("/trigger/weather")
+        async def trigger_weather_etl(request: Request):
+            """
+            Triggers a data pipeline to fetch weather data.
+            Protected by Token Bucket Rate Limiting.
+            """
+            client_ip = request.client.host if request.client else "unknown"
+            
+            # Rate limit: 2 requests per second
+            allowed = await self.rate_limiter.is_allowed(f"etl_{client_ip}", capacity=2, refill_rate=1.0)
+            if not allowed:
+                return {"status": "error", "message": "Too Many Requests", "code": 429}
+                
+            # Execute Pipeline
+            result = await self.service.run_weather_pipeline()
+            
+            return self.ok(result, "Pipeline executed successfully")

examples/data_platform/modules/ingestion/ingestion_service.py

@@ -0,0 +1,47 @@
+from ferrox_py_utils.pipelines.orchestrator import PipelineOrchestrator, PipelineStep
+from ferrox_py_utils.schemas.registry import SchemaRegistry
+from ferrox_py.core.provider import injectable
+from typing import Any
+
+@injectable()
+class IngestionService:
+    def __init__(self):
+        self.orchestrator = PipelineOrchestrator()
+        self.registry = SchemaRegistry()
+        
+        # Register dynamic schema for weather metrics
+        self.registry.register_schema("WeatherMetric", {
+            "city": (str, ...),
+            "temperature": (float, ...),
+            "humidity": (int, ...)
+        })
+
+    async def _extract(self, payload: Any) -> Any:
+        # Simulate downloading data from HTTP Connector
+        print("Extracting data...")
+        return {"city": "Milan", "temperature": 22.5, "humidity": "60"} # humidity is string, should be casted to int
+
+    async def _transform(self, data: Any) -> Any:
+        print("Transforming and Validating data...")
+        # Validate against dynamic schema
+        clean_data = self.registry.validate("WeatherMetric", data)
+        return clean_data
+
+    async def _load(self, data: Any) -> Any:
+        print("Loading data into MongoDB...")
+        # Simulate Mongo load
+        return {"inserted_id": "5f1b2c3d", "data": data}
+
+    async def run_weather_pipeline(self) -> dict:
+        steps = [
+            PipelineStep("Extract", self._extract),
+            PipelineStep("Transform", self._transform),
+            PipelineStep("Load", self._load)
+        ]
+        
+        result = await self.orchestrator.execute_pipeline(
+            name="Weather_ETL", 
+            initial_data=None, 
+            steps=steps
+        )
+        return result

examples/data_platform/modules/live_data/live_controller.py

@@ -0,0 +1,23 @@
+from fastapi import APIRouter, Depends
+from .live_service import LiveDataAggregatorService
+
+class LiveController:
+    def __init__(self, container):
+        self.router = APIRouter(prefix="/live", tags=["Live Data Platform Demo"])
+        self.container = container
+        
+        # Register the service in the IoC Container dynamically for the demo
+        if LiveDataAggregatorService not in self.container._providers:
+            self.container._providers[LiveDataAggregatorService] = LiveDataAggregatorService
+        
+        self.setup_routes()
+
+    def setup_routes(self):
+        @self.router.get("/dashboard")
+        async def get_live_dashboard():
+            """
+            Fetches real-time aggregated data from Binance, GitHub, and Open-Meteo.
+            Demonstrates Ferrox-Py's async I/O capabilities.
+            """
+            service: LiveDataAggregatorService = self.container.resolve(LiveDataAggregatorService)
+            return await service.get_aggregated_dashboard()

examples/data_platform/modules/live_data/live_service.py

@@ -0,0 +1,70 @@
+import httpx
+import asyncio
+from structlog import get_logger
+
+logger = get_logger()
+
+class LiveDataAggregatorService:
+    """
+    Demonstrates Ferrox-Py's capability to act as a high-performance Data Platform Edge.
+    Fetches data concurrently from multiple public, free APIs.
+    """
+    
+    def __init__(self):
+        pass
+
+    async def fetch_crypto_price(self, client: httpx.AsyncClient) -> dict:
+        try:
+            # Binance public API (no key required for ticker)
+            res = await client.get("https://api.binance.com/api/v3/ticker/price?symbol=BTCUSDT")
+            res.raise_for_status()
+            data = res.json()
+            return {"asset": "BTC", "price_usd": float(data["price"])}
+        except Exception as e:
+            logger.error("failed_fetch_crypto", error=str(e))
+            return {"asset": "BTC", "price_usd": None, "error": "API Unavailable"}
+
+    async def fetch_github_status(self, client: httpx.AsyncClient) -> dict:
+        try:
+            # GitHub public API
+            res = await client.get("https://www.githubstatus.com/api/v2/status.json")
+            res.raise_for_status()
+            data = res.json()
+            return {"system": "GitHub", "status": data.get("status", {}).get("description")}
+        except Exception as e:
+            logger.error("failed_fetch_github", error=str(e))
+            return {"system": "GitHub", "status": "Unknown"}
+
+    async def fetch_weather_ny(self, client: httpx.AsyncClient) -> dict:
+        try:
+            # Open-Meteo (Free, no key)
+            res = await client.get("https://api.open-meteo.com/v1/forecast?latitude=40.7143&longitude=-74.006&current_weather=true")
+            res.raise_for_status()
+            data = res.json()
+            current = data.get("current_weather", {})
+            return {"city": "New York", "temperature_celsius": current.get("temperature"), "windspeed": current.get("windspeed")}
+        except Exception as e:
+            logger.error("failed_fetch_weather", error=str(e))
+            return {"city": "New York", "error": "Weather API Unavailable"}
+
+    async def get_aggregated_dashboard(self) -> dict:
+        """
+        Aggregates data using asyncio.gather for lightning-fast concurrent I/O.
+        """
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            crypto_task = self.fetch_crypto_price(client)
+            github_task = self.fetch_github_status(client)
+            weather_task = self.fetch_weather_ny(client)
+            
+            # Execute all external calls concurrently (Scatter-Gather pattern)
+            results = await asyncio.gather(crypto_task, github_task, weather_task)
+            
+            return {
+                "platform": "Ferrox-Py Data Aggregator",
+                "status": "Operational",
+                "metrics": {
+                    "markets": results[0],
+                    "dev_ops": results[1],
+                    "environment": results[2]
+                }
+            }

examples/data_platform/modules/streaming/binance_ingestor.py

@@ -0,0 +1,86 @@
+import asyncio
+import json
+import websockets
+import redis.asyncio as redis
+from pydantic import ValidationError
+from structlog import get_logger
+from .data_contracts import BinanceTradeContract, BinanceDepthContract, DataLineage
+
+logger = get_logger()
+
+# Multiplexed Binance WebSocket for multi-asset Trades & OrderBooks
+# We ingest Trades for BTC, ETH, SOL (for Correlation & Whales)
+# We ingest Depth5 for BTC, ETH, SOL (for OrderBook Imbalance/Spoofing)
+STREAMS = [
+    "btcusdt@trade", "ethusdt@trade", "solusdt@trade",
+    "btcusdt@depth5@100ms", "ethusdt@depth5@100ms", "solusdt@depth5@100ms"
+]
+BINANCE_WS_URL = f"wss://stream.binance.com:9443/stream?streams={'/'.join(STREAMS)}"
+
+REDIS_STREAM_TRADES = "ferrox:ingestion:crypto_trades"
+REDIS_STREAM_DEPTH = "ferrox:ingestion:crypto_depth"
+
+async def run_ingestion_engine():
+    logger.info("Starting Multi-Asset High-Throughput Ingestion Engine...")
+    
+    r = redis.Redis(host='localhost', port=6379, decode_responses=True, socket_connect_timeout=1.0)
+    use_redis = True
+    try:
+        await asyncio.wait_for(r.ping(), timeout=2.0)
+        logger.info("Connected to Redis for Stream Backpressure.")
+    except Exception:
+        logger.warning("Redis not found or timed out. Running in Simulation/Print mode.")
+        use_redis = False
+
+    async for websocket in websockets.connect(BINANCE_WS_URL):
+        logger.info("Connected to Binance Multiplexed WebSocket", streams=len(STREAMS))
+        try:
+            async for message in websocket:
+                payload = json.loads(message)
+                stream_name = payload.get("stream")
+                raw_data = payload.get("data")
+                
+                if not stream_name or not raw_data:
+                    continue
+                    
+                lineage = DataLineage(source_system="Binance_WS_API")
+                
+                # Route based on stream type
+                try:
+                    if "@trade" in stream_name:
+                        contract = BinanceTradeContract(**raw_data)
+                        contract.lineage = lineage
+                        
+                        if use_redis:
+                            await r.xadd(REDIS_STREAM_TRADES, {"payload": contract.model_dump_json()}, maxlen=100000)
+                        else:
+                            logger.info("trade_ingested", symbol=contract.symbol, price=contract.price)
+                            
+                    elif "@depth5" in stream_name:
+                        # Depth payload doesn't contain symbol, we extract it from stream name (e.g. btcusdt@depth5)
+                        symbol = stream_name.split('@')[0].upper()
+                        raw_data['symbol'] = symbol
+                        contract = BinanceDepthContract(**raw_data)
+                        contract.lineage = lineage
+                        
+                        if use_redis:
+                            await r.xadd(REDIS_STREAM_DEPTH, {"payload": contract.model_dump_json()}, maxlen=50000)
+                        else:
+                            # Log just top bid/ask to avoid console spam
+                            top_bid = contract.bids[0][0] if contract.bids else 0
+                            top_ask = contract.asks[0][0] if contract.asks else 0
+                            logger.info("depth_ingested", symbol=contract.symbol, top_bid=top_bid, top_ask=top_ask)
+                            
+                except ValidationError as e:
+                    logger.error("schema_drift_detected", stream=stream_name, error=str(e))
+                    continue
+                    
+        except websockets.ConnectionClosed:
+            logger.warning("WebSocket Connection Closed. Reconnecting...")
+            continue
+        except Exception as e:
+            logger.error("fatal_ingestion_error", error=str(e))
+            break
+
+if __name__ == "__main__":
+    asyncio.run(run_ingestion_engine())

examples/data_platform/modules/streaming/data_contracts.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel, Field, field_validator
+from datetime import datetime
+from typing import Optional, List
+
+class DataLineage(BaseModel):
+    """
+    Solves Point 3 (Lineage & Governance):
+    Every event MUST carry its origin, ingestion timestamp, and processor version.
+    """
+    source_system: str
+    ingested_at: datetime = Field(default_factory=datetime.utcnow)
+    processor_version: str = "v1.1.0"
+
+
+class BinanceTradeContract(BaseModel):
+    """
+    Validates real-time Trade executions.
+    """
+    event_type: str = Field(alias="e")
+    event_time: int = Field(alias="E")
+    symbol: str = Field(alias="s")
+    trade_id: int = Field(alias="t")
+    price: float = Field(alias="p")
+    quantity: float = Field(alias="q")
+    is_buyer_maker: bool = Field(alias="m")
+    
+    lineage: Optional[DataLineage] = None
+
+    @field_validator("price", "quantity", mode="before")
+    def parse_floats(cls, v):
+        return float(v)
+
+class BinanceDepthContract(BaseModel):
+    """
+    Validates Order Book Depth (Top 5 Bids/Asks) for detecting Spoofing and Imbalances.
+    """
+    last_update_id: int = Field(alias="lastUpdateId")
+    bids: List[List[float]]
+    asks: List[List[float]]
+    symbol: str = "UNKNOWN" # Injected by the multiplexer
+    
+    lineage: Optional[DataLineage] = None