dataenginex 0.3.4__py3-none-any.whl

dataenginex/README.md

@@ -0,0 +1,35 @@
+# dataenginex
+
+`dataenginex` is the core DataEngineX framework package for building observable, production-ready data and API services.
+
+It provides:
+- FastAPI application primitives and API extensions
+- Middleware for structured logging, metrics, and tracing
+- Data quality and validation utilities
+- Lakehouse and warehouse building blocks
+- Reusable ML support modules for model-serving workflows
+
+## Install
+
+```bash
+pip install dataenginex
+```
+
+## Package Scope
+
+This package is the core library from the DEX monorepo.
+`careerdex` and `weatherdex` are maintained in the same repository but are not part of this package release flow.
+
+## Quick Usage
+
+```python
+from dataenginex import __version__
+
+print(__version__)
+```
+
+## Source and Docs
+
+- Repository: https://github.com/data-literate/DEX
+- CI/CD guide: `docs/CI_CD.md`
+- Release notes: `packages/dataenginex/src/dataenginex/RELEASE_NOTES.md`

dataenginex/RELEASE_NOTES.md

@@ -0,0 +1,38 @@
+# dataenginex Release Notes
+
+This document tracks published package releases for `dataenginex` only.
+Only include changes that modify files under `packages/dataenginex/src/dataenginex/**`.
+
+## v0.3.4 - 2026-02-20
+
+- Released package version `0.3.4`.
+- Tag: `v0.3.4`
+- Release title: `Release v0.3.4`
+- Changes in this release:
+	- Repo hygiene updates after monorepo/package-layout migration.
+	- Canonicalized package/app path references across docs and workflow guidance.
+	- Removed standalone `careerdex` and `weatherdex` package scaffolds from `packages/`.
+	- Updated CI/package validation and project metadata/docs to align with current structure.
+
+## v0.3.3 - 2026-02-16
+
+- Released package version `0.3.3`.
+- Tag: `v0.3.3`
+- Release title: `Release v0.3.3`
+- Changes in this release (`packages/dataenginex/src/dataenginex` only):
+	- Significant package expansion and refactor across API, core, data, lakehouse, middleware, ML, and warehouse modules.
+	- Added/expanded API modules including auth, pagination, rate limiting, and v1 router wiring.
+	- Consolidated middleware layout (logging, metrics, tracing) and moved logging config under middleware.
+	- Package diff from `v0.2.0` to `v0.3.3`: 39 files changed, 4213 insertions, 247 deletions.
+
+## v0.2.0 - 2026-02-12
+
+- Released package version `0.2.0`.
+- Tag: `v0.2.0`
+- Release title: `Release v0.2.0 - Production Hardening`
+- Changes in this release (`packages/dataenginex/src/dataenginex` only):
+	- Established core package structure and initial module organization.
+	- Added API readiness/health behavior and improved probe handling.
+	- Added structured request logging with request ID tracking.
+	- Added Prometheus metrics and OpenTelemetry tracing support in package middleware.
+	- Added validation and error-handling improvements in API/core paths.

dataenginex/__init__.py

@@ -0,0 +1,16 @@
+"""
+DataEngineX (DEX) — Core framework for data engineering projects.
+
+Submodules:
+    api        – FastAPI application, health checks, error handling
+    core       – Schemas, validators, medallion architecture, pipeline config
+    middleware – Logging, metrics, tracing, request middleware
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("dataenginex")
+except PackageNotFoundError:
+    __version__ = "0.3.4"
+

dataenginex/api/__init__.py

@@ -0,0 +1,11 @@
+"""Reusable API components (auth, health, errors) for product services."""
+
+from .errors import APIHTTPException, ServiceUnavailableError  # noqa: F401
+from .health import HealthChecker, HealthStatus  # noqa: F401
+
+__all__ = [
+    "HealthChecker",
+    "HealthStatus",
+    "APIHTTPException",
+    "ServiceUnavailableError",
+]

dataenginex/api/auth.py

@@ -0,0 +1,173 @@
+"""
+JWT authentication middleware for DEX API.
+
+Provides a ``JWTAuth`` dependency for FastAPI that validates bearer tokens
+using HMAC-SHA256 (symmetric) or RSA (asymmetric via ``DEX_JWT_PUBLIC_KEY``).
+
+Configuration is via environment variables:
+    DEX_JWT_SECRET   — HMAC shared secret (required unless RSA key is set)
+    DEX_JWT_ALGORITHM — Algorithm (default HS256)
+    DEX_AUTH_ENABLED  — "true" to enforce auth (default "false")
+"""
+
+from __future__ import annotations
+
+import hmac
+import json
+import os
+import time
+from base64 import urlsafe_b64decode, urlsafe_b64encode
+from dataclasses import dataclass
+from hashlib import sha256
+from typing import Any
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from loguru import logger
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.responses import Response
+
+# ---------------------------------------------------------------------------
+# Token helpers (pure-Python HS256 — no external ``pyjwt`` needed)
+# ---------------------------------------------------------------------------
+
+
+def _b64url_decode(data: str) -> bytes:
+    padding = 4 - len(data) % 4
+    return urlsafe_b64decode(data + "=" * padding)
+
+
+def _b64url_encode(data: bytes) -> str:
+    return urlsafe_b64encode(data).rstrip(b"=").decode()
+
+
+def create_token(payload: dict[str, Any], secret: str, ttl: int = 3600) -> str:
+    """Create a HS256 JWT token.
+
+    Parameters
+    ----------
+    payload:
+        Claims dict (e.g. ``{"sub": "user123", "roles": ["admin"]}``).
+    secret:
+        HMAC shared secret.
+    ttl:
+        Time-to-live in seconds (default 1 hour).
+    """
+    header = {"alg": "HS256", "typ": "JWT"}
+    now = int(time.time())
+    payload = {**payload, "iat": now, "exp": now + ttl}
+
+    segments = [
+        _b64url_encode(json.dumps(header).encode()),
+        _b64url_encode(json.dumps(payload, default=str).encode()),
+    ]
+    signing_input = f"{segments[0]}.{segments[1]}"
+    signature = hmac.new(secret.encode(), signing_input.encode(), sha256).digest()
+    segments.append(_b64url_encode(signature))
+    return ".".join(segments)
+
+
+def decode_token(token: str, secret: str) -> dict[str, Any]:
+    """Decode and verify a HS256 JWT token.  Raises ``ValueError`` on failure."""
+    parts = token.split(".")
+    if len(parts) != 3:
+        raise ValueError("Malformed JWT")
+
+    signing_input = f"{parts[0]}.{parts[1]}"
+    expected_sig = hmac.new(secret.encode(), signing_input.encode(), sha256).digest()
+    actual_sig = _b64url_decode(parts[2])
+
+    if not hmac.compare_digest(expected_sig, actual_sig):
+        raise ValueError("Invalid JWT signature")
+
+    payload: dict[str, Any] = json.loads(_b64url_decode(parts[1]))
+    exp = payload.get("exp")
+    if exp is not None and int(exp) < int(time.time()):
+        raise ValueError("Token expired")
+
+    return payload
+
+
+# ---------------------------------------------------------------------------
+# Dataclass carrying the authenticated user info
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class AuthUser:
+    """Resolved identity from a valid JWT."""
+
+    sub: str
+    roles: list[str]
+    claims: dict[str, Any]
+
+
+# ---------------------------------------------------------------------------
+# FastAPI middleware
+# ---------------------------------------------------------------------------
+
+# Paths that never require authentication
+_PUBLIC_PATHS: set[str] = {
+    "/",
+    "/health",
+    "/ready",
+    "/startup",
+    "/metrics",
+    "/docs",
+    "/redoc",
+    "/openapi.json",
+    "/openapi.yaml",
+}
+
+
+class AuthMiddleware(BaseHTTPMiddleware):
+    """Starlette middleware that enforces JWT auth when enabled.
+
+    When ``DEX_AUTH_ENABLED`` is ``"true"`` (case-insensitive), every request
+    to a non-public path must carry a valid ``Authorization: Bearer <token>``
+    header. The decoded claims are stored on ``request.state.auth_user``.
+    """
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        enabled = os.getenv("DEX_AUTH_ENABLED", "false").lower() == "true"
+        if not enabled:
+            return await call_next(request)
+
+        # Skip public endpoints
+        if request.url.path in _PUBLIC_PATHS:
+            return await call_next(request)
+
+        secret = os.getenv("DEX_JWT_SECRET", "")
+        if not secret:
+            logger.error("DEX_AUTH_ENABLED=true but DEX_JWT_SECRET is not set")
+            return JSONResponse(
+                status_code=500,
+                content={"error": "auth_config_error", "message": "Auth secret not configured"},
+            )
+
+        auth_header = request.headers.get("Authorization", "")
+        if not auth_header.startswith("Bearer "):
+            return JSONResponse(
+                status_code=401,
+                content={"error": "unauthorized", "message": "Missing bearer token"},
+            )
+
+        token = auth_header[7:]
+        try:
+            claims = decode_token(token, secret)
+        except ValueError:
+            logger.exception("JWT validation failed")
+            return JSONResponse(
+                status_code=401,
+                content={
+                    "error": "unauthorized",
+                    "message": "Invalid or expired authentication token",
+                },
+            )
+
+        request.state.auth_user = AuthUser(
+            sub=claims.get("sub", "anonymous"),
+            roles=claims.get("roles", []),
+            claims=claims,
+        )
+        return await call_next(request)

dataenginex/api/errors.py

@@ -0,0 +1,70 @@
+"""Custom exception types for API error handling."""
+
+from __future__ import annotations
+
+from fastapi import HTTPException, status
+
+from dataenginex.core.schemas import ErrorDetail
+
+
+class APIHTTPException(HTTPException):
+    """Base HTTP exception with error code and details."""
+
+    def __init__(
+        self,
+        status_code: int,
+        message: str,
+        code: str = "api_error",
+        details: list[ErrorDetail] | None = None,
+    ) -> None:
+        self.code = code
+        self.details = details
+        super().__init__(status_code=status_code, detail=message)
+
+
+class BadRequestError(APIHTTPException):
+    """Raised for 400 validation or malformed requests."""
+
+    def __init__(
+        self,
+        message: str = "Bad request",
+        details: list[ErrorDetail] | None = None,
+    ) -> None:
+        super().__init__(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            message=message,
+            code="bad_request",
+            details=details,
+        )
+
+
+class NotFoundError(APIHTTPException):
+    """Raised for 404 not found errors."""
+
+    def __init__(
+        self,
+        message: str = "Resource not found",
+        details: list[ErrorDetail] | None = None,
+    ) -> None:
+        super().__init__(
+            status_code=status.HTTP_404_NOT_FOUND,
+            message=message,
+            code="not_found",
+            details=details,
+        )
+
+
+class ServiceUnavailableError(APIHTTPException):
+    """Raised when a dependency is unavailable."""
+
+    def __init__(
+        self,
+        message: str = "Service unavailable",
+        details: list[ErrorDetail] | None = None,
+    ) -> None:
+        super().__init__(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            message=message,
+            code="service_unavailable",
+            details=details,
+        )

dataenginex/api/health.py

@@ -0,0 +1,133 @@
+"""Health check utilities for DEX."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from dataclasses import dataclass
+from enum import StrEnum
+
+import httpx
+
+
+class HealthStatus(StrEnum):
+    """Supported health statuses."""
+
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    SKIPPED = "skipped"
+
+
+@dataclass(frozen=True)
+class ComponentHealth:
+    name: str
+    status: HealthStatus
+    message: str | None = None
+    duration_ms: float | None = None
+
+    def to_dict(self) -> dict[str, object | None]:
+        return {
+            "name": self.name,
+            "status": self.status.value,
+            "message": self.message,
+            "duration_ms": self.duration_ms,
+        }
+
+
+class HealthChecker:
+    """Runs health checks for DEX dependencies."""
+
+    def __init__(self, timeout_seconds: float = 1.0) -> None:
+        self.timeout_seconds = timeout_seconds
+
+    async def check_all(self) -> list[ComponentHealth]:
+        return [
+            await self.check_database(),
+            await self.check_cache(),
+            await self.check_external_api(),
+        ]
+
+    def overall_status(self, components: list[ComponentHealth]) -> HealthStatus:
+        if any(c.status == HealthStatus.UNHEALTHY for c in components):
+            return HealthStatus.UNHEALTHY
+        if any(c.status == HealthStatus.DEGRADED for c in components):
+            return HealthStatus.DEGRADED
+        return HealthStatus.HEALTHY
+
+    async def check_database(self) -> ComponentHealth:
+        host = os.getenv("DEX_DB_HOST")
+        port = os.getenv("DEX_DB_PORT")
+        if not host or not port:
+            return ComponentHealth(
+                name="database",
+                status=HealthStatus.SKIPPED,
+                message="database not configured",
+            )
+
+        start = time.perf_counter()
+        ok, message = await self._tcp_check(host, int(port))
+        duration_ms = (time.perf_counter() - start) * 1000
+        return ComponentHealth(
+            name="database",
+            status=HealthStatus.HEALTHY if ok else HealthStatus.UNHEALTHY,
+            message=message,
+            duration_ms=round(duration_ms, 2),
+        )
+
+    async def check_cache(self) -> ComponentHealth:
+        host = os.getenv("DEX_CACHE_HOST")
+        port = os.getenv("DEX_CACHE_PORT")
+        if not host or not port:
+            return ComponentHealth(
+                name="cache",
+                status=HealthStatus.SKIPPED,
+                message="cache not configured",
+            )
+
+        start = time.perf_counter()
+        ok, message = await self._tcp_check(host, int(port))
+        duration_ms = (time.perf_counter() - start) * 1000
+        return ComponentHealth(
+            name="cache",
+            status=HealthStatus.HEALTHY if ok else HealthStatus.UNHEALTHY,
+            message=message,
+            duration_ms=round(duration_ms, 2),
+        )
+
+    async def check_external_api(self) -> ComponentHealth:
+        url = os.getenv("DEX_EXTERNAL_API_URL")
+        if not url:
+            return ComponentHealth(
+                name="external_api",
+                status=HealthStatus.SKIPPED,
+                message="external API not configured",
+            )
+
+        start = time.perf_counter()
+        timeout = httpx.Timeout(self.timeout_seconds)
+        try:
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.get(url)
+            ok = response.status_code < 500
+            message = f"status_code={response.status_code}"
+        except httpx.HTTPError as exc:
+            ok = False
+            message = f"error={exc.__class__.__name__}"
+        duration_ms = (time.perf_counter() - start) * 1000
+        return ComponentHealth(
+            name="external_api",
+            status=HealthStatus.HEALTHY if ok else HealthStatus.UNHEALTHY,
+            message=message,
+            duration_ms=round(duration_ms, 2),
+        )
+
+    async def _tcp_check(self, host: str, port: int) -> tuple[bool, str]:
+        try:
+            await asyncio.wait_for(
+                asyncio.open_connection(host, port), timeout=self.timeout_seconds
+            )
+            return True, "reachable"
+        except (TimeoutError, OSError) as exc:
+            return False, f"error={exc.__class__.__name__}"

dataenginex/api/pagination.py

@@ -0,0 +1,94 @@
+"""
+Cursor-based pagination utilities for the DEX API.
+
+Usage::
+
+    from dataenginex.api.pagination import PaginatedResponse, paginate
+
+    @app.get("/api/v1/items")
+    def list_items(cursor: str | None = None, limit: int = 20):
+        all_items = get_all_items()
+        return paginate(all_items, cursor=cursor, limit=limit)
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, Field
+
+T = TypeVar("T")
+
+
+class PaginationMeta(BaseModel):
+    """Pagination metadata returned alongside results."""
+
+    total: int = Field(description="Total number of items")
+    limit: int = Field(description="Page size")
+    has_next: bool = Field(description="Whether more items exist")
+    next_cursor: str | None = Field(None, description="Opaque cursor for next page")
+    has_previous: bool = Field(default=False, description="Whether previous items exist")
+
+
+class PaginatedResponse(BaseModel):
+    """Generic paginated response wrapper."""
+
+    data: list[Any] = Field(default_factory=list)
+    pagination: PaginationMeta
+
+
+def encode_cursor(offset: int) -> str:
+    """Encode an integer offset into an opaque base64 cursor."""
+    return base64.urlsafe_b64encode(json.dumps({"o": offset}).encode()).decode()
+
+
+def decode_cursor(cursor: str) -> int:
+    """Decode a cursor back to an integer offset.  Returns 0 on failure."""
+    try:
+        data = json.loads(base64.urlsafe_b64decode(cursor))
+        return int(data.get("o", 0))
+    except Exception:
+        return 0
+
+
+def paginate(
+    items: list[Any],
+    *,
+    cursor: str | None = None,
+    limit: int = 20,
+    max_limit: int = 100,
+) -> PaginatedResponse:
+    """Slice *items* and return a ``PaginatedResponse``.
+
+    Parameters
+    ----------
+    items:
+        Full list of items to paginate.
+    cursor:
+        Opaque cursor from a previous response (or *None* for the first page).
+    limit:
+        Number of items per page.
+    max_limit:
+        Hard ceiling on *limit* to prevent abuse.
+    """
+    limit = min(max(1, limit), max_limit)
+    offset = decode_cursor(cursor) if cursor else 0
+    total = len(items)
+
+    page = items[offset : offset + limit]
+    has_next = (offset + limit) < total
+    next_cursor = encode_cursor(offset + limit) if has_next else None
+    has_previous = offset > 0
+
+    return PaginatedResponse(
+        data=page,
+        pagination=PaginationMeta(
+            total=total,
+            limit=limit,
+            has_next=has_next,
+            next_cursor=next_cursor,
+            has_previous=has_previous,
+        ),
+    )

dataenginex/api/rate_limit.py

@@ -0,0 +1,122 @@
+"""
+Rate-limiting middleware for DEX API.
+
+Implements a token-bucket algorithm per client IP.  Configuration:
+
+    DEX_RATE_LIMIT_ENABLED   — "true" to enable (default "false")
+    DEX_RATE_LIMIT_RPM       — Requests per minute per IP (default 60)
+    DEX_RATE_LIMIT_BURST     — Maximum burst size (default 10)
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from loguru import logger
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.responses import Response
+
+
+@dataclass
+class _Bucket:
+    """Token bucket for a single client."""
+
+    tokens: float
+    last_refill: float
+    capacity: float
+    refill_rate: float  # tokens per second
+
+    def consume(self) -> bool:
+        now = time.monotonic()
+        elapsed = now - self.last_refill
+        self.tokens = min(self.capacity, self.tokens + elapsed * self.refill_rate)
+        self.last_refill = now
+        if self.tokens >= 1.0:
+            self.tokens -= 1.0
+            return True
+        return False
+
+
+class RateLimiter:
+    """In-memory token-bucket rate limiter.
+
+    Parameters
+    ----------
+    requests_per_minute:
+        Sustained request rate per client.
+    burst:
+        Maximum instantaneous burst size.
+    """
+
+    def __init__(self, requests_per_minute: int = 60, burst: int = 10) -> None:
+        self.rpm = requests_per_minute
+        self.burst = burst
+        self._buckets: dict[str, _Bucket] = {}
+
+    def allow(self, client_id: str) -> bool:
+        bucket = self._buckets.get(client_id)
+        if bucket is None:
+            bucket = _Bucket(
+                tokens=float(self.burst),
+                last_refill=time.monotonic(),
+                capacity=float(self.burst),
+                refill_rate=self.rpm / 60.0,
+            )
+            self._buckets[client_id] = bucket
+        return bucket.consume()
+
+    def get_stats(self) -> dict[str, Any]:
+        return {
+            "rpm": self.rpm,
+            "burst": self.burst,
+            "active_clients": len(self._buckets),
+        }
+
+    def cleanup(self, max_age_seconds: float = 300.0) -> int:
+        """Evict buckets idle for more than *max_age_seconds*."""
+        now = time.monotonic()
+        stale = [k for k, b in self._buckets.items() if now - b.last_refill > max_age_seconds]
+        for k in stale:
+            del self._buckets[k]
+        return len(stale)
+
+
+# Paths exempt from rate limiting
+_EXEMPT_PATHS: set[str] = {"/health", "/ready", "/startup", "/metrics"}
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Starlette middleware applying per-IP rate limiting."""
+
+    def __init__(self, app: Any) -> None:  # noqa: ANN401
+        super().__init__(app)
+        rpm = int(os.getenv("DEX_RATE_LIMIT_RPM", "60"))
+        burst = int(os.getenv("DEX_RATE_LIMIT_BURST", "10"))
+        self._limiter = RateLimiter(requests_per_minute=rpm, burst=burst)
+        self._enabled = os.getenv("DEX_RATE_LIMIT_ENABLED", "false").lower() == "true"
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if not self._enabled:
+            return await call_next(request)
+
+        if request.url.path in _EXEMPT_PATHS:
+            return await call_next(request)
+
+        client_ip = request.client.host if request.client else "unknown"
+        if not self._limiter.allow(client_ip):
+            logger.warning("Rate limit exceeded for %s", client_ip)
+            return JSONResponse(
+                status_code=429,
+                content={
+                    "error": "rate_limit_exceeded",
+                    "message": "Too many requests — please slow down",
+                },
+                headers={"Retry-After": "60"},
+            )
+
+        return await call_next(request)