sql-dialect-master 1.1.1__py3-none-any.whl

backend/api/middleware.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Middleware for SQL Dialect Master API.
+
+Provides rate limiting, request-size protection, logging, and security headers.
+"""
+import json
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Dict
+
+from fastapi import Request, Response
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from backend.core.exceptions import ErrorCode
+
+from .rate_limit_store import RateLimitStore, create_rate_limit_store
+from .readiness import health_probe_response
+
+logger = logging.getLogger(__name__)
+DEFAULT_MAX_REQUEST_BODY_BYTES = 512 * 1024
+
+
+def _sanitize_log_value(value: str, max_length: int = 500) -> str:
+    """Prevent control characters from forging or corrupting log records."""
+    return value.replace("\r", "\\r").replace("\n", "\\n")[:max_length]
+
+
+@dataclass
+class RateLimitEntry:
+    """Legacy compatibility entry."""
+    requests: int = 0
+    window_start: float = field(default_factory=time.time)
+
+
+class RateLimiter:
+    """Rate limiter backed by a local or shared store."""
+
+    def __init__(self, requests_per_window: int = 100, window_seconds: int = 60, store: RateLimitStore = None):
+        if requests_per_window <= 0:
+            raise ValueError("requests_per_window must be positive")
+        if window_seconds <= 0:
+            raise ValueError("window_seconds must be positive")
+        self._requests_per_window = requests_per_window
+        self._window_seconds = window_seconds
+        self._store = store or create_rate_limit_store()
+
+    @property
+    def _limits(self):
+        """Read-only compatibility view for the in-memory backend."""
+        return getattr(self._store, "_entries", {})
+
+    def is_allowed(self, client_id: str) -> tuple:
+        return self._store.check(client_id, self._requests_per_window, self._window_seconds)
+
+    def get_stats(self) -> Dict:
+        stats = self._store.stats()
+        stats.update({"requests_per_window": self._requests_per_window, "window_seconds": self._window_seconds})
+        return stats
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Rate limiting middleware with a bounded request-body guard."""
+
+    def __init__(self, app, limiter: RateLimiter = None, enabled: bool = True, max_body_bytes: int = DEFAULT_MAX_REQUEST_BODY_BYTES):
+        super().__init__(app)
+        if max_body_bytes <= 0:
+            raise ValueError("max_body_bytes must be positive")
+        self.limiter = limiter or RateLimiter()
+        self.enabled = enabled
+        self.max_body_bytes = max_body_bytes
+
+    async def _read_bounded_body(self, request: Request) -> Response | None:
+        """Consume and cache request bodies without trusting Content-Length."""
+        if request.method in {"GET", "HEAD"}:
+            return None
+        content_length = request.headers.get("content-length")
+        if content_length:
+            try:
+                declared_length = int(content_length)
+            except ValueError:
+                return JSONResponse(status_code=400, content={"success": False, "error": {"code": 400, "message": "Invalid Content-Length"}, "timestamp": datetime.now().isoformat()})
+            if declared_length > self.max_body_bytes:
+                return JSONResponse(status_code=413, content={"success": False, "error": {"code": 413, "message": "Request body exceeds the maximum allowed size", "max_bytes": self.max_body_bytes}, "timestamp": datetime.now().isoformat()})
+
+        total = 0
+        chunks = []
+        async for chunk in request.stream():
+            total += len(chunk)
+            if total > self.max_body_bytes:
+                return JSONResponse(status_code=413, content={"success": False, "error": {"code": 413, "message": "Request body exceeds the maximum allowed size", "max_bytes": self.max_body_bytes}, "timestamp": datetime.now().isoformat()})
+            chunks.append(chunk)
+        request._body = b"".join(chunks)
+        return None
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        body_error = await self._read_bounded_body(request)
+        if body_error is not None:
+            return body_error
+
+        if request.url.path == "/api/nl2sql" and request.method == "POST":
+            try:
+                body = await request.body()
+                payload = json.loads(body.decode("utf-8")) if body else {}
+            except (UnicodeDecodeError, json.JSONDecodeError):
+                payload = None
+            if isinstance(payload, dict):
+                from backend.core.column_hint_validation import validate_column_hints
+                error = validate_column_hints(payload.get("column_hints"))
+                if error:
+                    return JSONResponse(
+                        status_code=422,
+                        content={"success": False, "error": {"code": "VALIDATION_FAILED", "message": error}},
+                    )
+
+        if not self.enabled:
+            return await call_next(request)
+        if request.url.path in {"/health", "/", "/ready", "/health/deep"} and request.method in {"GET", "HEAD"}:
+            if request.url.path in {"/ready", "/health/deep"}:
+                return await health_probe_response(request)
+            return await call_next(request)
+
+        client_id = self._get_client_id(request)
+        is_allowed, remaining, reset_time = self.limiter.is_allowed(client_id)
+        if not is_allowed:
+            logger.warning("Rate limit exceeded for client: %s", _sanitize_log_value(client_id))
+            return JSONResponse(
+                status_code=429,
+                content={"success": False, "error": {"code": 429, "message": "Rate limit exceeded", "retry_after": reset_time}, "timestamp": datetime.now().isoformat()},
+                headers={"X-RateLimit-Limit": str(self.limiter._requests_per_window), "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": str(reset_time), "Retry-After": str(reset_time)},
+            )
+        response = await call_next(request)
+        response.headers["X-RateLimit-Limit"] = str(self.limiter._requests_per_window)
+        response.headers["X-RateLimit-Remaining"] = str(remaining)
+        response.headers["X-RateLimit-Reset"] = str(reset_time)
+        return response
+
+    def _get_client_id(self, request: Request) -> str:
+        return request.client.host if request.client else "unknown"
+
+
+class StructuredLoggingMiddleware(BaseHTTPMiddleware):
+    """Structured request logging and request-id propagation."""
+
+    def __init__(self, app, log_body: bool = False):
+        super().__init__(app)
+        self.log_body = log_body
+
+    @staticmethod
+    def _get_request_id(request: Request) -> str:
+        """Reuse a valid RFC 4122 request ID; otherwise issue a fresh UUID."""
+        candidate = request.headers.get("X-Request-ID")
+        if candidate:
+            try:
+                parsed = uuid.UUID(candidate)
+                if parsed.variant == uuid.RFC_4122:
+                    return candidate
+            except (ValueError, AttributeError, TypeError):
+                pass
+        return str(uuid.uuid4())
+
+    @staticmethod
+    async def _normalize_error_response(response: Response, request_id: str) -> Response:
+        """Normalize standard HTTP 4xx JSON responses while preserving domain-specific codes."""
+        code_by_status = {
+            400: ErrorCode.BAD_REQUEST.value,
+            404: ErrorCode.NOT_FOUND.value,
+            405: ErrorCode.METHOD_NOT_ALLOWED.value,
+            413: ErrorCode.PAYLOAD_TOO_LARGE.value,
+            422: ErrorCode.VALIDATION_FAILED.value,
+            429: ErrorCode.RATE_LIMITED.value,
+        }
+        if response.status_code not in code_by_status:
+            return response
+
+        try:
+            if hasattr(response, "body"):
+                raw_body = response.body
+            else:
+                chunks = []
+                async for chunk in response.body_iterator:
+                    chunks.append(chunk)
+                raw_body = b"".join(chunks)
+            payload = json.loads(raw_body.decode("utf-8"))
+        except (AttributeError, UnicodeDecodeError, json.JSONDecodeError):
+            return response
+
+        error = payload.get("error") if isinstance(payload, dict) else None
+        expected_code = code_by_status[response.status_code]
+        if not isinstance(error, dict):
+            detail = payload.get("detail") if isinstance(payload, dict) else None
+            if response.status_code == 422:
+                error = {
+                    "code": expected_code,
+                    "message": "Request validation failed",
+                    "details": detail if isinstance(detail, list) else ([detail] if detail else []),
+                }
+            elif response.status_code == 413:
+                error = {
+                    "code": expected_code,
+                    "message": str(detail or "Request body exceeds the maximum allowed size"),
+                    "details": {},
+                }
+            elif response.status_code == 429:
+                error = {
+                    "code": expected_code,
+                    "message": str(detail or "Rate limit exceeded"),
+                    "details": {},
+                }
+            elif response.status_code == 405:
+                error = {
+                    "code": expected_code,
+                    "message": str(detail or "Method not allowed"),
+                    "details": {},
+                }
+            else:
+                error = {
+                    "code": expected_code,
+                    "message": str(detail or ("Resource not found" if response.status_code == 404 else "Bad request")),
+                    "details": {},
+                }
+        else:
+            code = error.get("code")
+            if not isinstance(code, str):
+                error["code"] = expected_code
+            error.setdefault("details", {})
+
+        normalized = {
+            "success": False,
+            "error": error,
+            "timestamp": payload.get("timestamp", datetime.now().isoformat()) if isinstance(payload, dict) else datetime.now().isoformat(),
+            "request_id": request_id,
+        }
+        headers = dict(response.headers)
+        headers.pop("content-length", None)
+        headers.pop("content-type", None)
+        return JSONResponse(status_code=response.status_code, content=normalized, headers=headers)
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        start_time = time.time()
+        request_id = self._get_request_id(request)
+        request.state.request_id = request_id
+        status_code = 500
+        error = None
+        log_data = {"event": "request_start", "request_id": request_id, "method": _sanitize_log_value(request.method), "path": _sanitize_log_value(request.url.path), "query": _sanitize_log_value(str(request.query_params)), "client": _sanitize_log_value(request.client.host if request.client else "unknown"), "timestamp": datetime.now().isoformat()}
+        logger.info(json.dumps(log_data))
+        try:
+            response = await call_next(request)
+            response = await self._normalize_error_response(response, request_id)
+            status_code = response.status_code
+        except Exception as exc:
+            error = _sanitize_log_value(str(exc))
+            raise
+        finally:
+            process_time = time.time() - start_time
+            log_data = {"event": "request_end", "request_id": request_id, "method": _sanitize_log_value(request.method), "path": _sanitize_log_value(request.url.path), "status_code": status_code, "process_time_ms": round(process_time * 1000, 2), "error": error, "timestamp": datetime.now().isoformat()}
+            (logger.warning if status_code >= 400 else logger.info)(json.dumps(log_data))
+        response.headers["X-Request-ID"] = request_id
+        return response
+
+
+class SecurityHeadersMiddleware(BaseHTTPMiddleware):
+    """Add security headers to responses."""
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        response = await call_next(request)
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        response.headers["X-Frame-Options"] = "DENY"
+        response.headers["X-XSS-Protection"] = "1; mode=block"
+        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+        return response

backend/api/rate_limit_store.py

@@ -0,0 +1,105 @@
+"""Shared rate-limit backends.
+
+Redis is opt-in. The in-memory backend remains the default for local/single-process use.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from threading import Lock
+from typing import Protocol
+
+
+class RateLimitStore(Protocol):
+    """Minimal storage contract used by RateLimiter."""
+
+    def check(self, key: str, limit: int, window_seconds: int) -> tuple[bool, int, int]:
+        """Return allowed, remaining, and reset seconds."""
+
+    def stats(self) -> dict:
+        """Return storage metadata."""
+
+
+class InMemoryRateLimitStore:
+    """Thread-safe fixed-window store for a single process."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, tuple[int, float]] = {}
+        self._expirations: dict[str, float] = {}
+        self._next_cleanup_at = 0.0
+        self._lock = Lock()
+
+    def _prune_expired(self, now: float) -> None:
+        if now < self._next_cleanup_at:
+            return
+        expired_keys = [
+            key for key, expires_at in self._expirations.items() if expires_at <= now
+        ]
+        for key in expired_keys:
+            self._entries.pop(key, None)
+            self._expirations.pop(key, None)
+        self._next_cleanup_at = min(self._expirations.values(), default=float("inf"))
+
+    def check(self, key: str, limit: int, window_seconds: int) -> tuple[bool, int, int]:
+        with self._lock:
+            now = time.time()
+            self._prune_expired(now)
+            count, window_start = self._entries.get(key, (0, now))
+            if now - window_start >= window_seconds:
+                count, window_start = 0, now
+            count += 1
+            expires_at = window_start + window_seconds
+            self._entries[key] = (count, window_start)
+            self._expirations[key] = expires_at
+            self._next_cleanup_at = min(self._next_cleanup_at, expires_at)
+            return count <= limit, max(0, limit - count), max(0, int(expires_at - now))
+
+    def stats(self) -> dict:
+        with self._lock:
+            self._prune_expired(time.time())
+            return {"backend": "memory", "active_clients": len(self._entries)}
+
+
+class RedisRateLimitStore:
+    """Atomic fixed-window store backed by Redis for multi-worker deployments."""
+
+    _SCRIPT = """
+    local count = redis.call('INCR', KEYS[1])
+    if count == 1 then
+        redis.call('EXPIRE', KEYS[1], ARGV[1])
+    end
+    local ttl = redis.call('TTL', KEYS[1])
+    return {count, ttl}
+    """
+
+    def __init__(self, url: str, namespace: str = "sdm:rate-limit:") -> None:
+        if not url:
+            raise ValueError("Redis URL is required when rate-limit backend is redis")
+        try:
+            import redis
+        except ImportError as exc:
+            raise RuntimeError("Redis rate limiting requires the optional 'redis' package") from exc
+        self._client = redis.Redis.from_url(url, decode_responses=False)
+        self._client.ping()
+        self._script = self._client.register_script(self._SCRIPT)
+        self._namespace = namespace
+
+    def check(self, key: str, limit: int, window_seconds: int) -> tuple[bool, int, int]:
+        values = self._script(keys=[f"{self._namespace}{key}"], args=[window_seconds])
+        count = int(values[0])
+        ttl = max(0, int(values[1]))
+        return count <= limit, max(0, limit - count), ttl
+
+    def stats(self) -> dict:
+        return {"backend": "redis", "namespace": self._namespace}
+
+
+def create_rate_limit_store() -> RateLimitStore:
+    """Create the configured store; never silently downgrade Redis to memory."""
+    backend = os.getenv("SDM_RATE_LIMIT_BACKEND", "memory").strip().lower()
+    if backend == "memory":
+        return InMemoryRateLimitStore()
+    if backend == "redis":
+        return RedisRateLimitStore(os.getenv("SDM_REDIS_URL", ""))
+    raise ValueError("Unsupported SDM_RATE_LIMIT_BACKEND; expected 'memory' or 'redis'")

backend/api/readiness.py

@@ -0,0 +1,178 @@
+"""Dependency-backed readiness probe for the API service."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import secrets
+import time
+import weakref
+from datetime import datetime
+from typing import Any
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+logger = logging.getLogger(__name__)
+
+_READINESS_CACHE_TTL_SECONDS = 5.0
+_READINESS_TIMEOUT_SECONDS = 3.0
+_READINESS_LOCKS: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary()
+_READINESS_LOCKS_GUARD = __import__("threading").Lock()
+_READINESS_CACHE = {"checks": None, "cached_at": 0.0}
+_HEALTH_PROBE_PATHS = {"/ready", "/health/deep"}
+
+
+def _probe_allowed(request: Request) -> bool:
+    """Allow health probes only from loopback or with the configured probe secret."""
+    token = os.getenv("SDM_HEALTH_PROBE_TOKEN", "").strip()
+    supplied = request.headers.get("X-Health-Probe-Token", "")
+    if token and supplied and secrets.compare_digest(supplied, token):
+        return True
+    client = request.client
+    return bool(client and client.host in {"127.0.0.1", "::1", "localhost"})
+
+
+def _health_probe_denied_response() -> JSONResponse:
+    """Return a stable denial response for public health probe requests."""
+    return JSONResponse(
+        status_code=403,
+        content={
+            "success": False,
+            "error": {"code": 403, "message": "health probe access denied"},
+        },
+    )
+
+
+async def health_probe_response(request: Request) -> JSONResponse:
+    """Authorize and dispatch a readiness/deep-health probe request."""
+    if request.url.path not in _HEALTH_PROBE_PATHS or request.method not in {"GET", "HEAD"}:
+        raise ValueError("health_probe_response called for a non-probe request")
+    if not _probe_allowed(request):
+        return _health_probe_denied_response()
+    if request.url.path == "/ready":
+        return await readiness_response(request)
+    return await deep_health_response(request)
+
+
+def _get_readiness_lock() -> asyncio.Lock:
+    loop = asyncio.get_running_loop()
+    with _READINESS_LOCKS_GUARD:
+        lock = _READINESS_LOCKS.get(loop)
+        if lock is None:
+            lock = asyncio.Lock()
+            _READINESS_LOCKS[loop] = lock
+        return lock
+
+
+def _run_checks() -> dict[str, dict[str, Any]]:
+    """Run the lightweight-but-real component checks synchronously."""
+    from backend.core.functions_lookup import FunctionEncyclopedia
+    from backend.core.nl2sql import NL2SQLGenerator
+    from backend.core.transpiler import SQLTranspiler
+    from backend.core.type_mapping import TypeMapper
+
+    transpiler = SQLTranspiler()
+    function_encyclopedia = FunctionEncyclopedia()
+    type_mapper = TypeMapper()
+    nl2sql_generator = NL2SQLGenerator()
+
+    checks: dict[str, dict[str, Any]] = {}
+    try:
+        result = transpiler.transpile("SELECT 1 AS readiness_probe", "mysql", "postgres")
+        checks["transpiler"] = {"status": "ok" if result.success else "error", "test_result": result.success}
+    except Exception:
+        logger.exception("Readiness transpiler probe failed")
+        checks["transpiler"] = {"status": "error", "code": "probe_failed"}
+    try:
+        function = function_encyclopedia.get_function("CONCAT")
+        checks["functions"] = {"status": "ok" if function else "error", "sample_lookup": "CONCAT" if function else None}
+    except Exception:
+        logger.exception("Readiness function probe failed")
+        checks["functions"] = {"status": "error", "code": "probe_failed"}
+    try:
+        mapping = type_mapper.map_type("VARCHAR", "mysql", "postgres")
+        checks["types"] = {"status": "ok" if mapping.get("success") else "error", "sample_mapping": mapping.get("target_type")}
+    except Exception:
+        logger.exception("Readiness type probe failed")
+        checks["types"] = {"status": "error", "code": "probe_failed"}
+    try:
+        generated = nl2sql_generator.generate("查询所有用户", "mysql")
+        checks["nl2sql"] = {"status": "ok" if generated.success else "error", "confidence": generated.confidence}
+    except Exception:
+        logger.exception("Readiness NL2SQL probe failed")
+        checks["nl2sql"] = {"status": "error", "code": "probe_failed"}
+    return checks
+
+
+async def _get_checks() -> dict[str, dict[str, Any]]:
+    """Run at most one expensive probe at a time per event loop and reuse it briefly."""
+    now = time.monotonic()
+    cached_checks = globals().get("_cached_checks", _READINESS_CACHE["checks"])
+    cached_at = globals().get("_cached_at", _READINESS_CACHE["cached_at"])
+    if cached_checks is not None and now - cached_at < _READINESS_CACHE_TTL_SECONDS:
+        return cached_checks
+    async with _get_readiness_lock():
+        now = time.monotonic()
+        cached_checks = globals().get("_cached_checks", _READINESS_CACHE["checks"])
+        cached_at = globals().get("_cached_at", _READINESS_CACHE["cached_at"])
+        if cached_checks is not None and now - cached_at < _READINESS_CACHE_TTL_SECONDS:
+            return cached_checks
+        try:
+            checks = await asyncio.wait_for(asyncio.to_thread(_run_checks), timeout=_READINESS_TIMEOUT_SECONDS)
+        except asyncio.TimeoutError:
+            logger.error("Readiness probe timed out after %.1fs", _READINESS_TIMEOUT_SECONDS)
+            checks = {name: {"status": "error", "code": "probe_timeout"} for name in ("transpiler", "functions", "types", "nl2sql")}
+        except Exception:
+            logger.exception("Readiness probe failed unexpectedly")
+            checks = {"probe": {"status": "error", "code": "probe_failed"}}
+        _READINESS_CACHE["checks"] = checks
+        _READINESS_CACHE["cached_at"] = time.monotonic()
+        globals()["_cached_checks"] = checks
+        globals()["_cached_at"] = _READINESS_CACHE["cached_at"]
+        return checks
+
+
+def _response_status(checks: dict[str, dict[str, Any]]) -> tuple[int, bool]:
+    """Return the HTTP status and health flag for a probe result."""
+    failed = [name for name, check in checks.items() if check.get("status") != "ok"]
+    return (200 if not failed else 503, not failed)
+
+
+async def readiness_response(request: Request) -> JSONResponse:
+    """Return HTTP 200 only when all core service probes pass."""
+    checks = await _get_checks()
+    status_code, healthy = _response_status(checks)
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "status": "ready" if healthy else "not_ready",
+            "version": _api_version(),
+            "probe": "readiness",
+            "checks": checks,
+            "timestamp": datetime.now().isoformat(),
+        },
+    )
+
+
+async def deep_health_response(request: Request) -> JSONResponse:
+    """Return deep health using the same cached dependency probes as readiness."""
+    checks = await _get_checks()
+    status_code, healthy = _response_status(checks)
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "status": "✅ healthy" if healthy else "❌ unhealthy",
+            "version": _api_version(),
+            "checks": checks,
+            "timestamp": datetime.now().isoformat(),
+        },
+    )
+
+
+def _api_version() -> str:
+    """Resolve API version without importing the API module back into readiness."""
+    from backend.core.config import settings
+
+    return settings.api_version

backend/core/__init__.py

@@ -0,0 +1,57 @@
+# SQL Dialect Master - Core Module
+from .config import (
+    setup_logging,
+    SUPPORTED_DIALECTS,
+    DIALECT_METADATA,
+    settings,
+    AppSettings,
+    get_dialect_ui_info,
+    get_dialect_api_info,
+    get_dialect_label,
+)
+from .parser import SQLParser, ParseResult
+from .transpiler import SQLTranspiler, TranspileResult
+from .semantic_diff import SQLSemanticDiffer, SemanticDiff, diff_sql_ast
+from .post_processor import PostProcessor
+from .functions_lookup import FunctionEncyclopedia
+from .type_mapping import TypeMapper
+from .nl2sql import NL2SQLGenerator, NL2SQLResult
+from .cache import TTLCache
+from .exceptions import (
+    SDMException,
+    UnsupportedDialectError,
+    TranspileError,
+    ParseError,
+    SecurityViolationError,
+    ValidationError,
+)
+
+__all__ = [
+    "setup_logging",
+    "SUPPORTED_DIALECTS",
+    "DIALECT_METADATA",
+    "settings",
+    "AppSettings",
+    "get_dialect_ui_info",
+    "get_dialect_api_info",
+    "get_dialect_label",
+    "SQLParser",
+    "ParseResult",
+    "SQLTranspiler",
+    "TranspileResult",
+    "SQLSemanticDiffer",
+    "SemanticDiff",
+    "diff_sql_ast",
+    "TTLCache",
+    "PostProcessor",
+    "FunctionEncyclopedia",
+    "TypeMapper",
+    "NL2SQLGenerator",
+    "NL2SQLResult",
+    "SDMException",
+    "UnsupportedDialectError",
+    "TranspileError",
+    "ParseError",
+    "SecurityViolationError",
+    "ValidationError",
+]