sql-dialect-master 1.1.1__tar.gz → 1.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/PKG-INFO +11 -3
  2. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/README.md +8 -1
  3. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/api/main.py +51 -10
  4. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/api/middleware.py +34 -15
  5. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/api/rate_limit_store.py +44 -9
  6. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/api/readiness.py +25 -7
  7. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/cache.py +12 -2
  8. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/config.py +46 -14
  9. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/metadata.py +2 -2
  10. sql_dialect_master-1.1.3/backend/core/nl2sql_components/aggregations.py +52 -0
  11. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/core_extract.py +42 -9
  12. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/relations.py +99 -9
  13. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/templates.py +7 -2
  14. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/orchestrator.py +88 -10
  15. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/predicates.py +61 -1
  16. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/renderer.py +69 -24
  17. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/resolution.py +12 -3
  18. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/templates.py +55 -5
  19. sql_dialect_master-1.1.3/backend/core/nl2sql_pipeline/validation.py +76 -0
  20. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/p1_sql_scanner.py +18 -6
  21. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/parser.py +83 -3
  22. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/rules.py +216 -15
  23. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/transpiler.py +637 -66
  24. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/type_mapping.py +40 -3
  25. sql_dialect_master-1.1.3/backend/frontend_entrypoint.py +22 -0
  26. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/utils/validators.py +87 -10
  27. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/design_tokens.py +7 -3
  28. sql_dialect_master-1.1.3/frontend/core/fingerprints.py +24 -0
  29. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/navigation.py +22 -0
  30. sql_dialect_master-1.1.3/frontend/core/sql_inputs.py +29 -0
  31. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/state.py +81 -26
  32. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/themes.py +7 -1
  33. sql_dialect_master-1.1.3/frontend/core/type_status.py +26 -0
  34. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/viewmodels.py +22 -2
  35. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/convert.py +105 -16
  36. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/diff.py +15 -2
  37. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/functions.py +2 -3
  38. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/history.py +9 -6
  39. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/lineage.py +31 -14
  40. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/nl2sql.py +17 -17
  41. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/query_analysis.py +12 -4
  42. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/runtime.py +3 -3
  43. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/templates.py +21 -2
  44. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/types.py +2 -6
  45. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/ui/command_palette.py +21 -2
  46. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/ui/editor.py +1 -1
  47. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/ui/status.py +6 -6
  48. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/pyproject.toml +13 -2
  49. sql_dialect_master-1.1.3/sdm_local_v2.py +219 -0
  50. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/sql_dialect_master.egg-info/PKG-INFO +11 -3
  51. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/sql_dialect_master.egg-info/SOURCES.txt +6 -0
  52. sql_dialect_master-1.1.3/sql_dialect_master.egg-info/entry_points.txt +2 -0
  53. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/sql_dialect_master.egg-info/requires.txt +2 -1
  54. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/sql_dialect_master.egg-info/top_level.txt +1 -0
  55. sql_dialect_master-1.1.1/backend/core/nl2sql_components/aggregations.py +0 -35
  56. sql_dialect_master-1.1.1/backend/core/nl2sql_pipeline/validation.py +0 -28
  57. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/LICENSE +0 -0
  58. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/__init__.py +0 -0
  59. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/api/__init__.py +0 -0
  60. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/__init__.py +0 -0
  61. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/column_hint_validation.py +0 -0
  62. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/constants.py +0 -0
  63. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/exceptions.py +0 -0
  64. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/function_call_scanner.py +0 -0
  65. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/function_semantics.py +0 -0
  66. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/functions_db.json +0 -0
  67. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/functions_lookup.py +0 -0
  68. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql.py +0 -0
  69. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/__init__.py +0 -0
  70. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/boolean_conditions.py +0 -0
  71. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/evidence.py +0 -0
  72. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/mappings.py +0 -0
  73. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_components/tokenizer.py +0 -0
  74. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_legacy.py +0 -0
  75. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/__init__.py +0 -0
  76. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/aggregations.py +0 -0
  77. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/clause_extract.py +0 -0
  78. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/dialect.py +0 -0
  79. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/intent.py +0 -0
  80. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/ir.py +0 -0
  81. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/models.py +0 -0
  82. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/nl2sql_pipeline/suggestions.py +0 -0
  83. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/p2_data_validation.py +0 -0
  84. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/post_processor.py +0 -0
  85. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/semantic_diff.py +0 -0
  86. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/core/type_mapping.json +0 -0
  87. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/backend/utils/__init__.py +0 -0
  88. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/__init__.py +0 -0
  89. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/app_context_v2.py +0 -0
  90. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/__init__.py +0 -0
  91. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/escaping.py +0 -0
  92. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/core/styles.py +0 -0
  93. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/__init__.py +0 -0
  94. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/pages/settings.py +0 -0
  95. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/templates_v2.py +0 -0
  96. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/ui/__init__.py +0 -0
  97. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/frontend/ui/diff.py +0 -0
  98. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/setup.cfg +0 -0
  99. {sql_dialect_master-1.1.1 → sql_dialect_master-1.1.3}/sql_dialect_master.egg-info/dependency_links.txt +0 -0
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sql-dialect-master
3
- Version: 1.1.1
3
+ Version: 1.1.3
4
4
  Summary: SQL Dialect Master - Multi-database SQL conversion engine
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
7
7
  License-File: LICENSE
8
- Requires-Dist: sqlglot>=25.0.0
8
+ Requires-Dist: sqlglot<31.0.0,>=30.18.0
9
9
  Requires-Dist: streamlit>=1.38.0
10
10
  Requires-Dist: fastapi>=0.115.0
11
11
  Requires-Dist: uvicorn>=0.30.0
@@ -16,6 +16,7 @@ Requires-Dist: hypothesis>=6.112.0; extra == "dev"
16
16
  Requires-Dist: pytest>=8.3.0; extra == "dev"
17
17
  Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
18
18
  Requires-Dist: httpx>=0.27.0; extra == "dev"
19
+ Requires-Dist: requests>=2.31.0; extra == "dev"
19
20
  Provides-Extra: qa
20
21
  Requires-Dist: ruff>=0.6; extra == "qa"
21
22
  Requires-Dist: bandit>=1.7; extra == "qa"
@@ -31,7 +32,7 @@ Dynamic: license-file
31
32
 
32
33
  Enterprise-grade multi-database SQL conversion engine supporting 12 database dialects.
33
34
 
34
- Current release: 1.1.1
35
+ Current release: 1.1.3
35
36
 
36
37
  [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
37
38
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -95,6 +96,13 @@ python -m pip install -e ".[dev,semantic]"
95
96
  streamlit run sdm_local_v2.py
96
97
  ```
97
98
 
99
+ After installing the package, the same workspace can be launched from any
100
+ working directory with:
101
+
102
+ ```bash
103
+ sdm-streamlit
104
+ ```
105
+
98
106
  Open http://localhost:8501 in your browser.
99
107
 
100
108
  ### Run FastAPI Backend
@@ -2,7 +2,7 @@
2
2
 
3
3
  Enterprise-grade multi-database SQL conversion engine supporting 12 database dialects.
4
4
 
5
- Current release: 1.1.1
5
+ Current release: 1.1.3
6
6
 
7
7
  [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
8
8
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -66,6 +66,13 @@ python -m pip install -e ".[dev,semantic]"
66
66
  streamlit run sdm_local_v2.py
67
67
  ```
68
68
 
69
+ After installing the package, the same workspace can be launched from any
70
+ working directory with:
71
+
72
+ ```bash
73
+ sdm-streamlit
74
+ ```
75
+
69
76
  Open http://localhost:8501 in your browser.
70
77
 
71
78
  ### Run FastAPI Backend
@@ -11,6 +11,7 @@ Enterprise-grade multi-database SQL conversion API:
11
11
  Exact counts are derived from the canonical data files at runtime —
12
12
  see backend.core.metadata — and must not be restated here.
13
13
  """
14
+ import asyncio
14
15
  import logging
15
16
  import time
16
17
  from datetime import datetime
@@ -19,22 +20,41 @@ from typing import Any, Dict, List, Optional
19
20
  from fastapi import FastAPI, HTTPException, Query, Request
20
21
  from fastapi.middleware.cors import CORSMiddleware
21
22
  from fastapi.responses import JSONResponse
23
+ from starlette.concurrency import run_in_threadpool
22
24
  from pydantic import BaseModel, ConfigDict, Field
23
25
 
24
26
  from backend.core.config import get_dialect_api_info, settings, setup_logging
25
27
  from backend.core import metadata as meta
26
28
  from backend.core.functions_lookup import FunctionEncyclopedia
27
29
  from backend.core.nl2sql import NL2SQLGenerator
30
+ from backend.core.nl2sql_pipeline.orchestrator import SAFE_TABLE_HINT
28
31
  from backend.core.parser import SUPPORTED_DIALECTS, SQLParser
29
32
  from backend.core.transpiler import SQLTranspiler
30
33
  from backend.core.type_mapping import TypeMapper
31
34
 
32
35
  from backend.api.readiness import health_probe_response
33
36
 
34
- # Configure logging for the application
35
- logger = setup_logging(level=logging.INFO)
37
+ # Configure logging for the application from the validated settings object.
38
+ # Keep an INFO fallback for defensive compatibility with custom settings
39
+ # objects, while normal AppSettings values are resolved from SDM_LOG_LEVEL.
40
+ _log_level = getattr(logging, settings.log_level.upper(), logging.INFO)
41
+ logger = setup_logging(level=_log_level, log_file=settings.log_file or None)
36
42
  api_logger = logging.getLogger(__name__)
37
43
 
44
+
45
+ async def _run_cpu_bound(func, *args):
46
+ """Run parser/transpiler work off-loop with the configured timeout."""
47
+ try:
48
+ return await asyncio.wait_for(
49
+ run_in_threadpool(func, *args),
50
+ timeout=settings.request_timeout,
51
+ )
52
+ except asyncio.TimeoutError as exc:
53
+ raise HTTPException(
54
+ status_code=504,
55
+ detail="Request processing timed out",
56
+ ) from exc
57
+
38
58
  # API metadata
39
59
  API_VERSION = settings.api_version
40
60
  API_TITLE = settings.api_title
@@ -198,7 +218,10 @@ class ConvertRequest(BaseModel):
198
218
  sql: str = Field(..., description="Source SQL statement to convert")
199
219
  source_dialect: str = Field(..., description="Source database dialect")
200
220
  target_dialect: str = Field(..., description="Target database dialect")
201
- pretty: bool = Field(True, description="Format output SQL with indentation")
221
+ pretty: bool = Field(
222
+ settings.transpiler_pretty_default,
223
+ description="Format output SQL with indentation",
224
+ )
202
225
 
203
226
  model_config = ConfigDict(json_schema_extra={
204
227
  "example": {
@@ -401,11 +424,14 @@ async def convert_sql(request: ConvertRequest):
401
424
  detail=f"Unsupported target dialect: {request.target_dialect}. Supported: {SUPPORTED_DIALECTS}"
402
425
  )
403
426
 
404
- result = transpiler.transpile(
427
+ # sqlglot parsing/transpilation is CPU-bound. Keep it off the event loop
428
+ # so one large request cannot stall unrelated async requests.
429
+ result = await _run_cpu_bound(
430
+ transpiler.transpile,
405
431
  request.sql,
406
432
  source_dialect,
407
433
  target_dialect,
408
- request.pretty
434
+ request.pretty,
409
435
  )
410
436
  return ConvertResponse(
411
437
  success=result.success,
@@ -430,7 +456,10 @@ async def parse_sql(request: ParseRequest):
430
456
  """
431
457
  dialect = normalize_dialect(request.dialect)
432
458
  parser = SQLParser(dialect)
433
- elements = parser.extract_elements(request.sql)
459
+ parsed = await _run_cpu_bound(parser.parse, request.sql)
460
+ if not parsed.success:
461
+ raise HTTPException(status_code=422, detail=parsed.error or "SQL parsing failed")
462
+ elements = await _run_cpu_bound(parser.extract_elements, request.sql)
434
463
  return {
435
464
  "success": True,
436
465
  "dialect": dialect,
@@ -457,7 +486,7 @@ async def list_functions(
457
486
  if search:
458
487
  functions = func_encyclopedia.search(search, limit)
459
488
  elif category:
460
- functions = func_encyclopedia.list_by_category(category)
489
+ functions = func_encyclopedia.list_by_category(category)[:limit]
461
490
  else:
462
491
  functions = func_encyclopedia.functions[:limit]
463
492
 
@@ -568,7 +597,7 @@ async def map_type(request: TypeMapRequest):
568
597
  target_dialect,
569
598
  )
570
599
  return {
571
- "success": True,
600
+ "success": result.get("success", False),
572
601
  **result,
573
602
  "timestamp": datetime.now().isoformat()
574
603
  }
@@ -586,7 +615,19 @@ async def generate_sql(request: NL2SQLRequest):
586
615
  - "Get top 10 users by score" → SELECT * FROM users ORDER BY score DESC LIMIT 10
587
616
  """
588
617
  dialect = normalize_dialect(request.dialect)
589
- result = nl2sql_generator.generate(
618
+ # Validate table_hint at the API boundary: an illegal identifier is
619
+ # a client error (4xx), never an internal ValueError → 500. The
620
+ # same rule backs the orchestrator's direct-caller ValueError.
621
+ if request.table_hint and not SAFE_TABLE_HINT.fullmatch(request.table_hint.strip()):
622
+ raise HTTPException(
623
+ status_code=400,
624
+ detail=(
625
+ "Invalid table_hint: must be a simple SQL identifier or "
626
+ "dotted identifier"
627
+ ),
628
+ )
629
+ result = await _run_cpu_bound(
630
+ nl2sql_generator.generate,
590
631
  request.text,
591
632
  dialect,
592
633
  request.table_hint,
@@ -599,7 +640,7 @@ async def generate_sql(request: NL2SQLRequest):
599
640
  dialect=dialect,
600
641
  explanation=result.explanation,
601
642
  confidence=result.confidence,
602
- suggestions=result.suggestions
643
+ suggestions=list(result.suggestions[: settings.nl2sql_max_suggestions])
603
644
  )
604
645
 
605
646
  @app.get("/health", tags=["system"])
@@ -75,7 +75,11 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
75
75
 
76
76
  async def _read_bounded_body(self, request: Request) -> Response | None:
77
77
  """Consume and cache request bodies without trusting Content-Length."""
78
- if request.method in {"GET", "HEAD"}:
78
+ # HEAD has no response body by definition and some ASGI servers do
79
+ # not expose a request stream for it. GET bodies are unusual but
80
+ # still need the same bound so an oversized payload cannot bypass
81
+ # request-size protection by changing the HTTP method.
82
+ if request.method == "HEAD":
79
83
  return None
80
84
  content_length = request.headers.get("content-length")
81
85
  if content_length:
@@ -83,6 +87,8 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
83
87
  declared_length = int(content_length)
84
88
  except ValueError:
85
89
  return JSONResponse(status_code=400, content={"success": False, "error": {"code": 400, "message": "Invalid Content-Length"}, "timestamp": datetime.now().isoformat()})
90
+ if declared_length < 0:
91
+ return JSONResponse(status_code=400, content={"success": False, "error": {"code": 400, "message": "Invalid Content-Length"}, "timestamp": datetime.now().isoformat()})
86
92
  if declared_length > self.max_body_bytes:
87
93
  return JSONResponse(status_code=413, content={"success": False, "error": {"code": 413, "message": "Request body exceeds the maximum allowed size", "max_bytes": self.max_body_bytes}, "timestamp": datetime.now().isoformat()})
88
94
 
@@ -97,6 +103,28 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
97
103
  return None
98
104
 
99
105
  async def dispatch(self, request: Request, call_next) -> Response:
106
+ is_health_probe = (
107
+ request.url.path in {"/health", "/", "/ready", "/health/deep"}
108
+ and request.method in {"GET", "HEAD"}
109
+ )
110
+
111
+ # Apply rate limiting before JSON/body parsing. This prevents a
112
+ # client from spending unbounded CPU on invalid payloads while still
113
+ # keeping the request-size guard active when rate limiting is disabled.
114
+ rate_limit_checked = False
115
+ remaining = reset_time = None
116
+ if self.enabled and not is_health_probe:
117
+ client_id = self._get_client_id(request)
118
+ is_allowed, remaining, reset_time = self.limiter.is_allowed(client_id)
119
+ rate_limit_checked = True
120
+ if not is_allowed:
121
+ logger.warning("Rate limit exceeded for client: %s", _sanitize_log_value(client_id))
122
+ return JSONResponse(
123
+ status_code=429,
124
+ content={"success": False, "error": {"code": 429, "message": "Rate limit exceeded", "retry_after": reset_time}, "timestamp": datetime.now().isoformat()},
125
+ headers={"X-RateLimit-Limit": str(self.limiter._requests_per_window), "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": str(reset_time), "Retry-After": str(reset_time)},
126
+ )
127
+
100
128
  body_error = await self._read_bounded_body(request)
101
129
  if body_error is not None:
102
130
  return body_error
@@ -118,24 +146,15 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
118
146
 
119
147
  if not self.enabled:
120
148
  return await call_next(request)
121
- if request.url.path in {"/health", "/", "/ready", "/health/deep"} and request.method in {"GET", "HEAD"}:
149
+ if is_health_probe:
122
150
  if request.url.path in {"/ready", "/health/deep"}:
123
151
  return await health_probe_response(request)
124
152
  return await call_next(request)
125
-
126
- client_id = self._get_client_id(request)
127
- is_allowed, remaining, reset_time = self.limiter.is_allowed(client_id)
128
- if not is_allowed:
129
- logger.warning("Rate limit exceeded for client: %s", _sanitize_log_value(client_id))
130
- return JSONResponse(
131
- status_code=429,
132
- content={"success": False, "error": {"code": 429, "message": "Rate limit exceeded", "retry_after": reset_time}, "timestamp": datetime.now().isoformat()},
133
- headers={"X-RateLimit-Limit": str(self.limiter._requests_per_window), "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": str(reset_time), "Retry-After": str(reset_time)},
134
- )
135
153
  response = await call_next(request)
136
- response.headers["X-RateLimit-Limit"] = str(self.limiter._requests_per_window)
137
- response.headers["X-RateLimit-Remaining"] = str(remaining)
138
- response.headers["X-RateLimit-Reset"] = str(reset_time)
154
+ if rate_limit_checked:
155
+ response.headers["X-RateLimit-Limit"] = str(self.limiter._requests_per_window)
156
+ response.headers["X-RateLimit-Remaining"] = str(remaining)
157
+ response.headers["X-RateLimit-Reset"] = str(reset_time)
139
158
  return response
140
159
 
141
160
  def _get_client_id(self, request: Request) -> str:
@@ -5,10 +5,11 @@ Redis is opt-in. The in-memory backend remains the default for local/single-proc
5
5
 
6
6
  from __future__ import annotations
7
7
 
8
- import os
9
8
  import time
10
9
  from threading import Lock
11
- from typing import Protocol
10
+ from typing import Optional, Protocol
11
+
12
+ from backend.core.config import AppSettings, settings as _default_settings
12
13
 
13
14
 
14
15
  class RateLimitStore(Protocol):
@@ -64,6 +65,8 @@ class InMemoryRateLimitStore:
64
65
  class RedisRateLimitStore:
65
66
  """Atomic fixed-window store backed by Redis for multi-worker deployments."""
66
67
 
68
+ DEFAULT_TIMEOUT_SECONDS = 3.0
69
+
67
70
  _SCRIPT = """
68
71
  local count = redis.call('INCR', KEYS[1])
69
72
  if count == 1 then
@@ -73,14 +76,29 @@ class RedisRateLimitStore:
73
76
  return {count, ttl}
74
77
  """
75
78
 
76
- def __init__(self, url: str, namespace: str = "sdm:rate-limit:") -> None:
79
+ def __init__(
80
+ self,
81
+ url: str,
82
+ namespace: str = "sdm:rate-limit:",
83
+ timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
84
+ ) -> None:
77
85
  if not url:
78
86
  raise ValueError("Redis URL is required when rate-limit backend is redis")
87
+ if timeout_seconds <= 0:
88
+ raise ValueError("timeout_seconds must be positive")
79
89
  try:
80
90
  import redis
81
91
  except ImportError as exc:
82
92
  raise RuntimeError("Redis rate limiting requires the optional 'redis' package") from exc
83
- self._client = redis.Redis.from_url(url, decode_responses=False)
93
+ # Bound both connection establishment and command execution. A
94
+ # stalled Redis node must not block API startup or every request that
95
+ # checks the distributed rate-limit store.
96
+ self._client = redis.Redis.from_url(
97
+ url,
98
+ decode_responses=False,
99
+ socket_connect_timeout=timeout_seconds,
100
+ socket_timeout=timeout_seconds,
101
+ )
84
102
  self._client.ping()
85
103
  self._script = self._client.register_script(self._SCRIPT)
86
104
  self._namespace = namespace
@@ -95,11 +113,28 @@ class RedisRateLimitStore:
95
113
  return {"backend": "redis", "namespace": self._namespace}
96
114
 
97
115
 
98
- def create_rate_limit_store() -> RateLimitStore:
99
- """Create the configured store; never silently downgrade Redis to memory."""
100
- backend = os.getenv("SDM_RATE_LIMIT_BACKEND", "memory").strip().lower()
116
+ def create_rate_limit_store(config: Optional[AppSettings] = None) -> RateLimitStore:
117
+ """Create the configured store; never silently downgrade Redis to memory.
118
+
119
+ Reads configuration from ``config``, or from the unified
120
+ ``backend.core.config.settings`` singleton when ``config`` is None.
121
+ The factory never reads raw environment variables — all deployment
122
+ configuration flows through AppSettings (SDM_ prefix), so backend
123
+ selection and the Redis URL share one validated configuration path.
124
+
125
+ The Redis URL is a SecretStr in AppSettings; its actual value is
126
+ read here only to construct the Redis client, and it is never
127
+ embedded in error messages or logs.
128
+ """
129
+ settings_obj = config if config is not None else _default_settings
130
+ backend = settings_obj.rate_limit_backend
101
131
  if backend == "memory":
102
132
  return InMemoryRateLimitStore()
103
133
  if backend == "redis":
104
- return RedisRateLimitStore(os.getenv("SDM_REDIS_URL", ""))
105
- raise ValueError("Unsupported SDM_RATE_LIMIT_BACKEND; expected 'memory' or 'redis'")
134
+ url = settings_obj.redis_url
135
+ url_value = url.get_secret_value() if url is not None else ""
136
+ return RedisRateLimitStore(url_value)
137
+ # Boundary guard for untyped callers: only reachable when a caller
138
+ # passes a non-AppSettings object with an unexpected backend value
139
+ # (typed AppSettings validation rejects it earlier).
140
+ raise ValueError("Unsupported rate_limit_backend; expected 'memory' or 'redis'")
@@ -7,6 +7,7 @@ import logging
7
7
  import os
8
8
  import secrets
9
9
  import time
10
+ import uuid
10
11
  import weakref
11
12
  from datetime import datetime
12
13
  from typing import Any
@@ -16,6 +17,8 @@ from fastapi.responses import JSONResponse
16
17
 
17
18
  logger = logging.getLogger(__name__)
18
19
 
20
+ from backend.core.config import settings
21
+
19
22
  _READINESS_CACHE_TTL_SECONDS = 5.0
20
23
  _READINESS_TIMEOUT_SECONDS = 3.0
21
24
  _READINESS_LOCKS: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary()
@@ -26,22 +29,37 @@ _HEALTH_PROBE_PATHS = {"/ready", "/health/deep"}
26
29
 
27
30
  def _probe_allowed(request: Request) -> bool:
28
31
  """Allow health probes only from loopback or with the configured probe secret."""
29
- token = os.getenv("SDM_HEALTH_PROBE_TOKEN", "").strip()
32
+ configured = settings.health_probe_token
33
+ token = configured.get_secret_value().strip() if configured else ""
34
+ # Keep process-level test/dev overrides effective after module import;
35
+ # production configuration still comes from the validated singleton above
36
+ # (including values loaded from .env).
37
+ if not token:
38
+ token = os.environ.get("SDM_HEALTH_PROBE_TOKEN", "").strip()
30
39
  supplied = request.headers.get("X-Health-Probe-Token", "")
31
40
  if token and supplied and secrets.compare_digest(supplied, token):
32
41
  return True
42
+ # Once a token is configured, require it even for loopback callers. This
43
+ # avoids trusting an unverified X-Forwarded-For chain in front of a proxy.
44
+ if token:
45
+ return False
33
46
  client = request.client
34
47
  return bool(client and client.host in {"127.0.0.1", "::1", "localhost"})
35
48
 
36
49
 
37
- def _health_probe_denied_response() -> JSONResponse:
50
+ def _health_probe_denied_response(request: Request) -> JSONResponse:
38
51
  """Return a stable denial response for public health probe requests."""
52
+ request_id = getattr(request.state, "request_id", None) or str(uuid.uuid4())
53
+ headers = {"X-Request-ID": request_id}
54
+ content = {
55
+ "success": False,
56
+ "error": {"code": 403, "message": "health probe access denied"},
57
+ }
58
+ content["request_id"] = request_id
39
59
  return JSONResponse(
40
60
  status_code=403,
41
- content={
42
- "success": False,
43
- "error": {"code": 403, "message": "health probe access denied"},
44
- },
61
+ content=content,
62
+ headers=headers,
45
63
  )
46
64
 
47
65
 
@@ -50,7 +68,7 @@ async def health_probe_response(request: Request) -> JSONResponse:
50
68
  if request.url.path not in _HEALTH_PROBE_PATHS or request.method not in {"GET", "HEAD"}:
51
69
  raise ValueError("health_probe_response called for a non-probe request")
52
70
  if not _probe_allowed(request):
53
- return _health_probe_denied_response()
71
+ return _health_probe_denied_response(request)
54
72
  if request.url.path == "/ready":
55
73
  return await readiness_response(request)
56
74
  return await deep_health_response(request)
@@ -48,6 +48,8 @@ class TTLCache:
48
48
  """
49
49
  if max_size <= 0:
50
50
  raise ValueError("max_size must be greater than 0")
51
+ if ttl < 0:
52
+ raise ValueError("ttl must be greater than or equal to 0")
51
53
  self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
52
54
  self._max_size = max_size
53
55
  self._ttl = ttl
@@ -222,10 +224,18 @@ class TTLCache:
222
224
  except Exception as e:
223
225
  logger.error(f"Background cleanup error: {e}")
224
226
 
225
- thread = threading.Thread(target=cleanup_loop, daemon=True, name="CacheCleanup")
226
227
  with self._lock:
228
+ # Check, assign and start under the same lock. Keeping the
229
+ # assignment inside the critical section closes the TOCTOU window
230
+ # where two concurrent callers both observe no live thread and
231
+ # each start a cleanup worker.
232
+ if self._cleanup_thread is not None and self._cleanup_thread.is_alive():
233
+ raise RuntimeError(
234
+ "background cleanup is already running for this cache"
235
+ )
236
+ thread = threading.Thread(target=cleanup_loop, daemon=True, name="CacheCleanup")
227
237
  self._cleanup_thread = thread
228
- thread.start()
238
+ thread.start()
229
239
  logger.info(f"Started background cache cleanup (interval={interval}s)")
230
240
  return thread
231
241
 
@@ -12,9 +12,9 @@ Supports configuration via:
12
12
  import logging
13
13
  import sys
14
14
  from enum import Enum
15
- from typing import Any, Dict, List, Optional, TypedDict
15
+ from typing import Any, Dict, List, Literal, Optional, TypedDict
16
16
 
17
- from pydantic import Field
17
+ from pydantic import Field, SecretStr, field_validator, model_validator
18
18
  from pydantic_settings import BaseSettings, SettingsConfigDict
19
19
 
20
20
  # =============================================================================
@@ -220,23 +220,34 @@ class AppSettings(BaseSettings):
220
220
  "http://localhost:8501,http://localhost:8000,"
221
221
  "http://127.0.0.1:8501,http://127.0.0.1:8000"
222
222
  )
223
- max_batch_size: int = 100
224
- request_timeout: int = 30
223
+ max_batch_size: int = Field(100, gt=0)
224
+ request_timeout: int = Field(30, gt=0)
225
225
 
226
226
  # Cache settings
227
227
  cache_enabled: bool = True
228
- cache_ttl: int = 300
228
+ # Zero intentionally disables expiry (TTLCache contract); negatives are
229
+ # invalid because they are indistinguishable from that mode at runtime.
230
+ cache_ttl: int = Field(300, ge=0)
229
231
  cache_max_size: int = Field(1000, gt=0)
230
232
 
231
233
  # Rate limiting
232
234
  rate_limit_enabled: bool = True
233
- rate_limit_requests: int = 100
234
- rate_limit_window: int = 60
235
+ rate_limit_requests: int = Field(100, gt=0)
236
+ rate_limit_window: int = Field(60, gt=0)
237
+ # Storage backend selection: strict lowercase values; no case or
238
+ # whitespace normalization — a Literal rejects any other spelling at
239
+ # settings construction instead of silently coercing it.
240
+ rate_limit_backend: Literal["memory", "redis"] = "memory"
241
+ # Redis connection URL, required only when the backend is redis.
242
+ # SecretStr keeps the password out of repr / logs / serialization;
243
+ # the store factory reads the actual value only to build the Redis
244
+ # client. Default None = not configured.
245
+ redis_url: Optional[SecretStr] = None
235
246
 
236
247
  # NL2SQL settings
237
- nl2sql_confidence_threshold: float = 0.6
238
- nl2sql_max_suggestions: int = 5
239
- nl2sql_max_input_length: int = 8192
248
+ nl2sql_confidence_threshold: float = Field(0.6, ge=0.0, le=1.0)
249
+ nl2sql_max_suggestions: int = Field(5, ge=0)
250
+ nl2sql_max_input_length: int = Field(8192, gt=0)
240
251
 
241
252
  # SQL Transpiler / Parser settings
242
253
  #
@@ -245,12 +256,16 @@ class AppSettings(BaseSettings):
245
256
  # to ``transpiler_max_sql_length``. Business code must read the
246
257
  # settings object — never ``os.getenv`` directly.
247
258
  transpiler_pretty_default: bool = True
248
- transpiler_max_sql_length: int = 100000
249
- parser_max_sql_length: int = 100000
259
+ transpiler_max_sql_length: int = Field(100000, gt=0)
260
+ parser_max_sql_length: Optional[int] = Field(None, gt=0)
250
261
 
251
262
  # Function encyclopedia
252
- function_search_limit: int = 50
253
- function_fuzzy_threshold: float = 0.4
263
+ function_search_limit: int = Field(50, gt=0)
264
+ function_fuzzy_threshold: float = Field(0.4, ge=0.0, le=1.0)
265
+
266
+ # Readiness probes. SecretStr keeps the value out of settings reprs and
267
+ # structured logs while still allowing deployment through SDM_ env vars.
268
+ health_probe_token: Optional[SecretStr] = None
254
269
 
255
270
  # Security settings
256
271
  security_check_enabled: bool = True
@@ -259,6 +274,23 @@ class AppSettings(BaseSettings):
259
274
  # Logging settings
260
275
  log_level: str = "INFO"
261
276
  log_file: str = ""
277
+
278
+ @model_validator(mode="after")
279
+ def _apply_parser_length_fallback(self) -> "AppSettings":
280
+ """Keep parser and transpiler limits aligned unless explicitly set."""
281
+ if self.parser_max_sql_length is None:
282
+ self.parser_max_sql_length = self.transpiler_max_sql_length
283
+ return self
284
+
285
+ @field_validator("log_level", mode="before")
286
+ @classmethod
287
+ def _normalize_log_level(cls, value: str) -> str:
288
+ normalized = str(value).strip().upper()
289
+ if normalized not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}:
290
+ raise ValueError(
291
+ "log_level must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL"
292
+ )
293
+ return normalized
262
294
 
263
295
  def to_dict(self) -> Dict[str, Any]:
264
296
  """Convert settings to dictionary."""
@@ -17,8 +17,8 @@ Design rules:
17
17
  from __future__ import annotations
18
18
 
19
19
  # ── Release version facts (sole source of truth) ────────────────────────
20
- PROJECT_VERSION = "1.1.1"
21
- API_VERSION = "1.1.1"
20
+ PROJECT_VERSION = "1.1.3"
21
+ API_VERSION = "1.1.3"
22
22
  # Frontend v2 architecture — independent of the product release version.
23
23
  FRONTEND_ARCHITECTURE_VERSION = 2
24
24
 
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env python3
2
+ """NL2SQL aggregation extraction.
3
+
4
+ Extracts aggregate function calls (COUNT, SUM, AVG, MAX, MIN) from
5
+ natural-language text using keyword matching. Uses the shared
6
+ COLUMN_PATTERNS to resolve the target column for value aggregates.
7
+ """
8
+ import re
9
+ from typing import Dict, List
10
+
11
+
12
+ def extract_aggregations(text: str, column_patterns: Dict[str, str]) -> List[str]:
13
+ """Extract aggregation expressions from text.
14
+
15
+ Returns a list of SQL aggregate expressions, e.g.
16
+ ['COUNT(*)', 'SUM(amount)']. An empty list means no aggregation
17
+ was requested.
18
+ """
19
+ aggs: List[str] = []
20
+ text_lower = text.lower()
21
+ # Pick the user's named metric column, not the first pattern that
22
+ # happens to sit inside an aggregate keyword: the legacy first-hit
23
+ # walk let "average" contain "age" and produced AVG(age).
24
+ named_metrics = [
25
+ column
26
+ for pattern, column in column_patterns.items()
27
+ if pattern != "age" and pattern in text
28
+ ]
29
+ # Bare "age" is only a metric when it is not the trailing substring
30
+ # of "average": require a non-letter before it so "average" never
31
+ # self-matches, while "average age" still does.
32
+ if re.search(r"(?<![a-z])age", text_lower):
33
+ if "age" not in named_metrics:
34
+ named_metrics.append("age")
35
+ if "price" in text_lower:
36
+ named_metrics = [c for c in named_metrics if c != "age"]
37
+ agg_column = named_metrics[0] if named_metrics else None
38
+ col = agg_column or "amount"
39
+ if any(k in text for k in ["统计", "计数", "数量", "多少", "几个", "count", "total", "how many"]):
40
+ aggs.append("COUNT(*)")
41
+ if any(k in text for k in ["求和", "总和", "合计", "总计", "sum", "total of"]):
42
+ aggs.append(f"SUM({col})")
43
+ if any(k in text_lower for k in ["平均", "均值", "平均值", "average", "avg", "mean"]):
44
+ # "average" contains "age": the average metric is price unless
45
+ # the text explicitly names the age column as the target.
46
+ col = "price" if ("price" in text_lower and not re.search(r"(?<![a-z])age", text_lower)) else (agg_column or "amount")
47
+ aggs.append(f"AVG({col})")
48
+ if any(k in text_lower for k in ["最大", "最高", "最多", "max", "maximum", "highest", "largest"]):
49
+ aggs.append(f"MAX({col})")
50
+ if any(k in text_lower for k in ["最小", "最低", "最少", "min", "minimum", "lowest", "smallest"]):
51
+ aggs.append(f"MIN({col})")
52
+ return aggs