nl2sql-engine 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nl2sql_engine-0.1.0/PKG-INFO +295 -0
- nl2sql_engine-0.1.0/README.md +241 -0
- nl2sql_engine-0.1.0/pyproject.toml +76 -0
- nl2sql_engine-0.1.0/setup.cfg +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/__init__.py +38 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/duckdb/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/duckdb/adapter.py +71 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/mssql/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/mssql/adapter.py +122 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/mysql/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/mysql/adapter.py +123 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/postgres/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/postgres/adapter.py +115 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/sqlalchemy_base/__init__.py +17 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/sqlalchemy_base/adapter.py +476 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/sqlalchemy_base/models.py +36 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/sqlite/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/adapters/sqlite/adapter.py +88 -0
- nl2sql_engine-0.1.0/src/nl2sql/aggregation/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/aggregation/aggregator.py +98 -0
- nl2sql_engine-0.1.0/src/nl2sql/aggregation/engines/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/aggregation/engines/polars_duckdb.py +125 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/auth_api.py +60 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/benchmark_api.py +114 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/datasource_api.py +132 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/indexing_api.py +59 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/llm_api.py +82 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/policy_api.py +135 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/query_api.py +138 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/result_api.py +24 -0
- nl2sql_engine-0.1.0/src/nl2sql/api/settings_api.py +65 -0
- nl2sql_engine-0.1.0/src/nl2sql/auth/__init__.py +8 -0
- nl2sql_engine-0.1.0/src/nl2sql/auth/models.py +36 -0
- nl2sql_engine-0.1.0/src/nl2sql/auth/rbac.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/checks.py +53 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/benchmark.py +34 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/doctor.py +49 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/indexing.py +126 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/info.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/install.py +27 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/policy.py +57 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/run.py +166 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/setup.py +415 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/commands/visualize.py +34 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/common/decorators.py +34 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/config.py +24 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/console.py +52 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/__init__.py +1 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/data.py +87 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/defaults.py +122 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/factory.py +289 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/manager.py +230 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/schemas.py +336 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/writers/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/writers/docker.py +182 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/demo/writers/sqlite.py +88 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/datasources/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/datasources/generator.py +24 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/datasources/templates.py +7 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/env/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/env/generator.py +46 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/env/templates.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/llm/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/llm/generator.py +24 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/llm/templates.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/policies/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/policies/generator.py +20 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/generators/policies/templates.py +2 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/main.py +195 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/reporting.py +878 -0
- nl2sql_engine-0.1.0/src/nl2sql/cli/types.py +13 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/__init__.py +1 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/cancellation.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/context.py +5 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/errors.py +109 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/event_logger.py +88 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/exceptions.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/logger.py +119 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/metrics.py +50 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/resilience.py +59 -0
- nl2sql_engine-0.1.0/src/nl2sql/common/settings.py +195 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/__init__.py +6 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/datasources.py +10 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/llm.py +36 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/manager.py +176 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/policies.py +14 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/sample_questions.py +11 -0
- nl2sql_engine-0.1.0/src/nl2sql/configs/secrets.py +11 -0
- nl2sql_engine-0.1.0/src/nl2sql/context.py +106 -0
- nl2sql_engine-0.1.0/src/nl2sql/datasources/__init__.py +21 -0
- nl2sql_engine-0.1.0/src/nl2sql/datasources/discovery.py +28 -0
- nl2sql_engine-0.1.0/src/nl2sql/datasources/models.py +21 -0
- nl2sql_engine-0.1.0/src/nl2sql/datasources/protocols.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/datasources/registry.py +172 -0
- nl2sql_engine-0.1.0/src/nl2sql/evaluation/__init__.py +6 -0
- nl2sql_engine-0.1.0/src/nl2sql/evaluation/benchmark_runner.py +320 -0
- nl2sql_engine-0.1.0/src/nl2sql/evaluation/evaluator.py +134 -0
- nl2sql_engine-0.1.0/src/nl2sql/evaluation/types.py +22 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/artifacts/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/artifacts/parquet.py +41 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/artifacts/store.py +165 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/contracts.py +57 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/execution_store.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/executor/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/execution/executor/sql_executor.py +116 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/__init__.py +7 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/chunk_builder.py +227 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/embeddings.py +180 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/enrichment_service.py +316 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/models.py +209 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/orchestrator.py +90 -0
- nl2sql_engine-0.1.0/src/nl2sql/indexing/vector_store.py +422 -0
- nl2sql_engine-0.1.0/src/nl2sql/llm/__init__.py +8 -0
- nl2sql_engine-0.1.0/src/nl2sql/llm/models.py +10 -0
- nl2sql_engine-0.1.0/src/nl2sql/llm/registry.py +214 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/__init__.py +1 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/graph.py +73 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/graph_utils.py +141 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/__init__.py +25 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/aggregator/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/aggregator/node.py +55 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/aggregator/prompts.py +20 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/aggregator/schemas.py +28 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/answer_synthesizer/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/answer_synthesizer/node.py +98 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/answer_synthesizer/prompts.py +19 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/answer_synthesizer/schemas.py +24 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/ast_planner/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/ast_planner/node.py +104 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/ast_planner/prompts.py +138 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/ast_planner/schemas.py +236 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/datasource_resolver/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/datasource_resolver/node.py +253 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/datasource_resolver/schemas.py +21 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/decomposer/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/decomposer/node.py +219 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/decomposer/prompts.py +96 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/decomposer/schemas.py +143 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/executor/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/executor/node.py +107 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/generator/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/generator/node.py +267 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/generator/schemas.py +13 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/global_planner/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/global_planner/node.py +186 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/global_planner/schemas.py +101 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/refiner/__init__.py +4 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/refiner/node.py +132 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/refiner/prompts.py +28 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/refiner/schemas.py +13 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/schema_retriever/__init__.py +3 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/schema_retriever/node.py +252 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/schema_retriever/schema.py +27 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/validator/__init__.py +7 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/validator/node.py +839 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/nodes/validator/schemas.py +12 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/pipeline_runner.py +72 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/routes.py +72 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/runtime.py +153 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/state.py +92 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/subgraphs/__init__.py +5 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/subgraphs/schemas.py +23 -0
- nl2sql_engine-0.1.0/src/nl2sql/pipeline/subgraphs/sql_agent.py +167 -0
- nl2sql_engine-0.1.0/src/nl2sql/public_api.py +199 -0
- nl2sql_engine-0.1.0/src/nl2sql/schema/__init__.py +37 -0
- nl2sql_engine-0.1.0/src/nl2sql/schema/in_memory_store.py +173 -0
- nl2sql_engine-0.1.0/src/nl2sql/schema/protocol.py +88 -0
- nl2sql_engine-0.1.0/src/nl2sql/schema/sqlite_store.py +233 -0
- nl2sql_engine-0.1.0/src/nl2sql/schema/store.py +29 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/__init__.py +14 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/factory.py +85 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/interfaces.py +16 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/manager.py +139 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/models.py +56 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/providers/aws.py +30 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/providers/azure.py +49 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/providers/env.py +8 -0
- nl2sql_engine-0.1.0/src/nl2sql/secrets/providers/hashi.py +46 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/__init__.py +0 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/monitor.py +84 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/node_context.py +7 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/node_handlers.py +187 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/node_metrics.py +14 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/presenter.py +12 -0
- nl2sql_engine-0.1.0/src/nl2sql/services/callbacks/token_handler.py +56 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/PKG-INFO +295 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/SOURCES.txt +195 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/dependency_links.txt +1 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/entry_points.txt +9 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/requires.txt +59 -0
- nl2sql_engine-0.1.0/src/nl2sql_engine.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nl2sql-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Natural-language-to-SQL engine, CLI and database adapters
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: SQLAlchemy>=2.0
|
|
8
|
+
Requires-Dist: PyYAML>=6.0
|
|
9
|
+
Requires-Dist: langgraph<2,>=1.2
|
|
10
|
+
Requires-Dist: langchain>=0.2.0
|
|
11
|
+
Requires-Dist: langchain-core<2,>=1.6
|
|
12
|
+
Requires-Dist: langchain-chroma<2,>=1.1
|
|
13
|
+
Requires-Dist: langchain-openai<2,>=1.6
|
|
14
|
+
Requires-Dist: chromadb<2,>=1.5
|
|
15
|
+
Requires-Dist: nl2sql-adapter-sdk~=0.1
|
|
16
|
+
Requires-Dist: pydantic>=2.0
|
|
17
|
+
Requires-Dist: opentelemetry-api>=1.20.0
|
|
18
|
+
Requires-Dist: opentelemetry-sdk>=1.20.0
|
|
19
|
+
Requires-Dist: opentelemetry-exporter-otlp>=1.20.0
|
|
20
|
+
Requires-Dist: sqlglot>=23.0.0
|
|
21
|
+
Requires-Dist: pydantic-settings>=2.0.0
|
|
22
|
+
Requires-Dist: pandas>=1.5.0
|
|
23
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
24
|
+
Requires-Dist: pybreaker>=1.3.0
|
|
25
|
+
Requires-Dist: polars>=1.44
|
|
26
|
+
Requires-Dist: duckdb>=1.5
|
|
27
|
+
Requires-Dist: rich>=13.0.0
|
|
28
|
+
Requires-Dist: typer[all]>=0.9.0
|
|
29
|
+
Requires-Dist: inquirerpy>=0.3.4
|
|
30
|
+
Provides-Extra: aws
|
|
31
|
+
Requires-Dist: boto3; extra == "aws"
|
|
32
|
+
Provides-Extra: azure
|
|
33
|
+
Requires-Dist: azure-identity; extra == "azure"
|
|
34
|
+
Requires-Dist: azure-keyvault-secrets; extra == "azure"
|
|
35
|
+
Provides-Extra: duckdb
|
|
36
|
+
Requires-Dist: duckdb; extra == "duckdb"
|
|
37
|
+
Requires-Dist: duckdb-engine; extra == "duckdb"
|
|
38
|
+
Provides-Extra: hashicorp
|
|
39
|
+
Requires-Dist: hvac; extra == "hashicorp"
|
|
40
|
+
Provides-Extra: mssql
|
|
41
|
+
Requires-Dist: pyodbc>=4.0.0; extra == "mssql"
|
|
42
|
+
Requires-Dist: pymssql>=2.2.0; platform_system == "Linux" and extra == "mssql"
|
|
43
|
+
Provides-Extra: mysql
|
|
44
|
+
Requires-Dist: pymysql>=1.1.0; extra == "mysql"
|
|
45
|
+
Provides-Extra: postgres
|
|
46
|
+
Requires-Dist: psycopg2-binary>=2.9; extra == "postgres"
|
|
47
|
+
Provides-Extra: all
|
|
48
|
+
Requires-Dist: duckdb; extra == "all"
|
|
49
|
+
Requires-Dist: duckdb-engine; extra == "all"
|
|
50
|
+
Requires-Dist: pyodbc>=4.0.0; extra == "all"
|
|
51
|
+
Requires-Dist: pymssql>=2.2.0; platform_system == "Linux" and extra == "all"
|
|
52
|
+
Requires-Dist: pymysql>=1.1.0; extra == "all"
|
|
53
|
+
Requires-Dist: psycopg2-binary>=2.9; extra == "all"
|
|
54
|
+
|
|
55
|
+
# NL2SQL
|
|
56
|
+
|
|
57
|
+
The **`nl2sql`** package is the brain of the natural language to SQL engine. It orchestrates the entire query lifecycle using a graph-based agent architecture, and also ships the `nl2sql` CLI (`nl2sql.cli`) and the four database adapters (`nl2sql.adapters.*`).
|
|
58
|
+
|
|
59
|
+
## 🏗️ Architecture Overview
|
|
60
|
+
|
|
61
|
+
The NL2SQL Core is built around a **graph-based orchestration system** using LangGraph that treats text-to-SQL as a distributed systems problem. The architecture is organized around several key planes:
|
|
62
|
+
|
|
63
|
+
### 1. **The Control Plane (The Graph)**
|
|
64
|
+
- **Responsibility**: Reasoning, Planning, and Orchestration
|
|
65
|
+
- **Implementation**: Directed Cyclic Graph (LangGraph) with explicit state (`GraphState`)
|
|
66
|
+
- **Features**: Agentic graph with refinement loops for self-correction when plans fail validation
|
|
67
|
+
|
|
68
|
+
### 2. **The Security Plane (The Firewall)**
|
|
69
|
+
- **Responsibility**: Invariants Enforcement
|
|
70
|
+
- **Implementation**: Valid-by-Construction approach where LLM generates Abstract Syntax Tree (AST) rather than executing SQL
|
|
71
|
+
- **Features**: Static analysis through logical validators enforcing RBAC and schema constraints
|
|
72
|
+
|
|
73
|
+
### 3. **The Data Plane (The Sandbox)**
|
|
74
|
+
- **Responsibility**: Semantic Search and Execution
|
|
75
|
+
- **Implementation**: Sandboxed Process Pool for SQL driver isolation
|
|
76
|
+
- **Features**: Partitioned retrieval with schema store and vector-based context injection
|
|
77
|
+
|
|
78
|
+
### 4. **The Reliability Plane (The Guard)**
|
|
79
|
+
- **Responsibility**: Fault Tolerance and Stability
|
|
80
|
+
- **Implementation**: Layered defense with Circuit Breakers and Sandboxing
|
|
81
|
+
- **Features**: Fail-fast approach with strict timeouts preventing cascading failures
|
|
82
|
+
|
|
83
|
+
### 5. **The Observability Plane (The Watchtower)**
|
|
84
|
+
- **Responsibility**: Visibility, Forensics, and Compliance
|
|
85
|
+
- **Implementation**: Native OpenTelemetry integration
|
|
86
|
+
- **Features**: Distributed tracing (Jaeger), metrics (Prometheus), and forensic audit logs
|
|
87
|
+
|
|
88
|
+
## 🧠 Key Components
|
|
89
|
+
|
|
90
|
+
### **Context Management (`context.py`)**
|
|
91
|
+
- `NL2SQLContext`: Centralized application context managing initialization lifecycle
|
|
92
|
+
- Ensures proper ordering: secrets → datasources → LLMs → policies
|
|
93
|
+
- Coordinates all registries and stores
|
|
94
|
+
|
|
95
|
+
### **Graph Pipeline (`pipeline/`)**
|
|
96
|
+
- **Graph Orchestration**: LangGraph-based state machine managing query flow
|
|
97
|
+
- **Nodes**: DatasourceResolver, Decomposer, GlobalPlanner, Aggregator, AnswerSynthesizer
|
|
98
|
+
- **Subgraphs**: SQL Agent subgraph with AST planner, validators, and executor
|
|
99
|
+
- **State Management**: Shared `GraphState` for auditability and reproducibility
|
|
100
|
+
|
|
101
|
+
### **Schema Management (`schema/`)**
|
|
102
|
+
- **Schema Store**: Persistent storage for schema snapshots with versioning
|
|
103
|
+
- **Schema Contracts**: Typed representations of database schemas
|
|
104
|
+
- **Versioning**: Multiple schema versions with eviction policies
|
|
105
|
+
|
|
106
|
+
### **Indexing System (`indexing/`)**
|
|
107
|
+
- **Schema Indexing**: Vector-based indexing of schema information
|
|
108
|
+
- **Chunk Builder**: Breaks schema into searchable chunks
|
|
109
|
+
- **Enrichment Service**: Enhances schema with example questions
|
|
110
|
+
|
|
111
|
+
### **Data Sources (`datasources/`)**
|
|
112
|
+
- **Registry**: Dynamic registration and management of database adapters
|
|
113
|
+
- **Protocols**: Standardized interfaces for database connectivity
|
|
114
|
+
- **Discovery**: Automatic discovery of available adapter types
|
|
115
|
+
|
|
116
|
+
### **LLM Management (`llm/`)**
|
|
117
|
+
- **Registry**: Management of multiple LLM instances
|
|
118
|
+
- **Configuration**: Flexible LLM provider configuration (OpenAI, etc.)
|
|
119
|
+
- **Routing**: Intelligent routing to appropriate LLMs
|
|
120
|
+
|
|
121
|
+
### **Authentication & Authorization (`auth/`)**
|
|
122
|
+
- **RBAC**: Role-based access control for data access
|
|
123
|
+
- **User Context**: Identity and permission context propagation
|
|
124
|
+
- **Policy Engine**: Fine-grained access control rules
|
|
125
|
+
|
|
126
|
+
## 🚀 Public API
|
|
127
|
+
|
|
128
|
+
The main public interface is provided through the `NL2SQL` class:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from nl2sql import NL2SQL
|
|
132
|
+
|
|
133
|
+
# Initialize the engine
|
|
134
|
+
engine = NL2SQL(
|
|
135
|
+
ds_config_path="configs/datasources.yaml",
|
|
136
|
+
llm_config_path="configs/llm.yaml",
|
|
137
|
+
policies_path="configs/policies.json"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# Run a natural language query
|
|
141
|
+
result = engine.run_query("Show top 10 customers by revenue")
|
|
142
|
+
print(result.final_answer)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Two-Tier API Architecture
|
|
146
|
+
|
|
147
|
+
NL2SQL provides a two-tier API architecture:
|
|
148
|
+
|
|
149
|
+
#### 1. Core API (Python) - This Package
|
|
150
|
+
- **Interface**: Direct Python class interface (`NL2SQL` class)
|
|
151
|
+
- **Use Case**: Direct Python integration, embedded applications
|
|
152
|
+
- **Access**: Import and use directly in Python code
|
|
153
|
+
|
|
154
|
+
#### 2. REST API (HTTP)
|
|
155
|
+
- **Package**: API package (`nl2sql-api`)
|
|
156
|
+
- **Interface**: HTTP REST endpoints
|
|
157
|
+
- **Use Case**: Remote clients, web applications, TypeScript CLI
|
|
158
|
+
- **Access**: HTTP requests to API endpoints
|
|
159
|
+
|
|
160
|
+
Both APIs provide access to the same underlying NL2SQL engine functionality, allowing flexible integration options.
|
|
161
|
+
|
|
162
|
+
### Modular API Structure
|
|
163
|
+
|
|
164
|
+
The engine provides modular APIs for different functionality areas:
|
|
165
|
+
|
|
166
|
+
- `engine.query` - Query execution API (`run_query`, etc.)
|
|
167
|
+
- `engine.datasource` - Datasource management API (`add_datasource`, `list_datasources`, etc.)
|
|
168
|
+
- `engine.llm` - LLM configuration API (`configure_llm`, etc.)
|
|
169
|
+
- `engine.indexing` - Schema indexing API (`index_datasource`, `clear_index`, etc.)
|
|
170
|
+
- `engine.auth` - Authentication and RBAC API (`check_permissions`, `get_allowed_resources`, etc.)
|
|
171
|
+
- `engine.settings` - Configuration and settings API (`get_current_settings`, `validate_configuration`, etc.)
|
|
172
|
+
- `engine.results` - Result management API (`store_query_result`, `retrieve_query_result`, etc.)
|
|
173
|
+
- `engine.policy` - Policy validation API (`validate_policies`, etc.)
|
|
174
|
+
- `engine.benchmark` - Benchmarking API (`run_matrix`, etc.)
|
|
175
|
+
|
|
176
|
+
For complete Core API documentation, see `docs/api/core.md` in this repo
|
|
177
|
+
or the API section of the published MkDocs site.
|
|
178
|
+
|
|
179
|
+
## 📋 Public API Classes
|
|
180
|
+
|
|
181
|
+
The public API exports the following classes and types:
|
|
182
|
+
|
|
183
|
+
- `NL2SQL` - Main engine class
|
|
184
|
+
- `QueryResult` - Query result container
|
|
185
|
+
- `UserContext` - User authentication context
|
|
186
|
+
- `ErrorSeverity`, `ErrorCode`, `PipelineError` - Error handling types
|
|
187
|
+
- `QueryAPI`, `DatasourceAPI`, `LLM_API`, `IndexingAPI`, `AuthAPI`, `SettingsAPI`, `ResultAPI`, `PolicyAPI`, `BenchmarkAPI` - Modular API classes
|
|
188
|
+
- `BenchmarkConfig` - Benchmark configuration model
|
|
189
|
+
|
|
190
|
+
## 📦 Installation
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
# Engine, CLI and adapters; sqlite works out of the box
|
|
194
|
+
pip install nl2sql-engine
|
|
195
|
+
|
|
196
|
+
# Add the drivers for selected dialects
|
|
197
|
+
pip install "nl2sql-engine[mysql,mssql]"
|
|
198
|
+
|
|
199
|
+
# Add every database driver
|
|
200
|
+
pip install "nl2sql-engine[all]"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## 🔖 Versioning Policy
|
|
204
|
+
|
|
205
|
+
The three distributions in this monorepo -- `nl2sql-adapter-sdk`, `nl2sql-engine` and
|
|
206
|
+
`nl2sql-api` -- share a single version number and are released together. They
|
|
207
|
+
pin internal dependencies to the same version to prevent mismatches.
|
|
208
|
+
|
|
209
|
+
## 🚀 Usage (CLI)
|
|
210
|
+
|
|
211
|
+
The core package exposes the CLI entry point:
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
python -m nl2sql.cli --query "Show me all users" --id my_postgres_db
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
## 🛡️ Architectural Invariants
|
|
218
|
+
|
|
219
|
+
| Invariant | Rationale | Mechanism |
|
|
220
|
+
| :--- | :--- | :--- |
|
|
221
|
+
| **No Unvalidated SQL** | Prevent hallucinations & data leaks | All plans pass through `LogicalValidator` (AST). |
|
|
222
|
+
| **Zero Shared State** | Crash Safety | Execution happens in isolated processes; no shared memory with the Control Plane. |
|
|
223
|
+
| **Fail-Fast** | Reliability | Circuit Breakers and Strict Timeouts prevent cascading failures (Retry Storms). |
|
|
224
|
+
| **Determinism** | Debuggability | Temperature-0 generation + Strict Typing (Pydantic) for all LLM outputs. |
|
|
225
|
+
|
|
226
|
+
## 🏗️ Pipeline Flow
|
|
227
|
+
|
|
228
|
+
The main execution flow follows this sequence:
|
|
229
|
+
|
|
230
|
+
1. **Datasource Resolver** → **Decomposer** → **Global Planner** → **Layer Router**
|
|
231
|
+
2. **SQL Agent Subgraph**: Schema Retriever → AST Planner → Logical Validator → Generator → Executor
|
|
232
|
+
3. **Self-correction loops**: When validation fails, the system refines and retries
|
|
233
|
+
|
|
234
|
+
### SQL Agent Subgraph Details:
|
|
235
|
+
- **AST Planner**: Generates Abstract Syntax Tree instead of direct SQL
|
|
236
|
+
- **Logical Validator**: Enforces RBAC and schema constraints
|
|
237
|
+
- **Generator**: Converts AST to dialect-specific SQL
|
|
238
|
+
- **Executor**: Runs SQL in sandboxed environment
|
|
239
|
+
- **Refiner**: Self-correction when validation fails
|
|
240
|
+
|
|
241
|
+
## 🔐 Security Features
|
|
242
|
+
|
|
243
|
+
- **RBAC System**: Role-based access control for data access
|
|
244
|
+
- **Schema Validation**: All queries validated against schema before execution
|
|
245
|
+
- **Sandboxed Execution**: SQL runs in isolated processes
|
|
246
|
+
- **Query Limiting**: Row limits, timeout controls, and byte limits
|
|
247
|
+
- **Audit Logging**: Comprehensive logging of all operations
|
|
248
|
+
|
|
249
|
+
## 📊 Observability
|
|
250
|
+
|
|
251
|
+
- **OpenTelemetry Integration**: Native support for distributed tracing
|
|
252
|
+
- **Metrics Collection**: Performance and operational metrics
|
|
253
|
+
- **Audit Logs**: Persistent forensic logs for compliance
|
|
254
|
+
- **Structured Logging**: Rich, contextual log information
|
|
255
|
+
|
|
256
|
+
## 📁 Directory Structure
|
|
257
|
+
|
|
258
|
+
```
|
|
259
|
+
src/nl2sql/
|
|
260
|
+
├── api/ # Public API modules
|
|
261
|
+
│ ├── query_api.py # Query execution API
|
|
262
|
+
│ ├── datasource_api.py # Datasource management API
|
|
263
|
+
│ ├── llm_api.py # LLM configuration API
|
|
264
|
+
│ ├── indexing_api.py # Schema indexing API
|
|
265
|
+
│ ├── auth_api.py # Authentication API
|
|
266
|
+
│ ├── settings_api.py # Settings API
|
|
267
|
+
│ ├── result_api.py # Result management API
|
|
268
|
+
│ ├── policy_api.py # Policy validation API
|
|
269
|
+
│ └── benchmark_api.py # Benchmarking API
|
|
270
|
+
├── auth/ # Authentication and RBAC
|
|
271
|
+
├── common/ # Common utilities and settings
|
|
272
|
+
├── configs/ # Configuration management
|
|
273
|
+
├── adapters/ # Database adapters and the SQLAlchemy base
|
|
274
|
+
├── cli/ # `nl2sql` command line interface
|
|
275
|
+
├── datasources/ # Datasource management and discovery
|
|
276
|
+
├── execution/ # Execution engine and artifacts
|
|
277
|
+
├── indexing/ # Schema indexing system
|
|
278
|
+
├── llm/ # LLM management
|
|
279
|
+
├── pipeline/ # Graph orchestration
|
|
280
|
+
│ ├── nodes/ # Individual pipeline nodes
|
|
281
|
+
│ ├── subgraphs/ # Subgraph definitions
|
|
282
|
+
│ └── routes/ # Routing logic
|
|
283
|
+
├── schema/ # Schema management
|
|
284
|
+
├── secrets/ # Secret management
|
|
285
|
+
└── context.py # Application context
|
|
286
|
+
└── public_api.py # Public API facade
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
## 📋 Configuration
|
|
290
|
+
|
|
291
|
+
The engine requires configuration files for:
|
|
292
|
+
- `configs/datasources.yaml` - Database connection configurations
|
|
293
|
+
- `configs/llm.yaml` - LLM provider configurations
|
|
294
|
+
- `configs/secrets.yaml` - Secret management configurations
|
|
295
|
+
- `configs/policies.json` - RBAC policies and permissions
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# NL2SQL
|
|
2
|
+
|
|
3
|
+
The **`nl2sql`** package is the brain of the natural language to SQL engine. It orchestrates the entire query lifecycle using a graph-based agent architecture, and also ships the `nl2sql` CLI (`nl2sql.cli`) and the four database adapters (`nl2sql.adapters.*`).
|
|
4
|
+
|
|
5
|
+
## 🏗️ Architecture Overview
|
|
6
|
+
|
|
7
|
+
The NL2SQL Core is built around a **graph-based orchestration system** using LangGraph that treats text-to-SQL as a distributed systems problem. The architecture is organized around several key planes:
|
|
8
|
+
|
|
9
|
+
### 1. **The Control Plane (The Graph)**
|
|
10
|
+
- **Responsibility**: Reasoning, Planning, and Orchestration
|
|
11
|
+
- **Implementation**: Directed Cyclic Graph (LangGraph) with explicit state (`GraphState`)
|
|
12
|
+
- **Features**: Agentic graph with refinement loops for self-correction when plans fail validation
|
|
13
|
+
|
|
14
|
+
### 2. **The Security Plane (The Firewall)**
|
|
15
|
+
- **Responsibility**: Invariants Enforcement
|
|
16
|
+
- **Implementation**: Valid-by-Construction approach where LLM generates Abstract Syntax Tree (AST) rather than executing SQL
|
|
17
|
+
- **Features**: Static analysis through logical validators enforcing RBAC and schema constraints
|
|
18
|
+
|
|
19
|
+
### 3. **The Data Plane (The Sandbox)**
|
|
20
|
+
- **Responsibility**: Semantic Search and Execution
|
|
21
|
+
- **Implementation**: Sandboxed Process Pool for SQL driver isolation
|
|
22
|
+
- **Features**: Partitioned retrieval with schema store and vector-based context injection
|
|
23
|
+
|
|
24
|
+
### 4. **The Reliability Plane (The Guard)**
|
|
25
|
+
- **Responsibility**: Fault Tolerance and Stability
|
|
26
|
+
- **Implementation**: Layered defense with Circuit Breakers and Sandboxing
|
|
27
|
+
- **Features**: Fail-fast approach with strict timeouts preventing cascading failures
|
|
28
|
+
|
|
29
|
+
### 5. **The Observability Plane (The Watchtower)**
|
|
30
|
+
- **Responsibility**: Visibility, Forensics, and Compliance
|
|
31
|
+
- **Implementation**: Native OpenTelemetry integration
|
|
32
|
+
- **Features**: Distributed tracing (Jaeger), metrics (Prometheus), and forensic audit logs
|
|
33
|
+
|
|
34
|
+
## 🧠 Key Components
|
|
35
|
+
|
|
36
|
+
### **Context Management (`context.py`)**
|
|
37
|
+
- `NL2SQLContext`: Centralized application context managing initialization lifecycle
|
|
38
|
+
- Ensures proper ordering: secrets → datasources → LLMs → policies
|
|
39
|
+
- Coordinates all registries and stores
|
|
40
|
+
|
|
41
|
+
### **Graph Pipeline (`pipeline/`)**
|
|
42
|
+
- **Graph Orchestration**: LangGraph-based state machine managing query flow
|
|
43
|
+
- **Nodes**: DatasourceResolver, Decomposer, GlobalPlanner, Aggregator, AnswerSynthesizer
|
|
44
|
+
- **Subgraphs**: SQL Agent subgraph with AST planner, validators, and executor
|
|
45
|
+
- **State Management**: Shared `GraphState` for auditability and reproducibility
|
|
46
|
+
|
|
47
|
+
### **Schema Management (`schema/`)**
|
|
48
|
+
- **Schema Store**: Persistent storage for schema snapshots with versioning
|
|
49
|
+
- **Schema Contracts**: Typed representations of database schemas
|
|
50
|
+
- **Versioning**: Multiple schema versions with eviction policies
|
|
51
|
+
|
|
52
|
+
### **Indexing System (`indexing/`)**
|
|
53
|
+
- **Schema Indexing**: Vector-based indexing of schema information
|
|
54
|
+
- **Chunk Builder**: Breaks schema into searchable chunks
|
|
55
|
+
- **Enrichment Service**: Enhances schema with example questions
|
|
56
|
+
|
|
57
|
+
### **Data Sources (`datasources/`)**
|
|
58
|
+
- **Registry**: Dynamic registration and management of database adapters
|
|
59
|
+
- **Protocols**: Standardized interfaces for database connectivity
|
|
60
|
+
- **Discovery**: Automatic discovery of available adapter types
|
|
61
|
+
|
|
62
|
+
### **LLM Management (`llm/`)**
|
|
63
|
+
- **Registry**: Management of multiple LLM instances
|
|
64
|
+
- **Configuration**: Flexible LLM provider configuration (OpenAI, etc.)
|
|
65
|
+
- **Routing**: Intelligent routing to appropriate LLMs
|
|
66
|
+
|
|
67
|
+
### **Authentication & Authorization (`auth/`)**
|
|
68
|
+
- **RBAC**: Role-based access control for data access
|
|
69
|
+
- **User Context**: Identity and permission context propagation
|
|
70
|
+
- **Policy Engine**: Fine-grained access control rules
|
|
71
|
+
|
|
72
|
+
## 🚀 Public API
|
|
73
|
+
|
|
74
|
+
The main public interface is provided through the `NL2SQL` class:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from nl2sql import NL2SQL
|
|
78
|
+
|
|
79
|
+
# Initialize the engine
|
|
80
|
+
engine = NL2SQL(
|
|
81
|
+
ds_config_path="configs/datasources.yaml",
|
|
82
|
+
llm_config_path="configs/llm.yaml",
|
|
83
|
+
policies_path="configs/policies.json"
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# Run a natural language query
|
|
87
|
+
result = engine.run_query("Show top 10 customers by revenue")
|
|
88
|
+
print(result.final_answer)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Two-Tier API Architecture
|
|
92
|
+
|
|
93
|
+
NL2SQL provides a two-tier API architecture:
|
|
94
|
+
|
|
95
|
+
#### 1. Core API (Python) - This Package
|
|
96
|
+
- **Interface**: Direct Python class interface (`NL2SQL` class)
|
|
97
|
+
- **Use Case**: Direct Python integration, embedded applications
|
|
98
|
+
- **Access**: Import and use directly in Python code
|
|
99
|
+
|
|
100
|
+
#### 2. REST API (HTTP)
|
|
101
|
+
- **Package**: API package (`nl2sql-api`)
|
|
102
|
+
- **Interface**: HTTP REST endpoints
|
|
103
|
+
- **Use Case**: Remote clients, web applications, TypeScript CLI
|
|
104
|
+
- **Access**: HTTP requests to API endpoints
|
|
105
|
+
|
|
106
|
+
Both APIs provide access to the same underlying NL2SQL engine functionality, allowing flexible integration options.
|
|
107
|
+
|
|
108
|
+
### Modular API Structure
|
|
109
|
+
|
|
110
|
+
The engine provides modular APIs for different functionality areas:
|
|
111
|
+
|
|
112
|
+
- `engine.query` - Query execution API (`run_query`, etc.)
|
|
113
|
+
- `engine.datasource` - Datasource management API (`add_datasource`, `list_datasources`, etc.)
|
|
114
|
+
- `engine.llm` - LLM configuration API (`configure_llm`, etc.)
|
|
115
|
+
- `engine.indexing` - Schema indexing API (`index_datasource`, `clear_index`, etc.)
|
|
116
|
+
- `engine.auth` - Authentication and RBAC API (`check_permissions`, `get_allowed_resources`, etc.)
|
|
117
|
+
- `engine.settings` - Configuration and settings API (`get_current_settings`, `validate_configuration`, etc.)
|
|
118
|
+
- `engine.results` - Result management API (`store_query_result`, `retrieve_query_result`, etc.)
|
|
119
|
+
- `engine.policy` - Policy validation API (`validate_policies`, etc.)
|
|
120
|
+
- `engine.benchmark` - Benchmarking API (`run_matrix`, etc.)
|
|
121
|
+
|
|
122
|
+
For complete Core API documentation, see `docs/api/core.md` in this repo
|
|
123
|
+
or the API section of the published MkDocs site.
|
|
124
|
+
|
|
125
|
+
## 📋 Public API Classes
|
|
126
|
+
|
|
127
|
+
The public API exports the following classes and types:
|
|
128
|
+
|
|
129
|
+
- `NL2SQL` - Main engine class
|
|
130
|
+
- `QueryResult` - Query result container
|
|
131
|
+
- `UserContext` - User authentication context
|
|
132
|
+
- `ErrorSeverity`, `ErrorCode`, `PipelineError` - Error handling types
|
|
133
|
+
- `QueryAPI`, `DatasourceAPI`, `LLM_API`, `IndexingAPI`, `AuthAPI`, `SettingsAPI`, `ResultAPI`, `PolicyAPI`, `BenchmarkAPI` - Modular API classes
|
|
134
|
+
- `BenchmarkConfig` - Benchmark configuration model
|
|
135
|
+
|
|
136
|
+
## 📦 Installation
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
# Engine, CLI and adapters; sqlite works out of the box
|
|
140
|
+
pip install nl2sql-engine
|
|
141
|
+
|
|
142
|
+
# Add the drivers for selected dialects
|
|
143
|
+
pip install "nl2sql-engine[mysql,mssql]"
|
|
144
|
+
|
|
145
|
+
# Add every database driver
|
|
146
|
+
pip install "nl2sql-engine[all]"
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## 🔖 Versioning Policy
|
|
150
|
+
|
|
151
|
+
The three distributions in this monorepo -- `nl2sql-adapter-sdk`, `nl2sql-engine` and
|
|
152
|
+
`nl2sql-api` -- share a single version number and are released together. They
|
|
153
|
+
pin internal dependencies to the same version to prevent mismatches.
|
|
154
|
+
|
|
155
|
+
## 🚀 Usage (CLI)
|
|
156
|
+
|
|
157
|
+
The core package exposes the CLI entry point:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
python -m nl2sql.cli --query "Show me all users" --id my_postgres_db
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## 🛡️ Architectural Invariants
|
|
164
|
+
|
|
165
|
+
| Invariant | Rationale | Mechanism |
|
|
166
|
+
| :--- | :--- | :--- |
|
|
167
|
+
| **No Unvalidated SQL** | Prevent hallucinations & data leaks | All plans pass through `LogicalValidator` (AST). |
|
|
168
|
+
| **Zero Shared State** | Crash Safety | Execution happens in isolated processes; no shared memory with the Control Plane. |
|
|
169
|
+
| **Fail-Fast** | Reliability | Circuit Breakers and Strict Timeouts prevent cascading failures (Retry Storms). |
|
|
170
|
+
| **Determinism** | Debuggability | Temperature-0 generation + Strict Typing (Pydantic) for all LLM outputs. |
|
|
171
|
+
|
|
172
|
+
## 🏗️ Pipeline Flow
|
|
173
|
+
|
|
174
|
+
The main execution flow follows this sequence:
|
|
175
|
+
|
|
176
|
+
1. **Datasource Resolver** → **Decomposer** → **Global Planner** → **Layer Router**
|
|
177
|
+
2. **SQL Agent Subgraph**: Schema Retriever → AST Planner → Logical Validator → Generator → Executor
|
|
178
|
+
3. **Self-correction loops**: When validation fails, the system refines and retries
|
|
179
|
+
|
|
180
|
+
### SQL Agent Subgraph Details:
|
|
181
|
+
- **AST Planner**: Generates Abstract Syntax Tree instead of direct SQL
|
|
182
|
+
- **Logical Validator**: Enforces RBAC and schema constraints
|
|
183
|
+
- **Generator**: Converts AST to dialect-specific SQL
|
|
184
|
+
- **Executor**: Runs SQL in sandboxed environment
|
|
185
|
+
- **Refiner**: Self-correction when validation fails
|
|
186
|
+
|
|
187
|
+
## 🔐 Security Features
|
|
188
|
+
|
|
189
|
+
- **RBAC System**: Role-based access control for data access
|
|
190
|
+
- **Schema Validation**: All queries validated against schema before execution
|
|
191
|
+
- **Sandboxed Execution**: SQL runs in isolated processes
|
|
192
|
+
- **Query Limiting**: Row limits, timeout controls, and byte limits
|
|
193
|
+
- **Audit Logging**: Comprehensive logging of all operations
|
|
194
|
+
|
|
195
|
+
## 📊 Observability
|
|
196
|
+
|
|
197
|
+
- **OpenTelemetry Integration**: Native support for distributed tracing
|
|
198
|
+
- **Metrics Collection**: Performance and operational metrics
|
|
199
|
+
- **Audit Logs**: Persistent forensic logs for compliance
|
|
200
|
+
- **Structured Logging**: Rich, contextual log information
|
|
201
|
+
|
|
202
|
+
## 📁 Directory Structure
|
|
203
|
+
|
|
204
|
+
```
|
|
205
|
+
src/nl2sql/
|
|
206
|
+
├── api/ # Public API modules
|
|
207
|
+
│ ├── query_api.py # Query execution API
|
|
208
|
+
│ ├── datasource_api.py # Datasource management API
|
|
209
|
+
│ ├── llm_api.py # LLM configuration API
|
|
210
|
+
│ ├── indexing_api.py # Schema indexing API
|
|
211
|
+
│ ├── auth_api.py # Authentication API
|
|
212
|
+
│ ├── settings_api.py # Settings API
|
|
213
|
+
│ ├── result_api.py # Result management API
|
|
214
|
+
│ ├── policy_api.py # Policy validation API
|
|
215
|
+
│ └── benchmark_api.py # Benchmarking API
|
|
216
|
+
├── auth/ # Authentication and RBAC
|
|
217
|
+
├── common/ # Common utilities and settings
|
|
218
|
+
├── configs/ # Configuration management
|
|
219
|
+
├── adapters/ # Database adapters and the SQLAlchemy base
|
|
220
|
+
├── cli/ # `nl2sql` command line interface
|
|
221
|
+
├── datasources/ # Datasource management and discovery
|
|
222
|
+
├── execution/ # Execution engine and artifacts
|
|
223
|
+
├── indexing/ # Schema indexing system
|
|
224
|
+
├── llm/ # LLM management
|
|
225
|
+
├── pipeline/ # Graph orchestration
|
|
226
|
+
│ ├── nodes/ # Individual pipeline nodes
|
|
227
|
+
│ ├── subgraphs/ # Subgraph definitions
|
|
228
|
+
│ └── routes/ # Routing logic
|
|
229
|
+
├── schema/ # Schema management
|
|
230
|
+
├── secrets/ # Secret management
|
|
231
|
+
└── context.py # Application context
|
|
232
|
+
└── public_api.py # Public API facade
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
## 📋 Configuration
|
|
236
|
+
|
|
237
|
+
The engine requires configuration files for:
|
|
238
|
+
- `configs/datasources.yaml` - Database connection configurations
|
|
239
|
+
- `configs/llm.yaml` - LLM provider configurations
|
|
240
|
+
- `configs/secrets.yaml` - Secret management configurations
|
|
241
|
+
- `configs/policies.json` - RBAC policies and permissions
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nl2sql-engine"
|
|
7
|
+
version = "0.1.0" # x-release-please-version
|
|
8
|
+
description = "Natural-language-to-SQL engine, CLI and database adapters"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"SQLAlchemy>=2.0",
|
|
13
|
+
"PyYAML>=6.0",
|
|
14
|
+
"langgraph>=1.2,<2",
|
|
15
|
+
"langchain>=0.2.0",
|
|
16
|
+
"langchain-core>=1.6,<2",
|
|
17
|
+
"langchain-chroma>=1.1,<2",
|
|
18
|
+
"langchain-openai>=1.6,<2",
|
|
19
|
+
"chromadb>=1.5,<2",
|
|
20
|
+
"nl2sql-adapter-sdk~=0.1",
|
|
21
|
+
"pydantic>=2.0",
|
|
22
|
+
"opentelemetry-api>=1.20.0",
|
|
23
|
+
"opentelemetry-sdk>=1.20.0",
|
|
24
|
+
"opentelemetry-exporter-otlp>=1.20.0",
|
|
25
|
+
"sqlglot>=23.0.0",
|
|
26
|
+
"pydantic-settings>=2.0.0",
|
|
27
|
+
"pandas>=1.5.0", # For metrics/evals
|
|
28
|
+
"pyarrow>=14.0.0", # For the parquet artifact store
|
|
29
|
+
"pybreaker>=1.3.0",
|
|
30
|
+
"polars>=1.44",
|
|
31
|
+
"duckdb>=1.5",
|
|
32
|
+
# CLI
|
|
33
|
+
"rich>=13.0.0",
|
|
34
|
+
"typer[all]>=0.9.0",
|
|
35
|
+
"inquirerpy>=0.3.4",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
# Every dialect adapter ships in this distribution; an extra only adds the
|
|
39
|
+
# database driver it needs to connect. sqlite needs none -- the driver is in
|
|
40
|
+
# the standard library -- so it has no extra.
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
aws = ["boto3"]
|
|
43
|
+
azure = ["azure-identity", "azure-keyvault-secrets"]
|
|
44
|
+
duckdb = ["duckdb", "duckdb-engine"]
|
|
45
|
+
hashicorp = ["hvac"]
|
|
46
|
+
mssql = [
|
|
47
|
+
"pyodbc>=4.0.0",
|
|
48
|
+
"pymssql>=2.2.0; platform_system=='Linux'", # Optional fallback
|
|
49
|
+
]
|
|
50
|
+
mysql = ["pymysql>=1.1.0"]
|
|
51
|
+
postgres = ["psycopg2-binary>=2.9"]
|
|
52
|
+
all = [
|
|
53
|
+
"duckdb",
|
|
54
|
+
"duckdb-engine",
|
|
55
|
+
"pyodbc>=4.0.0",
|
|
56
|
+
"pymssql>=2.2.0; platform_system=='Linux'",
|
|
57
|
+
"pymysql>=1.1.0",
|
|
58
|
+
"psycopg2-binary>=2.9",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[project.scripts]
|
|
62
|
+
nl2sql = "nl2sql.cli.main:main"
|
|
63
|
+
|
|
64
|
+
[project.entry-points."nl2sql.adapters"]
|
|
65
|
+
duckdb = "nl2sql.adapters.duckdb.adapter:DuckdbAdapter"
|
|
66
|
+
mssql = "nl2sql.adapters.mssql.adapter:MssqlAdapter"
|
|
67
|
+
mysql = "nl2sql.adapters.mysql.adapter:MysqlAdapter"
|
|
68
|
+
postgres = "nl2sql.adapters.postgres.adapter:PostgresAdapter"
|
|
69
|
+
sqlite = "nl2sql.adapters.sqlite.adapter:SqliteAdapter"
|
|
70
|
+
|
|
71
|
+
[tool.setuptools.packages.find]
|
|
72
|
+
where = ["src"]
|
|
73
|
+
include = ["nl2sql*"]
|
|
74
|
+
|
|
75
|
+
[tool.setuptools.package-data]
|
|
76
|
+
"*" = ["configs/*.yaml", ".env"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# nl2sql package
|
|
2
|
+
|
|
3
|
+
from .public_api import NL2SQL, QueryResult
|
|
4
|
+
|
|
5
|
+
# Also expose individual API modules for more granular access
|
|
6
|
+
from .api.query_api import QueryAPI
|
|
7
|
+
from .api.datasource_api import DatasourceAPI
|
|
8
|
+
from .api.llm_api import LLM_API
|
|
9
|
+
from .api.indexing_api import IndexingAPI
|
|
10
|
+
from .api.auth_api import AuthAPI
|
|
11
|
+
from .api.settings_api import SettingsAPI
|
|
12
|
+
from .api.result_api import ResultAPI
|
|
13
|
+
from .api.policy_api import PolicyAPI
|
|
14
|
+
from .api.benchmark_api import BenchmarkAPI
|
|
15
|
+
|
|
16
|
+
# Also expose core models and enums
|
|
17
|
+
from .common.errors import ErrorSeverity, ErrorCode, PipelineError
|
|
18
|
+
from .auth.models import UserContext
|
|
19
|
+
from .evaluation.types import BenchmarkConfig
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"NL2SQL",
|
|
23
|
+
"QueryResult",
|
|
24
|
+
"QueryAPI",
|
|
25
|
+
"DatasourceAPI",
|
|
26
|
+
"LLM_API",
|
|
27
|
+
"IndexingAPI",
|
|
28
|
+
"AuthAPI",
|
|
29
|
+
"SettingsAPI",
|
|
30
|
+
"ResultAPI",
|
|
31
|
+
"PolicyAPI",
|
|
32
|
+
"BenchmarkAPI",
|
|
33
|
+
"ErrorSeverity",
|
|
34
|
+
"ErrorCode",
|
|
35
|
+
"PipelineError",
|
|
36
|
+
"UserContext",
|
|
37
|
+
"BenchmarkConfig",
|
|
38
|
+
]
|
|
File without changes
|
|
File without changes
|