nlp2sql 0.2.0rc10__tar.gz → 0.2.0rc12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/PKG-INFO +3 -2
  2. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/README.md +2 -1
  3. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/pyproject.toml +1 -1
  4. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/__init__.py +1 -1
  5. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/openai_embedding_adapter.py +10 -7
  6. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/redshift_adapter.py +215 -120
  7. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_redshift_integration.py +125 -0
  8. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/uv.lock +1 -1
  9. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/db.md +0 -0
  10. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/quality.md +0 -0
  11. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/test.md +0 -0
  12. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/adapters.md +0 -0
  13. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/architecture.md +0 -0
  14. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/python-style.md +0 -0
  15. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/testing.md +0 -0
  16. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/settings.json +0 -0
  17. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  18. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  19. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/ci.yml +0 -0
  20. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/publish-pypi.yml +0 -0
  21. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/publish-testpypi.yml +0 -0
  22. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.gitignore +0 -0
  23. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.mcp.json.example +0 -0
  24. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.python-version +0 -0
  25. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/CLAUDE.md +0 -0
  26. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/CONTRIBUTING.md +0 -0
  27. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/LICENSE +0 -0
  28. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/SECURITY.md +0 -0
  29. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/assets/nlp2sql-logo.png +0 -0
  30. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/benchmarks/enterprise_benchmark_results.json +0 -0
  31. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/.env.example +0 -0
  32. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/README.md +0 -0
  33. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/docker-compose.yml +0 -0
  34. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/get-db-urls.sh +0 -0
  35. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/init-schema.sql +0 -0
  36. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/large-schema.sql +0 -0
  37. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/localstack-init.sh +0 -0
  38. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/test-redshift.sh +0 -0
  39. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/API.md +0 -0
  40. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/ARCHITECTURE.md +0 -0
  41. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/CONFIGURATION.md +0 -0
  42. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/ENTERPRISE.md +0 -0
  43. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/Redshift.md +0 -0
  44. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/README.md +0 -0
  45. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/advanced/test_multiple_providers.py +0 -0
  46. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/advanced/test_simple_api.py +0 -0
  47. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/database_specific/test_odoo_integration.py +0 -0
  48. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/default_examples.json +0 -0
  49. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/documentation/real_world_example.py +0 -0
  50. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/basic_usage.py +0 -0
  51. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/simple_demo.py +0 -0
  52. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/test_api_setup.py +0 -0
  53. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/schema_management/test_auto_schema.py +0 -0
  54. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/schema_management/test_schema_filters_comprehensive.py +0 -0
  55. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/.dockerignore +0 -0
  56. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/.env.example +0 -0
  57. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_README.md +0 -0
  58. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_SETUP.md +0 -0
  59. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_TESTING_SUMMARY.md +0 -0
  60. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/Dockerfile +0 -0
  61. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/INDEX.md +0 -0
  62. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/README.md +0 -0
  63. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/docker/init-ecommerce-schema.sql +0 -0
  64. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/docker-compose.yml +0 -0
  65. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/main.py +0 -0
  66. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/pyproject.toml +0 -0
  67. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/test-docker-endpoints.sh +0 -0
  68. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/install-dev.sh +0 -0
  69. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/README.md +0 -0
  70. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/__init__.py +0 -0
  71. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/example_config.json +0 -0
  72. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/pyproject.toml +0 -0
  73. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/server.py +0 -0
  74. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/__init__.py +0 -0
  75. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/anthropic_adapter.py +0 -0
  76. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/gemini_adapter.py +0 -0
  77. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/local_embedding_adapter.py +0 -0
  78. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/openai_adapter.py +0 -0
  79. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/postgres_repository.py +0 -0
  80. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/regex_query_validator.py +0 -0
  81. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/cli.py +0 -0
  82. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/client.py +0 -0
  83. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/config/__init__.py +0 -0
  84. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/config/settings.py +0 -0
  85. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/__init__.py +0 -0
  86. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/database_prompts.py +0 -0
  87. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/entities.py +0 -0
  88. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/provider_config.py +0 -0
  89. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/result.py +0 -0
  90. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/sql_keywords.py +0 -0
  91. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/sql_safety.py +0 -0
  92. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/exceptions/__init__.py +0 -0
  93. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/factories/__init__.py +0 -0
  94. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/factories/repository_factory.py +0 -0
  95. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/__init__.py +0 -0
  96. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/ai_provider.py +0 -0
  97. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/cache.py +0 -0
  98. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/embedding_provider.py +0 -0
  99. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/example_repository.py +0 -0
  100. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_optimizer.py +0 -0
  101. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_safety.py +0 -0
  102. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_validator.py +0 -0
  103. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/schema_repository.py +0 -0
  104. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/schema_strategy.py +0 -0
  105. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/__init__.py +0 -0
  106. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/analyzer.py +0 -0
  107. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/embedding_manager.py +0 -0
  108. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/example_loader.py +0 -0
  109. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/example_store.py +0 -0
  110. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/manager.py +0 -0
  111. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/services/__init__.py +0 -0
  112. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/services/query_service.py +0 -0
  113. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/__init__.py +0 -0
  114. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/helpers.py +0 -0
  115. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/storage.py +0 -0
  116. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/test_tmp/.gitignore +0 -0
  117. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/conftest.py +0 -0
  118. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/mcp/__init__.py +0 -0
  119. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/mcp/test_sql_security.py +0 -0
  120. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_basic.py +0 -0
  121. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_batch_scoring.py +0 -0
  122. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_column_validation.py +0 -0
  123. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_database_prompts.py +0 -0
  124. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_dsl_integration.py +0 -0
  125. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_embedding_manager.py +0 -0
  126. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_integration.py +0 -0
  127. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_postgres_integration.py +0 -0
  128. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_repository_cache.py +0 -0
  129. {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_repository_factory.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nlp2sql
3
- Version: 0.2.0rc10
3
+ Version: 0.2.0rc12
4
4
  Summary: Enterprise-ready Natural Language to SQL converter with multi-provider support. Built for production scale (1000+ tables) with Clean Architecture.
5
5
  Project-URL: Homepage, https://github.com/luiscarbonel1991/nlp2sql
6
6
  Project-URL: Bug Tracker, https://github.com/luiscarbonel1991/nlp2sql/issues
@@ -81,12 +81,13 @@ Description-Content-Type: text/markdown
81
81
  </p>
82
82
 
83
83
  <p align="center">
84
+ <a href="https://pepy.tech/projects/nlp2sql"><img src="https://static.pepy.tech/personalized-badge/nlp2sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
84
85
  <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
85
86
  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
86
87
  <a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code style: black"></a>
87
88
  </p>
88
89
 
89
- # nlp2sql
90
+ # nlp2sql
90
91
 
91
92
  **Enterprise-ready Natural Language to SQL converter with multi-provider support**
92
93
 
@@ -3,12 +3,13 @@
3
3
  </p>
4
4
 
5
5
  <p align="center">
6
+ <a href="https://pepy.tech/projects/nlp2sql"><img src="https://static.pepy.tech/personalized-badge/nlp2sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
6
7
  <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
7
8
  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
8
9
  <a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code style: black"></a>
9
10
  </p>
10
11
 
11
- # nlp2sql
12
+ # nlp2sql
12
13
 
13
14
  **Enterprise-ready Natural Language to SQL converter with multi-provider support**
14
15
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "nlp2sql"
3
- version = "0.2.0rc10"
3
+ version = "0.2.0rc12"
4
4
  description = "Enterprise-ready Natural Language to SQL converter with multi-provider support. Built for production scale (1000+ tables) with Clean Architecture."
5
5
  readme = "README.md"
6
6
  license = {text = "MIT"}
@@ -20,7 +20,7 @@ from .ports.schema_repository import SchemaRepositoryPort
20
20
  from .schema.example_store import ExampleStore
21
21
  from .services.query_service import QueryGenerationService
22
22
 
23
- __version__ = "0.2.0rc10"
23
+ __version__ = "0.2.0rc12"
24
24
  __author__ = "Luis Carbonel"
25
25
  __email__ = "devhighlevel@gmail.com"
26
26
 
@@ -79,17 +79,19 @@ class OpenAIEmbeddingAdapter(EmbeddingProviderPort):
79
79
  processed_texts.append(text)
80
80
 
81
81
  try:
82
- response = await self.client.embeddings.create(model=self.model, input=processed_texts)
82
+ # OpenAI API limits input array to 2048 items per request
83
+ batch_size = 2000
84
+ all_embeddings: list[list[float]] = []
83
85
 
84
- # Extract embeddings from response
85
- embeddings = [item.embedding for item in response.data]
86
- embeddings_array = np.array(embeddings)
86
+ for i in range(0, len(processed_texts), batch_size):
87
+ batch = processed_texts[i : i + batch_size]
88
+ response = await self.client.embeddings.create(model=self.model, input=batch)
89
+ all_embeddings.extend(item.embedding for item in response.data)
90
+
91
+ embeddings_array = np.array(all_embeddings)
87
92
 
88
93
  # Normalize embeddings for cosine similarity with FAISS IndexFlatIP
89
- # This is critical: FAISS IndexFlatIP uses inner product which only works
90
- # as cosine similarity when vectors are normalized to unit length
91
94
  norms = np.linalg.norm(embeddings_array, axis=1, keepdims=True)
92
- # Avoid division by zero (though rare for real embeddings)
93
95
  norms = np.where(norms == 0, 1, norms)
94
96
  normalized_embeddings = embeddings_array / norms
95
97
 
@@ -97,6 +99,7 @@ class OpenAIEmbeddingAdapter(EmbeddingProviderPort):
97
99
  "OpenAI embeddings generated and normalized",
98
100
  model=self.model,
99
101
  texts_count=len(processed_texts),
102
+ batches=((len(processed_texts) - 1) // batch_size) + 1,
100
103
  dimension=normalized_embeddings.shape[1],
101
104
  )
102
105
 
@@ -39,10 +39,16 @@ class RedshiftRepository(SchemaRepositoryPort):
39
39
  def __init__(self, connection_string: str, schema_name: str = "public"):
40
40
  self.connection_string = connection_string
41
41
  self.database_url = connection_string
42
- self.schema_name = schema_name
42
+ self.schema_name = schema_name.lower() if schema_name else schema_name
43
43
  self._connection_params = self._parse_connection_string(connection_string)
44
44
  self._initialized = False
45
45
  self._cache_dir: Optional[Path] = None
46
+ # System view configuration (auto-detected at init)
47
+ self._table_view: str = "svv_tables"
48
+ self._column_view: str = "svv_columns"
49
+ self._schema_col: str = "table_schema"
50
+ self._db_filter: str = ""
51
+ self._db_filter_aliased: str = ""
46
52
 
47
53
  def _get_cache_dir(self) -> Path:
48
54
  """Get the cache directory for this database connection."""
@@ -106,7 +112,11 @@ class RedshiftRepository(SchemaRepositoryPort):
106
112
  return None
107
113
 
108
114
  def _save_tables_to_cache(self, tables: List[TableInfo], schema: str) -> None:
109
- """Save tables to disk cache."""
115
+ """Save tables to disk cache. Skips caching if tables list is empty."""
116
+ if not tables:
117
+ logger.debug("Skipping cache save for empty tables list", schema=schema)
118
+ return
119
+
110
120
  cache_path = self._get_tables_cache_path(schema)
111
121
  try:
112
122
  cache_data = {
@@ -169,12 +179,12 @@ class RedshiftRepository(SchemaRepositoryPort):
169
179
  return psycopg2.connect(**self._connection_params)
170
180
 
171
181
  async def initialize(self) -> None:
172
- """Initialize database connections."""
182
+ """Initialize database connections and detect available system views."""
173
183
  if self._initialized:
174
184
  return
175
185
 
176
186
  try:
177
- # Test connection
187
+
178
188
  def test_connection():
179
189
  conn = self._get_connection()
180
190
  cursor = conn.cursor()
@@ -183,14 +193,79 @@ class RedshiftRepository(SchemaRepositoryPort):
183
193
  conn.close()
184
194
 
185
195
  await asyncio.to_thread(test_connection)
196
+ await self._detect_system_views()
186
197
 
187
198
  self._initialized = True
188
- logger.info("Redshift repository initialized")
199
+ logger.info("Redshift repository initialized", table_view=self._table_view)
189
200
 
201
+ except SchemaException:
202
+ raise
190
203
  except Exception as e:
191
204
  logger.error("Failed to initialize Redshift repository", error=str(e))
192
205
  raise SchemaException(f"Database initialization failed: {e!s}")
193
206
 
207
+ async def _detect_system_views(self) -> None:
208
+ """Auto-detect which system views are available for this connection.
209
+
210
+ Fallback chain: svv_tables → svv_all_tables → information_schema
211
+ """
212
+ probes = [
213
+ {
214
+ "table_view": "svv_tables",
215
+ "column_view": "svv_columns",
216
+ "schema_col": "table_schema",
217
+ "db_filter": "",
218
+ "db_filter_aliased": "",
219
+ "query": "SELECT COUNT(*) as cnt FROM svv_tables WHERE table_schema = %s",
220
+ },
221
+ {
222
+ "table_view": "svv_all_tables",
223
+ "column_view": "svv_all_columns",
224
+ "schema_col": "schema_name",
225
+ "db_filter": "AND database_name = current_database()",
226
+ "db_filter_aliased": "AND t.database_name = current_database()",
227
+ "query": (
228
+ "SELECT COUNT(*) as cnt FROM svv_all_tables "
229
+ "WHERE schema_name = %s AND database_name = current_database()"
230
+ ),
231
+ },
232
+ {
233
+ "table_view": "information_schema.tables",
234
+ "column_view": "information_schema.columns",
235
+ "schema_col": "table_schema",
236
+ "db_filter": "AND table_type IN ('BASE TABLE', 'VIEW')",
237
+ "db_filter_aliased": "AND t.table_type IN ('BASE TABLE', 'VIEW')",
238
+ "query": (
239
+ "SELECT COUNT(*) as cnt FROM information_schema.tables "
240
+ "WHERE table_schema = %s AND table_type IN ('BASE TABLE', 'VIEW')"
241
+ ),
242
+ },
243
+ ]
244
+
245
+ for probe in probes:
246
+ try:
247
+ row = await asyncio.to_thread(self._execute_query_one, probe["query"], (self.schema_name,))
248
+ if row and row["cnt"] > 0:
249
+ self._table_view = probe["table_view"]
250
+ self._column_view = probe["column_view"]
251
+ self._schema_col = probe["schema_col"]
252
+ self._db_filter = probe["db_filter"]
253
+ self._db_filter_aliased = probe["db_filter_aliased"]
254
+ logger.info(
255
+ "Schema discovery source selected",
256
+ view=probe["table_view"],
257
+ schema=self.schema_name,
258
+ tables=row["cnt"],
259
+ )
260
+ return
261
+ except Exception as e:
262
+ logger.debug("Probe failed", view=probe["table_view"], error=str(e))
263
+
264
+ logger.warning(
265
+ "All schema discovery probes returned 0 tables",
266
+ schema=self.schema_name,
267
+ )
268
+
194
269
  def _execute_query(self, query: str, params: tuple = None) -> List[Dict]:
195
270
  """Execute a query and return results as list of dicts."""
196
271
  conn = self._get_connection()
@@ -209,18 +284,10 @@ class RedshiftRepository(SchemaRepositoryPort):
209
284
  return results[0] if results else None
210
285
 
211
286
  async def get_tables(self, schema_name: Optional[str] = None, force_refresh: bool = False) -> List[TableInfo]:
212
- """Get all tables in the schema using hybrid bulk query + disk cache.
213
-
214
- Uses SVV_TABLES and SVV_COLUMNS (Redshift system views) for better
215
- permission handling. See AWS docs:
216
- https://docs.aws.amazon.com/redshift/latest/dg/cm_chap_system-tables.html
217
-
218
- Args:
219
- schema_name: Schema name to query (defaults to self.schema_name)
220
- force_refresh: If True, bypass cache and fetch fresh data
287
+ """Get all tables in the schema using bulk query + disk cache.
221
288
 
222
- Returns:
223
- List of TableInfo objects
289
+ The system view source (svv_tables, svv_all_tables, or information_schema)
290
+ is auto-detected at initialization time.
224
291
  """
225
292
  if not self._initialized:
226
293
  await self.initialize()
@@ -242,18 +309,78 @@ class RedshiftRepository(SchemaRepositoryPort):
242
309
 
243
310
  return tables
244
311
 
245
- async def _get_tables_bulk(self, schema: str) -> List[TableInfo]:
246
- """Fetch all tables with columns and primary keys in a single bulk query.
312
+ def _process_bulk_rows(self, rows: List[Dict]) -> List[TableInfo]:
313
+ """Convert raw bulk query rows into TableInfo objects."""
314
+ tables_dict: Dict[str, Dict[str, Any]] = {}
315
+ for row in rows:
316
+ table_name = row["table_name"]
317
+
318
+ if table_name not in tables_dict:
319
+ pk_cols = row.get("pk_columns")
320
+ if pk_cols:
321
+ if isinstance(pk_cols, str):
322
+ pk_cols = pk_cols.strip("{}").split(",") if pk_cols != "{}" else []
323
+ elif isinstance(pk_cols, list):
324
+ pk_cols = list(pk_cols)
325
+ else:
326
+ pk_cols = []
327
+ else:
328
+ pk_cols = []
329
+
330
+ tables_dict[table_name] = {
331
+ "name": table_name,
332
+ "schema": row["table_schema"],
333
+ "description": row.get("table_comment", ""),
334
+ "columns": [],
335
+ "primary_keys": pk_cols,
336
+ "foreign_keys": [],
337
+ "indexes": [],
338
+ "_seen_columns": set(),
339
+ }
247
340
 
248
- Uses Redshift SVV views for better permission handling.
249
- """
250
- # Note: Redshift's information_schema views have limited support for
251
- # constraints, so we skip primary key detection in the bulk query.
252
- # Primary keys are not essential for SQL generation.
253
- bulk_query = """
341
+ col_name = row.get("column_name")
342
+ if col_name and col_name not in tables_dict[table_name]["_seen_columns"]:
343
+ tables_dict[table_name]["_seen_columns"].add(col_name)
344
+ tables_dict[table_name]["columns"].append(
345
+ {
346
+ "name": col_name,
347
+ "type": row.get("data_type", ""),
348
+ "nullable": row.get("is_nullable") == "YES",
349
+ "default": row.get("column_default"),
350
+ "max_length": row.get("character_maximum_length"),
351
+ "precision": row.get("numeric_precision"),
352
+ "scale": row.get("numeric_scale"),
353
+ "description": "",
354
+ }
355
+ )
356
+
357
+ tables = []
358
+ for table_data in tables_dict.values():
359
+ del table_data["_seen_columns"]
360
+ tables.append(
361
+ TableInfo(
362
+ name=table_data["name"],
363
+ schema=table_data["schema"],
364
+ columns=table_data["columns"],
365
+ primary_keys=table_data["primary_keys"],
366
+ foreign_keys=table_data["foreign_keys"],
367
+ indexes=table_data["indexes"],
368
+ row_count=0,
369
+ size_bytes=0,
370
+ description=table_data["description"],
371
+ last_updated=datetime.now(),
372
+ )
373
+ )
374
+
375
+ logger.info("Tables processed from bulk query", count=len(tables))
376
+ return tables
377
+
378
+ async def _get_tables_bulk(self, schema: str) -> List[TableInfo]:
379
+ """Fetch all tables with columns in a single bulk query."""
380
+ bulk_query = f"""
254
381
  SELECT
255
382
  t.table_name,
256
- t.table_schema,
383
+ t.{self._schema_col} as table_schema,
257
384
  '' as table_comment,
258
385
  c.column_name,
259
386
  c.data_type,
@@ -264,10 +391,10 @@ class RedshiftRepository(SchemaRepositoryPort):
264
391
  c.numeric_scale,
265
392
  c.ordinal_position,
266
393
  NULL as pk_columns
267
- FROM svv_tables t
268
- LEFT JOIN svv_columns c ON t.table_name = c.table_name
269
- AND t.table_schema = c.table_schema
270
- WHERE t.table_schema = %s
394
+ FROM {self._table_view} t
395
+ LEFT JOIN {self._column_view} c ON t.table_name = c.table_name
396
+ AND t.{self._schema_col} = c.{self._schema_col}
397
+ WHERE t.{self._schema_col} = %s {self._db_filter_aliased}
271
398
  ORDER BY t.table_name, c.ordinal_position
272
399
  """
273
400
 
@@ -277,79 +404,54 @@ class RedshiftRepository(SchemaRepositoryPort):
277
404
 
278
405
  elapsed_ms = round((time.time() - start_time) * 1000, 2)
279
406
  logger.info("Bulk query completed", rows=len(rows), elapsed_ms=elapsed_ms)
407
+ return self._process_bulk_rows(rows)
280
408
 
281
- # Process rows and group by table
282
- tables_dict: Dict[str, Dict[str, Any]] = {}
283
- for row in rows:
284
- table_name = row["table_name"]
285
-
286
- if table_name not in tables_dict:
287
- # Parse pk_columns - Redshift may return string representation
288
- pk_cols = row.get("pk_columns")
289
- if pk_cols:
290
- if isinstance(pk_cols, str):
291
- # Handle string like '{col1,col2}'
292
- pk_cols = pk_cols.strip("{}").split(",") if pk_cols != "{}" else []
293
- elif isinstance(pk_cols, list):
294
- pk_cols = list(pk_cols)
295
- else:
296
- pk_cols = []
297
- else:
298
- pk_cols = []
299
-
300
- tables_dict[table_name] = {
301
- "name": table_name,
302
- "schema": row["table_schema"],
303
- "description": row.get("table_comment", ""),
304
- "columns": [],
305
- "primary_keys": pk_cols,
306
- "foreign_keys": [],
307
- "indexes": [],
308
- "_seen_columns": set(),
309
- }
310
-
311
- # Add column (avoiding duplicates)
312
- col_name = row.get("column_name")
313
- if col_name and col_name not in tables_dict[table_name]["_seen_columns"]:
314
- tables_dict[table_name]["_seen_columns"].add(col_name)
315
- tables_dict[table_name]["columns"].append(
316
- {
317
- "name": col_name,
318
- "type": row.get("data_type", ""),
319
- "nullable": row.get("is_nullable") == "YES",
320
- "default": row.get("column_default"),
321
- "max_length": row.get("character_maximum_length"),
322
- "precision": row.get("numeric_precision"),
323
- "scale": row.get("numeric_scale"),
324
- "description": "",
325
- }
326
- )
327
-
328
- # Convert to TableInfo objects
329
- tables = []
330
- for table_data in tables_dict.values():
331
- del table_data["_seen_columns"]
332
-
333
- tables.append(
334
- TableInfo(
335
- name=table_data["name"],
336
- schema=table_data["schema"],
337
- columns=table_data["columns"],
338
- primary_keys=table_data["primary_keys"],
339
- foreign_keys=table_data["foreign_keys"],
340
- indexes=table_data["indexes"],
341
- row_count=0,
342
- size_bytes=0,
343
- description=table_data["description"],
344
- last_updated=datetime.now(),
345
- )
409
+ except Exception as e:
410
+ if self._table_view != "information_schema.tables":
411
+ logger.warning(
412
+ "Bulk query failed, retrying with information_schema",
413
+ error=str(e),
414
+ original_view=self._table_view,
346
415
  )
416
+ return await self._get_tables_bulk_information_schema(schema)
347
417
 
348
- logger.info("Tables processed from bulk query", count=len(tables))
349
- return tables
418
+ logger.error("Bulk query failed", error=str(e))
419
+ raise SchemaException(f"Failed to get tables: {e!s}")
350
420
 
421
+ async def _get_tables_bulk_information_schema(self, schema: str) -> List[TableInfo]:
422
+ """Fallback bulk query using information_schema (no state mutation)."""
423
+ fallback_query = """
424
+ SELECT
425
+ t.table_name,
426
+ t.table_schema,
427
+ '' as table_comment,
428
+ c.column_name,
429
+ c.data_type,
430
+ c.is_nullable,
431
+ c.column_default,
432
+ c.character_maximum_length,
433
+ c.numeric_precision,
434
+ c.numeric_scale,
435
+ c.ordinal_position,
436
+ NULL as pk_columns
437
+ FROM information_schema.tables t
438
+ LEFT JOIN information_schema.columns c ON t.table_name = c.table_name
439
+ AND t.table_schema = c.table_schema
440
+ WHERE t.table_schema = %s AND t.table_type IN ('BASE TABLE', 'VIEW')
441
+ ORDER BY t.table_name, c.ordinal_position
442
+ """
443
+ try:
444
+ start_time = time.time()
445
+ rows = await asyncio.to_thread(self._execute_query, fallback_query, (schema,))
446
+ elapsed_ms = round((time.time() - start_time) * 1000, 2)
447
+ logger.info(
448
+ "information_schema fallback completed",
449
+ rows=len(rows),
450
+ elapsed_ms=elapsed_ms,
451
+ )
452
+ return self._process_bulk_rows(rows)
351
453
  except Exception as e:
352
- logger.error("Bulk query failed", error=str(e))
454
+ logger.error("information_schema fallback also failed", error=str(e))
353
455
  raise SchemaException(f"Failed to get tables: {e!s}")
354
456
 
355
457
  async def get_table_info(self, table_name: str, schema_name: Optional[str] = None) -> TableInfo:
@@ -359,14 +461,13 @@ class RedshiftRepository(SchemaRepositoryPort):
359
461
 
360
462
  schema = schema_name or self.schema_name
361
463
 
362
- # Use SVV_TABLES for better Redshift compatibility
363
- query = """
464
+ query = f"""
364
465
  SELECT
365
466
  table_name,
366
- table_schema,
467
+ {self._schema_col} as table_schema,
367
468
  '' as table_comment
368
- FROM svv_tables
369
- WHERE table_name = %s AND table_schema = %s
469
+ FROM {self._table_view}
470
+ WHERE table_name = %s AND {self._schema_col} = %s {self._db_filter}
370
471
  """
371
472
 
372
473
  try:
@@ -388,15 +489,14 @@ class RedshiftRepository(SchemaRepositoryPort):
388
489
  if not self._initialized:
389
490
  await self.initialize()
390
491
 
391
- # Use SVV_TABLES for better Redshift compatibility
392
- query = """
492
+ query = f"""
393
493
  SELECT
394
494
  table_name,
395
- table_schema,
495
+ {self._schema_col} as table_schema,
396
496
  '' as table_comment
397
- FROM svv_tables
398
- WHERE table_schema = %s
399
- AND table_name ILIKE %s
497
+ FROM {self._table_view}
498
+ WHERE {self._schema_col} = %s
499
+ AND table_name ILIKE %s {self._db_filter}
400
500
  ORDER BY table_name
401
501
  """
402
502
 
@@ -427,14 +527,13 @@ class RedshiftRepository(SchemaRepositoryPort):
427
527
  if not self._initialized:
428
528
  await self.initialize()
429
529
 
430
- # Use SVV_TABLES for better Redshift compatibility
431
- query = """
530
+ query = f"""
432
531
  SELECT
433
532
  current_database() as database_name,
434
533
  version() as database_version,
435
534
  COUNT(*) as total_tables
436
- FROM svv_tables
437
- WHERE table_schema = %s
535
+ FROM {self._table_view}
536
+ WHERE {self._schema_col} = %s {self._db_filter}
438
537
  """
439
538
 
440
539
  try:
@@ -498,12 +597,8 @@ class RedshiftRepository(SchemaRepositoryPort):
498
597
  )
499
598
 
500
599
  async def _get_table_columns(self, table_name: str, schema: str) -> List[Dict[str, Any]]:
501
- """Get column information for a table.
502
-
503
- Uses SVV_COLUMNS (Redshift system view) for better permission handling.
504
- See: https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_COLUMNS.html
505
- """
506
- query = """
600
+ """Get column information for a table."""
601
+ query = f"""
507
602
  SELECT
508
603
  column_name,
509
604
  data_type,
@@ -512,8 +607,8 @@ class RedshiftRepository(SchemaRepositoryPort):
512
607
  character_maximum_length,
513
608
  numeric_precision,
514
609
  numeric_scale
515
- FROM svv_columns
516
- WHERE table_name = %s AND table_schema = %s
610
+ FROM {self._column_view}
611
+ WHERE table_name = %s AND {self._schema_col} = %s {self._db_filter}
517
612
  ORDER BY ordinal_position
518
613
  """
519
614
 
@@ -47,6 +47,131 @@ class TestRedshiftAdapter:
47
47
  assert "postgres" in db_types # Make sure we didn't break existing types
48
48
 
49
49
 
50
+ class TestRedshiftSystemViewDetection:
51
+ """Unit tests for _detect_system_views fallback chain."""
52
+
53
+ def _make_repo(self) -> RedshiftRepository:
54
+ repo = RedshiftRepository("redshift://user:pass@host:5439/db", schema_name="test_schema")
55
+ repo._initialized = True # Skip actual DB connection
56
+ return repo
57
+
58
+ @pytest.mark.asyncio
59
+ async def test_detect_probe1_wins(self, mocker):
60
+ """When svv_tables returns rows, it should be selected."""
61
+ repo = self._make_repo()
62
+ mocker.patch.object(repo, "_execute_query_one", return_value={"cnt": 10})
63
+ await repo._detect_system_views()
64
+ assert repo._table_view == "svv_tables"
65
+ assert repo._column_view == "svv_columns"
66
+ assert repo._schema_col == "table_schema"
67
+
68
+ @pytest.mark.asyncio
69
+ async def test_detect_probe2_wins(self, mocker):
70
+ """When svv_tables returns 0, svv_all_tables should be selected."""
71
+ calls = [0]
72
+
73
+ def side_effect(query, params):
74
+ calls[0] += 1
75
+ if calls[0] == 1:
76
+ return {"cnt": 0} # svv_tables empty
77
+ return {"cnt": 95} # svv_all_tables has data
78
+
79
+ mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
80
+ await repo._detect_system_views()
81
+ assert repo._table_view == "svv_all_tables"
82
+ assert repo._column_view == "svv_all_columns"
83
+ assert repo._schema_col == "schema_name"
84
+ assert "database_name" in repo._db_filter
85
+
86
+ @pytest.mark.asyncio
87
+ async def test_detect_probe3_wins(self, mocker):
88
+ """When both SVV views return 0, information_schema should be selected."""
89
+ calls = [0]
90
+
91
+ def side_effect(query, params):
92
+ calls[0] += 1
93
+ if calls[0] <= 2:
94
+ return {"cnt": 0}
95
+ return {"cnt": 50}
96
+
97
+ mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
98
+ await repo._detect_system_views()
99
+ assert repo._table_view == "information_schema.tables"
100
+ assert repo._column_view == "information_schema.columns"
101
+ assert repo._schema_col == "table_schema"
102
+
103
+ @pytest.mark.asyncio
104
+ async def test_detect_all_probes_fail(self, mocker):
105
+ """When all probes return 0, defaults should be kept with a warning."""
106
+ mocker.patch.object(repo := self._make_repo(), "_execute_query_one", return_value={"cnt": 0})
107
+ await repo._detect_system_views()
108
+ # Defaults are kept (svv_tables)
109
+ assert repo._table_view == "svv_tables"
110
+
111
+ @pytest.mark.asyncio
112
+ async def test_detect_exception_falls_through(self, mocker):
113
+ """When a probe throws, it should fall through to the next."""
114
+ calls = [0]
115
+
116
+ def side_effect(query, params):
117
+ calls[0] += 1
118
+ if calls[0] == 1:
119
+ raise Exception("permission denied")
120
+ return {"cnt": 42}
121
+
122
+ mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
123
+ await repo._detect_system_views()
124
+ assert repo._table_view == "svv_all_tables"
125
+
126
+ def test_schema_name_normalized_to_lowercase(self):
127
+ """Schema name should be lowercased on init."""
128
+ repo = RedshiftRepository("redshift://u:p@h:5439/db", schema_name="DWH_Antares")
129
+ assert repo.schema_name == "dwh_antares"
130
+
131
+ def test_process_bulk_rows_empty(self):
132
+ """Empty rows should produce empty table list."""
133
+ repo = self._make_repo()
134
+ assert repo._process_bulk_rows([]) == []
135
+
136
+ def test_process_bulk_rows_groups_by_table(self):
137
+ """Rows should be grouped into TableInfo by table_name."""
138
+ repo = self._make_repo()
139
+ rows = [
140
+ {
141
+ "table_name": "users",
142
+ "table_schema": "public",
143
+ "table_comment": "",
144
+ "column_name": "id",
145
+ "data_type": "integer",
146
+ "is_nullable": "NO",
147
+ "column_default": None,
148
+ "character_maximum_length": None,
149
+ "numeric_precision": 32,
150
+ "numeric_scale": 0,
151
+ "ordinal_position": 1,
152
+ "pk_columns": None,
153
+ },
154
+ {
155
+ "table_name": "users",
156
+ "table_schema": "public",
157
+ "table_comment": "",
158
+ "column_name": "name",
159
+ "data_type": "varchar",
160
+ "is_nullable": "YES",
161
+ "column_default": None,
162
+ "character_maximum_length": 255,
163
+ "numeric_precision": None,
164
+ "numeric_scale": None,
165
+ "ordinal_position": 2,
166
+ "pk_columns": None,
167
+ },
168
+ ]
169
+ tables = repo._process_bulk_rows(rows)
170
+ assert len(tables) == 1
171
+ assert tables[0].name == "users"
172
+ assert len(tables[0].columns) == 2
173
+
174
+
50
175
  @pytest.mark.integration
51
176
  @pytest.mark.skipif(os.getenv("SKIP_LOCALSTACK_TESTS") == "true", reason="LocalStack tests disabled")
52
177
  class TestRedshiftLocalStackIntegration:
@@ -2064,7 +2064,7 @@ wheels = [
2064
2064
 
2065
2065
  [[package]]
2066
2066
  name = "nlp2sql"
2067
- version = "0.2.0rc10"
2067
+ version = "0.2.0rc12"
2068
2068
  source = { editable = "." }
2069
2069
  dependencies = [
2070
2070
  { name = "anthropic" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes