nlp2sql 0.2.0rc10__tar.gz → 0.2.0rc12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/PKG-INFO +3 -2
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/README.md +2 -1
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/pyproject.toml +1 -1
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/__init__.py +1 -1
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/openai_embedding_adapter.py +10 -7
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/redshift_adapter.py +215 -120
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_redshift_integration.py +125 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/uv.lock +1 -1
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/db.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/quality.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/commands/test.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/adapters.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/architecture.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/python-style.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/rules/testing.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.claude/settings.json +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/ci.yml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/publish-pypi.yml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.github/workflows/publish-testpypi.yml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.gitignore +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.mcp.json.example +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/.python-version +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/CLAUDE.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/CONTRIBUTING.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/LICENSE +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/SECURITY.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/assets/nlp2sql-logo.png +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/benchmarks/enterprise_benchmark_results.json +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/.env.example +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/README.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/docker-compose.yml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/get-db-urls.sh +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/init-schema.sql +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/large-schema.sql +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/localstack-init.sh +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docker/test-redshift.sh +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/API.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/ARCHITECTURE.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/CONFIGURATION.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/ENTERPRISE.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/docs/Redshift.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/README.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/advanced/test_multiple_providers.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/advanced/test_simple_api.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/database_specific/test_odoo_integration.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/default_examples.json +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/documentation/real_world_example.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/basic_usage.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/simple_demo.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/getting_started/test_api_setup.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/schema_management/test_auto_schema.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/schema_management/test_schema_filters_comprehensive.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/.dockerignore +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/.env.example +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_README.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_SETUP.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_TESTING_SUMMARY.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/Dockerfile +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/INDEX.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/README.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/docker/init-ecommerce-schema.sql +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/docker-compose.yml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/main.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/pyproject.toml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/test-docker-endpoints.sh +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/install-dev.sh +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/README.md +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/example_config.json +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/pyproject.toml +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/mcp_server/server.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/anthropic_adapter.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/gemini_adapter.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/local_embedding_adapter.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/openai_adapter.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/postgres_repository.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/adapters/regex_query_validator.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/cli.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/client.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/config/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/config/settings.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/database_prompts.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/entities.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/provider_config.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/result.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/sql_keywords.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/core/sql_safety.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/exceptions/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/factories/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/factories/repository_factory.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/ai_provider.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/cache.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/embedding_provider.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/example_repository.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_optimizer.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_safety.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/query_validator.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/schema_repository.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/ports/schema_strategy.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/analyzer.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/embedding_manager.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/example_loader.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/example_store.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/schema/manager.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/services/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/services/query_service.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/helpers.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/src/nlp2sql/utils/storage.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/test_tmp/.gitignore +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/conftest.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/mcp/__init__.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/mcp/test_sql_security.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_basic.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_batch_scoring.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_column_validation.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_database_prompts.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_dsl_integration.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_embedding_manager.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_integration.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_postgres_integration.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_repository_cache.py +0 -0
- {nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/tests/test_repository_factory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp2sql
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.0rc12
|
|
4
4
|
Summary: Enterprise-ready Natural Language to SQL converter with multi-provider support. Built for production scale (1000+ tables) with Clean Architecture.
|
|
5
5
|
Project-URL: Homepage, https://github.com/luiscarbonel1991/nlp2sql
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/luiscarbonel1991/nlp2sql/issues
|
|
@@ -81,12 +81,13 @@ Description-Content-Type: text/markdown
|
|
|
81
81
|
</p>
|
|
82
82
|
|
|
83
83
|
<p align="center">
|
|
84
|
+
<a href="https://pepy.tech/projects/nlp2sql"><img src="https://static.pepy.tech/personalized-badge/nlp2sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
|
|
84
85
|
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
|
|
85
86
|
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
|
|
86
87
|
<a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code style: black"></a>
|
|
87
88
|
</p>
|
|
88
89
|
|
|
89
|
-
# nlp2sql
|
|
90
|
+
# nlp2sql
|
|
90
91
|
|
|
91
92
|
**Enterprise-ready Natural Language to SQL converter with multi-provider support**
|
|
92
93
|
|
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<p align="center">
|
|
6
|
+
<a href="https://pepy.tech/projects/nlp2sql"><img src="https://static.pepy.tech/personalized-badge/nlp2sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
|
|
6
7
|
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
|
|
7
8
|
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
|
|
8
9
|
<a href="https://github.com/psf/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code style: black"></a>
|
|
9
10
|
</p>
|
|
10
11
|
|
|
11
|
-
# nlp2sql
|
|
12
|
+
# nlp2sql
|
|
12
13
|
|
|
13
14
|
**Enterprise-ready Natural Language to SQL converter with multi-provider support**
|
|
14
15
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nlp2sql"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.0rc12"
|
|
4
4
|
description = "Enterprise-ready Natural Language to SQL converter with multi-provider support. Built for production scale (1000+ tables) with Clean Architecture."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = {text = "MIT"}
|
|
@@ -20,7 +20,7 @@ from .ports.schema_repository import SchemaRepositoryPort
|
|
|
20
20
|
from .schema.example_store import ExampleStore
|
|
21
21
|
from .services.query_service import QueryGenerationService
|
|
22
22
|
|
|
23
|
-
__version__ = "0.2.
|
|
23
|
+
__version__ = "0.2.0rc12"
|
|
24
24
|
__author__ = "Luis Carbonel"
|
|
25
25
|
__email__ = "devhighlevel@gmail.com"
|
|
26
26
|
|
|
@@ -79,17 +79,19 @@ class OpenAIEmbeddingAdapter(EmbeddingProviderPort):
|
|
|
79
79
|
processed_texts.append(text)
|
|
80
80
|
|
|
81
81
|
try:
|
|
82
|
-
|
|
82
|
+
# OpenAI API limits input array to 2048 items per request
|
|
83
|
+
batch_size = 2000
|
|
84
|
+
all_embeddings: list[list[float]] = []
|
|
83
85
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
for i in range(0, len(processed_texts), batch_size):
|
|
87
|
+
batch = processed_texts[i : i + batch_size]
|
|
88
|
+
response = await self.client.embeddings.create(model=self.model, input=batch)
|
|
89
|
+
all_embeddings.extend(item.embedding for item in response.data)
|
|
90
|
+
|
|
91
|
+
embeddings_array = np.array(all_embeddings)
|
|
87
92
|
|
|
88
93
|
# Normalize embeddings for cosine similarity with FAISS IndexFlatIP
|
|
89
|
-
# This is critical: FAISS IndexFlatIP uses inner product which only works
|
|
90
|
-
# as cosine similarity when vectors are normalized to unit length
|
|
91
94
|
norms = np.linalg.norm(embeddings_array, axis=1, keepdims=True)
|
|
92
|
-
# Avoid division by zero (though rare for real embeddings)
|
|
93
95
|
norms = np.where(norms == 0, 1, norms)
|
|
94
96
|
normalized_embeddings = embeddings_array / norms
|
|
95
97
|
|
|
@@ -97,6 +99,7 @@ class OpenAIEmbeddingAdapter(EmbeddingProviderPort):
|
|
|
97
99
|
"OpenAI embeddings generated and normalized",
|
|
98
100
|
model=self.model,
|
|
99
101
|
texts_count=len(processed_texts),
|
|
102
|
+
batches=((len(processed_texts) - 1) // batch_size) + 1,
|
|
100
103
|
dimension=normalized_embeddings.shape[1],
|
|
101
104
|
)
|
|
102
105
|
|
|
@@ -39,10 +39,16 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
39
39
|
def __init__(self, connection_string: str, schema_name: str = "public"):
|
|
40
40
|
self.connection_string = connection_string
|
|
41
41
|
self.database_url = connection_string
|
|
42
|
-
self.schema_name = schema_name
|
|
42
|
+
self.schema_name = schema_name.lower() if schema_name else schema_name
|
|
43
43
|
self._connection_params = self._parse_connection_string(connection_string)
|
|
44
44
|
self._initialized = False
|
|
45
45
|
self._cache_dir: Optional[Path] = None
|
|
46
|
+
# System view configuration (auto-detected at init)
|
|
47
|
+
self._table_view: str = "svv_tables"
|
|
48
|
+
self._column_view: str = "svv_columns"
|
|
49
|
+
self._schema_col: str = "table_schema"
|
|
50
|
+
self._db_filter: str = ""
|
|
51
|
+
self._db_filter_aliased: str = ""
|
|
46
52
|
|
|
47
53
|
def _get_cache_dir(self) -> Path:
|
|
48
54
|
"""Get the cache directory for this database connection."""
|
|
@@ -106,7 +112,11 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
106
112
|
return None
|
|
107
113
|
|
|
108
114
|
def _save_tables_to_cache(self, tables: List[TableInfo], schema: str) -> None:
|
|
109
|
-
"""Save tables to disk cache."""
|
|
115
|
+
"""Save tables to disk cache. Skips caching if tables list is empty."""
|
|
116
|
+
if not tables:
|
|
117
|
+
logger.debug("Skipping cache save for empty tables list", schema=schema)
|
|
118
|
+
return
|
|
119
|
+
|
|
110
120
|
cache_path = self._get_tables_cache_path(schema)
|
|
111
121
|
try:
|
|
112
122
|
cache_data = {
|
|
@@ -169,12 +179,12 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
169
179
|
return psycopg2.connect(**self._connection_params)
|
|
170
180
|
|
|
171
181
|
async def initialize(self) -> None:
|
|
172
|
-
"""Initialize database connections."""
|
|
182
|
+
"""Initialize database connections and detect available system views."""
|
|
173
183
|
if self._initialized:
|
|
174
184
|
return
|
|
175
185
|
|
|
176
186
|
try:
|
|
177
|
-
|
|
187
|
+
|
|
178
188
|
def test_connection():
|
|
179
189
|
conn = self._get_connection()
|
|
180
190
|
cursor = conn.cursor()
|
|
@@ -183,14 +193,79 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
183
193
|
conn.close()
|
|
184
194
|
|
|
185
195
|
await asyncio.to_thread(test_connection)
|
|
196
|
+
await self._detect_system_views()
|
|
186
197
|
|
|
187
198
|
self._initialized = True
|
|
188
|
-
logger.info("Redshift repository initialized")
|
|
199
|
+
logger.info("Redshift repository initialized", table_view=self._table_view)
|
|
189
200
|
|
|
201
|
+
except SchemaException:
|
|
202
|
+
raise
|
|
190
203
|
except Exception as e:
|
|
191
204
|
logger.error("Failed to initialize Redshift repository", error=str(e))
|
|
192
205
|
raise SchemaException(f"Database initialization failed: {e!s}")
|
|
193
206
|
|
|
207
|
+
async def _detect_system_views(self) -> None:
|
|
208
|
+
"""Auto-detect which system views are available for this connection.
|
|
209
|
+
|
|
210
|
+
Fallback chain: svv_tables → svv_all_tables → information_schema
|
|
211
|
+
"""
|
|
212
|
+
probes = [
|
|
213
|
+
{
|
|
214
|
+
"table_view": "svv_tables",
|
|
215
|
+
"column_view": "svv_columns",
|
|
216
|
+
"schema_col": "table_schema",
|
|
217
|
+
"db_filter": "",
|
|
218
|
+
"db_filter_aliased": "",
|
|
219
|
+
"query": "SELECT COUNT(*) as cnt FROM svv_tables WHERE table_schema = %s",
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"table_view": "svv_all_tables",
|
|
223
|
+
"column_view": "svv_all_columns",
|
|
224
|
+
"schema_col": "schema_name",
|
|
225
|
+
"db_filter": "AND database_name = current_database()",
|
|
226
|
+
"db_filter_aliased": "AND t.database_name = current_database()",
|
|
227
|
+
"query": (
|
|
228
|
+
"SELECT COUNT(*) as cnt FROM svv_all_tables "
|
|
229
|
+
"WHERE schema_name = %s AND database_name = current_database()"
|
|
230
|
+
),
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
"table_view": "information_schema.tables",
|
|
234
|
+
"column_view": "information_schema.columns",
|
|
235
|
+
"schema_col": "table_schema",
|
|
236
|
+
"db_filter": "AND table_type IN ('BASE TABLE', 'VIEW')",
|
|
237
|
+
"db_filter_aliased": "AND t.table_type IN ('BASE TABLE', 'VIEW')",
|
|
238
|
+
"query": (
|
|
239
|
+
"SELECT COUNT(*) as cnt FROM information_schema.tables "
|
|
240
|
+
"WHERE table_schema = %s AND table_type IN ('BASE TABLE', 'VIEW')"
|
|
241
|
+
),
|
|
242
|
+
},
|
|
243
|
+
]
|
|
244
|
+
|
|
245
|
+
for probe in probes:
|
|
246
|
+
try:
|
|
247
|
+
row = await asyncio.to_thread(self._execute_query_one, probe["query"], (self.schema_name,))
|
|
248
|
+
if row and row["cnt"] > 0:
|
|
249
|
+
self._table_view = probe["table_view"]
|
|
250
|
+
self._column_view = probe["column_view"]
|
|
251
|
+
self._schema_col = probe["schema_col"]
|
|
252
|
+
self._db_filter = probe["db_filter"]
|
|
253
|
+
self._db_filter_aliased = probe["db_filter_aliased"]
|
|
254
|
+
logger.info(
|
|
255
|
+
"Schema discovery source selected",
|
|
256
|
+
view=probe["table_view"],
|
|
257
|
+
schema=self.schema_name,
|
|
258
|
+
tables=row["cnt"],
|
|
259
|
+
)
|
|
260
|
+
return
|
|
261
|
+
except Exception as e:
|
|
262
|
+
logger.debug("Probe failed", view=probe["table_view"], error=str(e))
|
|
263
|
+
|
|
264
|
+
logger.warning(
|
|
265
|
+
"All schema discovery probes returned 0 tables",
|
|
266
|
+
schema=self.schema_name,
|
|
267
|
+
)
|
|
268
|
+
|
|
194
269
|
def _execute_query(self, query: str, params: tuple = None) -> List[Dict]:
|
|
195
270
|
"""Execute a query and return results as list of dicts."""
|
|
196
271
|
conn = self._get_connection()
|
|
@@ -209,18 +284,10 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
209
284
|
return results[0] if results else None
|
|
210
285
|
|
|
211
286
|
async def get_tables(self, schema_name: Optional[str] = None, force_refresh: bool = False) -> List[TableInfo]:
|
|
212
|
-
"""Get all tables in the schema using
|
|
213
|
-
|
|
214
|
-
Uses SVV_TABLES and SVV_COLUMNS (Redshift system views) for better
|
|
215
|
-
permission handling. See AWS docs:
|
|
216
|
-
https://docs.aws.amazon.com/redshift/latest/dg/cm_chap_system-tables.html
|
|
217
|
-
|
|
218
|
-
Args:
|
|
219
|
-
schema_name: Schema name to query (defaults to self.schema_name)
|
|
220
|
-
force_refresh: If True, bypass cache and fetch fresh data
|
|
287
|
+
"""Get all tables in the schema using bulk query + disk cache.
|
|
221
288
|
|
|
222
|
-
|
|
223
|
-
|
|
289
|
+
The system view source (svv_tables, svv_all_tables, or information_schema)
|
|
290
|
+
is auto-detected at initialization time.
|
|
224
291
|
"""
|
|
225
292
|
if not self._initialized:
|
|
226
293
|
await self.initialize()
|
|
@@ -242,18 +309,78 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
242
309
|
|
|
243
310
|
return tables
|
|
244
311
|
|
|
245
|
-
|
|
246
|
-
"""
|
|
312
|
+
def _process_bulk_rows(self, rows: List[Dict]) -> List[TableInfo]:
|
|
313
|
+
"""Convert raw bulk query rows into TableInfo objects."""
|
|
314
|
+
tables_dict: Dict[str, Dict[str, Any]] = {}
|
|
315
|
+
for row in rows:
|
|
316
|
+
table_name = row["table_name"]
|
|
317
|
+
|
|
318
|
+
if table_name not in tables_dict:
|
|
319
|
+
pk_cols = row.get("pk_columns")
|
|
320
|
+
if pk_cols:
|
|
321
|
+
if isinstance(pk_cols, str):
|
|
322
|
+
pk_cols = pk_cols.strip("{}").split(",") if pk_cols != "{}" else []
|
|
323
|
+
elif isinstance(pk_cols, list):
|
|
324
|
+
pk_cols = list(pk_cols)
|
|
325
|
+
else:
|
|
326
|
+
pk_cols = []
|
|
327
|
+
else:
|
|
328
|
+
pk_cols = []
|
|
329
|
+
|
|
330
|
+
tables_dict[table_name] = {
|
|
331
|
+
"name": table_name,
|
|
332
|
+
"schema": row["table_schema"],
|
|
333
|
+
"description": row.get("table_comment", ""),
|
|
334
|
+
"columns": [],
|
|
335
|
+
"primary_keys": pk_cols,
|
|
336
|
+
"foreign_keys": [],
|
|
337
|
+
"indexes": [],
|
|
338
|
+
"_seen_columns": set(),
|
|
339
|
+
}
|
|
247
340
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
341
|
+
col_name = row.get("column_name")
|
|
342
|
+
if col_name and col_name not in tables_dict[table_name]["_seen_columns"]:
|
|
343
|
+
tables_dict[table_name]["_seen_columns"].add(col_name)
|
|
344
|
+
tables_dict[table_name]["columns"].append(
|
|
345
|
+
{
|
|
346
|
+
"name": col_name,
|
|
347
|
+
"type": row.get("data_type", ""),
|
|
348
|
+
"nullable": row.get("is_nullable") == "YES",
|
|
349
|
+
"default": row.get("column_default"),
|
|
350
|
+
"max_length": row.get("character_maximum_length"),
|
|
351
|
+
"precision": row.get("numeric_precision"),
|
|
352
|
+
"scale": row.get("numeric_scale"),
|
|
353
|
+
"description": "",
|
|
354
|
+
}
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
tables = []
|
|
358
|
+
for table_data in tables_dict.values():
|
|
359
|
+
del table_data["_seen_columns"]
|
|
360
|
+
tables.append(
|
|
361
|
+
TableInfo(
|
|
362
|
+
name=table_data["name"],
|
|
363
|
+
schema=table_data["schema"],
|
|
364
|
+
columns=table_data["columns"],
|
|
365
|
+
primary_keys=table_data["primary_keys"],
|
|
366
|
+
foreign_keys=table_data["foreign_keys"],
|
|
367
|
+
indexes=table_data["indexes"],
|
|
368
|
+
row_count=0,
|
|
369
|
+
size_bytes=0,
|
|
370
|
+
description=table_data["description"],
|
|
371
|
+
last_updated=datetime.now(),
|
|
372
|
+
)
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
logger.info("Tables processed from bulk query", count=len(tables))
|
|
376
|
+
return tables
|
|
377
|
+
|
|
378
|
+
async def _get_tables_bulk(self, schema: str) -> List[TableInfo]:
|
|
379
|
+
"""Fetch all tables with columns in a single bulk query."""
|
|
380
|
+
bulk_query = f"""
|
|
254
381
|
SELECT
|
|
255
382
|
t.table_name,
|
|
256
|
-
t.table_schema,
|
|
383
|
+
t.{self._schema_col} as table_schema,
|
|
257
384
|
'' as table_comment,
|
|
258
385
|
c.column_name,
|
|
259
386
|
c.data_type,
|
|
@@ -264,10 +391,10 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
264
391
|
c.numeric_scale,
|
|
265
392
|
c.ordinal_position,
|
|
266
393
|
NULL as pk_columns
|
|
267
|
-
FROM
|
|
268
|
-
LEFT JOIN
|
|
269
|
-
AND t.
|
|
270
|
-
WHERE t.
|
|
394
|
+
FROM {self._table_view} t
|
|
395
|
+
LEFT JOIN {self._column_view} c ON t.table_name = c.table_name
|
|
396
|
+
AND t.{self._schema_col} = c.{self._schema_col}
|
|
397
|
+
WHERE t.{self._schema_col} = %s {self._db_filter_aliased}
|
|
271
398
|
ORDER BY t.table_name, c.ordinal_position
|
|
272
399
|
"""
|
|
273
400
|
|
|
@@ -277,79 +404,54 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
277
404
|
|
|
278
405
|
elapsed_ms = round((time.time() - start_time) * 1000, 2)
|
|
279
406
|
logger.info("Bulk query completed", rows=len(rows), elapsed_ms=elapsed_ms)
|
|
407
|
+
return self._process_bulk_rows(rows)
|
|
280
408
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
# Parse pk_columns - Redshift may return string representation
|
|
288
|
-
pk_cols = row.get("pk_columns")
|
|
289
|
-
if pk_cols:
|
|
290
|
-
if isinstance(pk_cols, str):
|
|
291
|
-
# Handle string like '{col1,col2}'
|
|
292
|
-
pk_cols = pk_cols.strip("{}").split(",") if pk_cols != "{}" else []
|
|
293
|
-
elif isinstance(pk_cols, list):
|
|
294
|
-
pk_cols = list(pk_cols)
|
|
295
|
-
else:
|
|
296
|
-
pk_cols = []
|
|
297
|
-
else:
|
|
298
|
-
pk_cols = []
|
|
299
|
-
|
|
300
|
-
tables_dict[table_name] = {
|
|
301
|
-
"name": table_name,
|
|
302
|
-
"schema": row["table_schema"],
|
|
303
|
-
"description": row.get("table_comment", ""),
|
|
304
|
-
"columns": [],
|
|
305
|
-
"primary_keys": pk_cols,
|
|
306
|
-
"foreign_keys": [],
|
|
307
|
-
"indexes": [],
|
|
308
|
-
"_seen_columns": set(),
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
# Add column (avoiding duplicates)
|
|
312
|
-
col_name = row.get("column_name")
|
|
313
|
-
if col_name and col_name not in tables_dict[table_name]["_seen_columns"]:
|
|
314
|
-
tables_dict[table_name]["_seen_columns"].add(col_name)
|
|
315
|
-
tables_dict[table_name]["columns"].append(
|
|
316
|
-
{
|
|
317
|
-
"name": col_name,
|
|
318
|
-
"type": row.get("data_type", ""),
|
|
319
|
-
"nullable": row.get("is_nullable") == "YES",
|
|
320
|
-
"default": row.get("column_default"),
|
|
321
|
-
"max_length": row.get("character_maximum_length"),
|
|
322
|
-
"precision": row.get("numeric_precision"),
|
|
323
|
-
"scale": row.get("numeric_scale"),
|
|
324
|
-
"description": "",
|
|
325
|
-
}
|
|
326
|
-
)
|
|
327
|
-
|
|
328
|
-
# Convert to TableInfo objects
|
|
329
|
-
tables = []
|
|
330
|
-
for table_data in tables_dict.values():
|
|
331
|
-
del table_data["_seen_columns"]
|
|
332
|
-
|
|
333
|
-
tables.append(
|
|
334
|
-
TableInfo(
|
|
335
|
-
name=table_data["name"],
|
|
336
|
-
schema=table_data["schema"],
|
|
337
|
-
columns=table_data["columns"],
|
|
338
|
-
primary_keys=table_data["primary_keys"],
|
|
339
|
-
foreign_keys=table_data["foreign_keys"],
|
|
340
|
-
indexes=table_data["indexes"],
|
|
341
|
-
row_count=0,
|
|
342
|
-
size_bytes=0,
|
|
343
|
-
description=table_data["description"],
|
|
344
|
-
last_updated=datetime.now(),
|
|
345
|
-
)
|
|
409
|
+
except Exception as e:
|
|
410
|
+
if self._table_view != "information_schema.tables":
|
|
411
|
+
logger.warning(
|
|
412
|
+
"Bulk query failed, retrying with information_schema",
|
|
413
|
+
error=str(e),
|
|
414
|
+
original_view=self._table_view,
|
|
346
415
|
)
|
|
416
|
+
return await self._get_tables_bulk_information_schema(schema)
|
|
347
417
|
|
|
348
|
-
logger.
|
|
349
|
-
|
|
418
|
+
logger.error("Bulk query failed", error=str(e))
|
|
419
|
+
raise SchemaException(f"Failed to get tables: {e!s}")
|
|
350
420
|
|
|
421
|
+
async def _get_tables_bulk_information_schema(self, schema: str) -> List[TableInfo]:
|
|
422
|
+
"""Fallback bulk query using information_schema (no state mutation)."""
|
|
423
|
+
fallback_query = """
|
|
424
|
+
SELECT
|
|
425
|
+
t.table_name,
|
|
426
|
+
t.table_schema,
|
|
427
|
+
'' as table_comment,
|
|
428
|
+
c.column_name,
|
|
429
|
+
c.data_type,
|
|
430
|
+
c.is_nullable,
|
|
431
|
+
c.column_default,
|
|
432
|
+
c.character_maximum_length,
|
|
433
|
+
c.numeric_precision,
|
|
434
|
+
c.numeric_scale,
|
|
435
|
+
c.ordinal_position,
|
|
436
|
+
NULL as pk_columns
|
|
437
|
+
FROM information_schema.tables t
|
|
438
|
+
LEFT JOIN information_schema.columns c ON t.table_name = c.table_name
|
|
439
|
+
AND t.table_schema = c.table_schema
|
|
440
|
+
WHERE t.table_schema = %s AND t.table_type IN ('BASE TABLE', 'VIEW')
|
|
441
|
+
ORDER BY t.table_name, c.ordinal_position
|
|
442
|
+
"""
|
|
443
|
+
try:
|
|
444
|
+
start_time = time.time()
|
|
445
|
+
rows = await asyncio.to_thread(self._execute_query, fallback_query, (schema,))
|
|
446
|
+
elapsed_ms = round((time.time() - start_time) * 1000, 2)
|
|
447
|
+
logger.info(
|
|
448
|
+
"information_schema fallback completed",
|
|
449
|
+
rows=len(rows),
|
|
450
|
+
elapsed_ms=elapsed_ms,
|
|
451
|
+
)
|
|
452
|
+
return self._process_bulk_rows(rows)
|
|
351
453
|
except Exception as e:
|
|
352
|
-
logger.error("
|
|
454
|
+
logger.error("information_schema fallback also failed", error=str(e))
|
|
353
455
|
raise SchemaException(f"Failed to get tables: {e!s}")
|
|
354
456
|
|
|
355
457
|
async def get_table_info(self, table_name: str, schema_name: Optional[str] = None) -> TableInfo:
|
|
@@ -359,14 +461,13 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
359
461
|
|
|
360
462
|
schema = schema_name or self.schema_name
|
|
361
463
|
|
|
362
|
-
|
|
363
|
-
query = """
|
|
464
|
+
query = f"""
|
|
364
465
|
SELECT
|
|
365
466
|
table_name,
|
|
366
|
-
table_schema,
|
|
467
|
+
{self._schema_col} as table_schema,
|
|
367
468
|
'' as table_comment
|
|
368
|
-
FROM
|
|
369
|
-
WHERE table_name = %s AND
|
|
469
|
+
FROM {self._table_view}
|
|
470
|
+
WHERE table_name = %s AND {self._schema_col} = %s {self._db_filter}
|
|
370
471
|
"""
|
|
371
472
|
|
|
372
473
|
try:
|
|
@@ -388,15 +489,14 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
388
489
|
if not self._initialized:
|
|
389
490
|
await self.initialize()
|
|
390
491
|
|
|
391
|
-
|
|
392
|
-
query = """
|
|
492
|
+
query = f"""
|
|
393
493
|
SELECT
|
|
394
494
|
table_name,
|
|
395
|
-
table_schema,
|
|
495
|
+
{self._schema_col} as table_schema,
|
|
396
496
|
'' as table_comment
|
|
397
|
-
FROM
|
|
398
|
-
WHERE
|
|
399
|
-
AND table_name ILIKE %s
|
|
497
|
+
FROM {self._table_view}
|
|
498
|
+
WHERE {self._schema_col} = %s
|
|
499
|
+
AND table_name ILIKE %s {self._db_filter}
|
|
400
500
|
ORDER BY table_name
|
|
401
501
|
"""
|
|
402
502
|
|
|
@@ -427,14 +527,13 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
427
527
|
if not self._initialized:
|
|
428
528
|
await self.initialize()
|
|
429
529
|
|
|
430
|
-
|
|
431
|
-
query = """
|
|
530
|
+
query = f"""
|
|
432
531
|
SELECT
|
|
433
532
|
current_database() as database_name,
|
|
434
533
|
version() as database_version,
|
|
435
534
|
COUNT(*) as total_tables
|
|
436
|
-
FROM
|
|
437
|
-
WHERE
|
|
535
|
+
FROM {self._table_view}
|
|
536
|
+
WHERE {self._schema_col} = %s {self._db_filter}
|
|
438
537
|
"""
|
|
439
538
|
|
|
440
539
|
try:
|
|
@@ -498,12 +597,8 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
498
597
|
)
|
|
499
598
|
|
|
500
599
|
async def _get_table_columns(self, table_name: str, schema: str) -> List[Dict[str, Any]]:
|
|
501
|
-
"""Get column information for a table.
|
|
502
|
-
|
|
503
|
-
Uses SVV_COLUMNS (Redshift system view) for better permission handling.
|
|
504
|
-
See: https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_COLUMNS.html
|
|
505
|
-
"""
|
|
506
|
-
query = """
|
|
600
|
+
"""Get column information for a table."""
|
|
601
|
+
query = f"""
|
|
507
602
|
SELECT
|
|
508
603
|
column_name,
|
|
509
604
|
data_type,
|
|
@@ -512,8 +607,8 @@ class RedshiftRepository(SchemaRepositoryPort):
|
|
|
512
607
|
character_maximum_length,
|
|
513
608
|
numeric_precision,
|
|
514
609
|
numeric_scale
|
|
515
|
-
FROM
|
|
516
|
-
WHERE table_name = %s AND
|
|
610
|
+
FROM {self._column_view}
|
|
611
|
+
WHERE table_name = %s AND {self._schema_col} = %s {self._db_filter}
|
|
517
612
|
ORDER BY ordinal_position
|
|
518
613
|
"""
|
|
519
614
|
|
|
@@ -47,6 +47,131 @@ class TestRedshiftAdapter:
|
|
|
47
47
|
assert "postgres" in db_types # Make sure we didn't break existing types
|
|
48
48
|
|
|
49
49
|
|
|
50
|
+
class TestRedshiftSystemViewDetection:
|
|
51
|
+
"""Unit tests for _detect_system_views fallback chain."""
|
|
52
|
+
|
|
53
|
+
def _make_repo(self) -> RedshiftRepository:
|
|
54
|
+
repo = RedshiftRepository("redshift://user:pass@host:5439/db", schema_name="test_schema")
|
|
55
|
+
repo._initialized = True # Skip actual DB connection
|
|
56
|
+
return repo
|
|
57
|
+
|
|
58
|
+
@pytest.mark.asyncio
|
|
59
|
+
async def test_detect_probe1_wins(self, mocker):
|
|
60
|
+
"""When svv_tables returns rows, it should be selected."""
|
|
61
|
+
repo = self._make_repo()
|
|
62
|
+
mocker.patch.object(repo, "_execute_query_one", return_value={"cnt": 10})
|
|
63
|
+
await repo._detect_system_views()
|
|
64
|
+
assert repo._table_view == "svv_tables"
|
|
65
|
+
assert repo._column_view == "svv_columns"
|
|
66
|
+
assert repo._schema_col == "table_schema"
|
|
67
|
+
|
|
68
|
+
@pytest.mark.asyncio
|
|
69
|
+
async def test_detect_probe2_wins(self, mocker):
|
|
70
|
+
"""When svv_tables returns 0, svv_all_tables should be selected."""
|
|
71
|
+
calls = [0]
|
|
72
|
+
|
|
73
|
+
def side_effect(query, params):
|
|
74
|
+
calls[0] += 1
|
|
75
|
+
if calls[0] == 1:
|
|
76
|
+
return {"cnt": 0} # svv_tables empty
|
|
77
|
+
return {"cnt": 95} # svv_all_tables has data
|
|
78
|
+
|
|
79
|
+
mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
|
|
80
|
+
await repo._detect_system_views()
|
|
81
|
+
assert repo._table_view == "svv_all_tables"
|
|
82
|
+
assert repo._column_view == "svv_all_columns"
|
|
83
|
+
assert repo._schema_col == "schema_name"
|
|
84
|
+
assert "database_name" in repo._db_filter
|
|
85
|
+
|
|
86
|
+
@pytest.mark.asyncio
|
|
87
|
+
async def test_detect_probe3_wins(self, mocker):
|
|
88
|
+
"""When both SVV views return 0, information_schema should be selected."""
|
|
89
|
+
calls = [0]
|
|
90
|
+
|
|
91
|
+
def side_effect(query, params):
|
|
92
|
+
calls[0] += 1
|
|
93
|
+
if calls[0] <= 2:
|
|
94
|
+
return {"cnt": 0}
|
|
95
|
+
return {"cnt": 50}
|
|
96
|
+
|
|
97
|
+
mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
|
|
98
|
+
await repo._detect_system_views()
|
|
99
|
+
assert repo._table_view == "information_schema.tables"
|
|
100
|
+
assert repo._column_view == "information_schema.columns"
|
|
101
|
+
assert repo._schema_col == "table_schema"
|
|
102
|
+
|
|
103
|
+
@pytest.mark.asyncio
|
|
104
|
+
async def test_detect_all_probes_fail(self, mocker):
|
|
105
|
+
"""When all probes return 0, defaults should be kept with a warning."""
|
|
106
|
+
mocker.patch.object(repo := self._make_repo(), "_execute_query_one", return_value={"cnt": 0})
|
|
107
|
+
await repo._detect_system_views()
|
|
108
|
+
# Defaults are kept (svv_tables)
|
|
109
|
+
assert repo._table_view == "svv_tables"
|
|
110
|
+
|
|
111
|
+
@pytest.mark.asyncio
|
|
112
|
+
async def test_detect_exception_falls_through(self, mocker):
|
|
113
|
+
"""When a probe throws, it should fall through to the next."""
|
|
114
|
+
calls = [0]
|
|
115
|
+
|
|
116
|
+
def side_effect(query, params):
|
|
117
|
+
calls[0] += 1
|
|
118
|
+
if calls[0] == 1:
|
|
119
|
+
raise Exception("permission denied")
|
|
120
|
+
return {"cnt": 42}
|
|
121
|
+
|
|
122
|
+
mocker.patch.object(repo := self._make_repo(), "_execute_query_one", side_effect=side_effect)
|
|
123
|
+
await repo._detect_system_views()
|
|
124
|
+
assert repo._table_view == "svv_all_tables"
|
|
125
|
+
|
|
126
|
+
def test_schema_name_normalized_to_lowercase(self):
|
|
127
|
+
"""Schema name should be lowercased on init."""
|
|
128
|
+
repo = RedshiftRepository("redshift://u:p@h:5439/db", schema_name="DWH_Antares")
|
|
129
|
+
assert repo.schema_name == "dwh_antares"
|
|
130
|
+
|
|
131
|
+
def test_process_bulk_rows_empty(self):
|
|
132
|
+
"""Empty rows should produce empty table list."""
|
|
133
|
+
repo = self._make_repo()
|
|
134
|
+
assert repo._process_bulk_rows([]) == []
|
|
135
|
+
|
|
136
|
+
def test_process_bulk_rows_groups_by_table(self):
|
|
137
|
+
"""Rows should be grouped into TableInfo by table_name."""
|
|
138
|
+
repo = self._make_repo()
|
|
139
|
+
rows = [
|
|
140
|
+
{
|
|
141
|
+
"table_name": "users",
|
|
142
|
+
"table_schema": "public",
|
|
143
|
+
"table_comment": "",
|
|
144
|
+
"column_name": "id",
|
|
145
|
+
"data_type": "integer",
|
|
146
|
+
"is_nullable": "NO",
|
|
147
|
+
"column_default": None,
|
|
148
|
+
"character_maximum_length": None,
|
|
149
|
+
"numeric_precision": 32,
|
|
150
|
+
"numeric_scale": 0,
|
|
151
|
+
"ordinal_position": 1,
|
|
152
|
+
"pk_columns": None,
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"table_name": "users",
|
|
156
|
+
"table_schema": "public",
|
|
157
|
+
"table_comment": "",
|
|
158
|
+
"column_name": "name",
|
|
159
|
+
"data_type": "varchar",
|
|
160
|
+
"is_nullable": "YES",
|
|
161
|
+
"column_default": None,
|
|
162
|
+
"character_maximum_length": 255,
|
|
163
|
+
"numeric_precision": None,
|
|
164
|
+
"numeric_scale": None,
|
|
165
|
+
"ordinal_position": 2,
|
|
166
|
+
"pk_columns": None,
|
|
167
|
+
},
|
|
168
|
+
]
|
|
169
|
+
tables = repo._process_bulk_rows(rows)
|
|
170
|
+
assert len(tables) == 1
|
|
171
|
+
assert tables[0].name == "users"
|
|
172
|
+
assert len(tables[0].columns) == 2
|
|
173
|
+
|
|
174
|
+
|
|
50
175
|
@pytest.mark.integration
|
|
51
176
|
@pytest.mark.skipif(os.getenv("SKIP_LOCALSTACK_TESTS") == "true", reason="LocalStack tests disabled")
|
|
52
177
|
class TestRedshiftLocalStackIntegration:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/DOCKER_TESTING_SUMMARY.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nlp2sql-0.2.0rc10 → nlp2sql-0.2.0rc12}/examples/strands-fastapi-agent/test-docker-endpoints.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|