quillsql 2.2.5__tar.gz → 2.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quillsql-2.2.5/quillsql.egg-info → quillsql-2.2.7}/PKG-INFO +2 -2
- quillsql-2.2.7/quillsql/__init__.py +14 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/core.py +27 -10
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/db/bigquery.py +1 -1
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/db/cached_connection.py +32 -9
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/db/db_helper.py +15 -11
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/__init__.py +1 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/pivot_template.py +0 -1
- quillsql-2.2.7/quillsql/utils/post_quill_executor.py +69 -0
- {quillsql-2.2.5 → quillsql-2.2.7/quillsql.egg-info}/PKG-INFO +1 -1
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql.egg-info/SOURCES.txt +1 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/setup.py +1 -1
- quillsql-2.2.5/quillsql/__init__.py +0 -5
- {quillsql-2.2.5 → quillsql-2.2.7}/README.md +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/assets/__init__.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/assets/pgtypes.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/db/__init__.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/db/postgres.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/error.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/filters.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/run_query_processes.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/schema_conversion.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql/utils/tenants.py +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql.egg-info/dependency_links.txt +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql.egg-info/requires.txt +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/quillsql.egg-info/top_level.txt +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/setup.cfg +0 -0
- {quillsql-2.2.5 → quillsql-2.2.7}/tests/test_core.py +0 -0
|
@@ -144,10 +144,14 @@ class Quill:
|
|
|
144
144
|
None
|
|
145
145
|
)
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
distinct_value_results
|
|
149
|
-
|
|
150
|
-
|
|
147
|
+
distinct_value_query_results = (
|
|
148
|
+
distinct_value_results or {}
|
|
149
|
+
).get("queryResults") or []
|
|
150
|
+
if distinct_value_query_results:
|
|
151
|
+
distinct_values = parse_distinct_values(
|
|
152
|
+
distinct_value_query_results[0],
|
|
153
|
+
config.get("databaseType")
|
|
154
|
+
)
|
|
151
155
|
|
|
152
156
|
try:
|
|
153
157
|
final_query = hydrate_pivot_template(template, distinct_values, config)
|
|
@@ -261,17 +265,16 @@ class Quill:
|
|
|
261
265
|
else:
|
|
262
266
|
tenant_flags = flags
|
|
263
267
|
|
|
264
|
-
|
|
265
|
-
self.run_queries(
|
|
268
|
+
if metadata.get("preQueries"):
|
|
269
|
+
pre_query_results = self.run_queries(
|
|
266
270
|
metadata.get("preQueries"),
|
|
267
271
|
self.target_connection.database_type,
|
|
268
272
|
metadata.get("databaseType"),
|
|
269
273
|
metadata,
|
|
270
274
|
metadata.get("runQueryConfig"),
|
|
271
275
|
)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
)
|
|
276
|
+
else:
|
|
277
|
+
pre_query_results = {}
|
|
275
278
|
|
|
276
279
|
if metadata.get("runQueryConfig") and metadata.get("runQueryConfig").get(
|
|
277
280
|
"overridePost"
|
|
@@ -402,7 +405,21 @@ class Quill:
|
|
|
402
405
|
results = {}
|
|
403
406
|
if not queries:
|
|
404
407
|
return {"queryResults": []}
|
|
405
|
-
|
|
408
|
+
def _normalize_db_type(db_type):
|
|
409
|
+
if not db_type:
|
|
410
|
+
return None
|
|
411
|
+
lowered = db_type.lower()
|
|
412
|
+
if lowered in ("postgresql", "postgres"):
|
|
413
|
+
return "postgres"
|
|
414
|
+
return lowered
|
|
415
|
+
|
|
416
|
+
normalized_metadata_db = _normalize_db_type(databaseType)
|
|
417
|
+
normalized_backend_db = _normalize_db_type(pkDatabaseType)
|
|
418
|
+
if (
|
|
419
|
+
normalized_metadata_db
|
|
420
|
+
and normalized_backend_db
|
|
421
|
+
and normalized_metadata_db != normalized_backend_db
|
|
422
|
+
):
|
|
406
423
|
return {"dbMismatched": True, "backendDatabaseType": pkDatabaseType}
|
|
407
424
|
if runQueryConfig and runQueryConfig.get("arrayToMap"):
|
|
408
425
|
mapped_array = array_to_map(
|
|
@@ -29,20 +29,43 @@ class CachedConnection:
|
|
|
29
29
|
)
|
|
30
30
|
return None
|
|
31
31
|
|
|
32
|
+
def _is_connection_closed(self):
|
|
33
|
+
return self.connection is None or getattr(self.connection, "closed", True)
|
|
34
|
+
|
|
35
|
+
def _open_connection(self):
|
|
36
|
+
self.connection = connect_to_db(
|
|
37
|
+
self.database_type, self.config, self.using_connection_string
|
|
38
|
+
)
|
|
39
|
+
return self.connection
|
|
40
|
+
|
|
41
|
+
def _close_connection(self):
|
|
42
|
+
if self.connection is None:
|
|
43
|
+
return
|
|
44
|
+
try:
|
|
45
|
+
self.connection.close()
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
finally:
|
|
49
|
+
self.connection = None
|
|
50
|
+
|
|
32
51
|
def exec_with_reconnect(self, sql):
|
|
33
52
|
reconnect_count = 0
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
self.database_type, self.config, self.using_connection_string
|
|
37
|
-
)
|
|
53
|
+
last_error = None
|
|
54
|
+
while reconnect_count < 10:
|
|
38
55
|
try:
|
|
56
|
+
if self._is_connection_closed():
|
|
57
|
+
self._open_connection()
|
|
39
58
|
return run_query_by_db(self.database_type, sql, self.connection)
|
|
40
59
|
except psycopg2.Error as err:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
60
|
+
last_error = err
|
|
61
|
+
reconnect_count += 1
|
|
62
|
+
self._close_connection()
|
|
63
|
+
except Exception:
|
|
64
|
+
self._close_connection()
|
|
65
|
+
raise
|
|
66
|
+
diag = getattr(last_error, "diag", None)
|
|
67
|
+
position = getattr(diag, "statement_position", None)
|
|
68
|
+
raise PgQueryError(last_error or "Failed to execute query", sql, position)
|
|
46
69
|
|
|
47
70
|
def exec(self, sql):
|
|
48
71
|
try:
|
|
@@ -15,47 +15,51 @@ from quillsql.db.bigquery import (
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
def _is_postgres(database_type):
|
|
19
|
+
return database_type and database_type.lower() in ("postgresql", "postgres")
|
|
20
|
+
|
|
21
|
+
|
|
18
22
|
def get_db_credentials(database_type, connection_string):
|
|
19
|
-
if database_type
|
|
23
|
+
if _is_postgres(database_type):
|
|
20
24
|
return format_postgres(connection_string)
|
|
21
|
-
elif database_type.lower() == "bigquery":
|
|
25
|
+
elif database_type and database_type.lower() == "bigquery":
|
|
22
26
|
return format_bigquery_config(connection_string)
|
|
23
27
|
return {}
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
def connect_to_db(database_type, config, using_connection_string):
|
|
27
|
-
if database_type
|
|
31
|
+
if _is_postgres(database_type):
|
|
28
32
|
return connect_to_postgres(config, using_connection_string)
|
|
29
|
-
elif database_type.lower() == "bigquery":
|
|
33
|
+
elif database_type and database_type.lower() == "bigquery":
|
|
30
34
|
return connect_to_bigquery(config, using_connection_string)
|
|
31
35
|
return None
|
|
32
36
|
|
|
33
37
|
|
|
34
38
|
def run_query_by_db(database_type, query, connection):
|
|
35
|
-
if database_type
|
|
39
|
+
if _is_postgres(database_type):
|
|
36
40
|
return run_query_postgres(query, connection)
|
|
37
|
-
elif database_type.lower() == "bigquery":
|
|
41
|
+
elif database_type and database_type.lower() == "bigquery":
|
|
38
42
|
return run_query_big_query(query, connection)
|
|
39
43
|
return None
|
|
40
44
|
|
|
41
45
|
|
|
42
46
|
def disconnect_from_db(database_type, connection):
|
|
43
|
-
if database_type
|
|
47
|
+
if _is_postgres(database_type):
|
|
44
48
|
return disconnect_from_postgres(connection)
|
|
45
49
|
return None
|
|
46
50
|
|
|
47
51
|
|
|
48
52
|
def get_schema_tables_by_db(database_type, connection, schema_name):
|
|
49
|
-
if database_type
|
|
53
|
+
if _is_postgres(database_type):
|
|
50
54
|
return get_tables_by_schema_postgres(connection, schema_name)
|
|
51
|
-
elif database_type.lower() == "bigquery":
|
|
55
|
+
elif database_type and database_type.lower() == "bigquery":
|
|
52
56
|
return get_tables_by_schema_big_query(connection, schema_name)
|
|
53
57
|
return None
|
|
54
58
|
|
|
55
59
|
|
|
56
60
|
def get_schema_column_info_by_db(database_type, connection, schema_name, table_names):
|
|
57
|
-
if database_type
|
|
61
|
+
if _is_postgres(database_type):
|
|
58
62
|
return get_schema_column_info_postgres(connection, schema_name, table_names)
|
|
59
|
-
elif database_type.lower() == "bigquery":
|
|
63
|
+
elif database_type and database_type.lower() == "bigquery":
|
|
60
64
|
return get_schema_column_info_big_query(connection, schema_name, table_names)
|
|
61
65
|
return None
|
|
@@ -198,7 +198,6 @@ def parse_distinct_values(query_result: Dict[str, Any], database_type: str) -> L
|
|
|
198
198
|
distinct_values = [v.strip() for v in row["string_values"].split(",")]
|
|
199
199
|
|
|
200
200
|
else:
|
|
201
|
-
print(f"Warning: Unknown database type: {database_type}")
|
|
202
201
|
distinct_values = []
|
|
203
202
|
|
|
204
203
|
# Filter out null/undefined/empty values
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
5
|
+
from typing import Callable, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ParallelPostQuill:
|
|
9
|
+
"""
|
|
10
|
+
Provides a thread-pooled, drop-in replacement for Quill.post_quill.
|
|
11
|
+
|
|
12
|
+
Example
|
|
13
|
+
-------
|
|
14
|
+
parallel_post = ParallelPostQuill(
|
|
15
|
+
quill_factory=lambda: Quill(...),
|
|
16
|
+
max_workers=4,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# Synchronous (behaves like the original post_quill)
|
|
20
|
+
response = parallel_post("report", payload)
|
|
21
|
+
|
|
22
|
+
# Async-style execution
|
|
23
|
+
future = parallel_post("report", payload, wait=False)
|
|
24
|
+
response = future.result()
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
quill_factory: Callable[[], "Quill"],
|
|
30
|
+
max_workers: int = 5,
|
|
31
|
+
executor: Optional[ThreadPoolExecutor] = None,
|
|
32
|
+
):
|
|
33
|
+
if executor is None:
|
|
34
|
+
self._executor = ThreadPoolExecutor(
|
|
35
|
+
max_workers=max_workers,
|
|
36
|
+
thread_name_prefix="post-quill",
|
|
37
|
+
)
|
|
38
|
+
self._owns_executor = True
|
|
39
|
+
else:
|
|
40
|
+
self._executor = executor
|
|
41
|
+
self._owns_executor = False
|
|
42
|
+
|
|
43
|
+
if not callable(quill_factory):
|
|
44
|
+
raise ValueError("quill_factory must be callable and create Quill instances.")
|
|
45
|
+
|
|
46
|
+
self._quill_factory = quill_factory
|
|
47
|
+
self._thread_local = threading.local()
|
|
48
|
+
|
|
49
|
+
def __call__(self, path: str, payload: dict, wait: bool = True):
|
|
50
|
+
future: Future = self._executor.submit(self._invoke, path, payload)
|
|
51
|
+
if wait:
|
|
52
|
+
return future.result()
|
|
53
|
+
return future
|
|
54
|
+
|
|
55
|
+
def shutdown(self, wait: bool = True):
|
|
56
|
+
if self._owns_executor:
|
|
57
|
+
self._executor.shutdown(wait=wait)
|
|
58
|
+
|
|
59
|
+
def _get_quill(self):
|
|
60
|
+
quill = getattr(self._thread_local, "quill", None)
|
|
61
|
+
if quill is None:
|
|
62
|
+
quill = self._quill_factory()
|
|
63
|
+
self._thread_local.quill = quill
|
|
64
|
+
return quill
|
|
65
|
+
|
|
66
|
+
def _invoke(self, path: str, payload: dict):
|
|
67
|
+
quill = self._get_quill()
|
|
68
|
+
return quill.post_quill(path, payload)
|
|
69
|
+
|
|
@@ -18,6 +18,7 @@ quillsql/db/postgres.py
|
|
|
18
18
|
quillsql/utils/__init__.py
|
|
19
19
|
quillsql/utils/filters.py
|
|
20
20
|
quillsql/utils/pivot_template.py
|
|
21
|
+
quillsql/utils/post_quill_executor.py
|
|
21
22
|
quillsql/utils/run_query_processes.py
|
|
22
23
|
quillsql/utils/schema_conversion.py
|
|
23
24
|
quillsql/utils/tenants.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|