quillsql 2.2.6__tar.gz → 2.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quillsql-2.2.6/quillsql.egg-info → quillsql-2.2.7}/PKG-INFO +2 -2
- quillsql-2.2.7/quillsql/__init__.py +14 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/core.py +27 -21
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/db/bigquery.py +2 -1
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/db/cached_connection.py +32 -9
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/__init__.py +1 -0
- quillsql-2.2.7/quillsql/utils/post_quill_executor.py +69 -0
- {quillsql-2.2.6 → quillsql-2.2.7/quillsql.egg-info}/PKG-INFO +1 -1
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql.egg-info/SOURCES.txt +1 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/setup.py +1 -1
- quillsql-2.2.6/quillsql/__init__.py +0 -5
- {quillsql-2.2.6 → quillsql-2.2.7}/README.md +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/assets/__init__.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/assets/pgtypes.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/db/__init__.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/db/db_helper.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/db/postgres.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/error.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/filters.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/pivot_template.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/run_query_processes.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/schema_conversion.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql/utils/tenants.py +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql.egg-info/dependency_links.txt +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql.egg-info/requires.txt +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/quillsql.egg-info/top_level.txt +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/setup.cfg +0 -0
- {quillsql-2.2.6 → quillsql-2.2.7}/tests/test_core.py +0 -0
|
@@ -144,10 +144,14 @@ class Quill:
|
|
|
144
144
|
None
|
|
145
145
|
)
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
distinct_value_results
|
|
149
|
-
|
|
150
|
-
|
|
147
|
+
distinct_value_query_results = (
|
|
148
|
+
distinct_value_results or {}
|
|
149
|
+
).get("queryResults") or []
|
|
150
|
+
if distinct_value_query_results:
|
|
151
|
+
distinct_values = parse_distinct_values(
|
|
152
|
+
distinct_value_query_results[0],
|
|
153
|
+
config.get("databaseType")
|
|
154
|
+
)
|
|
151
155
|
|
|
152
156
|
try:
|
|
153
157
|
final_query = hydrate_pivot_template(template, distinct_values, config)
|
|
@@ -261,17 +265,16 @@ class Quill:
|
|
|
261
265
|
else:
|
|
262
266
|
tenant_flags = flags
|
|
263
267
|
|
|
264
|
-
|
|
265
|
-
self.run_queries(
|
|
268
|
+
if metadata.get("preQueries"):
|
|
269
|
+
pre_query_results = self.run_queries(
|
|
266
270
|
metadata.get("preQueries"),
|
|
267
271
|
self.target_connection.database_type,
|
|
268
272
|
metadata.get("databaseType"),
|
|
269
273
|
metadata,
|
|
270
274
|
metadata.get("runQueryConfig"),
|
|
271
275
|
)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
)
|
|
276
|
+
else:
|
|
277
|
+
pre_query_results = {}
|
|
275
278
|
|
|
276
279
|
if metadata.get("runQueryConfig") and metadata.get("runQueryConfig").get(
|
|
277
280
|
"overridePost"
|
|
@@ -396,24 +399,27 @@ class Quill:
|
|
|
396
399
|
return query
|
|
397
400
|
return f"{query.rstrip(';')} limit {limit}"
|
|
398
401
|
|
|
399
|
-
def normalize_database_type(self, db_type):
|
|
400
|
-
if not db_type:
|
|
401
|
-
return None
|
|
402
|
-
lowered = db_type.lower()
|
|
403
|
-
if lowered in ("postgresql", "postgres"):
|
|
404
|
-
return "postgres"
|
|
405
|
-
return lowered
|
|
406
|
-
|
|
407
402
|
def run_queries(
|
|
408
403
|
self, queries, pkDatabaseType, databaseType=None, metadata=None, runQueryConfig=None
|
|
409
404
|
):
|
|
410
405
|
results = {}
|
|
411
406
|
if not queries:
|
|
412
407
|
return {"queryResults": []}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
408
|
+
def _normalize_db_type(db_type):
|
|
409
|
+
if not db_type:
|
|
410
|
+
return None
|
|
411
|
+
lowered = db_type.lower()
|
|
412
|
+
if lowered in ("postgresql", "postgres"):
|
|
413
|
+
return "postgres"
|
|
414
|
+
return lowered
|
|
415
|
+
|
|
416
|
+
normalized_metadata_db = _normalize_db_type(databaseType)
|
|
417
|
+
normalized_backend_db = _normalize_db_type(pkDatabaseType)
|
|
418
|
+
if (
|
|
419
|
+
normalized_metadata_db
|
|
420
|
+
and normalized_backend_db
|
|
421
|
+
and normalized_metadata_db != normalized_backend_db
|
|
422
|
+
):
|
|
417
423
|
return {"dbMismatched": True, "backendDatabaseType": pkDatabaseType}
|
|
418
424
|
if runQueryConfig and runQueryConfig.get("arrayToMap"):
|
|
419
425
|
mapped_array = array_to_map(
|
|
@@ -26,7 +26,8 @@ def format_bigquery_config(connection_string):
|
|
|
26
26
|
"project": service_account.get("project_id"),
|
|
27
27
|
"credentials": service_account,
|
|
28
28
|
}
|
|
29
|
-
except (ValueError, TypeError):
|
|
29
|
+
except (ValueError, TypeError) as e:
|
|
30
|
+
print("Invalid service account JSON.", e)
|
|
30
31
|
return connection_string
|
|
31
32
|
|
|
32
33
|
|
|
@@ -29,20 +29,43 @@ class CachedConnection:
|
|
|
29
29
|
)
|
|
30
30
|
return None
|
|
31
31
|
|
|
32
|
+
def _is_connection_closed(self):
|
|
33
|
+
return self.connection is None or getattr(self.connection, "closed", True)
|
|
34
|
+
|
|
35
|
+
def _open_connection(self):
|
|
36
|
+
self.connection = connect_to_db(
|
|
37
|
+
self.database_type, self.config, self.using_connection_string
|
|
38
|
+
)
|
|
39
|
+
return self.connection
|
|
40
|
+
|
|
41
|
+
def _close_connection(self):
|
|
42
|
+
if self.connection is None:
|
|
43
|
+
return
|
|
44
|
+
try:
|
|
45
|
+
self.connection.close()
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
finally:
|
|
49
|
+
self.connection = None
|
|
50
|
+
|
|
32
51
|
def exec_with_reconnect(self, sql):
|
|
33
52
|
reconnect_count = 0
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
self.database_type, self.config, self.using_connection_string
|
|
37
|
-
)
|
|
53
|
+
last_error = None
|
|
54
|
+
while reconnect_count < 10:
|
|
38
55
|
try:
|
|
56
|
+
if self._is_connection_closed():
|
|
57
|
+
self._open_connection()
|
|
39
58
|
return run_query_by_db(self.database_type, sql, self.connection)
|
|
40
59
|
except psycopg2.Error as err:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
60
|
+
last_error = err
|
|
61
|
+
reconnect_count += 1
|
|
62
|
+
self._close_connection()
|
|
63
|
+
except Exception:
|
|
64
|
+
self._close_connection()
|
|
65
|
+
raise
|
|
66
|
+
diag = getattr(last_error, "diag", None)
|
|
67
|
+
position = getattr(diag, "statement_position", None)
|
|
68
|
+
raise PgQueryError(last_error or "Failed to execute query", sql, position)
|
|
46
69
|
|
|
47
70
|
def exec(self, sql):
|
|
48
71
|
try:
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
5
|
+
from typing import Callable, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ParallelPostQuill:
|
|
9
|
+
"""
|
|
10
|
+
Provides a thread-pooled, drop-in replacement for Quill.post_quill.
|
|
11
|
+
|
|
12
|
+
Example
|
|
13
|
+
-------
|
|
14
|
+
parallel_post = ParallelPostQuill(
|
|
15
|
+
quill_factory=lambda: Quill(...),
|
|
16
|
+
max_workers=4,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# Synchronous (behaves like the original post_quill)
|
|
20
|
+
response = parallel_post("report", payload)
|
|
21
|
+
|
|
22
|
+
# Async-style execution
|
|
23
|
+
future = parallel_post("report", payload, wait=False)
|
|
24
|
+
response = future.result()
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
quill_factory: Callable[[], "Quill"],
|
|
30
|
+
max_workers: int = 5,
|
|
31
|
+
executor: Optional[ThreadPoolExecutor] = None,
|
|
32
|
+
):
|
|
33
|
+
if executor is None:
|
|
34
|
+
self._executor = ThreadPoolExecutor(
|
|
35
|
+
max_workers=max_workers,
|
|
36
|
+
thread_name_prefix="post-quill",
|
|
37
|
+
)
|
|
38
|
+
self._owns_executor = True
|
|
39
|
+
else:
|
|
40
|
+
self._executor = executor
|
|
41
|
+
self._owns_executor = False
|
|
42
|
+
|
|
43
|
+
if not callable(quill_factory):
|
|
44
|
+
raise ValueError("quill_factory must be callable and create Quill instances.")
|
|
45
|
+
|
|
46
|
+
self._quill_factory = quill_factory
|
|
47
|
+
self._thread_local = threading.local()
|
|
48
|
+
|
|
49
|
+
def __call__(self, path: str, payload: dict, wait: bool = True):
|
|
50
|
+
future: Future = self._executor.submit(self._invoke, path, payload)
|
|
51
|
+
if wait:
|
|
52
|
+
return future.result()
|
|
53
|
+
return future
|
|
54
|
+
|
|
55
|
+
def shutdown(self, wait: bool = True):
|
|
56
|
+
if self._owns_executor:
|
|
57
|
+
self._executor.shutdown(wait=wait)
|
|
58
|
+
|
|
59
|
+
def _get_quill(self):
|
|
60
|
+
quill = getattr(self._thread_local, "quill", None)
|
|
61
|
+
if quill is None:
|
|
62
|
+
quill = self._quill_factory()
|
|
63
|
+
self._thread_local.quill = quill
|
|
64
|
+
return quill
|
|
65
|
+
|
|
66
|
+
def _invoke(self, path: str, payload: dict):
|
|
67
|
+
quill = self._get_quill()
|
|
68
|
+
return quill.post_quill(path, payload)
|
|
69
|
+
|
|
@@ -18,6 +18,7 @@ quillsql/db/postgres.py
|
|
|
18
18
|
quillsql/utils/__init__.py
|
|
19
19
|
quillsql/utils/filters.py
|
|
20
20
|
quillsql/utils/pivot_template.py
|
|
21
|
+
quillsql/utils/post_quill_executor.py
|
|
21
22
|
quillsql/utils/run_query_processes.py
|
|
22
23
|
quillsql/utils/schema_conversion.py
|
|
23
24
|
quillsql/utils/tenants.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|