pytrilogy 0.0.2.57__py3-none-any.whl → 0.0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytrilogy-0.0.2.57.dist-info → pytrilogy-0.0.3.0.dist-info}/METADATA +9 -2
- pytrilogy-0.0.3.0.dist-info/RECORD +99 -0
- {pytrilogy-0.0.2.57.dist-info → pytrilogy-0.0.3.0.dist-info}/WHEEL +1 -1
- trilogy/__init__.py +2 -2
- trilogy/core/enums.py +1 -7
- trilogy/core/env_processor.py +17 -5
- trilogy/core/environment_helpers.py +11 -25
- trilogy/core/exceptions.py +4 -0
- trilogy/core/functions.py +695 -261
- trilogy/core/graph_models.py +10 -10
- trilogy/core/internal.py +11 -2
- trilogy/core/models/__init__.py +0 -0
- trilogy/core/models/author.py +2110 -0
- trilogy/core/models/build.py +1845 -0
- trilogy/core/models/build_environment.py +151 -0
- trilogy/core/models/core.py +370 -0
- trilogy/core/models/datasource.py +297 -0
- trilogy/core/models/environment.py +696 -0
- trilogy/core/models/execute.py +931 -0
- trilogy/core/optimization.py +17 -22
- trilogy/core/optimizations/base_optimization.py +1 -1
- trilogy/core/optimizations/inline_constant.py +6 -6
- trilogy/core/optimizations/inline_datasource.py +17 -11
- trilogy/core/optimizations/predicate_pushdown.py +17 -16
- trilogy/core/processing/concept_strategies_v3.py +181 -146
- trilogy/core/processing/graph_utils.py +1 -1
- trilogy/core/processing/node_generators/basic_node.py +19 -18
- trilogy/core/processing/node_generators/common.py +51 -45
- trilogy/core/processing/node_generators/filter_node.py +26 -13
- trilogy/core/processing/node_generators/group_node.py +26 -21
- trilogy/core/processing/node_generators/group_to_node.py +13 -10
- trilogy/core/processing/node_generators/multiselect_node.py +60 -43
- trilogy/core/processing/node_generators/node_merge_node.py +76 -38
- trilogy/core/processing/node_generators/rowset_node.py +59 -36
- trilogy/core/processing/node_generators/select_helpers/datasource_injection.py +27 -34
- trilogy/core/processing/node_generators/select_merge_node.py +161 -64
- trilogy/core/processing/node_generators/select_node.py +13 -13
- trilogy/core/processing/node_generators/union_node.py +12 -11
- trilogy/core/processing/node_generators/unnest_node.py +9 -7
- trilogy/core/processing/node_generators/window_node.py +19 -16
- trilogy/core/processing/nodes/__init__.py +21 -18
- trilogy/core/processing/nodes/base_node.py +92 -77
- trilogy/core/processing/nodes/filter_node.py +19 -13
- trilogy/core/processing/nodes/group_node.py +55 -40
- trilogy/core/processing/nodes/merge_node.py +47 -38
- trilogy/core/processing/nodes/select_node_v2.py +54 -40
- trilogy/core/processing/nodes/union_node.py +5 -7
- trilogy/core/processing/nodes/unnest_node.py +7 -11
- trilogy/core/processing/nodes/window_node.py +9 -4
- trilogy/core/processing/utility.py +108 -80
- trilogy/core/query_processor.py +67 -49
- trilogy/core/statements/__init__.py +0 -0
- trilogy/core/statements/author.py +413 -0
- trilogy/core/statements/build.py +0 -0
- trilogy/core/statements/common.py +30 -0
- trilogy/core/statements/execute.py +42 -0
- trilogy/dialect/base.py +152 -111
- trilogy/dialect/common.py +9 -10
- trilogy/dialect/duckdb.py +1 -1
- trilogy/dialect/enums.py +4 -2
- trilogy/dialect/presto.py +1 -1
- trilogy/dialect/sql_server.py +1 -1
- trilogy/executor.py +44 -32
- trilogy/hooks/base_hook.py +6 -4
- trilogy/hooks/query_debugger.py +110 -93
- trilogy/parser.py +1 -1
- trilogy/parsing/common.py +303 -64
- trilogy/parsing/parse_engine.py +263 -617
- trilogy/parsing/render.py +50 -26
- trilogy/scripts/trilogy.py +2 -1
- pytrilogy-0.0.2.57.dist-info/RECORD +0 -87
- trilogy/core/models.py +0 -4960
- {pytrilogy-0.0.2.57.dist-info → pytrilogy-0.0.3.0.dist-info}/LICENSE.md +0 -0
- {pytrilogy-0.0.2.57.dist-info → pytrilogy-0.0.3.0.dist-info}/entry_points.txt +0 -0
- {pytrilogy-0.0.2.57.dist-info → pytrilogy-0.0.3.0.dist-info}/top_level.txt +0 -0
trilogy/dialect/base.py
CHANGED
|
@@ -11,63 +11,80 @@ from trilogy.core.enums import (
|
|
|
11
11
|
WindowType,
|
|
12
12
|
)
|
|
13
13
|
from trilogy.core.internal import DEFAULT_CONCEPTS
|
|
14
|
-
from trilogy.core.models import (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
14
|
+
from trilogy.core.models.build import (
|
|
15
|
+
BuildAggregateWrapper,
|
|
16
|
+
BuildCaseElse,
|
|
17
|
+
BuildCaseWhen,
|
|
18
|
+
BuildComparison,
|
|
19
|
+
BuildConcept,
|
|
20
|
+
BuildConditional,
|
|
21
|
+
BuildFilterItem,
|
|
22
|
+
BuildFunction,
|
|
23
|
+
BuildMultiSelectLineage,
|
|
24
|
+
BuildOrderItem,
|
|
25
|
+
BuildParenthetical,
|
|
26
|
+
BuildRowsetItem,
|
|
27
|
+
BuildSubselectComparison,
|
|
28
|
+
BuildWindowItem,
|
|
29
|
+
Factory,
|
|
30
|
+
)
|
|
31
|
+
from trilogy.core.models.core import (
|
|
26
32
|
DataType,
|
|
27
|
-
Environment,
|
|
28
|
-
FilterItem,
|
|
29
|
-
Function,
|
|
30
|
-
ImportStatement,
|
|
31
33
|
ListType,
|
|
32
34
|
ListWrapper,
|
|
33
35
|
MapType,
|
|
34
36
|
MapWrapper,
|
|
35
|
-
MergeStatementV2,
|
|
36
|
-
MultiSelectStatement,
|
|
37
37
|
NumericType,
|
|
38
|
-
OrderItem,
|
|
39
|
-
Parenthetical,
|
|
40
|
-
PersistStatement,
|
|
41
|
-
ProcessedCopyStatement,
|
|
42
|
-
ProcessedQuery,
|
|
43
|
-
ProcessedQueryPersist,
|
|
44
|
-
ProcessedRawSQLStatement,
|
|
45
|
-
ProcessedShowStatement,
|
|
46
|
-
RawColumnExpr,
|
|
47
|
-
RawSQLStatement,
|
|
48
|
-
RowsetDerivationStatement,
|
|
49
|
-
RowsetItem,
|
|
50
|
-
SelectStatement,
|
|
51
|
-
ShowStatement,
|
|
52
38
|
StructType,
|
|
53
|
-
SubselectComparison,
|
|
54
39
|
TupleWrapper,
|
|
55
|
-
UnionCTE,
|
|
56
|
-
WindowItem,
|
|
57
40
|
)
|
|
41
|
+
from trilogy.core.models.datasource import Datasource, RawColumnExpr
|
|
42
|
+
from trilogy.core.models.environment import Environment
|
|
43
|
+
from trilogy.core.models.execute import CTE, CompiledCTE, UnionCTE
|
|
58
44
|
from trilogy.core.processing.utility import (
|
|
59
45
|
decompose_condition,
|
|
60
46
|
is_scalar_condition,
|
|
61
47
|
sort_select_output,
|
|
62
48
|
)
|
|
63
49
|
from trilogy.core.query_processor import process_copy, process_persist, process_query
|
|
50
|
+
from trilogy.core.statements.author import (
|
|
51
|
+
ConceptDeclarationStatement,
|
|
52
|
+
CopyStatement,
|
|
53
|
+
ImportStatement,
|
|
54
|
+
MergeStatementV2,
|
|
55
|
+
MultiSelectStatement,
|
|
56
|
+
PersistStatement,
|
|
57
|
+
RawSQLStatement,
|
|
58
|
+
RowsetDerivationStatement,
|
|
59
|
+
SelectStatement,
|
|
60
|
+
ShowStatement,
|
|
61
|
+
)
|
|
62
|
+
from trilogy.core.statements.execute import (
|
|
63
|
+
ProcessedCopyStatement,
|
|
64
|
+
ProcessedQuery,
|
|
65
|
+
ProcessedQueryPersist,
|
|
66
|
+
ProcessedRawSQLStatement,
|
|
67
|
+
ProcessedShowStatement,
|
|
68
|
+
)
|
|
64
69
|
from trilogy.dialect.common import render_join, render_unnest
|
|
65
70
|
from trilogy.hooks.base_hook import BaseHook
|
|
66
71
|
|
|
67
72
|
LOGGER_PREFIX = "[RENDERING]"
|
|
68
73
|
|
|
74
|
+
WINDOW_ITEMS = (BuildWindowItem,)
|
|
75
|
+
FILTER_ITEMS = (BuildFilterItem,)
|
|
76
|
+
AGGREGATE_ITEMS = (BuildAggregateWrapper,)
|
|
77
|
+
FUNCTION_ITEMS = (BuildFunction,)
|
|
78
|
+
PARENTHETICAL_ITEMS = (BuildParenthetical,)
|
|
79
|
+
CASE_WHEN_ITEMS = (BuildCaseWhen,)
|
|
80
|
+
CASE_ELSE_ITEMS = (BuildCaseElse,)
|
|
81
|
+
SUBSELECT_COMPARISON_ITEMS = (BuildSubselectComparison,)
|
|
82
|
+
COMPARISON_ITEMS = (BuildComparison,)
|
|
83
|
+
CONDITIONAL_ITEMS = (BuildConditional,)
|
|
84
|
+
|
|
69
85
|
|
|
70
86
|
def INVALID_REFERENCE_STRING(x: Any, callsite: str = ""):
|
|
87
|
+
# raise SyntaxError(x)
|
|
71
88
|
return f"INVALID_REFERENCE_BUG_{callsite}<{x}>"
|
|
72
89
|
|
|
73
90
|
|
|
@@ -237,7 +254,7 @@ def safe_quote(string: str, quote_char: str):
|
|
|
237
254
|
return ".".join([f"{quote_char}{string}{quote_char}" for string in components])
|
|
238
255
|
|
|
239
256
|
|
|
240
|
-
def safe_get_cte_value(coalesce, cte: CTE | UnionCTE, c:
|
|
257
|
+
def safe_get_cte_value(coalesce, cte: CTE | UnionCTE, c: BuildConcept, quote_char: str):
|
|
241
258
|
address = c.address
|
|
242
259
|
raw = cte.source_map.get(address, None)
|
|
243
260
|
|
|
@@ -265,25 +282,26 @@ class BaseDialect:
|
|
|
265
282
|
|
|
266
283
|
def render_order_item(
|
|
267
284
|
self,
|
|
268
|
-
order_item:
|
|
285
|
+
order_item: BuildOrderItem,
|
|
269
286
|
cte: CTE | UnionCTE,
|
|
270
287
|
final: bool = False,
|
|
271
288
|
alias: bool = True,
|
|
272
289
|
) -> str:
|
|
273
|
-
if final:
|
|
274
|
-
|
|
275
|
-
|
|
290
|
+
# if final:
|
|
291
|
+
# if not alias:
|
|
292
|
+
# return f"{self.QUOTE_CHARACTER}{order_item.expr.safe_address}{self.QUOTE_CHARACTER} {order_item.order.value}"
|
|
276
293
|
|
|
277
|
-
|
|
294
|
+
# return f"{cte.name}.{self.QUOTE_CHARACTER}{order_item.expr.safe_address}{self.QUOTE_CHARACTER} {order_item.order.value}"
|
|
278
295
|
|
|
279
|
-
return f"{self.
|
|
296
|
+
return f"{self.render_expr(order_item.expr, cte=cte, qualify=False)} {order_item.order.value}"
|
|
280
297
|
|
|
281
298
|
def render_concept_sql(
|
|
282
299
|
self,
|
|
283
|
-
c:
|
|
300
|
+
c: BuildConcept,
|
|
284
301
|
cte: CTE | UnionCTE,
|
|
285
302
|
alias: bool = True,
|
|
286
303
|
raise_invalid: bool = False,
|
|
304
|
+
qualify: bool = True,
|
|
287
305
|
) -> str:
|
|
288
306
|
result = None
|
|
289
307
|
if c.pseudonyms:
|
|
@@ -297,20 +315,26 @@ class BaseDialect:
|
|
|
297
315
|
f"{LOGGER_PREFIX} [{c.address}] Attempting rendering w/ candidate {candidate.address}"
|
|
298
316
|
)
|
|
299
317
|
result = self._render_concept_sql(
|
|
300
|
-
candidate, cte, raise_invalid=True
|
|
318
|
+
candidate, cte, raise_invalid=True, qualify=qualify
|
|
301
319
|
)
|
|
302
320
|
if result:
|
|
303
321
|
break
|
|
304
322
|
except ValueError:
|
|
305
323
|
continue
|
|
306
324
|
if not result:
|
|
307
|
-
result = self._render_concept_sql(
|
|
325
|
+
result = self._render_concept_sql(
|
|
326
|
+
c, cte, raise_invalid=raise_invalid, qualify=qualify
|
|
327
|
+
)
|
|
308
328
|
if alias:
|
|
309
329
|
return f"{result} as {self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
|
|
310
330
|
return result
|
|
311
331
|
|
|
312
332
|
def _render_concept_sql(
|
|
313
|
-
self,
|
|
333
|
+
self,
|
|
334
|
+
c: BuildConcept,
|
|
335
|
+
cte: CTE | UnionCTE,
|
|
336
|
+
raise_invalid: bool = False,
|
|
337
|
+
qualify: bool = True,
|
|
314
338
|
) -> str:
|
|
315
339
|
# only recurse while it's in sources of the current cte
|
|
316
340
|
logger.debug(
|
|
@@ -322,9 +346,9 @@ class BaseDialect:
|
|
|
322
346
|
logger.debug(
|
|
323
347
|
f"{LOGGER_PREFIX} [{c.address}] rendering concept with lineage that is not already existing, have {cte.source_map}"
|
|
324
348
|
)
|
|
325
|
-
if isinstance(c.lineage,
|
|
349
|
+
if isinstance(c.lineage, WINDOW_ITEMS):
|
|
326
350
|
rendered_order_components = [
|
|
327
|
-
f"{self.
|
|
351
|
+
f"{self.render_expr(x.expr, cte, raise_invalid=raise_invalid)} {x.order.value}"
|
|
328
352
|
for x in c.lineage.order_by
|
|
329
353
|
]
|
|
330
354
|
rendered_over_components = [
|
|
@@ -345,17 +369,17 @@ class BaseDialect:
|
|
|
345
369
|
sort=",".join(rendered_order_components),
|
|
346
370
|
offset=c.lineage.index,
|
|
347
371
|
)
|
|
348
|
-
elif isinstance(c.lineage,
|
|
372
|
+
elif isinstance(c.lineage, FILTER_ITEMS):
|
|
349
373
|
# for cases when we've optimized this
|
|
350
374
|
if cte.condition == c.lineage.where.conditional:
|
|
351
375
|
rval = self.render_expr(c.lineage.content, cte=cte)
|
|
352
376
|
else:
|
|
353
377
|
rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)} ELSE NULL END"
|
|
354
|
-
elif isinstance(c.lineage,
|
|
378
|
+
elif isinstance(c.lineage, BuildRowsetItem):
|
|
355
379
|
rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)}"
|
|
356
|
-
elif isinstance(c.lineage,
|
|
380
|
+
elif isinstance(c.lineage, BuildMultiSelectLineage):
|
|
357
381
|
rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
|
|
358
|
-
elif isinstance(c.lineage,
|
|
382
|
+
elif isinstance(c.lineage, AGGREGATE_ITEMS):
|
|
359
383
|
args = [
|
|
360
384
|
self.render_expr(v, cte) # , alias=False)
|
|
361
385
|
for v in c.lineage.function.arguments
|
|
@@ -369,21 +393,21 @@ class BaseDialect:
|
|
|
369
393
|
)
|
|
370
394
|
rval = f"{self.FUNCTION_GRAIN_MATCH_MAP[c.lineage.function.operator](args)}"
|
|
371
395
|
elif (
|
|
372
|
-
isinstance(c.lineage,
|
|
396
|
+
isinstance(c.lineage, FUNCTION_ITEMS)
|
|
373
397
|
and c.lineage.operator == FunctionType.UNION
|
|
374
398
|
):
|
|
375
399
|
local_matched = [
|
|
376
400
|
x
|
|
377
401
|
for x in c.lineage.arguments
|
|
378
|
-
if isinstance(x,
|
|
402
|
+
if isinstance(x, BuildConcept) and x.address in cte.output_columns
|
|
379
403
|
]
|
|
404
|
+
# if we're sorting by the output of the union
|
|
380
405
|
if not local_matched:
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
)
|
|
384
|
-
rval = self.render_expr(local_matched[0], cte)
|
|
406
|
+
rval = c.safe_address
|
|
407
|
+
else:
|
|
408
|
+
rval = self.render_expr(local_matched[0], cte)
|
|
385
409
|
elif (
|
|
386
|
-
isinstance(c.lineage,
|
|
410
|
+
isinstance(c.lineage, FUNCTION_ITEMS)
|
|
387
411
|
and c.lineage.operator == FunctionType.CONSTANT
|
|
388
412
|
and CONFIG.rendering.parameters is True
|
|
389
413
|
and c.datatype.data_type != DataType.MAP
|
|
@@ -393,9 +417,9 @@ class BaseDialect:
|
|
|
393
417
|
args = []
|
|
394
418
|
for arg in c.lineage.arguments:
|
|
395
419
|
if (
|
|
396
|
-
isinstance(arg,
|
|
420
|
+
isinstance(arg, BuildConcept)
|
|
397
421
|
and arg.lineage
|
|
398
|
-
and isinstance(arg.lineage,
|
|
422
|
+
and isinstance(arg.lineage, FUNCTION_ITEMS)
|
|
399
423
|
and arg.lineage.operator
|
|
400
424
|
in (
|
|
401
425
|
FunctionType.ADD,
|
|
@@ -406,7 +430,7 @@ class BaseDialect:
|
|
|
406
430
|
):
|
|
407
431
|
args.append(
|
|
408
432
|
self.render_expr(
|
|
409
|
-
|
|
433
|
+
BuildParenthetical(content=arg),
|
|
410
434
|
cte=cte,
|
|
411
435
|
raise_invalid=raise_invalid,
|
|
412
436
|
)
|
|
@@ -422,13 +446,13 @@ class BaseDialect:
|
|
|
422
446
|
rval = f"{self.FUNCTION_GRAIN_MATCH_MAP[c.lineage.operator](args)}"
|
|
423
447
|
else:
|
|
424
448
|
logger.debug(
|
|
425
|
-
f"{LOGGER_PREFIX} [{c.address}] Rendering basic lookup from {cte.source_map.get(c.address,
|
|
449
|
+
f"{LOGGER_PREFIX} [{c.address}] Rendering basic lookup from {cte.source_map.get(c.address,None)}"
|
|
426
450
|
)
|
|
427
451
|
|
|
428
452
|
raw_content = cte.get_alias(c)
|
|
429
453
|
if isinstance(raw_content, RawColumnExpr):
|
|
430
454
|
rval = raw_content.text
|
|
431
|
-
elif isinstance(raw_content,
|
|
455
|
+
elif isinstance(raw_content, FUNCTION_ITEMS):
|
|
432
456
|
rval = self.render_expr(
|
|
433
457
|
raw_content, cte=cte, raise_invalid=raise_invalid
|
|
434
458
|
)
|
|
@@ -440,23 +464,34 @@ class BaseDialect:
|
|
|
440
464
|
self.QUOTE_CHARACTER,
|
|
441
465
|
)
|
|
442
466
|
if not rval:
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
467
|
+
# unions won't have a specific source mapped; just use a generic column reference
|
|
468
|
+
# we shouldn't ever have an expression at this point, so will be safe
|
|
469
|
+
if isinstance(cte, UnionCTE):
|
|
470
|
+
rval = c.safe_address
|
|
471
|
+
else:
|
|
472
|
+
if raise_invalid:
|
|
473
|
+
raise ValueError(
|
|
474
|
+
f"Invalid reference string found in query: {rval}, this should never occur. Please report this issue."
|
|
475
|
+
)
|
|
476
|
+
rval = INVALID_REFERENCE_STRING(
|
|
477
|
+
f"Missing source reference to {c.address}"
|
|
446
478
|
)
|
|
447
|
-
rval = INVALID_REFERENCE_STRING(
|
|
448
|
-
f"Missing source reference to {c.address}"
|
|
449
|
-
)
|
|
450
479
|
return rval
|
|
451
480
|
|
|
452
481
|
def render_expr(
|
|
453
482
|
self,
|
|
454
483
|
e: Union[
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
484
|
+
BuildConcept,
|
|
485
|
+
BuildFunction,
|
|
486
|
+
BuildConditional,
|
|
487
|
+
BuildAggregateWrapper,
|
|
488
|
+
BuildComparison,
|
|
489
|
+
BuildCaseWhen,
|
|
490
|
+
BuildCaseElse,
|
|
491
|
+
BuildSubselectComparison,
|
|
492
|
+
BuildWindowItem,
|
|
493
|
+
BuildFilterItem,
|
|
494
|
+
BuildParenthetical,
|
|
460
495
|
str,
|
|
461
496
|
int,
|
|
462
497
|
list,
|
|
@@ -465,9 +500,6 @@ class BaseDialect:
|
|
|
465
500
|
date,
|
|
466
501
|
datetime,
|
|
467
502
|
DataType,
|
|
468
|
-
Function,
|
|
469
|
-
Parenthetical,
|
|
470
|
-
AggregateWrapper,
|
|
471
503
|
MagicConstants,
|
|
472
504
|
MapWrapper[Any, Any],
|
|
473
505
|
MapType,
|
|
@@ -477,18 +509,14 @@ class BaseDialect:
|
|
|
477
509
|
ListWrapper[Any],
|
|
478
510
|
TupleWrapper[Any],
|
|
479
511
|
DatePart,
|
|
480
|
-
CaseWhen,
|
|
481
|
-
CaseElse,
|
|
482
|
-
WindowItem,
|
|
483
|
-
FilterItem,
|
|
484
|
-
# FilterItem
|
|
485
512
|
],
|
|
486
513
|
cte: Optional[CTE | UnionCTE] = None,
|
|
487
514
|
cte_map: Optional[Dict[str, CTE | UnionCTE]] = None,
|
|
488
515
|
raise_invalid: bool = False,
|
|
516
|
+
qualify: bool = True,
|
|
489
517
|
) -> str:
|
|
490
|
-
if isinstance(e,
|
|
491
|
-
if isinstance(e.right,
|
|
518
|
+
if isinstance(e, SUBSELECT_COMPARISON_ITEMS):
|
|
519
|
+
if isinstance(e.right, BuildConcept):
|
|
492
520
|
# we won't always have an existnce map
|
|
493
521
|
# so fall back to the normal map
|
|
494
522
|
lookup_cte = cte
|
|
@@ -513,7 +541,10 @@ class BaseDialect:
|
|
|
513
541
|
f"Missing source CTE for {e.right.address}"
|
|
514
542
|
)
|
|
515
543
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
|
|
516
|
-
elif isinstance(
|
|
544
|
+
elif isinstance(
|
|
545
|
+
e.right,
|
|
546
|
+
(ListWrapper, TupleWrapper, BuildParenthetical, list),
|
|
547
|
+
):
|
|
517
548
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
|
|
518
549
|
|
|
519
550
|
elif isinstance(
|
|
@@ -528,12 +559,12 @@ class BaseDialect:
|
|
|
528
559
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} ({self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)})"
|
|
529
560
|
else:
|
|
530
561
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
|
|
531
|
-
elif isinstance(e,
|
|
562
|
+
elif isinstance(e, COMPARISON_ITEMS):
|
|
532
563
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
|
|
533
|
-
elif isinstance(e,
|
|
564
|
+
elif isinstance(e, CONDITIONAL_ITEMS):
|
|
534
565
|
# conditions need to be nested in parentheses
|
|
535
566
|
return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
|
|
536
|
-
elif isinstance(e,
|
|
567
|
+
elif isinstance(e, WINDOW_ITEMS):
|
|
537
568
|
rendered_order_components = [
|
|
538
569
|
f"{self.render_expr(x.expr, cte, cte_map=cte_map, raise_invalid=raise_invalid)} {x.order.value}"
|
|
539
570
|
for x in e.order_by
|
|
@@ -543,22 +574,22 @@ class BaseDialect:
|
|
|
543
574
|
for x in e.over
|
|
544
575
|
]
|
|
545
576
|
return f"{self.WINDOW_FUNCTION_MAP[e.type](concept = self.render_expr(e.content, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}" # noqa: E501
|
|
546
|
-
elif isinstance(e,
|
|
577
|
+
elif isinstance(e, PARENTHETICAL_ITEMS):
|
|
547
578
|
# conditions need to be nested in parentheses
|
|
548
579
|
if isinstance(e.content, list):
|
|
549
580
|
return f"( {','.join([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e.content])} )"
|
|
550
581
|
return f"( {self.render_expr(e.content, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} )"
|
|
551
|
-
elif isinstance(e,
|
|
582
|
+
elif isinstance(e, CASE_WHEN_ITEMS):
|
|
552
583
|
return f"WHEN {self.render_expr(e.comparison, cte=cte, cte_map=cte_map) } THEN {self.render_expr(e.expr, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) }"
|
|
553
|
-
elif isinstance(e,
|
|
584
|
+
elif isinstance(e, CASE_ELSE_ITEMS):
|
|
554
585
|
return f"ELSE {self.render_expr(e.expr, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) }"
|
|
555
|
-
elif isinstance(e,
|
|
586
|
+
elif isinstance(e, FUNCTION_ITEMS):
|
|
556
587
|
arguments = []
|
|
557
588
|
for arg in e.arguments:
|
|
558
589
|
if (
|
|
559
|
-
isinstance(arg,
|
|
590
|
+
isinstance(arg, BuildConcept)
|
|
560
591
|
and arg.lineage
|
|
561
|
-
and isinstance(arg.lineage,
|
|
592
|
+
and isinstance(arg.lineage, FUNCTION_ITEMS)
|
|
562
593
|
and arg.lineage.operator
|
|
563
594
|
in (
|
|
564
595
|
FunctionType.ADD,
|
|
@@ -569,7 +600,7 @@ class BaseDialect:
|
|
|
569
600
|
):
|
|
570
601
|
arguments.append(
|
|
571
602
|
self.render_expr(
|
|
572
|
-
|
|
603
|
+
BuildParenthetical(content=arg),
|
|
573
604
|
cte=cte,
|
|
574
605
|
cte_map=cte_map,
|
|
575
606
|
raise_invalid=raise_invalid,
|
|
@@ -586,15 +617,15 @@ class BaseDialect:
|
|
|
586
617
|
return self.FUNCTION_MAP[e.operator](arguments)
|
|
587
618
|
|
|
588
619
|
return self.FUNCTION_GRAIN_MATCH_MAP[e.operator](arguments)
|
|
589
|
-
elif isinstance(e,
|
|
620
|
+
elif isinstance(e, AGGREGATE_ITEMS):
|
|
590
621
|
return self.render_expr(
|
|
591
622
|
e.function, cte, cte_map=cte_map, raise_invalid=raise_invalid
|
|
592
623
|
)
|
|
593
|
-
elif isinstance(e,
|
|
624
|
+
elif isinstance(e, FILTER_ITEMS):
|
|
594
625
|
return f"CASE WHEN {self.render_expr(e.where.conditional,cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} THEN {self.render_expr(e.content, cte, cte_map=cte_map, raise_invalid=raise_invalid)} ELSE NULL END"
|
|
595
|
-
elif isinstance(e,
|
|
626
|
+
elif isinstance(e, BuildConcept):
|
|
596
627
|
if (
|
|
597
|
-
isinstance(e.lineage,
|
|
628
|
+
isinstance(e.lineage, FUNCTION_ITEMS)
|
|
598
629
|
and e.lineage.operator == FunctionType.CONSTANT
|
|
599
630
|
and CONFIG.rendering.parameters is True
|
|
600
631
|
and e.datatype.data_type != DataType.MAP
|
|
@@ -602,7 +633,11 @@ class BaseDialect:
|
|
|
602
633
|
return f":{e.safe_address}"
|
|
603
634
|
if cte:
|
|
604
635
|
return self.render_concept_sql(
|
|
605
|
-
e,
|
|
636
|
+
e,
|
|
637
|
+
cte,
|
|
638
|
+
alias=False,
|
|
639
|
+
raise_invalid=raise_invalid,
|
|
640
|
+
qualify=qualify,
|
|
606
641
|
)
|
|
607
642
|
elif cte_map:
|
|
608
643
|
return f"{cte_map[e.address].name}.{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
|
|
@@ -643,6 +678,7 @@ class BaseDialect:
|
|
|
643
678
|
[self.render_cte(child).statement for child in cte.internal_ctes]
|
|
644
679
|
)
|
|
645
680
|
if cte.order_by:
|
|
681
|
+
|
|
646
682
|
ordering = [
|
|
647
683
|
self.render_order_item(i, cte, final=True, alias=False)
|
|
648
684
|
for i in cte.order_by.items
|
|
@@ -660,7 +696,7 @@ class BaseDialect:
|
|
|
660
696
|
self.render_concept_sql(c, cte)
|
|
661
697
|
for c in cte.output_columns
|
|
662
698
|
if c.address not in [y.address for y in cte.join_derived_concepts]
|
|
663
|
-
and c.address not in
|
|
699
|
+
and c.address not in cte.hidden_concepts
|
|
664
700
|
] + [
|
|
665
701
|
f"{self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
|
|
666
702
|
for c in cte.join_derived_concepts
|
|
@@ -670,7 +706,7 @@ class BaseDialect:
|
|
|
670
706
|
select_columns = [
|
|
671
707
|
self.render_concept_sql(c, cte)
|
|
672
708
|
for c in cte.output_columns
|
|
673
|
-
if c.address not in
|
|
709
|
+
if c.address not in cte.hidden_concepts
|
|
674
710
|
]
|
|
675
711
|
if auto_sort:
|
|
676
712
|
select_columns = sorted(select_columns, key=lambda x: x)
|
|
@@ -703,8 +739,8 @@ class BaseDialect:
|
|
|
703
739
|
final_joins = []
|
|
704
740
|
else:
|
|
705
741
|
final_joins = cte.joins or []
|
|
706
|
-
where:
|
|
707
|
-
having:
|
|
742
|
+
where: BuildConditional | BuildParenthetical | BuildComparison | None = None
|
|
743
|
+
having: BuildConditional | BuildParenthetical | BuildComparison | None = None
|
|
708
744
|
materialized = {x for x, v in cte.source_map.items() if v}
|
|
709
745
|
if cte.condition:
|
|
710
746
|
if not cte.group_to_grain or is_scalar_condition(
|
|
@@ -807,6 +843,7 @@ class BaseDialect:
|
|
|
807
843
|
| ProcessedRawSQLStatement
|
|
808
844
|
| ProcessedCopyStatement
|
|
809
845
|
] = []
|
|
846
|
+
factory = Factory(environment=environment)
|
|
810
847
|
for statement in statements:
|
|
811
848
|
if isinstance(statement, PersistStatement):
|
|
812
849
|
if hooks:
|
|
@@ -837,12 +874,15 @@ class BaseDialect:
|
|
|
837
874
|
elif isinstance(statement, ShowStatement):
|
|
838
875
|
# TODO - encapsulate this a little better
|
|
839
876
|
if isinstance(statement.content, SelectStatement):
|
|
877
|
+
|
|
840
878
|
output.append(
|
|
841
879
|
ProcessedShowStatement(
|
|
842
880
|
output_columns=[
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
881
|
+
factory.build(
|
|
882
|
+
environment.concepts[
|
|
883
|
+
DEFAULT_CONCEPTS["query_text"].address
|
|
884
|
+
]
|
|
885
|
+
)
|
|
846
886
|
],
|
|
847
887
|
output_values=[
|
|
848
888
|
process_query(
|
|
@@ -886,9 +926,10 @@ class BaseDialect:
|
|
|
886
926
|
select_columns: Dict[str, str] = {}
|
|
887
927
|
cte_output_map = {}
|
|
888
928
|
selected = set()
|
|
889
|
-
hidden_addresses = [c.address for c in query.hidden_columns]
|
|
890
929
|
output_addresses = [
|
|
891
|
-
c.address
|
|
930
|
+
c.address
|
|
931
|
+
for c in query.output_columns
|
|
932
|
+
if c.address not in query.hidden_columns
|
|
892
933
|
]
|
|
893
934
|
|
|
894
935
|
for c in query.base.output_columns:
|
|
@@ -897,7 +938,7 @@ class BaseDialect:
|
|
|
897
938
|
f"{query.base.name}.{safe_quote(c.safe_address, self.QUOTE_CHARACTER)}"
|
|
898
939
|
)
|
|
899
940
|
cte_output_map[c.address] = query.base
|
|
900
|
-
if c.address not in
|
|
941
|
+
if c.address not in query.hidden_columns:
|
|
901
942
|
selected.add(c.address)
|
|
902
943
|
if not all([x in selected for x in output_addresses]):
|
|
903
944
|
missing = [x for x in output_addresses if x not in selected]
|
trilogy/dialect/common.py
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
from typing import Callable
|
|
2
2
|
|
|
3
3
|
from trilogy.core.enums import Modifier, UnnestMode
|
|
4
|
-
from trilogy.core.models import
|
|
4
|
+
from trilogy.core.models.build import BuildConcept, BuildFunction
|
|
5
|
+
from trilogy.core.models.datasource import RawColumnExpr
|
|
6
|
+
from trilogy.core.models.execute import (
|
|
5
7
|
CTE,
|
|
6
|
-
Concept,
|
|
7
|
-
Function,
|
|
8
8
|
InstantiatedUnnestJoin,
|
|
9
9
|
Join,
|
|
10
|
-
RawColumnExpr,
|
|
11
10
|
)
|
|
12
11
|
|
|
13
12
|
|
|
@@ -20,8 +19,8 @@ def null_wrapper(lval: str, rval: str, modifiers: list[Modifier]) -> str:
|
|
|
20
19
|
def render_unnest(
|
|
21
20
|
unnest_mode: UnnestMode,
|
|
22
21
|
quote_character: str,
|
|
23
|
-
concept:
|
|
24
|
-
render_func: Callable[[
|
|
22
|
+
concept: BuildConcept,
|
|
23
|
+
render_func: Callable[[BuildConcept, CTE, bool], str],
|
|
25
24
|
cte: CTE,
|
|
26
25
|
):
|
|
27
26
|
if unnest_mode == UnnestMode.CROSS_JOIN:
|
|
@@ -33,7 +32,7 @@ def render_join_concept(
|
|
|
33
32
|
name: str,
|
|
34
33
|
quote_character: str,
|
|
35
34
|
cte: CTE,
|
|
36
|
-
concept:
|
|
35
|
+
concept: BuildConcept,
|
|
37
36
|
render_expr,
|
|
38
37
|
inlined_ctes: set[str],
|
|
39
38
|
):
|
|
@@ -43,7 +42,7 @@ def render_join_concept(
|
|
|
43
42
|
if isinstance(raw_content, RawColumnExpr):
|
|
44
43
|
rval = raw_content.text
|
|
45
44
|
return rval
|
|
46
|
-
elif isinstance(raw_content,
|
|
45
|
+
elif isinstance(raw_content, BuildFunction):
|
|
47
46
|
rval = render_expr(raw_content, cte=cte)
|
|
48
47
|
return rval
|
|
49
48
|
return f"{name}.{quote_character}{raw_content}{quote_character}"
|
|
@@ -53,8 +52,8 @@ def render_join_concept(
|
|
|
53
52
|
def render_join(
|
|
54
53
|
join: Join | InstantiatedUnnestJoin,
|
|
55
54
|
quote_character: str,
|
|
56
|
-
render_func: Callable[[
|
|
57
|
-
render_expr_func: Callable[[
|
|
55
|
+
render_func: Callable[[BuildConcept, CTE, bool], str],
|
|
56
|
+
render_expr_func: Callable[[BuildConcept, CTE], str],
|
|
58
57
|
cte: CTE,
|
|
59
58
|
unnest_mode: UnnestMode = UnnestMode.CROSS_APPLY,
|
|
60
59
|
) -> str | None:
|
trilogy/dialect/duckdb.py
CHANGED
|
@@ -3,7 +3,7 @@ from typing import Any, Callable, Mapping
|
|
|
3
3
|
from jinja2 import Template
|
|
4
4
|
|
|
5
5
|
from trilogy.core.enums import FunctionType, UnnestMode, WindowType
|
|
6
|
-
from trilogy.core.models import DataType
|
|
6
|
+
from trilogy.core.models.core import DataType
|
|
7
7
|
from trilogy.dialect.base import BaseDialect
|
|
8
8
|
|
|
9
9
|
WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
|
trilogy/dialect/enums.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from enum import Enum
|
|
2
2
|
from typing import TYPE_CHECKING, Callable, List, Optional
|
|
3
3
|
|
|
4
|
+
from trilogy.core.models.environment import Environment
|
|
5
|
+
|
|
4
6
|
if TYPE_CHECKING:
|
|
5
|
-
from trilogy import
|
|
7
|
+
from trilogy import Executor
|
|
6
8
|
from trilogy.hooks.base_hook import BaseHook
|
|
7
9
|
|
|
8
10
|
from trilogy.constants import logger
|
|
@@ -98,7 +100,7 @@ class Dialects(Enum):
|
|
|
98
100
|
conf: DialectConfig | None = None,
|
|
99
101
|
_engine_factory: Callable | None = None,
|
|
100
102
|
) -> "Executor":
|
|
101
|
-
from trilogy import
|
|
103
|
+
from trilogy import Executor
|
|
102
104
|
|
|
103
105
|
if _engine_factory is not None:
|
|
104
106
|
return Executor(
|
trilogy/dialect/presto.py
CHANGED
|
@@ -3,7 +3,7 @@ from typing import Any, Callable, Mapping
|
|
|
3
3
|
from jinja2 import Template
|
|
4
4
|
|
|
5
5
|
from trilogy.core.enums import FunctionType, UnnestMode, WindowType
|
|
6
|
-
from trilogy.core.models import DataType
|
|
6
|
+
from trilogy.core.models.core import DataType
|
|
7
7
|
from trilogy.dialect.base import BaseDialect
|
|
8
8
|
|
|
9
9
|
WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
|
trilogy/dialect/sql_server.py
CHANGED
|
@@ -3,7 +3,7 @@ from typing import Any, Callable, Mapping
|
|
|
3
3
|
from jinja2 import Template
|
|
4
4
|
|
|
5
5
|
from trilogy.core.enums import FunctionType, WindowType
|
|
6
|
-
from trilogy.core.
|
|
6
|
+
from trilogy.core.statements.execute import (
|
|
7
7
|
ProcessedQuery,
|
|
8
8
|
ProcessedQueryPersist,
|
|
9
9
|
ProcessedRawSQLStatement,
|