pytrilogy 0.0.2.49__py3-none-any.whl → 0.0.2.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pytrilogy might be problematic. Click here for more details.
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/METADATA +1 -1
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/RECORD +43 -41
- trilogy/__init__.py +1 -1
- trilogy/core/enums.py +11 -0
- trilogy/core/functions.py +4 -1
- trilogy/core/internal.py +5 -1
- trilogy/core/models.py +135 -263
- trilogy/core/processing/concept_strategies_v3.py +14 -7
- trilogy/core/processing/node_generators/basic_node.py +7 -3
- trilogy/core/processing/node_generators/common.py +8 -5
- trilogy/core/processing/node_generators/filter_node.py +5 -8
- trilogy/core/processing/node_generators/group_node.py +24 -9
- trilogy/core/processing/node_generators/group_to_node.py +0 -2
- trilogy/core/processing/node_generators/multiselect_node.py +4 -5
- trilogy/core/processing/node_generators/node_merge_node.py +14 -3
- trilogy/core/processing/node_generators/rowset_node.py +3 -5
- trilogy/core/processing/node_generators/select_helpers/__init__.py +0 -0
- trilogy/core/processing/node_generators/select_helpers/datasource_injection.py +203 -0
- trilogy/core/processing/node_generators/select_merge_node.py +153 -66
- trilogy/core/processing/node_generators/union_node.py +0 -1
- trilogy/core/processing/node_generators/unnest_node.py +0 -2
- trilogy/core/processing/node_generators/window_node.py +0 -2
- trilogy/core/processing/nodes/base_node.py +2 -36
- trilogy/core/processing/nodes/filter_node.py +0 -3
- trilogy/core/processing/nodes/group_node.py +19 -13
- trilogy/core/processing/nodes/merge_node.py +2 -5
- trilogy/core/processing/nodes/select_node_v2.py +0 -4
- trilogy/core/processing/nodes/union_node.py +0 -3
- trilogy/core/processing/nodes/unnest_node.py +0 -3
- trilogy/core/processing/nodes/window_node.py +0 -3
- trilogy/core/processing/utility.py +3 -0
- trilogy/core/query_processor.py +0 -1
- trilogy/dialect/base.py +14 -2
- trilogy/dialect/duckdb.py +7 -0
- trilogy/hooks/graph_hook.py +17 -1
- trilogy/parsing/common.py +68 -17
- trilogy/parsing/parse_engine.py +70 -20
- trilogy/parsing/render.py +8 -1
- trilogy/parsing/trilogy.lark +3 -1
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/LICENSE.md +0 -0
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/WHEEL +0 -0
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/entry_points.txt +0 -0
- {pytrilogy-0.0.2.49.dist-info → pytrilogy-0.0.2.51.dist-info}/top_level.txt +0 -0
|
@@ -34,7 +34,6 @@ class SelectNode(StrategyNode):
|
|
|
34
34
|
input_concepts: List[Concept],
|
|
35
35
|
output_concepts: List[Concept],
|
|
36
36
|
environment: Environment,
|
|
37
|
-
g,
|
|
38
37
|
datasource: Datasource | None = None,
|
|
39
38
|
whole_grain: bool = False,
|
|
40
39
|
parents: List["StrategyNode"] | None = None,
|
|
@@ -52,7 +51,6 @@ class SelectNode(StrategyNode):
|
|
|
52
51
|
input_concepts=input_concepts,
|
|
53
52
|
output_concepts=output_concepts,
|
|
54
53
|
environment=environment,
|
|
55
|
-
g=g,
|
|
56
54
|
whole_grain=whole_grain,
|
|
57
55
|
parents=parents,
|
|
58
56
|
depth=depth,
|
|
@@ -197,7 +195,6 @@ class SelectNode(StrategyNode):
|
|
|
197
195
|
input_concepts=list(self.input_concepts),
|
|
198
196
|
output_concepts=list(self.output_concepts),
|
|
199
197
|
environment=self.environment,
|
|
200
|
-
g=self.g,
|
|
201
198
|
datasource=self.datasource,
|
|
202
199
|
depth=self.depth,
|
|
203
200
|
parents=self.parents,
|
|
@@ -221,7 +218,6 @@ class ConstantNode(SelectNode):
|
|
|
221
218
|
input_concepts=list(self.input_concepts),
|
|
222
219
|
output_concepts=list(self.output_concepts),
|
|
223
220
|
environment=self.environment,
|
|
224
|
-
g=self.g,
|
|
225
221
|
datasource=self.datasource,
|
|
226
222
|
depth=self.depth,
|
|
227
223
|
partial_concepts=list(self.partial_concepts),
|
|
@@ -18,7 +18,6 @@ class UnionNode(StrategyNode):
|
|
|
18
18
|
input_concepts: List[Concept],
|
|
19
19
|
output_concepts: List[Concept],
|
|
20
20
|
environment,
|
|
21
|
-
g,
|
|
22
21
|
whole_grain: bool = False,
|
|
23
22
|
parents: List["StrategyNode"] | None = None,
|
|
24
23
|
depth: int = 0,
|
|
@@ -27,7 +26,6 @@ class UnionNode(StrategyNode):
|
|
|
27
26
|
input_concepts=input_concepts,
|
|
28
27
|
output_concepts=output_concepts,
|
|
29
28
|
environment=environment,
|
|
30
|
-
g=g,
|
|
31
29
|
whole_grain=whole_grain,
|
|
32
30
|
parents=parents,
|
|
33
31
|
depth=depth,
|
|
@@ -43,7 +41,6 @@ class UnionNode(StrategyNode):
|
|
|
43
41
|
input_concepts=list(self.input_concepts),
|
|
44
42
|
output_concepts=list(self.output_concepts),
|
|
45
43
|
environment=self.environment,
|
|
46
|
-
g=self.g,
|
|
47
44
|
whole_grain=self.whole_grain,
|
|
48
45
|
parents=self.parents,
|
|
49
46
|
depth=self.depth,
|
|
@@ -23,7 +23,6 @@ class UnnestNode(StrategyNode):
|
|
|
23
23
|
input_concepts: List[Concept],
|
|
24
24
|
output_concepts: List[Concept],
|
|
25
25
|
environment,
|
|
26
|
-
g,
|
|
27
26
|
whole_grain: bool = False,
|
|
28
27
|
parents: List["StrategyNode"] | None = None,
|
|
29
28
|
depth: int = 0,
|
|
@@ -32,7 +31,6 @@ class UnnestNode(StrategyNode):
|
|
|
32
31
|
input_concepts=input_concepts,
|
|
33
32
|
output_concepts=output_concepts,
|
|
34
33
|
environment=environment,
|
|
35
|
-
g=g,
|
|
36
34
|
whole_grain=whole_grain,
|
|
37
35
|
parents=parents,
|
|
38
36
|
depth=depth,
|
|
@@ -62,7 +60,6 @@ class UnnestNode(StrategyNode):
|
|
|
62
60
|
input_concepts=list(self.input_concepts),
|
|
63
61
|
output_concepts=list(self.output_concepts),
|
|
64
62
|
environment=self.environment,
|
|
65
|
-
g=self.g,
|
|
66
63
|
whole_grain=self.whole_grain,
|
|
67
64
|
parents=self.parents,
|
|
68
65
|
depth=self.depth,
|
|
@@ -12,7 +12,6 @@ class WindowNode(StrategyNode):
|
|
|
12
12
|
input_concepts: List[Concept],
|
|
13
13
|
output_concepts: List[Concept],
|
|
14
14
|
environment,
|
|
15
|
-
g,
|
|
16
15
|
whole_grain: bool = False,
|
|
17
16
|
parents: List["StrategyNode"] | None = None,
|
|
18
17
|
depth: int = 0,
|
|
@@ -21,7 +20,6 @@ class WindowNode(StrategyNode):
|
|
|
21
20
|
input_concepts=input_concepts,
|
|
22
21
|
output_concepts=output_concepts,
|
|
23
22
|
environment=environment,
|
|
24
|
-
g=g,
|
|
25
23
|
whole_grain=whole_grain,
|
|
26
24
|
parents=parents,
|
|
27
25
|
depth=depth,
|
|
@@ -36,7 +34,6 @@ class WindowNode(StrategyNode):
|
|
|
36
34
|
input_concepts=list(self.input_concepts),
|
|
37
35
|
output_concepts=list(self.output_concepts),
|
|
38
36
|
environment=self.environment,
|
|
39
|
-
g=self.g,
|
|
40
37
|
whole_grain=self.whole_grain,
|
|
41
38
|
parents=self.parents,
|
|
42
39
|
depth=self.depth,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
|
+
from datetime import date, datetime
|
|
2
3
|
from enum import Enum
|
|
3
4
|
from logging import Logger
|
|
4
5
|
from typing import Any, Dict, List, Set, Tuple
|
|
@@ -380,6 +381,8 @@ def is_scalar_condition(
|
|
|
380
381
|
int
|
|
381
382
|
| str
|
|
382
383
|
| float
|
|
384
|
+
| date
|
|
385
|
+
| datetime
|
|
383
386
|
| list[Any]
|
|
384
387
|
| WindowItem
|
|
385
388
|
| FilterItem
|
trilogy/core/query_processor.py
CHANGED
trilogy/dialect/base.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from datetime import date, datetime
|
|
1
2
|
from typing import Any, Callable, Dict, List, Optional, Sequence, Union
|
|
2
3
|
|
|
3
4
|
from jinja2 import Template
|
|
@@ -102,13 +103,15 @@ WINDOW_FUNCTION_MAP = {
|
|
|
102
103
|
WindowType.AVG: window_factory("avg", include_concept=True),
|
|
103
104
|
}
|
|
104
105
|
|
|
105
|
-
DATATYPE_MAP = {
|
|
106
|
+
DATATYPE_MAP: dict[DataType, str] = {
|
|
106
107
|
DataType.STRING: "string",
|
|
107
108
|
DataType.INTEGER: "int",
|
|
108
109
|
DataType.FLOAT: "float",
|
|
109
110
|
DataType.BOOL: "bool",
|
|
110
111
|
DataType.NUMERIC: "numeric",
|
|
111
112
|
DataType.MAP: "map",
|
|
113
|
+
DataType.DATE: "date",
|
|
114
|
+
DataType.DATETIME: "datetime",
|
|
112
115
|
}
|
|
113
116
|
|
|
114
117
|
|
|
@@ -131,6 +134,7 @@ FUNCTION_MAP = {
|
|
|
131
134
|
FunctionType.SPLIT: lambda x: f"split({x[0]}, {x[1]})",
|
|
132
135
|
FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
|
|
133
136
|
FunctionType.BOOL: lambda x: f"CASE WHEN {x[0]} THEN TRUE ELSE FALSE END",
|
|
137
|
+
FunctionType.PARENTHETICAL: lambda x: f"({x[0]})",
|
|
134
138
|
# Complex
|
|
135
139
|
FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
|
|
136
140
|
FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
|
|
@@ -138,6 +142,8 @@ FUNCTION_MAP = {
|
|
|
138
142
|
FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
|
|
139
143
|
FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
|
|
140
144
|
FunctionType.ARRAY: lambda x: f"[{', '.join(x)}]",
|
|
145
|
+
FunctionType.DATE_LITERAL: lambda x: f"date '{x}'",
|
|
146
|
+
FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
|
|
141
147
|
# math
|
|
142
148
|
FunctionType.ADD: lambda x: " + ".join(x),
|
|
143
149
|
FunctionType.SUBTRACT: lambda x: " - ".join(x),
|
|
@@ -454,6 +460,8 @@ class BaseDialect:
|
|
|
454
460
|
list,
|
|
455
461
|
bool,
|
|
456
462
|
float,
|
|
463
|
+
date,
|
|
464
|
+
datetime,
|
|
457
465
|
DataType,
|
|
458
466
|
Function,
|
|
459
467
|
Parenthetical,
|
|
@@ -612,7 +620,7 @@ class BaseDialect:
|
|
|
612
620
|
elif isinstance(e, list):
|
|
613
621
|
return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])}"
|
|
614
622
|
elif isinstance(e, DataType):
|
|
615
|
-
return
|
|
623
|
+
return self.DATATYPE_MAP.get(e, e.value)
|
|
616
624
|
elif isinstance(e, DatePart):
|
|
617
625
|
return str(e.value)
|
|
618
626
|
elif isinstance(e, NumericType):
|
|
@@ -620,6 +628,10 @@ class BaseDialect:
|
|
|
620
628
|
elif isinstance(e, MagicConstants):
|
|
621
629
|
if e == MagicConstants.NULL:
|
|
622
630
|
return "null"
|
|
631
|
+
elif isinstance(e, date):
|
|
632
|
+
return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
|
|
633
|
+
elif isinstance(e, datetime):
|
|
634
|
+
return self.FUNCTION_MAP[FunctionType.DATETIME_LITERAL](e)
|
|
623
635
|
else:
|
|
624
636
|
raise ValueError(f"Unable to render type {type(e)} {e}")
|
|
625
637
|
|
trilogy/dialect/duckdb.py
CHANGED
|
@@ -3,6 +3,7 @@ from typing import Any, Callable, Mapping
|
|
|
3
3
|
from jinja2 import Template
|
|
4
4
|
|
|
5
5
|
from trilogy.core.enums import FunctionType, UnnestMode, WindowType
|
|
6
|
+
from trilogy.core.models import DataType
|
|
6
7
|
from trilogy.dialect.base import BaseDialect
|
|
7
8
|
|
|
8
9
|
WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
|
|
@@ -30,6 +31,8 @@ FUNCTION_MAP = {
|
|
|
30
31
|
FunctionType.DATE_PART: lambda x: f"date_part('{x[1]}', {x[0]})",
|
|
31
32
|
FunctionType.DATE_DIFF: lambda x: f"date_diff('{x[2]}', {x[0]}, {x[1]})",
|
|
32
33
|
FunctionType.CONCAT: lambda x: f"({' || '.join(x)})",
|
|
34
|
+
FunctionType.DATE_LITERAL: lambda x: f"date '{x}'",
|
|
35
|
+
FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
# if an aggregate function is called on a source that is at the same grain as the aggregate
|
|
@@ -44,6 +47,9 @@ FUNCTION_GRAIN_MATCH_MAP = {
|
|
|
44
47
|
FunctionType.MIN: lambda args: f"{args[0]}",
|
|
45
48
|
}
|
|
46
49
|
|
|
50
|
+
DATATYPE_MAP: dict[DataType, str] = {}
|
|
51
|
+
|
|
52
|
+
|
|
47
53
|
DUCKDB_TEMPLATE = Template(
|
|
48
54
|
"""{%- if output %}
|
|
49
55
|
CREATE OR REPLACE TABLE {{ output.address.location }} AS
|
|
@@ -84,6 +90,7 @@ class DuckDBDialect(BaseDialect):
|
|
|
84
90
|
**BaseDialect.FUNCTION_GRAIN_MATCH_MAP,
|
|
85
91
|
**FUNCTION_GRAIN_MATCH_MAP,
|
|
86
92
|
}
|
|
93
|
+
DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, **DATATYPE_MAP}
|
|
87
94
|
QUOTE_CHARACTER = '"'
|
|
88
95
|
SQL_TEMPLATE = DUCKDB_TEMPLATE
|
|
89
96
|
UNNEST_MODE = UnnestMode.DIRECT
|
trilogy/hooks/graph_hook.py
CHANGED
|
@@ -1,7 +1,19 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from os import environ
|
|
3
|
+
|
|
1
4
|
import networkx as nx
|
|
2
5
|
|
|
3
6
|
from trilogy.hooks.base_hook import BaseHook
|
|
4
7
|
|
|
8
|
+
if not environ.get("TCL_LIBRARY"):
|
|
9
|
+
minor = sys.version_info.minor
|
|
10
|
+
if minor == 13:
|
|
11
|
+
environ["TCL_LIBRARY"] = r"C:\Program Files\Python313\tcl\tcl8.6"
|
|
12
|
+
elif minor == 12:
|
|
13
|
+
environ["TCL_LIBRARY"] = r"C:\Program Files\Python312\tcl\tcl8.6"
|
|
14
|
+
else:
|
|
15
|
+
pass
|
|
16
|
+
|
|
5
17
|
|
|
6
18
|
class GraphHook(BaseHook):
|
|
7
19
|
def __init__(self):
|
|
@@ -11,11 +23,14 @@ class GraphHook(BaseHook):
|
|
|
11
23
|
except ImportError:
|
|
12
24
|
raise ImportError("GraphHook requires matplotlib and scipy to be installed")
|
|
13
25
|
|
|
26
|
+
# https://github.com/python/cpython/issues/125235#issuecomment-2412948604
|
|
27
|
+
|
|
14
28
|
def query_graph_built(
|
|
15
29
|
self,
|
|
16
30
|
graph: nx.DiGraph,
|
|
17
31
|
target: str | None = None,
|
|
18
32
|
highlight_nodes: list[str] | None = None,
|
|
33
|
+
remove_isolates: bool = True,
|
|
19
34
|
):
|
|
20
35
|
from matplotlib import pyplot as plt
|
|
21
36
|
|
|
@@ -24,7 +39,8 @@ class GraphHook(BaseHook):
|
|
|
24
39
|
for node in nodes:
|
|
25
40
|
if "__preql_internal" in node:
|
|
26
41
|
graph.remove_node(node)
|
|
27
|
-
|
|
42
|
+
if remove_isolates:
|
|
43
|
+
graph.remove_nodes_from(list(nx.isolates(graph)))
|
|
28
44
|
color_map = []
|
|
29
45
|
highlight_nodes = highlight_nodes or []
|
|
30
46
|
for node in graph:
|
trilogy/parsing/common.py
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
|
+
from datetime import date, datetime
|
|
1
2
|
from typing import List, Tuple
|
|
2
3
|
|
|
3
4
|
from trilogy.constants import (
|
|
4
5
|
VIRTUAL_CONCEPT_PREFIX,
|
|
5
6
|
)
|
|
6
|
-
from trilogy.core.enums import
|
|
7
|
+
from trilogy.core.enums import (
|
|
8
|
+
FunctionType,
|
|
9
|
+
Granularity,
|
|
10
|
+
Modifier,
|
|
11
|
+
PurposeLineage,
|
|
12
|
+
WindowType,
|
|
13
|
+
)
|
|
7
14
|
from trilogy.core.functions import arg_to_datatype, function_args_to_output_purpose
|
|
8
15
|
from trilogy.core.models import (
|
|
9
16
|
AggregateWrapper,
|
|
@@ -37,14 +44,22 @@ def process_function_args(
|
|
|
37
44
|
args,
|
|
38
45
|
meta: Meta | None,
|
|
39
46
|
environment: Environment,
|
|
40
|
-
):
|
|
41
|
-
final: List[Concept | Function] = []
|
|
47
|
+
) -> List[Concept | Function | str | int | float | date | datetime]:
|
|
48
|
+
final: List[Concept | Function | str | int | float | date | datetime] = []
|
|
42
49
|
for arg in args:
|
|
43
50
|
# if a function has an anonymous function argument
|
|
44
51
|
# create an implicit concept
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
52
|
+
if isinstance(arg, Parenthetical):
|
|
53
|
+
processed = process_function_args([arg.content], meta, environment)
|
|
54
|
+
final.append(
|
|
55
|
+
Function(
|
|
56
|
+
operator=FunctionType.PARENTHETICAL,
|
|
57
|
+
arguments=processed,
|
|
58
|
+
output_datatype=arg_to_datatype(processed[0]),
|
|
59
|
+
output_purpose=function_args_to_output_purpose(processed),
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
elif isinstance(arg, Function):
|
|
48
63
|
# if it's not an aggregate function, we can skip the virtual concepts
|
|
49
64
|
# to simplify anonymous function handling
|
|
50
65
|
if (
|
|
@@ -57,7 +72,7 @@ def process_function_args(
|
|
|
57
72
|
concept = function_to_concept(
|
|
58
73
|
arg,
|
|
59
74
|
name=f"{VIRTUAL_CONCEPT_PREFIX}_{arg.operator.value}_{id_hash}",
|
|
60
|
-
|
|
75
|
+
environment=environment,
|
|
61
76
|
)
|
|
62
77
|
# to satisfy mypy, concept will always have metadata
|
|
63
78
|
if concept.metadata and meta:
|
|
@@ -71,7 +86,7 @@ def process_function_args(
|
|
|
71
86
|
concept = arbitrary_to_concept(
|
|
72
87
|
arg,
|
|
73
88
|
name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
|
|
74
|
-
|
|
89
|
+
environment=environment,
|
|
75
90
|
)
|
|
76
91
|
if concept.metadata and meta:
|
|
77
92
|
concept.metadata.line_number = meta.line
|
|
@@ -130,10 +145,39 @@ def constant_to_concept(
|
|
|
130
145
|
)
|
|
131
146
|
|
|
132
147
|
|
|
148
|
+
def concepts_to_grain_concepts(
|
|
149
|
+
concepts: List[Concept] | List[str] | set[str], environment: Environment | None
|
|
150
|
+
) -> list[Concept]:
|
|
151
|
+
environment = Environment() if environment is None else environment
|
|
152
|
+
pconcepts: list[Concept] = [
|
|
153
|
+
c if isinstance(c, Concept) else environment.concepts[c] for c in concepts
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
final: List[Concept] = []
|
|
157
|
+
for sub in pconcepts:
|
|
158
|
+
if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
|
|
159
|
+
if any([c.address in pconcepts for c in sub.keys]):
|
|
160
|
+
continue
|
|
161
|
+
if sub.purpose in (Purpose.METRIC,):
|
|
162
|
+
if all([c in pconcepts for c in sub.grain.components]):
|
|
163
|
+
continue
|
|
164
|
+
if sub.granularity == Granularity.SINGLE_ROW:
|
|
165
|
+
continue
|
|
166
|
+
final.append(sub)
|
|
167
|
+
final = unique(final, "address")
|
|
168
|
+
v2 = sorted(final, key=lambda x: x.name)
|
|
169
|
+
return v2
|
|
170
|
+
|
|
171
|
+
|
|
133
172
|
def function_to_concept(
|
|
134
|
-
parent: Function,
|
|
173
|
+
parent: Function,
|
|
174
|
+
name: str,
|
|
175
|
+
environment: Environment,
|
|
176
|
+
namespace: str | None = None,
|
|
177
|
+
metadata: Metadata | None = None,
|
|
135
178
|
) -> Concept:
|
|
136
179
|
pkeys: List[Concept] = []
|
|
180
|
+
namespace = namespace or environment.namespace
|
|
137
181
|
for x in parent.arguments:
|
|
138
182
|
pkeys += [
|
|
139
183
|
x
|
|
@@ -153,7 +197,7 @@ def function_to_concept(
|
|
|
153
197
|
key_grain += [*x.keys]
|
|
154
198
|
else:
|
|
155
199
|
key_grain.append(x)
|
|
156
|
-
keys = tuple(
|
|
200
|
+
keys = tuple(concepts_to_grain_concepts(key_grain, environment))
|
|
157
201
|
if not pkeys:
|
|
158
202
|
purpose = Purpose.CONSTANT
|
|
159
203
|
else:
|
|
@@ -247,7 +291,7 @@ def window_item_to_concept(
|
|
|
247
291
|
lineage=parent,
|
|
248
292
|
metadata=fmetadata,
|
|
249
293
|
# filters are implicitly at the grain of the base item
|
|
250
|
-
grain=Grain(
|
|
294
|
+
grain=Grain.from_concepts(grain),
|
|
251
295
|
namespace=namespace,
|
|
252
296
|
keys=keys,
|
|
253
297
|
modifiers=modifiers,
|
|
@@ -259,9 +303,8 @@ def agg_wrapper_to_concept(
|
|
|
259
303
|
namespace: str,
|
|
260
304
|
name: str,
|
|
261
305
|
metadata: Metadata | None = None,
|
|
262
|
-
purpose: Purpose | None = None,
|
|
263
306
|
) -> Concept:
|
|
264
|
-
|
|
307
|
+
_, keys = get_purpose_and_keys(
|
|
265
308
|
Purpose.METRIC, tuple(parent.by) if parent.by else None
|
|
266
309
|
)
|
|
267
310
|
# anything grouped to a grain should be a property
|
|
@@ -275,7 +318,7 @@ def agg_wrapper_to_concept(
|
|
|
275
318
|
purpose=Purpose.METRIC,
|
|
276
319
|
metadata=fmetadata,
|
|
277
320
|
lineage=parent,
|
|
278
|
-
grain=Grain(
|
|
321
|
+
grain=Grain.from_concepts(parent.by) if parent.by else Grain(),
|
|
279
322
|
namespace=namespace,
|
|
280
323
|
keys=tuple(parent.by) if parent.by else keys,
|
|
281
324
|
modifiers=modifiers,
|
|
@@ -295,15 +338,17 @@ def arbitrary_to_concept(
|
|
|
295
338
|
| float
|
|
296
339
|
| str
|
|
297
340
|
),
|
|
298
|
-
|
|
341
|
+
environment: Environment,
|
|
342
|
+
namespace: str | None = None,
|
|
299
343
|
name: str | None = None,
|
|
300
344
|
metadata: Metadata | None = None,
|
|
301
345
|
purpose: Purpose | None = None,
|
|
302
346
|
) -> Concept:
|
|
347
|
+
namespace = namespace or environment.namespace
|
|
303
348
|
if isinstance(parent, AggregateWrapper):
|
|
304
349
|
if not name:
|
|
305
350
|
name = f"{VIRTUAL_CONCEPT_PREFIX}_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
|
|
306
|
-
return agg_wrapper_to_concept(parent, namespace, name, metadata
|
|
351
|
+
return agg_wrapper_to_concept(parent, namespace, name, metadata)
|
|
307
352
|
elif isinstance(parent, WindowItem):
|
|
308
353
|
if not name:
|
|
309
354
|
name = f"{VIRTUAL_CONCEPT_PREFIX}_window_{parent.type.value}_{string_to_hash(str(parent))}"
|
|
@@ -315,7 +360,13 @@ def arbitrary_to_concept(
|
|
|
315
360
|
elif isinstance(parent, Function):
|
|
316
361
|
if not name:
|
|
317
362
|
name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
|
|
318
|
-
return function_to_concept(
|
|
363
|
+
return function_to_concept(
|
|
364
|
+
parent,
|
|
365
|
+
name,
|
|
366
|
+
metadata=metadata,
|
|
367
|
+
environment=environment,
|
|
368
|
+
namespace=namespace,
|
|
369
|
+
)
|
|
319
370
|
elif isinstance(parent, ListWrapper):
|
|
320
371
|
if not name:
|
|
321
372
|
name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
|
trilogy/parsing/parse_engine.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
|
+
from datetime import date, datetime
|
|
2
3
|
from os.path import dirname, join
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from re import IGNORECASE
|
|
@@ -462,7 +463,7 @@ class ParseToObjects(Transformer):
|
|
|
462
463
|
datatype=args[2],
|
|
463
464
|
purpose=args[0],
|
|
464
465
|
metadata=metadata,
|
|
465
|
-
grain=Grain(components=parents),
|
|
466
|
+
grain=Grain(components={x.address for x in parents}),
|
|
466
467
|
namespace=namespace,
|
|
467
468
|
keys=parents,
|
|
468
469
|
modifiers=modifiers,
|
|
@@ -529,6 +530,7 @@ class ParseToObjects(Transformer):
|
|
|
529
530
|
source_value,
|
|
530
531
|
name=name,
|
|
531
532
|
namespace=namespace,
|
|
533
|
+
environment=self.environment,
|
|
532
534
|
purpose=purpose,
|
|
533
535
|
metadata=metadata,
|
|
534
536
|
)
|
|
@@ -597,7 +599,7 @@ class ParseToObjects(Transformer):
|
|
|
597
599
|
output_purpose=Purpose.CONSTANT,
|
|
598
600
|
arguments=[constant],
|
|
599
601
|
),
|
|
600
|
-
grain=Grain(components=
|
|
602
|
+
grain=Grain(components=set()),
|
|
601
603
|
namespace=namespace,
|
|
602
604
|
)
|
|
603
605
|
if concept.metadata:
|
|
@@ -622,7 +624,9 @@ class ParseToObjects(Transformer):
|
|
|
622
624
|
return args
|
|
623
625
|
|
|
624
626
|
def grain_clause(self, args) -> Grain:
|
|
625
|
-
return Grain(
|
|
627
|
+
return Grain(
|
|
628
|
+
components=set([self.environment.concepts[a].address for a in args[0]])
|
|
629
|
+
)
|
|
626
630
|
|
|
627
631
|
def whole_grain_clause(self, args) -> WholeGrainWrapper:
|
|
628
632
|
return WholeGrainWrapper(where=args[0])
|
|
@@ -669,6 +673,12 @@ class ParseToObjects(Transformer):
|
|
|
669
673
|
)
|
|
670
674
|
for column in columns:
|
|
671
675
|
column.concept = column.concept.with_grain(datasource.grain)
|
|
676
|
+
if datasource.where:
|
|
677
|
+
for x in datasource.where.concept_arguments:
|
|
678
|
+
if x.address not in datasource.output_concepts:
|
|
679
|
+
raise ValueError(
|
|
680
|
+
f"Datasource {name} where condition depends on concept {x.address} that does not exist on the datasource, line {meta.line}."
|
|
681
|
+
)
|
|
672
682
|
self.environment.add_datasource(datasource, meta=meta)
|
|
673
683
|
return datasource
|
|
674
684
|
|
|
@@ -708,7 +718,11 @@ class ParseToObjects(Transformer):
|
|
|
708
718
|
)
|
|
709
719
|
elif isinstance(transformation, Function):
|
|
710
720
|
concept = function_to_concept(
|
|
711
|
-
transformation,
|
|
721
|
+
transformation,
|
|
722
|
+
namespace=namespace,
|
|
723
|
+
name=output,
|
|
724
|
+
metadata=metadata,
|
|
725
|
+
environment=self.environment,
|
|
712
726
|
)
|
|
713
727
|
else:
|
|
714
728
|
raise SyntaxError("Invalid transformation")
|
|
@@ -764,10 +778,8 @@ class ParseToObjects(Transformer):
|
|
|
764
778
|
def handle_order_item(x, namespace: str):
|
|
765
779
|
if not isinstance(x, Concept):
|
|
766
780
|
x = arbitrary_to_concept(
|
|
767
|
-
x,
|
|
768
|
-
namespace=namespace,
|
|
781
|
+
x, namespace=namespace, environment=self.environment
|
|
769
782
|
)
|
|
770
|
-
self.environment.add_concept(x)
|
|
771
783
|
return x
|
|
772
784
|
|
|
773
785
|
return [
|
|
@@ -1022,12 +1034,32 @@ class ParseToObjects(Transformer):
|
|
|
1022
1034
|
order_by=order_by,
|
|
1023
1035
|
meta=Metadata(line_number=meta.line),
|
|
1024
1036
|
)
|
|
1025
|
-
for parse_pass in [
|
|
1037
|
+
for parse_pass in [
|
|
1038
|
+
1,
|
|
1039
|
+
2,
|
|
1040
|
+
]:
|
|
1026
1041
|
# the first pass will result in all concepts being defined
|
|
1027
1042
|
# the second will get grains appropriately
|
|
1028
1043
|
# eg if someone does sum(x)->a, b+c -> z - we don't know if Z is a key to group by or an aggregate
|
|
1029
1044
|
# until after the first pass, and so don't know the grain of a
|
|
1030
|
-
|
|
1045
|
+
|
|
1046
|
+
if parse_pass == 1:
|
|
1047
|
+
grain = Grain.from_concepts(
|
|
1048
|
+
[
|
|
1049
|
+
x.content
|
|
1050
|
+
for x in output.selection
|
|
1051
|
+
if isinstance(x.content, Concept)
|
|
1052
|
+
],
|
|
1053
|
+
where_clause=output.where_clause,
|
|
1054
|
+
)
|
|
1055
|
+
if parse_pass == 2:
|
|
1056
|
+
grain = Grain.from_concepts(
|
|
1057
|
+
output.output_components, where_clause=output.where_clause
|
|
1058
|
+
)
|
|
1059
|
+
print(
|
|
1060
|
+
f"end pass {parse_pass} grain {grain} from {output.output_components}"
|
|
1061
|
+
)
|
|
1062
|
+
output.grain = grain
|
|
1031
1063
|
for item in select_items:
|
|
1032
1064
|
# we don't know the grain of an aggregate at assignment time
|
|
1033
1065
|
# so rebuild at this point in the tree
|
|
@@ -1057,7 +1089,7 @@ class ParseToObjects(Transformer):
|
|
|
1057
1089
|
environment=self.environment,
|
|
1058
1090
|
)
|
|
1059
1091
|
output.local_concepts[item.content.address] = item.content
|
|
1060
|
-
|
|
1092
|
+
|
|
1061
1093
|
if order_by:
|
|
1062
1094
|
output.order_by = order_by.with_select_context(
|
|
1063
1095
|
local_concepts=output.local_concepts,
|
|
@@ -1174,7 +1206,7 @@ class ParseToObjects(Transformer):
|
|
|
1174
1206
|
if isinstance(args[0], AggregateWrapper):
|
|
1175
1207
|
left = arbitrary_to_concept(
|
|
1176
1208
|
args[0],
|
|
1177
|
-
|
|
1209
|
+
environment=self.environment,
|
|
1178
1210
|
)
|
|
1179
1211
|
self.environment.add_concept(left)
|
|
1180
1212
|
else:
|
|
@@ -1182,7 +1214,7 @@ class ParseToObjects(Transformer):
|
|
|
1182
1214
|
if isinstance(args[2], AggregateWrapper):
|
|
1183
1215
|
right = arbitrary_to_concept(
|
|
1184
1216
|
args[2],
|
|
1185
|
-
|
|
1217
|
+
environment=self.environment,
|
|
1186
1218
|
)
|
|
1187
1219
|
self.environment.add_concept(right)
|
|
1188
1220
|
else:
|
|
@@ -1220,10 +1252,7 @@ class ParseToObjects(Transformer):
|
|
|
1220
1252
|
):
|
|
1221
1253
|
right = right.content
|
|
1222
1254
|
if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
|
|
1223
|
-
right = arbitrary_to_concept(
|
|
1224
|
-
right,
|
|
1225
|
-
namespace=self.environment.namespace,
|
|
1226
|
-
)
|
|
1255
|
+
right = arbitrary_to_concept(right, environment=self.environment)
|
|
1227
1256
|
self.environment.add_concept(right, meta=meta)
|
|
1228
1257
|
return SubselectComparison(
|
|
1229
1258
|
left=args[0],
|
|
@@ -1292,10 +1321,7 @@ class ParseToObjects(Transformer):
|
|
|
1292
1321
|
elif isinstance(item, WindowType):
|
|
1293
1322
|
type = item
|
|
1294
1323
|
else:
|
|
1295
|
-
concept = arbitrary_to_concept(
|
|
1296
|
-
item,
|
|
1297
|
-
namespace=self.environment.namespace,
|
|
1298
|
-
)
|
|
1324
|
+
concept = arbitrary_to_concept(item, environment=self.environment)
|
|
1299
1325
|
self.environment.add_concept(concept, meta=meta)
|
|
1300
1326
|
assert concept
|
|
1301
1327
|
return WindowItem(
|
|
@@ -1783,6 +1809,30 @@ class ParseToObjects(Transformer):
|
|
|
1783
1809
|
@v_args(meta=True)
|
|
1784
1810
|
def fcast(self, meta, args) -> Function:
|
|
1785
1811
|
args = process_function_args(args, meta=meta, environment=self.environment)
|
|
1812
|
+
if isinstance(args[0], str):
|
|
1813
|
+
processed: date | datetime | int | float | bool | str
|
|
1814
|
+
if args[1] == DataType.DATE:
|
|
1815
|
+
processed = date.fromisoformat(args[0])
|
|
1816
|
+
elif args[1] == DataType.DATETIME:
|
|
1817
|
+
processed = datetime.fromisoformat(args[0])
|
|
1818
|
+
elif args[1] == DataType.TIMESTAMP:
|
|
1819
|
+
processed = datetime.fromisoformat(args[0])
|
|
1820
|
+
elif args[1] == DataType.INTEGER:
|
|
1821
|
+
processed = int(args[0])
|
|
1822
|
+
elif args[1] == DataType.FLOAT:
|
|
1823
|
+
processed = float(args[0])
|
|
1824
|
+
elif args[1] == DataType.BOOL:
|
|
1825
|
+
processed = args[0].capitalize() == "True"
|
|
1826
|
+
elif args[1] == DataType.STRING:
|
|
1827
|
+
processed = args[0]
|
|
1828
|
+
else:
|
|
1829
|
+
raise SyntaxError(f"Invalid cast type {args[1]}")
|
|
1830
|
+
return Function(
|
|
1831
|
+
operator=FunctionType.CONSTANT,
|
|
1832
|
+
output_datatype=args[1],
|
|
1833
|
+
output_purpose=Purpose.CONSTANT,
|
|
1834
|
+
arguments=[processed],
|
|
1835
|
+
)
|
|
1786
1836
|
output_datatype = args[1]
|
|
1787
1837
|
return Function(
|
|
1788
1838
|
operator=FunctionType.CAST,
|
trilogy/parsing/render.py
CHANGED
|
@@ -158,7 +158,14 @@ class Renderer:
|
|
|
158
158
|
|
|
159
159
|
@to_string.register
|
|
160
160
|
def _(self, arg: "Grain"):
|
|
161
|
-
|
|
161
|
+
final = []
|
|
162
|
+
for comp in arg.components:
|
|
163
|
+
if comp.startswith(DEFAULT_NAMESPACE):
|
|
164
|
+
final.append(comp.split(".", 1)[1])
|
|
165
|
+
else:
|
|
166
|
+
final.append(comp)
|
|
167
|
+
final = sorted(final)
|
|
168
|
+
components = ",".join(x for x in final)
|
|
162
169
|
return f"grain ({components})"
|
|
163
170
|
|
|
164
171
|
@to_string.register
|
trilogy/parsing/trilogy.lark
CHANGED
|
@@ -193,7 +193,9 @@
|
|
|
193
193
|
_math_functions: fmul | fdiv | fadd | fsub | fround | fmod | fabs
|
|
194
194
|
|
|
195
195
|
//generic
|
|
196
|
-
|
|
196
|
+
_fcast_primary: "cast"i "(" expr "as"i data_type ")"
|
|
197
|
+
_fcast_alt: expr "::" data_type
|
|
198
|
+
fcast: _fcast_primary | _fcast_alt
|
|
197
199
|
concat: ("concat"i "(" (expr ",")* expr ")") | (expr "||" expr)
|
|
198
200
|
fcoalesce: "coalesce"i "(" (expr ",")* expr ")"
|
|
199
201
|
fcase_when: "WHEN"i conditional "THEN"i expr
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|