pytrilogy 0.0.3.48__py3-none-any.whl → 0.0.3.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pytrilogy might be problematic. Click here for more details.
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/METADATA +1 -1
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/RECORD +24 -24
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/WHEEL +1 -1
- trilogy/__init__.py +1 -1
- trilogy/core/enums.py +3 -0
- trilogy/core/functions.py +22 -0
- trilogy/core/models/author.py +6 -2
- trilogy/core/models/build.py +15 -3
- trilogy/core/models/execute.py +4 -2
- trilogy/core/processing/concept_strategies_v3.py +1 -1
- trilogy/core/processing/node_generators/common.py +3 -4
- trilogy/core/processing/node_generators/filter_node.py +142 -91
- trilogy/core/processing/node_generators/group_node.py +3 -4
- trilogy/core/processing/nodes/base_node.py +4 -1
- trilogy/core/statements/author.py +0 -2
- trilogy/dialect/base.py +7 -2
- trilogy/dialect/bigquery.py +2 -0
- trilogy/dialect/duckdb.py +2 -0
- trilogy/parsing/common.py +25 -15
- trilogy/parsing/parse_engine.py +35 -7
- trilogy/parsing/trilogy.lark +10 -4
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/entry_points.txt +0 -0
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/licenses/LICENSE.md +0 -0
- {pytrilogy-0.0.3.48.dist-info → pytrilogy-0.0.3.51.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
pytrilogy-0.0.3.
|
|
2
|
-
trilogy/__init__.py,sha256=
|
|
1
|
+
pytrilogy-0.0.3.51.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
|
|
2
|
+
trilogy/__init__.py,sha256=jsdVvojVM5ErpcYeajtXLHstXjkkMaZ73Xhh5lccD6g,303
|
|
3
3
|
trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
|
|
5
5
|
trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
|
|
@@ -11,37 +11,37 @@ trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
|
|
|
11
11
|
trilogy/authoring/__init__.py,sha256=v9PRuZs4fTnxhpXAnwTxCDwlLasUax6g2FONidcujR4,2369
|
|
12
12
|
trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
13
|
trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
|
|
14
|
-
trilogy/core/enums.py,sha256=
|
|
14
|
+
trilogy/core/enums.py,sha256=uZTi9K6PEpQ1oFV4OpHlC1NUSxrmAFdQBfRyy9Rba-8,7440
|
|
15
15
|
trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0,3170
|
|
16
16
|
trilogy/core/environment_helpers.py,sha256=VvPIiFemqaLLpIpLIqprfu63K7muZ1YzNg7UZIUph8w,8267
|
|
17
17
|
trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
|
|
18
18
|
trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
|
|
19
|
-
trilogy/core/functions.py,sha256=
|
|
19
|
+
trilogy/core/functions.py,sha256=_75wCQgu8XuoKtKN1AMbBJP62YMsKD3IkMaznpPd6kQ,28470
|
|
20
20
|
trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
|
|
21
21
|
trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
|
|
22
22
|
trilogy/core/optimization.py,sha256=O7ag0IVQlJyWdAXBi_hHeU3Df5DRyd75Vlz6pks2J10,8197
|
|
23
23
|
trilogy/core/query_processor.py,sha256=NNzYPKN5HzivQFXugSbJC_MaupkwOYii7A_vnXuBIK4,20063
|
|
24
24
|
trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
-
trilogy/core/models/author.py,sha256=
|
|
26
|
-
trilogy/core/models/build.py,sha256=
|
|
25
|
+
trilogy/core/models/author.py,sha256=x1yuyX4unahnH4JPsEEmtu00UMDHiYF-wGbTycswhb8,77228
|
|
26
|
+
trilogy/core/models/build.py,sha256=PJ2S8uWsOo38K60zh2OfY7dl_WYnnvJCtyqihxClejQ,63218
|
|
27
27
|
trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
|
|
28
28
|
trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
|
|
29
29
|
trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
|
|
30
30
|
trilogy/core/models/environment.py,sha256=AVSrvjNcNX535GhCPtYhCRY2Lp_Hj0tdY3VVt_kZb9Q,27260
|
|
31
|
-
trilogy/core/models/execute.py,sha256=
|
|
31
|
+
trilogy/core/models/execute.py,sha256=6eud57-Cg8biJRTRFzfDYqa6XMB22d1c8rC-LXibUqk,34748
|
|
32
32
|
trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
|
|
33
33
|
trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
|
|
34
34
|
trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
|
|
35
35
|
trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
|
|
36
36
|
trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
|
-
trilogy/core/processing/concept_strategies_v3.py,sha256=
|
|
37
|
+
trilogy/core/processing/concept_strategies_v3.py,sha256=8Wos5d9_tzfnzSbejb36QL4uoGPQ3GiwP27u_a4JrcE,44097
|
|
38
38
|
trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
|
|
39
39
|
trilogy/core/processing/utility.py,sha256=rfzdgl-vWkCyhLzXNNuWgPLK59eiYypQb6TdZKymUqk,21469
|
|
40
40
|
trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
|
|
41
41
|
trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
|
|
42
|
-
trilogy/core/processing/node_generators/common.py,sha256=
|
|
43
|
-
trilogy/core/processing/node_generators/filter_node.py,sha256=
|
|
44
|
-
trilogy/core/processing/node_generators/group_node.py,sha256=
|
|
42
|
+
trilogy/core/processing/node_generators/common.py,sha256=PdysdroW9DUADP7f5Wv_GKPUyCTROZV1g3L45fawxi8,9443
|
|
43
|
+
trilogy/core/processing/node_generators/filter_node.py,sha256=0hdfiS2I-Jvr6P-il3jnAJK-g-DMG7_cFbZGCnLnJAo,10032
|
|
44
|
+
trilogy/core/processing/node_generators/group_node.py,sha256=nIfiMrJQEksUfqAeeA3X5PS1343y4lmPTipYuCa-rvs,6141
|
|
45
45
|
trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
|
|
46
46
|
trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
|
|
47
47
|
trilogy/core/processing/node_generators/node_merge_node.py,sha256=sv55oynfqgpHEpo1OEtVDri-5fywzPhDlR85qaWikvY,16195
|
|
@@ -55,7 +55,7 @@ trilogy/core/processing/node_generators/window_node.py,sha256=RUHgpYovQObFod1xRI
|
|
|
55
55
|
trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
56
56
|
trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
|
|
57
57
|
trilogy/core/processing/nodes/__init__.py,sha256=xPFF7x3TFs1Z4IcfthCykZgrksb-UhN-pc_oIigfFSo,6014
|
|
58
|
-
trilogy/core/processing/nodes/base_node.py,sha256=
|
|
58
|
+
trilogy/core/processing/nodes/base_node.py,sha256=z-aZEVjnLdFm6TpmneEm2bnRXj-tRFr7mN7DYG4zH9A,16967
|
|
59
59
|
trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
|
|
60
60
|
trilogy/core/processing/nodes/group_node.py,sha256=MUvcOg9U5J6TnWBel8eht9PdI9BfAKjUxmfjP_ZXx9o,10484
|
|
61
61
|
trilogy/core/processing/nodes/merge_node.py,sha256=02oWRca0ba41U6PSAB14jwnWWxoyrvxRPLwkli259SY,15865
|
|
@@ -64,17 +64,17 @@ trilogy/core/processing/nodes/union_node.py,sha256=fDFzLAUh5876X6_NM7nkhoMvHEdGJ
|
|
|
64
64
|
trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
|
|
65
65
|
trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
|
|
66
66
|
trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
|
-
trilogy/core/statements/author.py,sha256=
|
|
67
|
+
trilogy/core/statements/author.py,sha256=jCwPXmnjj8u2ytBRAS_NU7ga0uB7k3_TZY6dZSIMl9Y,15253
|
|
68
68
|
trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
69
69
|
trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
|
|
70
70
|
trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
|
|
71
71
|
trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
|
-
trilogy/dialect/base.py,sha256=
|
|
73
|
-
trilogy/dialect/bigquery.py,sha256=
|
|
72
|
+
trilogy/dialect/base.py,sha256=lZup3hq-DyYczpG260a0wBByHyOGkbw4-yrPJvXKOM4,42300
|
|
73
|
+
trilogy/dialect/bigquery.py,sha256=mGnBl5A3rVi4f1gt74jnaxSOCheA07OcRi6ZD8KWOGg,3436
|
|
74
74
|
trilogy/dialect/common.py,sha256=JQ8ONloalaWEXsTTWUhZcYyzMRaZ9HdUw7cN6QWtY5c,5295
|
|
75
75
|
trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
|
|
76
76
|
trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
|
|
77
|
-
trilogy/dialect/duckdb.py,sha256=
|
|
77
|
+
trilogy/dialect/duckdb.py,sha256=IQzaRaCv5c6TUDERhbsLM4uTW0aGkO_DrAMR5k_j7TU,3861
|
|
78
78
|
trilogy/dialect/enums.py,sha256=FRNYQ5-w-B6-X0yXKNU5g9GowsMlERFogTC5u2nxL_s,4740
|
|
79
79
|
trilogy/dialect/postgres.py,sha256=VH4EB4myjIeZTHeFU6vK00GxY9c53rCBjg2mLbdaCEE,3254
|
|
80
80
|
trilogy/dialect/presto.py,sha256=Mw7_F8h19mWfuZHkHQJizQWbpu1lIHe6t2PA0r88gsY,3392
|
|
@@ -86,13 +86,13 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
|
|
|
86
86
|
trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
|
|
87
87
|
trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
88
88
|
trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
89
|
-
trilogy/parsing/common.py,sha256=
|
|
89
|
+
trilogy/parsing/common.py,sha256=hrsK432wf26n3TCWT_D6BWHemcNbj_19gbTyyyc3tVg,29709
|
|
90
90
|
trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
|
|
91
91
|
trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
|
|
92
92
|
trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
93
|
-
trilogy/parsing/parse_engine.py,sha256=
|
|
93
|
+
trilogy/parsing/parse_engine.py,sha256=Fn_XjY-8Igp_jk-IzNwZF5rIEj4PVb3fdNKT3our73I,69750
|
|
94
94
|
trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
|
|
95
|
-
trilogy/parsing/trilogy.lark,sha256=
|
|
95
|
+
trilogy/parsing/trilogy.lark,sha256=ijY6220e2hV21F1XFsvpYRimSrpNGIdjP7b0TVz7caI,13814
|
|
96
96
|
trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
97
97
|
trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
|
|
98
98
|
trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -101,8 +101,8 @@ trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
|
|
|
101
101
|
trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
|
|
102
102
|
trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
|
|
103
103
|
trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
|
|
104
|
-
pytrilogy-0.0.3.
|
|
105
|
-
pytrilogy-0.0.3.
|
|
106
|
-
pytrilogy-0.0.3.
|
|
107
|
-
pytrilogy-0.0.3.
|
|
108
|
-
pytrilogy-0.0.3.
|
|
104
|
+
pytrilogy-0.0.3.51.dist-info/METADATA,sha256=0rBGrA1yLDvWA2CUe6fR_sfSaRn8c5b11qdVoIRGxhE,9095
|
|
105
|
+
pytrilogy-0.0.3.51.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
|
|
106
|
+
pytrilogy-0.0.3.51.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
|
|
107
|
+
pytrilogy-0.0.3.51.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
|
|
108
|
+
pytrilogy-0.0.3.51.dist-info/RECORD,,
|
trilogy/__init__.py
CHANGED
trilogy/core/enums.py
CHANGED
|
@@ -131,6 +131,7 @@ class FunctionType(Enum):
|
|
|
131
131
|
CONSTANT = "constant"
|
|
132
132
|
COALESCE = "coalesce"
|
|
133
133
|
IS_NULL = "isnull"
|
|
134
|
+
NULLIF = "nullif"
|
|
134
135
|
BOOL = "bool"
|
|
135
136
|
|
|
136
137
|
# COMPLEX
|
|
@@ -156,6 +157,8 @@ class FunctionType(Enum):
|
|
|
156
157
|
ABS = "abs"
|
|
157
158
|
SQRT = "sqrt"
|
|
158
159
|
RANDOM = "random"
|
|
160
|
+
FLOOR = "floor"
|
|
161
|
+
CEIL = "ceil"
|
|
159
162
|
|
|
160
163
|
# Aggregates
|
|
161
164
|
## group is not a real aggregate - it just means group by this + some other set of fields
|
trilogy/core/functions.py
CHANGED
|
@@ -279,6 +279,12 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
|
|
|
279
279
|
output_purpose=Purpose.PROPERTY,
|
|
280
280
|
arg_count=1,
|
|
281
281
|
),
|
|
282
|
+
FunctionType.NULLIF: FunctionConfig(
|
|
283
|
+
valid_inputs={*DataType},
|
|
284
|
+
output_purpose=Purpose.PROPERTY,
|
|
285
|
+
output_type_function=lambda args: get_output_type_at_index(args, 0),
|
|
286
|
+
arg_count=2,
|
|
287
|
+
),
|
|
282
288
|
FunctionType.COALESCE: FunctionConfig(
|
|
283
289
|
valid_inputs={*DataType},
|
|
284
290
|
output_purpose=Purpose.PROPERTY,
|
|
@@ -637,6 +643,22 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
|
|
|
637
643
|
output_type_function=lambda args: get_output_type_at_index(args, 0),
|
|
638
644
|
arg_count=2,
|
|
639
645
|
),
|
|
646
|
+
FunctionType.FLOOR: FunctionConfig(
|
|
647
|
+
valid_inputs=[
|
|
648
|
+
{DataType.INTEGER, DataType.FLOAT, DataType.NUMBER, DataType.NUMERIC},
|
|
649
|
+
],
|
|
650
|
+
output_purpose=Purpose.PROPERTY,
|
|
651
|
+
output_type=DataType.INTEGER,
|
|
652
|
+
arg_count=1,
|
|
653
|
+
),
|
|
654
|
+
FunctionType.CEIL: FunctionConfig(
|
|
655
|
+
valid_inputs=[
|
|
656
|
+
{DataType.INTEGER, DataType.FLOAT, DataType.NUMBER, DataType.NUMERIC},
|
|
657
|
+
],
|
|
658
|
+
output_purpose=Purpose.PROPERTY,
|
|
659
|
+
output_type=DataType.INTEGER,
|
|
660
|
+
arg_count=1,
|
|
661
|
+
),
|
|
640
662
|
FunctionType.CUSTOM: FunctionConfig(
|
|
641
663
|
output_purpose=Purpose.PROPERTY,
|
|
642
664
|
arg_count=InfiniteFunctionArgs,
|
trilogy/core/models/author.py
CHANGED
|
@@ -1945,11 +1945,15 @@ class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
|
|
|
1945
1945
|
|
|
1946
1946
|
@property
|
|
1947
1947
|
def output_datatype(self):
|
|
1948
|
-
return self.content
|
|
1948
|
+
return arg_to_datatype(self.content)
|
|
1949
1949
|
|
|
1950
1950
|
@property
|
|
1951
1951
|
def concept_arguments(self):
|
|
1952
|
-
|
|
1952
|
+
if isinstance(self.content, ConceptRef):
|
|
1953
|
+
return [self.content] + self.where.concept_arguments
|
|
1954
|
+
elif isinstance(self.content, ConceptArgs):
|
|
1955
|
+
return self.content.concept_arguments + self.where.concept_arguments
|
|
1956
|
+
return self.where.concept_arguments
|
|
1953
1957
|
|
|
1954
1958
|
|
|
1955
1959
|
class RowsetLineage(Namespaced, Mergeable, BaseModel):
|
trilogy/core/models/build.py
CHANGED
|
@@ -1173,7 +1173,7 @@ class BuildAggregateWrapper(BuildConceptArgs, DataTyped, BaseModel):
|
|
|
1173
1173
|
|
|
1174
1174
|
|
|
1175
1175
|
class BuildFilterItem(BuildConceptArgs, BaseModel):
|
|
1176
|
-
content:
|
|
1176
|
+
content: "BuildExpr"
|
|
1177
1177
|
where: BuildWhereClause
|
|
1178
1178
|
|
|
1179
1179
|
def __str__(self):
|
|
@@ -1181,15 +1181,27 @@ class BuildFilterItem(BuildConceptArgs, BaseModel):
|
|
|
1181
1181
|
|
|
1182
1182
|
@property
|
|
1183
1183
|
def output_datatype(self):
|
|
1184
|
-
return self.content
|
|
1184
|
+
return arg_to_datatype(self.content)
|
|
1185
1185
|
|
|
1186
1186
|
@property
|
|
1187
1187
|
def output_purpose(self):
|
|
1188
1188
|
return self.content.purpose
|
|
1189
1189
|
|
|
1190
|
+
@property
|
|
1191
|
+
def content_concept_arguments(self):
|
|
1192
|
+
if isinstance(self.content, BuildConcept):
|
|
1193
|
+
return [self.content]
|
|
1194
|
+
elif isinstance(self.content, BuildConceptArgs):
|
|
1195
|
+
return self.content.concept_arguments
|
|
1196
|
+
return []
|
|
1197
|
+
|
|
1190
1198
|
@property
|
|
1191
1199
|
def concept_arguments(self):
|
|
1192
|
-
|
|
1200
|
+
if isinstance(self.content, BuildConcept):
|
|
1201
|
+
return [self.content] + self.where.concept_arguments
|
|
1202
|
+
elif isinstance(self.content, BuildConceptArgs):
|
|
1203
|
+
return self.content.concept_arguments + self.where.concept_arguments
|
|
1204
|
+
return self.where.concept_arguments
|
|
1193
1205
|
|
|
1194
1206
|
|
|
1195
1207
|
class BuildRowsetLineage(BuildConceptArgs, BaseModel):
|
trilogy/core/models/execute.py
CHANGED
|
@@ -695,7 +695,7 @@ class QueryDatasource(BaseModel):
|
|
|
695
695
|
"can only merge two datasources if the force_group flag is the same"
|
|
696
696
|
)
|
|
697
697
|
logger.debug(
|
|
698
|
-
f"
|
|
698
|
+
f"[Query Datasource] merging {self.name} with"
|
|
699
699
|
f" {[c.address for c in self.output_concepts]} concepts and"
|
|
700
700
|
f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
|
|
701
701
|
)
|
|
@@ -759,7 +759,9 @@ class QueryDatasource(BaseModel):
|
|
|
759
759
|
hidden_concepts=hidden,
|
|
760
760
|
ordering=self.ordering,
|
|
761
761
|
)
|
|
762
|
-
|
|
762
|
+
logger.debug(
|
|
763
|
+
f"[Query Datasource] merged with {[c.address for c in qds.output_concepts]} concepts"
|
|
764
|
+
)
|
|
763
765
|
return qds
|
|
764
766
|
|
|
765
767
|
@property
|
|
@@ -319,7 +319,7 @@ def generate_node(
|
|
|
319
319
|
]
|
|
320
320
|
|
|
321
321
|
logger.info(
|
|
322
|
-
f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating aggregate node with {[x
|
|
322
|
+
f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating aggregate node with {[x for x in agg_optional]}"
|
|
323
323
|
)
|
|
324
324
|
return gen_group_node(
|
|
325
325
|
concept,
|
|
@@ -67,14 +67,14 @@ def resolve_condition_parent_concepts(
|
|
|
67
67
|
def resolve_filter_parent_concepts(
|
|
68
68
|
concept: BuildConcept,
|
|
69
69
|
environment: BuildEnvironment,
|
|
70
|
-
) -> Tuple[
|
|
70
|
+
) -> Tuple[List[BuildConcept], List[Tuple[BuildConcept, ...]]]:
|
|
71
71
|
if not isinstance(concept.lineage, (BuildFilterItem,)):
|
|
72
72
|
raise ValueError(
|
|
73
73
|
f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
|
|
74
74
|
)
|
|
75
75
|
direct_parent = concept.lineage.content
|
|
76
76
|
base_existence = []
|
|
77
|
-
base_rows = [direct_parent]
|
|
77
|
+
base_rows = [direct_parent] if isinstance(direct_parent, BuildConcept) else []
|
|
78
78
|
condition_rows, condition_existence = resolve_condition_parent_concepts(
|
|
79
79
|
concept.lineage.where
|
|
80
80
|
)
|
|
@@ -90,11 +90,10 @@ def resolve_filter_parent_concepts(
|
|
|
90
90
|
|
|
91
91
|
if concept.lineage.where.existence_arguments:
|
|
92
92
|
return (
|
|
93
|
-
concept.lineage.content,
|
|
94
93
|
unique(base_rows, "address"),
|
|
95
94
|
base_existence,
|
|
96
95
|
)
|
|
97
|
-
return
|
|
96
|
+
return unique(base_rows, "address"), []
|
|
98
97
|
|
|
99
98
|
|
|
100
99
|
def gen_property_enrichment_node(
|
|
@@ -25,71 +25,140 @@ LOGGER_PREFIX = "[GEN_FILTER_NODE]"
|
|
|
25
25
|
FILTER_TYPES = (BuildFilterItem,)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
def
|
|
29
|
-
concept: BuildConcept,
|
|
28
|
+
def pushdown_filter_to_parent(
|
|
30
29
|
local_optional: List[BuildConcept],
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
conditions: BuildWhereClause | None,
|
|
31
|
+
filter_where: BuildWhereClause,
|
|
32
|
+
same_filter_optional: list[BuildConcept],
|
|
33
33
|
depth: int,
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
) -> bool:
|
|
35
|
+
optimized_pushdown = False
|
|
36
|
+
if not is_scalar_condition(filter_where.conditional):
|
|
37
|
+
optimized_pushdown = False
|
|
38
|
+
elif not local_optional:
|
|
39
|
+
optimized_pushdown = True
|
|
40
|
+
elif conditions and conditions == filter_where:
|
|
41
|
+
logger.info(
|
|
42
|
+
f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
|
|
43
|
+
)
|
|
44
|
+
optimized_pushdown = True
|
|
45
|
+
elif same_filter_optional == local_optional:
|
|
46
|
+
logger.info(
|
|
47
|
+
f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
|
|
48
|
+
)
|
|
49
|
+
optimized_pushdown = True
|
|
50
|
+
|
|
51
|
+
return optimized_pushdown
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def build_parent_concepts(
|
|
55
|
+
concept: BuildConcept,
|
|
56
|
+
environment: BuildEnvironment,
|
|
57
|
+
local_optional: List[BuildConcept],
|
|
36
58
|
conditions: BuildWhereClause | None = None,
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
59
|
+
depth: int = 0,
|
|
60
|
+
):
|
|
61
|
+
parent_row_concepts, parent_existence_concepts = resolve_filter_parent_concepts(
|
|
62
|
+
concept, environment
|
|
40
63
|
)
|
|
41
64
|
if not isinstance(concept.lineage, FILTER_TYPES):
|
|
42
65
|
raise SyntaxError('Filter node must have a filter type lineage"')
|
|
43
|
-
|
|
66
|
+
filter_where = concept.lineage.where
|
|
44
67
|
|
|
45
|
-
|
|
68
|
+
same_filter_optional: list[BuildConcept] = []
|
|
46
69
|
|
|
47
70
|
for x in local_optional:
|
|
48
71
|
if isinstance(x.lineage, FILTER_TYPES):
|
|
49
|
-
if concept.lineage.where ==
|
|
72
|
+
if concept.lineage.where == filter_where:
|
|
50
73
|
logger.info(
|
|
51
|
-
f"{padding(depth)}{LOGGER_PREFIX} fetching
|
|
74
|
+
f"{padding(depth)}{LOGGER_PREFIX} fetching parents for peer {x} with same filter conditions"
|
|
52
75
|
)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
76
|
+
|
|
77
|
+
for arg in x.lineage.content_concept_arguments:
|
|
78
|
+
if arg.address not in parent_row_concepts:
|
|
79
|
+
parent_row_concepts.append(arg)
|
|
80
|
+
same_filter_optional.append(x)
|
|
56
81
|
continue
|
|
57
|
-
|
|
58
|
-
|
|
82
|
+
elif conditions and conditions == filter_where:
|
|
83
|
+
same_filter_optional.append(x)
|
|
59
84
|
|
|
60
85
|
# sometimes, it's okay to include other local optional above the filter
|
|
61
86
|
# in case it is, prep our list
|
|
62
87
|
extra_row_level_optional: list[BuildConcept] = []
|
|
88
|
+
|
|
63
89
|
for x in local_optional:
|
|
64
|
-
if x.address in
|
|
90
|
+
if x.address in same_filter_optional:
|
|
65
91
|
continue
|
|
66
92
|
extra_row_level_optional.append(x)
|
|
93
|
+
is_optimized_pushdown = pushdown_filter_to_parent(
|
|
94
|
+
local_optional, conditions, filter_where, same_filter_optional, depth
|
|
95
|
+
)
|
|
96
|
+
if not is_optimized_pushdown:
|
|
97
|
+
parent_row_concepts += extra_row_level_optional
|
|
98
|
+
return (
|
|
99
|
+
parent_row_concepts,
|
|
100
|
+
parent_existence_concepts,
|
|
101
|
+
same_filter_optional,
|
|
102
|
+
is_optimized_pushdown,
|
|
103
|
+
)
|
|
67
104
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
105
|
+
|
|
106
|
+
def add_existence_sources(
|
|
107
|
+
core_parent_nodes: list[StrategyNode],
|
|
108
|
+
parent_existence_concepts: list[tuple[BuildConcept, ...]],
|
|
109
|
+
source_concepts,
|
|
110
|
+
environment,
|
|
111
|
+
g,
|
|
112
|
+
depth,
|
|
113
|
+
history,
|
|
114
|
+
):
|
|
115
|
+
for existence_tuple in parent_existence_concepts:
|
|
116
|
+
if not existence_tuple:
|
|
117
|
+
continue
|
|
76
118
|
logger.info(
|
|
77
|
-
f"{padding(depth)}{LOGGER_PREFIX}
|
|
119
|
+
f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in existence_tuple]}"
|
|
78
120
|
)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
121
|
+
parent_existence = source_concepts(
|
|
122
|
+
mandatory_list=list(existence_tuple),
|
|
123
|
+
environment=environment,
|
|
124
|
+
g=g,
|
|
125
|
+
depth=depth + 1,
|
|
126
|
+
history=history,
|
|
83
127
|
)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
128
|
+
if not parent_existence:
|
|
129
|
+
logger.info(
|
|
130
|
+
f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
|
|
131
|
+
)
|
|
132
|
+
return None
|
|
133
|
+
core_parent_nodes.append(parent_existence)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def gen_filter_node(
|
|
137
|
+
concept: BuildConcept,
|
|
138
|
+
local_optional: List[BuildConcept],
|
|
139
|
+
environment: BuildEnvironment,
|
|
140
|
+
g,
|
|
141
|
+
depth: int,
|
|
142
|
+
source_concepts,
|
|
143
|
+
history: History | None = None,
|
|
144
|
+
conditions: BuildWhereClause | None = None,
|
|
145
|
+
) -> StrategyNode | None:
|
|
146
|
+
if not isinstance(concept.lineage, FILTER_TYPES):
|
|
147
|
+
raise SyntaxError('Filter node must have a filter type lineage"')
|
|
148
|
+
where = concept.lineage.where
|
|
149
|
+
|
|
150
|
+
(
|
|
151
|
+
parent_row_concepts,
|
|
152
|
+
parent_existence_concepts,
|
|
153
|
+
same_filter_optional,
|
|
154
|
+
optimized_pushdown,
|
|
155
|
+
) = build_parent_concepts(
|
|
156
|
+
concept,
|
|
157
|
+
environment=environment,
|
|
158
|
+
local_optional=local_optional,
|
|
159
|
+
conditions=conditions,
|
|
160
|
+
depth=depth,
|
|
87
161
|
)
|
|
88
|
-
# we'll populate this with the row parent
|
|
89
|
-
# and the existence parent(s)
|
|
90
|
-
core_parents = []
|
|
91
|
-
if not optimized_pushdown:
|
|
92
|
-
parent_row_concepts += extra_row_level_optional
|
|
93
162
|
|
|
94
163
|
row_parent: StrategyNode = source_concepts(
|
|
95
164
|
mandatory_list=parent_row_concepts,
|
|
@@ -100,27 +169,19 @@ def gen_filter_node(
|
|
|
100
169
|
conditions=conditions,
|
|
101
170
|
)
|
|
102
171
|
|
|
172
|
+
core_parent_nodes: list[StrategyNode] = []
|
|
103
173
|
flattened_existence = [x for y in parent_existence_concepts for x in y]
|
|
104
174
|
if parent_existence_concepts:
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
depth=depth + 1,
|
|
116
|
-
history=history,
|
|
117
|
-
)
|
|
118
|
-
if not parent_existence:
|
|
119
|
-
logger.info(
|
|
120
|
-
f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
|
|
121
|
-
)
|
|
122
|
-
return None
|
|
123
|
-
core_parents.append(parent_existence)
|
|
175
|
+
add_existence_sources(
|
|
176
|
+
core_parent_nodes,
|
|
177
|
+
parent_existence_concepts,
|
|
178
|
+
source_concepts,
|
|
179
|
+
environment,
|
|
180
|
+
g,
|
|
181
|
+
depth,
|
|
182
|
+
history,
|
|
183
|
+
)
|
|
184
|
+
|
|
124
185
|
if not row_parent:
|
|
125
186
|
logger.info(
|
|
126
187
|
f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
|
|
@@ -129,7 +190,7 @@ def gen_filter_node(
|
|
|
129
190
|
|
|
130
191
|
if optimized_pushdown:
|
|
131
192
|
logger.info(
|
|
132
|
-
f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
|
|
193
|
+
f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional} across {[concept] + same_filter_optional + row_parent.output_concepts} "
|
|
133
194
|
)
|
|
134
195
|
if isinstance(row_parent, SelectNode):
|
|
135
196
|
logger.info(
|
|
@@ -137,7 +198,9 @@ def gen_filter_node(
|
|
|
137
198
|
)
|
|
138
199
|
parent = StrategyNode(
|
|
139
200
|
input_concepts=row_parent.output_concepts,
|
|
140
|
-
output_concepts=[concept]
|
|
201
|
+
output_concepts=[concept]
|
|
202
|
+
+ same_filter_optional
|
|
203
|
+
+ row_parent.output_concepts,
|
|
141
204
|
environment=row_parent.environment,
|
|
142
205
|
parents=[row_parent],
|
|
143
206
|
depth=row_parent.depth,
|
|
@@ -146,46 +209,34 @@ def gen_filter_node(
|
|
|
146
209
|
)
|
|
147
210
|
else:
|
|
148
211
|
parent = row_parent
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
x
|
|
152
|
-
for x in local_optional
|
|
153
|
-
if x.address in [y for y in parent.output_concepts]
|
|
154
|
-
or x.address in [y for y in optional_included]
|
|
155
|
-
]
|
|
156
|
-
parent.add_parents(core_parents)
|
|
212
|
+
parent.add_output_concepts([concept] + same_filter_optional)
|
|
213
|
+
parent.add_parents(core_parent_nodes)
|
|
157
214
|
parent.add_condition(where.conditional)
|
|
158
|
-
parent.add_existence_concepts(flattened_existence, False)
|
|
159
|
-
expected_output, False
|
|
160
|
-
)
|
|
215
|
+
parent.add_existence_concepts(flattened_existence, False)
|
|
161
216
|
parent.grain = BuildGrain.from_concepts(
|
|
162
|
-
|
|
163
|
-
[environment.concepts[k] for k in immediate_parent.keys]
|
|
164
|
-
if immediate_parent.keys
|
|
165
|
-
else [immediate_parent]
|
|
166
|
-
)
|
|
167
|
-
+ [
|
|
168
|
-
x
|
|
169
|
-
for x in local_optional
|
|
170
|
-
if x.address in [y.address for y in parent.output_concepts]
|
|
171
|
-
],
|
|
217
|
+
parent.output_concepts,
|
|
172
218
|
environment=environment,
|
|
173
219
|
)
|
|
174
220
|
parent.rebuild_cache()
|
|
175
221
|
filter_node = parent
|
|
176
222
|
else:
|
|
177
|
-
|
|
178
|
-
|
|
223
|
+
core_parent_nodes.append(row_parent)
|
|
224
|
+
filters = [concept] + same_filter_optional
|
|
225
|
+
parents_for_grain = [
|
|
226
|
+
x.lineage.content
|
|
227
|
+
for x in filters
|
|
228
|
+
if isinstance(x.lineage.content, BuildConcept)
|
|
229
|
+
]
|
|
179
230
|
filter_node = FilterNode(
|
|
180
231
|
input_concepts=unique(
|
|
181
|
-
|
|
232
|
+
parent_row_concepts + flattened_existence,
|
|
182
233
|
"address",
|
|
183
234
|
),
|
|
184
|
-
output_concepts=[concept
|
|
235
|
+
output_concepts=[concept] + same_filter_optional + parent_row_concepts,
|
|
185
236
|
environment=environment,
|
|
186
|
-
parents=
|
|
237
|
+
parents=core_parent_nodes,
|
|
187
238
|
grain=BuildGrain.from_concepts(
|
|
188
|
-
|
|
239
|
+
parents_for_grain + parent_row_concepts, environment=environment
|
|
189
240
|
),
|
|
190
241
|
preexisting_conditions=conditions.conditional if conditions else None,
|
|
191
242
|
)
|
|
@@ -211,7 +262,8 @@ def gen_filter_node(
|
|
|
211
262
|
)
|
|
212
263
|
enrich_node: StrategyNode = source_concepts( # this fetches the parent + join keys
|
|
213
264
|
# to then connect to the rest of the query
|
|
214
|
-
mandatory_list=
|
|
265
|
+
mandatory_list=parent_row_concepts
|
|
266
|
+
+ [x for x in local_optional if x.address not in filter_node.output_concepts],
|
|
215
267
|
environment=environment,
|
|
216
268
|
g=g,
|
|
217
269
|
depth=depth + 1,
|
|
@@ -227,14 +279,13 @@ def gen_filter_node(
|
|
|
227
279
|
f"{padding(depth)}{LOGGER_PREFIX} returning filter node and enrich node with {enrich_node.output_concepts} and {enrich_node.input_concepts}"
|
|
228
280
|
)
|
|
229
281
|
return MergeNode(
|
|
230
|
-
input_concepts=
|
|
282
|
+
input_concepts=filter_node.output_concepts + enrich_node.output_concepts,
|
|
231
283
|
output_concepts=[
|
|
232
284
|
concept,
|
|
233
285
|
]
|
|
234
286
|
+ local_optional,
|
|
235
287
|
environment=environment,
|
|
236
288
|
parents=[
|
|
237
|
-
# this node fetches only what we need to filter
|
|
238
289
|
filter_node,
|
|
239
290
|
enrich_node,
|
|
240
291
|
],
|
|
@@ -51,6 +51,7 @@ def gen_group_node(
|
|
|
51
51
|
):
|
|
52
52
|
grain_components = [environment.concepts[c] for c in concept.grain.components]
|
|
53
53
|
parent_concepts += grain_components
|
|
54
|
+
build_grain_parents = BuildGrain.from_concepts(parent_concepts)
|
|
54
55
|
output_concepts += grain_components
|
|
55
56
|
for possible_agg in local_optional:
|
|
56
57
|
|
|
@@ -76,9 +77,7 @@ def gen_group_node(
|
|
|
76
77
|
logger.info(
|
|
77
78
|
f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
|
|
78
79
|
)
|
|
79
|
-
elif BuildGrain.from_concepts(agg_parents) ==
|
|
80
|
-
parent_concepts
|
|
81
|
-
):
|
|
80
|
+
elif BuildGrain.from_concepts(agg_parents) == build_grain_parents:
|
|
82
81
|
extra = [x for x in agg_parents if x.address not in parent_concepts]
|
|
83
82
|
parent_concepts += extra
|
|
84
83
|
output_concepts.append(possible_agg)
|
|
@@ -87,7 +86,7 @@ def gen_group_node(
|
|
|
87
86
|
)
|
|
88
87
|
else:
|
|
89
88
|
logger.info(
|
|
90
|
-
f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; mismatched grain {BuildGrain.from_concepts(agg_parents)} vs {BuildGrain.from_concepts(parent_concepts)}"
|
|
89
|
+
f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; mismatched parent grain {BuildGrain.from_concepts(agg_parents)} vs local parent {BuildGrain.from_concepts(parent_concepts)}"
|
|
91
90
|
)
|
|
92
91
|
if parent_concepts:
|
|
93
92
|
logger.info(
|
|
@@ -194,15 +194,18 @@ class StrategyNode:
|
|
|
194
194
|
if not self.parents:
|
|
195
195
|
return
|
|
196
196
|
non_hidden = set()
|
|
197
|
+
hidden = set()
|
|
197
198
|
for x in self.parents:
|
|
198
199
|
for z in x.usable_outputs:
|
|
199
200
|
non_hidden.add(z.address)
|
|
200
201
|
for psd in z.pseudonyms:
|
|
201
202
|
non_hidden.add(psd)
|
|
203
|
+
for z in x.hidden_concepts:
|
|
204
|
+
hidden.add(z)
|
|
202
205
|
if not all([x.address in non_hidden for x in self.input_concepts]):
|
|
203
206
|
missing = [x for x in self.input_concepts if x.address not in non_hidden]
|
|
204
207
|
raise ValueError(
|
|
205
|
-
f"Invalid input concepts; {missing} are missing non-hidden parent nodes"
|
|
208
|
+
f"Invalid input concepts; {missing} are missing non-hidden parent nodes; have {non_hidden} and hidden {hidden}"
|
|
206
209
|
)
|
|
207
210
|
|
|
208
211
|
def add_parents(self, parents: list["StrategyNode"]):
|
|
@@ -162,9 +162,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
|
|
|
162
162
|
output.local_concepts[x.content.address] = environment.concepts[
|
|
163
163
|
x.content.address
|
|
164
164
|
]
|
|
165
|
-
|
|
166
165
|
output.grain = output.calculate_grain(environment, output.local_concepts)
|
|
167
|
-
|
|
168
166
|
output.validate_syntax(environment)
|
|
169
167
|
return output
|
|
170
168
|
|
trilogy/dialect/base.py
CHANGED
|
@@ -161,6 +161,7 @@ FUNCTION_MAP = {
|
|
|
161
161
|
FunctionType.GROUP: lambda x: f"{x[0]}",
|
|
162
162
|
FunctionType.CONSTANT: lambda x: f"{x[0]}",
|
|
163
163
|
FunctionType.COALESCE: lambda x: f"coalesce({','.join(x)})",
|
|
164
|
+
FunctionType.NULLIF: lambda x: f"nullif({x[0]},{x[1]})",
|
|
164
165
|
FunctionType.CAST: lambda x: f"cast({x[0]} as {x[1]})",
|
|
165
166
|
FunctionType.CASE: lambda x: render_case(x),
|
|
166
167
|
FunctionType.SPLIT: lambda x: f"split({x[0]}, {x[1]})",
|
|
@@ -183,6 +184,8 @@ FUNCTION_MAP = {
|
|
|
183
184
|
FunctionType.DIVIDE: lambda x: " / ".join(x),
|
|
184
185
|
FunctionType.MULTIPLY: lambda x: " * ".join(x),
|
|
185
186
|
FunctionType.ROUND: lambda x: f"round({x[0]},{x[1]})",
|
|
187
|
+
FunctionType.FLOOR: lambda x: f"floor({x[0]})",
|
|
188
|
+
FunctionType.CEIL: lambda x: f"ceil({x[0]})",
|
|
186
189
|
FunctionType.MOD: lambda x: f"({x[0]} % {x[1]})",
|
|
187
190
|
FunctionType.SQRT: lambda x: f"sqrt({x[0]})",
|
|
188
191
|
FunctionType.RANDOM: lambda x: "random()",
|
|
@@ -400,9 +403,11 @@ class BaseDialect:
|
|
|
400
403
|
elif isinstance(c.lineage, FILTER_ITEMS):
|
|
401
404
|
# for cases when we've optimized this
|
|
402
405
|
if cte.condition == c.lineage.where.conditional:
|
|
403
|
-
rval = self.render_expr(
|
|
406
|
+
rval = self.render_expr(
|
|
407
|
+
c.lineage.content, cte=cte, raise_invalid=raise_invalid
|
|
408
|
+
)
|
|
404
409
|
else:
|
|
405
|
-
rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.
|
|
410
|
+
rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_expr(c.lineage.content, cte=cte, raise_invalid=raise_invalid)} ELSE NULL END"
|
|
406
411
|
elif isinstance(c.lineage, BuildRowsetItem):
|
|
407
412
|
rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)}"
|
|
408
413
|
elif isinstance(c.lineage, BuildMultiSelectLineage):
|
trilogy/dialect/bigquery.py
CHANGED
|
@@ -30,6 +30,8 @@ FUNCTION_MAP = {
|
|
|
30
30
|
# math
|
|
31
31
|
FunctionType.DIVIDE: lambda x: f"COALESCE(SAFE_DIVIDE({x[0]},{x[1]}),0)",
|
|
32
32
|
FunctionType.DATE_ADD: lambda x: f"DATE_ADD({x[0]}, INTERVAL {x[2]} {x[1]})",
|
|
33
|
+
# string
|
|
34
|
+
FunctionType.CONTAINS: lambda x: f"CONTAINS_SUBSTR({x[0]}, {x[1]})",
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
FUNCTION_GRAIN_MATCH_MAP = {
|
trilogy/dialect/duckdb.py
CHANGED
|
@@ -35,6 +35,8 @@ FUNCTION_MAP = {
|
|
|
35
35
|
FunctionType.CONCAT: lambda x: f"({' || '.join(x)})",
|
|
36
36
|
FunctionType.DATE_LITERAL: lambda x: f"date '{x}'",
|
|
37
37
|
FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
|
|
38
|
+
# string
|
|
39
|
+
FunctionType.CONTAINS: lambda x: f"CONTAINS(LOWER({x[0]}), LOWER({x[1]}))",
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
# if an aggregate function is called on a source that is at the same grain as the aggregate
|
trilogy/parsing/common.py
CHANGED
|
@@ -281,28 +281,38 @@ def concepts_to_grain_concepts(
|
|
|
281
281
|
environment: Environment | None,
|
|
282
282
|
local_concepts: dict[str, Concept] | None = None,
|
|
283
283
|
) -> list[Concept]:
|
|
284
|
-
|
|
284
|
+
preconcepts: list[Concept] = []
|
|
285
285
|
for c in concepts:
|
|
286
286
|
if isinstance(c, Concept):
|
|
287
|
-
|
|
287
|
+
preconcepts.append(c)
|
|
288
|
+
|
|
288
289
|
elif isinstance(c, ConceptRef) and environment:
|
|
289
290
|
if local_concepts and c.address in local_concepts:
|
|
290
|
-
|
|
291
|
+
preconcepts.append(local_concepts[c.address])
|
|
291
292
|
else:
|
|
292
|
-
|
|
293
|
+
preconcepts.append(environment.concepts[c.address])
|
|
293
294
|
elif isinstance(c, str) and environment:
|
|
294
295
|
if local_concepts and c in local_concepts:
|
|
295
|
-
|
|
296
|
+
preconcepts.append(local_concepts[c])
|
|
296
297
|
else:
|
|
297
|
-
|
|
298
|
+
preconcepts.append(environment.concepts[c])
|
|
298
299
|
else:
|
|
299
300
|
raise ValueError(
|
|
300
301
|
f"Unable to resolve input {c} without environment provided to concepts_to_grain call"
|
|
301
302
|
)
|
|
302
|
-
|
|
303
|
+
pconcepts = []
|
|
304
|
+
for x in preconcepts:
|
|
305
|
+
if (
|
|
306
|
+
x.lineage
|
|
307
|
+
and isinstance(x.lineage, Function)
|
|
308
|
+
and x.lineage.operator == FunctionType.ALIAS
|
|
309
|
+
):
|
|
310
|
+
# if the function is an alias, use the unaliased concept to calculate grain
|
|
311
|
+
pconcepts.append(environment.concepts[x.lineage.arguments[0].address]) # type: ignore
|
|
312
|
+
else:
|
|
313
|
+
pconcepts.append(x)
|
|
303
314
|
final: List[Concept] = []
|
|
304
315
|
for sub in pconcepts:
|
|
305
|
-
|
|
306
316
|
if not concept_is_relevant(sub, pconcepts, environment): # type: ignore
|
|
307
317
|
continue
|
|
308
318
|
final.append(sub)
|
|
@@ -515,8 +525,10 @@ def filter_item_to_concept(
|
|
|
515
525
|
metadata: Metadata | None = None,
|
|
516
526
|
) -> Concept:
|
|
517
527
|
fmetadata = metadata or Metadata()
|
|
528
|
+
fallback_keys = set()
|
|
518
529
|
if isinstance(parent.content, ConceptRef):
|
|
519
530
|
cparent = environment.concepts[parent.content.address]
|
|
531
|
+
fallback_keys = set([cparent.address])
|
|
520
532
|
elif isinstance(
|
|
521
533
|
parent.content,
|
|
522
534
|
(
|
|
@@ -527,9 +539,13 @@ def filter_item_to_concept(
|
|
|
527
539
|
Function,
|
|
528
540
|
ListWrapper,
|
|
529
541
|
MapWrapper,
|
|
542
|
+
int,
|
|
543
|
+
str,
|
|
544
|
+
float,
|
|
530
545
|
),
|
|
531
546
|
):
|
|
532
547
|
cparent = arbitrary_to_concept(parent.content, environment, namespace=namespace)
|
|
548
|
+
|
|
533
549
|
else:
|
|
534
550
|
raise NotImplementedError(
|
|
535
551
|
f"Filter item with non ref content {parent.content} not yet supported"
|
|
@@ -547,13 +563,7 @@ def filter_item_to_concept(
|
|
|
547
563
|
metadata=fmetadata,
|
|
548
564
|
namespace=namespace,
|
|
549
565
|
# filtered copies cannot inherit keys
|
|
550
|
-
keys=(
|
|
551
|
-
cparent.keys
|
|
552
|
-
if cparent.purpose == Purpose.PROPERTY
|
|
553
|
-
else {
|
|
554
|
-
cparent.address,
|
|
555
|
-
}
|
|
556
|
-
),
|
|
566
|
+
keys=(cparent.keys if cparent.purpose == Purpose.PROPERTY else fallback_keys),
|
|
557
567
|
grain=grain,
|
|
558
568
|
modifiers=modifiers,
|
|
559
569
|
derivation=Derivation.FILTER,
|
trilogy/parsing/parse_engine.py
CHANGED
|
@@ -190,16 +190,25 @@ def parse_concept_reference(
|
|
|
190
190
|
|
|
191
191
|
def expr_to_boolean(
|
|
192
192
|
root,
|
|
193
|
+
function_factory: FunctionFactory,
|
|
193
194
|
) -> Union[Comparison, SubselectComparison, Conditional]:
|
|
194
195
|
if not isinstance(root, (Comparison, SubselectComparison, Conditional)):
|
|
195
196
|
if arg_to_datatype(root) == DataType.BOOL:
|
|
196
197
|
root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
|
|
198
|
+
elif arg_to_datatype(root) == DataType.INTEGER:
|
|
199
|
+
root = Comparison(
|
|
200
|
+
left=function_factory.create_function(
|
|
201
|
+
[root],
|
|
202
|
+
FunctionType.BOOL,
|
|
203
|
+
),
|
|
204
|
+
right=True,
|
|
205
|
+
operator=ComparisonOperator.EQ,
|
|
206
|
+
)
|
|
197
207
|
else:
|
|
198
208
|
root = Comparison(
|
|
199
|
-
left=root,
|
|
200
|
-
right=MagicConstants.NULL,
|
|
201
|
-
operator=ComparisonOperator.IS_NOT,
|
|
209
|
+
left=root, right=NULL_VALUE, operator=ComparisonOperator.IS_NOT
|
|
202
210
|
)
|
|
211
|
+
|
|
203
212
|
return root
|
|
204
213
|
|
|
205
214
|
|
|
@@ -1271,7 +1280,7 @@ class ParseToObjects(Transformer):
|
|
|
1271
1280
|
|
|
1272
1281
|
def where(self, args):
|
|
1273
1282
|
root = args[0]
|
|
1274
|
-
root = expr_to_boolean(root)
|
|
1283
|
+
root = expr_to_boolean(root, self.function_factory)
|
|
1275
1284
|
return WhereClause(conditional=root)
|
|
1276
1285
|
|
|
1277
1286
|
def having(self, args):
|
|
@@ -1490,7 +1499,14 @@ class ParseToObjects(Transformer):
|
|
|
1490
1499
|
def parenthetical(self, args):
|
|
1491
1500
|
return Parenthetical(content=args[0])
|
|
1492
1501
|
|
|
1493
|
-
|
|
1502
|
+
@v_args(meta=True)
|
|
1503
|
+
def condition_parenthetical(self, meta, args):
|
|
1504
|
+
if len(args) == 2:
|
|
1505
|
+
return Comparison(
|
|
1506
|
+
left=Parenthetical(content=args[1]),
|
|
1507
|
+
right=False,
|
|
1508
|
+
operator=ComparisonOperator.EQ,
|
|
1509
|
+
)
|
|
1494
1510
|
return Parenthetical(content=args[0])
|
|
1495
1511
|
|
|
1496
1512
|
def conditional(self, args):
|
|
@@ -1573,7 +1589,7 @@ class ParseToObjects(Transformer):
|
|
|
1573
1589
|
if isinstance(raw, WhereClause):
|
|
1574
1590
|
where = raw
|
|
1575
1591
|
else:
|
|
1576
|
-
where = WhereClause(conditional=raw)
|
|
1592
|
+
where = WhereClause(conditional=expr_to_boolean(raw, self.function_factory))
|
|
1577
1593
|
if isinstance(expr, str):
|
|
1578
1594
|
expr = self.environment.concepts[expr].reference
|
|
1579
1595
|
return FilterItem(content=expr, where=where)
|
|
@@ -1632,6 +1648,10 @@ class ParseToObjects(Transformer):
|
|
|
1632
1648
|
def fcoalesce(self, meta, args):
|
|
1633
1649
|
return self.function_factory.create_function(args, FunctionType.COALESCE, meta)
|
|
1634
1650
|
|
|
1651
|
+
@v_args(meta=True)
|
|
1652
|
+
def fnullif(self, meta, args):
|
|
1653
|
+
return self.function_factory.create_function(args, FunctionType.NULLIF, meta)
|
|
1654
|
+
|
|
1635
1655
|
@v_args(meta=True)
|
|
1636
1656
|
def unnest(self, meta, args):
|
|
1637
1657
|
return self.function_factory.create_function(args, FunctionType.UNNEST, meta)
|
|
@@ -1861,6 +1881,14 @@ class ParseToObjects(Transformer):
|
|
|
1861
1881
|
args.append(0)
|
|
1862
1882
|
return self.function_factory.create_function(args, FunctionType.ROUND, meta)
|
|
1863
1883
|
|
|
1884
|
+
@v_args(meta=True)
|
|
1885
|
+
def ffloor(self, meta, args) -> Function:
|
|
1886
|
+
return self.function_factory.create_function(args, FunctionType.FLOOR, meta)
|
|
1887
|
+
|
|
1888
|
+
@v_args(meta=True)
|
|
1889
|
+
def fceil(self, meta, args) -> Function:
|
|
1890
|
+
return self.function_factory.create_function(args, FunctionType.CEIL, meta)
|
|
1891
|
+
|
|
1864
1892
|
@v_args(meta=True)
|
|
1865
1893
|
def fcase(self, meta, args: List[Union[CaseWhen, CaseElse]]) -> Function:
|
|
1866
1894
|
return self.function_factory.create_function(args, FunctionType.CASE, meta)
|
|
@@ -1868,7 +1896,7 @@ class ParseToObjects(Transformer):
|
|
|
1868
1896
|
@v_args(meta=True)
|
|
1869
1897
|
def fcase_when(self, meta, args) -> CaseWhen:
|
|
1870
1898
|
args = process_function_args(args, meta=meta, environment=self.environment)
|
|
1871
|
-
root = expr_to_boolean(args[0])
|
|
1899
|
+
root = expr_to_boolean(args[0], self.function_factory)
|
|
1872
1900
|
return CaseWhen(comparison=root, expr=args[1])
|
|
1873
1901
|
|
|
1874
1902
|
@v_args(meta=True)
|
trilogy/parsing/trilogy.lark
CHANGED
|
@@ -99,7 +99,7 @@
|
|
|
99
99
|
type_declaration: "type" IDENTIFIER data_type
|
|
100
100
|
|
|
101
101
|
// user_id where state = Mexico
|
|
102
|
-
_filter_alt: (IDENTIFIER | "(" expr ")") "?" conditional
|
|
102
|
+
_filter_alt: (IDENTIFIER | literal | "(" expr ")") "?" conditional
|
|
103
103
|
_filter_base: "filter"i IDENTIFIER where
|
|
104
104
|
filter_item: _filter_base | _filter_alt
|
|
105
105
|
|
|
@@ -146,7 +146,8 @@
|
|
|
146
146
|
_and_condition: _condition_unit
|
|
147
147
|
| (_and_condition LOGICAL_AND _condition_unit)
|
|
148
148
|
|
|
149
|
-
|
|
149
|
+
CONDITION_NOT: "NOT"i
|
|
150
|
+
condition_parenthetical: CONDITION_NOT? "(" conditional ")"
|
|
150
151
|
|
|
151
152
|
_condition_unit: expr
|
|
152
153
|
| condition_parenthetical
|
|
@@ -221,13 +222,17 @@
|
|
|
221
222
|
fmod: ( "mod"i "(" expr "," (int_lit | concept_lit ) ")")
|
|
222
223
|
_ROUND.1: "round"i "("
|
|
223
224
|
fround: _ROUND expr ("," expr)? ")"
|
|
225
|
+
_FLOOR.1: "floor"i "("
|
|
226
|
+
ffloor: _FLOOR expr ")"
|
|
227
|
+
_CEIL.1: "ceil"i "("
|
|
228
|
+
fceil: _CEIL expr ")"
|
|
224
229
|
fabs: "abs"i "(" expr ")"
|
|
225
230
|
_SQRT.1: "sqrt("
|
|
226
231
|
fsqrt: _SQRT expr ")"
|
|
227
232
|
_RANDOM.1: "random("i
|
|
228
233
|
frandom: _RANDOM expr ")"
|
|
229
234
|
|
|
230
|
-
_math_functions: fmul | fdiv | fadd | fsub | fround | fmod | fabs | fsqrt | frandom
|
|
235
|
+
_math_functions: fmul | fdiv | fadd | fsub | fround | ffloor | fceil | fmod | fabs | fsqrt | frandom
|
|
231
236
|
|
|
232
237
|
//generic
|
|
233
238
|
_fcast_primary: "cast"i "(" expr "as"i data_type ")"
|
|
@@ -241,8 +246,9 @@
|
|
|
241
246
|
len: "len"i "(" expr ")"
|
|
242
247
|
fnot: "NOT"i expr
|
|
243
248
|
fbool: "bool"i "(" expr ")"
|
|
249
|
+
fnullif: "nullif"i "(" expr "," expr ")"
|
|
244
250
|
|
|
245
|
-
_generic_functions: fcast | concat | fcoalesce | fcase | len | fnot | fbool
|
|
251
|
+
_generic_functions: fcast | concat | fcoalesce | fnullif | fcase | len | fnot | fbool
|
|
246
252
|
|
|
247
253
|
//constant
|
|
248
254
|
CURRENT_DATE.1: /current_date\(\)/
|
|
File without changes
|
|
File without changes
|
|
File without changes
|