pytrilogy 0.0.2.13__py3-none-any.whl → 0.0.2.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pytrilogy might be problematic. Click here for more details.
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/METADATA +1 -1
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/RECORD +28 -28
- trilogy/__init__.py +1 -1
- trilogy/constants.py +12 -2
- trilogy/core/models.py +120 -11
- trilogy/core/optimizations/predicate_pushdown.py +1 -1
- trilogy/core/processing/node_generators/common.py +0 -13
- trilogy/core/processing/node_generators/filter_node.py +0 -14
- trilogy/core/processing/node_generators/group_node.py +19 -1
- trilogy/core/processing/node_generators/group_to_node.py +0 -12
- trilogy/core/processing/node_generators/multiselect_node.py +1 -10
- trilogy/core/processing/node_generators/rowset_node.py +17 -18
- trilogy/core/processing/node_generators/select_node.py +26 -0
- trilogy/core/processing/node_generators/window_node.py +1 -1
- trilogy/core/processing/nodes/base_node.py +28 -1
- trilogy/core/processing/nodes/group_node.py +31 -18
- trilogy/core/processing/nodes/merge_node.py +13 -4
- trilogy/core/processing/nodes/select_node_v2.py +4 -0
- trilogy/core/processing/utility.py +91 -3
- trilogy/core/query_processor.py +1 -2
- trilogy/dialect/common.py +10 -8
- trilogy/parsing/common.py +18 -2
- trilogy/parsing/parse_engine.py +13 -9
- trilogy/parsing/trilogy.lark +2 -2
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/LICENSE.md +0 -0
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/WHEEL +0 -0
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/entry_points.txt +0 -0
- {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/top_level.txt +0 -0
|
@@ -32,6 +32,7 @@ class DatasourceMatch:
|
|
|
32
32
|
datasource: Datasource
|
|
33
33
|
matched: LooseConceptList
|
|
34
34
|
partial: LooseConceptList
|
|
35
|
+
nullable: LooseConceptList
|
|
35
36
|
|
|
36
37
|
def __repr__(self):
|
|
37
38
|
return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
|
|
@@ -76,6 +77,7 @@ def dm_to_strategy_node(
|
|
|
76
77
|
parents=[],
|
|
77
78
|
depth=depth,
|
|
78
79
|
partial_concepts=dm.partial.concepts,
|
|
80
|
+
nullable_concepts=dm.nullable.concepts,
|
|
79
81
|
accept_partial=accept_partial,
|
|
80
82
|
datasource=datasource,
|
|
81
83
|
grain=datasource.grain,
|
|
@@ -189,6 +191,13 @@ def gen_select_nodes_from_tables_v2(
|
|
|
189
191
|
if not c.is_complete and c.concept.address in all_lcl
|
|
190
192
|
]
|
|
191
193
|
),
|
|
194
|
+
nullable=LooseConceptList(
|
|
195
|
+
concepts=[
|
|
196
|
+
c.concept
|
|
197
|
+
for c in datasource.columns
|
|
198
|
+
if c.is_nullable and c.concept in all_lcl
|
|
199
|
+
]
|
|
200
|
+
),
|
|
192
201
|
)
|
|
193
202
|
if not matched:
|
|
194
203
|
continue
|
|
@@ -318,6 +327,12 @@ def gen_select_node_from_table(
|
|
|
318
327
|
if not c.is_complete and c.concept in all_lcl
|
|
319
328
|
]
|
|
320
329
|
partial_lcl = LooseConceptList(concepts=partial_concepts)
|
|
330
|
+
nullable_concepts = [
|
|
331
|
+
c.concept
|
|
332
|
+
for c in datasource.columns
|
|
333
|
+
if c.is_nullable and c.concept in all_lcl
|
|
334
|
+
]
|
|
335
|
+
nullable_lcl = LooseConceptList(concepts=nullable_concepts)
|
|
321
336
|
if not accept_partial and target_concept in partial_lcl:
|
|
322
337
|
continue
|
|
323
338
|
logger.info(
|
|
@@ -359,6 +374,7 @@ def gen_select_node_from_table(
|
|
|
359
374
|
parents=[],
|
|
360
375
|
depth=depth,
|
|
361
376
|
partial_concepts=[c for c in all_concepts if c in partial_lcl],
|
|
377
|
+
nullable_concepts=[c for c in all_concepts if c in nullable_lcl],
|
|
362
378
|
accept_partial=accept_partial,
|
|
363
379
|
datasource=datasource,
|
|
364
380
|
grain=Grain(components=all_concepts),
|
|
@@ -377,6 +393,7 @@ def gen_select_node_from_table(
|
|
|
377
393
|
parents=[bcandidate],
|
|
378
394
|
depth=depth,
|
|
379
395
|
partial_concepts=bcandidate.partial_concepts,
|
|
396
|
+
nullable_concepts=bcandidate.nullable_concepts,
|
|
380
397
|
)
|
|
381
398
|
else:
|
|
382
399
|
candidate = bcandidate
|
|
@@ -469,6 +486,14 @@ def gen_select_node(
|
|
|
469
486
|
)
|
|
470
487
|
]
|
|
471
488
|
|
|
489
|
+
all_nullable = [
|
|
490
|
+
c
|
|
491
|
+
for c in all_concepts
|
|
492
|
+
if any(
|
|
493
|
+
[c.address in [x.address for x in p.nullable_concepts] for p in parents]
|
|
494
|
+
)
|
|
495
|
+
]
|
|
496
|
+
|
|
472
497
|
if all_found:
|
|
473
498
|
logger.info(
|
|
474
499
|
f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
|
|
@@ -494,6 +519,7 @@ def gen_select_node(
|
|
|
494
519
|
parents=parents,
|
|
495
520
|
depth=depth,
|
|
496
521
|
partial_concepts=all_partial,
|
|
522
|
+
nullable_concepts=all_nullable,
|
|
497
523
|
grain=inferred_grain,
|
|
498
524
|
)
|
|
499
525
|
|
|
@@ -12,8 +12,8 @@ from trilogy.constants import logger
|
|
|
12
12
|
from trilogy.core.processing.utility import padding, create_log_lambda
|
|
13
13
|
from trilogy.core.processing.node_generators.common import (
|
|
14
14
|
gen_enrichment_node,
|
|
15
|
-
concept_to_relevant_joins,
|
|
16
15
|
)
|
|
16
|
+
from trilogy.core.processing.utility import concept_to_relevant_joins
|
|
17
17
|
|
|
18
18
|
LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
|
|
19
19
|
|
|
@@ -13,6 +13,7 @@ from trilogy.core.models import (
|
|
|
13
13
|
Comparison,
|
|
14
14
|
Parenthetical,
|
|
15
15
|
LooseConceptList,
|
|
16
|
+
ConceptPair,
|
|
16
17
|
)
|
|
17
18
|
from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
|
|
18
19
|
from trilogy.utility import unique
|
|
@@ -122,6 +123,26 @@ def get_all_parent_partial(
|
|
|
122
123
|
)
|
|
123
124
|
|
|
124
125
|
|
|
126
|
+
def get_all_parent_nullable(
|
|
127
|
+
all_concepts: List[Concept], parents: List["StrategyNode"]
|
|
128
|
+
) -> List[Concept]:
|
|
129
|
+
return unique(
|
|
130
|
+
[
|
|
131
|
+
c
|
|
132
|
+
for c in all_concepts
|
|
133
|
+
if len(
|
|
134
|
+
[
|
|
135
|
+
p
|
|
136
|
+
for p in parents
|
|
137
|
+
if c.address in [x.address for x in p.nullable_concepts]
|
|
138
|
+
]
|
|
139
|
+
)
|
|
140
|
+
>= 1
|
|
141
|
+
],
|
|
142
|
+
"address",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
125
146
|
class StrategyNode:
|
|
126
147
|
source_type = SourceType.ABSTRACT
|
|
127
148
|
|
|
@@ -134,6 +155,7 @@ class StrategyNode:
|
|
|
134
155
|
whole_grain: bool = False,
|
|
135
156
|
parents: List["StrategyNode"] | None = None,
|
|
136
157
|
partial_concepts: List[Concept] | None = None,
|
|
158
|
+
nullable_concepts: List[Concept] | None = None,
|
|
137
159
|
depth: int = 0,
|
|
138
160
|
conditions: Conditional | Comparison | Parenthetical | None = None,
|
|
139
161
|
force_group: bool | None = None,
|
|
@@ -157,6 +179,9 @@ class StrategyNode:
|
|
|
157
179
|
self.partial_concepts = partial_concepts or get_all_parent_partial(
|
|
158
180
|
self.output_concepts, self.parents
|
|
159
181
|
)
|
|
182
|
+
self.nullable_concepts = nullable_concepts or get_all_parent_nullable(
|
|
183
|
+
self.output_concepts, self.parents
|
|
184
|
+
)
|
|
160
185
|
|
|
161
186
|
self.depth = depth
|
|
162
187
|
self.conditions = conditions
|
|
@@ -269,6 +294,7 @@ class StrategyNode:
|
|
|
269
294
|
grain=grain,
|
|
270
295
|
condition=self.conditions,
|
|
271
296
|
partial_concepts=self.partial_concepts,
|
|
297
|
+
nullable_concepts=self.nullable_concepts,
|
|
272
298
|
force_group=self.force_group,
|
|
273
299
|
hidden_concepts=self.hidden_concepts,
|
|
274
300
|
)
|
|
@@ -297,6 +323,7 @@ class StrategyNode:
|
|
|
297
323
|
whole_grain=self.whole_grain,
|
|
298
324
|
parents=list(self.parents),
|
|
299
325
|
partial_concepts=list(self.partial_concepts),
|
|
326
|
+
nullable_concepts=list(self.nullable_concepts),
|
|
300
327
|
depth=self.depth,
|
|
301
328
|
conditions=self.conditions,
|
|
302
329
|
force_group=self.force_group,
|
|
@@ -314,7 +341,7 @@ class NodeJoin:
|
|
|
314
341
|
concepts: List[Concept]
|
|
315
342
|
join_type: JoinType
|
|
316
343
|
filter_to_mutual: bool = False
|
|
317
|
-
concept_pairs: list[
|
|
344
|
+
concept_pairs: list[ConceptPair] | None = None
|
|
318
345
|
|
|
319
346
|
def __post_init__(self):
|
|
320
347
|
if self.concept_pairs:
|
|
@@ -20,6 +20,7 @@ from trilogy.core.processing.nodes.base_node import (
|
|
|
20
20
|
)
|
|
21
21
|
from trilogy.utility import unique
|
|
22
22
|
from trilogy.core.processing.utility import is_scalar_condition
|
|
23
|
+
from trilogy.core.processing.utility import find_nullable_concepts
|
|
23
24
|
|
|
24
25
|
LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
|
|
25
26
|
|
|
@@ -37,6 +38,7 @@ class GroupNode(StrategyNode):
|
|
|
37
38
|
parents: List["StrategyNode"] | None = None,
|
|
38
39
|
depth: int = 0,
|
|
39
40
|
partial_concepts: Optional[List[Concept]] = None,
|
|
41
|
+
nullable_concepts: Optional[List[Concept]] = None,
|
|
40
42
|
force_group: bool | None = None,
|
|
41
43
|
conditions: Conditional | Comparison | Parenthetical | None = None,
|
|
42
44
|
existence_concepts: List[Concept] | None = None,
|
|
@@ -50,6 +52,7 @@ class GroupNode(StrategyNode):
|
|
|
50
52
|
parents=parents,
|
|
51
53
|
depth=depth,
|
|
52
54
|
partial_concepts=partial_concepts,
|
|
55
|
+
nullable_concepts=nullable_concepts,
|
|
53
56
|
force_group=force_group,
|
|
54
57
|
conditions=conditions,
|
|
55
58
|
existence_concepts=existence_concepts,
|
|
@@ -113,27 +116,34 @@ class GroupNode(StrategyNode):
|
|
|
113
116
|
f" {parent.grain}"
|
|
114
117
|
)
|
|
115
118
|
source_type = SourceType.GROUP
|
|
116
|
-
|
|
119
|
+
source_map = resolve_concept_map(
|
|
120
|
+
parent_sources,
|
|
121
|
+
targets=(
|
|
122
|
+
unique(
|
|
123
|
+
self.output_concepts + self.conditions.concept_arguments,
|
|
124
|
+
"address",
|
|
125
|
+
)
|
|
126
|
+
if self.conditions
|
|
127
|
+
else self.output_concepts
|
|
128
|
+
),
|
|
129
|
+
inherited_inputs=self.input_concepts + self.existence_concepts,
|
|
130
|
+
)
|
|
131
|
+
nullable_addresses = find_nullable_concepts(
|
|
132
|
+
source_map=source_map, joins=[], datasources=parent_sources
|
|
133
|
+
)
|
|
134
|
+
nullable_concepts = [
|
|
135
|
+
x for x in self.output_concepts if x.address in nullable_addresses
|
|
136
|
+
]
|
|
117
137
|
base = QueryDatasource(
|
|
118
138
|
input_concepts=self.input_concepts,
|
|
119
139
|
output_concepts=self.output_concepts,
|
|
120
140
|
datasources=parent_sources,
|
|
121
141
|
source_type=source_type,
|
|
122
|
-
source_map=
|
|
123
|
-
parent_sources,
|
|
124
|
-
targets=(
|
|
125
|
-
unique(
|
|
126
|
-
self.output_concepts + self.conditions.concept_arguments,
|
|
127
|
-
"address",
|
|
128
|
-
)
|
|
129
|
-
if self.conditions
|
|
130
|
-
else self.output_concepts
|
|
131
|
-
),
|
|
132
|
-
inherited_inputs=self.input_concepts + self.existence_concepts,
|
|
133
|
-
),
|
|
142
|
+
source_map=source_map,
|
|
134
143
|
joins=[],
|
|
135
144
|
grain=grain,
|
|
136
145
|
partial_concepts=self.partial_concepts,
|
|
146
|
+
nullable_concepts=nullable_concepts,
|
|
137
147
|
condition=self.conditions,
|
|
138
148
|
)
|
|
139
149
|
# if there is a condition on a group node and it's not scalar
|
|
@@ -141,18 +151,20 @@ class GroupNode(StrategyNode):
|
|
|
141
151
|
if self.conditions and not is_scalar_condition(self.conditions):
|
|
142
152
|
base.condition = None
|
|
143
153
|
base.output_concepts = self.output_concepts + self.conditions.row_arguments
|
|
154
|
+
source_map = resolve_concept_map(
|
|
155
|
+
[base],
|
|
156
|
+
targets=self.output_concepts,
|
|
157
|
+
inherited_inputs=base.output_concepts,
|
|
158
|
+
)
|
|
144
159
|
return QueryDatasource(
|
|
145
160
|
input_concepts=base.output_concepts,
|
|
146
161
|
output_concepts=self.output_concepts,
|
|
147
162
|
datasources=[base],
|
|
148
163
|
source_type=SourceType.SELECT,
|
|
149
|
-
source_map=
|
|
150
|
-
[base],
|
|
151
|
-
targets=self.output_concepts,
|
|
152
|
-
inherited_inputs=base.output_concepts,
|
|
153
|
-
),
|
|
164
|
+
source_map=source_map,
|
|
154
165
|
joins=[],
|
|
155
166
|
grain=grain,
|
|
167
|
+
nullable_concepts=base.nullable_concepts,
|
|
156
168
|
partial_concepts=self.partial_concepts,
|
|
157
169
|
condition=self.conditions,
|
|
158
170
|
)
|
|
@@ -168,6 +180,7 @@ class GroupNode(StrategyNode):
|
|
|
168
180
|
parents=self.parents,
|
|
169
181
|
depth=self.depth,
|
|
170
182
|
partial_concepts=list(self.partial_concepts),
|
|
183
|
+
nullable_concepts=list(self.nullable_concepts),
|
|
171
184
|
force_group=self.force_group,
|
|
172
185
|
conditions=self.conditions,
|
|
173
186
|
existence_concepts=list(self.existence_concepts),
|
|
@@ -22,7 +22,7 @@ from trilogy.core.processing.nodes.base_node import (
|
|
|
22
22
|
resolve_concept_map,
|
|
23
23
|
NodeJoin,
|
|
24
24
|
)
|
|
25
|
-
from trilogy.core.processing.utility import get_node_joins
|
|
25
|
+
from trilogy.core.processing.utility import get_node_joins, find_nullable_concepts
|
|
26
26
|
|
|
27
27
|
LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
|
|
28
28
|
|
|
@@ -110,6 +110,7 @@ class MergeNode(StrategyNode):
|
|
|
110
110
|
join_concepts: Optional[List] = None,
|
|
111
111
|
force_join_type: Optional[JoinType] = None,
|
|
112
112
|
partial_concepts: Optional[List[Concept]] = None,
|
|
113
|
+
nullable_concepts: Optional[List[Concept]] = None,
|
|
113
114
|
force_group: bool | None = None,
|
|
114
115
|
depth: int = 0,
|
|
115
116
|
grain: Grain | None = None,
|
|
@@ -127,6 +128,7 @@ class MergeNode(StrategyNode):
|
|
|
127
128
|
parents=parents,
|
|
128
129
|
depth=depth,
|
|
129
130
|
partial_concepts=partial_concepts,
|
|
131
|
+
nullable_concepts=nullable_concepts,
|
|
130
132
|
force_group=force_group,
|
|
131
133
|
grain=grain,
|
|
132
134
|
conditions=conditions,
|
|
@@ -192,7 +194,7 @@ class MergeNode(StrategyNode):
|
|
|
192
194
|
pregrain: Grain,
|
|
193
195
|
grain: Grain,
|
|
194
196
|
environment: Environment,
|
|
195
|
-
) -> List[BaseJoin]:
|
|
197
|
+
) -> List[BaseJoin | UnnestJoin]:
|
|
196
198
|
# only finally, join between them for unique values
|
|
197
199
|
dataset_list: List[QueryDatasource] = sorted(
|
|
198
200
|
final_datasets, key=lambda x: -len(x.grain.components_copy)
|
|
@@ -308,7 +310,7 @@ class MergeNode(StrategyNode):
|
|
|
308
310
|
)
|
|
309
311
|
join_candidates = [x for x in final_datasets if x not in existence_final]
|
|
310
312
|
if len(join_candidates) > 1:
|
|
311
|
-
joins = self.generate_joins(
|
|
313
|
+
joins: List[BaseJoin | UnnestJoin] = self.generate_joins(
|
|
312
314
|
join_candidates, final_joins, pregrain, grain, self.environment
|
|
313
315
|
)
|
|
314
316
|
else:
|
|
@@ -318,7 +320,7 @@ class MergeNode(StrategyNode):
|
|
|
318
320
|
)
|
|
319
321
|
full_join_concepts = []
|
|
320
322
|
for join in joins:
|
|
321
|
-
if join.join_type == JoinType.FULL:
|
|
323
|
+
if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
|
|
322
324
|
full_join_concepts += join.concepts
|
|
323
325
|
if self.whole_grain:
|
|
324
326
|
force_group = False
|
|
@@ -341,6 +343,9 @@ class MergeNode(StrategyNode):
|
|
|
341
343
|
inherited_inputs=self.input_concepts + self.existence_concepts,
|
|
342
344
|
full_joins=full_join_concepts,
|
|
343
345
|
)
|
|
346
|
+
nullable_concepts = find_nullable_concepts(
|
|
347
|
+
source_map=source_map, joins=joins, datasources=final_datasets
|
|
348
|
+
)
|
|
344
349
|
qds = QueryDatasource(
|
|
345
350
|
input_concepts=unique(self.input_concepts, "address"),
|
|
346
351
|
output_concepts=unique(self.output_concepts, "address"),
|
|
@@ -349,6 +354,9 @@ class MergeNode(StrategyNode):
|
|
|
349
354
|
source_map=source_map,
|
|
350
355
|
joins=qd_joins,
|
|
351
356
|
grain=grain,
|
|
357
|
+
nullable_concepts=[
|
|
358
|
+
x for x in self.output_concepts if x.address in nullable_concepts
|
|
359
|
+
],
|
|
352
360
|
partial_concepts=self.partial_concepts,
|
|
353
361
|
force_group=force_group,
|
|
354
362
|
condition=self.conditions,
|
|
@@ -369,6 +377,7 @@ class MergeNode(StrategyNode):
|
|
|
369
377
|
force_group=self.force_group,
|
|
370
378
|
grain=self.grain,
|
|
371
379
|
conditions=self.conditions,
|
|
380
|
+
nullable_concepts=list(self.nullable_concepts),
|
|
372
381
|
hidden_concepts=list(self.hidden_concepts),
|
|
373
382
|
virtual_output_concepts=list(self.virtual_output_concepts),
|
|
374
383
|
node_joins=list(self.node_joins) if self.node_joins else None,
|
|
@@ -43,6 +43,7 @@ class SelectNode(StrategyNode):
|
|
|
43
43
|
parents: List["StrategyNode"] | None = None,
|
|
44
44
|
depth: int = 0,
|
|
45
45
|
partial_concepts: List[Concept] | None = None,
|
|
46
|
+
nullable_concepts: List[Concept] | None = None,
|
|
46
47
|
accept_partial: bool = False,
|
|
47
48
|
grain: Optional[Grain] = None,
|
|
48
49
|
force_group: bool | None = False,
|
|
@@ -58,6 +59,7 @@ class SelectNode(StrategyNode):
|
|
|
58
59
|
parents=parents,
|
|
59
60
|
depth=depth,
|
|
60
61
|
partial_concepts=partial_concepts,
|
|
62
|
+
nullable_concepts=nullable_concepts,
|
|
61
63
|
force_group=force_group,
|
|
62
64
|
grain=grain,
|
|
63
65
|
conditions=conditions,
|
|
@@ -115,6 +117,7 @@ class SelectNode(StrategyNode):
|
|
|
115
117
|
partial_concepts=[
|
|
116
118
|
c.concept for c in datasource.columns if not c.is_complete
|
|
117
119
|
],
|
|
120
|
+
nullable_concepts=[c.concept for c in datasource.columns if c.is_nullable],
|
|
118
121
|
source_type=SourceType.DIRECT_SELECT,
|
|
119
122
|
condition=self.conditions,
|
|
120
123
|
# select nodes should never group
|
|
@@ -183,6 +186,7 @@ class SelectNode(StrategyNode):
|
|
|
183
186
|
parents=self.parents,
|
|
184
187
|
whole_grain=self.whole_grain,
|
|
185
188
|
partial_concepts=list(self.partial_concepts),
|
|
189
|
+
nullable_concepts=list(self.nullable_concepts),
|
|
186
190
|
accept_partial=self.accept_partial,
|
|
187
191
|
grain=self.grain,
|
|
188
192
|
force_group=self.force_group,
|
|
@@ -18,9 +18,11 @@ from trilogy.core.models import (
|
|
|
18
18
|
WindowItem,
|
|
19
19
|
AggregateWrapper,
|
|
20
20
|
DataType,
|
|
21
|
+
ConceptPair,
|
|
22
|
+
UnnestJoin,
|
|
21
23
|
)
|
|
22
24
|
|
|
23
|
-
from trilogy.core.enums import Purpose, Granularity, BooleanOperator
|
|
25
|
+
from trilogy.core.enums import Purpose, Granularity, BooleanOperator, Modifier
|
|
24
26
|
from trilogy.core.constants import CONSTANT_DATASET
|
|
25
27
|
from enum import Enum
|
|
26
28
|
from trilogy.utility import unique
|
|
@@ -243,8 +245,23 @@ def get_node_joins(
|
|
|
243
245
|
local_concepts = [
|
|
244
246
|
c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
|
|
245
247
|
]
|
|
246
|
-
|
|
248
|
+
elif any(
|
|
249
|
+
[
|
|
250
|
+
c.address in [x.address for x in identifier_map[right].partial_concepts]
|
|
251
|
+
for c in local_concepts
|
|
252
|
+
]
|
|
253
|
+
) or any(
|
|
254
|
+
[
|
|
255
|
+
c.address in [x.address for x in identifier_map[left].nullable_concepts]
|
|
256
|
+
for c in local_concepts
|
|
257
|
+
]
|
|
258
|
+
):
|
|
247
259
|
join_type = JoinType.LEFT_OUTER
|
|
260
|
+
local_concepts = [
|
|
261
|
+
c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
|
|
262
|
+
]
|
|
263
|
+
else:
|
|
264
|
+
join_type = JoinType.INNER
|
|
248
265
|
# remove any constants if other join keys exist
|
|
249
266
|
local_concepts = [
|
|
250
267
|
c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
|
|
@@ -287,7 +304,18 @@ def get_node_joins(
|
|
|
287
304
|
)
|
|
288
305
|
narg = (left_arg, right_arg)
|
|
289
306
|
if narg not in join_tuples:
|
|
290
|
-
|
|
307
|
+
modifiers = set()
|
|
308
|
+
if left_arg.address in [
|
|
309
|
+
x.address for x in left_datasource.nullable_concepts
|
|
310
|
+
] and right_arg.address in [
|
|
311
|
+
x.address for x in right_datasource.nullable_concepts
|
|
312
|
+
]:
|
|
313
|
+
modifiers.add(Modifier.NULLABLE)
|
|
314
|
+
join_tuples.append(
|
|
315
|
+
ConceptPair(
|
|
316
|
+
left=left_arg, right=right_arg, modifiers=list(modifiers)
|
|
317
|
+
)
|
|
318
|
+
)
|
|
291
319
|
final_joins_pre.append(
|
|
292
320
|
BaseJoin(
|
|
293
321
|
left_datasource=identifier_map[left],
|
|
@@ -412,3 +440,63 @@ def decompose_condition(
|
|
|
412
440
|
else:
|
|
413
441
|
chunks.append(conditional)
|
|
414
442
|
return chunks
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def find_nullable_concepts(
|
|
446
|
+
source_map: Dict[str, set[Datasource | QueryDatasource | UnnestJoin]],
|
|
447
|
+
datasources: List[Datasource | QueryDatasource],
|
|
448
|
+
joins: List[BaseJoin | UnnestJoin],
|
|
449
|
+
) -> List[str]:
|
|
450
|
+
"""give a set of datasources and joins, find the concepts
|
|
451
|
+
that may contain nulls in the output set
|
|
452
|
+
"""
|
|
453
|
+
nullable_datasources = set()
|
|
454
|
+
datasource_map = {
|
|
455
|
+
x.identifier: x
|
|
456
|
+
for x in datasources
|
|
457
|
+
if isinstance(x, (Datasource, QueryDatasource))
|
|
458
|
+
}
|
|
459
|
+
for join in joins:
|
|
460
|
+
is_on_nullable_condition = False
|
|
461
|
+
if not isinstance(join, BaseJoin):
|
|
462
|
+
continue
|
|
463
|
+
if not join.concept_pairs:
|
|
464
|
+
continue
|
|
465
|
+
for pair in join.concept_pairs:
|
|
466
|
+
if pair.right.address in [
|
|
467
|
+
y.address
|
|
468
|
+
for y in datasource_map[
|
|
469
|
+
join.right_datasource.identifier
|
|
470
|
+
].nullable_concepts
|
|
471
|
+
]:
|
|
472
|
+
is_on_nullable_condition = True
|
|
473
|
+
break
|
|
474
|
+
if pair.left.address in [
|
|
475
|
+
y.address
|
|
476
|
+
for y in datasource_map[
|
|
477
|
+
join.left_datasource.identifier
|
|
478
|
+
].nullable_concepts
|
|
479
|
+
]:
|
|
480
|
+
is_on_nullable_condition = True
|
|
481
|
+
break
|
|
482
|
+
if is_on_nullable_condition:
|
|
483
|
+
nullable_datasources.add(datasource_map[join.right_datasource.identifier])
|
|
484
|
+
final_nullable = set()
|
|
485
|
+
|
|
486
|
+
for k, v in source_map.items():
|
|
487
|
+
local_nullable = [
|
|
488
|
+
x for x in datasources if k in [v.address for v in x.nullable_concepts]
|
|
489
|
+
]
|
|
490
|
+
if all(
|
|
491
|
+
[
|
|
492
|
+
k in [v.address for v in x.nullable_concepts]
|
|
493
|
+
for x in datasources
|
|
494
|
+
if k in [z.address for z in x.output_concepts]
|
|
495
|
+
]
|
|
496
|
+
):
|
|
497
|
+
final_nullable.add(k)
|
|
498
|
+
all_ds = set([ds for ds in local_nullable]).union(nullable_datasources)
|
|
499
|
+
if nullable_datasources:
|
|
500
|
+
if set(v).issubset(all_ds):
|
|
501
|
+
final_nullable.add(k)
|
|
502
|
+
return list(sorted(final_nullable))
|
trilogy/core/query_processor.py
CHANGED
|
@@ -192,8 +192,6 @@ def resolve_cte_base_name_and_alias_v2(
|
|
|
192
192
|
raw_joins: List[Join | InstantiatedUnnestJoin],
|
|
193
193
|
) -> Tuple[str | None, str | None]:
|
|
194
194
|
joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
|
|
195
|
-
# INFO trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
|
|
196
|
-
# 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
|
|
197
195
|
if (
|
|
198
196
|
len(source.datasources) == 1
|
|
199
197
|
and isinstance(source.datasources[0], Datasource)
|
|
@@ -297,6 +295,7 @@ def datasource_to_ctes(
|
|
|
297
295
|
parent_ctes=parents,
|
|
298
296
|
condition=query_datasource.condition,
|
|
299
297
|
partial_concepts=query_datasource.partial_concepts,
|
|
298
|
+
nullable_concepts=query_datasource.nullable_concepts,
|
|
300
299
|
join_derived_concepts=query_datasource.join_derived_concepts,
|
|
301
300
|
hidden_concepts=query_datasource.hidden_concepts,
|
|
302
301
|
base_name_override=base_name,
|
trilogy/dialect/common.py
CHANGED
|
@@ -3,9 +3,9 @@ from trilogy.core.enums import UnnestMode, Modifier
|
|
|
3
3
|
from typing import Optional, Callable
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def null_wrapper(lval: str, rval: str,
|
|
7
|
-
if
|
|
8
|
-
return f"(
|
|
6
|
+
def null_wrapper(lval: str, rval: str, modifiers: list[Modifier]) -> str:
|
|
7
|
+
if Modifier.NULLABLE in modifiers:
|
|
8
|
+
return f"({lval} = {rval} or ({lval} is null and {rval} is null))"
|
|
9
9
|
return f"{lval} = {rval}"
|
|
10
10
|
|
|
11
11
|
|
|
@@ -48,7 +48,7 @@ def render_join(
|
|
|
48
48
|
null_wrapper(
|
|
49
49
|
f"{left_name}.{quote_character}{join.left_cte.get_alias(key.concept) if isinstance(join.left_cte, Datasource) else key.concept.safe_address}{quote_character}",
|
|
50
50
|
f"{right_name}.{quote_character}{join.right_cte.get_alias(key.concept) if isinstance(join.right_cte, Datasource) else key.concept.safe_address}{quote_character}",
|
|
51
|
-
key.concept,
|
|
51
|
+
modifiers=key.concept.modifiers or [],
|
|
52
52
|
)
|
|
53
53
|
for key in join.joinkeys
|
|
54
54
|
]
|
|
@@ -56,11 +56,13 @@ def render_join(
|
|
|
56
56
|
base_joinkeys.extend(
|
|
57
57
|
[
|
|
58
58
|
null_wrapper(
|
|
59
|
-
f"{left_name}.{quote_character}{join.left_cte.get_alias(
|
|
60
|
-
f"{right_name}.{quote_character}{join.right_cte.get_alias(
|
|
61
|
-
|
|
59
|
+
f"{left_name}.{quote_character}{join.left_cte.get_alias(pair.left) if isinstance(join.left_cte, Datasource) else pair.left.safe_address}{quote_character}",
|
|
60
|
+
f"{right_name}.{quote_character}{join.right_cte.get_alias(pair.right) if isinstance(join.right_cte, Datasource) else pair.right.safe_address}{quote_character}",
|
|
61
|
+
modifiers=pair.modifiers
|
|
62
|
+
+ (pair.left.modifiers or [])
|
|
63
|
+
+ (pair.right.modifiers or []),
|
|
62
64
|
)
|
|
63
|
-
for
|
|
65
|
+
for pair in join.joinkey_pairs
|
|
64
66
|
]
|
|
65
67
|
)
|
|
66
68
|
if not base_joinkeys:
|
trilogy/parsing/common.py
CHANGED
|
@@ -25,6 +25,15 @@ from trilogy.core.enums import PurposeLineage
|
|
|
25
25
|
from trilogy.constants import (
|
|
26
26
|
VIRTUAL_CONCEPT_PREFIX,
|
|
27
27
|
)
|
|
28
|
+
from trilogy.core.enums import Modifier
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_upstream_modifiers(keys: List[Concept]) -> list[Modifier]:
|
|
32
|
+
modifiers = set()
|
|
33
|
+
for pkey in keys:
|
|
34
|
+
if pkey.modifiers:
|
|
35
|
+
modifiers.update(pkey.modifiers)
|
|
36
|
+
return list(modifiers)
|
|
28
37
|
|
|
29
38
|
|
|
30
39
|
def process_function_args(
|
|
@@ -125,7 +134,7 @@ def constant_to_concept(
|
|
|
125
134
|
|
|
126
135
|
|
|
127
136
|
def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
|
|
128
|
-
pkeys = []
|
|
137
|
+
pkeys: List[Concept] = []
|
|
129
138
|
for x in parent.arguments:
|
|
130
139
|
pkeys += [
|
|
131
140
|
x
|
|
@@ -135,7 +144,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
|
|
|
135
144
|
grain = Grain()
|
|
136
145
|
for x in pkeys:
|
|
137
146
|
grain += x.grain
|
|
138
|
-
|
|
147
|
+
modifiers = get_upstream_modifiers(pkeys)
|
|
139
148
|
key_grain = []
|
|
140
149
|
for x in pkeys:
|
|
141
150
|
if x.keys:
|
|
@@ -155,6 +164,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
|
|
|
155
164
|
namespace=namespace,
|
|
156
165
|
grain=grain,
|
|
157
166
|
keys=keys,
|
|
167
|
+
modifiers=modifiers,
|
|
158
168
|
)
|
|
159
169
|
|
|
160
170
|
|
|
@@ -166,6 +176,7 @@ def filter_item_to_concept(
|
|
|
166
176
|
metadata: Metadata | None = None,
|
|
167
177
|
) -> Concept:
|
|
168
178
|
fmetadata = metadata or Metadata()
|
|
179
|
+
modifiers = get_upstream_modifiers(parent.content.concept_arguments)
|
|
169
180
|
return Concept(
|
|
170
181
|
name=name,
|
|
171
182
|
datatype=parent.content.datatype,
|
|
@@ -184,6 +195,7 @@ def filter_item_to_concept(
|
|
|
184
195
|
if parent.content.purpose == Purpose.PROPERTY
|
|
185
196
|
else Grain()
|
|
186
197
|
),
|
|
198
|
+
modifiers=modifiers,
|
|
187
199
|
)
|
|
188
200
|
|
|
189
201
|
|
|
@@ -202,6 +214,7 @@ def window_item_to_concept(
|
|
|
202
214
|
grain += [item.expr.output]
|
|
203
215
|
else:
|
|
204
216
|
grain = parent.over + [parent.content.output]
|
|
217
|
+
modifiers = get_upstream_modifiers(parent.content.concept_arguments)
|
|
205
218
|
return Concept(
|
|
206
219
|
name=name,
|
|
207
220
|
datatype=parent.content.datatype,
|
|
@@ -212,6 +225,7 @@ def window_item_to_concept(
|
|
|
212
225
|
grain=Grain(components=grain),
|
|
213
226
|
namespace=namespace,
|
|
214
227
|
keys=keys,
|
|
228
|
+
modifiers=modifiers,
|
|
215
229
|
)
|
|
216
230
|
|
|
217
231
|
|
|
@@ -229,6 +243,7 @@ def agg_wrapper_to_concept(
|
|
|
229
243
|
# at that grain
|
|
230
244
|
fmetadata = metadata or Metadata()
|
|
231
245
|
aggfunction = parent.function
|
|
246
|
+
modifiers = get_upstream_modifiers(parent.concept_arguments)
|
|
232
247
|
out = Concept(
|
|
233
248
|
name=name,
|
|
234
249
|
datatype=aggfunction.output_datatype,
|
|
@@ -238,6 +253,7 @@ def agg_wrapper_to_concept(
|
|
|
238
253
|
grain=Grain(components=parent.by) if parent.by else Grain(),
|
|
239
254
|
namespace=namespace,
|
|
240
255
|
keys=tuple(parent.by) if parent.by else keys,
|
|
256
|
+
modifiers=modifiers,
|
|
241
257
|
)
|
|
242
258
|
return out
|
|
243
259
|
|