pytrilogy 0.0.2.13__py3-none-any.whl → 0.0.2.15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pytrilogy might be problematic. Click here for more details.

Files changed (28) hide show
  1. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/METADATA +1 -1
  2. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/RECORD +28 -28
  3. trilogy/__init__.py +1 -1
  4. trilogy/constants.py +12 -2
  5. trilogy/core/models.py +120 -11
  6. trilogy/core/optimizations/predicate_pushdown.py +1 -1
  7. trilogy/core/processing/node_generators/common.py +0 -13
  8. trilogy/core/processing/node_generators/filter_node.py +0 -14
  9. trilogy/core/processing/node_generators/group_node.py +19 -1
  10. trilogy/core/processing/node_generators/group_to_node.py +0 -12
  11. trilogy/core/processing/node_generators/multiselect_node.py +1 -10
  12. trilogy/core/processing/node_generators/rowset_node.py +17 -18
  13. trilogy/core/processing/node_generators/select_node.py +26 -0
  14. trilogy/core/processing/node_generators/window_node.py +1 -1
  15. trilogy/core/processing/nodes/base_node.py +28 -1
  16. trilogy/core/processing/nodes/group_node.py +31 -18
  17. trilogy/core/processing/nodes/merge_node.py +13 -4
  18. trilogy/core/processing/nodes/select_node_v2.py +4 -0
  19. trilogy/core/processing/utility.py +91 -3
  20. trilogy/core/query_processor.py +1 -2
  21. trilogy/dialect/common.py +10 -8
  22. trilogy/parsing/common.py +18 -2
  23. trilogy/parsing/parse_engine.py +13 -9
  24. trilogy/parsing/trilogy.lark +2 -2
  25. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/LICENSE.md +0 -0
  26. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/WHEEL +0 -0
  27. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/entry_points.txt +0 -0
  28. {pytrilogy-0.0.2.13.dist-info → pytrilogy-0.0.2.15.dist-info}/top_level.txt +0 -0
@@ -32,6 +32,7 @@ class DatasourceMatch:
32
32
  datasource: Datasource
33
33
  matched: LooseConceptList
34
34
  partial: LooseConceptList
35
+ nullable: LooseConceptList
35
36
 
36
37
  def __repr__(self):
37
38
  return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
@@ -76,6 +77,7 @@ def dm_to_strategy_node(
76
77
  parents=[],
77
78
  depth=depth,
78
79
  partial_concepts=dm.partial.concepts,
80
+ nullable_concepts=dm.nullable.concepts,
79
81
  accept_partial=accept_partial,
80
82
  datasource=datasource,
81
83
  grain=datasource.grain,
@@ -189,6 +191,13 @@ def gen_select_nodes_from_tables_v2(
189
191
  if not c.is_complete and c.concept.address in all_lcl
190
192
  ]
191
193
  ),
194
+ nullable=LooseConceptList(
195
+ concepts=[
196
+ c.concept
197
+ for c in datasource.columns
198
+ if c.is_nullable and c.concept in all_lcl
199
+ ]
200
+ ),
192
201
  )
193
202
  if not matched:
194
203
  continue
@@ -318,6 +327,12 @@ def gen_select_node_from_table(
318
327
  if not c.is_complete and c.concept in all_lcl
319
328
  ]
320
329
  partial_lcl = LooseConceptList(concepts=partial_concepts)
330
+ nullable_concepts = [
331
+ c.concept
332
+ for c in datasource.columns
333
+ if c.is_nullable and c.concept in all_lcl
334
+ ]
335
+ nullable_lcl = LooseConceptList(concepts=nullable_concepts)
321
336
  if not accept_partial and target_concept in partial_lcl:
322
337
  continue
323
338
  logger.info(
@@ -359,6 +374,7 @@ def gen_select_node_from_table(
359
374
  parents=[],
360
375
  depth=depth,
361
376
  partial_concepts=[c for c in all_concepts if c in partial_lcl],
377
+ nullable_concepts=[c for c in all_concepts if c in nullable_lcl],
362
378
  accept_partial=accept_partial,
363
379
  datasource=datasource,
364
380
  grain=Grain(components=all_concepts),
@@ -377,6 +393,7 @@ def gen_select_node_from_table(
377
393
  parents=[bcandidate],
378
394
  depth=depth,
379
395
  partial_concepts=bcandidate.partial_concepts,
396
+ nullable_concepts=bcandidate.nullable_concepts,
380
397
  )
381
398
  else:
382
399
  candidate = bcandidate
@@ -469,6 +486,14 @@ def gen_select_node(
469
486
  )
470
487
  ]
471
488
 
489
+ all_nullable = [
490
+ c
491
+ for c in all_concepts
492
+ if any(
493
+ [c.address in [x.address for x in p.nullable_concepts] for p in parents]
494
+ )
495
+ ]
496
+
472
497
  if all_found:
473
498
  logger.info(
474
499
  f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
@@ -494,6 +519,7 @@ def gen_select_node(
494
519
  parents=parents,
495
520
  depth=depth,
496
521
  partial_concepts=all_partial,
522
+ nullable_concepts=all_nullable,
497
523
  grain=inferred_grain,
498
524
  )
499
525
 
@@ -12,8 +12,8 @@ from trilogy.constants import logger
12
12
  from trilogy.core.processing.utility import padding, create_log_lambda
13
13
  from trilogy.core.processing.node_generators.common import (
14
14
  gen_enrichment_node,
15
- concept_to_relevant_joins,
16
15
  )
16
+ from trilogy.core.processing.utility import concept_to_relevant_joins
17
17
 
18
18
  LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
19
19
 
@@ -13,6 +13,7 @@ from trilogy.core.models import (
13
13
  Comparison,
14
14
  Parenthetical,
15
15
  LooseConceptList,
16
+ ConceptPair,
16
17
  )
17
18
  from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
18
19
  from trilogy.utility import unique
@@ -122,6 +123,26 @@ def get_all_parent_partial(
122
123
  )
123
124
 
124
125
 
126
+ def get_all_parent_nullable(
127
+ all_concepts: List[Concept], parents: List["StrategyNode"]
128
+ ) -> List[Concept]:
129
+ return unique(
130
+ [
131
+ c
132
+ for c in all_concepts
133
+ if len(
134
+ [
135
+ p
136
+ for p in parents
137
+ if c.address in [x.address for x in p.nullable_concepts]
138
+ ]
139
+ )
140
+ >= 1
141
+ ],
142
+ "address",
143
+ )
144
+
145
+
125
146
  class StrategyNode:
126
147
  source_type = SourceType.ABSTRACT
127
148
 
@@ -134,6 +155,7 @@ class StrategyNode:
134
155
  whole_grain: bool = False,
135
156
  parents: List["StrategyNode"] | None = None,
136
157
  partial_concepts: List[Concept] | None = None,
158
+ nullable_concepts: List[Concept] | None = None,
137
159
  depth: int = 0,
138
160
  conditions: Conditional | Comparison | Parenthetical | None = None,
139
161
  force_group: bool | None = None,
@@ -157,6 +179,9 @@ class StrategyNode:
157
179
  self.partial_concepts = partial_concepts or get_all_parent_partial(
158
180
  self.output_concepts, self.parents
159
181
  )
182
+ self.nullable_concepts = nullable_concepts or get_all_parent_nullable(
183
+ self.output_concepts, self.parents
184
+ )
160
185
 
161
186
  self.depth = depth
162
187
  self.conditions = conditions
@@ -269,6 +294,7 @@ class StrategyNode:
269
294
  grain=grain,
270
295
  condition=self.conditions,
271
296
  partial_concepts=self.partial_concepts,
297
+ nullable_concepts=self.nullable_concepts,
272
298
  force_group=self.force_group,
273
299
  hidden_concepts=self.hidden_concepts,
274
300
  )
@@ -297,6 +323,7 @@ class StrategyNode:
297
323
  whole_grain=self.whole_grain,
298
324
  parents=list(self.parents),
299
325
  partial_concepts=list(self.partial_concepts),
326
+ nullable_concepts=list(self.nullable_concepts),
300
327
  depth=self.depth,
301
328
  conditions=self.conditions,
302
329
  force_group=self.force_group,
@@ -314,7 +341,7 @@ class NodeJoin:
314
341
  concepts: List[Concept]
315
342
  join_type: JoinType
316
343
  filter_to_mutual: bool = False
317
- concept_pairs: list[tuple[Concept, Concept]] | None = None
344
+ concept_pairs: list[ConceptPair] | None = None
318
345
 
319
346
  def __post_init__(self):
320
347
  if self.concept_pairs:
@@ -20,6 +20,7 @@ from trilogy.core.processing.nodes.base_node import (
20
20
  )
21
21
  from trilogy.utility import unique
22
22
  from trilogy.core.processing.utility import is_scalar_condition
23
+ from trilogy.core.processing.utility import find_nullable_concepts
23
24
 
24
25
  LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
25
26
 
@@ -37,6 +38,7 @@ class GroupNode(StrategyNode):
37
38
  parents: List["StrategyNode"] | None = None,
38
39
  depth: int = 0,
39
40
  partial_concepts: Optional[List[Concept]] = None,
41
+ nullable_concepts: Optional[List[Concept]] = None,
40
42
  force_group: bool | None = None,
41
43
  conditions: Conditional | Comparison | Parenthetical | None = None,
42
44
  existence_concepts: List[Concept] | None = None,
@@ -50,6 +52,7 @@ class GroupNode(StrategyNode):
50
52
  parents=parents,
51
53
  depth=depth,
52
54
  partial_concepts=partial_concepts,
55
+ nullable_concepts=nullable_concepts,
53
56
  force_group=force_group,
54
57
  conditions=conditions,
55
58
  existence_concepts=existence_concepts,
@@ -113,27 +116,34 @@ class GroupNode(StrategyNode):
113
116
  f" {parent.grain}"
114
117
  )
115
118
  source_type = SourceType.GROUP
116
-
119
+ source_map = resolve_concept_map(
120
+ parent_sources,
121
+ targets=(
122
+ unique(
123
+ self.output_concepts + self.conditions.concept_arguments,
124
+ "address",
125
+ )
126
+ if self.conditions
127
+ else self.output_concepts
128
+ ),
129
+ inherited_inputs=self.input_concepts + self.existence_concepts,
130
+ )
131
+ nullable_addresses = find_nullable_concepts(
132
+ source_map=source_map, joins=[], datasources=parent_sources
133
+ )
134
+ nullable_concepts = [
135
+ x for x in self.output_concepts if x.address in nullable_addresses
136
+ ]
117
137
  base = QueryDatasource(
118
138
  input_concepts=self.input_concepts,
119
139
  output_concepts=self.output_concepts,
120
140
  datasources=parent_sources,
121
141
  source_type=source_type,
122
- source_map=resolve_concept_map(
123
- parent_sources,
124
- targets=(
125
- unique(
126
- self.output_concepts + self.conditions.concept_arguments,
127
- "address",
128
- )
129
- if self.conditions
130
- else self.output_concepts
131
- ),
132
- inherited_inputs=self.input_concepts + self.existence_concepts,
133
- ),
142
+ source_map=source_map,
134
143
  joins=[],
135
144
  grain=grain,
136
145
  partial_concepts=self.partial_concepts,
146
+ nullable_concepts=nullable_concepts,
137
147
  condition=self.conditions,
138
148
  )
139
149
  # if there is a condition on a group node and it's not scalar
@@ -141,18 +151,20 @@ class GroupNode(StrategyNode):
141
151
  if self.conditions and not is_scalar_condition(self.conditions):
142
152
  base.condition = None
143
153
  base.output_concepts = self.output_concepts + self.conditions.row_arguments
154
+ source_map = resolve_concept_map(
155
+ [base],
156
+ targets=self.output_concepts,
157
+ inherited_inputs=base.output_concepts,
158
+ )
144
159
  return QueryDatasource(
145
160
  input_concepts=base.output_concepts,
146
161
  output_concepts=self.output_concepts,
147
162
  datasources=[base],
148
163
  source_type=SourceType.SELECT,
149
- source_map=resolve_concept_map(
150
- [base],
151
- targets=self.output_concepts,
152
- inherited_inputs=base.output_concepts,
153
- ),
164
+ source_map=source_map,
154
165
  joins=[],
155
166
  grain=grain,
167
+ nullable_concepts=base.nullable_concepts,
156
168
  partial_concepts=self.partial_concepts,
157
169
  condition=self.conditions,
158
170
  )
@@ -168,6 +180,7 @@ class GroupNode(StrategyNode):
168
180
  parents=self.parents,
169
181
  depth=self.depth,
170
182
  partial_concepts=list(self.partial_concepts),
183
+ nullable_concepts=list(self.nullable_concepts),
171
184
  force_group=self.force_group,
172
185
  conditions=self.conditions,
173
186
  existence_concepts=list(self.existence_concepts),
@@ -22,7 +22,7 @@ from trilogy.core.processing.nodes.base_node import (
22
22
  resolve_concept_map,
23
23
  NodeJoin,
24
24
  )
25
- from trilogy.core.processing.utility import get_node_joins
25
+ from trilogy.core.processing.utility import get_node_joins, find_nullable_concepts
26
26
 
27
27
  LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
28
28
 
@@ -110,6 +110,7 @@ class MergeNode(StrategyNode):
110
110
  join_concepts: Optional[List] = None,
111
111
  force_join_type: Optional[JoinType] = None,
112
112
  partial_concepts: Optional[List[Concept]] = None,
113
+ nullable_concepts: Optional[List[Concept]] = None,
113
114
  force_group: bool | None = None,
114
115
  depth: int = 0,
115
116
  grain: Grain | None = None,
@@ -127,6 +128,7 @@ class MergeNode(StrategyNode):
127
128
  parents=parents,
128
129
  depth=depth,
129
130
  partial_concepts=partial_concepts,
131
+ nullable_concepts=nullable_concepts,
130
132
  force_group=force_group,
131
133
  grain=grain,
132
134
  conditions=conditions,
@@ -192,7 +194,7 @@ class MergeNode(StrategyNode):
192
194
  pregrain: Grain,
193
195
  grain: Grain,
194
196
  environment: Environment,
195
- ) -> List[BaseJoin]:
197
+ ) -> List[BaseJoin | UnnestJoin]:
196
198
  # only finally, join between them for unique values
197
199
  dataset_list: List[QueryDatasource] = sorted(
198
200
  final_datasets, key=lambda x: -len(x.grain.components_copy)
@@ -308,7 +310,7 @@ class MergeNode(StrategyNode):
308
310
  )
309
311
  join_candidates = [x for x in final_datasets if x not in existence_final]
310
312
  if len(join_candidates) > 1:
311
- joins = self.generate_joins(
313
+ joins: List[BaseJoin | UnnestJoin] = self.generate_joins(
312
314
  join_candidates, final_joins, pregrain, grain, self.environment
313
315
  )
314
316
  else:
@@ -318,7 +320,7 @@ class MergeNode(StrategyNode):
318
320
  )
319
321
  full_join_concepts = []
320
322
  for join in joins:
321
- if join.join_type == JoinType.FULL:
323
+ if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
322
324
  full_join_concepts += join.concepts
323
325
  if self.whole_grain:
324
326
  force_group = False
@@ -341,6 +343,9 @@ class MergeNode(StrategyNode):
341
343
  inherited_inputs=self.input_concepts + self.existence_concepts,
342
344
  full_joins=full_join_concepts,
343
345
  )
346
+ nullable_concepts = find_nullable_concepts(
347
+ source_map=source_map, joins=joins, datasources=final_datasets
348
+ )
344
349
  qds = QueryDatasource(
345
350
  input_concepts=unique(self.input_concepts, "address"),
346
351
  output_concepts=unique(self.output_concepts, "address"),
@@ -349,6 +354,9 @@ class MergeNode(StrategyNode):
349
354
  source_map=source_map,
350
355
  joins=qd_joins,
351
356
  grain=grain,
357
+ nullable_concepts=[
358
+ x for x in self.output_concepts if x.address in nullable_concepts
359
+ ],
352
360
  partial_concepts=self.partial_concepts,
353
361
  force_group=force_group,
354
362
  condition=self.conditions,
@@ -369,6 +377,7 @@ class MergeNode(StrategyNode):
369
377
  force_group=self.force_group,
370
378
  grain=self.grain,
371
379
  conditions=self.conditions,
380
+ nullable_concepts=list(self.nullable_concepts),
372
381
  hidden_concepts=list(self.hidden_concepts),
373
382
  virtual_output_concepts=list(self.virtual_output_concepts),
374
383
  node_joins=list(self.node_joins) if self.node_joins else None,
@@ -43,6 +43,7 @@ class SelectNode(StrategyNode):
43
43
  parents: List["StrategyNode"] | None = None,
44
44
  depth: int = 0,
45
45
  partial_concepts: List[Concept] | None = None,
46
+ nullable_concepts: List[Concept] | None = None,
46
47
  accept_partial: bool = False,
47
48
  grain: Optional[Grain] = None,
48
49
  force_group: bool | None = False,
@@ -58,6 +59,7 @@ class SelectNode(StrategyNode):
58
59
  parents=parents,
59
60
  depth=depth,
60
61
  partial_concepts=partial_concepts,
62
+ nullable_concepts=nullable_concepts,
61
63
  force_group=force_group,
62
64
  grain=grain,
63
65
  conditions=conditions,
@@ -115,6 +117,7 @@ class SelectNode(StrategyNode):
115
117
  partial_concepts=[
116
118
  c.concept for c in datasource.columns if not c.is_complete
117
119
  ],
120
+ nullable_concepts=[c.concept for c in datasource.columns if c.is_nullable],
118
121
  source_type=SourceType.DIRECT_SELECT,
119
122
  condition=self.conditions,
120
123
  # select nodes should never group
@@ -183,6 +186,7 @@ class SelectNode(StrategyNode):
183
186
  parents=self.parents,
184
187
  whole_grain=self.whole_grain,
185
188
  partial_concepts=list(self.partial_concepts),
189
+ nullable_concepts=list(self.nullable_concepts),
186
190
  accept_partial=self.accept_partial,
187
191
  grain=self.grain,
188
192
  force_group=self.force_group,
@@ -18,9 +18,11 @@ from trilogy.core.models import (
18
18
  WindowItem,
19
19
  AggregateWrapper,
20
20
  DataType,
21
+ ConceptPair,
22
+ UnnestJoin,
21
23
  )
22
24
 
23
- from trilogy.core.enums import Purpose, Granularity, BooleanOperator
25
+ from trilogy.core.enums import Purpose, Granularity, BooleanOperator, Modifier
24
26
  from trilogy.core.constants import CONSTANT_DATASET
25
27
  from enum import Enum
26
28
  from trilogy.utility import unique
@@ -243,8 +245,23 @@ def get_node_joins(
243
245
  local_concepts = [
244
246
  c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
245
247
  ]
246
- else:
248
+ elif any(
249
+ [
250
+ c.address in [x.address for x in identifier_map[right].partial_concepts]
251
+ for c in local_concepts
252
+ ]
253
+ ) or any(
254
+ [
255
+ c.address in [x.address for x in identifier_map[left].nullable_concepts]
256
+ for c in local_concepts
257
+ ]
258
+ ):
247
259
  join_type = JoinType.LEFT_OUTER
260
+ local_concepts = [
261
+ c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
262
+ ]
263
+ else:
264
+ join_type = JoinType.INNER
248
265
  # remove any constants if other join keys exist
249
266
  local_concepts = [
250
267
  c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
@@ -287,7 +304,18 @@ def get_node_joins(
287
304
  )
288
305
  narg = (left_arg, right_arg)
289
306
  if narg not in join_tuples:
290
- join_tuples.append((left_arg, right_arg))
307
+ modifiers = set()
308
+ if left_arg.address in [
309
+ x.address for x in left_datasource.nullable_concepts
310
+ ] and right_arg.address in [
311
+ x.address for x in right_datasource.nullable_concepts
312
+ ]:
313
+ modifiers.add(Modifier.NULLABLE)
314
+ join_tuples.append(
315
+ ConceptPair(
316
+ left=left_arg, right=right_arg, modifiers=list(modifiers)
317
+ )
318
+ )
291
319
  final_joins_pre.append(
292
320
  BaseJoin(
293
321
  left_datasource=identifier_map[left],
@@ -412,3 +440,63 @@ def decompose_condition(
412
440
  else:
413
441
  chunks.append(conditional)
414
442
  return chunks
443
+
444
+
445
+ def find_nullable_concepts(
446
+ source_map: Dict[str, set[Datasource | QueryDatasource | UnnestJoin]],
447
+ datasources: List[Datasource | QueryDatasource],
448
+ joins: List[BaseJoin | UnnestJoin],
449
+ ) -> List[str]:
450
+ """give a set of datasources and joins, find the concepts
451
+ that may contain nulls in the output set
452
+ """
453
+ nullable_datasources = set()
454
+ datasource_map = {
455
+ x.identifier: x
456
+ for x in datasources
457
+ if isinstance(x, (Datasource, QueryDatasource))
458
+ }
459
+ for join in joins:
460
+ is_on_nullable_condition = False
461
+ if not isinstance(join, BaseJoin):
462
+ continue
463
+ if not join.concept_pairs:
464
+ continue
465
+ for pair in join.concept_pairs:
466
+ if pair.right.address in [
467
+ y.address
468
+ for y in datasource_map[
469
+ join.right_datasource.identifier
470
+ ].nullable_concepts
471
+ ]:
472
+ is_on_nullable_condition = True
473
+ break
474
+ if pair.left.address in [
475
+ y.address
476
+ for y in datasource_map[
477
+ join.left_datasource.identifier
478
+ ].nullable_concepts
479
+ ]:
480
+ is_on_nullable_condition = True
481
+ break
482
+ if is_on_nullable_condition:
483
+ nullable_datasources.add(datasource_map[join.right_datasource.identifier])
484
+ final_nullable = set()
485
+
486
+ for k, v in source_map.items():
487
+ local_nullable = [
488
+ x for x in datasources if k in [v.address for v in x.nullable_concepts]
489
+ ]
490
+ if all(
491
+ [
492
+ k in [v.address for v in x.nullable_concepts]
493
+ for x in datasources
494
+ if k in [z.address for z in x.output_concepts]
495
+ ]
496
+ ):
497
+ final_nullable.add(k)
498
+ all_ds = set([ds for ds in local_nullable]).union(nullable_datasources)
499
+ if nullable_datasources:
500
+ if set(v).issubset(all_ds):
501
+ final_nullable.add(k)
502
+ return list(sorted(final_nullable))
@@ -192,8 +192,6 @@ def resolve_cte_base_name_and_alias_v2(
192
192
  raw_joins: List[Join | InstantiatedUnnestJoin],
193
193
  ) -> Tuple[str | None, str | None]:
194
194
  joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
195
- # INFO trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
196
- # 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
197
195
  if (
198
196
  len(source.datasources) == 1
199
197
  and isinstance(source.datasources[0], Datasource)
@@ -297,6 +295,7 @@ def datasource_to_ctes(
297
295
  parent_ctes=parents,
298
296
  condition=query_datasource.condition,
299
297
  partial_concepts=query_datasource.partial_concepts,
298
+ nullable_concepts=query_datasource.nullable_concepts,
300
299
  join_derived_concepts=query_datasource.join_derived_concepts,
301
300
  hidden_concepts=query_datasource.hidden_concepts,
302
301
  base_name_override=base_name,
trilogy/dialect/common.py CHANGED
@@ -3,9 +3,9 @@ from trilogy.core.enums import UnnestMode, Modifier
3
3
  from typing import Optional, Callable
4
4
 
5
5
 
6
- def null_wrapper(lval: str, rval: str, concept: Concept) -> str:
7
- if concept.modifiers and Modifier.NULLABLE in concept.modifiers:
8
- return f"(({lval} is null and {rval} is null) or ({lval} = {rval}))"
6
+ def null_wrapper(lval: str, rval: str, modifiers: list[Modifier]) -> str:
7
+ if Modifier.NULLABLE in modifiers:
8
+ return f"({lval} = {rval} or ({lval} is null and {rval} is null))"
9
9
  return f"{lval} = {rval}"
10
10
 
11
11
 
@@ -48,7 +48,7 @@ def render_join(
48
48
  null_wrapper(
49
49
  f"{left_name}.{quote_character}{join.left_cte.get_alias(key.concept) if isinstance(join.left_cte, Datasource) else key.concept.safe_address}{quote_character}",
50
50
  f"{right_name}.{quote_character}{join.right_cte.get_alias(key.concept) if isinstance(join.right_cte, Datasource) else key.concept.safe_address}{quote_character}",
51
- key.concept,
51
+ modifiers=key.concept.modifiers or [],
52
52
  )
53
53
  for key in join.joinkeys
54
54
  ]
@@ -56,11 +56,13 @@ def render_join(
56
56
  base_joinkeys.extend(
57
57
  [
58
58
  null_wrapper(
59
- f"{left_name}.{quote_character}{join.left_cte.get_alias(left_concept) if isinstance(join.left_cte, Datasource) else left_concept.safe_address}{quote_character}",
60
- f"{right_name}.{quote_character}{join.right_cte.get_alias(right_concept) if isinstance(join.right_cte, Datasource) else right_concept.safe_address}{quote_character}",
61
- left_concept,
59
+ f"{left_name}.{quote_character}{join.left_cte.get_alias(pair.left) if isinstance(join.left_cte, Datasource) else pair.left.safe_address}{quote_character}",
60
+ f"{right_name}.{quote_character}{join.right_cte.get_alias(pair.right) if isinstance(join.right_cte, Datasource) else pair.right.safe_address}{quote_character}",
61
+ modifiers=pair.modifiers
62
+ + (pair.left.modifiers or [])
63
+ + (pair.right.modifiers or []),
62
64
  )
63
- for left_concept, right_concept in join.joinkey_pairs
65
+ for pair in join.joinkey_pairs
64
66
  ]
65
67
  )
66
68
  if not base_joinkeys:
trilogy/parsing/common.py CHANGED
@@ -25,6 +25,15 @@ from trilogy.core.enums import PurposeLineage
25
25
  from trilogy.constants import (
26
26
  VIRTUAL_CONCEPT_PREFIX,
27
27
  )
28
+ from trilogy.core.enums import Modifier
29
+
30
+
31
+ def get_upstream_modifiers(keys: List[Concept]) -> list[Modifier]:
32
+ modifiers = set()
33
+ for pkey in keys:
34
+ if pkey.modifiers:
35
+ modifiers.update(pkey.modifiers)
36
+ return list(modifiers)
28
37
 
29
38
 
30
39
  def process_function_args(
@@ -125,7 +134,7 @@ def constant_to_concept(
125
134
 
126
135
 
127
136
  def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
128
- pkeys = []
137
+ pkeys: List[Concept] = []
129
138
  for x in parent.arguments:
130
139
  pkeys += [
131
140
  x
@@ -135,7 +144,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
135
144
  grain = Grain()
136
145
  for x in pkeys:
137
146
  grain += x.grain
138
-
147
+ modifiers = get_upstream_modifiers(pkeys)
139
148
  key_grain = []
140
149
  for x in pkeys:
141
150
  if x.keys:
@@ -155,6 +164,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
155
164
  namespace=namespace,
156
165
  grain=grain,
157
166
  keys=keys,
167
+ modifiers=modifiers,
158
168
  )
159
169
 
160
170
 
@@ -166,6 +176,7 @@ def filter_item_to_concept(
166
176
  metadata: Metadata | None = None,
167
177
  ) -> Concept:
168
178
  fmetadata = metadata or Metadata()
179
+ modifiers = get_upstream_modifiers(parent.content.concept_arguments)
169
180
  return Concept(
170
181
  name=name,
171
182
  datatype=parent.content.datatype,
@@ -184,6 +195,7 @@ def filter_item_to_concept(
184
195
  if parent.content.purpose == Purpose.PROPERTY
185
196
  else Grain()
186
197
  ),
198
+ modifiers=modifiers,
187
199
  )
188
200
 
189
201
 
@@ -202,6 +214,7 @@ def window_item_to_concept(
202
214
  grain += [item.expr.output]
203
215
  else:
204
216
  grain = parent.over + [parent.content.output]
217
+ modifiers = get_upstream_modifiers(parent.content.concept_arguments)
205
218
  return Concept(
206
219
  name=name,
207
220
  datatype=parent.content.datatype,
@@ -212,6 +225,7 @@ def window_item_to_concept(
212
225
  grain=Grain(components=grain),
213
226
  namespace=namespace,
214
227
  keys=keys,
228
+ modifiers=modifiers,
215
229
  )
216
230
 
217
231
 
@@ -229,6 +243,7 @@ def agg_wrapper_to_concept(
229
243
  # at that grain
230
244
  fmetadata = metadata or Metadata()
231
245
  aggfunction = parent.function
246
+ modifiers = get_upstream_modifiers(parent.concept_arguments)
232
247
  out = Concept(
233
248
  name=name,
234
249
  datatype=aggfunction.output_datatype,
@@ -238,6 +253,7 @@ def agg_wrapper_to_concept(
238
253
  grain=Grain(components=parent.by) if parent.by else Grain(),
239
254
  namespace=namespace,
240
255
  keys=tuple(parent.by) if parent.by else keys,
256
+ modifiers=modifiers,
241
257
  )
242
258
  return out
243
259