pytrilogy 0.0.2.7__tar.gz → 0.0.2.9__tar.gz

{pytrilogy-0.0.2.7/pytrilogy.egg-info → pytrilogy-0.0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.7
+Version: 0.0.2.9
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.7
+Version: 0.0.2.9
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.7"
+__version__ = "0.0.2.9"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/constants.py

@@ -24,6 +24,7 @@ class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
     constant_inlining: bool = True
+    constant_inline_cutoff: int = 2
     direct_return: bool = True
 
 

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/core/enums.py

@@ -123,6 +123,7 @@ class FunctionType(Enum):
     MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
     STRUCT = "struct"
+    ARRAY = "array"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/core/models.py

@@ -154,7 +154,10 @@ class ConceptArgs(ABC):
 class SelectContext(ABC):
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ):
         raise NotImplementedError
 
@@ -166,6 +169,7 @@ class ConstantInlineable(ABC):
 
 class SelectTypeMixin(BaseModel):
     where_clause: Union["WhereClause", None] = Field(default=None)
+    having_clause: Union["HavingClause", None] = Field(default=None)
 
     @property
     def output_components(self) -> List[Concept]:
@@ -595,13 +599,16 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         self,
         grain: Optional["Grain"] = None,
         conditional: Conditional | Comparison | Parenthetical | None = None,
+        environment: Environment | None = None,
     ) -> "Concept":
         if not all([isinstance(x, Concept) for x in self.keys or []]):
             raise ValueError(f"Invalid keys {self.keys} for concept {self.address}")
         new_grain = grain or self.grain
         new_lineage = self.lineage
         if isinstance(self.lineage, SelectContext):
-            new_lineage = self.lineage.with_select_context(new_grain, conditional)
+            new_lineage = self.lineage.with_select_context(
+                new_grain, conditional, environment=environment
+            )
         return self.__class__(
             name=self.name,
             datatype=self.datatype,
@@ -788,7 +795,9 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         return Granularity.MULTI_ROW
 
     def with_filter(
-        self, condition: "Conditional | Comparison | Parenthetical"
+        self,
+        condition: "Conditional | Comparison | Parenthetical",
+        environment: Environment | None = None,
     ) -> "Concept":
         from trilogy.utility import string_to_hash
 
@@ -805,12 +814,15 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
             modifiers=self.modifiers,
             pseudonyms=self.pseudonyms,
         )
+        if environment:
+            environment.add_concept(new)
         return new
 
 
 class Grain(Mergeable, BaseModel):
     nested: bool = False
     components: List[Concept] = Field(default_factory=list, validate_default=True)
+    where_clause: Optional[WhereClause] = Field(default=None)
 
     @field_validator("components")
     def component_validator(cls, v, info: ValidationInfo):
@@ -836,10 +848,12 @@ class Grain(Mergeable, BaseModel):
 
     def __str__(self):
         if self.abstract:
-            return (
-                "Grain<Abstract" + ",".join([c.address for c in self.components]) + ">"
-            )
-        return "Grain<" + ",".join([c.address for c in self.components]) + ">"
+            base = "Grain<Abstract>"
+        else:
+            base = "Grain<" + ",".join([c.address for c in self.components]) + ">"
+        if self.where_clause:
+            base += f"|{str(self.where_clause)}"
+        return base
 
     def with_namespace(self, namespace: str) -> "Grain":
         return Grain(
@@ -1046,12 +1060,15 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
         return self.output_datatype
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> Function:
         if self.operator in FunctionClass.AGGREGATE_FUNCTIONS.value and conditional:
             base = [
                 (
-                    c.with_select_context(grain, conditional)
+                    c.with_select_context(grain, conditional, environment)
                     if isinstance(
                         c,
                         SelectContext,
@@ -1061,7 +1078,7 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
                 for c in self.arguments
             ]
             final = [
-                c.with_filter(conditional) if isinstance(c, Concept) else c
+                c.with_filter(conditional, environment) if isinstance(c, Concept) else c
                 for c in base
             ]
             return Function(
@@ -1077,7 +1094,7 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
             operator=self.operator,
             arguments=[
                 (
-                    c.with_select_context(grain, conditional)
+                    c.with_select_context(grain, conditional, environment)
                     if isinstance(
                         c,
                         SelectContext,
@@ -1293,13 +1310,22 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> "WindowItem":
         return WindowItem(
             type=self.type,
-            content=self.content.with_select_context(grain, conditional),
-            over=[x.with_select_context(grain, conditional) for x in self.over],
-            order_by=[x.with_select_context(grain, conditional) for x in self.order_by],
+            content=self.content.with_select_context(grain, conditional, environment),
+            over=[
+                x.with_select_context(grain, conditional, environment)
+                for x in self.over
+            ],
+            order_by=[
+                x.with_select_context(grain, conditional, environment)
+                for x in self.order_by
+            ],
         )
 
     @property
@@ -1368,11 +1394,14 @@ class FilterItem(Namespaced, SelectContext, BaseModel):
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> FilterItem:
         return FilterItem(
-            content=self.content.with_select_context(grain, conditional),
-            where=self.where.with_select_context(grain, conditional),
+            content=self.content.with_select_context(grain, conditional, environment),
+            where=self.where.with_select_context(grain, conditional, environment),
         )
 
     @property
@@ -1452,9 +1481,17 @@ class OrderItem(Mergeable, SelectContext, Namespaced, BaseModel):
         return OrderItem(expr=self.expr.with_namespace(namespace), order=self.order)
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> "OrderItem":
-        return OrderItem(expr=self.expr.with_grain(grain), order=self.order)
+        return OrderItem(
+            expr=self.expr.with_select_context(
+                grain, conditional=conditional, environment=environment
+            ),
+            order=self.order,
+        )
 
     def with_merge(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
@@ -1643,7 +1680,9 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
                 )
             ):
                 output.append(item)
-        return Grain(components=unique(output, "address"))
+        return Grain(
+            components=unique(output, "address"), where_clause=self.where_clause
+        )
 
     def with_namespace(self, namespace: str) -> "SelectStatement":
         return SelectStatement(
@@ -2066,6 +2105,7 @@ class Datasource(Namespaced, BaseModel):
 class UnnestJoin(BaseModel):
     concept: Concept
     alias: str = "unnest"
+    rendering_required: bool = True
 
     def __hash__(self):
         return (self.alias + self.concept.address).__hash__()
@@ -2228,17 +2268,14 @@ class QueryDatasource(BaseModel):
     @classmethod
     def validate_source_map(cls, v, info: ValidationInfo):
         values = info.data
-        expected = {c.address for c in values["output_concepts"]}.union(
-            c.address for c in values["input_concepts"]
-        )
-        seen = set()
-        for k, _ in v.items():
-            seen.add(k)
-        for x in expected:
-            if x not in seen and CONFIG.validate_missing:
-                raise SyntaxError(
-                    f"source map missing {x} on (expected {expected}, have {seen})"
-                )
+        for key in ("input_concepts", "output_concepts"):
+            if not values.get(key):
+                continue
+            for concept in values[key]:
+                if concept.address not in v and CONFIG.validate_missing:
+                    raise SyntaxError(
+                        f"Missing source map for {concept.address} on {key}, have {v}"
+                    )
         return v
 
     def __str__(self):
@@ -2460,10 +2497,17 @@ class CTE(BaseModel):
                     for join in self.joins
                     if not isinstance(join, Join)
                     or (
-                        join.right_cte.name != removed_cte
-                        and join.left_cte.name != removed_cte
+                        isinstance(join, Join)
+                        and (
+                            join.right_cte.name != removed_cte
+                            and join.left_cte.name != removed_cte
+                        )
                     )
                 ]
+                for join in self.joins:
+                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                        join.rendering_required = False
+
                 self.parent_ctes = [
                     x for x in self.parent_ctes if x.name != removed_cte
                 ]
@@ -2841,6 +2885,7 @@ class UndefinedConcept(Concept, Mergeable, Namespaced):
         self,
         grain: Optional["Grain"] = None,
         conditional: Conditional | Comparison | Parenthetical | None = None,
+        environment: Environment | None = None,
     ) -> "UndefinedConcept":
         if not all([isinstance(x, Concept) for x in self.keys or []]):
             raise ValueError(f"Invalid keys {self.keys} for concept {self.address}")
@@ -2848,7 +2893,9 @@ class UndefinedConcept(Concept, Mergeable, Namespaced):
         if self.lineage:
             new_lineage = self.lineage
             if isinstance(self.lineage, SelectContext):
-                new_lineage = self.lineage.with_select_context(new_grain, conditional)
+                new_lineage = self.lineage.with_select_context(
+                    new_grain, conditional, environment
+                )
         else:
             new_lineage = None
         return self.__class__(
@@ -3291,7 +3338,9 @@ class LazyEnvironment(Environment):
         ) or name.startswith("_"):
             return super().__getattribute__(name)
         if not self.loaded:
-            print(f"lazily evaluating load path {self.load_path} to access {name}")
+            logger.info(
+                f"lazily evaluating load path {self.load_path} to access {name}"
+            )
             from trilogy import parse
 
             env = Environment(working_path=str(self.working_path))
@@ -3433,16 +3482,23 @@ class Comparison(
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ):
         return self.__class__(
             left=(
-                self.left.with_select_context(grain, conditional)
+                self.left.with_select_context(grain, conditional, environment)
                 if isinstance(self.left, SelectContext)
                 else self.left
             ),
             # the right side does NOT need to inherit select grain
-            right=self.right,
+            right=(
+                self.right.with_select_context(grain, conditional, environment)
+                if isinstance(self.right, SelectContext)
+                else self.right
+            ),
             operator=self.operator,
         )
 
@@ -3526,12 +3582,15 @@ class SubselectComparison(Comparison):
         return [tuple(get_concept_arguments(self.right))]
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ):
-        # there's no need to pass the select grain through to a subselect comparison
+        # there's no need to pass the select grain through to a subselect comparison on the right
         return self.__class__(
             left=(
-                self.left.with_select_context(grain, conditional)
+                self.left.with_select_context(grain, conditional, environment)
                 if isinstance(self.left, SelectContext)
                 else self.left
             ),
@@ -3562,12 +3621,17 @@ class CaseWhen(Namespaced, SelectContext, BaseModel):
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> CaseWhen:
         return CaseWhen(
-            comparison=self.comparison.with_select_context(grain, conditional),
+            comparison=self.comparison.with_select_context(
+                grain, conditional, environment
+            ),
             expr=(
-                (self.expr.with_select_context(grain, conditional))
+                (self.expr.with_select_context(grain, conditional, environment))
                 if isinstance(self.expr, SelectContext)
                 else self.expr
             ),
@@ -3584,12 +3648,15 @@ class CaseElse(Namespaced, SelectContext, BaseModel):
         return get_concept_arguments(self.expr)
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> CaseElse:
         return CaseElse(
             discriminant=self.discriminant,
             expr=(
-                self.expr.with_select_context(grain, conditional)
+                self.expr.with_select_context(grain, conditional, environment)
                 if isinstance(
                     self.expr,
                     SelectContext,
@@ -3729,16 +3796,19 @@ class Conditional(
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ):
         return Conditional(
             left=(
-                self.left.with_select_context(grain, conditional)
+                self.left.with_select_context(grain, conditional, environment)
                 if isinstance(self.left, SelectContext)
                 else self.left
             ),
             right=(
-                self.right.with_select_context(grain, conditional)
+                self.right.with_select_context(grain, conditional, environment)
                 if isinstance(self.right, SelectContext)
                 else self.right
             ),
@@ -3847,13 +3917,16 @@ class AggregateWrapper(Mergeable, Namespaced, SelectContext, BaseModel):
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> AggregateWrapper:
         if not self.by:
             by = grain.components_copy
         else:
             by = self.by
-        parent = self.function.with_select_context(grain, conditional)
+        parent = self.function.with_select_context(grain, conditional, environment)
         return AggregateWrapper(function=parent, by=by)
 
 
@@ -3885,10 +3958,15 @@ class WhereClause(Mergeable, ConceptArgs, Namespaced, SelectContext, BaseModel):
         return WhereClause(conditional=self.conditional.with_namespace(namespace))
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ) -> WhereClause:
         return WhereClause(
-            conditional=self.conditional.with_select_context(grain, conditional)
+            conditional=self.conditional.with_select_context(
+                grain, conditional, environment
+            )
         )
 
     @property
@@ -3901,6 +3979,22 @@ class WhereClause(Mergeable, ConceptArgs, Namespaced, SelectContext, BaseModel):
                 output += item.grain.components if item.grain else []
         return Grain(components=list(set(output)))
 
+    @property
+    def components(self):
+        from trilogy.core.processing.utility import decompose_condition
+
+        return decompose_condition(self.conditional)
+
+    @property
+    def is_scalar(self):
+        from trilogy.core.processing.utility import is_scalar_condition
+
+        return is_scalar_condition(self.conditional)
+
+
+class HavingClause(WhereClause):
+    pass
+
 
 class MaterializedDataset(BaseModel):
     address: Address
@@ -3920,6 +4014,7 @@ class ProcessedQuery(BaseModel):
     hidden_columns: List[Concept] = Field(default_factory=list)
     limit: Optional[int] = None
     where_clause: Optional[WhereClause] = None
+    having_clause: Optional[HavingClause] = None
     order_by: Optional[OrderBy] = None
 
 
@@ -4122,11 +4217,14 @@ class Parenthetical(
         )
 
     def with_select_context(
-        self, grain: Grain, conditional: Conditional | Comparison | Parenthetical | None
+        self,
+        grain: Grain,
+        conditional: Conditional | Comparison | Parenthetical | None,
+        environment: Environment | None = None,
     ):
         return Parenthetical(
             content=(
-                self.content.with_select_context(grain, conditional)
+                self.content.with_select_context(grain, conditional, environment)
                 if isinstance(self.content, SelectContext)
                 else self.content
             )

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/core/optimization.py

@@ -17,19 +17,58 @@ from trilogy.core.optimizations import (
 MAX_OPTIMIZATION_LOOPS = 100
 
 
+# other optimizations may make a CTE a pure passthrough
+# remove those
+# def is_locally_irrelevant(cte: CTE) -> CTE | bool:
+#     if not len(cte.parent_ctes) == 1:
+#         return False
+#     parent = cte.parent_ctes[0]
+#     if not parent.output_columns == cte.output_columns:
+#         return False
+#     if cte.condition is not None:
+#         return False
+#     if cte.group_to_grain:
+#         return False
+#     if len(cte.joins)>1:
+#         return False
+#     return parent
+
+
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
 ):
     relevant_ctes = set()
 
-    def recurse(cte: CTE):
+    def recurse(cte: CTE, inverse_map: dict[str, list[CTE]]):
+        # TODO: revisit this
+        # if parent := is_locally_irrelevant(cte):
+        #     logger.info(
+        #         f"[Optimization][Irrelevent CTE filtering] Removing redundant CTE {cte.name} and replacing with {parent.name}"
+        #     )
+        #     for child in inverse_map.get(cte.name, []):
+        #         child.parent_ctes = [
+        #             x for x in child.parent_ctes if x.name != cte.name
+        #         ] + [parent]
+        #         for x in child.source_map:
+        #             if cte.name in child.source_map[x]:
+        #                 child.source_map[x].remove(cte.name)
+        #                 child.source_map[x].append(parent.name)
+        #         for x2 in child.existence_source_map:
+        #             if cte.name in child.existence_source_map[x2]:
+        #                 child.existence_source_map[x2].remove(cte.name)
+        #                 child.existence_source_map[x2].append(parent.name)
+        # else:
         relevant_ctes.add(cte.name)
         for cte in cte.parent_ctes:
-            recurse(cte)
-
-    recurse(root_cte)
-    return [cte for cte in input if cte.name in relevant_ctes]
+            recurse(cte, inverse_map)
+
+    inverse_map = gen_inverse_map(input)
+    recurse(root_cte, inverse_map)
+    final = [cte for cte in input if cte.name in relevant_ctes]
+    if len(final) == len(input):
+        return input
+    return filter_irrelevant_ctes(final, root_cte)
 
 
 def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:

{pytrilogy-0.0.2.7 → pytrilogy-0.0.2.9}/trilogy/core/optimizations/inline_datasource.py

@@ -5,6 +5,7 @@ from trilogy.core.models import (
 
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 from collections import defaultdict
+from trilogy.constants import CONFIG
 
 
 class InlineDatasource(OptimizationRule):
@@ -18,28 +19,28 @@ class InlineDatasource(OptimizationRule):
         if not cte.parent_ctes:
             return False
 
-        self.log(
+        self.debug(
             f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
         )
         to_inline: list[CTE] = []
         force_group = False
         for parent_cte in cte.parent_ctes:
             if not parent_cte.is_root_datasource:
-                self.log(f"parent {parent_cte.name} is not root")
+                self.debug(f"parent {parent_cte.name} is not root")
                 continue
             if parent_cte.parent_ctes:
-                self.log(f"parent {parent_cte.name} has parents")
+                self.debug(f"parent {parent_cte.name} has parents")
                 continue
             if parent_cte.condition:
-                self.log(f"parent {parent_cte.name} has condition, cannot be inlined")
+                self.debug(f"parent {parent_cte.name} has condition, cannot be inlined")
                 continue
             raw_root = parent_cte.source.datasources[0]
             if not isinstance(raw_root, Datasource):
-                self.log(f"parent {parent_cte.name} is not datasource")
+                self.debug(f"Parent {parent_cte.name} is not datasource")
                 continue
             root: Datasource = raw_root
             if not root.can_be_inlined:
-                self.log(f"parent {parent_cte.name} datasource is not inlineable")
+                self.debug(f"Parent {parent_cte.name} datasource is not inlineable")
                 continue
             root_outputs = {x.address for x in root.output_concepts}
             inherited = {
@@ -52,7 +53,9 @@ class InlineDatasource(OptimizationRule):
                 )
                 continue
             if not root.grain.issubset(parent_cte.grain):
-                self.log(f"Not all {parent_cte.name} is at wrong grain to inline")
+                self.log(
+                    f"{parent_cte.name} is at wrong grain to inline ({root.grain} vs {parent_cte.grain})"
+                )
                 continue
             to_inline.append(parent_cte)
 
@@ -62,7 +65,10 @@ class InlineDatasource(OptimizationRule):
                 self.candidates[cte.name].add(replaceable.name)
                 self.count[replaceable.source.name] += 1
                 return True
-            if self.count[replaceable.source.name] > 1:
+            if (
+                self.count[replaceable.source.name]
+                > CONFIG.optimizations.constant_inline_cutoff
+            ):
                 self.log(
                     f"Skipping inlining raw datasource {replaceable.source.name} ({replaceable.name}) due to multiple references"
                 )