pytrilogy 0.0.3.101__tar.gz → 0.0.3.103__tar.gz

{pytrilogy-0.0.3.101/pytrilogy.egg-info → pytrilogy-0.0.3.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.101
+Version: 0.0.3.103
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.101
+Version: 0.0.3.103
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/pytrilogy.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ trilogy/core/models/environment.py
 trilogy/core/models/execute.py
 trilogy/core/optimizations/__init__.py
 trilogy/core/optimizations/base_optimization.py
+trilogy/core/optimizations/hide_unused_concept.py
 trilogy/core/optimizations/inline_datasource.py
 trilogy/core/optimizations/predicate_pushdown.py
 trilogy/core/processing/__init__.py

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/pytrilogy.egg-info/requires.txt

@@ -4,6 +4,7 @@ sqlalchemy<2.0.0
 networkx
 pyodbc
 pydantic
+duckdb<1.4.0
 duckdb-engine
 click
 

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.101"
+__version__ = "0.0.3.103"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/models/author.py

@@ -259,6 +259,15 @@ class Parenthetical(
             )
         )
 
+    def with_reference_replacement(self, source, target):
+        return Parenthetical.model_construct(
+            content=(
+                self.content.with_reference_replacement(source, target)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            )
+        )
+
     @property
     def concept_arguments(self) -> Sequence[ConceptRef]:
         base: List[ConceptRef] = []

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/models/build.py

@@ -1511,7 +1511,10 @@ def requires_concept_nesting(
 ) -> AggregateWrapper | WindowItem | FilterItem | Function | None:
     if isinstance(expr, (AggregateWrapper, WindowItem, FilterItem)):
         return expr
-    if isinstance(expr, Function) and expr.operator == FunctionType.GROUP:
+    if isinstance(expr, Function) and expr.operator in (
+        FunctionType.GROUP,
+        FunctionType.PARENTHETICAL,
+    ):
         # group by requires nesting
         return expr
     return None
@@ -1696,13 +1699,12 @@ class Factory:
         return self._build_case_when(base)
 
     def _build_case_when(self, base: CaseWhen) -> BuildCaseWhen:
-        comparison = base.comparison
         expr: Concept | FuncArgs = base.expr
         validation = requires_concept_nesting(expr)
         if validation:
             expr, _ = self.instantiate_concept(validation)
         return BuildCaseWhen(
-            comparison=self.build(comparison),
+            comparison=self.build(base.comparison),
             expr=self.build(expr),
         )
 
@@ -2019,7 +2021,12 @@ class Factory:
         return self._build_parenthetical(base)
 
     def _build_parenthetical(self, base: Parenthetical) -> BuildParenthetical:
-        return BuildParenthetical(content=(self.build(base.content)))
+        validate = requires_concept_nesting(base.content)
+        if validate:
+            content, _ = self.instantiate_concept(validate)
+            return BuildParenthetical(content=self.build(content))
+        else:
+            return BuildParenthetical(content=self.build(base.content))
 
     @build.register
     def _(self, base: SelectLineage) -> BuildSelectLineage:

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/optimization.py

@@ -5,6 +5,7 @@ from trilogy.core.models.build import (
 )
 from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations import (
+    HideUnusedConcepts,
     InlineDatasource,
     OptimizationRule,
     PredicatePushdown,
@@ -84,11 +85,18 @@ def filter_irrelevant_ctes(
         #                 child.existence_source_map[x2].append(parent.name)
         # else:
         relevant_ctes.add(cte.name)
-        for cte in cte.parent_ctes:
-            recurse(cte, inverse_map)
+
+        for parent in cte.parent_ctes:
+            if parent.name in relevant_ctes:
+                logger.info(
+                    f"[Optimization][Irrelevent CTE filtering] Already visited {parent.name} when visting {cte.name}, potential recursive dag"
+                )
+                continue
+
+            recurse(parent, inverse_map)
         if isinstance(cte, UnionCTE):
-            for cte in cte.internal_ctes:
-                recurse(cte, inverse_map)
+            for internal in cte.internal_ctes:
+                recurse(internal, inverse_map)
 
     inverse_map = gen_inverse_map(input)
     recurse(root_cte, inverse_map)
@@ -220,6 +228,7 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
+    REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/optimizations/__init__.py

@@ -1,4 +1,5 @@
 from .base_optimization import OptimizationRule
+from .hide_unused_concept import HideUnusedConcepts
 from .inline_datasource import InlineDatasource
 from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 
@@ -7,4 +8,5 @@ __all__ = [
     "InlineDatasource",
     "PredicatePushdown",
     "PredicatePushdownRemove",
+    "HideUnusedConcepts",
 ]

pytrilogy-0.0.3.103/trilogy/core/optimizations/hide_unused_concept.py

@@ -0,0 +1,51 @@
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class HideUnusedConcepts(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        used = set()
+        from trilogy.dialect.base import BaseDialect
+
+        renderer = BaseDialect()
+        children = inverse_map.get(cte.name, [])
+        if not children:
+            return False
+        for v in children:
+            self.log(f"Analyzing usage of {cte.name} in {v.name}")
+            renderer.render_cte(v)
+        used = renderer.used_map.get(cte.name, set())
+        self.log(f"Used concepts for {cte.name}: {used} from {renderer.used_map}")
+        add_to_hidden: list[BuildConcept] = []
+        for concept in cte.output_columns:
+            if concept.address not in used:
+                add_to_hidden.append(concept)
+        newly_hidden = [
+            x.address for x in add_to_hidden if x.address not in cte.hidden_concepts
+        ]
+        non_hidden = [
+            x for x in cte.output_columns if x.address not in cte.hidden_concepts
+        ]
+        if not newly_hidden or len(non_hidden) <= 1:
+            return False
+        self.log(
+            f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
+        )
+        candidates = [
+            x.address
+            for x in cte.output_columns
+            if x.address not in used and x.address not in cte.hidden_concepts
+        ]
+        if len(candidates) == len(set([x.address for x in cte.output_columns])):
+            # pop one out
+            candidates.pop()
+        cte.hidden_concepts = set(candidates)
+        return True

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/processing/discovery_node_factory.py

@@ -376,11 +376,6 @@ class RootNodeHandler:
 
         if pseudonyms:
             expanded.add_output_concepts(pseudonyms)
-            logger.info(
-                f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
-                f"Hiding pseudonyms {[c.address for c in pseudonyms]}"
-            )
-            expanded.hide_output_concepts(pseudonyms)
 
         logger.info(
             f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/processing/node_generators/rowset_node.py

@@ -64,14 +64,6 @@ def gen_rowset_node(
         v for v in concept_pool if v.address in rowset_outputs
     ]
 
-    select_hidden = node.hidden_concepts
-    rowset_hidden = [
-        x
-        for x in rowset_relevant
-        if x.address in lineage.rowset.derived_concepts
-        and isinstance(x.lineage, BuildRowsetItem)
-        and x.lineage.content.address in select_hidden
-    ]
     additional_relevant = [
         factory.build(x) for x in select.output_components if x.address in enrichment
     ]
@@ -84,18 +76,6 @@ def gen_rowset_node(
             )
             node.partial_concepts.append(item)
 
-    final_hidden = rowset_hidden + [
-        x
-        for x in node.output_concepts
-        if x.address not in local_optional + [concept]
-        and x.derivation != Derivation.ROWSET
-        and not any(z in lineage.rowset.derived_concepts for z in x.pseudonyms)
-    ]
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} hiding {final_hidden} local optional {local_optional}"
-    )
-    node.hide_output_concepts(final_hidden)
-
     node.grain = BuildGrain.from_concepts(
         [
             x

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/processing/node_generators/union_node.py

@@ -74,7 +74,7 @@ def gen_union_node(
             history=history,
             conditions=conditions,
         )
-        parent.hide_output_concepts(parent.output_concepts)
+
         parent.add_output_concepts(resolved)
         parent_nodes.append(parent)
         if not parent:

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/processing/node_generators/unnest_node.py

@@ -1,9 +1,19 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
-from trilogy.core.processing.nodes import History, StrategyNode, UnnestNode
+from trilogy.core.processing.nodes import (
+    History,
+    StrategyNode,
+    UnnestNode,
+    WhereSafetyNode,
+)
 from trilogy.core.processing.utility import padding
 
 LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
@@ -71,7 +81,9 @@ def gen_unnest_node(
                 return None
     else:
         parent = None
-
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} got parent {parent}"
+    )
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
@@ -83,7 +95,7 @@ def gen_unnest_node(
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
     conditional = conditions.conditional if conditions else None
-    new = StrategyNode(
+    new = WhereSafetyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,
         environment=environment,
@@ -92,9 +104,13 @@ def gen_unnest_node(
         preexisting_conditions=(
             conditional if conditional and local_conditions is False else None
         ),
+        grain=BuildGrain.from_concepts(
+            concepts=base.output_concepts,
+            environment=environment,
+        ),
     )
-    qds = new.resolve()
-    assert qds.source_map[concept.address] == {base.resolve()}
-    for x in equivalent_optional:
-        assert qds.source_map[x.address] == {base.resolve()}
+    # qds = new.resolve()
+    # assert qds.source_map[concept.address] == {base.resolve()}
+    # for x in equivalent_optional:
+    #     assert qds.source_map[x.address] == {base.resolve()}
     return new

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/core/processing/nodes/base_node.py

@@ -489,7 +489,6 @@ class WhereSafetyNode(StrategyNode):
             parent = parent.copy()
             # avoid performance hit by not rebuilding until end
             parent.set_output_concepts(self.output_concepts, rebuild=False)
-            parent.hide_output_concepts(self.hidden_concepts, rebuild=False)
 
             # these conditions
             if self.preexisting_conditions:

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/dialect/base.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from datetime import date, datetime
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
@@ -311,6 +312,7 @@ def safe_get_cte_value(
     c: BuildConcept,
     quote_char: str,
     render_expr: Callable,
+    use_map: dict[str, set[str]],
 ) -> Optional[str]:
     address = c.address
     raw = cte.source_map.get(address, None)
@@ -319,13 +321,17 @@ def safe_get_cte_value(
         return None
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
+        use_map[raw].add(c.address)
         return f"{quote_char}{raw}{quote_char}.{safe_quote(rendered, quote_char)}"
     if isinstance(raw, list) and len(raw) == 1:
         rendered = cte.get_alias(c, raw[0])
         if isinstance(rendered, FUNCTION_ITEMS):
             # if it's a function, we need to render it as a function
             return f"{render_expr(rendered, cte=cte, raise_invalid=True)}"
+        use_map[raw[0]].add(c.address)
         return f"{quote_char}{raw[0]}{quote_char}.{safe_quote(rendered, quote_char)}"
+    for x in raw:
+        use_map[x].add(c.address)
     return coalesce(
         sorted(
             [
@@ -350,13 +356,12 @@ class BaseDialect:
 
     def __init__(self, rendering: Rendering | None = None):
         self.rendering = rendering or CONFIG.rendering
+        self.used_map: dict[str, set[str]] = defaultdict(set)
 
     def render_order_item(
         self,
         order_item: BuildOrderItem,
         cte: CTE | UnionCTE,
-        final: bool = False,
-        alias: bool = True,
     ) -> str:
         # if final:
         #     if not alias:
@@ -527,6 +532,9 @@ class BaseDialect:
             )
 
             raw_content = cte.get_alias(c)
+            parent = cte.source_map.get(c.address, None)
+            if parent:
+                self.used_map[parent[0]].add(c.address)
             if isinstance(raw_content, RawColumnExpr):
                 rval = raw_content.text
             elif isinstance(raw_content, FUNCTION_ITEMS):
@@ -540,6 +548,7 @@ class BaseDialect:
                     c,
                     self.QUOTE_CHARACTER,
                     self.render_expr,
+                    self.used_map,
                 )
                 if not rval:
                     # unions won't have a specific source mapped; just use a generic column reference
@@ -615,6 +624,7 @@ class BaseDialect:
                 lookup_cte = cte
                 if cte_map and not lookup_cte:
                     lookup_cte = cte_map.get(e.right.address)
+
                 assert lookup_cte, "Subselects must be rendered with a CTE in context"
                 if e.right.address not in lookup_cte.existence_source_map:
                     lookup = lookup_cte.source_map.get(
@@ -634,6 +644,7 @@ class BaseDialect:
                         f"Missing source CTE for {e.right.address}"
                     )
                 assert cte, "CTE must be provided for inlined CTEs"
+                self.used_map[target].add(e.right.address)
                 if target in cte.inlined_ctes:
                     info = cte.inlined_ctes[target]
                     return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {info.new_base} as {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
@@ -738,6 +749,7 @@ class BaseDialect:
                     raise_invalid=raise_invalid,
                 )
             elif cte_map:
+                self.used_map[cte_map[e.address].name].add(e.address)
                 return f"{cte_map[e.address].name}.{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
             return f"{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
         elif isinstance(e, bool):
@@ -822,10 +834,7 @@ class BaseDialect:
             )
             if cte.order_by:
 
-                ordering = [
-                    self.render_order_item(i, cte, final=True, alias=False)
-                    for i in cte.order_by.items
-                ]
+                ordering = [self.render_order_item(i, cte) for i in cte.order_by.items]
                 base_statement += "\nORDER BY " + ",".join(ordering)
             return CompiledCTE(name=cte.name, statement=base_statement)
         elif isinstance(cte, RecursiveCTE):
@@ -950,7 +959,8 @@ class BaseDialect:
                             self.QUOTE_CHARACTER,
                             self.render_expr,
                             cte,
-                            self.UNNEST_MODE,
+                            use_map=self.used_map,
+                            unnest_mode=self.UNNEST_MODE,
                         )
                         for join in final_joins
                     ]

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/dialect/common.py

@@ -62,10 +62,12 @@ def render_join_concept(
     concept: BuildConcept,
     render_expr,
     inlined_ctes: set[str],
+    use_map: dict[str, set[str]],
 ):
     if cte.name in inlined_ctes:
         base = render_expr(concept, cte)
         return base
+    use_map[name].add(concept.address)
     return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 
@@ -85,6 +87,7 @@ def render_join(
         str,
     ],
     cte: CTE,
+    use_map: dict[str, set[str]],
     unnest_mode: UnnestMode = UnnestMode.CROSS_APPLY,
 ) -> str | None:
     # {% for key in join.joinkeys %}{{ key.inner }} = {{ key.outer}}{% endfor %}
@@ -121,6 +124,7 @@ def render_join(
                         pair.left,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     render_join_concept(
                         right_name,
@@ -129,6 +133,7 @@ def render_join(
                         pair.right,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     modifiers=pair.modifiers
                     + (pair.left.modifiers or [])

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/parsing/common.py

@@ -249,11 +249,14 @@ def atom_is_relevant(
         return atom_is_relevant(atom.left, others, environment) or atom_is_relevant(
             atom.right, others, environment
         )
+    elif isinstance(atom, Parenthetical):
+        return atom_is_relevant(atom.content, others, environment)
     elif isinstance(atom, ConceptArgs):
         # use atom is relevant here to trigger the early exit behavior for concepts in set
         return any(
             [atom_is_relevant(x, others, environment) for x in atom.concept_arguments]
         )
+
     return False
 
 
@@ -294,12 +297,18 @@ def concept_is_relevant(
         if all([c in others for c in concept.grain.components]):
             return False
     if concept.derivation in (Derivation.BASIC,) and isinstance(
-        concept.lineage, Function
+        concept.lineage, (Function, CaseWhen)
     ):
         relevant = False
         for arg in concept.lineage.arguments:
             relevant = atom_is_relevant(arg, others, environment) or relevant
+
         return relevant
+    if concept.derivation in (Derivation.BASIC,) and isinstance(
+        concept.lineage, Parenthetical
+    ):
+        return atom_is_relevant(concept.lineage.content, others, environment)
+
     if concept.granularity == Granularity.SINGLE_ROW:
         return False
     return True
@@ -346,6 +355,7 @@ def concepts_to_grain_concepts(
         if sub.address in seen:
             continue
         if not concept_is_relevant(sub, pconcepts, environment):  # type: ignore
+
             continue
         seen.add(sub.address)
 

{pytrilogy-0.0.3.101 → pytrilogy-0.0.3.103}/trilogy/parsing/parse_engine.py

@@ -992,6 +992,9 @@ class ParseToObjects(Transformer):
     def order_by(self, args):
         return OrderBy(items=args[0])
 
+    def over_component(self, args):
+        return ConceptRef(address=args[0].value.lstrip(",").strip())
+
     def over_list(self, args):
         return [x for x in args]