pytrilogy 0.3.148__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

trilogy/core/optimization.py

@@ -0,0 +1,251 @@
+from trilogy.constants import CONFIG, logger
+from trilogy.core.enums import BooleanOperator, Derivation
+from trilogy.core.models.build import (
+    BuildConditional,
+)
+from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
+from trilogy.core.optimizations import (
+    HideUnusedConcepts,
+    InlineDatasource,
+    OptimizationRule,
+    PredicatePushdown,
+    PredicatePushdownRemove,
+)
+from trilogy.core.processing.utility import sort_select_output
+from trilogy.core.statements.author import MultiSelectStatement, SelectStatement
+
+MAX_OPTIMIZATION_LOOPS = 100
+
+
+# other optimizations may make a CTE a pure passthrough
+# remove those
+# def is_locally_irrelevant(cte: CTE) -> CTE | bool:
+#     if not len(cte.parent_ctes) == 1:
+#         return False
+#     parent = cte.parent_ctes[0]
+#     if not parent.output_columns == cte.output_columns:
+#         return False
+#     if cte.condition is not None:
+#         return False
+#     if cte.group_to_grain:
+#         return False
+#     if len(cte.joins)>1:
+#         return False
+#     return parent
+
+
+def reorder_ctes(
+    input: list[CTE],
+):
+    import networkx as nx
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    mapping: dict[str, CTE] = {}
+    for cte in input:
+        mapping[cte.name] = cte
+        for parent in cte.parent_ctes:
+            G.add_edge(parent.name, cte.name)
+    # Perform topological sort (only works for DAGs)
+    try:
+        topological_order = list(nx.topological_sort(G))
+        if not topological_order:
+            return input
+        return [mapping[x] for x in topological_order]
+    except nx.NetworkXUnfeasible as e:
+        logger.error(
+            "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
+        )
+        raise e
+
+
+def filter_irrelevant_ctes(
+    input: list[CTE | UnionCTE],
+    root_cte: CTE | UnionCTE,
+):
+    relevant_ctes = set()
+
+    def recurse(cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]):
+        # TODO: revisit this
+        # if parent := is_locally_irrelevant(cte):
+        #     logger.info(
+        #         f"[Optimization][Irrelevent CTE filtering] Removing redundant CTE {cte.name} and replacing with {parent.name}"
+        #     )
+        #     for child in inverse_map.get(cte.name, []):
+        #         child.parent_ctes = [
+        #             x for x in child.parent_ctes if x.name != cte.name
+        #         ] + [parent]
+        #         for x in child.source_map:
+        #             if cte.name in child.source_map[x]:
+        #                 child.source_map[x].remove(cte.name)
+        #                 child.source_map[x].append(parent.name)
+        #         for x2 in child.existence_source_map:
+        #             if cte.name in child.existence_source_map[x2]:
+        #                 child.existence_source_map[x2].remove(cte.name)
+        #                 child.existence_source_map[x2].append(parent.name)
+        # else:
+        relevant_ctes.add(cte.name)
+
+        for parent in cte.parent_ctes:
+            if parent.name in relevant_ctes:
+                logger.info(
+                    f"[Optimization][Irrelevent CTE filtering] Already visited {parent.name} when visting {cte.name}, potential recursive dag"
+                )
+                continue
+
+            recurse(parent, inverse_map)
+        if isinstance(cte, UnionCTE):
+            for internal in cte.internal_ctes:
+                recurse(internal, inverse_map)
+
+    inverse_map = gen_inverse_map(input)
+    recurse(root_cte, inverse_map)
+    final = [cte for cte in input if cte.name in relevant_ctes]
+    filtered = [cte for cte in input if cte.name not in relevant_ctes]
+    if filtered:
+        logger.info(
+            f"[Optimization][Irrelevent CTE filtering] Removing redundant CTEs {[x.name for x in filtered]}"
+        )
+    if len(final) == len(input):
+        return input
+    return filter_irrelevant_ctes(final, root_cte)
+
+
+def gen_inverse_map(input: list[CTE | UnionCTE]) -> dict[str, list[CTE | UnionCTE]]:
+    inverse_map: dict[str, list[CTE | UnionCTE]] = {}
+    for cte in input:
+        if isinstance(cte, UnionCTE):
+            for internal in cte.internal_ctes:
+                if internal.name not in inverse_map:
+                    inverse_map[internal.name] = []
+                inverse_map[internal.name].append(cte)
+        else:
+            for parent in cte.parent_ctes:
+                if parent.name not in inverse_map:
+                    inverse_map[parent.name] = []
+                inverse_map[parent.name].append(cte)
+
+    return inverse_map
+
+
+SENSITIVE_DERIVATIONS = [
+    Derivation.UNNEST,
+    Derivation.WINDOW,
+    Derivation.RECURSIVE,
+]
+
+
+def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
+    # if isinstance(select, (PersistStatement, MultiSelectStatement)):
+    #     return False
+    if len(cte.parent_ctes) != 1:
+        return None
+    direct_parent = cte.parent_ctes[0]
+    if isinstance(direct_parent, (UnionCTE, RecursiveCTE)):
+        return None
+
+    output_addresses = set([x.address for x in cte.output_columns])
+    parent_output_addresses = set([x.address for x in direct_parent.output_columns])
+    if not output_addresses.issubset(parent_output_addresses):
+        return None
+    if not direct_parent.grain == cte.grain:
+        logger.info("[Direct Return] grain mismatch, cannot early exit")
+        return None
+
+    assert isinstance(cte, CTE)
+    derived_concepts = [
+        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
+    ]
+
+    parent_derived_concepts = [
+        c
+        for c in direct_parent.source.output_concepts
+        if c not in direct_parent.source.input_concepts
+    ]
+    condition_arguments = cte.condition.row_arguments if cte.condition else []
+    for x in derived_concepts:
+        if x.derivation in SENSITIVE_DERIVATIONS:
+            return None
+    for x in parent_derived_concepts:
+        if x.address not in condition_arguments:
+            continue
+        if x.derivation in SENSITIVE_DERIVATIONS:
+            return None
+    for x in condition_arguments:
+        # if it's derived in the parent
+        if x.address in parent_derived_concepts:
+            if x.derivation in SENSITIVE_DERIVATIONS:
+                return None
+            # this maybe needs to be recursive if we flatten a ton of derivation
+            # into one CTE
+            if not x.lineage:
+                continue
+            for z in x.lineage.concept_arguments:
+                # if it was preexisting in the parent, it's safe
+                if z.address in direct_parent.source.input_concepts:
+                    continue
+                # otherwise if it's dangerous, play it safe.
+                if z.derivation in SENSITIVE_DERIVATIONS:
+                    return None
+    logger.info(
+        f"[Optimization][EarlyReturn] Removing redundant output CTE {cte.name} with derived_concepts {[x.address for x in derived_concepts]}"
+    )
+    return direct_parent
+
+
+def optimize_ctes(
+    input: list[CTE | UnionCTE],
+    root_cte: CTE | UnionCTE,
+    select: SelectStatement | MultiSelectStatement,
+) -> list[CTE | UnionCTE]:
+    direct_parent: CTE | UnionCTE | None = root_cte
+    while CONFIG.optimizations.direct_return and (
+        direct_parent := is_direct_return_eligible(root_cte)
+    ):
+        direct_parent.order_by = root_cte.order_by
+        direct_parent.limit = root_cte.limit
+        direct_parent.hidden_concepts = root_cte.hidden_concepts.union(
+            direct_parent.hidden_concepts
+        )
+        if root_cte.condition:
+            if direct_parent.condition:
+                direct_parent.condition = BuildConditional(
+                    left=direct_parent.condition,
+                    operator=BooleanOperator.AND,
+                    right=root_cte.condition,
+                )
+            else:
+                direct_parent.condition = root_cte.condition
+        root_cte = direct_parent
+
+        sort_select_output(root_cte, select)
+
+    REGISTERED_RULES: list["OptimizationRule"] = []
+
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdownRemove())
+    if CONFIG.optimizations.hide_unused_concepts:
+        REGISTERED_RULES.append(HideUnusedConcepts())
+    for rule in REGISTERED_RULES:
+        loops = 0
+        complete = False
+        while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+            actions_taken = False
+            # assume we go through all CTEs once
+            look_at = [root_cte, *reversed(input)]
+            inverse_map = gen_inverse_map(look_at)
+            for cte in look_at:
+                opt = rule.optimize(cte, inverse_map)
+                actions_taken = actions_taken or opt
+            complete = not actions_taken
+            loops += 1
+        input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
+        logger.info(
+            f"[Optimization] Finished checking for {type(rule).__name__} after {loops} loop(s)"
+        )
+
+    return reorder_ctes(filter_irrelevant_ctes(input, root_cte))

trilogy/core/optimizations/__init__.py

@@ -0,0 +1,12 @@
+from .base_optimization import OptimizationRule
+from .hide_unused_concept import HideUnusedConcepts
+from .inline_datasource import InlineDatasource
+from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
+
+__all__ = [
+    "OptimizationRule",
+    "InlineDatasource",
+    "PredicatePushdown",
+    "PredicatePushdownRemove",
+    "HideUnusedConcepts",
+]

trilogy/core/optimizations/base_optimization.py

@@ -0,0 +1,17 @@
+from abc import ABC
+
+from trilogy.constants import logger
+from trilogy.core.models.execute import CTE, UnionCTE
+
+
+class OptimizationRule(ABC):
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        raise NotImplementedError
+
+    def log(self, message: str):
+        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
+
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")

trilogy/core/optimizations/hide_unused_concept.py

@@ -0,0 +1,47 @@
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class HideUnusedConcepts(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        used = set()
+        from trilogy.dialect.base import BaseDialect
+
+        renderer = BaseDialect()
+        children = inverse_map.get(cte.name, [])
+        if not children:
+            return False
+        for v in children:
+            self.log(f"Analyzing usage of {cte.name} in {v.name}")
+            renderer.render_cte(v)
+        used = renderer.used_map.get(cte.name, set())
+        self.log(f"Used concepts for {cte.name}: {used} from {renderer.used_map}")
+        add_to_hidden: list[BuildConcept] = []
+        for concept in cte.output_columns:
+            if concept.address not in used:
+                add_to_hidden.append(concept)
+        newly_hidden = [
+            x.address for x in add_to_hidden if x.address not in cte.hidden_concepts
+        ]
+        non_hidden = [
+            x for x in cte.output_columns if x.address not in cte.hidden_concepts
+        ]
+        if not newly_hidden or len(non_hidden) <= 1:
+            return False
+        self.log(
+            f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
+        )
+        candidates = [x.address for x in cte.output_columns if x.address not in used]
+        if len(candidates) == len(set([x.address for x in cte.output_columns])):
+            # pop one out
+            candidates.pop()
+        cte.hidden_concepts = set(candidates)
+        return True

trilogy/core/optimizations/inline_datasource.py

@@ -0,0 +1,102 @@
+from collections import defaultdict
+
+from trilogy.constants import CONFIG
+from trilogy.core.models.build import BuildDatasource
+from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineDatasource(OptimizationRule):
+    def __init__(self):
+        super().__init__()
+        self.candidates = defaultdict(lambda: set())
+        self.count = defaultdict(lambda: 0)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        if isinstance(cte, UnionCTE):
+            return any(
+                self.optimize(x, inverse_map=inverse_map) for x in cte.internal_ctes
+            )
+        if isinstance(cte, RecursiveCTE):
+            return False
+        if not cte.parent_ctes:
+            return False
+
+        self.debug(
+            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
+        )
+        to_inline: list[CTE] = []
+        force_group = False
+        for parent_cte in cte.parent_ctes:
+            if isinstance(parent_cte, UnionCTE):
+                continue
+            if isinstance(parent_cte, RecursiveCTE):
+                continue
+            if not parent_cte.is_root_datasource:
+                self.debug(f"Cannot inline: parent {parent_cte.name} is not root")
+                continue
+            if parent_cte.parent_ctes:
+                self.debug(f"Cannot inline: parent {parent_cte.name} has parents")
+                continue
+            if parent_cte.condition:
+                self.debug(
+                    f"Cannot inline: parent {parent_cte.name} has condition, cannot be inlined"
+                )
+                continue
+            raw_root = parent_cte.source.datasources[0]
+            if not isinstance(raw_root, BuildDatasource):
+                self.debug(f"Cannot inline: Parent {parent_cte.name} is not datasource")
+                continue
+            root: BuildDatasource = raw_root
+            if not root.can_be_inlined:
+                self.debug(
+                    f"Cannot inline: Parent {parent_cte.name} datasource is not inlineable"
+                )
+                continue
+            root_outputs = {x.address for x in root.output_concepts}
+            inherited = {
+                x for x, v in cte.source_map.items() if v and parent_cte.name in v
+            }
+            if not inherited.issubset(root_outputs):
+                cte_missing = inherited - root_outputs
+                self.log(
+                    f"Cannot inline: Not all required inputs to {parent_cte.name} are found on datasource, missing {cte_missing}"
+                )
+                continue
+            if not root.grain.issubset(parent_cte.grain):
+                self.log(
+                    f"Cannot inline: {parent_cte.name} is at wrong grain to inline ({root.grain} vs {parent_cte.grain})"
+                )
+                continue
+            to_inline.append(parent_cte)
+
+        optimized = False
+        for replaceable in to_inline:
+            if replaceable.name not in self.candidates[cte.name]:
+                self.candidates[cte.name].add(replaceable.name)
+                self.count[replaceable.source.identifier] += 1
+                return True
+            if (
+                self.count[replaceable.source.identifier]
+                > CONFIG.optimizations.constant_inline_cutoff
+            ):
+                self.log(
+                    f"Skipping inlining raw datasource {replaceable.source.identifier} ({replaceable.name}) due to multiple references"
+                )
+                continue
+            if not replaceable.source.datasources[0].grain.issubset(replaceable.grain):
+                self.log(
+                    f"Forcing group ({parent_cte.grain} being replaced by inlined source {root.grain})"
+                )
+                force_group = True
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(
+                    f"Inlined parent {replaceable.name} with {replaceable.source.identifier}"
+                )
+                optimized = True
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
+        return optimized

trilogy/core/optimizations/predicate_pushdown.py

@@ -0,0 +1,245 @@
+from trilogy.core.enums import (
+    BooleanOperator,
+    SourceType,
+)
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConceptArgs,
+    BuildConditional,
+    BuildDatasource,
+    BuildParenthetical,
+    BuildWindowItem,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+from trilogy.core.processing.utility import is_scalar_condition
+from trilogy.utility import unique
+
+
+def is_child_of(a, comparison):
+    base = comparison == a
+    if base:
+        return True
+    if isinstance(comparison, BuildConditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return base
+
+
+class PredicatePushdown(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def _check_parent(
+        self,
+        cte: CTE | UnionCTE,
+        parent_cte: CTE | UnionCTE,
+        candidate: BuildConditional | BuildComparison | BuildParenthetical | None,
+        inverse_map: dict[str, list[CTE | UnionCTE]],
+    ):
+        if not isinstance(candidate, BuildConceptArgs):
+            return False
+        if not isinstance(parent_cte, CTE):
+            return False
+        row_conditions = {x.address for x in candidate.row_arguments}
+        existence_conditions = {
+            y.address for x in candidate.existence_arguments for y in x
+        }
+        all_inputs = {x.address for x in candidate.concept_arguments}
+        if is_child_of(candidate, parent_cte.condition):
+            return False
+        non_materialized = [k for k, v in parent_cte.source_map.items() if v == []]
+        concrete = [
+            x for x in parent_cte.output_columns if x.address in non_materialized
+        ]
+        if any(isinstance(x.lineage, BuildWindowItem) for x in concrete):
+            self.debug(
+                f"CTE {parent_cte.name} has window clause calculation, cannot push up to this without changing results"
+            )
+            return False
+        materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+
+        if not row_conditions or not materialized:
+            return False
+        output_addresses = {x.address for x in parent_cte.output_columns}
+        # if any of the existence conditions are created on the asset, we can't push up to it
+        if existence_conditions and existence_conditions.intersection(output_addresses):
+            return False
+        if existence_conditions:
+            self.log(
+                f"Not pushing up existence {candidate} to {parent_cte.name} as it is a filter node"
+            )
+            if parent_cte.source.source_type == SourceType.FILTER:
+                return False
+        # if it's a root datasource, we can filter on _any_ of the output concepts
+        if parent_cte.is_root_datasource:
+            extra_check = {
+                x.address for x in parent_cte.source.datasources[0].output_concepts
+            }
+            if row_conditions.issubset(extra_check):
+                for x in row_conditions:
+                    if x not in materialized:
+                        materialized.add(x)
+                        parent_cte.source_map[x] = [
+                            parent_cte.source.datasources[0].name
+                        ]
+        if row_conditions.issubset(materialized):
+            children = inverse_map.get(parent_cte.name, [])
+            if all([is_child_of(candidate, child.condition) for child in children]):
+                self.log(
+                    f"All concepts [{row_conditions}] and existence conditions [{existence_conditions}] not block pushup of [{output_addresses}]found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
+                )
+                if parent_cte.condition and not is_scalar_condition(
+                    parent_cte.condition
+                ):
+                    self.log("Parent condition is not scalar, not safe to push up")
+                    return False
+                if parent_cte.condition:
+                    parent_cte.condition = BuildConditional(
+                        left=parent_cte.condition,
+                        operator=BooleanOperator.AND,
+                        right=candidate,
+                    )
+                else:
+                    parent_cte.condition = candidate
+                # promote up existence sources
+                if all_inputs.difference(row_conditions):
+                    for x in all_inputs.difference(row_conditions):
+                        if x not in parent_cte.source_map and x in cte.source_map:
+                            sources = [
+                                parent
+                                for parent in cte.parent_ctes
+                                if parent.name in cte.source_map[x]
+                            ]
+                            parent_cte.source_map[x] = cte.source_map[x]
+                            parent_cte.parent_ctes = unique(
+                                parent_cte.parent_ctes + sources, "name"
+                            )
+                return True
+        self.debug(
+            f"conditions {row_conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+        )
+        return False
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        # TODO - pushdown through unions
+        if isinstance(cte, UnionCTE):
+            return False
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        if self.complete.get(cte.name):
+            self.debug("Have done this CTE before")
+            return False
+
+        self.debug(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, BuildConditional):
+            candidates = cte.condition.decompose()
+        else:
+            candidates = [cte.condition]
+        self.debug(
+            f"Have {len(candidates)} candidates to try to push down from parent {type(cte.condition)}"
+        )
+        optimized = False
+        for candidate in candidates:
+            if not is_scalar_condition(candidate):
+                self.debug(
+                    f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
+                )
+                continue
+            self.debug(
+                f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
+            )
+            for parent_cte in cte.parent_ctes:
+                local_pushdown = self._check_parent(
+                    cte=cte,
+                    parent_cte=parent_cte,
+                    candidate=candidate,
+                    inverse_map=inverse_map,
+                )
+                optimized = optimized or local_pushdown
+                if local_pushdown:
+                    # taint a CTE again when something is pushed up to it.
+                    self.complete[parent_cte.name] = False
+                self.debug(
+                    f"Pushed down {candidate} from {cte.name} to {parent_cte.name}"
+                )
+
+        self.complete[cte.name] = True
+        return optimized
+
+
+class PredicatePushdownRemove(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        if isinstance(cte, UnionCTE):
+            return False
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        parent_filter_status = {
+            parent.name: is_child_of(cte.condition, parent.condition)
+            for parent in cte.parent_ctes
+        }
+        # flatten existnce argument tuples to a list
+
+        flattened_existence = [
+            x.address for y in cte.condition.existence_arguments for x in y
+        ]
+
+        existence_only = [
+            parent.name
+            for parent in cte.parent_ctes
+            if all([x.address in flattened_existence for x in parent.output_columns])
+            and len(flattened_existence) > 0
+        ]
+        if all(
+            [
+                value
+                for key, value in parent_filter_status.items()
+                if key not in existence_only
+            ]
+        ) and not any([isinstance(x, BuildDatasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            # remove any "parent" CTEs that provided only existence inputs
+            if existence_only:
+                original = [y.name for y in cte.parent_ctes]
+                cte.parent_ctes = [
+                    x for x in cte.parent_ctes if x.name not in existence_only
+                ]
+                self.log(
+                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
+                )
+            return True
+
+        self.complete[cte.name] = True
+        return optimized