pytrilogy 0.0.3.102__tar.gz → 0.0.3.104__tar.gz

{pytrilogy-0.0.3.102/pytrilogy.egg-info → pytrilogy-0.0.3.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.102
+Version: 0.0.3.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.102
+Version: 0.0.3.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/pytrilogy.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ trilogy/core/models/environment.py
 trilogy/core/models/execute.py
 trilogy/core/optimizations/__init__.py
 trilogy/core/optimizations/base_optimization.py
+trilogy/core/optimizations/hide_unused_concept.py
 trilogy/core/optimizations/inline_datasource.py
 trilogy/core/optimizations/predicate_pushdown.py
 trilogy/core/processing/__init__.py
@@ -146,6 +147,7 @@ trilogy/parsing/trilogy.lark
 trilogy/scripts/__init__.py
 trilogy/scripts/trilogy.py
 trilogy/std/__init__.py
+trilogy/std/color.preql
 trilogy/std/date.preql
 trilogy/std/display.preql
 trilogy/std/geography.preql

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/pytrilogy.egg-info/requires.txt

@@ -4,6 +4,7 @@ sqlalchemy<2.0.0
 networkx
 pyodbc
 pydantic
+duckdb<1.4.0
 duckdb-engine
 click
 

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/tests/test_partial_handling.py

@@ -2,7 +2,7 @@ from sqlalchemy import create_engine
 
 from trilogy import Dialects, Executor
 from trilogy.core.enums import Purpose
-from trilogy.core.models.author import Concept
+from trilogy.core.models.author import Concept, Grain
 from trilogy.core.models.core import (
     DataType,
 )
@@ -90,6 +90,7 @@ def setup_titanic(env: Environment):
                 ColumnAssignment(alias="name", concept=name),
                 ColumnAssignment(alias="fare", concept=fare),
             ],
+            grain=Grain(components=[id.address]),
         ),
     )
     return env
@@ -140,7 +141,6 @@ def test_partial_assignment():
         depth=0,
     )
     assert isinstance(sourced, FilterNode)
-    assert len(sourced.parents) == 1
 
 
 def test_filter_query():

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.102"
+__version__ = "0.0.3.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/constants.py

@@ -37,7 +37,7 @@ class Comments:
 
     show: bool = False
     basic: bool = True
-    joins: bool = True
+    joins: bool = False
     nullable: bool = True
     partial: bool = True
     source_map: bool = False

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/models/execute.py

@@ -711,8 +711,6 @@ class QueryDatasource(BaseModel):
             f" {[c.address for c in self.output_concepts]} concepts and"
             f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
         )
-        logger.info(self.source_map)
-        logger.info(other.source_map)
 
         merged_datasources: dict[str, Union[BuildDatasource, "QueryDatasource"]] = {}
 
@@ -816,10 +814,7 @@ class QueryDatasource(BaseModel):
                     use_raw_name,
                     force_alias=force_alias,
                 )
-            except ValueError as e:
-                from trilogy.constants import logger
-
-                logger.debug(e)
+            except ValueError:
                 continue
         existing = [c.with_grain(self.grain) for c in self.output_concepts]
         if concept in existing:

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/optimization.py

@@ -5,6 +5,7 @@ from trilogy.core.models.build import (
 )
 from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations import (
+    HideUnusedConcepts,
     InlineDatasource,
     OptimizationRule,
     PredicatePushdown,
@@ -84,11 +85,18 @@ def filter_irrelevant_ctes(
         #                 child.existence_source_map[x2].append(parent.name)
         # else:
         relevant_ctes.add(cte.name)
-        for cte in cte.parent_ctes:
-            recurse(cte, inverse_map)
+
+        for parent in cte.parent_ctes:
+            if parent.name in relevant_ctes:
+                logger.info(
+                    f"[Optimization][Irrelevent CTE filtering] Already visited {parent.name} when visting {cte.name}, potential recursive dag"
+                )
+                continue
+
+            recurse(parent, inverse_map)
         if isinstance(cte, UnionCTE):
-            for cte in cte.internal_ctes:
-                recurse(cte, inverse_map)
+            for internal in cte.internal_ctes:
+                recurse(internal, inverse_map)
 
     inverse_map = gen_inverse_map(input)
     recurse(root_cte, inverse_map)
@@ -220,6 +228,7 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
+    REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/optimizations/__init__.py

@@ -1,4 +1,5 @@
 from .base_optimization import OptimizationRule
+from .hide_unused_concept import HideUnusedConcepts
 from .inline_datasource import InlineDatasource
 from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 
@@ -7,4 +8,5 @@ __all__ = [
     "InlineDatasource",
     "PredicatePushdown",
     "PredicatePushdownRemove",
+    "HideUnusedConcepts",
 ]

pytrilogy-0.0.3.104/trilogy/core/optimizations/hide_unused_concept.py

@@ -0,0 +1,51 @@
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class HideUnusedConcepts(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        used = set()
+        from trilogy.dialect.base import BaseDialect
+
+        renderer = BaseDialect()
+        children = inverse_map.get(cte.name, [])
+        if not children:
+            return False
+        for v in children:
+            self.log(f"Analyzing usage of {cte.name} in {v.name}")
+            renderer.render_cte(v)
+        used = renderer.used_map.get(cte.name, set())
+        self.log(f"Used concepts for {cte.name}: {used} from {renderer.used_map}")
+        add_to_hidden: list[BuildConcept] = []
+        for concept in cte.output_columns:
+            if concept.address not in used:
+                add_to_hidden.append(concept)
+        newly_hidden = [
+            x.address for x in add_to_hidden if x.address not in cte.hidden_concepts
+        ]
+        non_hidden = [
+            x for x in cte.output_columns if x.address not in cte.hidden_concepts
+        ]
+        if not newly_hidden or len(non_hidden) <= 1:
+            return False
+        self.log(
+            f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
+        )
+        candidates = [
+            x.address
+            for x in cte.output_columns
+            if x.address not in used and x.address not in cte.hidden_concepts
+        ]
+        if len(candidates) == len(set([x.address for x in cte.output_columns])):
+            # pop one out
+            candidates.pop()
+        cte.hidden_concepts = set(candidates)
+        return True

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/optimizations/predicate_pushdown.py

@@ -1,5 +1,6 @@
 from trilogy.core.enums import (
     BooleanOperator,
+    SourceType,
 )
 from trilogy.core.models.build import (
     BuildComparison,
@@ -59,12 +60,19 @@ class PredicatePushdown(OptimizationRule):
             )
             return False
         materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+
         if not row_conditions or not materialized:
             return False
         output_addresses = {x.address for x in parent_cte.output_columns}
         # if any of the existence conditions are created on the asset, we can't push up to it
         if existence_conditions and existence_conditions.intersection(output_addresses):
             return False
+        if existence_conditions:
+            self.log(
+                f"Not pushing up existence {candidate} to {parent_cte.name} as it is a filter node"
+            )
+            if parent_cte.source.source_type == SourceType.FILTER:
+                return False
         # if it's a root datasource, we can filter on _any_ of the output concepts
         if parent_cte.is_root_datasource:
             extra_check = {
@@ -81,7 +89,7 @@ class PredicatePushdown(OptimizationRule):
             children = inverse_map.get(parent_cte.name, [])
             if all([is_child_of(candidate, child.condition) for child in children]):
                 self.log(
-                    f"All concepts are found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
+                    f"All concepts [{row_conditions}] and existence conditions [{existence_conditions}] not block pushup of [{output_addresses}]found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
                 )
                 if parent_cte.condition and not is_scalar_condition(
                     parent_cte.condition

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/processing/concept_strategies_v3.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.discovery_utility import (
     LOGGER_PREFIX,
     depth_to_prefix,
     get_priority_concept,
-    group_if_required,
+    group_if_required_v2,
 )
 from trilogy.core.processing.discovery_validation import (
     ValidationResult,
@@ -66,7 +66,19 @@ def generate_candidates_restrictive(
 
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
-        return [], conditions
+        logger.info("Have single row concept, including only other single row optional")
+        optional = (
+            [
+                x
+                for x in candidates
+                if x.granularity == Granularity.SINGLE_ROW
+                and x.address not in priority_concept.pseudonyms
+                and priority_concept.address not in x.pseudonyms
+            ]
+            if priority_concept.derivation == Derivation.AGGREGATE
+            else []
+        )
+        return optional, conditions
 
     if conditions and priority_concept.derivation in ROOT_DERIVATIONS:
         logger.info(
@@ -374,15 +386,21 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             logger.info(
                 f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
             )
-            output.set_output_concepts(
-                [
-                    x
-                    for x in output.output_concepts
-                    if x.address in non_virtual_output
-                    or any(c in non_virtual_output for c in x.pseudonyms)
-                ],
-                rebuild=False,
-            )
+            # output.set_output_concepts(
+            #     [
+            #         x
+            #         for x in output.output_concepts
+            #         if x.address not in non_virtual_difference_values
+            #         or any(c in non_virtual_output for c in x.pseudonyms)
+            #     ],
+            #     rebuild=True,
+            #     change_visibility=False
+            # )
+            # output.set_output_concepts(context.original_mandatory)
+
+            # if isinstance(output, MergeNode):
+            #     output.force_group = True
+            #     output.rebuild_cache()
 
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
@@ -416,14 +434,17 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
     logger.info(
         f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts or []]} with {context.conditions}"
     )
+    from trilogy.core.processing.discovery_utility import group_if_required_v2
+
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
-        return group_if_required(
+        return group_if_required_v2(
             output, context.original_mandatory, context.environment
         )
-    return output
+
+    return group_if_required_v2(output, context.original_mandatory, context.environment)
 
 
 def _search_concepts(
@@ -588,4 +609,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    return group_if_required(root, output_concepts, environment)
+    return group_if_required_v2(root, output_concepts, environment)

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/processing/discovery_node_factory.py

@@ -177,7 +177,12 @@ def _generate_union_node(ctx: NodeGenerationContext) -> StrategyNode | None:
 def _generate_aggregate_node(ctx: NodeGenerationContext) -> StrategyNode | None:
     # Filter out constants to avoid multiplication issues
     agg_optional = [
-        x for x in ctx.local_optional if x.granularity != Granularity.SINGLE_ROW
+        x
+        for x in ctx.local_optional
+        if not (
+            x.granularity == Granularity.SINGLE_ROW
+            and x.derivation != Derivation.AGGREGATE
+        )
     ]
 
     logger.info(
@@ -376,11 +381,6 @@ class RootNodeHandler:
 
         if pseudonyms:
             expanded.add_output_concepts(pseudonyms)
-            logger.info(
-                f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
-                f"Hiding pseudonyms {[c.address for c in pseudonyms]}"
-            )
-            expanded.hide_output_concepts(pseudonyms)
 
         logger.info(
             f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "

pytrilogy-0.0.3.104/trilogy/core/processing/discovery_utility.py

@@ -0,0 +1,294 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildDatasource,
+    BuildFilterItem,
+    BuildGrain,
+    BuildRowsetItem,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import QueryDatasource, UnnestJoin
+from trilogy.core.processing.nodes import GroupNode, MergeNode, StrategyNode
+from trilogy.core.processing.utility import GroupRequiredResponse
+
+
+def depth_to_prefix(depth: int) -> str:
+    return "\t" * depth
+
+
+LOGGER_PREFIX = "[DISCOVERY LOOP]"
+
+
+def calculate_effective_parent_grain(
+    node: QueryDatasource | BuildDatasource,
+) -> BuildGrain:
+    # calculate the effective grain of the parent node
+    # this is the union of all parent grains
+    if isinstance(node, MergeNode):
+        grain = BuildGrain()
+        qds = node.resolve()
+        if not qds.joins:
+            return qds.datasources[0].grain
+        for join in qds.joins:
+            if isinstance(join, UnnestJoin):
+                continue
+            pairs = join.concept_pairs or []
+            for key in pairs:
+                left = key.existing_datasource
+                grain += left.grain
+            keys = [key.right for key in pairs]
+            join_grain = BuildGrain.from_concepts(keys)
+            if join_grain == join.right_datasource.grain:
+                logger.info(f"irrelevant right join {join}, does not change grain")
+            else:
+                logger.info(
+                    f"join changes grain, adding {join.right_datasource.grain} to {grain}"
+                )
+                grain += join.right_datasource.grain
+        return grain
+    else:
+        return node.grain or BuildGrain()
+
+
+def check_if_group_required(
+    downstream_concepts: List[BuildConcept],
+    parents: list[QueryDatasource | BuildDatasource],
+    environment: BuildEnvironment,
+    depth: int = 0,
+) -> GroupRequiredResponse:
+    padding = "\t" * depth
+    target_grain = BuildGrain.from_concepts(
+        downstream_concepts,
+        environment=environment,
+    )
+
+    comp_grain = BuildGrain()
+    for source in parents:
+        # comp_grain += source.grain
+        comp_grain += calculate_effective_parent_grain(source)
+
+    # dynamically select if we need to group
+    # we must avoid grouping if we are already at grain
+    if comp_grain.issubset(target_grain):
+
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, target: {target_grain}, grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    # find out what extra is in the comp grain vs target grain
+    difference = [
+        environment.concepts[c] for c in (comp_grain - target_grain).components
+    ]
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Group requirement check: upstream grain: {comp_grain}, desired grain: {target_grain} from , difference {[x.address for x in difference]}"
+    )
+    for x in difference:
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Difference concept {x.address} purpose {x.purpose} keys {x.keys}"
+        )
+
+    # if the difference is all unique properties whose keys are in the source grain
+    # we can also suppress the group
+    if all(
+        [
+            x.keys
+            and all(
+                environment.concepts[z].address in comp_grain.components for z in x.keys
+            )
+            for x in difference
+        ]
+    ):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    if all([x.purpose == Purpose.KEY for x in difference]):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
+        )
+        replaced_grain_raw: list[set[str]] = [
+            (
+                x.keys or set()
+                if x.purpose == Purpose.UNIQUE_PROPERTY
+                else set([x.address])
+            )
+            for x in downstream_concepts
+            if x.address in target_grain.components
+        ]
+        # flatten the list of lists
+        replaced_grain = [item for sublist in replaced_grain_raw for item in sublist]
+        # if the replaced grain is a subset of the comp grain, we can skip the group
+        unique_grain_comp = BuildGrain.from_concepts(
+            replaced_grain, environment=environment
+        )
+        if comp_grain.issubset(unique_grain_comp):
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+            )
+            return GroupRequiredResponse(target_grain, comp_grain, False)
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Checking for grain equivalence for filters and rowsets"
+    )
+    ngrain = []
+    for con in target_grain.components:
+        full = environment.concepts[con]
+        if full.derivation == Derivation.ROWSET:
+            ngrain.append(full.address.split(".", 1)[1])
+        elif full.derivation == Derivation.FILTER:
+            assert isinstance(full.lineage, BuildFilterItem)
+            if isinstance(full.lineage.content, BuildConcept):
+                ngrain.append(full.lineage.content.address)
+        else:
+            ngrain.append(full.address)
+    target_grain2 = BuildGrain.from_concepts(
+        ngrain,
+        environment=environment,
+    )
+    if comp_grain.issubset(target_grain2):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain2}, pre rowset grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain2, comp_grain, False)
+
+    logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
+    return GroupRequiredResponse(target_grain, comp_grain, True)
+
+
+def group_if_required_v2(
+    root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
+):
+    required = check_if_group_required(
+        downstream_concepts=final, parents=[root.resolve()], environment=environment
+    )
+    targets = [
+        x
+        for x in root.output_concepts
+        if x.address in final or any(c in final for c in x.pseudonyms)
+    ]
+    if required.required:
+        if isinstance(root, MergeNode):
+            root.force_group = True
+            root.set_output_concepts(targets, rebuild=False, change_visibility=False)
+            root.rebuild_cache()
+            return root
+        elif isinstance(root, GroupNode):
+            # root.set_output_concepts(final, rebuild=False)
+            # root.rebuild_cache()
+            return root
+        return GroupNode(
+            output_concepts=targets,
+            input_concepts=targets,
+            environment=environment,
+            parents=[root],
+            partial_concepts=root.partial_concepts,
+            preexisting_conditions=root.preexisting_conditions,
+        )
+    elif isinstance(root, GroupNode):
+
+        return root
+    else:
+        root.set_output_concepts(targets, rebuild=False, change_visibility=False)
+    return root
+
+
+def get_upstream_concepts(base: BuildConcept, nested: bool = False) -> set[str]:
+    upstream = set()
+    if nested:
+        upstream.add(base.address)
+    if not base.lineage:
+        return upstream
+    for x in base.lineage.concept_arguments:
+        # if it's derived from any value in a rowset, ALL rowset items are upstream
+        if x.derivation == Derivation.ROWSET:
+            assert isinstance(x.lineage, BuildRowsetItem), type(x.lineage)
+            for y in x.lineage.rowset.select.output_components:
+                upstream.add(f"{x.lineage.rowset.name}.{y.address}")
+                # upstream = upstream.union(get_upstream_concepts(y, nested=True))
+        upstream = upstream.union(get_upstream_concepts(x, nested=True))
+    return upstream
+
+
+def get_priority_concept(
+    all_concepts: List[BuildConcept],
+    attempted_addresses: set[str],
+    found_concepts: set[str],
+    depth: int,
+) -> BuildConcept:
+    # optimized search for missing concepts
+    pass_one = sorted(
+        [
+            c
+            for c in all_concepts
+            if c.address not in attempted_addresses and c.address not in found_concepts
+        ],
+        key=lambda x: x.address,
+    )
+    # sometimes we need to scan intermediate concepts to get merge keys or filter keys,
+    # so do an exhaustive search
+    # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
+    for remaining_concept in (pass_one,):
+        priority = (
+            # then multiselects to remove them from scope
+            [c for c in remaining_concept if c.derivation == Derivation.MULTISELECT]
+            +
+            # then rowsets to remove them from scope, as they cannot get partials
+            [c for c in remaining_concept if c.derivation == Derivation.ROWSET]
+            +
+            # then rowsets to remove them from scope, as they cannot get partials
+            [c for c in remaining_concept if c.derivation == Derivation.UNION]
+            # we should be home-free here
+            +
+            # then aggregates to remove them from scope, as they cannot get partials
+            [c for c in remaining_concept if c.derivation == Derivation.AGGREGATE]
+            # then windows to remove them from scope, as they cannot get partials
+            + [c for c in remaining_concept if c.derivation == Derivation.WINDOW]
+            # then filters to remove them from scope, also cannot get partials
+            + [c for c in remaining_concept if c.derivation == Derivation.FILTER]
+            # unnests are weird?
+            + [c for c in remaining_concept if c.derivation == Derivation.UNNEST]
+            + [c for c in remaining_concept if c.derivation == Derivation.RECURSIVE]
+            + [c for c in remaining_concept if c.derivation == Derivation.BASIC]
+            + [c for c in remaining_concept if c.derivation == Derivation.GROUP_TO]
+            + [c for c in remaining_concept if c.derivation == Derivation.CONSTANT]
+            # finally our plain selects
+            + [
+                c for c in remaining_concept if c.derivation == Derivation.ROOT
+            ]  # and any non-single row constants
+        )
+
+        priority += [
+            c
+            for c in remaining_concept
+            if c.address not in [x.address for x in priority]
+        ]
+        final = []
+        # if any thing is derived from another concept
+        # get the derived copy first
+        # as this will usually resolve cleaner
+        for x in priority:
+            if any(
+                [
+                    x.address
+                    in get_upstream_concepts(
+                        c,
+                    )
+                    for c in priority
+                ]
+            ):
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} delaying fetch of {x.address} as parent of another concept"
+                )
+                continue
+            final.append(x)
+        # then append anything we didn't get
+        for x2 in priority:
+            if x2 not in final:
+                final.append(x2)
+        if final:
+            return final[0]
+    raise ValueError(
+        f"Cannot resolve query. No remaining priority concepts, have attempted {attempted_addresses}"
+    )

{pytrilogy-0.0.3.102 → pytrilogy-0.0.3.104}/trilogy/core/processing/node_generators/basic_node.py

@@ -143,4 +143,5 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
     )
+
     return parent_node