pytrilogy 0.0.2.48__tar.gz → 0.0.2.49__tar.gz

{pytrilogy-0.0.2.48/pytrilogy.egg-info → pytrilogy-0.0.2.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.48
+Version: 0.0.2.49
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.48
+Version: 0.0.2.49
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/tests/test_query_processing.py

@@ -130,9 +130,7 @@ def test_join_aggregate(test_environment: Environment, test_environment_graph):
 
 def test_query_aggregation(test_environment, test_environment_graph):
     select = SelectStatement(selection=[test_environment.concepts["total_revenue"]])
-    datasource = get_query_datasources(
-        environment=test_environment, graph=test_environment_graph, statement=select
-    )
+    datasource = get_query_datasources(environment=test_environment, statement=select)
 
     assert {datasource.identifier} == {"revenue_at_local_order_id_at_abstract"}
     check = datasource
@@ -150,9 +148,7 @@ def test_query_datasources(test_environment, test_environment_graph):
             test_environment.concepts["total_revenue"],
         ]
     )
-    get_query_datasources(
-        environment=test_environment, graph=test_environment_graph, statement=select
-    )
+    get_query_datasources(environment=test_environment, statement=select)
 
 
 def test_full_query(test_environment, test_environment_graph):

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.2.48"
+__version__ = "0.0.2.49"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/models.py

@@ -467,7 +467,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         )
 
     def __repr__(self):
-        base = f"{self.namespace}.{self.address}@{self.grain}"
+        base = f"{self.address}@{self.grain}"
         return base
 
     @property
@@ -662,11 +662,24 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
                 local_concepts=local_concepts, grain=grain, environment=environment
             )
         final_grain = self.grain
-
+        keys = (
+            tuple(
+                [
+                    x.with_select_context(local_concepts, grain, environment)
+                    for x in self.keys
+                ]
+            )
+            if self.keys
+            else None
+        )
         if self.is_aggregate and isinstance(new_lineage, Function):
             new_lineage = AggregateWrapper(function=new_lineage, by=grain.components)
             final_grain = grain
-
+            keys = tuple(grain.components)
+        elif (
+            self.is_aggregate and not keys and isinstance(new_lineage, AggregateWrapper)
+        ):
+            keys = tuple(new_lineage.by)
         return self.__class__(
             name=self.name,
             datatype=self.datatype,
@@ -675,16 +688,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
             lineage=new_lineage,
             grain=final_grain,
             namespace=self.namespace,
-            keys=(
-                tuple(
-                    [
-                        x.with_select_context(local_concepts, grain, environment)
-                        for x in self.keys
-                    ]
-                )
-                if self.keys
-                else None
-            ),
+            keys=keys,
             modifiers=self.modifiers,
             # a select needs to always defer to the environment for pseudonyms
             # TODO: evaluate if this should be cached
@@ -2626,7 +2630,7 @@ class QueryDatasource(BaseModel):
                     and CONFIG.validate_missing
                 ):
                     raise SyntaxError(
-                        f"Missing source map for {concept.address} on {key}, have {v}"
+                        f"On query datasource missing source map for {concept.address} on {key}, have {v}"
                     )
         return v
 

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/concept_strategies_v3.py

@@ -569,6 +569,8 @@ def validate_stack(
         resolved = node.resolve()
 
         for concept in resolved.output_concepts:
+            if concept in resolved.hidden_concepts:
+                continue
             validate_concept(
                 concept,
                 node,
@@ -836,6 +838,7 @@ def _search_concepts(
                     PurposeLineage.ROWSET,
                     PurposeLineage.BASIC,
                     PurposeLineage.MULTISELECT,
+                    PurposeLineage.UNION,
                 ]:
                     skip.add(priority_concept.address)
                 break

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/node_generators/rowset_node.py

@@ -38,7 +38,7 @@ def gen_rowset_node(
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
 
-    node = get_query_node(environment, select, graph=g, history=history)
+    node = get_query_node(environment, select)
 
     if not node:
         logger.info(
@@ -94,15 +94,22 @@ def gen_rowset_node(
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional found or no optional; exiting early."
         )
-        # node.set_preexisting_conditions(conditions.conditional if conditions else None)
         return node
-
-    possible_joins = concept_to_relevant_joins(node.output_concepts)
+    possible_joins = concept_to_relevant_joins(
+        [x for x in node.output_concepts if x.derivation != PurposeLineage.ROWSET]
+    )
+    logger.info({x.address: x.keys for x in possible_joins})
     if not possible_joins:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
         )
         return node
+    if any(x.derivation == PurposeLineage.ROWSET for x in possible_joins):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} cannot enrich rowset node with rowset concepts; exiting early"
+        )
+        return node
+    logger.info([x.address for x in possible_joins + local_optional])
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=possible_joins + local_optional,
@@ -110,15 +117,28 @@ def gen_rowset_node(
         g=g,
         depth=depth + 1,
         conditions=conditions,
+        history=history,
     )
     if not enrich_node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset enrichment node for {concept} with optional {local_optional}, returning just rowset node"
         )
         return node
+
+    non_hidden = [
+        x for x in node.output_concepts if x.address not in node.hidden_concepts
+    ]
+    for x in possible_joins:
+        if x.address in node.hidden_concepts:
+            node.unhide_output_concepts([x])
+    non_hidden_enrich = [
+        x
+        for x in enrich_node.output_concepts
+        if x.address not in enrich_node.hidden_concepts
+    ]
     return MergeNode(
-        input_concepts=enrich_node.output_concepts + node.output_concepts,
-        output_concepts=node.output_concepts + local_optional,
+        input_concepts=non_hidden + non_hidden_enrich,
+        output_concepts=non_hidden + local_optional,
         environment=environment,
         g=g,
         depth=depth,
@@ -126,6 +146,6 @@ def gen_rowset_node(
             node,
             enrich_node,
         ],
-        partial_concepts=node.partial_concepts,
+        partial_concepts=node.partial_concepts + enrich_node.partial_concepts,
         preexisting_conditions=conditions.conditional if conditions else None,
     )

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/nodes/base_node.py

@@ -211,8 +211,22 @@ class StrategyNode:
                 operator=BooleanOperator.AND,
             )
         self.validate_parents()
+        self.validate_inputs()
         self.log = True
 
+    def validate_inputs(self):
+        if not self.parents:
+            return
+        non_hidden = set()
+        for x in self.parents:
+            for z in x.usable_outputs:
+                non_hidden.add(z.address)
+        if not all([x.address in non_hidden for x in self.input_concepts]):
+            missing = [x for x in self.input_concepts if x.address not in non_hidden]
+            raise ValueError(
+                f"Invalid input concepts; {missing} are missing non-hidden parent nodes"
+            )
+
     def add_parents(self, parents: list["StrategyNode"]):
         self.parents += parents
         self.validate_parents()
@@ -288,6 +302,14 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
+    def unhide_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+        self.hidden_concepts = [
+            x for x in self.hidden_concepts if x.address not in concepts
+        ]
+        if rebuild:
+            self.rebuild_cache()
+        return self
+
     def remove_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
         for x in concepts:
             self.hidden_concepts.append(x)
@@ -299,6 +321,12 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
+    @property
+    def usable_outputs(self) -> list[Concept]:
+        return [
+            x for x in self.output_concepts if x.address not in self.hidden_concepts
+        ]
+
     @property
     def logging_prefix(self) -> str:
         return "\t" * self.depth

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/nodes/group_node.py

@@ -105,9 +105,9 @@ class GroupNode(StrategyNode):
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} Parent node"
                     f" {[c.address for c in parent.output_concepts[:2]]}... has"
-                    " grain"
+                    " set node grain"
                     f" {parent.grain}"
-                    f" resolved grain {parent.resolve().grain}"
+                    f" and resolved grain {parent.resolve().grain}"
                     f" {type(parent)}"
                 )
             source_type = SourceType.GROUP
@@ -146,7 +146,13 @@ class GroupNode(StrategyNode):
         # inject an additional CTE
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
-            base.output_concepts = self.output_concepts + self.conditions.row_arguments
+            base.output_concepts = unique(
+                base.output_concepts + self.conditions.row_arguments, "address"
+            )
+            # re-visible any hidden concepts
+            base.hidden_concepts = [
+                x for x in base.hidden_concepts if x not in base.output_concepts
+            ]
             source_map = resolve_concept_map(
                 [base],
                 targets=self.output_concepts,

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/nodes/merge_node.py

@@ -330,8 +330,9 @@ class MergeNode(StrategyNode):
             force_group = None
 
         qd_joins: List[BaseJoin | UnnestJoin] = [*joins]
+
         source_map = resolve_concept_map(
-            list(merged.values()),
+            final_datasets,
             targets=self.output_concepts,
             inherited_inputs=self.input_concepts + self.existence_concepts,
             full_joins=full_join_concepts,
@@ -339,6 +340,7 @@ class MergeNode(StrategyNode):
         nullable_concepts = find_nullable_concepts(
             source_map=source_map, joins=joins, datasources=final_datasets
         )
+
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/nodes/select_node_v2.py

@@ -67,6 +67,11 @@ class SelectNode(StrategyNode):
         self.accept_partial = accept_partial
         self.datasource = datasource
 
+    def validate_inputs(self):
+        # we do not need to validate inputs for a select node
+        # as it will be a root
+        return
+
     def resolve_from_provided_datasource(
         self,
     ) -> QueryDatasource:

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/processing/utility.py

@@ -214,7 +214,7 @@ def concept_to_relevant_joins(concepts: list[Concept]) -> List[Concept]:
             x for x in concepts if x.keys and all([key in addresses for key in x.keys])
         ]
     )
-    final = [c for c in concepts if c not in sub_props]
+    final = [c for c in concepts if c.address not in sub_props]
     return unique(final, "address")
 
 

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/core/query_processor.py

@@ -7,7 +7,6 @@ from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.enums import BooleanOperator, SourceType
 from trilogy.core.env_processor import generate_graph
 from trilogy.core.ergonomics import generate_cte_names
-from trilogy.core.graph_models import ReferenceGraph
 from trilogy.core.models import (
     CTE,
     BaseJoin,
@@ -353,13 +352,12 @@ def datasource_to_cte(
 def get_query_node(
     environment: Environment,
     statement: SelectStatement | MultiSelectStatement,
-    graph: Optional[ReferenceGraph] = None,
     history: History | None = None,
 ) -> StrategyNode:
     environment = environment.duplicate()
     for k, v in statement.local_concepts.items():
         environment.concepts[k] = v
-    graph = graph or generate_graph(environment)
+    graph = generate_graph(environment)
     logger.info(
         f"{LOGGER_PREFIX} getting source datasource for query with filtering {statement.where_clause_category} and output {[str(c) for c in statement.output_components]}"
     )
@@ -403,11 +401,10 @@ def get_query_node(
 def get_query_datasources(
     environment: Environment,
     statement: SelectStatement | MultiSelectStatement,
-    graph: Optional[ReferenceGraph] = None,
     hooks: Optional[List[BaseHook]] = None,
 ) -> QueryDatasource:
 
-    ds = get_query_node(environment, statement, graph)
+    ds = get_query_node(environment, statement)
     final_qds = ds.resolve()
     if hooks:
         for hook in hooks:
@@ -479,10 +476,9 @@ def process_query(
     hooks: List[BaseHook] | None = None,
 ) -> ProcessedQuery:
     hooks = hooks or []
-    graph = generate_graph(environment)
 
     root_datasource = get_query_datasources(
-        environment=environment, graph=graph, statement=statement, hooks=hooks
+        environment=environment, statement=statement, hooks=hooks
     )
     for hook in hooks:
         hook.process_root_datasource(root_datasource)

{pytrilogy-0.0.2.48 → pytrilogy-0.0.2.49}/trilogy/parsing/parse_engine.py

@@ -570,6 +570,7 @@ class ParseToObjects(Transformer):
         for new_concept in output.derived_concepts:
             if new_concept.metadata:
                 new_concept.metadata.line_number = meta.line
+            # output.select.local_concepts[new_concept.address] = new_concept
             self.environment.add_concept(new_concept)
 
         return output