pytrilogy 0.0.2.50__py3-none-any.whl → 0.0.2.51__py3-none-any.whl

trilogy/core/processing/nodes/group_node.py

@@ -15,10 +15,10 @@ from trilogy.core.models import (
 )
 from trilogy.core.processing.nodes.base_node import (
     StrategyNode,
-    concept_list_to_grain,
     resolve_concept_map,
 )
 from trilogy.core.processing.utility import find_nullable_concepts, is_scalar_condition
+from trilogy.parsing.common import concepts_to_grain_concepts
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
@@ -64,19 +64,27 @@ class GroupNode(StrategyNode):
             p.resolve() for p in self.parents
         ]
 
-        grain = self.grain or concept_list_to_grain(self.output_concepts, [])
+        target_grain = self.grain or Grain.from_concepts(
+            concepts_to_grain_concepts(
+                self.output_concepts, environment=self.environment
+            )
+        )
         comp_grain = Grain()
         for source in parent_sources:
             comp_grain += source.grain
-
+        comp_grain = Grain.from_concepts(
+            concepts_to_grain_concepts(
+                comp_grain.components, environment=self.environment
+            )
+        )
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
-        if comp_grain == grain and self.force_group is not True:
+        if comp_grain == target_grain and self.force_group is not True:
             # if there is no group by, and inputs equal outputs
             # return the parent
             logger.info(
                 f"{self.logging_prefix}{LOGGER_PREFIX} Grain of group by equals output"
-                f" grains {comp_grain} and {grain}"
+                f" grains {comp_grain} and {target_grain}"
             )
             if (
                 len(parent_sources) == 1
@@ -94,10 +102,11 @@ class GroupNode(StrategyNode):
             source_type = SourceType.SELECT
         else:
             logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} Group node has different grain than parents; forcing group"
-                f" upstream grains {[str(source.grain) for source in parent_sources]}"
+                f"{self.logging_prefix}{LOGGER_PREFIX} Group node has different grain than parents; group is required."
+                f" Upstream grains {[str(source.grain) for source in parent_sources]}"
                 f" with final grain {comp_grain} vs"
-                f" target grain {grain}"
+                f" target grain {target_grain}"
+                f" delta: {comp_grain - target_grain}"
             )
             for parent in self.parents:
                 logger.info(
@@ -134,7 +143,7 @@ class GroupNode(StrategyNode):
             source_type=source_type,
             source_map=source_map,
             joins=[],
-            grain=grain,
+            grain=target_grain,
             partial_concepts=self.partial_concepts,
             nullable_concepts=nullable_concepts,
             hidden_concepts=self.hidden_concepts,
@@ -163,7 +172,7 @@ class GroupNode(StrategyNode):
                 source_type=SourceType.SELECT,
                 source_map=source_map,
                 joins=[],
-                grain=grain,
+                grain=target_grain,
                 nullable_concepts=base.nullable_concepts,
                 partial_concepts=self.partial_concepts,
                 condition=self.conditions,

trilogy/core/processing/nodes/merge_node.py

@@ -58,7 +58,7 @@ def deduplicate_nodes(
                 og = merged[k1]
                 subset_to = merged[k2]
                 logger.info(
-                    f"{logging_prefix}{LOGGER_PREFIX} extraneous parent node that is subset of another parent node {og.grain.issubset(subset_to.grain)} {og.grain.set} {subset_to.grain.set}"
+                    f"{logging_prefix}{LOGGER_PREFIX} extraneous parent node that is subset of another parent node {og.grain.issubset(subset_to.grain)} {og.grain.components} {subset_to.grain.components}"
                 )
                 merged = {k: v for k, v in merged.items() if k != k1}
                 removed.add(k1)
@@ -197,7 +197,7 @@ class MergeNode(StrategyNode):
     ) -> List[BaseJoin | UnnestJoin]:
         # only finally, join between them for unique values
         dataset_list: List[QueryDatasource | Datasource] = sorted(
-            final_datasets, key=lambda x: -len(x.grain.components_copy)
+            final_datasets, key=lambda x: -len(x.grain.components)
         )
 
         logger.info(

trilogy/hooks/graph_hook.py

@@ -30,6 +30,7 @@ class GraphHook(BaseHook):
         graph: nx.DiGraph,
         target: str | None = None,
         highlight_nodes: list[str] | None = None,
+        remove_isolates: bool = True,
     ):
         from matplotlib import pyplot as plt
 
@@ -38,7 +39,8 @@ class GraphHook(BaseHook):
         for node in nodes:
             if "__preql_internal" in node:
                 graph.remove_node(node)
-        graph.remove_nodes_from(list(nx.isolates(graph)))
+        if remove_isolates:
+            graph.remove_nodes_from(list(nx.isolates(graph)))
         color_map = []
         highlight_nodes = highlight_nodes or []
         for node in graph:

trilogy/parsing/common.py

@@ -4,7 +4,13 @@ from typing import List, Tuple
 from trilogy.constants import (
     VIRTUAL_CONCEPT_PREFIX,
 )
-from trilogy.core.enums import FunctionType, Modifier, PurposeLineage, WindowType
+from trilogy.core.enums import (
+    FunctionType,
+    Granularity,
+    Modifier,
+    PurposeLineage,
+    WindowType,
+)
 from trilogy.core.functions import arg_to_datatype, function_args_to_output_purpose
 from trilogy.core.models import (
     AggregateWrapper,
@@ -66,7 +72,7 @@ def process_function_args(
             concept = function_to_concept(
                 arg,
                 name=f"{VIRTUAL_CONCEPT_PREFIX}_{arg.operator.value}_{id_hash}",
-                namespace=environment.namespace,
+                environment=environment,
             )
             # to satisfy mypy, concept will always have metadata
             if concept.metadata and meta:
@@ -80,7 +86,7 @@ def process_function_args(
             concept = arbitrary_to_concept(
                 arg,
                 name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
-                namespace=environment.namespace,
+                environment=environment,
             )
             if concept.metadata and meta:
                 concept.metadata.line_number = meta.line
@@ -139,10 +145,39 @@ def constant_to_concept(
     )
 
 
+def concepts_to_grain_concepts(
+    concepts: List[Concept] | List[str] | set[str], environment: Environment | None
+) -> list[Concept]:
+    environment = Environment() if environment is None else environment
+    pconcepts: list[Concept] = [
+        c if isinstance(c, Concept) else environment.concepts[c] for c in concepts
+    ]
+
+    final: List[Concept] = []
+    for sub in pconcepts:
+        if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
+            if any([c.address in pconcepts for c in sub.keys]):
+                continue
+        if sub.purpose in (Purpose.METRIC,):
+            if all([c in pconcepts for c in sub.grain.components]):
+                continue
+        if sub.granularity == Granularity.SINGLE_ROW:
+            continue
+        final.append(sub)
+    final = unique(final, "address")
+    v2 = sorted(final, key=lambda x: x.name)
+    return v2
+
+
 def function_to_concept(
-    parent: Function, name: str, namespace: str, metadata: Metadata | None = None
+    parent: Function,
+    name: str,
+    environment: Environment,
+    namespace: str | None = None,
+    metadata: Metadata | None = None,
 ) -> Concept:
     pkeys: List[Concept] = []
+    namespace = namespace or environment.namespace
     for x in parent.arguments:
         pkeys += [
             x
@@ -162,7 +197,7 @@ def function_to_concept(
             key_grain += [*x.keys]
         else:
             key_grain.append(x)
-    keys = tuple(Grain(components=key_grain).components_copy)
+    keys = tuple(concepts_to_grain_concepts(key_grain, environment))
     if not pkeys:
         purpose = Purpose.CONSTANT
     else:
@@ -256,7 +291,7 @@ def window_item_to_concept(
         lineage=parent,
         metadata=fmetadata,
         # filters are implicitly at the grain of the base item
-        grain=Grain(components=grain),
+        grain=Grain.from_concepts(grain),
         namespace=namespace,
         keys=keys,
         modifiers=modifiers,
@@ -268,9 +303,8 @@ def agg_wrapper_to_concept(
     namespace: str,
     name: str,
     metadata: Metadata | None = None,
-    purpose: Purpose | None = None,
 ) -> Concept:
-    local_purpose, keys = get_purpose_and_keys(
+    _, keys = get_purpose_and_keys(
         Purpose.METRIC, tuple(parent.by) if parent.by else None
     )
     # anything grouped to a grain should be a property
@@ -284,7 +318,7 @@ def agg_wrapper_to_concept(
         purpose=Purpose.METRIC,
         metadata=fmetadata,
         lineage=parent,
-        grain=Grain(components=parent.by) if parent.by else Grain(),
+        grain=Grain.from_concepts(parent.by) if parent.by else Grain(),
         namespace=namespace,
         keys=tuple(parent.by) if parent.by else keys,
         modifiers=modifiers,
@@ -304,15 +338,17 @@ def arbitrary_to_concept(
         | float
         | str
     ),
-    namespace: str,
+    environment: Environment,
+    namespace: str | None = None,
     name: str | None = None,
     metadata: Metadata | None = None,
     purpose: Purpose | None = None,
 ) -> Concept:
+    namespace = namespace or environment.namespace
     if isinstance(parent, AggregateWrapper):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
-        return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
+        return agg_wrapper_to_concept(parent, namespace, name, metadata)
     elif isinstance(parent, WindowItem):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_window_{parent.type.value}_{string_to_hash(str(parent))}"
@@ -324,7 +360,13 @@ def arbitrary_to_concept(
     elif isinstance(parent, Function):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
-        return function_to_concept(parent, name, namespace, metadata=metadata)
+        return function_to_concept(
+            parent,
+            name,
+            metadata=metadata,
+            environment=environment,
+            namespace=namespace,
+        )
     elif isinstance(parent, ListWrapper):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"

trilogy/parsing/parse_engine.py

@@ -463,7 +463,7 @@ class ParseToObjects(Transformer):
             datatype=args[2],
             purpose=args[0],
             metadata=metadata,
-            grain=Grain(components=parents),
+            grain=Grain(components={x.address for x in parents}),
             namespace=namespace,
             keys=parents,
             modifiers=modifiers,
@@ -530,6 +530,7 @@ class ParseToObjects(Transformer):
                 source_value,
                 name=name,
                 namespace=namespace,
+                environment=self.environment,
                 purpose=purpose,
                 metadata=metadata,
             )
@@ -598,7 +599,7 @@ class ParseToObjects(Transformer):
                 output_purpose=Purpose.CONSTANT,
                 arguments=[constant],
             ),
-            grain=Grain(components=[]),
+            grain=Grain(components=set()),
             namespace=namespace,
         )
         if concept.metadata:
@@ -623,7 +624,9 @@ class ParseToObjects(Transformer):
         return args
 
     def grain_clause(self, args) -> Grain:
-        return Grain(components=[self.environment.concepts[a] for a in args[0]])
+        return Grain(
+            components=set([self.environment.concepts[a].address for a in args[0]])
+        )
 
     def whole_grain_clause(self, args) -> WholeGrainWrapper:
         return WholeGrainWrapper(where=args[0])
@@ -715,7 +718,11 @@ class ParseToObjects(Transformer):
             )
         elif isinstance(transformation, Function):
             concept = function_to_concept(
-                transformation, namespace=namespace, name=output, metadata=metadata
+                transformation,
+                namespace=namespace,
+                name=output,
+                metadata=metadata,
+                environment=self.environment,
             )
         else:
             raise SyntaxError("Invalid transformation")
@@ -771,10 +778,8 @@ class ParseToObjects(Transformer):
         def handle_order_item(x, namespace: str):
             if not isinstance(x, Concept):
                 x = arbitrary_to_concept(
-                    x,
-                    namespace=namespace,
+                    x, namespace=namespace, environment=self.environment
                 )
-                self.environment.add_concept(x)
             return x
 
         return [
@@ -1029,12 +1034,32 @@ class ParseToObjects(Transformer):
             order_by=order_by,
             meta=Metadata(line_number=meta.line),
         )
-        for parse_pass in [1, 2]:
+        for parse_pass in [
+            1,
+            2,
+        ]:
             # the first pass will result in all concepts being defined
             # the second will get grains appropriately
             # eg if someone does sum(x)->a, b+c -> z - we don't know if Z is a key to group by or an aggregate
             # until after the first pass, and so don't know the grain of a
-            nselect = []
+
+            if parse_pass == 1:
+                grain = Grain.from_concepts(
+                    [
+                        x.content
+                        for x in output.selection
+                        if isinstance(x.content, Concept)
+                    ],
+                    where_clause=output.where_clause,
+                )
+            if parse_pass == 2:
+                grain = Grain.from_concepts(
+                    output.output_components, where_clause=output.where_clause
+                )
+            print(
+                f"end pass {parse_pass} grain {grain} from {output.output_components}"
+            )
+            output.grain = grain
             for item in select_items:
                 # we don't know the grain of an aggregate at assignment time
                 # so rebuild at this point in the tree
@@ -1064,7 +1089,7 @@ class ParseToObjects(Transformer):
                         environment=self.environment,
                     )
                     output.local_concepts[item.content.address] = item.content
-                nselect.append(item)
+
         if order_by:
             output.order_by = order_by.with_select_context(
                 local_concepts=output.local_concepts,
@@ -1181,7 +1206,7 @@ class ParseToObjects(Transformer):
         if isinstance(args[0], AggregateWrapper):
             left = arbitrary_to_concept(
                 args[0],
-                namespace=self.environment.namespace,
+                environment=self.environment,
             )
             self.environment.add_concept(left)
         else:
@@ -1189,7 +1214,7 @@ class ParseToObjects(Transformer):
         if isinstance(args[2], AggregateWrapper):
             right = arbitrary_to_concept(
                 args[2],
-                namespace=self.environment.namespace,
+                environment=self.environment,
             )
             self.environment.add_concept(right)
         else:
@@ -1227,10 +1252,7 @@ class ParseToObjects(Transformer):
         ):
             right = right.content
         if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
-            right = arbitrary_to_concept(
-                right,
-                namespace=self.environment.namespace,
-            )
+            right = arbitrary_to_concept(right, environment=self.environment)
             self.environment.add_concept(right, meta=meta)
         return SubselectComparison(
             left=args[0],
@@ -1299,10 +1321,7 @@ class ParseToObjects(Transformer):
             elif isinstance(item, WindowType):
                 type = item
             else:
-                concept = arbitrary_to_concept(
-                    item,
-                    namespace=self.environment.namespace,
-                )
+                concept = arbitrary_to_concept(item, environment=self.environment)
                 self.environment.add_concept(concept, meta=meta)
         assert concept
         return WindowItem(

trilogy/parsing/render.py

@@ -158,7 +158,14 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "Grain"):
-        components = ",".join(self.to_string(x) for x in arg.components)
+        final = []
+        for comp in arg.components:
+            if comp.startswith(DEFAULT_NAMESPACE):
+                final.append(comp.split(".", 1)[1])
+            else:
+                final.append(comp)
+        final = sorted(final)
+        components = ",".join(x for x in final)
         return f"grain ({components})"
 
     @to_string.register