pytrilogy 0.0.3.37__py3-none-any.whl → 0.0.3.39__py3-none-any.whl

trilogy/core/processing/node_generators/filter_node.py

@@ -48,9 +48,10 @@ def gen_filter_node(
         if isinstance(x.lineage, FILTER_TYPES):
             if concept.lineage.where == where:
                 logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent from optional {x} with same filter conditions "
                 )
-                parent_row_concepts.append(x.lineage.content)
+                if x.lineage.content.address not in parent_row_concepts:
+                    parent_row_concepts.append(x.lineage.content)
                 optional_included.append(x)
                 continue
         if conditions and conditions == where:

trilogy/core/processing/node_generators/group_to_node.py

@@ -3,6 +3,8 @@ from typing import List
 from trilogy.constants import logger
 from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
+
+# C:\Users\ethan\coding_projects\pytrilogy\trilogy\core\processing\node_generators\group_to_node.py
 from trilogy.core.processing.nodes import (
     GroupNode,
     History,
@@ -30,6 +32,7 @@ def gen_group_to_node(
             f"Group to should have function lineage, is {type(concept.lineage)}"
         )
     group_arg = concept.lineage.arguments[0]
+
     parent_concepts: List[BuildConcept] = concept.lineage.concept_arguments
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} group by node has required parents {[x.address for x in parent_concepts]}"

trilogy/core/processing/node_generators/union_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import FunctionType, Purpose
+from trilogy.core.enums import FunctionType
 from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
 from trilogy.core.processing.nodes import History, StrategyNode, UnionNode
 from trilogy.core.processing.utility import padding
@@ -16,6 +16,38 @@ def is_union(c: BuildConcept):
     )
 
 
+def build_layers(
+    concepts: list[BuildConcept],
+) -> tuple[list[list[BuildConcept]], list[BuildConcept]]:
+    sources = {
+        x.address: x.lineage.concept_arguments if x.lineage else [] for x in concepts
+    }
+    root = concepts[0]
+
+    built_layers = []
+    layers = root.lineage.concept_arguments if root.lineage else []
+    sourced = set()
+    while layers:
+        layer = []
+        current = layers.pop()
+        sourced.add(current.address)
+        layer.append(current)
+        for key, values in sources.items():
+            if key == current.address:
+                continue
+            for value in values:
+                if value.address in (current.keys or []) or current.address in (
+                    value.keys or []
+                ):
+                    layer.append(value)
+                    sourced.add(value.address)
+        built_layers.append(layer)
+    complete = [
+        x for x in concepts if all([x.address in sourced for x in sources[x.address]])
+    ]
+    return built_layers, complete
+
+
 def gen_union_node(
     concept: BuildConcept,
     local_optional: List[BuildConcept],
@@ -27,27 +59,15 @@ def gen_union_node(
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     all_unions = [x for x in local_optional if is_union(x)] + [concept]
-
-    parents = []
-    keys = [x for x in all_unions if x.purpose == Purpose.KEY]
-    base = keys.pop()
-    remaining = [x for x in all_unions if x.address != base.address]
-    arguments = []
-    if isinstance(base.lineage, BuildFunction):
-        arguments = base.lineage.concept_arguments
-    for arg in arguments:
-        relevant_parents: list[BuildConcept] = []
-        for other_union in remaining:
-            assert other_union.lineage
-            potential_parents = [z for z in other_union.lineage.concept_arguments]
-            relevant_parents += [
-                x for x in potential_parents if x.keys and arg.address in x.keys
-            ]
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} found unions {all_unions}")
+    parent_nodes = []
+    layers, resolved = build_layers(all_unions)
+    for layer in layers:
         logger.info(
-            f"For parent arg {arg.address}, including additional union inputs {[c.address for c in relevant_parents]}"
+            f"{padding(depth)}{LOGGER_PREFIX} fetching layer {layer} with resolved {resolved}"
         )
         parent: StrategyNode = source_concepts(
-            mandatory_list=[arg] + relevant_parents,
+            mandatory_list=layer,
             environment=environment,
             g=g,
             depth=depth + 1,
@@ -55,12 +75,8 @@ def gen_union_node(
             conditions=conditions,
         )
         parent.hide_output_concepts(parent.output_concepts)
-        # parent.remove_output_concepts(parent.output_concepts)
-        parent.add_output_concept(concept)
-        for x in remaining:
-            parent.add_output_concept(x)
-
-        parents.append(parent)
+        parent.add_output_concepts(resolved)
+        parent_nodes.append(parent)
         if not parent:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} could not find union node parents"
@@ -68,8 +84,8 @@ def gen_union_node(
             return None
 
     return UnionNode(
-        input_concepts=[concept] + local_optional,
-        output_concepts=[concept] + local_optional,
+        input_concepts=resolved,
+        output_concepts=resolved,
         environment=environment,
-        parents=parents,
+        parents=parent_nodes,
     )

trilogy/core/processing/nodes/__init__.py

@@ -45,6 +45,11 @@ class History(BaseModel):
         self.history[
             self._concepts_to_lookup(search, accept_partial, conditions=conditions)
         ] = output
+        self.log_end(
+            search,
+            accept_partial=accept_partial,
+            conditions=conditions,
+        )
 
     def get_history(
         self,
@@ -83,6 +88,20 @@ class History(BaseModel):
             )
         )
 
+    def log_end(
+        self,
+        search: list[BuildConcept],
+        accept_partial: bool = False,
+        conditions: BuildWhereClause | None = None,
+    ):
+        self.started.discard(
+            self._concepts_to_lookup(
+                search,
+                accept_partial=accept_partial,
+                conditions=conditions,
+            )
+        )
+
     def check_started(
         self,
         search: list[BuildConcept],

trilogy/core/processing/nodes/merge_node.py

@@ -272,6 +272,8 @@ class MergeNode(StrategyNode):
                     f"{self.logging_prefix}{LOGGER_PREFIX} Merge node has only one parent with the same"
                     " outputs as this merge node, dropping merge node "
                 )
+                # push up any conditions we need
+                final.ordering = self.ordering
                 return final
 
         # if we have multiple candidates, see if one is good enough
@@ -293,6 +295,7 @@ class MergeNode(StrategyNode):
                     f" has all required output properties with partial {[c.address for c in dataset.partial_concepts]}"
                     f" and self has no conditions ({self.conditions})"
                 )
+                dataset.ordering = self.ordering
                 return dataset
 
         pregrain = BuildGrain()

trilogy/core/processing/utility.py

@@ -603,15 +603,29 @@ def find_nullable_concepts(
 def sort_select_output_processed(
     cte: CTE | UnionCTE, query: ProcessedQuery
 ) -> CTE | UnionCTE:
-    output_addresses = [
-        c.address for c in query.output_columns if c.address not in query.hidden_columns
-    ]
+    output_addresses = [c.address for c in query.output_columns]
 
     mapping = {x.address: x for x in cte.output_columns}
 
-    new_output = []
+    new_output: list[BuildConcept] = []
     for x in output_addresses:
         new_output.append(mapping[x])
+
+    for oc in cte.output_columns:
+        # add hidden back
+        if oc.address not in output_addresses:
+            new_output.append(oc)
+
+    cte.hidden_concepts = set(
+        [
+            c.address
+            for c in cte.output_columns
+            if (
+                c.address not in query.output_columns
+                or c.address in query.hidden_columns
+            )
+        ]
+    )
     cte.output_columns = new_output
     return cte
 
@@ -619,18 +633,28 @@ def sort_select_output_processed(
 def sort_select_output(
     cte: CTE | UnionCTE, query: SelectStatement | MultiSelectStatement | ProcessedQuery
 ) -> CTE | UnionCTE:
+
     if isinstance(query, ProcessedQuery):
         return sort_select_output_processed(cte, query)
+
     output_addresses = [
         c.address
         for c in query.output_components
-        if c.address not in query.hidden_components
+        # if c.address not in query.hidden_components
     ]
 
     mapping = {x.address: x for x in cte.output_columns}
 
-    new_output = []
+    new_output: list[BuildConcept] = []
     for x in output_addresses:
         new_output.append(mapping[x])
     cte.output_columns = new_output
+    cte.hidden_concepts = set(
+        [
+            c.address
+            for c in query.output_components
+            if c.address in query.hidden_components
+        ]
+    )
+
     return cte

trilogy/core/query_processor.py

@@ -431,7 +431,9 @@ def get_query_datasources(
     hooks: Optional[List[BaseHook]] = None,
 ) -> QueryDatasource:
     ds = get_query_node(environment, statement.as_lineage(environment))
+
     final_qds = ds.resolve()
+
     if hooks:
         for hook in hooks:
             hook.process_root_strategy_node(ds)
@@ -510,6 +512,7 @@ def process_query(
         hook.process_root_datasource(root_datasource)
     # this should always return 1 - TODO, refactor
     root_cte = datasource_to_cte(root_datasource, environment.cte_name_map)
+
     for hook in hooks:
         hook.process_root_cte(root_cte)
     raw_ctes: List[CTE | UnionCTE] = list(reversed(flatten_ctes(root_cte)))

trilogy/core/statements/author.py

@@ -135,6 +135,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
         )
 
         output.grain = output.calculate_grain(environment)
+
         for x in selection:
             if x.is_undefined and environment.concepts.fail_on_missing:
                 environment.concepts.raise_undefined(
@@ -166,6 +167,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
         targets = []
         for x in self.selection:
             targets.append(x.concept)
+
         result = Grain.from_concepts(
             targets, where_clause=self.where_clause, environment=environment
         )

trilogy/dialect/base.py

@@ -171,6 +171,7 @@ FUNCTION_MAP = {
     FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
     # math
     FunctionType.ADD: lambda x: " + ".join(x),
+    FunctionType.ABS: lambda x: f"abs({x[0]})",
     FunctionType.SUBTRACT: lambda x: " - ".join(x),
     FunctionType.DIVIDE: lambda x: " / ".join(x),
     FunctionType.MULTIPLY: lambda x: " * ".join(x),

trilogy/parsing/common.py

@@ -102,8 +102,10 @@ def process_function_arg(
             concept.metadata.line_number = meta.line
         environment.add_concept(concept, meta=meta)
         return concept
+    elif isinstance(arg, Concept):
+        return arg.reference
     elif isinstance(arg, ConceptRef):
-        return environment.concepts[arg.address]
+        return environment.concepts[arg.address].reference
     return arg
 
 
@@ -111,8 +113,8 @@ def process_function_args(
     args,
     meta: Meta | None,
     environment: Environment,
-) -> List[Concept | Function | str | int | float | date | datetime]:
-    final: List[Concept | Function | str | int | float | date | datetime] = []
+) -> List[ConceptRef | Function | str | int | float | date | datetime]:
+    final: List[ConceptRef | Function | str | int | float | date | datetime] = []
     for arg in args:
         final.append(process_function_arg(arg, meta, environment))
     return final
@@ -220,7 +222,6 @@ def concept_is_relevant(
         if all([c in others for c in concept.grain.components]):
             return False
     if concept.derivation in (Derivation.BASIC,):
-
         return any(
             concept_is_relevant(c, others, environment)
             for c in concept.concept_arguments
@@ -235,6 +236,7 @@ def concepts_to_grain_concepts(
 ) -> list[Concept]:
     pconcepts: list[Concept] = []
     for c in concepts:
+
         if isinstance(c, Concept):
             pconcepts.append(c)
         elif isinstance(c, ConceptRef) and environment:
@@ -281,6 +283,76 @@ def get_relevant_parent_concepts(arg):
     return results
 
 
+def group_function_to_concept(
+    parent: Function,
+    name: str,
+    environment: Environment,
+    namespace: str | None = None,
+    metadata: Metadata | None = None,
+):
+    pkeys: List[Concept] = []
+    namespace = namespace or environment.namespace
+    is_metric = False
+    ref_args, is_metric = get_relevant_parent_concepts(parent)
+    concrete_args = [environment.concepts[c.address] for c in ref_args]
+    pkeys += [x for x in concrete_args if not x.derivation == Derivation.CONSTANT]
+    modifiers = get_upstream_modifiers(pkeys, environment)
+    key_grain: list[str] = []
+    for x in pkeys:
+        # for a group to, if we have a dynamic metric, ignore it
+        # it will end up with the group target grain
+        if x.purpose == Purpose.METRIC and not x.keys:
+            continue
+        # metrics will group to keys, so do no do key traversal
+        elif is_metric:
+            key_grain.append(x.address)
+        else:
+            key_grain.append(x.address)
+    keys = set(key_grain)
+
+    grain = Grain.from_concepts(keys, environment)
+    if is_metric:
+        purpose = Purpose.METRIC
+    elif not pkeys:
+        purpose = Purpose.CONSTANT
+    else:
+        purpose = parent.output_purpose
+    fmetadata = metadata or Metadata()
+    granularity = Granularity.MULTI_ROW
+
+    if grain is not None:
+        # deduplicte
+        grain = Grain.from_concepts(grain.components, environment)
+
+        r = Concept(
+            name=name,
+            datatype=parent.output_datatype,
+            purpose=purpose,
+            lineage=parent,
+            namespace=namespace,
+            keys=keys,
+            modifiers=modifiers,
+            grain=grain,
+            metadata=fmetadata,
+            derivation=Derivation.BASIC,
+            granularity=granularity,
+        )
+        return r
+
+    return Concept(
+        name=name,
+        datatype=parent.output_datatype,
+        purpose=purpose,
+        lineage=parent,
+        namespace=namespace,
+        keys=keys,
+        modifiers=modifiers,
+        metadata=fmetadata,
+        derivation=Derivation.BASIC,
+        granularity=granularity,
+    )
+
+
 def function_to_concept(
     parent: Function,
     name: str,
@@ -347,7 +419,6 @@ def function_to_concept(
     else:
         derivation = Derivation.BASIC
         granularity = Granularity.MULTI_ROW
-    # granularity = Concept.calculate_granularity(derivation, grain, parent)
 
     if grain is not None:
         r = Concept(
@@ -695,6 +766,14 @@ def arbitrary_to_concept(
     elif isinstance(parent, Function):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
+        if parent.operator == FunctionType.GROUP:
+            return group_function_to_concept(
+                parent,
+                name,
+                environment=environment,
+                namespace=namespace,
+                metadata=metadata,
+            )
         return function_to_concept(
             parent,
             name,

trilogy/parsing/parse_engine.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from datetime import date, datetime
 from enum import Enum
+from logging import getLogger
 from os.path import dirname, join
 from pathlib import Path
 from re import IGNORECASE
@@ -137,6 +138,8 @@ from trilogy.parsing.common import (
 )
 from trilogy.parsing.exceptions import ParseError
 
+perf_logger = getLogger("trilogy.parse.performance")
+
 
 class ParsePass(Enum):
     INITIAL = 1
@@ -407,6 +410,7 @@ class ParseToObjects(Transformer):
         mapping = self.environment.concepts[address]
         datatype = mapping.output_datatype
         return ConceptRef(
+            # this is load-bearing to handle pseudonyms
             address=mapping.address,
             metadata=Metadata(line_number=meta.line),
             datatype=datatype,
@@ -754,6 +758,11 @@ class ParseToObjects(Transformer):
             components=set([self.environment.concepts[a].address for a in args[0]])
         )
 
+    @v_args(meta=True)
+    def aggregate_by(self, meta: Meta, args):
+        args = [self.environment.concepts[a] for a in args]
+        return self.function_factory.create_function(args, FunctionType.GROUP, meta)
+
     def whole_grain_clause(self, args) -> WholeGrainWrapper:
         return WholeGrainWrapper(where=args[0])
 
@@ -1005,6 +1014,7 @@ class ParseToObjects(Transformer):
         return text
 
     def import_statement(self, args: list[str]) -> ImportStatement:
+        start = datetime.now()
         if len(args) == 2:
             alias = args[-1]
             cache_key = args[-1]
@@ -1041,9 +1051,11 @@ class ParseToObjects(Transformer):
             )
 
         if token_lookup in self.tokens:
+            perf_logger.debug(f"\tTokens cached for {token_lookup}")
             raw_tokens = self.tokens[token_lookup]
             text = self.text_lookup[token_lookup]
         else:
+            perf_logger.debug(f"\tTokens not cached for {token_lookup}, resolving")
             text = self.resolve_import_address(target, is_stdlib)
             self.text_lookup[token_lookup] = text
 
@@ -1056,12 +1068,19 @@ class ParseToObjects(Transformer):
             self.tokens[token_lookup] = raw_tokens
 
         if cache_lookup in self.parsed:
+            perf_logger.debug(f"\tEnvironment cached for {token_lookup}")
             nparser = self.parsed[cache_lookup]
             new_env = nparser.environment
             if nparser.parse_pass != ParsePass.VALIDATION:
                 # nparser.transform(raw_tokens)
+                second_pass_start = datetime.now()
                 nparser.run_second_parse_pass()
+                second_pass_end = datetime.now()
+                perf_logger.debug(
+                    f"{second_pass_end - second_pass_start} seconds | Import {alias} key ({cache_key}) second pass took {second_pass_end - second_pass_start} to parse, {len(new_env.concepts)} concepts"
+                )
         else:
+            perf_logger.debug(f"\tParsing new for {token_lookup}")
             try:
                 new_env = Environment(
                     working_path=dirname(target),
@@ -1093,6 +1112,10 @@ class ParseToObjects(Transformer):
         self.environment.add_import(
             alias, new_env, Import(alias=alias, path=parsed_path)
         )
+        end = datetime.now()
+        perf_logger.debug(
+            f"{end - start} seconds | Import {alias} key ({cache_key}) took  to parse, {len(new_env.concepts)} concepts"
+        )
         return imps
 
     @v_args(meta=True)
@@ -1268,8 +1291,8 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def function_binding_item(self, meta: Meta, args) -> ArgBinding:
         if len(args) == 2:
-            return ArgBinding(name=args[0], default=args[1])
-        return ArgBinding(name=args[0], default=None)
+            return ArgBinding.model_construct(name=args[0], default=args[1])
+        return ArgBinding.model_construct(name=args[0], default=None)
 
     @v_args(meta=True)
     def raw_function(self, meta: Meta, args) -> FunctionDeclaration:
@@ -1436,12 +1459,14 @@ class ParseToObjects(Transformer):
         return args[0]
 
     def window(self, args):
+
         return Window(count=args[1].value, window_order=args[0])
 
     def WINDOW_TYPE(self, args):
         return WindowType(args.strip())
 
     def window_item_over(self, args):
+
         return WindowItemOver(contents=args[0])
 
     def window_item_order(self, args):
@@ -1850,7 +1875,7 @@ def parse_text(
     parser = ParseToObjects(
         environment=environment, import_keys=["root"], parse_config=parse_config
     )
-
+    start = datetime.now()
     try:
         parser.set_text(text)
         # disable fail on missing to allow for circular dependencies
@@ -1860,6 +1885,10 @@ def parse_text(
         pass_two = parser.run_second_parse_pass()
         output = [v for v in pass_two if v]
         environment.concepts.fail_on_missing = True
+        end = datetime.now()
+        perf_logger.debug(
+            f"Parse time: {end - start} for {len(text)} characters, {len(output)} objects"
+        )
     except VisitError as e:
         unpack_visit_error(e)
         # this will never be reached