pytrilogy 0.0.3.55__py3-none-any.whl → 0.0.3.57__py3-none-any.whl

trilogy/core/processing/discovery_validation.py

@@ -0,0 +1,155 @@
+from collections import defaultdict
+from enum import Enum
+from typing import List
+
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    StrategyNode,
+)
+from trilogy.core.processing.utility import (
+    get_disconnected_components,
+)
+
+
+class ValidationResult(Enum):
+    COMPLETE = 1
+    DISCONNECTED = 2
+    INCOMPLETE = 3
+    INCOMPLETE_CONDITION = 4
+
+
+def validate_concept(
+    concept: BuildConcept,
+    node: StrategyNode,
+    found_addresses: set[str],
+    non_partial_addresses: set[str],
+    partial_addresses: set[str],
+    virtual_addresses: set[str],
+    found_map: dict[str, set[BuildConcept]],
+    accept_partial: bool,
+    seen: set[str],
+    environment: BuildEnvironment,
+):
+    found_map[str(node)].add(concept)
+    seen.add(concept.address)
+    if concept not in node.partial_concepts:
+        found_addresses.add(concept.address)
+        non_partial_addresses.add(concept.address)
+        # remove it from our partial tracking
+        if concept.address in partial_addresses:
+            partial_addresses.remove(concept.address)
+        if concept.address in virtual_addresses:
+            virtual_addresses.remove(concept.address)
+    if concept in node.partial_concepts:
+        if concept.address in non_partial_addresses:
+            return None
+        partial_addresses.add(concept.address)
+        if accept_partial:
+            found_addresses.add(concept.address)
+            found_map[str(node)].add(concept)
+    for v_address in concept.pseudonyms:
+        if v_address in seen:
+            return
+        v = environment.concepts[v_address]
+        if v.address in seen:
+            return
+        if v.address == concept.address:
+            return
+        validate_concept(
+            v,
+            node,
+            found_addresses,
+            non_partial_addresses,
+            partial_addresses,
+            virtual_addresses,
+            found_map,
+            accept_partial,
+            seen=seen,
+            environment=environment,
+        )
+
+
+def validate_stack(
+    environment: BuildEnvironment,
+    stack: List[StrategyNode],
+    concepts: List[BuildConcept],
+    mandatory_with_filter: List[BuildConcept],
+    conditions: BuildWhereClause | None = None,
+    accept_partial: bool = False,
+) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
+    found_map: dict[str, set[BuildConcept]] = defaultdict(set)
+    found_addresses: set[str] = set()
+    non_partial_addresses: set[str] = set()
+    partial_addresses: set[str] = set()
+    virtual_addresses: set[str] = set()
+    seen: set[str] = set()
+
+    for node in stack:
+        resolved = node.resolve()
+
+        for concept in resolved.output_concepts:
+            if concept.address in resolved.hidden_concepts:
+                continue
+
+            validate_concept(
+                concept,
+                node,
+                found_addresses,
+                non_partial_addresses,
+                partial_addresses,
+                virtual_addresses,
+                found_map,
+                accept_partial,
+                seen,
+                environment,
+            )
+        for concept in node.virtual_output_concepts:
+            if concept.address in non_partial_addresses:
+                continue
+            found_addresses.add(concept.address)
+            virtual_addresses.add(concept.address)
+    if not conditions:
+        conditions_met = True
+    else:
+        conditions_met = all(
+            [node.preexisting_conditions == conditions.conditional for node in stack]
+        ) or all([c.address in found_addresses for c in mandatory_with_filter])
+    # zip in those we know we found
+    if not all([c.address in found_addresses for c in concepts]) or not conditions_met:
+        if not all([c.address in found_addresses for c in concepts]):
+            return (
+                ValidationResult.INCOMPLETE,
+                found_addresses,
+                {c.address for c in concepts if c.address not in found_addresses},
+                partial_addresses,
+                virtual_addresses,
+            )
+        return (
+            ValidationResult.INCOMPLETE_CONDITION,
+            found_addresses,
+            {c.address for c in concepts if c.address not in mandatory_with_filter},
+            partial_addresses,
+            virtual_addresses,
+        )
+
+    graph_count, _ = get_disconnected_components(found_map)
+    if graph_count in (0, 1):
+        return (
+            ValidationResult.COMPLETE,
+            found_addresses,
+            set(),
+            partial_addresses,
+            virtual_addresses,
+        )
+    # if we have too many subgraphs, we need to keep searching
+    return (
+        ValidationResult.DISCONNECTED,
+        found_addresses,
+        set(),
+        partial_addresses,
+        virtual_addresses,
+    )

trilogy/core/processing/node_generators/basic_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import FunctionClass, SourceType
+from trilogy.core.enums import FunctionClass, FunctionType, SourceType
 from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
@@ -47,13 +47,25 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} with lineage {concept.lineage} has parents {[x for x in parent_concepts]}"
     )
-
+    synonyms: list[BuildConcept] = []
+    ignored_optional: set[str] = set()
+    assert isinstance(concept.lineage, BuildFunction)
+    if concept.lineage.operator == FunctionType.ATTR_ACCESS:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} checking for synonyms for attribute access"
+        )
+        for x in local_optional:
+            for z in x.pseudonyms:
+                s_concept = environment.alias_origin_lookup[z]
+                if is_equivalent_basic_function_lineage(concept, s_concept):
+                    synonyms.append(s_concept)
+                    ignored_optional.add(x.address)
     equivalent_optional = [
         x
         for x in local_optional
         if is_equivalent_basic_function_lineage(concept, x)
         and x.address != concept.address
-    ]
+    ] + synonyms
 
     if equivalent_optional:
         logger.info(
@@ -66,6 +78,7 @@ def gen_basic_node(
         for x in local_optional
         if x not in equivalent_optional
         and not any(x.address in y.pseudonyms for y in equivalent_optional)
+        and x.address not in ignored_optional
     ]
     all_parents: list[BuildConcept] = unique(
         parent_concepts + non_equivalent_optional, "address"
@@ -73,7 +86,7 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Fetching parents {[x.address for x in all_parents]}"
     )
-    parent_node: StrategyNode = source_concepts(
+    parent_node: StrategyNode | None = source_concepts(
         mandatory_list=all_parents,
         environment=environment,
         g=g,
@@ -92,14 +105,19 @@ def gen_basic_node(
     parent_node.add_output_concept(concept)
     for x in equivalent_optional:
         parent_node.add_output_concept(x)
-
-    parent_node.remove_output_concepts(
-        [
-            x
-            for x in parent_node.output_concepts
-            if x.address not in [concept] + local_optional
-        ]
+    targets = [concept] + local_optional
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"
     )
+    should_hide = [
+        x
+        for x in parent_node.output_concepts
+        if (
+            x.address not in targets
+            and not any(x.address in y.pseudonyms for y in targets)
+        )
+    ]
+    parent_node.hide_output_concepts(should_hide)
 
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"

trilogy/core/processing/node_generators/node_merge_node.py

@@ -143,7 +143,7 @@ def determine_induced_minimal_nodes(
     if not all([node in final.nodes for node in nodelist]):
         missing = [node for node in nodelist if node not in final.nodes]
         logger.debug(
-            f"Skipping graph for {nodelist} as missing nodes {missing} from {final.nodes}"
+            f"Skipping graph for initial list {nodelist} as missing nodes {missing} from final graph {final.nodes}"
         )
         return None
     logger.debug(f"Found final graph {final.nodes}")

trilogy/core/processing/node_generators/select_node.py

@@ -19,8 +19,7 @@ LOGGER_PREFIX = "[GEN_SELECT_NODE]"
 
 
 def gen_select_node(
-    concept: BuildConcept,
-    local_optional: list[BuildConcept],
+    concepts: list[BuildConcept],
     environment: BuildEnvironment,
     g,
     depth: int,
@@ -28,12 +27,11 @@ def gen_select_node(
     fail_if_not_found: bool = True,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
-    all_concepts = [concept] + local_optional
-    all_lcl = LooseBuildConceptList(concepts=all_concepts)
+    all_lcl = LooseBuildConceptList(concepts=concepts)
     materialized_lcl = LooseBuildConceptList(
         concepts=[
             x
-            for x in all_concepts
+            for x in concepts
             if x.address in environment.materialized_concepts
             or x.derivation == Derivation.CONSTANT
         ]
@@ -41,15 +39,15 @@ def gen_select_node(
     if materialized_lcl != all_lcl:
         missing = all_lcl.difference(materialized_lcl)
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
+            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concepts}"
             f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing}) "
         )
         if fail_if_not_found:
-            raise NoDatasourceException(f"No datasource exists for {concept}")
+            raise NoDatasourceException(f"No datasource exists for {concepts}")
         return None
 
     return gen_select_merge_node(
-        [concept] + local_optional,
+        concepts,
         g=g,
         environment=environment,
         depth=depth,

trilogy/core/processing/node_generators/synonym_node.py

@@ -29,7 +29,7 @@ def gen_synonym_node(
     conditions: BuildWhereClause | None = None,
     accept_partial: bool = False,
 ) -> StrategyNode | None:
-    local_prefix = f"[GEN_SYNONYM_NODE] {padding(depth)}"
+    local_prefix = f"{padding(depth)}[GEN_SYNONYM_NODE]"
     base_fingerprint = tuple([x.address for x in all_concepts])
     synonyms = defaultdict(list)
     synonym_count = 0
@@ -64,5 +64,6 @@ def gen_synonym_node(
         )
         if attempt:
             logger.info(f"{local_prefix} found inputs with {combo}")
+            print(attempt.output_concepts)
             return attempt
     return None

trilogy/core/processing/node_generators/unnest_node.py

@@ -20,16 +20,22 @@ def gen_unnest_node(
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     arguments = []
+    depth_prefix = "\t" * depth
     if isinstance(concept.lineage, BuildFunction):
         arguments = concept.lineage.concept_arguments
 
     equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
+    all_parents = arguments + non_equivalent_optional
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents {all_parents} and equivalent optional {equivalent_optional}"
+    )
     if arguments or local_optional:
         parent = source_concepts(
-            mandatory_list=arguments + non_equivalent_optional,
+            mandatory_list=all_parents,
             environment=environment,
             g=g,
             depth=depth + 1,

trilogy/core/processing/nodes/__init__.py

@@ -150,7 +150,6 @@ class History(BaseModel):
         environment: BuildEnvironment,
         g,
         depth: int,
-        source_concepts,
         fail_if_not_found: bool = False,
         accept_partial: bool = False,
         accept_partial_optional: bool = False,
@@ -169,8 +168,7 @@ class History(BaseModel):
         if fingerprint in self.select_history:
             return self.select_history[fingerprint]
         gen = gen_select_node(
-            concept,
-            local_optional,
+            [concept] + local_optional,
             environment,
             g,
             depth + 1,
@@ -190,8 +188,8 @@ __all__ = [
     "WindowNode",
     "StrategyNode",
     "NodeJoin",
-    "ConstantNode",
     "UnnestNode",
+    "ConstantNode",
     "UnionNode",
     "History",
     "WhereSafetyNode",

trilogy/core/processing/nodes/base_node.py

@@ -311,19 +311,6 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
-    def remove_output_concepts(
-        self, concepts: List[BuildConcept], rebuild: bool = True
-    ):
-        for x in concepts:
-            self.hidden_concepts.add(x.address)
-        addresses = [x.address for x in concepts]
-        self.output_concepts = [
-            x for x in self.output_concepts if x.address not in addresses
-        ]
-        if rebuild:
-            self.rebuild_cache()
-        return self
-
     @property
     def usable_outputs(self) -> list[BuildConcept]:
         return [

trilogy/core/processing/nodes/group_node.py

@@ -105,7 +105,7 @@ class GroupNode(StrategyNode):
         if comp_grain.issubset(target_grain):
 
             logger.info(
-                f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, is subset, no grain required"
+                f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, grain is subset of target, no group node required"
             )
             return GroupRequiredResponse(target_grain, comp_grain, False)
         # find out what extra is in the comp grain vs target grain

trilogy/core/processing/utility.py

@@ -409,7 +409,7 @@ def get_node_joins(
 
 
 def get_disconnected_components(
-    concept_map: Dict[str, Set[BuildConcept]]
+    concept_map: Dict[str, Set[BuildConcept]],
 ) -> Tuple[int, List]:
     """Find if any of the datasources are not linked"""
     import networkx as nx
@@ -608,8 +608,24 @@ def sort_select_output_processed(
     mapping = {x.address: x for x in cte.output_columns}
 
     new_output: list[BuildConcept] = []
-    for x in output_addresses:
-        new_output.append(mapping[x])
+    for x in query.output_columns:
+        if x.address in mapping:
+            new_output.append(mapping[x.address])
+        for oc in cte.output_columns:
+            if x.address in oc.pseudonyms:
+                # create a wrapper BuildConcept to render the pseudonym under the original name
+                new_output.append(
+                    BuildConcept(
+                        name=x.name,
+                        namespace=x.namespace,
+                        pseudonyms={oc.address},
+                        datatype=oc.datatype,
+                        purpose=oc.purpose,
+                        grain=oc.grain,
+                        build_is_aggregate=oc.build_is_aggregate,
+                    )
+                )
+                break
 
     for oc in cte.output_columns:
         # add hidden back
@@ -637,17 +653,28 @@ def sort_select_output(
     if isinstance(query, ProcessedQuery):
         return sort_select_output_processed(cte, query)
 
-    output_addresses = [
-        c.address
-        for c in query.output_components
-        # if c.address not in query.hidden_components
-    ]
-
     mapping = {x.address: x for x in cte.output_columns}
 
     new_output: list[BuildConcept] = []
-    for x in output_addresses:
-        new_output.append(mapping[x])
+    for x in query.output_components:
+        if x.address in mapping:
+            new_output.append(mapping[x.address])
+        else:
+            for oc in cte.output_columns:
+                if x.address in oc.pseudonyms:
+                    # create a wrapper BuildConcept to render the pseudonym under the original name
+                    new_output.append(
+                        BuildConcept(
+                            name=x.name,
+                            namespace=x.namespace,
+                            pseudonyms={oc.address},
+                            datatype=oc.datatype,
+                            purpose=oc.purpose,
+                            grain=oc.grain,
+                            build_is_aggregate=oc.build_is_aggregate,
+                        )
+                    )
+                    break
     cte.output_columns = new_output
     cte.hidden_concepts = set(
         [

trilogy/core/query_processor.py

@@ -432,7 +432,7 @@ def get_query_node(
             )
         ds = SelectNode(
             output_concepts=build_statement.output_components,
-            input_concepts=ds.output_concepts,
+            input_concepts=ds.usable_outputs,
             parents=[ds],
             environment=ds.environment,
             partial_concepts=ds.partial_concepts,
@@ -553,11 +553,11 @@ def process_query(
     root_cte.hidden_concepts = statement.hidden_components
 
     final_ctes = optimize_ctes(deduped_ctes, root_cte, statement)
-    mapping = {x.address: x for x in cte.output_columns}
+
     return ProcessedQuery(
         order_by=root_cte.order_by,
         limit=statement.limit,
-        output_columns=[mapping[x.address] for x in statement.output_components],
+        output_columns=statement.output_components,
         ctes=final_ctes,
         base=root_cte,
         hidden_columns=set([x for x in statement.hidden_components]),

trilogy/core/statements/author.py

@@ -5,7 +5,7 @@ from typing import Annotated, List, Optional, Union
 from pydantic import BaseModel, Field, computed_field, field_validator
 from pydantic.functional_validators import PlainValidator
 
-from trilogy.constants import CONFIG
+from trilogy.constants import CONFIG, DEFAULT_NAMESPACE
 from trilogy.core.enums import (
     ConceptSource,
     FunctionClass,
@@ -281,7 +281,11 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
             # if the concept is a locally derived concept, it cannot ever be partial
             # but if it's a concept pulled in from upstream and we have a where clause, it should be partial
             ColumnAssignment(
-                alias=c.address.replace(".", "_"),
+                alias=(
+                    c.address.replace(".", "_")
+                    if c.namespace != DEFAULT_NAMESPACE
+                    else c.name
+                ),
                 concept=environment.concepts[c.address].reference,
                 modifiers=modifiers if c.address not in self.locally_derived else [],
             )

trilogy/core/statements/execute.py

@@ -3,6 +3,7 @@ from typing import Annotated, List, Optional, Union
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import PlainValidator
 
+from trilogy.core.models.author import ConceptRef
 from trilogy.core.models.build import (
     BuildConcept,
     BuildDatasource,
@@ -14,7 +15,7 @@ from trilogy.core.statements.common import CopyQueryMixin, PersistQueryMixin
 
 
 class ProcessedQuery(BaseModel):
-    output_columns: List[BuildConcept]
+    output_columns: List[ConceptRef]
     ctes: List[CTE | UnionCTE]
     base: CTE | UnionCTE
     hidden_columns: set[str] = Field(default_factory=set)
@@ -38,5 +39,5 @@ class ProcessedRawSQLStatement(BaseModel):
 
 
 class ProcessedShowStatement(BaseModel):
-    output_columns: List[BuildConcept]
+    output_columns: List[ConceptRef]
     output_values: List[Union[BuildConcept, BuildDatasource, ProcessedQuery]]

trilogy/dialect/base.py

@@ -33,7 +33,6 @@ from trilogy.core.models.build import (
     BuildRowsetItem,
     BuildSubselectComparison,
     BuildWindowItem,
-    Factory,
 )
 from trilogy.core.models.core import (
     DataType,
@@ -904,7 +903,6 @@ class BaseDialect:
             | ProcessedRawSQLStatement
             | ProcessedCopyStatement
         ] = []
-        factory = Factory(environment=environment)
         for statement in statements:
             if isinstance(statement, PersistStatement):
                 if hooks:
@@ -939,11 +937,9 @@ class BaseDialect:
                     output.append(
                         ProcessedShowStatement(
                             output_columns=[
-                                factory.build(
-                                    environment.concepts[
-                                        DEFAULT_CONCEPTS["query_text"].address
-                                    ]
-                                )
+                                environment.concepts[
+                                    DEFAULT_CONCEPTS["query_text"].address
+                                ].reference
                             ],
                             output_values=[
                                 process_query(
@@ -984,29 +980,6 @@ class BaseDialect:
             return ";\n".join([str(x) for x in query.output_values])
         elif isinstance(query, ProcessedRawSQLStatement):
             return query.text
-        select_columns: Dict[str, str] = {}
-        cte_output_map = {}
-        selected = set()
-        output_addresses = [
-            c.address
-            for c in query.output_columns
-            if c.address not in query.hidden_columns
-        ]
-
-        for c in query.base.output_columns:
-            if c.address not in selected:
-                select_columns[c.address] = (
-                    f"{query.base.name}.{safe_quote(c.safe_address, self.QUOTE_CHARACTER)}"
-                )
-                cte_output_map[c.address] = query.base
-                if c.address not in query.hidden_columns:
-                    selected.add(c.address)
-        if not all([x in selected for x in output_addresses]):
-            missing = [x for x in output_addresses if x not in selected]
-            raise ValueError(
-                f"Did not get all output addresses in select - missing: {missing}, have"
-                f" {selected}"
-            )
 
         recursive = any(isinstance(x, RecursiveCTE) for x in query.ctes)
 

trilogy/dialect/snowflake.py

@@ -47,7 +47,7 @@ SNOWFLAKE_SQL_TEMPLATE = Template(
 CREATE OR REPLACE TABLE {{ output.address.location }} AS
 {% endif %}{%- if ctes %}
 WITH {% if recursive%}RECURSIVE{% endif %}{% for cte in ctes %}
-{{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% else %}
+"{{cte.name}}" as ({{cte.statement}}){% if not loop.last %},{% endif %}{% else %}
 {% endfor %}{% endif %}
 {%- if full_select -%}
 {{full_select}}

trilogy/executor.py

@@ -8,8 +8,8 @@ from sqlalchemy.engine import CursorResult
 
 from trilogy.constants import Rendering, logger
 from trilogy.core.enums import FunctionType, Granularity, IOType
-from trilogy.core.models.author import Concept, Function
-from trilogy.core.models.build import BuildConcept, BuildFunction
+from trilogy.core.models.author import Concept, ConceptRef, Function
+from trilogy.core.models.build import BuildFunction
 from trilogy.core.models.core import ListWrapper, MapWrapper
 from trilogy.core.models.datasource import Datasource
 from trilogy.core.models.environment import Environment
@@ -61,7 +61,7 @@ class MockResult:
 
 
 def generate_result_set(
-    columns: List[BuildConcept], output_data: list[Any]
+    columns: List[ConceptRef], output_data: list[Any]
 ) -> MockResult:
     names = [x.address.replace(".", "_") for x in columns]
     return MockResult(
@@ -90,7 +90,16 @@ class Executor(object):
         if self.dialect == Dialects.DATAFRAME:
             self.engine.setup(self.environment, self.connection)
 
-    def execute_statement(self, statement) -> Optional[CursorResult]:
+    def execute_statement(
+        self,
+        statement: (
+            ProcessedQuery
+            | ProcessedCopyStatement
+            | ProcessedRawSQLStatement
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+        ),
+    ) -> Optional[CursorResult]:
         if not isinstance(
             statement,
             (

trilogy/parsing/common.py

@@ -397,7 +397,7 @@ def group_function_to_concept(
             modifiers=modifiers,
             grain=grain,
             metadata=fmetadata,
-            derivation=Derivation.BASIC,
+            derivation=Derivation.GROUP_TO,
             granularity=granularity,
         )
         return r
@@ -654,7 +654,6 @@ def agg_wrapper_to_concept(
     fmetadata = metadata or Metadata()
     aggfunction = parent.function
     modifiers = get_upstream_modifiers(parent.concept_arguments, environment)
-    # derivation = Concept.calculate_derivation(parent, Purpose.PROPERTY)
     grain = Grain.from_concepts(parent.by, environment) if parent.by else Grain()
     granularity = Concept.calculate_granularity(Derivation.AGGREGATE, grain, parent)
 
@@ -778,7 +777,6 @@ def rowset_to_concepts(rowset: RowsetDerivationStatement, environment: Environme
     for x in pre_output:
         x.lineage = RowsetItem(
             content=orig_map[x.address].reference,
-            # where=rowset.select.where_clause,
             rowset=RowsetLineage(
                 name=rowset.name,
                 derived_concepts=[x.reference for x in pre_output],