pytrilogy 0.0.3.63__tar.gz → 0.0.3.65__tar.gz

{pytrilogy-0.0.3.63/pytrilogy.egg-info → pytrilogy-0.0.3.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.63
+Version: 0.0.3.65
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.63
+Version: 0.0.3.65
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/tests/test_select.py

@@ -122,6 +122,7 @@ def test_double_aggregate():
 
 
 def test_modifiers():
+
     q1 = """
     const a <- 1;
     const b <- 2;

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.63"
+__version__ = "0.0.3.65"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/graph_models.py

@@ -1,6 +1,50 @@
 import networkx as nx
 
-from trilogy.core.models.build import BuildConcept, BuildDatasource
+from trilogy.core.models.build import BuildConcept, BuildDatasource, BuildWhereClause
+
+
+def get_graph_exact_match(
+    g: nx.DiGraph, conditions: BuildWhereClause | None
+) -> set[str]:
+    datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
+        nx.get_node_attributes(g, "datasource")
+    )
+    exact: set[str] = set()
+    for node in g.nodes:
+        if node in datasources:
+            ds = datasources[node]
+            if isinstance(ds, list):
+                exact.add(node)
+                continue
+
+            if not conditions and not ds.non_partial_for:
+                exact.add(node)
+                continue
+            elif conditions:
+                if not ds.non_partial_for:
+                    continue
+                if ds.non_partial_for and conditions == ds.non_partial_for:
+                    exact.add(node)
+                    continue
+            else:
+                continue
+
+    return exact
+
+
+def prune_sources_for_conditions(
+    g: nx.DiGraph,
+    conditions: BuildWhereClause | None,
+):
+
+    complete = get_graph_exact_match(g, conditions)
+    to_remove = []
+    for node in g.nodes:
+        if node.startswith("ds~") and node not in complete:
+            to_remove.append(node)
+
+    for node in to_remove:
+        g.remove_node(node)
 
 
 def concept_to_node(input: BuildConcept) -> str:

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/models/build.py

@@ -1586,7 +1586,10 @@ class Factory:
 
                 return BuildFunction.model_construct(
                     operator=base.operator,
-                    arguments=[rval, *[self.build(c) for c in raw_args[1:]]],
+                    arguments=[
+                        rval,
+                        *[self.handle_constant(self.build(c)) for c in raw_args[1:]],
+                    ],
                     output_datatype=base.output_datatype,
                     output_purpose=base.output_purpose,
                     valid_inputs=base.valid_inputs,
@@ -2042,4 +2045,6 @@ class Factory:
             and base.lineage.operator == FunctionType.CONSTANT
         ):
             return BuildParamaterizedConceptReference(concept=base)
+        elif isinstance(base, ConceptRef):
+            return self.handle_constant(self.build(base))
         return base

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/models/environment.py

@@ -603,7 +603,7 @@ class Environment(BaseModel):
             # too hacky for maintainability
             if current_derivation not in (Derivation.ROOT, Derivation.CONSTANT):
                 logger.info(
-                    f"A datasource has been added which will persist derived concept {new_persisted_concept.address}"
+                    f"A datasource has been added which will persist derived concept {new_persisted_concept.address} with derivation {current_derivation}"
                 )
                 persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + new_persisted_concept.name
                 # override the current concept source to reflect that it's now coming from a datasource
@@ -622,17 +622,21 @@ class Environment(BaseModel):
                         meta=meta,
                         force=True,
                     )
+                    base = {
+                        "lineage": None,
+                        "metadata": new_persisted_concept.metadata.model_copy(
+                            update={"concept_source": ConceptSource.PERSIST_STATEMENT}
+                        ),
+                        "derivation": Derivation.ROOT,
+                        "purpose": new_persisted_concept.purpose,
+                    }
+                    # purpose is used in derivation calculation
+                    # which should be fixed, but we'll do in a followup
+                    # so override here
+                    if new_persisted_concept.purpose == Purpose.CONSTANT:
+                        base["purpose"] = Purpose.KEY
                     new_persisted_concept = new_persisted_concept.model_copy(
-                        deep=True,
-                        update={
-                            "lineage": None,
-                            "metadata": new_persisted_concept.metadata.model_copy(
-                                update={
-                                    "concept_source": ConceptSource.PERSIST_STATEMENT
-                                }
-                            ),
-                            "derivation": Derivation.ROOT,
-                        },
+                        deep=True, update=base
                     )
                     self.add_concept(
                         new_persisted_concept,

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/models/execute.py

@@ -56,6 +56,12 @@ LOGGER_PREFIX = "[MODELS_EXECUTE]"
 DATASOURCE_TYPES = (BuildDatasource, BuildDatasource)
 
 
+class InlinedCTE(BaseModel):
+    original_alias: str
+    new_alias: str
+    new_base: str
+
+
 class CTE(BaseModel):
     name: str
     source: "QueryDatasource"
@@ -78,6 +84,7 @@ class CTE(BaseModel):
     limit: Optional[int] = None
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
+    inlined_ctes: dict[str, InlinedCTE] = Field(default_factory=dict)
 
     @field_validator("join_derived_concepts")
     def validate_join_derived_concepts(cls, v):
@@ -104,62 +111,6 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
-    def inline_constant(self, concept: BuildConcept):
-        if not concept.derivation == Derivation.CONSTANT:
-            return False
-        if not isinstance(concept.lineage, BuildFunction):
-            return False
-        if not concept.lineage.operator == FunctionType.CONSTANT:
-            return False
-        # remove the constant
-        removed: set = set()
-        if concept.address in self.source_map:
-            removed = removed.union(self.source_map[concept.address])
-            del self.source_map[concept.address]
-
-        if self.condition:
-            self.condition = self.condition.inline_constant(concept)
-        # if we've entirely removed the need to join to someplace to get the concept
-        # drop the join as well.
-        for removed_cte in removed:
-            still_required = any(
-                [
-                    removed_cte in x
-                    for x in self.source_map.values()
-                    or self.existence_source_map.values()
-                ]
-            )
-            if not still_required:
-                self.joins = [
-                    join
-                    for join in self.joins
-                    if not isinstance(join, Join)
-                    or (
-                        isinstance(join, Join)
-                        and (
-                            join.right_cte.name != removed_cte
-                            and any(
-                                [
-                                    x.cte.name != removed_cte
-                                    for x in (join.joinkey_pairs or [])
-                                ]
-                            )
-                        )
-                    )
-                ]
-                for join in self.joins:
-                    if isinstance(join, UnnestJoin) and concept in join.concepts:
-                        join.rendering_required = False
-
-                self.parent_ctes = [
-                    x for x in self.parent_ctes if x.name != removed_cte
-                ]
-                if removed_cte == self.base_name_override:
-                    candidates = [x.name for x in self.parent_ctes]
-                    self.base_name_override = candidates[0] if candidates else None
-                    self.base_alias_override = candidates[0] if candidates else None
-        return True
-
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}. Group: {self.group_to_grain}"
@@ -240,7 +191,18 @@ class CTE(BaseModel):
                 ]
             elif v == parent.safe_identifier:
                 self.source_map[k] = [ds_being_inlined.safe_identifier]
-
+        for k, v in self.existence_source_map.items():
+            if isinstance(v, list):
+                self.existence_source_map[k] = [
+                    (
+                        ds_being_inlined.safe_identifier
+                        if x == parent.safe_identifier
+                        else x
+                    )
+                    for x in v
+                ]
+            elif v == parent.safe_identifier:
+                self.existence_source_map[k] = [ds_being_inlined.safe_identifier]
         # zip in any required values for lookups
         for k in ds_being_inlined.output_lcl.addresses:
             if k in self.source_map and self.source_map[k]:
@@ -251,6 +213,11 @@ class CTE(BaseModel):
         ]
         if force_group:
             self.group_to_grain = True
+        self.inlined_ctes[ds_being_inlined.safe_identifier] = InlinedCTE(
+            original_alias=parent.name,
+            new_alias=ds_being_inlined.safe_identifier,
+            new_base=ds_being_inlined.safe_location,
+        )
         return True
 
     def __add__(self, other: "CTE" | "UnionCTE"):
@@ -303,6 +270,10 @@ class CTE(BaseModel):
             **self.existence_source_map,
             **other.existence_source_map,
         }
+        self.inlined_ctes = {
+            **self.inlined_ctes,
+            **other.inlined_ctes,
+        }
 
         return self
 
@@ -672,7 +643,7 @@ class QueryDatasource(BaseModel):
                     and CONFIG.validate_missing
                 ):
                     raise SyntaxError(
-                        f"On query datasource from {values} missing source map entry (map: {v}) for {concept.address} on {key} with pseudonyms {concept.pseudonyms}, have {v}"
+                        f"Missing source map entry for {concept.address} on {key} with pseudonyms {concept.pseudonyms}, have map: {v}"
                     )
         return v
 
@@ -1057,6 +1028,7 @@ class UnionCTE(BaseModel):
     hidden_concepts: set[str] = Field(default_factory=set)
     partial_concepts: list[BuildConcept] = Field(default_factory=list)
     existence_source_map: Dict[str, list[str]] = Field(default_factory=dict)
+    inlined_ctes: Dict[str, InlinedCTE] = Field(default_factory=dict)
 
     @computed_field  # type: ignore
     @property

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/concept_strategies_v3.py

@@ -54,11 +54,7 @@ def generate_candidates_restrictive(
     exhausted: set[str],
     depth: int,
     conditions: BuildWhereClause | None = None,
-) -> List[BuildConcept]:
-    # if it's single row, joins are irrelevant. Fetch without keys.
-    if priority_concept.granularity == Granularity.SINGLE_ROW:
-        return []
-
+) -> tuple[list[BuildConcept], BuildWhereClause | None]:
     local_candidates = [
         x
         for x in list(candidates)
@@ -71,8 +67,16 @@ def generate_candidates_restrictive(
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Injecting additional conditional row arguments as all remaining concepts are roots or constant"
         )
-        return unique(list(conditions.row_arguments) + local_candidates, "address")
-    return local_candidates
+        # otherwise, we can ignore the conditions now that we've injected inputs
+        return (
+            unique(list(conditions.row_arguments) + local_candidates, "address"),
+            None,
+        )
+    # if it's single row, joins are irrelevant. Fetch without keys.
+    if priority_concept.granularity == Granularity.SINGLE_ROW:
+        return [], conditions
+
+    return local_candidates, conditions
 
 
 def append_existence_check(
@@ -104,9 +108,7 @@ def append_existence_check(
             )
             assert parent, "Could not resolve existence clause"
             node.add_parents([parent])
-            logger.info(
-                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
-            )
+            logger.info(f"{LOGGER_PREFIX} found {[str(c) for c in subselect]}")
             node.add_existence_concepts([*subselect])
 
 
@@ -440,7 +442,19 @@ def _search_concepts(
     accept_partial: bool = False,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
+    # check for direct materialization first
+    candidate = history.gen_select_node(
+        mandatory_list,
+        environment,
+        g,
+        depth + 1,
+        fail_if_not_found=False,
+        accept_partial=accept_partial,
+        conditions=conditions,
+    )
 
+    if candidate:
+        return candidate
     context = initialize_loop_context(
         mandatory_list=mandatory_list,
         environment=environment,
@@ -460,19 +474,21 @@ def _search_concepts(
         )
 
         local_conditions = evaluate_loop_conditions(context, priority_concept)
-        logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)} derivation {priority_concept.derivation} granularity {priority_concept.granularity} with conditions {local_conditions}"
-        )
 
         candidates = [
             c for c in context.mandatory_list if c.address != priority_concept.address
         ]
-        candidate_list = generate_candidates_restrictive(
+        # the local conditions list may be override if we end up injecting conditions
+        candidate_list, local_conditions = generate_candidates_restrictive(
             priority_concept,
             candidates,
             context.skip,
             depth=depth,
-            conditions=context.conditions,
+            conditions=local_conditions,
+        )
+
+        logger.info(
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)} derivation {priority_concept.derivation} granularity {priority_concept.granularity} with conditions {local_conditions}"
         )
 
         logger.info(

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/discovery_node_factory.py

@@ -438,15 +438,14 @@ def generate_node(
     )
 
     # Try materialized concept first
+    # this is worth checking every loop iteration
     candidate = history.gen_select_node(
-        concept,
-        local_optional,
+        [concept] + local_optional,
         environment,
         g,
         depth + 1,
         fail_if_not_found=False,
         accept_partial=accept_partial,
-        accept_partial_optional=False,
         conditions=conditions,
     )
 

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -6,7 +6,7 @@ from networkx.algorithms import approximation as ax
 from trilogy.constants import logger
 from trilogy.core.enums import Derivation
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
-from trilogy.core.graph_models import concept_to_node
+from trilogy.core.graph_models import concept_to_node, prune_sources_for_conditions
 from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
@@ -222,10 +222,12 @@ def resolve_weak_components(
     environment_graph: nx.DiGraph,
     filter_downstream: bool = True,
     accept_partial: bool = False,
+    search_conditions: BuildWhereClause | None = None,
 ) -> list[list[BuildConcept]] | None:
     break_flag = False
     found = []
     search_graph = environment_graph.copy()
+    prune_sources_for_conditions(search_graph, conditions=search_conditions)
     reduced_concept_sets: list[set[str]] = []
 
     # loop through, removing new nodes we find
@@ -239,7 +241,6 @@ def resolve_weak_components(
             if "__preql_internal" not in c.address
         ]
     )
-    logger.debug(f"Resolving weak components for {node_list} in {search_graph.nodes}")
     synonyms: set[str] = set()
     for x in all_concepts:
         synonyms = synonyms.union(x.pseudonyms)
@@ -407,6 +408,7 @@ def gen_merge_node(
             g,
             filter_downstream=filter_downstream,
             accept_partial=accept_partial,
+            search_conditions=search_conditions,
         )
         if not weak_resolve:
             logger.info(

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -169,7 +169,9 @@ def is_fully_covered(
     return current_end >= end
 
 
-def get_union_sources(datasources: list[BuildDatasource], concepts: list[BuildConcept]):
+def get_union_sources(
+    datasources: list[BuildDatasource], concepts: list[BuildConcept]
+) -> List[list[BuildDatasource]]:
     candidates: list[BuildDatasource] = []
     for x in datasources:
         if all([c.address in x.output_concepts for c in concepts]):

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/node_generators/select_merge_node.py

@@ -5,7 +5,11 @@ import networkx as nx
 
 from trilogy.constants import logger
 from trilogy.core.enums import Derivation
-from trilogy.core.graph_models import concept_to_node
+from trilogy.core.graph_models import (
+    concept_to_node,
+    get_graph_exact_match,
+    prune_sources_for_conditions,
+)
 from trilogy.core.models.build import (
     BuildConcept,
     BuildDatasource,
@@ -57,26 +61,6 @@ def get_graph_partial_nodes(
     return partial
 
 
-def get_graph_exact_match(
-    g: nx.DiGraph, conditions: BuildWhereClause | None
-) -> set[str]:
-    datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
-        nx.get_node_attributes(g, "datasource")
-    )
-    exact: set[str] = set()
-    for node in g.nodes:
-        if node in datasources:
-            ds = datasources[node]
-            if not isinstance(ds, list):
-                if ds.non_partial_for and conditions == ds.non_partial_for:
-                    exact.add(node)
-                    continue
-            else:
-                continue
-
-    return exact
-
-
 def get_graph_grains(g: nx.DiGraph) -> dict[str, list[str]]:
     datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
         nx.get_node_attributes(g, "datasource")
@@ -95,6 +79,34 @@ def get_graph_grains(g: nx.DiGraph) -> dict[str, list[str]]:
     return grain_length
 
 
+def subgraph_is_complete(
+    nodes: list[str], targets: set[str], mapping: dict[str, str], g: nx.DiGraph
+) -> bool:
+    mapped = set([mapping.get(n, n) for n in nodes])
+    passed = all([t in mapped for t in targets])
+    if not passed:
+        logger.info(
+            f"Subgraph {nodes} is not complete, missing targets {targets} - mapped {mapped}"
+        )
+        return False
+    # check if all concepts have a datasource edge
+    has_ds_edge = {
+        mapping.get(n, n): any(x.startswith("ds~") for x in nx.neighbors(g, n))
+        for n in nodes
+        if n.startswith("c~")
+    }
+    has_ds_edge = {k: False for k in targets}
+    # check at least one instance of concept has a datasource edge
+    for n in nodes:
+        if n.startswith("c~"):
+            neighbors = nx.neighbors(g, n)
+            for neighbor in neighbors:
+                if neighbor.startswith("ds~"):
+                    has_ds_edge[mapping.get(n, n)] = True
+                    break
+    return all(has_ds_edge.values()) and passed
+
+
 def create_pruned_concept_graph(
     g: nx.DiGraph,
     all_concepts: List[BuildConcept],
@@ -104,6 +116,7 @@ def create_pruned_concept_graph(
     depth: int = 0,
 ) -> nx.DiGraph:
     orig_g = g
+
     g = g.copy()
     union_options = get_union_sources(datasources, all_concepts)
     for ds_list in union_options:
@@ -114,7 +127,8 @@ def create_pruned_concept_graph(
         g.add_node(node_address, datasource=ds_list)
         for c in common:
             g.add_edge(node_address, concept_to_node(c))
-
+            g.add_edge(concept_to_node(c), node_address)
+    prune_sources_for_conditions(g, conditions)
     target_addresses = set([c.address for c in all_concepts])
     concepts: dict[str, BuildConcept] = nx.get_node_attributes(orig_g, "concept")
     datasource_map: dict[str, BuildDatasource | list[BuildDatasource]] = (
@@ -126,8 +140,7 @@ def create_pruned_concept_graph(
         # filter out synonyms
         if (x := concepts.get(n, None)) and x.address in target_addresses
     }
-    # from trilogy.hooks.graph_hook import GraphHook
-    # GraphHook().query_graph_built(g)
+
     relevant_concepts: list[str] = list(relevant_concepts_pre.keys())
     relevent_datasets: list[str] = []
     if not accept_partial:
@@ -149,6 +162,7 @@ def create_pruned_concept_graph(
                 to_remove.append(edge)
         for edge in to_remove:
             g.remove_edge(*edge)
+
     for n in g.nodes():
         if not n.startswith("ds~"):
             continue
@@ -181,8 +195,15 @@ def create_pruned_concept_graph(
             if n not in relevent_datasets and n not in relevant_concepts
         ]
     )
-
+    # from trilogy.hooks.graph_hook import GraphHook
+    # GraphHook().query_graph_built(g)
     subgraphs = list(nx.connected_components(g.to_undirected()))
+    subgraphs = [
+        s
+        for s in subgraphs
+        if subgraph_is_complete(s, target_addresses, relevant_concepts_pre, g)
+    ]
+
     if not subgraphs:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} cannot resolve root graph - no subgraphs after node prune"
@@ -486,6 +507,24 @@ def gen_select_merge_node(
     non_constant = [c for c in all_concepts if c.derivation != Derivation.CONSTANT]
     constants = [c for c in all_concepts if c.derivation == Derivation.CONSTANT]
     if not non_constant and constants:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} only constant inputs to discovery ({constants}), returning constant node directly"
+        )
+        for x in constants:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} {x} {x.lineage} {x.derivation}"
+            )
+        if conditions:
+            if not all(
+                [x.derivation == Derivation.CONSTANT for x in conditions.row_arguments]
+            ):
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} conditions being passed in to constant node {conditions}, but not all concepts are constants."
+                )
+                return None
+            else:
+                constants += conditions.row_arguments
+
         return ConstantNode(
             output_concepts=constants,
             input_concepts=[],
@@ -494,7 +533,7 @@ def gen_select_merge_node(
             depth=depth,
             partial_concepts=[],
             force_group=False,
-            preexisting_conditions=conditions.conditional if conditions else None,
+            conditions=conditions.conditional if conditions else None,
         )
     for attempt in [False, True]:
         pruned_concept_graph = create_pruned_concept_graph(

{pytrilogy-0.0.3.63 → pytrilogy-0.0.3.65}/trilogy/core/processing/node_generators/synonym_node.py

@@ -50,7 +50,9 @@ def gen_synonym_node(
 
     logger.info(f"{local_prefix} Generating Synonym Node with {len(synonyms)} synonyms")
     sorted_keys = sorted(synonyms.keys())
-    combinations_list = list(itertools.product(*(synonyms[obj] for obj in sorted_keys)))
+    combinations_list: list[tuple[BuildConcept, ...]] = list(
+        itertools.product(*(synonyms[obj] for obj in sorted_keys))
+    )
 
     def similarity_sort_key(combo):
         addresses = [x.address for x in combo]
@@ -83,7 +85,7 @@ def gen_synonym_node(
             f"{local_prefix} checking combination {fingerprint} with {len(combo)} concepts"
         )
         attempt: StrategyNode | None = source_concepts(
-            combo,
+            list(combo),
             history=history,
             environment=environment,
             depth=depth,