pytrilogy 0.0.1.102__tar.gz → 0.0.1.103__tar.gz

{pytrilogy-0.0.1.102/pytrilogy.egg-info → pytrilogy-0.0.1.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.102
+Version: 0.0.1.103
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -27,7 +27,7 @@ Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/README.md

@@ -1,4 +1,4 @@
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.102
+Version: 0.0.1.103
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -27,7 +27,7 @@ Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/tests/test_models.py

@@ -1,4 +1,4 @@
-from trilogy.core.enums import BooleanOperator, Purpose, JoinType
+from trilogy.core.enums import BooleanOperator, Purpose, JoinType, ComparisonOperator
 from trilogy.core.models import (
     CTE,
     Grain,
@@ -9,6 +9,7 @@ from trilogy.core.models import (
     Address,
     UndefinedConcept,
     BaseJoin,
+    Comparison,
 )
 
 
@@ -73,11 +74,17 @@ def test_conditional(test_environment, test_environment_graph):
     condition_b = Conditional(
         left=test_concept, right=test_concept, operator=BooleanOperator.AND
     )
-
     merged = condition_a + condition_b
-    assert merged.left == condition_a
-    assert merged.right == condition_b
-    assert merged.operator == BooleanOperator.AND
+    assert merged == condition_a
+
+    test_concept_two = list(test_environment.concepts.values())[-2]
+    condition_c = Conditional(
+        left=test_concept, right=test_concept_two, operator=BooleanOperator.AND
+    )
+    merged_two = condition_a + condition_c
+    assert merged_two.left == condition_a
+    assert merged_two.right == condition_c
+    assert merged_two.operator == BooleanOperator.AND
 
 
 def test_grain(test_environment):
@@ -177,3 +184,10 @@ def test_base_join(test_environment: Environment):
         exc3 = exc4
         pass
     assert isinstance(exc3, SyntaxError)
+
+
+def test_comparison():
+    try:
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    except Exception as exc:
+        assert isinstance(exc, SyntaxError)

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/__init__.py

@@ -3,6 +3,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.1.102"
+__version__ = "0.0.1.103"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment"]

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/core/env_processor.py

@@ -1,4 +1,8 @@
-from trilogy.core.graph_models import ReferenceGraph, concept_to_node, datasource_to_node
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
 from trilogy.core.models import Environment
 from trilogy.core.enums import PurposeLineage
 

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/core/models.py

@@ -338,7 +338,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
 
     def __eq__(self, other: object):
         if isinstance(other, str):
-            if self.address == str:
+            if self.address == other:
                 return True
         if not isinstance(other, Concept):
             return False
@@ -1887,7 +1887,9 @@ class QueryDatasource(BaseModel):
                 else None
             ),
             source_type=self.source_type,
-            partial_concepts=self.partial_concepts + other.partial_concepts,
+            partial_concepts=unique(
+                self.partial_concepts + other.partial_concepts, "address"
+            ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
         )
@@ -2530,7 +2532,7 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
 
     def __post_init__(self):
         if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise ValueError(
+            raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
 
@@ -2704,6 +2706,8 @@ class Conditional(Namespaced, SelectGrain, BaseModel):
     def __add__(self, other) -> "Conditional":
         if other is None:
             return self
+        elif str(other) == str(self):
+            return self
         elif isinstance(other, (Comparison, Conditional, Parenthetical)):
             return Conditional(left=self, right=other, operator=BooleanOperator.AND)
         raise ValueError(f"Cannot add {self.__class__} and {type(other)}")

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/core/processing/concept_strategies_v3.py

@@ -318,7 +318,7 @@ def generate_node(
         )
     elif concept.derivation == PurposeLineage.ROOT:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
         )
         return gen_select_node(
             concept,

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/core/processing/node_generators/rowset_node.py

@@ -35,26 +35,30 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    if where := select.where_clause:
+        targets = select.output_components + where.conditional.concept_arguments
+    else:
+        targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=select.output_components,
+        mandatory_list=targets,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
-    node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-    node.rebuild_cache()
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
         )
         return None
+    node.conditions = select.where_clause.conditional if select.where_clause else None
+    # rebuild any cached info with the new condition clause
+    node.rebuild_cache()
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [
         x
         for x in rowset.derived_concepts
-        if x.address == concept.address or x.address in enrichment
+        # if x.address == concept.address or x.address in enrichment
     ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
@@ -68,7 +72,7 @@ def gen_rowset_node(
         for item in additional_relevant:
             node.partial_concepts.append(item)
 
-    # assume grain to be outoput of select
+    # assume grain to be output of select
     # but don't include anything aggregate at this point
     assert node.resolution_cache
     node.resolution_cache.grain = concept_list_to_grain(

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/core/processing/node_generators/select_node.py

@@ -50,26 +50,21 @@ def gen_select_node_from_table(
     candidates: dict[str, StrategyNode] = {}
     scores: dict[str, int] = {}
     # otherwise, we need to look for a table
+    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_concepts]
     for datasource in environment.datasources.values():
         all_found = True
-        for raw_concept in all_concepts:
-            # look for connection to abstract grain
-            req_concept = raw_concept.with_default_grain()
-            # if we don't have a concept in the graph
-            # exit early
-            if concept_to_node(req_concept) not in g.nodes:
-                raise ValueError(concept_to_node(req_concept))
+        for idx, req_concept in enumerate(nodes_to_find):
             try:
                 path = nx.shortest_path(
                     g,
                     source=datasource_to_node(datasource),
-                    target=concept_to_node(req_concept),
+                    target=req_concept,
                 )
             except nx.NodeNotFound as e:
                 # just to provide better error
                 ncandidates = [
                     datasource_to_node(datasource),
-                    concept_to_node(req_concept),
+                    req_concept,
                 ]
                 for ncandidate in ncandidates:
                     try:
@@ -94,75 +89,77 @@ def gen_select_node_from_table(
             for node in path:
                 if g.nodes[node]["type"] == "datasource":
                     continue
-                if g.nodes[node]["concept"].address == raw_concept.address:
+                if g.nodes[node]["concept"].address == all_concepts[idx].address:
                     continue
                 all_found = False
                 break
 
-        if all_found:
-            partial_concepts = [
-                c.concept
-                for c in datasource.columns
-                if not c.is_complete and c.concept in all_lcl
-            ]
-            partial_lcl = LooseConceptList(concepts=partial_concepts)
-            if not accept_partial and target_concept in partial_lcl:
-                continue
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} target grain is {str(target_grain)}"
-            )
-            if target_grain and target_grain.issubset(datasource.grain):
+        if not all_found:
+            # skip to next node
+            continue
+        partial_concepts = [
+            c.concept
+            for c in datasource.columns
+            if not c.is_complete and c.concept in all_lcl
+        ]
+        partial_lcl = LooseConceptList(concepts=partial_concepts)
+        if not accept_partial and target_concept in partial_lcl:
+            continue
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} target grain is {str(target_grain)}"
+        )
+        if target_grain and target_grain.issubset(datasource.grain):
 
-                if all([x in all_lcl for x in target_grain.components]):
-                    force_group = False
-                # if we are not returning the grain
-                # we have to group
-                else:
-                    logger.info(
-                        f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(all_lcl)}, group to actual grain"
-                    )
-                    force_group = True
-            elif all([x in all_lcl for x in datasource.grain.components]):
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
-                )
+            if all([x in all_lcl for x in target_grain.components]):
                 force_group = False
+            # if we are not returning the grain
+            # we have to group
             else:
                 logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+                    f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(all_lcl)}, group to actual grain"
                 )
                 force_group = True
+        elif all([x in all_lcl for x in datasource.grain.components]):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
+            )
+            force_group = False
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+            )
+            force_group = True
 
-            bcandidate: StrategyNode = SelectNode(
-                input_concepts=[c.concept for c in datasource.columns],
+        bcandidate: StrategyNode = SelectNode(
+            input_concepts=[c.concept for c in datasource.columns],
+            output_concepts=all_concepts,
+            environment=environment,
+            g=g,
+            parents=[],
+            depth=depth,
+            partial_concepts=[c for c in all_concepts if c in partial_lcl],
+            accept_partial=accept_partial,
+            datasource=datasource,
+            grain=Grain(components=all_concepts),
+        )
+        # we need to nest the group node one further
+        if force_group is True:
+            candidate: StrategyNode = GroupNode(
                 output_concepts=all_concepts,
+                input_concepts=all_concepts,
                 environment=environment,
                 g=g,
-                parents=[],
+                parents=[bcandidate],
                 depth=depth,
-                partial_concepts=[c for c in all_concepts if c in partial_lcl],
-                accept_partial=accept_partial,
-                datasource=datasource,
-                grain=Grain(components=all_concepts),
+                partial_concepts=bcandidate.partial_concepts,
             )
-            # we need to ntest the group node one further
-            if force_group is True:
-                candidate: StrategyNode = GroupNode(
-                    output_concepts=all_concepts,
-                    input_concepts=all_concepts,
-                    environment=environment,
-                    g=g,
-                    parents=[bcandidate],
-                    depth=depth,
-                    partial_concepts=bcandidate.partial_concepts,
-                )
-            else:
-                candidate = bcandidate
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, returning {candidate.output_lcl}"
-            )
-            candidates[datasource.identifier] = candidate
-            scores[datasource.identifier] = -len(partial_concepts)
+        else:
+            candidate = bcandidate
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, returning {candidate.output_lcl}"
+        )
+        candidates[datasource.identifier] = candidate
+        scores[datasource.identifier] = -len(partial_concepts)
     if not candidates:
         return None
     final = max(candidates, key=lambda x: scores[x])
@@ -227,6 +224,21 @@ def gen_select_node(
         f"{padding(depth)}{LOGGER_PREFIX} looking for multiple sources that can satisfy"
     )
     all_found = False
+    unreachable: list[str] = []
+    # first pass
+    for opt_con in local_optional:
+        ds = gen_select_node_from_table(
+            concept,
+            [concept, opt_con],
+            g=g,
+            environment=environment,
+            depth=depth + 1,
+            accept_partial=accept_partial,
+            target_grain=Grain(components=all_concepts),
+        )
+        if not ds:
+            unreachable.append(opt_con.address)
+    # actual search
     for x in reversed(range(1, len(local_optional) + 1)):
         if all_found:
             break
@@ -234,7 +246,9 @@ def gen_select_node(
             if all_found:
                 break
             # filter to just the original ones we need to get
-            local_combo = [x for x in combo if x not in found]
+            local_combo = [
+                x for x in combo if x not in found and x.address not in unreachable
+            ]
             # skip if nothing new in this combo
             if not local_combo:
                 continue

{pytrilogy-0.0.1.102 → pytrilogy-0.0.1.103}/trilogy/scripts/trilogy.py

@@ -115,7 +115,7 @@ def run(ctx, input, dialect: str, conn_args):
 
             print_tabulate(results, tabulate.tabulate)
         except ImportError:
-            print('Install tabulate (pip install tabulate) for a prettier output')
+            print("Install tabulate (pip install tabulate) for a prettier output")
             print(", ".join(results.keys()))
             for row in results:
                 print(row)