pytrilogy 0.0.3.69__tar.gz → 0.0.3.71__tar.gz

{pytrilogy-0.0.3.69/pytrilogy.egg-info → pytrilogy-0.0.3.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.69
+Version: 0.0.3.71
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.69
+Version: 0.0.3.71
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/pytrilogy.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ tests/test_derived_concepts.py
 tests/test_discovery_nodes.py
 tests/test_enums.py
 tests/test_environment.py
+tests/test_execute_models.py
 tests/test_executor.py
 tests/test_failure.py
 tests/test_functions.py

pytrilogy-0.0.3.71/tests/test_execute_models.py

@@ -0,0 +1,61 @@
+from pytest import raises
+
+from trilogy.core.exceptions import InvalidSyntaxException
+from trilogy.core.models.execute import (
+    BuildConcept,
+    BuildDatasource,
+    BuildGrain,
+    DataType,
+    Purpose,
+    raise_helpful_join_validation_error,
+)
+
+
+def test_raise_helpful_join_validation_error():
+
+    with raises(InvalidSyntaxException):
+        raise_helpful_join_validation_error(
+            concepts=[
+                BuildConcept(
+                    name="test_concept",
+                    datatype=DataType.INTEGER,
+                    purpose=Purpose.KEY,
+                    build_is_aggregate=False,
+                    grain=BuildGrain(),
+                )
+            ],
+            left_datasource=BuildDatasource(name="left_ds", columns=[], address="agsg"),
+            right_datasource=BuildDatasource(
+                name="right_ds", columns=[], address="agsg"
+            ),
+        )
+    with raises(InvalidSyntaxException):
+        raise_helpful_join_validation_error(
+            concepts=[
+                BuildConcept(
+                    name="test_concept",
+                    datatype=DataType.INTEGER,
+                    purpose=Purpose.KEY,
+                    build_is_aggregate=False,
+                    grain=BuildGrain(),
+                )
+            ],
+            left_datasource=None,
+            right_datasource=BuildDatasource(
+                name="right_ds", columns=[], address="agsg"
+            ),
+        )
+    with raises(InvalidSyntaxException):
+        raise_helpful_join_validation_error(
+            concepts=[
+                BuildConcept(
+                    name="test_concept",
+                    datatype=DataType.INTEGER,
+                    purpose=Purpose.KEY,
+                    build_is_aggregate=False,
+                    grain=BuildGrain(),
+                )
+            ],
+            left_datasource=BuildDatasource(name="left_ds", columns=[], address="agsg"),
+            right_datasource=None,
+        )

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.69"
+__version__ = "0.0.3.71"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/functions.py

@@ -370,10 +370,10 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
         arg_count=2,
     ),
     FunctionType.REGEXP_EXTRACT: FunctionConfig(
-        valid_inputs={DataType.STRING},
+        valid_inputs=[{DataType.STRING}, {DataType.STRING}, {DataType.INTEGER}],
         output_purpose=Purpose.PROPERTY,
         output_type=DataType.STRING,
-        arg_count=2,
+        arg_count=3,
     ),
     FunctionType.REGEXP_REPLACE: FunctionConfig(
         valid_inputs={DataType.STRING},

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/graph_models.py

@@ -4,7 +4,7 @@ from trilogy.core.models.build import BuildConcept, BuildDatasource, BuildWhereC
 
 
 def get_graph_exact_match(
-    g: nx.DiGraph, conditions: BuildWhereClause | None
+    g: nx.DiGraph, accept_partial: bool, conditions: BuildWhereClause | None
 ) -> set[str]:
     datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
         nx.get_node_attributes(g, "datasource")
@@ -20,6 +20,9 @@ def get_graph_exact_match(
             if not conditions and not ds.non_partial_for:
                 exact.add(node)
                 continue
+            elif not conditions and accept_partial and ds.non_partial_for:
+                exact.add(node)
+                continue
             elif conditions:
                 if not ds.non_partial_for:
                     continue
@@ -34,10 +37,11 @@ def get_graph_exact_match(
 
 def prune_sources_for_conditions(
     g: nx.DiGraph,
+    accept_partial: bool,
     conditions: BuildWhereClause | None,
 ):
 
-    complete = get_graph_exact_match(g, conditions)
+    complete = get_graph_exact_match(g, accept_partial, conditions)
     to_remove = []
     for node in g.nodes:
         if node.startswith("ds~") and node not in complete:

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/models/datasource.py

@@ -249,6 +249,11 @@ class Datasource(HasUUID, Namespaced, BaseModel):
             address=self.address,
             columns=[c.with_namespace(namespace) for c in self.columns],
             where=self.where.with_namespace(namespace) if self.where else None,
+            non_partial_for=(
+                self.non_partial_for.with_namespace(namespace)
+                if self.non_partial_for
+                else None
+            ),
         )
         return new
 

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/models/execute.py

@@ -24,12 +24,12 @@ from trilogy.core.enums import (
     ComparisonOperator,
     Derivation,
     FunctionType,
-    Granularity,
     JoinType,
     Modifier,
     Purpose,
     SourceType,
 )
+from trilogy.core.exceptions import InvalidSyntaxException
 from trilogy.core.models.build import (
     BuildCaseElse,
     BuildCaseWhen,
@@ -438,7 +438,7 @@ class ConceptPair(BaseModel):
 
 
 class CTEConceptPair(ConceptPair):
-    cte: CTE
+    cte: CTE | UnionCTE
 
 
 class InstantiatedUnnestJoin(BaseModel):
@@ -460,6 +460,30 @@ class UnnestJoin(BaseModel):
         return self.alias + "".join([str(s.address) for s in self.concepts])
 
 
+def raise_helpful_join_validation_error(
+    concepts: List[BuildConcept],
+    left_datasource: BuildDatasource | QueryDatasource | None,
+    right_datasource: BuildDatasource | QueryDatasource | None,
+):
+
+    if not left_datasource or not right_datasource:
+        raise InvalidSyntaxException(
+            "No mutual keys found, and not two valid datasources"
+        )
+    left_keys = [c.address for c in left_datasource.output_concepts]
+    right_keys = [c.address for c in right_datasource.output_concepts]
+    match_concepts = [c.address for c in concepts]
+    assert left_datasource
+    assert right_datasource
+    raise InvalidSyntaxException(
+        "No mutual join keys found between"
+        f" {left_datasource.identifier} and"
+        f" {right_datasource.identifier}, left_keys {left_keys},"
+        f" right_keys {right_keys},"
+        f" provided join concepts {match_concepts}"
+    )
+
+
 class BaseJoin(BaseModel):
     right_datasource: Union[BuildDatasource, "QueryDatasource"]
     join_type: JoinType
@@ -479,27 +503,24 @@ class BaseJoin(BaseModel):
             )
 
         # Early returns maintained as in original code
-        if self.concept_pairs:
-            return self
-
-        if self.concepts == []:
+        if self.concept_pairs or self.concepts == []:
             return self
 
-        # Validation logic
+        # reduce concept list to just the mutual keys
         final_concepts = []
-        assert self.left_datasource and self.right_datasource
-
         for concept in self.concepts or []:
             include = True
             for ds in [self.left_datasource, self.right_datasource]:
                 synonyms = []
+                if not ds:
+                    continue
                 for c in ds.output_concepts:
                     synonyms += list(c.pseudonyms)
                 if (
-                    concept.address not in [c.address for c in ds.output_concepts]
+                    concept.address not in ds.output_concepts
                     and concept.address not in synonyms
                 ):
-                    raise SyntaxError(
+                    raise InvalidSyntaxException(
                         f"Invalid join, missing {concept} on {ds.name}, have"
                         f" {[c.address for c in ds.output_concepts]}"
                     )
@@ -507,32 +528,10 @@ class BaseJoin(BaseModel):
                 final_concepts.append(concept)
 
         if not final_concepts and self.concepts:
-            # if one datasource only has constants
-            # we can join on 1=1
-            for ds in [self.left_datasource, self.right_datasource]:
-                # single rows
-                if all(
-                    [
-                        c.granularity == Granularity.SINGLE_ROW
-                        for c in ds.output_concepts
-                    ]
-                ):
-                    self.concepts = []
-                    return self
-                # if everything is at abstract grain, we can skip joins
-                if all([c.grain.abstract for c in ds.output_concepts]):
-                    self.concepts = []
-                    return self
-
-            left_keys = [c.address for c in self.left_datasource.output_concepts]
-            right_keys = [c.address for c in self.right_datasource.output_concepts]
-            match_concepts = [c.address for c in self.concepts]
-            raise SyntaxError(
-                "No mutual join keys found between"
-                f" {self.left_datasource.identifier} and"
-                f" {self.right_datasource.identifier}, left_keys {left_keys},"
-                f" right_keys {right_keys},"
-                f" provided join concepts {match_concepts}"
+            raise_helpful_join_validation_error(
+                self.concepts,
+                self.left_datasource,
+                self.right_datasource,
             )
 
         self.concepts = final_concepts
@@ -1087,7 +1086,7 @@ class UnionCTE(BaseModel):
 class Join(BaseModel):
     right_cte: CTE | UnionCTE
     jointype: JoinType
-    left_cte: CTE | None = None
+    left_cte: CTE | UnionCTE | None = None
     joinkey_pairs: List[CTEConceptPair] | None = None
     inlined_ctes: set[str] = Field(default_factory=set)
     quote: str | None = None
@@ -1096,7 +1095,7 @@ class Join(BaseModel):
     def inline_cte(self, cte: CTE):
         self.inlined_ctes.add(cte.name)
 
-    def get_name(self, cte: CTE):
+    def get_name(self, cte: CTE | UnionCTE) -> str:
         if cte.identifier in self.inlined_ctes:
             return cte.source.datasources[0].safe_identifier
         return cte.safe_identifier

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/processing/concept_strategies_v3.py

@@ -291,7 +291,7 @@ def evaluate_loop_conditions(
 
 
 def check_for_early_exit(
-    complete, partial, context: LoopContext, priority_concept: BuildConcept
+    complete, partial, missing, context: LoopContext, priority_concept: BuildConcept
 ) -> bool:
     if complete == ValidationResult.INCOMPLETE_CONDITION:
         cond_dict = {str(node): node.preexisting_conditions for node in context.stack}
@@ -321,7 +321,7 @@ def check_for_early_exit(
         )
     else:
         logger.info(
-            f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Not complete, continuing search"
+            f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Not complete (missing {missing}), continuing search"
         )
     # if we have attempted on root node, we've tried them all.
     # inject in another search with filter concepts
@@ -412,7 +412,7 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
     elif context.conditions:
         output.preexisting_conditions = context.conditions.conditional
     logger.info(
-        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts]} with {context.conditions}"
+        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts or []]} with {context.conditions}"
     )
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
@@ -532,7 +532,9 @@ def _search_concepts(
         )
         # assign
         context.found = found_c
-        early_exit = check_for_early_exit(complete, partial, context, priority_concept)
+        early_exit = check_for_early_exit(
+            complete, partial, missing_c, context, priority_concept
+        )
         if early_exit:
             break
 

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -253,7 +253,9 @@ def resolve_weak_components(
     break_flag = False
     found = []
     search_graph = environment_graph.copy()
-    prune_sources_for_conditions(search_graph, conditions=search_conditions)
+    prune_sources_for_conditions(
+        search_graph, accept_partial, conditions=search_conditions
+    )
     reduced_concept_sets: list[set[str]] = []
 
     # loop through, removing new nodes we find

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/processing/node_generators/rowset_node.py

@@ -181,6 +181,5 @@ def gen_rowset_node(
             node,
             enrich_node,
         ],
-        partial_concepts=node.partial_concepts + enrich_node.partial_concepts,
         preexisting_conditions=conditions.conditional if conditions else None,
     )

{pytrilogy-0.0.3.69 → pytrilogy-0.0.3.71}/trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -1,3 +1,4 @@
+import sys
 from collections import defaultdict
 from datetime import date, datetime, timedelta
 from typing import List, Tuple, TypeVar
@@ -14,7 +15,7 @@ from trilogy.core.models.build import (
 from trilogy.core.models.core import DataType
 
 # Define a generic type that ensures start and end are the same type
-T = TypeVar("T", int, date, datetime)
+T = TypeVar("T", int, float, date, datetime)
 
 
 def reduce_expression(
@@ -37,21 +38,26 @@ def reduce_expression(
     elif var.datatype == DataType.DATETIME:
         lower_check = datetime.min  # type: ignore
         upper_check = datetime.max  # type: ignore
+    elif var.datatype == DataType.BOOL:
+        lower_check = False  # type: ignore
+        upper_check = True  # type: ignore
+    elif var.datatype == DataType.FLOAT:
+        lower_check = float("-inf")  # type: ignore
+        upper_check = float("inf")  # type: ignore
     else:
-        raise ValueError(f"Invalid datatype: {var.datatype}")
+        return False
 
     ranges: list[Tuple[T, T]] = []
     for op, value in group_tuple:
-        increment: int | timedelta
+        increment: int | timedelta | float
         if isinstance(value, date):
             increment = timedelta(days=1)
         elif isinstance(value, datetime):
             increment = timedelta(seconds=1)
         elif isinstance(value, int):
             increment = 1
-        # elif isinstance(value, float):
-        #     value = Decimal(value)
-        #     increment = Decimal(0.0000000001)
+        elif isinstance(value, float):
+            increment = sys.float_info.epsilon
 
         if op == ">":
             ranges.append(
@@ -88,24 +94,43 @@ def reduce_expression(
                     value,
                 )
             )
+        elif op == ComparisonOperator.IS:
+            ranges.append(
+                (
+                    value,
+                    value,
+                )
+            )
+        elif op == ComparisonOperator.NE:
+            pass
         else:
-            raise ValueError(f"Invalid operator: {op}")
+            return False
     return is_fully_covered(lower_check, upper_check, ranges, increment)
 
 
+TARGET_TYPES = (
+    int,
+    date,
+    float,
+    datetime,
+    bool,
+)
+REDUCABLE_TYPES = (int, float, date, bool, datetime, BuildFunction)
+
+
 def simplify_conditions(
     conditions: list[BuildComparison | BuildConditional | BuildParenthetical],
 ) -> bool:
     # Group conditions by variable
     grouped: dict[
-        BuildConcept, list[tuple[ComparisonOperator, datetime | int | date]]
+        BuildConcept, list[tuple[ComparisonOperator, datetime | int | date | float]]
     ] = defaultdict(list)
     for condition in conditions:
         if not isinstance(condition, BuildComparison):
             return False
-        if not isinstance(
-            condition.left, (int, date, datetime, BuildFunction)
-        ) and not isinstance(condition.right, (int, date, datetime, BuildFunction)):
+        if not isinstance(condition.left, REDUCABLE_TYPES) and not isinstance(
+            condition.right, REDUCABLE_TYPES
+        ):
             return False
         if not isinstance(condition.left, BuildConcept) and not isinstance(
             condition.right, BuildConcept
@@ -113,15 +138,20 @@ def simplify_conditions(
             return False
         vars = [condition.left, condition.right]
         concept = [x for x in vars if isinstance(x, BuildConcept)][0]
-        comparison = [x for x in vars if not isinstance(x, BuildConcept)][0]
-        if isinstance(comparison, BuildFunction):
-            if not comparison.operator == FunctionType.CONSTANT:
+        raw_comparison = [x for x in vars if not isinstance(x, BuildConcept)][0]
+        if isinstance(raw_comparison, BuildFunction):
+            if not raw_comparison.operator == FunctionType.CONSTANT:
                 return False
-            first_arg = comparison.arguments[0]
-            if not isinstance(first_arg, (int, date, datetime)):
+            first_arg = raw_comparison.arguments[0]
+            if not isinstance(first_arg, TARGET_TYPES):
                 return False
             comparison = first_arg
-        if not isinstance(comparison, (int, date, datetime)):
+        else:
+            if not isinstance(raw_comparison, TARGET_TYPES):
+                return False
+            comparison = raw_comparison
+
+        if not isinstance(comparison, REDUCABLE_TYPES):
             return False
 
         var = concept
@@ -136,11 +166,25 @@ def simplify_conditions(
     return True if all(isinstance(s, bool) and s for s in simplified) else False
 
 
+def boolean_fully_covered(
+    start: bool,
+    end: bool,
+    ranges: List[Tuple[bool, bool]],
+):
+    all = []
+    for r_start, r_end in ranges:
+        if r_start is True and r_end is True:
+            all.append(True)
+        elif r_start is False and r_end is False:
+            all.append(False)
+    return set(all) == {False, True}
+
+
 def is_fully_covered(
     start: T,
     end: T,
     ranges: List[Tuple[T, T]],
-    increment: int | timedelta,
+    increment: int | timedelta | float,
 ):
     """
     Check if the list of range pairs fully covers the set [start, end].
@@ -153,6 +197,11 @@ def is_fully_covered(
     Returns:
     - bool: True if the ranges fully cover [start, end], False otherwise.
     """
+    if isinstance(start, bool) and isinstance(end, bool):
+        # convert each element of each tuple to a boolean
+        bool_ranges = [(bool(r_start), bool(r_end)) for r_start, r_end in ranges]
+
+        return boolean_fully_covered(start, end, bool_ranges)
     # Sort ranges by their start values (and by end values for ties)
     ranges.sort()
 
@@ -173,14 +222,14 @@ def get_union_sources(
     datasources: list[BuildDatasource], concepts: list[BuildConcept]
 ) -> List[list[BuildDatasource]]:
     candidates: list[BuildDatasource] = []
+
     for x in datasources:
-        if all([c.address in x.output_concepts for c in concepts]):
+        if any([c.address in x.output_concepts for c in concepts]):
             if (
                 any([c.address in x.partial_concepts for c in concepts])
                 and x.non_partial_for
             ):
                 candidates.append(x)
-
     assocs: dict[str, list[BuildDatasource]] = defaultdict(list[BuildDatasource])
     for x in candidates:
         if not x.non_partial_for: