pytrilogy 0.0.2.13__tar.gz → 0.0.2.14__tar.gz

{pytrilogy-0.0.2.13/pytrilogy.egg-info → pytrilogy-0.0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.13
+Version: 0.0.2.14
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.13
+Version: 0.0.2.14
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

pytrilogy-0.0.2.14/tests/test_datatypes.py

@@ -0,0 +1,58 @@
+from trilogy.core.models import (
+    NumericType,
+)
+from trilogy.parsing.parse_engine import (
+    parse_text,
+)
+from trilogy.core.exceptions import InvalidSyntaxException
+
+
+def test_numeric():
+    env, _ = parse_text(
+        "const order_id numeric(12,2); const rounded <- cast(order_id as numeric(15,2));"
+    )
+    assert env.concepts["order_id"].datatype == NumericType(precision=12, scale=2)
+
+
+def test_cast_error():
+    found = False
+    try:
+        env, _ = parse_text(
+            """
+    const x <- 1;
+    const y <- 'fun';
+
+    select 
+        x
+    where 
+        x = y;
+
+    """
+        )
+    except InvalidSyntaxException as e:
+        assert "Cannot compare DataType.INTEGER and DataType.STRING" in str(e)
+        found = True
+    if not found:
+        assert False, "Expected InvalidSyntaxException not raised"
+
+
+def test_is_error():
+    found = False
+    try:
+        env, _ = parse_text(
+            """
+    const x <- 1;
+    const y <- 'fun';
+
+    select 
+        x
+    where 
+        x is [1,2];
+
+    """
+        )
+    except InvalidSyntaxException as e:
+        assert "Cannot use is with non-null or boolean value [1, 2]" in str(e)
+        found = True
+    if not found:
+        assert False, "Expected InvalidSyntaxException not raised"

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/tests/test_models.py

@@ -73,17 +73,15 @@ def test_concept(test_environment, test_environment_graph):
 def test_concept_filter(test_environment, test_environment_graph):
     test_concept: Concept = list(test_environment.concepts.values())[0]
     new = test_concept.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+        Comparison(left=1, right=2, operator=ComparisonOperator.EQ)
     )
     new2 = test_concept.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+        Comparison(left=1, right=2, operator=ComparisonOperator.EQ)
     )
 
     assert new.name == new2.name != test_concept.name
 
-    new3 = new.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
-    )
+    new3 = new.with_filter(Comparison(left=1, right=2, operator=ComparisonOperator.EQ))
     assert new3 == new
 
 

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/tests/test_parsing.py

@@ -34,7 +34,7 @@ def test_in():
     assert rendered.strip() == "( 1,2,3 )".strip()
 
     _, parsed = parse_text(
-        "const order_id <- 3; SELECT order_id  WHERE order_id IN (1);"
+        "const order_id <- 3; SELECT order_id  WHERE order_id IN (1,);"
     )
     query = parsed[-1]
     right = query.where_clause.conditional.right
@@ -42,7 +42,7 @@ def test_in():
         right,
         Parenthetical,
     ), type(right)
-    assert right.content == 1
+    assert right.content[0] == 1
     rendered = BaseDialect().render_expr(right)
     assert rendered.strip() == "( 1 )".strip()
 

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/tests/test_partial_handling.py

@@ -59,7 +59,7 @@ def setup_titanic(env: Environment):
         name="survived",
         namespace=namespace,
         purpose=Purpose.PROPERTY,
-        datatype=DataType.BOOL,
+        datatype=DataType.INTEGER,
         keys=(id,),
     )
     fare = Concept(

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/tests/test_where_clause.py

@@ -185,7 +185,7 @@ where
 def test_like_filter(test_environment):
     declarations = """
 property special_order <- filter order_id where like(category_name, 'test') = True;
-property special_order_2 <- filter order_id where like(category_name, 'test') = 1;
+property special_order_2 <- filter order_id where like(category_name, 'test') is True;
 
 select
     special_order

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.13"
+__version__ = "0.0.2.14"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/constants.py

@@ -30,7 +30,13 @@ class Optimizations:
 
 @dataclass
 class Comments:
-    pass
+    """Control what is placed in CTE comments"""
+
+    show: bool = False
+    basic: bool = True
+    joins: bool = True
+    nullable: bool = False
+    partial: bool = False
 
 
 # TODO: support loading from environments
@@ -39,9 +45,13 @@ class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     validate_missing: bool = True
-    show_comments: bool = False
+    comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)
 
+    @property
+    def show_comments(self) -> bool:
+        return self.comments.show
+
     def set_random_seed(self, seed: int):
         random.seed(seed)
 

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/models.py

@@ -79,6 +79,18 @@ VT = TypeVar("VT")
 LT = TypeVar("LT")
 
 
+def is_compatible_datatype(left, right):
+    if left == right:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.FLOAT}:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.INTEGER}:
+        return True
+    if {left, right} == {DataType.FLOAT, DataType.INTEGER}:
+        return True
+    return False
+
+
 def get_version():
     from trilogy import __version__
 
@@ -220,6 +232,7 @@ class DataType(Enum):
     ARRAY = "array"
     DATE_PART = "date_part"
     STRUCT = "struct"
+    NULL = "null"
 
     # GRANULAR
     UNIX_SECONDS = "unix_seconds"
@@ -960,6 +973,10 @@ class ColumnAssignment(BaseModel):
     def is_complete(self) -> bool:
         return Modifier.PARTIAL not in self.modifiers
 
+    @property
+    def is_nullable(self) -> bool:
+        return Modifier.NULLABLE in self.modifiers
+
     def with_namespace(self, namespace: str) -> "ColumnAssignment":
         return ColumnAssignment(
             alias=(
@@ -1065,9 +1082,12 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
         ]
     ]
 
-    def __str__(self):
+    def __repr__(self):
         return f'{self.operator.value}({",".join([str(a) for a in self.arguments])})'
 
+    def __str__(self):
+        return self.__repr__()
+
     @property
     def datatype(self):
         return self.output_datatype
@@ -2072,6 +2092,10 @@ class Datasource(Namespaced, BaseModel):
     def full_concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns if Modifier.PARTIAL not in c.modifiers]
 
+    @property
+    def nullable_concepts(self) -> List[Concept]:
+        return [c.concept for c in self.columns if Modifier.NULLABLE in c.modifiers]
+
     @property
     def output_concepts(self) -> List[Concept]:
         return self.concepts
@@ -2135,13 +2159,27 @@ class InstantiatedUnnestJoin(BaseModel):
     alias: str = "unnest"
 
 
+class ConceptPair(BaseModel):
+    left: Concept
+    right: Concept
+    modifiers: List[Modifier] = Field(default_factory=list)
+
+    @property
+    def is_partial(self):
+        return Modifier.PARTIAL in self.modifiers
+
+    @property
+    def is_nullable(self):
+        return Modifier.NULLABLE in self.modifiers
+
+
 class BaseJoin(BaseModel):
     left_datasource: Union[Datasource, "QueryDatasource"]
     right_datasource: Union[Datasource, "QueryDatasource"]
     concepts: List[Concept]
     join_type: JoinType
     filter_to_mutual: bool = False
-    concept_pairs: list[tuple[Concept, Concept]] | None = None
+    concept_pairs: list[ConceptPair] | None = None
 
     def __init__(self, **data: Any):
         super().__init__(**data)
@@ -2219,7 +2257,7 @@ class BaseJoin(BaseModel):
             return (
                 f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
                 f" {self.right_datasource.identifier} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.concept_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
             f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
@@ -2243,8 +2281,9 @@ class QueryDatasource(BaseModel):
     filter_concepts: List[Concept] = Field(default_factory=list)
     source_type: SourceType = SourceType.SELECT
     partial_concepts: List[Concept] = Field(default_factory=list)
-    join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
+    join_derived_concepts: List[Concept] = Field(default_factory=list)
     force_group: bool | None = None
     existence_source_map: Dict[str, Set[Union[Datasource, "QueryDatasource"]]] = Field(
         default_factory=dict
@@ -2500,6 +2539,7 @@ class CTE(BaseModel):
     joins: List[Union["Join", "InstantiatedUnnestJoin"]] = Field(default_factory=list)
     condition: Optional[Union["Conditional", "Comparison", "Parenthetical"]] = None
     partial_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
     join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
@@ -2583,6 +2623,10 @@ class CTE(BaseModel):
         base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
         if self.hidden_concepts:
             base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+        if self.nullable_concepts:
+            base += (
+                f"\n-- Nullable: {', '.join([str(x) for x in self.nullable_concepts])}."
+            )
 
         return base
 
@@ -2677,6 +2721,9 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
+        self.nullable_concepts = unique(
+            self.nullable_concepts + other.nullable_concepts, "address"
+        )
         self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
@@ -2842,7 +2889,7 @@ class Join(BaseModel):
     right_cte: CTE | Datasource
     jointype: JoinType
     joinkeys: List[JoinKey]
-    joinkey_pairs: List[tuple[Concept, Concept]] | None = None
+    joinkey_pairs: List[ConceptPair] | None = None
 
     @property
     def left_name(self) -> str:
@@ -2877,7 +2924,7 @@ class Join(BaseModel):
             return (
                 f"{self.jointype.value} JOIN {self.left_name} and"
                 f" {self.right_name} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.joinkey_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right)+str(k.modifiers) for k in self.joinkey_pairs])}"
             )
         return (
             f"{self.jointype.value} JOIN {self.left_name} and"
@@ -3450,11 +3497,41 @@ class Comparison(
     ]
     operator: ComparisonOperator
 
-    def __post_init__(self):
-        if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise SyntaxError(
-                f"Cannot compare {self.left} and {self.right} of different types"
-            )
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        if self.operator in (ComparisonOperator.IS, ComparisonOperator.IS_NOT):
+            if self.right != MagicConstants.NULL and DataType.BOOL != arg_to_datatype(
+                self.right
+            ):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-null or boolean value {self.right}"
+                )
+        elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
+            right = arg_to_datatype(self.right)
+            if not isinstance(self.right, Concept) and not isinstance(right, ListType):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-list type {right} in {str(self)}"
+                )
+
+            elif isinstance(right, ListType) and not is_compatible_datatype(
+                arg_to_datatype(self.left), right.value_data_type
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {right} with operator {self.operator} in {str(self)}"
+                )
+            elif isinstance(self.right, Concept) and not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} with operator {self.operator} in {str(self)}"
+                )
+        else:
+            if not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} of different types with operator {self.operator} in {str(self)}"
+                )
 
     def __add__(self, other):
         if other is None:
@@ -3655,6 +3732,12 @@ class CaseWhen(Namespaced, SelectContext, BaseModel):
     comparison: Conditional | SubselectComparison | Comparison
     expr: "Expr"
 
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        return f"WHEN {str(self.comparison)} THEN {str(self.expr)}"
+
     @property
     def concept_arguments(self):
         return get_concept_arguments(self.comparison) + get_concept_arguments(self.expr)
@@ -4347,6 +4430,7 @@ class ShowStatement(BaseModel):
 
 Expr = (
     bool
+    | MagicConstants
     | int
     | str
     | float
@@ -4401,9 +4485,34 @@ def dict_to_map_wrapper(arg):
     return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
 
 
+def merge_datatypes(
+    inputs: list[DataType | ListType | StructType | MapType | NumericType],
+) -> DataType | ListType | StructType | MapType | NumericType:
+    """This is a temporary hack for doing between
+    allowable datatype transformation matrix"""
+    if len(inputs) == 1:
+        return inputs[0]
+    if set(inputs) == {DataType.INTEGER, DataType.FLOAT}:
+        return DataType.FLOAT
+    if set(inputs) == {DataType.INTEGER, DataType.NUMERIC}:
+        return DataType.NUMERIC
+    if any(isinstance(x, NumericType) for x in inputs) and all(
+        isinstance(x, NumericType)
+        or x in (DataType.INTEGER, DataType.FLOAT, DataType.NUMERIC)
+        for x in inputs
+    ):
+        candidate = next(x for x in inputs if isinstance(x, NumericType))
+        return candidate
+    return inputs[0]
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
+    elif isinstance(arg, MagicConstants):
+        if arg == MagicConstants.NULL:
+            return DataType.NULL
+        raise ValueError(f"Cannot parse arg datatype for arg of type {arg}")
     elif isinstance(arg, Concept):
         return arg.datatype
     elif isinstance(arg, bool):

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/optimizations/predicate_pushdown.py

@@ -135,7 +135,7 @@ class PredicatePushdown(OptimizationRule):
                     f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
                 )
                 continue
-            self.log(
+            self.debug(
                 f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
             )
             for parent_cte in cte.parent_ctes:

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/common.py

@@ -15,12 +15,10 @@ from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import StrategyNode
 from trilogy.core.processing.nodes.merge_node import MergeNode
 from trilogy.core.processing.nodes import History
-from trilogy.core.enums import JoinType
 from trilogy.core.processing.nodes import (
     NodeJoin,
 )
 from collections import defaultdict
-from trilogy.core.processing.utility import concept_to_relevant_joins
 
 
 def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
@@ -218,17 +216,6 @@ def gen_enrichment_node(
         g=g,
         parents=[enrich_node, base_node],
         force_group=False,
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=base_node,
-                concepts=concept_to_relevant_joins(
-                    [x for x in join_keys if x in enrich_node.output_lcl]
-                ),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
     )
 
 

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/filter_node.py

@@ -1,12 +1,10 @@
 from typing import List
 
 
-from trilogy.core.enums import JoinType
 from trilogy.core.models import Concept, Environment, FilterItem, Grain, WhereClause
 from trilogy.core.processing.nodes import (
     FilterNode,
     MergeNode,
-    NodeJoin,
     History,
     StrategyNode,
     SelectNode,
@@ -16,7 +14,6 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, unique
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
@@ -215,16 +212,5 @@ def gen_filter_node(
             # this node fetches only what we need to filter
             filter_node,
             enrich_node,
-        ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=filter_node,
-                concepts=concept_to_relevant_joins(
-                    [immediate_parent] + parent_row_concepts
-                ),
-                join_type=JoinType.LEFT_OUTER,
-                filter_to_mutual=True,
-            )
-        ],
+        ]
     )

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/group_node.py

@@ -1,4 +1,11 @@
-from trilogy.core.models import Concept, Environment, LooseConceptList, WhereClause
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    LooseConceptList,
+    WhereClause,
+    Function,
+    AggregateWrapper,
+)
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
 from typing import List
@@ -42,6 +49,17 @@ def gen_group_node(
         )
         parent_concepts += grain_components
         output_concepts += grain_components
+        for possible_agg in local_optional:
+            if possible_agg.grain and possible_agg.grain == concept.grain:
+                if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
+                    continue
+                agg_parents: List[Concept] = resolve_function_parent_concepts(
+                    possible_agg
+                )
+                if set([x.address for x in agg_parents]).issubset(
+                    set([x.address for x in parent_concepts])
+                ):
+                    output_concepts.append(possible_agg)
 
     if parent_concepts:
         logger.info(

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/group_to_node.py

@@ -3,15 +3,12 @@ from trilogy.core.processing.nodes import (
     GroupNode,
     StrategyNode,
     MergeNode,
-    NodeJoin,
     History,
 )
 from typing import List
-from trilogy.core.enums import JoinType
 
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 LOGGER_PREFIX = "[GEN_GROUP_TO_NODE]"
 
@@ -84,15 +81,6 @@ def gen_group_to_node(
             # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=group_node,
-                right_node=enrich_node,
-                concepts=concept_to_relevant_joins(parent_concepts),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         whole_grain=True,
         depth=depth,
     )

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/multiselect_node.py

@@ -10,7 +10,7 @@ from typing import List
 from trilogy.core.enums import JoinType
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import concept_to_relevant_joins
 from collections import defaultdict
 from itertools import combinations
 from trilogy.core.enums import Purpose
@@ -176,14 +176,5 @@ def gen_multiselect_node(
             # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=node,
-                concepts=possible_joins,
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         partial_concepts=node.partial_concepts,
     )

{pytrilogy-0.0.2.13 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/rowset_node.py

@@ -6,14 +6,14 @@ from trilogy.core.models import (
     RowsetItem,
     MultiSelectStatement,
 )
-from trilogy.core.processing.nodes import MergeNode, NodeJoin, History, StrategyNode
+from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 from typing import List
 
-from trilogy.core.enums import JoinType, PurposeLineage
+from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import concept_to_relevant_joins
 
 
 LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
@@ -113,19 +113,8 @@ def gen_rowset_node(
         g=g,
         depth=depth,
         parents=[
-            # this node gets the window
             node,
-            # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=node,
-                concepts=concept_to_relevant_joins(additional_relevant),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         partial_concepts=node.partial_concepts,
     )