pytrilogy 0.0.2.12__tar.gz → 0.0.2.14__tar.gz

{pytrilogy-0.0.2.12/pytrilogy.egg-info → pytrilogy-0.0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.12
+Version: 0.0.2.14
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.12
+Version: 0.0.2.14
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

pytrilogy-0.0.2.14/tests/test_datatypes.py

@@ -0,0 +1,58 @@
+from trilogy.core.models import (
+    NumericType,
+)
+from trilogy.parsing.parse_engine import (
+    parse_text,
+)
+from trilogy.core.exceptions import InvalidSyntaxException
+
+
+def test_numeric():
+    env, _ = parse_text(
+        "const order_id numeric(12,2); const rounded <- cast(order_id as numeric(15,2));"
+    )
+    assert env.concepts["order_id"].datatype == NumericType(precision=12, scale=2)
+
+
+def test_cast_error():
+    found = False
+    try:
+        env, _ = parse_text(
+            """
+    const x <- 1;
+    const y <- 'fun';
+
+    select 
+        x
+    where 
+        x = y;
+
+    """
+        )
+    except InvalidSyntaxException as e:
+        assert "Cannot compare DataType.INTEGER and DataType.STRING" in str(e)
+        found = True
+    if not found:
+        assert False, "Expected InvalidSyntaxException not raised"
+
+
+def test_is_error():
+    found = False
+    try:
+        env, _ = parse_text(
+            """
+    const x <- 1;
+    const y <- 'fun';
+
+    select 
+        x
+    where 
+        x is [1,2];
+
+    """
+        )
+    except InvalidSyntaxException as e:
+        assert "Cannot use is with non-null or boolean value [1, 2]" in str(e)
+        found = True
+    if not found:
+        assert False, "Expected InvalidSyntaxException not raised"

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/tests/test_models.py

@@ -73,17 +73,15 @@ def test_concept(test_environment, test_environment_graph):
 def test_concept_filter(test_environment, test_environment_graph):
     test_concept: Concept = list(test_environment.concepts.values())[0]
     new = test_concept.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+        Comparison(left=1, right=2, operator=ComparisonOperator.EQ)
     )
     new2 = test_concept.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+        Comparison(left=1, right=2, operator=ComparisonOperator.EQ)
     )
 
     assert new.name == new2.name != test_concept.name
 
-    new3 = new.with_filter(
-        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
-    )
+    new3 = new.with_filter(Comparison(left=1, right=2, operator=ComparisonOperator.EQ))
     assert new3 == new
 
 

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/tests/test_parsing.py

@@ -34,7 +34,7 @@ def test_in():
     assert rendered.strip() == "( 1,2,3 )".strip()
 
     _, parsed = parse_text(
-        "const order_id <- 3; SELECT order_id  WHERE order_id IN (1);"
+        "const order_id <- 3; SELECT order_id  WHERE order_id IN (1,);"
     )
     query = parsed[-1]
     right = query.where_clause.conditional.right
@@ -42,7 +42,7 @@ def test_in():
         right,
         Parenthetical,
     ), type(right)
-    assert right.content == 1
+    assert right.content[0] == 1
     rendered = BaseDialect().render_expr(right)
     assert rendered.strip() == "( 1 )".strip()
 

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/tests/test_partial_handling.py

@@ -59,7 +59,7 @@ def setup_titanic(env: Environment):
         name="survived",
         namespace=namespace,
         purpose=Purpose.PROPERTY,
-        datatype=DataType.BOOL,
+        datatype=DataType.INTEGER,
         keys=(id,),
     )
     fare = Concept(

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/tests/test_where_clause.py

@@ -185,7 +185,7 @@ where
 def test_like_filter(test_environment):
     declarations = """
 property special_order <- filter order_id where like(category_name, 'test') = True;
-property special_order_2 <- filter order_id where like(category_name, 'test') = 1;
+property special_order_2 <- filter order_id where like(category_name, 'test') is True;
 
 select
     special_order

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.12"
+__version__ = "0.0.2.14"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/constants.py

@@ -28,15 +28,30 @@ class Optimizations:
     direct_return: bool = True
 
 
+@dataclass
+class Comments:
+    """Control what is placed in CTE comments"""
+
+    show: bool = False
+    basic: bool = True
+    joins: bool = True
+    nullable: bool = False
+    partial: bool = False
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     validate_missing: bool = True
-    show_comments: bool = False
+    comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)
 
+    @property
+    def show_comments(self) -> bool:
+        return self.comments.show
+
     def set_random_seed(self, seed: int):
         random.seed(seed)
 

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/core/enums.py

@@ -62,6 +62,8 @@ class Modifier(Enum):
         strval = str(value)
         if strval == "~":
             return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
 
@@ -273,6 +275,7 @@ class SourceType(Enum):
     CONSTANT = "constant"
     ROWSET = "rowset"
     MERGE = "merge"
+    BASIC = "basic"
 
 
 class ShowCategory(Enum):

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/core/models.py

@@ -79,6 +79,18 @@ VT = TypeVar("VT")
 LT = TypeVar("LT")
 
 
+def is_compatible_datatype(left, right):
+    if left == right:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.FLOAT}:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.INTEGER}:
+        return True
+    if {left, right} == {DataType.FLOAT, DataType.INTEGER}:
+        return True
+    return False
+
+
 def get_version():
     from trilogy import __version__
 
@@ -220,6 +232,7 @@ class DataType(Enum):
     ARRAY = "array"
     DATE_PART = "date_part"
     STRUCT = "struct"
+    NULL = "null"
 
     # GRANULAR
     UNIX_SECONDS = "unix_seconds"
@@ -960,6 +973,10 @@ class ColumnAssignment(BaseModel):
     def is_complete(self) -> bool:
         return Modifier.PARTIAL not in self.modifiers
 
+    @property
+    def is_nullable(self) -> bool:
+        return Modifier.NULLABLE in self.modifiers
+
     def with_namespace(self, namespace: str) -> "ColumnAssignment":
         return ColumnAssignment(
             alias=(
@@ -1065,9 +1082,12 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
         ]
     ]
 
-    def __str__(self):
+    def __repr__(self):
         return f'{self.operator.value}({",".join([str(a) for a in self.arguments])})'
 
+    def __str__(self):
+        return self.__repr__()
+
     @property
     def datatype(self):
         return self.output_datatype
@@ -2072,6 +2092,10 @@ class Datasource(Namespaced, BaseModel):
     def full_concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns if Modifier.PARTIAL not in c.modifiers]
 
+    @property
+    def nullable_concepts(self) -> List[Concept]:
+        return [c.concept for c in self.columns if Modifier.NULLABLE in c.modifiers]
+
     @property
     def output_concepts(self) -> List[Concept]:
         return self.concepts
@@ -2135,13 +2159,27 @@ class InstantiatedUnnestJoin(BaseModel):
     alias: str = "unnest"
 
 
+class ConceptPair(BaseModel):
+    left: Concept
+    right: Concept
+    modifiers: List[Modifier] = Field(default_factory=list)
+
+    @property
+    def is_partial(self):
+        return Modifier.PARTIAL in self.modifiers
+
+    @property
+    def is_nullable(self):
+        return Modifier.NULLABLE in self.modifiers
+
+
 class BaseJoin(BaseModel):
     left_datasource: Union[Datasource, "QueryDatasource"]
     right_datasource: Union[Datasource, "QueryDatasource"]
     concepts: List[Concept]
     join_type: JoinType
     filter_to_mutual: bool = False
-    concept_pairs: list[tuple[Concept, Concept]] | None = None
+    concept_pairs: list[ConceptPair] | None = None
 
     def __init__(self, **data: Any):
         super().__init__(**data)
@@ -2219,7 +2257,7 @@ class BaseJoin(BaseModel):
             return (
                 f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
                 f" {self.right_datasource.identifier} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.concept_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
             f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
@@ -2243,8 +2281,9 @@ class QueryDatasource(BaseModel):
     filter_concepts: List[Concept] = Field(default_factory=list)
     source_type: SourceType = SourceType.SELECT
     partial_concepts: List[Concept] = Field(default_factory=list)
-    join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
+    join_derived_concepts: List[Concept] = Field(default_factory=list)
     force_group: bool | None = None
     existence_source_map: Dict[str, Set[Union[Datasource, "QueryDatasource"]]] = Field(
         default_factory=dict
@@ -2264,6 +2303,7 @@ class QueryDatasource(BaseModel):
     @field_validator("joins")
     @classmethod
     def validate_joins(cls, v):
+        unique_pairs = set()
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
@@ -2271,7 +2311,16 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
-
+            pairing = "".join(
+                sorted(
+                    [join.left_datasource.identifier, join.right_datasource.identifier]
+                )
+            )
+            if pairing in unique_pairs:
+                raise SyntaxError(
+                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
+                )
+            unique_pairs.add(pairing)
         return v
 
     @field_validator("input_concepts")
@@ -2386,6 +2435,11 @@ class QueryDatasource(BaseModel):
                 final_source_map[key] = other.source_map[key]
         for k, v in final_source_map.items():
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
+        self_hidden = self.hidden_concepts or []
+        other_hidden = other.hidden_concepts or []
+        hidden = [
+            x for x in self_hidden if x.address in [y.address for y in other_hidden]
+        ]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2409,9 +2463,7 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
-            hidden_concepts=unique(
-                self.hidden_concepts + other.hidden_concepts, "address"
-            ),
+            hidden_concepts=hidden,
         )
 
         return qds
@@ -2487,6 +2539,7 @@ class CTE(BaseModel):
     joins: List[Union["Join", "InstantiatedUnnestJoin"]] = Field(default_factory=list)
     condition: Optional[Union["Conditional", "Comparison", "Parenthetical"]] = None
     partial_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
     join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
@@ -2557,6 +2610,7 @@ class CTE(BaseModel):
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}."
+        base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
         if self.joins:
@@ -2565,6 +2619,15 @@ class CTE(BaseModel):
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
+        base += f"\n-- Source Map: {self.source_map}."
+        base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.hidden_concepts:
+            base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+        if self.nullable_concepts:
+            base += (
+                f"\n-- Nullable: {', '.join([str(x) for x in self.nullable_concepts])}."
+            )
+
         return base
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
@@ -2632,6 +2695,10 @@ class CTE(BaseModel):
                 f" {self.name} {other.name} conditions {self.condition} {other.condition}"
             )
             raise ValueError(error)
+        mutually_hidden = []
+        for concept in self.hidden_concepts:
+            if concept in other.hidden_concepts:
+                mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
         )
@@ -2654,9 +2721,10 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
-        self.hidden_concepts = unique(
-            self.hidden_concepts + other.hidden_concepts, "address"
+        self.nullable_concepts = unique(
+            self.nullable_concepts + other.nullable_concepts, "address"
         )
+        self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
             **other.existence_source_map,
@@ -2821,7 +2889,7 @@ class Join(BaseModel):
     right_cte: CTE | Datasource
     jointype: JoinType
     joinkeys: List[JoinKey]
-    joinkey_pairs: List[tuple[Concept, Concept]] | None = None
+    joinkey_pairs: List[ConceptPair] | None = None
 
     @property
     def left_name(self) -> str:
@@ -2856,7 +2924,7 @@ class Join(BaseModel):
             return (
                 f"{self.jointype.value} JOIN {self.left_name} and"
                 f" {self.right_name} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.joinkey_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right)+str(k.modifiers) for k in self.joinkey_pairs])}"
             )
         return (
             f"{self.jointype.value} JOIN {self.left_name} and"
@@ -3012,6 +3080,9 @@ class EnvironmentDatasourceDict(dict):
     def values(self) -> ValuesView[Datasource]:  # type: ignore
         return super().values()
 
+    def items(self) -> ItemsView[str, Datasource]:  # type: ignore
+        return super().items()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
@@ -3426,11 +3497,41 @@ class Comparison(
     ]
     operator: ComparisonOperator
 
-    def __post_init__(self):
-        if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise SyntaxError(
-                f"Cannot compare {self.left} and {self.right} of different types"
-            )
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        if self.operator in (ComparisonOperator.IS, ComparisonOperator.IS_NOT):
+            if self.right != MagicConstants.NULL and DataType.BOOL != arg_to_datatype(
+                self.right
+            ):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-null or boolean value {self.right}"
+                )
+        elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
+            right = arg_to_datatype(self.right)
+            if not isinstance(self.right, Concept) and not isinstance(right, ListType):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-list type {right} in {str(self)}"
+                )
+
+            elif isinstance(right, ListType) and not is_compatible_datatype(
+                arg_to_datatype(self.left), right.value_data_type
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {right} with operator {self.operator} in {str(self)}"
+                )
+            elif isinstance(self.right, Concept) and not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} with operator {self.operator} in {str(self)}"
+                )
+        else:
+            if not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} of different types with operator {self.operator} in {str(self)}"
+                )
 
     def __add__(self, other):
         if other is None:
@@ -3631,6 +3732,12 @@ class CaseWhen(Namespaced, SelectContext, BaseModel):
     comparison: Conditional | SubselectComparison | Comparison
     expr: "Expr"
 
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        return f"WHEN {str(self.comparison)} THEN {str(self.expr)}"
+
     @property
     def concept_arguments(self):
         return get_concept_arguments(self.comparison) + get_concept_arguments(self.expr)
@@ -4323,6 +4430,7 @@ class ShowStatement(BaseModel):
 
 Expr = (
     bool
+    | MagicConstants
     | int
     | str
     | float
@@ -4377,9 +4485,34 @@ def dict_to_map_wrapper(arg):
     return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
 
 
+def merge_datatypes(
+    inputs: list[DataType | ListType | StructType | MapType | NumericType],
+) -> DataType | ListType | StructType | MapType | NumericType:
+    """This is a temporary hack for doing between
+    allowable datatype transformation matrix"""
+    if len(inputs) == 1:
+        return inputs[0]
+    if set(inputs) == {DataType.INTEGER, DataType.FLOAT}:
+        return DataType.FLOAT
+    if set(inputs) == {DataType.INTEGER, DataType.NUMERIC}:
+        return DataType.NUMERIC
+    if any(isinstance(x, NumericType) for x in inputs) and all(
+        isinstance(x, NumericType)
+        or x in (DataType.INTEGER, DataType.FLOAT, DataType.NUMERIC)
+        for x in inputs
+    ):
+        candidate = next(x for x in inputs if isinstance(x, NumericType))
+        return candidate
+    return inputs[0]
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
+    elif isinstance(arg, MagicConstants):
+        if arg == MagicConstants.NULL:
+            return DataType.NULL
+        raise ValueError(f"Cannot parse arg datatype for arg of type {arg}")
     elif isinstance(arg, Concept):
         return arg.datatype
     elif isinstance(arg, bool):

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/core/optimizations/predicate_pushdown.py

@@ -135,7 +135,7 @@ class PredicatePushdown(OptimizationRule):
                     f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
                 )
                 continue
-            self.log(
+            self.debug(
                 f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
             )
             for parent_cte in cte.parent_ctes:

{pytrilogy-0.0.2.12 → pytrilogy-0.0.2.14}/trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from trilogy.core.enums import SourceType
 from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
@@ -35,7 +36,11 @@ def gen_basic_node(
     attempts: List[tuple[list[Concept], list[Concept]]] = [
         (parent_concepts, [concept] + local_optional_redundant)
     ]
-    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if x.lineage == concept.lineage and x.address != concept.address
+    ]
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
@@ -61,8 +66,10 @@ def gen_basic_node(
             depth=depth + 1,
             history=history,
         )
+
         if not parent_node:
             continue
+        parent_node.source_type = SourceType.BASIC
         parents: List[StrategyNode] = [parent_node]
         for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]