pytrilogy 0.0.2.2__tar.gz → 0.0.2.4__tar.gz

{pytrilogy-0.0.2.2/pytrilogy.egg-info → pytrilogy-0.0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.2
+Version: 0.0.2.4
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -247,7 +247,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -284,7 +284,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/README.md

@@ -218,7 +218,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -255,7 +255,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.2
+Version: 0.0.2.4
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -247,7 +247,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -284,7 +284,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/tests/test_parsing.py

@@ -16,6 +16,7 @@ from trilogy.parsing.parse_engine import (
 from trilogy.constants import MagicConstants
 from trilogy.dialect.base import BaseDialect
 from trilogy.core.enums import BooleanOperator
+from trilogy import Dialects
 
 
 def test_in():
@@ -314,3 +315,136 @@ select 1 as test;
 """
     )
     assert parsed[0].text == "select 1"
+
+
+def test_circular_aliasing():
+    from trilogy.hooks.query_debugger import DebuggingHook
+
+    executor = Dialects.DUCK_DB.default_executor(hooks=[DebuggingHook()])
+    test_case = """key composite_id string;
+
+property composite_id.first <- split(composite_id, '-')[1];
+property composite_id.second <- split(composite_id, '-')[2];
+
+key composite_id_alt <- concat(first, '-', second);
+
+merge composite_id_alt into composite_id;
+
+datasource random (
+    first:first,
+    second:second
+)
+grain (composite_id)
+query '''
+select '123' as first, 'abc' as second
+'''
+;
+
+datasource metrics (
+    composite_id: composite_id,
+)
+grain (composite_id)
+query '''
+select '123-abc' as composite_id
+'''
+;
+
+
+select first, second;
+
+
+"""
+    executor.parse_text(test_case)
+
+    results = executor.execute_text(test_case)[0].fetchall()
+
+    assert results == [("123", "abc")]
+
+
+def test_circular_aliasing_inverse():
+    from trilogy.hooks.query_debugger import DebuggingHook
+
+    executor = Dialects.DUCK_DB.default_executor(hooks=[DebuggingHook()])
+    test_case = """key composite_id string;
+
+property composite_id.first <- split(composite_id, '-')[1];
+property composite_id.second <- split(composite_id, '-')[2];
+
+key composite_id_alt <- concat(first, '-', second);
+
+merge composite_id_alt into composite_id;
+
+
+datasource metrics (
+    first:first,
+    second:second
+)
+grain (composite_id)
+query '''
+select '123' as first, 'abc' as second
+'''
+;
+
+
+select composite_id;
+
+
+"""
+    executor.parse_text(test_case)
+
+    results = executor.execute_text(test_case)[0].fetchall()
+
+    assert results == [("123-abc",)]
+
+
+def test_map_definition():
+    env, parsed = parse_text(
+        """
+key id int;
+property id.labels map<string, int>;
+
+"""
+    )
+    assert env.concepts["labels"].datatype.key_type == DataType.STRING
+    assert env.concepts["labels"].datatype.value_type == DataType.INTEGER
+
+
+def test_map_string_access():
+    env, parsed = parse_text(
+        """
+const labels <- { 'a': 1, 'b': 2, 'c': 3 };
+
+
+select
+    labels['a'] as label_a,
+;
+
+"""
+    )
+    assert env.concepts["labels"].datatype.key_type == DataType.STRING
+    assert env.concepts["labels"].datatype.value_type == DataType.INTEGER
+
+
+def test_struct_attr_access():
+
+    text = """
+const labels <- struct(a=1, b=2, c=3);
+
+
+select
+    labels,
+    labels.a as label_a
+;
+
+"""
+    env, parsed = parse_text(text)
+    assert env.concepts["labels"].datatype.fields_map["a"] == 1
+
+    assert env.concepts["labels.a"].concept_arguments[0].name == "labels"
+
+    results = Dialects.DUCK_DB.default_executor().execute_text(text)[0]
+
+    assert results.fetchall()[0] == (
+        {"a": 1, "b": 2, "c": 3},
+        1,
+    )

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/tests/test_where_clause.py

@@ -3,6 +3,7 @@ from trilogy.core.models import SelectStatement, Grain, Parenthetical
 from trilogy.core.query_processor import process_query
 from trilogy.dialect.base import BaseDialect
 from trilogy.parser import parse
+from trilogy.core.processing.utility import is_scalar_condition
 
 
 def test_select_where(test_environment):
@@ -44,6 +45,8 @@ where
 
 
 def test_select_where_agg(test_environment):
+    from trilogy.hooks.query_debugger import DebuggingHook
+
     declarations = """
 property my_favorite_order_revenue <- filter revenue where order_id in (1,2,3);
 
@@ -54,10 +57,15 @@ select
 
 
     """
-    env, parsed = parse(declarations, environment=test_environment)
+    env, parsed = parse(
+        declarations,
+        environment=test_environment,
+    )
     select: SelectStatement = parsed[-1]
 
-    BaseDialect().compile_statement(process_query(test_environment, select))
+    BaseDialect().compile_statement(
+        process_query(test_environment, select, hooks=[DebuggingHook()])
+    )
 
 
 def test_select_where_joins(test_environment):
@@ -205,3 +213,19 @@ where
 
     """
     env, parsed = parse(declarations, environment=test_environment)
+
+
+def test_where_scalar(test_environment):
+    declarations = """
+select
+    category_name
+where
+    count(order_id) > 1
+;
+"""
+    env, parsed = parse(declarations, environment=test_environment)
+    select: SelectStatement = parsed[-1]
+
+    assert is_scalar_condition(select.where_clause.conditional) is False
+    query = BaseDialect().compile_statement(process_query(test_environment, select))
+    print(query)

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.2"
+__version__ = "0.0.2.4"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/core/enums.py

@@ -119,7 +119,9 @@ class FunctionType(Enum):
 
     # COMPLEX
     INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/core/env_processor.py

@@ -17,6 +17,11 @@ def add_concept(concept: Concept, g: ReferenceGraph):
     for _, pseudonym in concept.pseudonyms.items():
         pseudonym = pseudonym.with_default_grain()
         pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
         g.add_edge(pseudonym_node, node_name, pseudonym=True)

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/core/functions.py

@@ -202,15 +202,21 @@ def Split(args: list[Concept]) -> Function:
     )
 
 
+def get_index_output_type(
+    arg: Concept,
+) -> DataType | StructType | MapType | ListType | NumericType:
+    if isinstance(arg.datatype, ListType):
+        return arg.datatype.value_data_type
+    elif isinstance(arg.datatype, MapType):
+        return arg.datatype.value_data_type
+    return arg.datatype
+
+
 def IndexAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.INDEX_ACCESS,
         arguments=args,
-        output_datatype=(
-            args[0].datatype.value_data_type
-            if isinstance(args[0].datatype, ListType)
-            else args[0].datatype
-        ),
+        output_datatype=get_index_output_type(args[0]),
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {
@@ -226,11 +232,38 @@ def IndexAccess(args: list[Concept]):
     )
 
 
+def MapAccess(args: list[Concept]):
+    return Function(
+        operator=FunctionType.MAP_ACCESS,
+        arguments=args,
+        output_datatype=get_index_output_type(args[0]),
+        output_purpose=Purpose.PROPERTY,
+        valid_inputs=[
+            {
+                DataType.MAP,
+            },
+            {
+                DataType.INTEGER,
+                DataType.STRING,
+            },
+        ],
+        arg_count=2,
+    )
+
+
+def get_attr_datatype(
+    arg: Concept, lookup
+) -> DataType | ListType | StructType | MapType | NumericType:
+    if isinstance(arg.datatype, StructType):
+        return arg_to_datatype(arg.datatype.fields_map[lookup])
+    return arg.datatype
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,
         arguments=args,
-        output_datatype=args[0].field_map[args[1]].datatype,  # type: ignore
+        output_datatype=get_attr_datatype(args[0], args[1]),  # type: ignore
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {DataType.STRUCT},

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/core/models.py

@@ -67,7 +67,7 @@ from trilogy.core.enums import (
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
-from collections import UserList
+from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 
@@ -267,7 +267,7 @@ class ListType(BaseModel):
 
 class MapType(BaseModel):
     key_type: DataType
-    content_type: ALL_TYPES
+    value_type: ALL_TYPES
 
     @property
     def data_type(self):
@@ -277,10 +277,26 @@ class MapType(BaseModel):
     def value(self):
         return self.data_type.value
 
+    @property
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.value_type, Concept):
+            return self.value_type.datatype
+        return self.value_type
+
+    @property
+    def key_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.key_type, Concept):
+            return self.key_type.datatype
+        return self.key_type
+
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
 
     @property
     def data_type(self):
@@ -314,6 +330,34 @@ class ListWrapper(Generic[VT], UserList):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
+class MapWrapper(Generic[KT, VT], UserDict):
+    """Used to distinguish parsed map objects from other dicts"""
+
+    def __init__(self, *args, key_type: DataType, value_type: DataType, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.key_type = key_type
+        self.value_type = value_type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Dict[args])  # type: ignore
+        else:
+            schema = handler(Dict)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(
+            v,
+            key_type=arg_to_datatype(list(v.keys()).pop()),
+            value_type=arg_to_datatype(list(v.values()).pop()),
+        )
+
+
 class Metadata(BaseModel):
     """Metadata container object.
     TODO: support arbitrary tags"""
@@ -375,6 +419,23 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def __hash__(self):
         return hash(str(self))
 
+    @property
+    def is_aggregate(self):
+        if (
+            self.lineage
+            and isinstance(self.lineage, Function)
+            and self.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        if (
+            self.lineage
+            and isinstance(self.lineage, AggregateWrapper)
+            and self.lineage.function.operator
+            in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        return False
+
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
@@ -949,8 +1010,8 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType | NumericType],
-            List[Set[DataType | ListType | StructType] | NumericType],
+            Set[DataType | ListType | StructType | MapType | NumericType],
+            List[Set[DataType | ListType | StructType | MapType | NumericType]],
         ]
     ] = None
     arguments: Sequence[
@@ -961,17 +1022,17 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
             int,
             float,
             str,
+            MapWrapper[Any, Any],
             DataType,
             ListType,
+            MapType,
             NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
             "CaseElse",
             list,
-            ListWrapper[int],
-            ListWrapper[str],
-            ListWrapper[float],
+            ListWrapper[Any],
         ]
     ]
 
@@ -2342,7 +2403,6 @@ class CTE(BaseModel):
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
-    requires_nesting: bool = True
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
 
@@ -3141,6 +3201,7 @@ class Environment(BaseModel):
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                v.pseudonyms[target.address] = target
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():
@@ -4115,6 +4176,15 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def dict_to_map_wrapper(arg):
+    key_types = [arg_to_datatype(arg) for arg in arg.keys()]
+
+    value_types = [arg_to_datatype(arg) for arg in arg.values()]
+    assert len(set(key_types)) == 1
+    assert len(set(key_types)) == 1
+    return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
@@ -4143,5 +4213,7 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
     elif isinstance(arg, list):
         wrapper = list_to_wrapper(arg)
         return ListType(type=wrapper.type)
+    elif isinstance(arg, MapWrapper):
+        return MapType(key_type=arg.key_type, value_type=arg.value_type)
     else:
         raise ValueError(f"Cannot parse arg datatype for arg of raw type {type(arg)}")

{pytrilogy-0.0.2.2 → pytrilogy-0.0.2.4}/trilogy/core/optimization.py

@@ -1,10 +1,10 @@
 from trilogy.core.models import (
     CTE,
     SelectStatement,
-    PersistStatement,
     MultiSelectStatement,
+    Conditional,
 )
-from trilogy.core.enums import PurposeLineage
+from trilogy.core.enums import PurposeLineage, BooleanOperator
 from trilogy.constants import logger, CONFIG
 from trilogy.core.optimizations import (
     OptimizationRule,
@@ -42,34 +42,45 @@ def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
     return inverse_map
 
 
-def is_direct_return_eligible(
-    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
-) -> bool:
-    if isinstance(select, (PersistStatement, MultiSelectStatement)):
-        return False
+def is_direct_return_eligible(cte: CTE) -> CTE | None:
+    # if isinstance(select, (PersistStatement, MultiSelectStatement)):
+    #     return False
+    if len(cte.parent_ctes) != 1:
+        return None
+    direct_parent = cte.parent_ctes[0]
+
+    output_addresses = set([x.address for x in cte.output_columns])
+    parent_output_addresses = set([x.address for x in direct_parent.output_columns])
+    if not output_addresses.issubset(parent_output_addresses):
+        return None
+    if not direct_parent.grain == cte.grain:
+        return None
     derived_concepts = [
         c
         for c in cte.source.output_concepts + cte.source.hidden_concepts
         if c not in cte.source.input_concepts
     ]
-    eligible = True
     conditions = (
-        set(x.address for x in select.where_clause.concept_arguments)
-        if select.where_clause
+        set(x.address for x in direct_parent.condition.concept_arguments)
+        if direct_parent.condition
         else set()
     )
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
-            return False
+            return None
         if x.derivation == PurposeLineage.UNNEST:
-            return False
+            return None
         if x.derivation == PurposeLineage.AGGREGATE:
             if x.address in conditions:
-                return False
+                return None
+    # handling top level nodes that require unpacking
+    for x in cte.output_columns:
+        if x.derivation == PurposeLineage.UNNEST:
+            return None
     logger.info(
-        f"[Optimization][EarlyReturn] Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
-    return eligible
+    return direct_parent
 
 
 def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
@@ -90,23 +101,27 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:
 
-    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
-        root_cte, select
+    direct_parent: CTE | None = root_cte
+    while CONFIG.optimizations.direct_return and (
+        direct_parent := is_direct_return_eligible(root_cte)
     ):
-        root_cte.order_by = select.order_by
-        root_cte.limit = select.limit
-        # if select.where_clause:
-
-        #     if root_cte.condition:
-        #         root_cte.condition = Conditional(
-        #             left=root_cte.condition,
-        #             operator=BooleanOperator.AND,
-        #             right=select.where_clause.conditional,
-        #         )
-        #     else:
-        #         root_cte.condition = select.where_clause.conditional
-        root_cte.requires_nesting = False
-        sort_select_output(root_cte, select)
+        direct_parent.order_by = root_cte.order_by
+        direct_parent.limit = root_cte.limit
+        direct_parent.hidden_concepts = (
+            root_cte.hidden_concepts + direct_parent.hidden_concepts
+        )
+        if root_cte.condition:
+            if direct_parent.condition:
+                direct_parent.condition = Conditional(
+                    left=direct_parent.condition,
+                    operator=BooleanOperator.AND,
+                    right=root_cte.condition,
+                )
+            else:
+                direct_parent.condition = root_cte.condition
+        root_cte = direct_parent
+
+    sort_select_output(root_cte, select)
 
     REGISTERED_RULES: list["OptimizationRule"] = []
     if CONFIG.optimizations.constant_inlining: