pytrilogy 0.0.3.60__tar.gz → 0.0.3.63__tar.gz

{pytrilogy-0.0.3.60/pytrilogy.egg-info → pytrilogy-0.0.3.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.60
+Version: 0.0.3.63
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.60
+Version: 0.0.3.63
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/tests/test_imports.py

@@ -52,3 +52,23 @@ def test_import_concept_resolution():
     materialized = basic.materialize_for_select()
     assert "one.two.import_key" in materialized.materialized_concepts
     assert "two.two.import_key" in materialized.materialized_concepts
+
+
+def test_import_basics():
+    basic = Environment(working_path=Path(__file__).parent)
+
+    basic.parse(
+        """
+import test_env;
+
+key id2 int;
+
+
+""",
+    )
+
+    assert len(basic.imports["local"]) == 1, basic.imports
+    importz = basic.imports["local"][0]
+    assert importz.path == Path("test_env")
+    expected = Path(__file__).parent / "test_env.preql"
+    assert importz.input_path == expected

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/tests/test_models.py

@@ -2,7 +2,7 @@ from copy import deepcopy
 
 from pytest import raises
 
-from trilogy import parse
+from trilogy import Dialects, parse
 from trilogy.core.enums import (
     BooleanOperator,
     ComparisonOperator,
@@ -360,3 +360,81 @@ def test_parenthetical(test_environment: Environment):
 
     merged = x + x
     assert isinstance(merged, Conditional)
+
+
+def test_datasource_grain_application():
+    env, statements = parse(
+        """
+key x string;
+property x.part_1 string;
+property x.part_2 string;
+auto _x <- concat(x.part_1, '.', x.part_2);
+
+merge _x into x;
+
+datasource xes (
+    x:x,
+    )
+grain (x)
+address x_table;
+
+datasource parts (
+    part_1: part_1,
+    part_2:part_2,
+    concat(x.part_1, '.', x.part_2): x
+    )
+grain (part_1, part_2)
+address parts;
+
+
+
+"""
+    )
+
+    test_grain = Grain.from_concepts(
+        concepts=["x", "part_1", "part_2"], environment=env
+    )
+    assert test_grain.components == {
+        "local.x",
+    }
+
+
+def test_datasource_merge_generation():
+    env, statements = parse(
+        """
+key x string;
+property x.part_1 string;
+property x.part_2 string;
+auto _x <- concat(part_1, '.', part_2);
+
+merge _x into x;
+
+datasource xes (
+    x:x,
+    )
+grain (x)
+address x_table;
+
+datasource parts (
+    part_1: part_1,
+    part_2:part_2,
+    concat(part_1, '.', part_2): x
+    )
+grain (part_1, part_2)
+address parts;
+
+
+select x,
+    part_1,
+    part_2
+;
+
+
+"""
+    )
+
+    q = Dialects.DUCK_DB.default_executor(environment=env).generate_sql(statements[-1])[
+        0
+    ]
+
+    assert "JOIN" not in q, q

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/tests/test_parsing.py

@@ -426,6 +426,19 @@ property id.labels map<string, int>;
     assert env.concepts["labels"].datatype.value_type == DataType.INTEGER
 
 
+def test_map_concept_definition():
+    env, parsed = parse_text(
+        """
+key id int;
+property id.label string;
+key map_store map<id, string>;
+
+"""
+    )
+    assert env.concepts["map_store"].datatype.key_data_type == DataType.INTEGER
+    assert env.concepts["map_store"].datatype.value_data_type == DataType.STRING
+
+
 def test_map_string_access():
     env, parsed = parse_text(
         """

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.60"
+__version__ = "0.0.3.63"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/enums.py

@@ -9,6 +9,7 @@ class UnnestMode(Enum):
     CROSS_JOIN = "cross_join"
     CROSS_JOIN_UNNEST = "cross_join_unnest"
     CROSS_JOIN_ALIAS = "cross_join_alias"
+    PRESTO = "presto"
     SNOWFLAKE = "snowflake"
 
 

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/models/build.py

@@ -131,7 +131,7 @@ def concept_is_relevant(
     if (
         concept.purpose == Purpose.KEY
         and concept.keys
-        and all([c in others for c in concept.keys])
+        and all([c in others and c != concept.address for c in concept.keys])
     ):
         return False
     if concept.purpose in (Purpose.METRIC,):

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/models/core.py

@@ -175,13 +175,17 @@ class ListType(BaseModel):
 
 
 class MapType(BaseModel):
-    key_type: DataType
+    key_type: TYPEDEF_TYPES
     value_type: TYPEDEF_TYPES
 
     @field_validator("value_type", mode="plain")
     def validate_type(cls, v):
         return v
 
+    @field_validator("key_type", mode="plain")
+    def validate_key_type(cls, v):
+        return v
+
     @property
     def data_type(self):
         return DataType.MAP

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/models/environment.py

@@ -58,7 +58,9 @@ if TYPE_CHECKING:
 class Import:
     alias: str
     path: Path
-    input_path: str | None = None
+    input_path: Path | None = (
+        None  # filepath where the text came from (path is the import path, but may be resolved from a dictionary for some resolvers)
+    )
 
 
 class BaseImportResolver(BaseModel):

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/optimization.py

@@ -52,7 +52,7 @@ def reorder_ctes(
             return input
         return [mapping[x] for x in topological_order]
     except nx.NetworkXUnfeasible as e:
-        print(
+        logger.error(
             "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
         )
         raise e

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/processing/node_generators/basic_node.py

@@ -21,6 +21,8 @@ def is_equivalent_basic_function_lineage(
         y.lineage, BuildFunction
     ):
         return False
+    if x.lineage.operator == y.lineage.operator == FunctionType.ATTR_ACCESS:
+        return x.lineage.concept_arguments == y.lineage.concept_arguments
     if x.lineage.operator == y.lineage.operator:
         return True
     if (
@@ -50,14 +52,25 @@ def gen_basic_node(
     synonyms: list[BuildConcept] = []
     ignored_optional: set[str] = set()
     assert isinstance(concept.lineage, BuildFunction)
+    # when we are getting an attribute, if there is anything else
+    # that is an attribute of the same struct in local optional
+    # select that value for discovery as well
     if concept.lineage.operator == FunctionType.ATTR_ACCESS:
         logger.info(
             f"{depth_prefix}{LOGGER_PREFIX} checking for synonyms for attribute access"
         )
         for x in local_optional:
+            found = False
             for z in x.pseudonyms:
-                s_concept = environment.alias_origin_lookup[z]
+                # gate to ensure we don't match to multiple synonyms
+                if found:
+                    continue
+                if z in environment.concepts:
+                    s_concept = environment.concepts[z]
+                else:
+                    s_concept = environment.alias_origin_lookup[z]
                 if is_equivalent_basic_function_lineage(concept, s_concept):
+                    found = True
                     synonyms.append(s_concept)
                     ignored_optional.add(x.address)
     equivalent_optional = [
@@ -72,7 +85,11 @@ def gen_basic_node(
             f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
         )
     for eo in equivalent_optional:
-        parent_concepts += resolve_function_parent_concepts(eo, environment=environment)
+        new_parents = resolve_function_parent_concepts(eo, environment=environment)
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} equivalent optional {eo.address} has parents {[x.address for x in new_parents]}"
+        )
+        parent_concepts += new_parents
     non_equivalent_optional = [
         x
         for x in local_optional

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/processing/node_generators/synonym_node.py

@@ -43,16 +43,45 @@ def gen_synonym_node(
             elif y in environment.concepts:
                 synonyms[x.address].append(environment.concepts[y])
                 synonym_count += 1
+    for address in synonyms:
+        synonyms[address].sort(key=lambda obj: obj.address)
     if synonym_count == 0:
         return None
 
     logger.info(f"{local_prefix} Generating Synonym Node with {len(synonyms)} synonyms")
+    sorted_keys = sorted(synonyms.keys())
+    combinations_list = list(itertools.product(*(synonyms[obj] for obj in sorted_keys)))
 
-    combinations = itertools.product(*(synonyms[obj] for obj in synonyms.keys()))
-    for combo in combinations:
+    def similarity_sort_key(combo):
+        addresses = [x.address for x in combo]
+
+        # Calculate similarity score - count how many pairs share prefixes
+        similarity_score = 0
+        for i in range(len(addresses)):
+            for j in range(i + 1, len(addresses)):
+                # Find common prefix length
+                addr1_parts = addresses[i].split(".")
+                addr2_parts = addresses[j].split(".")
+                common_prefix_len = 0
+                for k in range(min(len(addr1_parts), len(addr2_parts))):
+                    if addr1_parts[k] == addr2_parts[k]:
+                        common_prefix_len += 1
+                    else:
+                        break
+                similarity_score += common_prefix_len
+
+        # Sort by similarity (descending), then by addresses (ascending) for ties
+        return (-similarity_score, addresses)
+
+    combinations_list.sort(key=similarity_sort_key)
+    logger.info(combinations_list)
+    for combo in combinations_list:
         fingerprint = tuple([x.address for x in combo])
         if fingerprint == base_fingerprint:
             continue
+        logger.info(
+            f"{local_prefix} checking combination {fingerprint} with {len(combo)} concepts"
+        )
         attempt: StrategyNode | None = source_concepts(
             combo,
             history=history,
@@ -64,6 +93,5 @@ def gen_synonym_node(
         )
         if attempt:
             logger.info(f"{local_prefix} found inputs with {combo}")
-            print(attempt.output_concepts)
             return attempt
     return None

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/core/processing/node_generators/unnest_node.py

@@ -31,7 +31,7 @@ def gen_unnest_node(
     ]
     all_parents = arguments + non_equivalent_optional
     logger.info(
-        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents {all_parents} and equivalent optional {equivalent_optional}"
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents + optional {all_parents} and equivalent optional {equivalent_optional}"
     )
     if arguments or local_optional:
         parent = source_concepts(

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/dialect/base.py

@@ -757,7 +757,7 @@ class BaseDialect:
                 f"{self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
                 for c in cte.join_derived_concepts
             ]
-        elif self.UNNEST_MODE == UnnestMode.CROSS_JOIN_UNNEST:
+        elif self.UNNEST_MODE in (UnnestMode.CROSS_JOIN_UNNEST, UnnestMode.PRESTO):
             select_columns = [
                 self.render_concept_sql(c, cte)
                 for c in cte.output_columns
@@ -787,12 +787,14 @@ class BaseDialect:
                 ):
 
                     source = f"{render_unnest(self.UNNEST_MODE, self.QUOTE_CHARACTER, cte.join_derived_concepts[0], self.render_expr, cte)}"
-                elif (
-                    cte.join_derived_concepts
-                    and self.UNNEST_MODE == UnnestMode.CROSS_JOIN_UNNEST
+                elif cte.join_derived_concepts and self.UNNEST_MODE in (
+                    UnnestMode.CROSS_JOIN_UNNEST,
                 ):
                     source = f"{self.render_expr(cte.join_derived_concepts[0], cte)} as {self.QUOTE_CHARACTER}{UNNEST_NAME}{self.QUOTE_CHARACTER}"
-
+                elif cte.join_derived_concepts and self.UNNEST_MODE in (
+                    UnnestMode.PRESTO,
+                ):
+                    source = f"{self.render_expr(cte.join_derived_concepts[0], cte)} as t({self.QUOTE_CHARACTER}{UNNEST_NAME}{self.QUOTE_CHARACTER})"
                 elif (
                     cte.join_derived_concepts
                     and self.UNNEST_MODE == UnnestMode.SNOWFLAKE

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/dialect/common.py

@@ -35,7 +35,6 @@ def render_unnest(
     cte: CTE,
 ):
     if not isinstance(concept, (BuildConcept, BuildParamaterizedConceptReference)):
-        print(type(concept))
         address = UNNEST_NAME
     else:
         address = concept.safe_address
@@ -43,6 +42,8 @@ def render_unnest(
         return f"{render_func(concept, cte)} as {quote_character}{address}{quote_character}"
     elif unnest_mode == UnnestMode.CROSS_JOIN_UNNEST:
         return f"unnest({render_func(concept, cte)}) as {quote_character}{address}{quote_character}"
+    elif unnest_mode == UnnestMode.PRESTO:
+        return f"unnest({render_func(concept, cte)}) as t({quote_character}{UNNEST_NAME}{quote_character})"
     elif unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
         return f"{render_func(concept, cte)} as unnest_wrapper ({quote_character}{address}{quote_character})"
     elif unnest_mode == UnnestMode.SNOWFLAKE:
@@ -103,6 +104,7 @@ def render_join(
             UnnestMode.CROSS_JOIN,
             UnnestMode.CROSS_JOIN_UNNEST,
             UnnestMode.CROSS_JOIN_ALIAS,
+            UnnestMode.PRESTO,
         ):
             return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.object_to_unnest, render_expr_func, cte)}"
         if unnest_mode == UnnestMode.SNOWFLAKE:

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/dialect/presto.py

@@ -33,6 +33,8 @@ FUNCTION_MAP = {
     FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
     FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
     FunctionType.ARRAY: lambda x: f"ARRAY[{', '.join(x)}]",
+    # regex
+    FunctionType.REGEXP_CONTAINS: lambda x: f"REGEXP_LIKE({x[0]}, {x[1]})",
 }
 
 FUNCTION_GRAIN_MATCH_MAP = {
@@ -85,8 +87,12 @@ class PrestoDialect(BaseDialect):
     }
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = SQL_TEMPLATE
-    DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, DataType.NUMERIC: "DECIMAL"}
-    UNNEST_MODE = UnnestMode.CROSS_JOIN_ALIAS
+    DATATYPE_MAP = {
+        **BaseDialect.DATATYPE_MAP,
+        DataType.NUMERIC: "DECIMAL",
+        DataType.STRING: "VARCHAR",
+    }
+    UNNEST_MODE = UnnestMode.PRESTO
 
 
 class TrinoDialect(PrestoDialect):

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/parsing/parse_engine.py

@@ -460,13 +460,22 @@ class ParseToObjects(Transformer):
         )
 
     def list_type(self, args) -> ListType:
-        return ListType(type=args[0])
+        content = args[0]
+        if isinstance(content, str):
+            content = self.environment.concepts[content]
+        return ListType(type=content)
 
     def numeric_type(self, args) -> NumericType:
         return NumericType(precision=args[0], scale=args[1])
 
     def map_type(self, args) -> MapType:
-        return MapType(key_type=args[0], value_type=args[1])
+        key = args[0]
+        value = args[1]
+        if isinstance(key, str):
+            key = self.environment.concepts[key]
+        elif isinstance(value, str):
+            value = self.environment.concepts[value]
+        return MapType(key_type=key, value_type=value)
 
     @v_args(meta=True)
     def data_type(
@@ -842,6 +851,17 @@ class ParseToObjects(Transformer):
                     continue
 
                 key_inputs = grain.components
+                eligible = True
+                for key in key_inputs:
+                    # never overwrite a key with a dependency on a property
+                    # for example - binding a datasource with a grain of <x>.fun should
+                    # never override the grain of x to <fun>
+                    if column.concept.address in (
+                        self.environment.concepts[key].keys or set()
+                    ):
+                        eligible = False
+                if not eligible:
+                    continue
                 keys = [self.environment.concepts[grain] for grain in key_inputs]
                 # target_c.purpose = Purpose.PROPERTY
                 target_c.keys = set([x.address for x in keys])
@@ -1030,6 +1050,9 @@ class ParseToObjects(Transformer):
 
     def import_statement(self, args: list[str]) -> ImportStatement:
         start = datetime.now()
+        is_file_resolver = isinstance(
+            self.environment.config.import_resolver, FileSystemImportResolver
+        )
         if len(args) == 2:
             alias = args[-1]
             cache_key = args[-1]
@@ -1043,9 +1066,7 @@ class ParseToObjects(Transformer):
             is_stdlib = True
             target = join(STDLIB_ROOT, *path) + ".preql"
             token_lookup: Path | str = Path(target)
-        elif isinstance(
-            self.environment.config.import_resolver, FileSystemImportResolver
-        ):
+        elif is_file_resolver:
             target = join(self.environment.working_path, *path) + ".preql"
             # tokens + text are cached by path
             token_lookup = Path(target)
@@ -1125,7 +1146,13 @@ class ParseToObjects(Transformer):
         imps = ImportStatement(alias=alias, input_path=input_path, path=parsed_path)
 
         self.environment.add_import(
-            alias, new_env, Import(alias=alias, path=parsed_path)
+            alias,
+            new_env,
+            Import(
+                alias=alias,
+                path=parsed_path,
+                input_path=Path(target) if is_file_resolver else None,
+            ),
         )
         end = datetime.now()
         perf_logger.debug(
@@ -1677,6 +1704,7 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def unnest(self, meta, args):
+
         return self.function_factory.create_function(args, FunctionType.UNNEST, meta)
 
     @v_args(meta=True)

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/parsing/render.py

@@ -31,6 +31,7 @@ from trilogy.core.models.core import (
     DataType,
     ListType,
     ListWrapper,
+    MapWrapper,
     NumericType,
     TraitDataType,
     TupleWrapper,
@@ -250,6 +251,17 @@ class Renderer:
     def _(self, arg: TupleWrapper):
         return "(" + ", ".join([self.to_string(x) for x in arg]) + ")"
 
+    @to_string.register
+    def _(self, arg: MapWrapper):
+        def process_key_value(key, value):
+            return f"{self.to_string(key)}: {self.to_string(value)}"
+
+        return (
+            "{"
+            + ", ".join([process_key_value(key, value) for key, value in arg.items()])
+            + "}"
+        )
+
     @to_string.register
     def _(self, arg: DatePart):
         return arg.value

{pytrilogy-0.0.3.60 → pytrilogy-0.0.3.63}/trilogy/parsing/trilogy.lark

@@ -402,11 +402,11 @@
 
     struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
 
-    list_type: ("list"i "<" data_type ">" ) | ("array"i "<" data_type ">" )
+    list_type: ("list"i | "array"i) "<" (data_type | IDENTIFIER) ">" 
 
     numeric_type: "numeric"i "(" int_lit "," int_lit ")"
 
-    map_type: "map"i "<" data_type "," data_type ">"
+    map_type: "map"i "<" (data_type | IDENTIFIER) "," (data_type | IDENTIFIER) ">"
 
     !data_type:  ("string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | map_type | struct_type | list_type) ("::" IDENTIFIER)?