pytrilogy 0.0.2.22__tar.gz → 0.0.2.25__tar.gz

{pytrilogy-0.0.2.22/pytrilogy.egg-info → pytrilogy-0.0.2.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.22
+Version: 0.0.2.25
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.22
+Version: 0.0.2.25
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

pytrilogy-0.0.2.25/tests/test_imports.py

@@ -0,0 +1,42 @@
+from trilogy import Environment
+from pathlib import Path
+
+
+def test_multi_environment():
+    basic = Environment()
+
+    basic.parse(
+        """
+const pi <- 3.14;
+                     
+
+""",
+        namespace="math",
+    )
+
+    basic.parse(
+        """
+                
+            select math.pi;
+                """
+    )
+
+    assert basic.concepts["math.pi"].name == "pi"
+
+
+def test_test_alias_free_import():
+    basic = Environment(working_path=Path(__file__).parent)
+
+    basic.parse(
+        """
+import test_env;
+
+key id2 int;
+
+
+""",
+    )
+
+    assert basic.concepts["id"].name == "id"
+    assert basic.concepts["id2"].name == "id2"
+    assert basic.concepts["id"].namespace == basic.concepts["id2"].namespace

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/tests/test_models.py

@@ -15,8 +15,11 @@ from trilogy.core.models import (
     Concept,
     AggregateWrapper,
     RowsetItem,
+    TupleWrapper,
+    DataType,
 )
 from trilogy import parse
+from copy import deepcopy
 
 
 def test_cte_merge(test_environment, test_environment_graph):
@@ -312,3 +315,9 @@ select avg_greater_ten;
     lineage = env.concepts["avg_greater_ten"].lineage
     assert isinstance(lineage, AggregateWrapper)
     assert isinstance(lineage.function.concept_arguments[0].lineage, RowsetItem)
+
+
+def test_tuple_clone():
+    x = TupleWrapper([1, 2, 3], type=DataType.INTEGER)
+    y = deepcopy(x)
+    assert y == x

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/tests/test_parsing.py

@@ -1,12 +1,12 @@
 from trilogy.core.enums import Purpose, ComparisonOperator
 from trilogy.core.models import (
     DataType,
-    Parenthetical,
     ProcessedQuery,
     ShowStatement,
     SelectStatement,
     Environment,
     Comparison,
+    TupleWrapper,
 )
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
 from trilogy.parsing.parse_engine import (
@@ -27,11 +27,11 @@ def test_in():
     right = query.where_clause.conditional.right
     assert isinstance(
         right,
-        Parenthetical,
+        TupleWrapper,
     ), type(right)
-    assert right.content[0] == 1
+    assert right[0] == 1
     rendered = BaseDialect().render_expr(right)
-    assert rendered.strip() == "( 1,2,3 )".strip()
+    assert rendered.strip() == "(1,2,3)".strip()
 
     _, parsed = parse_text(
         "const order_id <- 3; SELECT order_id  WHERE order_id IN (1,);"
@@ -40,11 +40,11 @@ def test_in():
     right = query.where_clause.conditional.right
     assert isinstance(
         right,
-        Parenthetical,
+        TupleWrapper,
     ), type(right)
-    assert right.content[0] == 1
+    assert right[0] == 1
     rendered = BaseDialect().render_expr(right)
-    assert rendered.strip() == "( 1 )".strip()
+    assert rendered.strip() == "(1)".strip()
 
 
 def test_not_in():
@@ -53,13 +53,10 @@ def test_not_in():
     )
     query: ProcessedQuery = parsed[-1]
     right = query.where_clause.conditional.right
-    assert isinstance(
-        right,
-        Parenthetical,
-    ), type(right)
-    assert right.content[0] == 1
+    assert isinstance(right, TupleWrapper), type(right)
+    assert right[0] == 1
     rendered = BaseDialect().render_expr(right)
-    assert rendered.strip() == "( 1,2,3 )".strip()
+    assert rendered.strip() == "(1,2,3)".strip()
 
 
 def test_is_not_null():
@@ -516,3 +513,24 @@ select filtered_test;
     results = Dialects.DUCK_DB.default_executor().generate_sql(text)[0]
 
     assert "filtered_test" in results, results
+
+
+def test_unnest_parsing():
+    x = """
+key scalar int;    
+property scalar.int_array list<int>;
+
+key split <- unnest(int_array);
+
+datasource avalues (
+    int_array: int_array,
+	scalar: scalar
+    ) 
+grain (scalar) 
+query '''(
+select [1,2,3,4] as int_array, 2 as scalar
+)''';
+"""
+
+    env, parsed = parse_text(x)
+    assert env.concepts["split"].datatype == DataType.INTEGER

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/tests/test_show.py

@@ -1,5 +1,4 @@
 from trilogy.core.models import ShowStatement
-from trilogy.core.query_processor import process_query
 from trilogy import Dialects
 from trilogy.parser import parse
 

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/tests/test_statements.py

@@ -1,4 +1,7 @@
 from trilogy.parser import parse
+from trilogy import Dialects
+from trilogy.core.models import ProcessedCopyStatement
+from pathlib import Path
 
 # from trilogy.compiler import compile
 
@@ -43,6 +46,24 @@ datasource posts (
     parse(text)
 
 
+def test_io_statement():
+    target = Path(__file__).parent / "test_io_statement.csv"
+    if target.exists():
+        target.unlink()
+    text = f"""const array <- [1,2,3,4];
+
+auto x <- unnest(array);
+
+copy into csv '{target}' from select x -> test;
+"""
+    exec = Dialects.DUCK_DB.default_executor()
+    results = exec.parse_text(text)
+    assert isinstance(results[-1], ProcessedCopyStatement)
+    for z in results:
+        exec.execute_query(z)
+    assert target.exists(), "csv file was not created"
+
+
 def test_datasource_where():
     text = """key user_id int metadata(description="the description");
 property user_id.display_name string metadata(description="The display name ");

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.22"
+__version__ = "0.0.2.25"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/enums.py

@@ -292,3 +292,13 @@ class SelectFiltering(Enum):
     NONE = "none"
     EXPLICIT = "explicit"  # the filtering contains only selected values
     IMPLICIT = "implicit"  # the filtering contains unselected values
+
+
+class IOType(Enum):
+    CSV = "csv"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return IOType(value.lower())
+        return super()._missing_(value)

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/env_processor.py

@@ -6,17 +6,20 @@ from trilogy.core.graph_models import (
 from trilogy.core.models import Environment, Concept, Datasource
 
 
-def add_concept(concept: Concept, g: ReferenceGraph):
+def add_concept(
+    concept: Concept, g: ReferenceGraph, concept_mapping: dict[str, Concept]
+):
     g.add_node(concept)
     # if we have sources, recursively add them
     node_name = concept_to_node(concept)
     if concept.concept_arguments:
         for source in concept.concept_arguments:
             generic = source.with_default_grain()
-            add_concept(generic, g)
+            add_concept(generic, g, concept_mapping)
 
             g.add_edge(generic, node_name)
-    for _, pseudonym in concept.pseudonyms.items():
+    for ps_address in concept.pseudonyms:
+        pseudonym = concept_mapping[ps_address]
         pseudonym = pseudonym.with_default_grain()
         pseudonym_node = concept_to_node(pseudonym)
         if (pseudonym_node, node_name) in g.edges and (
@@ -28,7 +31,7 @@ def add_concept(concept: Concept, g: ReferenceGraph):
             continue
         g.add_edge(pseudonym_node, node_name, pseudonym=True)
         g.add_edge(node_name, pseudonym_node, pseudonym=True)
-        add_concept(pseudonym, g)
+        add_concept(pseudonym, g, concept_mapping)
 
 
 def generate_adhoc_graph(
@@ -37,10 +40,11 @@ def generate_adhoc_graph(
     restrict_to_listed: bool = False,
 ) -> ReferenceGraph:
     g = ReferenceGraph()
+    concept_mapping = {x.address: x for x in concepts}
 
     # add all parsed concepts
     for concept in concepts:
-        add_concept(concept, g)
+        add_concept(concept, g, concept_mapping)
 
     for dataset in datasources:
         node = datasource_to_node(dataset)
@@ -66,5 +70,7 @@ def generate_graph(
 ) -> ReferenceGraph:
 
     return generate_adhoc_graph(
-        list(environment.concepts.values()), list(environment.datasources.values())
+        list(environment.concepts.values())
+        + list(environment.alias_origin_lookup.values()),
+        list(environment.datasources.values()),
     )

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/environment_helpers.py

@@ -191,4 +191,3 @@ def generate_related_concepts(
             environment.add_concept(auto, meta=meta)
             if isinstance(value, Concept):
                 environment.merge_concept(auto, value, modifiers=[])
-                assert value.pseudonyms is not None

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/functions.py

@@ -104,6 +104,8 @@ def Unnest(args: list[Concept]) -> Function:
     output = arg_to_datatype(args[0])
     if isinstance(output, (ListType)):
         output = output.value_data_type
+    else:
+        output = DataType.STRING
     return Function(
         operator=FunctionType.UNNEST,
         arguments=args,

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/models.py

@@ -65,6 +65,7 @@ from trilogy.core.enums import (
     ShowCategory,
     Granularity,
     SelectFiltering,
+    IOType,
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
@@ -81,6 +82,9 @@ LT = TypeVar("LT")
 
 
 def is_compatible_datatype(left, right):
+    # for unknown types, we can't make any assumptions
+    if right == DataType.UNKNOWN or left == DataType.UNKNOWN:
+        return True
     if left == right:
         return True
     if {left, right} == {DataType.NUMERIC, DataType.FLOAT}:
@@ -98,6 +102,12 @@ def get_version():
     return __version__
 
 
+def address_with_namespace(address: str, namespace: str) -> str:
+    if address.split(".", 1)[0] == DEFAULT_NAMESPACE:
+        return f"{namespace}.{address.split('.',1)[1]}"
+    return f"{namespace}.{address}"
+
+
 def get_concept_arguments(expr) -> List["Concept"]:
     output = []
     if isinstance(expr, Concept):
@@ -432,7 +442,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     keys: Optional[Tuple["Concept", ...]] = None
     grain: "Grain" = Field(default=None, validate_default=True)
     modifiers: Optional[List[Modifier]] = Field(default_factory=list)
-    pseudonyms: Dict[str, Concept] = Field(default_factory=dict)
+    pseudonyms: set[str] = Field(default_factory=set)
     _address_cache: str | None = None
 
     def __hash__(self):
@@ -458,7 +468,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
-            new.pseudonyms[self.address] = self
+            new.pseudonyms.add(self.address)
             return new
         return self.__class__(
             name=self.name,
@@ -612,9 +622,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
                 else None
             ),
             modifiers=self.modifiers,
-            pseudonyms={
-                k: v.with_namespace(namespace) for k, v in self.pseudonyms.items()
-            },
+            pseudonyms={address_with_namespace(v, namespace) for v in self.pseudonyms},
         )
 
     def with_select_context(
@@ -858,7 +866,7 @@ class Grain(Mergeable, BaseModel):
             )
         else:
             v2 = unique(v, "address")
-        final = []
+        final: List[Concept] = []
         for sub in v2:
             if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
                 if all([c in v2 for c in sub.keys]):
@@ -912,6 +920,20 @@ class Grain(Mergeable, BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
+    @property
+    def synonym_set(self) -> set[str]:
+        base = []
+        for x in self.components_copy:
+            if isinstance(x.lineage, RowsetItem):
+                base.append(x.lineage.content.address)
+                for c in x.lineage.content.pseudonyms:
+                    base.append(c)
+            else:
+                base.append(x.address)
+                for c in x.pseudonyms:
+                    base.append(c)
+        return set(base)
+
     @cached_property
     def set(self) -> set[str]:
         base = []
@@ -927,7 +949,11 @@ class Grain(Mergeable, BaseModel):
             return self.set == set([c.address for c in other])
         if not isinstance(other, Grain):
             return False
-        return self.set == other.set
+        if self.set == other.set:
+            return True
+        elif self.synonym_set == other.synonym_set:
+            return True
+        return False
 
     def issubset(self, other: "Grain"):
         return self.set.issubset(other.set)
@@ -1778,6 +1804,16 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
         )
 
 
+class CopyStatement(BaseModel):
+    target: str
+    target_type: IOType
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+    select: SelectStatement
+
+    def refresh_bindings(self, environment: Environment):
+        self.select.refresh_bindings(environment)
+
+
 class AlignItem(Namespaced, BaseModel):
     alias: str
     concepts: List[Concept]
@@ -2248,7 +2284,7 @@ class BaseJoin(BaseModel):
             for ds in [self.left_datasource, self.right_datasource]:
                 synonyms = []
                 for c in ds.output_concepts:
-                    synonyms += list(c.pseudonyms.keys())
+                    synonyms += list(c.pseudonyms)
                 if (
                     concept.address not in [c.address for c in ds.output_concepts]
                     and concept.address not in synonyms
@@ -2823,9 +2859,21 @@ class CTE(BaseModel):
             return self.parent_ctes[0].name
         return self.name
 
+    def get_concept(self, address: str) -> Concept | None:
+        for cte in self.parent_ctes:
+            if address in cte.output_columns:
+                match = [x for x in cte.output_columns if x.address == address].pop()
+                return match
+
+        for array in [self.source.input_concepts, self.source.output_concepts]:
+            match_list = [x for x in array if x.address == address]
+            if match_list:
+                return match_list.pop()
+        return None
+
     def get_alias(self, concept: Concept, source: str | None = None) -> str:
         for cte in self.parent_ctes:
-            if concept.address in [x.address for x in cte.output_columns]:
+            if concept.address in cte.output_columns:
                 if source and source != cte.name:
                     continue
                 return concept.safe_address
@@ -2977,7 +3025,7 @@ class UndefinedConcept(Concept, Mergeable, Namespaced):
     ) -> "UndefinedConcept" | Concept:
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
-            new.pseudonyms[self.address] = self
+            new.pseudonyms.add(self.address)
             return new
         return self.__class__(
             name=self.name,
@@ -3509,6 +3557,7 @@ class Environment(BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         replacements = {}
+
         # exit early if we've run this
         if source.address in self.alias_origin_lookup:
             if self.concepts[source.address] == target:
@@ -3517,11 +3566,11 @@ class Environment(BaseModel):
         for k, v in self.concepts.items():
 
             if v.address == target.address:
-                v.pseudonyms[source.address] = source
+                v.pseudonyms.add(source.address)
             if v.address == source.address:
                 replacements[k] = target
                 self.canonical_map[k] = target.address
-                v.pseudonyms[target.address] = target
+                v.pseudonyms.add(target.address)
             # we need to update keys and grains of all concepts
             else:
                 replacements[k] = v.with_merge(source, target, modifiers)
@@ -3599,6 +3648,7 @@ class Comparison(
         MagicConstants,
         WindowItem,
         AggregateWrapper,
+        TupleWrapper,
     ]
     operator: ComparisonOperator
 
@@ -4258,13 +4308,23 @@ class ProcessedQuery(BaseModel):
     order_by: Optional[OrderBy] = None
 
 
-class ProcessedQueryMixin(BaseModel):
+class PersistQueryMixin(BaseModel):
     output_to: MaterializedDataset
     datasource: Datasource
     # base:Dataset
 
 
-class ProcessedQueryPersist(ProcessedQuery, ProcessedQueryMixin):
+class ProcessedQueryPersist(ProcessedQuery, PersistQueryMixin):
+    pass
+
+
+class CopyQueryMixin(BaseModel):
+    target: str
+    target_type: IOType
+    # base:Dataset
+
+
+class ProcessedCopyStatement(ProcessedQuery, CopyQueryMixin):
     pass
 
 
@@ -4523,6 +4583,37 @@ class Parenthetical(
         return base
 
 
+class TupleWrapper(Generic[VT], tuple):
+    """Used to distinguish parsed tuple objects from other tuples"""
+
+    def __init__(self, val, type: DataType, **kwargs):
+        super().__init__()
+        self.type = type
+        self.val = val
+
+    def __getnewargs__(self):
+        return (self.val, self.type)
+
+    def __new__(cls, val, type: DataType, **kwargs):
+        return super().__new__(cls, tuple(val))
+        # self.type = type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Tuple[args])  # type: ignore
+        else:
+            schema = handler(Tuple)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(v, type=arg_to_datatype(v[0]))
+
+
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
@@ -4589,6 +4680,12 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def tuple_to_wrapper(args):
+    types = [arg_to_datatype(arg) for arg in args]
+    assert len(set(types)) == 1
+    return TupleWrapper(args, type=types[0])
+
+
 def dict_to_map_wrapper(arg):
     key_types = [arg_to_datatype(arg) for arg in arg.keys()]
 
@@ -4644,6 +4741,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
         return arg.function.output_datatype
     elif isinstance(arg, Parenthetical):
         return arg_to_datatype(arg.content)
+    elif isinstance(arg, TupleWrapper):
+        return ListType(type=arg.type)
     elif isinstance(arg, WindowItem):
         if arg.type in (WindowType.RANK, WindowType.ROW_NUMBER):
             return DataType.INTEGER

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/processing/concept_strategies_v3.py

@@ -455,7 +455,20 @@ def generate_node(
                         if x.address not in [y.address for y in root_targets]
                         and x not in ex_resolve.grain.components
                     ]
-                    expanded.set_output_concepts(root_targets)
+
+                    pseudonyms = [
+                        x
+                        for x in extra
+                        if any(x.address in y.pseudonyms for y in root_targets)
+                    ]
+                    # if we're only connected by a pseudonym, keep those in output
+                    expanded.set_output_concepts(root_targets + pseudonyms)
+                    # but hide them
+                    if pseudonyms:
+                        logger.info(
+                            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Hiding pseudonyms{[c.address for c in pseudonyms]}"
+                        )
+                        expanded.hide_output_concepts(pseudonyms)
 
                     logger.info(
                         f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in root_targets]} via concept addition; removing extra {[c.address for c in extra]}"
@@ -480,6 +493,7 @@ def validate_concept(
     found_map: dict[str, set[Concept]],
     accept_partial: bool,
     seen: set[str],
+    environment: Environment,
 ):
 
     found_map[str(node)].add(concept)
@@ -500,10 +514,11 @@ def validate_concept(
         if accept_partial:
             found_addresses.add(concept.address)
             found_map[str(node)].add(concept)
-    for _, v in concept.pseudonyms.items():
-        if v.address == concept.address:
+    for v_address in concept.pseudonyms:
+        v = environment.concepts[v_address]
+        if v == concept.address:
             return
-        if v.address in seen:
+        if v in seen:
             return
         validate_concept(
             v,
@@ -515,10 +530,12 @@ def validate_concept(
             found_map,
             accept_partial,
             seen=seen,
+            environment=environment,
         )
 
 
 def validate_stack(
+    environment: Environment,
     stack: List[StrategyNode],
     concepts: List[Concept],
     mandatory_with_filter: List[Concept],
@@ -546,6 +563,7 @@ def validate_stack(
                 found_map,
                 accept_partial,
                 seen,
+                environment,
             )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses:
@@ -807,6 +825,7 @@ def _search_concepts(
                 break
         attempted.add(priority_concept.address)
         complete, found, missing, partial, virtual = validate_stack(
+            environment,
             stack,
             mandatory_list,
             completion_mandatory,

{pytrilogy-0.0.2.22 → pytrilogy-0.0.2.25}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -209,9 +209,9 @@ def resolve_weak_components(
         for c in all_concepts
         if "__preql_internal" not in c.address
     ]
-    synonyms: list[Concept] = []
+    synonyms: set[str] = set()
     for x in all_concepts:
-        synonyms += x.pseudonyms.values()
+        synonyms = synonyms.union(x.pseudonyms)
     while break_flag is not True:
         count += 1
         if count > AMBIGUITY_CHECK_LIMIT:
@@ -385,9 +385,9 @@ def gen_merge_node(
     # one concept handling may need to be kicked to alias
     if len(all_concepts) == 1:
         concept = all_concepts[0]
-        for k, v in concept.pseudonyms.items():
+        for v in concept.pseudonyms:
             test = subgraphs_to_merge_node(
-                [[concept, v]],
+                [[concept, environment.alias_origin_lookup[v]]],
                 g=g,
                 all_concepts=[concept],
                 environment=environment,