pytrilogy 0.0.2.21__py3-none-any.whl → 0.0.2.23__py3-none-any.whl

{pytrilogy-0.0.2.21.dist-info → pytrilogy-0.0.2.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.21
+Version: 0.0.2.23
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -145,7 +145,7 @@ Run the following from the directory the file is in.
 trilogy run hello.trilogy duckdb
 ```
 
-![UI Preview](./hello_world.png)
+![UI Preview](hello-world.png)
 
 ## Backends
 
@@ -158,9 +158,9 @@ The current Trilogy implementation supports these backends:
 
 ## Basic Example - Python
 
-Trilogy can be run directly in python.
+Trilogy can be run directly in python through the core SDK. Trilogy code can be defined and parsed inline or parsed out of files.
 
-A bigquery example, similar to bigquery [the quickstart](https://cloud.google.com/bigquery/docs/quickstarts/query-public-dataset-console)
+A bigquery example, similar to bigquery [the quickstart](https://cloud.google.com/bigquery/docs/quickstarts/query-public-dataset-console).
 
 ```python
 
@@ -224,7 +224,7 @@ and second the dialect to run.
 To pass arguments to a backend, append additional --<option> flags after specifying the dialect.
 
 Example:
-`trilogy run key in int; datasource test_source ( i:in) grain(in) address test; select in;" duckdb --path <path/to/duckdb>`
+`trilogy run "key x int; datasource test_source ( i:x) grain(in) address test; select x;" duckdb --path <path/to/database>`
 
 ### Bigquery Args
 N/A, only supports default auth. In python you can pass in a custom client.
@@ -267,7 +267,7 @@ Clone repository and install requirements.txt and requirements-test.txt.
 Please open an issue first to discuss what you would like to change, and then create a PR against that issue.
 
 ## Similar in space
-Trilogy combines two aspects; a semantic layer and a query language. We've covered examples of both below:
+Trilogy combines two aspects; a semantic layer and a query language. Examples of both are linked below:
 
 Python "semantic layers" are tools for defining data access to a warehouse in a more abstract way.
 
@@ -284,20 +284,26 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### IMPORT
 
-`import <path> as <alias>;`
+`import [path] as [alias];`
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval | list<[type]> | map<[type], [type]> | struct<name:[type], name:[type]>`;
 
 Key:
-`key <name> <type>;`
+`key [name] [type];`
 
 Property:
-`property <key>.<name> <type>;`
+`property [key>].[name] [type];`
+`property x.y int;`
+or 
+`property <[key](,[key])?>.<name> [type];`
+`property <x,y>.z int;`
+
 
 Transformation:
-`auto <name> <- <expression>;`
+`auto [name] <- [expression];`
+`auto x <- y + 1;`
 
 #### DATASOURCE
 ```sql

{pytrilogy-0.0.2.21.dist-info → pytrilogy-0.0.2.23.dist-info}/RECORD

@@ -1,33 +1,33 @@
-trilogy/__init__.py,sha256=OCXbyl49qGgFMeTRpc5o4WNKO8b9oOs7BpkoN7kt4sg,291
+trilogy/__init__.py,sha256=ubM_nAcusmpKBFKEh_KcrJJbLAvprPMo6sNjwVkytbQ,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=pZkOneh_65f9Ua6NICu1bHAFAbmQxmiXRXS7tsmCWbQ,1235
+trilogy/constants.py,sha256=rHCe0Pe3LuB-VwCr2765QhzkUrTqZKEYPJ7rS0ykxYw,1273
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=An6YLpHQOt96E7ozRQhwZels2hMsDbh0WV767kKCGU0,11294
+trilogy/executor.py,sha256=b2pUL_Ha1H7pyhqssc2-hTd0OUO2KIcS0x6BLMPckZw,11822
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
-trilogy/core/enums.py,sha256=IHi-Z656detnp2MKMeCphsnVL7-uaogehulJT3JA64I,6087
-trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
+trilogy/core/enums.py,sha256=y0Z0m-xtcVw1ktkQ5yD3fJYWfOa4ncN_MzCTpREAxy0,6374
+trilogy/core/env_processor.py,sha256=z8pYgl5XpprA4ZzRvn7CVIG0hbMu04BlNkugKlT6i3o,2333
 trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
-trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
+trilogy/core/ergonomics.py,sha256=ASLDd0RqKWrZiG3XcKHo8nyTjaB_8xfE9t4NZ1UvGpc,1639
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=ShFTStIKbgI-3EZIU0xTumI78AC5QlvARwnBM53P2O0,10677
+trilogy/core/functions.py,sha256=IhVpt3n6wEanKHnGu3oA2w6-hKIlxWpEyz7fHN66mpo,10720
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=2YoDJz0cK5EljGtWCt1uFtcV56VWS9v8RfHw1nvaFrc,153416
+trilogy/core/models.py,sha256=Q3lhch_w1YNrBx-jl6ch5tJKFBYop42En5bghdCGWXw,155612
 trilogy/core/optimization.py,sha256=od_60A9F8J8Nj24MHgrxl4vwRwmBFH13TMdoMQvgVKs,7717
-trilogy/core/query_processor.py,sha256=D4-FKc-wP6c98-35DPcBE7-Gp6vbyj9DTZrubgtqACA,16561
+trilogy/core/query_processor.py,sha256=sdG0XcHNBS0kuqUPztDZ1i-kpDV5LJLrO55Og2Y8hSg,17140
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
 trilogy/core/optimizations/predicate_pushdown.py,sha256=1l9WnFOSv79e341typG3tTdk0XGl1J_ToQih3LYoGIY,8435
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=DO9gybVLku8GEkO3uNPaCeqhalnufsjYYbvDs-gkwNc,35295
+trilogy/core/processing/concept_strategies_v3.py,sha256=J4efhZCSGSo_CXVRObn7p3Lxr5Ry_G01265amsr2iIU,35294
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=v06sqXpnuYct_MMZXxEaiP0WwkeblWpO81QG1Ns3yGc,19420
+trilogy/core/processing/utility.py,sha256=KkbyMyDucbvK6YuLfUNVlDZ-1Adl7hthHsZAXeIbWm8,19466
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=WQNgJ1MwrMS_BQ-b3XwGGB6eToDykelAVj_fesJuqe0,2069
 trilogy/core/processing/node_generators/common.py,sha256=LwDgPlhWeuw0t07f3kX9IE5LXBdZhXfh-aY0XGk50ak,8946
@@ -35,8 +35,8 @@ trilogy/core/processing/node_generators/filter_node.py,sha256=Vz9Rb67e1dfZgnliek
 trilogy/core/processing/node_generators/group_node.py,sha256=r54IVEhXW-tzod6uEHIQObrxgQt6aNySk5emWkWyqCU,4938
 trilogy/core/processing/node_generators/group_to_node.py,sha256=R9i_wHipxjXJyfYEwfeTw2EPpuanXVA327XyfcP2tBg,2537
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=_KO9lqzHQoy4VAviO0ttQlmK0tjaqrJj4SJPhmoIYm8,6229
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=4aoSkynWYcKAxeN4fU5jnCdxausa5rNgFokoVhPXI80,13511
-trilogy/core/processing/node_generators/rowset_node.py,sha256=gU_ybfYXO9tZqHjUSABIioVpb8AWtITpegj3IGSf2GI,4587
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=ArwE7QoLgSoc2WUR_z9kyWJROxn9qq3KhMcdMZdH3k0,13977
+trilogy/core/processing/node_generators/rowset_node.py,sha256=KtdN6t2xM8CJxobc4aQX4W8uX98U6IabeuBF_FtBLR4,4583
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=MKjlXqFBSin6cTnS6n5lEcNBJsMvSefDIXOwYNVbM0s,10371
 trilogy/core/processing/node_generators/select_node.py,sha256=nwXHQF6C-aQUIelx9dyxN2pK3muL-4-6RIqnqQqNwtw,1808
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cZ26CN338CBnd6asML1OBUtNcDzmNlFpY0Vnade4yrc,2256
@@ -50,7 +50,7 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=gS9OQgS2TSEK59BQ9R0i83pTH
 trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=ro0QfMFi4ZmIn5Q4D0M_vJWfnHH_C0MN7XkVkx8Gygg,1214
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=QDeKbc7vgqfbR649-87cwsOz8G3VY63V19zH6I-WITo,33103
+trilogy/dialect/base.py,sha256=BDxL4eFEmkcT8Nj8W9P4anCYkAAvfsl9G01-NpI7r6w,33802
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=Hr0mxcNxjSvhpBM5Wvb_Q7aklAuYj5aBDrW433py0Zs,4403
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
@@ -66,18 +66,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=Pe-Kw1JGngeLqQOMQb0E3-24jXEavqnPCQ-KOfTfjP8,4357
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=-4LM71ocidA8DI2RngqFEOmhzBrIt8VdBTO4x2BpD8E,9502
+trilogy/parsing/common.py,sha256=kbqWy30nnVc7ID-sdSDwxYomnxd3guyuIJF3yvlpQwg,9960
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=_orHmZTdExJYkuNRmAaazdXpZcvtPMLKIBI9vBAgq6w,64871
-trilogy/parsing/render.py,sha256=8yxerPAi4AhlhPBlAfbYbOM3F9rz6HzpWVEWPtK2VEg,12321
-trilogy/parsing/trilogy.lark,sha256=0JAvQBACFNL-X61I0tB_0QPZgsguZgerfHBv903oKh0,11623
+trilogy/parsing/parse_engine.py,sha256=tcBgjfew0kAfSEt1aFo9Pu3yacEBB1KFm7v_Iobz52g,64467
+trilogy/parsing/render.py,sha256=7mEEe5DWVAafaGl__oQE7FPn_4QhcsGT2VVp-nk1Lr8,13078
+trilogy/parsing/trilogy.lark,sha256=ZP9USPgD8-Fq5UzIl4iGpAeGuh2JLGzSoYJhvEGOi2c,12188
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.21.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.21.dist-info/METADATA,sha256=zUKc5pQLdUDjCdk3crvgw6wTA_caPxs0qbHGdm_kD8E,8132
-pytrilogy-0.0.2.21.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-pytrilogy-0.0.2.21.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.21.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.21.dist-info/RECORD,,
+pytrilogy-0.0.2.23.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.23.dist-info/METADATA,sha256=w2tvs68fbIBkngB-SzrUElriNjj5eXa8PfsPniRU2vY,8403
+pytrilogy-0.0.2.23.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+pytrilogy-0.0.2.23.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.23.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.23.dist-info/RECORD,,

{pytrilogy-0.0.2.21.dist-info → pytrilogy-0.0.2.23.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.21"
+__version__ = "0.0.2.23"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -44,6 +44,7 @@ class Comments:
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
+    randomize_cte_names: bool = False
     validate_missing: bool = True
     comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)

trilogy/core/enums.py

@@ -12,6 +12,7 @@ class UnnestMode(Enum):
 
 class ConceptSource(Enum):
     MANUAL = "manual"
+    CTE = "cte"
     PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 
@@ -206,6 +207,8 @@ class FunctionClass(Enum):
         FunctionType.CURRENT_DATETIME,
     ]
 
+    ONE_TO_MANY = [FunctionType.UNNEST]
+
 
 class Boolean(Enum):
     TRUE = "true"
@@ -289,3 +292,13 @@ class SelectFiltering(Enum):
     NONE = "none"
     EXPLICIT = "explicit"  # the filtering contains only selected values
     IMPLICIT = "implicit"  # the filtering contains unselected values
+
+
+class IOType(Enum):
+    CSV = "csv"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return IOType(value.lower())
+        return super()._missing_(value)

trilogy/core/env_processor.py

@@ -10,9 +10,11 @@ def add_concept(concept: Concept, g: ReferenceGraph):
     g.add_node(concept)
     # if we have sources, recursively add them
     node_name = concept_to_node(concept)
-    if concept.sources:
-        for source in concept.sources:
+    if concept.concept_arguments:
+        for source in concept.concept_arguments:
             generic = source.with_default_grain()
+            add_concept(generic, g)
+
             g.add_edge(generic, node_name)
     for _, pseudonym in concept.pseudonyms.items():
         pseudonym = pseudonym.with_default_grain()

trilogy/core/ergonomics.py

@@ -1,3 +1,5 @@
+from trilogy.constants import CONFIG
+
 # source: https://github.com/aaronbassett/Pass-phrase
 CTE_NAMES = """quizzical
 highfalutin
@@ -103,8 +105,6 @@ mandrill
 marlin
 monitor
 ocelot
-osprey
-owl
 petrel
 python
 ray
@@ -132,7 +132,6 @@ cuckoo
 darter
 dove
 duck
-eagle
 falcon
 finch
 flamingo
@@ -184,4 +183,12 @@ warbler""".split(
     "\n"
 )
 
-CTE_NAMES = list(set(CTE_NAMES))
+
+def generate_cte_names():
+    if CONFIG.randomize_cte_names:
+        from random import shuffle
+
+        new = [*CTE_NAMES]
+        shuffle(new)
+        return new
+    return CTE_NAMES

trilogy/core/functions.py

@@ -104,6 +104,8 @@ def Unnest(args: list[Concept]) -> Function:
     output = arg_to_datatype(args[0])
     if isinstance(output, (ListType)):
         output = output.value_data_type
+    else:
+        output = DataType.STRING
     return Function(
         operator=FunctionType.UNNEST,
         arguments=args,

trilogy/core/models.py

@@ -65,6 +65,7 @@ from trilogy.core.enums import (
     ShowCategory,
     Granularity,
     SelectFiltering,
+    IOType,
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
@@ -81,6 +82,9 @@ LT = TypeVar("LT")
 
 
 def is_compatible_datatype(left, right):
+    # for unknown types, we can't make any assumptions
+    if right == DataType.UNKNOWN or left == DataType.UNKNOWN:
+        return True
     if left == right:
         return True
     if {left, right} == {DataType.NUMERIC, DataType.FLOAT}:
@@ -1580,6 +1584,13 @@ class RawSQLStatement(BaseModel):
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
 
+class CopyStatement(BaseModel):
+    target: str
+    target_type: IOType
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+    select: SelectStatement
+
+
 class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
     selection: List[SelectItem]
     order_by: Optional[OrderBy] = None
@@ -3478,10 +3489,18 @@ class Environment(BaseModel):
                     # to make this a root for discovery purposes
                     # as it now "exists" in a table
                     current_concept.lineage = None
+                    current_concept = current_concept.with_default_grain()
                     self.add_concept(
                         current_concept, meta=meta, force=True, _ignore_cache=True
                     )
                     self.merge_concept(new_concept, current_concept, [])
+                else:
+                    self.add_concept(current_concept, meta=meta, _ignore_cache=True)
+
+            # else:
+            #     self.add_concept(
+            #         current_concept, meta=meta, _ignore_cache=True
+            #     )
         if not _ignore_cache:
             self.gen_concept_list_caches()
         return datasource
@@ -3591,6 +3610,7 @@ class Comparison(
         MagicConstants,
         WindowItem,
         AggregateWrapper,
+        TupleWrapper,
     ]
     operator: ComparisonOperator
 
@@ -4250,13 +4270,23 @@ class ProcessedQuery(BaseModel):
     order_by: Optional[OrderBy] = None
 
 
-class ProcessedQueryMixin(BaseModel):
+class PersistQueryMixin(BaseModel):
     output_to: MaterializedDataset
     datasource: Datasource
     # base:Dataset
 
 
-class ProcessedQueryPersist(ProcessedQuery, ProcessedQueryMixin):
+class ProcessedQueryPersist(ProcessedQuery, PersistQueryMixin):
+    pass
+
+
+class CopyQueryMixin(BaseModel):
+    target: str
+    target_type: IOType
+    # base:Dataset
+
+
+class ProcessedCopyStatement(ProcessedQuery, CopyQueryMixin):
     pass
 
 
@@ -4289,6 +4319,9 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
     def __repr__(self):
         return f"RowsetDerivation<{str(self.select)}>"
 
+    def __str__(self):
+        return self.__repr__()
+
     @property
     def derived_concepts(self) -> List[Concept]:
         output: list[Concept] = []
@@ -4307,7 +4340,8 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
                     content=orig_concept, where=self.select.where_clause, rowset=self
                 ),
                 grain=orig_concept.grain,
-                metadata=orig_concept.metadata,
+                # TODO: add proper metadata
+                metadata=Metadata(concept_source=ConceptSource.CTE),
                 namespace=(
                     f"{self.name}.{orig_concept.namespace}"
                     if orig_concept.namespace != self.namespace
@@ -4334,6 +4368,7 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
                     components=[orig[c.address] for c in x.grain.components_copy]
                 )
             else:
+
                 x.grain = default_grain
         return output
 
@@ -4359,6 +4394,9 @@ class RowsetItem(Mergeable, Namespaced, BaseModel):
             f"<Rowset<{self.rowset.name}>: {str(self.content)} where {str(self.where)}>"
         )
 
+    def __str__(self):
+        return self.__repr__()
+
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         return RowsetItem(
             content=self.content.with_merge(source, target, modifiers),
@@ -4507,6 +4545,37 @@ class Parenthetical(
         return base
 
 
+class TupleWrapper(Generic[VT], tuple):
+    """Used to distinguish parsed tuple objects from other tuples"""
+
+    def __init__(self, val, type: DataType, **kwargs):
+        super().__init__()
+        self.type = type
+        self.val = val
+
+    def __getnewargs__(self):
+        return (self.val, self.type)
+
+    def __new__(cls, val, type: DataType, **kwargs):
+        return super().__new__(cls, tuple(val))
+        # self.type = type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Tuple[args])  # type: ignore
+        else:
+            schema = handler(Tuple)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(v, type=arg_to_datatype(v[0]))
+
+
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
@@ -4573,6 +4642,12 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def tuple_to_wrapper(args):
+    types = [arg_to_datatype(arg) for arg in args]
+    assert len(set(types)) == 1
+    return TupleWrapper(args, type=types[0])
+
+
 def dict_to_map_wrapper(arg):
     key_types = [arg_to_datatype(arg) for arg in arg.keys()]
 
@@ -4628,6 +4703,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
         return arg.function.output_datatype
     elif isinstance(arg, Parenthetical):
         return arg_to_datatype(arg.content)
+    elif isinstance(arg, TupleWrapper):
+        return ListType(type=arg.type)
     elif isinstance(arg, WindowItem):
         if arg.type in (WindowType.RANK, WindowType.ROW_NUMBER):
             return DataType.INTEGER

trilogy/core/processing/concept_strategies_v3.py

@@ -432,7 +432,6 @@ def generate_node(
             conditions=conditions,
         )
         if not check:
-
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Could not resolve root concepts, checking for expanded concepts"
             )

trilogy/core/processing/node_generators/node_merge_node.py

@@ -87,12 +87,10 @@ def determine_induced_minimal_nodes(
     for node in G.nodes:
         if concepts.get(node):
             lookup: Concept = concepts[node]
-            if lookup.derivation not in (PurposeLineage.BASIC, PurposeLineage.ROOT):
-                nodes_to_remove.append(node)
-            elif lookup.derivation == PurposeLineage.BASIC and G.out_degree(node) == 0:
+            if lookup.derivation in (PurposeLineage.CONSTANT,):
                 nodes_to_remove.append(node)
             # purge a node if we're already looking for all it's parents
-            elif filter_downstream and lookup.derivation == PurposeLineage.BASIC:
+            if filter_downstream and lookup.derivation not in (PurposeLineage.ROOT,):
                 nodes_to_remove.append(node)
 
     H.remove_nodes_from(nodes_to_remove)
@@ -105,11 +103,12 @@ def determine_induced_minimal_nodes(
         zero_out = list(
             x for x in H.nodes if G.out_degree(x) == 0 and x not in nodelist
         )
+
     try:
         paths = nx.multi_source_dijkstra_path(H, nodelist)
     except nx.exception.NodeNotFound:
+        logger.debug(f"Unable to find paths for {nodelist}")
         return None
-
     H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
     final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
@@ -126,12 +125,24 @@ def determine_induced_minimal_nodes(
     # all concept nodes must have a parent
 
     if not all(
-        [final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
+        [
+            final.in_degree(node) > 0
+            for node in final.nodes
+            if node.startswith("c~") and node in nodelist
+        ]
     ):
+        missing = [
+            node
+            for node in final.nodes
+            if node.startswith("c~") and final.in_degree(node) == 0
+        ]
+        logger.debug(f"Skipping graph for {nodelist} as no in_degree {missing}")
         return None
 
     if not all([node in final.nodes for node in nodelist]):
+        logger.debug(f"Skipping graph for {nodelist} as missing nodes")
         return None
+    logger.debug(f"Found final graph {final.nodes}")
     return final
 
 
@@ -256,7 +267,9 @@ def resolve_weak_components(
 
     subgraphs: list[list[Concept]] = []
     # components = nx.strongly_connected_components(g)
+    node_list = [x for x in g.nodes if x.startswith("c~")]
     components = extract_ds_components(g, node_list)
+    logger.debug(f"Extracted components {components} from {node_list}")
     for component in components:
         # we need to take unique again as different addresses may map to the same concept
         sub_component = unique(

trilogy/core/processing/node_generators/rowset_node.py

@@ -96,7 +96,7 @@ def gen_rowset_node(
         # node.set_preexisting_conditions(conditions.conditional if conditions else None)
         return node
 
-    possible_joins = concept_to_relevant_joins(additional_relevant)
+    possible_joins = concept_to_relevant_joins(node.output_concepts)
     if not possible_joins:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
@@ -104,7 +104,7 @@ def gen_rowset_node(
         return node
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
-        mandatory_list=additional_relevant + local_optional,
+        mandatory_list=possible_joins + local_optional,
         environment=environment,
         g=g,
         depth=depth + 1,

trilogy/core/processing/utility.py

@@ -28,6 +28,7 @@ from trilogy.core.models import (
     DatePart,
     NumericType,
     ListType,
+    TupleWrapper,
 )
 
 from trilogy.core.enums import Purpose, Granularity, BooleanOperator, Modifier
@@ -422,6 +423,7 @@ def is_scalar_condition(
         | NumericType
         | DatePart
         | ListWrapper[Any]
+        | TupleWrapper[Any]
     ),
     materialized: set[str] | None = None,
 ) -> bool:

trilogy/core/query_processor.py

@@ -26,13 +26,15 @@ from trilogy.core.models import (
     BaseJoin,
     InstantiatedUnnestJoin,
     Conditional,
+    ProcessedCopyStatement,
+    CopyStatement,
 )
 
 from trilogy.utility import unique
 
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.constants import logger
-from trilogy.core.ergonomics import CTE_NAMES
+from trilogy.core.ergonomics import generate_cte_names
 from trilogy.core.optimization import optimize_ctes
 from math import ceil
 from collections import defaultdict
@@ -169,15 +171,16 @@ def datasource_to_query_datasource(datasource: Datasource) -> QueryDatasource:
 
 
 def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
+    cte_names = generate_cte_names()
     if CONFIG.human_identifiers:
         if full_name in name_map:
             return name_map[full_name]
         suffix = ""
         idx = len(name_map)
-        if idx >= len(CTE_NAMES):
-            int = ceil(idx / len(CTE_NAMES))
+        if idx >= len(cte_names):
+            int = ceil(idx / len(cte_names))
             suffix = f"_{int}"
-        valid = [x for x in CTE_NAMES if x + suffix not in name_map.values()]
+        valid = [x for x in cte_names if x + suffix not in name_map.values()]
         lookup = valid[0]
         new_name = f"{lookup}{suffix}"
         name_map[full_name] = new_name
@@ -417,6 +420,24 @@ def process_persist(
     )
 
 
+def process_copy(
+    environment: Environment,
+    statement: CopyStatement,
+    hooks: List[BaseHook] | None = None,
+) -> ProcessedCopyStatement:
+    select = process_query(
+        environment=environment, statement=statement.select, hooks=hooks
+    )
+
+    # build our object to return
+    arg_dict = {k: v for k, v in select.__dict__.items()}
+    return ProcessedCopyStatement(
+        **arg_dict,
+        target=statement.target,
+        target_type=statement.target_type,
+    )
+
+
 def process_query(
     environment: Environment,
     statement: SelectStatement | MultiSelectStatement,

trilogy/dialect/base.py

@@ -35,6 +35,7 @@ from trilogy.core.models import (
     Environment,
     RawColumnExpr,
     ListWrapper,
+    TupleWrapper,
     MapWrapper,
     ShowStatement,
     RowsetItem,
@@ -48,8 +49,11 @@ from trilogy.core.models import (
     MapType,
     StructType,
     MergeStatementV2,
+    Datasource,
+    CopyStatement,
+    ProcessedCopyStatement,
 )
-from trilogy.core.query_processor import process_query, process_persist
+from trilogy.core.query_processor import process_query, process_persist, process_copy
 from trilogy.dialect.common import render_join, render_unnest
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.core.enums import UnnestMode
@@ -390,6 +394,7 @@ class BaseDialect:
             StructType,
             ListType,
             ListWrapper[Any],
+            TupleWrapper[Any],
             DatePart,
             CaseWhen,
             CaseElse,
@@ -429,7 +434,7 @@ class BaseDialect:
                         f"Missing source CTE for {e.right.address}"
                     )
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
-            elif isinstance(e.right, (ListWrapper, Parenthetical, list)):
+            elif isinstance(e.right, (ListWrapper, TupleWrapper, Parenthetical, list)):
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
 
             elif isinstance(
@@ -510,6 +515,8 @@ class BaseDialect:
             return str(e)
         elif isinstance(e, ListWrapper):
             return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])}]"
+        elif isinstance(e, TupleWrapper):
+            return f"({','.join([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])})"
         elif isinstance(e, MapWrapper):
             return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}:{self.render_expr(v, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}' for k, v in e.items()])}}}"
         elif isinstance(e, list):
@@ -661,6 +668,7 @@ class BaseDialect:
             | ImportStatement
             | RawSQLStatement
             | MergeStatementV2
+            | CopyStatement
         ],
         hooks: Optional[List[BaseHook]] = None,
     ) -> List[
@@ -674,6 +682,7 @@ class BaseDialect:
             | ProcessedQueryPersist
             | ProcessedShowStatement
             | ProcessedRawSQLStatement
+            | ProcessedCopyStatement
         ] = []
         for statement in statements:
             if isinstance(statement, PersistStatement):
@@ -682,6 +691,12 @@ class BaseDialect:
                         hook.process_persist_info(statement)
                 persist = process_persist(environment, statement, hooks=hooks)
                 output.append(persist)
+            elif isinstance(statement, CopyStatement):
+                if hooks:
+                    for hook in hooks:
+                        hook.process_select_info(statement.select)
+                copy = process_copy(environment, statement, hooks=hooks)
+                output.append(copy)
             elif isinstance(statement, SelectStatement):
                 if hooks:
                     for hook in hooks:
@@ -724,6 +739,7 @@ class BaseDialect:
                     MergeStatementV2,
                     ImportStatement,
                     RowsetDerivationStatement,
+                    Datasource,
                 ),
             ):
                 continue

trilogy/executor.py

@@ -10,6 +10,7 @@ from trilogy.core.models import (
     ProcessedShowStatement,
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
+    ProcessedCopyStatement,
     RawSQLStatement,
     MultiSelectStatement,
     SelectStatement,
@@ -18,12 +19,14 @@ from trilogy.core.models import (
     Concept,
     ConceptDeclarationStatement,
     Datasource,
+    CopyStatement,
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
+from trilogy.core.enums import IOType
 from trilogy.parser import parse_text
 from trilogy.hooks.base_hook import BaseHook
-
+from pathlib import Path
 from dataclasses import dataclass
 
 
@@ -94,7 +97,15 @@ class Executor(object):
         self.connection = self.engine.connect()
 
     def execute_statement(self, statement) -> Optional[CursorResult]:
-        if not isinstance(statement, (ProcessedQuery, ProcessedQueryPersist)):
+        if not isinstance(
+            statement,
+            (
+                ProcessedQuery,
+                ProcessedShowStatement,
+                ProcessedQueryPersist,
+                ProcessedCopyStatement,
+            ),
+        ):
             return None
         return self.execute_query(statement)
 
@@ -152,6 +163,13 @@ class Executor(object):
     def _(self, query: RawSQLStatement) -> CursorResult:
         return self.execute_raw_sql(query.text)
 
+    @execute_query.register
+    def _(self, query: ShowStatement) -> CursorResult:
+        sql = self.generator.generate_queries(
+            self.environment, [query], hooks=self.hooks
+        )
+        return self.execute_query(sql[0])
+
     @execute_query.register
     def _(self, query: ProcessedShowStatement) -> CursorResult:
         return generate_result_set(
@@ -176,12 +194,33 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ProcessedQueryPersist) -> CursorResult:
+
         sql = self.generator.compile_statement(query)
-        # connection = self.engine.connect()
+
         output = self.connection.execute(text(sql))
         self.environment.add_datasource(query.datasource)
         return output
 
+    @execute_query.register
+    def _(self, query: ProcessedCopyStatement) -> CursorResult:
+        sql = self.generator.compile_statement(query)
+        output: CursorResult = self.connection.execute(text(sql))
+        if query.target_type == IOType.CSV:
+            import csv
+
+            with open(query.target, "w", newline="", encoding="utf-8") as f:
+                outcsv = csv.writer(f)
+                outcsv.writerow(output.keys())
+                outcsv.writerows(output)
+        else:
+            raise NotImplementedError(f"Unsupported IOType {query.target_type}")
+        # now return the query we ran through IO
+        # TODO: instead return how many rows were written?
+        return generate_result_set(
+            query.output_columns,
+            [self.generator.compile_statement(query)],
+        )
+
     @singledispatchmethod
     def generate_sql(self, command) -> list[str]:
         raise NotImplementedError(
@@ -244,39 +283,17 @@ class Executor(object):
         | ProcessedQueryPersist
         | ProcessedShowStatement
         | ProcessedRawSQLStatement
+        | ProcessedCopyStatement
     ]:
-        """Process a preql text command"""
-        _, parsed = parse_text(command, self.environment)
-        generatable = [
-            x
-            for x in parsed
-            if isinstance(
-                x,
-                (
-                    SelectStatement,
-                    PersistStatement,
-                    MultiSelectStatement,
-                    ShowStatement,
-                    RawSQLStatement,
-                ),
-            )
-        ]
-        sql = []
-        while generatable:
-            t = generatable.pop(0)
-            x = self.generator.generate_queries(
-                self.environment, [t], hooks=self.hooks
-            )[0]
-            if persist and isinstance(x, ProcessedQueryPersist):
-                self.environment.add_datasource(x.datasource)
-            sql.append(x)
-        return sql
+
+        return list(self.parse_text_generator(command, persist=persist))
 
     def parse_text_generator(self, command: str, persist: bool = False) -> Generator[
         ProcessedQuery
         | ProcessedQueryPersist
         | ProcessedShowStatement
-        | ProcessedRawSQLStatement,
+        | ProcessedRawSQLStatement
+        | ProcessedCopyStatement,
         None,
         None,
     ]:
@@ -293,6 +310,7 @@ class Executor(object):
                     MultiSelectStatement,
                     ShowStatement,
                     RawSQLStatement,
+                    CopyStatement,
                 ),
             )
         ]
@@ -333,11 +351,11 @@ class Executor(object):
                     )
                 )
                 continue
-            compiled_sql = self.generator.compile_statement(statement)
-            logger.debug(compiled_sql)
-
-            output.append(self.connection.execute(text(compiled_sql)))
-            # generalize post-run success hooks
-            if isinstance(statement, ProcessedQueryPersist):
-                self.environment.add_datasource(statement.datasource)
+            output.append(self.execute_query(statement))
         return output
+
+    def execute_file(self, file: str | Path) -> List[CursorResult]:
+        file = Path(file)
+        with open(file, "r") as f:
+            command = f.read()
+        return self.execute_text(command)

trilogy/parsing/common.py

@@ -141,9 +141,12 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
             for x in parent.concept_arguments
             if not x.derivation == PurposeLineage.CONSTANT
         ]
-    grain = Grain()
+    grain: Grain | None = Grain()
     for x in pkeys:
         grain += x.grain
+    if parent.operator in FunctionClass.ONE_TO_MANY.value:
+        # if the function will create more rows, we don't know what grain this is at
+        grain = None
     modifiers = get_upstream_modifiers(pkeys)
     key_grain = []
     for x in pkeys:
@@ -156,13 +159,24 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         purpose = Purpose.CONSTANT
     else:
         purpose = parent.output_purpose
+    if grain is not None:
+        return Concept(
+            name=name,
+            datatype=parent.output_datatype,
+            purpose=purpose,
+            lineage=parent,
+            namespace=namespace,
+            keys=keys,
+            modifiers=modifiers,
+            grain=grain,
+        )
+
     return Concept(
         name=name,
         datatype=parent.output_datatype,
         purpose=purpose,
         lineage=parent,
         namespace=namespace,
-        grain=grain,
         keys=keys,
         modifiers=modifiers,
     )

trilogy/parsing/parse_engine.py

@@ -1,7 +1,7 @@
 from os.path import dirname, join
 from typing import List, Optional, Tuple, Union
 from re import IGNORECASE
-from lark import Lark, Transformer, v_args
+from lark import Lark, Transformer, v_args, Tree
 from lark.exceptions import (
     UnexpectedCharacters,
     UnexpectedEOF,
@@ -31,6 +31,7 @@ from trilogy.core.enums import (
     DatePart,
     ShowCategory,
     FunctionClass,
+    IOType,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
@@ -84,6 +85,7 @@ from trilogy.core.models import (
     PersistStatement,
     Query,
     RawSQLStatement,
+    CopyStatement,
     SelectStatement,
     SelectItem,
     WhereClause,
@@ -105,9 +107,11 @@ from trilogy.core.models import (
     ConceptDerivation,
     RowsetDerivationStatement,
     list_to_wrapper,
+    tuple_to_wrapper,
     dict_to_map_wrapper,
     NumericType,
     HavingClause,
+    TupleWrapper,
 )
 from trilogy.parsing.exceptions import ParseError
 from trilogy.parsing.common import (
@@ -461,40 +465,7 @@ class ParseToObjects(Transformer):
         while isinstance(source_value, Parenthetical):
             source_value = source_value.content
 
-        if (
-            isinstance(source_value, Function)
-            and source_value.operator == FunctionType.STRUCT
-        ):
-            concept = arbitrary_to_concept(
-                source_value,
-                name=name,
-                namespace=namespace,
-                purpose=purpose,
-                metadata=metadata,
-            )
-
-            if concept.metadata:
-                concept.metadata.line_number = meta.line
-            self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
-        elif (
-            isinstance(source_value, Function)
-            and source_value.operator == FunctionType.ALIAS
-        ):
-            concept = arbitrary_to_concept(
-                source_value,
-                name=name,
-                namespace=namespace,
-                purpose=purpose,
-                metadata=metadata,
-            )
-
-            if concept.metadata:
-                concept.metadata.line_number = meta.line
-            self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
-
-        elif isinstance(
+        if isinstance(
             source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
         ):
             concept = arbitrary_to_concept(
@@ -781,13 +752,29 @@ class ParseToObjects(Transformer):
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
         return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
 
+    def COPY_TYPE(self, args) -> IOType:
+        return IOType(args.value)
+
+    @v_args(meta=True)
+    def copy_statement(self, meta: Meta, args) -> CopyStatement:
+
+        return CopyStatement(
+            target=args[1],
+            target_type=args[0],
+            meta=Metadata(line_number=meta.line),
+            select=args[-1],
+        )
+
     def resolve_import_address(self, address) -> str:
         with open(address, "r", encoding="utf-8") as f:
             text = f.read()
         return text
 
     def import_statement(self, args: list[str]) -> ImportStatement:
-        alias = args[-1]
+        if len(args) == 2:
+            alias = args[-1]
+        else:
+            alias = self.environment.namespace
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
@@ -1097,6 +1084,9 @@ class ParseToObjects(Transformer):
     def array_lit(self, args):
         return list_to_wrapper(args)
 
+    def tuple_lit(self, args):
+        return tuple_to_wrapper(args)
+
     def struct_lit(self, args):
 
         zipped = dict(zip(args[::2], args[1::2]))
@@ -1157,12 +1147,18 @@ class ParseToObjects(Transformer):
 
         while isinstance(right, Parenthetical) and isinstance(
             right.content,
-            (Concept, Function, FilterItem, WindowItem, AggregateWrapper, ListWrapper),
+            (
+                Concept,
+                Function,
+                FilterItem,
+                WindowItem,
+                AggregateWrapper,
+                ListWrapper,
+                TupleWrapper,
+            ),
         ):
             right = right.content
-        if isinstance(
-            right, (Function, FilterItem, WindowItem, AggregateWrapper, ListWrapper)
-        ):
+        if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
             right = arbitrary_to_concept(
                 right,
                 namespace=self.environment.namespace,
@@ -1175,7 +1171,7 @@ class ParseToObjects(Transformer):
         )
 
     def expr_tuple(self, args):
-        return Parenthetical(content=args)
+        return TupleWrapper(content=tuple(args))
 
     def parenthetical(self, args):
         return Parenthetical(content=args[0])
@@ -1873,10 +1869,12 @@ def unpack_visit_error(e: VisitError):
         unpack_visit_error(e.orig_exc)
     elif isinstance(e.orig_exc, (UndefinedConceptException, ImportError)):
         raise e.orig_exc
-    elif isinstance(e.orig_exc, SyntaxError):
-        raise InvalidSyntaxException(str(e.orig_exc) + str(e.rule) + str(e.obj))
-    elif isinstance(e.orig_exc, (ValidationError, TypeError)):
-        raise InvalidSyntaxException(str(e.orig_exc) + str(e.rule) + str(e.obj))
+    elif isinstance(e.orig_exc, (SyntaxError, TypeError)):
+        if isinstance(e.obj, Tree):
+            raise InvalidSyntaxException(
+                str(e.orig_exc) + " in " + str(e.rule) + f" Line: {e.obj.meta.line}"
+            )
+        raise InvalidSyntaxException(str(e.orig_exc))
     raise e
 
 

trilogy/parsing/render.py

@@ -32,6 +32,8 @@ from trilogy.core.models import (
     AggregateWrapper,
     PersistStatement,
     ListWrapper,
+    ListType,
+    TupleWrapper,
     RowsetDerivationStatement,
     MultiSelectStatement,
     OrderBy,
@@ -40,6 +42,7 @@ from trilogy.core.models import (
     RawSQLStatement,
     NumericType,
     MergeStatementV2,
+    CopyStatement,
 )
 from trilogy.core.enums import Modifier
 
@@ -180,12 +183,18 @@ class Renderer:
     def _(self, arg: ListWrapper):
         return "[" + ", ".join([self.to_string(x) for x in arg]) + "]"
 
+    @to_string.register
+    def _(self, arg: TupleWrapper):
+        return "(" + ", ".join([self.to_string(x) for x in arg]) + ")"
+
     @to_string.register
     def _(self, arg: DatePart):
         return arg.value
 
     @to_string.register
     def _(self, arg: "Address"):
+        if arg.is_query:
+            return f"query '''{arg.location}'''"
         return f"address {arg.location}"
 
     @to_string.register
@@ -209,21 +218,30 @@ class Renderer:
             base_description = concept.metadata.description
         else:
             base_description = None
-        if concept.namespace:
+        if concept.namespace and concept.namespace != DEFAULT_NAMESPACE:
             namespace = f"{concept.namespace}."
         else:
             namespace = ""
         if not concept.lineage:
             if concept.purpose == Purpose.PROPERTY and concept.keys:
-                output = f"{concept.purpose.value} {namespace}{concept.keys[0].name}.{concept.name} {concept.datatype.value};"
+                keys = ",".join([self.to_string(key) for key in concept.keys])
+                output = f"{concept.purpose.value} <{keys}>.{namespace}{concept.name} {self.to_string(concept.datatype)};"
             else:
-                output = f"{concept.purpose.value} {namespace}{concept.name} {concept.datatype.value};"
+                output = f"{concept.purpose.value} {namespace}{concept.name} {self.to_string(concept.datatype)};"
         else:
             output = f"{concept.purpose.value} {namespace}{concept.name} <- {self.to_string(concept.lineage)};"
         if base_description:
             output += f" # {base_description}"
         return output
 
+    @to_string.register
+    def _(self, arg: ListType):
+        return f"list<{self.to_string(arg.value_data_type)}>"
+
+    @to_string.register
+    def _(self, arg: DataType):
+        return arg.value
+
     @to_string.register
     def _(self, arg: ConceptDerivation):
         # this is identical rendering;
@@ -269,6 +287,10 @@ class Renderer:
         base += "\n;"
         return base
 
+    @to_string.register
+    def _(self, arg: CopyStatement):
+        return f"COPY INTO {arg.target_type.value.upper()} '{arg.target}' FROM {self.to_string(arg.select)}"
+
     @to_string.register
     def _(self, arg: AlignClause):
         return "\nALIGN\n\t" + ",\n\t".join([self.to_string(c) for c in arg.items])

trilogy/parsing/trilogy.lark

@@ -8,7 +8,7 @@
     | persist_statement
     | rowset_derivation_statement
     | import_statement
-
+    | copy_statement
     | merge_statement_v2
     | rawsql_statement
     
@@ -57,7 +57,7 @@
     
     column_list : (IDENTIFIER "," )* IDENTIFIER ","?
     
-    import_statement: "import" (IDENTIFIER ".") * IDENTIFIER "as" IDENTIFIER
+    import_statement: "import" IDENTIFIER ("."  IDENTIFIER)* ("as" IDENTIFIER)?
 
     // persist_statement
     persist_statement: "persist"i IDENTIFIER "into"i IDENTIFIER "from"i select_statement grain_clause?
@@ -78,6 +78,12 @@
     // raw sql statement
     rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
 
+    // copy statement
+
+    COPY_TYPE: "csv"i
+
+    copy_statement: "copy"i "into"i COPY_TYPE _string_lit "from"i select_statement
+
     // FUNCTION blocks
     function: raw_function
     function_binding_item: IDENTIFIER ":" data_type
@@ -200,29 +206,42 @@
     _constant_functions: fcurrent_date | fcurrent_datetime
     
     //string
-    like: "like"i "(" expr "," _string_lit ")"
-    ilike: "ilike"i "(" expr "," _string_lit ")"
-    alt_like: expr "like"i expr
-    upper: "upper"i "(" expr ")"
-    lower: "lower"i "(" expr ")"    
-    fsplit: "split"i "(" expr "," _string_lit ")"
-    fstrpos: "strpos"i "(" expr "," expr ")"
+    _LIKE.1: "like("i
+    like:  _LIKE expr "," _string_lit ")"
+    _ILIKE.1: "ilike("i
+    ilike:   _ILIKE expr "," _string_lit ")"
+    alt_like: expr  "like"i expr
+    _UPPER.1: "upper("i
+    upper: _UPPER expr ")"
+   _LOWER.1: "lower("i
+    lower: _LOWER expr ")"  
+    _SPLIT.1: "split("i  
+    fsplit: _SPLIT expr "," _string_lit ")"
+   _STRPOS.1: "strpos("i
+    fstrpos: _STRPOS expr "," expr ")"
     _SUBSTRING.1: "substring("i
     fsubstring: _SUBSTRING expr "," expr "," expr ")"
     
     _string_functions: like | ilike | upper | lower | fsplit | fstrpos | fsubstring 
     
     // special aggregate
-    fgroup: "group"i "(" expr ")" aggregate_over?
+    _GROUP.1: "group("i
+    fgroup: _GROUP expr ")" aggregate_over?
     //aggregates
     _COUNT.1: "count("i
     count:   _COUNT  expr ")"
+    
     count_distinct: "count_distinct"i "(" expr ")"
     
-    sum: "sum"i "(" expr ")"
-    avg: "avg"i "(" expr ")"
-    max: "max"i "(" expr ")"
-    min: "min"i "(" expr ")"
+    // avoid conflicts with the window
+    _SUM.1: "sum("i
+    sum: _SUM expr ")"
+    _AVG.1: "avg("i
+    avg: _AVG expr ")"
+    _MAX.1: "max("i
+    max: _MAX expr ")"
+    _MIN.1: "min("i
+    min: _MIN expr ")"
     
     //aggregates can force a grain
     aggregate_all: "*"
@@ -255,10 +274,14 @@
     fyear: _YEAR expr ")"
     
     DATE_PART: "DAY"i | "WEEK"i | "MONTH"i | "QUARTER"i | "YEAR"i | "MINUTE"i | "HOUR"i | "SECOND"i
-    fdate_trunc: "date_trunc"i "(" expr "," DATE_PART ")"
-    fdate_part: "date_part"i "(" expr "," DATE_PART ")"
-    fdate_add: "date_add"i "(" expr "," DATE_PART "," int_lit ")"
-    fdate_diff: "date_diff"i "(" expr "," expr "," DATE_PART ")"
+    _DATE_TRUNC.1: "date_trunc("i
+    fdate_trunc: _DATE_TRUNC expr "," DATE_PART ")"
+    _DATE_PART.1: "date_part("i
+    fdate_part: _DATE_PART expr "," DATE_PART ")"
+    _DATE_ADD.1: "date_add("i
+    fdate_add: _DATE_ADD expr "," DATE_PART "," int_lit ")"
+   _DATE_DIFF.1: "date_diff("i
+    fdate_diff: _DATE_DIFF expr "," expr "," DATE_PART ")"
     
     _date_functions: fdate | fdate_add | fdate_diff | fdatetime | ftimestamp | fsecond | fminute | fhour | fday | fday_of_week | fweek | fmonth | fquarter | fyear | fdate_part | fdate_trunc
     
@@ -286,16 +309,18 @@
 
     array_lit: "[" (literal ",")* literal ","? "]"()
 
+    tuple_lit: "(" (literal ",")* literal ","? ")"
+
     map_lit: "{" (literal ":" literal ",")* literal ":" literal ","? "}"
 
-   
-    struct_lit: "struct" "(" (IDENTIFIER "=" literal ",")* IDENTIFIER "=" literal ","? ")"
+    _STRUCT.1: "struct("i
+    struct_lit: _STRUCT (IDENTIFIER "=" literal ",")* IDENTIFIER "=" literal ","? ")"
     
     !bool_lit: "True"i | "False"i
 
     !null_lit.1: "null"i
     
-    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit
+    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit | tuple_lit
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i