pytrilogy 0.0.2.31__py3-none-any.whl → 0.0.2.33__py3-none-any.whl

{pytrilogy-0.0.2.31.dist-info → pytrilogy-0.0.2.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.31
+Version: 0.0.2.33
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.31.dist-info → pytrilogy-0.0.2.33.dist-info}/RECORD

@@ -1,6 +1,6 @@
-trilogy/__init__.py,sha256=VblpsLULNBnUhw05evBTHbgeqv65Ler6brQxBYz86lM,291
+trilogy/__init__.py,sha256=a83gD8nszaMaJ-H2JcfjmxDVs4wHl6s8zGL9z5zyk_Y,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=KiyYnctoZen4Hzv8WG2jeN-IE-dfQbWHdVCUeTZYjBg,1270
+trilogy/constants.py,sha256=HQAnGUqJ5uMri7TWtqXHhz8iVWBzi2LCfRG8vKnBIB8,1269
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
 trilogy/executor.py,sha256=VcZ2U3RUU2al_VJ75AKVwmCJQLltYouxlgTjq4oxPB0,12577
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
@@ -8,7 +8,7 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
-trilogy/core/enums.py,sha256=y0Z0m-xtcVw1ktkQ5yD3fJYWfOa4ncN_MzCTpREAxy0,6374
+trilogy/core/enums.py,sha256=GG51K-bP3HTVuvJQFCjbmVt7iBOGpmF_qtnRVf0_obI,6396
 trilogy/core/env_processor.py,sha256=SHVB3nkidIlFc5dz-sofRMKXx66stpLQNuVdQSjC-So,2586
 trilogy/core/environment_helpers.py,sha256=DIsoo-GcXmXVPB1JbNh8Oku25Nyef9mexPIdy2ur_sk,7159
 trilogy/core/ergonomics.py,sha256=ASLDd0RqKWrZiG3XcKHo8nyTjaB_8xfE9t4NZ1UvGpc,1639
@@ -16,7 +16,7 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=IhVpt3n6wEanKHnGu3oA2w6-hKIlxWpEyz7fHN66mpo,10720
 trilogy/core/graph_models.py,sha256=mameUTiuCajtihDw_2-W218xyJlvTusOWrEKP1yAWgk,2003
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=d0nmbSrR7LObYOpfx9Q5GYrXg-hV0OrBDwrIj90eGX8,159625
+trilogy/core/models.py,sha256=vuy6sGOZ4jI_96wf6bqa7M8CyuD79fKd5lYWkhE0Z98,160393
 trilogy/core/optimization.py,sha256=VFSvJLNoCCOraip-PZUKeE4qrlxtXARjQUzJZiW-yRk,7325
 trilogy/core/query_processor.py,sha256=mbcZlgjChrRjDHkdmMbKe-T70UpbBkJhS09MyU5a6UY,17785
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
@@ -27,7 +27,7 @@ trilogy/core/optimizations/predicate_pushdown.py,sha256=1l9WnFOSv79e341typG3tTdk
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=Qy3WHiKaF371dDmbA0DWVQqSMKptH_gK714Uc41wbqc,36132
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=VFArfoUY5EiacspwQ03uWiKRr5SzEDhIB_iMrOIPBAg,18540
+trilogy/core/processing/utility.py,sha256=_sWLDuV8WAEWdEIcAPM8-FcXgBai30UV12i4O9u5Mpc,18680
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=WQNgJ1MwrMS_BQ-b3XwGGB6eToDykelAVj_fesJuqe0,2069
 trilogy/core/processing/node_generators/common.py,sha256=eslHTTPFTkmwHwKIuUsbFn54jxj-Avtt-QScqtNwzdg,8945
@@ -50,9 +50,9 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=7WoFxeGEAzhpS4y4Zw2nH2tt7
 trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=ro0QfMFi4ZmIn5Q4D0M_vJWfnHH_C0MN7XkVkx8Gygg,1214
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=VqBnDwPRzJkb1HPIlZ2X4lLsZzYrgJIOqr8x0UQuESg,34226
+trilogy/dialect/base.py,sha256=V3YSzuyom8PLRS5Qru4iGatlB0huNLKorPKsYoAVWLk,34308
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
-trilogy/dialect/common.py,sha256=Pm9naWL5eo-BfZTYFMWhX9lpsGuxEyvvn_EuXbFeos0,3817
+trilogy/dialect/common.py,sha256=eqJi_Si1iyb2sV0siTf8g8JOHueWu6RkdtQZtutKazk,3826
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=_0a5HBU8zRNtZj7YED3ju4fHXRYG9jNeKwnlZwUDvwI,3419
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
@@ -66,18 +66,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=787umJjdGA057DCC714dqFstzJRUbwmz3MNr66IdpQI,4404
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=t7yiL_3f6rz_rouF9et84v5orAgs-EprV4V9ghQ6ql4,10024
+trilogy/parsing/common.py,sha256=_GW9LU6_4RuUgcdcr8EE1ybCRd-7cz3idZtjHZ66pYA,10182
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=goN7oQqvSjU3Vqm4lXs4bNVAoHEMT-JkL_o_fhYy9vo,64591
+trilogy/parsing/parse_engine.py,sha256=N6HnbAru28-uOKrLPKoZnWU7sTboFH8BKFx_TmFFJHA,66519
 trilogy/parsing/render.py,sha256=VKyo8wEOuiOzUtJ6w9EoGGmkhlqDQyy8wFj_Q_h6EfE,15263
-trilogy/parsing/trilogy.lark,sha256=_z5px2N-e8oLUf7SpPMXXNqbAykDkZOvP4_lPgf5-Uk,12245
+trilogy/parsing/trilogy.lark,sha256=Tuqw5oGMwOYt3TYOEx_hZqGpsAp-PiAKiMW8S3EFRcg,12236
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.31.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.31.dist-info/METADATA,sha256=ouEJ1harU8vFjegOhfJuIsAfDrwyyhA8Tb1rbQiH4lU,8403
-pytrilogy-0.0.2.31.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-pytrilogy-0.0.2.31.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.31.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.31.dist-info/RECORD,,
+pytrilogy-0.0.2.33.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.33.dist-info/METADATA,sha256=IFglmRAOBjSrRj19t9GNj1P_sHVNlZRpc3TqQM0lN_c,8403
+pytrilogy-0.0.2.33.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+pytrilogy-0.0.2.33.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.33.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.33.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.31"
+__version__ = "0.0.2.33"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -36,7 +36,7 @@ class Comments:
     basic: bool = True
     joins: bool = True
     nullable: bool = False
-    partial: bool = False
+    partial: bool = True
 
 
 # TODO: support loading from environments

trilogy/core/enums.py

@@ -13,6 +13,7 @@ class UnnestMode(Enum):
 class ConceptSource(Enum):
     MANUAL = "manual"
     CTE = "cte"
+    SELECT = "select"
     PERSIST_STATEMENT = "persist_statement"
     AUTO_DERIVED = "auto_derived"
 

trilogy/core/models.py

@@ -2077,6 +2077,11 @@ class Datasource(HasUUID, Namespaced, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         original = [c for c in self.columns if c.concept.address == source.address]
+        early_exit_check = [
+            c for c in self.columns if c.concept.address == target.address
+        ]
+        if early_exit_check:
+            return None
         if len(original) != 1:
             raise ValueError(
                 f"Expected exactly one column to merge, got {len(original)} for {source.address}, {[x.alias for x in original]}"
@@ -3332,7 +3337,6 @@ class Environment(BaseModel):
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
-    _parse_count: int = 0
 
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
@@ -3442,19 +3446,22 @@ class Environment(BaseModel):
         exists = False
         existing = self.imports[alias]
         if imp_stm:
-            if any([x.path == imp_stm.path for x in existing]):
+            if any(
+                [x.path == imp_stm.path and x.alias == imp_stm.alias for x in existing]
+            ):
                 exists = True
-
         else:
-            if any([x.path == source.working_path for x in existing]):
+            if any(
+                [x.path == source.working_path and x.alias == alias for x in existing]
+            ):
                 exists = True
             imp_stm = ImportStatement(alias=alias, path=Path(source.working_path))
-
         same_namespace = alias == self.namespace
 
         if not exists:
             self.imports[alias].append(imp_stm)
-
+        else:
+            return self
         for k, concept in source.concepts.items():
             if same_namespace:
                 new = self.add_concept(concept, _ignore_cache=True)
@@ -3485,13 +3492,25 @@ class Environment(BaseModel):
         self.gen_concept_list_caches()
         return self
 
-    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
-        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
-
-        apath = path.split(".")
-        apath[-1] = apath[-1] + ".preql"
+    def add_file_import(
+        self, path: str | Path, alias: str, env: Environment | None = None
+    ):
+        from trilogy.parsing.parse_engine import (
+            ParseToObjects,
+            PARSER,
+            gen_cache_lookup,
+        )
 
-        target: Path = Path(self.working_path, *apath)
+        if isinstance(path, str):
+            if path.endswith(".preql"):
+                path = path.rsplit(".", 1)[0]
+            if "." not in path:
+                target = Path(self.working_path, path)
+            else:
+                target = Path(self.working_path, *path.split("."))
+            target = target.with_suffix(".preql")
+        else:
+            target = path
         if alias in self.imports:
             imports = self.imports[alias]
             for x in imports:
@@ -3502,17 +3521,19 @@ class Environment(BaseModel):
                 ImportStatement(alias=alias, path=target, environment=env)
             )
         else:
+            parse_address = gen_cache_lookup(str(target), alias, str(self.working_path))
             try:
                 with open(target, "r", encoding="utf-8") as f:
                     text = f.read()
                 nparser = ParseToObjects(
                     visit_tokens=True,
-                    text=text,
                     environment=Environment(
                         working_path=target.parent,
                     ),
-                    parse_address=str(target),
+                    parse_address=parse_address,
+                    token_address=target,
                 )
+                nparser.set_text(text)
                 nparser.transform(PARSER.parse(text))
             except Exception as e:
                 raise ImportError(

trilogy/core/processing/utility.py

@@ -425,6 +425,8 @@ def is_scalar_condition(
             return True
         if element.lineage and isinstance(element.lineage, AggregateWrapper):
             return is_scalar_condition(element.lineage, materialized)
+        if element.lineage and isinstance(element.lineage, Function):
+            return is_scalar_condition(element.lineage, materialized)
         return True
     elif isinstance(element, AggregateWrapper):
         return is_scalar_condition(element.function, materialized)

trilogy/dialect/base.py

@@ -646,12 +646,14 @@ class BaseDialect:
                     else None
                 ),
                 group_by=(
-                    list(
-                        set(
-                            [
-                                self.render_concept_sql(c, cte, alias=False)
-                                for c in cte.group_concepts
-                            ]
+                    sorted(
+                        list(
+                            set(
+                                [
+                                    self.render_concept_sql(c, cte, alias=False)
+                                    for c in cte.group_concepts
+                                ]
+                            )
                         )
                     )
                     if cte.group_to_grain

trilogy/dialect/common.py

@@ -101,5 +101,6 @@ def render_join(
         )
     if not base_joinkeys:
         base_joinkeys = ["1=1"]
-    joinkeys = " AND ".join(base_joinkeys)
+
+    joinkeys = " AND ".join(sorted(base_joinkeys))
     return f"{join.jointype.value.upper()} JOIN {right_base} on {joinkeys}"

trilogy/parsing/common.py

@@ -133,7 +133,9 @@ def constant_to_concept(
     )
 
 
-def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
+def function_to_concept(
+    parent: Function, name: str, namespace: str, metadata: Metadata | None = None
+) -> Concept:
     pkeys: List[Concept] = []
     for x in parent.arguments:
         pkeys += [
@@ -159,6 +161,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         purpose = Purpose.CONSTANT
     else:
         purpose = parent.output_purpose
+    fmetadata = metadata or Metadata()
     if grain is not None:
         return Concept(
             name=name,
@@ -169,6 +172,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
             keys=keys,
             modifiers=modifiers,
             grain=grain,
+            metadata=fmetadata,
         )
 
     return Concept(
@@ -179,6 +183,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         namespace=namespace,
         keys=keys,
         modifiers=modifiers,
+        metadata=fmetadata,
     )
 
 
@@ -305,7 +310,7 @@ def arbitrary_to_concept(
     elif isinstance(parent, Function):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
-        return function_to_concept(parent, name, namespace)
+        return function_to_concept(parent, name, namespace, metadata=metadata)
     elif isinstance(parent, ListWrapper):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"

trilogy/parsing/parse_engine.py

@@ -1,7 +1,7 @@
 from os.path import dirname, join
 from typing import List, Optional, Tuple, Union
 from re import IGNORECASE
-from lark import Lark, Transformer, v_args, Tree
+from lark import Lark, Transformer, v_args, Tree, ParseTree
 from lark.exceptions import (
     UnexpectedCharacters,
     UnexpectedEOF,
@@ -32,6 +32,7 @@ from trilogy.core.enums import (
     ShowCategory,
     FunctionClass,
     IOType,
+    ConceptSource,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
@@ -126,13 +127,16 @@ from trilogy.parsing.common import (
 from dataclasses import dataclass
 
 
+CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
+
+SELF_LABEL = "root"
+
+
 @dataclass
 class WholeGrainWrapper:
     where: WhereClause
 
 
-CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
-
 with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
     PARSER = Lark(
         f.read(),
@@ -144,6 +148,10 @@ with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
     )
 
 
+def gen_cache_lookup(path: str, alias: str, parent: str) -> str:
+    return path + alias + parent
+
+
 def parse_concept_reference(
     name: str, environment: Environment, purpose: Optional[Purpose] = None
 ) -> Tuple[str, str, str, str | None]:
@@ -164,8 +172,8 @@ def parse_concept_reference(
 
 def expr_to_boolean(
     root,
-) -> Union[Comparison, Conditional, SubselectComparison, Parenthetical]:
-    if not isinstance(root, (Comparison, Conditional, Parenthetical)):
+) -> Union[Comparison, SubselectComparison, Conditional]:
+    if not isinstance(root, (Comparison, SubselectComparison, Conditional)):
         if arg_to_datatype(root) == DataType.BOOL:
             root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
         else:
@@ -220,25 +228,30 @@ class ParseToObjects(Transformer):
     def __init__(
         self,
         visit_tokens,
-        text,
         environment: Environment,
         parse_address: str | None = None,
+        token_address: Path | None = None,
         parsed: dict[str, "ParseToObjects"] | None = None,
+        tokens: dict[Path | str, ParseTree] | None = None,
+        text_lookup: dict[Path | str, str] | None = None,
     ):
         Transformer.__init__(self, visit_tokens)
-        self.text = text
         self.environment: Environment = environment
-        self.parse_address = parse_address or "root"
+        self.parse_address: str = parse_address or SELF_LABEL
+        self.token_address: Path | str = token_address or SELF_LABEL
         self.parsed: dict[str, ParseToObjects] = parsed if parsed else {}
+        self.tokens: dict[Path | str, ParseTree] = tokens or {}
+        self.text_lookup: dict[Path | str, str] = text_lookup or {}
         # we do a second pass to pick up circular dependencies
         # after initial parsing
         self.pass_count = 1
-        self._results_stash = None
 
-    def transform(self, tree):
+    def set_text(self, text: str):
+        self.text_lookup[self.token_address] = text
+
+    def transform(self, tree: Tree):
         results = super().transform(tree)
-        self._results_stash = results
-        self.environment._parse_count += 1
+        self.tokens[self.token_address] = tree
         return results
 
     def hydrate_missing(self):
@@ -248,10 +261,10 @@ class ParseToObjects(Transformer):
             if v.pass_count == 2:
                 continue
             v.hydrate_missing()
-        self.environment.concepts.fail_on_missing = True
+        # self.environment.concepts.fail_on_missing = True
         # if not self.environment.concepts.undefined:
         #     return self._results_stash
-        reparsed = self.transform(PARSER.parse(self.text))
+        reparsed = self.transform(self.tokens[self.token_address])
         self.environment.concepts.undefined = {}
         return reparsed
 
@@ -266,6 +279,18 @@ class ParseToObjects(Transformer):
                     output.concept.metadata.description
                     or args[1].text.split("#")[1].strip()
                 )
+        if isinstance(output, ImportStatement):
+            if len(args) > 1 and isinstance(args[1], Comment):
+                comment = args[1].text.split("#")[1].strip()
+                namespace = output.alias
+                for _, v in self.environment.concepts.items():
+                    if v.namespace == namespace:
+                        if v.metadata.description:
+                            v.metadata.description = (
+                                f"{comment}: {v.metadata.description}"
+                            )
+                        else:
+                            v.metadata.description = comment
 
         return args[0]
 
@@ -647,45 +672,41 @@ class ParseToObjects(Transformer):
         return Comment(text=args.value)
 
     @v_args(meta=True)
-    def select_transform(self, meta, args) -> ConceptTransform:
+    def select_transform(self, meta: Meta, args) -> ConceptTransform:
 
         output: str = args[1]
-        function = unwrap_transformation(args[0])
+        transformation = unwrap_transformation(args[0])
         lookup, namespace, output, parent = parse_concept_reference(
             output, self.environment
         )
 
-        if isinstance(function, AggregateWrapper):
-            concept = agg_wrapper_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, WindowItem):
-            concept = window_item_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, FilterItem):
-            concept = filter_item_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, CONSTANT_TYPES):
-            concept = constant_to_concept(function, namespace=namespace, name=output)
-        elif isinstance(function, Function):
-            concept = function_to_concept(function, namespace=namespace, name=output)
-        else:
-            if function.output_purpose == Purpose.PROPERTY:
-                pkeys = [x for x in function.arguments if isinstance(x, Concept)]
-                grain = Grain(components=pkeys)
-                keys = tuple(grain.components_copy)
-            else:
-                grain = None
-                keys = None
-            concept = Concept(
-                name=output,
-                datatype=function.output_datatype,
-                purpose=function.output_purpose,
-                lineage=function,
-                namespace=namespace,
-                grain=Grain(components=[]) if not grain else grain,
-                keys=keys,
+        metadata = Metadata(line_number=meta.line, concept_source=ConceptSource.SELECT)
+
+        if isinstance(transformation, AggregateWrapper):
+            concept = agg_wrapper_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
             )
-        if concept.metadata:
-            concept.metadata.line_number = meta.line
+        elif isinstance(transformation, WindowItem):
+            concept = window_item_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, FilterItem):
+            concept = filter_item_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, CONSTANT_TYPES):
+            concept = constant_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        elif isinstance(transformation, Function):
+            concept = function_to_concept(
+                transformation, namespace=namespace, name=output, metadata=metadata
+            )
+        else:
+            raise SyntaxError("Invalid transformation")
+
         self.environment.add_concept(concept, meta=meta)
-        return ConceptTransform(function=function, output=concept)
+        return ConceptTransform(function=transformation, output=concept)
 
     @v_args(meta=True)
     def concept_nullable_modifier(self, meta: Meta, args) -> Modifier:
@@ -709,7 +730,7 @@ class ParseToObjects(Transformer):
         if len(args) != 1:
             raise ParseError(
                 "Malformed select statement"
-                f" {args} {self.text[meta.start_pos:meta.end_pos]}"
+                f" {args} {self.text_lookup[self.parse_address][meta.start_pos:meta.end_pos]}"
             )
         content = args[0]
         if isinstance(content, ConceptTransform):
@@ -807,25 +828,46 @@ class ParseToObjects(Transformer):
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
-        if target in self.parsed:
-            nparser = self.parsed[target]
+
+        # tokens + text are cached by path
+        token_lookup = Path(target)
+
+        # cache lookups by the target, the alias, and the file we're importing it from
+        cache_lookup = gen_cache_lookup(
+            path=target, alias=alias, parent=str(self.token_address)
+        )
+        if token_lookup in self.tokens:
+            raw_tokens = self.tokens[token_lookup]
+            text = self.text_lookup[token_lookup]
+        else:
+            text = self.resolve_import_address(target)
+            self.text_lookup[token_lookup] = text
+
+            raw_tokens = PARSER.parse(text)
+            self.tokens[token_lookup] = raw_tokens
+
+        if cache_lookup in self.parsed:
+            nparser = self.parsed[cache_lookup]
         else:
             try:
-                text = self.resolve_import_address(target)
                 nparser = ParseToObjects(
                     visit_tokens=True,
-                    text=text,
                     environment=Environment(
                         working_path=dirname(target),
                         # namespace=alias,
                     ),
-                    parse_address=target,
+                    parse_address=cache_lookup,
+                    token_address=token_lookup,
                     parsed={**self.parsed, **{self.parse_address: self}},
+                    tokens={**self.tokens, **{token_lookup: raw_tokens}},
+                    text_lookup={**self.text_lookup, **{token_lookup: text}},
                 )
-                nparser.transform(PARSER.parse(text))
-                self.parsed[target] = nparser
+                nparser.transform(raw_tokens)
+                self.parsed[cache_lookup] = nparser
                 # add the parsed objects of the import in
                 self.parsed = {**self.parsed, **nparser.parsed}
+                self.tokens = {**self.tokens, **nparser.tokens}
+                self.text_lookup = {**self.text_lookup, **nparser.text_lookup}
             except Exception as e:
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
@@ -1098,6 +1140,11 @@ class ParseToObjects(Transformer):
     def tuple_lit(self, args):
         return tuple_to_wrapper(args)
 
+    def string_lit(self, args) -> str:
+        if not args:
+            return ""
+        return args[0]
+
     def struct_lit(self, args):
 
         zipped = dict(zip(args[::2], args[1::2]))
@@ -1907,9 +1954,10 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
     ],
 ]:
     environment = environment or Environment()
-    parser = ParseToObjects(visit_tokens=True, text=text, environment=environment)
+    parser = ParseToObjects(visit_tokens=True, environment=environment)
 
     try:
+        parser.set_text(text)
         parser.transform(PARSER.parse(text))
         # handle circular dependencies
         pass_two = parser.hydrate_missing()

trilogy/parsing/trilogy.lark

@@ -84,7 +84,7 @@
 
     COPY_TYPE: "csv"i
 
-    copy_statement: "copy"i "into"i COPY_TYPE _string_lit "from"i select_statement
+    copy_statement: "copy"i "into"i COPY_TYPE string_lit "from"i select_statement
 
     // FUNCTION blocks
     function: raw_function
@@ -118,7 +118,7 @@
     _assignment: ("->") | "as"i
     select_transform : expr _assignment IDENTIFIER metadata?
     
-    metadata: "metadata" "(" IDENTIFIER "=" _string_lit ")"
+    metadata: "metadata" "(" IDENTIFIER "=" string_lit ")"
     
     limit: "LIMIT"i /[0-9]+/
     
@@ -169,8 +169,8 @@
     unnest: _UNNEST expr ")"
     //indexing into an expression is a function
     index_access: expr "[" int_lit "]"
-    map_key_access: expr "[" _string_lit "]"
-    attr_access: expr  "." _string_lit 
+    map_key_access: expr "[" string_lit "]"
+    attr_access: expr  "." string_lit 
 
     expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | _static_functions |  literal |  concept_lit  | index_access |  map_key_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like 
     
@@ -209,16 +209,16 @@
     
     //string
     _LIKE.1: "like("i
-    like:  _LIKE expr "," _string_lit ")"
+    like:  _LIKE expr "," string_lit ")"
     _ILIKE.1: "ilike("i
-    ilike:   _ILIKE expr "," _string_lit ")"
+    ilike:   _ILIKE expr "," string_lit ")"
     alt_like: expr  "like"i expr
     _UPPER.1: "upper("i
     upper: _UPPER expr ")"
    _LOWER.1: "lower("i
     lower: _LOWER expr ")"  
     _SPLIT.1: "split("i  
-    fsplit: _SPLIT expr "," _string_lit ")"
+    fsplit: _SPLIT expr "," string_lit ")"
    _STRPOS.1: "strpos("i
     fstrpos: _STRPOS expr "," expr ")"
     _SUBSTRING.1: "substring("i
@@ -301,7 +301,7 @@
     SINGLE_STRING_CHARS: /(?:(?!\${)([^'\\]|\\.))+/+ // any character except '
     _single_quote: "'" ( SINGLE_STRING_CHARS )* "'" 
     _double_quote: "\"" ( DOUBLE_STRING_CHARS )* "\"" 
-    _string_lit: _single_quote | _double_quote
+    string_lit: _single_quote | _double_quote
 
     MINUS: "-"
 
@@ -322,7 +322,7 @@
 
     !null_lit.1: "null"i
     
-    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit | tuple_lit
+    literal: null_lit | string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit | tuple_lit
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i