pytrilogy 0.0.3.3__py3-none-any.whl → 0.0.3.5__py3-none-any.whl

{pytrilogy-0.0.3.3.dist-info → pytrilogy-0.0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: pytrilogy
-Version: 0.0.3.3
+Version: 0.0.3.5
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.3.dist-info → pytrilogy-0.0.3.5.dist-info}/RECORD

@@ -1,11 +1,12 @@
-trilogy/__init__.py,sha256=nHzZPfctd2gKdQ2jsPOuU0higL8uy586PocOwP0TzdY,302
+trilogy/__init__.py,sha256=7rK8M4Aw3BDqS9a5ou3PjyRkVMzEhaVMf6a7V-8ll4E,302
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=qZ1d0hoKPPV2HHCoFwPYTVB7b6bXjpWvXd3lE-zEhy8,1494
 trilogy/engine.py,sha256=yOPnR7XCjWG82Gym_LLZBkYKKJdLCvqdCyt8zguNcnM,1103
-trilogy/executor.py,sha256=nvi8F8ls7stAXvYUIRs6zh8X4q6O_plZcfazPnL-hKw,16745
+trilogy/executor.py,sha256=sssEPDnIDPiQtMSrt5pFiJXUfcDc6gSi4m2Eliod_BM,16844
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
+trilogy/authoring/__init__.py,sha256=6eIJewNIbrMdP6_dyLpyZ0uU62_pSj9wCP80Pkz2Bl8,1922
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
 trilogy/core/enums.py,sha256=aTFXL6nckmG0hpNLdCxI2kAT26cIsZPahF-pHzLQ9uc,7085
@@ -17,14 +18,14 @@ trilogy/core/functions.py,sha256=7Pq9jYSJd45L2pxT7AI-_rXVZmeLnmTPp8d1lA4z4Vk,244
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=xGO8piVsLrpqrx-Aid_Y56_5slSv4eZmlP64hCHRiEc,7957
-trilogy/core/query_processor.py,sha256=HyDxBhQsD9KX-Y7pYznlpCAW6AvI76RqPTQNa1mreoE,19450
+trilogy/core/query_processor.py,sha256=Do8YpdPBdsbKtl9n37hobzk8SORMGqH-e_zNNxd-BE4,19456
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=ydg1FwMjL58AWsM5cJTB0B8AR8jFaxAZdXfQroiTC0M,67264
+trilogy/core/models/author.py,sha256=oRCKWhz-i1fO1LlHWiHE3l1awCHdQ3yx6FKH9n9RxRU,67188
 trilogy/core/models/build.py,sha256=kiq31T8LtUtgmT37m617Q2MlMvQTuAxJzwb6947EiWU,56127
 trilogy/core/models/build_environment.py,sha256=8UggvlPU708GZWYPJMc_ou2r7M3TY2g69eqGvz03YX0,5528
 trilogy/core/models/core.py,sha256=yie1uuq62uOQ5fjob9NMJbdvQPrCErXUT7JTCuYRyjI,9697
 trilogy/core/models/datasource.py,sha256=c0tGxyH2WwTmAD047tr69U0a6GNVf-ug26H68yii7DA,9257
-trilogy/core/models/environment.py,sha256=QSl-H6nwarzKbQgNRjtwDKMJtA4F_GVQpRs-NMNt-6Q,24983
+trilogy/core/models/environment.py,sha256=GU8D3cn6lCFAseVPYfW_a-cnBbD1sYEeDVOkbZSWCxk,25943
 trilogy/core/models/execute.py,sha256=ABylFQgtavjjCfFkEsFdUwfMB4UBQLHjdzQ9E67QlAE,33521
 trilogy/core/optimizations/__init__.py,sha256=EBanqTXEzf1ZEYjAneIWoIcxtMDite5-n2dQ5xcfUtg,356
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
@@ -32,7 +33,7 @@ trilogy/core/optimizations/inline_constant.py,sha256=lvNTIXaLNkw3HseJyXyDNk5R52d
 trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=0rFnasSQlkXTRIfAFMHyHVux1pTQ3ryKeQds-SFSot0,40290
+trilogy/core/processing/concept_strategies_v3.py,sha256=wPlpg4L7uw-f0DgJBkI8VRdcisjDT1X6iApjEE6CmfA,40291
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
 trilogy/core/processing/utility.py,sha256=Oc5tLGeDDpzhbfo2ZcF8ex1kez-NcJDMcG2Lm5BjS4c,20548
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
@@ -83,18 +84,18 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=lZSSu6Q8HvJVB0xCYsNpoTlkCgNNEVVCwHExVBWCDO8,20524
+trilogy/parsing/common.py,sha256=yAE3x4SyO4PfAb7HhZ_l9sNPYaf_pcM1K8ioEy76SCU,20301
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=rtlwl-WzcPCd5pm0WhXHwe8JGOJCuQm_EAnRj6qG93w,54318
+trilogy/parsing/parse_engine.py,sha256=uazcnUhY3pERY8Xa116IHl1JhCPSlvQG6i4vP7I4Qpk,54638
 trilogy/parsing/render.py,sha256=o_XuQWhcwx1lD9eGVqkqZEwkmQK0HdmWWokGBtdeH4I,17837
 trilogy/parsing/trilogy.lark,sha256=EazfEvYPuvkPkNjUnVzFi0uD9baavugbSI8CyfawShk,12573
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
-pytrilogy-0.0.3.3.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.3.3.dist-info/METADATA,sha256=v5lMZ7w4aJJNsIOCQ0fTTgUkyg37u_SYC7GZAcptleg,8983
-pytrilogy-0.0.3.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-pytrilogy-0.0.3.3.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.3.3.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.3.dist-info/RECORD,,
+pytrilogy-0.0.3.5.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.3.5.dist-info/METADATA,sha256=W8SsLe2qtgpl4SDr_xlL3lOwReWbKYBClHXkjLQSIqk,8983
+pytrilogy-0.0.3.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+pytrilogy-0.0.3.5.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.3.5.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.5.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.3"
+__version__ = "0.0.3.5"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/authoring/__init__.py

@@ -0,0 +1,90 @@
+from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.core.enums import (
+    BooleanOperator,
+    ComparisonOperator,
+    FunctionClass,
+    FunctionType,
+    InfiniteFunctionArgs,
+    Ordering,
+    Purpose,
+)
+from trilogy.core.functions import FunctionFactory
+from trilogy.core.models.author import (
+    AggregateWrapper,
+    CaseElse,
+    CaseWhen,
+    Comparison,
+    Concept,
+    ConceptRef,
+    Conditional,
+    FilterItem,
+    Function,
+    HavingClause,
+    MagicConstants,
+    Metadata,
+    OrderBy,
+    OrderItem,
+    Parenthetical,
+    SubselectComparison,
+    WhereClause,
+    WindowItem,
+    WindowItemOrder,
+    WindowItemOver,
+    WindowOrder,
+    WindowType,
+)
+from trilogy.core.models.core import DataType, ListType, ListWrapper, StructType
+from trilogy.core.models.environment import Environment
+from trilogy.core.statements.author import (
+    ConceptDeclarationStatement,
+    ConceptTransform,
+    SelectItem,
+    SelectStatement,
+)
+from trilogy.parsing.common import arbitrary_to_concept, arg_to_datatype
+
+__all__ = [
+    "Concept",
+    "Function",
+    "WhereClause",
+    "Comparison",
+    "FilterItem",
+    "CaseWhen",
+    "CaseElse",
+    "AggregateWrapper",
+    "WindowItem",
+    "WindowOrder",
+    "WindowType",
+    "WindowItemOrder",
+    "WindowItemOver",
+    "DataType",
+    "StructType",
+    "ListType",
+    "ListWrapper",
+    "FunctionType",
+    "FunctionFactory",
+    "ConceptDeclarationStatement",
+    "ConceptTransform",
+    "SelectItem",
+    "SelectStatement",
+    "Environment",
+    "ConceptRef",
+    "HavingClause",
+    "MagicConstants",
+    "Metadata",
+    "OrderBy",
+    "OrderItem",
+    "Parenthetical",
+    "SubselectComparison",
+    "Conditional",
+    "BooleanOperator",
+    "ComparisonOperator",
+    "FunctionClass",
+    "FunctionType",
+    "InfiniteFunctionArgs",
+    "Ordering",
+    "Purpose",
+    "DEFAULT_NAMESPACE",
+    "arbitrary_to_concept",
+    "arg_to_datatype",
+]

trilogy/core/models/author.py

@@ -166,10 +166,8 @@ class UndefinedConcept(ConceptRef):
 
 
 def address_with_namespace(address: str, namespace: str) -> str:
-    ns = address.split(".", 1)[0]
-    if ns == namespace:
-        return address
-    if ns == DEFAULT_NAMESPACE:
+    existing_ns = address.split(".", 1)[0]
+    if existing_ns == DEFAULT_NAMESPACE:
         return f"{namespace}.{address.split('.',1)[1]}"
     return f"{namespace}.{address}"
 
@@ -203,7 +201,7 @@ class Parenthetical(
     def __repr__(self):
         return f"({str(self.content)})"
 
-    def with_namespace(self, namespace: str):
+    def with_namespace(self, namespace: str) -> Parenthetical:
         return Parenthetical.model_construct(
             content=(
                 self.content.with_namespace(namespace)
@@ -917,8 +915,6 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         return self.name.replace(".", "_")
 
     def with_namespace(self, namespace: str) -> Self:
-        if namespace == self.namespace:
-            return self
         return self.__class__.model_construct(
             name=self.name,
             datatype=self.datatype,

trilogy/core/models/environment.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Annotated,
+    Any,
     Dict,
     ItemsView,
     List,
@@ -383,14 +384,13 @@ class Environment(BaseModel):
             ):
                 exists = True
             imp_stm = Import(alias=alias, path=Path(source.working_path))
-        same_namespace = alias == self.namespace
+        same_namespace = alias == DEFAULT_NAMESPACE
 
         if not exists:
             self.imports[alias].append(imp_stm)
         # we can't exit early
         # as there may be new concepts
         for k, concept in source.concepts.items():
-
             # skip internal namespace
             if INTERNAL_NAMESPACE in concept.address:
                 continue
@@ -416,7 +416,6 @@ class Environment(BaseModel):
                 self.alias_origin_lookup[address_with_namespace(key, alias)] = (
                     val.with_namespace(alias)
                 )
-
         return self
 
     def add_file_import(
@@ -457,8 +456,10 @@ class Environment(BaseModel):
                     token_address=target,
                 )
                 nparser.set_text(text)
+                nparser.environment.concepts.fail_on_missing = False
                 nparser.transform(PARSER.parse(text))
                 nparser.hydrate_missing()
+                nparser.environment.concepts.fail_on_missing = True
 
             except Exception as e:
                 raise ImportError(
@@ -664,37 +665,59 @@ class LazyEnvironment(Environment):
     until relevant attributes accessed."""
 
     load_path: Path
+    setup_queries: list[Any] = Field(default_factory=list)
     loaded: bool = False
 
+    @property
+    def setup_path(self) -> Path:
+        return self.load_path.parent / "setup.preql"
+
     def __init__(self, **data):
+        if not data.get("working_path"):
+            data["working_path"] = data["load_path"].parent
         super().__init__(**data)
+        assert self.working_path == self.load_path.parent
 
     def _add_path_concepts(self):
         pass
 
+    def _load(self):
+        if self.loaded:
+            return
+        from trilogy import parse
+
+        env = Environment(working_path=self.load_path.parent)
+        assert env.working_path == self.load_path.parent
+        with open(self.load_path, "r") as f:
+            env, _ = parse(f.read(), env)
+        if self.setup_path.exists():
+            with open(self.setup_path, "r") as f2:
+                env, q = parse(f2.read(), env)
+                for q in q:
+                    self.setup_queries.append(q)
+        self.loaded = True
+        self.datasources = env.datasources
+        self.concepts = env.concepts
+        self.imports = env.imports
+        self.alias_origin_lookup = env.alias_origin_lookup
+        self.materialized_concepts = env.materialized_concepts
+        self.functions = env.functions
+        self.data_types = env.data_types
+        self.cte_name_map = env.cte_name_map
+
     def __getattribute__(self, name):
-        if name in (
-            "load_path",
-            "loaded",
-            "working_path",
-            "model_config",
-            "model_fields",
-            "model_post_init",
+        if name not in (
+            "datasources",
+            "concepts",
+            "imports",
+            "materialized_concepts",
+            "functions",
+            "datatypes",
+            "cte_name_map",
         ) or name.startswith("_"):
             return super().__getattribute__(name)
         if not self.loaded:
-            logger.info(
-                f"lazily evaluating load path {self.load_path} to access {name}"
-            )
-            from trilogy import parse
-
-            env = Environment(working_path=str(self.working_path))
-            with open(self.load_path, "r") as f:
-                parse(f.read(), env)
-            self.loaded = True
-            self.datasources = env.datasources
-            self.concepts = env.concepts
-            self.imports = env.imports
+            self._load()
         return super().__getattribute__(name)
 
 

trilogy/core/processing/concept_strategies_v3.py

@@ -1046,7 +1046,7 @@ def source_query_concepts(
             f"{c.address}<{c.purpose}>{c.derivation}>" for c in output_concepts
         ]
         raise ValueError(
-            f"Could not resolve conections between {error_strings} from environment graph."
+            f"Could not resolve connections between {error_strings} from environment graph."
         )
     final = [x for x in root.output_concepts if x.address not in root.hidden_concepts]
     logger.info(

trilogy/core/query_processor.py

@@ -385,7 +385,7 @@ def get_query_node(
     )
     graph = generate_graph(build_environment)
     logger.info(
-        f"{LOGGER_PREFIX} getting source datasource for outputs {statement.output_components} grain {build_statement.grain}"
+        f"{LOGGER_PREFIX} getting source datasource for outputs {build_statement.output_components} grain {build_statement.grain}"
     )
 
     search_concepts: list[BuildConcept] = build_statement.output_components

trilogy/executor.py

@@ -346,10 +346,10 @@ class Executor(object):
         file = Path(file)
         with open(file, "r") as f:
             command = f.read()
-            return self.parse_text_generator(command, persist=persist)
+            return self.parse_text_generator(command, persist=persist, root=file)
 
     def parse_text(
-        self, command: str, persist: bool = False
+        self, command: str, persist: bool = False, root: Path | None = None
     ) -> List[
         ProcessedQuery
         | ProcessedQueryPersist
@@ -357,9 +357,11 @@ class Executor(object):
         | ProcessedRawSQLStatement
         | ProcessedCopyStatement
     ]:
-        return list(self.parse_text_generator(command, persist=persist))
+        return list(self.parse_text_generator(command, persist=persist, root=root))
 
-    def parse_text_generator(self, command: str, persist: bool = False) -> Generator[
+    def parse_text_generator(
+        self, command: str, persist: bool = False, root: Path | None = None
+    ) -> Generator[
         ProcessedQuery
         | ProcessedQueryPersist
         | ProcessedShowStatement
@@ -369,7 +371,7 @@ class Executor(object):
         None,
     ]:
         """Process a preql text command"""
-        _, parsed = parse_text(command, self.environment)
+        _, parsed = parse_text(command, self.environment, root=root)
         generatable = [
             x
             for x in parsed

trilogy/parsing/common.py

@@ -152,7 +152,6 @@ def constant_to_concept(
     parent: ListWrapper | MapWrapper | list | int | float | str,
     name: str,
     namespace: str,
-    purpose: Purpose | None = None,
     metadata: Metadata | None = None,
 ) -> Concept:
     const_function: Function = Function(
@@ -340,7 +339,6 @@ def filter_item_to_concept(
     name: str,
     namespace: str,
     environment: Environment,
-    purpose: Purpose | None = None,
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
@@ -377,14 +375,13 @@ def window_item_to_concept(
     name: str,
     namespace: str,
     environment: Environment,
-    purpose: Purpose | None = None,
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
     bcontent = environment.concepts[parent.content.address]
     if isinstance(bcontent, UndefinedConcept):
         return UndefinedConcept(address=f"{namespace}.{name}", metadata=fmetadata)
-    local_purpose, keys = get_purpose_and_keys(purpose, (bcontent,), environment)
+    local_purpose, keys = get_purpose_and_keys(None, (bcontent,), environment)
 
     if parent.order_by:
         grain_components = parent.over + [bcontent.output]
@@ -568,7 +565,6 @@ def arbitrary_to_concept(
     namespace: str | None = None,
     name: str | None = None,
     metadata: Metadata | None = None,
-    purpose: Purpose | None = None,
 ) -> Concept:
     namespace = namespace or environment.namespace
     if isinstance(parent, AggregateWrapper):
@@ -585,7 +581,6 @@ def arbitrary_to_concept(
             name,
             namespace,
             environment=environment,
-            purpose=purpose,
             metadata=metadata,
         )
     elif isinstance(parent, FilterItem):
@@ -596,7 +591,6 @@ def arbitrary_to_concept(
             name,
             namespace,
             environment=environment,
-            purpose=purpose,
             metadata=metadata,
         )
     elif isinstance(parent, Function):
@@ -612,8 +606,8 @@ def arbitrary_to_concept(
     elif isinstance(parent, ListWrapper):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
-        return constant_to_concept(parent, name, namespace, purpose, metadata)
+        return constant_to_concept(parent, name, namespace, metadata)
     else:
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
-        return constant_to_concept(parent, name, namespace, purpose, metadata)
+        return constant_to_concept(parent, name, namespace, metadata)

trilogy/parsing/parse_engine.py

@@ -147,6 +147,9 @@ with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
 
 
 def gen_cache_lookup(path: str, alias: str, parent: str) -> str:
+    # path is the path of the file
+    # alias is what it's being imported under
+    # parent is the...direct parnet?
     return path + alias + parent
 
 
@@ -252,10 +255,12 @@ class ParseToObjects(Transformer):
         for _, v in self.parsed.items():
             v.prepare_parse()
 
-    def hydrate_missing(self):
+    def hydrate_missing(self, force: bool = False):
+        if self.token_address not in self.tokens:
+            return []
         self.parse_pass = ParsePass.VALIDATION
-        for k, v in self.parsed.items():
-            if v.parse_pass == ParsePass.VALIDATION:
+        for _, v in list(self.parsed.items()):
+            if v.parse_pass == ParsePass.VALIDATION and not force:
                 continue
             v.hydrate_missing()
         reparsed = self.transform(self.tokens[self.token_address])
@@ -390,7 +395,6 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def column_assignment(self, meta: Meta, args):
-        # TODO -> deal with conceptual modifiers
         modifiers = []
         alias = args[0]
         concept_list = args[1]
@@ -398,6 +402,7 @@ class ParseToObjects(Transformer):
         if len(concept_list) > 1:
             modifiers += concept_list[:-1]
         concept = concept_list[-1]
+        assert not self.environment.concepts.fail_on_missing
         resolved = self.environment.concepts.__getitem__(  # type: ignore
             key=concept, line_no=meta.line, file=self.token_address
         )
@@ -508,8 +513,6 @@ class ParseToObjects(Transformer):
         else:
             metadata = None
         purpose = args[0]
-        if purpose == Purpose.AUTO:
-            purpose = None
         raw_name = args[1]
         # abc.def.property pattern
         if isinstance(raw_name, str):
@@ -538,10 +541,14 @@ class ParseToObjects(Transformer):
                 name=name,
                 namespace=namespace,
                 environment=self.environment,
-                purpose=purpose,
                 metadata=metadata,
             )
 
+            if purpose and purpose != Purpose.AUTO and concept.purpose != purpose:
+                raise SyntaxError(
+                    f'Concept {name} purpose {concept.purpose} does not match declared purpose {purpose}. Suggest defaulting to "auto"'
+                )
+
             if concept.metadata:
                 concept.metadata.line_number = meta.line
             self.environment.add_concept(concept, meta=meta)
@@ -552,7 +559,6 @@ class ParseToObjects(Transformer):
                 source_value,
                 name=name,
                 namespace=namespace,
-                purpose=purpose,
                 metadata=metadata,
             )
             if concept.metadata:
@@ -577,15 +583,6 @@ class ParseToObjects(Transformer):
             namespace=self.environment.namespace or DEFAULT_NAMESPACE,
         )
 
-        # clean up current definitions
-        # to_delete = set()
-        # if output.name in self.environment.named_statements:
-        #     for k, v in self.environment.concepts.items():
-        #         if v.derivation == Derivation.ROWSET and v.lineage.rowset.name == name:
-        #             to_delete.add(k)
-        # for k in to_delete:
-        #     self.environment.concepts.pop(k)
-
         for new_concept in rowset_to_concepts(output, self.environment):
             if new_concept.metadata:
                 new_concept.metadata.line_number = meta.line
@@ -887,10 +884,14 @@ class ParseToObjects(Transformer):
         if cache_lookup in self.parsed:
             nparser = self.parsed[cache_lookup]
             new_env = nparser.environment
+            if nparser.parse_pass != ParsePass.VALIDATION:
+                # nparser.transform(raw_tokens)
+                nparser.hydrate_missing()
         else:
             try:
                 new_env = Environment(
                     working_path=dirname(target),
+                    env_file_path=token_lookup,
                 )
                 new_env.concepts.fail_on_missing = False
                 self.parsed[self.parse_address] = self
@@ -908,10 +909,11 @@ class ParseToObjects(Transformer):
                 raise ImportError(
                     f"Unable to import file {target}, parsing error: {e}"
                 ) from e
+        parsed_path = Path(args[0])
+        imps = ImportStatement(alias=alias, path=parsed_path)
 
-        imps = ImportStatement(alias=alias, path=Path(args[0]))
         self.environment.add_import(
-            alias, new_env, Import(alias=alias, path=Path(args[0]))
+            alias, new_env, Import(alias=alias, path=parsed_path)
         )
         return imps
 
@@ -1602,7 +1604,9 @@ def parse_text_raw(text: str, environment: Optional[Environment] = None):
     PARSER.parse(text)
 
 
-def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
+def parse_text(
+    text: str, environment: Optional[Environment] = None, root: Path | None = None
+) -> Tuple[
     Environment,
     List[
         Datasource
@@ -1614,7 +1618,9 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
         | None
     ],
 ]:
-    environment = environment or Environment()
+    environment = environment or (
+        Environment(working_path=root) if root else Environment()
+    )
     parser = ParseToObjects(environment=environment)
 
     try:
@@ -1623,7 +1629,7 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
         parser.prepare_parse()
         parser.transform(PARSER.parse(text))
         # this will reset fail on missing
-        pass_two = parser.hydrate_missing()
+        pass_two = parser.hydrate_missing(force=True)
         output = [v for v in pass_two if v]
         environment.concepts.fail_on_missing = True
     except VisitError as e: