pytrilogy 0.0.1.103__tar.gz → 0.0.1.104__tar.gz

{pytrilogy-0.0.1.103/pytrilogy.egg-info → pytrilogy-0.0.1.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.103
+Version: 0.0.1.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.103
+Version: 0.0.1.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/__init__.py

@@ -3,6 +3,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.1.103"
+__version__ = "0.0.1.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment"]

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/core/models.py

@@ -355,7 +355,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
         return f"{self.namespace}.{self.name}"
 
@@ -436,7 +436,8 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
-    def with_default_grain(self) -> "Concept":
+    @cached_property
+    def _with_default_grain(self) -> "Concept":
         if self.purpose == Purpose.KEY:
             # we need to make this abstract
             grain = Grain(components=[self.with_grain(Grain())], nested=True)
@@ -473,6 +474,9 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
+    def with_default_grain(self) -> "Concept":
+        return self._with_default_grain
+
     @property
     def sources(self) -> List["Concept"]:
         if self.lineage:
@@ -610,7 +614,7 @@ class Grain(BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
-    @property
+    @cached_property
     def set(self):
         return set([c.address for c in self.components_copy])
 
@@ -1585,7 +1589,7 @@ class Datasource(Namespaced, BaseModel):
             columns=[c.with_namespace(namespace) for c in self.columns],
         )
 
-    @property
+    @cached_property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -1780,7 +1784,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info=ValidationInfo):
+    def validate_source_map(cls, v, info: ValidationInfo):
         values = info.data
         expected = {c.address for c in values["output_concepts"]}.union(
             c.address for c in values["input_concepts"]
@@ -2288,8 +2292,8 @@ class EnvironmentConceptDict(dict):
 
 class ImportStatement(BaseModel):
     alias: str
-    path: str
-    # environment: "Environment" | None = None
+    path: Path
+    environment: Union["Environment", None] = None
     # TODO: this might result in a lot of duplication
     # environment:"Environment"
 
@@ -2324,6 +2328,9 @@ class Environment(BaseModel):
     version: str = Field(default_factory=get_version)
     cte_name_map: Dict[str, str] = Field(default_factory=dict)
 
+    materialized_concepts: List[Concept] = Field(default_factory=list)
+    _parse_count: int = 0
+
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
         with open(path, "r") as f:
@@ -2349,20 +2356,14 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    @property
-    def materialized_concepts(self) -> List[Concept]:
-        output = []
-        for concept in self.concepts.values():
-            found = False
-            # basic concepts are effectively materialized
-            # and can be found via join paths
-            for datasource in self.datasources.values():
-                if concept.address in [x.address for x in datasource.output_concepts]:
-                    found = True
-                    break
-            if found:
-                output.append(concept)
-        return output
+    def gen_materialized_concepts(self) -> None:
+        concrete_addresses = set()
+        for datasource in self.datasources.values():
+            for concept in datasource.output_concepts:
+                concrete_addresses.add(concept.address)
+        self.materialized_concepts = [
+            c for c in self.concepts.values() if c.address in concrete_addresses
+        ]
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -2392,12 +2393,61 @@ class Environment(BaseModel):
 
     def add_import(self, alias: str, environment: Environment):
         self.imports[alias] = ImportStatement(
-            alias=alias, path=str(environment.working_path)
+            alias=alias, path=Path(environment.working_path)
         )
         for key, concept in environment.concepts.items():
             self.concepts[f"{alias}.{key}"] = concept.with_namespace(alias)
         for key, datasource in environment.datasources.items():
             self.datasources[f"{alias}.{key}"] = datasource.with_namespace(alias)
+        self.gen_materialized_concepts()
+        return self
+
+    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
+        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
+
+        apath = path.split(".")
+        apath[-1] = apath[-1] + ".preql"
+
+        target: Path = Path(self.working_path, *apath)
+        if env:
+            self.imports[alias] = ImportStatement(
+                alias=alias, path=target, environment=env
+            )
+
+        elif alias in self.imports:
+            current = self.imports[alias]
+            env = self.imports[alias].environment
+            if current.path != target:
+                raise ImportError(
+                    f"Attempted to import {target} with alias {alias} but {alias} is already imported from {current.path}"
+                )
+        else:
+            try:
+                with open(target, "r", encoding="utf-8") as f:
+                    text = f.read()
+                nparser = ParseToObjects(
+                    visit_tokens=True,
+                    text=text,
+                    environment=Environment(
+                        working_path=target.parent,
+                    ),
+                    parse_address=str(target),
+                )
+                nparser.transform(PARSER.parse(text))
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import file {target.parent}, parsing error: {e}"
+                )
+            env = nparser.environment
+        if env:
+            for _, concept in env.concepts.items():
+                self.add_concept(concept.with_namespace(alias))
+
+            for _, datasource in env.datasources.items():
+                self.add_datasource(datasource.with_namespace(alias))
+        imps = ImportStatement(alias=alias, path=target, environment=env)
+        self.imports[alias] = imps
+        return imps
 
     def parse(
         self, input: str, namespace: str | None = None, persist: bool = False
@@ -2448,21 +2498,22 @@ class Environment(BaseModel):
             from trilogy.core.environment_helpers import generate_related_concepts
 
             generate_related_concepts(concept, self)
+        self.gen_materialized_concepts()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
+        meta: Meta | None = None,
     ):
-        if datasource.namespace == DEFAULT_NAMESPACE:
-            self.datasources[datasource.name] = datasource
-            return datasource
-        if not datasource.namespace:
+        if not datasource.namespace or datasource.namespace == DEFAULT_NAMESPACE:
             self.datasources[datasource.name] = datasource
+            self.gen_materialized_concepts()
             return datasource
         self.datasources[datasource.namespace + "." + datasource.identifier] = (
             datasource
         )
+        self.gen_materialized_concepts()
         return datasource
 
 

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/core/processing/concept_strategies_v3.py

@@ -23,7 +23,6 @@ from trilogy.core.processing.node_generators import (
     gen_window_node,
     gen_group_node,
     gen_basic_node,
-    gen_select_node,
     gen_unnest_node,
     gen_merge_node,
     gen_group_to_node,
@@ -208,7 +207,8 @@ def generate_node(
     history: History | None = None,
 ) -> StrategyNode | None:
     # first check in case there is a materialized_concept
-    candidate = gen_select_node(
+    history = history or History()
+    candidate = history.gen_select_node(
         concept,
         local_optional,
         environment,
@@ -218,6 +218,7 @@ def generate_node(
         accept_partial=accept_partial,
         accept_partial_optional=False,
     )
+
     if candidate:
         return candidate
 
@@ -320,7 +321,7 @@ def generate_node(
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
         )
-        return gen_select_node(
+        return history.gen_select_node(
             concept,
             local_optional,
             environment,
@@ -328,6 +329,7 @@ def generate_node(
             depth + 1,
             fail_if_not_found=False,
             accept_partial=accept_partial,
+            accept_partial_optional=True,
         )
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/core/processing/node_generators/filter_node.py

@@ -55,6 +55,8 @@ def gen_filter_node(
         depth=depth + 1,
         history=history,
     )
+    if not enrich_node:
+        return filter_node
     x = MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/core/processing/node_generators/select_node.py

@@ -2,7 +2,13 @@ from itertools import combinations
 from typing import List, Optional
 
 from trilogy.core.enums import PurposeLineage
-from trilogy.core.models import Concept, Environment, Grain, LooseConceptList
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    Grain,
+    LooseConceptList,
+    Datasource,
+)
 from trilogy.core.processing.nodes import (
     StrategyNode,
     SelectNode,
@@ -15,10 +21,211 @@ import networkx as nx
 from trilogy.core.graph_models import concept_to_node, datasource_to_node
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
+from dataclasses import dataclass
 
 LOGGER_PREFIX = "[GEN_SELECT_NODE]"
 
 
+@dataclass
+class DatasourceMatch:
+    key: str
+    datasource: Datasource
+    matched: LooseConceptList
+    partial: LooseConceptList
+
+    def __repr__(self):
+        return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
+
+
+def dm_to_strategy_node(
+    dm: DatasourceMatch,
+    target_grain: Grain,
+    environment: Environment,
+    g: nx.DiGraph,
+    depth: int,
+    accept_partial: bool = False,
+) -> StrategyNode:
+    datasource = dm.datasource
+    if target_grain and target_grain.issubset(datasource.grain):
+        if all([x in dm.matched for x in target_grain.components]):
+            force_group = False
+        # if we are not returning the grain
+        # we have to group
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(dm.matched)}, group to actual grain"
+            )
+            force_group = True
+    elif all([x in dm.matched for x in datasource.grain.components]):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
+        )
+        force_group = False
+    else:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+        )
+        force_group = True
+    bcandidate: StrategyNode = SelectNode(
+        input_concepts=[c.concept for c in datasource.columns],
+        output_concepts=dm.matched.concepts,
+        environment=environment,
+        g=g,
+        parents=[],
+        depth=depth,
+        partial_concepts=dm.partial.concepts,
+        accept_partial=accept_partial,
+        datasource=datasource,
+        grain=Grain(components=dm.matched.concepts),
+    )
+    # we need to nest the group node one further
+    if force_group is True:
+        candidate: StrategyNode = GroupNode(
+            output_concepts=dm.matched.concepts,
+            input_concepts=dm.matched.concepts,
+            environment=environment,
+            g=g,
+            parents=[bcandidate],
+            depth=depth,
+            partial_concepts=bcandidate.partial_concepts,
+        )
+    else:
+        candidate = bcandidate
+    return candidate
+
+
+def gen_select_nodes_from_tables_v2(
+    mandatory_concept: Concept,
+    all_concepts: List[Concept],
+    g: nx.DiGraph,
+    environment: Environment,
+    depth: int,
+    target_grain: Grain,
+    accept_partial: bool = False,
+) -> tuple[bool, list[Concept], list[StrategyNode]]:
+    # if we have only constants
+    # we don't need a table
+    # so verify nothing, select node will render
+    all_lcl = LooseConceptList(concepts=all_concepts)
+    if all([c.derivation == PurposeLineage.CONSTANT for c in all_lcl]):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} All concepts {[x.address for x in all_lcl]} are constants, returning constant node"
+        )
+        return (
+            True,
+            all_lcl.concepts,
+            [
+                ConstantNode(
+                    output_concepts=all_lcl.concepts,
+                    input_concepts=[],
+                    environment=environment,
+                    g=g,
+                    parents=[],
+                    depth=depth,
+                    # no partial for constants
+                    partial_concepts=[],
+                    force_group=False,
+                )
+            ],
+        )
+    # otherwise, we need to look for a table
+    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_lcl.concepts]
+    matches: dict[str, DatasourceMatch] = {}
+    for k, datasource in environment.datasources.items():
+        matched = []
+        matched_paths = []
+        for idx, req_concept in enumerate(nodes_to_find):
+            try:
+                path = nx.shortest_path(
+                    g,
+                    source=datasource_to_node(datasource),
+                    target=req_concept,
+                )
+                ds_valid = (
+                    sum(
+                        [
+                            1 if g.nodes[node]["type"] == "datasource" else 0
+                            for node in path
+                        ]
+                    )
+                    == 1
+                )
+                address_valid = (
+                    sum(
+                        [
+                            (
+                                1
+                                if g.nodes[node]["type"] == "concept"
+                                and g.nodes[node]["concept"].address
+                                != all_lcl.concepts[idx].address
+                                else 0
+                            )
+                            for node in path
+                        ]
+                    )
+                    == 0
+                )
+                if ds_valid and address_valid:
+                    matched_paths.append(path)
+                    matched.append(all_lcl.concepts[idx])
+            except nx.NodeNotFound:
+                continue
+            except nx.exception.NetworkXNoPath:
+                continue
+        dm = DatasourceMatch(
+            key=k,
+            datasource=datasource,
+            matched=LooseConceptList(concepts=matched),
+            partial=LooseConceptList(
+                concepts=[
+                    c.concept
+                    for c in datasource.columns
+                    if not c.is_complete and c.concept.address in all_lcl
+                ]
+            ),
+        )
+        if not matched:
+            continue
+        if mandatory_concept.address not in dm.matched:
+            continue
+        if not accept_partial and dm.partial.addresses:
+            continue
+        matches[k] = dm
+    found: set[str] = set()
+    all_found = False
+    all_checked = False
+    to_return: list[StrategyNode] = []
+    if not matches:
+        return False, [], []
+    while not all_found and not all_checked:
+        final_key: str = max(
+            matches,
+            key=lambda x: len(
+                [m for m in matches[x].matched.addresses if m not in found]
+            )
+            - 0.1 * len(matches[x].partial.addresses),
+        )
+        final: DatasourceMatch = matches[final_key]
+        candidate = dm_to_strategy_node(
+            final,
+            target_grain=Grain(
+                components=[
+                    x for x in target_grain.components if x.address in final.matched
+                ]
+            ),
+            environment=environment,
+            g=g,
+            depth=depth,
+            accept_partial=accept_partial,
+        )
+        to_return.append(candidate)
+        del matches[final_key]
+        found = found.union(final.matched.addresses)
+        all_found = all_lcl.addresses.issubset(found)
+        all_checked = len(matches) == 0
+    return all_found, [x for x in all_concepts if x.address in found], to_return
+
+
 def gen_select_node_from_table(
     target_concept: Concept,
     all_concepts: List[Concept],
@@ -166,58 +373,15 @@ def gen_select_node_from_table(
     return candidates[final]
 
 
-def gen_select_node(
-    concept: Concept,
+def gen_select_nodes_from_tables(
     local_optional: List[Concept],
-    environment: Environment,
-    g,
     depth: int,
-    accept_partial: bool = False,
-    fail_if_not_found: bool = True,
-    accept_partial_optional: bool = True,
-    target_grain: Grain | None = None,
-) -> StrategyNode | None:
-    all_concepts = [concept] + local_optional
-    all_lcl = LooseConceptList(concepts=all_concepts)
-    materialized_lcl = LooseConceptList(
-        concepts=[
-            x
-            for x in all_concepts
-            if x.address in [z.address for z in environment.materialized_concepts]
-            or x.derivation == PurposeLineage.CONSTANT
-        ]
-    )
-    if not target_grain:
-        target_grain = Grain()
-        for ac in all_concepts:
-            target_grain += ac.grain
-    if materialized_lcl != all_lcl:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address} "
-            f" as it + optional (looking for all {all_lcl}) includes non-materialized concepts {all_lcl.difference(materialized_lcl)} vs materialized: {materialized_lcl}"
-        )
-        if fail_if_not_found:
-            raise NoDatasourceException(f"No datasource exists for {concept}")
-        return None
-
-    ds: StrategyNode | None = None
-
-    # attempt to select all concepts from table
-    ds = gen_select_node_from_table(
-        concept,
-        [concept] + local_optional,
-        g=g,
-        environment=environment,
-        depth=depth,
-        accept_partial=accept_partial,
-        target_grain=target_grain,
-    )
-    if ds:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Found select node with all target concepts, force group is {ds.force_group}, target grain {target_grain}"
-        )
-        return ds
-    # if we cannot find a match
+    concept: Concept,
+    environment: Environment,
+    g: nx.DiGraph,
+    accept_partial: bool,
+    all_concepts: List[Concept],
+) -> tuple[bool, list[Concept], list[StrategyNode]]:
     parents: List[StrategyNode] = []
     found: List[Concept] = []
     logger.info(
@@ -238,7 +402,7 @@ def gen_select_node(
         )
         if not ds:
             unreachable.append(opt_con.address)
-    # actual search
+    all_found = False
     for x in reversed(range(1, len(local_optional) + 1)):
         if all_found:
             break
@@ -275,6 +439,64 @@ def gen_select_node(
                         f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]}"
                     )
                     all_found = True
+    return all_found, found, parents
+
+
+def gen_select_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    accept_partial: bool = False,
+    fail_if_not_found: bool = True,
+    accept_partial_optional: bool = True,
+    target_grain: Grain | None = None,
+) -> StrategyNode | None:
+    all_concepts = [concept] + local_optional
+    all_lcl = LooseConceptList(concepts=all_concepts)
+    materialized_lcl = LooseConceptList(
+        concepts=[
+            x
+            for x in all_concepts
+            if x.address in [z.address for z in environment.materialized_concepts]
+            or x.derivation == PurposeLineage.CONSTANT
+        ]
+    )
+    if not target_grain:
+        target_grain = Grain()
+        for ac in all_concepts:
+            target_grain += ac.grain
+    if materialized_lcl != all_lcl:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address} "
+            f" as it + optional (looking for all {all_lcl}) includes non-materialized concepts {all_lcl.difference(materialized_lcl)} vs materialized: {materialized_lcl}"
+        )
+        if fail_if_not_found:
+            raise NoDatasourceException(f"No datasource exists for {concept}")
+        return None
+
+    ds: StrategyNode | None = None
+
+    # attempt to select all concepts from table
+    ds = gen_select_node_from_table(
+        concept,
+        [concept] + local_optional,
+        g=g,
+        environment=environment,
+        depth=depth,
+        accept_partial=accept_partial,
+        target_grain=target_grain,
+    )
+    if ds:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Found select node with all target concepts, force group is {ds.force_group}, target grain {target_grain}"
+        )
+        return ds
+    # if we cannot find a match
+    all_found, found, parents = gen_select_nodes_from_tables_v2(
+        concept, all_concepts, g, environment, depth, target_grain, accept_partial
+    )
     if parents and (all_found or accept_partial_optional):
         if all_found:
             logger.info(
@@ -282,7 +504,7 @@ def gen_select_node(
             )
         else:
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found some optional {[c.address for c in found]}, returning"
+                f"{padding(depth)}{LOGGER_PREFIX} found some optional, returning"
             )
         all_partial = [
             c

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/core/processing/nodes/__init__.py

@@ -6,11 +6,12 @@ from .window_node import WindowNode
 from .base_node import StrategyNode, NodeJoin
 from .unnest_node import UnnestNode
 from pydantic import BaseModel, Field, ConfigDict
-from trilogy.core.models import Concept
+from trilogy.core.models import Concept, Environment
 
 
 class History(BaseModel):
     history: dict[str, StrategyNode | None] = Field(default_factory=dict)
+    select_history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     started: set[str] = Field(default_factory=set)
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -60,6 +61,58 @@ class History(BaseModel):
             in self.started
         )
 
+    def _select_concepts_to_lookup(
+        self,
+        main: Concept,
+        search: list[Concept],
+        accept_partial: bool,
+        fail_if_not_found: bool,
+        accept_partial_optional: bool,
+    ) -> str:
+        return (
+            str(main.address)
+            + "|"
+            + "-".join([c.address for c in search])
+            + str(accept_partial)
+            + str(fail_if_not_found)
+            + str(accept_partial_optional)
+        )
+
+    def gen_select_node(
+        self,
+        concept: Concept,
+        local_optional: list[Concept],
+        environment: Environment,
+        g,
+        depth: int,
+        fail_if_not_found: bool = False,
+        accept_partial: bool = False,
+        accept_partial_optional: bool = False,
+    ) -> StrategyNode | None:
+        from trilogy.core.processing.node_generators.select_node import gen_select_node
+
+        fingerprint = self._select_concepts_to_lookup(
+            concept,
+            local_optional,
+            accept_partial,
+            fail_if_not_found,
+            accept_partial_optional,
+        )
+        if fingerprint in self.select_history:
+            return self.select_history[fingerprint]
+        gen = gen_select_node(
+            concept,
+            local_optional,
+            environment,
+            g,
+            depth + 1,
+            fail_if_not_found=fail_if_not_found,
+            accept_partial=accept_partial,
+            accept_partial_optional=accept_partial_optional,
+        )
+        self.select_history[fingerprint] = gen
+        return gen
+
 
 __all__ = [
     "FilterNode",

{pytrilogy-0.0.1.103 → pytrilogy-0.0.1.104}/trilogy/parsing/parse_engine.py

@@ -9,6 +9,7 @@ from lark.exceptions import (
     UnexpectedToken,
     VisitError,
 )
+from pathlib import Path
 from lark.tree import Meta
 from pydantic import ValidationError
 from trilogy.core.internal import INTERNAL_NAMESPACE, ALL_ROWS_CONCEPT
@@ -466,25 +467,34 @@ class ParseToObjects(Transformer):
         text,
         environment: Environment,
         parse_address: str | None = None,
-        parsed: dict | None = None,
+        parsed: dict[str, "ParseToObjects"] | None = None,
     ):
         Transformer.__init__(self, visit_tokens)
         self.text = text
         self.environment: Environment = environment
-        self.imported: set[str] = set()
         self.parse_address = parse_address or "root"
         self.parsed: dict[str, ParseToObjects] = parsed if parsed else {}
         # we do a second pass to pick up circular dependencies
         # after initial parsing
         self.pass_count = 1
+        self._results_stash = None
+
+    def transform(self, tree):
+        results = super().transform(tree)
+        self._results_stash = results
+        self.environment._parse_count += 1
+        return results
 
     def hydrate_missing(self):
         self.pass_count = 2
         for k, v in self.parsed.items():
+
             if v.pass_count == 2:
                 continue
             v.hydrate_missing()
         self.environment.concepts.fail_on_missing = True
+        # if not self.environment.concepts.undefined:
+        #     return self._results_stash
         reparsed = self.transform(PARSER.parse(self.text))
         self.environment.concepts.undefined = {}
         return reparsed
@@ -932,7 +942,7 @@ class ParseToObjects(Transformer):
         )
         for column in columns:
             column.concept = column.concept.with_grain(datasource.grain)
-        self.environment.datasources[datasource.identifier] = datasource
+        self.environment.add_datasource(datasource, meta=meta)
         return datasource
 
     @v_args(meta=True)
@@ -1046,12 +1056,11 @@ class ParseToObjects(Transformer):
         self.environment.add_concept(new, meta=meta)
         return merge
 
-    def import_statement(self, args: list[str]):
+    def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
-        self.imported.add(target)
         if target in self.parsed:
             nparser = self.parsed[target]
         else:
@@ -1070,21 +1079,23 @@ class ParseToObjects(Transformer):
                 )
                 nparser.transform(PARSER.parse(text))
                 self.parsed[target] = nparser
+                # add the parsed objects of the import in
+                self.parsed = {**self.parsed, **nparser.parsed}
             except Exception as e:
                 raise ImportError(
                     f"Unable to import file {dirname(target)}, parsing error: {e}"
                 )
 
-        for key, concept in nparser.environment.concepts.items():
-            # self.environment.concepts[f"{alias}.{key}"] = concept.with_namespace(new_namespace)
+        for _, concept in nparser.environment.concepts.items():
             self.environment.add_concept(concept.with_namespace(alias))
 
-        for key, datasource in nparser.environment.datasources.items():
+        for _, datasource in nparser.environment.datasources.items():
             self.environment.add_datasource(datasource.with_namespace(alias))
-            # self.environment.datasources[f"{alias}.{key}"] = datasource.with_namespace(new_namespace)
-
-        self.environment.imports[alias] = ImportStatement(alias=alias, path=args[0])
-        return None
+        imps = ImportStatement(
+            alias=alias, path=Path(args[0]), environment=nparser.environment
+        )
+        self.environment.imports[alias] = imps
+        return imps
 
     @v_args(meta=True)
     def show_category(self, meta: Meta, args) -> ShowCategory: