pytrilogy 0.0.1.102__py3-none-any.whl → 0.0.1.104__py3-none-any.whl

{pytrilogy-0.0.1.102.dist-info → pytrilogy-0.0.1.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.102
+Version: 0.0.1.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -27,7 +27,7 @@ Requires-Dist: psycopg2-binary ; extra == 'postgres'
 Provides-Extra: snowflake
 Requires-Dist: snowflake-sqlalchemy ; extra == 'snowflake'
 
-##Trilogy
+## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)
 [![Discord](https://img.shields.io/badge/DISCORD-CHAT-red?logo=discord)](https://discord.gg/Z4QSSuqGEd)
 

{pytrilogy-0.0.1.102.dist-info → pytrilogy-0.0.1.104.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=oDahoPZQX0IDYqdzVJrJ5IU0M0QfuwEhv8kb2xlJ7yY,245
+trilogy/__init__.py,sha256=SFMMJIYUKTFdaXEFolvT4DhBf1i_Id3pbS5lCdo1kE0,245
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=MphpyZXP4URq7R1vP8EUtT_ZfGPSKuqKKFGMMjfMRtA,482
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -9,33 +9,33 @@ trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
 trilogy/core/enums.py,sha256=KEZQTzJ8tlGIukuUwQUIG1FTHOP1B4i0EeCgFjfsbDw,5394
-trilogy/core/env_processor.py,sha256=iMYkOOmErxvMwD7JvNMGjsdKt5nSZeZgPydtGRIKqKg,1384
+trilogy/core/env_processor.py,sha256=dfiUp5QxcrgG5YI3Py4xP4OTDRZqltWJGQp8PwukkfY,1401
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
 trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=QOD8QdjjYvZN_2MPV9MSOztuh6Ix8W9Q0oaoVQ_CxuM,100790
+trilogy/core/models.py,sha256=I0RJVMRzRawznOFasphMfFa31WqJvZhXw5VtkWA-KgA,102941
 trilogy/core/query_processor.py,sha256=x3fjs1Vhg_G1FHhjLomZ3kH16PU9RPcediRz-Lu_QZg,11625
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=K-3fhJXnEilVAsSyUs6PV_zoMPes5B1NGlNYz54tLXk,22243
+trilogy/core/processing/concept_strategies_v3.py,sha256=L0lBiiZ1D3a1zsJfnXa0vNlV3EM6Nyp936OzzUT5u24,22368
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
 trilogy/core/processing/utility.py,sha256=Gk35HgyIG2SSUyI5OHZcB0bw1PZUVC_aNc9Sre6xPQU,10535
 trilogy/core/processing/node_generators/__init__.py,sha256=1L1TWnGkrSQlLe9ZuMG8DMGfS755v0fjCdz_W1ofCJQ,747
 trilogy/core/processing/node_generators/basic_node.py,sha256=tVPmg0r0kDdABkmn6z4sxsk1hKy9yTT_Xvl1eVN2Zck,2162
 trilogy/core/processing/node_generators/common.py,sha256=o2nLcAv2jwjynOhgixoCFN46v_p3s5Zw_M8iDiMNvi8,8444
 trilogy/core/processing/node_generators/concept_merge.py,sha256=oKxRuc7m6wnsrxAaLg3q2-dM2dyFgAvaXCp6nmG--jU,5113
-trilogy/core/processing/node_generators/filter_node.py,sha256=nhbbqTI-oZT387JsYS6UhhKD1hf6HubabTth9PIrBUM,2460
+trilogy/core/processing/node_generators/filter_node.py,sha256=VCafGBvFQgBfkqclR-Y9TLNwZeOucsBkhXy3jLV_TPc,2511
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/merge_node.py,sha256=gF1AELkkakP6NDdntQMAba7Efhyd4b3QS07aQXyQE-c,5359
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/rowset_node.py,sha256=Kri-bLqJUYAcefhlejZxDsHMaMkmeWuJA2iG9dMoeE8,4529
-trilogy/core/processing/node_generators/select_node.py,sha256=4izz-cstBgiIsiKEJp70AdRf-Iw1PonC1xRucHE2exM,12556
+trilogy/core/processing/node_generators/rowset_node.py,sha256=K-aoLi0OSfNADXR5_vxDcNv2dJeFy30XNp_IaaWWJ6o,4684
+trilogy/core/processing/node_generators/select_node.py,sha256=xeCqIUEubrf3u_QQfbGdf1BG4fO0HYQ64hiFur8NUqY,20080
 trilogy/core/processing/node_generators/unnest_node.py,sha256=s1VXQZSf1LnX3ISeQ5JzmzmCKUw30-5OK_f0YTB9_48,1031
 trilogy/core/processing/node_generators/window_node.py,sha256=ekazi5eXxnShpcp-qukXNG4DHFdULoXrX-YWUWLNEpM,2527
-trilogy/core/processing/nodes/__init__.py,sha256=7z5ElK_6U6n2cMCdeRQpF6Vqwlo7-QzTR6daQuJf9Yo,2022
+trilogy/core/processing/nodes/__init__.py,sha256=ZkDGQksvsM5uNia5rhXFCUJcpTRhoYYFdyfJw-Eiu8s,3674
 trilogy/core/processing/nodes/base_node.py,sha256=ovFM4r8QG2sy5d2X1MTCk5BiYUwvr0chxSaIopkO6Fc,8890
 trilogy/core/processing/nodes/filter_node.py,sha256=DqSRv8voEajPZqzeeiIsxuv4ubvsmeQcCW6x_v2CmOk,1359
 trilogy/core/processing/nodes/group_node.py,sha256=-xaZuAkCXHWP_K5BmoJ5jTM_53wYWtYPEkrr4LaWib0,3735
@@ -65,13 +65,13 @@ trilogy/parsing/common.py,sha256=wLxfqoLYdpAJ6I-6HB951-8kNOOUbnEhwgX61xlz0_0,479
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=CQr7kzHtybMPSjhJ5M5u30z_Nh_2Vu8RZqLjfmjh-NE,65491
+trilogy/parsing/parse_engine.py,sha256=2-wCQhJXaPhPtnnS31oR3NwLZ5MoH20N5IHo_RPG6KQ,65779
 trilogy/parsing/render.py,sha256=OlkJs6LnQsSNAc94j_rcxT6KI3VU6wzZK1gDHgjL_Rs,16133
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/scripts/trilogy.py,sha256=CkMTK5RU2jTYjrnZoI-wvASr5WYGYmlRLdYvdoi1ICE,3729
-pytrilogy-0.0.1.102.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.102.dist-info/METADATA,sha256=J7wRo7eRvnQvlMufW5R0-VmdRyw16AGbAJoG6uvUXi0,6882
-pytrilogy-0.0.1.102.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-pytrilogy-0.0.1.102.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.102.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.102.dist-info/RECORD,,
+trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
+pytrilogy-0.0.1.104.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.104.dist-info/METADATA,sha256=RhbBJJRMUSJQW0csiwwJ-aE-HBtrpI0aOjrza5WGfvM,6883
+pytrilogy-0.0.1.104.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+pytrilogy-0.0.1.104.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.104.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.104.dist-info/RECORD,,

{pytrilogy-0.0.1.102.dist-info → pytrilogy-0.0.1.104.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.1)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -3,6 +3,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.1.102"
+__version__ = "0.0.1.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment"]

trilogy/core/env_processor.py

@@ -1,4 +1,8 @@
-from trilogy.core.graph_models import ReferenceGraph, concept_to_node, datasource_to_node
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
 from trilogy.core.models import Environment
 from trilogy.core.enums import PurposeLineage
 

trilogy/core/models.py

@@ -338,7 +338,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
 
     def __eq__(self, other: object):
         if isinstance(other, str):
-            if self.address == str:
+            if self.address == other:
                 return True
         if not isinstance(other, Concept):
             return False
@@ -355,7 +355,7 @@ class Concept(Namespaced, SelectGrain, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
         return f"{self.namespace}.{self.name}"
 
@@ -436,7 +436,8 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
-    def with_default_grain(self) -> "Concept":
+    @cached_property
+    def _with_default_grain(self) -> "Concept":
         if self.purpose == Purpose.KEY:
             # we need to make this abstract
             grain = Grain(components=[self.with_grain(Grain())], nested=True)
@@ -473,6 +474,9 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             modifiers=self.modifiers,
         )
 
+    def with_default_grain(self) -> "Concept":
+        return self._with_default_grain
+
     @property
     def sources(self) -> List["Concept"]:
         if self.lineage:
@@ -610,7 +614,7 @@ class Grain(BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
-    @property
+    @cached_property
     def set(self):
         return set([c.address for c in self.components_copy])
 
@@ -1585,7 +1589,7 @@ class Datasource(Namespaced, BaseModel):
             columns=[c.with_namespace(namespace) for c in self.columns],
         )
 
-    @property
+    @cached_property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -1780,7 +1784,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info=ValidationInfo):
+    def validate_source_map(cls, v, info: ValidationInfo):
         values = info.data
         expected = {c.address for c in values["output_concepts"]}.union(
             c.address for c in values["input_concepts"]
@@ -1887,7 +1891,9 @@ class QueryDatasource(BaseModel):
                 else None
             ),
             source_type=self.source_type,
-            partial_concepts=self.partial_concepts + other.partial_concepts,
+            partial_concepts=unique(
+                self.partial_concepts + other.partial_concepts, "address"
+            ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
         )
@@ -2286,8 +2292,8 @@ class EnvironmentConceptDict(dict):
 
 class ImportStatement(BaseModel):
     alias: str
-    path: str
-    # environment: "Environment" | None = None
+    path: Path
+    environment: Union["Environment", None] = None
     # TODO: this might result in a lot of duplication
     # environment:"Environment"
 
@@ -2322,6 +2328,9 @@ class Environment(BaseModel):
     version: str = Field(default_factory=get_version)
     cte_name_map: Dict[str, str] = Field(default_factory=dict)
 
+    materialized_concepts: List[Concept] = Field(default_factory=list)
+    _parse_count: int = 0
+
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
         with open(path, "r") as f:
@@ -2347,20 +2356,14 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    @property
-    def materialized_concepts(self) -> List[Concept]:
-        output = []
-        for concept in self.concepts.values():
-            found = False
-            # basic concepts are effectively materialized
-            # and can be found via join paths
-            for datasource in self.datasources.values():
-                if concept.address in [x.address for x in datasource.output_concepts]:
-                    found = True
-                    break
-            if found:
-                output.append(concept)
-        return output
+    def gen_materialized_concepts(self) -> None:
+        concrete_addresses = set()
+        for datasource in self.datasources.values():
+            for concept in datasource.output_concepts:
+                concrete_addresses.add(concept.address)
+        self.materialized_concepts = [
+            c for c in self.concepts.values() if c.address in concrete_addresses
+        ]
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -2390,12 +2393,61 @@ class Environment(BaseModel):
 
     def add_import(self, alias: str, environment: Environment):
         self.imports[alias] = ImportStatement(
-            alias=alias, path=str(environment.working_path)
+            alias=alias, path=Path(environment.working_path)
         )
         for key, concept in environment.concepts.items():
             self.concepts[f"{alias}.{key}"] = concept.with_namespace(alias)
         for key, datasource in environment.datasources.items():
             self.datasources[f"{alias}.{key}"] = datasource.with_namespace(alias)
+        self.gen_materialized_concepts()
+        return self
+
+    def add_file_import(self, path: str, alias: str, env: Environment | None = None):
+        from trilogy.parsing.parse_engine import ParseToObjects, PARSER
+
+        apath = path.split(".")
+        apath[-1] = apath[-1] + ".preql"
+
+        target: Path = Path(self.working_path, *apath)
+        if env:
+            self.imports[alias] = ImportStatement(
+                alias=alias, path=target, environment=env
+            )
+
+        elif alias in self.imports:
+            current = self.imports[alias]
+            env = self.imports[alias].environment
+            if current.path != target:
+                raise ImportError(
+                    f"Attempted to import {target} with alias {alias} but {alias} is already imported from {current.path}"
+                )
+        else:
+            try:
+                with open(target, "r", encoding="utf-8") as f:
+                    text = f.read()
+                nparser = ParseToObjects(
+                    visit_tokens=True,
+                    text=text,
+                    environment=Environment(
+                        working_path=target.parent,
+                    ),
+                    parse_address=str(target),
+                )
+                nparser.transform(PARSER.parse(text))
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import file {target.parent}, parsing error: {e}"
+                )
+            env = nparser.environment
+        if env:
+            for _, concept in env.concepts.items():
+                self.add_concept(concept.with_namespace(alias))
+
+            for _, datasource in env.datasources.items():
+                self.add_datasource(datasource.with_namespace(alias))
+        imps = ImportStatement(alias=alias, path=target, environment=env)
+        self.imports[alias] = imps
+        return imps
 
     def parse(
         self, input: str, namespace: str | None = None, persist: bool = False
@@ -2446,21 +2498,22 @@ class Environment(BaseModel):
             from trilogy.core.environment_helpers import generate_related_concepts
 
             generate_related_concepts(concept, self)
+        self.gen_materialized_concepts()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
+        meta: Meta | None = None,
     ):
-        if datasource.namespace == DEFAULT_NAMESPACE:
-            self.datasources[datasource.name] = datasource
-            return datasource
-        if not datasource.namespace:
+        if not datasource.namespace or datasource.namespace == DEFAULT_NAMESPACE:
             self.datasources[datasource.name] = datasource
+            self.gen_materialized_concepts()
             return datasource
         self.datasources[datasource.namespace + "." + datasource.identifier] = (
             datasource
         )
+        self.gen_materialized_concepts()
         return datasource
 
 
@@ -2530,7 +2583,7 @@ class Comparison(Namespaced, SelectGrain, BaseModel):
 
     def __post_init__(self):
         if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise ValueError(
+            raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
 
@@ -2704,6 +2757,8 @@ class Conditional(Namespaced, SelectGrain, BaseModel):
     def __add__(self, other) -> "Conditional":
         if other is None:
             return self
+        elif str(other) == str(self):
+            return self
         elif isinstance(other, (Comparison, Conditional, Parenthetical)):
             return Conditional(left=self, right=other, operator=BooleanOperator.AND)
         raise ValueError(f"Cannot add {self.__class__} and {type(other)}")

trilogy/core/processing/concept_strategies_v3.py

@@ -23,7 +23,6 @@ from trilogy.core.processing.node_generators import (
     gen_window_node,
     gen_group_node,
     gen_basic_node,
-    gen_select_node,
     gen_unnest_node,
     gen_merge_node,
     gen_group_to_node,
@@ -208,7 +207,8 @@ def generate_node(
     history: History | None = None,
 ) -> StrategyNode | None:
     # first check in case there is a materialized_concept
-    candidate = gen_select_node(
+    history = history or History()
+    candidate = history.gen_select_node(
         concept,
         local_optional,
         environment,
@@ -218,6 +218,7 @@ def generate_node(
         accept_partial=accept_partial,
         accept_partial_optional=False,
     )
+
     if candidate:
         return candidate
 
@@ -318,9 +319,9 @@ def generate_node(
         )
     elif concept.derivation == PurposeLineage.ROOT:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
         )
-        return gen_select_node(
+        return history.gen_select_node(
             concept,
             local_optional,
             environment,
@@ -328,6 +329,7 @@ def generate_node(
             depth + 1,
             fail_if_not_found=False,
             accept_partial=accept_partial,
+            accept_partial_optional=True,
         )
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")

trilogy/core/processing/node_generators/filter_node.py

@@ -55,6 +55,8 @@ def gen_filter_node(
         depth=depth + 1,
         history=history,
     )
+    if not enrich_node:
+        return filter_node
     x = MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[

trilogy/core/processing/node_generators/rowset_node.py

@@ -35,26 +35,30 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    if where := select.where_clause:
+        targets = select.output_components + where.conditional.concept_arguments
+    else:
+        targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=select.output_components,
+        mandatory_list=targets,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
-    node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-    node.rebuild_cache()
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
         )
         return None
+    node.conditions = select.where_clause.conditional if select.where_clause else None
+    # rebuild any cached info with the new condition clause
+    node.rebuild_cache()
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [
         x
         for x in rowset.derived_concepts
-        if x.address == concept.address or x.address in enrichment
+        # if x.address == concept.address or x.address in enrichment
     ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
@@ -68,7 +72,7 @@ def gen_rowset_node(
         for item in additional_relevant:
             node.partial_concepts.append(item)
 
-    # assume grain to be outoput of select
+    # assume grain to be output of select
     # but don't include anything aggregate at this point
     assert node.resolution_cache
     node.resolution_cache.grain = concept_list_to_grain(

trilogy/core/processing/node_generators/select_node.py

@@ -2,7 +2,13 @@ from itertools import combinations
 from typing import List, Optional
 
 from trilogy.core.enums import PurposeLineage
-from trilogy.core.models import Concept, Environment, Grain, LooseConceptList
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    Grain,
+    LooseConceptList,
+    Datasource,
+)
 from trilogy.core.processing.nodes import (
     StrategyNode,
     SelectNode,
@@ -15,10 +21,211 @@ import networkx as nx
 from trilogy.core.graph_models import concept_to_node, datasource_to_node
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
+from dataclasses import dataclass
 
 LOGGER_PREFIX = "[GEN_SELECT_NODE]"
 
 
+@dataclass
+class DatasourceMatch:
+    key: str
+    datasource: Datasource
+    matched: LooseConceptList
+    partial: LooseConceptList
+
+    def __repr__(self):
+        return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
+
+
+def dm_to_strategy_node(
+    dm: DatasourceMatch,
+    target_grain: Grain,
+    environment: Environment,
+    g: nx.DiGraph,
+    depth: int,
+    accept_partial: bool = False,
+) -> StrategyNode:
+    datasource = dm.datasource
+    if target_grain and target_grain.issubset(datasource.grain):
+        if all([x in dm.matched for x in target_grain.components]):
+            force_group = False
+        # if we are not returning the grain
+        # we have to group
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(dm.matched)}, group to actual grain"
+            )
+            force_group = True
+    elif all([x in dm.matched for x in datasource.grain.components]):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
+        )
+        force_group = False
+    else:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+        )
+        force_group = True
+    bcandidate: StrategyNode = SelectNode(
+        input_concepts=[c.concept for c in datasource.columns],
+        output_concepts=dm.matched.concepts,
+        environment=environment,
+        g=g,
+        parents=[],
+        depth=depth,
+        partial_concepts=dm.partial.concepts,
+        accept_partial=accept_partial,
+        datasource=datasource,
+        grain=Grain(components=dm.matched.concepts),
+    )
+    # we need to nest the group node one further
+    if force_group is True:
+        candidate: StrategyNode = GroupNode(
+            output_concepts=dm.matched.concepts,
+            input_concepts=dm.matched.concepts,
+            environment=environment,
+            g=g,
+            parents=[bcandidate],
+            depth=depth,
+            partial_concepts=bcandidate.partial_concepts,
+        )
+    else:
+        candidate = bcandidate
+    return candidate
+
+
+def gen_select_nodes_from_tables_v2(
+    mandatory_concept: Concept,
+    all_concepts: List[Concept],
+    g: nx.DiGraph,
+    environment: Environment,
+    depth: int,
+    target_grain: Grain,
+    accept_partial: bool = False,
+) -> tuple[bool, list[Concept], list[StrategyNode]]:
+    # if we have only constants
+    # we don't need a table
+    # so verify nothing, select node will render
+    all_lcl = LooseConceptList(concepts=all_concepts)
+    if all([c.derivation == PurposeLineage.CONSTANT for c in all_lcl]):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} All concepts {[x.address for x in all_lcl]} are constants, returning constant node"
+        )
+        return (
+            True,
+            all_lcl.concepts,
+            [
+                ConstantNode(
+                    output_concepts=all_lcl.concepts,
+                    input_concepts=[],
+                    environment=environment,
+                    g=g,
+                    parents=[],
+                    depth=depth,
+                    # no partial for constants
+                    partial_concepts=[],
+                    force_group=False,
+                )
+            ],
+        )
+    # otherwise, we need to look for a table
+    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_lcl.concepts]
+    matches: dict[str, DatasourceMatch] = {}
+    for k, datasource in environment.datasources.items():
+        matched = []
+        matched_paths = []
+        for idx, req_concept in enumerate(nodes_to_find):
+            try:
+                path = nx.shortest_path(
+                    g,
+                    source=datasource_to_node(datasource),
+                    target=req_concept,
+                )
+                ds_valid = (
+                    sum(
+                        [
+                            1 if g.nodes[node]["type"] == "datasource" else 0
+                            for node in path
+                        ]
+                    )
+                    == 1
+                )
+                address_valid = (
+                    sum(
+                        [
+                            (
+                                1
+                                if g.nodes[node]["type"] == "concept"
+                                and g.nodes[node]["concept"].address
+                                != all_lcl.concepts[idx].address
+                                else 0
+                            )
+                            for node in path
+                        ]
+                    )
+                    == 0
+                )
+                if ds_valid and address_valid:
+                    matched_paths.append(path)
+                    matched.append(all_lcl.concepts[idx])
+            except nx.NodeNotFound:
+                continue
+            except nx.exception.NetworkXNoPath:
+                continue
+        dm = DatasourceMatch(
+            key=k,
+            datasource=datasource,
+            matched=LooseConceptList(concepts=matched),
+            partial=LooseConceptList(
+                concepts=[
+                    c.concept
+                    for c in datasource.columns
+                    if not c.is_complete and c.concept.address in all_lcl
+                ]
+            ),
+        )
+        if not matched:
+            continue
+        if mandatory_concept.address not in dm.matched:
+            continue
+        if not accept_partial and dm.partial.addresses:
+            continue
+        matches[k] = dm
+    found: set[str] = set()
+    all_found = False
+    all_checked = False
+    to_return: list[StrategyNode] = []
+    if not matches:
+        return False, [], []
+    while not all_found and not all_checked:
+        final_key: str = max(
+            matches,
+            key=lambda x: len(
+                [m for m in matches[x].matched.addresses if m not in found]
+            )
+            - 0.1 * len(matches[x].partial.addresses),
+        )
+        final: DatasourceMatch = matches[final_key]
+        candidate = dm_to_strategy_node(
+            final,
+            target_grain=Grain(
+                components=[
+                    x for x in target_grain.components if x.address in final.matched
+                ]
+            ),
+            environment=environment,
+            g=g,
+            depth=depth,
+            accept_partial=accept_partial,
+        )
+        to_return.append(candidate)
+        del matches[final_key]
+        found = found.union(final.matched.addresses)
+        all_found = all_lcl.addresses.issubset(found)
+        all_checked = len(matches) == 0
+    return all_found, [x for x in all_concepts if x.address in found], to_return
+
+
 def gen_select_node_from_table(
     target_concept: Concept,
     all_concepts: List[Concept],
@@ -50,26 +257,21 @@ def gen_select_node_from_table(
     candidates: dict[str, StrategyNode] = {}
     scores: dict[str, int] = {}
     # otherwise, we need to look for a table
+    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_concepts]
     for datasource in environment.datasources.values():
         all_found = True
-        for raw_concept in all_concepts:
-            # look for connection to abstract grain
-            req_concept = raw_concept.with_default_grain()
-            # if we don't have a concept in the graph
-            # exit early
-            if concept_to_node(req_concept) not in g.nodes:
-                raise ValueError(concept_to_node(req_concept))
+        for idx, req_concept in enumerate(nodes_to_find):
             try:
                 path = nx.shortest_path(
                     g,
                     source=datasource_to_node(datasource),
-                    target=concept_to_node(req_concept),
+                    target=req_concept,
                 )
             except nx.NodeNotFound as e:
                 # just to provide better error
                 ncandidates = [
                     datasource_to_node(datasource),
-                    concept_to_node(req_concept),
+                    req_concept,
                 ]
                 for ncandidate in ncandidates:
                     try:
@@ -94,81 +296,152 @@ def gen_select_node_from_table(
             for node in path:
                 if g.nodes[node]["type"] == "datasource":
                     continue
-                if g.nodes[node]["concept"].address == raw_concept.address:
+                if g.nodes[node]["concept"].address == all_concepts[idx].address:
                     continue
                 all_found = False
                 break
 
-        if all_found:
-            partial_concepts = [
-                c.concept
-                for c in datasource.columns
-                if not c.is_complete and c.concept in all_lcl
-            ]
-            partial_lcl = LooseConceptList(concepts=partial_concepts)
-            if not accept_partial and target_concept in partial_lcl:
-                continue
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} target grain is {str(target_grain)}"
-            )
-            if target_grain and target_grain.issubset(datasource.grain):
+        if not all_found:
+            # skip to next node
+            continue
+        partial_concepts = [
+            c.concept
+            for c in datasource.columns
+            if not c.is_complete and c.concept in all_lcl
+        ]
+        partial_lcl = LooseConceptList(concepts=partial_concepts)
+        if not accept_partial and target_concept in partial_lcl:
+            continue
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} target grain is {str(target_grain)}"
+        )
+        if target_grain and target_grain.issubset(datasource.grain):
 
-                if all([x in all_lcl for x in target_grain.components]):
-                    force_group = False
-                # if we are not returning the grain
-                # we have to group
-                else:
-                    logger.info(
-                        f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(all_lcl)}, group to actual grain"
-                    )
-                    force_group = True
-            elif all([x in all_lcl for x in datasource.grain.components]):
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
-                )
+            if all([x in all_lcl for x in target_grain.components]):
                 force_group = False
+            # if we are not returning the grain
+            # we have to group
             else:
                 logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+                    f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(all_lcl)}, group to actual grain"
                 )
                 force_group = True
+        elif all([x in all_lcl for x in datasource.grain.components]):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
+            )
+            force_group = False
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
+            )
+            force_group = True
 
-            bcandidate: StrategyNode = SelectNode(
-                input_concepts=[c.concept for c in datasource.columns],
+        bcandidate: StrategyNode = SelectNode(
+            input_concepts=[c.concept for c in datasource.columns],
+            output_concepts=all_concepts,
+            environment=environment,
+            g=g,
+            parents=[],
+            depth=depth,
+            partial_concepts=[c for c in all_concepts if c in partial_lcl],
+            accept_partial=accept_partial,
+            datasource=datasource,
+            grain=Grain(components=all_concepts),
+        )
+        # we need to nest the group node one further
+        if force_group is True:
+            candidate: StrategyNode = GroupNode(
                 output_concepts=all_concepts,
+                input_concepts=all_concepts,
                 environment=environment,
                 g=g,
-                parents=[],
+                parents=[bcandidate],
                 depth=depth,
-                partial_concepts=[c for c in all_concepts if c in partial_lcl],
-                accept_partial=accept_partial,
-                datasource=datasource,
-                grain=Grain(components=all_concepts),
-            )
-            # we need to ntest the group node one further
-            if force_group is True:
-                candidate: StrategyNode = GroupNode(
-                    output_concepts=all_concepts,
-                    input_concepts=all_concepts,
-                    environment=environment,
-                    g=g,
-                    parents=[bcandidate],
-                    depth=depth,
-                    partial_concepts=bcandidate.partial_concepts,
-                )
-            else:
-                candidate = bcandidate
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, returning {candidate.output_lcl}"
+                partial_concepts=bcandidate.partial_concepts,
             )
-            candidates[datasource.identifier] = candidate
-            scores[datasource.identifier] = -len(partial_concepts)
+        else:
+            candidate = bcandidate
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, returning {candidate.output_lcl}"
+        )
+        candidates[datasource.identifier] = candidate
+        scores[datasource.identifier] = -len(partial_concepts)
     if not candidates:
         return None
     final = max(candidates, key=lambda x: scores[x])
     return candidates[final]
 
 
+def gen_select_nodes_from_tables(
+    local_optional: List[Concept],
+    depth: int,
+    concept: Concept,
+    environment: Environment,
+    g: nx.DiGraph,
+    accept_partial: bool,
+    all_concepts: List[Concept],
+) -> tuple[bool, list[Concept], list[StrategyNode]]:
+    parents: List[StrategyNode] = []
+    found: List[Concept] = []
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} looking for multiple sources that can satisfy"
+    )
+    all_found = False
+    unreachable: list[str] = []
+    # first pass
+    for opt_con in local_optional:
+        ds = gen_select_node_from_table(
+            concept,
+            [concept, opt_con],
+            g=g,
+            environment=environment,
+            depth=depth + 1,
+            accept_partial=accept_partial,
+            target_grain=Grain(components=all_concepts),
+        )
+        if not ds:
+            unreachable.append(opt_con.address)
+    all_found = False
+    for x in reversed(range(1, len(local_optional) + 1)):
+        if all_found:
+            break
+        for combo in combinations(local_optional, x):
+            if all_found:
+                break
+            # filter to just the original ones we need to get
+            local_combo = [
+                x for x in combo if x not in found and x.address not in unreachable
+            ]
+            # skip if nothing new in this combo
+            if not local_combo:
+                continue
+            # include core concept as join
+            all_concepts = [concept, *local_combo]
+
+            ds = gen_select_node_from_table(
+                concept,
+                all_concepts,
+                g=g,
+                environment=environment,
+                depth=depth + 1,
+                accept_partial=accept_partial,
+                target_grain=Grain(components=all_concepts),
+            )
+            if ds:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found a source with {[x.address for x in all_concepts]}"
+                )
+                parents.append(ds)
+                found += [x for x in ds.output_concepts if x != concept]
+                if {x.address for x in found} == {c.address for c in local_optional}:
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]}"
+                    )
+                    all_found = True
+    return all_found, found, parents
+
+
 def gen_select_node(
     concept: Concept,
     local_optional: List[Concept],
@@ -221,46 +494,9 @@ def gen_select_node(
         )
         return ds
     # if we cannot find a match
-    parents: List[StrategyNode] = []
-    found: List[Concept] = []
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} looking for multiple sources that can satisfy"
+    all_found, found, parents = gen_select_nodes_from_tables_v2(
+        concept, all_concepts, g, environment, depth, target_grain, accept_partial
     )
-    all_found = False
-    for x in reversed(range(1, len(local_optional) + 1)):
-        if all_found:
-            break
-        for combo in combinations(local_optional, x):
-            if all_found:
-                break
-            # filter to just the original ones we need to get
-            local_combo = [x for x in combo if x not in found]
-            # skip if nothing new in this combo
-            if not local_combo:
-                continue
-            # include core concept as join
-            all_concepts = [concept, *local_combo]
-
-            ds = gen_select_node_from_table(
-                concept,
-                all_concepts,
-                g=g,
-                environment=environment,
-                depth=depth + 1,
-                accept_partial=accept_partial,
-                target_grain=Grain(components=all_concepts),
-            )
-            if ds:
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} found a source with {[x.address for x in all_concepts]}"
-                )
-                parents.append(ds)
-                found += [x for x in ds.output_concepts if x != concept]
-                if {x.address for x in found} == {c.address for c in local_optional}:
-                    logger.info(
-                        f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]}"
-                    )
-                    all_found = True
     if parents and (all_found or accept_partial_optional):
         if all_found:
             logger.info(
@@ -268,7 +504,7 @@ def gen_select_node(
             )
         else:
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found some optional {[c.address for c in found]}, returning"
+                f"{padding(depth)}{LOGGER_PREFIX} found some optional, returning"
             )
         all_partial = [
             c

trilogy/core/processing/nodes/__init__.py

@@ -6,11 +6,12 @@ from .window_node import WindowNode
 from .base_node import StrategyNode, NodeJoin
 from .unnest_node import UnnestNode
 from pydantic import BaseModel, Field, ConfigDict
-from trilogy.core.models import Concept
+from trilogy.core.models import Concept, Environment
 
 
 class History(BaseModel):
     history: dict[str, StrategyNode | None] = Field(default_factory=dict)
+    select_history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     started: set[str] = Field(default_factory=set)
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -60,6 +61,58 @@ class History(BaseModel):
             in self.started
         )
 
+    def _select_concepts_to_lookup(
+        self,
+        main: Concept,
+        search: list[Concept],
+        accept_partial: bool,
+        fail_if_not_found: bool,
+        accept_partial_optional: bool,
+    ) -> str:
+        return (
+            str(main.address)
+            + "|"
+            + "-".join([c.address for c in search])
+            + str(accept_partial)
+            + str(fail_if_not_found)
+            + str(accept_partial_optional)
+        )
+
+    def gen_select_node(
+        self,
+        concept: Concept,
+        local_optional: list[Concept],
+        environment: Environment,
+        g,
+        depth: int,
+        fail_if_not_found: bool = False,
+        accept_partial: bool = False,
+        accept_partial_optional: bool = False,
+    ) -> StrategyNode | None:
+        from trilogy.core.processing.node_generators.select_node import gen_select_node
+
+        fingerprint = self._select_concepts_to_lookup(
+            concept,
+            local_optional,
+            accept_partial,
+            fail_if_not_found,
+            accept_partial_optional,
+        )
+        if fingerprint in self.select_history:
+            return self.select_history[fingerprint]
+        gen = gen_select_node(
+            concept,
+            local_optional,
+            environment,
+            g,
+            depth + 1,
+            fail_if_not_found=fail_if_not_found,
+            accept_partial=accept_partial,
+            accept_partial_optional=accept_partial_optional,
+        )
+        self.select_history[fingerprint] = gen
+        return gen
+
 
 __all__ = [
     "FilterNode",

trilogy/parsing/parse_engine.py

@@ -9,6 +9,7 @@ from lark.exceptions import (
     UnexpectedToken,
     VisitError,
 )
+from pathlib import Path
 from lark.tree import Meta
 from pydantic import ValidationError
 from trilogy.core.internal import INTERNAL_NAMESPACE, ALL_ROWS_CONCEPT
@@ -466,25 +467,34 @@ class ParseToObjects(Transformer):
         text,
         environment: Environment,
         parse_address: str | None = None,
-        parsed: dict | None = None,
+        parsed: dict[str, "ParseToObjects"] | None = None,
     ):
         Transformer.__init__(self, visit_tokens)
         self.text = text
         self.environment: Environment = environment
-        self.imported: set[str] = set()
         self.parse_address = parse_address or "root"
         self.parsed: dict[str, ParseToObjects] = parsed if parsed else {}
         # we do a second pass to pick up circular dependencies
         # after initial parsing
         self.pass_count = 1
+        self._results_stash = None
+
+    def transform(self, tree):
+        results = super().transform(tree)
+        self._results_stash = results
+        self.environment._parse_count += 1
+        return results
 
     def hydrate_missing(self):
         self.pass_count = 2
         for k, v in self.parsed.items():
+
             if v.pass_count == 2:
                 continue
             v.hydrate_missing()
         self.environment.concepts.fail_on_missing = True
+        # if not self.environment.concepts.undefined:
+        #     return self._results_stash
         reparsed = self.transform(PARSER.parse(self.text))
         self.environment.concepts.undefined = {}
         return reparsed
@@ -932,7 +942,7 @@ class ParseToObjects(Transformer):
         )
         for column in columns:
             column.concept = column.concept.with_grain(datasource.grain)
-        self.environment.datasources[datasource.identifier] = datasource
+        self.environment.add_datasource(datasource, meta=meta)
         return datasource
 
     @v_args(meta=True)
@@ -1046,12 +1056,11 @@ class ParseToObjects(Transformer):
         self.environment.add_concept(new, meta=meta)
         return merge
 
-    def import_statement(self, args: list[str]):
+    def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
-        self.imported.add(target)
         if target in self.parsed:
             nparser = self.parsed[target]
         else:
@@ -1070,21 +1079,23 @@ class ParseToObjects(Transformer):
                 )
                 nparser.transform(PARSER.parse(text))
                 self.parsed[target] = nparser
+                # add the parsed objects of the import in
+                self.parsed = {**self.parsed, **nparser.parsed}
             except Exception as e:
                 raise ImportError(
                     f"Unable to import file {dirname(target)}, parsing error: {e}"
                 )
 
-        for key, concept in nparser.environment.concepts.items():
-            # self.environment.concepts[f"{alias}.{key}"] = concept.with_namespace(new_namespace)
+        for _, concept in nparser.environment.concepts.items():
             self.environment.add_concept(concept.with_namespace(alias))
 
-        for key, datasource in nparser.environment.datasources.items():
+        for _, datasource in nparser.environment.datasources.items():
             self.environment.add_datasource(datasource.with_namespace(alias))
-            # self.environment.datasources[f"{alias}.{key}"] = datasource.with_namespace(new_namespace)
-
-        self.environment.imports[alias] = ImportStatement(alias=alias, path=args[0])
-        return None
+        imps = ImportStatement(
+            alias=alias, path=Path(args[0]), environment=nparser.environment
+        )
+        self.environment.imports[alias] = imps
+        return imps
 
     @v_args(meta=True)
     def show_category(self, meta: Meta, args) -> ShowCategory:

trilogy/scripts/trilogy.py

@@ -115,7 +115,7 @@ def run(ctx, input, dialect: str, conn_args):
 
             print_tabulate(results, tabulate.tabulate)
         except ImportError:
-            print('Install tabulate (pip install tabulate) for a prettier output')
+            print("Install tabulate (pip install tabulate) for a prettier output")
             print(", ".join(results.keys()))
             for row in results:
                 print(row)