pytrilogy 0.0.2.25__py3-none-any.whl → 0.0.2.26__py3-none-any.whl

{pytrilogy-0.0.2.25.dist-info → pytrilogy-0.0.2.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.25
+Version: 0.0.2.26
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.25.dist-info → pytrilogy-0.0.2.26.dist-info}/RECORD

@@ -1,8 +1,8 @@
-trilogy/__init__.py,sha256=5XwNYAlRMOuSKIFUPwSVWUdNK1RpSEPCsc_H7W06R7w,291
+trilogy/__init__.py,sha256=Cc2rIa67kJZaRPTgItA7vo9mKdPdLIskVp1BDwgHWbc,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=rHCe0Pe3LuB-VwCr2765QhzkUrTqZKEYPJ7rS0ykxYw,1273
+trilogy/constants.py,sha256=KiyYnctoZen4Hzv8WG2jeN-IE-dfQbWHdVCUeTZYjBg,1270
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=b2pUL_Ha1H7pyhqssc2-hTd0OUO2KIcS0x6BLMPckZw,11822
+trilogy/executor.py,sha256=Gd9KRT1rNAQyF1oDtKMcidg6XWqGMBhPnErrzFpf7Ew,12139
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
@@ -16,9 +16,9 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=IhVpt3n6wEanKHnGu3oA2w6-hKIlxWpEyz7fHN66mpo,10720
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=LUaoxk4twHRY9_Qatdbo1GjjCZRo_91Hql7BwKjLbfM,156934
+trilogy/core/models.py,sha256=ZPMOWmN4vDvXyLZvyiaN-WZnMDukDwr2nJYFIe6vJKo,158251
 trilogy/core/optimization.py,sha256=od_60A9F8J8Nj24MHgrxl4vwRwmBFH13TMdoMQvgVKs,7717
-trilogy/core/query_processor.py,sha256=sdG0XcHNBS0kuqUPztDZ1i-kpDV5LJLrO55Og2Y8hSg,17140
+trilogy/core/query_processor.py,sha256=-fKPlygk3aX1cY60dl4tKNQofKRFl3zhqz5klRIbtq0,17683
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
@@ -27,17 +27,17 @@ trilogy/core/optimizations/predicate_pushdown.py,sha256=1l9WnFOSv79e341typG3tTdk
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=7MT_x6QFHrbSDmjz21pYdQB5ux419ES4QS-8lO16eyw,36091
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=hzuEsNqP5dq1GBT96lWYX1UkakMl6XItp-nIKFH6wLg,19617
+trilogy/core/processing/utility.py,sha256=xNl310JhBda0Vv1SSwg5EtMPmMDAWucxfWd7M53Sb9k,17422
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=WQNgJ1MwrMS_BQ-b3XwGGB6eToDykelAVj_fesJuqe0,2069
-trilogy/core/processing/node_generators/common.py,sha256=LwDgPlhWeuw0t07f3kX9IE5LXBdZhXfh-aY0XGk50ak,8946
+trilogy/core/processing/node_generators/common.py,sha256=eslHTTPFTkmwHwKIuUsbFn54jxj-Avtt-QScqtNwzdg,8945
 trilogy/core/processing/node_generators/filter_node.py,sha256=Vz9Rb67e1dfZgnliekwwLeDPVkthMbdrnrKRdz7J1ik,7654
 trilogy/core/processing/node_generators/group_node.py,sha256=r54IVEhXW-tzod6uEHIQObrxgQt6aNySk5emWkWyqCU,4938
 trilogy/core/processing/node_generators/group_to_node.py,sha256=R9i_wHipxjXJyfYEwfeTw2EPpuanXVA327XyfcP2tBg,2537
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=_KO9lqzHQoy4VAviO0ttQlmK0tjaqrJj4SJPhmoIYm8,6229
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=dIEv5P2MTViAES2MzqJgccYzM3HldjHrQYFwH00cqyc,14003
 trilogy/core/processing/node_generators/rowset_node.py,sha256=KtdN6t2xM8CJxobc4aQX4W8uX98U6IabeuBF_FtBLR4,4583
-trilogy/core/processing/node_generators/select_merge_node.py,sha256=MKjlXqFBSin6cTnS6n5lEcNBJsMvSefDIXOwYNVbM0s,10371
+trilogy/core/processing/node_generators/select_merge_node.py,sha256=z1sF04MQsMYbLjE__co5Nwi5hTvMeTZACzTnuBe7lsk,11341
 trilogy/core/processing/node_generators/select_node.py,sha256=nwXHQF6C-aQUIelx9dyxN2pK3muL-4-6RIqnqQqNwtw,1808
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cZ26CN338CBnd6asML1OBUtNcDzmNlFpY0Vnade4yrc,2256
 trilogy/core/processing/node_generators/window_node.py,sha256=jy3FF8uN0VA7yyrBeR40B9CAqR_5qBP4PiS6Gr-f-7w,2590
@@ -45,14 +45,14 @@ trilogy/core/processing/nodes/__init__.py,sha256=qS5EJDRwwIrCEfS7ibCA2ESE0RPzsAI
 trilogy/core/processing/nodes/base_node.py,sha256=sc3HrXkWk-xpsAQ7B7ltX1ZejYAkqFiv8Ei8Jg5VGkQ,15579
 trilogy/core/processing/nodes/filter_node.py,sha256=GfZ9eghpFDI-s7iQP2UqTljCmn25LT_T5TAxDlh7PkQ,2343
 trilogy/core/processing/nodes/group_node.py,sha256=PrBHaGq_f8RmokUw9lXLGJ5YbjdP77P7Ag0pgR6e2cU,7293
-trilogy/core/processing/nodes/merge_node.py,sha256=wlqh0suFfZBJOgkn7vy0OiDs5jza3NCX7eHTEEb6mBQ,14799
+trilogy/core/processing/nodes/merge_node.py,sha256=2BjE2bTyoMHLfn_pnl1fioJkm1AfWtVKnuzzL4aWS5I,14799
 trilogy/core/processing/nodes/select_node_v2.py,sha256=gS9OQgS2TSEK59BQ9R0i83pTHfGJUxv7AkAmT21sYxI,8067
 trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=ro0QfMFi4ZmIn5Q4D0M_vJWfnHH_C0MN7XkVkx8Gygg,1214
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=zqPBWv5gsnk4KhyzHdxcpDkVXU7luEVvHepUWRzjUfo,34019
+trilogy/dialect/base.py,sha256=JpC-ig7iRzZxcuHmKRInKfFAoNTzFOcclAvmvUJiWOo,34087
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
-trilogy/dialect/common.py,sha256=Hr0mxcNxjSvhpBM5Wvb_Q7aklAuYj5aBDrW433py0Zs,4403
+trilogy/dialect/common.py,sha256=Pm9naWL5eo-BfZTYFMWhX9lpsGuxEyvvn_EuXbFeos0,3817
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=_0a5HBU8zRNtZj7YED3ju4fHXRYG9jNeKwnlZwUDvwI,3419
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
@@ -66,18 +66,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=Pe-Kw1JGngeLqQOMQb0E3-24jXEavqnPCQ-KOfTfjP8,4357
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=kbqWy30nnVc7ID-sdSDwxYomnxd3guyuIJF3yvlpQwg,9960
+trilogy/parsing/common.py,sha256=t7yiL_3f6rz_rouF9et84v5orAgs-EprV4V9ghQ6ql4,10024
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=tcBgjfew0kAfSEt1aFo9Pu3yacEBB1KFm7v_Iobz52g,64467
-trilogy/parsing/render.py,sha256=7mEEe5DWVAafaGl__oQE7FPn_4QhcsGT2VVp-nk1Lr8,13078
-trilogy/parsing/trilogy.lark,sha256=ZP9USPgD8-Fq5UzIl4iGpAeGuh2JLGzSoYJhvEGOi2c,12188
+trilogy/parsing/parse_engine.py,sha256=nyNyClknlHVYeHHGrSXWDAuV_E_XQSvFLUZqZ2q97kE,64513
+trilogy/parsing/render.py,sha256=_Jb1or0XFmrj2mHHv7My1VNdWkcpOAYWnRwFW2sh4U0,14052
+trilogy/parsing/trilogy.lark,sha256=NZgFchImZsQ3fyyBh8kwq8esTQOR5QlZ9n6k-F5H8nI,12184
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.25.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.25.dist-info/METADATA,sha256=GtN5gVcDusbMKaXTn5O_CLcZAk00h5Xe5df4i5QqZzs,8403
-pytrilogy-0.0.2.25.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-pytrilogy-0.0.2.25.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.25.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.25.dist-info/RECORD,,
+pytrilogy-0.0.2.26.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.26.dist-info/METADATA,sha256=1tD8kmlqzPcPdU5SiPHkiiSvXXwSqEm4sBvjp1LIDY4,8403
+pytrilogy-0.0.2.26.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+pytrilogy-0.0.2.26.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.26.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.26.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.25"
+__version__ = "0.0.2.26"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -7,7 +7,7 @@ logger = getLogger("trilogy")
 
 DEFAULT_NAMESPACE = "local"
 
-VIRTUAL_CONCEPT_PREFIX = "_virtual"
+VIRTUAL_CONCEPT_PREFIX = "_virt"
 
 ENV_CACHE_NAME = ".preql_cache.json"
 

trilogy/core/models.py

@@ -73,6 +73,7 @@ from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 from collections import defaultdict
+import hashlib
 
 LOGGER_PREFIX = "[MODELS]"
 
@@ -190,6 +191,13 @@ class ConstantInlineable(ABC):
         raise NotImplementedError
 
 
+class HasUUID(ABC):
+
+    @property
+    def uuid(self) -> str:
+        return hashlib.md5(str(self).encode()).hexdigest()
+
+
 class SelectTypeMixin(BaseModel):
     where_clause: Union["WhereClause", None] = Field(default=None)
     having_clause: Union["HavingClause", None] = Field(default=None)
@@ -1606,7 +1614,7 @@ class RawSQLStatement(BaseModel):
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
 
-class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
+class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel):
     selection: List[SelectItem]
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
@@ -1724,19 +1732,34 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
             # if the concept is a locally derived concept, it cannot ever be partial
             # but if it's a concept pulled in from upstream and we have a where clause, it should be partial
             ColumnAssignment(
-                alias=c.address.replace(".", "_"),
+                alias=(
+                    c.name.replace(".", "_")
+                    if c.namespace == DEFAULT_NAMESPACE
+                    else c.address.replace(".", "_")
+                ),
                 concept=c,
                 modifiers=modifiers if c.address not in self.locally_derived else [],
             )
             for c in self.output_components
         ]
 
+        condition = None
+        if self.where_clause:
+            condition = self.where_clause.conditional
+        if self.having_clause:
+            if condition:
+                condition = self.having_clause.conditional + condition
+            else:
+                condition = self.having_clause.conditional
+
         new_datasource = Datasource(
             identifier=identifier,
             address=address,
             grain=grain or self.grain,
             columns=columns,
             namespace=namespace,
+            where=WhereClause(conditional=condition) if condition else None,
+            non_partial_for=WhereClause(conditional=condition) if condition else None,
         )
         for column in columns:
             column.concept = column.concept.with_grain(new_datasource.grain)
@@ -1859,7 +1882,7 @@ class AlignClause(Namespaced, BaseModel):
         return AlignClause(items=[x.with_namespace(namespace) for x in self.items])
 
 
-class MultiSelectStatement(SelectTypeMixin, Mergeable, Namespaced, BaseModel):
+class MultiSelectStatement(HasUUID, SelectTypeMixin, Mergeable, Namespaced, BaseModel):
     selects: List[SelectStatement]
     align: AlignClause
     namespace: str
@@ -2021,7 +2044,7 @@ class DatasourceMetadata(BaseModel):
     line_no: int | None = None
 
 
-class MergeStatementV2(Namespaced, BaseModel):
+class MergeStatementV2(HasUUID, Namespaced, BaseModel):
     source: Concept
     target: Concept
     modifiers: List[Modifier] = Field(default_factory=list)
@@ -2035,7 +2058,7 @@ class MergeStatementV2(Namespaced, BaseModel):
         return new
 
 
-class Datasource(Namespaced, BaseModel):
+class Datasource(HasUUID, Namespaced, BaseModel):
     identifier: str
     columns: List[ColumnAssignment]
     address: Union[Address, str]
@@ -2047,6 +2070,7 @@ class Datasource(Namespaced, BaseModel):
         default_factory=lambda: DatasourceMetadata(freshness_concept=None)
     )
     where: Optional[WhereClause] = None
+    non_partial_for: Optional[WhereClause] = None
 
     def merge_concept(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
@@ -2247,6 +2271,7 @@ class InstantiatedUnnestJoin(BaseModel):
 class ConceptPair(BaseModel):
     left: Concept
     right: Concept
+    existing_datasource: Union[Datasource, "QueryDatasource"]
     modifiers: List[Modifier] = Field(default_factory=list)
 
     @property
@@ -2258,17 +2283,23 @@ class ConceptPair(BaseModel):
         return Modifier.NULLABLE in self.modifiers
 
 
+class CTEConceptPair(ConceptPair):
+    cte: CTE
+
+
 class BaseJoin(BaseModel):
-    left_datasource: Union[Datasource, "QueryDatasource"]
     right_datasource: Union[Datasource, "QueryDatasource"]
-    concepts: List[Concept]
     join_type: JoinType
-    filter_to_mutual: bool = False
+    concepts: Optional[List[Concept]] = None
+    left_datasource: Optional[Union[Datasource, "QueryDatasource"]] = None
     concept_pairs: list[ConceptPair] | None = None
 
     def __init__(self, **data: Any):
         super().__init__(**data)
-        if self.left_datasource.full_name == self.right_datasource.full_name:
+        if (
+            self.left_datasource
+            and self.left_datasource.full_name == self.right_datasource.full_name
+        ):
             raise SyntaxError(
                 f"Cannot join a dataself to itself, joining {self.left_datasource} and"
                 f" {self.right_datasource}"
@@ -2278,8 +2309,10 @@ class BaseJoin(BaseModel):
         # if we have a list of concept pairs
         if self.concept_pairs:
             return
-
-        for concept in self.concepts:
+        if self.concepts == []:
+            return
+        assert self.left_datasource and self.right_datasource
+        for concept in self.concepts or []:
             include = True
             for ds in [self.left_datasource, self.right_datasource]:
                 synonyms = []
@@ -2289,13 +2322,10 @@ class BaseJoin(BaseModel):
                     concept.address not in [c.address for c in ds.output_concepts]
                     and concept.address not in synonyms
                 ):
-                    if self.filter_to_mutual:
-                        include = False
-                    else:
-                        raise SyntaxError(
-                            f"Invalid join, missing {concept} on {ds.name}, have"
-                            f" {[c.address for c in ds.output_concepts]}"
-                        )
+                    raise SyntaxError(
+                        f"Invalid join, missing {concept} on {ds.name}, have"
+                        f" {[c.address for c in ds.output_concepts]}"
+                    )
             if include:
                 final_concepts.append(concept)
         if not final_concepts and self.concepts:
@@ -2312,7 +2342,7 @@ class BaseJoin(BaseModel):
                     self.concepts = []
                     return
                 # if everything is at abstract grain, we can skip joins
-                if all([c.grain == Grain() for c in ds.output_concepts]):
+                if all([c.grain.abstract for c in ds.output_concepts]):
                     self.concepts = []
                     return
 
@@ -2330,21 +2360,27 @@ class BaseJoin(BaseModel):
 
     @property
     def unique_id(self) -> str:
-        # TODO: include join type?
-        return (
-            self.left_datasource.name
-            + self.right_datasource.name
-            + self.join_type.value
-        )
+        return str(self)
+
+    @property
+    def input_concepts(self) -> List[Concept]:
+        base = []
+        if self.concept_pairs:
+            for pair in self.concept_pairs:
+                base += [pair.left, pair.right]
+        elif self.concepts:
+            base += self.concepts
+        return base
 
     def __str__(self):
         if self.concept_pairs:
             return (
-                f"{self.join_type.value} on"
-                f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
+                f"{self.join_type.value} {self.right_datasource.name} on"
+                f" {','.join([str(k.existing_datasource.name) + '.'+ str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
-            f"{self.join_type.value} on" f" {','.join([str(k) for k in self.concepts])}"
+            f"{self.join_type.value} {self.right_datasource.name} on"
+            f" {','.join([str(k) for k in self.concepts])}"
         )
 
 
@@ -2389,19 +2425,9 @@ class QueryDatasource(BaseModel):
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
-            if join.left_datasource.identifier == join.right_datasource.identifier:
-                raise SyntaxError(
-                    f"Cannot join a datasource to itself, joining {join.left_datasource}"
-                )
-            pairing = "".join(
-                sorted(
-                    [join.left_datasource.identifier, join.right_datasource.identifier]
-                )
-            )
+            pairing = str(join)
             if pairing in unique_pairs:
-                raise SyntaxError(
-                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
-                )
+                raise SyntaxError(f"Duplicate join {str(join)}")
             unique_pairs.add(pairing)
         return v
 
@@ -2666,7 +2692,12 @@ class CTE(BaseModel):
                         isinstance(join, Join)
                         and (
                             join.right_cte.name != removed_cte
-                            and join.left_cte.name != removed_cte
+                            and any(
+                                [
+                                    x.cte.name != removed_cte
+                                    for x in (join.joinkey_pairs or [])
+                                ]
+                            )
                         )
                     )
                 ]
@@ -2737,8 +2768,12 @@ class CTE(BaseModel):
         for join in self.joins:
             if isinstance(join, InstantiatedUnnestJoin):
                 continue
-            if join.left_cte.name == parent.name:
+            if join.left_cte and join.left_cte.name == parent.name:
                 join.inline_cte(parent)
+            if join.joinkey_pairs:
+                for pair in join.joinkey_pairs:
+                    if pair.cte and pair.cte.name == parent.name:
+                        join.inline_cte(parent)
             if join.right_cte.name == parent.name:
                 join.inline_cte(parent)
         for k, v in self.source_map.items():
@@ -2961,21 +2996,26 @@ class JoinKey(BaseModel):
 
 
 class Join(BaseModel):
-    left_cte: CTE
+
     right_cte: CTE
     jointype: JoinType
-    joinkeys: List[JoinKey]
-    joinkey_pairs: List[ConceptPair] | None = None
+    left_cte: CTE | None = None
+    joinkey_pairs: List[CTEConceptPair] | None = None
     inlined_ctes: set[str] = Field(default_factory=set)
 
     def inline_cte(self, cte: CTE):
         self.inlined_ctes.add(cte.name)
 
-    @property
-    def left_name(self) -> str:
-        if self.left_cte.name in self.inlined_ctes:
-            return self.left_cte.source.datasources[0].identifier
-        return self.left_cte.name
+    # @property
+    # def left_name(self) -> str:
+    #     if self.left_cte.name in self.inlined_ctes:
+    #         return self.left_cte.source.datasources[0].identifier
+    #     return self.left_cte.name
+
+    def get_name(self, cte: CTE):
+        if cte.name in self.inlined_ctes:
+            return cte.source.datasources[0].identifier
+        return cte.name
 
     @property
     def right_name(self) -> str:
@@ -2983,12 +3023,6 @@ class Join(BaseModel):
             return self.right_cte.source.datasources[0].identifier
         return self.right_cte.name
 
-    @property
-    def left_ref(self) -> str:
-        if self.left_cte.name in self.inlined_ctes:
-            return f"{self.left_cte.source.datasources[0].safe_location} as {self.left_cte.source.datasources[0].identifier}"
-        return self.left_cte.name
-
     @property
     def right_ref(self) -> str:
         if self.right_cte.name in self.inlined_ctes:
@@ -2997,19 +3031,21 @@ class Join(BaseModel):
 
     @property
     def unique_id(self) -> str:
-        return self.left_name + self.right_name + self.jointype.value
+        return str(self)
 
     def __str__(self):
         if self.joinkey_pairs:
             return (
-                f"{self.jointype.value} JOIN {self.left_name} and"
+                f"{self.jointype.value} join"
                 f" {self.right_name} on"
-                f" {','.join([str(k.left)+'='+str(k.right)+str(k.modifiers) for k in self.joinkey_pairs])}"
+                f" {','.join([k.cte.name + '.'+str(k.left.address)+'='+str(k.right.address) for k in self.joinkey_pairs])}"
             )
-        return (
-            f"{self.jointype.value} JOIN {self.left_name} and"
-            f" {self.right_name} on {','.join([str(k) for k in self.joinkeys])}"
-        )
+        elif self.left_cte:
+            return (
+                f"{self.jointype.value} JOIN {self.left_cte.name} and"
+                f" {self.right_name} on {','.join([str(k) for k in self.joinkey_pairs])}"
+            )
+        return f"{self.jointype.value} JOIN  {self.right_name} on {','.join([str(k) for k in self.joinkey_pairs])}"
 
 
 class UndefinedConcept(Concept, Mergeable, Namespaced):
@@ -3227,7 +3263,7 @@ class EnvironmentConceptDict(dict):
         return super().items()
 
 
-class ImportStatement(BaseModel):
+class ImportStatement(HasUUID, BaseModel):
     alias: str
     path: Path
     environment: Union["Environment", None] = None
@@ -4223,6 +4259,9 @@ class AggregateWrapper(Mergeable, Namespaced, SelectContext, BaseModel):
 class WhereClause(Mergeable, ConceptArgs, Namespaced, SelectContext, BaseModel):
     conditional: Union[SubselectComparison, Comparison, Conditional, "Parenthetical"]
 
+    def __repr__(self):
+        return str(self.conditional)
+
     @property
     def input(self) -> List[Concept]:
         return self.conditional.input
@@ -4341,7 +4380,7 @@ class Limit(BaseModel):
     count: int
 
 
-class ConceptDeclarationStatement(BaseModel):
+class ConceptDeclarationStatement(HasUUID, BaseModel):
     concept: Concept
 
 
@@ -4349,7 +4388,7 @@ class ConceptDerivation(BaseModel):
     concept: Concept
 
 
-class RowsetDerivationStatement(Namespaced, BaseModel):
+class RowsetDerivationStatement(HasUUID, Namespaced, BaseModel):
     name: str
     select: SelectStatement | MultiSelectStatement
     namespace: str
@@ -4614,7 +4653,7 @@ class TupleWrapper(Generic[VT], tuple):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
-class PersistStatement(BaseModel):
+class PersistStatement(HasUUID, BaseModel):
     datasource: Datasource
     select: SelectStatement
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())

trilogy/core/processing/node_generators/common.py

@@ -1,6 +1,5 @@
 from typing import List, Tuple, Callable
 
-
 from trilogy.core.enums import PurposeLineage, Purpose
 from trilogy.core.models import (
     Concept,

trilogy/core/processing/node_generators/select_merge_node.py

@@ -28,14 +28,18 @@ def extract_address(node: str):
     return node.split("~")[1].split("@")[0]
 
 
-def get_graph_partial_nodes(g: nx.DiGraph) -> dict[str, list[str]]:
+def get_graph_partial_nodes(
+    g: nx.DiGraph, conditions: WhereClause | None
+) -> dict[str, list[str]]:
     datasources: dict[str, Datasource] = nx.get_node_attributes(g, "datasource")
     partial: dict[str, list[str]] = {}
     for node in g.nodes:
         if node in datasources:
-            partial[node] = [
-                concept_to_node(c) for c in datasources[node].partial_concepts
-            ]
+            ds = datasources[node]
+            partial[node] = [concept_to_node(c) for c in ds.partial_concepts]
+            if ds.non_partial_for and conditions == ds.non_partial_for:
+                partial[node] = []
+
     return partial
 
 
@@ -49,7 +53,10 @@ def get_graph_grain_length(g: nx.DiGraph) -> dict[str, int]:
 
 
 def create_pruned_concept_graph(
-    g: nx.DiGraph, all_concepts: List[Concept], accept_partial: bool = False
+    g: nx.DiGraph,
+    all_concepts: List[Concept],
+    accept_partial: bool = False,
+    conditions: WhereClause | None = None,
 ) -> nx.DiGraph:
     orig_g = g
     g = g.copy()
@@ -66,11 +73,7 @@ def create_pruned_concept_graph(
     relevent_datasets: list[str] = []
     if not accept_partial:
         partial = {}
-        for node in g.nodes:
-            if node in datasources:
-                partial[node] = [
-                    concept_to_node(c) for c in datasources[node].partial_concepts
-                ]
+        partial = get_graph_partial_nodes(g, conditions)
         to_remove = []
         for edge in g.edges:
             if (
@@ -133,31 +136,53 @@ def create_pruned_concept_graph(
     return g
 
 
-def resolve_subgraphs(g: nx.DiGraph) -> dict[str, list[str]]:
+def resolve_subgraphs(
+    g: nx.DiGraph, conditions: WhereClause | None
+) -> dict[str, list[str]]:
     datasources = [n for n in g.nodes if n.startswith("ds~")]
-    subgraphs = {ds: list(set(list(nx.all_neighbors(g, ds)))) for ds in datasources}
-    partial_map = get_graph_partial_nodes(g)
+    subgraphs: dict[str, list[str]] = {
+        ds: list(set(list(nx.all_neighbors(g, ds)))) for ds in datasources
+    }
+    partial_map = get_graph_partial_nodes(g, conditions)
     grain_length = get_graph_grain_length(g)
-    non_partial = {
-        ds: [c for c in subgraphs[ds] if c not in partial_map[ds]] for ds in datasources
+    concepts: dict[str, Concept] = nx.get_node_attributes(g, "concept")
+    non_partial_map = {
+        ds: [concepts[c].address for c in subgraphs[ds] if c not in partial_map[ds]]
+        for ds in datasources
+    }
+    concept_map = {
+        ds: [concepts[c].address for c in subgraphs[ds]] for ds in datasources
     }
     pruned_subgraphs = {}
-    for key, value in subgraphs.items():
+    for key, nodes in subgraphs.items():
+        value = non_partial_map[key]
+        all_concepts = concept_map[key]
         is_subset = False
         matches = set()
         # Compare current list with other lists
-        for other_key, other_value in non_partial.items():
-            if key != other_key and set(value).issubset(set(other_value)):
+        for other_key, other_all_concepts in concept_map.items():
+            other_value = non_partial_map[other_key]
+            # needs to be a subset of non partial and a subset of all
+            if (
+                key != other_key
+                and set(value).issubset(set(other_value))
+                and set(all_concepts).issubset(set(other_all_concepts))
+            ):
                 if len(value) < len(other_value):
                     is_subset = True
+                    logger.debug(
+                        f"Dropping subgraph {key} with {value} as it is a subset of {other_key} with {other_value}"
+                    )
                     break
-                elif len(value) == len(other_value):
+                elif len(value) == len(other_value) and len(all_concepts) == len(
+                    other_all_concepts
+                ):
                     matches.add(other_key)
                     matches.add(key)
         if matches:
             is_subset = key is not min(matches, key=lambda x: (grain_length[x], x))
         if not is_subset:
-            pruned_subgraphs[key] = value
+            pruned_subgraphs[key] = nodes
     return pruned_subgraphs
 
 
@@ -261,7 +286,9 @@ def gen_select_merge_node(
             force_group=False,
         )
     for attempt in [False, True]:
-        pruned_concept_graph = create_pruned_concept_graph(g, non_constant, attempt)
+        pruned_concept_graph = create_pruned_concept_graph(
+            g, non_constant, attempt, conditions
+        )
         if pruned_concept_graph:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} found covering graph w/ partial flag {attempt}"
@@ -274,7 +301,7 @@ def gen_select_merge_node(
         )
         return None
 
-    sub_nodes = resolve_subgraphs(pruned_concept_graph)
+    sub_nodes = resolve_subgraphs(pruned_concept_graph, conditions)
 
     logger.info(f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {sub_nodes}")
     parents = [

trilogy/core/processing/nodes/merge_node.py

@@ -215,7 +215,7 @@ class MergeNode(StrategyNode):
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} inferring node joins to target grain {str(grain)}"
                 )
-                joins = get_node_joins(dataset_list, grain.components, environment)
+                joins = get_node_joins(dataset_list, environment=environment)
         elif final_joins:
             logger.info(
                 f"{self.logging_prefix}{LOGGER_PREFIX} translating provided node joins {len(final_joins)}"
@@ -314,7 +314,7 @@ class MergeNode(StrategyNode):
         full_join_concepts = []
         for join in joins:
             if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
-                full_join_concepts += join.concepts
+                full_join_concepts += join.input_concepts
         if self.whole_grain:
             force_group = False
         elif self.force_group is False: