pytrilogy 0.0.3.64__py3-none-any.whl → 0.0.3.66__py3-none-any.whl

{pytrilogy-0.0.3.64.dist-info → pytrilogy-0.0.3.66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.64
+Version: 0.0.3.66
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.64.dist-info → pytrilogy-0.0.3.66.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.64.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=qOtWjDJ0j0xbQBLf8qO2rhFCUnbYN4Ds9lNcOIOte1k,303
+pytrilogy-0.0.3.66.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=aRLUllOSoHKDIW__yBx1aitL_E96Qtj4AGz-mFHB16g,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=lv_aJWP6dn6e2aF4BAE72jbnNtceFddfqtiDSsvzno0,1692
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -17,19 +17,19 @@ trilogy/core/environment_helpers.py,sha256=VvPIiFemqaLLpIpLIqprfu63K7muZ1YzNg7UZ
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
 trilogy/core/functions.py,sha256=poVfAwet1xdxTkC7WL38UmGRDpUVO9iSMNWSagl9_r4,29302
-trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
+trilogy/core/graph_models.py,sha256=wIT-oBchHWE46GLDkgN5K7EzhOBEo8LfaeWV5G5cYcE,3302
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
-trilogy/core/query_processor.py,sha256=QiE_w5HgheT4GLZFnaLssJ4plf4voK0TeTd6N3jhR6A,20188
+trilogy/core/query_processor.py,sha256=kBvQex0Xi2-u8DvAbAo_LXFTMKzMkFvwL-SCrIXU41k,20310
 trilogy/core/utility.py,sha256=3VC13uSQWcZNghgt7Ot0ZTeEmNqs__cx122abVq9qhM,410
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=8XbIsQr6cQrgo9uzee5qRoYiMdEG7yKF4FiiWImW7U0,77490
-trilogy/core/models/build.py,sha256=80v9rxwl41O8_7uIJoHK4tnTUfgR6u8EPrwHg4ySqO4,63323
+trilogy/core/models/author.py,sha256=44VyEInWied287lJp99T14Ahq8RSQK4q3Y-AzAByMTY,77818
+trilogy/core/models/build.py,sha256=yR3GtkIgLrT88B_wu9geqHWxnEI4joUWQSYW7_9Gjk0,63489
 trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
 trilogy/core/models/core.py,sha256=EMAuWTngoNVGCdfNrAY7_k6g528iodNQLwPRVip-8DA,10980
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
-trilogy/core/models/environment.py,sha256=tM8SwH9r1ZSc-F0Enod7r1NuqzGpQfLcPxnrtK5Hqpk,27398
-trilogy/core/models/execute.py,sha256=hOilC-lka4W-C8Pakb0Vd1-T0oskeWdC8Ls0bm8_388,43109
+trilogy/core/models/environment.py,sha256=7bkxUob5pNgvK7Om-qvlJgsDiCh5iSPlMHI7tN_OZhU,27717
+trilogy/core/models/execute.py,sha256=94CZVY_EdW675n8SihDnhaGTV0dq7BAAsl2Anf1mPmk,41815
 trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
@@ -49,17 +49,17 @@ trilogy/core/processing/node_generators/filter_node.py,sha256=0hdfiS2I-Jvr6P-il3
 trilogy/core/processing/node_generators/group_node.py,sha256=nIfiMrJQEksUfqAeeA3X5PS1343y4lmPTipYuCa-rvs,6141
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=-suq5RSkI9SZLH7p69lLL5nMaUDVfsRvpIZ9wiuoRps,16647
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=-sVhRHB3NqNIOk_e7LLVhW17Rjcf82hxaHjEvpF5Q-w,16858
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=2BiSsegbRF9csJ_Xl8P_CxIm4dAAb7dF29u6v_Odr-A,6709
-trilogy/core/processing/node_generators/select_merge_node.py,sha256=Usa58nRrp-nww9g2QtihKa_5B_duY3nMCjXnqT-bIY0,21557
+trilogy/core/processing/node_generators/select_merge_node.py,sha256=3GDGi1tNIfuKO_FMrNCfp-G1c3lxdRuuufcmomYLt4s,21446
 trilogy/core/processing/node_generators/select_node.py,sha256=3dvw0d53eUtCRCUPN6J48I3qBEX1Wha7saQ_ndPu6_I,1777
 trilogy/core/processing/node_generators/synonym_node.py,sha256=CN2swdGPEP_Irx4GykHp4gyLCK0dWd2vX7PYJUGxw7w,3548
 trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
 trilogy/core/processing/node_generators/unnest_node.py,sha256=ueOQtoTf2iJHO09RzWHDFQ5iKZq2fVhGf2KAF2U2kU8,2677
 trilogy/core/processing/node_generators/window_node.py,sha256=GP3Hvkbb0TDA6ef7W7bmvQEHVH-NRIfBT_0W4fcH3g4,6529
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
+trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=Dw0pjdhuJX0p-18CjelLrIJoevAPKzZOTN9uSLNPOF4,6553
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
 trilogy/core/processing/nodes/base_node.py,sha256=p6yljFNLQsXz277c5wTATMNqsKUbsdP_3e7--tezBMw,17691
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
@@ -76,7 +76,7 @@ trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
 trilogy/core/statements/execute.py,sha256=rqfuoMuXPcH7L7TmE1dSiZ_K_A1ohB8whVMfGimZBOk,1294
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=5lmOd6MOkM0H-Yss9l6uyEJ4JTainPiEv1xVL9sEAh8,43102
+trilogy/dialect/base.py,sha256=fM5tPJA5yi6tTHUTWpHldh4RApJUmrosDorbrRQibe4,43613
 trilogy/dialect/bigquery.py,sha256=6ghCqy-k7UioIJc1EEQ7gRo_PHaO8Vm7yYbiQ-kgpzs,3629
 trilogy/dialect/common.py,sha256=hhzuMTFW9QQIP7TKLT9BlJy6lw2R03a68jKQ-7t4-2c,6070
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -89,11 +89,11 @@ trilogy/dialect/snowflake.py,sha256=LQIcHuyuGZXbxrv6sH17aLXLzw7yFVuRoE9M4doNk5k,
 trilogy/dialect/sql_server.py,sha256=z2Vg7Qvw83rbGiEFIvHHLqVWJTWiz2xs76kpQj4HdTU,3131
 trilogy/hooks/__init__.py,sha256=T3SF3phuUDPLXKGRVE_Lf9mzuwoXWyaLolncR_1kY30,144
 trilogy/hooks/base_hook.py,sha256=I_l-NBMNC7hKTDx1JgHZPVOOCvLQ36m2oIGaR5EUMXY,1180
-trilogy/hooks/graph_hook.py,sha256=4jeC3ot4u_FyqiYiHr3Eit2ktD0zPk7_pcG8JgFoqMc,4844
+trilogy/hooks/graph_hook.py,sha256=5BfR7Dt0bgEsCLgwjowgCsVkboGYfVJGOz8g9mqpnos,4756
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=yuKN3fQEtftRMZlJb0ESUX4TLOVFcAE0vw2CfImYG1A,29980
+trilogy/parsing/common.py,sha256=_-KWPP3NvNAHiB4B6jyGSnYKqlLs2XqXeTZiX4MCtWU,30861
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -104,14 +104,14 @@ trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/std/date.preql,sha256=HWZm4t4HWyxr5geWRsY05RnHBVDMci8z8YA2cu0-OOw,188
-trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
+trilogy/std/display.preql,sha256=nm7lox87Xf6lBvXCVCS6x2HskguMKzndEBucJ5pktzk,175
 trilogy/std/geography.preql,sha256=qLnHmDU5EnvjTbfqZF-NEclSYM5_e9rZra7QjV01rZ4,582
 trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=-bMV6dyofskl4Kvows-iQ4JCxjVUwsZOeWCy8JO5Ftw,135
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.64.dist-info/METADATA,sha256=-m9RjEszSqiTRWWSld1SMzy7JKn6e9SPxLUUrjsgqs8,9095
-pytrilogy-0.0.3.64.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.64.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.64.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.64.dist-info/RECORD,,
+pytrilogy-0.0.3.66.dist-info/METADATA,sha256=cQJUWN4bTjEYsCxAcAOKmkzEkkmpxB7mBmMQPx0O4Cw,9095
+pytrilogy-0.0.3.66.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.66.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.66.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.66.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.64"
+__version__ = "0.0.3.66"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/graph_models.py

@@ -1,6 +1,50 @@
 import networkx as nx
 
-from trilogy.core.models.build import BuildConcept, BuildDatasource
+from trilogy.core.models.build import BuildConcept, BuildDatasource, BuildWhereClause
+
+
+def get_graph_exact_match(
+    g: nx.DiGraph, conditions: BuildWhereClause | None
+) -> set[str]:
+    datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
+        nx.get_node_attributes(g, "datasource")
+    )
+    exact: set[str] = set()
+    for node in g.nodes:
+        if node in datasources:
+            ds = datasources[node]
+            if isinstance(ds, list):
+                exact.add(node)
+                continue
+
+            if not conditions and not ds.non_partial_for:
+                exact.add(node)
+                continue
+            elif conditions:
+                if not ds.non_partial_for:
+                    continue
+                if ds.non_partial_for and conditions == ds.non_partial_for:
+                    exact.add(node)
+                    continue
+            else:
+                continue
+
+    return exact
+
+
+def prune_sources_for_conditions(
+    g: nx.DiGraph,
+    conditions: BuildWhereClause | None,
+):
+
+    complete = get_graph_exact_match(g, conditions)
+    to_remove = []
+    for node in g.nodes:
+        if node.startswith("ds~") and node not in complete:
+            to_remove.append(node)
+
+    for node in to_remove:
+        g.remove_node(node)
 
 
 def concept_to_node(input: BuildConcept) -> str:

trilogy/core/models/author.py

@@ -271,6 +271,20 @@ class Conditional(Mergeable, ConceptArgs, Namespaced, DataTyped, BaseModel):
     right: Expr
     operator: BooleanOperator
 
+    @field_validator("left", mode="before")
+    @classmethod
+    def left_validator(cls, v, info: ValidationInfo):
+        if isinstance(v, Concept):
+            return v.reference
+        return v
+
+    @field_validator("right", mode="before")
+    @classmethod
+    def right_validator(cls, v, info: ValidationInfo):
+        if isinstance(v, Concept):
+            return v.reference
+        return v
+
     def __add__(self, other) -> "Conditional":
         if other is None:
             return self
@@ -346,7 +360,6 @@ class Conditional(Mergeable, ConceptArgs, Namespaced, DataTyped, BaseModel):
 
     @property
     def concept_arguments(self) -> Sequence[ConceptRef]:
-        """Return concepts directly referenced in where clause"""
         output = []
         output += get_concept_arguments(self.left)
         output += get_concept_arguments(self.right)
@@ -575,11 +588,11 @@ class Comparison(ConceptArgs, Mergeable, DataTyped, Namespaced, BaseModel):
         date,
         Function,
         ConceptRef,
-        "Conditional",
+        Conditional,
         DataType,
-        "Comparison",
+        Comparison,
         FunctionCallWrapper,
-        "Parenthetical",
+        Parenthetical,
         MagicConstants,
         WindowItem,
         AggregateWrapper,

trilogy/core/models/build.py

@@ -1631,7 +1631,10 @@ class Factory:
 
     @build.register
     def _(self, base: CaseElse) -> BuildCaseElse:
-        return BuildCaseElse.model_construct(expr=self.build(base.expr))
+        expr: Concept | FuncArgs = base.expr
+        if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
+            expr, _ = self.instantiate_concept(expr)
+        return BuildCaseElse.model_construct(expr=self.build(expr))
 
     @build.register
     def _(self, base: Concept) -> BuildConcept:

trilogy/core/models/environment.py

@@ -603,7 +603,7 @@ class Environment(BaseModel):
             # too hacky for maintainability
             if current_derivation not in (Derivation.ROOT, Derivation.CONSTANT):
                 logger.info(
-                    f"A datasource has been added which will persist derived concept {new_persisted_concept.address}"
+                    f"A datasource has been added which will persist derived concept {new_persisted_concept.address} with derivation {current_derivation}"
                 )
                 persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + new_persisted_concept.name
                 # override the current concept source to reflect that it's now coming from a datasource
@@ -622,17 +622,21 @@ class Environment(BaseModel):
                         meta=meta,
                         force=True,
                     )
+                    base = {
+                        "lineage": None,
+                        "metadata": new_persisted_concept.metadata.model_copy(
+                            update={"concept_source": ConceptSource.PERSIST_STATEMENT}
+                        ),
+                        "derivation": Derivation.ROOT,
+                        "purpose": new_persisted_concept.purpose,
+                    }
+                    # purpose is used in derivation calculation
+                    # which should be fixed, but we'll do in a followup
+                    # so override here
+                    if new_persisted_concept.purpose == Purpose.CONSTANT:
+                        base["purpose"] = Purpose.KEY
                     new_persisted_concept = new_persisted_concept.model_copy(
-                        deep=True,
-                        update={
-                            "lineage": None,
-                            "metadata": new_persisted_concept.metadata.model_copy(
-                                update={
-                                    "concept_source": ConceptSource.PERSIST_STATEMENT
-                                }
-                            ),
-                            "derivation": Derivation.ROOT,
-                        },
+                        deep=True, update=base
                     )
                     self.add_concept(
                         new_persisted_concept,

trilogy/core/models/execute.py

@@ -56,6 +56,12 @@ LOGGER_PREFIX = "[MODELS_EXECUTE]"
 DATASOURCE_TYPES = (BuildDatasource, BuildDatasource)
 
 
+class InlinedCTE(BaseModel):
+    original_alias: str
+    new_alias: str
+    new_base: str
+
+
 class CTE(BaseModel):
     name: str
     source: "QueryDatasource"
@@ -78,6 +84,7 @@ class CTE(BaseModel):
     limit: Optional[int] = None
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
+    inlined_ctes: dict[str, InlinedCTE] = Field(default_factory=dict)
 
     @field_validator("join_derived_concepts")
     def validate_join_derived_concepts(cls, v):
@@ -104,62 +111,6 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
-    def inline_constant(self, concept: BuildConcept):
-        if not concept.derivation == Derivation.CONSTANT:
-            return False
-        if not isinstance(concept.lineage, BuildFunction):
-            return False
-        if not concept.lineage.operator == FunctionType.CONSTANT:
-            return False
-        # remove the constant
-        removed: set = set()
-        if concept.address in self.source_map:
-            removed = removed.union(self.source_map[concept.address])
-            del self.source_map[concept.address]
-
-        if self.condition:
-            self.condition = self.condition.inline_constant(concept)
-        # if we've entirely removed the need to join to someplace to get the concept
-        # drop the join as well.
-        for removed_cte in removed:
-            still_required = any(
-                [
-                    removed_cte in x
-                    for x in self.source_map.values()
-                    or self.existence_source_map.values()
-                ]
-            )
-            if not still_required:
-                self.joins = [
-                    join
-                    for join in self.joins
-                    if not isinstance(join, Join)
-                    or (
-                        isinstance(join, Join)
-                        and (
-                            join.right_cte.name != removed_cte
-                            and any(
-                                [
-                                    x.cte.name != removed_cte
-                                    for x in (join.joinkey_pairs or [])
-                                ]
-                            )
-                        )
-                    )
-                ]
-                for join in self.joins:
-                    if isinstance(join, UnnestJoin) and concept in join.concepts:
-                        join.rendering_required = False
-
-                self.parent_ctes = [
-                    x for x in self.parent_ctes if x.name != removed_cte
-                ]
-                if removed_cte == self.base_name_override:
-                    candidates = [x.name for x in self.parent_ctes]
-                    self.base_name_override = candidates[0] if candidates else None
-                    self.base_alias_override = candidates[0] if candidates else None
-        return True
-
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}. Group: {self.group_to_grain}"
@@ -240,7 +191,18 @@ class CTE(BaseModel):
                 ]
             elif v == parent.safe_identifier:
                 self.source_map[k] = [ds_being_inlined.safe_identifier]
-
+        for k, v in self.existence_source_map.items():
+            if isinstance(v, list):
+                self.existence_source_map[k] = [
+                    (
+                        ds_being_inlined.safe_identifier
+                        if x == parent.safe_identifier
+                        else x
+                    )
+                    for x in v
+                ]
+            elif v == parent.safe_identifier:
+                self.existence_source_map[k] = [ds_being_inlined.safe_identifier]
         # zip in any required values for lookups
         for k in ds_being_inlined.output_lcl.addresses:
             if k in self.source_map and self.source_map[k]:
@@ -251,6 +213,11 @@ class CTE(BaseModel):
         ]
         if force_group:
             self.group_to_grain = True
+        self.inlined_ctes[ds_being_inlined.safe_identifier] = InlinedCTE(
+            original_alias=parent.name,
+            new_alias=ds_being_inlined.safe_identifier,
+            new_base=ds_being_inlined.safe_location,
+        )
         return True
 
     def __add__(self, other: "CTE" | "UnionCTE"):
@@ -303,6 +270,10 @@ class CTE(BaseModel):
             **self.existence_source_map,
             **other.existence_source_map,
         }
+        self.inlined_ctes = {
+            **self.inlined_ctes,
+            **other.inlined_ctes,
+        }
 
         return self
 
@@ -672,7 +643,7 @@ class QueryDatasource(BaseModel):
                     and CONFIG.validate_missing
                 ):
                     raise SyntaxError(
-                        f"On query datasource from {values} missing source map entry (map: {v}) for {concept.address} on {key} with pseudonyms {concept.pseudonyms}, have {v}"
+                        f"Missing source map entry for {concept.address} on {key} with pseudonyms {concept.pseudonyms}, have map: {v}"
                     )
         return v
 
@@ -1057,6 +1028,7 @@ class UnionCTE(BaseModel):
     hidden_concepts: set[str] = Field(default_factory=set)
     partial_concepts: list[BuildConcept] = Field(default_factory=list)
     existence_source_map: Dict[str, list[str]] = Field(default_factory=dict)
+    inlined_ctes: Dict[str, InlinedCTE] = Field(default_factory=dict)
 
     @computed_field  # type: ignore
     @property

trilogy/core/processing/node_generators/node_merge_node.py

@@ -6,7 +6,7 @@ from networkx.algorithms import approximation as ax
 from trilogy.constants import logger
 from trilogy.core.enums import Derivation
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
-from trilogy.core.graph_models import concept_to_node
+from trilogy.core.graph_models import concept_to_node, prune_sources_for_conditions
 from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
@@ -222,10 +222,12 @@ def resolve_weak_components(
     environment_graph: nx.DiGraph,
     filter_downstream: bool = True,
     accept_partial: bool = False,
+    search_conditions: BuildWhereClause | None = None,
 ) -> list[list[BuildConcept]] | None:
     break_flag = False
     found = []
     search_graph = environment_graph.copy()
+    prune_sources_for_conditions(search_graph, conditions=search_conditions)
     reduced_concept_sets: list[set[str]] = []
 
     # loop through, removing new nodes we find
@@ -406,6 +408,7 @@ def gen_merge_node(
             g,
             filter_downstream=filter_downstream,
             accept_partial=accept_partial,
+            search_conditions=search_conditions,
         )
         if not weak_resolve:
             logger.info(

trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -169,7 +169,9 @@ def is_fully_covered(
     return current_end >= end
 
 
-def get_union_sources(datasources: list[BuildDatasource], concepts: list[BuildConcept]):
+def get_union_sources(
+    datasources: list[BuildDatasource], concepts: list[BuildConcept]
+) -> List[list[BuildDatasource]]:
     candidates: list[BuildDatasource] = []
     for x in datasources:
         if all([c.address in x.output_concepts for c in concepts]):

trilogy/core/processing/node_generators/select_merge_node.py

@@ -5,7 +5,11 @@ import networkx as nx
 
 from trilogy.constants import logger
 from trilogy.core.enums import Derivation
-from trilogy.core.graph_models import concept_to_node
+from trilogy.core.graph_models import (
+    concept_to_node,
+    get_graph_exact_match,
+    prune_sources_for_conditions,
+)
 from trilogy.core.models.build import (
     BuildConcept,
     BuildDatasource,
@@ -57,28 +61,6 @@ def get_graph_partial_nodes(
     return partial
 
 
-def get_graph_exact_match(
-    g: nx.DiGraph, conditions: BuildWhereClause | None
-) -> set[str]:
-    datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
-        nx.get_node_attributes(g, "datasource")
-    )
-    exact: set[str] = set()
-    for node in g.nodes:
-        if node in datasources:
-            ds = datasources[node]
-            if not isinstance(ds, list):
-                if not ds.non_partial_for:
-                    continue
-                if ds.non_partial_for and conditions == ds.non_partial_for:
-                    exact.add(node)
-                    continue
-            else:
-                continue
-
-    return exact
-
-
 def get_graph_grains(g: nx.DiGraph) -> dict[str, list[str]]:
     datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
         nx.get_node_attributes(g, "datasource")
@@ -98,28 +80,31 @@ def get_graph_grains(g: nx.DiGraph) -> dict[str, list[str]]:
 
 
 def subgraph_is_complete(
-    nodes: list[str], targets: set[str], mapping: dict[str, str]
+    nodes: list[str], targets: set[str], mapping: dict[str, str], g: nx.DiGraph
 ) -> bool:
     mapped = set([mapping.get(n, n) for n in nodes])
-    return all([t in mapped for t in targets])
-
-
-def prune_sources_for_conditions(
-    g: nx.DiGraph,
-    depth: int,
-    conditions: BuildWhereClause | None,
-):
-
-    complete = get_graph_exact_match(g, conditions)
-    to_remove = []
-    for node in g.nodes:
-        if node.startswith("ds~") and node not in complete:
-            to_remove.append(node)
-            logger.debug(
-                f"{padding(depth)}{LOGGER_PREFIX} removing datasource {node} as it is not a match for conditions {conditions}"
-            )
-    for node in to_remove:
-        g.remove_node(node)
+    passed = all([t in mapped for t in targets])
+    if not passed:
+        logger.info(
+            f"Subgraph {nodes} is not complete, missing targets {targets} - mapped {mapped}"
+        )
+        return False
+    # check if all concepts have a datasource edge
+    has_ds_edge = {
+        mapping.get(n, n): any(x.startswith("ds~") for x in nx.neighbors(g, n))
+        for n in nodes
+        if n.startswith("c~")
+    }
+    has_ds_edge = {k: False for k in targets}
+    # check at least one instance of concept has a datasource edge
+    for n in nodes:
+        if n.startswith("c~"):
+            neighbors = nx.neighbors(g, n)
+            for neighbor in neighbors:
+                if neighbor.startswith("ds~"):
+                    has_ds_edge[mapping.get(n, n)] = True
+                    break
+    return all(has_ds_edge.values()) and passed
 
 
 def create_pruned_concept_graph(
@@ -133,8 +118,6 @@ def create_pruned_concept_graph(
     orig_g = g
 
     g = g.copy()
-    if conditions:
-        prune_sources_for_conditions(g, depth, conditions)
     union_options = get_union_sources(datasources, all_concepts)
     for ds_list in union_options:
         node_address = "ds~" + "-".join([x.name for x in ds_list])
@@ -144,7 +127,8 @@ def create_pruned_concept_graph(
         g.add_node(node_address, datasource=ds_list)
         for c in common:
             g.add_edge(node_address, concept_to_node(c))
-
+            g.add_edge(concept_to_node(c), node_address)
+    prune_sources_for_conditions(g, conditions)
     target_addresses = set([c.address for c in all_concepts])
     concepts: dict[str, BuildConcept] = nx.get_node_attributes(orig_g, "concept")
     datasource_map: dict[str, BuildDatasource | list[BuildDatasource]] = (
@@ -156,8 +140,7 @@ def create_pruned_concept_graph(
         # filter out synonyms
         if (x := concepts.get(n, None)) and x.address in target_addresses
     }
-    # from trilogy.hooks.graph_hook import GraphHook
-    # GraphHook().query_graph_built(g)
+
     relevant_concepts: list[str] = list(relevant_concepts_pre.keys())
     relevent_datasets: list[str] = []
     if not accept_partial:
@@ -179,6 +162,7 @@ def create_pruned_concept_graph(
                 to_remove.append(edge)
         for edge in to_remove:
             g.remove_edge(*edge)
+
     for n in g.nodes():
         if not n.startswith("ds~"):
             continue
@@ -211,13 +195,13 @@ def create_pruned_concept_graph(
             if n not in relevent_datasets and n not in relevant_concepts
         ]
     )
-
+    # from trilogy.hooks.graph_hook import GraphHook
+    # GraphHook().query_graph_built(g)
     subgraphs = list(nx.connected_components(g.to_undirected()))
-
     subgraphs = [
         s
         for s in subgraphs
-        if subgraph_is_complete(s, target_addresses, relevant_concepts_pre)
+        if subgraph_is_complete(s, target_addresses, relevant_concepts_pre, g)
     ]
 
     if not subgraphs:
@@ -524,8 +508,12 @@ def gen_select_merge_node(
     constants = [c for c in all_concepts if c.derivation == Derivation.CONSTANT]
     if not non_constant and constants:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} only constant inputs to discovery, returning constant node directly"
+            f"{padding(depth)}{LOGGER_PREFIX} only constant inputs to discovery ({constants}), returning constant node directly"
         )
+        for x in constants:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} {x} {x.lineage} {x.derivation}"
+            )
         if conditions:
             if not all(
                 [x.derivation == Derivation.CONSTANT for x in conditions.row_arguments]

trilogy/core/query_processor.py

@@ -395,6 +395,9 @@ def get_query_node(
     if not statement.output_components:
         raise ValueError(f"Statement has no output components {statement}")
     history = history or History(base_environment=environment)
+    print(
+        f"{LOGGER_PREFIX} building query node for {statement.output_components} grain {statement.grain}"
+    )
     build_statement: BuildSelectLineage | BuildMultiSelectLineage = Factory(
         environment=environment,
     ).build(statement)

trilogy/dialect/base.py

@@ -580,7 +580,12 @@ class BaseDialect:
                     target = INVALID_REFERENCE_STRING(
                         f"Missing source CTE for {e.right.address}"
                     )
+                assert cte, "CTE must be provided for inlined CTEs"
+                if target in cte.inlined_ctes:
+                    info = cte.inlined_ctes[target]
+                    return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {info.new_base} as {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
+
             elif isinstance(
                 e.right,
                 (ListWrapper, TupleWrapper, BuildParenthetical, list),

trilogy/hooks/graph_hook.py

@@ -85,7 +85,6 @@ class GraphHook(BaseHook):
         plt.show()
 
     def _draw_labels_with_manual_spacing(self, graph, pos):
-        """Fallback method for manual label spacing when adjustText is not available"""
         import numpy as np
 
         pos_labels = {}

trilogy/parsing/common.py

@@ -23,9 +23,13 @@ from trilogy.core.models.author import (
     AggregateWrapper,
     AlignClause,
     AlignItem,
+    CaseElse,
+    CaseWhen,
+    Comparison,
     Concept,
     ConceptArgs,
     ConceptRef,
+    Conditional,
     FilterItem,
     Function,
     FunctionCallWrapper,
@@ -38,6 +42,7 @@ from trilogy.core.models.author import (
     Parenthetical,
     RowsetItem,
     RowsetLineage,
+    SubselectComparison,
     TraitDataType,
     UndefinedConcept,
     WhereClause,
@@ -198,6 +203,7 @@ def atom_is_relevant(
     others: list[Concept | ConceptRef],
     environment: Environment | None = None,
 ):
+
     if isinstance(atom, (ConceptRef, Concept)):
         # when we are looking at atoms, if there is a concept that is in others
         # return directly
@@ -210,9 +216,10 @@ def atom_is_relevant(
     elif isinstance(atom, AggregateWrapper):
         return any(atom_is_relevant(x, others, environment) for x in atom.by)
 
-    if isinstance(atom, Function):
+    elif isinstance(atom, Function):
         relevant = False
         for arg in atom.arguments:
+
             relevant = relevant or atom_is_relevant(arg, others, environment)
         return relevant
     elif isinstance(atom, FunctionCallWrapper):
@@ -220,8 +227,27 @@ def atom_is_relevant(
             [atom_is_relevant(atom.content, others, environment)]
             + [atom_is_relevant(x, others, environment) for x in atom.args]
         )
+    elif isinstance(atom, CaseWhen):
+        rval = atom_is_relevant(atom.expr, others, environment) or atom_is_relevant(
+            atom.comparison, others, environment
+        )
+        return rval
+    elif isinstance(atom, CaseElse):
+
+        rval = atom_is_relevant(atom.expr, others, environment)
+        return rval
+    elif isinstance(atom, SubselectComparison):
+        return atom_is_relevant(atom.left, others, environment)
+    elif isinstance(atom, Comparison):
+        return atom_is_relevant(atom.left, others, environment) or atom_is_relevant(
+            atom.right, others, environment
+        )
+    elif isinstance(atom, Conditional):
+        return atom_is_relevant(atom.left, others, environment) or atom_is_relevant(
+            atom.right, others, environment
+        )
     elif isinstance(atom, ConceptArgs):
-        # use atom is relevant here to trigger the early exit behavior for concpets in set
+        # use atom is relevant here to trigger the early exit behavior for concepts in set
         return any(
             [atom_is_relevant(x, others, environment) for x in atom.concept_arguments]
         )
@@ -233,7 +259,6 @@ def concept_is_relevant(
     others: list[Concept | ConceptRef],
     environment: Environment | None = None,
 ) -> bool:
-
     if isinstance(concept, UndefinedConcept):
         return False
     if concept.datatype == DataType.UNKNOWN:

trilogy/std/display.preql

@@ -1,3 +1,6 @@
 
 
-type percent float; # Percentage value
+type percent float; # Percentage value
+
+def calc_percent(a, b, digits=-1) -> case when digits =-1 then (a/b):: float::percent
+else round((a/b):: float::percent, digits) end;