pytrilogy 0.0.2.15__py3-none-any.whl → 0.0.2.18__py3-none-any.whl

{pytrilogy-0.0.2.15.dist-info → pytrilogy-0.0.2.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.15
+Version: 0.0.2.18
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -42,15 +42,15 @@ Installation: `pip install pytrilogy`
 
 `pytrilogy` can be run locally to parse and execute trilogy model [.preql] files using the `trilogy` CLI tool, or can be run in python by importing the `trilogy` package.
 
-You can read more about the project [here](https://trilogydata.dev/) and try out an interactive demo on the page an interactive demo [here](https://trilogydata.dev/demo). 
+You can read more about the project [here](https://trilogydata.dev/) and try out an interactive demo [here](https://trilogydata.dev/demo/). 
 
 Trilogy:
 ```sql
+WHERE 
+    name like '%lvis%'
 SELECT
     name,
     count(name) as name_count
-WHERE 
-    name='Elvis'
 ORDER BY
     name_count desc
 LIMIT 10;
@@ -145,7 +145,7 @@ Run the following from the directory the file is in.
 trilogy run hello.trilogy duckdb
 ```
 
-![UI Preview](./docs/hello_world.png)
+![UI Preview](./hello_world.png)
 
 ## Backends
 
@@ -214,7 +214,7 @@ for row in results:
 
 ## Basic Example - CLI
 
-Trilogy can be run through a CLI tool, appropriately named 'trilogy'.
+Trilogy can be run through a CLI tool, also named 'trilogy'.
 
 After installing trilogy, you can run the trilogy CLI with two required positional arguments; the first the path to a file or a direct command,
 and second the dialect to run.
@@ -252,7 +252,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 ## More Examples
 
-[Interactive demo](https://trilogydata.dev/demo). 
+[Interactive demo](https://trilogydata.dev/demo/). 
 
 Additional examples can be found in the [public model repository](https://github.com/trilogydata/trilogy-public-models).
 
@@ -267,11 +267,15 @@ Clone repository and install requirements.txt and requirements-test.txt.
 Please open an issue first to discuss what you would like to change, and then create a PR against that issue.
 
 ## Similar in space
+Trilogy combines two aspects; a semantic layer and a query language. We've covered examples of both below:
+
+Python "semantic layers" are tools for defining data access to a warehouse in a more abstract way.
+
+- [metricsflow](https://github.com/dbt-labs/metricflow)
 
 "Better SQL" has been a popular space. We believe Trilogy takes a different approach then the following,
 but all are worth checking out. Please open PRs/comment for anything missed!
 
-
 - [malloy](https://github.com/malloydata/malloy)
 - [preql](https://github.com/erezsh/Preql)
 - [PREQL](https://github.com/PRQL/prql)

pytrilogy-0.0.2.18.dist-info/RECORD

@@ -0,0 +1,83 @@
+trilogy/__init__.py,sha256=NVclSieaZqXKRfCCzUhXoqSrNUdoiMn3ytKw0jCWj7A,291
+trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/constants.py,sha256=pZkOneh_65f9Ua6NICu1bHAFAbmQxmiXRXS7tsmCWbQ,1235
+trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
+trilogy/executor.py,sha256=An6YLpHQOt96E7ozRQhwZels2hMsDbh0WV767kKCGU0,11294
+trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
+trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
+trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
+trilogy/core/enums.py,sha256=A9VC0lbP5eo9sndm2TzA-nNJRRmvbjE918ZiEXtcQ_c,6043
+trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
+trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
+trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
+trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
+trilogy/core/functions.py,sha256=ShFTStIKbgI-3EZIU0xTumI78AC5QlvARwnBM53P2O0,10677
+trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
+trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
+trilogy/core/models.py,sha256=nSCOXedYRFgYdZa45dputERCvgPRQpRvCyiGglyeM0g,150985
+trilogy/core/optimization.py,sha256=od_60A9F8J8Nj24MHgrxl4vwRwmBFH13TMdoMQvgVKs,7717
+trilogy/core/query_processor.py,sha256=kXuBsIaRHu1s7zB_rAnT_gRe4-VgRSrPE1TnVJXFLtc,16447
+trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
+trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
+trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
+trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
+trilogy/core/optimizations/predicate_pushdown.py,sha256=1l9WnFOSv79e341typG3tTdk0XGl1J_ToQih3LYoGIY,8435
+trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/core/processing/concept_strategies_v3.py,sha256=DO9gybVLku8GEkO3uNPaCeqhalnufsjYYbvDs-gkwNc,35295
+trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
+trilogy/core/processing/utility.py,sha256=plpaAmssWjcxPee8_J4hleazTzQIBL6mBLJ33FKfwOM,19421
+trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
+trilogy/core/processing/node_generators/basic_node.py,sha256=WQNgJ1MwrMS_BQ-b3XwGGB6eToDykelAVj_fesJuqe0,2069
+trilogy/core/processing/node_generators/common.py,sha256=LwDgPlhWeuw0t07f3kX9IE5LXBdZhXfh-aY0XGk50ak,8946
+trilogy/core/processing/node_generators/filter_node.py,sha256=Vz9Rb67e1dfZgnliekwwLeDPVkthMbdrnrKRdz7J1ik,7654
+trilogy/core/processing/node_generators/group_node.py,sha256=Dn9vEY-WPFHNN-LtXfgWiHIXspzHDKfkKL5a2KE2gD0,4252
+trilogy/core/processing/node_generators/group_to_node.py,sha256=R9i_wHipxjXJyfYEwfeTw2EPpuanXVA327XyfcP2tBg,2537
+trilogy/core/processing/node_generators/multiselect_node.py,sha256=_KO9lqzHQoy4VAviO0ttQlmK0tjaqrJj4SJPhmoIYm8,6229
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=yRDfY8muZN7G2vsdYXF2X1iqbQ2zDUNGlxvSIyKVoWU,13512
+trilogy/core/processing/node_generators/rowset_node.py,sha256=gU_ybfYXO9tZqHjUSABIioVpb8AWtITpegj3IGSf2GI,4587
+trilogy/core/processing/node_generators/select_merge_node.py,sha256=ipSxw1Oqk-hVVGhPhZlvRbptC0Vpwh52hZ7z8oOj2yk,10065
+trilogy/core/processing/node_generators/select_node.py,sha256=vUg3gXHGvagdbniIAE7DdqJcQ0V1VAfHtTrw3edYPso,1734
+trilogy/core/processing/node_generators/unnest_node.py,sha256=cZ26CN338CBnd6asML1OBUtNcDzmNlFpY0Vnade4yrc,2256
+trilogy/core/processing/node_generators/window_node.py,sha256=jy3FF8uN0VA7yyrBeR40B9CAqR_5qBP4PiS6Gr-f-7w,2590
+trilogy/core/processing/nodes/__init__.py,sha256=qS5EJDRwwIrCEfS7ibCA2ESE0RPzsAIii1UWd_wNsHA,4760
+trilogy/core/processing/nodes/base_node.py,sha256=sc3HrXkWk-xpsAQ7B7ltX1ZejYAkqFiv8Ei8Jg5VGkQ,15579
+trilogy/core/processing/nodes/filter_node.py,sha256=GfZ9eghpFDI-s7iQP2UqTljCmn25LT_T5TAxDlh7PkQ,2343
+trilogy/core/processing/nodes/group_node.py,sha256=PrBHaGq_f8RmokUw9lXLGJ5YbjdP77P7Ag0pgR6e2cU,7293
+trilogy/core/processing/nodes/merge_node.py,sha256=wlqh0suFfZBJOgkn7vy0OiDs5jza3NCX7eHTEEb6mBQ,14799
+trilogy/core/processing/nodes/select_node_v2.py,sha256=gS9OQgS2TSEK59BQ9R0i83pTHfGJUxv7AkAmT21sYxI,8067
+trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
+trilogy/core/processing/nodes/window_node.py,sha256=ro0QfMFi4ZmIn5Q4D0M_vJWfnHH_C0MN7XkVkx8Gygg,1214
+trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/dialect/base.py,sha256=QDeKbc7vgqfbR649-87cwsOz8G3VY63V19zH6I-WITo,33103
+trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
+trilogy/dialect/common.py,sha256=Hr0mxcNxjSvhpBM5Wvb_Q7aklAuYj5aBDrW433py0Zs,4403
+trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
+trilogy/dialect/duckdb.py,sha256=_0a5HBU8zRNtZj7YED3ju4fHXRYG9jNeKwnlZwUDvwI,3419
+trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
+trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
+trilogy/dialect/presto.py,sha256=2Rs53UfPxKU0rJTcEbiS-Lxm-CDiqUGojh7yRpQgyRE,3416
+trilogy/dialect/snowflake.py,sha256=_Bf4XO7-nImMv9XCSsTfVM3g2f_KHdO17VTa9J-HgSM,2989
+trilogy/dialect/sql_server.py,sha256=owUZbMFrooYIMj1DSLstPWxPO7K7WAUEWNvDKM-DMt0,3118
+trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
+trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2467
+trilogy/hooks/query_debugger.py,sha256=Pe-Kw1JGngeLqQOMQb0E3-24jXEavqnPCQ-KOfTfjP8,4357
+trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/parsing/common.py,sha256=-4LM71ocidA8DI2RngqFEOmhzBrIt8VdBTO4x2BpD8E,9502
+trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
+trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
+trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+trilogy/parsing/parse_engine.py,sha256=r0B7tXAIfTMJ42ZJwP70ulQBrOoM1AAiXFbCQ2rOpQs,64720
+trilogy/parsing/render.py,sha256=8yxerPAi4AhlhPBlAfbYbOM3F9rz6HzpWVEWPtK2VEg,12321
+trilogy/parsing/trilogy.lark,sha256=0JAvQBACFNL-X61I0tB_0QPZgsguZgerfHBv903oKh0,11623
+trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
+pytrilogy-0.0.2.18.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.18.dist-info/METADATA,sha256=erqF5E59Qz6Xr2ZKiBKejkmQUFMGpDxK9NerqOoT6K0,8132
+pytrilogy-0.0.2.18.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.18.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.18.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.18.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.15"
+__version__ = "0.0.2.18"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -24,7 +24,7 @@ class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
     constant_inlining: bool = True
-    constant_inline_cutoff: int = 3
+    constant_inline_cutoff: int = 10
     direct_return: bool = True
 
 

trilogy/core/enums.py

@@ -119,6 +119,7 @@ class FunctionType(Enum):
     CONSTANT = "constant"
     COALESCE = "coalesce"
     IS_NULL = "isnull"
+    BOOL = "bool"
 
     # COMPLEX
     INDEX_ACCESS = "index_access"

trilogy/core/functions.py

@@ -340,6 +340,17 @@ def IsNull(args: list[Concept]) -> Function:
     )
 
 
+def Bool(args: list[Concept]) -> Function:
+    return Function(
+        operator=FunctionType.BOOL,
+        arguments=args,
+        output_datatype=DataType.BOOL,
+        output_purpose=function_args_to_output_purpose(args),
+        arg_count=1,
+        # output_grain=Grain(components=arguments),
+    )
+
+
 def StrPos(args: list[Concept]) -> Function:
     return Function(
         operator=FunctionType.STRPOS,

trilogy/core/models.py

@@ -432,6 +432,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     grain: "Grain" = Field(default=None, validate_default=True)
     modifiers: Optional[List[Modifier]] = Field(default_factory=list)
     pseudonyms: Dict[str, Concept] = Field(default_factory=dict)
+    _address_cache: str | None = None
 
     def __hash__(self):
         return hash(str(self))
@@ -558,9 +559,19 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @cached_property
+    @property
     def address(self) -> str:
-        return f"{self.namespace}.{self.name}"
+        if not self._address_cache:
+            self._address_cache = f"{self.namespace}.{self.name}"
+        return self._address_cache
+
+    @address.setter
+    def address(self, address: str) -> None:
+        self._address_cache = address
+
+    def set_name(self, name: str):
+        self.name = name
+        self.address = f"{self.namespace}.{self.name}"
 
     @property
     def output(self) -> "Concept":
@@ -905,7 +916,13 @@ class Grain(Mergeable, BaseModel):
 
     @cached_property
     def set(self):
-        return set([c.address for c in self.components_copy])
+        base = []
+        for x in self.components_copy:
+            if x.derivation == PurposeLineage.ROWSET:
+                base.append(x.lineage.content.address)
+            else:
+                base.append(x.address)
+        return set(base)
 
     def __eq__(self, other: object):
         if isinstance(other, list):
@@ -1555,6 +1572,10 @@ class OrderBy(Mergeable, Namespaced, BaseModel):
             items=[x.with_merge(source, target, modifiers) for x in self.items]
         )
 
+    @property
+    def concept_arguments(self):
+        return [x.expr for x in self.items]
+
 
 class RawSQLStatement(BaseModel):
     text: str
@@ -1567,6 +1588,13 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
     limit: Optional[int] = None
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
+    def refresh_bindings(self, environment: Environment):
+        for item in self.selection:
+            if isinstance(item.content, Concept):
+                item.content = environment.concepts[item.content.address].with_grain(
+                    self.grain
+                )
+
     def __str__(self):
         from trilogy.parsing.render import render_query
 
@@ -1605,6 +1633,14 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
             limit=self.limit,
         )
 
+    @property
+    def locally_derived(self) -> set[str]:
+        locally_derived: set[str] = set()
+        for item in self.selection:
+            if isinstance(item.content, ConceptTransform):
+                locally_derived.add(item.content.output.address)
+        return locally_derived
+
     @property
     def input_components(self) -> List[Concept]:
         output = set()
@@ -1655,11 +1691,22 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
         address: Address,
         grain: Grain | None = None,
     ) -> Datasource:
+        if self.where_clause or self.having_clause:
+            modifiers = [Modifier.PARTIAL]
+        else:
+            modifiers = []
         columns = [
             # TODO: replace hardcoded replacement here
-            ColumnAssignment(alias=c.address.replace(".", "_"), concept=c)
+            # if the concept is a locally derived concept, it cannot ever be partial
+            # but if it's a concept pulled in from upstream and we have a where clause, it should be partial
+            ColumnAssignment(
+                alias=c.address.replace(".", "_"),
+                concept=c,
+                modifiers=modifiers if c.address not in self.locally_derived else [],
+            )
             for c in self.output_components
         ]
+
         new_datasource = Datasource(
             identifier=identifier,
             address=address,
@@ -1786,6 +1833,10 @@ class MultiSelectStatement(SelectTypeMixin, Mergeable, Namespaced, BaseModel):
     limit: Optional[int] = None
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
+    def refresh_bindings(self, environment: Environment):
+        for select in self.selects:
+            select.refresh_bindings(environment)
+
     def __repr__(self):
         return "MultiSelect<" + " MERGE ".join([str(s) for s in self.selects]) + ">"
 
@@ -2255,14 +2306,11 @@ class BaseJoin(BaseModel):
     def __str__(self):
         if self.concept_pairs:
             return (
-                f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
-                f" {self.right_datasource.identifier} on"
+                f"{self.join_type.value} on"
                 f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
-            f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
-            f" {self.right_datasource.identifier} on"
-            f" {','.join([str(k) for k in self.concepts])}"
+            f"{self.join_type.value} on" f" {','.join([str(k) for k in self.concepts])}"
         )
 
 
@@ -2395,10 +2443,6 @@ class QueryDatasource(BaseModel):
             raise SyntaxError(
                 "Can only merge two query datasources with identical grain"
             )
-        if not self.source_type == other.source_type:
-            raise SyntaxError(
-                "Can only merge two query datasources with identical source type"
-            )
         if not self.group_required == other.group_required:
             raise SyntaxError(
                 "can only merge two datasources if the group required flag is the same"
@@ -2437,9 +2481,7 @@ class QueryDatasource(BaseModel):
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
         self_hidden = self.hidden_concepts or []
         other_hidden = other.hidden_concepts or []
-        hidden = [
-            x for x in self_hidden if x.address in [y.address for y in other_hidden]
-        ]
+        hidden = [x for x in self_hidden if x.address in other_hidden]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2609,7 +2651,7 @@ class CTE(BaseModel):
 
     @property
     def comment(self) -> str:
-        base = f"Target: {str(self.grain)}."
+        base = f"Target: {str(self.grain)}. Group: {self.group_to_grain}"
         base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
@@ -2621,6 +2663,8 @@ class CTE(BaseModel):
             )
         base += f"\n-- Source Map: {self.source_map}."
         base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.source.input_concepts:
+            base += f"\n-- Inputs: {', '.join([str(x) for x in self.source.input_concepts])}."
         if self.hidden_concepts:
             base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
         if self.nullable_concepts:
@@ -2660,9 +2704,9 @@ class CTE(BaseModel):
             if isinstance(join, InstantiatedUnnestJoin):
                 continue
             if join.left_cte.name == parent.name:
-                join.left_cte = ds_being_inlined
+                join.inline_cte(parent)
             if join.right_cte.name == parent.name:
-                join.right_cte = ds_being_inlined
+                join.inline_cte(parent)
         for k, v in self.source_map.items():
             if isinstance(v, list):
                 self.source_map[k] = [
@@ -2697,7 +2741,7 @@ class CTE(BaseModel):
             raise ValueError(error)
         mutually_hidden = []
         for concept in self.hidden_concepts:
-            if concept in other.hidden_concepts:
+            if concept.address in other.hidden_concepts:
                 mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
@@ -2797,37 +2841,23 @@ class CTE(BaseModel):
 
     @property
     def group_concepts(self) -> List[Concept]:
+        def check_is_not_in_group(c: Concept):
+            if len(self.source_map.get(c.address, [])) > 0:
+                return False
+            if c.derivation == PurposeLineage.ROWSET:
+                return False
+            if c.derivation == PurposeLineage.CONSTANT:
+                return False
+            if c.purpose == Purpose.METRIC:
+                return True
+            elif c.derivation == PurposeLineage.BASIC and c.lineage:
+                if all([check_is_not_in_group(x) for x in c.lineage.concept_arguments]):
+                    return True
+            return False
+
         return (
             unique(
-                self.grain.components
-                + [
-                    c
-                    for c in self.output_columns
-                    if c.purpose in (Purpose.PROPERTY, Purpose.KEY)
-                    and c.address not in [x.address for x in self.grain.components]
-                ]
-                + [
-                    c
-                    for c in self.output_columns
-                    if c.purpose == Purpose.METRIC
-                    and (
-                        any(
-                            [
-                                c.with_grain(cte.grain) in cte.output_columns
-                                for cte in self.parent_ctes
-                            ]
-                        )
-                        # if we have this metric from a source
-                        # it isn't derived here and must be grouped on
-                        or len(self.source_map[c.address]) > 0
-                    )
-                ]
-                + [
-                    c
-                    for c in self.output_columns
-                    if c.purpose == Purpose.CONSTANT
-                    and self.source_map[c.address] != []
-                ],
+                [c for c in self.output_columns if not check_is_not_in_group(c)],
                 "address",
             )
             if self.group_to_grain
@@ -2885,34 +2915,38 @@ class JoinKey(BaseModel):
 
 
 class Join(BaseModel):
-    left_cte: CTE | Datasource
-    right_cte: CTE | Datasource
+    left_cte: CTE
+    right_cte: CTE
     jointype: JoinType
     joinkeys: List[JoinKey]
     joinkey_pairs: List[ConceptPair] | None = None
+    inlined_ctes: set[str] = Field(default_factory=set)
+
+    def inline_cte(self, cte: CTE):
+        self.inlined_ctes.add(cte.name)
 
     @property
     def left_name(self) -> str:
-        if isinstance(self.left_cte, Datasource):
-            return self.left_cte.identifier
+        if self.left_cte.name in self.inlined_ctes:
+            return self.left_cte.source.datasources[0].identifier
         return self.left_cte.name
 
     @property
     def right_name(self) -> str:
-        if isinstance(self.right_cte, Datasource):
-            return self.right_cte.identifier
+        if self.right_cte.name in self.inlined_ctes:
+            return self.right_cte.source.datasources[0].identifier
         return self.right_cte.name
 
     @property
     def left_ref(self) -> str:
-        if isinstance(self.left_cte, Datasource):
-            return f"{self.left_cte.safe_location} as {self.left_cte.identifier}"
+        if self.left_cte.name in self.inlined_ctes:
+            return f"{self.left_cte.source.datasources[0].safe_location} as {self.left_cte.source.datasources[0].identifier}"
         return self.left_cte.name
 
     @property
     def right_ref(self) -> str:
-        if isinstance(self.right_cte, Datasource):
-            return f"{self.right_cte.safe_location} as {self.right_cte.identifier}"
+        if self.right_cte.name in self.inlined_ctes:
+            return f"{self.right_cte.source.datasources[0].safe_location} as {self.right_cte.source.datasources[0].identifier}"
         return self.right_cte.name
 
     @property
@@ -3386,6 +3420,18 @@ class Environment(BaseModel):
     ):
 
         self.datasources[datasource.env_label] = datasource
+        for column in datasource.columns:
+            current_concept = column.concept
+            current_derivation = current_concept.derivation
+            if current_derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT):
+                new_concept = current_concept.model_copy(deep=True)
+                new_concept.set_name("_pre_persist_" + current_concept.name)
+                # remove the associated lineage
+                current_concept.lineage = None
+                self.add_concept(new_concept, meta=meta, force=True)
+                self.add_concept(current_concept, meta=meta, force=True)
+                self.merge_concept(new_concept, current_concept, [])
+
         self.gen_concept_list_caches()
         return datasource
 

trilogy/core/optimization.py

@@ -125,23 +125,29 @@ def is_direct_return_eligible(cte: CTE) -> CTE | None:
         for c in cte.source.output_concepts + cte.source.hidden_concepts
         if c not in cte.source.input_concepts
     ]
-    conditions = (
-        set(x.address for x in direct_parent.condition.concept_arguments)
-        if direct_parent.condition
-        else set()
-    )
+
+    parent_derived_concepts = [
+        c
+        for c in direct_parent.source.output_concepts
+        + direct_parent.source.hidden_concepts
+        if c not in direct_parent.source.input_concepts
+    ]
+    condition_arguments = cte.condition.row_arguments if cte.condition else []
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
             return None
         if x.derivation == PurposeLineage.UNNEST:
             return None
         if x.derivation == PurposeLineage.AGGREGATE:
-            if x.address in conditions:
-                return None
-    # handling top level nodes that require unpacking
-    for x in cte.output_columns:
+            return None
+    for x in parent_derived_concepts:
+        if x.address not in condition_arguments:
+            continue
         if x.derivation == PurposeLineage.UNNEST:
             return None
+        if x.derivation == PurposeLineage.WINDOW:
+            return None
+
     logger.info(
         f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )