pytrilogy 0.0.3.52__py3-none-any.whl → 0.0.3.54__py3-none-any.whl

{pytrilogy-0.0.3.52.dist-info → pytrilogy-0.0.3.54.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.52
+Version: 0.0.3.54
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.52.dist-info → pytrilogy-0.0.3.54.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.52.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=3lpAbpHzxQ6c0SUaZfiuOcDzIS6IZ5DyK1fl8FmZ3hE,303
+pytrilogy-0.0.3.54.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=Wz4e3LWZ4MWyBm3EuLrz5-n5-yyu96rHR7k-4JPA4RE,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -11,16 +11,17 @@ trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
 trilogy/authoring/__init__.py,sha256=v9PRuZs4fTnxhpXAnwTxCDwlLasUax6g2FONidcujR4,2369
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
-trilogy/core/enums.py,sha256=uZTi9K6PEpQ1oFV4OpHlC1NUSxrmAFdQBfRyy9Rba-8,7440
+trilogy/core/enums.py,sha256=QVylGAe6epdGGpOKkeJ4cbx0mIZb0aARAKhsoZaGhoA,7576
 trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0,3170
 trilogy/core/environment_helpers.py,sha256=VvPIiFemqaLLpIpLIqprfu63K7muZ1YzNg7UZIUph8w,8267
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
-trilogy/core/functions.py,sha256=OIcaftda-afXrHMSvPksLbRTwPUwQHAIpy9l78EBZVU,28643
+trilogy/core/functions.py,sha256=IvqHyuO__o6Th8tkDWjb9cDxQDly6l3ZEfJ9y8YrTRU,29227
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
-trilogy/core/optimization.py,sha256=O7ag0IVQlJyWdAXBi_hHeU3Df5DRyd75Vlz6pks2J10,8197
+trilogy/core/optimization.py,sha256=ClDPMuuLBjuZCHdRGsuNhKNNTkbev1I22SbnLMzHT60,8839
 trilogy/core/query_processor.py,sha256=NNzYPKN5HzivQFXugSbJC_MaupkwOYii7A_vnXuBIK4,20063
+trilogy/core/utility.py,sha256=3VC13uSQWcZNghgt7Ot0ZTeEmNqs__cx122abVq9qhM,410
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/models/author.py,sha256=KKW_3A1hdwq7D2dFwI6xZanukPuCQQ23R4GzE5VRJ6c,77206
 trilogy/core/models/build.py,sha256=yBiOQ4Bhjz09pSD1jSGhhf9QFFQuplrvZ0JQB5-iXHk,63104
@@ -28,7 +29,7 @@ trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
 trilogy/core/models/environment.py,sha256=AVSrvjNcNX535GhCPtYhCRY2Lp_Hj0tdY3VVt_kZb9Q,27260
-trilogy/core/models/execute.py,sha256=ucxMwsu5OMoP0E4pVKtkCNU0nogElJKQAqfu3arE4Jo,34879
+trilogy/core/models/execute.py,sha256=F7-9VyUz5MC__VUSXd4U7gUb23Dc5PH5FdMUt6FqCPM,35214
 trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
@@ -45,17 +46,17 @@ trilogy/core/processing/node_generators/group_node.py,sha256=nIfiMrJQEksUfqAeeA3
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=sv55oynfqgpHEpo1OEtVDri-5fywzPhDlR85qaWikvY,16195
-trilogy/core/processing/node_generators/rowset_node.py,sha256=YmBs6ZQ7azLXRFEmeoecpGjK4pMHsUCovuBxfb3UKZI,6848
+trilogy/core/processing/node_generators/rowset_node.py,sha256=2BiSsegbRF9csJ_Xl8P_CxIm4dAAb7dF29u6v_Odr-A,6709
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=lxXhMhDKGbu67QFNbbAT-BO8gbWppIvjn_hAXpLEPe0,19953
 trilogy/core/processing/node_generators/select_node.py,sha256=Y-zO0AFkTrpi2LyebjpyHU7WWANr7nKZSS9rY7DH4Wo,1888
 trilogy/core/processing/node_generators/synonym_node.py,sha256=9LHK2XHDjbyTLjmDQieskG8fqbiSpRnFOkfrutDnOTE,2258
 trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cOEKnMRzXUW3bwmiOlgn3E1-B38osng0dh2pDykwITY,2410
-trilogy/core/processing/node_generators/window_node.py,sha256=RUHgpYovQObFod1xRIMWtDzMcxwlm4-1Fdrf_Cuw5W4,6346
+trilogy/core/processing/node_generators/window_node.py,sha256=GP3Hvkbb0TDA6ef7W7bmvQEHVH-NRIfBT_0W4fcH3g4,6529
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
-trilogy/core/processing/nodes/__init__.py,sha256=xPFF7x3TFs1Z4IcfthCykZgrksb-UhN-pc_oIigfFSo,6014
-trilogy/core/processing/nodes/base_node.py,sha256=z-aZEVjnLdFm6TpmneEm2bnRXj-tRFr7mN7DYG4zH9A,16967
+trilogy/core/processing/nodes/__init__.py,sha256=oHDknqAZFm6KSDDbzJXj8HDGDh-dn-YjHDGqiXc8Psw,6054
+trilogy/core/processing/nodes/base_node.py,sha256=IdKR2yaQGY1iRgKXgxF1UtlyuJEmPXWRh0rGFXv7Z_U,18111
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
 trilogy/core/processing/nodes/group_node.py,sha256=MUvcOg9U5J6TnWBel8eht9PdI9BfAKjUxmfjP_ZXx9o,10484
 trilogy/core/processing/nodes/merge_node.py,sha256=02oWRca0ba41U6PSAB14jwnWWxoyrvxRPLwkli259SY,15865
@@ -69,9 +70,9 @@ trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
 trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=lZup3hq-DyYczpG260a0wBByHyOGkbw4-yrPJvXKOM4,42300
-trilogy/dialect/bigquery.py,sha256=mGnBl5A3rVi4f1gt74jnaxSOCheA07OcRi6ZD8KWOGg,3436
-trilogy/dialect/common.py,sha256=JQ8ONloalaWEXsTTWUhZcYyzMRaZ9HdUw7cN6QWtY5c,5295
+trilogy/dialect/base.py,sha256=Y4m4RQdYI3usjeTLJKUM_SIIyuxXZfe2hboG5JSxDLU,42412
+trilogy/dialect/bigquery.py,sha256=MyUumO8CeyPLh2JquoKPp6yQSZYEzUQ2mM07LCps-CA,3526
+trilogy/dialect/common.py,sha256=EFf2Ye7XcwTti7IsFRwMo_4AW2CF8eaxSk8XA0mA5qw,5400
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
 trilogy/dialect/duckdb.py,sha256=IQzaRaCv5c6TUDERhbsLM4uTW0aGkO_DrAMR5k_j7TU,3861
@@ -90,9 +91,9 @@ trilogy/parsing/common.py,sha256=g1RmQF4fS_OgkcC6j4hnKIcn_ap0fFa_kzNUlH5D0nA,297
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=p2YnE-YV-Dt0FlC6rP7Rq8phNxzk_O4ukNzVIDyHyu4,70054
+trilogy/parsing/parse_engine.py,sha256=mQzUSWOX2QdUng2ozAkqLDfMLI_NoqPjAiauhe_mHz4,70606
 trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
-trilogy/parsing/trilogy.lark,sha256=ijY6220e2hV21F1XFsvpYRimSrpNGIdjP7b0TVz7caI,13814
+trilogy/parsing/trilogy.lark,sha256=ybs65Ckb89PCitK4hcwy6znqElcWvIeMDQzsI2p_3YI,14197
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -101,8 +102,8 @@ trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.52.dist-info/METADATA,sha256=TIB3nLBjPqlhtsB-ZgL61mCM8MPkXdJkhB-nAIuhdcA,9095
-pytrilogy-0.0.3.52.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-pytrilogy-0.0.3.52.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.52.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.52.dist-info/RECORD,,
+pytrilogy-0.0.3.54.dist-info/METADATA,sha256=_Bv_Q8dZSrH750iNkW7KtpUP1vSAwZRnBzsG8KDXdXE,9095
+pytrilogy-0.0.3.54.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+pytrilogy-0.0.3.54.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.54.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.54.dist-info/RECORD,,

{pytrilogy-0.0.3.52.dist-info → pytrilogy-0.0.3.54.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.52"
+__version__ = "0.0.3.54"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -181,6 +181,11 @@ class FunctionType(Enum):
     STRPOS = "strpos"
     CONTAINS = "contains"
 
+    # STRING REGEX
+    REGEXP_CONTAINS = "regexp_contains"
+    REGEXP_EXTRACT = "regexp_extract"
+    REGEXP_REPLACE = "regexp_replace"
+
     # Dates
     DATE = "date"
     DATETIME = "datetime"

trilogy/core/functions.py

@@ -360,6 +360,24 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
         output_type=DataType.STRING,
         arg_count=1,
     ),
+    FunctionType.REGEXP_CONTAINS: FunctionConfig(
+        valid_inputs={DataType.STRING},
+        output_purpose=Purpose.PROPERTY,
+        output_type=DataType.BOOL,
+        arg_count=2,
+    ),
+    FunctionType.REGEXP_EXTRACT: FunctionConfig(
+        valid_inputs={DataType.STRING},
+        output_purpose=Purpose.PROPERTY,
+        output_type=DataType.STRING,
+        arg_count=2,
+    ),
+    FunctionType.REGEXP_REPLACE: FunctionConfig(
+        valid_inputs={DataType.STRING},
+        output_purpose=Purpose.PROPERTY,
+        output_type=DataType.STRING,
+        arg_count=3,
+    ),
     FunctionType.DATE: FunctionConfig(
         valid_inputs={
             DataType.DATE,

trilogy/core/models/execute.py

@@ -37,6 +37,7 @@ from trilogy.core.models.build import (
     LooseBuildConceptList,
 )
 from trilogy.core.models.datasource import Address
+from trilogy.core.utility import safe_quote
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[MODELS_EXECUTE]"
@@ -201,6 +202,7 @@ class CTE(BaseModel):
             self.base_name_override = ds_being_inlined.safe_location
             self.base_alias_override = ds_being_inlined.safe_identifier
 
+        # if we have a join to the parent, we need to remove it
         for join in self.joins:
             if isinstance(join, InstantiatedUnnestJoin):
                 continue
@@ -322,17 +324,20 @@ class CTE(BaseModel):
         return self.source.name
 
     @property
-    def quote_address(self) -> dict[str, bool]:
-
-        return {
-            candidate.safe_identifier: (
-                candidate.address.quoted and not candidate.address.is_query
-                if isinstance(candidate, BuildDatasource)
-                and isinstance(candidate.address, Address)
-                else False
-            )
-            for candidate in self.source.datasources
-        }
+    def quote_address(self) -> bool:
+        if self.is_root_datasource:
+            root = self.source.datasources[0]
+            if isinstance(root, BuildDatasource) and isinstance(root.address, Address):
+                return not root.address.is_query
+            return True
+        elif not self.source.datasources:
+            return False
+        base = self.source.datasources[0]
+        if isinstance(base, BuildDatasource):
+            if isinstance(base.address, Address):
+                return not base.address.is_query
+            return True
+        return True
 
     @property
     def base_alias(self) -> str:
@@ -926,8 +931,8 @@ class Join(BaseModel):
     def right_ref(self) -> str:
         if self.quote:
             if self.right_cte.identifier in self.inlined_ctes:
-                return f"{self.quote}{self.right_cte.source.datasources[0].safe_location}{self.quote} as {self.right_cte.source.datasources[0].safe_identifier}"
-            return self.right_cte.safe_identifier
+                return f"{safe_quote(self.right_cte.source.datasources[0].safe_location, self.quote)} as {self.quote}{self.right_cte.source.datasources[0].safe_identifier}{self.quote}"
+            return f"{self.quote}{self.right_cte.safe_identifier}{self.quote}"
         if self.right_cte.identifier in self.inlined_ctes:
             return f"{self.right_cte.source.datasources[0].safe_location} as {self.right_cte.source.datasources[0].safe_identifier}"
         return self.right_cte.safe_identifier

trilogy/core/optimization.py

@@ -120,6 +120,13 @@ def gen_inverse_map(input: list[CTE | UnionCTE]) -> dict[str, list[CTE | UnionCT
     return inverse_map
 
 
+SENSITIVE_DERIVATIONS = [
+    Derivation.UNNEST,
+    Derivation.WINDOW,
+    # Derivation.AGGREGATE,
+]
+
+
 def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
     # if isinstance(select, (PersistStatement, MultiSelectStatement)):
     #     return False
@@ -151,21 +158,31 @@ def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
     ]
     condition_arguments = cte.condition.row_arguments if cte.condition else []
     for x in derived_concepts:
-        if x.derivation == Derivation.WINDOW:
-            return None
-        if x.derivation == Derivation.UNNEST:
-            return None
-        if x.derivation == Derivation.AGGREGATE:
+        if x.derivation in SENSITIVE_DERIVATIONS:
             return None
     for x in parent_derived_concepts:
         if x.address not in condition_arguments:
             continue
-        if x.derivation == Derivation.UNNEST:
-            return None
-        if x.derivation == Derivation.WINDOW:
+        if x.derivation in SENSITIVE_DERIVATIONS:
             return None
+    for x in condition_arguments:
+        # if it's derived in the parent
+        if x.address in parent_derived_concepts:
+            if x.derivation in SENSITIVE_DERIVATIONS:
+                return None
+            # this maybe needs to be recursive if we flatten a ton of derivation
+            # into one CTE
+            if not x.lineage:
+                continue
+            for z in x.lineage.concept_arguments:
+                # if it was preexisting in the parent, it's safe
+                if z.address in direct_parent.source.input_concepts:
+                    continue
+                # otherwise if it's dangerous, play it safe.
+                if z.derivation in SENSITIVE_DERIVATIONS:
+                    return None
     logger.info(
-        f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Removing redundant output CTE {cte.name} with derived_concepts {[x.address for x in derived_concepts]}"
     )
     return direct_parent
 

trilogy/core/processing/node_generators/rowset_node.py

@@ -95,9 +95,7 @@ def gen_rowset_node(
         f"{padding(depth)}{LOGGER_PREFIX} hiding {final_hidden} local optional {local_optional}"
     )
     node.hide_output_concepts(final_hidden)
-    assert node.resolution_cache
-    # assume grain to be output of select
-    # but don't include anything hidden(the non-rowset concepts)
+
     node.grain = BuildGrain.from_concepts(
         [
             x

trilogy/core/processing/node_generators/window_node.py

@@ -11,7 +11,12 @@ from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     gen_enrichment_node,
 )
-from trilogy.core.processing.nodes import History, StrategyNode, WindowNode
+from trilogy.core.processing.nodes import (
+    History,
+    StrategyNode,
+    WhereSafetyNode,
+    WindowNode,
+)
 from trilogy.core.processing.utility import create_log_lambda, padding
 from trilogy.utility import unique
 
@@ -71,10 +76,13 @@ def gen_window_node(
     if equivalent_optional:
         for x in equivalent_optional:
             assert isinstance(x.lineage, WINDOW_TYPES)
+            base, parents = resolve_window_parent_concepts(x, environment)
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found equivalent optional {x} with parents {resolve_window_parent_concepts(x, environment)[1]}"
+                f"{padding(depth)}{LOGGER_PREFIX} found equivalent optional {x} with parents {parents}"
             )
             additional_outputs.append(x)
+            # also append the base concept it's being grouped over
+            targets.append(base)
 
     grain_equivalents = [
         x
@@ -85,7 +93,8 @@ def gen_window_node(
     ]
 
     for x in grain_equivalents:
-        logger.info("Appending grain equivalent %s", x)
+        if x.address in additional_outputs:
+            continue
         targets.append(x)
 
     # finally, the ones we'll need to enrich
@@ -134,7 +143,7 @@ def gen_window_node(
     _window_node.rebuild_cache()
     _window_node.resolve()
 
-    window_node = StrategyNode(
+    window_node = WhereSafetyNode(
         input_concepts=[concept] + additional_outputs + parent_concepts + targets,
         output_concepts=[concept] + additional_outputs + parent_concepts + targets,
         environment=environment,

trilogy/core/processing/nodes/__init__.py

@@ -6,7 +6,7 @@ from trilogy.core.models.build import BuildConcept, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.models.environment import Environment
 
-from .base_node import NodeJoin, StrategyNode
+from .base_node import NodeJoin, StrategyNode, WhereSafetyNode
 from .filter_node import FilterNode
 from .group_node import GroupNode
 from .merge_node import MergeNode
@@ -193,4 +193,5 @@ __all__ = [
     "UnnestNode",
     "UnionNode",
     "History",
+    "WhereSafetyNode",
 ]

trilogy/core/processing/nodes/base_node.py

@@ -291,9 +291,14 @@ class StrategyNode:
     def add_output_concept(self, concept: BuildConcept, rebuild: bool = True):
         return self.add_output_concepts([concept], rebuild)
 
-    def hide_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
+    def hide_output_concepts(
+        self, concepts: List[BuildConcept] | list[str] | set[str], rebuild: bool = True
+    ):
         for x in concepts:
-            self.hidden_concepts.add(x.address)
+            if isinstance(x, BuildConcept):
+                self.hidden_concepts.add(x.address)
+            else:
+                self.hidden_concepts.add(x)
         if rebuild:
             self.rebuild_cache()
         return self
@@ -471,3 +476,28 @@ class NodeJoin:
             f" {self.right_node} on"
             f" {','.join([str(k) for k in self.concepts])}"
         )
+
+
+class WhereSafetyNode(StrategyNode):
+    """Specialized node to be used to pad certain
+    select outputs that can't be immediately used in a where
+    clause; eg window functions. Will remove itself if not required."""
+
+    def resolve(self) -> QueryDatasource:
+        if not self.conditions and len(self.parents) == 1:
+            parent = self.parents[0]
+            parent = parent.copy()
+            # avoid performance hit by not rebuilding until end
+            parent.set_output_concepts(self.output_concepts, rebuild=False)
+            parent.hide_output_concepts(self.hidden_concepts, rebuild=False)
+
+            # these conditions
+            if self.preexisting_conditions:
+                parent.set_preexisting_conditions(self.preexisting_conditions)
+            # TODO: add a helper for this
+            parent.ordering = self.ordering
+
+            # actually build the node
+            parent.rebuild_cache()
+            return parent.resolve()
+        return super().resolve()

trilogy/core/utility.py

@@ -0,0 +1,8 @@
+def safe_quote(string: str, quote_char: str):
+    # split dotted identifiers
+    # TODO: evaluate if we need smarter parsing for strings that could actually include .
+    if string.startswith("https://"):
+        # it's a url, no splitting
+        return f"{quote_char}{string}{quote_char}"
+    components = string.split(".")
+    return ".".join([f"{quote_char}{string}{quote_char}" for string in components])

trilogy/dialect/base.py

@@ -74,6 +74,7 @@ from trilogy.core.statements.execute import (
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
 )
+from trilogy.core.utility import safe_quote
 from trilogy.dialect.common import render_join, render_unnest
 from trilogy.hooks.base_hook import BaseHook
 
@@ -204,6 +205,9 @@ FUNCTION_MAP = {
     FunctionType.SUBSTRING: lambda x: f"SUBSTRING({x[0]},{x[1]},{x[2]})",
     FunctionType.STRPOS: lambda x: f"STRPOS({x[0]},{x[1]})",
     FunctionType.CONTAINS: lambda x: f"CONTAINS({x[0]},{x[1]})",
+    FunctionType.REGEXP_CONTAINS: lambda x: f"REGEXP_CONTAINS({x[0]},{x[1]})",
+    FunctionType.REGEXP_EXTRACT: lambda x: f"REGEXP_EXTRACT({x[0]},{x[1]})",
+    FunctionType.REGEXP_REPLACE: lambda x: f"REGEXP_REPLACE({x[0]},{x[1]}, {x[2]})",
     # FunctionType.NOT_LIKE: lambda x: f" CASE WHEN {x[0]} like {x[1]} THEN 0 ELSE 1 END",
     # date types
     FunctionType.DATE_TRUNCATE: lambda x: f"date_trunc({x[0]},{x[1]})",
@@ -270,13 +274,6 @@ ORDER BY{% for order in order_by %}
 )
 
 
-def safe_quote(string: str, quote_char: str):
-    # split dotted identifiers
-    # TODO: evaluate if we need smarter parsing for strings that could actually include .
-    components = string.split(".")
-    return ".".join([f"{quote_char}{string}{quote_char}" for string in components])
-
-
 def safe_get_cte_value(coalesce, cte: CTE | UnionCTE, c: BuildConcept, quote_char: str):
     address = c.address
     raw = cte.source_map.get(address, None)
@@ -285,12 +282,17 @@ def safe_get_cte_value(coalesce, cte: CTE | UnionCTE, c: BuildConcept, quote_cha
         return None
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
-        return f"{raw}.{safe_quote(rendered, quote_char)}"
+        return f"{quote_char}{raw}{quote_char}.{safe_quote(rendered, quote_char)}"
     if isinstance(raw, list) and len(raw) == 1:
         rendered = cte.get_alias(c, raw[0])
-        return f"{raw[0]}.{safe_quote(rendered, quote_char)}"
+        return f"{quote_char}{raw[0]}{quote_char}.{safe_quote(rendered, quote_char)}"
     return coalesce(
-        sorted([f"{x}.{safe_quote(cte.get_alias(c, x), quote_char)}" for x in raw])
+        sorted(
+            [
+                f"{quote_char}{x}{quote_char}.{safe_quote(cte.get_alias(c, x), quote_char)}"
+                for x in raw
+            ]
+        )
     )
 
 
@@ -783,12 +785,12 @@ class BaseDialect:
             else:
                 source = None
         else:
-            if cte.quote_address.get(cte.source.datasources[0].safe_identifier, False):
-                source = f"{self.QUOTE_CHARACTER}{cte.base_name}{self.QUOTE_CHARACTER}"
+            if cte.quote_address:
+                source = safe_quote(cte.base_name, self.QUOTE_CHARACTER)
             else:
                 source = cte.base_name
             if cte.base_name != cte.base_alias:
-                source = f"{source} as {cte.base_alias}"
+                source = f"{source} as {self.QUOTE_CHARACTER}{cte.base_alias}{self.QUOTE_CHARACTER}"
         if not cte.render_from_clause:
             final_joins = []
         else:

trilogy/dialect/bigquery.py

@@ -18,6 +18,7 @@ FUNCTION_MAP = {
     FunctionType.LIKE: lambda x: (
         f" CASE WHEN {x[0]} like {x[1]} THEN True ELSE False END"
     ),
+    FunctionType.IS_NULL: lambda x: f"CASE WHEN {x[0]} IS NULL THEN True ELSE False END",
     FunctionType.MINUTE: lambda x: f"EXTRACT(MINUTE from {x[0]})",
     FunctionType.SECOND: lambda x: f"EXTRACT(SECOND from {x[0]})",
     FunctionType.HOUR: lambda x: f"EXTRACT(HOUR from {x[0]})",

trilogy/dialect/common.py

@@ -63,8 +63,8 @@ def render_join_concept(
         elif isinstance(raw_content, BuildFunction):
             rval = render_expr(raw_content, cte=cte)
             return rval
-        return f"{name}.{quote_character}{raw_content}{quote_character}"
-    return f"{name}.{quote_character}{concept.safe_address}{quote_character}"
+        return f"{quote_character}{name}{quote_character}.{quote_character}{raw_content}{quote_character}"
+    return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 
 def render_join(
@@ -91,8 +91,9 @@ def render_join(
         return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.object_to_unnest, render_expr_func, cte)}"
     # left_name = join.left_name
     right_name = join.right_name
-    if cte.quote_address.get(join.right_name, False):
-        join.quote = quote_character
+    join.quote = quote_character
+    # if cte.quote_address.get(join.right_name, False):
+    #     join.quote = quote_character
     right_base = join.right_ref
     base_joinkeys = []
     if join.joinkey_pairs:

trilogy/parsing/parse_engine.py

@@ -1744,9 +1744,27 @@ class ParseToObjects(Transformer):
         return self.function_factory.create_function(args, FunctionType.SUBSTRING, meta)
 
     @v_args(meta=True)
-    def lower(self, meta, args):
+    def flower(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.LOWER, meta)
 
+    @v_args(meta=True)
+    def fregexp_contains(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_CONTAINS, meta
+        )
+
+    @v_args(meta=True)
+    def fregexp_extract(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_EXTRACT, meta
+        )
+
+    @v_args(meta=True)
+    def fregexp_replace(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_REPLACE, meta
+        )
+
     # date functions
     @v_args(meta=True)
     def fdate(self, meta, args):

trilogy/parsing/trilogy.lark

@@ -267,7 +267,7 @@
     _UPPER.1: "upper("i
     upper: _UPPER expr ")"
     _LOWER.1: "lower("i
-    lower: _LOWER expr ")"  
+    flower: _LOWER expr ")"  
     _SPLIT.1: "split("i  
     fsplit: _SPLIT expr "," string_lit ")"
     _STRPOS.1: "strpos("i
@@ -276,8 +276,14 @@
     fcontains: _CONTAINS expr "," expr ")"
     _SUBSTRING.1: "substring("i
     fsubstring: _SUBSTRING expr "," expr "," expr ")"
-    
-    _string_functions: like | ilike | upper | lower | fsplit | fstrpos | fsubstring | fcontains
+    _REGEXP_EXTRACT.1: "regexp_extract("
+    fregexp_extract: _REGEXP_EXTRACT expr "," expr ")"
+    _REGEXP_CONTAINS.1: "regexp_contains("
+    fregexp_contains: _REGEXP_CONTAINS expr "," expr  ")"
+    _REGEXP_REPLACE.1: "regexp_replace("
+    fregexp_replace: _REGEXP_REPLACE expr "," expr "," expr   ")"
+
+    _string_functions: like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains | fregexp_extract | fregexp_contains | fregexp_replace
     
     // special aggregate
     _GROUP.1: "group("i
@@ -311,7 +317,8 @@
     _DATE.1: "date("i
     fdate:  _DATE expr ")"
     fdatetime: "datetime"i "(" expr ")"
-    ftimestamp: "timestamp"i "(" expr ")"
+    _TIMESTAMP.1: "timestamp("i
+    ftimestamp: _TIMESTAMP expr ")"
     
     _SECOND.1: "second("i
     fsecond: _SECOND expr ")"