pytrilogy 0.0.3.115__py3-none-any.whl → 0.0.3.116__py3-none-any.whl

{pytrilogy-0.0.3.115.dist-info → pytrilogy-0.0.3.116.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.115
+Version: 0.0.3.116
 Summary: Declarative, typed query language that compiles to SQL.
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3

{pytrilogy-0.0.3.115.dist-info → pytrilogy-0.0.3.116.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pytrilogy-0.0.3.115.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=L11AizxIDnbCCjCvAE55VTsy0d4QmvAZfNp61uAPu00,304
-trilogy/constants.py,sha256=2ERigKZBmULg3Qr43nWKgdwqiCnT8XPQVeH-O-k66wc,2640
+pytrilogy-0.0.3.116.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=4zCLL3ve6Z14PQ3LocAoJnyiTflxDn90gE8yUR-T9-w,304
+trilogy/constants.py,sha256=_Tm7YGaAZuxH77X5ve0TajU0dQD7RcGV6ECrTHRL3qQ,2678
 trilogy/engine.py,sha256=v4TpNktM4zZ9OX7jZH2nde4dpX5uAH2U23ELfULTCSg,2280
-trilogy/executor.py,sha256=q3EsAjzgxNxPn-yTHD_FTFzm7bJ2mlf9CrJEjyt6-pE,17884
+trilogy/executor.py,sha256=uKlCnPp4FHkgsa_dDcQJ4y-ObtvKat2KFx05c-z1mZo,17885
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/render.py,sha256=qQWwduymauOlB517UtM-VGbVe8Cswa4UJub5aGbSO6c,1512
@@ -23,45 +23,45 @@ trilogy/ai/providers/utils.py,sha256=yttP6y2E_XzdytBCwhaKekfXfxM6gE6MRce4AtyLL60
 trilogy/authoring/__init__.py,sha256=TABMOETSMERrWuyDLR0nK4ISlqR0yaqeXrmuOdrSvAY,3060
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=nizWYDCJQ1bigQMtkNIEMNTcN0NoEAXiIHLzpelxQ24,201
-trilogy/core/enums.py,sha256=R_iNsK8j0uGJ5fzfFUwoe1vhIHTX28VJfDQd99WBQ4E,9064
+trilogy/core/enums.py,sha256=SkuyRrqqS59LO08CWwbFaZ5kYw_w0WWWcigd9MSXdaA,9082
 trilogy/core/env_processor.py,sha256=H-rr2ALj31l5oh3FqeI47Qju6OOfiXBacXNJGNZ92zQ,4521
 trilogy/core/environment_helpers.py,sha256=TRlqVctqIRBxzfjRBmpQsAVoiCcsEKBhG1B6PUE0l1M,12743
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=axkVXYJYQXCCwMHwlyDA232g4tCOwdCZUt7eHeUMDMg,2829
-trilogy/core/functions.py,sha256=hURJBM5dNcHfXuPHaHM_tz_ahsEktnmL4At0zebfjMk,34668
+trilogy/core/functions.py,sha256=QidWH7dDjvLjXA0ujCkemUzYjkYmtMD2T78hnvk86K8,34876
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
-trilogy/core/optimization.py,sha256=Km0ITEx9n6Iv5ReX6tm4uXO5uniSv_ooahycNNiET3g,9212
+trilogy/core/optimization.py,sha256=eKieOaWXUtoNTVQbThGA5tqrI06ZR6SUFOqGe4Jw0k4,9262
 trilogy/core/query_processor.py,sha256=rMrtLSQxVm7yeyh0nWjDNI9nnu4Xi0NgHvBJ14gvu4I,20384
 trilogy/core/utility.py,sha256=3VC13uSQWcZNghgt7Ot0ZTeEmNqs__cx122abVq9qhM,410
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=DFpIGlIruz0JYIOLJ0qZQrsdT4BvQX0augow1G8pD54,81841
-trilogy/core/models/build.py,sha256=LN8G1JRk8RDSFQwqS3CUszIIUgB5v9Nxy6EpxvAcJ1A,71514
+trilogy/core/models/author.py,sha256=YnBwxanbn9uBAExz5POM0_RFw9GIHmjfLZRVzA10Gws,85862
+trilogy/core/models/build.py,sha256=zVHD8jo8V8E0aGWtobzXG7hY3VjLChiW4s1QBx9sD5k,72938
 trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYycCrP4zF-rHU,5258
 trilogy/core/models/core.py,sha256=iT9WdZoiXeglmUHWn6bZyXCTBpkApTGPKtNm_Mhbu_g,12987
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
-trilogy/core/models/environment.py,sha256=do1Xvr9oyjDj0knAxgIqexSbt0HMrHbLJNyl9utkSvs,28760
+trilogy/core/models/environment.py,sha256=-r6_imqkJ10iPTbWGrdfeF2Th480UKLsG8ihACyqO8w,28823
 trilogy/core/models/execute.py,sha256=3fgEdho2e7S0outq91cCzb9jFwz6L1hTbsTrJwGvIFs,42311
 trilogy/core/optimizations/__init__.py,sha256=yspWc25M5SgAuvXYoSt5J8atyPbDlOfsKjIo5yGD9s4,368
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
-trilogy/core/optimizations/hide_unused_concept.py,sha256=DbsP8NqQOxmPv9omDOoFNPUGObUkqsRRNrr5d1xDxx4,1962
+trilogy/core/optimizations/hide_unused_concept.py,sha256=THp6byyh64dArSz6EuY7unik5WKgQQfWxNGToEqjqE0,1875
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
 trilogy/core/optimizations/predicate_pushdown.py,sha256=5ubatgq1IwWQ4L2FDt4--y168YLuGP-vwqH0m8IeTIw,9786
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=Y6AG10NGU_mUiTfXmxkss7QSO5iLpOdzGN6-7Hc5XiQ,23294
+trilogy/core/processing/concept_strategies_v3.py,sha256=MBQeJGBDW2w3xxwano43-5MVBqfbIU9M3K7RFbnSKNA,23367
 trilogy/core/processing/discovery_node_factory.py,sha256=llnLxZo1NqBRIuuPz0GUohym6LZFhVkPT3xSiORi3k4,15446
 trilogy/core/processing/discovery_utility.py,sha256=mO0npZMRlQSzxt3l4m8garKBAOrXFkzt3eiiUyUSoIU,13528
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
 trilogy/core/processing/utility.py,sha256=ESs6pKqVP2c9eMdfB2JNjw7D7YnoezVwbLFx1D6OUYA,26088
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
-trilogy/core/processing/node_generators/basic_node.py,sha256=rv53il-W633by0plvbN5qaqznJfl60yPinLZGBAC_MI,5707
-trilogy/core/processing/node_generators/common.py,sha256=xF32Kf6B08dZgKs2SOow1HomptSiSC057GCUCHFlS5s,9464
+trilogy/core/processing/node_generators/basic_node.py,sha256=IeR00MV6Ay1TDuebcbwNTp1WC9kT23_JIWYpeUqtn_M,5992
+trilogy/core/processing/node_generators/common.py,sha256=oGUW4_RhSpfGu9PNUj0JsHB8iMeTz76FQHMcfVRHP30,9516
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
 trilogy/core/processing/node_generators/filter_node.py,sha256=EiP_tafx-X0gM-BIVRCy2rnp1_apt2cbhVfv8cg9dVg,11259
 trilogy/core/processing/node_generators/group_node.py,sha256=sIm1QYrF4EY6sk56A48B6MieCZqvaJLSQebih_aiKnQ,8567
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
-trilogy/core/processing/node_generators/multiselect_node.py,sha256=dHPDoSKU0FF6Ue_t_LkZxTd0Q-Sf-EpYdsMYdyUlFQc,7120
+trilogy/core/processing/node_generators/multiselect_node.py,sha256=qrgx8ofExbigipFBXKufu279vKWBLSBP91TyJW6g1qE,7135
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=hNcZxnDLTZyYJWfojg769zH9HB9PfZfESmpN1lcHWXg,23172
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=MuVNIexXhqGONho_mewqMOwaYXNUnjjvyPvk_RDGNYE,5943
@@ -84,7 +84,7 @@ trilogy/core/processing/nodes/union_node.py,sha256=hLAXXVWqEgMWi7dlgSHfCF59fon64
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
 trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/statements/author.py,sha256=zLd-1IZKF5UE7oWQYqifn3b3JcW2wCbonlELjJiZhfo,16436
+trilogy/core/statements/author.py,sha256=uEYJ9Q39TdGP0cxQEv3Ue4pGp_dmZ-NiT6k2-rfZERQ,16525
 trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/statements/common.py,sha256=VnVLULQg1TJLNUFzJaROT1tsf2ewk3IpuhvZaP36R6A,535
 trilogy/core/statements/execute.py,sha256=kiwJcVeMa4wZR-xLfM2oYOJ9DeyJkP8An38WFyJxktM,2413
@@ -95,7 +95,7 @@ trilogy/core/validation/datasource.py,sha256=nJeEFyb6iMBwlEVdYVy1vLzAbdRZwOsUjGx
 trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
 trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=Eo8n5fJiHOYkfDDDFgnrqMuIZTxRorWwVer_1-DRG4c,51178
+trilogy/dialect/base.py,sha256=i_rdrO3TmKYZK4K8k6ggpvRmzaCl1oR65n89_yHNEPo,52445
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
 trilogy/dialect/common.py,sha256=I5Ku_TR5MwJTB3ZhcQenrtvXhH2RvTQ8wQe9w5lfkfA,5708
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -113,13 +113,13 @@ trilogy/hooks/graph_hook.py,sha256=5BfR7Dt0bgEsCLgwjowgCsVkboGYfVJGOz8g9mqpnos,4
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=r9EmWzN2ozMKtWCY1HfjOREJnLKvaURT6h6edS5U4x8,32508
+trilogy/parsing/common.py,sha256=GijDRpysULL6vQWpFcjgxVASuTWXUVUi5fILHvjzkbg,35534
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=lMo73QD37ih5kgvAOToMLCBbCkBzo_G7QMOaQaHprV8,84333
+trilogy/parsing/parse_engine.py,sha256=iHBRUatv4hXSdDgAjllgTe1OtD8WE7V0Gu-ZTT2YugY,86301
 trilogy/parsing/render.py,sha256=k7MNp8EBTqVBSVqFlgTHSwIhfSKLyJfSeb2fSbt9dVA,24307
-trilogy/parsing/trilogy.lark,sha256=kPOIrwa__kdXvxS3uM5GzdlgIa2cQxNW_0DNRqsRBeg,16939
+trilogy/parsing/trilogy.lark,sha256=6nQ4fLpiL0pNMAlmmIEwN8Am4DlIIYpBVajp4jHveA8,17185
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -132,8 +132,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.115.dist-info/METADATA,sha256=pB3OdY3t5s8GkX73rniQKQkMSy8JcXjzKZ0w1va3k5E,12911
-pytrilogy-0.0.3.115.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.115.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.115.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.115.dist-info/RECORD,,
+pytrilogy-0.0.3.116.dist-info/METADATA,sha256=N3aeyGXHRF0etmB8YiAXUHOfidTmQgqDcxUm8enTumw,12911
+pytrilogy-0.0.3.116.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.116.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.116.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.116.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.115"
+__version__ = "0.0.3.116"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -31,6 +31,7 @@ class Optimizations:
     constant_inlining: bool = True
     constant_inline_cutoff: int = 10
     direct_return: bool = True
+    hide_unused_concepts: bool = True
 
 
 @dataclass

trilogy/core/enums.py

@@ -217,6 +217,7 @@ class FunctionType(Enum):
     CONTAINS = "contains"
     TRIM = "trim"
     REPLACE = "replace"
+    HASH = "hash"
 
     # STRING REGEX
     REGEXP_CONTAINS = "regexp_contains"

trilogy/core/functions.py

@@ -928,6 +928,14 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
         output_type=DataType.TIMESTAMP,
         arg_count=1,
     ),
+    FunctionType.HASH: FunctionConfig(
+        valid_inputs={
+            DataType.STRING,
+        },
+        output_purpose=Purpose.PROPERTY,
+        output_type=DataType.STRING,
+        arg_count=2,
+    ),
 }
 
 EXCLUDED_FUNCTIONS = {

trilogy/core/models/author.py

@@ -865,6 +865,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
             AggregateWrapper,
             RowsetItem,
             MultiSelectLineage,
+            Comparison,
         ]
     ] = None
     namespace: str = Field(default=DEFAULT_NAMESPACE, validate_default=True)
@@ -1079,6 +1080,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         | AggregateWrapper
         | RowsetItem
         | MultiSelectLineage
+        | Comparison
         | None,
         Grain,
         set[str] | None,
@@ -1178,6 +1180,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
                     AggregateWrapper,
                     RowsetItem,
                     MultiSelectLineage,
+                    Comparison,
                 ],
                 output: List[ConceptRef],
             ):
@@ -1204,6 +1207,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
     def calculate_derivation(self, lineage, purpose: Purpose) -> Derivation:
         from trilogy.core.models.build import (
             BuildAggregateWrapper,
+            BuildComparison,
             BuildFilterItem,
             BuildFunction,
             BuildMultiSelectLineage,
@@ -1221,6 +1225,8 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         #     return Derivation.PARENTHETICAL
         elif lineage and isinstance(lineage, (BuildRowsetItem, RowsetItem)):
             return Derivation.ROWSET
+        elif lineage and isinstance(lineage, BuildComparison):
+            return Derivation.BASIC
         elif lineage and isinstance(
             lineage, (BuildMultiSelectLineage, MultiSelectLineage)
         ):
@@ -2133,6 +2139,89 @@ class AlignClause(Namespaced, BaseModel):
         )
 
 
+class DeriveItem(Namespaced, DataTyped, ConceptArgs, Mergeable, BaseModel):
+    expr: Expr
+    name: str
+    namespace: str
+
+    @property
+    def derived_concept(self) -> str:
+        return f"{self.namespace}.{self.name}"
+        # return ConceptRef(
+        #     address=f"{self.namespace}.{self.name}",
+        #     datatype=arg_to_datatype(self.expr),
+        # )
+
+    def with_namespace(self, namespace):
+        return DeriveItem.model_construct(
+            expr=(self.expr.with_namespace(namespace) if self.expr else None),
+            name=self.name,
+            namespace=namespace,
+        )
+
+    def with_merge(
+        self, source: Concept, target: Concept, modifiers: List[Modifier]
+    ) -> "DeriveItem":
+        return DeriveItem.model_construct(
+            expr=(
+                self.expr.with_merge(source, target, modifiers)
+                if isinstance(self.expr, Mergeable)
+                else self.expr
+            ),
+            name=self.name,
+            namespace=self.namespace,
+        )
+
+    def with_reference_replacement(self, source, target):
+        return DeriveItem.model_construct(
+            expr=(
+                self.expr.with_reference_replacement(source, target)
+                if isinstance(self.expr, Mergeable)
+                else self.expr
+            ),
+            name=self.name,
+            namespace=self.namespace,
+        )
+
+
+class DeriveClause(Mergeable, Namespaced, BaseModel):
+    items: List[DeriveItem]
+
+    def with_namespace(self, namespace: str) -> "DeriveClause":
+        return DeriveClause.model_construct(
+            items=[
+                x.with_namespace(namespace) if isinstance(x, Namespaced) else x
+                for x in self.items
+            ]
+        )
+
+    def with_merge(
+        self, source: Concept, target: Concept, modifiers: List[Modifier]
+    ) -> "DeriveClause":
+        return DeriveClause.model_construct(
+            items=[
+                (
+                    x.with_merge(source, target, modifiers)
+                    if isinstance(x, Mergeable)
+                    else x
+                )
+                for x in self.items
+            ]
+        )
+
+    def with_reference_replacement(self, source, target):
+        return DeriveClause.model_construct(
+            items=[
+                (
+                    x.with_reference_replacement(source, target)
+                    if isinstance(x, Mergeable)
+                    else x
+                )
+                for x in self.items
+            ]
+        )
+
+
 class SelectLineage(Mergeable, Namespaced, BaseModel):
     selection: List[ConceptRef]
     hidden_components: set[str]
@@ -2177,15 +2266,40 @@ class SelectLineage(Mergeable, Namespaced, BaseModel):
             ),
         )
 
+    def with_namespace(self, namespace):
+        return SelectLineage.model_construct(
+            selection=[x.with_namespace(namespace) for x in self.selection],
+            hidden_components=self.hidden_components,
+            local_concepts={
+                x: y.with_namespace(namespace) for x, y in self.local_concepts.items()
+            },
+            order_by=self.order_by.with_namespace(namespace) if self.order_by else None,
+            limit=self.limit,
+            meta=self.meta,
+            grain=self.grain.with_namespace(namespace),
+            where_clause=(
+                self.where_clause.with_namespace(namespace)
+                if self.where_clause
+                else None
+            ),
+            having_clause=(
+                self.having_clause.with_namespace(namespace)
+                if self.having_clause
+                else None
+            ),
+        )
+
 
 class MultiSelectLineage(Mergeable, ConceptArgs, Namespaced, BaseModel):
     selects: List[SelectLineage]
     align: AlignClause
+
     namespace: str
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
     where_clause: Union["WhereClause", None] = Field(default=None)
     having_clause: Union["HavingClause", None] = Field(default=None)
+    derive: DeriveClause | None = None
     hidden_components: set[str]
 
     @property
@@ -2211,6 +2325,11 @@ class MultiSelectLineage(Mergeable, ConceptArgs, Namespaced, BaseModel):
         new = MultiSelectLineage.model_construct(
             selects=[s.with_merge(source, target, modifiers) for s in self.selects],
             align=self.align,
+            derive=(
+                self.derive.with_merge(source, target, modifiers)
+                if self.derive
+                else None
+            ),
             namespace=self.namespace,
             hidden_components=self.hidden_components,
             order_by=(
@@ -2236,6 +2355,7 @@ class MultiSelectLineage(Mergeable, ConceptArgs, Namespaced, BaseModel):
         return MultiSelectLineage.model_construct(
             selects=[c.with_namespace(namespace) for c in self.selects],
             align=self.align.with_namespace(namespace),
+            derive=self.derive.with_namespace(namespace) if self.derive else None,
             namespace=namespace,
             hidden_components=self.hidden_components,
             order_by=self.order_by.with_namespace(namespace) if self.order_by else None,
@@ -2257,6 +2377,9 @@ class MultiSelectLineage(Mergeable, ConceptArgs, Namespaced, BaseModel):
         output = set()
         for item in self.align.items:
             output.add(item.aligned_concept)
+        if self.derive:
+            for ditem in self.derive.items:
+                output.add(ditem.derived_concept)
         return output
 
     @property

trilogy/core/models/build.py

@@ -48,6 +48,8 @@ from trilogy.core.models.author import (
     Concept,
     ConceptRef,
     Conditional,
+    DeriveClause,
+    DeriveItem,
     FilterItem,
     FuncArgs,
     Function,
@@ -247,7 +249,7 @@ class BuildParamaterizedConceptReference(DataTyped):
     concept: BuildConcept
 
     def __str__(self):
-        return f":{self.concept.address}"
+        return f":{self.concept.address.replace('.', '_')}"
 
     @property
     def safe_address(self) -> str:
@@ -1268,6 +1270,22 @@ class BuildAlignClause:
     items: List[BuildAlignItem]
 
 
+@dataclass
+class BuildDeriveClause:
+    items: List[BuildDeriveItem]
+
+
+@dataclass
+class BuildDeriveItem:
+    expr: BuildExpr
+    name: str
+    namespace: str = field(default=DEFAULT_NAMESPACE)
+
+    @property
+    def address(self) -> str:
+        return f"{self.namespace}.{self.name}"
+
+
 @dataclass
 class BuildSelectLineage:
     selection: List[BuildConcept]
@@ -1299,12 +1317,16 @@ class BuildMultiSelectLineage(BuildConceptArgs):
     limit: Optional[int] = None
     where_clause: Union["BuildWhereClause", None] = field(default=None)
     having_clause: Union["BuildHavingClause", None] = field(default=None)
+    derive: BuildDeriveClause | None = None
 
     @property
     def derived_concepts(self) -> set[str]:
         output = set()
         for item in self.align.items:
             output.add(item.aligned_concept)
+        if self.derive:
+            for ditem in self.derive.items:
+                output.add(ditem.address)
         return output
 
     @property
@@ -1312,10 +1334,12 @@ class BuildMultiSelectLineage(BuildConceptArgs):
         return self.build_output_components
 
     @property
-    def derived_concept(self) -> set[str]:
-        output = set()
-        for item in self.align.items:
-            output.add(item.aligned_concept)
+    def calculated_derivations(self) -> set[str]:
+        output: set[str] = set()
+        if not self.derive:
+            return output
+        for item in self.derive.items:
+            output.add(item.address)
         return output
 
     @property
@@ -1335,6 +1359,7 @@ class BuildMultiSelectLineage(BuildConceptArgs):
                 for c in x.concepts:
                     if c.address in cte.output_lcl:
                         return c
+
         raise SyntaxError(
             f"Could not find upstream map for multiselect {str(concept)} on cte ({cte})"
         )
@@ -1961,6 +1986,28 @@ class Factory:
     def _build_align_clause(self, base: AlignClause) -> BuildAlignClause:
         return BuildAlignClause(items=[self._build_align_item(x) for x in base.items])
 
+    @build.register
+    def _(self, base: DeriveItem) -> BuildDeriveItem:
+        return self._build_derive_item(base)
+
+    def _build_derive_item(self, base: DeriveItem) -> BuildDeriveItem:
+        expr: Concept | FuncArgs = base.expr
+        validation = requires_concept_nesting(expr)
+        if validation:
+            expr, _ = self.instantiate_concept(validation)
+        return BuildDeriveItem(
+            expr=self.build(expr),
+            name=base.name,
+            namespace=base.namespace,
+        )
+
+    @build.register
+    def _(self, base: DeriveClause) -> BuildDeriveClause:
+        return self._build_derive_clause(base)
+
+    def _build_derive_clause(self, base: DeriveClause) -> BuildDeriveClause:
+        return BuildDeriveClause(items=[self.build(x) for x in base.items])
+
     @build.register
     def _(self, base: RowsetItem) -> BuildRowsetItem:
         return self._build_rowset_item(base)
@@ -2162,6 +2209,7 @@ class Factory:
             selects=base.selects,
             grain=final_grain,
             align=factory.build(base.align),
+            derive=factory.build(base.derive) if base.derive else None,
             # self.align.with_select_context(
             #     local_build_cache, self.grain, environment
             # ),

trilogy/core/models/environment.py

@@ -413,7 +413,8 @@ class Environment(BaseModel):
             self.imports[alias].append(imp_stm)
         # we can't exit early
         # as there may be new concepts
-        for k, concept in source.concepts.items():
+        iteration: list[tuple[str, Concept]] = list(source.concepts.items())
+        for k, concept in iteration:
             # skip internal namespace
             if INTERNAL_NAMESPACE in concept.address:
                 continue

trilogy/core/optimization.py

@@ -228,7 +228,8 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
-    REGISTERED_RULES.append(HideUnusedConcepts())
+    if CONFIG.optimizations.hide_unused_concepts:
+        REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
@@ -242,7 +243,7 @@ def optimize_ctes(
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
-            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
+        input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(
             f"[Optimization] Finished checking for {type(rule).__name__} after {loops} loop(s)"
         )

trilogy/core/optimizations/hide_unused_concept.py

@@ -39,11 +39,7 @@ class HideUnusedConcepts(OptimizationRule):
         self.log(
             f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
         )
-        candidates = [
-            x.address
-            for x in cte.output_columns
-            if x.address not in used and x.address not in cte.hidden_concepts
-        ]
+        candidates = [x.address for x in cte.output_columns if x.address not in used]
         if len(candidates) == len(set([x.address for x in cte.output_columns])):
             # pop one out
             candidates.pop()

trilogy/core/processing/concept_strategies_v3.py

@@ -496,6 +496,8 @@ def _search_concepts(
         conditions=conditions,
     )
     partial: set[str] = set()
+    virtual: set[str] = set()
+    complete = ValidationResult.INCOMPLETE
     while context.incomplete:
 
         priority_concept = get_priority_concept(

trilogy/core/processing/node_generators/basic_node.py

@@ -7,7 +7,7 @@ from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
 )
-from trilogy.core.processing.nodes import History, StrategyNode
+from trilogy.core.processing.nodes import ConstantNode, History, StrategyNode
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
@@ -51,11 +51,14 @@ def gen_basic_node(
     )
     synonyms: list[BuildConcept] = []
     ignored_optional: set[str] = set()
-    assert isinstance(concept.lineage, BuildFunction)
+
     # when we are getting an attribute, if there is anything else
     # that is an attribute of the same struct in local optional
     # select that value for discovery as well
-    if concept.lineage.operator == FunctionType.ATTR_ACCESS:
+    if (
+        isinstance(concept.lineage, BuildFunction)
+        and concept.lineage.operator == FunctionType.ATTR_ACCESS
+    ):
         logger.info(
             f"{depth_prefix}{LOGGER_PREFIX} checking for synonyms for attribute access"
         )
@@ -106,20 +109,28 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Fetching parents {[x.address for x in all_parents]}"
     )
-    parent_node: StrategyNode | None = source_concepts(
-        mandatory_list=all_parents,
-        environment=environment,
-        g=g,
-        depth=depth + 1,
-        history=history,
-        conditions=conditions,
-    )
+    if all_parents:
+        parent_node: StrategyNode | None = source_concepts(
+            mandatory_list=all_parents,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
 
-    if not parent_node:
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+        if not parent_node:
+            logger.info(
+                f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+            )
+            return None
+    else:
+        return ConstantNode(
+            input_concepts=[],
+            output_concepts=[concept],
+            environment=environment,
+            depth=depth,
         )
-        return None
     if parent_node.source_type != SourceType.CONSTANT:
         parent_node.source_type = SourceType.BASIC
     parent_node.add_output_concept(concept)

trilogy/core/processing/node_generators/common.py

@@ -4,6 +4,7 @@ from typing import Callable, List, Tuple
 from trilogy.core.enums import Derivation, Purpose
 from trilogy.core.models.build import (
     BuildAggregateWrapper,
+    BuildComparison,
     BuildConcept,
     BuildFilterItem,
     BuildFunction,
@@ -26,7 +27,9 @@ FUNCTION_TYPES = (BuildFunction,)
 def resolve_function_parent_concepts(
     concept: BuildConcept, environment: BuildEnvironment
 ) -> List[BuildConcept]:
-    if not isinstance(concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES)):
+    if not isinstance(
+        concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES, BuildComparison)
+    ):
         raise ValueError(
             f"Concept {concept} lineage is not function or aggregate, is {type(concept.lineage)}"
         )

trilogy/core/processing/node_generators/multiselect_node.py

@@ -157,19 +157,19 @@ def gen_multiselect_node(
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for multiselect node; exiting early"
         )
         return node
     if not possible_joins:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for multiselect node; exiting early"
         )
         return node
     if all(
         [x.address in [y.address for y in node.output_concepts] for x in local_optional]
     ):
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base multiselect node; exiting early"
         )
         return node
     logger.info(

trilogy/core/statements/author.py

@@ -21,6 +21,7 @@ from trilogy.core.models.author import (
     Concept,
     ConceptRef,
     CustomType,
+    DeriveClause,
     Expr,
     FilterItem,
     Function,
@@ -352,11 +353,13 @@ class MultiSelectStatement(HasUUID, SelectTypeMixin, BaseModel):
     local_concepts: Annotated[
         EnvironmentConceptDict, PlainValidator(validate_concepts)
     ] = Field(default_factory=EnvironmentConceptDict)
+    derive: DeriveClause | None = None
 
     def as_lineage(self, environment: Environment):
         return MultiSelectLineage(
             selects=[x.as_lineage(environment) for x in self.selects],
             align=self.align,
+            derive=self.derive,
             namespace=self.namespace,
             # derived_concepts = self.derived_concepts,
             limit=self.limit,

trilogy/dialect/base.py

@@ -176,6 +176,20 @@ def struct_arg(args):
     return [f"{x[1]}: {x[0]}" for x in zip(args[::2], args[1::2])]
 
 
+def hash_from_args(val, hash_type):
+    hash_type = hash_type[1:-1]
+    if hash_type.lower() == "md5":
+        return f"md5({val})"
+    elif hash_type.lower() == "sha1":
+        return f"sha1({val})"
+    elif hash_type.lower() == "sha256":
+        return f"sha256({val})"
+    elif hash_type.lower() == "sha512":
+        return f"sha512({val})"
+    else:
+        raise ValueError(f"Unsupported hash type: {hash_type}")
+
+
 FUNCTION_MAP = {
     # generic types
     FunctionType.ALIAS: lambda x: f"{x[0]}",
@@ -260,6 +274,7 @@ FUNCTION_MAP = {
     FunctionType.REGEXP_REPLACE: lambda x: f"REGEXP_REPLACE({x[0]},{x[1]}, {x[2]})",
     FunctionType.TRIM: lambda x: f"TRIM({x[0]})",
     FunctionType.REPLACE: lambda x: f"REPLACE({x[0]},{x[1]},{x[2]})",
+    FunctionType.HASH: lambda x: hash_from_args(x[0], x[1]),
     # FunctionType.NOT_LIKE: lambda x: f" CASE WHEN {x[0]} like {x[1]} THEN 0 ELSE 1 END",
     # date types
     FunctionType.DATE_TRUNCATE: lambda x: f"date_trunc({x[0]},{x[1]})",
@@ -484,7 +499,20 @@ class BaseDialect:
             elif isinstance(c.lineage, BuildRowsetItem):
                 rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, BuildMultiSelectLineage):
-                rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
+                if c.address in c.lineage.calculated_derivations:
+                    assert c.lineage.derive is not None
+                    for x in c.lineage.derive.items:
+                        if x.address == c.address:
+                            rval = self.render_expr(
+                                x.expr,
+                                cte=cte,
+                                raise_invalid=raise_invalid,
+                            )
+                            break
+                else:
+                    rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
+            elif isinstance(c.lineage, BuildComparison):
+                rval = f"{self.render_expr(c.lineage.left, cte=cte, raise_invalid=raise_invalid)} {c.lineage.operator.value} {self.render_expr(c.lineage.right, cte=cte, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, AGGREGATE_ITEMS):
                 args = [
                     self.render_expr(v, cte)  # , alias=False)
@@ -816,7 +844,7 @@ class BaseDialect:
             if self.rendering.parameters:
                 if e.concept.namespace == DEFAULT_NAMESPACE:
                     return f":{e.concept.name}"
-                return f":{e.concept.address}"
+                return f":{e.concept.address.replace('.', '_')}"
             elif e.concept.lineage:
                 return self.render_expr(e.concept.lineage, cte=cte, cte_map=cte_map)
             return f"{self.QUOTE_CHARACTER}{e.concept.address}{self.QUOTE_CHARACTER}"

trilogy/executor.py

@@ -397,7 +397,7 @@ class Executor(object):
                 if v.safe_address == param or v.address == param
             ]
         if not matched:
-            raise SyntaxError(f"No concept found for parameter {param}")
+            raise SyntaxError(f"No concept found for parameter {param};")
 
         concept: Concept = matched.pop()
         return self._concept_to_value(concept, local_concepts=local_concepts)

trilogy/parsing/common.py

@@ -3,9 +3,7 @@ from typing import Iterable, List, Sequence, Tuple
 
 from lark.tree import Meta
 
-from trilogy.constants import (
-    VIRTUAL_CONCEPT_PREFIX,
-)
+from trilogy.constants import DEFAULT_NAMESPACE, VIRTUAL_CONCEPT_PREFIX
 from trilogy.core.constants import ALL_ROWS_CONCEPT
 from trilogy.core.enums import (
     ConceptSource,
@@ -64,10 +62,12 @@ ARBITRARY_INPUTS = (
     | Parenthetical
     | ListWrapper
     | MapWrapper
+    | Comparison
     | int
     | float
     | str
     | date
+    | bool
 )
 
 
@@ -399,6 +399,10 @@ def _get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
         return [x.reference for x in arg.by], True
     elif isinstance(arg, FunctionCallWrapper):
         return get_relevant_parent_concepts(arg.content)
+    elif isinstance(arg, Comparison):
+        left, lflag = get_relevant_parent_concepts(arg.left)
+        right, rflag = get_relevant_parent_concepts(arg.right)
+        return left + right, lflag or rflag
     return get_concept_arguments(arg), False
 
 
@@ -775,6 +779,27 @@ def align_item_to_concept(
     return new
 
 
+def derive_item_to_concept(
+    parent: ARBITRARY_INPUTS,
+    name: str,
+    lineage: MultiSelectLineage,
+    namespace: str | None = None,
+) -> Concept:
+    datatype = arg_to_datatype(parent)
+    grain = Grain()
+    new = Concept(
+        name=name,
+        datatype=datatype,
+        purpose=Purpose.PROPERTY,
+        lineage=lineage,
+        grain=grain,
+        namespace=namespace or DEFAULT_NAMESPACE,
+        granularity=Granularity.MULTI_ROW,
+        derivation=Derivation.MULTISELECT,
+    )
+    return new
+
+
 def rowset_concept(
     orig_address: ConceptRef,
     environment: Environment,
@@ -856,20 +881,7 @@ def rowset_to_concepts(rowset: RowsetDerivationStatement, environment: Environme
 
 
 def generate_concept_name(
-    parent: (
-        AggregateWrapper
-        | FunctionCallWrapper
-        | WindowItem
-        | FilterItem
-        | Function
-        | ListWrapper
-        | MapWrapper
-        | Parenthetical
-        | int
-        | float
-        | str
-        | date
-    ),
+    parent: ARBITRARY_INPUTS,
 ) -> str:
     """Generate a name for a concept based on its parent type and content."""
     if isinstance(parent, AggregateWrapper):
@@ -890,6 +902,8 @@ def generate_concept_name(
         return f"{VIRTUAL_CONCEPT_PREFIX}_paren_{string_to_hash(str(parent))}"
     elif isinstance(parent, FunctionCallWrapper):
         return f"{VIRTUAL_CONCEPT_PREFIX}_{parent.name}_{string_to_hash(str(parent))}"
+    elif isinstance(parent, Comparison):
+        return f"{VIRTUAL_CONCEPT_PREFIX}_comp_{string_to_hash(str(parent))}"
     else:  # ListWrapper, MapWrapper, or primitive types
         return f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
 
@@ -914,6 +928,82 @@ def parenthetical_to_concept(
     )
 
 
+def comparison_to_concept(
+    parent: Comparison,
+    name: str,
+    namespace: str,
+    environment: Environment,
+    metadata: Metadata | None = None,
+):
+    fmetadata = metadata or Metadata()
+
+    pkeys: List[Concept] = []
+    namespace = namespace or environment.namespace
+    is_metric = False
+    ref_args, is_metric = get_relevant_parent_concepts(parent)
+    concrete_args = [environment.concepts[c.address] for c in ref_args]
+    pkeys += [
+        x
+        for x in concrete_args
+        if not x.derivation == Derivation.CONSTANT
+        and not (x.derivation == Derivation.AGGREGATE and not x.grain.components)
+    ]
+    grain: Grain | None = Grain()
+    for x in pkeys:
+        grain += x.grain
+    if parent.operator in FunctionClass.ONE_TO_MANY.value:
+        # if the function will create more rows, we don't know what grain this is at
+        grain = None
+    modifiers = get_upstream_modifiers(pkeys, environment)
+    key_grain: list[str] = []
+    for x in pkeys:
+        # metrics will group to keys, so do not do key traversal
+        if is_metric:
+            key_grain.append(x.address)
+        # otherwse, for row ops, assume keys are transitive
+        elif x.keys:
+            key_grain += [*x.keys]
+        else:
+            key_grain.append(x.address)
+    keys = set(key_grain)
+    if is_metric:
+        purpose = Purpose.METRIC
+    elif not pkeys:
+        purpose = Purpose.CONSTANT
+    else:
+        purpose = Purpose.PROPERTY
+    fmetadata = metadata or Metadata()
+
+    if grain is not None:
+        r = Concept(
+            name=name,
+            datatype=parent.output_datatype,
+            purpose=purpose,
+            lineage=parent,
+            namespace=namespace,
+            keys=keys,
+            modifiers=modifiers,
+            grain=grain,
+            metadata=fmetadata,
+            derivation=Derivation.BASIC,
+            granularity=Granularity.MULTI_ROW,
+        )
+        return r
+
+    return Concept(
+        name=name,
+        datatype=parent.output_datatype,
+        purpose=purpose,
+        lineage=parent,
+        namespace=namespace,
+        keys=keys,
+        modifiers=modifiers,
+        metadata=fmetadata,
+        derivation=Derivation.BASIC,
+        granularity=Granularity.MULTI_ROW,
+    )
+
+
 def arbitrary_to_concept(
     parent: ARBITRARY_INPUTS,
     environment: Environment,
@@ -973,5 +1063,7 @@ def arbitrary_to_concept(
         return constant_to_concept(parent, name, namespace, metadata)
     elif isinstance(parent, Parenthetical):
         return parenthetical_to_concept(parent, name, namespace, environment, metadata)
+    elif isinstance(parent, Comparison):
+        return comparison_to_concept(parent, name, namespace, environment, metadata)
     else:
         return constant_to_concept(parent, name, namespace, metadata)

trilogy/parsing/parse_engine.py

@@ -62,6 +62,8 @@ from trilogy.core.models.author import (
     Conditional,
     CustomFunctionFactory,
     CustomType,
+    DeriveClause,
+    DeriveItem,
     Expr,
     FilterItem,
     Function,
@@ -69,6 +71,7 @@ from trilogy.core.models.author import (
     Grain,
     HavingClause,
     Metadata,
+    MultiSelectLineage,
     OrderBy,
     OrderItem,
     Parenthetical,
@@ -135,6 +138,7 @@ from trilogy.parsing.common import (
     align_item_to_concept,
     arbitrary_to_concept,
     constant_to_concept,
+    derive_item_to_concept,
     process_function_args,
     rowset_to_concepts,
 )
@@ -603,6 +607,9 @@ class ParseToObjects(Transformer):
     def PROPERTY(self, args):
         return Purpose.PROPERTY
 
+    def HASH_TYPE(self, args):
+        return args.value
+
     @v_args(meta=True)
     def prop_ident(self, meta: Meta, args) -> Tuple[List[Concept], str]:
         return [self.environment.concepts[grain] for grain in args[:-1]], args[-1]
@@ -707,7 +714,14 @@ class ParseToObjects(Transformer):
 
         if isinstance(
             source_value,
-            (FilterItem, WindowItem, AggregateWrapper, Function, FunctionCallWrapper),
+            (
+                FilterItem,
+                WindowItem,
+                AggregateWrapper,
+                Function,
+                FunctionCallWrapper,
+                Comparison,
+            ),
         ):
             concept = arbitrary_to_concept(
                 source_value,
@@ -1275,6 +1289,17 @@ class ParseToObjects(Transformer):
     def align_clause(self, meta: Meta, args) -> AlignClause:
         return AlignClause(items=args)
 
+    @v_args(meta=True)
+    def derive_item(self, meta: Meta, args) -> DeriveItem:
+        return DeriveItem(
+            expr=args[0], name=args[1], namespace=self.environment.namespace
+        )
+
+    @v_args(meta=True)
+    def derive_clause(self, meta: Meta, args) -> DeriveClause:
+
+        return DeriveClause(items=args)
+
     @v_args(meta=True)
     def multi_select_statement(self, meta: Meta, args) -> MultiSelectStatement:
 
@@ -1284,6 +1309,7 @@ class ParseToObjects(Transformer):
         order_by: OrderBy | None = None
         where: WhereClause | None = None
         having: HavingClause | None = None
+        derive: DeriveClause | None = None
         for arg in args:
             if isinstance(arg, SelectStatement):
                 selects.append(arg)
@@ -1297,11 +1323,24 @@ class ParseToObjects(Transformer):
                 having = arg
             elif isinstance(arg, AlignClause):
                 align = arg
+            elif isinstance(arg, DeriveClause):
+                derive = arg
 
         assert align
         assert align is not None
 
         derived_concepts = []
+        new_selects = [x.as_lineage(self.environment) for x in selects]
+        lineage = MultiSelectLineage(
+            selects=new_selects,
+            align=align,
+            derive=derive,
+            namespace=self.environment.namespace,
+            where_clause=where,
+            having_clause=having,
+            limit=limit,
+            hidden_components=set(y for x in new_selects for y in x.hidden_components),
+        )
         for x in align.items:
             concept = align_item_to_concept(
                 x,
@@ -1314,6 +1353,19 @@ class ParseToObjects(Transformer):
             )
             derived_concepts.append(concept)
             self.environment.add_concept(concept, meta=meta)
+        if derive:
+            for derived in derive.items:
+                derivation = derived.expr
+                name = derived.name
+                if not isinstance(derivation, (Function, Comparison, WindowItem)):
+                    raise SyntaxError(
+                        f"Invalid derive expression {derivation} in {meta.line}, must be a function or conditional"
+                    )
+                concept = derive_item_to_concept(
+                    derivation, name, lineage, self.environment.namespace
+                )
+                derived_concepts.append(concept)
+                self.environment.add_concept(concept, meta=meta)
         multi = MultiSelectStatement(
             selects=selects,
             align=align,
@@ -1323,6 +1375,7 @@ class ParseToObjects(Transformer):
             limit=limit,
             meta=Metadata(line_number=meta.line),
             derived_concepts=derived_concepts,
+            derive=derive,
         )
         return multi
 
@@ -1883,6 +1936,10 @@ class ParseToObjects(Transformer):
     def ftrim(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.TRIM, meta)
 
+    @v_args(meta=True)
+    def fhash(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.HASH, meta)
+
     @v_args(meta=True)
     def fsubstring(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.SUBSTRING, meta)

trilogy/parsing/trilogy.lark

@@ -74,12 +74,16 @@
     select_statement: where? "select"i select_list where?  having?  order_by?  limit? 
 
     // multiple_selects
-    multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause  where? order_by?  limit? 
+    multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause ("derive" derive_clause)?  where? order_by?  limit? 
 
     align_item: IDENTIFIER ":" IDENTIFIER ("," IDENTIFIER)*  ","?
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
+    derive_item: expr "->" IDENTIFIER
+
+    derive_clause: derive_item ("," derive_item)* ","?
+
     merge_statement: "merge"i  WILDCARD_IDENTIFIER "into"i SHORTHAND_MODIFIER? WILDCARD_IDENTIFIER
 
     // raw sql statement
@@ -308,8 +312,11 @@
     fregexp_contains: _REGEXP_CONTAINS expr "," expr  ")"
     _REGEXP_REPLACE.1: "regexp_replace("
     fregexp_replace: _REGEXP_REPLACE expr "," expr "," expr   ")"
+    _HASH.1: "hash("
+    HASH_TYPE: "md5"i | "sha1"i | "sha256"i | "sha512"i
+    fhash: _HASH expr "," HASH_TYPE ")"
 
-    _string_functions: like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains | ftrim | freplace | fregexp_extract | fregexp_contains | fregexp_replace
+    _string_functions: like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains | ftrim | freplace | fregexp_extract | fregexp_contains | fregexp_replace | fhash
 
     //array_functions
     _ARRAY_SUM.1: "array_sum("i