pytrilogy 0.0.1.118__py3-none-any.whl → 0.0.2.2__py3-none-any.whl

trilogy/parsing/parse_engine.py

@@ -32,6 +32,7 @@ from trilogy.core.enums import (
     WindowType,
     DatePart,
     ShowCategory,
+    SelectFiltering,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
@@ -68,7 +69,7 @@ from trilogy.core.models import (
     ConceptTransform,
     Conditional,
     Datasource,
-    MergeStatement,
+    MergeStatementV2,
     Environment,
     FilterItem,
     Function,
@@ -400,7 +401,7 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def concept_property_declaration(self, meta: Meta, args) -> Concept:
 
-        metadata = None
+        metadata = Metadata()
         modifiers = []
         for arg in args:
             if isinstance(arg, Metadata):
@@ -439,7 +440,7 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def concept_declaration(self, meta: Meta, args) -> ConceptDeclarationStatement:
-        metadata = None
+        metadata = Metadata()
         modifiers = []
         for arg in args:
             if isinstance(arg, Metadata):
@@ -447,9 +448,7 @@ class ParseToObjects(Transformer):
             if isinstance(arg, Modifier):
                 modifiers.append(arg)
         name = args[1]
-        lookup, namespace, name, parent = parse_concept_reference(
-            name, self.environment
-        )
+        _, namespace, name, _ = parse_concept_reference(name, self.environment)
         concept = Concept(
             name=name,
             datatype=args[2],
@@ -606,6 +605,7 @@ class ParseToObjects(Transformer):
         columns: List[ColumnAssignment] = args[1]
         grain: Optional[Grain] = None
         address: Optional[Address] = None
+        where: Optional[WhereClause] = None
         for val in args[1:]:
             if isinstance(val, Address):
                 address = val
@@ -613,6 +613,8 @@ class ParseToObjects(Transformer):
                 grain = val
             elif isinstance(val, Query):
                 address = Address(location=f"({val.text})", is_query=True)
+            elif isinstance(val, WhereClause):
+                where = val
         if not address:
             raise ValueError(
                 "Malformed datasource, missing address or query declaration"
@@ -625,6 +627,7 @@ class ParseToObjects(Transformer):
             grain=grain,  # type: ignore
             address=address,
             namespace=self.environment.namespace,
+            where=where,
         )
         for column in columns:
             column.concept = column.concept.with_grain(datasource.grain)
@@ -754,19 +757,21 @@ class ParseToObjects(Transformer):
         return [x for x in args]
 
     @v_args(meta=True)
-    def merge_statement(self, meta: Meta, args) -> MergeStatement:
-        parsed = [self.environment.concepts[x] for x in args]
-        datatypes = {x.datatype for x in parsed}
-        if not len(datatypes) == 1 and self.environment.concepts.fail_on_missing:
-            type_dict = {x.address: x.datatype for x in parsed}
-            raise SyntaxError(
-                f"Cannot merge concepts with different datatype"
-                f"line: {meta.line} concepts: {type_dict}"
-            )
-        merge = MergeStatement(concepts=parsed, datatype=datatypes.pop())
-        new = merge.merge_concept
-        self.environment.add_concept(new, meta=meta)
-        return merge
+    def merge_statement_v2(self, meta: Meta, args) -> MergeStatementV2:
+        modifiers = []
+        concepts = []
+        for arg in args:
+            if isinstance(arg, Modifier):
+                modifiers.append(arg)
+            else:
+                concepts.append(self.environment.concepts[arg])
+        new = MergeStatementV2(
+            source=concepts[0], target=concepts[1], modifiers=modifiers
+        )
+
+        self.environment.merge_concept(new.source, new.target, modifiers)
+
+        return new
 
     @v_args(meta=True)
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
@@ -892,7 +897,7 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def select_statement(self, meta: Meta, args) -> SelectStatement:
-        select_items = None
+        select_items: List[SelectItem] | None = None
         limit = None
         order_by = None
         where = None
@@ -919,16 +924,24 @@ class ParseToObjects(Transformer):
             # so rebuild at this point in the tree
             # TODO: simplify
             if isinstance(item.content, ConceptTransform):
-                new_concept = item.content.output.with_select_grain(output.grain)
+                new_concept = item.content.output.with_select_context(
+                    output.grain,
+                    conditional=(
+                        output.where_clause.conditional
+                        if output.where_clause
+                        and output.where_clause_category == SelectFiltering.IMPLICIT
+                        else None
+                    ),
+                )
                 self.environment.add_concept(new_concept, meta=meta)
                 item.content.output = new_concept
         if order_by:
-            for item in order_by.items:
+            for orderitem in order_by.items:
                 if (
-                    isinstance(item.expr, Concept)
-                    and item.expr.purpose == Purpose.METRIC
+                    isinstance(orderitem.expr, Concept)
+                    and orderitem.expr.purpose == Purpose.METRIC
                 ):
-                    item.expr = item.expr.with_grain(output.grain)
+                    orderitem.expr = orderitem.expr.with_grain(output.grain)
         return output
 
     @v_args(meta=True)
@@ -1045,8 +1058,20 @@ class ParseToObjects(Transformer):
     def parenthetical(self, args):
         return Parenthetical(content=args[0])
 
+    def condition_parenthetical(self, args):
+        return Parenthetical(content=args[0])
+
     def conditional(self, args):
-        return Conditional(left=args[0], right=args[2], operator=args[1])
+        def munch_args(args):
+            while args:
+                if len(args) == 1:
+                    return args[0]
+                else:
+                    return Conditional(
+                        left=args[0], operator=args[1], right=munch_args(args[2:])
+                    )
+
+        return munch_args(args)
 
     def window_order(self, args):
         return WindowOrder(args[0])
@@ -1723,7 +1748,7 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
         | None
     ],
 ]:
-    environment = environment or Environment(datasources={})
+    environment = environment or Environment()
     parser = ParseToObjects(visit_tokens=True, text=text, environment=environment)
 
     try:

trilogy/parsing/render.py

@@ -33,13 +33,13 @@ from trilogy.core.models import (
     PersistStatement,
     ListWrapper,
     RowsetDerivationStatement,
-    MergeStatement,
     MultiSelectStatement,
     OrderBy,
     AlignClause,
     AlignItem,
     RawSQLStatement,
     NumericType,
+    MergeStatementV2,
 )
 from trilogy.core.enums import Modifier
 
@@ -131,22 +131,21 @@ class Renderer:
     @to_string.register
     def _(self, arg: Datasource):
         assignments = ",\n\t".join([self.to_string(x) for x in arg.columns])
-        return f"""datasource {arg.name} (
+        base = f"""datasource {arg.name} (
     {assignments}
     ) 
 {self.to_string(arg.grain)} 
-{self.to_string(arg.address)};"""
+{self.to_string(arg.address)}"""
+        if arg.where:
+            base += f"\n{self.to_string(arg.where)}"
+        base += ";"
+        return base
 
     @to_string.register
     def _(self, arg: "Grain"):
         components = ",".join(self.to_string(x) for x in arg.components)
         return f"grain ({components})"
 
-    @to_string.register
-    def _(self, arg: MergeStatement):
-        components = ", ".join(self.to_string(x) for x in arg.concepts)
-        return f"merge {components};"
-
     @to_string.register
     def _(self, arg: "Query"):
         return f"""query {arg.text}"""
@@ -201,7 +200,7 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "ColumnAssignment"):
-        return f"{arg.alias}:{self.to_string(arg.concept)}"
+        return f"{arg.alias}: {self.to_string(arg.concept)}"
 
     @to_string.register
     def _(self, arg: "ConceptDeclarationStatement"):
@@ -341,6 +340,18 @@ class Renderer:
             return f"{self.to_string(arg.function)} by {self.to_string(arg.by)}"
         return f"{self.to_string(arg.function)}"
 
+    @to_string.register
+    def _(self, arg: MergeStatementV2):
+        return f"MERGE {self.to_string(arg.source)} into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{self.to_string(arg.target)};"
+
+    @to_string.register
+    def _(self, arg: Modifier):
+        if arg == Modifier.PARTIAL:
+            return "~"
+        if arg == Modifier.HIDDEN:
+            return "--"
+        return arg.value
+
     @to_string.register
     def _(self, arg: int):
         return f"{arg}"

trilogy/parsing/trilogy.lark

@@ -8,7 +8,8 @@
     | persist_statement
     | rowset_derivation_statement
     | import_statement
-    | merge_statement
+
+    | merge_statement_v2
     | rawsql_statement
     
     _TERMINATOR:  ";"i /\s*/
@@ -34,7 +35,7 @@
     prop_ident: "<" IDENTIFIER ("," IDENTIFIER )* ","? ">" "." IDENTIFIER
 
     // datasource concepts
-    datasource: "datasource" IDENTIFIER  "("  column_assignment_list ")"  grain_clause? (address | query)
+    datasource: "datasource" IDENTIFIER  "("  column_assignment_list ")"  grain_clause? (address | query) where?
     
     grain_clause: "grain" "(" column_list ")"
     
@@ -69,8 +70,8 @@
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
-    // merge statemment
-    merge_statement: "merge" IDENTIFIER ("," IDENTIFIER)* ","? 
+    // merge_v2
+    merge_statement_v2: "merge"i  IDENTIFIER "into"i SHORTHAND_MODIFIER? IDENTIFIER 
 
     // raw sql statement
     rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
@@ -125,9 +126,13 @@
     
     !logical_operator: "and"i | "or"i
     
-    conditional: expr logical_operator expr
+    condition_parenthetical: "(" conditional ")"
+
+    _condition_unit: (expr | condition_parenthetical)
+
+    conditional: _condition_unit (logical_operator _condition_unit)*
     
-    where: "WHERE"i expr+
+    where: "WHERE"i conditional
     
     !array_comparison: ( ("NOT"i "IN"i) | "IN"i)
 
@@ -150,7 +155,7 @@
     index_access: expr "[" int_lit "]"
     attr_access: expr  "[" _string_lit "]"
 
-    expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | _static_functions |  literal |  concept_lit  | index_access | attr_access | parenthetical | expr_tuple | comparison | conditional | alt_like 
+    expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | _static_functions |  literal |  concept_lit  | index_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like 
     
     // functions
     
@@ -168,7 +173,7 @@
     fcast: "cast"i "(" expr "as"i data_type ")"
     concat: ("concat"i "(" (expr ",")* expr ")") | (expr "||" expr)
     fcoalesce: "coalesce"i "(" (expr ",")* expr ")"
-    fcase_when: "WHEN"i (expr | conditional) "THEN"i expr
+    fcase_when: "WHEN"i conditional "THEN"i expr
     fcase_else: "ELSE"i expr
     fcase: "CASE"i (fcase_when)* (fcase_else)? "END"i
     len: "len"i "(" expr ")"

pytrilogy-0.0.1.118.dist-info/RECORD

@@ -1,83 +0,0 @@
-trilogy/__init__.py,sha256=fYN8J2EkHXTX5cKVWpbBMQHilscbkhmZF-pMLoMqHFc,292
-trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=u2dNxhwy0v-6HrvG1GcpDVvuhzdTH5fuyYNCxDPlr2E,770
-trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
-trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
-trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
-trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=9IzCv0Hzzr3OTJQWzg9yHV_b46Jvk47RNoZjdBi2Its,5678
-trilogy/core/env_processor.py,sha256=SU-jpaGfoWLe9sGTeQYG1qjVnwGQ7TwctmnJRlfzluc,1459
-trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
-trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
-trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=fWuuPOeWRAiukTkXvTEu_NCSbu6eUmogtLGpC4ey2FI,9465
-trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
-trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=Mp6MB8Gl2CZcPDWkofPX8Mym1ZvT0OUS0dKAUWCj7K8,116419
-trilogy/core/optimization.py,sha256=oM3Ry7UpbpTSm2xNkmWx70OHd2V2vWRjM72sZpsZfb8,4116
-trilogy/core/query_processor.py,sha256=clIRJ6IcsqIVBPKFsxt8bqCLsLyajvAu02MUIcKQhTo,15713
-trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
-trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
-trilogy/core/optimizations/inline_constant.py,sha256=neZOFjX7M2pzQ-8m-f8nApy_MfJuowX6SzcGwGFt5w4,927
-trilogy/core/optimizations/inline_datasource.py,sha256=BSp54fwF4RRwInd-09pggemC7JuXj-uqGzi32ufeqYo,2171
-trilogy/core/optimizations/predicate_pushdown.py,sha256=sIojWvoYp6k_ANCyVqxCpEyLY_GLmzsG-Sghj0cbk3k,4135
-trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=MYrpNMidqvPOg123RekOcqVTjcj03i_538gBo0MzoWE,23432
-trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
-trilogy/core/processing/utility.py,sha256=acxH5448-j8JXqxMRibyAxjz1Wqu7QudbR0PfMuucww,9902
-trilogy/core/processing/node_generators/__init__.py,sha256=LIs6uBEum8LDc-26zjyAwjxa-ay2ok9tKtPjDNvbVkE,757
-trilogy/core/processing/node_generators/basic_node.py,sha256=HJnIhZLgkUdorKYcofe-QnKSM3Lf_3QO91cbSJhsqf4,2242
-trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
-trilogy/core/processing/node_generators/concept_merge_node.py,sha256=x4M8VVZZmBcqHDY1uq7M9KGKCBwjU6mcE_x2BOEk2Mg,7328
-trilogy/core/processing/node_generators/filter_node.py,sha256=y_tqYe2So18vWHASMwVPLzDO-PnyQCO-MAlI4B-rY3Y,4526
-trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
-trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
-trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=wNDHAbRrKSjsns-EROM_G12mRyOMjbcWpYav2uefXOE,6045
-trilogy/core/processing/node_generators/rowset_node.py,sha256=eNG6rfLifUKraoRGxE8pesQMy5cKT6R5XNIaa3Wuiwk,6081
-trilogy/core/processing/node_generators/select_node.py,sha256=Qb00Kizsv-877UMkGfusl5jXKXMZtZTtLks5pxU07SU,20698
-trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
-trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOqhOZPNzCUxw-lkxt0R7HORI6ss,2582
-trilogy/core/processing/nodes/__init__.py,sha256=baODkJfvUoWEEbu843GEd7snubwLeOG5FQ8l-CwIaC8,3928
-trilogy/core/processing/nodes/base_node.py,sha256=yhjmsAUmhHDqgbQjz_9YdfP-M5pj4xbrPRDF6Y4XVuw,10498
-trilogy/core/processing/nodes/filter_node.py,sha256=rDw4vfE6tqWxuKT0arihVmIOoOWDDCyzRA-2yONX_Ek,1860
-trilogy/core/processing/nodes/group_node.py,sha256=vzeU9J4xMhRrPj4-KPJTgNbH-KFu2ZS8b57SOynsdw0,4448
-trilogy/core/processing/nodes/merge_node.py,sha256=FvSiTWKOzaUsXBkf6wJD8QQqQxp_aphS_I5VzNRw8Yo,13600
-trilogy/core/processing/nodes/select_node_v2.py,sha256=ERCflBFzKpD5SzweMevnJLyQnxmF_-IQ6VRu5yVeiBg,6552
-trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
-trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
-trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=ycsxbUL68DsodOsHEgEoNLKMn5vgRN3sDIRfiH9fQDs,31719
-trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
-trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
-trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=Ddyt68sr8IL2HnZMenyytoD65FXwY_O2pz1McyS0bis,3075
-trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
-trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=UxBodRiV3szpFcQlcjoJaGXEwAhZJf_OT7dHczYvO80,3092
-trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
-trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
-trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
-trilogy/hooks/graph_hook.py,sha256=i-Tv9sxZU0sMc-God8bLLz-nAg4-wYafogZtHaU8LXw,801
-trilogy/hooks/query_debugger.py,sha256=D2VJUcyvQrVJ8sT6FCMKR3NKTfVrgZQ7gly91avHHpw,4222
-trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=iR3fiiZ7w8VJuUGrQ0v06XGDXov81f4z1ZlFnj6y40E,5804
-trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
-trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
-trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=F1ok96qT6EhKRKV1Q_YzfHxMFtNV8qAXopK8NaePgU4,57080
-trilogy/parsing/render.py,sha256=TnLf5fg4wimpd9EvhLU-FMDwpyW9pesoedBZ0RrmWD4,11810
-trilogy/parsing/trilogy.lark,sha256=1AZbQGpNmpm4KamAXA5IWcuOr2B8Gb8kUJcAOmKf_zY,10862
-trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.118.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.118.dist-info/METADATA,sha256=tZ1kQcOTJ5JTYb5tqlfdo7g-o94TC0tKmcdqLtJ35bI,7878
-pytrilogy-0.0.1.118.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-pytrilogy-0.0.1.118.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.118.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.118.dist-info/RECORD,,

trilogy/core/processing/node_generators/concept_merge_node.py

@@ -1,214 +0,0 @@
-from trilogy.core.models import (
-    Concept,
-    Environment,
-    MergeStatement,
-)
-from trilogy.core.processing.nodes import MergeNode, NodeJoin, History
-from trilogy.core.processing.nodes.base_node import concept_list_to_grain, StrategyNode
-from typing import List
-
-from trilogy.core.enums import JoinType, PurposeLineage
-from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
-from itertools import combinations
-from trilogy.core.processing.node_generators.common import resolve_join_order
-from trilogy.utility import unique
-
-
-LOGGER_PREFIX = "[GEN_CONCEPT_MERGE_NODE]"
-
-
-def merge_joins(
-    parents: List[StrategyNode], merge_concepts: List[Concept]
-) -> List[NodeJoin]:
-    output = []
-    for left, right in combinations(parents, 2):
-        output.append(
-            NodeJoin(
-                left_node=left,
-                right_node=right,
-                concepts=merge_concepts,
-                join_type=JoinType.FULL,
-                filter_to_mutual=True,
-            )
-        )
-    return resolve_join_order(output)
-
-
-def gen_concept_merge_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
-    g,
-    depth: int,
-    source_concepts,
-    history: History | None = None,
-) -> MergeNode | None:
-    if not isinstance(concept.lineage, MergeStatement):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge node for {concept}"
-        )
-        return None
-    lineage: MergeStatement = concept.lineage
-
-    base_parents: List[StrategyNode] = []
-
-    # get additional concepts that should be merged across the environments
-    additional_merge: List[Concept] = [*lineage.concepts]
-    target_namespaces = set(x.namespace for x in [concept] + local_optional)
-    for x in local_optional:
-        if x.address in environment.merged_concepts:
-            ms = environment.merged_concepts[x.address].lineage
-            assert isinstance(ms, MergeStatement)
-            additional_merge += ms.concepts
-
-    for select in lineage.concepts:
-        # if it's a merge concept, filter it out of the optional
-        if select.namespace not in target_namespaces:
-            continue
-        sub_optional = [
-            x
-            for x in local_optional
-            if x.address not in environment.merged_concepts
-            and x.namespace == select.namespace
-        ]
-
-        sub_additional_merge = [
-            x for x in additional_merge if x.namespace == select.namespace
-        ]
-        sub_optional += sub_additional_merge
-        final: List[Concept] = unique([select] + sub_optional, "address")
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} generating concept merge parent node with {[x.address for x in final]}"
-        )
-        snode: StrategyNode = source_concepts(
-            mandatory_list=final,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-        )
-
-        if not snode:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge node for {concept}"
-            )
-            return None
-        for x in sub_additional_merge:
-            snode.add_output_concept(environment.merged_concepts[x.address])
-        base_parents.append(snode)
-
-    node_joins = merge_joins(
-        base_parents,
-        unique(
-            [environment.merged_concepts[x.address] for x in additional_merge],
-            "address",
-        ),
-    )
-
-    enrichment = set([x.address for x in local_optional])
-    outputs = [
-        x
-        for y in base_parents
-        for x in y.output_concepts
-        if x.derivation != PurposeLineage.MERGE
-    ]
-
-    additional_relevant = [x for x in outputs if x.address in enrichment]
-    final_outputs = outputs + additional_relevant + [concept]
-    virtual_outputs = [x for x in final_outputs if x.derivation == PurposeLineage.MERGE]
-    node = MergeNode(
-        input_concepts=[x for y in base_parents for x in y.output_concepts],
-        output_concepts=[
-            x for x in final_outputs if x.derivation != PurposeLineage.MERGE
-        ],
-        environment=environment,
-        g=g,
-        depth=depth,
-        parents=base_parents,
-        node_joins=node_joins,
-        virtual_output_concepts=virtual_outputs,
-    )
-
-    qds = node.rebuild_cache()
-
-    # assume grain to be outoput of select
-    # but don't include anything aggregate at this point
-    qds.grain = concept_list_to_grain(
-        node.output_concepts, parent_sources=qds.datasources
-    )
-    possible_joins = concept_to_relevant_joins(additional_relevant)
-    if not local_optional:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for merge concept node; exiting early"
-        )
-        return node
-    if not possible_joins:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for merge concept node; exiting early"
-        )
-        return node
-    if all(
-        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
-    ):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base merge concept node; exiting early"
-        )
-        return node
-    missing = [
-        x
-        for x in local_optional
-        if x.address not in [y.address for y in node.output_concepts]
-    ]
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} generating merge concept enrichment node for missing {[x.address for x in missing]}"
-    )
-    enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
-        # to then connect to the rest of the query
-        mandatory_list=additional_relevant + missing,
-        environment=environment,
-        g=g,
-        depth=depth + 1,
-        history=history,
-    )
-    if not enrich_node:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge concept enrichment node for {concept.address} with optional {[x.address for x in local_optional]}, returning just merge concept"
-        )
-        return node
-
-    # we still need the hidden concepts to be returned to the search
-    # since they must be on the final node
-    # to avoid further recursion
-    # TODO: let the downstream search know they were found
-    return MergeNode(
-        input_concepts=enrich_node.output_concepts + node.output_concepts,
-        # also filter out the
-        output_concepts=[
-            x
-            for x in node.output_concepts + local_optional
-            if x.derivation != PurposeLineage.MERGE
-        ],
-        hidden_concepts=[],
-        environment=environment,
-        g=g,
-        depth=depth,
-        parents=[
-            # this node gets the window
-            node,
-            # this node gets enrichment
-            enrich_node,
-        ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=node,
-                concepts=possible_joins,
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
-        partial_concepts=node.partial_concepts,
-        virtual_output_concepts=virtual_outputs,
-    )