pytrilogy 0.0.1.103__py3-none-any.whl → 0.0.1.105__py3-none-any.whl

trilogy/parsing/parse_engine.py

@@ -9,6 +9,7 @@ from lark.exceptions import (
     UnexpectedToken,
     VisitError,
 )
+from pathlib import Path
 from lark.tree import Meta
 from pydantic import ValidationError
 from trilogy.core.internal import INTERNAL_NAMESPACE, ALL_ROWS_CONCEPT
@@ -16,6 +17,7 @@ from trilogy.constants import (
     DEFAULT_NAMESPACE,
     NULL_VALUE,
     VIRTUAL_CONCEPT_PREFIX,
+    MagicConstants,
 )
 from trilogy.core.enums import (
     BooleanOperator,
@@ -61,6 +63,7 @@ from trilogy.core.models import (
     ColumnAssignment,
     Comment,
     Comparison,
+    SubselectComparison,
     Concept,
     ConceptTransform,
     Conditional,
@@ -107,6 +110,7 @@ from trilogy.parsing.common import (
     function_to_concept,
     filter_item_to_concept,
     constant_to_concept,
+    arbitrary_to_concept,
 )
 
 CONSTANT_TYPES = (int, float, str, bool, ListWrapper)
@@ -186,9 +190,9 @@ grammar = r"""
 
     // FUNCTION blocks
     function: raw_function
-    function_binding_item: IDENTIFIER data_type
+    function_binding_item: IDENTIFIER ":" data_type
     function_binding_list: (function_binding_item ",")* function_binding_item ","?
-    raw_function: "def" "rawsql"  IDENTIFIER  "("  function_binding_list ")" "-" ">" data_type "as"i MULTILINE_STRING
+    raw_function: "bind" "sql"  IDENTIFIER  "("  function_binding_list ")" "-" ">" data_type "as"i MULTILINE_STRING
 
     
     // user_id where state = Mexico
@@ -245,7 +249,9 @@ grammar = r"""
 
     COMPARISON_OPERATOR: (/is[\s]+not/ | "is" |"=" | ">" | "<" | ">=" | "<=" | "!="  )
     
-    comparison: (expr COMPARISON_OPERATOR expr) | (expr array_comparison expr_tuple) 
+    comparison: (expr COMPARISON_OPERATOR expr) | (expr array_comparison expr_tuple)
+
+    subselect_comparison: expr array_comparison expr
     
     expr_tuple: "("  (expr ",")* expr ","?  ")"
 
@@ -257,7 +263,7 @@ grammar = r"""
 
     parenthetical: "(" (conditional | expr) ")"
     
-    expr:  window_item | filter_item | comparison | fgroup |  aggregate_functions | unnest | _string_functions | _math_functions | _generic_functions | _constant_functions| _date_functions |  literal |  expr_reference  | index_access | attr_access |  parenthetical
+    expr:  window_item | filter_item | comparison | subselect_comparison | fgroup |  aggregate_functions | unnest | _string_functions | _math_functions | _generic_functions | _constant_functions| _date_functions |  literal |  expr_reference  | index_access | attr_access |  parenthetical
     
     // functions
     
@@ -466,25 +472,34 @@ class ParseToObjects(Transformer):
         text,
         environment: Environment,
         parse_address: str | None = None,
-        parsed: dict | None = None,
+        parsed: dict[str, "ParseToObjects"] | None = None,
     ):
         Transformer.__init__(self, visit_tokens)
         self.text = text
         self.environment: Environment = environment
-        self.imported: set[str] = set()
         self.parse_address = parse_address or "root"
         self.parsed: dict[str, ParseToObjects] = parsed if parsed else {}
         # we do a second pass to pick up circular dependencies
         # after initial parsing
         self.pass_count = 1
+        self._results_stash = None
+
+    def transform(self, tree):
+        results = super().transform(tree)
+        self._results_stash = results
+        self.environment._parse_count += 1
+        return results
 
     def hydrate_missing(self):
         self.pass_count = 2
         for k, v in self.parsed.items():
+
             if v.pass_count == 2:
                 continue
             v.hydrate_missing()
         self.environment.concepts.fail_on_missing = True
+        # if not self.environment.concepts.undefined:
+        #     return self._results_stash
         reparsed = self.transform(PARSER.parse(self.text))
         self.environment.concepts.undefined = {}
         return reparsed
@@ -515,43 +530,11 @@ class ParseToObjects(Transformer):
                     concept.metadata.line_number = meta.line
                 self.environment.add_concept(concept, meta=meta)
                 final.append(concept)
-            elif isinstance(arg, FilterItem):
+            elif isinstance(
+                arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper)
+            ):
                 id_hash = string_to_hash(str(arg))
-                concept = filter_item_to_concept(
-                    arg,
-                    name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
-                    namespace=self.environment.namespace,
-                )
-                if concept.metadata:
-                    concept.metadata.line_number = meta.line
-                self.environment.add_concept(concept, meta=meta)
-                final.append(concept)
-            elif isinstance(arg, WindowItem):
-                id_hash = string_to_hash(str(arg))
-                concept = window_item_to_concept(
-                    arg,
-                    namespace=self.environment.namespace,
-                    name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
-                )
-                if concept.metadata:
-                    concept.metadata.line_number = meta.line
-                self.environment.add_concept(concept, meta=meta)
-                final.append(concept)
-            elif isinstance(arg, AggregateWrapper):
-                id_hash = string_to_hash(str(arg))
-                concept = agg_wrapper_to_concept(
-                    arg,
-                    namespace=self.environment.namespace,
-                    name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
-                )
-                if concept.metadata:
-                    concept.metadata.line_number = meta.line
-                self.environment.add_concept(concept, meta=meta)
-                final.append(concept)
-            # we don't need virtual types for most constants
-            elif isinstance(arg, (ListWrapper)):
-                id_hash = string_to_hash(str(arg))
-                concept = constant_to_concept(
+                concept = arbitrary_to_concept(
                     arg,
                     name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
                     namespace=self.environment.namespace,
@@ -560,6 +543,7 @@ class ParseToObjects(Transformer):
                     concept.metadata.line_number = meta.line
                 self.environment.add_concept(concept, meta=meta)
                 final.append(concept)
+
             else:
                 final.append(arg)
         return final
@@ -763,8 +747,10 @@ class ParseToObjects(Transformer):
         while isinstance(source_value, Parenthetical):
             source_value = source_value.content
 
-        if isinstance(source_value, FilterItem):
-            concept = filter_item_to_concept(
+        if isinstance(
+            source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
+        ):
+            concept = arbitrary_to_concept(
                 source_value,
                 name=name,
                 namespace=namespace,
@@ -772,31 +758,6 @@ class ParseToObjects(Transformer):
                 metadata=metadata,
             )
 
-            if concept.metadata:
-                concept.metadata.line_number = meta.line
-            self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
-        elif isinstance(source_value, WindowItem):
-
-            concept = window_item_to_concept(
-                source_value,
-                name=name,
-                namespace=namespace,
-                purpose=purpose,
-                metadata=metadata,
-            )
-            if concept.metadata:
-                concept.metadata.line_number = meta.line
-            self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
-        elif isinstance(source_value, AggregateWrapper):
-            concept = agg_wrapper_to_concept(
-                source_value,
-                namespace=namespace,
-                name=name,
-                metadata=metadata,
-                purpose=purpose,
-            )
             if concept.metadata:
                 concept.metadata.line_number = meta.line
             self.environment.add_concept(concept, meta=meta)
@@ -814,19 +775,6 @@ class ParseToObjects(Transformer):
             self.environment.add_concept(concept, meta=meta)
             return ConceptDerivation(concept=concept)
 
-        elif isinstance(source_value, Function):
-            function: Function = source_value
-
-            concept = function_to_concept(
-                function,
-                name=name,
-                namespace=namespace,
-            )
-            if concept.metadata:
-                concept.metadata.line_number = meta.line
-            self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
-
         raise SyntaxError(
             f"Received invalid type {type(args[2])} {args[2]} as input to select"
             " transform"
@@ -932,7 +880,7 @@ class ParseToObjects(Transformer):
         )
         for column in columns:
             column.concept = column.concept.with_grain(datasource.grain)
-        self.environment.datasources[datasource.identifier] = datasource
+        self.environment.add_datasource(datasource, meta=meta)
         return datasource
 
     @v_args(meta=True)
@@ -1046,12 +994,11 @@ class ParseToObjects(Transformer):
         self.environment.add_concept(new, meta=meta)
         return merge
 
-    def import_statement(self, args: list[str]):
+    def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
 
         target = join(self.environment.working_path, *path) + ".preql"
-        self.imported.add(target)
         if target in self.parsed:
             nparser = self.parsed[target]
         else:
@@ -1070,21 +1017,23 @@ class ParseToObjects(Transformer):
                 )
                 nparser.transform(PARSER.parse(text))
                 self.parsed[target] = nparser
+                # add the parsed objects of the import in
+                self.parsed = {**self.parsed, **nparser.parsed}
             except Exception as e:
                 raise ImportError(
                     f"Unable to import file {dirname(target)}, parsing error: {e}"
                 )
 
-        for key, concept in nparser.environment.concepts.items():
-            # self.environment.concepts[f"{alias}.{key}"] = concept.with_namespace(new_namespace)
+        for _, concept in nparser.environment.concepts.items():
             self.environment.add_concept(concept.with_namespace(alias))
 
-        for key, datasource in nparser.environment.datasources.items():
+        for _, datasource in nparser.environment.datasources.items():
             self.environment.add_datasource(datasource.with_namespace(alias))
-            # self.environment.datasources[f"{alias}.{key}"] = datasource.with_namespace(new_namespace)
-
-        self.environment.imports[alias] = ImportStatement(alias=alias, path=args[0])
-        return None
+        imps = ImportStatement(
+            alias=alias, path=Path(args[0]), environment=nparser.environment
+        )
+        self.environment.imports[alias] = imps
+        return imps
 
     @v_args(meta=True)
     def show_category(self, meta: Meta, args) -> ShowCategory:
@@ -1208,7 +1157,14 @@ class ParseToObjects(Transformer):
     def where(self, args):
         root = args[0]
         if not isinstance(root, (Comparison, Conditional, Parenthetical)):
-            root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
+            if arg_to_datatype(root) == DataType.BOOL:
+                root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
+            else:
+                root = Comparison(
+                    left=root,
+                    right=MagicConstants.NULL,
+                    operator=ComparisonOperator.IS_NOT,
+                )
         return WhereClause(conditional=root)
 
     @v_args(meta=True)
@@ -1221,7 +1177,6 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def raw_function(self, meta: Meta, args) -> Function:
-        print(args)
         identity = args[0]
         fargs = args[1]
         output = args[2]
@@ -1262,6 +1217,22 @@ class ParseToObjects(Transformer):
     def comparison(self, args) -> Comparison:
         return Comparison(left=args[0], right=args[2], operator=args[1])
 
+    @v_args(meta=True)
+    def subselect_comparison(self, meta: Meta, args) -> SubselectComparison:
+        right = args[2]
+        if not isinstance(right, Concept):
+            right = arbitrary_to_concept(
+                right,
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(right))}",
+            )
+            self.environment.add_concept(right)
+        return SubselectComparison(
+            left=args[0],
+            right=right,
+            operator=args[1],
+        )
+
     def expr_tuple(self, args):
         return Parenthetical(content=args)
 

trilogy/parsing/render.py

@@ -298,128 +298,6 @@ class Renderer:
     def _(self, arg: "FilterItem"):
         return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
 
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
-    @to_string.register
-    def _(self, arg: "ImportStatement"):
-        return f"import {arg.path} as {arg.alias};"
-
-    @to_string.register
-    def _(self, arg: "WindowItem"):
-        over = ",".join(self.to_string(c) for c in arg.over)
-        order = ",".join(self.to_string(c) for c in arg.order_by)
-        if over:
-            return (
-                f"{arg.type.value} {self.to_string(arg.content)} by {order} OVER {over}"
-            )
-        return f"{arg.type.value} {self.to_string(arg.content)} by {order}"
-
-    @to_string.register
-    def _(self, arg: "FilterItem"):
-        return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
-
     @to_string.register
     def _(self, arg: "ImportStatement"):
         return f"import {arg.path} as {arg.alias};"