relationalai 0.13.0.dev0__py3-none-any.whl → 0.13.2__py3-none-any.whl

relationalai/rel_emitter.py

@@ -0,0 +1,373 @@
+from __future__ import annotations
+import json
+from typing import Any, List, Set, cast
+from datetime import datetime, date
+import pandas as pd
+from zoneinfo import ZoneInfo
+
+from relationalai.errors import RAIException
+from relationalai.rel_utils import RESERVED_WORDS, sanitize_identifier
+
+from .metamodel import Builtins, Action, ActionType, Namer, Property, Type, Var, Task, Value
+from . import metamodel as m
+from . import compiler as c
+import re
+
+gather_vars = m.Utils.gather_vars
+
+#--------------------------------------------------
+# Emitter
+#--------------------------------------------------
+
+rel_infix = [">", "<", ">=", "<=", "=", "!=", "+", "-", "*", "/", "^", "~="]
+rel_sanitize_re = re.compile(r'[^\w:\[\]\^" ,]|^(?=\d)', re.UNICODE)
+unsafe_symbol_pattern = re.compile(r"[^a-zA-Z0-9_]", re.UNICODE)
+
+def sanitize(input_string, is_rel_name_or_symbol = False):
+    # Replace non-alphanumeric characters with underscores
+    if is_rel_name_or_symbol and "[" in input_string:
+        string_parts = input_string.split('[', 1)
+        sanitized_name_or_symbol = sanitize_identifier(string_parts[0])
+        sanitized_rest = re.sub(rel_sanitize_re, "_", string_parts[1])
+        sanitized = f"{sanitized_name_or_symbol}[{sanitized_rest}"
+    else:
+        if "::" in input_string: # TODO: This is a temp solution to avoid sanitizing the namespace
+            sanitized = re.sub(rel_sanitize_re, "_", input_string)
+        else:
+            sanitized = sanitize_identifier(input_string)
+
+    # Check if the resulting string is a keyword and append an underscore if it is
+    if sanitized in RESERVED_WORDS:
+        sanitized += "_"
+
+    return sanitized
+
+class Emitter(c.Emitter):
+
+    def __init__(self, config):
+        super().__init__()
+        self.namer = Namer()
+        self.stack:List[Set[Var]] = []
+        self.mapped = {}
+        self.config = config
+
+    def reset(self):
+        self.namer.reset()
+        self.stack = []
+        self.mapped = {}
+
+    #--------------------------------------------------
+    # Emit
+    #--------------------------------------------------
+
+    def emit(self, task: Task|Var):
+        self.mapped.clear()
+        code = ""
+        try:
+            if isinstance(task, Task):
+                code = getattr(self, task.behavior.value)(task)
+            elif isinstance(task, Var):
+                code = self.emit_var(task)
+        except Exception as e:
+            # print("EMIT FAILED:", e)
+            raise e
+        return code
+
+    #--------------------------------------------------
+    # Helpers
+    #--------------------------------------------------
+
+    def sanitize(self, input_string, is_rel_name = False):
+        return sanitize(input_string, is_rel_name)
+
+    #--------------------------------------------------
+    # Vars
+    #--------------------------------------------------
+
+    def emit_val(self, value: Value, var:Var|None=None):
+        if value is True:
+            return 'boolean_true'
+        if value is False:
+            return 'boolean_false'
+        if isinstance(value, list):
+            return f"{', '.join([self.emit_var(v) for v in value])}"
+        if isinstance(value, Task) and (value.items or not value.name):
+            return self.to_inline_relation(value)
+        if isinstance(value, Task) and value.name:
+            return f"{{{value.name}}}"
+        if isinstance(value, bytes):
+            if var and var.type.isa(Builtins.ValueType):
+                return f"^{self.sanitize(var.name).capitalize()}Type[0x{value.hex()}]"
+            return f"0x{value.hex()}"
+        if isinstance(value, Property):
+            return "{" + self.sanitize(value.name) + "}"
+        if isinstance(value, Type):
+            return "{" + self.sanitize(value.name, value.isa(Builtins.Relation)) + "}"
+        if isinstance(value, pd.Timestamp):
+            t = value.tz_localize('UTC')
+            return t.isoformat()
+        if isinstance(value, datetime):
+            t = value.astimezone(ZoneInfo('UTC'))
+            return t.isoformat(timespec='milliseconds')
+        if isinstance(value, date):
+            t = value
+            return t.isoformat()
+        if isinstance(value, str):
+            # % is the string interpolation character in rel
+            return json.dumps(value).replace("%", "\\%")
+        return json.dumps(value)
+
+    def emit_var(self, var: Var|Value):
+        if not isinstance(var, Var):
+            return self.emit_val(var)
+        if var in self.mapped:
+            return self.mapped[var]
+        if var.type.isa(Builtins.Symbol):
+            if unsafe_symbol_pattern.search(cast(str, var.value)):
+                return f":\"{var.value}\""
+            return f":{self.sanitize(var.value)}"
+        if var.value is not None:
+            return self.emit_val(var.value, var)
+        if var.name == "_":
+            return "_"
+        name = self.namer.get(var)
+        return f"_{self.sanitize(name)}"
+
+    def to_name(self, value: Value|Var|None) -> str:
+        if value is not None and hasattr(value, "name"):
+            name = getattr(value, "name") # @NOTE: Working around pyright
+            if name:
+                return name
+
+        raise RAIException(f"Unable to determine the appropriate name for value of type: {type(value)} ({value})")
+
+    #--------------------------------------------------
+    # Sequence
+    #--------------------------------------------------
+
+    def sequence_action(self, action: Action):
+        if action.entity.value == Builtins.RawCode:
+            return str([*action.bindings.values()][0].value)
+        elif isinstance(action.entity.value, Task):
+            return self.emit(action.entity.value)
+        else:
+            raise Exception(f"TODO: Rel emit for action type {action.action}")
+
+    def sequence(self, task: Task):
+        items = [ self.sequence_action(i) for i in task.items ]
+        return "\n\n".join(items)
+
+    #--------------------------------------------------
+    # Query helpers
+    #--------------------------------------------------
+
+    def to_relation(self, action: Action, vars: set, body:List[str], in_head = False):
+        root = cast(Type, action.entity.value)
+        rel_name = root.name
+        if not rel_name and isinstance(root, Task):
+            rel_name = f"T{root.id}"
+        elif not rel_name:
+            rel_name = f"R_{root.id}"
+        args = []
+        for var in action.bindings.values():
+            emitted = self.emit_var(var)
+            vars.add(var)
+            if in_head and emitted in args:
+                name = self.namer.get(var)
+                orig = emitted
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {orig}")
+            elif in_head and var.value is not None and var.isa(Builtins.ValueType):
+                name = self.namer.get(var)
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {self.emit_val(var.value, var)}")
+            elif in_head and isinstance(var.value, bool):
+                name = self.namer.get(var)
+                orig = emitted
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {orig}")
+            args.append(emitted)
+        if rel_name in rel_infix:
+            if len(args) == 2:
+                return f"{args[0]} {rel_name} {args[1]}"
+            else:
+                return f"{args[2]} = {args[0]} {rel_name} {args[1]}"
+        rel_name = self.sanitize(rel_name, True)
+        if isinstance(root, Task) and root.isa(Builtins.Quantifier):
+            # TODO: handle quantifier grouping
+            args = args[1:]
+            rel_name = root.name.lower()
+            if root.isa(Builtins.Exists):
+                rel_name = "not empty"
+        if action.action == ActionType.Persist:
+            rel_name = f"insert[:{rel_name}]"
+        elif action.action == ActionType.Unpersist:
+            rel_name = f"delete[:{rel_name}]"
+        partial_appl = rel_name.find('[')
+        if partial_appl >= 0:
+            final_rel_name = rel_name[:partial_appl]
+            if len(args):
+                final_args = f"{rel_name[partial_appl+1:-1]}, {', '.join(args)}"
+            else:
+                final_args = ', '.join(args)
+            final = f"{final_rel_name}({final_args})"
+        else:
+            final = f"{rel_name}({', '.join(args)})"
+        return final
+
+    def to_return(self, action: Action, vars: set, body:List[str]):
+        args = []
+        for var in action.bindings.values():
+            emitted = self.emit_var(var)
+            vars.add(var)
+            if emitted in args:
+                name = self.namer.get(var)
+                orig = emitted
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {orig}")
+            elif var.value is not None and var.isa(Builtins.ValueType):
+                name = self.namer.get(var)
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {self.emit_val(var.value, var)}")
+            elif isinstance(var.value, bool):
+                name = self.namer.get(var)
+                orig = emitted
+                emitted = self.sanitize(f"_{self.namer.get_safe_name(name)}")
+                body.append(f"{emitted} = {orig}")
+            args.append(emitted)
+        return f"output({', '.join(args)})"
+
+    def to_set(self, action: Action, vars: set):
+        vals:Any = [v for v in action.bindings.values()]
+        mid_point = len(vals) // 2
+        vars_str = ", ".join([self.emit_var(i) for i in vals[mid_point:]])
+        rows = []
+        for ix in range(len(vals[0].value)):
+            row_vals = vals[:mid_point]
+            product_str = ",".join([self.emit_var(col.value[ix]) for col in row_vals])
+            if len(row_vals) > 1:
+                rows.append(f"({product_str})")
+            else:
+                rows.append(product_str)
+        row_str = "; ".join(rows)
+        return f"{{{row_str}}}({vars_str})"
+
+    def to_inline_set(self, action: Action, vars: set):
+        vals:Any = [v for v in action.bindings.values()]
+        rows = []
+        for ix in range(len(vals[0].value)):
+            row_vals = vals[:-1]
+            product_str = ",".join([self.emit_var(col.value[ix]) for col in row_vals])
+            if len(row_vals) > 1:
+                rows.append(f"({product_str})")
+            else:
+                rows.append(product_str)
+        row_str = "; ".join(rows)
+        return (vals[-1], f"{{{row_str}}}")
+
+    def to_inline_relation(self, task:Task, existing_vars=set()):
+        return f"{{{self.query(task, True)}}}"
+
+    #--------------------------------------------------
+    # Query
+    #--------------------------------------------------
+
+    def query(self, task: Task, inline = False):
+        supporting_rules = []
+        head = ""
+        body = []
+        body_vars = set()
+        head_vars = set()
+
+        task_vars = set(gather_vars(task.items))
+        self.stack.append(task_vars)
+
+        for i in task.items:
+            if i.action in [ActionType.Get, ActionType.Call]:
+                if i.entity.value == Builtins.RawData:
+                    body.append(self.to_set(i, body_vars))
+                elif i.entity.value == Builtins.InlineRawData:
+                    (v, data) = self.to_inline_set(i, body_vars)
+                    self.mapped[v] = data
+                    head_vars.add(v)
+                elif i.entity.value == Builtins.Install:
+                    (item,) = i.bindings.values()
+                    supporting_rules.append(f"declare {self.sanitize(self.to_name(item.value), True)}")
+                    if isinstance(item.value, Type) and item.value.isa(Builtins.ValueType):
+                        vtype = self.sanitize(item.value.name, True).capitalize() + "Type"
+                        supporting_rules.append(f"value type {vtype} {{ UInt128 }}")
+                        supporting_rules.append(f"@inline def pyrel_unwrap(y in {vtype}, x): ^{vtype}(x, y)")
+                        supporting_rules.append(f"def ::std::common::uuid_string(x in {vtype}, y): exists((z) | pyrel_unwrap(x, z) and uuid_string(z, y))")
+                        for op in ["<", ">"]:
+                            supporting_rules.append(f"def ::std::common::({op})[x in {vtype}, y in {vtype}]: exists((a, b) | ^{vtype}(a, x) and ^{vtype}(b, y) and a {op} b)")
+                            if self.config.get("compiler.use_monotype_operators", False):
+                                supporting_rules.append(f"def ::std::monotype::({op})[x in {vtype}, y in {vtype}]: exists((a, b) | ^{vtype}(a, x) and ^{vtype}(b, y) and a {op} b)")
+                        for op in ["minimum", "maximum"]:
+                            supporting_rules.append(f"def ::std::common::{op}(x in {vtype}, y in {vtype}, z in {vtype}): exists((a, b) | ^{vtype}(minimum[a, b], z) and ^{vtype}(a, x) and ^{vtype}(b, y))")
+                else:
+                    body.append(self.to_relation(i, body_vars, body))
+            elif i.action == ActionType.Bind and i.entity.value == Builtins.Return:
+                head_rel = self.to_return(i, head_vars, body)
+                head = f"def {head_rel}:\n    "
+            elif i.action in [ActionType.Bind, ActionType.Persist, ActionType.Unpersist]:
+                if not inline:
+                    head_rel = self.to_relation(i, head_vars, body, True)
+                    annotations = set(["from_pyrel"])
+                    # If optimized rel is requested, add desired attribute
+                    opt = self.config.get(name="describe_optimization", strict=False)
+                    if opt == "simple" :
+                        annotations.add(Builtins.InspectOptimizedSimple.name.lower())
+                    elif opt == "verbose" :
+                        annotations.add(Builtins.InspectOptimizedVerbose.name.lower())
+                    ent = i.entity.value
+                    if isinstance(ent, Type) or isinstance(ent, Property):
+                        for parent in ent.parents:
+                            if parent.isa(Builtins.Annotation):
+                                annotations.add(parent.name.lower())
+                    if annotations:
+                        annotation_str = " ".join([f"@{a}" for a in annotations])
+                        head = f"{annotation_str}\ndef {head_rel}:\n    "
+                    else:
+                        head = f"def {head_rel}:\n    "
+                else:
+                    props = list(i.bindings.values())
+                    head_vars.update(props)
+                    head = f"({', '.join([self.emit_var(v) for v in props])}): "
+            elif i.action == ActionType.Construct:
+                inputs = [*i.bindings.values()]
+                input_str =', '.join([self.emit_var(v) for v in inputs[:-1]])
+                output = self.emit_var(inputs[-1])
+                body_vars.update(inputs)
+                body.append(f"{output} = ^{self.sanitize(self.to_name(i.entity.value)).capitalize()}Type[{input_str}]")
+            else:
+                raise Exception(f"TODO: Rel emit for action type {i.action}")
+        body_str = " and\n    ".join(body)
+        existing_vars = set()
+        for vars in self.stack[:-1]:
+            existing_vars.update(vars)
+        from_vars = [self.emit_var(v) for v in (body_vars - head_vars - existing_vars) if v.value is None]
+        if len(from_vars) and not inline:
+            body_str = f"exists(({', '.join(from_vars)}) |\n    {body_str})"
+        if head and len(from_vars) and inline:
+            body_str = f"exists(({', '.join(from_vars)}) | {body_str})"
+
+
+        self.stack.pop()
+
+        if not head and inline:
+            return body_str
+        elif not head:
+            head = f"def T{task.id}():\n    "
+
+        support_str = ""
+        if len(supporting_rules):
+            support_str = "\n\n".join(supporting_rules) + "\n\n"
+
+        if support_str and not head_vars and not body_str:
+            return support_str
+
+        if not body_str:
+            body_str = "{()}"
+
+
+        return f"{support_str}{head}{body_str}"

relationalai/rel_utils.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+from typing import Any, Iterable, List, Sequence
+
+import pandas as pd
+
+from relationalai.clients.types import TransactionAsyncResponse
+from relationalai.clients.util import IdentityParser, ParseError
+from relationalai.dsl import safe_symbol
+from relationalai.errors import RAIException, RAIExceptionSet, RelQueryError
+
+#-------------------------------------------------------------------------------
+# Emitting
+#-------------------------------------------------------------------------------
+
+class Char(str):
+    def __new__(cls, value):
+        if value is None:
+            raise ValueError("Char cannot be None")
+        if len(value) != 1:
+            raise ValueError("A Char must be a single character")
+        return str.__new__(cls, value)
+
+def emit_literal(v: Any):
+    """Emit `v` as its equivalent literal representation in rel."""
+    if isinstance(v, Char):
+        sanitized = v.replace("'", "\\'")
+        return f"'{sanitized}'"
+    if isinstance(v, str):
+        sanitized = v.replace('"', '\\"').replace("%", "\\%")
+        return f'"{sanitized}"'
+    if isinstance(v, tuple):
+        return ", ".join(emit_literal(item) for item in v)
+    return v
+
+def emit_nested_relation(prefix: str, obj: dict|None = None, keys: Iterable[str]|None = None, raw = False) -> str:
+    """Emit a set of defs encoding `obj` in GNF on `prefix`."""
+    obj = obj or {}
+    result = ""
+    for k in keys or obj.keys():
+        v = obj.get(k, None)
+        if isinstance(v, dict):
+            result += emit_nested_relation(f"{prefix}{safe_symbol(k)}, ", v)
+        elif v is not None:
+            result += f"def {prefix}{safe_symbol(k)}]: {emit_literal(v) if not raw else v}\n"
+    return result
+
+# SEE: REL:      https://docs.relational.ai/rel/ref/lexical-symbols#keywords
+#      CORE REL: https://docs.google.com/document/d/12LUQdRed7P5EqQI1D7AYG4Q5gno9uKqy32i3kvAWPCA
+RESERVED_WORDS = [
+    "and",
+    "as",
+    "bound",
+    "declare",
+    "def",
+    "else",
+    "end",
+    "entity",
+    "exists",
+    "false",
+    "for",
+    "forall",
+    "from",
+    "ic",
+    "if",
+    "iff",
+    "implies",
+    "in",
+    "module",
+    "namespace",
+    "not",
+    "or",
+    "requires",
+    "then",
+    "true",
+    "use",
+    "where",
+    "with",
+    "xor"
+]
+
+def sanitize_identifier(name: str) -> str:
+    """
+    Return a string safe to use as a top level identifier in rel, such as a variable or relation name.
+
+    Args:
+        name (str): The input identifier string.
+
+    Returns:
+        str: The sanitized identifier string.
+    """
+
+    if not name:
+        return name
+
+    safe_name = ''.join(c if c.isalnum() else '_' for c in name)
+    if safe_name[0].isdigit():
+        safe_name = '_' + safe_name
+    if  safe_name in RESERVED_WORDS:
+        safe_name = safe_name + "_" # preferring the pythonic pattern of `from_` vs `_from`
+    return safe_name
+
+def to_fqn_relation_name(fqn: str) -> str:
+    """
+    Wrapper around `sanitize_identifier` which also lowers the string if no double quotes are present
+    (for backwards compatibility), otherwise leaves it as is.
+
+    Args:
+        fqn (str): The fully qualified name ('db.schema.object').
+
+    Returns:
+        str: A string safe to use as a relation name in rel and matching the format in SF NA.
+    """
+    # In this case we do not want to upper case the non unique/double quoted parts of the identifier
+    # This is required currently for for backwards compatibility
+    parser = IdentityParser(fqn, force_upper_case = False)
+    if not parser.is_complete:
+        raise ParseError(f"Failed to parse {fqn}. Incomplete relation identifier. Must be fully qualified e.g. 'db.schema.object'.")
+    if not parser.identity:
+        raise ParseError(f"Failed to parse {fqn}. Empty identifier.")
+
+    _name = parser.identity if parser.has_double_quoted_identifier else parser.identity.lower()
+    return sanitize_identifier(_name)
+
+#-------------------------------------------------------------------------------
+# Result Handling
+#-------------------------------------------------------------------------------
+
+_known_problems: list[Any] = []
+def ignore_known_problems(problems: Sequence[Any]):
+    """Filter out issues already in `problems` when checking responses for rel problems."""
+    global _known_problems
+    _known_problems.clear()
+    _known_problems.extend(problems)
+
+def assert_no_problems(res: TransactionAsyncResponse):
+    """Throw a vaguely-readable exception if rel problems were reported with the given transaction."""
+    new_problems = []
+    for problem in res.problems or []:
+        if problem not in _known_problems:
+            new_problems.append(problem)
+
+    if new_problems:
+        errs: List[RAIException] = [RelQueryError(problem) for problem in new_problems]
+        if len(errs) > 1:
+            raise RAIExceptionSet(errs) from None
+        else:
+            raise errs[0] from None
+
+def maybe_scalarize(v):
+    if getattr(v, "__len__", None) and len(v) == 1:
+        return v[0]
+    return v
+
+def process_gnf_results(gnf: Sequence, *key_names: str):
+    """Process GNF results into a nested object keyed by the key(s) in `key_names`"""
+    assert len(key_names) > 0, "Must supply a name for the GNF key(s)"
+    key_size = len(key_names)
+
+    obj = {}
+
+    for rel in gnf:
+        sig = rel["relationId"].split("/")
+        if sig[1] != ":output":
+            continue
+
+        table: pd.DataFrame = rel["table"].to_pandas()
+
+        key_cols = table.columns[0:key_size]
+
+        for _, row in table.iterrows():
+            raw_keys = tuple(row[col] for col in key_cols)
+            keys = maybe_scalarize(raw_keys)
+            vals = [row[col] for col in table.columns[key_size:]]
+
+            entry = obj.setdefault(keys, {k: v for k, v in zip(key_names, raw_keys)})
+
+            cur = 4
+            field = sig[cur - 2][1:]
+            while len(sig) > cur and sig[cur][0] == ":":
+                entry = entry.setdefault(field, {})
+                field = sig[cur][1:]
+                cur += 2
+            entry[field] = maybe_scalarize(vals)
+
+    return obj