relationalai 1.0.0a1__py3-none-any.whl → 1.0.0a3__py3-none-any.whl

relationalai/semantics/std/aggregates.py

@@ -10,7 +10,6 @@ AggValue = Value | Distinct
 # Aggregates
 #------------------------------------------------------
 
-# TODO - overloads
 _sum = library.Relation("sum", fields=[Field.input("value", Numeric), Field("result", Numeric)],
                         overloads=[[Number, Number], [Float, Float]])
 _count = library.Relation("count", fields=[Field("result", Integer)])

relationalai/semantics/std/datetime.py

@@ -3,8 +3,9 @@ from __future__ import annotations
 from relationalai.semantics.std import floats
 
 from . import StringValue, IntegerValue, DateValue, DateTimeValue, math, common
-from ..frontend.base import Aggregate, Library, Concept, NumberConcept, Expression, Field, Literal, Variable
+from ..frontend.base import Aggregate, Library, Concept, MetaRef, NumberConcept, Expression, Field, Literal, Variable
 from ..frontend.core import Float, Number, String, Integer, Date, DateTime
+from ..frontend import core
 from .. import select
 
 from typing import Union, Literal
@@ -162,6 +163,7 @@ class date:
             num_days = cls.period_days(start, end)
             if freq in ["W", "M", "Y"]:
                 range_end = math.ceil(num_days * _days[freq])
+                range_end = core.cast(MetaRef(Integer), range_end)
             else:
                 range_end = num_days
             # date_range is inclusive. add 1 since std.range is exclusive
@@ -348,6 +350,7 @@ class datetime:
                 _end = num_ms
             else:
                 _end = math.ceil(num_ms * Float(_milliseconds[freq]))
+            _end = core.cast(MetaRef(Integer), _end)
             # datetime_range is inclusive. add 1 since common.range is exclusive
             ix = common.range(0, _end + 1, 1)
         else:

relationalai/shims/executor.py

@@ -10,7 +10,10 @@ from v0.relationalai.semantics.rel.executor import RelExecutor
 from v0.relationalai.semantics.metamodel import ir as v0, factory as v0_factory
 from v0.relationalai.semantics.metamodel.visitor import collect_by_type
 from v0.relationalai.semantics.snowflake import Table as v0Table
-from v0.relationalai.clients.snowflake import Provider as v0Provider
+try:
+    from v0.relationalai.clients.snowflake import Provider as v0Provider #type: ignore
+except ImportError:
+    from v0.relationalai.clients.resources.snowflake import Provider as v0Provider
 from v0.relationalai.clients.config import Config
 
 # from ..config import Config
@@ -29,10 +32,28 @@ TYPER_DEBUGGER=False
 # PRINT_RESULT=True
 # TYPER_DEBUGGER=True
 
+@lru_cache()
+def get_config():
+    return Config()
+
+def with_source(item: mm.Node):
+    if not hasattr(item, "source"):
+        raise ValueError(f"Item {item} has no source")
+    elif item.source is None:
+        return {}
+    elif debugging.DEBUG:
+        source = item.source.block
+        if source:
+            return { "file": source.file, "line": source.line, "source": source.source }
+        else:
+            return {"file":item.source.file, "line":item.source.line}
+    else:
+        return {"file":item.source.file, "line":item.source.line}
+
 def execute(query: Fragment, model: Model|None = None, executor=None, export_to="", update=False):
     if not executor:
         # use_lqp = Config().reasoner.rule.use_lqp
-        use_lqp = bool(Config().get("reasoner.rule.use_lqp", True))
+        use_lqp = bool(get_config().get("reasoner.rule.use_lqp", True))
         executor = "lqp" if use_lqp else "rel"
     mm_model = model.to_metamodel() if model else None
     mm_query = query.to_metamodel()
@@ -41,7 +62,7 @@ def execute(query: Fragment, model: Model|None = None, executor=None, export_to=
 
 def execute_mm(mm_query: mm.Task, mm_model: mm.Model|None = None, executor="lqp", export_to="", update=False, model: Model|None = None):
     # perform type inference
-    typer = Typer()
+    typer = Typer(enforce=False)
     # normalize the metamodel
     normalizer = Normalize()
     # translate the metamodel into a v0 query
@@ -117,7 +138,7 @@ def execute_mm(mm_query: mm.Task, mm_model: mm.Model|None = None, executor="lqp"
                     f.write(msg)
                     f.write('\n')
 
-    if DRY_RUN:
+    if DRY_RUN or get_config().get("compiler.dry_run", False):
         results = []
     else:
         # create snowflake tables for all the tables that have been used
@@ -132,7 +153,7 @@ def execute_mm(mm_query: mm.Task, mm_model: mm.Model|None = None, executor="lqp"
 
         # get an executor and execute
         executor = _get_executor(executor, model.name if model else "")
-        with debugging.span("query", tag=None, export_to=export_to) as query_span:
+        with debugging.span("query", tag=None, export_to=export_to, dsl="", **with_source(mm_query)) as query_span:
             if isinstance(executor, (LQPExecutor, RelExecutor)):
                 results = executor.execute(v0_model, v0_query, export_to=export_table, update=update)
             else:

relationalai/shims/mm2v0.py

@@ -21,6 +21,8 @@ from ..semantics.backends.lqp import annotations as lqp_annotations
 from v0.relationalai.semantics.metamodel import ir as v0, builtins as v0_builtins, types as v0_types, factory as f
 from v0.relationalai.semantics.metamodel.util import FrozenOrderedSet, frozen, ordered_set, filter_by_type, OrderedSet
 from v0.relationalai.semantics.internal.internal import literal_value_to_type
+from v0.relationalai.semantics.metamodel.typer import typer as v0_typer
+from v0.relationalai.clients.util import IdentityParser
 
 from .hoister import Hoister
 from .helpers import is_output_update, is_main_output
@@ -131,6 +133,16 @@ class Translator():
     def translate_frozen(self, nodes: seq[mm.Node], parent, ctx) -> FrozenOrderedSet[v0.Node]:
         return frozen(*self.translate_seq(nodes, parent, ctx))
 
+    #------------------------------------------------------
+    # Helper
+    #------------------------------------------------------
+
+    # NOTE: This has to match what is done in the v0 snowflake.Table class as that is what CDC
+    # produces. If there's a quote in the table name, then we take it verbatim, otherwise we lowercase it.
+    def translate_table_name(self, table: mm.Table) -> str:
+        name = IdentityParser(table.name).identifier
+        name = name.lower() if '"' not in table.name else table.name.replace('"', '_')
+        return sanitize(name)
 
     #-----------------------------------------------------------------------------
     # Capabilities, Reasoners
@@ -279,12 +291,22 @@ class Translator():
                         else:
                             overloads.update(x) # type: ignore
             overloads.update(self.translate_frozen(r.overloads, r, ctx))  # type: ignore
+            annotations = self.translate_seq(r.annotations, r, ctx)  # type: ignore
+            name = r.name
+            fields = self.translate_seq(r.fields, r, ctx)  # type: ignore
+            # We need to turn column relations into what the CDC pass would otherwise produce
+            # by making the relation name be the table name and adding a symbol field at the front
+            # representing the column name
+            if r.fields and isinstance(r.fields[0].type, mm.Table) and r in r.fields[0].type.columns:
+                name = self.translate_table_name(r.fields[0].type)
+                fields = (v0.Field("symbol", v0_types.Symbol, False), *fields) # type: ignore
+                annotations = annotations + (v0_builtins.external_annotation,)
 
             return v0.Relation(
-                name=r.name,
-                fields=self.translate_frozen(r.fields, r, ctx), # type: ignore
+                name=name,
+                fields=frozen(*fields), # type: ignore
                 requires=self.translate_frozen(r.requires, r, ctx), # type: ignore
-                annotations=self.translate_frozen(r.annotations, r, ctx), # type: ignore
+                annotations=frozen(*annotations), # type: ignore
                 overloads=overloads.frozen(), # type: ignore
             )
 
@@ -321,6 +343,7 @@ class Translator():
         b.core.Date: v0_types.Date,
         b.core.DateTime: v0_types.DateTime,
         b.core.Float: v0_types.Float,
+        b.core.Hash: v0_types.Hash,
     }
 
     def translate_scalartype(self, t: mm.ScalarType, parent: mm.Node, ctx) -> v0.ScalarType|v0.Relation|None:
@@ -334,18 +357,14 @@ class Translator():
             assert isinstance(actual_type, v0.ScalarType)
             fields = [v0.Field(name="entity", type=actual_type, input=False)] # type: ignore
             annotations = [v0_builtins.concept_relation_annotation]
+            name = t.name
             if isinstance(t, mm.Table):
                 annotations.append(v0_builtins.external_annotation)
-                annotations.append(v0_builtins.from_cdc_annotation)
-                for col in t.columns:
-                    fields.append(v0.Field(
-                        name=col.name,
-                        type=self.translate_node(col.fields[-1].type, col, Context.MODEL), # type: ignore
-                        input=False
-                    ))
+                name = self.translate_table_name(t)
+                fields.insert(0, v0.Field(name="symbol", type=v0_types.Symbol, input=False)) # type: ignore
 
             type_relation = v0.Relation(
-                name=t.name,
+                name=name,
                 fields=tuple(fields), # type: ignore
                 requires=frozen(),
                 annotations=frozen(*annotations), # type: ignore
@@ -437,6 +456,17 @@ class Translator():
     def translate_table(self, t: mm.Table, parent, ctx):
         return self.translate_scalartype(t, parent, ctx)
 
+    def rewrite_cdc_args(self, l: mm.Lookup, args, parent, ctx):
+        # If this is a lookup for an external table column,
+        # we have to prepend the column symbol to the args
+        root_type = l.relation.fields[0].type
+        if isinstance(root_type, mm.Table):
+            self.used_tables.add(root_type)
+            is_table = l.relation == root_type
+            if is_table or l.relation in root_type.columns:
+                sym = "METADATA$KEY" if is_table else l.relation.name
+                args = (v0.Literal(type=v0_types.Symbol, value=sym), *args)
+        return args
 
     # -----------------------------------------------------------------------------
     # Values
@@ -541,7 +571,7 @@ class Translator():
         # inline logicals if possible
         new_children = []
         for c in children:
-            if isinstance(c, v0.Logical) and not c.hoisted and len(c.body) == 1:
+            if isinstance(c, v0.Logical) and not c.hoisted and len(c.body) == 1 and not isinstance(c.body[0], (v0.Aggregate, v0.Rank)):
                 new_children.extend(c.body)
             else:
                 new_children.append(c)
@@ -590,10 +620,49 @@ class Translator():
                 return outputs
 
             # if this is an optional logical but we're not hoisting anything and not updating,
-            # then it's effectively a no-op since it cannot affect the query. Just return the outputs.
+            # then it's possible we're filtering an outer variable, but only for this column. If
+            # so, we need to alias the output and hoist it.
             # this is important because the LQP stack blows up if there's a logical with no effect
             if l.optional and not hoisted and not any(isinstance(c, v0.Update) for c in children):
-                return outputs
+                # if there are no lookups, then this really is a no-op, just return outputs
+                # LQP blows up with e.g. a match-only logical
+                if not any(isinstance(c, v0.Lookup) for c in children):
+                    return outputs
+                # otherwise, make sure we filter the outer variable through aliasing
+                new_children = [*children]
+                new_hoists = []
+                new_outputs = []
+                # add an eq to a new var for the output, hoist the new var, change the output to use the new var
+                for output in outputs:
+                    # shim outputs always only have one alias
+                    (name, orig_var) = output.aliases.data[0]
+                    # if the original var is not a Var or Literal, or it's already in the keys, skip (we can't filter keys)
+                    if not isinstance(orig_var, (v0.Var, v0.Literal)) or (output.keys and orig_var in output.keys): #type: ignore
+                        new_outputs.append(output)
+                        continue
+                    new_var = v0.Var(
+                        type=orig_var.type,
+                        name=f"{orig_var.name if isinstance(orig_var, v0.Var) else 'literal'}_hoisted"
+                    )
+                    new_output = v0.Output(
+                        engine=None,
+                        aliases=frozen((name, new_var)),
+                        keys=output.keys,
+                        annotations=output.annotations
+                    )
+                    eq = v0.Lookup(
+                        engine=None,
+                        relation=v0_builtins.eq,
+                        args=(new_var, orig_var),
+                        annotations=frozen()
+                    )
+                    new_children.append(eq)
+                    new_hoists.append(v0.Default(new_var, None))
+                    new_outputs.append(new_output)
+                outputs = new_outputs
+                hoisted = tuple(new_hoists)
+                children = tuple(new_children)
+                # return outputs
 
             # return outputs + a logical with the other children
             outputs.append(
@@ -782,44 +851,22 @@ class Translator():
                     args=(var, col_var), # type: ignore
                     annotations=self.translate_frozen(l.annotations, l, ctx) # type: ignore
                 )
-            else:
+            # if this is a data column, we just ignore it as the data node already binds the variables
+            elif isinstance(l.args[0].type, mm.Data):
                 return None
+            # Otherwise we keep the lookup because that's what the LQP stack expect (the lookup gets repeated for
+            # each column)
+
 
         relation, args = self._resolve_reading(l, ctx)
         if relation is None:
             return None
 
-        # External Table Column lookups
-        # we have to take the 6nf column relations and pull them into a single wide lookup
-        # making sure that the variable get mapped correctly. To match the expectations of
-        # v0, we also have to make sure that if we're looking up the table row itself, that
-        # it is wrapped in its own logical
-        root_type = l.relation.fields[0].type
-        if isinstance(root_type, mm.Table):
-            self.used_tables.add(root_type)
-            assert isinstance(l.args[0], mm.Var)
-            is_col = l.relation in root_type.columns
-            is_table = l.relation == root_type
-            if is_col:
-                self.column_map[l.args[0]][l.relation] = args[-1]
-                # we always lookup the full table, so replace the relation and args
-                relation = self.translate_node(root_type, l, ctx)
-
-            # this is a lookup on the table itself or the columns, translate to the column vars
-            if is_col or is_table:
-                mapped = self.column_map.get(l.args[0], {})
-                col_args = []
-                for col in root_type.columns:
-                    v = mapped.setdefault(col, v0.Var(
-                        type=self.translate_node(col.fields[-1].type, col, Context.MODEL), # type: ignore
-                        name=f"{col.name}"
-                    ))
-                    col_args.append(v)
-                args = tuple([args[0], *col_args]) # type: ignore
 
         # Specific rewrites
         rewrite = self.rewrite_lookup(l, parent, ctx)
         if rewrite is None:
+            args = self.rewrite_cdc_args(l, args, parent, ctx)
             # General translation
             rewrite = v0.Lookup(
                 engine=self.translate_reasoner(l.reasoner, l, Context.MODEL),
@@ -1056,10 +1103,12 @@ class Translator():
             else:
                 replaced_args.append(arg)
         # translate the lookup
+        args = tuple(self.translate_value(arg, l, ctx) for arg in replaced_args)
+        args = self.rewrite_cdc_args(l, args, l, ctx)
         lookup = v0.Lookup(
             engine=self.translate_reasoner(l.reasoner, l, Context.MODEL),
             relation=self.translate_node(l.relation), # type: ignore
-            args=tuple(self.translate_value(arg, l, ctx) for arg in replaced_args),
+            args=args,
             annotations=self.translate_frozen(l.annotations, l, ctx) # type: ignore
         )
         # subtract 1 from the index to convert from 1-based to 0-based
@@ -1109,6 +1158,18 @@ class Translator():
     def decrement(self, l: mm.Lookup, index: int, ctx):
         """ Rewrite the lookup such that the arg at `index` is decremented by 1 before the
         lookup.  """
+        x = l.args[index]
+        if isinstance(x, mm.Literal):
+            # if the arg is a literal, just decrement the literal directly
+            new_literal = mm.Literal(type=x.type, value=x.value - 1) # type: ignore
+            return v0.Lookup(
+                    engine=None,
+                    relation=self.translate_node(l.relation), # type: ignore
+                    args=tuple(self.translate_value(arg, l, ctx) if i != index else self.translate_value(new_literal, l, ctx) for i, arg in enumerate(l.args)),
+                    annotations=self.translate_frozen(l.annotations, l, ctx) # type: ignore
+            )
+
+        # arg is not a literal, so we need to create a tmp var to store the decremented value
         tmp = self.translate_value(mm.Var(type=b.core.Number, name="tmp"), l, ctx)
         # lookup(..., tmp, ...)
         new = v0.Lookup(
@@ -1137,7 +1198,21 @@ class Translator():
         inputs = []
         outputs = []
         args = []
-        for arg, field in zip(l.args, l.relation.fields):
+        fields = l.relation.fields
+        # if there are overloads, we need to cast based on the most compatible overload
+        if l.relation.overloads:
+            inf = float("inf")
+            min_cost = inf
+            for overload in l.relation.overloads:
+                total = 0
+                for arg, field in zip(l.args, overload.fields):
+                    if v0_typer.to_type(arg) != field.type:
+                        total += 1
+                if total < min_cost:
+                    min_cost = total
+                    fields = overload.fields
+
+        for arg, field in zip(l.args, fields):
             target_type = field.type
             if target_type is None or not isinstance(arg, (v0.Var, v0.Literal)) or arg.type == target_type:
                 args.append(arg)

{relationalai-1.0.0a1.dist-info → relationalai-1.0.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: relationalai
-Version: 1.0.0a1
+Version: 1.0.0a3
 Summary: RelationalAI Library and CLI
 Author-email: RelationalAI <support@relational.ai>
 Requires-Python: >=3.10