relationalai 0.11.2__py3-none-any.whl → 0.11.4__py3-none-any.whl

relationalai/early_access/builder/std/__init__.py

@@ -1,6 +1,6 @@
 import warnings
 
-from relationalai.semantics.std import range, hash, cast, dates, math, strings, decimals, integers, pragmas, \
+from relationalai.semantics.std import range, hash, cast, datetime as dates, math, strings, decimals, integers, pragmas, \
     constraints, uuid_to_string
 
 __all__ = [

relationalai/early_access/dsl/bindings/csv.py

@@ -1,5 +1,5 @@
 from io import StringIO
-from typing import Optional
+from typing import Optional, Hashable
 
 import numpy as np
 import pandas as pd
@@ -76,7 +76,7 @@ class BindableCsvColumn(BindableColumn, b.Relationship):
 
 
 class CsvTable(AbstractBindableTable[BindableCsvColumn]):
-    _basic_type_schema: dict[str, str]
+    _basic_type_schema: dict[Hashable, str]
     _csv_data: list[pd.DataFrame]
     _num_rows: int
 
@@ -157,13 +157,13 @@ class CsvSourceModule:
     def _row_to_date_value_rule(column, row, value):
         parse_date = b.Relationship.builtins['parse_date']
         rez = b.Date.ref()
-        where(parse_date(value, 'Y-m-d', rez)).define(column(row, rez))
+        where(parse_date(value, 'yyyy-mm-dd', rez)).define(column(row, rez))
 
     @staticmethod
     def _row_to_date_time_value_rule(column, row, value):
         parse_datetime = b.Relationship.builtins['parse_datetime']
         rez = b.DateTime.ref()
-        where(parse_datetime(value, 'Y-m-d HH:MM:SS z', rez)).define(column(row, rez))
+        where(parse_datetime(value, 'yyy-mm-dd HH:MM:SS z', rez)).define(column(row, rez))
 
     @staticmethod
     def _row_to_decimal_value_rule(column, row, value):

relationalai/semantics/internal/internal.py

@@ -514,11 +514,13 @@ class Producer:
     #--------------------------------------------------
 
     def in_(self, values:list[Any]|Fragment) -> Expression:
+        columns = None
         if isinstance(values, Fragment):
             return self == values
         if not isinstance(values[0], tuple):
             values = [tuple([v]) for v in values]
-        d = data(values)
+            columns = [f"v{i}" for i in range(len(values[0]))]
+        d = data(values, columns)
         return self == d[0]
 
     #--------------------------------------------------
@@ -907,9 +909,9 @@ class Concept(Producer):
                 if python_types_to_concepts.get(v):
                     v = python_types_to_concepts[v]
                 if isinstance(v, Concept):
-                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._name}}}", short_name=k, model=self._model))
+                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._name}}}", parent=self, short_name=k, model=self._model))
                 elif isinstance(v, type) and issubclass(v, self._model.Enum): #type: ignore
-                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._concept._name}}}", short_name=k, model=self._model))
+                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._concept._name}}}", parent=self, short_name=k, model=self._model))
                 elif isinstance(v, Relationship):
                     self._validate_identifier_relationship(v)
                     setattr(self, k, v)
@@ -1189,6 +1191,7 @@ def is_decimal(concept: Concept) -> bool:
 Concept.builtins["Int"] = Concept.builtins["Int128"]
 Concept.builtins["Integer"] = Concept.builtins["Int128"]
 
+_np_datetime = np.dtype('datetime64[ns]')
 python_types_to_concepts : dict[Any, Concept] = {
     int: Concept.builtins["Int128"],
     float: Concept.builtins["Float"],
@@ -1213,6 +1216,7 @@ python_types_to_concepts : dict[Any, Concept] = {
     np.dtype('float32'): Concept.builtins["Float"],
     np.dtype('bool'): Concept.builtins["Bool"],
     np.dtype('object'): Concept.builtins["String"],  # Often strings are stored as object dtype
+    _np_datetime: Concept.builtins["DateTime"],
 
     # Pandas extension dtypes
     pd.Int64Dtype(): Concept.builtins["Int128"],
@@ -1655,7 +1659,9 @@ class Expression(Producer):
             raise ValueError(f"Argument index should be positive, got {idx}")
         if len(self._params) <= idx:
             raise ValueError(f"Expression '{self.__str__()}' has only {len(self._params)} arguments")
-        return ArgumentRef(self, self._params[idx])
+        param = self._params[idx]
+        # if param is an Expression then refer the last param of this expression
+        return ArgumentRef(self, param._params[-1] if isinstance(param, Expression) else param)
 
     def __getattr__(self, name: str):
         last = self._params[-1]
@@ -2090,8 +2096,20 @@ class DataColumn(Producer):
         self._data = data
         self._type = _type
         self._name = name if isinstance(name, str) else f"v{name}"
+        if pd.api.types.is_datetime64_any_dtype(_type):
+            _type = _np_datetime
+        # dates are objects in pandas
+        elif pd.api.types.is_object_dtype(_type) and self._is_date_column():
+            _type = date
         self._ref = python_types_to_concepts[_type].ref(self._name)
 
+    def _is_date_column(self) -> bool:
+        sample = self._data._data[self._name].dropna()
+        if sample.empty:
+            return False
+        sample_value = sample.iloc[0]
+        return isinstance(sample_value, date) and not isinstance(sample_value, datetime)
+
     def __str__(self):
         return f"DataColumn({self._name}, {self._type})"
 

relationalai/semantics/lqp/executor.py

@@ -12,6 +12,7 @@ from relationalai import debugging
 from relationalai.semantics.lqp import result_helpers
 from relationalai.semantics.metamodel import ir, factory as f, executor as e
 from relationalai.semantics.lqp.compiler import Compiler
+from relationalai.semantics.lqp.intrinsics import mk_intrinsic_datetime_now
 from relationalai.semantics.lqp.types import lqp_type_to_sql
 from lqp import print as lqp_print, ir as lqp_ir
 from lqp.parser import construct_configure
@@ -120,13 +121,15 @@ class LQPExecutor(e.Executor):
                     pyrel_errors[problem["props"]["pyrel_id"]].append(problem)
                 elif abort_on_error:
                     e = errors.RelQueryError(problem, source)
-                    supplementary_message = "Troubleshooting:\n" + \
-                        "  1. Please retry with a new name for your model. This can work around state-related issues.\n" + \
-                        "  2. If the error persists, please retry with the `use_lqp` flag set to `False`, for example:\n" + \
-                        "       `model = Model(..., use_lqp=False)`\n" + \
-                        "     This will switch the execution to the legacy backend, which may avoid the issue with some performance cost.\n"
 
-                    e.content = f"{e.content}{supplementary_message}"
+                    if code == 'SYSTEM_INTERNAL':
+                        supplementary_message = "Troubleshooting:\n" + \
+                            "  1. Please retry with a new name for your model. This can work around state-related issues.\n" + \
+                            "  2. If the error persists, please retry with the `use_lqp` flag set to `False`, for example:\n" + \
+                            "       `model = Model(..., use_lqp=False)`\n" + \
+                            "     This will switch the execution to the legacy backend, which may avoid the issue with some performance cost.\n"
+
+                        e.content = f"{e.content}{supplementary_message}"
                     all_errors.append(e)
             else:
                 if code == "ARITY_MISMATCH":
@@ -256,11 +259,47 @@ class LQPExecutor(e.Executor):
 
         return ", ".join(fields)
 
+    def _construct_configure(self):
+        config_dict = {}
+        # Only set the IVM flag if there is a value in `config`. Otherwise, let
+        # `construct_configure` set the default value.
+        ivm_flag = self.config.get('reasoner.rule.incremental_maintenance', None)
+        if ivm_flag:
+            config_dict['ivm.maintenance_level'] = lqp_ir.Value(value=ivm_flag, meta=None)
+        return construct_configure(config_dict, None)
+    
+    def _compile_intrinsics(self) -> lqp_ir.Epoch:
+        """Construct an epoch that defines a number of built-in definitions used by the
+        emitter."""
+        with debugging.span("compile_intrinsics") as span:
+            debug_info = lqp_ir.DebugInfo(id_to_orig_name={}, meta=None)
+            intrinsics_fragment = lqp_ir.Fragment(
+                id = lqp_ir.FragmentId(id=b"__pyrel_lqp_intrinsics", meta=None),
+                declarations = [
+                    mk_intrinsic_datetime_now(),
+                ],
+                debug_info = debug_info,
+                meta = None,
+            )
+
+            span["compile_type"] = "intrinsics"
+            span["lqp"] = lqp_print.to_string(intrinsics_fragment, {"print_names": True, "print_debug": False, "print_csv_filename": False})
+
+            return lqp_ir.Epoch(
+                writes=[
+                    lqp_ir.Write(write_type=lqp_ir.Define(fragment=intrinsics_fragment, meta=None), meta=None)
+                ],
+                meta=None,
+            )
+
     def compile_lqp(self, model: ir.Model, task: ir.Task):
+        configure = self._construct_configure()
+
         model_txn = None
         if self._last_model != model:
             with debugging.span("compile", metamodel=model) as install_span:
                 _, model_txn = self.compiler.compile(model, {"fragment_id": b"model"})
+                model_txn = txn_with_configure(model_txn, configure)
                 install_span["compile_type"] = "model"
                 install_span["lqp"] = lqp_print.to_string(model_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
                 self._last_model = model
@@ -273,23 +312,26 @@ class LQPExecutor(e.Executor):
             }
             result, final_model = self.compiler.compile_inner(query, options)
             export_info, query_txn = result
+            query_txn = txn_with_configure(query_txn, configure)
             compile_span["compile_type"] = "query"
             compile_span["lqp"] = lqp_print.to_string(query_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
 
-        txn = query_txn
+        # Merge the epochs into a single transactions. Long term the query bits should all
+        # go into a WhatIf action and the intrinsics could be fused with either of them. But
+        # for now we just use separate epochs.
+        epochs = []
+
+        epochs.append(self._compile_intrinsics())
+
         if model_txn is not None:
-            # Merge the two LQP transactions into one. Long term the query bits should all
-            # go into a WhatIf action. But for now we just use two separate epochs.
-            model_epoch = model_txn.epochs[0]
-            query_epoch = query_txn.epochs[0]
-            txn = lqp_ir.Transaction(
-                epochs=[model_epoch, query_epoch],
-                configure=construct_configure({}, None),
-                meta=None,
-            )
+            epochs.append(model_txn.epochs[0])
 
-            # Revalidate now that we've joined two epochs
-            validate_lqp(txn)
+        epochs.append(query_txn.epochs[0])
+
+        txn = lqp_ir.Transaction(epochs=epochs, configure=configure, meta=None)
+
+        # Revalidate now that we've joined all the epochs.
+        validate_lqp(txn)
 
         txn_proto = convert_transaction(txn)
         # TODO (azreika): Should export_info be encoded as part of the txn_proto? [RAI-40312]
@@ -350,3 +392,12 @@ class LQPExecutor(e.Executor):
             # If processing the results failed, revert to the previous model.
             self._last_model = previous_model
             raise e
+
+def txn_with_configure(txn: lqp_ir.Transaction, configure: lqp_ir.Configure) -> lqp_ir.Transaction:
+    """ Return a new transaction with the given configure. If the transaction already has
+    a configure, it is replaced. """
+    return lqp_ir.Transaction(
+        epochs=txn.epochs,
+        configure=configure,
+        meta=txn.meta,
+    )

relationalai/semantics/lqp/intrinsics.py

@@ -0,0 +1,23 @@
+from datetime import datetime, timezone
+
+from relationalai.semantics.lqp import ir as lqp
+from relationalai.semantics.lqp.constructors import mk_abstraction, mk_value, mk_var, mk_type, mk_primitive
+from relationalai.semantics.lqp.utils import lqp_hash
+
+def mk_intrinsic_datetime_now() -> lqp.Def:
+    """Constructs a definition of the current datetime."""
+    id = lqp_hash("__pyrel_lqp_intrinsic_datetime_now")
+    out = mk_var("out")
+    out_type = mk_type(lqp.TypeName.DATETIME)
+    now = mk_value(lqp.DateTimeValue(value=datetime.now(timezone.utc), meta=None))
+    datetime_now = mk_abstraction(
+        [(out, out_type)],
+        mk_primitive("rel_primitive_eq", [out, now]),
+    )
+
+    return lqp.Def(
+        name = lqp.RelationId(id=id, meta=None),
+        body = datetime_now,
+        attrs = [],
+        meta = None,
+    )

relationalai/semantics/lqp/model2lqp.py

@@ -13,7 +13,7 @@ from relationalai.semantics.lqp.utils import TranslationCtx, gen_unique_var
 from relationalai.semantics.lqp.validators import assert_valid_input
 
 from decimal import Decimal as PyDecimal
-from datetime import datetime, date
+from datetime import datetime, date, timezone
 from typing import Tuple, cast, Union, Optional
 from warnings import warn
 import re
@@ -192,12 +192,21 @@ def _translate_effect(ctx: TranslationCtx, effect: Union[ir.Output, ir.Update],
     elif isinstance(effect, ir.Output):
         ctx.output_ids.append((rel_id, def_name))
 
+    # First we collect annotations on the effect itself, e.g. from something like
+    # `select(...).annotate(...)`.
+    annotations = effect.annotations
+    if isinstance(effect, ir.Update):
+        # Then we translate annotations on the relation itself, e.g.
+        # ```
+        # Bar.foo = model.Relationship(...)
+        # Bar.foo.annotate(...)
+        # ```
+        annotations = annotations | effect.relation.annotations
+
     return lqp.Def(
         name = rel_id,
         body = mk_abstraction(projection, new_body),
-        # TODO this only covers the annotations on the effect itself. Annotations on the
-        # relation are not included yet.
-        attrs = _translate_annotations(effect.annotations),
+        attrs = _translate_annotations(annotations),
         meta = None,
     )
 
@@ -627,7 +636,8 @@ def to_lqp_value(value: ir.PyValue, value_type: ir.Type) -> lqp.Value:
     elif typ.type_name == lqp.TypeName.DATE and isinstance(value, date):
         val = lqp.DateValue(value=value, meta=None)
     elif typ.type_name == lqp.TypeName.DATETIME and isinstance(value, datetime):
-        val = lqp.DateTimeValue(value=value, meta=None)
+        utc_value = value.astimezone(timezone.utc) if value.tzinfo is not None else value # Convert to UTC cf. Iceberg
+        val = lqp.DateTimeValue(value=utc_value, meta=None)
     elif typ.type_name == lqp.TypeName.BOOLEAN and isinstance(value, bool):
         val = lqp.BooleanValue(value=value, meta=None)
     else:
@@ -696,4 +706,4 @@ def _translate_join(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
 
     output_term = _translate_term(ctx, target)[0]
 
-    return lqp.Reduce(meta=None, op=op, body=body, terms=[output_term])
+    return lqp.Reduce(meta=None, op=op, body=body, terms=[output_term])

relationalai/semantics/lqp/passes.py

@@ -8,7 +8,7 @@ from relationalai.semantics.metamodel.util import FrozenOrderedSet
 
 from relationalai.semantics.metamodel.rewrite import Flatten
 # TODO: Move this into metamodel.rewrite
-from relationalai.semantics.rel.rewrite import QuantifyVars, CDC
+from relationalai.semantics.rel.rewrite import QuantifyVars, CDC, ExtractCommon
 
 from relationalai.semantics.lqp.utils import output_names
 
@@ -25,8 +25,7 @@ def lqp_passes() -> list[Pass]:
         InferTypes(),
         DNFUnionSplitter(),
         ExtractKeys(),
-        # Broken
-        # ExtractCommon(),
+        ExtractCommon(),
         Flatten(),
         Splinter(), # Splits multi-headed rules into multiple rules
         QuantifyVars(), # Adds missing existentials
@@ -548,7 +547,7 @@ class PeriodMath(Pass):
                 return node
 
             if node.relation.name not in {
-                "year", "month", "week", "day", "hour", "minute", "second", "millisecond"
+                "year", "month", "week", "day", "hour", "minute", "second", "millisecond", "microsecond", "nanosecond"
             }:
                 return node
 

relationalai/semantics/lqp/primitives.py

@@ -1,7 +1,7 @@
 from relationalai.semantics.metamodel.types import digits_to_bits
 from relationalai.semantics.lqp import ir as lqp
 from relationalai.semantics.lqp.types import is_numeric
-from relationalai.semantics.lqp.utils import UniqueNames
+from relationalai.semantics.lqp.utils import UniqueNames, lqp_hash
 from relationalai.semantics.lqp.constructors import mk_primitive, mk_specialized_value, mk_type, mk_value, mk_var
 
 rel_to_lqp = {
@@ -22,12 +22,18 @@ rel_to_lqp = {
     "sinh": "rel_primitive_sinh",
     "asin": "rel_primitive_asin",
     "asinh": "rel_primitive_asinh",
+    "tan": "rel_primitive_tan",
+    "tanh": "rel_primitive_tanh",
+    "atan": "rel_primitive_atan",
+    "atanh": "rel_primitive_atanh",
+    "cot": "rel_primitive_cot",
+    "acot": "rel_primitive_acot",
     "construct_date": "rel_primitive_construct_date",
     "construct_date_from_datetime": "rel_primitive_datetime_date_convert",
-    "construct_datetime": "rel_primitive_construct_datetime",
     "construct_datetime_ms_tz": "rel_primitive_construct_datetime",
     "hash": "rel_primitive_hash_tuple_uint128",
     "uuid_to_string": "rel_primitive_uuid_string",
+    "parse_uuid": "rel_primitive_parse_uuid",
     "parse_date": "rel_primitive_parse_date",
     "parse_datetime": "rel_primitive_parse_datetime",
     "parse_decimal": "rel_primitive_parse_decimal",
@@ -47,23 +53,42 @@ rel_to_lqp = {
     "replace": "rel_primitive_replace",
     "strip": "rel_primitive_trim",
     "date_year": "rel_primitive_date_year",
+    "date_quarter": "rel_primitive_date_quarterofyear",
     "date_month": "rel_primitive_date_month",
     "date_week": "rel_primitive_date_week",
     "date_day": "rel_primitive_date_day",
+    "date_dayofyear": "rel_primitive_date_dayofyear",
+    "date_weekday": "rel_primitive_date_dayofweek",
     "date_add": "rel_primitive_typed_add_date_period",
     "date_subtract": "rel_primitive_typed_subtract_date_period",
     "dates_period_days": "rel_primitive_date_days_between",
+    "datetime_now": "__pyrel_lqp_intrinsic_datetime_now",
     "datetime_add": "rel_primitive_typed_add_datetime_period",
     "datetime_subtract": "rel_primitive_typed_subtract_datetime_period",
+    "datetime_year": "rel_primitive_datetime_year",
+    "datetime_quarter": "rel_primitive_datetime_quarterofyear",
+    "datetime_month": "rel_primitive_datetime_month",
     "datetime_week": "rel_primitive_datetime_week",
+    "datetime_day": "rel_primitive_datetime_day",
+    "datetime_dayofyear": "rel_primitive_datetime_dayofyear",
+    "datetime_hour": "rel_primitive_datetime_hour",
+    "datetime_minute": "rel_primitive_datetime_minute",
+    "datetime_second": "rel_primitive_datetime_second",
+    "datetime_weekday": "rel_primitive_datetime_dayofweek",
     "datetimes_period_milliseconds": "rel_primitive_datetime_milliseconds_between",
     "date_format": "rel_primitive_format_date",
     "datetime_format": "rel_primitive_format_datetime",
     "range": "rel_primitive_range",
     "natural_log": "rel_primitive_natural_log",
+    "log": "rel_primitive_log",
+    "log2": "rel_primitive_log2",
+    "log10": "rel_primitive_log10",
     "sqrt": "rel_primitive_sqrt",
     "isinf": "rel_primitive_isinf",
     "isnan": "rel_primitive_isnan",
+    "exp": "rel_primitive_natural_exp",
+    "erf": "rel_primitive_error_function",
+    "erfinv": "rel_primitive_error_function_inverse",
     # Division is monotype, but only on the input args. Until we distinguish between input
     # and output args, we can't use the same assertions for monotype-ness as the other ops.
     "/": "rel_primitive_divide_monotype",
@@ -101,18 +126,7 @@ rel_to_lqp_monotype = {
 # Insert extra terms where a raicode primitive expects more terms, and there are possible
 # defaults.
 def _extend_primitive_terms(name: str, terms: list[lqp.RelTerm], term_types: list[lqp.Type]) -> tuple[list[lqp.RelTerm], list[lqp.Type]]:
-    if name == "rel_primitive_construct_datetime" and len(terms) == 7:
-        # construct_datetime does not provide a timezone or milliseconds so we
-        # default to 0 milliseconds and UTC timezone.
-        terms = [*terms[:-1], mk_value(0), mk_value("UTC"), terms[-1]]
-        term_types = [
-            *term_types[:-1],
-            mk_type(lqp.TypeName.INT),
-            mk_type(lqp.TypeName.STRING),
-            term_types[-1],
-        ]
-
-    elif name == "rel_primitive_parse_decimal" and len(terms) == 2:
+    if name == "rel_primitive_parse_decimal" and len(terms) == 2:
         assert term_types
         py_precision = term_types[1].parameters[0].value
         bit_value = mk_value(digits_to_bits(py_precision))
@@ -162,6 +176,16 @@ def build_primitive(
     terms, term_types = _reorder_primitive_terms(lqp_name, terms, term_types)
     _assert_primitive_terms(lqp_name, terms, term_types)
 
+    # Handle intrinsics. To callers of `build_primitive` the distinction between intrinsic
+    # and primitive doesn't matter, so we don't want to burden them with that detail.
+    # Intrinsics are built-in definitions added by the LQP emitter, that user logic can just
+    # refer to.
+    if lqp_name == "__pyrel_lqp_intrinsic_datetime_now":
+        id = lqp.RelationId(id=lqp_hash(lqp_name), meta=None)
+        assert len(terms) == 1
+        assert isinstance(terms[0], lqp.Term)
+        return lqp.Atom(name=id, terms=[terms[0]], meta=None)
+
     return mk_primitive(lqp_name, terms)
 
 def relname_to_lqp_name(name: str) -> str: