relationalai 0.11.3__py3-none-any.whl → 0.12.0__py3-none-any.whl

relationalai/early_access/dsl/bindings/csv.py

@@ -1,5 +1,5 @@
 from io import StringIO
-from typing import Optional
+from typing import Optional, Hashable
 
 import numpy as np
 import pandas as pd
@@ -76,7 +76,7 @@ class BindableCsvColumn(BindableColumn, b.Relationship):
 
 
 class CsvTable(AbstractBindableTable[BindableCsvColumn]):
-    _basic_type_schema: dict[str, str]
+    _basic_type_schema: dict[Hashable, str]
     _csv_data: list[pd.DataFrame]
     _num_rows: int
 

relationalai/early_access/metamodel/rewrite/__init__.py

@@ -1,5 +1,7 @@
-from relationalai.semantics.metamodel.rewrite import Splinter, RewriteListTypes, GarbageCollectNodes, Flatten, \
-    DNFUnionSplitter, ExtractKeys, ExtractNestedLogicals, FDConstraints, flatten
+from relationalai.semantics.metamodel.rewrite import Flatten, \
+    DNFUnionSplitter, ExtractNestedLogicals, flatten
+from relationalai.semantics.lqp.rewrite import Splinter,  \
+    ExtractKeys, FDConstraints
 
-__all__ = ["Splinter", "RewriteListTypes", "GarbageCollectNodes", "Flatten", "DNFUnionSplitter", "ExtractKeys",
+__all__ = ["Splinter", "Flatten", "DNFUnionSplitter", "ExtractKeys",
            "ExtractNestedLogicals", "FDConstraints", "flatten"]

relationalai/early_access/rel/rewrite/__init__.py

@@ -1,4 +1,4 @@
-from relationalai.semantics.rel.rewrite import CDC, ExtractCommon, QuantifyVars
+from relationalai.semantics.lqp.rewrite import CDC, ExtractCommon, QuantifyVars
 
 __all__ = [
     "CDC",

relationalai/errors.py

@@ -2397,17 +2397,18 @@ class UnsupportedColumnTypesWarning(RAIWarning):
 """)
 
 class QueryTimeoutExceededException(RAIException):
-    def __init__(self, timeout_mins: int, config_file_path: str | None = None):
+    def __init__(self, timeout_mins: int, query_id: str | None = None, config_file_path: str | None = None):
         self.timeout_mins = timeout_mins
-        self.message = f"Query execution time exceeded the specified timeout of {timeout_mins} minutes."
         self.name = "Query Timeout Exceeded"
+        self.message = f"Query execution time exceeded the specified timeout of {self.timeout_mins} minutes."
+        self.query_id = query_id or ""
         self.config_file_path = config_file_path or ""
         self.content = self.format_message()
         super().__init__(self.message, self.name, self.content)
 
     def format_message(self):
         return textwrap.dedent(f"""
-        {self.message}
+        Query execution time exceeded the specified timeout of {self.timeout_mins} minutes{f' for query with ID: {self.query_id}' if self.query_id else ''}.
 
         Consider increasing the 'query_timeout_mins' parameter in your configuration file{f' (stored in {self.config_file_path})' if self.config_file_path else ''} to allow more time for query execution.
         """)
@@ -2432,3 +2433,23 @@ class AzureUnsupportedQueryTimeoutException(RAIException):
         Please remove the 'query_timeout_mins' from your configuration file{f' (stored in {self.config_file_path})' if self.config_file_path else ''} when running on platform Azure.
         """)
 
+class AzureLegacyDependencyMissingException(RAIException):
+    def __init__(self):
+        self.message = "The Azure platform requires the 'legacy' extras to be installed."
+        self.name = "Azure Legacy Dependency Missing"
+        self.content = self.format_message()
+        super().__init__(self.message, self.name, self.content)
+
+    def format_message(self):
+        return textwrap.dedent("""
+        The Azure platform requires the 'rai-sdk' package, which is not installed.
+
+        To use the Azure platform, please install the legacy extras:
+
+            pip install relationalai[legacy]
+
+        Or if upgrading an existing installation:
+
+            pip install --upgrade relationalai[legacy]
+        """)
+

relationalai/semantics/internal/annotations.py

@@ -5,3 +5,4 @@ external = Relationship.builtins["external"]
 concept_population = Relationship.builtins["concept_population"]
 function = Relationship.builtins["function"]
 from_cdc = Relationship.builtins["from_cdc"]
+track = Relationship.builtins["track"]

relationalai/semantics/internal/internal.py

@@ -514,11 +514,13 @@ class Producer:
     #--------------------------------------------------
 
     def in_(self, values:list[Any]|Fragment) -> Expression:
+        columns = None
         if isinstance(values, Fragment):
             return self == values
         if not isinstance(values[0], tuple):
             values = [tuple([v]) for v in values]
-        d = data(values)
+            columns = [f"v{i}" for i in range(len(values[0]))]
+        d = data(values, columns)
         return self == d[0]
 
     #--------------------------------------------------
@@ -907,9 +909,9 @@ class Concept(Producer):
                 if python_types_to_concepts.get(v):
                     v = python_types_to_concepts[v]
                 if isinstance(v, Concept):
-                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._name}}}", short_name=k, model=self._model))
+                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._name}}}", parent=self, short_name=k, model=self._model))
                 elif isinstance(v, type) and issubclass(v, self._model.Enum): #type: ignore
-                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._concept._name}}}", short_name=k, model=self._model))
+                    setattr(self, k, Property(f"{{{self._name}}} has {{{k}:{v._concept._name}}}", parent=self, short_name=k, model=self._model))
                 elif isinstance(v, Relationship):
                     self._validate_identifier_relationship(v)
                     setattr(self, k, v)
@@ -1189,6 +1191,7 @@ def is_decimal(concept: Concept) -> bool:
 Concept.builtins["Int"] = Concept.builtins["Int128"]
 Concept.builtins["Integer"] = Concept.builtins["Int128"]
 
+_np_datetime = np.dtype('datetime64[ns]')
 python_types_to_concepts : dict[Any, Concept] = {
     int: Concept.builtins["Int128"],
     float: Concept.builtins["Float"],
@@ -1213,6 +1216,7 @@ python_types_to_concepts : dict[Any, Concept] = {
     np.dtype('float32'): Concept.builtins["Float"],
     np.dtype('bool'): Concept.builtins["Bool"],
     np.dtype('object'): Concept.builtins["String"],  # Often strings are stored as object dtype
+    _np_datetime: Concept.builtins["DateTime"],
 
     # Pandas extension dtypes
     pd.Int64Dtype(): Concept.builtins["Int128"],
@@ -1655,7 +1659,9 @@ class Expression(Producer):
             raise ValueError(f"Argument index should be positive, got {idx}")
         if len(self._params) <= idx:
             raise ValueError(f"Expression '{self.__str__()}' has only {len(self._params)} arguments")
-        return ArgumentRef(self, self._params[idx])
+        param = self._params[idx]
+        # if param is an Expression then refer the last param of this expression
+        return ArgumentRef(self, param._params[-1] if isinstance(param, Expression) else param)
 
     def __getattr__(self, name: str):
         last = self._params[-1]
@@ -2090,8 +2096,20 @@ class DataColumn(Producer):
         self._data = data
         self._type = _type
         self._name = name if isinstance(name, str) else f"v{name}"
+        if pd.api.types.is_datetime64_any_dtype(_type):
+            _type = _np_datetime
+        # dates are objects in pandas
+        elif pd.api.types.is_object_dtype(_type) and self._is_date_column():
+            _type = date
         self._ref = python_types_to_concepts[_type].ref(self._name)
 
+    def _is_date_column(self) -> bool:
+        sample = self._data._data[self._name].dropna()
+        if sample.empty:
+            return False
+        sample_value = sample.iloc[0]
+        return isinstance(sample_value, date) and not isinstance(sample_value, datetime)
+
     def __str__(self):
         return f"DataColumn({self._name}, {self._type})"
 

relationalai/semantics/lqp/builtins.py

@@ -11,4 +11,5 @@ adhoc_annotation = f.annotation(adhoc, [])
 annotations_to_emit = FrozenOrderedSet([
     adhoc.name,
     builtins.function.name,
+    builtins.track.name,
 ])

relationalai/semantics/lqp/executor.py

@@ -12,6 +12,7 @@ from relationalai import debugging
 from relationalai.semantics.lqp import result_helpers
 from relationalai.semantics.metamodel import ir, factory as f, executor as e
 from relationalai.semantics.lqp.compiler import Compiler
+from relationalai.semantics.lqp.intrinsics import mk_intrinsic_datetime_now
 from relationalai.semantics.lqp.types import lqp_type_to_sql
 from lqp import print as lqp_print, ir as lqp_ir
 from lqp.parser import construct_configure
@@ -258,11 +259,47 @@ class LQPExecutor(e.Executor):
 
         return ", ".join(fields)
 
+    def _construct_configure(self):
+        config_dict = {}
+        # Only set the IVM flag if there is a value in `config`. Otherwise, let
+        # `construct_configure` set the default value.
+        ivm_flag = self.config.get('reasoner.rule.incremental_maintenance', None)
+        if ivm_flag:
+            config_dict['ivm.maintenance_level'] = lqp_ir.Value(value=ivm_flag, meta=None)
+        return construct_configure(config_dict, None)
+    
+    def _compile_intrinsics(self) -> lqp_ir.Epoch:
+        """Construct an epoch that defines a number of built-in definitions used by the
+        emitter."""
+        with debugging.span("compile_intrinsics") as span:
+            debug_info = lqp_ir.DebugInfo(id_to_orig_name={}, meta=None)
+            intrinsics_fragment = lqp_ir.Fragment(
+                id = lqp_ir.FragmentId(id=b"__pyrel_lqp_intrinsics", meta=None),
+                declarations = [
+                    mk_intrinsic_datetime_now(),
+                ],
+                debug_info = debug_info,
+                meta = None,
+            )
+
+            span["compile_type"] = "intrinsics"
+            span["lqp"] = lqp_print.to_string(intrinsics_fragment, {"print_names": True, "print_debug": False, "print_csv_filename": False})
+
+            return lqp_ir.Epoch(
+                writes=[
+                    lqp_ir.Write(write_type=lqp_ir.Define(fragment=intrinsics_fragment, meta=None), meta=None)
+                ],
+                meta=None,
+            )
+
     def compile_lqp(self, model: ir.Model, task: ir.Task):
+        configure = self._construct_configure()
+
         model_txn = None
         if self._last_model != model:
             with debugging.span("compile", metamodel=model) as install_span:
                 _, model_txn = self.compiler.compile(model, {"fragment_id": b"model"})
+                model_txn = txn_with_configure(model_txn, configure)
                 install_span["compile_type"] = "model"
                 install_span["lqp"] = lqp_print.to_string(model_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
                 self._last_model = model
@@ -275,23 +312,26 @@ class LQPExecutor(e.Executor):
             }
             result, final_model = self.compiler.compile_inner(query, options)
             export_info, query_txn = result
+            query_txn = txn_with_configure(query_txn, configure)
             compile_span["compile_type"] = "query"
             compile_span["lqp"] = lqp_print.to_string(query_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
 
-        txn = query_txn
+        # Merge the epochs into a single transactions. Long term the query bits should all
+        # go into a WhatIf action and the intrinsics could be fused with either of them. But
+        # for now we just use separate epochs.
+        epochs = []
+
+        epochs.append(self._compile_intrinsics())
+
         if model_txn is not None:
-            # Merge the two LQP transactions into one. Long term the query bits should all
-            # go into a WhatIf action. But for now we just use two separate epochs.
-            model_epoch = model_txn.epochs[0]
-            query_epoch = query_txn.epochs[0]
-            txn = lqp_ir.Transaction(
-                epochs=[model_epoch, query_epoch],
-                configure=construct_configure({}, None),
-                meta=None,
-            )
+            epochs.append(model_txn.epochs[0])
 
-            # Revalidate now that we've joined two epochs
-            validate_lqp(txn)
+        epochs.append(query_txn.epochs[0])
+
+        txn = lqp_ir.Transaction(epochs=epochs, configure=configure, meta=None)
+
+        # Revalidate now that we've joined all the epochs.
+        validate_lqp(txn)
 
         txn_proto = convert_transaction(txn)
         # TODO (azreika): Should export_info be encoded as part of the txn_proto? [RAI-40312]
@@ -352,3 +392,12 @@ class LQPExecutor(e.Executor):
             # If processing the results failed, revert to the previous model.
             self._last_model = previous_model
             raise e
+
+def txn_with_configure(txn: lqp_ir.Transaction, configure: lqp_ir.Configure) -> lqp_ir.Transaction:
+    """ Return a new transaction with the given configure. If the transaction already has
+    a configure, it is replaced. """
+    return lqp_ir.Transaction(
+        epochs=txn.epochs,
+        configure=configure,
+        meta=txn.meta,
+    )

relationalai/semantics/lqp/intrinsics.py

@@ -0,0 +1,23 @@
+from datetime import datetime, timezone
+
+from relationalai.semantics.lqp import ir as lqp
+from relationalai.semantics.lqp.constructors import mk_abstraction, mk_value, mk_var, mk_type, mk_primitive
+from relationalai.semantics.lqp.utils import lqp_hash
+
+def mk_intrinsic_datetime_now() -> lqp.Def:
+    """Constructs a definition of the current datetime."""
+    id = lqp_hash("__pyrel_lqp_intrinsic_datetime_now")
+    out = mk_var("out")
+    out_type = mk_type(lqp.TypeName.DATETIME)
+    now = mk_value(lqp.DateTimeValue(value=datetime.now(timezone.utc), meta=None))
+    datetime_now = mk_abstraction(
+        [(out, out_type)],
+        mk_primitive("rel_primitive_eq", [out, now]),
+    )
+
+    return lqp.Def(
+        name = lqp.RelationId(id=id, meta=None),
+        body = datetime_now,
+        attrs = [],
+        meta = None,
+    )

relationalai/semantics/lqp/model2lqp.py

@@ -192,12 +192,21 @@ def _translate_effect(ctx: TranslationCtx, effect: Union[ir.Output, ir.Update],
     elif isinstance(effect, ir.Output):
         ctx.output_ids.append((rel_id, def_name))
 
+    # First we collect annotations on the effect itself, e.g. from something like
+    # `select(...).annotate(...)`.
+    annotations = effect.annotations
+    if isinstance(effect, ir.Update):
+        # Then we translate annotations on the relation itself, e.g.
+        # ```
+        # Bar.foo = model.Relationship(...)
+        # Bar.foo.annotate(...)
+        # ```
+        annotations = annotations | effect.relation.annotations
+
     return lqp.Def(
         name = rel_id,
         body = mk_abstraction(projection, new_body),
-        # TODO this only covers the annotations on the effect itself. Annotations on the
-        # relation are not included yet.
-        attrs = _translate_annotations(effect.annotations),
+        attrs = _translate_annotations(annotations),
         meta = None,
     )
 
@@ -697,4 +706,4 @@ def _translate_join(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
 
     output_term = _translate_term(ctx, target)[0]
 
-    return lqp.Reduce(meta=None, op=op, body=body, terms=[output_term])
+    return lqp.Reduce(meta=None, op=op, body=body, terms=[output_term])

relationalai/semantics/lqp/passes.py

@@ -2,13 +2,12 @@ from relationalai.semantics.metamodel.compiler import Pass
 from relationalai.semantics.metamodel import ir, builtins as rel_builtins, factory as f, visitor
 from relationalai.semantics.metamodel.typer import Checker, InferTypes, typer
 from relationalai.semantics.metamodel import helpers, types
-from relationalai.semantics.metamodel.rewrite import (Splinter, ExtractNestedLogicals, ExtractKeys, FDConstraints,
-                                                      DNFUnionSplitter, DischargeConstraints)
 from relationalai.semantics.metamodel.util import FrozenOrderedSet
 
 from relationalai.semantics.metamodel.rewrite import Flatten
-# TODO: Move this into metamodel.rewrite
-from relationalai.semantics.rel.rewrite import QuantifyVars, CDC
+
+from ..metamodel.rewrite import DischargeConstraints, DNFUnionSplitter, ExtractNestedLogicals
+from .rewrite import CDC, ExtractCommon, ExtractKeys, FDConstraints, QuantifyVars, Splinter
 
 from relationalai.semantics.lqp.utils import output_names
 
@@ -25,8 +24,7 @@ def lqp_passes() -> list[Pass]:
         InferTypes(),
         DNFUnionSplitter(),
         ExtractKeys(),
-        # Broken
-        # ExtractCommon(),
+        ExtractCommon(),
         Flatten(),
         Splinter(), # Splits multi-headed rules into multiple rules
         QuantifyVars(), # Adds missing existentials

relationalai/semantics/lqp/primitives.py

@@ -1,7 +1,7 @@
 from relationalai.semantics.metamodel.types import digits_to_bits
 from relationalai.semantics.lqp import ir as lqp
 from relationalai.semantics.lqp.types import is_numeric
-from relationalai.semantics.lqp.utils import UniqueNames
+from relationalai.semantics.lqp.utils import UniqueNames, lqp_hash
 from relationalai.semantics.lqp.constructors import mk_primitive, mk_specialized_value, mk_type, mk_value, mk_var
 
 rel_to_lqp = {
@@ -62,6 +62,7 @@ rel_to_lqp = {
     "date_add": "rel_primitive_typed_add_date_period",
     "date_subtract": "rel_primitive_typed_subtract_date_period",
     "dates_period_days": "rel_primitive_date_days_between",
+    "datetime_now": "__pyrel_lqp_intrinsic_datetime_now",
     "datetime_add": "rel_primitive_typed_add_datetime_period",
     "datetime_subtract": "rel_primitive_typed_subtract_datetime_period",
     "datetime_year": "rel_primitive_datetime_year",
@@ -175,6 +176,16 @@ def build_primitive(
     terms, term_types = _reorder_primitive_terms(lqp_name, terms, term_types)
     _assert_primitive_terms(lqp_name, terms, term_types)
 
+    # Handle intrinsics. To callers of `build_primitive` the distinction between intrinsic
+    # and primitive doesn't matter, so we don't want to burden them with that detail.
+    # Intrinsics are built-in definitions added by the LQP emitter, that user logic can just
+    # refer to.
+    if lqp_name == "__pyrel_lqp_intrinsic_datetime_now":
+        id = lqp.RelationId(id=lqp_hash(lqp_name), meta=None)
+        assert len(terms) == 1
+        assert isinstance(terms[0], lqp.Term)
+        return lqp.Atom(name=id, terms=[terms[0]], meta=None)
+
     return mk_primitive(lqp_name, terms)
 
 def relname_to_lqp_name(name: str) -> str:

relationalai/semantics/{rel → lqp}/rewrite/__init__.py

@@ -1,9 +1,15 @@
 from .cdc import CDC
 from .extract_common import ExtractCommon
+from .extract_keys import ExtractKeys
+from .fd_constraints import FDConstraints
 from .quantify_vars import QuantifyVars
+from .splinter import Splinter
 
 __all__ = [
     "CDC",
     "ExtractCommon",
+    "ExtractKeys",
+    "FDConstraints",
     "QuantifyVars",
+    "Splinter",
 ]