relationalai 0.12.8__py3-none-any.whl → 0.12.10__py3-none-any.whl

relationalai/clients/snowflake.py

@@ -441,7 +441,8 @@ class Resources(ResourcesBase):
         code: str,
         params: List[Any] | Any | None = None,
         raw: bool = False,
-        help: bool = True
+        help: bool = True,
+        skip_auto_create: bool = False
     ) -> Any:
         # print(f"\n--- sql---\n{code}\n--- end sql---\n")
         if not self._session:
@@ -458,7 +459,6 @@ class Resources(ResourcesBase):
             rai_app = self.config.get("rai_app_name", "")
             current_role = self.config.get("role")
             engine = self.get_default_engine_name()
-            engine_size = self.config.get_default_engine_size()
             assert isinstance(rai_app, str), f"rai_app_name must be a string, not {type(rai_app)}"
             assert isinstance(engine, str), f"engine must be a string, not {type(engine)}"
             print("\n")
@@ -467,15 +467,10 @@ class Resources(ResourcesBase):
             if re.search(f"database '{rai_app}' does not exist or not authorized.".lower(), orig_message):
                 exception = SnowflakeAppMissingException(rai_app, current_role)
                 raise exception from None
-            if _is_engine_issue(orig_message) or _is_database_issue(orig_message):
+            # skip creating the engine if the query is a user transaction. exec_async_v2 will handle that case.
+            if _is_engine_issue(orig_message) and not skip_auto_create:
                 try:
-                    self._poll_use_index(
-                        app_name=self.get_app_name(),
-                        sources=self.sources,
-                        model=self.database,
-                        engine_name=engine,
-                        engine_size=engine_size
-                    )
+                    self.auto_create_engine(engine)
                     return self._exec(code, params, raw=raw, help=help)
                 except EngineNameValidationException as e:
                     raise EngineNameValidationException(engine) from e
@@ -1612,6 +1607,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
         response = self._exec(
             sql_string,
             raw_code,
+            skip_auto_create=True,
         )
         if not response:
             raise Exception("Failed to create transaction")
@@ -1629,6 +1625,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
         bypass_index=False,
         language: str = "rel",
         query_timeout_mins: int | None = None,
+        gi_setup_skipped: bool = False,
     ):
         if inputs is None:
             inputs = {}
@@ -1638,6 +1635,8 @@ Otherwise, remove it from your '{profile}' configuration profile.
         with debugging.span("transaction", **query_attrs_dict) as txn_span:
             with debugging.span("create_v2", **query_attrs_dict) as create_span:
                 request_headers['user-agent'] = get_pyrel_version(self.generation)
+                request_headers['gi_setup_skipped'] = str(gi_setup_skipped)
+                request_headers['pyrel_program_id'] = debugging.get_program_span_id() or ""
                 response = self._exec_rai_app(
                     database=database,
                     engine=engine,
@@ -1897,26 +1896,29 @@ Otherwise, remove it from your '{profile}' configuration profile.
     # Exec
     #--------------------------------------------------
 
-    def exec_lqp(
+    def _exec_with_gi_retry(
         self,
         database: str,
         engine: str | None,
-        raw_code: bytes,
-        readonly=True,
-        *,
-        inputs: Dict | None = None,
-        nowait_durable=False,
-        headers: Dict | None = None,
-        bypass_index=False,
-        query_timeout_mins: int | None = None,
+        raw_code: str,
+        inputs: Dict | None,
+        readonly: bool,
+        nowait_durable: bool,
+        headers: Dict | None,
+        bypass_index: bool,
+        language: str,
+        query_timeout_mins: int | None,
     ):
-        raw_code_b64 = base64.b64encode(raw_code).decode("utf-8")
+        """Execute with graph index retry logic.
 
+        Attempts execution with gi_setup_skipped=True first. If an engine or database
+        issue occurs, polls use_index and retries with gi_setup_skipped=False.
+        """
         try:
             return self._exec_async_v2(
-                database, engine, raw_code_b64, inputs, readonly, nowait_durable,
-                headers=headers, bypass_index=bypass_index, language='lqp',
-                query_timeout_mins=query_timeout_mins,
+                database, engine, raw_code, inputs, readonly, nowait_durable,
+                headers=headers, bypass_index=bypass_index, language=language,
+                query_timeout_mins=query_timeout_mins, gi_setup_skipped=True,
             )
         except Exception as e:
             err_message = str(e).lower()
@@ -1933,13 +1935,32 @@ Otherwise, remove it from your '{profile}' configuration profile.
                 )
 
                 return self._exec_async_v2(
-                    database, engine, raw_code_b64, inputs, readonly, nowait_durable,
-                    headers=headers, bypass_index=bypass_index, language='lqp',
-                    query_timeout_mins=query_timeout_mins,
+                    database, engine, raw_code, inputs, readonly, nowait_durable,
+                    headers=headers, bypass_index=bypass_index, language=language,
+                    query_timeout_mins=query_timeout_mins, gi_setup_skipped=False,
                 )
             else:
                 raise e
 
+    def exec_lqp(
+        self,
+        database: str,
+        engine: str | None,
+        raw_code: bytes,
+        readonly=True,
+        *,
+        inputs: Dict | None = None,
+        nowait_durable=False,
+        headers: Dict | None = None,
+        bypass_index=False,
+        query_timeout_mins: int | None = None,
+    ):
+        raw_code_b64 = base64.b64encode(raw_code).decode("utf-8")
+        return self._exec_with_gi_retry(
+            database, engine, raw_code_b64, inputs, readonly, nowait_durable,
+            headers, bypass_index, 'lqp', query_timeout_mins
+        )
+
 
     def exec_raw(
         self,
@@ -1955,45 +1976,10 @@ Otherwise, remove it from your '{profile}' configuration profile.
         query_timeout_mins: int | None = None,
     ):
         raw_code = raw_code.replace("'", "\\'")
-
-        try:
-            return self._exec_async_v2(
-                database,
-                engine,
-                raw_code,
-                inputs,
-                readonly,
-                nowait_durable,
-                headers=headers,
-                bypass_index=bypass_index,
-                query_timeout_mins=query_timeout_mins,
-            )
-        except Exception as e:
-            err_message = str(e).lower()
-            if _is_engine_issue(err_message) or _is_database_issue(err_message):
-                engine_name = engine or self.get_default_engine_name()
-                engine_size = self.config.get_default_engine_size()
-                self._poll_use_index(
-                    app_name=self.get_app_name(),
-                    sources=self.sources,
-                    model=database,
-                    engine_name=engine_name,
-                    engine_size=engine_size,
-                    headers=headers,
-                )
-                return self._exec_async_v2(
-                    database,
-                    engine,
-                    raw_code,
-                    inputs,
-                    readonly,
-                    nowait_durable,
-                    headers=headers,
-                    bypass_index=bypass_index,
-                    query_timeout_mins=query_timeout_mins,
-                )
-            else:
-                raise e
+        return self._exec_with_gi_retry(
+            database, engine, raw_code, inputs, readonly, nowait_durable,
+            headers, bypass_index, 'rel', query_timeout_mins
+        )
 
 
     def format_results(self, results, task:m.Task|None=None) -> Tuple[DataFrame, List[Any]]:
@@ -3314,19 +3300,10 @@ class DirectAccessResources(Resources):
                         message = ""  # Not used when we check status_code directly
 
                     # fix engine on engine error and retry
-                    # Skip auto-retry if skip_auto_create is True to avoid recursion
-                    if (_is_engine_issue(message) and not skip_auto_create) or _is_database_issue(message):
-                        engine_name = payload.get("caller_engine_name", "") if payload else ""
-                        engine_name = engine_name or self.get_default_engine_name()
-                        engine_size = self.config.get_default_engine_size()
-                        self._poll_use_index(
-                            app_name=self.get_app_name(),
-                            sources=self.sources,
-                            model=self.database,
-                            engine_name=engine_name,
-                            engine_size=engine_size,
-                            headers=headers,
-                        )
+                    # Skip auto-retry if skip_auto_create is True to avoid recursion or to let _exec_async_v2 poll the index.
+                    if _is_engine_issue(message) and not skip_auto_create:
+                        engine = payload.get("engine_name", "") if payload else ""
+                        self.auto_create_engine(engine)
                         response = _send_request()
             except requests.exceptions.ConnectionError as e:
                 if "NameResolutionError" in str(e):
@@ -3340,6 +3317,48 @@ class DirectAccessResources(Resources):
                 raise e
             return response
 
+    def _txn_request_with_gi_retry(
+        self,
+        payload: Dict,
+        headers: Dict[str, str],
+        query_params: Dict,
+        engine: Union[str, None],
+    ):
+        """Make request with graph index retry logic.
+
+        Attempts request with gi_setup_skipped=True first. If an engine or database
+        issue occurs, polls use_index and retries with gi_setup_skipped=False.
+        """
+        response = self.request(
+            "create_txn", payload=payload, headers=headers, query_params=query_params, skip_auto_create=True
+        )
+
+        if response.status_code != 200:
+            try:
+                message = response.json().get("message", "")
+            except requests.exceptions.JSONDecodeError:
+                message = ""
+
+            if _is_engine_issue(message) or _is_database_issue(message):
+                engine_name = engine or self.get_default_engine_name()
+                engine_size = self.config.get_default_engine_size()
+                self._poll_use_index(
+                    app_name=self.get_app_name(),
+                    sources=self.sources,
+                    model=self.database,
+                    engine_name=engine_name,
+                    engine_size=engine_size,
+                    headers=headers,
+                )
+                headers['gi_setup_skipped'] = 'False'
+                response = self.request(
+                    "create_txn", payload=payload, headers=headers, query_params=query_params, skip_auto_create=True
+                )
+            else:
+                raise ResponseStatusException("Failed to create transaction.", response)
+
+        return response
+
     def _exec_async_v2(
         self,
         database: str,
@@ -3352,6 +3371,7 @@ class DirectAccessResources(Resources):
         bypass_index=False,
         language: str = "rel",
         query_timeout_mins: int | None = None,
+        gi_setup_skipped: bool = False,
     ):
 
         with debugging.span("transaction") as txn_span:
@@ -3374,12 +3394,15 @@ class DirectAccessResources(Resources):
                     payload["timeout_mins"] = query_timeout_mins
                 query_params={"use_graph_index": str(use_graph_index and not bypass_index)}
 
-                response = self.request(
-                    "create_txn", payload=payload, headers=headers, query_params=query_params,
-                )
+                # Add gi_setup_skipped to headers
+                if headers is None:
+                    headers = {}
+                headers["gi_setup_skipped"] = str(gi_setup_skipped)
+                headers['pyrel_program_id'] = debugging.get_program_span_id() or ""
 
-                if response.status_code != 200:
-                    raise ResponseStatusException("Failed to create transaction.", response)
+                response = self._txn_request_with_gi_retry(
+                    payload, headers, query_params, engine
+                )
 
                 artifact_info = {}
                 response_content = response.json()

relationalai/debugging.py

@@ -26,6 +26,9 @@ find_block_in = find_block_in # re-export
 DEBUG = True
 handled_error = None
 
+# Configurable debug log file location
+DEBUG_LOG_FILE = os.environ.get('RAI_DEBUG_LOG', 'debug.jsonl')
+
 #--------------------------------------------------
 # Log Formatters
 #--------------------------------------------------
@@ -70,7 +73,7 @@ class FlushingFileHandler(logging.FileHandler):
     def emit(self, record):
         if not self._initialized:
             self._initialized = True
-            with open('debug.jsonl', 'w'):
+            with open(DEBUG_LOG_FILE, 'w'):
                 pass
         super().emit(record)
         self.flush()
@@ -78,7 +81,7 @@ class FlushingFileHandler(logging.FileHandler):
 try:
     # keep the old file-based debugger around and working until it's fully replaced.
     if DEBUG:
-        file_handler = FlushingFileHandler('debug.jsonl', mode='a')
+        file_handler = FlushingFileHandler(DEBUG_LOG_FILE, mode='a')
         file_handler.setFormatter(JsonFormatter())
         logger.addHandler(file_handler)
 except Exception:

relationalai/semantics/__init__.py

@@ -8,7 +8,7 @@ __include_in_docs__ = True
 
 from .internal import (
     Model, Concept, Relationship, RelationshipReading, Expression, Fragment, Error, Field,
-    String, Integer, Int64, Int128, Float, Decimal, Bool,
+    AnyEntity, String, Integer, Int64, Int128, Float, Decimal, Bool,
     Date, DateTime,
     RawSource, Hash,
     select, where, require, define, distinct, union, data,
@@ -19,7 +19,7 @@ from .internal import (
 
 __all__ = [
     "Model", "Concept", "Relationship", "RelationshipReading", "Expression", "Fragment", "Error", "Field",
-    "String", "Integer", "Int64", "Int128", "Float", "Decimal", "Bool",
+    "AnyEntity", "String", "Integer", "Int64", "Int128", "Float", "Decimal", "Bool",
     "Date", "DateTime",
     "RawSource", "Hash",
     "select", "where", "require", "define", "distinct", "union", "data",

relationalai/semantics/internal/__init__.py

@@ -4,7 +4,7 @@ API for RelationalAI.
 
 from .internal import (
     Model, Concept, Relationship, RelationshipReading, Expression, Fragment, Error, Field,
-    String, Integer, Int64, Int128, Float, Decimal, Bool,
+    AnyEntity, String, Integer, Int64, Int128, Float, Decimal, Bool,
     Date, DateTime,
     RawSource, Hash,
     select, where, require, define, distinct, union, data,
@@ -15,7 +15,7 @@ from .internal import (
 
 __all__ = [
     "Model", "Concept", "Relationship", "RelationshipReading", "Expression", "Fragment", "Error", "Field",
-    "String", "Integer", "Int64", "Int128", "Float", "Decimal", "Bool",
+    "AnyEntity", "String", "Integer", "Int64", "Int128", "Float", "Decimal", "Bool",
     "Date", "DateTime",
     "RawSource", "Hash",
     "select", "where", "require", "define", "distinct", "union", "data",

relationalai/semantics/internal/internal.py

@@ -40,7 +40,7 @@ _global_id = peekable(itertools.count(0))
 
 # Single context variable with default values
 _overrides = ContextVar("overrides", default = {})
-def overrides(key: str, default: bool | str | dict):
+def overrides(key: str, default: bool | str | dict | datetime | None):
     return _overrides.get().get(key, default)
 
 # Flag that users set in the config or directly on the model, but that can still be
@@ -60,6 +60,13 @@ def with_overrides(**kwargs):
     finally:
         _overrides.reset(token)
 
+# Intrinsic values to override for stable snapshots.
+def get_intrinsic_overrides() -> dict[str, Any]:
+    datetime_now = overrides('datetime_now', None)
+    if datetime_now is not None:
+        return {'datetime_now': datetime_now}
+    return {}
+
 #--------------------------------------------------
 # Root tracking
 #--------------------------------------------------
@@ -614,7 +621,7 @@ class Producer:
 
                 if self._model and self._model._strict:
                     raise AttributeError(f"{self._name} has no relationship `{name}`")
-                if topmost_parent is not concept:
+                if topmost_parent is not concept and topmost_parent not in Concept.builtin_concepts:
                     topmost_parent._relationships[name] = topmost_parent._get_relationship(name)
                     rich.print(f"[red bold][Implicit Subtype Relationship][/red bold] [yellow]{concept}.{name}[/yellow] appended to topmost parent [yellow]{topmost_parent}[/yellow] instead")
 
@@ -953,12 +960,25 @@ class Concept(Producer):
                 self._validate_identifier_relationship(rel)
         self._add_ref_scheme(*args)
 
-    def _add_ref_scheme(self, *args: Relationship|RelationshipReading):
-        self._reference_schemes.append(args)
-        # assumed that all Relationship|RelationshipReading are defined on the identified Concept
-        fields = tuple([arg.__getitem__(0) for arg in args])
-        uc = Unique(*fields, model=self._model)
-        require(uc.to_expressions())
+    def _add_ref_scheme(self, *rels: Relationship|RelationshipReading):
+        # thanks to prior validation we we can safely assume that
+        # * the input types are correct due to prior validation
+        # * all relationships are binary and defined on this concept
+
+        self._reference_schemes.append(rels)
+
+        # for every concept x every field f has at most one value y.
+        # f(x,y): x -> y holds
+        concept_fields = tuple([rel.__getitem__(0) for rel in rels])
+        for field in concept_fields:
+            concept_uc = Unique(field, model=self._model)
+            require(concept_uc.to_expressions())
+
+        # for any combination of field values there is at most one concept x.
+        # f₁(x,y₁) ∧ … ∧ fₙ(x,yₙ): {y₁,…,yₙ} → {x}
+        key_fields = tuple([rel.__getitem__(1) for rel in rels])
+        key_uc = Unique(*key_fields, model=self._model)
+        require(key_uc.to_expressions())
 
     def _validate_identifier_relationship(self, rel:Relationship|RelationshipReading):
         if rel._arity() != 2:
@@ -1145,7 +1165,10 @@ Primitive = Concept.builtins["Primitive"] = Concept("Primitive")
 Error = Concept.builtins["Error"] = ErrorConcept("Error")
 
 def _register_builtin(name):
-    c = Concept(name, extends=[Primitive])
+    if name == "AnyEntity":
+        c = Concept(name)
+    else:
+        c = Concept(name, extends=[Primitive])
     Concept.builtin_concepts.add(c)
     Concept.builtins[name] = c
 
@@ -1154,6 +1177,7 @@ for builtin in types.builtin_types:
     if isinstance(builtin, ir.ScalarType):
         _register_builtin(builtin.name)
 
+AnyEntity = Concept.builtins["AnyEntity"]
 Float = Concept.builtins["Float"]
 Number = Concept.builtins["Number"]
 Int64 = Concept.builtins["Int64"]
@@ -2603,6 +2627,7 @@ class Model():
         config_overrides = overrides('config', {})
         for k, v in config_overrides.items():
             self._config.set(k, v)
+        self._intrinsic_overrides = get_intrinsic_overrides()
         self._strict = cast(bool, overrides('strict', strict))
         self._use_lqp = overridable_flag('reasoner.rule.use_lqp', self._config, use_lqp, default=not self._use_sql)
         self._enable_otel_handler = overridable_flag('enable_otel_handler', self._config, enable_otel_handler, default=False)
@@ -2644,6 +2669,7 @@ class Model():
                     wide_outputs=self._wide_outputs,
                     connection=self._connection,
                     config=self._config,
+                    intrinsic_overrides=self._intrinsic_overrides,
                 )
             elif self._use_sql:
                 self._executor = SnowflakeExecutor(
@@ -2874,10 +2900,9 @@ class Compiler():
                 if concept not in self.types:
                     self.to_type(concept)
                     self.to_relation(concept)
-                if concept._extends:
-                    rule = self.concept_inheritance_rule(concept)
-                    if rule:
-                        rules.append(rule)
+                rule = self.concept_inheritance_rule(concept)
+                if rule:
+                    rules.append(rule)
         unresolved = []
         for relationship in model.relationships:
             if relationship not in self.relations:
@@ -3182,8 +3207,11 @@ class Compiler():
         # filter extends to get only non-primitive parents
         parents = []
         for parent in concept._extends:
-            if not parent._is_primitive():
+            if not parent._is_primitive() and parent is not AnyEntity:
                 parents.append(parent)
+        # always extend AnyEntity for non-primitive types that are not built-in
+        if not concept._is_primitive() and concept not in Concept.builtin_concepts:
+            parents.append(AnyEntity)
         # only extends primitive types, no need for inheritance rules
         if not parents:
             return None
@@ -3196,6 +3224,17 @@ class Compiler():
             *[f.derive(self.to_relation(parent), [var]) for parent in parents]
         ])
 
+    def concept_any_entity_rule(self, entities:list[Concept]):
+        """
+        Generate an inheritance rule for all these entities to AnyEntity.
+        """
+        any_entity_relation = self.to_relation(AnyEntity)
+        var = f.var("v", types.Any)
+        return f.logical([
+            f.union([f.lookup(self.to_relation(e), [var]) for e in entities]),
+            f.derive(any_entity_relation, [var])
+        ])
+
     def relation_dict(self, items:dict[Relationship|Concept, Producer], ctx:CompilerContext) -> dict[ir.Relation, list[ir.Var]]:
         return {self.to_relation(k): unwrap_list(self.lookup(v, ctx)) for k, v in items.items()}
 

relationalai/semantics/lqp/README.md

@@ -0,0 +1,34 @@
+# Logic Engine LQP Backend
+
+The logic engine runs the *Logical Query Protocol* (short *LQP*). This module includes a
+compiler from the semantic metamodel to LQP along with an executor.
+
+## Running against a local logic engine
+
+For development and testing, it is possible to run PyRel models against a local logic engine
+server process.
+
+To start your local server, please refer to the [logic engine
+docs](https://github.com/RelationalAI/raicode/tree/master/src/Server#starting-the-server).
+
+With the local server running, add this to your `raiconfig.toml`:
+
+```toml
+[profile.local]
+platform = "local"
+engine = "local"
+host = "localhost"
+port = 8010
+```
+
+Then set `active_profile = "local"` at the top of the file.
+
+**Known limitations:**
+
+Local execution does not support running against Snowflake source tables.
+
+At the moment, locally created databases cannot be cleaned up by the client. Eventually you
+will need to clear your local pager directory.
+
+At the moment, local execution is only supported for fast-path transactions, i.e. those
+which complete in less than 5 seconds. Polling support will be added soon.

relationalai/semantics/lqp/compiler.py

@@ -14,7 +14,7 @@ class Compiler(c.Compiler):
         super().__init__(lqp_passes())
         self.def_names = UniqueNames()
 
-    def do_compile(self, model: ir.Model, options:dict={}) -> tuple[Optional[tuple], lqp.Transaction]:
+    def do_compile(self, model: ir.Model, options:dict={}) -> tuple[Optional[tuple], lqp.Epoch]:
         fragment_id: bytes = options.get("fragment_id", bytes(404))
         # Reset the var context for each compilation
         # TODO: Change to unique var names per lookup

relationalai/semantics/lqp/constructors.py

@@ -59,3 +59,10 @@ def mk_pragma(name: str, terms: list[lqp.Var]) -> lqp.Pragma:
 
 def mk_attribute(name: str, args: list[lqp.Value]) -> lqp.Attribute:
     return lqp.Attribute(name=name, args=args, meta=None)
+
+def mk_transaction(
+    epochs: list[lqp.Epoch],
+    configure: lqp.Configure = lqp.construct_configure({}, None),
+    sync = None
+) -> lqp.Transaction:
+    return lqp.Transaction(epochs=epochs, configure=configure, sync=sync, meta=None)