relationalai 0.13.4__py3-none-any.whl → 0.13.5__py3-none-any.whl

relationalai/semantics/reasoners/optimization/solvers_pb.py

@@ -14,9 +14,11 @@ import textwrap
 import time
 import uuid
 from typing import Any, Optional
+from concurrent.futures import ThreadPoolExecutor
 
 from relationalai.experimental.solvers import Solver
 from relationalai.semantics.internal import internal as b
+from relationalai.semantics import std
 from relationalai.semantics.rel.executor import RelExecutor
 from relationalai.tools.constants import DEFAULT_QUERY_TIMEOUT_MINS
 from relationalai.util.timeout import calc_remaining_timeout_minutes
@@ -126,6 +128,9 @@ class SolverModelPB:
         self._model = model
         self._num_type = num_type
         self._id = next(b._global_id)
+        self._model_id = str(uuid.uuid4()).upper().replace('-', '_')
+        self._load_point_counter = 0
+
         # Maps relationships to their corresponding variable concepts
         self._variable_relationships: dict[b.Relationship, b.Concept] = {}
         prefix_uppercase = f"SolverModel_{self._id}_"
@@ -160,6 +165,17 @@ class SolverModelPB:
             f"{{var}} has {{value:{result_type}}}",
             short_name=(prefix_lowercase + "point"),
         )
+        # TODO (dba) Remove in favor of Loopy when possible.
+
+        # This is a helper relation that allows us to model
+        # `.load_point()` behavior. Results are loaded from SF views
+        # and we cannot swap which solution `.point` currently points
+        # to. So we keep a trace of updates. Each time a user calls
+        # `.load_point()` we derive a new tuple into this relation and
+        # use `max` to select the latest update.
+        self._point_updates = model.Property(
+            "at {update:int} we load {point:int}"
+        )
         self.points = model.Property(
             f"point {{i:int}} for {{var}} has {{value:{result_type}}}",
             short_name=(prefix_lowercase + "points"),
@@ -190,7 +206,6 @@ class SolverModelPB:
         """
         b.define(b.RawSource("rel", textwrap.dedent(install_rel)))
 
-
     # -------------------------------------------------------------------------
     # Variable Handling
     # -------------------------------------------------------------------------
@@ -601,101 +616,69 @@ class SolverModelPB:
 
         executor.execute_raw(export_rel, query_timeout_mins=query_timeout_mins)
 
-    def _import_solver_results_from_csv(
-        self,
-        model_id: str,
-        executor: RelExecutor,
-        prefix_lowercase: str,
-        query_timeout_mins: Optional[int] = None
-    ) -> None:
-        """Import solver results from CSV files in Snowflake stage.
-
-        Loads and extracts CSV files in a single transaction to minimize overhead.
-
-        Args:
-            model_id: Unique model identifier for stage paths.
-            executor: RelExecutor instance.
-            prefix_lowercase: Prefix for relation names.
-            query_timeout_mins: Query timeout in minutes.
-        """
-        result_stage_base = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/SOLVERS/job_{model_id}/results"
-        value_parse_fn = "parse_int" if self._num_type == "int" else "parse_float"
-
-        # Single transaction: Load CSV files and extract/map results
-        # Use inline definitions to avoid needing declared relations
-        load_and_extract_rel = textwrap.dedent(f"""
-        // Define CSV loading inline (no declare needed)
-        // Load ancillary.csv - contains solver metadata (NAME, VALUE columns)
-        def ancillary_config[:path]: "{result_stage_base}/ancillary.csv.gz"
-        def ancillary_config[:syntax, :header_row]: 1
-        def ancillary_config[:schema, :NAME]: "string"
-        def ancillary_config[:schema, :VALUE]: "string"
-        def {prefix_lowercase}solver_ancillary_raw {{load_csv[ancillary_config]}}
-
-        // Load objective_values.csv - contains objective values (SOL_INDEX, VALUE columns)
-        def objective_values_config[:path]: "{result_stage_base}/objective_values.csv.gz"
-        def objective_values_config[:syntax, :header_row]: 1
-        def objective_values_config[:schema, :SOL_INDEX]: "string"
-        def objective_values_config[:schema, :VALUE]: "string"
-        def {prefix_lowercase}solver_objective_values_raw {{load_csv[objective_values_config]}}
-
-        // Load points.csv.gz - contains solution points (SOL_INDEX, VAR_HASH, VALUE columns)
-        def points_config[:path]: "{result_stage_base}/points.csv.gz"
-        def points_config[:syntax, :header_row]: 1
-        def points_config[:schema, :SOL_INDEX]: "string"
-        def points_config[:schema, :VAR_HASH]: "string"
-        def points_config[:schema, :VALUE]: "string"
-        def {prefix_lowercase}solver_points_raw {{load_csv[points_config]}}
-
-        // Clear existing result data
-        def delete[:{self.result_info._name}]: {self.result_info._name}
-        def delete[:{self.point._name}]: {self.point._name}
-        def delete[:{self.points._name}]: {self.points._name}
-
-        // Extract ancillary data (result info) - NAME and VALUE columns
-        def insert(:{self.result_info._name}, key, val): {{
-            exists((row) |
-                {prefix_lowercase}solver_ancillary_raw(:NAME, row, key) and
-                {prefix_lowercase}solver_ancillary_raw(:VALUE, row, val))
-        }}
-
-        // Extract objective value from objective_values CSV (first solution)
-        def insert(:{self.result_info._name}, "objective_value", val): {{
-            exists((row) |
-                {prefix_lowercase}solver_objective_values_raw(:SOL_INDEX, row, "1") and
-                {prefix_lowercase}solver_objective_values_raw(:VALUE, row, val))
-        }}
-
-        // Extract solution points from points.csv.gz into points property
-        // This file has SOL_INDEX, VAR_HASH, VALUE columns
-        // Convert CSV string index to Int128 for points property signature
-        // Convert value to Int128 (for int) or Float64 (for float)
-        def insert(:{self.points._name}, sol_idx_int128, var, val_converted): {{
-            exists((row, sol_idx_str, var_hash_str, val_str, sol_idx_int, val) |
-                {prefix_lowercase}solver_points_raw(:SOL_INDEX, row, sol_idx_str) and
-                {prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
-                {prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
-                parse_int(sol_idx_str, sol_idx_int) and
-                parse_uuid(var_hash_str, var) and
-                {value_parse_fn}(val_str, val) and
-                ::std::mirror::convert(std::mirror::typeof[Int128], sol_idx_int, sol_idx_int128) and
-                {'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
-        }}
-
-        // Extract first solution into point property (default solution)
-        // Filter to SOL_INDEX = 1
-        def insert(:{self.point._name}, var, val_converted): {{
-            exists((row, var_hash_str, val_str, val) |
-                {prefix_lowercase}solver_points_raw(:SOL_INDEX, row, "1") and
-                {prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
-                {prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
-                parse_uuid(var_hash_str, var) and
-                {value_parse_fn}(val_str, val) and
-                {'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
-        }}
-        """)
-
-        executor.execute_raw(load_and_extract_rel, query_timeout_mins=query_timeout_mins)
+    def _import_solver_results_from_views(self) -> None:
+        from relationalai.semantics.internal.snowflake import Table
+
+        resources = self._model._to_executor().resources
+        app_name = resources.get_app_name()
+        logic_engine_name = resources.get_default_engine_name()
+        model_id = self._model_id
+
+        query = f"select {resources.get_app_name()}.api.get_index_db_name(?)"
+
+        # To prevent sql injection, pass the variable as a separate argument to the executor
+        index_db_name = resources._exec(query, [self._model.name])[0][0]
+
+        # Result tables.
+        ancillary_tbl = Table(f"{resources.get_app_name()}.RESULTS.SOLVERS_{model_id}_ANCILLARY")
+        obj_vals_tbl = Table(f"{resources.get_app_name()}.RESULTS.SOLVERS_{model_id}_OBJECTIVE_VALUES")
+        points_tbl = Table(f"{resources.get_app_name()}.RESULTS.SOLVERS_{model_id}_POINTS")
+
+        # Skip cdc as we already make sure the tables are loaded.
+        ancillary_tbl._skip_cdc = True
+        obj_vals_tbl._skip_cdc = True
+        points_tbl._skip_cdc = True
+
+        # Load all tables into the GI database in parallel.
+        load_statements = []
+        for tbl in [ancillary_tbl, obj_vals_tbl, points_tbl]:
+            rel_name = tbl._fqn.replace(".", "_").lower()
+            load_statements.append(f"call {app_name}.api.load_data('{index_db_name}', '{logic_engine_name}', '{rel_name}', '{tbl._fqn}', '')")
+
+        with ThreadPoolExecutor(max_workers=3) as executor:
+            futures = [executor.submit(resources._exec, stmt) for stmt in load_statements]
+            for future in futures:
+                future.result()
+
+        NumericType = b.Integer if self._num_type == "int" else b.Float
+
+        # Metadata.
+        key, val = b.String.ref(), b.String.ref()
+        b.where(ancillary_tbl.name == key, ancillary_tbl.value == val).define(
+            self.result_info(key, val)
+               )
+
+        b.where(obj_vals_tbl.sol_index == 1, std.strings.string(obj_vals_tbl.value) == val).define(
+            self.result_info("objective_value", val)
+               )
+
+        # Points.
+        val_ref = NumericType.ref()
+        b.where(
+            points_tbl.sol_index == b.Integer,
+            points_tbl.var_hash == b.String,
+            points_tbl.value == val_ref,
+               ).define(self.points(b.Integer, std.parse_uuid(b.String), val_ref))
+
+        # Default to loading solution `1`.
+        b.define(self._point_updates(self._load_point_counter, 1))
+
+        # Derive into `point` the latest user selection from
+        # `_point_updates`.
+        point_index = b.Integer.ref()
+        b.where(self._point_updates(b.Integer, point_index), b.Integer == b.max(b.Integer),
+                self.points(point_index, b.Hash, val_ref))\
+         .define(self.point(b.Hash, val_ref))
 
     def _export_model_to_protobuf(
         self,
@@ -805,9 +788,11 @@ class SolverModelPB:
             **kwargs: Solver options and parameters.
         """
 
+        self._solver = solver
+
         use_csv_store = solver.engine_settings.get("store", {})\
                                         .get("csv", {})\
-                                        .get("enabled", False)
+                                        .get("enabled", True)
 
         print(f"Using {'csv' if use_csv_store else 'protobuf'} store...")
 
@@ -853,15 +838,13 @@ class SolverModelPB:
         payload: dict[str, Any] = {"solver": solver.solver_name.lower(), "options": options}
 
         if use_csv_store:
-            # CSV format: model and results are exchanged via CSV files
-            model_id = str(uuid.uuid4()).upper().replace('-', '_')
-            payload["model_uri"] = f"snowflake://SOLVERS/job_{model_id}/model"
+            payload["model_uri"] = f"snowflake://SOLVERS/job_{self._model_id}/model"
 
             print("Exporting model to CSV...")
             remaining_timeout_minutes = calc_remaining_timeout_minutes(
                 start_time, query_timeout_mins, config_file_path=config_file_path
             )
-            self._export_model_to_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
+            self._export_model_to_csv(self._model_id, executor, prefix_lowercase, remaining_timeout_minutes)
             print("Model CSV export completed")
 
             print("Execute solver job")
@@ -874,7 +857,7 @@ class SolverModelPB:
             remaining_timeout_minutes = calc_remaining_timeout_minutes(
                 start_time, query_timeout_mins, config_file_path=config_file_path
             )
-            self._import_solver_results_from_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
+            self._import_solver_results_from_views()
 
         else:  # protobuf format
             # Protobuf format: model and results are exchanged via binary protobuf
@@ -924,11 +907,22 @@ class SolverModelPB:
             raise ValueError(
                 f"Expected RelExecutor, but got {type(executor).__name__}."
             )
-        load_point_relation = f"""
-        def delete[:{self.point._name}]: {self.point._name}
-        def insert(:{self.point._name}, variable, value): {self.points._name}(int128[{point_index}], variable, value)
-        """
-        executor.execute_raw(textwrap.dedent(load_point_relation))
+
+        use_csv_store = True
+        if self._solver is not None:
+            use_csv_store = self._solver.engine_settings.get("store", {})\
+                                                        .get("csv", {})\
+                                                        .get("enabled", True)
+
+        if use_csv_store:
+            self._load_point_counter += 1
+            b.define(self._point_updates(self._load_point_counter, point_index))
+        else:
+            load_point_relation = f"""
+            def delete[:{self.point._name}]: {self.point._name}
+            def insert(:{self.point._name}, variable, value): {self.points._name}(int128[{point_index}], variable, value)
+            """
+            executor.execute_raw(textwrap.dedent(load_point_relation))
 
     def summarize_result(self) -> Any:
         """Print solver result summary.

relationalai/semantics/rel/compiler.py

@@ -12,7 +12,10 @@ from relationalai.semantics.metamodel.util import OrderedSet, group_by, NameCach
 
 from relationalai.semantics.rel import rel, rel_utils as u, builtins as rel_bt
 
-from ..metamodel.rewrite import (Flatten, ExtractNestedLogicals, DNFUnionSplitter, DischargeConstraints, FormatOutputs)
+from ..metamodel.rewrite import (
+    Flatten, DNFUnionSplitter, DischargeConstraints, FormatOutputs, ExtractNestedLogicals,
+    # HandleAggregationsAndRanks
+)
 from ..lqp.rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter, SplitMultiCheckRequires
 
 import math
@@ -30,13 +33,14 @@ class Compiler(c.Compiler):
             DischargeConstraints(),
             Checker(),
             CDC(), # specialize to physical relations before extracting nested and typing
-            ExtractNestedLogicals(), # before InferTypes to avoid extracting casts
+            ExtractNestedLogicals(),
             InferTypes(),
             DNFUnionSplitter(),
             ExtractKeys(),
-            FormatOutputs(),
+            FormatOutputs(use_rel=True),
             ExtractCommon(),
             Flatten(),
+            # HandleAggregationsAndRanks(),
             Splinter(),
             QuantifyVars(),
         ])

relationalai/semantics/rel/executor.py

@@ -318,7 +318,7 @@ class RelExecutor(e.Executor):
         if self.dry_run:
             return DataFrame()
 
-        cols, extra_cols = self._compute_cols(task, task_model)
+        cols, extra_cols, _ = self._compute_cols(task, task_model)
 
         if not export_to:
             if format == "pandas":

relationalai/tools/txn_progress.py

@@ -0,0 +1,188 @@
+from datetime import datetime
+from collections import defaultdict
+from rich.text import Text
+from rich.console import Console
+from relationalai.util.format import format_duration
+
+# Create a console for rendering Rich text to ANSI codes
+# Force terminal mode and no pager to ensure ANSI codes are always generated
+_console = Console(force_terminal=True, legacy_windows=False)
+
+def render_line(line_data):
+    """Convert structured line data to rendered string with optional ANSI color codes.
+    
+    Args:
+        line_data: dict with 'text' (str) and 'color' (str|None) fields.
+                   color can be 'green', or None
+    
+    Returns:
+        str: Plain text or ANSI-colored text
+    """
+    color = line_data.get("color")
+    if color:
+        text = Text(line_data["text"], style=color)
+        with _console.capture() as capture:
+            _console.print(text, end="")
+        return capture.get()
+    return line_data["text"]
+
+def parse_ts(ts):
+    ts = ts.replace("Z", "")
+
+    # Handle fractional seconds - normalize to 6 digits (microseconds)
+    if "." in ts:
+        parts = ts.split(".")
+        if len(parts) == 2:
+            # Pad or truncate fractional seconds to 6 digits
+            fractional = parts[1].ljust(6, '0')[:6]
+            ts = f"{parts[0]}.{fractional}"
+
+    return datetime.fromisoformat(ts)
+
+def duration_ms(task):
+    if "start_time" not in task or "end_time" not in task:
+        return 0
+    return int((parse_ts(task["end_time"]) - parse_ts(task["start_time"])).total_seconds() * 1000)
+
+def build_graph(tasks):
+    children = defaultdict(list)
+    parents = defaultdict(list)
+
+    for tid, t in tasks.items():
+        if "origin_id" in t:
+            parent = t["origin_id"]
+            children[parent].append(tid)
+            parents[tid].append(parent)
+
+    return parents, children
+
+def find_roots(tasks, parents):
+    # roots = nodes with no parents (or parents not in tasks)
+    return [tid for tid in tasks if tid not in parents or not any(p in tasks for p in parents[tid])]
+
+def node_label(tid, task):
+    """Create structured label data for a task node.
+    
+    Returns:
+        list of dicts with 'text' (str) and 'color' (str|None) fields.
+        First item is the task line, subsequent items are warning details.
+    """
+    name = task.get("task_name", tid)
+    is_finished = "start_time" in task and "end_time" in task
+    has_warnings = "warnings" in task and task["warnings"]
+    warning_count = 0
+    warning_text = ""
+    text_color = 'green' if is_finished else None
+    
+    lines = []
+
+    if has_warnings:
+        warning_count = len(task["warnings"])
+        warning_text = f" ⚠️  {warning_count} warning" + ("s" if warning_count > 1 else "")
+    
+    if is_finished:
+        dur_seconds = duration_ms(task) / 1000.0
+        duration_str = format_duration(dur_seconds)
+        
+        lines.append({"text": f"{name}  ({duration_str}){warning_text}", "color": text_color})
+    else:
+        lines.append({"text": f"{name}  (Evaluating...){warning_text}", "color": text_color})
+    
+    # Add warning details as separate lines (for both finished and evaluating tasks)
+    if has_warnings:
+        for warning_code, warning_data in task["warnings"].items():
+            message = warning_data.get("message", "")
+            lines.append({"text": f"  ⚠️  {warning_code}: {message}", "color": text_color})
+    
+    return lines
+
+def build_tree(tid, tasks, children, prefix="", is_last=True, seen=None):
+    """Build tree structure and return as list of structured line data dicts.
+    
+    Returns:
+        list of dicts with 'text' (str) and 'color' (str|None) fields
+    """
+    if seen is None:
+        seen = set()
+
+    if tid not in tasks:
+        return []
+
+    if tid in seen:
+        return []
+    seen.add(tid)
+
+    task = tasks[tid]
+    connector = "└─ " if is_last else "├─ "
+    label_lines = node_label(tid, task)
+    
+    # Build lines with proper prefix
+    lines = []
+    for i, label_data in enumerate(label_lines):
+        if i == 0:
+            # First line gets the tree connector
+            full_text = prefix + connector + label_data["text"]
+        else:
+            # Warning detail lines get continuation prefix
+            continuation_prefix = prefix + ("   " if is_last else "│  ")
+            full_text = continuation_prefix + label_data["text"]
+        lines.append({"text": full_text, "color": label_data["color"]})
+
+    new_prefix = prefix + ("   " if is_last else "│  ")
+
+    if tid not in children:
+        return lines
+
+    child_list = [c for c in children[tid] if c in tasks and should_include_task(c, tasks[c])]
+    for i, child in enumerate(child_list):
+        last = i == len(child_list) - 1
+        lines.extend(build_tree(child, tasks, children, new_prefix, last, seen))
+    
+    return lines
+
+def should_include_task(task_id, task):
+    """Filter out error tasks that aren't useful for execution flow analysis"""
+    task_name = task.get("task_name", task_id)
+    return not task_name.startswith(":error_pyrel_error_attrs")
+
+def format_execution_tree(progress):
+    """Format execution tree and return as a string."""
+    if not progress or "tasks" not in progress:
+        return ""
+
+    tasks = progress["tasks"]
+    parents, children = build_graph(tasks)
+    roots = find_roots(tasks, parents)
+
+    # Filter out error tasks from roots
+    filtered_roots = [root for root in roots if should_include_task(root, tasks[root])]
+
+    # Sort roots: in-progress tasks first, then finished tasks, alphabetically within each group
+    def sort_key(task_id):
+        task = tasks[task_id]
+        is_finished = "start_time" in task and "end_time" in task
+        task_name = task.get("task_name", task_id)
+        return (is_finished, task_name)
+    
+    sorted_roots = sorted(filtered_roots, key=sort_key)
+
+    # Build structured data first
+    all_line_data = []
+    for root in sorted_roots:
+        # Add root node (may include warning lines)
+        label_lines = node_label(root, tasks[root])
+        all_line_data.extend(label_lines)
+            
+        # Add children
+        if root in children:
+            child_list = [c for c in children[root] if c in tasks and should_include_task(c, tasks[c])]
+            for i, child in enumerate(child_list):
+                last = i == len(child_list) - 1
+                all_line_data.extend(build_tree(child, tasks, children, "", last, set()))
+        
+        # Empty line after each root tree
+        all_line_data.append({"text": "", "color": None})
+
+    # Render structured data to strings
+    rendered_lines = [render_line(line_data) for line_data in all_line_data]
+    return "\n\n" + "\n".join(rendered_lines)

{relationalai-0.13.4.dist-info → relationalai-0.13.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: relationalai
-Version: 0.13.4
+Version: 0.13.5
 Summary: RelationalAI Library and CLI
 Author-email: RelationalAI <support@relational.ai>
 License-File: LICENSE