relationalai 1.0.0a3__py3-none-any.whl → 1.0.0a5__py3-none-any.whl

v0/relationalai/semantics/reasoners/optimization/solvers_pb.py

@@ -1,8 +1,8 @@
-"""Solver model implementation using protobuf format.
+"""Solver model implementation supporting protobuf and CSV formats.
 
 This module provides the SolverModelPB class for defining optimization and
-constraint programming problems that are serialized to protobuf format and
-solved by external solver engines.
+constraint programming problems that are serialized and solved by external
+solver engines. Supports both protobuf (default) and CSV (future) exchange formats.
 
 Note: This protobuf-based implementation will be deprecated in favor of the
       development version (solvers_dev.py) in future releases.
@@ -23,7 +23,6 @@ from v0.relationalai.util.timeout import calc_remaining_timeout_minutes
 
 from .common import make_name
 
-
 # =============================================================================
 # Solver ProtoBuf Format Constants and Helpers
 # =============================================================================
@@ -191,6 +190,7 @@ class SolverModelPB:
         """
         b.define(b.RawSource("rel", textwrap.dedent(install_rel)))
 
+
     # -------------------------------------------------------------------------
     # Variable Handling
     # -------------------------------------------------------------------------
@@ -501,69 +501,218 @@ class SolverModelPB:
     # Solving and Result Handling
     # -------------------------------------------------------------------------
 
-    def solve(
-        self, solver: Solver, log_to_console: bool = False, **kwargs: Any
+    def _export_model_to_csv(
+        self,
+        model_id: str,
+        executor: RelExecutor,
+        prefix_lowercase: str,
+        query_timeout_mins: Optional[int] = None
     ) -> None:
-        """Solve the model.
+        """Export model to CSV files in Snowflake stage.
 
         Args:
-            solver: Solver instance.
-            log_to_console: Whether to show solver output.
-            **kwargs: Solver options and parameters.
+            model_id: Unique model identifier for stage paths.
+            executor: RelExecutor instance.
+            prefix_lowercase: Prefix for relation names.
+            query_timeout_mins: Query timeout in minutes.
         """
-        options = {**kwargs, "version": 1}
+        stage_base_no_txn = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/SOLVERS/job_{model_id}"
+
+        # Export all model relations using Rel-native export_csv in a single transaction
+        # Transformations (uuid_string, encode_base64) are done inline in the export query
+        export_rel = textwrap.dedent(f"""
+        // Get transaction ID for folder naming - solver service validates ownership
+        // Use uuid_string to get proper UUID format, then replace hyphens with underscores
+        def txn_id_str {{string_replace[uuid_string[current_transaction_id], "-", "_"]}}
+
+        // Define base path with txn_id in folder name: model_{{txn_id}}/
+        def base_path {{"{stage_base_no_txn}/model"}}
+
+        // Export variable_hash.csv - single column: HASH (UUID string)
+        // Transformation: convert Variable UInt128 to UUID string inline
+        def variable_hash_data(:HASH, v, h):
+            {self.Variable._name}(v) and uuid_string(v, h)
+
+        def export[:variable_hash]: {{export_csv[{{
+            (:path, base_path ++ "/variable_hash_" ++ txn_id_str ++ ".csv");
+            (:data, variable_hash_data);
+            (:compression, "gzip")
+        }}]}}
+
+        // Export variable_name.csv - columns: HASH (UUID string), VALUE (name string)
+        // Transformation: convert Variable UInt128 to UUID string inline
+        def variable_name_data(:HASH, v, h):
+            {prefix_lowercase}variable_name(v, _) and uuid_string(v, h)
+        def variable_name_data(:VALUE, v, name):
+            {prefix_lowercase}variable_name(v, name)
+
+        def export[:variable_name]: {{export_csv[{{
+            (:path, base_path ++ "/variable_name_" ++ txn_id_str ++ ".csv");
+            (:data, variable_name_data);
+            (:compression, "gzip")
+        }}]}}
+
+        // Export constraint.csv - single column: VALUE (base64 encoded constraint)
+        // Transformation: encode_base64 done inline
+        def constraint_data(:VALUE, c, e):
+            exists((s) |
+                {self.Constraint._name}(c) and
+                {prefix_lowercase}constraint_serialized(c, s) and
+                encode_base64(s, e))
+
+        def export[:constraint]: {{export_csv[{{
+            (:path, base_path ++ "/constraint_" ++ txn_id_str ++ ".csv");
+            (:data, constraint_data);
+            (:compression, "gzip")
+        }}]}}
+
+        // Export min_objective.csv - columns: HASH (UUID string), VALUE (base64 encoded)
+        // Transformations: uuid_string and encode_base64 done inline
+        def min_objective_data(:HASH, obj, h):
+            {self.MinObjective._name}(obj) and uuid_string(obj, h)
+        def min_objective_data(:VALUE, obj, e):
+            exists((s) |
+                {self.MinObjective._name}(obj) and
+                {prefix_lowercase}minobjective_serialized(obj, s) and
+                encode_base64(s, e))
+
+        def export[:min_objective]: {{export_csv[{{
+            (:path, base_path ++ "/min_objective_" ++ txn_id_str ++ ".csv");
+            (:data, min_objective_data);
+            (:compression, "gzip")
+        }}]}}
+
+        // Export max_objective.csv - columns: HASH (UUID string), VALUE (base64 encoded)
+        // Transformations: uuid_string and encode_base64 done inline
+        def max_objective_data(:HASH, obj, h):
+            {self.MaxObjective._name}(obj) and uuid_string(obj, h)
+        def max_objective_data(:VALUE, obj, e):
+            exists((s) |
+                {self.MaxObjective._name}(obj) and
+                {prefix_lowercase}maxobjective_serialized(obj, s) and
+                encode_base64(s, e))
+
+        def export[:max_objective]: {{export_csv[{{
+            (:path, base_path ++ "/max_objective_" ++ txn_id_str ++ ".csv");
+            (:data, max_objective_data);
+            (:compression, "gzip")
+        }}]}}
+        """)
+
+        executor.execute_raw(export_rel, query_timeout_mins=query_timeout_mins)
+
+    def _import_solver_results_from_csv(
+        self,
+        model_id: str,
+        executor: RelExecutor,
+        prefix_lowercase: str,
+        query_timeout_mins: Optional[int] = None
+    ) -> None:
+        """Import solver results from CSV files in Snowflake stage.
 
-        # Validate solver options
-        for option_key, option_value in options.items():
-            if not isinstance(option_key, str):
-                raise TypeError(
-                    f"Solver option keys must be strings, but got {type(option_key).__name__} for key {option_key!r}."
-                )
-            if not isinstance(option_value, (int, float, str, bool)):
-                raise TypeError(
-                    f"Solver option values must be int, float, str, or bool, "
-                    f"but got {type(option_value).__name__} for option {option_key!r}."
-                )
+        Loads and extracts CSV files in a single transaction to minimize overhead.
 
-        # Three-phase solve process:
-        # 1. Export model to Snowflake as protobuf
-        # 2. Execute solver job (external solver reads from Snowflake)
-        # 3. Extract and load results back into the model
-        input_id = uuid.uuid4()
-        model_uri = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/job-inputs/solver/{input_id}/model.binpb"
-        sf_input_uri = f"snowflake://job-inputs/solver/{input_id}/model.binpb"
-        payload: dict[str, Any] = {"solver": solver.solver_name.lower()}
-        payload["options"] = options
-        payload["model_uri"] = sf_input_uri
+        Args:
+            model_id: Unique model identifier for stage paths.
+            executor: RelExecutor instance.
+            prefix_lowercase: Prefix for relation names.
+            query_timeout_mins: Query timeout in minutes.
+        """
+        result_stage_base = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/SOLVERS/job_{model_id}/results"
+        value_parse_fn = "parse_int" if self._num_type == "int" else "parse_float"
+
+        # Single transaction: Load CSV files and extract/map results
+        # Use inline definitions to avoid needing declared relations
+        load_and_extract_rel = textwrap.dedent(f"""
+        // Define CSV loading inline (no declare needed)
+        // Load ancillary.csv - contains solver metadata (NAME, VALUE columns)
+        def ancillary_config[:path]: "{result_stage_base}/ancillary.csv.gz"
+        def ancillary_config[:syntax, :header_row]: 1
+        def ancillary_config[:schema, :NAME]: "string"
+        def ancillary_config[:schema, :VALUE]: "string"
+        def {prefix_lowercase}solver_ancillary_raw {{load_csv[ancillary_config]}}
+
+        // Load objective_values.csv - contains objective values (SOL_INDEX, VALUE columns)
+        def objective_values_config[:path]: "{result_stage_base}/objective_values.csv.gz"
+        def objective_values_config[:syntax, :header_row]: 1
+        def objective_values_config[:schema, :SOL_INDEX]: "string"
+        def objective_values_config[:schema, :VALUE]: "string"
+        def {prefix_lowercase}solver_objective_values_raw {{load_csv[objective_values_config]}}
+
+        // Load points.csv.gz - contains solution points (SOL_INDEX, VAR_HASH, VALUE columns)
+        def points_config[:path]: "{result_stage_base}/points.csv.gz"
+        def points_config[:syntax, :header_row]: 1
+        def points_config[:schema, :SOL_INDEX]: "string"
+        def points_config[:schema, :VAR_HASH]: "string"
+        def points_config[:schema, :VALUE]: "string"
+        def {prefix_lowercase}solver_points_raw {{load_csv[points_config]}}
+
+        // Clear existing result data
+        def delete[:{self.result_info._name}]: {self.result_info._name}
+        def delete[:{self.point._name}]: {self.point._name}
+        def delete[:{self.points._name}]: {self.points._name}
 
-        executor = self._model._to_executor()
-        if not isinstance(executor, RelExecutor):
-            raise ValueError(f"Expected RelExecutor, got {type(executor).__name__}.")
-        prefix_lowercase = f"solvermodel_{self._id}_"
+        // Extract ancillary data (result info) - NAME and VALUE columns
+        def insert(:{self.result_info._name}, key, val): {{
+            exists((row) |
+                {prefix_lowercase}solver_ancillary_raw(:NAME, row, key) and
+                {prefix_lowercase}solver_ancillary_raw(:VALUE, row, val))
+        }}
 
-        query_timeout_mins = kwargs.get("query_timeout_mins", None)
-        config = self._model._config
-        if (
-            query_timeout_mins is None
-            and (
-                timeout_value := config.get(
-                    "query_timeout_mins", DEFAULT_QUERY_TIMEOUT_MINS
-                )
-            )
-            is not None
-        ):
-            query_timeout_mins = int(timeout_value)
-        config_file_path = getattr(config, "file_path", None)
-        start_time = time.monotonic()
-        remaining_timeout_minutes = query_timeout_mins
+        // Extract objective value from objective_values CSV (first solution)
+        def insert(:{self.result_info._name}, "objective_value", val): {{
+            exists((row) |
+                {prefix_lowercase}solver_objective_values_raw(:SOL_INDEX, row, "1") and
+                {prefix_lowercase}solver_objective_values_raw(:VALUE, row, val))
+        }}
 
-        # Step 1: Materialize the model and store it in Snowflake
-        print("export model")
-        # TODO(coey): Weird hack to avoid uninitialized properties error
-        # This forces evaluation of the Variable concept before export
-        b.select(b.count(self.Variable)).to_df()
-        export_model_relation = f"""
-        // TODO maybe only want to pass names if printing - like in old setup
+        // Extract solution points from points.csv.gz into points property
+        // This file has SOL_INDEX, VAR_HASH, VALUE columns
+        // Convert CSV string index to Int128 for points property signature
+        // Convert value to Int128 (for int) or Float64 (for float)
+        def insert(:{self.points._name}, sol_idx_int128, var, val_converted): {{
+            exists((row, sol_idx_str, var_hash_str, val_str, sol_idx_int, val) |
+                {prefix_lowercase}solver_points_raw(:SOL_INDEX, row, sol_idx_str) and
+                {prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
+                {prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
+                parse_int(sol_idx_str, sol_idx_int) and
+                parse_uuid(var_hash_str, var) and
+                {value_parse_fn}(val_str, val) and
+                ::std::mirror::convert(std::mirror::typeof[Int128], sol_idx_int, sol_idx_int128) and
+                {'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
+        }}
+
+        // Extract first solution into point property (default solution)
+        // Filter to SOL_INDEX = 1
+        def insert(:{self.point._name}, var, val_converted): {{
+            exists((row, var_hash_str, val_str, val) |
+                {prefix_lowercase}solver_points_raw(:SOL_INDEX, row, "1") and
+                {prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
+                {prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
+                parse_uuid(var_hash_str, var) and
+                {value_parse_fn}(val_str, val) and
+                {'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
+        }}
+        """)
+
+        executor.execute_raw(load_and_extract_rel, query_timeout_mins=query_timeout_mins)
+
+    def _export_model_to_protobuf(
+        self,
+        model_uri: str,
+        executor: RelExecutor,
+        prefix_lowercase: str,
+        query_timeout_mins: Optional[int] = None
+    ) -> None:
+        """Export model to protobuf format in Snowflake stage.
+
+        Args:
+            model_uri: Snowflake URI for the protobuf file.
+            executor: RelExecutor instance.
+            prefix_lowercase: Prefix for relation names.
+            query_timeout_mins: Query timeout in minutes.
+        """
+        export_rel = f"""
         // Collect all model components into a relation for serialization
         def model_relation {{
             (:variable, {self.Variable._name});
@@ -584,31 +733,24 @@ class SolverModelPB:
         def export {{ config }}
         """
         executor.execute_raw(
-            textwrap.dedent(export_model_relation),
-            query_timeout_mins=remaining_timeout_minutes,
+            textwrap.dedent(export_rel),
+            query_timeout_mins=query_timeout_mins
         )
 
-        # Step 2: Execute solver job and wait for completion
-        print("execute solver job")
-        remaining_timeout_minutes = calc_remaining_timeout_minutes(
-            start_time,
-            query_timeout_mins,
-            config_file_path=config_file_path,
-        )
-        job_id = solver._exec_job(
-            payload,
-            log_to_console=log_to_console,
-            query_timeout_mins=remaining_timeout_minutes,
-        )
+    def _import_solver_results_from_protobuf(
+        self,
+        job_id: str,
+        executor: RelExecutor,
+        query_timeout_mins: Optional[int] = None
+    ) -> None:
+        """Import solver results from protobuf format.
 
-        # Step 3: Extract and insert solver results into the model
-        print("extract result")
-        remaining_timeout_minutes = calc_remaining_timeout_minutes(
-            start_time,
-            query_timeout_mins,
-            config_file_path=config_file_path,
-        )
-        extract_results_relation = f"""
+        Args:
+            job_id: Job identifier for result location.
+            executor: RelExecutor instance.
+            query_timeout_mins: Query timeout in minutes.
+        """
+        extract_rel = f"""
         def raw_result {{
             load_binary["snowflake://APP_STATE.RAI_INTERNAL_STAGE/job-results/{job_id}/result.binpb"]
         }}
@@ -625,6 +767,7 @@ class SolverModelPB:
         def insert(:{self.result_info._name}, key, value):
             exists((original_key) | string(extracted[original_key], value) and ::std::mirror::lower(original_key, key))
         """
+
         if self._num_type == "int":
             insert_points_relation = f"""
             def insert(:{self.point._name}, variable, value):
@@ -645,15 +788,122 @@ class SolverModelPB:
                     ::std::mirror::convert(std::mirror::typeof[Int128], float_index, point_index)
                 )
             """
+
         executor.execute_raw(
-            textwrap.dedent(extract_results_relation)
-            + textwrap.dedent(insert_points_relation),
-            readonly=False,
-            query_timeout_mins=remaining_timeout_minutes,
+            textwrap.dedent(extract_rel) + textwrap.dedent(insert_points_relation),
+            query_timeout_mins=query_timeout_mins
         )
 
-        print("finished solve")
+    def solve(
+        self, solver: Solver, log_to_console: bool = False, **kwargs: Any
+    ) -> None:
+        """Solve the model.
+
+        Args:
+            solver: Solver instance.
+            log_to_console: Whether to show solver output.
+            **kwargs: Solver options and parameters.
+        """
+
+        use_csv_store = solver.engine_settings.get("store", {})\
+                                        .get("csv", {})\
+                                        .get("enabled", False)
+
+        print(f"Using {'csv' if use_csv_store else 'protobuf'} store...")
+
+        options = {**kwargs, "version": 1}
+
+        # Validate solver options
+        for option_key, option_value in options.items():
+            if not isinstance(option_key, str):
+                raise TypeError(
+                    f"Solver option keys must be strings, but got {type(option_key).__name__} for key {option_key!r}."
+                )
+            if not isinstance(option_value, (int, float, str, bool)):
+                raise TypeError(
+                    f"Solver option values must be int, float, str, or bool, "
+                    f"but got {type(option_value).__name__} for option {option_key!r}."
+                )
+
+        executor = self._model._to_executor()
+        if not isinstance(executor, RelExecutor):
+            raise ValueError(f"Expected RelExecutor, got {type(executor).__name__}.")
+        prefix_lowercase = f"solvermodel_{self._id}_"
+
+        # Initialize timeout from config
+        query_timeout_mins = kwargs.get("query_timeout_mins", None)
+        config = self._model._config
+        if (
+            query_timeout_mins is None
+            and (
+                timeout_value := config.get(
+                    "query_timeout_mins", DEFAULT_QUERY_TIMEOUT_MINS
+                )
+            )
+            is not None
+        ):
+            query_timeout_mins = int(timeout_value)
+        config_file_path = getattr(config, "file_path", None)
+        start_time = time.monotonic()
+
+        # Force evaluation of Variable concept before export
+        b.select(b.count(self.Variable)).to_df()
+
+        # Prepare payload for solver service
+        payload: dict[str, Any] = {"solver": solver.solver_name.lower(), "options": options}
+
+        if use_csv_store:
+            # CSV format: model and results are exchanged via CSV files
+            model_id = str(uuid.uuid4()).upper().replace('-', '_')
+            payload["model_uri"] = f"snowflake://SOLVERS/job_{model_id}/model"
+
+            print("Exporting model to CSV...")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            self._export_model_to_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
+            print("Model CSV export completed")
+
+            print("Execute solver job")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            solver._exec_job(payload, log_to_console=log_to_console, query_timeout_mins=remaining_timeout_minutes)
+
+            print("Loading and extracting solver results...")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            self._import_solver_results_from_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
+
+        else:  # protobuf format
+            # Protobuf format: model and results are exchanged via binary protobuf
+            input_id = uuid.uuid4()
+            model_uri = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/job-inputs/solver/{input_id}/model.binpb"
+            sf_input_uri = f"snowflake://job-inputs/solver/{input_id}/model.binpb"
+            payload["model_uri"] = sf_input_uri
+
+            print("Export model...")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            self._export_model_to_protobuf(model_uri, executor, prefix_lowercase, remaining_timeout_minutes)
+
+            print("Execute solver job...")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            job_id = solver._exec_job(payload, log_to_console=log_to_console, query_timeout_mins=remaining_timeout_minutes)
+
+            print("Extract result...")
+            remaining_timeout_minutes = calc_remaining_timeout_minutes(
+                start_time, query_timeout_mins, config_file_path=config_file_path
+            )
+            self._import_solver_results_from_protobuf(job_id, executor, remaining_timeout_minutes)
+
+        print("Finished solve")
         print()
+        return None
 
     def load_point(self, point_index: int) -> None:
         """Load a solution point.
@@ -678,7 +928,7 @@ class SolverModelPB:
         def delete[:{self.point._name}]: {self.point._name}
         def insert(:{self.point._name}, variable, value): {self.points._name}(int128[{point_index}], variable, value)
         """
-        executor.execute_raw(textwrap.dedent(load_point_relation), readonly=False)
+        executor.execute_raw(textwrap.dedent(load_point_relation))
 
     def summarize_result(self) -> Any:
         """Print solver result summary.

v0/relationalai/semantics/rel/compiler.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from typing import Any, Iterable, Sequence as PySequence, cast, Tuple, Union
 from dataclasses import dataclass, field
 from decimal import Decimal as PyDecimal
@@ -843,7 +844,7 @@ class ModelToRel:
 
     def _effect_name(self, n: ir.Task):
         """ Return the name to be used for the effect (e.g. the relation name, output, etc). """
-        if isinstance(n, ir.Output) and bt.export_annotation in n.annotations:
+        if helpers.is_export(n):
             return "Export_Relation"
         elif isinstance(n, ir.Output):
             return "output"

v0/relationalai/semantics/rel/executor.py

@@ -355,8 +355,9 @@ class RelExecutor(e.Executor):
 
     # NOTE(coey): this is added temporarily to support executing Rel for the solvers library in EA.
     # It can be removed once this is no longer needed by the solvers library.
-    def execute_raw(self, raw_rel:str, readonly:bool=True, query_timeout_mins:int|None=None) -> DataFrame:
-        raw_results = self.resources.exec_raw(self.database, self.engine, raw_rel, readonly, nowait_durable=True, query_timeout_mins=query_timeout_mins)
+    def execute_raw(self, raw_rel:str, query_timeout_mins:int|None=None) -> DataFrame:
+        # NOTE intentionally hard-coding to read-only=False, because read-only Rel queries are deprecated.
+        raw_results = self.resources.exec_raw(self.database, self.engine, raw_rel, False, nowait_durable=True, query_timeout_mins=query_timeout_mins)
         df, errs = result_helpers.format_results(raw_results, None, generation=Generation.QB)  # Pass None for task parameter
         self.report_errors(errs)
         return df