relationalai 1.0.0a2__py3-none-any.whl → 1.0.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- relationalai/config/shims.py +1 -0
- relationalai/semantics/__init__.py +7 -1
- relationalai/semantics/frontend/base.py +19 -13
- relationalai/semantics/frontend/core.py +30 -2
- relationalai/semantics/frontend/front_compiler.py +38 -11
- relationalai/semantics/frontend/pprint.py +1 -1
- relationalai/semantics/metamodel/rewriter.py +6 -2
- relationalai/semantics/metamodel/typer.py +70 -26
- relationalai/semantics/reasoners/__init__.py +11 -0
- relationalai/semantics/reasoners/graph/__init__.py +38 -0
- relationalai/semantics/reasoners/graph/core.py +9015 -0
- relationalai/shims/executor.py +4 -1
- relationalai/shims/hoister.py +9 -0
- relationalai/shims/mm2v0.py +47 -34
- relationalai/tools/cli/cli.py +138 -0
- relationalai/tools/cli/docs.py +394 -0
- {relationalai-1.0.0a2.dist-info → relationalai-1.0.0a4.dist-info}/METADATA +5 -3
- {relationalai-1.0.0a2.dist-info → relationalai-1.0.0a4.dist-info}/RECORD +57 -43
- v0/relationalai/__init__.py +69 -22
- v0/relationalai/clients/__init__.py +15 -2
- v0/relationalai/clients/client.py +4 -4
- v0/relationalai/clients/exec_txn_poller.py +91 -0
- v0/relationalai/clients/local.py +5 -5
- v0/relationalai/clients/resources/__init__.py +8 -0
- v0/relationalai/clients/{azure.py → resources/azure/azure.py} +12 -12
- v0/relationalai/clients/resources/snowflake/__init__.py +20 -0
- v0/relationalai/clients/resources/snowflake/cli_resources.py +87 -0
- v0/relationalai/clients/resources/snowflake/direct_access_resources.py +717 -0
- v0/relationalai/clients/resources/snowflake/engine_state_handlers.py +309 -0
- v0/relationalai/clients/resources/snowflake/error_handlers.py +199 -0
- v0/relationalai/clients/resources/snowflake/resources_factory.py +99 -0
- v0/relationalai/clients/{snowflake.py → resources/snowflake/snowflake.py} +642 -1399
- v0/relationalai/clients/{use_index_poller.py → resources/snowflake/use_index_poller.py} +51 -12
- v0/relationalai/clients/resources/snowflake/use_index_resources.py +188 -0
- v0/relationalai/clients/resources/snowflake/util.py +387 -0
- v0/relationalai/early_access/dsl/ir/executor.py +4 -4
- v0/relationalai/early_access/dsl/snow/api.py +2 -1
- v0/relationalai/errors.py +18 -0
- v0/relationalai/experimental/solvers.py +7 -7
- v0/relationalai/semantics/devtools/benchmark_lqp.py +4 -5
- v0/relationalai/semantics/devtools/extract_lqp.py +1 -1
- v0/relationalai/semantics/internal/snowflake.py +1 -1
- v0/relationalai/semantics/lqp/executor.py +7 -12
- v0/relationalai/semantics/lqp/rewrite/extract_keys.py +25 -3
- v0/relationalai/semantics/metamodel/util.py +6 -5
- v0/relationalai/semantics/reasoners/optimization/solvers_pb.py +335 -84
- v0/relationalai/semantics/rel/executor.py +14 -11
- v0/relationalai/semantics/sql/executor/snowflake.py +9 -5
- v0/relationalai/semantics/tests/test_snapshot_abstract.py +1 -1
- v0/relationalai/tools/cli.py +26 -30
- v0/relationalai/tools/cli_helpers.py +10 -2
- v0/relationalai/util/otel_configuration.py +2 -1
- v0/relationalai/util/otel_handler.py +1 -1
- {relationalai-1.0.0a2.dist-info → relationalai-1.0.0a4.dist-info}/WHEEL +0 -0
- {relationalai-1.0.0a2.dist-info → relationalai-1.0.0a4.dist-info}/entry_points.txt +0 -0
- {relationalai-1.0.0a2.dist-info → relationalai-1.0.0a4.dist-info}/top_level.txt +0 -0
- /v0/relationalai/clients/{cache_store.py → resources/snowflake/cache_store.py} +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
from typing import Callable, Generator, Generic, IO, Iterable, Optional, Sequence, Tuple, TypeVar, cast, Hashable
|
|
2
|
+
from typing import Callable, Generator, Generic, IO, Iterable, Optional, Sequence, Tuple, TypeVar, cast, Hashable, Union
|
|
3
|
+
import types
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
|
|
5
6
|
#--------------------------------------------------
|
|
@@ -345,7 +346,7 @@ def rewrite_set(t: type[T], f: Callable[[T], T], items: FrozenOrderedSet[T]) ->
|
|
|
345
346
|
return items
|
|
346
347
|
return ordered_set(*new_items).frozen()
|
|
347
348
|
|
|
348
|
-
def rewrite_list(t: type[T], f: Callable[[T], T], items: Tuple[T, ...]) -> Tuple[T, ...]:
|
|
349
|
+
def rewrite_list(t: Union[type[T], types.UnionType], f: Callable[[T], T], items: Tuple[T, ...]) -> Tuple[T, ...]:
|
|
349
350
|
""" Map a function over a list, returning a new list with the results. Avoid allocating a new list if the function is the identity. """
|
|
350
351
|
new_items: Optional[list[T]] = None
|
|
351
352
|
for i in range(len(items)):
|
|
@@ -359,15 +360,15 @@ def rewrite_list(t: type[T], f: Callable[[T], T], items: Tuple[T, ...]) -> Tuple
|
|
|
359
360
|
return items
|
|
360
361
|
return tuple(new_items)
|
|
361
362
|
|
|
362
|
-
def flatten_iter(items: Iterable[object], t: type[T]) -> Generator[T, None, None]:
|
|
363
|
+
def flatten_iter(items: Iterable[object], t: Union[type[T], types.UnionType]) -> Generator[T, None, None]:
|
|
363
364
|
"""Yield items from a nested iterable structure one at a time."""
|
|
364
365
|
for item in items:
|
|
365
366
|
if isinstance(item, (list, tuple, OrderedSet)):
|
|
366
367
|
yield from flatten_iter(item, t)
|
|
367
368
|
elif isinstance(item, t):
|
|
368
|
-
yield item
|
|
369
|
+
yield cast(T, item)
|
|
369
370
|
|
|
370
|
-
def flatten_tuple(items: Iterable[object], t: type[T]) -> tuple[T, ...]:
|
|
371
|
+
def flatten_tuple(items: Iterable[object], t: Union[type[T], types.UnionType]) -> tuple[T, ...]:
|
|
371
372
|
""" Flatten the nested iterable structure into a tuple."""
|
|
372
373
|
return tuple(flatten_iter(items, t))
|
|
373
374
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"""Solver model implementation
|
|
1
|
+
"""Solver model implementation supporting protobuf and CSV formats.
|
|
2
2
|
|
|
3
3
|
This module provides the SolverModelPB class for defining optimization and
|
|
4
|
-
constraint programming problems that are serialized
|
|
5
|
-
|
|
4
|
+
constraint programming problems that are serialized and solved by external
|
|
5
|
+
solver engines. Supports both protobuf (default) and CSV (future) exchange formats.
|
|
6
6
|
|
|
7
7
|
Note: This protobuf-based implementation will be deprecated in favor of the
|
|
8
8
|
development version (solvers_dev.py) in future releases.
|
|
@@ -23,7 +23,6 @@ from v0.relationalai.util.timeout import calc_remaining_timeout_minutes
|
|
|
23
23
|
|
|
24
24
|
from .common import make_name
|
|
25
25
|
|
|
26
|
-
|
|
27
26
|
# =============================================================================
|
|
28
27
|
# Solver ProtoBuf Format Constants and Helpers
|
|
29
28
|
# =============================================================================
|
|
@@ -191,6 +190,7 @@ class SolverModelPB:
|
|
|
191
190
|
"""
|
|
192
191
|
b.define(b.RawSource("rel", textwrap.dedent(install_rel)))
|
|
193
192
|
|
|
193
|
+
|
|
194
194
|
# -------------------------------------------------------------------------
|
|
195
195
|
# Variable Handling
|
|
196
196
|
# -------------------------------------------------------------------------
|
|
@@ -501,69 +501,218 @@ class SolverModelPB:
|
|
|
501
501
|
# Solving and Result Handling
|
|
502
502
|
# -------------------------------------------------------------------------
|
|
503
503
|
|
|
504
|
-
def
|
|
505
|
-
self,
|
|
504
|
+
def _export_model_to_csv(
|
|
505
|
+
self,
|
|
506
|
+
model_id: str,
|
|
507
|
+
executor: RelExecutor,
|
|
508
|
+
prefix_lowercase: str,
|
|
509
|
+
query_timeout_mins: Optional[int] = None
|
|
506
510
|
) -> None:
|
|
507
|
-
"""
|
|
511
|
+
"""Export model to CSV files in Snowflake stage.
|
|
508
512
|
|
|
509
513
|
Args:
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
514
|
+
model_id: Unique model identifier for stage paths.
|
|
515
|
+
executor: RelExecutor instance.
|
|
516
|
+
prefix_lowercase: Prefix for relation names.
|
|
517
|
+
query_timeout_mins: Query timeout in minutes.
|
|
513
518
|
"""
|
|
514
|
-
|
|
519
|
+
stage_base_no_txn = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/SOLVERS/job_{model_id}"
|
|
520
|
+
|
|
521
|
+
# Export all model relations using Rel-native export_csv in a single transaction
|
|
522
|
+
# Transformations (uuid_string, encode_base64) are done inline in the export query
|
|
523
|
+
export_rel = textwrap.dedent(f"""
|
|
524
|
+
// Get transaction ID for folder naming - solver service validates ownership
|
|
525
|
+
// Use uuid_string to get proper UUID format, then replace hyphens with underscores
|
|
526
|
+
def txn_id_str {{string_replace[uuid_string[current_transaction_id], "-", "_"]}}
|
|
527
|
+
|
|
528
|
+
// Define base path with txn_id in folder name: model_{{txn_id}}/
|
|
529
|
+
def base_path {{"{stage_base_no_txn}/model"}}
|
|
530
|
+
|
|
531
|
+
// Export variable_hash.csv - single column: HASH (UUID string)
|
|
532
|
+
// Transformation: convert Variable UInt128 to UUID string inline
|
|
533
|
+
def variable_hash_data(:HASH, v, h):
|
|
534
|
+
{self.Variable._name}(v) and uuid_string(v, h)
|
|
535
|
+
|
|
536
|
+
def export[:variable_hash]: {{export_csv[{{
|
|
537
|
+
(:path, base_path ++ "/variable_hash_" ++ txn_id_str ++ ".csv");
|
|
538
|
+
(:data, variable_hash_data);
|
|
539
|
+
(:compression, "gzip")
|
|
540
|
+
}}]}}
|
|
541
|
+
|
|
542
|
+
// Export variable_name.csv - columns: HASH (UUID string), VALUE (name string)
|
|
543
|
+
// Transformation: convert Variable UInt128 to UUID string inline
|
|
544
|
+
def variable_name_data(:HASH, v, h):
|
|
545
|
+
{prefix_lowercase}variable_name(v, _) and uuid_string(v, h)
|
|
546
|
+
def variable_name_data(:VALUE, v, name):
|
|
547
|
+
{prefix_lowercase}variable_name(v, name)
|
|
548
|
+
|
|
549
|
+
def export[:variable_name]: {{export_csv[{{
|
|
550
|
+
(:path, base_path ++ "/variable_name_" ++ txn_id_str ++ ".csv");
|
|
551
|
+
(:data, variable_name_data);
|
|
552
|
+
(:compression, "gzip")
|
|
553
|
+
}}]}}
|
|
554
|
+
|
|
555
|
+
// Export constraint.csv - single column: VALUE (base64 encoded constraint)
|
|
556
|
+
// Transformation: encode_base64 done inline
|
|
557
|
+
def constraint_data(:VALUE, c, e):
|
|
558
|
+
exists((s) |
|
|
559
|
+
{self.Constraint._name}(c) and
|
|
560
|
+
{prefix_lowercase}constraint_serialized(c, s) and
|
|
561
|
+
encode_base64(s, e))
|
|
562
|
+
|
|
563
|
+
def export[:constraint]: {{export_csv[{{
|
|
564
|
+
(:path, base_path ++ "/constraint_" ++ txn_id_str ++ ".csv");
|
|
565
|
+
(:data, constraint_data);
|
|
566
|
+
(:compression, "gzip")
|
|
567
|
+
}}]}}
|
|
568
|
+
|
|
569
|
+
// Export min_objective.csv - columns: HASH (UUID string), VALUE (base64 encoded)
|
|
570
|
+
// Transformations: uuid_string and encode_base64 done inline
|
|
571
|
+
def min_objective_data(:HASH, obj, h):
|
|
572
|
+
{self.MinObjective._name}(obj) and uuid_string(obj, h)
|
|
573
|
+
def min_objective_data(:VALUE, obj, e):
|
|
574
|
+
exists((s) |
|
|
575
|
+
{self.MinObjective._name}(obj) and
|
|
576
|
+
{prefix_lowercase}minobjective_serialized(obj, s) and
|
|
577
|
+
encode_base64(s, e))
|
|
578
|
+
|
|
579
|
+
def export[:min_objective]: {{export_csv[{{
|
|
580
|
+
(:path, base_path ++ "/min_objective_" ++ txn_id_str ++ ".csv");
|
|
581
|
+
(:data, min_objective_data);
|
|
582
|
+
(:compression, "gzip")
|
|
583
|
+
}}]}}
|
|
584
|
+
|
|
585
|
+
// Export max_objective.csv - columns: HASH (UUID string), VALUE (base64 encoded)
|
|
586
|
+
// Transformations: uuid_string and encode_base64 done inline
|
|
587
|
+
def max_objective_data(:HASH, obj, h):
|
|
588
|
+
{self.MaxObjective._name}(obj) and uuid_string(obj, h)
|
|
589
|
+
def max_objective_data(:VALUE, obj, e):
|
|
590
|
+
exists((s) |
|
|
591
|
+
{self.MaxObjective._name}(obj) and
|
|
592
|
+
{prefix_lowercase}maxobjective_serialized(obj, s) and
|
|
593
|
+
encode_base64(s, e))
|
|
594
|
+
|
|
595
|
+
def export[:max_objective]: {{export_csv[{{
|
|
596
|
+
(:path, base_path ++ "/max_objective_" ++ txn_id_str ++ ".csv");
|
|
597
|
+
(:data, max_objective_data);
|
|
598
|
+
(:compression, "gzip")
|
|
599
|
+
}}]}}
|
|
600
|
+
""")
|
|
601
|
+
|
|
602
|
+
executor.execute_raw(export_rel, readonly=False, query_timeout_mins=query_timeout_mins)
|
|
603
|
+
|
|
604
|
+
def _import_solver_results_from_csv(
|
|
605
|
+
self,
|
|
606
|
+
model_id: str,
|
|
607
|
+
executor: RelExecutor,
|
|
608
|
+
prefix_lowercase: str,
|
|
609
|
+
query_timeout_mins: Optional[int] = None
|
|
610
|
+
) -> None:
|
|
611
|
+
"""Import solver results from CSV files in Snowflake stage.
|
|
515
612
|
|
|
516
|
-
|
|
517
|
-
for option_key, option_value in options.items():
|
|
518
|
-
if not isinstance(option_key, str):
|
|
519
|
-
raise TypeError(
|
|
520
|
-
f"Solver option keys must be strings, but got {type(option_key).__name__} for key {option_key!r}."
|
|
521
|
-
)
|
|
522
|
-
if not isinstance(option_value, (int, float, str, bool)):
|
|
523
|
-
raise TypeError(
|
|
524
|
-
f"Solver option values must be int, float, str, or bool, "
|
|
525
|
-
f"but got {type(option_value).__name__} for option {option_key!r}."
|
|
526
|
-
)
|
|
613
|
+
Loads and extracts CSV files in a single transaction to minimize overhead.
|
|
527
614
|
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
615
|
+
Args:
|
|
616
|
+
model_id: Unique model identifier for stage paths.
|
|
617
|
+
executor: RelExecutor instance.
|
|
618
|
+
prefix_lowercase: Prefix for relation names.
|
|
619
|
+
query_timeout_mins: Query timeout in minutes.
|
|
620
|
+
"""
|
|
621
|
+
result_stage_base = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/SOLVERS/job_{model_id}/results"
|
|
622
|
+
value_parse_fn = "parse_int" if self._num_type == "int" else "parse_float"
|
|
623
|
+
|
|
624
|
+
# Single transaction: Load CSV files and extract/map results
|
|
625
|
+
# Use inline definitions to avoid needing declared relations
|
|
626
|
+
load_and_extract_rel = textwrap.dedent(f"""
|
|
627
|
+
// Define CSV loading inline (no declare needed)
|
|
628
|
+
// Load ancillary.csv - contains solver metadata (NAME, VALUE columns)
|
|
629
|
+
def ancillary_config[:path]: "{result_stage_base}/ancillary.csv.gz"
|
|
630
|
+
def ancillary_config[:syntax, :header_row]: 1
|
|
631
|
+
def ancillary_config[:schema, :NAME]: "string"
|
|
632
|
+
def ancillary_config[:schema, :VALUE]: "string"
|
|
633
|
+
def {prefix_lowercase}solver_ancillary_raw {{load_csv[ancillary_config]}}
|
|
634
|
+
|
|
635
|
+
// Load objective_values.csv - contains objective values (SOL_INDEX, VALUE columns)
|
|
636
|
+
def objective_values_config[:path]: "{result_stage_base}/objective_values.csv.gz"
|
|
637
|
+
def objective_values_config[:syntax, :header_row]: 1
|
|
638
|
+
def objective_values_config[:schema, :SOL_INDEX]: "string"
|
|
639
|
+
def objective_values_config[:schema, :VALUE]: "string"
|
|
640
|
+
def {prefix_lowercase}solver_objective_values_raw {{load_csv[objective_values_config]}}
|
|
641
|
+
|
|
642
|
+
// Load points.csv.gz - contains solution points (SOL_INDEX, VAR_HASH, VALUE columns)
|
|
643
|
+
def points_config[:path]: "{result_stage_base}/points.csv.gz"
|
|
644
|
+
def points_config[:syntax, :header_row]: 1
|
|
645
|
+
def points_config[:schema, :SOL_INDEX]: "string"
|
|
646
|
+
def points_config[:schema, :VAR_HASH]: "string"
|
|
647
|
+
def points_config[:schema, :VALUE]: "string"
|
|
648
|
+
def {prefix_lowercase}solver_points_raw {{load_csv[points_config]}}
|
|
649
|
+
|
|
650
|
+
// Clear existing result data
|
|
651
|
+
def delete[:{self.result_info._name}]: {self.result_info._name}
|
|
652
|
+
def delete[:{self.point._name}]: {self.point._name}
|
|
653
|
+
def delete[:{self.points._name}]: {self.points._name}
|
|
538
654
|
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
655
|
+
// Extract ancillary data (result info) - NAME and VALUE columns
|
|
656
|
+
def insert(:{self.result_info._name}, key, val): {{
|
|
657
|
+
exists((row) |
|
|
658
|
+
{prefix_lowercase}solver_ancillary_raw(:NAME, row, key) and
|
|
659
|
+
{prefix_lowercase}solver_ancillary_raw(:VALUE, row, val))
|
|
660
|
+
}}
|
|
543
661
|
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
"query_timeout_mins", DEFAULT_QUERY_TIMEOUT_MINS
|
|
551
|
-
)
|
|
552
|
-
)
|
|
553
|
-
is not None
|
|
554
|
-
):
|
|
555
|
-
query_timeout_mins = int(timeout_value)
|
|
556
|
-
config_file_path = getattr(config, "file_path", None)
|
|
557
|
-
start_time = time.monotonic()
|
|
558
|
-
remaining_timeout_minutes = query_timeout_mins
|
|
662
|
+
// Extract objective value from objective_values CSV (first solution)
|
|
663
|
+
def insert(:{self.result_info._name}, "objective_value", val): {{
|
|
664
|
+
exists((row) |
|
|
665
|
+
{prefix_lowercase}solver_objective_values_raw(:SOL_INDEX, row, "1") and
|
|
666
|
+
{prefix_lowercase}solver_objective_values_raw(:VALUE, row, val))
|
|
667
|
+
}}
|
|
559
668
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
669
|
+
// Extract solution points from points.csv.gz into points property
|
|
670
|
+
// This file has SOL_INDEX, VAR_HASH, VALUE columns
|
|
671
|
+
// Convert CSV string index to Int128 for points property signature
|
|
672
|
+
// Convert value to Int128 (for int) or Float64 (for float)
|
|
673
|
+
def insert(:{self.points._name}, sol_idx_int128, var, val_converted): {{
|
|
674
|
+
exists((row, sol_idx_str, var_hash_str, val_str, sol_idx_int, val) |
|
|
675
|
+
{prefix_lowercase}solver_points_raw(:SOL_INDEX, row, sol_idx_str) and
|
|
676
|
+
{prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
|
|
677
|
+
{prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
|
|
678
|
+
parse_int(sol_idx_str, sol_idx_int) and
|
|
679
|
+
parse_uuid(var_hash_str, var) and
|
|
680
|
+
{value_parse_fn}(val_str, val) and
|
|
681
|
+
::std::mirror::convert(std::mirror::typeof[Int128], sol_idx_int, sol_idx_int128) and
|
|
682
|
+
{'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
|
|
683
|
+
}}
|
|
684
|
+
|
|
685
|
+
// Extract first solution into point property (default solution)
|
|
686
|
+
// Filter to SOL_INDEX = 1
|
|
687
|
+
def insert(:{self.point._name}, var, val_converted): {{
|
|
688
|
+
exists((row, var_hash_str, val_str, val) |
|
|
689
|
+
{prefix_lowercase}solver_points_raw(:SOL_INDEX, row, "1") and
|
|
690
|
+
{prefix_lowercase}solver_points_raw(:VAR_HASH, row, var_hash_str) and
|
|
691
|
+
{prefix_lowercase}solver_points_raw(:VALUE, row, val_str) and
|
|
692
|
+
parse_uuid(var_hash_str, var) and
|
|
693
|
+
{value_parse_fn}(val_str, val) and
|
|
694
|
+
{'::std::mirror::convert(std::mirror::typeof[Int128], val, val_converted)' if self._num_type == 'int' else '::std::mirror::convert(std::mirror::typeof[Float64], val, val_converted)'})
|
|
695
|
+
}}
|
|
696
|
+
""")
|
|
697
|
+
|
|
698
|
+
executor.execute_raw(load_and_extract_rel, readonly=False, query_timeout_mins=query_timeout_mins)
|
|
699
|
+
|
|
700
|
+
def _export_model_to_protobuf(
|
|
701
|
+
self,
|
|
702
|
+
model_uri: str,
|
|
703
|
+
executor: RelExecutor,
|
|
704
|
+
prefix_lowercase: str,
|
|
705
|
+
query_timeout_mins: Optional[int] = None
|
|
706
|
+
) -> None:
|
|
707
|
+
"""Export model to protobuf format in Snowflake stage.
|
|
708
|
+
|
|
709
|
+
Args:
|
|
710
|
+
model_uri: Snowflake URI for the protobuf file.
|
|
711
|
+
executor: RelExecutor instance.
|
|
712
|
+
prefix_lowercase: Prefix for relation names.
|
|
713
|
+
query_timeout_mins: Query timeout in minutes.
|
|
714
|
+
"""
|
|
715
|
+
export_rel = f"""
|
|
567
716
|
// Collect all model components into a relation for serialization
|
|
568
717
|
def model_relation {{
|
|
569
718
|
(:variable, {self.Variable._name});
|
|
@@ -584,31 +733,24 @@ class SolverModelPB:
|
|
|
584
733
|
def export {{ config }}
|
|
585
734
|
"""
|
|
586
735
|
executor.execute_raw(
|
|
587
|
-
textwrap.dedent(
|
|
588
|
-
query_timeout_mins=
|
|
736
|
+
textwrap.dedent(export_rel),
|
|
737
|
+
query_timeout_mins=query_timeout_mins
|
|
589
738
|
)
|
|
590
739
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
job_id = solver._exec_job(
|
|
599
|
-
payload,
|
|
600
|
-
log_to_console=log_to_console,
|
|
601
|
-
query_timeout_mins=remaining_timeout_minutes,
|
|
602
|
-
)
|
|
740
|
+
def _import_solver_results_from_protobuf(
|
|
741
|
+
self,
|
|
742
|
+
job_id: str,
|
|
743
|
+
executor: RelExecutor,
|
|
744
|
+
query_timeout_mins: Optional[int] = None
|
|
745
|
+
) -> None:
|
|
746
|
+
"""Import solver results from protobuf format.
|
|
603
747
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
)
|
|
611
|
-
extract_results_relation = f"""
|
|
748
|
+
Args:
|
|
749
|
+
job_id: Job identifier for result location.
|
|
750
|
+
executor: RelExecutor instance.
|
|
751
|
+
query_timeout_mins: Query timeout in minutes.
|
|
752
|
+
"""
|
|
753
|
+
extract_rel = f"""
|
|
612
754
|
def raw_result {{
|
|
613
755
|
load_binary["snowflake://APP_STATE.RAI_INTERNAL_STAGE/job-results/{job_id}/result.binpb"]
|
|
614
756
|
}}
|
|
@@ -625,6 +767,7 @@ class SolverModelPB:
|
|
|
625
767
|
def insert(:{self.result_info._name}, key, value):
|
|
626
768
|
exists((original_key) | string(extracted[original_key], value) and ::std::mirror::lower(original_key, key))
|
|
627
769
|
"""
|
|
770
|
+
|
|
628
771
|
if self._num_type == "int":
|
|
629
772
|
insert_points_relation = f"""
|
|
630
773
|
def insert(:{self.point._name}, variable, value):
|
|
@@ -645,15 +788,123 @@ class SolverModelPB:
|
|
|
645
788
|
::std::mirror::convert(std::mirror::typeof[Int128], float_index, point_index)
|
|
646
789
|
)
|
|
647
790
|
"""
|
|
791
|
+
|
|
648
792
|
executor.execute_raw(
|
|
649
|
-
textwrap.dedent(
|
|
650
|
-
+ textwrap.dedent(insert_points_relation),
|
|
793
|
+
textwrap.dedent(extract_rel) + textwrap.dedent(insert_points_relation),
|
|
651
794
|
readonly=False,
|
|
652
|
-
query_timeout_mins=
|
|
795
|
+
query_timeout_mins=query_timeout_mins
|
|
653
796
|
)
|
|
654
797
|
|
|
655
|
-
|
|
798
|
+
def solve(
|
|
799
|
+
self, solver: Solver, log_to_console: bool = False, **kwargs: Any
|
|
800
|
+
) -> None:
|
|
801
|
+
"""Solve the model.
|
|
802
|
+
|
|
803
|
+
Args:
|
|
804
|
+
solver: Solver instance.
|
|
805
|
+
log_to_console: Whether to show solver output.
|
|
806
|
+
**kwargs: Solver options and parameters.
|
|
807
|
+
"""
|
|
808
|
+
|
|
809
|
+
use_csv_store = solver.engine_settings.get("store", {})\
|
|
810
|
+
.get("csv", {})\
|
|
811
|
+
.get("enabled", False)
|
|
812
|
+
|
|
813
|
+
print(f"Using {'csv' if use_csv_store else 'protobuf'} store...")
|
|
814
|
+
|
|
815
|
+
options = {**kwargs, "version": 1}
|
|
816
|
+
|
|
817
|
+
# Validate solver options
|
|
818
|
+
for option_key, option_value in options.items():
|
|
819
|
+
if not isinstance(option_key, str):
|
|
820
|
+
raise TypeError(
|
|
821
|
+
f"Solver option keys must be strings, but got {type(option_key).__name__} for key {option_key!r}."
|
|
822
|
+
)
|
|
823
|
+
if not isinstance(option_value, (int, float, str, bool)):
|
|
824
|
+
raise TypeError(
|
|
825
|
+
f"Solver option values must be int, float, str, or bool, "
|
|
826
|
+
f"but got {type(option_value).__name__} for option {option_key!r}."
|
|
827
|
+
)
|
|
828
|
+
|
|
829
|
+
executor = self._model._to_executor()
|
|
830
|
+
if not isinstance(executor, RelExecutor):
|
|
831
|
+
raise ValueError(f"Expected RelExecutor, got {type(executor).__name__}.")
|
|
832
|
+
prefix_lowercase = f"solvermodel_{self._id}_"
|
|
833
|
+
|
|
834
|
+
# Initialize timeout from config
|
|
835
|
+
query_timeout_mins = kwargs.get("query_timeout_mins", None)
|
|
836
|
+
config = self._model._config
|
|
837
|
+
if (
|
|
838
|
+
query_timeout_mins is None
|
|
839
|
+
and (
|
|
840
|
+
timeout_value := config.get(
|
|
841
|
+
"query_timeout_mins", DEFAULT_QUERY_TIMEOUT_MINS
|
|
842
|
+
)
|
|
843
|
+
)
|
|
844
|
+
is not None
|
|
845
|
+
):
|
|
846
|
+
query_timeout_mins = int(timeout_value)
|
|
847
|
+
config_file_path = getattr(config, "file_path", None)
|
|
848
|
+
start_time = time.monotonic()
|
|
849
|
+
|
|
850
|
+
# Force evaluation of Variable concept before export
|
|
851
|
+
b.select(b.count(self.Variable)).to_df()
|
|
852
|
+
|
|
853
|
+
# Prepare payload for solver service
|
|
854
|
+
payload: dict[str, Any] = {"solver": solver.solver_name.lower(), "options": options}
|
|
855
|
+
|
|
856
|
+
if use_csv_store:
|
|
857
|
+
# CSV format: model and results are exchanged via CSV files
|
|
858
|
+
model_id = str(uuid.uuid4()).upper().replace('-', '_')
|
|
859
|
+
payload["model_uri"] = f"snowflake://SOLVERS/job_{model_id}/model"
|
|
860
|
+
|
|
861
|
+
print("Exporting model to CSV...")
|
|
862
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
863
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
864
|
+
)
|
|
865
|
+
self._export_model_to_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
|
|
866
|
+
print("Model CSV export completed")
|
|
867
|
+
|
|
868
|
+
print("Execute solver job")
|
|
869
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
870
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
871
|
+
)
|
|
872
|
+
solver._exec_job(payload, log_to_console=log_to_console, query_timeout_mins=remaining_timeout_minutes)
|
|
873
|
+
|
|
874
|
+
print("Loading and extracting solver results...")
|
|
875
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
876
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
877
|
+
)
|
|
878
|
+
self._import_solver_results_from_csv(model_id, executor, prefix_lowercase, remaining_timeout_minutes)
|
|
879
|
+
|
|
880
|
+
else: # protobuf format
|
|
881
|
+
# Protobuf format: model and results are exchanged via binary protobuf
|
|
882
|
+
input_id = uuid.uuid4()
|
|
883
|
+
model_uri = f"snowflake://APP_STATE.RAI_INTERNAL_STAGE/job-inputs/solver/{input_id}/model.binpb"
|
|
884
|
+
sf_input_uri = f"snowflake://job-inputs/solver/{input_id}/model.binpb"
|
|
885
|
+
payload["model_uri"] = sf_input_uri
|
|
886
|
+
|
|
887
|
+
print("Export model...")
|
|
888
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
889
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
890
|
+
)
|
|
891
|
+
self._export_model_to_protobuf(model_uri, executor, prefix_lowercase, remaining_timeout_minutes)
|
|
892
|
+
|
|
893
|
+
print("Execute solver job...")
|
|
894
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
895
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
896
|
+
)
|
|
897
|
+
job_id = solver._exec_job(payload, log_to_console=log_to_console, query_timeout_mins=remaining_timeout_minutes)
|
|
898
|
+
|
|
899
|
+
print("Extract result...")
|
|
900
|
+
remaining_timeout_minutes = calc_remaining_timeout_minutes(
|
|
901
|
+
start_time, query_timeout_mins, config_file_path=config_file_path
|
|
902
|
+
)
|
|
903
|
+
self._import_solver_results_from_protobuf(job_id, executor, remaining_timeout_minutes)
|
|
904
|
+
|
|
905
|
+
print("Finished solve")
|
|
656
906
|
print()
|
|
907
|
+
return None
|
|
657
908
|
|
|
658
909
|
def load_point(self, point_index: int) -> None:
|
|
659
910
|
"""Load a solution point.
|
|
@@ -9,16 +9,15 @@ import uuid
|
|
|
9
9
|
from pandas import DataFrame
|
|
10
10
|
from typing import Any, Optional, Literal, TYPE_CHECKING
|
|
11
11
|
from snowflake.snowpark import Session
|
|
12
|
-
import v0.relationalai as rai
|
|
13
12
|
|
|
14
13
|
from v0.relationalai import debugging
|
|
15
14
|
from v0.relationalai.clients import result_helpers
|
|
16
15
|
from v0.relationalai.clients.util import IdentityParser, escape_for_f_string
|
|
17
|
-
from v0.relationalai.clients.snowflake import APP_NAME
|
|
16
|
+
from v0.relationalai.clients.resources.snowflake import APP_NAME, create_resources_instance
|
|
18
17
|
from v0.relationalai.semantics.metamodel import ir, executor as e, factory as f
|
|
19
18
|
from v0.relationalai.semantics.rel import Compiler
|
|
20
19
|
from v0.relationalai.clients.config import Config
|
|
21
|
-
from v0.relationalai.tools.constants import
|
|
20
|
+
from v0.relationalai.tools.constants import Generation, QUERY_ATTRIBUTES_HEADER
|
|
22
21
|
from v0.relationalai.tools.query_utils import prepare_metadata_for_headers
|
|
23
22
|
|
|
24
23
|
if TYPE_CHECKING:
|
|
@@ -53,15 +52,11 @@ class RelExecutor(e.Executor):
|
|
|
53
52
|
if not self._resources:
|
|
54
53
|
with debugging.span("create_session"):
|
|
55
54
|
self.dry_run |= bool(self.config.get("compiler.dry_run", False))
|
|
56
|
-
resource_class = rai.clients.snowflake.Resources
|
|
57
|
-
if self.config.get("use_direct_access", USE_DIRECT_ACCESS):
|
|
58
|
-
resource_class = rai.clients.snowflake.DirectAccessResources
|
|
59
55
|
# NOTE: language="rel" is required for Rel execution. It is the default, but
|
|
60
56
|
# we set it explicitly here to be sure.
|
|
61
|
-
self._resources =
|
|
62
|
-
dry_run=self.dry_run,
|
|
57
|
+
self._resources = create_resources_instance(
|
|
63
58
|
config=self.config,
|
|
64
|
-
|
|
59
|
+
dry_run=self.dry_run,
|
|
65
60
|
connection=self.connection,
|
|
66
61
|
language="rel",
|
|
67
62
|
)
|
|
@@ -163,13 +158,20 @@ class RelExecutor(e.Executor):
|
|
|
163
158
|
raise errors.RAIExceptionSet(all_errors)
|
|
164
159
|
|
|
165
160
|
def _export(self, raw_code: str, dest: Table, actual_cols: list[str], declared_cols: list[str], update:bool, headers: dict[str, Any] | None = None):
|
|
161
|
+
# _export is Snowflake-specific and requires Snowflake Resources
|
|
162
|
+
# It calls Snowflake stored procedures (APP_NAME.api.exec_into_table, etc.)
|
|
163
|
+
# LocalResources doesn't support this functionality
|
|
164
|
+
from v0.relationalai.clients.local import LocalResources
|
|
165
|
+
if isinstance(self.resources, LocalResources):
|
|
166
|
+
raise NotImplementedError("Export functionality is not supported in local mode. Use Snowflake Resources instead.")
|
|
167
|
+
|
|
166
168
|
_exec = self.resources._exec
|
|
167
169
|
output_table = "out" + str(uuid.uuid4()).replace("-", "_")
|
|
168
170
|
txn_id = None
|
|
169
171
|
artifacts = None
|
|
170
172
|
dest_database, dest_schema, dest_table, _ = IdentityParser(dest._fqn, require_all_parts=True).to_list()
|
|
171
173
|
dest_fqn = dest._fqn
|
|
172
|
-
assert self.resources._session
|
|
174
|
+
assert self.resources._session # All Snowflake Resources have _session
|
|
173
175
|
with debugging.span("transaction"):
|
|
174
176
|
try:
|
|
175
177
|
with debugging.span("exec_format") as span:
|
|
@@ -258,7 +260,7 @@ class RelExecutor(e.Executor):
|
|
|
258
260
|
SELECT 1
|
|
259
261
|
FROM {dest_database}.INFORMATION_SCHEMA.TABLES
|
|
260
262
|
WHERE table_schema = '{dest_schema}'
|
|
261
|
-
|
|
263
|
+
AND table_name = '{dest_table}'
|
|
262
264
|
)) THEN
|
|
263
265
|
EXECUTE IMMEDIATE 'TRUNCATE TABLE {dest_fqn}';
|
|
264
266
|
END IF;
|
|
@@ -267,6 +269,7 @@ class RelExecutor(e.Executor):
|
|
|
267
269
|
else:
|
|
268
270
|
raise e
|
|
269
271
|
if txn_id:
|
|
272
|
+
# These methods are available on all Snowflake Resources
|
|
270
273
|
artifact_info = self.resources._list_exec_async_artifacts(txn_id, headers=headers)
|
|
271
274
|
with debugging.span("fetch"):
|
|
272
275
|
artifacts = self.resources._download_results(artifact_info, txn_id, "ABORTED")
|
|
@@ -17,6 +17,7 @@ from v0.relationalai.semantics.sql.executor.result_helpers import format_columns
|
|
|
17
17
|
from v0.relationalai.semantics.metamodel.visitor import collect_by_type
|
|
18
18
|
from v0.relationalai.semantics.metamodel.typer import typer
|
|
19
19
|
from v0.relationalai.tools.constants import USE_DIRECT_ACCESS
|
|
20
|
+
from v0.relationalai.clients.resources.snowflake import Resources, DirectAccessResources, Provider
|
|
20
21
|
|
|
21
22
|
if TYPE_CHECKING:
|
|
22
23
|
from v0.relationalai.semantics.snowflake import Table
|
|
@@ -51,17 +52,20 @@ class SnowflakeExecutor(e.Executor):
|
|
|
51
52
|
if not self._resources:
|
|
52
53
|
with debugging.span("create_session"):
|
|
53
54
|
self.dry_run |= bool(self.config.get("compiler.dry_run", False))
|
|
54
|
-
resource_class =
|
|
55
|
+
resource_class: type = Resources
|
|
55
56
|
if self.config.get("use_direct_access", USE_DIRECT_ACCESS):
|
|
56
|
-
resource_class =
|
|
57
|
-
self._resources = resource_class(
|
|
58
|
-
|
|
57
|
+
resource_class = DirectAccessResources
|
|
58
|
+
self._resources = resource_class(
|
|
59
|
+
dry_run=self.dry_run, config=self.config, generation=rai.Generation.QB,
|
|
60
|
+
connection=self.connection,
|
|
61
|
+
language="sql",
|
|
62
|
+
)
|
|
59
63
|
return self._resources
|
|
60
64
|
|
|
61
65
|
@property
|
|
62
66
|
def provider(self):
|
|
63
67
|
if not self._provider:
|
|
64
|
-
self._provider =
|
|
68
|
+
self._provider = Provider(resources=self.resources)
|
|
65
69
|
return self._provider
|
|
66
70
|
|
|
67
71
|
def execute(self, model: ir.Model, task: ir.Task, format:Literal["pandas", "snowpark"]="pandas",
|
|
@@ -10,7 +10,7 @@ from v0.relationalai.semantics.tests.utils import reset_state
|
|
|
10
10
|
from v0.relationalai.semantics.internal import internal
|
|
11
11
|
from v0.relationalai.clients.result_helpers import sort_data_frame_result
|
|
12
12
|
from v0.relationalai.clients.util import IdentityParser
|
|
13
|
-
from v0.relationalai.clients.snowflake import Provider as SFProvider
|
|
13
|
+
from v0.relationalai.clients.resources.snowflake import Provider as SFProvider
|
|
14
14
|
from v0.relationalai import Provider
|
|
15
15
|
from typing import cast, Dict, Union
|
|
16
16
|
from pathlib import Path
|