relationalai 1.0.0a3__py3-none-any.whl → 1.0.0a5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- relationalai/config/config.py +47 -21
- relationalai/config/connections/__init__.py +5 -2
- relationalai/config/connections/duckdb.py +2 -2
- relationalai/config/connections/local.py +31 -0
- relationalai/config/connections/snowflake.py +0 -1
- relationalai/config/external/raiconfig_converter.py +235 -0
- relationalai/config/external/raiconfig_models.py +202 -0
- relationalai/config/external/utils.py +31 -0
- relationalai/config/shims.py +1 -0
- relationalai/semantics/__init__.py +10 -8
- relationalai/semantics/backends/sql/sql_compiler.py +1 -4
- relationalai/semantics/experimental/__init__.py +0 -0
- relationalai/semantics/experimental/builder.py +295 -0
- relationalai/semantics/experimental/builtins.py +154 -0
- relationalai/semantics/frontend/base.py +67 -42
- relationalai/semantics/frontend/core.py +34 -6
- relationalai/semantics/frontend/front_compiler.py +209 -37
- relationalai/semantics/frontend/pprint.py +6 -2
- relationalai/semantics/metamodel/__init__.py +7 -0
- relationalai/semantics/metamodel/metamodel.py +2 -0
- relationalai/semantics/metamodel/metamodel_analyzer.py +58 -16
- relationalai/semantics/metamodel/pprint.py +6 -1
- relationalai/semantics/metamodel/rewriter.py +11 -7
- relationalai/semantics/metamodel/typer.py +116 -41
- relationalai/semantics/reasoners/__init__.py +11 -0
- relationalai/semantics/reasoners/graph/__init__.py +35 -0
- relationalai/semantics/reasoners/graph/core.py +9028 -0
- relationalai/semantics/std/__init__.py +30 -10
- relationalai/semantics/std/aggregates.py +641 -12
- relationalai/semantics/std/common.py +146 -13
- relationalai/semantics/std/constraints.py +71 -1
- relationalai/semantics/std/datetime.py +904 -21
- relationalai/semantics/std/decimals.py +143 -2
- relationalai/semantics/std/floats.py +57 -4
- relationalai/semantics/std/integers.py +98 -4
- relationalai/semantics/std/math.py +857 -35
- relationalai/semantics/std/numbers.py +216 -20
- relationalai/semantics/std/re.py +213 -5
- relationalai/semantics/std/strings.py +437 -44
- relationalai/shims/executor.py +60 -52
- relationalai/shims/fixtures.py +85 -0
- relationalai/shims/helpers.py +26 -2
- relationalai/shims/hoister.py +28 -9
- relationalai/shims/mm2v0.py +204 -173
- relationalai/tools/cli/cli.py +192 -10
- relationalai/tools/cli/components/progress_reader.py +1 -1
- relationalai/tools/cli/docs.py +394 -0
- relationalai/tools/debugger.py +11 -4
- relationalai/tools/qb_debugger.py +435 -0
- relationalai/tools/typer_debugger.py +1 -2
- relationalai/util/dataclasses.py +3 -5
- relationalai/util/docutils.py +1 -2
- relationalai/util/error.py +2 -5
- relationalai/util/python.py +23 -0
- relationalai/util/runtime.py +1 -2
- relationalai/util/schema.py +2 -4
- relationalai/util/structures.py +4 -2
- relationalai/util/tracing.py +8 -2
- {relationalai-1.0.0a3.dist-info → relationalai-1.0.0a5.dist-info}/METADATA +8 -5
- {relationalai-1.0.0a3.dist-info → relationalai-1.0.0a5.dist-info}/RECORD +118 -95
- {relationalai-1.0.0a3.dist-info → relationalai-1.0.0a5.dist-info}/WHEEL +1 -1
- v0/relationalai/__init__.py +1 -1
- v0/relationalai/clients/client.py +52 -18
- v0/relationalai/clients/exec_txn_poller.py +122 -0
- v0/relationalai/clients/local.py +23 -8
- v0/relationalai/clients/resources/azure/azure.py +36 -11
- v0/relationalai/clients/resources/snowflake/__init__.py +4 -4
- v0/relationalai/clients/resources/snowflake/cli_resources.py +12 -1
- v0/relationalai/clients/resources/snowflake/direct_access_resources.py +124 -100
- v0/relationalai/clients/resources/snowflake/engine_service.py +381 -0
- v0/relationalai/clients/resources/snowflake/engine_state_handlers.py +35 -29
- v0/relationalai/clients/resources/snowflake/error_handlers.py +43 -2
- v0/relationalai/clients/resources/snowflake/snowflake.py +277 -179
- v0/relationalai/clients/resources/snowflake/use_index_poller.py +8 -0
- v0/relationalai/clients/types.py +5 -0
- v0/relationalai/errors.py +19 -1
- v0/relationalai/semantics/lqp/algorithms.py +173 -0
- v0/relationalai/semantics/lqp/builtins.py +199 -2
- v0/relationalai/semantics/lqp/executor.py +68 -37
- v0/relationalai/semantics/lqp/ir.py +28 -2
- v0/relationalai/semantics/lqp/model2lqp.py +215 -45
- v0/relationalai/semantics/lqp/passes.py +13 -658
- v0/relationalai/semantics/lqp/rewrite/__init__.py +12 -0
- v0/relationalai/semantics/lqp/rewrite/algorithm.py +385 -0
- v0/relationalai/semantics/lqp/rewrite/constants_to_vars.py +70 -0
- v0/relationalai/semantics/lqp/rewrite/deduplicate_vars.py +104 -0
- v0/relationalai/semantics/lqp/rewrite/eliminate_data.py +108 -0
- v0/relationalai/semantics/lqp/rewrite/extract_keys.py +25 -3
- v0/relationalai/semantics/lqp/rewrite/period_math.py +77 -0
- v0/relationalai/semantics/lqp/rewrite/quantify_vars.py +65 -31
- v0/relationalai/semantics/lqp/rewrite/unify_definitions.py +317 -0
- v0/relationalai/semantics/lqp/utils.py +11 -1
- v0/relationalai/semantics/lqp/validators.py +14 -1
- v0/relationalai/semantics/metamodel/builtins.py +2 -1
- v0/relationalai/semantics/metamodel/compiler.py +2 -1
- v0/relationalai/semantics/metamodel/dependency.py +12 -3
- v0/relationalai/semantics/metamodel/executor.py +11 -1
- v0/relationalai/semantics/metamodel/factory.py +2 -2
- v0/relationalai/semantics/metamodel/helpers.py +7 -0
- v0/relationalai/semantics/metamodel/ir.py +3 -2
- v0/relationalai/semantics/metamodel/rewrite/dnf_union_splitter.py +30 -20
- v0/relationalai/semantics/metamodel/rewrite/flatten.py +50 -13
- v0/relationalai/semantics/metamodel/rewrite/format_outputs.py +9 -3
- v0/relationalai/semantics/metamodel/typer/checker.py +6 -4
- v0/relationalai/semantics/metamodel/typer/typer.py +4 -3
- v0/relationalai/semantics/metamodel/visitor.py +4 -3
- v0/relationalai/semantics/reasoners/optimization/solvers_dev.py +1 -1
- v0/relationalai/semantics/reasoners/optimization/solvers_pb.py +336 -86
- v0/relationalai/semantics/rel/compiler.py +2 -1
- v0/relationalai/semantics/rel/executor.py +3 -2
- v0/relationalai/semantics/tests/lqp/__init__.py +0 -0
- v0/relationalai/semantics/tests/lqp/algorithms.py +345 -0
- v0/relationalai/tools/cli.py +339 -186
- v0/relationalai/tools/cli_controls.py +216 -67
- v0/relationalai/tools/cli_helpers.py +410 -6
- v0/relationalai/util/format.py +5 -2
- {relationalai-1.0.0a3.dist-info → relationalai-1.0.0a5.dist-info}/entry_points.txt +0 -0
- {relationalai-1.0.0a3.dist-info → relationalai-1.0.0a5.dist-info}/top_level.txt +0 -0
|
@@ -189,6 +189,9 @@ class UseIndexPoller:
|
|
|
189
189
|
# on every 5th iteration we reset the cdc status, so it will be checked again
|
|
190
190
|
self.should_check_cdc = True
|
|
191
191
|
|
|
192
|
+
# Flag to only check data stream health once in the first call
|
|
193
|
+
self.check_data_stream_health = True
|
|
194
|
+
|
|
192
195
|
self.wait_for_stream_sync = self.res.config.get(
|
|
193
196
|
"wait_for_stream_sync", WAIT_FOR_STREAM_SYNC
|
|
194
197
|
)
|
|
@@ -503,6 +506,7 @@ class UseIndexPoller:
|
|
|
503
506
|
"init_engine_async": self.init_engine_async,
|
|
504
507
|
"language": self.language,
|
|
505
508
|
"data_freshness_mins": self.data_freshness,
|
|
509
|
+
"check_data_stream_health": self.check_data_stream_health
|
|
506
510
|
})
|
|
507
511
|
|
|
508
512
|
request_headers = debugging.add_current_propagation_headers(self.headers)
|
|
@@ -535,6 +539,7 @@ class UseIndexPoller:
|
|
|
535
539
|
errors = use_index_data.get("errors", [])
|
|
536
540
|
relations = use_index_data.get("relations", {})
|
|
537
541
|
cdc_enabled = use_index_data.get("cdcEnabled", False)
|
|
542
|
+
health_checked = use_index_data.get("healthChecked", False)
|
|
538
543
|
if self.check_ready_count % ERP_CHECK_FREQUENCY == 0 or not cdc_enabled:
|
|
539
544
|
self.should_check_cdc = True
|
|
540
545
|
else:
|
|
@@ -542,6 +547,9 @@ class UseIndexPoller:
|
|
|
542
547
|
|
|
543
548
|
if engines and self.init_engine_async:
|
|
544
549
|
self.init_engine_async = False
|
|
550
|
+
|
|
551
|
+
if self.check_data_stream_health and health_checked:
|
|
552
|
+
self.check_data_stream_health = False
|
|
545
553
|
|
|
546
554
|
break_loop = False
|
|
547
555
|
has_stream_errors = False
|
v0/relationalai/clients/types.py
CHANGED
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
from abc import ABC
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from typing import Any, Optional, TypedDict
|
|
5
|
+
from typing_extensions import NotRequired
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from urllib.parse import urlparse
|
|
7
8
|
|
|
@@ -27,6 +28,7 @@ class AvailableModel(TypedDict):
|
|
|
27
28
|
|
|
28
29
|
class EngineState(TypedDict):
|
|
29
30
|
name: str
|
|
31
|
+
type: str
|
|
30
32
|
id: str
|
|
31
33
|
size: str
|
|
32
34
|
state: str
|
|
@@ -38,6 +40,9 @@ class EngineState(TypedDict):
|
|
|
38
40
|
auto_suspend: int|None
|
|
39
41
|
suspends_at: datetime|None
|
|
40
42
|
|
|
43
|
+
# Optional JSON settings (engine configuration)
|
|
44
|
+
settings: NotRequired[dict | None]
|
|
45
|
+
|
|
41
46
|
class SourceInfo(TypedDict, total=False):
|
|
42
47
|
type: str|None
|
|
43
48
|
state: str
|
v0/relationalai/errors.py
CHANGED
|
@@ -1780,7 +1780,7 @@ class SnowflakeRaiAppNotStarted(RAIException):
|
|
|
1780
1780
|
def __init__(self, app_name: str):
|
|
1781
1781
|
self.app_name = app_name
|
|
1782
1782
|
self.message = f"The RelationalAI app '{app_name}' isn't started."
|
|
1783
|
-
self.name = "The RelationalAI application
|
|
1783
|
+
self.name = "The RelationalAI application isn't running"
|
|
1784
1784
|
self.content = self.format_message()
|
|
1785
1785
|
|
|
1786
1786
|
super().__init__(self.message, self.name, self.content)
|
|
@@ -2436,6 +2436,24 @@ class QueryTimeoutExceededException(RAIException):
|
|
|
2436
2436
|
Consider increasing the 'query_timeout_mins' parameter in your configuration file{f' (stored in {self.config_file_path})' if self.config_file_path else ''} to allow more time for query execution.
|
|
2437
2437
|
""")
|
|
2438
2438
|
|
|
2439
|
+
class GuardRailsException(RAIException):
|
|
2440
|
+
def __init__(self, progress: dict[str, Any]={}):
|
|
2441
|
+
self.name = "Guard Rails Violation"
|
|
2442
|
+
self.message = "Transaction aborted due to guard rails violation."
|
|
2443
|
+
self.progress = progress
|
|
2444
|
+
self.content = self.format_message()
|
|
2445
|
+
super().__init__(self.message, self.name, self.content)
|
|
2446
|
+
|
|
2447
|
+
def format_message(self):
|
|
2448
|
+
messages = [] if self.progress else [self.message]
|
|
2449
|
+
for task in self.progress.get("tasks", {}).values():
|
|
2450
|
+
for warning_type, warning_data in task.get("warnings", {}).items():
|
|
2451
|
+
messages.append(textwrap.dedent(f"""
|
|
2452
|
+
Relation Name: [yellow]{task["task_name"]}[/yellow]
|
|
2453
|
+
Warning: {warning_type}
|
|
2454
|
+
Message: {warning_data["message"]}
|
|
2455
|
+
"""))
|
|
2456
|
+
return "\n".join(messages)
|
|
2439
2457
|
|
|
2440
2458
|
#--------------------------------------------------
|
|
2441
2459
|
# Azure Exceptions
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
from typing import TypeGuard
|
|
2
|
+
from v0.relationalai.semantics.metamodel import ir, factory, types
|
|
3
|
+
from v0.relationalai.semantics.metamodel.visitor import Rewriter, collect_by_type
|
|
4
|
+
from v0.relationalai.semantics.lqp import ir as lqp
|
|
5
|
+
from v0.relationalai.semantics.lqp.types import meta_type_to_lqp
|
|
6
|
+
from v0.relationalai.semantics.lqp.builtins import (
|
|
7
|
+
has_empty_annotation, has_assign_annotation, has_upsert_annotation,
|
|
8
|
+
has_monoid_annotation, has_monus_annotation, has_script_annotation,
|
|
9
|
+
has_algorithm_annotation, has_while_annotation, global_annotation,
|
|
10
|
+
empty_annotation, assign_annotation, upsert_annotation, monoid_annotation,
|
|
11
|
+
monus_annotation
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
# Complex tests for Loopy constructs in the metamodel
|
|
15
|
+
def is_script(task: ir.Task) -> TypeGuard[ir.Sequence]:
|
|
16
|
+
""" Check if it is a script i.e., a Sequence with @script annotation. """
|
|
17
|
+
if not isinstance(task, ir.Sequence):
|
|
18
|
+
return False
|
|
19
|
+
return has_script_annotation(task)
|
|
20
|
+
|
|
21
|
+
def is_algorithm_logical(task: ir.Task) -> TypeGuard[ir.Logical]:
|
|
22
|
+
""" Check if it is an algorithm logical i.e., a Logical task with all subtasks being
|
|
23
|
+
algorithm scripts. """
|
|
24
|
+
if not isinstance(task, ir.Logical):
|
|
25
|
+
return False
|
|
26
|
+
return all(is_algorithm_script(subtask) for subtask in task.body)
|
|
27
|
+
|
|
28
|
+
def is_algorithm_script(task: ir.Task) -> TypeGuard[ir.Sequence]:
|
|
29
|
+
""" Check if it is an algorithm script i.e., a Sequence with @script and @algorithm annotations. """
|
|
30
|
+
if not isinstance(task, ir.Sequence):
|
|
31
|
+
return False
|
|
32
|
+
return is_script(task) and has_algorithm_annotation(task)
|
|
33
|
+
|
|
34
|
+
def is_while_loop(task: ir.Task) -> TypeGuard[ir.Loop]:
|
|
35
|
+
""" Check if input is is a while loop i.e., a Loop with @while annotation. """
|
|
36
|
+
if not isinstance(task, ir.Loop):
|
|
37
|
+
return False
|
|
38
|
+
return has_while_annotation(task)
|
|
39
|
+
|
|
40
|
+
def is_while_script(task: ir.Task) -> TypeGuard[ir.Sequence]:
|
|
41
|
+
""" Check if input is a while script i.e., a Sequence with @script and @while annotations. """
|
|
42
|
+
if not isinstance(task, ir.Sequence):
|
|
43
|
+
return False
|
|
44
|
+
return is_script(task) and has_while_annotation(task)
|
|
45
|
+
|
|
46
|
+
# Tools for annotating Loopy constructs
|
|
47
|
+
class LoopyAnnoAdder(Rewriter):
|
|
48
|
+
""" Rewrites a node by adding the given annotation to all Update nodes. """
|
|
49
|
+
def __init__(self, anno: ir.Annotation):
|
|
50
|
+
self.anno = anno
|
|
51
|
+
super().__init__()
|
|
52
|
+
|
|
53
|
+
def handle_update(self, node: ir.Update, parent: ir.Node) -> ir.Update:
|
|
54
|
+
new_annos = list(node.annotations) + [self.anno]
|
|
55
|
+
return factory.update(node.relation, node.args, node.effect, new_annos, node.engine)
|
|
56
|
+
|
|
57
|
+
def mk_global(i: ir.Node):
|
|
58
|
+
return LoopyAnnoAdder(global_annotation()).walk(i)
|
|
59
|
+
|
|
60
|
+
def mk_empty(i: ir.Node):
|
|
61
|
+
return LoopyAnnoAdder(empty_annotation()).walk(i)
|
|
62
|
+
|
|
63
|
+
def mk_assign(i: ir.Node):
|
|
64
|
+
return LoopyAnnoAdder(assign_annotation()).walk(i)
|
|
65
|
+
|
|
66
|
+
def mk_upsert(i: ir.Node, arity: int):
|
|
67
|
+
return LoopyAnnoAdder(upsert_annotation(arity)).walk(i)
|
|
68
|
+
|
|
69
|
+
def mk_monoid(i: ir.Node, monoid_type: ir.ScalarType, monoid_op: str, arity: int):
|
|
70
|
+
return LoopyAnnoAdder(monoid_annotation(monoid_type, monoid_op, arity)).walk(i)
|
|
71
|
+
|
|
72
|
+
def mk_monus(i: ir.Node, monoid_type: ir.ScalarType, monoid_op: str, arity: int):
|
|
73
|
+
return LoopyAnnoAdder(monus_annotation(monoid_type, monoid_op, arity)).walk(i)
|
|
74
|
+
|
|
75
|
+
def construct_monoid(i: ir.Annotation):
|
|
76
|
+
base_type = None
|
|
77
|
+
op = None
|
|
78
|
+
for arg in i.args:
|
|
79
|
+
if isinstance(arg, ir.ScalarType):
|
|
80
|
+
base_type = meta_type_to_lqp(arg)
|
|
81
|
+
elif isinstance(arg, ir.Literal) and arg.type == types.String:
|
|
82
|
+
op = arg.value
|
|
83
|
+
assert isinstance(base_type, lqp.Type) and isinstance(op, str), "Failed to get monoid"
|
|
84
|
+
if op.lower() == "or":
|
|
85
|
+
return lqp.OrMonoid(meta=None)
|
|
86
|
+
elif op.lower() == "sum":
|
|
87
|
+
return lqp.SumMonoid(type=base_type, meta=None)
|
|
88
|
+
elif op.lower() == "min":
|
|
89
|
+
return lqp.MinMonoid(type=base_type, meta=None)
|
|
90
|
+
elif op.lower() == "max":
|
|
91
|
+
return lqp.MaxMonoid(type=base_type, meta=None)
|
|
92
|
+
else:
|
|
93
|
+
assert False, "Failed to get monoid"
|
|
94
|
+
|
|
95
|
+
# Tools for analyzing Loopy constructs
|
|
96
|
+
def is_logical_instruction(node: ir.Node) -> TypeGuard[ir.Logical]:
|
|
97
|
+
if not isinstance(node, ir.Logical):
|
|
98
|
+
return False
|
|
99
|
+
return any(collect_by_type(ir.Update, node)) and not any(collect_by_type(ir.Sequence, node))
|
|
100
|
+
|
|
101
|
+
def get_instruction_body_rels(node: ir.Logical) -> set[ir.Relation]:
|
|
102
|
+
assert is_logical_instruction(node)
|
|
103
|
+
body: set[ir.Relation] = set()
|
|
104
|
+
for update in collect_by_type(ir.Lookup, node):
|
|
105
|
+
body.add(update.relation)
|
|
106
|
+
return body
|
|
107
|
+
|
|
108
|
+
def get_instruction_head_rels(node: ir.Logical) -> set[ir.Relation]:
|
|
109
|
+
assert is_logical_instruction(node)
|
|
110
|
+
heads: set[ir.Relation] = set()
|
|
111
|
+
for update in collect_by_type(ir.Update, node):
|
|
112
|
+
heads.add(update.relation)
|
|
113
|
+
return heads
|
|
114
|
+
|
|
115
|
+
# base Loopy instruction: @empty, @assign, @upsert, @monoid, @monus
|
|
116
|
+
def is_instruction(update: ir.Task) -> TypeGuard[ir.Logical]:
|
|
117
|
+
if not is_logical_instruction(update):
|
|
118
|
+
return False
|
|
119
|
+
for u in collect_by_type(ir.Update, update):
|
|
120
|
+
if (has_empty_annotation(u) or
|
|
121
|
+
has_assign_annotation(u) or
|
|
122
|
+
has_upsert_annotation(u) or
|
|
123
|
+
has_monoid_annotation(u) or
|
|
124
|
+
has_monus_annotation(u)):
|
|
125
|
+
return True
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
# update Loopy instruction @upsert, @monoid, @monus
|
|
129
|
+
def is_update_instruction(task: ir.Task) -> TypeGuard[ir.Logical]:
|
|
130
|
+
if not is_logical_instruction(task):
|
|
131
|
+
return False
|
|
132
|
+
for u in collect_by_type(ir.Update, task):
|
|
133
|
+
if (has_upsert_annotation(u) or
|
|
134
|
+
has_monoid_annotation(u) or
|
|
135
|
+
has_monus_annotation(u)):
|
|
136
|
+
return True
|
|
137
|
+
return False
|
|
138
|
+
|
|
139
|
+
def is_empty_instruction(node: ir.Node) -> TypeGuard[ir.Logical]:
|
|
140
|
+
""" Check if input is an empty Loopy instruction `empty rel = ∅`"""
|
|
141
|
+
if not is_logical_instruction(node):
|
|
142
|
+
return False
|
|
143
|
+
updates = collect_by_type(ir.Update, node)
|
|
144
|
+
if not any(has_empty_annotation(update) for update in updates):
|
|
145
|
+
return False
|
|
146
|
+
|
|
147
|
+
# At this point, we have the prerequisites for an empty instruction. We check it is
|
|
148
|
+
# well-formed:
|
|
149
|
+
# 1. It has only a single @empty Update operation
|
|
150
|
+
# 2. Has no other operations
|
|
151
|
+
assert len(updates) == 1, "[Loopy] Empty instruction must have single Update operation"
|
|
152
|
+
assert len(node.body) == 1, "[Loopy] Empty instruction must have only a single Update operation"
|
|
153
|
+
|
|
154
|
+
return True
|
|
155
|
+
|
|
156
|
+
# Splits a Loopy instruction into its head updates, body lookups, and other body tasks
|
|
157
|
+
def split_instruction(update_logical: ir.Logical) -> tuple[ir.Update,list[ir.Lookup],list[ir.Task]]:
|
|
158
|
+
assert is_instruction(update_logical)
|
|
159
|
+
lookups = []
|
|
160
|
+
update = None
|
|
161
|
+
others = []
|
|
162
|
+
for task in update_logical.body:
|
|
163
|
+
if isinstance(task, ir.Lookup):
|
|
164
|
+
lookups.append(task)
|
|
165
|
+
elif isinstance(task, ir.Update):
|
|
166
|
+
if update is not None:
|
|
167
|
+
raise AssertionError("[Loopy] Update instruction must have exactly one Update operation")
|
|
168
|
+
update = task
|
|
169
|
+
else:
|
|
170
|
+
others.append(task)
|
|
171
|
+
assert update is not None, "[Loopy] Update instruction must have exactly one Update operation"
|
|
172
|
+
|
|
173
|
+
return update, lookups, others
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from
|
|
1
|
+
from typing import TypeGuard
|
|
2
|
+
from v0.relationalai.semantics.metamodel import factory as f, ir, types
|
|
2
3
|
from v0.relationalai.semantics.metamodel.util import FrozenOrderedSet
|
|
3
4
|
from v0.relationalai.semantics.metamodel import builtins
|
|
4
5
|
|
|
@@ -8,9 +9,205 @@ adhoc = f.relation("adhoc", [])
|
|
|
8
9
|
adhoc_annotation = f.annotation(adhoc, [])
|
|
9
10
|
|
|
10
11
|
# We only want to emit attributes for a known set of annotations.
|
|
11
|
-
|
|
12
|
+
supported_lqp_annotations = FrozenOrderedSet([
|
|
12
13
|
adhoc.name,
|
|
13
14
|
builtins.function.name,
|
|
14
15
|
builtins.track.name,
|
|
15
16
|
builtins.recursion_config.name,
|
|
16
17
|
])
|
|
18
|
+
|
|
19
|
+
# [LoopyIR] Annotations used to mark metamodel IR elements as Loopy constructs.
|
|
20
|
+
# 1. Programming structures:
|
|
21
|
+
# * @script marks Sequence blocks `begin ... end`
|
|
22
|
+
# * @algorithm additionally marks the top-level script
|
|
23
|
+
# * @while marks Loop as a `while(true) {...}`; its sole Task is a @script @while Sequence
|
|
24
|
+
# 2. Base instructions (Update's with derive Effects)
|
|
25
|
+
# * @global marks instructions that write to a global relation (only used in top-level script)
|
|
26
|
+
# * @empty marks instructions that initialize relations to an empty relation
|
|
27
|
+
# * @assign marks instructions that are standard assignments
|
|
28
|
+
# * @upsert marks instructions that perform in-place upserts
|
|
29
|
+
# * @monoid marks instructions that perform in-place monoid updates
|
|
30
|
+
# * @monus marks instructions that perform in-place monus updates
|
|
31
|
+
|
|
32
|
+
# These tasks require dedicated handling and currently are only supported in LQP.
|
|
33
|
+
|
|
34
|
+
# Here we only provide basic inspection functions. Functions for creating these annotations
|
|
35
|
+
# and more complex analysis are in the module relationalai.semantics.lqp.algorithms
|
|
36
|
+
|
|
37
|
+
# Algorithm: for top-level script of an algorithm
|
|
38
|
+
_algorithm_anno_name = "algorithm"
|
|
39
|
+
algorithm = f.relation(_algorithm_anno_name, [])
|
|
40
|
+
|
|
41
|
+
def algorithm_annotation():
|
|
42
|
+
return f.annotation(algorithm, [])
|
|
43
|
+
|
|
44
|
+
def has_algorithm_annotation(node: ir.Node) -> bool:
|
|
45
|
+
if not hasattr(node, "annotations"):
|
|
46
|
+
return False
|
|
47
|
+
annotations = getattr(node, "annotations", [])
|
|
48
|
+
for anno in annotations:
|
|
49
|
+
if anno.relation.name == _algorithm_anno_name:
|
|
50
|
+
return True
|
|
51
|
+
return False
|
|
52
|
+
|
|
53
|
+
# Script: for Sequence blocks (algorithm or while loop)
|
|
54
|
+
_script_anno_name = "script"
|
|
55
|
+
script = f.relation(_script_anno_name, [])
|
|
56
|
+
|
|
57
|
+
def script_annotation():
|
|
58
|
+
return f.annotation(script, [])
|
|
59
|
+
|
|
60
|
+
def has_script_annotation(node: ir.Node) -> bool:
|
|
61
|
+
if not hasattr(node, "annotations"):
|
|
62
|
+
return False
|
|
63
|
+
annotations = getattr(node, "annotations", [])
|
|
64
|
+
for anno in annotations:
|
|
65
|
+
if anno.relation.name == _script_anno_name:
|
|
66
|
+
return True
|
|
67
|
+
return False
|
|
68
|
+
|
|
69
|
+
# While: for a while Loop or its script body (Sequence)
|
|
70
|
+
_while_anno_name = "while"
|
|
71
|
+
while_ = f.relation(_while_anno_name, [])
|
|
72
|
+
|
|
73
|
+
def while_annotation():
|
|
74
|
+
return f.annotation(while_, [])
|
|
75
|
+
|
|
76
|
+
def has_while_annotation(node: ir.Node) -> bool:
|
|
77
|
+
if not hasattr(node, "annotations"):
|
|
78
|
+
return False
|
|
79
|
+
annotations = getattr(node, "annotations", [])
|
|
80
|
+
for anno in annotations:
|
|
81
|
+
if anno.relation.name == _while_anno_name:
|
|
82
|
+
return True
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
# Global: marks instructions that write to relation that is the result of an algorithm
|
|
86
|
+
_global_anno_name = "global"
|
|
87
|
+
global_ = f.relation(_global_anno_name, [])
|
|
88
|
+
|
|
89
|
+
def global_annotation():
|
|
90
|
+
return f.annotation(global_, [])
|
|
91
|
+
|
|
92
|
+
def has_global_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
93
|
+
if not hasattr(node, "annotations"):
|
|
94
|
+
return False
|
|
95
|
+
annotations = getattr(node, "annotations", [])
|
|
96
|
+
for anno in annotations:
|
|
97
|
+
if anno.relation.name == _global_anno_name:
|
|
98
|
+
return True
|
|
99
|
+
return False
|
|
100
|
+
|
|
101
|
+
# Empty: Initializes a relation to an empty relation
|
|
102
|
+
_empty_anno_name = "empty"
|
|
103
|
+
empty = f.relation(_empty_anno_name, [])
|
|
104
|
+
|
|
105
|
+
def empty_annotation():
|
|
106
|
+
return f.annotation(empty, [])
|
|
107
|
+
|
|
108
|
+
def has_empty_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
109
|
+
if not hasattr(node, "annotations"):
|
|
110
|
+
return False
|
|
111
|
+
annotations = getattr(node, "annotations", [])
|
|
112
|
+
for anno in annotations:
|
|
113
|
+
if anno.relation.name == _empty_anno_name:
|
|
114
|
+
return True
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
# Assign: overwrites the target relation
|
|
118
|
+
_assign_anno_name = "assign"
|
|
119
|
+
assign = f.relation(_assign_anno_name, [])
|
|
120
|
+
|
|
121
|
+
def assign_annotation():
|
|
122
|
+
return f.annotation(assign, [])
|
|
123
|
+
|
|
124
|
+
def has_assign_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
125
|
+
if not hasattr(node, "annotations"):
|
|
126
|
+
return False
|
|
127
|
+
annotations = getattr(node, "annotations", [])
|
|
128
|
+
for anno in annotations:
|
|
129
|
+
if anno.relation.name == _assign_anno_name:
|
|
130
|
+
return True
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
# Upsert: In-place update of relation
|
|
134
|
+
_upsert_anno_name = "upsert"
|
|
135
|
+
upsert = f.relation(_upsert_anno_name, [])
|
|
136
|
+
|
|
137
|
+
def upsert_annotation(arity: int):
|
|
138
|
+
return f.annotation(upsert, [f.literal(arity, type=types.Int64)])
|
|
139
|
+
|
|
140
|
+
def has_upsert_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
141
|
+
if not hasattr(node, "annotations"):
|
|
142
|
+
return False
|
|
143
|
+
annotations = getattr(node, "annotations", [])
|
|
144
|
+
for anno in annotations:
|
|
145
|
+
if anno.relation.name == _upsert_anno_name:
|
|
146
|
+
return True
|
|
147
|
+
return False
|
|
148
|
+
|
|
149
|
+
def get_upsert_annotation(i: ir.Update):
|
|
150
|
+
for anno in i.annotations:
|
|
151
|
+
if anno.relation.name == _upsert_anno_name:
|
|
152
|
+
return anno
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
# Monoid: In-place update of relation by another by a monoid operation (e.g. Integer addition)
|
|
156
|
+
_monoid_anno_name = "monoid"
|
|
157
|
+
monoid = f.relation(_monoid_anno_name, [])
|
|
158
|
+
|
|
159
|
+
def monoid_annotation(monoid_type: ir.ScalarType, monoid_op: str, arity: int):
|
|
160
|
+
return f.annotation(monoid, [f.literal(arity, type=types.Int64), monoid_type, f.literal(monoid_op, type=types.String)])
|
|
161
|
+
|
|
162
|
+
def has_monoid_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
163
|
+
if not hasattr(node, "annotations"):
|
|
164
|
+
return False
|
|
165
|
+
annotations = getattr(node, "annotations", [])
|
|
166
|
+
for anno in annotations:
|
|
167
|
+
if anno.relation.name == _monoid_anno_name:
|
|
168
|
+
return True
|
|
169
|
+
return False
|
|
170
|
+
|
|
171
|
+
def get_monoid_annotation(i: ir.Update):
|
|
172
|
+
for anno in i.annotations:
|
|
173
|
+
if anno.relation.name == _monoid_anno_name:
|
|
174
|
+
return anno
|
|
175
|
+
return None
|
|
176
|
+
|
|
177
|
+
# Monus: In-place update of relation by another by "subtraction" operation, if it exists (e.g. Integer subtraction)
|
|
178
|
+
_monus_anno_name = "monus"
|
|
179
|
+
monus = f.relation(_monus_anno_name, [])
|
|
180
|
+
|
|
181
|
+
def monus_annotation(monoid_type: ir.ScalarType, monoid_op: str, arity: int):
|
|
182
|
+
return f.annotation(monus, [f.literal(arity, type=types.Int64), monoid_type, f.literal(monoid_op, type=types.String)])
|
|
183
|
+
|
|
184
|
+
def has_monus_annotation(node: ir.Node) -> TypeGuard[ir.Update]:
|
|
185
|
+
if not hasattr(node, "annotations"):
|
|
186
|
+
return False
|
|
187
|
+
annotations = getattr(node, "annotations", [])
|
|
188
|
+
for anno in annotations:
|
|
189
|
+
if anno.relation.name == _monus_anno_name:
|
|
190
|
+
return True
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
def get_monus_annotation(i: ir.Update):
|
|
194
|
+
for anno in i.annotations:
|
|
195
|
+
if anno.relation.name == _monus_anno_name:
|
|
196
|
+
return anno
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
# Get arity from annotation (for @upsert, @monoid, and @monus)
|
|
200
|
+
def get_arity(i: ir.Annotation):
|
|
201
|
+
for arg in i.args:
|
|
202
|
+
if isinstance(arg, ir.Literal) and (arg.type == types.Int64 or arg.type == types.Int128 or arg.type == types.Number):
|
|
203
|
+
return arg.value
|
|
204
|
+
assert False, "Failed to get arity"
|
|
205
|
+
|
|
206
|
+
# All Loopy instructions
|
|
207
|
+
loopy_instructions = [
|
|
208
|
+
empty,
|
|
209
|
+
assign,
|
|
210
|
+
upsert,
|
|
211
|
+
monoid,
|
|
212
|
+
monus
|
|
213
|
+
]
|
|
@@ -12,6 +12,9 @@ from v0.relationalai import debugging
|
|
|
12
12
|
from v0.relationalai.errors import NonDefaultLQPSemanticsVersionWarning
|
|
13
13
|
from v0.relationalai.semantics.lqp import result_helpers
|
|
14
14
|
from v0.relationalai.semantics.metamodel import ir, factory as f, executor as e
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from v0.relationalai.semantics.internal.internal import Model as InternalModel
|
|
15
18
|
from v0.relationalai.semantics.lqp.compiler import Compiler
|
|
16
19
|
from v0.relationalai.semantics.lqp.intrinsics import mk_intrinsic_datetime_now
|
|
17
20
|
from v0.relationalai.semantics.lqp.constructors import mk_transaction
|
|
@@ -31,7 +34,9 @@ if TYPE_CHECKING:
|
|
|
31
34
|
|
|
32
35
|
# Whenever the logic engine introduces a breaking change in behaviour, we bump this version
|
|
33
36
|
# once the client is ready to handle it.
|
|
34
|
-
|
|
37
|
+
#
|
|
38
|
+
# [2026-01-09] bumping to 1 to opt-into hard validation errors from the engine
|
|
39
|
+
DEFAULT_LQP_SEMANTICS_VERSION = "1"
|
|
35
40
|
|
|
36
41
|
class LQPExecutor(e.Executor):
|
|
37
42
|
"""Executes LQP using the RAI client."""
|
|
@@ -278,10 +283,11 @@ class LQPExecutor(e.Executor):
|
|
|
278
283
|
fields.append(f"NULL as \"{name}\"")
|
|
279
284
|
continue
|
|
280
285
|
|
|
281
|
-
|
|
286
|
+
# Get the actual physical column name from column_fields
|
|
287
|
+
colname = column_fields[ix][0]
|
|
282
288
|
ix += 1
|
|
283
289
|
|
|
284
|
-
if colname in sample_keys:
|
|
290
|
+
if colname.lower() in sample_keys:
|
|
285
291
|
# Actual column exists in sample
|
|
286
292
|
fields.append(f"{colname} as \"{name}\"")
|
|
287
293
|
else:
|
|
@@ -400,38 +406,15 @@ class LQPExecutor(e.Executor):
|
|
|
400
406
|
txn_proto = convert_transaction(txn)
|
|
401
407
|
return final_model, export_info, txn_proto
|
|
402
408
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
df.columns = cols[: len(df.columns)]
|
|
413
|
-
|
|
414
|
-
# Process exports
|
|
415
|
-
if export_to and not self.dry_run:
|
|
416
|
-
assert cols, "No columns found in the output"
|
|
417
|
-
assert isinstance(raw_results, TransactionAsyncResponse) and raw_results.transaction, "Invalid transaction result"
|
|
418
|
-
|
|
419
|
-
result_cols = export_to._col_names
|
|
420
|
-
|
|
421
|
-
if result_cols is not None:
|
|
422
|
-
assert all(col in result_cols or col in extra_cols for col in cols)
|
|
423
|
-
else:
|
|
424
|
-
result_cols = [col for col in cols if col not in extra_cols]
|
|
425
|
-
assert result_cols
|
|
426
|
-
|
|
427
|
-
assert export_info, "Export info should be populated if we are exporting results"
|
|
428
|
-
self._export(raw_results.transaction['id'], export_info, export_to, cols, result_cols, update)
|
|
429
|
-
|
|
430
|
-
return self._postprocess_df(self.config, df, extra_cols)
|
|
431
|
-
|
|
432
|
-
def execute(self, model: ir.Model, task: ir.Task, format: Literal["pandas", "snowpark"] = "pandas",
|
|
433
|
-
export_to: Optional[Table] = None,
|
|
434
|
-
update: bool = False, meta: dict[str, Any] | None = None) -> DataFrame:
|
|
409
|
+
def execute(
|
|
410
|
+
self,
|
|
411
|
+
model: ir.Model,
|
|
412
|
+
task: ir.Task,
|
|
413
|
+
format: Literal["pandas", "snowpark", "csv"] = "pandas",
|
|
414
|
+
export_to: Optional[Table] = None,
|
|
415
|
+
update: bool = False,
|
|
416
|
+
meta: dict[str, Any] | None = None,
|
|
417
|
+
) -> DataFrame:
|
|
435
418
|
self.prepare_data()
|
|
436
419
|
previous_model = self._last_model
|
|
437
420
|
|
|
@@ -440,7 +423,7 @@ class LQPExecutor(e.Executor):
|
|
|
440
423
|
if self.dry_run:
|
|
441
424
|
return DataFrame()
|
|
442
425
|
|
|
443
|
-
if format
|
|
426
|
+
if format == "snowpark":
|
|
444
427
|
raise ValueError(f"Unsupported format: {format}")
|
|
445
428
|
|
|
446
429
|
# Format meta as headers
|
|
@@ -460,8 +443,56 @@ class LQPExecutor(e.Executor):
|
|
|
460
443
|
assert isinstance(raw_results, TransactionAsyncResponse)
|
|
461
444
|
|
|
462
445
|
try:
|
|
463
|
-
|
|
446
|
+
cols, extra_cols = self._compute_cols(task, final_model)
|
|
447
|
+
df, errs = result_helpers.format_results(raw_results, cols)
|
|
448
|
+
self.report_errors(errs)
|
|
449
|
+
|
|
450
|
+
# Rename columns if wide outputs is enabled
|
|
451
|
+
if self.wide_outputs and len(cols) - len(extra_cols) == len(df.columns):
|
|
452
|
+
df.columns = cols[: len(df.columns)]
|
|
453
|
+
|
|
454
|
+
if export_to:
|
|
455
|
+
assert cols, "No columns found in the output"
|
|
456
|
+
assert raw_results.transaction, "Invalid transaction result"
|
|
457
|
+
assert export_info, "Export info should be populated if we are exporting results"
|
|
458
|
+
result_cols = export_to._col_names
|
|
459
|
+
if result_cols is not None:
|
|
460
|
+
assert all(col in result_cols or col in extra_cols for col in cols)
|
|
461
|
+
else:
|
|
462
|
+
result_cols = [col for col in cols if col not in extra_cols]
|
|
463
|
+
assert result_cols
|
|
464
|
+
self._export(raw_results.transaction['id'], export_info, export_to, cols, result_cols, update)
|
|
465
|
+
|
|
466
|
+
if format == "csv":
|
|
467
|
+
if export_info is not None and isinstance(export_info, tuple) and isinstance(export_info[0], str):
|
|
468
|
+
return DataFrame([export_info[0]], columns=["path"])
|
|
469
|
+
else:
|
|
470
|
+
raise ValueError("The CSV export was not successful!")
|
|
471
|
+
|
|
472
|
+
return self._postprocess_df(self.config, df, extra_cols)
|
|
473
|
+
|
|
464
474
|
except Exception as e:
|
|
465
475
|
# If processing the results failed, revert to the previous model.
|
|
466
476
|
self._last_model = previous_model
|
|
467
477
|
raise e
|
|
478
|
+
|
|
479
|
+
def export_to_csv(self, model: "InternalModel", query) -> str:
|
|
480
|
+
### Exports the result of the given query fragment to a CSV file in
|
|
481
|
+
### the Snowflake stage area and returns the path to the CSV file.
|
|
482
|
+
|
|
483
|
+
from v0.relationalai.semantics.internal.internal import Fragment, with_source
|
|
484
|
+
from v0.relationalai.environments import runtime_env
|
|
485
|
+
|
|
486
|
+
if not query._select:
|
|
487
|
+
raise ValueError("Cannot export empty selection to CSV")
|
|
488
|
+
|
|
489
|
+
clone = Fragment(parent=query)
|
|
490
|
+
clone._is_export = True
|
|
491
|
+
clone._source = runtime_env.get_source_pos()
|
|
492
|
+
ir_model = model._to_ir()
|
|
493
|
+
with debugging.span("query", dsl=str(clone), **with_source(clone), meta=clone._meta):
|
|
494
|
+
query_task = model._compiler.fragment(clone)
|
|
495
|
+
csv_info = self.execute(ir_model, query_task, format="csv", meta=clone._meta)
|
|
496
|
+
path = csv_info.at[0, "path"]
|
|
497
|
+
assert isinstance(path, str)
|
|
498
|
+
return path
|
|
@@ -6,7 +6,6 @@ __all__ = [
|
|
|
6
6
|
"Declaration",
|
|
7
7
|
"FunctionalDependency",
|
|
8
8
|
"Def",
|
|
9
|
-
"Loop",
|
|
10
9
|
"Abstraction",
|
|
11
10
|
"Formula",
|
|
12
11
|
"Exists",
|
|
@@ -55,6 +54,20 @@ __all__ = [
|
|
|
55
54
|
"convert_transaction",
|
|
56
55
|
"validate_lqp",
|
|
57
56
|
"construct_configure",
|
|
57
|
+
"Algorithm",
|
|
58
|
+
"Script",
|
|
59
|
+
"Construct",
|
|
60
|
+
"Loop",
|
|
61
|
+
"Instruction",
|
|
62
|
+
"Assign",
|
|
63
|
+
"Break",
|
|
64
|
+
"Upsert",
|
|
65
|
+
"MonoidDef",
|
|
66
|
+
"MonusDef",
|
|
67
|
+
"OrMonoid",
|
|
68
|
+
"MinMonoid",
|
|
69
|
+
"MaxMonoid",
|
|
70
|
+
"SumMonoid",
|
|
58
71
|
]
|
|
59
72
|
|
|
60
73
|
from lqp.ir import (
|
|
@@ -63,7 +76,6 @@ from lqp.ir import (
|
|
|
63
76
|
Declaration,
|
|
64
77
|
FunctionalDependency,
|
|
65
78
|
Def,
|
|
66
|
-
Loop,
|
|
67
79
|
Abstraction,
|
|
68
80
|
Formula,
|
|
69
81
|
Exists,
|
|
@@ -108,6 +120,20 @@ from lqp.ir import (
|
|
|
108
120
|
Configure,
|
|
109
121
|
IVMConfig,
|
|
110
122
|
MaintenanceLevel,
|
|
123
|
+
Algorithm,
|
|
124
|
+
Script,
|
|
125
|
+
Construct,
|
|
126
|
+
Loop,
|
|
127
|
+
Instruction,
|
|
128
|
+
Assign,
|
|
129
|
+
Break,
|
|
130
|
+
Upsert,
|
|
131
|
+
MonoidDef,
|
|
132
|
+
MonusDef,
|
|
133
|
+
OrMonoid,
|
|
134
|
+
MinMonoid,
|
|
135
|
+
MaxMonoid,
|
|
136
|
+
SumMonoid,
|
|
111
137
|
)
|
|
112
138
|
|
|
113
139
|
from lqp.emit import (
|