relationalai 0.12.8__py3-none-any.whl → 0.12.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- relationalai/semantics/internal/internal.py +29 -7
- relationalai/semantics/lqp/compiler.py +1 -1
- relationalai/semantics/lqp/constructors.py +6 -0
- relationalai/semantics/lqp/executor.py +23 -38
- relationalai/semantics/lqp/intrinsics.py +4 -3
- relationalai/semantics/lqp/model2lqp.py +6 -12
- relationalai/semantics/lqp/passes.py +2 -1
- relationalai/semantics/lqp/rewrite/__init__.py +2 -1
- relationalai/semantics/lqp/rewrite/function_annotations.py +91 -56
- relationalai/semantics/lqp/rewrite/functional_dependencies.py +282 -0
- relationalai/semantics/metamodel/builtins.py +5 -0
- relationalai/semantics/metamodel/rewrite/extract_nested_logicals.py +5 -4
- relationalai/semantics/rel/compiler.py +19 -1
- relationalai/semantics/tests/test_snapshot_abstract.py +3 -0
- relationalai/util/otel_handler.py +10 -4
- {relationalai-0.12.8.dist-info → relationalai-0.12.9.dist-info}/METADATA +1 -1
- {relationalai-0.12.8.dist-info → relationalai-0.12.9.dist-info}/RECORD +20 -19
- {relationalai-0.12.8.dist-info → relationalai-0.12.9.dist-info}/WHEEL +0 -0
- {relationalai-0.12.8.dist-info → relationalai-0.12.9.dist-info}/entry_points.txt +0 -0
- {relationalai-0.12.8.dist-info → relationalai-0.12.9.dist-info}/licenses/LICENSE +0 -0
|
@@ -40,7 +40,7 @@ _global_id = peekable(itertools.count(0))
|
|
|
40
40
|
|
|
41
41
|
# Single context variable with default values
|
|
42
42
|
_overrides = ContextVar("overrides", default = {})
|
|
43
|
-
def overrides(key: str, default: bool | str | dict):
|
|
43
|
+
def overrides(key: str, default: bool | str | dict | datetime | None):
|
|
44
44
|
return _overrides.get().get(key, default)
|
|
45
45
|
|
|
46
46
|
# Flag that users set in the config or directly on the model, but that can still be
|
|
@@ -60,6 +60,13 @@ def with_overrides(**kwargs):
|
|
|
60
60
|
finally:
|
|
61
61
|
_overrides.reset(token)
|
|
62
62
|
|
|
63
|
+
# Intrinsic values to override for stable snapshots.
|
|
64
|
+
def get_intrinsic_overrides() -> dict[str, Any]:
|
|
65
|
+
datetime_now = overrides('datetime_now', None)
|
|
66
|
+
if datetime_now is not None:
|
|
67
|
+
return {'datetime_now': datetime_now}
|
|
68
|
+
return {}
|
|
69
|
+
|
|
63
70
|
#--------------------------------------------------
|
|
64
71
|
# Root tracking
|
|
65
72
|
#--------------------------------------------------
|
|
@@ -953,12 +960,25 @@ class Concept(Producer):
|
|
|
953
960
|
self._validate_identifier_relationship(rel)
|
|
954
961
|
self._add_ref_scheme(*args)
|
|
955
962
|
|
|
956
|
-
def _add_ref_scheme(self, *
|
|
957
|
-
|
|
958
|
-
#
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
963
|
+
def _add_ref_scheme(self, *rels: Relationship|RelationshipReading):
|
|
964
|
+
# thanks to prior validation we we can safely assume that
|
|
965
|
+
# * the input types are correct due to prior validation
|
|
966
|
+
# * all relationships are binary and defined on this concept
|
|
967
|
+
|
|
968
|
+
self._reference_schemes.append(rels)
|
|
969
|
+
|
|
970
|
+
# for every concept x every field f has at most one value y.
|
|
971
|
+
# f(x,y): x -> y holds
|
|
972
|
+
concept_fields = tuple([rel.__getitem__(0) for rel in rels])
|
|
973
|
+
for field in concept_fields:
|
|
974
|
+
concept_uc = Unique(field, model=self._model)
|
|
975
|
+
require(concept_uc.to_expressions())
|
|
976
|
+
|
|
977
|
+
# for any combination of field values there is at most one concept x.
|
|
978
|
+
# f₁(x,y₁) ∧ … ∧ fₙ(x,yₙ): {y₁,…,yₙ} → {x}
|
|
979
|
+
key_fields = tuple([rel.__getitem__(1) for rel in rels])
|
|
980
|
+
key_uc = Unique(*key_fields, model=self._model)
|
|
981
|
+
require(key_uc.to_expressions())
|
|
962
982
|
|
|
963
983
|
def _validate_identifier_relationship(self, rel:Relationship|RelationshipReading):
|
|
964
984
|
if rel._arity() != 2:
|
|
@@ -2603,6 +2623,7 @@ class Model():
|
|
|
2603
2623
|
config_overrides = overrides('config', {})
|
|
2604
2624
|
for k, v in config_overrides.items():
|
|
2605
2625
|
self._config.set(k, v)
|
|
2626
|
+
self._intrinsic_overrides = get_intrinsic_overrides()
|
|
2606
2627
|
self._strict = cast(bool, overrides('strict', strict))
|
|
2607
2628
|
self._use_lqp = overridable_flag('reasoner.rule.use_lqp', self._config, use_lqp, default=not self._use_sql)
|
|
2608
2629
|
self._enable_otel_handler = overridable_flag('enable_otel_handler', self._config, enable_otel_handler, default=False)
|
|
@@ -2644,6 +2665,7 @@ class Model():
|
|
|
2644
2665
|
wide_outputs=self._wide_outputs,
|
|
2645
2666
|
connection=self._connection,
|
|
2646
2667
|
config=self._config,
|
|
2668
|
+
intrinsic_overrides=self._intrinsic_overrides,
|
|
2647
2669
|
)
|
|
2648
2670
|
elif self._use_sql:
|
|
2649
2671
|
self._executor = SnowflakeExecutor(
|
|
@@ -14,7 +14,7 @@ class Compiler(c.Compiler):
|
|
|
14
14
|
super().__init__(lqp_passes())
|
|
15
15
|
self.def_names = UniqueNames()
|
|
16
16
|
|
|
17
|
-
def do_compile(self, model: ir.Model, options:dict={}) -> tuple[Optional[tuple], lqp.
|
|
17
|
+
def do_compile(self, model: ir.Model, options:dict={}) -> tuple[Optional[tuple], lqp.Epoch]:
|
|
18
18
|
fragment_id: bytes = options.get("fragment_id", bytes(404))
|
|
19
19
|
# Reset the var context for each compilation
|
|
20
20
|
# TODO: Change to unique var names per lookup
|
|
@@ -59,3 +59,9 @@ def mk_pragma(name: str, terms: list[lqp.Var]) -> lqp.Pragma:
|
|
|
59
59
|
|
|
60
60
|
def mk_attribute(name: str, args: list[lqp.Value]) -> lqp.Attribute:
|
|
61
61
|
return lqp.Attribute(name=name, args=args, meta=None)
|
|
62
|
+
|
|
63
|
+
def mk_transaction(
|
|
64
|
+
epochs: list[lqp.Epoch],
|
|
65
|
+
configure: lqp.Configure = lqp.construct_configure({}, None),
|
|
66
|
+
) -> lqp.Transaction:
|
|
67
|
+
return lqp.Transaction(epochs=epochs, configure=configure, meta=None)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
from collections import defaultdict
|
|
3
|
+
from datetime import datetime, timezone
|
|
3
4
|
import atexit
|
|
4
5
|
import re
|
|
5
6
|
|
|
@@ -13,6 +14,7 @@ from relationalai.semantics.lqp import result_helpers
|
|
|
13
14
|
from relationalai.semantics.metamodel import ir, factory as f, executor as e
|
|
14
15
|
from relationalai.semantics.lqp.compiler import Compiler
|
|
15
16
|
from relationalai.semantics.lqp.intrinsics import mk_intrinsic_datetime_now
|
|
17
|
+
from relationalai.semantics.lqp.constructors import mk_transaction
|
|
16
18
|
from relationalai.semantics.lqp.types import lqp_type_to_sql
|
|
17
19
|
from lqp import print as lqp_print, ir as lqp_ir
|
|
18
20
|
from lqp.parser import construct_configure
|
|
@@ -39,6 +41,9 @@ class LQPExecutor(e.Executor):
|
|
|
39
41
|
wide_outputs: bool = False,
|
|
40
42
|
connection: Session | None = None,
|
|
41
43
|
config: Config | None = None,
|
|
44
|
+
# In order to facilitate snapshot testing, we allow overriding intrinsic definitions
|
|
45
|
+
# like the current time, which would otherwise change between runs.
|
|
46
|
+
intrinsic_overrides: dict = {},
|
|
42
47
|
) -> None:
|
|
43
48
|
super().__init__()
|
|
44
49
|
self.database = database
|
|
@@ -48,6 +53,7 @@ class LQPExecutor(e.Executor):
|
|
|
48
53
|
self.compiler = Compiler()
|
|
49
54
|
self.connection = connection
|
|
50
55
|
self.config = config or Config()
|
|
56
|
+
self.intrinsic_overrides = intrinsic_overrides
|
|
51
57
|
self._resources = None
|
|
52
58
|
self._last_model = None
|
|
53
59
|
self._last_sources_version = (-1, None)
|
|
@@ -311,19 +317,16 @@ class LQPExecutor(e.Executor):
|
|
|
311
317
|
with debugging.span("compile_intrinsics") as span:
|
|
312
318
|
span["compile_type"] = "intrinsics"
|
|
313
319
|
|
|
320
|
+
now = self.intrinsic_overrides.get('datetime_now', datetime.now(timezone.utc))
|
|
321
|
+
|
|
314
322
|
debug_info = lqp_ir.DebugInfo(id_to_orig_name={}, meta=None)
|
|
315
323
|
intrinsics_fragment = lqp_ir.Fragment(
|
|
316
324
|
id = lqp_ir.FragmentId(id=b"__pyrel_lqp_intrinsics", meta=None),
|
|
317
|
-
declarations = [
|
|
318
|
-
mk_intrinsic_datetime_now(),
|
|
319
|
-
],
|
|
325
|
+
declarations = [mk_intrinsic_datetime_now(now)],
|
|
320
326
|
debug_info = debug_info,
|
|
321
327
|
meta = None,
|
|
322
328
|
)
|
|
323
329
|
|
|
324
|
-
|
|
325
|
-
span["lqp"] = lqp_print.to_string(intrinsics_fragment, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
326
|
-
|
|
327
330
|
return lqp_ir.Epoch(
|
|
328
331
|
writes=[
|
|
329
332
|
lqp_ir.Write(write_type=lqp_ir.Define(fragment=intrinsics_fragment, meta=None), meta=None)
|
|
@@ -354,47 +357,38 @@ class LQPExecutor(e.Executor):
|
|
|
354
357
|
|
|
355
358
|
def compile_lqp(self, model: ir.Model, task: ir.Task):
|
|
356
359
|
configure = self._construct_configure()
|
|
360
|
+
# Merge the epochs into a single transaction. Long term the query bits should all
|
|
361
|
+
# go into a WhatIf action and the intrinsics could be fused with either of them. But
|
|
362
|
+
# for now we just use separate epochs.
|
|
363
|
+
epochs = []
|
|
364
|
+
epochs.append(self._compile_intrinsics())
|
|
357
365
|
|
|
358
|
-
model_txn = None
|
|
359
366
|
if self._last_model != model:
|
|
360
367
|
with debugging.span("compile", metamodel=model) as install_span:
|
|
361
368
|
install_span["compile_type"] = "model"
|
|
362
|
-
_,
|
|
363
|
-
|
|
364
|
-
install_span["lqp"] = lqp_print.to_string(model_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
369
|
+
_, model_epoch = self.compiler.compile(model, {"fragment_id": b"model"})
|
|
370
|
+
epochs.append(model_epoch)
|
|
365
371
|
self._last_model = model
|
|
366
372
|
|
|
367
|
-
with debugging.span("compile", metamodel=task) as
|
|
368
|
-
compile_span["compile_type"] = "query"
|
|
373
|
+
with debugging.span("compile", metamodel=task) as txn_span:
|
|
369
374
|
query = f.compute_model(f.logical([task]))
|
|
370
375
|
options = {
|
|
371
376
|
"wide_outputs": self.wide_outputs,
|
|
372
377
|
"fragment_id": b"query",
|
|
373
378
|
}
|
|
374
379
|
result, final_model = self.compiler.compile_inner(query, options)
|
|
375
|
-
export_info,
|
|
376
|
-
query_txn = txn_with_configure(query_txn, configure)
|
|
377
|
-
compile_span["lqp"] = lqp_print.to_string(query_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
380
|
+
export_info, query_epoch = result
|
|
378
381
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
# for now we just use separate epochs.
|
|
382
|
-
epochs = []
|
|
382
|
+
epochs.append(query_epoch)
|
|
383
|
+
epochs.append(self._compile_undefine_query(query_epoch))
|
|
383
384
|
|
|
384
|
-
|
|
385
|
+
txn_span["compile_type"] = "query"
|
|
386
|
+
txn = mk_transaction(epochs=epochs, configure=configure)
|
|
387
|
+
txn_span["lqp"] = lqp_print.to_string(txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
385
388
|
|
|
386
|
-
if model_txn is not None:
|
|
387
|
-
epochs.append(model_txn.epochs[0])
|
|
388
|
-
|
|
389
|
-
query_txn_epoch = query_txn.epochs[0]
|
|
390
|
-
epochs.append(query_txn_epoch)
|
|
391
|
-
epochs.append(self._compile_undefine_query(query_txn_epoch))
|
|
392
|
-
|
|
393
|
-
txn = lqp_ir.Transaction(epochs=epochs, configure=configure, meta=None)
|
|
394
389
|
validate_lqp(txn)
|
|
395
390
|
|
|
396
391
|
txn_proto = convert_transaction(txn)
|
|
397
|
-
# TODO (azreika): Should export_info be encoded as part of the txn_proto? [RAI-40312]
|
|
398
392
|
return final_model, export_info, txn_proto
|
|
399
393
|
|
|
400
394
|
# TODO (azreika): This should probably be split up into exporting and other processing. There are quite a lot of arguments here...
|
|
@@ -462,12 +456,3 @@ class LQPExecutor(e.Executor):
|
|
|
462
456
|
# If processing the results failed, revert to the previous model.
|
|
463
457
|
self._last_model = previous_model
|
|
464
458
|
raise e
|
|
465
|
-
|
|
466
|
-
def txn_with_configure(txn: lqp_ir.Transaction, configure: lqp_ir.Configure) -> lqp_ir.Transaction:
|
|
467
|
-
""" Return a new transaction with the given configure. If the transaction already has
|
|
468
|
-
a configure, it is replaced. """
|
|
469
|
-
return lqp_ir.Transaction(
|
|
470
|
-
epochs=txn.epochs,
|
|
471
|
-
configure=configure,
|
|
472
|
-
meta=txn.meta,
|
|
473
|
-
)
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
from datetime import datetime
|
|
1
|
+
from datetime import datetime
|
|
2
2
|
|
|
3
3
|
from relationalai.semantics.lqp import ir as lqp
|
|
4
4
|
from relationalai.semantics.lqp.constructors import mk_abstraction, mk_value, mk_type, mk_primitive
|
|
5
5
|
from relationalai.semantics.lqp.utils import lqp_hash
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
# Constructs a definition of the current datetime.
|
|
8
|
+
def mk_intrinsic_datetime_now(dt: datetime) -> lqp.Def:
|
|
8
9
|
"""Constructs a definition of the current datetime."""
|
|
9
10
|
id = lqp_hash("__pyrel_lqp_intrinsic_datetime_now")
|
|
10
11
|
out = lqp.Var(name="out", meta=None)
|
|
11
12
|
out_type = mk_type(lqp.TypeName.DATETIME)
|
|
12
|
-
now = mk_value(lqp.DateTimeValue(value=
|
|
13
|
+
now = mk_value(lqp.DateTimeValue(value=dt, meta=None))
|
|
13
14
|
datetime_now = mk_abstraction(
|
|
14
15
|
[(out, out_type)],
|
|
15
16
|
mk_primitive("rel_primitive_eq", [out, now]),
|
|
@@ -19,8 +19,8 @@ from warnings import warn
|
|
|
19
19
|
import re
|
|
20
20
|
import uuid
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
def to_lqp(model: ir.Model, fragment_name: bytes, ctx: TranslationCtx) -> tuple[Optional[tuple], lqp.
|
|
22
|
+
# Main access point for translating metamodel to lqp. Converts the model IR to an LQP epoch.
|
|
23
|
+
def to_lqp(model: ir.Model, fragment_name: bytes, ctx: TranslationCtx) -> tuple[Optional[tuple], lqp.Epoch]:
|
|
24
24
|
assert_valid_input(model)
|
|
25
25
|
decls: list[lqp.Declaration] = []
|
|
26
26
|
reads: list[lqp.Read] = []
|
|
@@ -50,16 +50,10 @@ def to_lqp(model: ir.Model, fragment_name: bytes, ctx: TranslationCtx) -> tuple[
|
|
|
50
50
|
fragment = lqp.Fragment(id=fragment_id, declarations=decls, meta=None, debug_info=debug_info)
|
|
51
51
|
define_op = lqp.Define(fragment=fragment, meta=None)
|
|
52
52
|
|
|
53
|
-
txn = lqp.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
writes=[lqp.Write(write_type=define_op, meta=None)],
|
|
58
|
-
meta=None
|
|
59
|
-
)
|
|
60
|
-
],
|
|
61
|
-
configure=lqp.construct_configure({}, None),
|
|
62
|
-
meta=None,
|
|
53
|
+
txn = lqp.Epoch(
|
|
54
|
+
reads=reads,
|
|
55
|
+
writes=[lqp.Write(write_type=define_op, meta=None)],
|
|
56
|
+
meta=None
|
|
63
57
|
)
|
|
64
58
|
|
|
65
59
|
return (export_info, txn)
|
|
@@ -7,7 +7,7 @@ from relationalai.semantics.metamodel.util import FrozenOrderedSet
|
|
|
7
7
|
from relationalai.semantics.metamodel.rewrite import Flatten
|
|
8
8
|
|
|
9
9
|
from ..metamodel.rewrite import DischargeConstraints, DNFUnionSplitter, ExtractNestedLogicals, FormatOutputs
|
|
10
|
-
from .rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter
|
|
10
|
+
from .rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter, SplitMultiCheckRequires
|
|
11
11
|
|
|
12
12
|
from relationalai.semantics.lqp.utils import output_names
|
|
13
13
|
|
|
@@ -18,6 +18,7 @@ import hashlib
|
|
|
18
18
|
|
|
19
19
|
def lqp_passes() -> list[Pass]:
|
|
20
20
|
return [
|
|
21
|
+
SplitMultiCheckRequires(),
|
|
21
22
|
FunctionAnnotations(),
|
|
22
23
|
DischargeConstraints(),
|
|
23
24
|
Checker(),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from .cdc import CDC
|
|
2
2
|
from .extract_common import ExtractCommon
|
|
3
3
|
from .extract_keys import ExtractKeys
|
|
4
|
-
from .function_annotations import FunctionAnnotations
|
|
4
|
+
from .function_annotations import FunctionAnnotations, SplitMultiCheckRequires
|
|
5
5
|
from .quantify_vars import QuantifyVars
|
|
6
6
|
from .splinter import Splinter
|
|
7
7
|
|
|
@@ -12,4 +12,5 @@ __all__ = [
|
|
|
12
12
|
"FunctionAnnotations",
|
|
13
13
|
"QuantifyVars",
|
|
14
14
|
"Splinter",
|
|
15
|
+
"SplitMultiCheckRequires",
|
|
15
16
|
]
|
|
@@ -1,79 +1,114 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
from relationalai.semantics.metamodel import
|
|
5
|
-
from relationalai.semantics.metamodel.
|
|
3
|
+
from typing import Optional
|
|
4
|
+
from relationalai.semantics.metamodel import builtins
|
|
5
|
+
from relationalai.semantics.metamodel.ir import (
|
|
6
|
+
Node, Model, Require, Logical, Relation, Annotation, Update
|
|
7
|
+
)
|
|
8
|
+
from relationalai.semantics.metamodel.compiler import Pass
|
|
9
|
+
from relationalai.semantics.metamodel.visitor import Rewriter, Visitor
|
|
10
|
+
from relationalai.semantics.lqp.rewrite.functional_dependencies import (
|
|
11
|
+
is_valid_unique_constraint, normalized_fd
|
|
12
|
+
)
|
|
6
13
|
|
|
14
|
+
# In the future iterations of PyRel metamodel, `Require` nodes will have a single `Check`
|
|
15
|
+
# (and no `errors`). Currently, however, the unique constraints may result in multiple
|
|
16
|
+
# `Check` nodes and for simplicity we split them in to separate `Require` nodes. This step
|
|
17
|
+
# will be removed in the future.
|
|
18
|
+
#
|
|
19
|
+
# Note that unique constraints always have an empty `domain` so apply the splitting only
|
|
20
|
+
# to such `Require` nodes.
|
|
21
|
+
class SplitMultiCheckRequires(Pass):
|
|
22
|
+
"""
|
|
23
|
+
Pass splits unique Require nodes that have empty domain but multiple checks into multiple
|
|
24
|
+
Require nodes with single check each.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def rewrite(self, model: Model, options: dict = {}) -> Model:
|
|
28
|
+
return SplitMultiCheckRequiresRewriter().walk(model)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SplitMultiCheckRequiresRewriter(Rewriter):
|
|
32
|
+
"""
|
|
33
|
+
Splits unique Require nodes that have empty domain but multiple checks into multiple
|
|
34
|
+
Require nodes with single check each.
|
|
35
|
+
"""
|
|
36
|
+
def handle_require(self, node: Require, parent: Node):
|
|
7
37
|
|
|
8
|
-
|
|
38
|
+
if isinstance(node.domain, Logical) and not node.domain.body and len(node.checks) > 1:
|
|
39
|
+
require_nodes = []
|
|
40
|
+
for check in node.checks:
|
|
41
|
+
single_check = self.walk(check, node)
|
|
42
|
+
require_nodes.append(
|
|
43
|
+
node.reconstruct(node.engine, node.domain, (single_check,), node.annotations)
|
|
44
|
+
)
|
|
45
|
+
return require_nodes
|
|
46
|
+
|
|
47
|
+
return node
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class FunctionAnnotations(Pass):
|
|
9
51
|
"""
|
|
10
|
-
Pass marks all appropriate relations with `function` annotation.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
- `unique` declared for all the fields in a derived relation expect the last
|
|
52
|
+
Pass marks all appropriate relations with `function` annotation. Collects functional
|
|
53
|
+
dependencies from unique Require nodes and uses this information to identify functional
|
|
54
|
+
relations.
|
|
14
55
|
"""
|
|
15
56
|
|
|
16
|
-
def rewrite(self, model:
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
return
|
|
57
|
+
def rewrite(self, model: Model, options: dict = {}) -> Model:
|
|
58
|
+
collect_fds = CollectFDsVisitor()
|
|
59
|
+
collect_fds.visit_model(model, None)
|
|
60
|
+
annotated_model = FunctionalAnnotationsRewriter(collect_fds.functional_relations).walk(model)
|
|
61
|
+
return annotated_model
|
|
21
62
|
|
|
22
63
|
|
|
23
|
-
|
|
24
|
-
class CollectFunctionalRelationsVisitor(v.Rewriter):
|
|
64
|
+
class CollectFDsVisitor(Visitor):
|
|
25
65
|
"""
|
|
26
|
-
Visitor collects all
|
|
66
|
+
Visitor collects all unique constraints.
|
|
27
67
|
"""
|
|
28
68
|
|
|
69
|
+
# Currently, only information about k-functional fd is collected.
|
|
29
70
|
def __init__(self):
|
|
30
71
|
super().__init__()
|
|
31
|
-
self.functional_relations =
|
|
32
|
-
|
|
33
|
-
def
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
functional_rel = []
|
|
46
|
-
# mark relations as functional when at least 1 `unique` builtin
|
|
47
|
-
if len(unique_vars) > 0:
|
|
48
|
-
for item in check.body:
|
|
49
|
-
if isinstance(item, ir.Lookup) and not item.relation.name == builtins.unique.name:
|
|
50
|
-
for var_set in unique_vars:
|
|
51
|
-
# when unique declared for all the fields except the last one in the relation mark it as functional
|
|
52
|
-
if var_set == set(item.args[:-1]):
|
|
53
|
-
functional_rel.append(item.relation)
|
|
54
|
-
|
|
55
|
-
self.functional_relations.update(functional_rel)
|
|
56
|
-
return node.reconstruct(check, node.error, node.annotations)
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
@dataclass
|
|
60
|
-
class FunctionalAnnotationsVisitor(v.Rewriter):
|
|
72
|
+
self.functional_relations:dict[Relation, int] = {}
|
|
73
|
+
|
|
74
|
+
def visit_require(self, node: Require, parent: Optional[Node]):
|
|
75
|
+
if is_valid_unique_constraint(node):
|
|
76
|
+
fd = normalized_fd(node)
|
|
77
|
+
assert fd is not None
|
|
78
|
+
if fd.is_structural:
|
|
79
|
+
relation = fd.structural_relation
|
|
80
|
+
k = fd.structural_rank
|
|
81
|
+
current_k = self.functional_relations.get(relation, 0)
|
|
82
|
+
self.functional_relations[relation] = max(current_k, k)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class FunctionalAnnotationsRewriter(Rewriter):
|
|
61
86
|
"""
|
|
62
|
-
This visitor marks functional_relations with
|
|
87
|
+
This visitor marks functional_relations with `@function(:checked [, k])` annotation.
|
|
63
88
|
"""
|
|
64
89
|
|
|
65
|
-
def __init__(self, functional_relations:
|
|
90
|
+
def __init__(self, functional_relations: dict[Relation, int]):
|
|
66
91
|
super().__init__()
|
|
67
|
-
self.
|
|
92
|
+
self.functional_relations = functional_relations
|
|
93
|
+
|
|
94
|
+
def get_functional_annotation(self, rel: Relation) -> Optional[Annotation]:
|
|
95
|
+
k = self.functional_relations.get(rel, None)
|
|
96
|
+
if k is None:
|
|
97
|
+
return None
|
|
98
|
+
if k == 1:
|
|
99
|
+
return builtins.function_checked_annotation
|
|
100
|
+
return builtins.function_ranked_checked_annotation(k)
|
|
68
101
|
|
|
69
|
-
def handle_relation(self, node:
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
102
|
+
def handle_relation(self, node: Relation, parent: Node):
|
|
103
|
+
function_annotation = self.get_functional_annotation(node)
|
|
104
|
+
if function_annotation:
|
|
105
|
+
return node.reconstruct(node.name, node.fields, node.requires,
|
|
106
|
+
node.annotations | [function_annotation], node.overloads)
|
|
73
107
|
return node.reconstruct(node.name, node.fields, node.requires, node.annotations, node.overloads)
|
|
74
108
|
|
|
75
|
-
def handle_update(self, node:
|
|
76
|
-
|
|
109
|
+
def handle_update(self, node: Update, parent: Node):
|
|
110
|
+
function_annotation = self.get_functional_annotation(node.relation)
|
|
111
|
+
if function_annotation:
|
|
77
112
|
return node.reconstruct(node.engine, node.relation, node.args, node.effect,
|
|
78
|
-
node.annotations | [
|
|
113
|
+
node.annotations | [function_annotation])
|
|
79
114
|
return node.reconstruct(node.engine, node.relation, node.args, node.effect, node.annotations)
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from typing import Optional, Sequence
|
|
3
|
+
from relationalai.semantics.internal import internal
|
|
4
|
+
from relationalai.semantics.metamodel.ir import (
|
|
5
|
+
Require, Logical, Var, Relation, Lookup, ScalarType
|
|
6
|
+
)
|
|
7
|
+
from relationalai.semantics.metamodel import builtins
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
"""
|
|
11
|
+
Helper functions for converting `Require` nodes with unique constraints to functional
|
|
12
|
+
dependencies. The main functionalities provided are:
|
|
13
|
+
1. Check whether a `Require` node is a valid unique constraint representation
|
|
14
|
+
2. Represent the uniqueness constraint as a functional dependency
|
|
15
|
+
3. Check if the functional dependency is structural i.e., can be represented with
|
|
16
|
+
`@function(k)` annotation on a single relation.
|
|
17
|
+
|
|
18
|
+
=========================== Structure of unique constraints ================================
|
|
19
|
+
A `Require` node represents a _unique constraint_ if it meets the following criteria:
|
|
20
|
+
* the `Require` node's `domain` is an empty `Logical` node
|
|
21
|
+
* the `Require` node's `checks` has a single `Check` node
|
|
22
|
+
* the single `Check` node has `Logical` task that is a list of `Lookup` tasks
|
|
23
|
+
* precisely one `Lookup` task in the `Check` uses the `unique` builtin relation name
|
|
24
|
+
* the `unique` lookup has precisely one argument, which is a `TupleArg` or a `tuple`
|
|
25
|
+
containing at least one `Var`
|
|
26
|
+
* all `Lookup` nodes use variables only (no constants)
|
|
27
|
+
* the variables used in the `unique` lookup are a subset of the variables used in other
|
|
28
|
+
lookups
|
|
29
|
+
============================================================================================
|
|
30
|
+
|
|
31
|
+
We use the following unique constraint as the running example.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
Require
|
|
35
|
+
domain
|
|
36
|
+
Logical
|
|
37
|
+
checks:
|
|
38
|
+
Check
|
|
39
|
+
check:
|
|
40
|
+
Logical
|
|
41
|
+
Person(person::Person)
|
|
42
|
+
first_name(person::Person, firstname::String)
|
|
43
|
+
last_name(person::Person, lastname::String)
|
|
44
|
+
unique((firstname::String, lastname::String))
|
|
45
|
+
error:
|
|
46
|
+
...
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
=========================== Semantics of unique constraints ================================
|
|
50
|
+
A unique constraint states that the columns declared in the `unique` predicate must be
|
|
51
|
+
unique in the result of the conjunctive query consisting of all remaining predicates.
|
|
52
|
+
============================================================================================
|
|
53
|
+
|
|
54
|
+
In the running example, the conjunctive query computes a table with 3 columns, the person id
|
|
55
|
+
`person::Person`, the first name `firstname::String`, and the last name `lastname::String`.
|
|
56
|
+
The uniqueness predicate `unique((firstname::String, lastname::String))` states that no person
|
|
57
|
+
can have more than a single combination of first and last name.
|
|
58
|
+
|
|
59
|
+
The unique constraint in the running example above corresponds to the following functional
|
|
60
|
+
dependency.
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
Person(x) ∧ first_name(x, y) ∧ last_name(x, z): {y, z} -> {x}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
------------------------------ Redundant Type Atoms ----------------------------------------
|
|
67
|
+
At the time of writing, PyRel does not yet remove redundant unary atoms. For instance, in
|
|
68
|
+
the running example, the atom `Person(person::Person)` is redundant because the type of the
|
|
69
|
+
`person` variable is specified in the other two atoms `first_name` and `last_name`.
|
|
70
|
+
Consequently, we identify redundant atoms and remove them from the definition of the
|
|
71
|
+
corresponding functional dependency.
|
|
72
|
+
|
|
73
|
+
Formally, a _guard_ atom is any `Lookup` node whose relation name is not `unique`. Now, a
|
|
74
|
+
unary guard atom `T(x::T)` is _redundant_ if the uniqueness constraint has a non-unary guard
|
|
75
|
+
atom `R(...,x::T,...)`.
|
|
76
|
+
|
|
77
|
+
================================ Normalized FDs ============================================
|
|
78
|
+
Now, the _(normalized)_ functional dependency_ corresponding to a unique constraint is an
|
|
79
|
+
object of the form `φ: X → Y`, where :
|
|
80
|
+
1. `φ` is the set of all non-redundant guard atoms.
|
|
81
|
+
2. `X` is the set of variables used in the `unique` atom
|
|
82
|
+
3. `Y` is the set of all other variables used in the constraint
|
|
83
|
+
============================================================================================
|
|
84
|
+
|
|
85
|
+
The normalized functional dependency corresponding to the unique constraints from the running
|
|
86
|
+
example is :
|
|
87
|
+
```
|
|
88
|
+
first_name(person::Person, firstname::String) ∧ last_name(person::Person, lastname::String): {firstname:String, lastname:String} -> {person:Person}
|
|
89
|
+
```
|
|
90
|
+
Note that the unary atom `Person(person::Person)` is redundant and thus omitted from the
|
|
91
|
+
decomposition.
|
|
92
|
+
|
|
93
|
+
Some simple functional dependencies can, however, be expressed simply with `@function(k)`
|
|
94
|
+
attribute of a single relation. Specifically, a functional dependency `φ: X → Y` is
|
|
95
|
+
_structural_ if φ consists of a single atom `R(x1,...,xm,y1,...,yk)` and `X = {x1,...,xm}`.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
#
|
|
99
|
+
# Checks that an input `Require` node is a valid unique constraint. Returns `None` if not.
|
|
100
|
+
# If yes, we return the decomposition of the unique constraint as a tuple
|
|
101
|
+
# `(all_vars, unique_vars, guard)`, where
|
|
102
|
+
# - `all_vars` is the list of all variables used in the constraint
|
|
103
|
+
# - `unique_vars` is the list of variables used in the `unique` atom
|
|
104
|
+
# - `guard` is the list of all other `Lookup` atoms
|
|
105
|
+
#
|
|
106
|
+
def _split_unique_require_node(node: Require) -> Optional[tuple[list[Var], list[Var], list[Lookup]]]:
|
|
107
|
+
if not isinstance(node.domain, Logical):
|
|
108
|
+
return None
|
|
109
|
+
if len(node.domain.body) != 0:
|
|
110
|
+
return None
|
|
111
|
+
if len(node.checks) != 1:
|
|
112
|
+
return None
|
|
113
|
+
check = node.checks[0]
|
|
114
|
+
if not isinstance(check.check, Logical):
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
unique_atom: Optional[Lookup] = None
|
|
118
|
+
guard: list[Lookup] = []
|
|
119
|
+
for task in check.check.body:
|
|
120
|
+
if not isinstance(task, Lookup):
|
|
121
|
+
return None
|
|
122
|
+
if task.relation.name == builtins.unique.name:
|
|
123
|
+
if unique_atom is not None:
|
|
124
|
+
return None
|
|
125
|
+
unique_atom = task
|
|
126
|
+
else:
|
|
127
|
+
guard.append(task)
|
|
128
|
+
|
|
129
|
+
if unique_atom is None:
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
# collect variables
|
|
133
|
+
all_vars: set[Var] = set()
|
|
134
|
+
for lookup in guard:
|
|
135
|
+
for arg in lookup.args:
|
|
136
|
+
if not isinstance(arg, Var):
|
|
137
|
+
return None
|
|
138
|
+
all_vars.add(arg)
|
|
139
|
+
|
|
140
|
+
unique_vars: set[Var] = set()
|
|
141
|
+
if len(unique_atom.args) != 1:
|
|
142
|
+
return None
|
|
143
|
+
if not isinstance(unique_atom.args[0], (internal.TupleArg, tuple)):
|
|
144
|
+
return None
|
|
145
|
+
if len(unique_atom.args[0]) == 0:
|
|
146
|
+
return None
|
|
147
|
+
for arg in unique_atom.args[0]:
|
|
148
|
+
if not isinstance(arg, Var):
|
|
149
|
+
return None
|
|
150
|
+
unique_vars.add(arg)
|
|
151
|
+
|
|
152
|
+
# check that unique vars are a subset of other vars
|
|
153
|
+
if not unique_vars.issubset(all_vars):
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
return list(all_vars), list(unique_vars), guard
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def is_valid_unique_constraint(node: Require) -> bool:
|
|
160
|
+
"""
|
|
161
|
+
Checks whether the input `Require` node is a valid unique constraint. See description at
|
|
162
|
+
the top of the file for details.
|
|
163
|
+
"""
|
|
164
|
+
return _split_unique_require_node(node) is not None
|
|
165
|
+
|
|
166
|
+
#
|
|
167
|
+
# A unary guard atom `T(x::T)` is redundant if the constraint contains a non-unary atom
|
|
168
|
+
# `R(...,x::T,...)`. We discard all redundant guard atoms in the constructed fd.
|
|
169
|
+
#
|
|
170
|
+
def normalized_fd(node: Require) -> Optional[FunctionalDependency]:
|
|
171
|
+
"""
|
|
172
|
+
If the input `Require` node is a uniqueness constraint, constructs its reduced
|
|
173
|
+
functional dependency `φ: X -> Y`, where `φ` contains all non-redundant guard atoms,
|
|
174
|
+
`X` are the variables used in the `unique` atom, and `Y` are the remaining variables.
|
|
175
|
+
Returns `None` if the input node is not a valid uniqueness constraint.
|
|
176
|
+
"""
|
|
177
|
+
parts = _split_unique_require_node(node)
|
|
178
|
+
if parts is None:
|
|
179
|
+
return None
|
|
180
|
+
all_vars, unique_vars, guard_atoms = parts
|
|
181
|
+
|
|
182
|
+
# remove redundant lookups
|
|
183
|
+
redundant_guard_atoms: list[Lookup] = []
|
|
184
|
+
for atom in guard_atoms:
|
|
185
|
+
# the atom is unary A(x::T)
|
|
186
|
+
if len(atom.args) != 1:
|
|
187
|
+
continue
|
|
188
|
+
var = atom.args[0]
|
|
189
|
+
assert isinstance(var, Var)
|
|
190
|
+
# T is a scalar type (which includes entity types)
|
|
191
|
+
var_type = var.type
|
|
192
|
+
if not isinstance(var_type, ScalarType):
|
|
193
|
+
continue
|
|
194
|
+
# the atom is a entity typing T(x::T) i.e., T = A (and hence not a Boolean property)
|
|
195
|
+
var_type_name = var_type.name
|
|
196
|
+
rel_name = atom.relation.name
|
|
197
|
+
if rel_name != var_type_name:
|
|
198
|
+
continue
|
|
199
|
+
# Found an atom of the form T(x::T)
|
|
200
|
+
# check if there is another atom R(...,x::T,...)
|
|
201
|
+
for typed_atom in guard_atoms:
|
|
202
|
+
if len(typed_atom.args) == 1:
|
|
203
|
+
continue
|
|
204
|
+
if var in typed_atom.args:
|
|
205
|
+
redundant_guard_atoms.append(atom)
|
|
206
|
+
break
|
|
207
|
+
|
|
208
|
+
guard = [atom for atom in guard_atoms if atom not in redundant_guard_atoms]
|
|
209
|
+
keys = unique_vars
|
|
210
|
+
values = [v for v in all_vars if v not in keys]
|
|
211
|
+
|
|
212
|
+
return FunctionalDependency(guard, keys, values)
|
|
213
|
+
|
|
214
|
+
class FunctionalDependency:
|
|
215
|
+
"""
|
|
216
|
+
Represents a functional dependency of the form `φ: X -> Y`, where
|
|
217
|
+
- `φ` is a set of `Lookup` atoms
|
|
218
|
+
- `X` and `Y` are disjoint and covering sets of variables used in `φ`
|
|
219
|
+
"""
|
|
220
|
+
def __init__(self, guard: Sequence[Lookup], keys: Sequence[Var], values: Sequence[Var]):
|
|
221
|
+
self.guard = frozenset(guard)
|
|
222
|
+
self.keys = frozenset(keys)
|
|
223
|
+
self.values = frozenset(values)
|
|
224
|
+
assert self.keys.isdisjoint(self.values), "Keys and values must be disjoint"
|
|
225
|
+
|
|
226
|
+
# for structural fd check
|
|
227
|
+
self._is_structural:bool = False
|
|
228
|
+
self._structural_relation:Optional[Relation] = None
|
|
229
|
+
self._structural_rank:Optional[int] = None
|
|
230
|
+
|
|
231
|
+
self._determine_is_structural()
|
|
232
|
+
|
|
233
|
+
# A functional dependency `φ: X → Y` is _k-functional_ if `φ` consists of a single atom
|
|
234
|
+
# `R(x1,...,xm,y1,...,yk)` and `X = {x1,...,xm}`. Not all functional dependencies are
|
|
235
|
+
# k-functional. For instance, `R(x, y, z): {y, z} → {x}` cannot be expressed with
|
|
236
|
+
# `@function`. neither can `R(x, y) ∧ P(x, z) : {x} → {y, z}`.
|
|
237
|
+
def _determine_is_structural(self):
|
|
238
|
+
if len(self.guard) != 1:
|
|
239
|
+
self._is_structural = False
|
|
240
|
+
return
|
|
241
|
+
atom = next(iter(self.guard))
|
|
242
|
+
atom_vars = atom.args
|
|
243
|
+
if len(atom_vars) <= len(self.keys): # @function(0) provides no information
|
|
244
|
+
self._is_structural = False
|
|
245
|
+
return
|
|
246
|
+
prefix_vars = atom_vars[:len(self.keys)]
|
|
247
|
+
if set(prefix_vars) != set(self.keys):
|
|
248
|
+
self._is_structural = False
|
|
249
|
+
return
|
|
250
|
+
self._is_structural = True
|
|
251
|
+
self._structural_relation = atom.relation
|
|
252
|
+
self._structural_rank = len(atom_vars) - len(self.keys)
|
|
253
|
+
|
|
254
|
+
@property
|
|
255
|
+
def is_structural(self) -> bool:
|
|
256
|
+
"""
|
|
257
|
+
Whether the functional dependency is functional, i.e., can be represented
|
|
258
|
+
with `@function(k)` annotation on a single relation.
|
|
259
|
+
"""
|
|
260
|
+
return self._is_structural
|
|
261
|
+
|
|
262
|
+
@property
|
|
263
|
+
def structural_relation(self) -> Relation:
|
|
264
|
+
"""
|
|
265
|
+
The structural relation of a functional dependency. Raises ValueError if the functional
|
|
266
|
+
dependency is not structural.
|
|
267
|
+
"""
|
|
268
|
+
if not self._is_structural:
|
|
269
|
+
raise ValueError("Functional dependency is not structural")
|
|
270
|
+
assert self._structural_relation is not None
|
|
271
|
+
return self._structural_relation
|
|
272
|
+
|
|
273
|
+
@property
|
|
274
|
+
def structural_rank(self) -> int:
|
|
275
|
+
"""
|
|
276
|
+
The structural rank k of k-structural fd. Raises ValueError if the structural
|
|
277
|
+
dependency is not k-structural.
|
|
278
|
+
"""
|
|
279
|
+
if not self._is_structural:
|
|
280
|
+
raise ValueError("Functional dependency is not structural")
|
|
281
|
+
assert self._structural_rank is not None
|
|
282
|
+
return self._structural_rank
|
|
@@ -495,6 +495,11 @@ output_keys_annotation = f.annotation(output_keys, [])
|
|
|
495
495
|
function = f.relation("function", [f.input_field("code", types.Symbol)])
|
|
496
496
|
function_checked_annotation = f.annotation(function, [f.lit("checked")])
|
|
497
497
|
function_annotation = f.annotation(function, [])
|
|
498
|
+
function_ranked = f.relation("function", [f.input_field("code", types.Symbol), f.input_field("rank", types.Int64)])
|
|
499
|
+
def function_ranked_checked_annotation(k:int) -> ir.Annotation:
|
|
500
|
+
return f.annotation(function_ranked, [f.lit("checked"), f.lit(k)])
|
|
501
|
+
def function_ranked_annotation(k:int) -> ir.Annotation:
|
|
502
|
+
return f.annotation(function_ranked, [f.lit(k)])
|
|
498
503
|
|
|
499
504
|
# Indicates this relation should be tracked in telemetry. Supported for Relationships and Concepts.
|
|
500
505
|
# `RAI_BackIR.with_relation_tracking` produces log messages at the start and end of each
|
|
@@ -48,10 +48,10 @@ class LogicalExtractor(Rewriter):
|
|
|
48
48
|
# variables (which is currently done by flatten), such as when the parent is a Match
|
|
49
49
|
# or a Union, of if the logical has a Rank.
|
|
50
50
|
if not (
|
|
51
|
-
|
|
51
|
+
logical.hoisted and
|
|
52
52
|
not isinstance(parent, (ir.Match, ir.Union)) and
|
|
53
|
-
all(isinstance(v, ir.Var) for v in
|
|
54
|
-
not any(isinstance(c, ir.Rank) for c in
|
|
53
|
+
all(isinstance(v, ir.Var) for v in logical.hoisted) and
|
|
54
|
+
not any(isinstance(c, ir.Rank) for c in logical.body)
|
|
55
55
|
):
|
|
56
56
|
return logical
|
|
57
57
|
|
|
@@ -61,10 +61,11 @@ class LogicalExtractor(Rewriter):
|
|
|
61
61
|
|
|
62
62
|
# if there are aggregations, make sure we don't expose the projected and input vars,
|
|
63
63
|
# but expose groupbys
|
|
64
|
-
for agg in collect_by_type(ir.Aggregate,
|
|
64
|
+
for agg in collect_by_type(ir.Aggregate, logical):
|
|
65
65
|
exposed_vars.difference_update(agg.projection)
|
|
66
66
|
exposed_vars.difference_update(helpers.aggregate_inputs(agg))
|
|
67
67
|
exposed_vars.update(agg.group)
|
|
68
|
+
|
|
68
69
|
# add the values (hoisted)
|
|
69
70
|
exposed_vars.update(helpers.hoisted_vars(logical.hoisted))
|
|
70
71
|
|
|
@@ -12,7 +12,7 @@ from relationalai.semantics.metamodel.util import OrderedSet, group_by, NameCach
|
|
|
12
12
|
from relationalai.semantics.rel import rel, rel_utils as u, builtins as rel_bt
|
|
13
13
|
|
|
14
14
|
from ..metamodel.rewrite import (Flatten, ExtractNestedLogicals, DNFUnionSplitter, DischargeConstraints, FormatOutputs)
|
|
15
|
-
from ..lqp.rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter
|
|
15
|
+
from ..lqp.rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter, SplitMultiCheckRequires
|
|
16
16
|
|
|
17
17
|
import math
|
|
18
18
|
|
|
@@ -24,6 +24,7 @@ import math
|
|
|
24
24
|
class Compiler(c.Compiler):
|
|
25
25
|
def __init__(self):
|
|
26
26
|
super().__init__([
|
|
27
|
+
SplitMultiCheckRequires(),
|
|
27
28
|
FunctionAnnotations(),
|
|
28
29
|
DischargeConstraints(),
|
|
29
30
|
Checker(),
|
|
@@ -696,7 +697,24 @@ class ModelToRel:
|
|
|
696
697
|
rel_annos = cast(Tuple[rel.Annotation, ...], self.handle_list(filtered_annos))
|
|
697
698
|
return rel_annos
|
|
698
699
|
|
|
700
|
+
# standard handling mistreats the integer arg of ranked `@function(:checked, k)`
|
|
701
|
+
def handle_ranked_function_annotation(self, n: ir.Annotation):
|
|
702
|
+
assert n.relation == bt.function_ranked and len(n.args) == 2
|
|
703
|
+
checked_lit = n.args[0]
|
|
704
|
+
rank_lit = n.args[1]
|
|
705
|
+
assert isinstance(checked_lit, ir.Literal) and isinstance(rank_lit, ir.Literal)
|
|
706
|
+
checked = rel.MetaValue(checked_lit.value)
|
|
707
|
+
rank = rank_lit.value
|
|
708
|
+
return rel.Annotation(
|
|
709
|
+
n.relation.name,
|
|
710
|
+
(checked, rank)
|
|
711
|
+
)
|
|
712
|
+
|
|
699
713
|
def handle_annotation(self, n: ir.Annotation):
|
|
714
|
+
# special treatment for (ranked) @function(:checked, k)
|
|
715
|
+
if n.relation == bt.function_ranked:
|
|
716
|
+
return self.handle_ranked_function_annotation(n)
|
|
717
|
+
|
|
700
718
|
# we know that annotations won't have vars, so we can ignore that type warning
|
|
701
719
|
return rel.Annotation(
|
|
702
720
|
n.relation.name,
|
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
import os
|
|
3
3
|
import uuid
|
|
4
4
|
import sys
|
|
5
|
+
import datetime
|
|
5
6
|
from abc import abstractmethod, ABC
|
|
6
7
|
|
|
7
8
|
from relationalai.semantics.tests.logging import Capturer
|
|
@@ -47,6 +48,8 @@ class AbstractSnapshotTest(ABC):
|
|
|
47
48
|
'use_sql': use_sql,
|
|
48
49
|
'reasoner.rule.use_lqp': use_lqp,
|
|
49
50
|
'keep_model': False,
|
|
51
|
+
# fix the current time to keep snapshots stable
|
|
52
|
+
'datetime_now': datetime.datetime.fromisoformat("2025-12-01T12:00:00+00:00"),
|
|
50
53
|
}
|
|
51
54
|
if use_direct_access:
|
|
52
55
|
# for direct access we can not use user/password authentication
|
|
@@ -20,6 +20,7 @@ from relationalai.debugging import logger, Span, filter_span_attrs, otel_traceid
|
|
|
20
20
|
from relationalai.clients.snowflake import Resources
|
|
21
21
|
|
|
22
22
|
MAX_PAYLOAD_SIZE = 25*1024 # 25KB
|
|
23
|
+
MAX_ATTRIBUTE_LENGTH = 1000
|
|
23
24
|
_otel_initialized = False
|
|
24
25
|
|
|
25
26
|
|
|
@@ -44,8 +45,6 @@ class CachedSpanExporter(SpanExporter):
|
|
|
44
45
|
|
|
45
46
|
def get_spans(self):
|
|
46
47
|
return self.spans
|
|
47
|
-
|
|
48
|
-
|
|
49
48
|
class NativeAppSpanExporter(SpanExporter):
|
|
50
49
|
def __init__(self, resource:Resources, app_name:str):
|
|
51
50
|
self.resource = resource
|
|
@@ -252,6 +251,12 @@ def encode_otlp_attribute(k, v):
|
|
|
252
251
|
try:
|
|
253
252
|
valueObj = {}
|
|
254
253
|
if isinstance(v, str):
|
|
254
|
+
# Truncate metamodel attribute value if it's too large
|
|
255
|
+
if k == "metamodel" and len(v) > MAX_ATTRIBUTE_LENGTH:
|
|
256
|
+
original_length = len(v)
|
|
257
|
+
v = v[:MAX_ATTRIBUTE_LENGTH] + f"... (truncated from {original_length} chars)"
|
|
258
|
+
logging.warning(f"{k} attribute truncated from {original_length} to {MAX_ATTRIBUTE_LENGTH} characters")
|
|
259
|
+
|
|
255
260
|
valueObj = {
|
|
256
261
|
"stringValue": html.escape(v).replace("\n", "; ")
|
|
257
262
|
}
|
|
@@ -369,7 +374,7 @@ def encode_spans_to_otlp_json(spans: Sequence[ReadableSpan], max_bytes: int) ->
|
|
|
369
374
|
except Exception as e:
|
|
370
375
|
logging.warning(f"Error encoding resource to OTLP JSON: {e}")
|
|
371
376
|
return "", spans # Return the unencoded spans if resource encoding fails
|
|
372
|
-
|
|
377
|
+
|
|
373
378
|
# TODO encode pyrel version for real
|
|
374
379
|
header_str = f'{{"resourceSpans": [{{"resource":{resource_str}, "scopeSpans": [{{"scope":{{"name":"pyrel", "version":"v0.4.0"}},"spans":['
|
|
375
380
|
footer_str = ']}]}]}'
|
|
@@ -378,7 +383,7 @@ def encode_spans_to_otlp_json(spans: Sequence[ReadableSpan], max_bytes: int) ->
|
|
|
378
383
|
encoded_spans = []
|
|
379
384
|
for span in spans:
|
|
380
385
|
try:
|
|
381
|
-
span_str = encode_span_to_otlp_json(span)
|
|
386
|
+
span_str = encode_span_to_otlp_json(span)
|
|
382
387
|
except Exception as e:
|
|
383
388
|
logging.warning(f"Error encoding span {span}: {e}")
|
|
384
389
|
continue # Skip this span if encoding fails
|
|
@@ -460,6 +465,7 @@ def enable_otel_export(client_resources: Resources, app_name):
|
|
|
460
465
|
if _otel_initialized:
|
|
461
466
|
logging.warning("OTel already initialized, skipping.")
|
|
462
467
|
return
|
|
468
|
+
|
|
463
469
|
_otel_initialized = True
|
|
464
470
|
|
|
465
471
|
if TRACE_PROVIDER is None:
|
|
@@ -302,32 +302,33 @@ relationalai/semantics/devtools/compilation_manager.py,sha256=XBqG_nYWtK3s_J6MeC
|
|
|
302
302
|
relationalai/semantics/devtools/extract_lqp.py,sha256=gxI3EvPUTPAkwgnkCKAkEm2vA6QkLfoM8AXXiVz0c34,3696
|
|
303
303
|
relationalai/semantics/internal/__init__.py,sha256=JXrpFaL-fdZrvKpWTEn1UoLXITOoTGnAYwmgeiglhSk,774
|
|
304
304
|
relationalai/semantics/internal/annotations.py,sha256=PkrRN-gHO2ksh1hDKB1VVIB39dONvLdTd8_Y0rCR3fE,367
|
|
305
|
-
relationalai/semantics/internal/internal.py,sha256=
|
|
305
|
+
relationalai/semantics/internal/internal.py,sha256=hElTyxtnTfNsmX9kJZ2I5yU4yA3udFyu2xk-qq8G3T0,149986
|
|
306
306
|
relationalai/semantics/internal/snowflake.py,sha256=8D6WYDFKtt8R-sc9o1Oxgtl6Xwehs2Txw_lKNBid7UA,13467
|
|
307
307
|
relationalai/semantics/lqp/__init__.py,sha256=XgcQZxK-zz_LqPDVtwREhsIvjTuUIt4BZhIedCeMY-s,48
|
|
308
308
|
relationalai/semantics/lqp/builtins.py,sha256=IWRYJ1J-HGEQqBn8QVOyjZvgEiq6W9tZ0nBLdHz5wjA,576
|
|
309
|
-
relationalai/semantics/lqp/compiler.py,sha256=
|
|
310
|
-
relationalai/semantics/lqp/constructors.py,sha256=
|
|
311
|
-
relationalai/semantics/lqp/executor.py,sha256=
|
|
312
|
-
relationalai/semantics/lqp/intrinsics.py,sha256=
|
|
309
|
+
relationalai/semantics/lqp/compiler.py,sha256=oOGlN03NVvltKN5KSDOqvb0TsAER_igUe9CnOcHAuoY,943
|
|
310
|
+
relationalai/semantics/lqp/constructors.py,sha256=KWXW3OxpZE8E-OYYjrjE_uFAXRekuGHn0TNSPPQyY7g,2336
|
|
311
|
+
relationalai/semantics/lqp/executor.py,sha256=7Jk2eKlGZ0H4wXInjnHKZehDVGOq09W6zY1mFALwIyE,21160
|
|
312
|
+
relationalai/semantics/lqp/intrinsics.py,sha256=oKPIcW8PYgU-yPTO21iSF00RBsFKPFFP5MICe6izjKk,871
|
|
313
313
|
relationalai/semantics/lqp/ir.py,sha256=DUw0ltul0AS9CRjntNlmllWTwXpxMyYg4iJ9t7NFYMA,1791
|
|
314
|
-
relationalai/semantics/lqp/model2lqp.py,sha256=
|
|
315
|
-
relationalai/semantics/lqp/passes.py,sha256=
|
|
314
|
+
relationalai/semantics/lqp/model2lqp.py,sha256=mtxXXOMWTMrLYyB84x0OqdmwAACfQ7au1IhiKZ2_lqY,35262
|
|
315
|
+
relationalai/semantics/lqp/passes.py,sha256=WNVuGxf1jU-UYyoEW5XvnREwJ-NW4_iMECW-Gaphaq8,28531
|
|
316
316
|
relationalai/semantics/lqp/pragmas.py,sha256=FzzldrJEAZ1AIcEw6D-FfaVg3CoahRYgPCFo7xHfg1g,375
|
|
317
317
|
relationalai/semantics/lqp/primitives.py,sha256=9Hjow-Yp06jt0xatuUrH1dw0ErnzknIr9K0TB_AwdjU,11029
|
|
318
318
|
relationalai/semantics/lqp/result_helpers.py,sha256=oYpLoTBnzsiyOVIWA2rLMHlgs7P7BoEkqthQ2aMosnk,10123
|
|
319
319
|
relationalai/semantics/lqp/types.py,sha256=3TZ61ybwNV8lDyUMujZIWNFz3Fgn4uifsJb8ExfoMDg,4508
|
|
320
320
|
relationalai/semantics/lqp/utils.py,sha256=iOoS-f8kyFjrgAnpK4cWDvAA-WmPgDRggSKUXm_JdTc,6317
|
|
321
321
|
relationalai/semantics/lqp/validators.py,sha256=YO_ciSgEVNILWUbkxIagKpIxI4oqV0fRSTO2Ok0rPJk,1526
|
|
322
|
-
relationalai/semantics/lqp/rewrite/__init__.py,sha256=
|
|
322
|
+
relationalai/semantics/lqp/rewrite/__init__.py,sha256=3LxYHZm6-vaqs-5Co9DQ8awxk838f5huiR5OEaLc8Ww,411
|
|
323
323
|
relationalai/semantics/lqp/rewrite/cdc.py,sha256=I6DeMOZScx-3UAVoSCMn9cuOgLzwdvJVKNwsgFa6R_k,10390
|
|
324
324
|
relationalai/semantics/lqp/rewrite/extract_common.py,sha256=sbihURqk4wtc1ekDWXWltq9LrO42XTLfOHl5D6nT5vw,18371
|
|
325
325
|
relationalai/semantics/lqp/rewrite/extract_keys.py,sha256=dSr5SVkYmrhiR0XPY5eRAnWD66dcZYgXdilXcERv634,18682
|
|
326
|
-
relationalai/semantics/lqp/rewrite/function_annotations.py,sha256=
|
|
326
|
+
relationalai/semantics/lqp/rewrite/function_annotations.py,sha256=9ZzLASvXh_OgQ04eup0AyoMIh2HxWHkoRETLm1-XtWs,4660
|
|
327
|
+
relationalai/semantics/lqp/rewrite/functional_dependencies.py,sha256=pQo7a7oS1wsep3ObdxPPaV962RlR0iBWzBmd7lCzCvQ,11564
|
|
327
328
|
relationalai/semantics/lqp/rewrite/quantify_vars.py,sha256=wYMEXzCW_D_Y_1rSLvuAAqw9KN1oIOn_vIMxELzRVb4,11568
|
|
328
329
|
relationalai/semantics/lqp/rewrite/splinter.py,sha256=oeDjP_F2PVLVexAKFn8w7CLtO9oy-R-tS2IOmzw_Ujk,3199
|
|
329
330
|
relationalai/semantics/metamodel/__init__.py,sha256=I-XqQAGycD0nKkKYvnF3G9d0QK_1LIM4xXICw8g8fBA,805
|
|
330
|
-
relationalai/semantics/metamodel/builtins.py,sha256=
|
|
331
|
+
relationalai/semantics/metamodel/builtins.py,sha256=lW8VdaSICyr8gxhFzaE-fD8wHSLBuErDOzntEC9FNL0,38407
|
|
331
332
|
relationalai/semantics/metamodel/compiler.py,sha256=XBsAnbFwgZ_TcRry6yXGWLyw_MaO2WJDp1EnC_ubhps,4525
|
|
332
333
|
relationalai/semantics/metamodel/dataflow.py,sha256=wfj1tARrR4yEAaTwUTrAcxEcz81VkUal4U_AX1esovk,3929
|
|
333
334
|
relationalai/semantics/metamodel/dependency.py,sha256=iJLx-w_zqde7CtbGcXxLxZBdUKZYl7AUykezPI9ccck,33926
|
|
@@ -341,7 +342,7 @@ relationalai/semantics/metamodel/visitor.py,sha256=DFY0DACLhxlZ0e4p0vWqbK6ZJr_GW
|
|
|
341
342
|
relationalai/semantics/metamodel/rewrite/__init__.py,sha256=9ONWFSdMPHkWpObDMSljt8DywhpFf4Ehsq1aT3fTPt8,344
|
|
342
343
|
relationalai/semantics/metamodel/rewrite/discharge_constraints.py,sha256=0v613BqCLlo4sgWuZjcLSxxakp3d34mYWbG4ldhzGno,1949
|
|
343
344
|
relationalai/semantics/metamodel/rewrite/dnf_union_splitter.py,sha256=ZXX190gCKXhdB-Iyi4MGowc4FS9P0PIJTtTT0LrTr6A,7970
|
|
344
|
-
relationalai/semantics/metamodel/rewrite/extract_nested_logicals.py,sha256=
|
|
345
|
+
relationalai/semantics/metamodel/rewrite/extract_nested_logicals.py,sha256=vQ0-7t_GORskB1ZG50KuzM4phm6YNPvehfFn3v_LbgI,3354
|
|
345
346
|
relationalai/semantics/metamodel/rewrite/flatten.py,sha256=uw7pKBGdrGqGYZU9NewAjvqRuhJk7As49hA_Wcwfp2k,21918
|
|
346
347
|
relationalai/semantics/metamodel/rewrite/format_outputs.py,sha256=n0IxC3RL3UMly6MWsq342EGfL2yGj3vOgVG_wg7kt-o,6225
|
|
347
348
|
relationalai/semantics/metamodel/typer/__init__.py,sha256=E3ydmhWRdm-cAqWsNR24_Qd3NcwiHx8ElO2tzNysAXc,143
|
|
@@ -360,7 +361,7 @@ relationalai/semantics/reasoners/optimization/solvers_dev.py,sha256=lbw3c8Z6PlHR
|
|
|
360
361
|
relationalai/semantics/reasoners/optimization/solvers_pb.py,sha256=ESwraHU9c4NCEVRZ16tnBZsUCmJg7lUhy-v0-GGq0qo,48000
|
|
361
362
|
relationalai/semantics/rel/__init__.py,sha256=pMlVTC_TbQ45mP1LpzwFBBgPxpKc0H3uJDvvDXEWzvs,55
|
|
362
363
|
relationalai/semantics/rel/builtins.py,sha256=kQToiELc4NnvCmXyFtu9CsGZNdTQtSzTB-nuyIfQcsM,1562
|
|
363
|
-
relationalai/semantics/rel/compiler.py,sha256
|
|
364
|
+
relationalai/semantics/rel/compiler.py,sha256=-gyu_kL1p4Z-SXI2fwLXSbGfUy-ejQcKVrc66rVvOyg,43044
|
|
364
365
|
relationalai/semantics/rel/executor.py,sha256=v-yHl9R8AV0AA2xnm5YZDzue83pr8j2Q97Ky1MKkU70,17309
|
|
365
366
|
relationalai/semantics/rel/rel.py,sha256=9I_V6dQ83QRaLzq04Tt-KjBWhmNxNO3tFzeornBK4zc,15738
|
|
366
367
|
relationalai/semantics/rel/rel_utils.py,sha256=EH-NBROA4vIJXajLKniapt4Dxt7cXSqY4NEjD-wD8Mc,9566
|
|
@@ -390,7 +391,7 @@ relationalai/semantics/std/std.py,sha256=Ql27y2Rs0d1kluktWi-t6_M_uYIxQUfO94GjlVf
|
|
|
390
391
|
relationalai/semantics/std/strings.py,sha256=Q_7kvx5dud6gppNURHOi4SMvgZNPRuWwEKDIPSEIOJI,2702
|
|
391
392
|
relationalai/semantics/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
392
393
|
relationalai/semantics/tests/logging.py,sha256=oJTrUS_Bq6WxqqO8QLtrjdkUB02Uu5erZ7FTl__-lNY,1432
|
|
393
|
-
relationalai/semantics/tests/test_snapshot_abstract.py,sha256=
|
|
394
|
+
relationalai/semantics/tests/test_snapshot_abstract.py,sha256=ob_5p43CQzz9nfB7Q9W1geKqwNmLyUmQx-mpJfTOwhg,6305
|
|
394
395
|
relationalai/semantics/tests/test_snapshot_base.py,sha256=vlqqSyQf_IsDb7feDnkiHZKSCsJqDq0Ep4V3CKtKWns,466
|
|
395
396
|
relationalai/semantics/tests/utils.py,sha256=h4GPNCJGmAWvwskGWxc523BtFEd2ayff4zfBNHapoCo,1825
|
|
396
397
|
relationalai/std/__init__.py,sha256=U8-DpdOcxeGXVSf3LqJl9mSXKztNgB9iwropQcXp960,2178
|
|
@@ -422,7 +423,7 @@ relationalai/util/format.py,sha256=fLRovumUa2cu0_2gy3O5vaEbpND4p9_IIgr-vDaIGDc,3
|
|
|
422
423
|
relationalai/util/graph.py,sha256=eT8s0yCiJIu6D1T1fjZsLSPCcuQb2Mzl6qnljtQ5TuA,1504
|
|
423
424
|
relationalai/util/list_databases.py,sha256=xJZGHzE0VLaDItWo5XvQSx75OwV045h2rjCBBnhNB3o,152
|
|
424
425
|
relationalai/util/otel_configuration.py,sha256=SV1SFqdKm8q-OD1TsmiIB0x5VxtAFsBRyTwD8zNtnQc,1066
|
|
425
|
-
relationalai/util/otel_handler.py,sha256=
|
|
426
|
+
relationalai/util/otel_handler.py,sha256=2h3ROnra798RRRPLTlEBxEcj-jXXk1ri_l10RAFaiWk,18109
|
|
426
427
|
relationalai/util/snowflake_handler.py,sha256=8HUo6eU5wPxa6m_EocYuEZD_XpVIh9EqBnGwaS06Usw,2942
|
|
427
428
|
relationalai/util/span_format_test.py,sha256=1rU-M3RXhRt8vCX6rtuIJogEVkB164N3qiKcTmb-sl8,1345
|
|
428
429
|
relationalai/util/span_tracker.py,sha256=SKj4UB43clUxWShq3lFChl3dZ4JGRhcnY_xAh-7Xzvg,7888
|
|
@@ -438,8 +439,8 @@ frontend/debugger/dist/index.html,sha256=0wIQ1Pm7BclVV1wna6Mj8OmgU73B9rSEGPVX-Wo
|
|
|
438
439
|
frontend/debugger/dist/assets/favicon-Dy0ZgA6N.png,sha256=tPXOEhOrM4tJyZVJQVBO_yFgNAlgooY38ZsjyrFstgg,620
|
|
439
440
|
frontend/debugger/dist/assets/index-Cssla-O7.js,sha256=MxgIGfdKQyBWgufck1xYggQNhW5nj6BPjCF6Wleo-f0,298886
|
|
440
441
|
frontend/debugger/dist/assets/index-DlHsYx1V.css,sha256=21pZtAjKCcHLFjbjfBQTF6y7QmOic-4FYaKNmwdNZVE,60141
|
|
441
|
-
relationalai-0.12.
|
|
442
|
-
relationalai-0.12.
|
|
443
|
-
relationalai-0.12.
|
|
444
|
-
relationalai-0.12.
|
|
445
|
-
relationalai-0.12.
|
|
442
|
+
relationalai-0.12.9.dist-info/METADATA,sha256=r7ebl7OUAHSnHZxFdPUL8woKPut0wWKjJ6Z1wvHc6Ko,2562
|
|
443
|
+
relationalai-0.12.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
444
|
+
relationalai-0.12.9.dist-info/entry_points.txt,sha256=fo_oLFJih3PUgYuHXsk7RnCjBm9cqRNR--ab6DgI6-0,88
|
|
445
|
+
relationalai-0.12.9.dist-info/licenses/LICENSE,sha256=pPyTVXFYhirkEW9VsnHIgUjT0Vg8_xsE6olrF5SIgpc,11343
|
|
446
|
+
relationalai-0.12.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|