relationalai 0.12.3__py3-none-any.whl → 0.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- relationalai/clients/snowflake.py +117 -28
- relationalai/clients/use_index_poller.py +3 -0
- relationalai/semantics/internal/snowflake.py +2 -3
- relationalai/semantics/lqp/executor.py +15 -10
- relationalai/semantics/lqp/model2lqp.py +0 -1
- relationalai/semantics/lqp/rewrite/extract_common.py +26 -6
- relationalai/semantics/metamodel/dependency.py +14 -11
- relationalai/semantics/metamodel/rewrite/flatten.py +1 -13
- relationalai/semantics/reasoners/graph/core.py +72 -26
- relationalai/semantics/rel/executor.py +13 -6
- relationalai/semantics/sql/executor/snowflake.py +2 -2
- {relationalai-0.12.3.dist-info → relationalai-0.12.4.dist-info}/METADATA +1 -1
- {relationalai-0.12.3.dist-info → relationalai-0.12.4.dist-info}/RECORD +16 -16
- {relationalai-0.12.3.dist-info → relationalai-0.12.4.dist-info}/WHEEL +0 -0
- {relationalai-0.12.3.dist-info → relationalai-0.12.4.dist-info}/entry_points.txt +0 -0
- {relationalai-0.12.3.dist-info → relationalai-0.12.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -83,6 +83,7 @@ FIELD_MAP = {
|
|
|
83
83
|
VALID_IMPORT_STATES = ["PENDING", "PROCESSING", "QUARANTINED", "LOADED"]
|
|
84
84
|
ENGINE_ERRORS = ["engine is suspended", "create/resume", "engine not found", "no engines found", "engine was deleted"]
|
|
85
85
|
ENGINE_NOT_READY_MSGS = ["engine is in pending", "engine is provisioning"]
|
|
86
|
+
DATABASE_ERRORS = ["database not found"]
|
|
86
87
|
PYREL_ROOT_DB = 'pyrel_root_db'
|
|
87
88
|
|
|
88
89
|
TERMINAL_TXN_STATES = ["COMPLETED", "ABORTED"]
|
|
@@ -281,6 +282,8 @@ def _sanitize_user_name(user: str) -> str:
|
|
|
281
282
|
def _is_engine_issue(response_message: str) -> bool:
|
|
282
283
|
return any(kw in response_message.lower() for kw in ENGINE_ERRORS + ENGINE_NOT_READY_MSGS)
|
|
283
284
|
|
|
285
|
+
def _is_database_issue(response_message: str) -> bool:
|
|
286
|
+
return any(kw in response_message.lower() for kw in DATABASE_ERRORS)
|
|
284
287
|
|
|
285
288
|
|
|
286
289
|
#--------------------------------------------------
|
|
@@ -298,6 +301,7 @@ class Resources(ResourcesBase):
|
|
|
298
301
|
dry_run: bool = False,
|
|
299
302
|
reset_session: bool = False,
|
|
300
303
|
generation: Generation | None = None,
|
|
304
|
+
language: str = "rel",
|
|
301
305
|
):
|
|
302
306
|
super().__init__(profile, config=config)
|
|
303
307
|
self._token_handler: TokenHandler | None = None
|
|
@@ -315,6 +319,8 @@ class Resources(ResourcesBase):
|
|
|
315
319
|
# self.sources contains fully qualified Snowflake table/view names
|
|
316
320
|
self.sources: set[str] = set()
|
|
317
321
|
self._sproc_models = None
|
|
322
|
+
self.database = ""
|
|
323
|
+
self.language = language
|
|
318
324
|
atexit.register(self.cancel_pending_transactions)
|
|
319
325
|
|
|
320
326
|
@property
|
|
@@ -452,6 +458,7 @@ class Resources(ResourcesBase):
|
|
|
452
458
|
rai_app = self.config.get("rai_app_name", "")
|
|
453
459
|
current_role = self.config.get("role")
|
|
454
460
|
engine = self.get_default_engine_name()
|
|
461
|
+
engine_size = self.config.get_default_engine_size()
|
|
455
462
|
assert isinstance(rai_app, str), f"rai_app_name must be a string, not {type(rai_app)}"
|
|
456
463
|
assert isinstance(engine, str), f"engine must be a string, not {type(engine)}"
|
|
457
464
|
print("\n")
|
|
@@ -460,9 +467,15 @@ class Resources(ResourcesBase):
|
|
|
460
467
|
if re.search(f"database '{rai_app}' does not exist or not authorized.".lower(), orig_message):
|
|
461
468
|
exception = SnowflakeAppMissingException(rai_app, current_role)
|
|
462
469
|
raise exception from None
|
|
463
|
-
if
|
|
470
|
+
if _is_engine_issue(orig_message) or _is_database_issue(orig_message):
|
|
464
471
|
try:
|
|
465
|
-
self.
|
|
472
|
+
self._poll_use_index(
|
|
473
|
+
app_name=self.get_app_name(),
|
|
474
|
+
sources=self.sources,
|
|
475
|
+
model=self.database,
|
|
476
|
+
engine_name=engine,
|
|
477
|
+
engine_size=engine_size
|
|
478
|
+
)
|
|
466
479
|
return self._exec(code, params, raw=raw, help=help)
|
|
467
480
|
except EngineNameValidationException as e:
|
|
468
481
|
raise EngineNameValidationException(engine) from e
|
|
@@ -767,7 +780,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
|
|
|
767
780
|
keep_database = not force and self.config.get("reuse_model", True)
|
|
768
781
|
with debugging.span("release_index", name=name, keep_database=keep_database, language=language):
|
|
769
782
|
#TODO add headers to release_index
|
|
770
|
-
response = self._exec(f"call {APP_NAME}.api.release_index('{name}', OBJECT_CONSTRUCT('keep_database', {keep_database}, 'language', '{language}'));")
|
|
783
|
+
response = self._exec(f"call {APP_NAME}.api.release_index('{name}', OBJECT_CONSTRUCT('keep_database', {keep_database}, 'language', '{language}', 'user_agent', '{get_pyrel_version(self.generation)}'));")
|
|
771
784
|
if response:
|
|
772
785
|
result = next(iter(response))
|
|
773
786
|
obj = json.loads(result["RELEASE_INDEX"])
|
|
@@ -788,14 +801,13 @@ Otherwise, remove it from your '{profile}' configuration profile.
|
|
|
788
801
|
headers = debugging.gen_current_propagation_headers()
|
|
789
802
|
self._exec(f"call {APP_NAME}.api.clone_database('{target_name}', '{source_name}', {nowait_durable}, {headers});")
|
|
790
803
|
|
|
791
|
-
def
|
|
804
|
+
def _poll_use_index(
|
|
792
805
|
self,
|
|
793
806
|
app_name: str,
|
|
794
807
|
sources: Iterable[str],
|
|
795
808
|
model: str,
|
|
796
809
|
engine_name: str,
|
|
797
810
|
engine_size: str | None = None,
|
|
798
|
-
language: str = "rel",
|
|
799
811
|
program_span_id: str | None = None,
|
|
800
812
|
headers: Dict | None = None,
|
|
801
813
|
):
|
|
@@ -806,12 +818,36 @@ Otherwise, remove it from your '{profile}' configuration profile.
|
|
|
806
818
|
model,
|
|
807
819
|
engine_name,
|
|
808
820
|
engine_size,
|
|
809
|
-
language,
|
|
821
|
+
self.language,
|
|
810
822
|
program_span_id,
|
|
811
823
|
headers,
|
|
812
824
|
self.generation
|
|
813
825
|
).poll()
|
|
814
826
|
|
|
827
|
+
def maybe_poll_use_index(
|
|
828
|
+
self,
|
|
829
|
+
app_name: str,
|
|
830
|
+
sources: Iterable[str],
|
|
831
|
+
model: str,
|
|
832
|
+
engine_name: str,
|
|
833
|
+
engine_size: str | None = None,
|
|
834
|
+
program_span_id: str | None = None,
|
|
835
|
+
headers: Dict | None = None,
|
|
836
|
+
):
|
|
837
|
+
"""Only call _poll_use_index if there are sources to process."""
|
|
838
|
+
sources_list = list(sources)
|
|
839
|
+
self.database = model
|
|
840
|
+
if sources_list:
|
|
841
|
+
return self._poll_use_index(
|
|
842
|
+
app_name=app_name,
|
|
843
|
+
sources=sources_list,
|
|
844
|
+
model=model,
|
|
845
|
+
engine_name=engine_name,
|
|
846
|
+
engine_size=engine_size,
|
|
847
|
+
program_span_id=program_span_id,
|
|
848
|
+
headers=headers,
|
|
849
|
+
)
|
|
850
|
+
|
|
815
851
|
#--------------------------------------------------
|
|
816
852
|
# Models
|
|
817
853
|
#--------------------------------------------------
|
|
@@ -1868,9 +1904,19 @@ Otherwise, remove it from your '{profile}' configuration profile.
|
|
|
1868
1904
|
)
|
|
1869
1905
|
except Exception as e:
|
|
1870
1906
|
err_message = str(e).lower()
|
|
1871
|
-
if _is_engine_issue(err_message):
|
|
1872
|
-
self.
|
|
1873
|
-
self.
|
|
1907
|
+
if _is_engine_issue(err_message) or _is_database_issue(err_message):
|
|
1908
|
+
engine_name = engine or self.get_default_engine_name()
|
|
1909
|
+
engine_size = self.config.get_default_engine_size()
|
|
1910
|
+
self._poll_use_index(
|
|
1911
|
+
app_name=self.get_app_name(),
|
|
1912
|
+
sources=self.sources,
|
|
1913
|
+
model=database,
|
|
1914
|
+
engine_name=engine_name,
|
|
1915
|
+
engine_size=engine_size,
|
|
1916
|
+
headers=headers,
|
|
1917
|
+
)
|
|
1918
|
+
|
|
1919
|
+
return self._exec_async_v2(
|
|
1874
1920
|
database, engine, raw_code_b64, inputs, readonly, nowait_durable,
|
|
1875
1921
|
headers=headers, bypass_index=bypass_index, language='lqp',
|
|
1876
1922
|
query_timeout_mins=query_timeout_mins,
|
|
@@ -1908,8 +1954,17 @@ Otherwise, remove it from your '{profile}' configuration profile.
|
|
|
1908
1954
|
)
|
|
1909
1955
|
except Exception as e:
|
|
1910
1956
|
err_message = str(e).lower()
|
|
1911
|
-
if _is_engine_issue(err_message):
|
|
1912
|
-
self.
|
|
1957
|
+
if _is_engine_issue(err_message) or _is_database_issue(err_message):
|
|
1958
|
+
engine_name = engine or self.get_default_engine_name()
|
|
1959
|
+
engine_size = self.config.get_default_engine_size()
|
|
1960
|
+
self._poll_use_index(
|
|
1961
|
+
app_name=self.get_app_name(),
|
|
1962
|
+
sources=self.sources,
|
|
1963
|
+
model=database,
|
|
1964
|
+
engine_name=engine_name,
|
|
1965
|
+
engine_size=engine_size,
|
|
1966
|
+
headers=headers,
|
|
1967
|
+
)
|
|
1913
1968
|
return self._exec_async_v2(
|
|
1914
1969
|
database,
|
|
1915
1970
|
engine,
|
|
@@ -2972,13 +3027,12 @@ class SnowflakeClient(Client):
|
|
|
2972
3027
|
|
|
2973
3028
|
query_attrs_dict = json.loads(headers.get("X-Query-Attributes", "{}")) if headers else {}
|
|
2974
3029
|
with debugging.span("poll_use_index", sources=self.resources.sources, model=model, engine=engine_name, **query_attrs_dict):
|
|
2975
|
-
self.
|
|
3030
|
+
self.maybe_poll_use_index(
|
|
2976
3031
|
app_name=app_name,
|
|
2977
3032
|
sources=self.resources.sources,
|
|
2978
3033
|
model=model,
|
|
2979
3034
|
engine_name=engine_name,
|
|
2980
3035
|
engine_size=engine_size,
|
|
2981
|
-
language="rel",
|
|
2982
3036
|
program_span_id=program_span_id,
|
|
2983
3037
|
headers=headers
|
|
2984
3038
|
)
|
|
@@ -2989,29 +3043,24 @@ class SnowflakeClient(Client):
|
|
|
2989
3043
|
if isolated and not self.keep_model:
|
|
2990
3044
|
atexit.register(self.delete_database)
|
|
2991
3045
|
|
|
2992
|
-
|
|
2993
|
-
# if data is ready, break the loop
|
|
2994
|
-
# if data is not ready, print the status of the tables or engines
|
|
2995
|
-
# if data is not ready and there are errors, collect the errors and raise exceptions
|
|
2996
|
-
def poll_use_index(
|
|
3046
|
+
def maybe_poll_use_index(
|
|
2997
3047
|
self,
|
|
2998
3048
|
app_name: str,
|
|
2999
3049
|
sources: Iterable[str],
|
|
3000
3050
|
model: str,
|
|
3001
3051
|
engine_name: str,
|
|
3002
3052
|
engine_size: str | None = None,
|
|
3003
|
-
language: str = "rel",
|
|
3004
3053
|
program_span_id: str | None = None,
|
|
3005
3054
|
headers: Dict | None = None,
|
|
3006
3055
|
):
|
|
3056
|
+
"""Only call _poll_use_index if there are sources to process."""
|
|
3007
3057
|
assert isinstance(self.resources, Resources)
|
|
3008
|
-
return self.resources.
|
|
3058
|
+
return self.resources.maybe_poll_use_index(
|
|
3009
3059
|
app_name=app_name,
|
|
3010
3060
|
sources=sources,
|
|
3011
3061
|
model=model,
|
|
3012
3062
|
engine_name=engine_name,
|
|
3013
3063
|
engine_size=engine_size,
|
|
3014
|
-
language=language,
|
|
3015
3064
|
program_span_id=program_span_id,
|
|
3016
3065
|
headers=headers
|
|
3017
3066
|
)
|
|
@@ -3136,6 +3185,7 @@ class DirectAccessResources(Resources):
|
|
|
3136
3185
|
dry_run: bool = False,
|
|
3137
3186
|
reset_session: bool = False,
|
|
3138
3187
|
generation: Optional[Generation] = None,
|
|
3188
|
+
language: str = "rel",
|
|
3139
3189
|
):
|
|
3140
3190
|
super().__init__(
|
|
3141
3191
|
generation=generation,
|
|
@@ -3144,11 +3194,13 @@ class DirectAccessResources(Resources):
|
|
|
3144
3194
|
connection=connection,
|
|
3145
3195
|
reset_session=reset_session,
|
|
3146
3196
|
dry_run=dry_run,
|
|
3197
|
+
language=language,
|
|
3147
3198
|
)
|
|
3148
3199
|
self._endpoint_info = ConfigStore(ENDPOINT_FILE)
|
|
3149
3200
|
self._service_endpoint = ""
|
|
3150
3201
|
self._direct_access_client = None
|
|
3151
3202
|
self.generation = generation
|
|
3203
|
+
self.database = ""
|
|
3152
3204
|
|
|
3153
3205
|
@property
|
|
3154
3206
|
def service_endpoint(self) -> str:
|
|
@@ -3226,9 +3278,18 @@ class DirectAccessResources(Resources):
|
|
|
3226
3278
|
|
|
3227
3279
|
# fix engine on engine error and retry
|
|
3228
3280
|
# Skip auto-retry if skip_auto_create is True to avoid recursion
|
|
3229
|
-
if _is_engine_issue(message) and not skip_auto_create:
|
|
3230
|
-
|
|
3231
|
-
self.
|
|
3281
|
+
if (_is_engine_issue(message) and not skip_auto_create) or _is_database_issue(message):
|
|
3282
|
+
engine_name = payload.get("caller_engine_name", "") if payload else ""
|
|
3283
|
+
engine_name = engine_name or self.get_default_engine_name()
|
|
3284
|
+
engine_size = self.config.get_default_engine_size()
|
|
3285
|
+
self._poll_use_index(
|
|
3286
|
+
app_name=self.get_app_name(),
|
|
3287
|
+
sources=self.sources,
|
|
3288
|
+
model=self.database,
|
|
3289
|
+
engine_name=engine_name,
|
|
3290
|
+
engine_size=engine_size,
|
|
3291
|
+
headers=headers,
|
|
3292
|
+
)
|
|
3232
3293
|
response = _send_request()
|
|
3233
3294
|
except requests.exceptions.ConnectionError as e:
|
|
3234
3295
|
if "NameResolutionError" in str(e):
|
|
@@ -3356,14 +3417,13 @@ class DirectAccessResources(Resources):
|
|
|
3356
3417
|
|
|
3357
3418
|
return response.json()
|
|
3358
3419
|
|
|
3359
|
-
def
|
|
3420
|
+
def _poll_use_index(
|
|
3360
3421
|
self,
|
|
3361
3422
|
app_name: str,
|
|
3362
3423
|
sources: Iterable[str],
|
|
3363
3424
|
model: str,
|
|
3364
3425
|
engine_name: str,
|
|
3365
3426
|
engine_size: str | None = None,
|
|
3366
|
-
language: str = "rel",
|
|
3367
3427
|
program_span_id: str | None = None,
|
|
3368
3428
|
headers: Dict | None = None,
|
|
3369
3429
|
):
|
|
@@ -3374,12 +3434,36 @@ class DirectAccessResources(Resources):
|
|
|
3374
3434
|
model=model,
|
|
3375
3435
|
engine_name=engine_name,
|
|
3376
3436
|
engine_size=engine_size,
|
|
3377
|
-
language=language,
|
|
3437
|
+
language=self.language,
|
|
3378
3438
|
program_span_id=program_span_id,
|
|
3379
3439
|
headers=headers,
|
|
3380
3440
|
generation=self.generation,
|
|
3381
3441
|
).poll()
|
|
3382
3442
|
|
|
3443
|
+
def maybe_poll_use_index(
|
|
3444
|
+
self,
|
|
3445
|
+
app_name: str,
|
|
3446
|
+
sources: Iterable[str],
|
|
3447
|
+
model: str,
|
|
3448
|
+
engine_name: str,
|
|
3449
|
+
engine_size: str | None = None,
|
|
3450
|
+
program_span_id: str | None = None,
|
|
3451
|
+
headers: Dict | None = None,
|
|
3452
|
+
):
|
|
3453
|
+
"""Only call _poll_use_index if there are sources to process."""
|
|
3454
|
+
sources_list = list(sources)
|
|
3455
|
+
self.database = model
|
|
3456
|
+
if sources_list:
|
|
3457
|
+
return self._poll_use_index(
|
|
3458
|
+
app_name=app_name,
|
|
3459
|
+
sources=sources_list,
|
|
3460
|
+
model=model,
|
|
3461
|
+
engine_name=engine_name,
|
|
3462
|
+
engine_size=engine_size,
|
|
3463
|
+
program_span_id=program_span_id,
|
|
3464
|
+
headers=headers,
|
|
3465
|
+
)
|
|
3466
|
+
|
|
3383
3467
|
def _check_exec_async_status(self, txn_id: str, headers: Dict[str, str] | None = None) -> bool:
|
|
3384
3468
|
"""Check whether the given transaction has completed."""
|
|
3385
3469
|
|
|
@@ -3522,7 +3606,12 @@ class DirectAccessResources(Resources):
|
|
|
3522
3606
|
with debugging.span("release_index", name=name, keep_database=keep_database, language=language):
|
|
3523
3607
|
response = self.request(
|
|
3524
3608
|
"release_index",
|
|
3525
|
-
payload={
|
|
3609
|
+
payload={
|
|
3610
|
+
"model_name": name,
|
|
3611
|
+
"keep_database": keep_database,
|
|
3612
|
+
"language": language,
|
|
3613
|
+
"user_agent": get_pyrel_version(self.generation),
|
|
3614
|
+
},
|
|
3526
3615
|
headers=prop_hdrs,
|
|
3527
3616
|
)
|
|
3528
3617
|
if (
|
|
@@ -264,12 +264,11 @@ class Table():
|
|
|
264
264
|
else:
|
|
265
265
|
me = self._rel._field_refs[0]
|
|
266
266
|
b.where(self).define(concept(me))
|
|
267
|
-
#
|
|
268
|
-
rel_func = b.Relationship if keys else b.Property
|
|
267
|
+
# All the fields are treated as properties
|
|
269
268
|
for field in self._rel._fields[1:]:
|
|
270
269
|
field_name = sanitize_identifier(field.name.lower())
|
|
271
270
|
if field_name not in key_dict:
|
|
272
|
-
r =
|
|
271
|
+
r = b.Property(
|
|
273
272
|
f"{{{concept}}} has {{{field_name}:{field.type_str}}}",
|
|
274
273
|
parent=concept,
|
|
275
274
|
short_name=field_name,
|
|
@@ -56,7 +56,15 @@ class LQPExecutor(e.Executor):
|
|
|
56
56
|
resource_class = rai.clients.snowflake.Resources
|
|
57
57
|
if self.config.get("use_direct_access", USE_DIRECT_ACCESS):
|
|
58
58
|
resource_class = rai.clients.snowflake.DirectAccessResources
|
|
59
|
-
|
|
59
|
+
# NOTE: language="lqp" is not strictly required for LQP execution, but it
|
|
60
|
+
# will significantly improve performance.
|
|
61
|
+
self._resources = resource_class(
|
|
62
|
+
dry_run=self.dry_run,
|
|
63
|
+
config=self.config,
|
|
64
|
+
generation=rai.Generation.QB,
|
|
65
|
+
connection=self.connection,
|
|
66
|
+
language="lqp",
|
|
67
|
+
)
|
|
60
68
|
if not self.dry_run:
|
|
61
69
|
self.engine = self._resources.get_default_engine_name()
|
|
62
70
|
if not self.keep_model:
|
|
@@ -88,13 +96,12 @@ class LQPExecutor(e.Executor):
|
|
|
88
96
|
assert self.engine is not None
|
|
89
97
|
|
|
90
98
|
with debugging.span("poll_use_index", sources=sources, model=model, engine=engine_name):
|
|
91
|
-
resources.
|
|
99
|
+
resources.maybe_poll_use_index(
|
|
92
100
|
app_name=app_name,
|
|
93
101
|
sources=sources,
|
|
94
102
|
model=model,
|
|
95
103
|
engine_name=self.engine,
|
|
96
104
|
engine_size=engine_size,
|
|
97
|
-
language="lqp",
|
|
98
105
|
program_span_id=program_span_id,
|
|
99
106
|
)
|
|
100
107
|
|
|
@@ -280,6 +287,8 @@ class LQPExecutor(e.Executor):
|
|
|
280
287
|
"""Construct an epoch that defines a number of built-in definitions used by the
|
|
281
288
|
emitter."""
|
|
282
289
|
with debugging.span("compile_intrinsics") as span:
|
|
290
|
+
span["compile_type"] = "intrinsics"
|
|
291
|
+
|
|
283
292
|
debug_info = lqp_ir.DebugInfo(id_to_orig_name={}, meta=None)
|
|
284
293
|
intrinsics_fragment = lqp_ir.Fragment(
|
|
285
294
|
id = lqp_ir.FragmentId(id=b"__pyrel_lqp_intrinsics", meta=None),
|
|
@@ -290,7 +299,7 @@ class LQPExecutor(e.Executor):
|
|
|
290
299
|
meta = None,
|
|
291
300
|
)
|
|
292
301
|
|
|
293
|
-
|
|
302
|
+
|
|
294
303
|
span["lqp"] = lqp_print.to_string(intrinsics_fragment, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
295
304
|
|
|
296
305
|
return lqp_ir.Epoch(
|
|
@@ -327,13 +336,14 @@ class LQPExecutor(e.Executor):
|
|
|
327
336
|
model_txn = None
|
|
328
337
|
if self._last_model != model:
|
|
329
338
|
with debugging.span("compile", metamodel=model) as install_span:
|
|
339
|
+
install_span["compile_type"] = "model"
|
|
330
340
|
_, model_txn = self.compiler.compile(model, {"fragment_id": b"model"})
|
|
331
341
|
model_txn = txn_with_configure(model_txn, configure)
|
|
332
|
-
install_span["compile_type"] = "model"
|
|
333
342
|
install_span["lqp"] = lqp_print.to_string(model_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
334
343
|
self._last_model = model
|
|
335
344
|
|
|
336
345
|
with debugging.span("compile", metamodel=task) as compile_span:
|
|
346
|
+
compile_span["compile_type"] = "query"
|
|
337
347
|
query = f.compute_model(f.logical([task]))
|
|
338
348
|
options = {
|
|
339
349
|
"wide_outputs": self.wide_outputs,
|
|
@@ -342,7 +352,6 @@ class LQPExecutor(e.Executor):
|
|
|
342
352
|
result, final_model = self.compiler.compile_inner(query, options)
|
|
343
353
|
export_info, query_txn = result
|
|
344
354
|
query_txn = txn_with_configure(query_txn, configure)
|
|
345
|
-
compile_span["compile_type"] = "query"
|
|
346
355
|
compile_span["lqp"] = lqp_print.to_string(query_txn, {"print_names": True, "print_debug": False, "print_csv_filename": False})
|
|
347
356
|
|
|
348
357
|
# Merge the epochs into a single transactions. Long term the query bits should all
|
|
@@ -356,14 +365,10 @@ class LQPExecutor(e.Executor):
|
|
|
356
365
|
epochs.append(model_txn.epochs[0])
|
|
357
366
|
|
|
358
367
|
query_txn_epoch = query_txn.epochs[0]
|
|
359
|
-
|
|
360
368
|
epochs.append(query_txn_epoch)
|
|
361
|
-
|
|
362
369
|
epochs.append(self._compile_undefine_query(query_txn_epoch))
|
|
363
370
|
|
|
364
371
|
txn = lqp_ir.Transaction(epochs=epochs, configure=configure, meta=None)
|
|
365
|
-
|
|
366
|
-
# Revalidate now that we've joined all the epochs.
|
|
367
372
|
validate_lqp(txn)
|
|
368
373
|
|
|
369
374
|
txn_proto = convert_transaction(txn)
|
|
@@ -2,8 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from typing import Optional
|
|
5
|
-
from relationalai.semantics.metamodel
|
|
6
|
-
from relationalai.semantics.metamodel import ir, factory as f, helpers
|
|
5
|
+
from relationalai.semantics.metamodel import ir, factory as f, helpers, visitor
|
|
7
6
|
from relationalai.semantics.metamodel.compiler import Pass, group_tasks
|
|
8
7
|
from relationalai.semantics.metamodel.util import OrderedSet, ordered_set
|
|
9
8
|
from relationalai.semantics.metamodel import dependency
|
|
@@ -52,7 +51,7 @@ class ExtractCommon(Pass):
|
|
|
52
51
|
#--------------------------------------------------
|
|
53
52
|
def rewrite(self, model: ir.Model, options:dict={}) -> ir.Model:
|
|
54
53
|
# create the context
|
|
55
|
-
ctx = ExtractCommon.Context(model)
|
|
54
|
+
ctx = ExtractCommon.Context(model, options)
|
|
56
55
|
|
|
57
56
|
# rewrite the root
|
|
58
57
|
replacement = self.handle(model.root, ctx)
|
|
@@ -76,9 +75,10 @@ class ExtractCommon(Pass):
|
|
|
76
75
|
#--------------------------------------------------
|
|
77
76
|
|
|
78
77
|
class Context():
|
|
79
|
-
def __init__(self, model: ir.Model):
|
|
78
|
+
def __init__(self, model: ir.Model, options: dict):
|
|
80
79
|
self.rewrite_ctx = helpers.RewriteContext()
|
|
81
80
|
self.info = dependency.analyze(model.root)
|
|
81
|
+
self.options = options
|
|
82
82
|
|
|
83
83
|
def handle(self, task: ir.Task, ctx: Context):
|
|
84
84
|
# currently we only extract if it's a sequence of Logicals, but we could in the
|
|
@@ -107,7 +107,7 @@ class ExtractCommon(Pass):
|
|
|
107
107
|
# extracted logic).
|
|
108
108
|
plan = None
|
|
109
109
|
if len(binders) > 1 and composites_and_effects:
|
|
110
|
-
extractables =
|
|
110
|
+
extractables = self._get_extractables(ctx, composites_and_effects)
|
|
111
111
|
# only makes sense to extract common if at least one nested composite will be
|
|
112
112
|
# extracted during Flatten
|
|
113
113
|
if extractables:
|
|
@@ -325,7 +325,6 @@ class ExtractCommon(Pass):
|
|
|
325
325
|
|
|
326
326
|
return ExtractCommon.ExtractionPlan(common_body, remaining, exposed_vars, local_dependencies, distribute_common_reference)
|
|
327
327
|
|
|
328
|
-
|
|
329
328
|
def _compute_local_dependencies(self, ctx: Context, binders: OrderedSet[ir.Task], composite: ir.Task, exposed_vars: OrderedSet[ir.Var]):
|
|
330
329
|
"""
|
|
331
330
|
The tasks in common_body will be extracted into a logical that will expose the exposed_vars.
|
|
@@ -362,3 +361,24 @@ class ExtractCommon(Pass):
|
|
|
362
361
|
if inputs:
|
|
363
362
|
vars_needed.update(inputs - vars_exposed)
|
|
364
363
|
return local_body
|
|
364
|
+
|
|
365
|
+
def _get_extractables(self, ctx: Context, composites: OrderedSet[ir.Task]):
|
|
366
|
+
"""
|
|
367
|
+
Extractables are tasks that will eventually be extracted by the Flatten pass later.
|
|
368
|
+
Given a set of tasks, return the extractable ones.
|
|
369
|
+
"""
|
|
370
|
+
def _extractable(t: ir.Task):
|
|
371
|
+
# With GNF outputs (i.e., wide_outputs = False), the output tasks will be
|
|
372
|
+
# extracted into separate top-level single-column outputs.
|
|
373
|
+
if isinstance(t, ir.Output) and not ctx.options.get("wide_outputs", False):
|
|
374
|
+
return True
|
|
375
|
+
|
|
376
|
+
extractable_types = (ir.Update, ir.Aggregate, ir.Match, ir.Union, ir.Rank)
|
|
377
|
+
return isinstance(t, ir.Logical) and len(visitor.collect_by_type(extractable_types, t)) > 0
|
|
378
|
+
|
|
379
|
+
extractables = []
|
|
380
|
+
for t in composites:
|
|
381
|
+
if _extractable(t):
|
|
382
|
+
extractables.append(t)
|
|
383
|
+
|
|
384
|
+
return extractables
|
|
@@ -96,8 +96,12 @@ class DependencyInfo():
|
|
|
96
96
|
# start with the cluster dependencies, because cluster represents the task we
|
|
97
97
|
# care about
|
|
98
98
|
queue.extend(cluster.dependencies)
|
|
99
|
+
seen = set()
|
|
99
100
|
while queue:
|
|
100
101
|
cluster = queue.pop()
|
|
102
|
+
if cluster.id in seen:
|
|
103
|
+
continue
|
|
104
|
+
seen.add(cluster.id)
|
|
101
105
|
deps.update(cluster.content)
|
|
102
106
|
queue.extend(cluster.dependencies)
|
|
103
107
|
|
|
@@ -625,12 +629,14 @@ class BindingAnalysis(visitor.Visitor):
|
|
|
625
629
|
# TODO: this is similar to what's done below in visit_lookup, modularize
|
|
626
630
|
if builtins.is_eq(child.relation):
|
|
627
631
|
x, y = child.args[0], child.args[1]
|
|
632
|
+
# Compute input/output vars of the equality
|
|
628
633
|
if isinstance(x, ir.Var) and not isinstance(y, ir.Var):
|
|
629
|
-
|
|
630
|
-
|
|
634
|
+
# Variable x is potentially grounded by other expressions at
|
|
635
|
+
# level in the Logical. If it is, then we should mark it as
|
|
636
|
+
# input (which is done later).
|
|
637
|
+
potentially_grounded.add((child, x, x))
|
|
631
638
|
elif not isinstance(x, ir.Var) and isinstance(y, ir.Var):
|
|
632
|
-
|
|
633
|
-
self.output(child, y)
|
|
639
|
+
potentially_grounded.add((child, y, y))
|
|
634
640
|
elif isinstance(x, ir.Var) and isinstance(y, ir.Var):
|
|
635
641
|
# mark as potentially grounded, if any is grounded in other atoms then we later ground both
|
|
636
642
|
potentially_grounded.add((child, x, y))
|
|
@@ -779,22 +785,19 @@ class BindingAnalysis(visitor.Visitor):
|
|
|
779
785
|
if builtins.is_eq(node.relation):
|
|
780
786
|
# Most cases are covered already at the parent level if the equality is part of
|
|
781
787
|
# a Logical. The remaining cases are when the equality is a child of a
|
|
782
|
-
# non-Logical.
|
|
788
|
+
# non-Logical, or if its variables are not ground elsewhere in the Logical.
|
|
783
789
|
if self.info.task_inputs(node) or self.info.task_outputs(node):
|
|
784
790
|
# already covered
|
|
785
791
|
pass
|
|
786
792
|
else:
|
|
787
793
|
x, y = node.args[0], node.args[1]
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
elif not isinstance(x, ir.Var) and isinstance(y, ir.Var):
|
|
791
|
-
self.output(node, y)
|
|
792
|
-
elif isinstance(x, ir.Var) and isinstance(y, ir.Var):
|
|
793
|
-
grounds = self._grounded[-1] if self._grounded else ordered_set()
|
|
794
|
+
grounds = self._grounded[-1] if self._grounded else ordered_set()
|
|
795
|
+
if isinstance(x, ir.Var):
|
|
794
796
|
if x in grounds:
|
|
795
797
|
self.input(node, x)
|
|
796
798
|
else:
|
|
797
799
|
self.output(node, x)
|
|
800
|
+
if isinstance(y, ir.Var):
|
|
798
801
|
if y in grounds:
|
|
799
802
|
self.input(node, y)
|
|
800
803
|
else:
|
|
@@ -3,7 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from typing import cast, Optional, TypeVar
|
|
4
4
|
from typing import Tuple
|
|
5
5
|
|
|
6
|
-
from relationalai.semantics.metamodel import builtins, ir, factory as f, helpers, types
|
|
6
|
+
from relationalai.semantics.metamodel import builtins, ir, factory as f, helpers, types
|
|
7
7
|
from relationalai.semantics.metamodel.compiler import Pass, group_tasks
|
|
8
8
|
from relationalai.semantics.metamodel.util import OrderedSet, ordered_set, NameCache
|
|
9
9
|
from relationalai.semantics.metamodel import dependency
|
|
@@ -611,18 +611,6 @@ def set_union(s1: Optional[OrderedSet[T]], s2: Optional[OrderedSet[T]]) -> list:
|
|
|
611
611
|
return s2.get_list()
|
|
612
612
|
return []
|
|
613
613
|
|
|
614
|
-
def extractable(t: ir.Task):
|
|
615
|
-
"""
|
|
616
|
-
Whether this task is a Logical that will be extracted as a top level by this
|
|
617
|
-
pass, because it has an aggregation, effects, match, union, etc.
|
|
618
|
-
"""
|
|
619
|
-
extractable_types = (ir.Update, ir.Aggregate, ir.Match, ir.Union, ir.Rank)
|
|
620
|
-
return isinstance(t, ir.Logical) and len(visitor.collect_by_type(extractable_types, t)) > 0
|
|
621
|
-
|
|
622
|
-
def extractables(composites: OrderedSet[ir.Task]):
|
|
623
|
-
""" Filter the set of composites, keeping only the extractable ones. """
|
|
624
|
-
return list(filter(extractable, composites))
|
|
625
|
-
|
|
626
614
|
def negate(lookup: ir.Lookup, values: int):
|
|
627
615
|
"""
|
|
628
616
|
Return a negation of this reference, where the last `values` arguments are to
|
|
@@ -353,14 +353,16 @@ class Graph():
|
|
|
353
353
|
@cached_property
|
|
354
354
|
def Node(self) -> Concept:
|
|
355
355
|
"""Lazily define and cache the self.Node concept."""
|
|
356
|
-
|
|
357
|
-
|
|
356
|
+
_Node = self._user_node_concept or self._model.Concept(self._NodeConceptStr)
|
|
357
|
+
_Node.annotate(annotations.track("graphs", "Node"))
|
|
358
|
+
return _Node
|
|
358
359
|
|
|
359
360
|
@cached_property
|
|
360
361
|
def Edge(self):
|
|
361
362
|
"""Lazily define and cache the self.Edge concept and friends,
|
|
362
363
|
by passing through to self._EdgeComplex."""
|
|
363
364
|
_Edge, _, _, _ = self._EdgeComplex
|
|
365
|
+
_Edge.annotate(annotations.track("graphs", "Edge"))
|
|
364
366
|
return _Edge
|
|
365
367
|
|
|
366
368
|
@cached_property
|
|
@@ -594,6 +596,7 @@ class Graph():
|
|
|
594
596
|
consuming the `Edge` concept's `EdgeSrc` and `EdgeDst` relationships.
|
|
595
597
|
"""
|
|
596
598
|
_edge_rel = self._model.Relationship(f"{{src:{self._NodeConceptStr}}} has edge to {{dst:{self._NodeConceptStr}}}")
|
|
599
|
+
_edge_rel.annotate(annotations.track("graphs", "_edge"))
|
|
597
600
|
|
|
598
601
|
Edge, EdgeSrc, EdgeDst = self.Edge, self.EdgeSrc, self.EdgeDst
|
|
599
602
|
src, dst = self.Node.ref(), self.Node.ref()
|
|
@@ -620,6 +623,7 @@ class Graph():
|
|
|
620
623
|
consuming the `Edge` concept's `EdgeSrc`, `EdgeDst`, and `EdgeWeight` relationships.
|
|
621
624
|
"""
|
|
622
625
|
_weight_rel = self._model.Relationship(f"{{src:{self._NodeConceptStr}}} has edge to {{dst:{self._NodeConceptStr}}} with weight {{weight:Float}}")
|
|
626
|
+
_weight_rel.annotate(annotations.track("graphs", "_weight"))
|
|
623
627
|
|
|
624
628
|
Edge, EdgeSrc, EdgeDst, EdgeWeight = self.Edge, self.EdgeSrc, self.EdgeDst, self.EdgeWeight
|
|
625
629
|
src, dst, weight = self.Node.ref(), self.Node.ref(), Float.ref()
|
|
@@ -3920,17 +3924,31 @@ class Graph():
|
|
|
3920
3924
|
|
|
3921
3925
|
|
|
3922
3926
|
@include_in_docs
|
|
3923
|
-
def triangle(self):
|
|
3927
|
+
def triangle(self, *, full: Optional[bool] = None):
|
|
3924
3928
|
"""Returns a ternary relationship containing all triangles in the graph.
|
|
3925
3929
|
|
|
3926
3930
|
Unlike `unique_triangle`, this relationship contains all permutations
|
|
3927
3931
|
of the nodes for each triangle found.
|
|
3928
3932
|
|
|
3933
|
+
Parameters
|
|
3934
|
+
----------
|
|
3935
|
+
full : bool, optional
|
|
3936
|
+
If ``True``, computes triangles for all triplets of nodes in the graph.
|
|
3937
|
+
This computation can be expensive for large graphs. Must be set to ``True``
|
|
3938
|
+
to compute the full triangle relationship.
|
|
3939
|
+
Default is ``None``.
|
|
3940
|
+
|
|
3929
3941
|
Returns
|
|
3930
3942
|
-------
|
|
3931
3943
|
Relationship
|
|
3932
3944
|
A ternary relationship where each tuple represents a triangle.
|
|
3933
3945
|
|
|
3946
|
+
Raises
|
|
3947
|
+
------
|
|
3948
|
+
ValueError
|
|
3949
|
+
If ``full`` is not provided.
|
|
3950
|
+
If ``full`` is not ``True``.
|
|
3951
|
+
|
|
3934
3952
|
Relationship Schema
|
|
3935
3953
|
-------------------
|
|
3936
3954
|
``triangle(node_a, node_b, node_c)``
|
|
@@ -3970,7 +3988,7 @@ class Graph():
|
|
|
3970
3988
|
>>>
|
|
3971
3989
|
>>> # 3. Select all triangles and inspect
|
|
3972
3990
|
>>> a,b,c = Node.ref("a"), Node.ref("b"), Node.ref("c")
|
|
3973
|
-
>>> triangle = graph.triangle()
|
|
3991
|
+
>>> triangle = graph.triangle(full=True)
|
|
3974
3992
|
>>> select(a.id, b.id, c.id).where(triangle(a, b, c)).inspect()
|
|
3975
3993
|
▰▰▰▰ Setup complete
|
|
3976
3994
|
id id2 id3
|
|
@@ -3999,7 +4017,7 @@ class Graph():
|
|
|
3999
4017
|
>>>
|
|
4000
4018
|
>>> # 3. Select all triangles and inspect
|
|
4001
4019
|
>>> a,b,c = Node.ref("a"), Node.ref("b"), Node.ref("c")
|
|
4002
|
-
>>> triangle = graph.triangle()
|
|
4020
|
+
>>> triangle = graph.triangle(full=True)
|
|
4003
4021
|
>>> select(a.id, b.id, c.id).where(triangle(a, b, c)).inspect()
|
|
4004
4022
|
▰▰▰▰ Setup complete
|
|
4005
4023
|
id id2 id3
|
|
@@ -4017,15 +4035,22 @@ class Graph():
|
|
|
4017
4035
|
triangle_count
|
|
4018
4036
|
|
|
4019
4037
|
"""
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
"
|
|
4024
|
-
"
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4028
|
-
|
|
4038
|
+
# Validate full parameter
|
|
4039
|
+
if full is None:
|
|
4040
|
+
raise ValueError(
|
|
4041
|
+
"Computing triangle for all triplets can be expensive. To confirm "
|
|
4042
|
+
"that you would like to compute the full triangle relationship, "
|
|
4043
|
+
"please call `triangle(full=True)`. "
|
|
4044
|
+
"(Domain constraints are not available for `triangle` at this time. "
|
|
4045
|
+
"If you need domain constraints for `triangle`, please reach out.)"
|
|
4046
|
+
)
|
|
4047
|
+
|
|
4048
|
+
if full is not True:
|
|
4049
|
+
raise ValueError(
|
|
4050
|
+
f"Invalid value (`{full}`) for 'full' parameter. Use `full=True` "
|
|
4051
|
+
"to compute the full triangle relationship. "
|
|
4052
|
+
)
|
|
4053
|
+
|
|
4029
4054
|
return self._triangle
|
|
4030
4055
|
|
|
4031
4056
|
@cached_property
|
|
@@ -4052,15 +4077,29 @@ class Graph():
|
|
|
4052
4077
|
|
|
4053
4078
|
|
|
4054
4079
|
@include_in_docs
|
|
4055
|
-
def unique_triangle(self):
|
|
4080
|
+
def unique_triangle(self, *, full: Optional[bool] = None):
|
|
4056
4081
|
"""Returns a ternary relationship containing all unique triangles in the graph.
|
|
4057
4082
|
|
|
4083
|
+
Parameters
|
|
4084
|
+
----------
|
|
4085
|
+
full : bool, optional
|
|
4086
|
+
If ``True``, computes unique triangles for all triplets of nodes in the graph.
|
|
4087
|
+
This computation can be expensive for large graphs. Must be set to ``True``
|
|
4088
|
+
to compute the full unique_triangle relationship.
|
|
4089
|
+
Default is ``None``.
|
|
4090
|
+
|
|
4058
4091
|
Returns
|
|
4059
4092
|
-------
|
|
4060
4093
|
Relationship
|
|
4061
4094
|
A ternary relationship where each tuple represents a unique
|
|
4062
4095
|
triangle.
|
|
4063
4096
|
|
|
4097
|
+
Raises
|
|
4098
|
+
------
|
|
4099
|
+
ValueError
|
|
4100
|
+
If ``full`` is not provided.
|
|
4101
|
+
If ``full`` is not ``True``.
|
|
4102
|
+
|
|
4064
4103
|
Relationship Schema
|
|
4065
4104
|
-------------------
|
|
4066
4105
|
``unique_triangle(node_a, node_b, node_c)``
|
|
@@ -4117,7 +4156,7 @@ class Graph():
|
|
|
4117
4156
|
>>>
|
|
4118
4157
|
>>> # 3. Select the unique triangles and inspect
|
|
4119
4158
|
>>> a,b,c = Node.ref("a"), Node.ref("b"), Node.ref("c")
|
|
4120
|
-
>>> unique_triangle = graph.unique_triangle()
|
|
4159
|
+
>>> unique_triangle = graph.unique_triangle(full=True)
|
|
4121
4160
|
>>> select(a.id, b.id, c.id).where(unique_triangle(a, b, c)).inspect()
|
|
4122
4161
|
▰▰▰▰ Setup complete
|
|
4123
4162
|
id id2 id3
|
|
@@ -4146,7 +4185,7 @@ class Graph():
|
|
|
4146
4185
|
>>>
|
|
4147
4186
|
>>> # 3. Select the unique triangles and inspect
|
|
4148
4187
|
>>> a,b,c = Node.ref("a"), Node.ref("b"), Node.ref("c")
|
|
4149
|
-
>>> unique_triangle = graph.unique_triangle()
|
|
4188
|
+
>>> unique_triangle = graph.unique_triangle(full=True)
|
|
4150
4189
|
>>> select(a.id, b.id, c.id).where(unique_triangle(a, b, c)).inspect()
|
|
4151
4190
|
▰▰▰▰ Setup complete
|
|
4152
4191
|
id id2 id3
|
|
@@ -4160,15 +4199,22 @@ class Graph():
|
|
|
4160
4199
|
triangle_count
|
|
4161
4200
|
|
|
4162
4201
|
"""
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
"
|
|
4167
|
-
"
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4202
|
+
# Validate full parameter
|
|
4203
|
+
if full is None:
|
|
4204
|
+
raise ValueError(
|
|
4205
|
+
"Computing unique_triangle for all triplets can be expensive. To confirm "
|
|
4206
|
+
"that you would like to compute the full unique_triangle relationship, "
|
|
4207
|
+
"please call `unique_triangle(full=True)`. "
|
|
4208
|
+
"(Domain constraints are not available for `unique_triangle` at this time. "
|
|
4209
|
+
"If you need domain constraints for `unique_triangle`, please reach out.)"
|
|
4210
|
+
)
|
|
4211
|
+
|
|
4212
|
+
if full is not True:
|
|
4213
|
+
raise ValueError(
|
|
4214
|
+
f"Invalid value (`{full}`) for 'full' parameter. Use `full=True` "
|
|
4215
|
+
"to compute the full unique_triangle relationship."
|
|
4216
|
+
)
|
|
4217
|
+
|
|
4172
4218
|
return self._unique_triangle
|
|
4173
4219
|
|
|
4174
4220
|
@cached_property
|
|
@@ -54,7 +54,15 @@ class RelExecutor(e.Executor):
|
|
|
54
54
|
resource_class = rai.clients.snowflake.Resources
|
|
55
55
|
if self.config.get("use_direct_access", USE_DIRECT_ACCESS):
|
|
56
56
|
resource_class = rai.clients.snowflake.DirectAccessResources
|
|
57
|
-
|
|
57
|
+
# NOTE: language="rel" is required for Rel execution. It is the default, but
|
|
58
|
+
# we set it explicitly here to be sure.
|
|
59
|
+
self._resources = resource_class(
|
|
60
|
+
dry_run=self.dry_run,
|
|
61
|
+
config=self.config,
|
|
62
|
+
generation=rai.Generation.QB,
|
|
63
|
+
connection=self.connection,
|
|
64
|
+
language="rel",
|
|
65
|
+
)
|
|
58
66
|
if not self.dry_run:
|
|
59
67
|
self.engine = self._resources.get_default_engine_name()
|
|
60
68
|
if not self.keep_model:
|
|
@@ -85,13 +93,12 @@ class RelExecutor(e.Executor):
|
|
|
85
93
|
assert self.engine is not None
|
|
86
94
|
|
|
87
95
|
with debugging.span("poll_use_index", sources=sources, model=model, engine=engine_name):
|
|
88
|
-
resources.
|
|
96
|
+
resources.maybe_poll_use_index(
|
|
89
97
|
app_name=app_name,
|
|
90
98
|
sources=sources,
|
|
91
99
|
model=model,
|
|
92
100
|
engine_name=self.engine,
|
|
93
101
|
engine_size=engine_size,
|
|
94
|
-
language="rel",
|
|
95
102
|
program_span_id=program_span_id,
|
|
96
103
|
headers=headers,
|
|
97
104
|
)
|
|
@@ -249,25 +256,26 @@ class RelExecutor(e.Executor):
|
|
|
249
256
|
# Format meta as headers
|
|
250
257
|
json_meta = prepare_metadata_for_headers(meta)
|
|
251
258
|
headers = {QUERY_ATTRIBUTES_HEADER: json_meta} if json_meta else {}
|
|
252
|
-
|
|
259
|
+
|
|
253
260
|
self.check_graph_index(headers)
|
|
254
261
|
resources= self.resources
|
|
255
262
|
|
|
256
263
|
rules_code = ""
|
|
257
264
|
if self._last_model != model:
|
|
258
265
|
with debugging.span("compile", metamodel=model) as install_span:
|
|
266
|
+
install_span["compile_type"] = "model"
|
|
259
267
|
base = textwrap.dedent("""
|
|
260
268
|
declare pyrel_error_attrs(err in ::std::common::UInt128, attr in ::std::common::String, v) requires true
|
|
261
269
|
|
|
262
270
|
""")
|
|
263
271
|
rules_code = base + self.compiler.compile(model, {"wide_outputs": self.wide_outputs})
|
|
264
|
-
install_span["compile_type"] = "model"
|
|
265
272
|
install_span["rel"] = rules_code
|
|
266
273
|
rules_code = resources.create_models_code([("pyrel_qb_0", rules_code)])
|
|
267
274
|
self._last_model = model
|
|
268
275
|
|
|
269
276
|
|
|
270
277
|
with debugging.span("compile", metamodel=task) as compile_span:
|
|
278
|
+
compile_span["compile_type"] = "query"
|
|
271
279
|
base = textwrap.dedent("""
|
|
272
280
|
def output(:pyrel_error, err, attr, val):
|
|
273
281
|
pyrel_error_attrs(err, attr, val)
|
|
@@ -276,7 +284,6 @@ class RelExecutor(e.Executor):
|
|
|
276
284
|
task_model = f.compute_model(f.logical([task]))
|
|
277
285
|
task_code, task_model = self.compiler.compile_inner(task_model, {"no_declares": True, "wide_outputs": self.wide_outputs})
|
|
278
286
|
task_code = base + task_code
|
|
279
|
-
compile_span["compile_type"] = "query"
|
|
280
287
|
compile_span["rel"] = task_code
|
|
281
288
|
|
|
282
289
|
full_code = textwrap.dedent(f"""
|
|
@@ -73,19 +73,19 @@ class SnowflakeExecutor(e.Executor):
|
|
|
73
73
|
|
|
74
74
|
if self._last_model != model:
|
|
75
75
|
with debugging.span("compile", metamodel=model) as model_span:
|
|
76
|
-
model_sql, _ = self.compiler.compile(model, options)
|
|
77
76
|
model_span["compile_type"] = "model"
|
|
77
|
+
model_sql, _ = self.compiler.compile(model, options)
|
|
78
78
|
model_span["sql"] = model_sql
|
|
79
79
|
self._last_model = model
|
|
80
80
|
self._last_model_sql = model_sql
|
|
81
81
|
|
|
82
82
|
with debugging.span("compile", metamodel=task) as compile_span:
|
|
83
|
+
compile_span["compile_type"] = "query"
|
|
83
84
|
# compile into sql and keep the new_task, which is the task model after rewrites,
|
|
84
85
|
# as it may contain results of type inference, which is useful for determining
|
|
85
86
|
# how to format the outputs
|
|
86
87
|
query_options = {**options, "query_compilation": True}
|
|
87
88
|
query_sql, new_task = self.compiler.compile(f.compute_model(f.logical([task])), query_options)
|
|
88
|
-
compile_span["compile_type"] = "query"
|
|
89
89
|
compile_span["sql"] = query_sql
|
|
90
90
|
|
|
91
91
|
if self.dry_run:
|
|
@@ -28,9 +28,9 @@ relationalai/clients/export_procedure.py.jinja,sha256=nhvVcs5hQyWExFDuROQbi1VyYz
|
|
|
28
28
|
relationalai/clients/hash_util.py,sha256=pZVR1FX3q4G_19p_r6wpIR2tIM8_WUlfAR7AVZJjIYM,1495
|
|
29
29
|
relationalai/clients/profile_polling.py,sha256=pUH7WKH4nYDD0SlQtg3wsWdj0K7qt6nZqUw8jTthCBs,2565
|
|
30
30
|
relationalai/clients/result_helpers.py,sha256=wDSD02Ngx6W-YQqBIGKnpXD4Ju3pA1e9Nz6ORRI6SRI,17808
|
|
31
|
-
relationalai/clients/snowflake.py,sha256=
|
|
31
|
+
relationalai/clients/snowflake.py,sha256=qybrFowe_s3bAPOsFCh-j-ARpdivW3g6_4HjlcN_Fvw,163091
|
|
32
32
|
relationalai/clients/types.py,sha256=eNo6akcMTbnBFbBbHd5IgVeY-zuAgtXlOs8Bo1SWmVU,2890
|
|
33
|
-
relationalai/clients/use_index_poller.py,sha256=
|
|
33
|
+
relationalai/clients/use_index_poller.py,sha256=OFeTJ3fF98UoVk6MA8v8ri8WPbmmCquyA4T8MD_T1Xc,46398
|
|
34
34
|
relationalai/clients/util.py,sha256=NJC8fnrWHR01NydwESPSetIHRWf7jQJURYpaWJjmDyE,12311
|
|
35
35
|
relationalai/early_access/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
36
36
|
relationalai/early_access/builder/__init__.py,sha256=mrR-aGLPquUGc_e9-DOkVCCCo8QRE5A56GTTtgILNa4,993
|
|
@@ -257,15 +257,15 @@ relationalai/semantics/devtools/extract_lqp.py,sha256=gxI3EvPUTPAkwgnkCKAkEm2vA6
|
|
|
257
257
|
relationalai/semantics/internal/__init__.py,sha256=JXrpFaL-fdZrvKpWTEn1UoLXITOoTGnAYwmgeiglhSk,774
|
|
258
258
|
relationalai/semantics/internal/annotations.py,sha256=P_nRpBm4wLmE_8L0VO3TDORL1p2flXaPOVDC0GG7KsQ,306
|
|
259
259
|
relationalai/semantics/internal/internal.py,sha256=Vh4LFatRA8xO6sIBxRSlbAv6HOMJPU65t2c_ZEh9lug,148972
|
|
260
|
-
relationalai/semantics/internal/snowflake.py,sha256=
|
|
260
|
+
relationalai/semantics/internal/snowflake.py,sha256=3t_GVU61WRUtVrOCpTJ7RlVTZLm0DPpMKU7Q66IiatY,12966
|
|
261
261
|
relationalai/semantics/lqp/__init__.py,sha256=XgcQZxK-zz_LqPDVtwREhsIvjTuUIt4BZhIedCeMY-s,48
|
|
262
262
|
relationalai/semantics/lqp/builtins.py,sha256=bRmQ6fdceWU-4xf4l-W-YiuyDxJTPey1s6O4xlyW6iM,540
|
|
263
263
|
relationalai/semantics/lqp/compiler.py,sha256=Nury1gPw_-Oi_mqT1-rhr13L4UmyIP2BGuotbuklQKA,949
|
|
264
264
|
relationalai/semantics/lqp/constructors.py,sha256=8U4eUL8-m1wYRQnws_YWC1coGquTugVH5YC0Zek6VT8,2309
|
|
265
|
-
relationalai/semantics/lqp/executor.py,sha256=
|
|
265
|
+
relationalai/semantics/lqp/executor.py,sha256=P5_YITIGUfLlkblzzOB1CdprxhuVYA5pn5kow50Mnik,20636
|
|
266
266
|
relationalai/semantics/lqp/intrinsics.py,sha256=Pb1mLIme7Q-5Y-CVacUOEvapfhKs076bgtRNi3f0ayY,833
|
|
267
267
|
relationalai/semantics/lqp/ir.py,sha256=DUw0ltul0AS9CRjntNlmllWTwXpxMyYg4iJ9t7NFYMA,1791
|
|
268
|
-
relationalai/semantics/lqp/model2lqp.py,sha256=
|
|
268
|
+
relationalai/semantics/lqp/model2lqp.py,sha256=V0Gavuk1eBsZaCP9YSq6r-XSwfqpVGo1KABkk6XObuE,31727
|
|
269
269
|
relationalai/semantics/lqp/passes.py,sha256=nLppoHvIQkGP6VuG56OAZ1oOrYhEqpR_0w91gfJ7t_s,27540
|
|
270
270
|
relationalai/semantics/lqp/pragmas.py,sha256=FzzldrJEAZ1AIcEw6D-FfaVg3CoahRYgPCFo7xHfg1g,375
|
|
271
271
|
relationalai/semantics/lqp/primitives.py,sha256=Gbh6cohoAArhqEJTN_TgIRc7wmtdxXt231NRW0beEj0,10898
|
|
@@ -275,7 +275,7 @@ relationalai/semantics/lqp/utils.py,sha256=x8dcfVoyqzCznNFtI4qSNSL-vWcdxzC-QrnFJ
|
|
|
275
275
|
relationalai/semantics/lqp/validators.py,sha256=YO_ciSgEVNILWUbkxIagKpIxI4oqV0fRSTO2Ok0rPJk,1526
|
|
276
276
|
relationalai/semantics/lqp/rewrite/__init__.py,sha256=NQMkrZMUrLksScGXs0EiZTT-In5VltvswfeB4kCkeZM,337
|
|
277
277
|
relationalai/semantics/lqp/rewrite/cdc.py,sha256=I6DeMOZScx-3UAVoSCMn9cuOgLzwdvJVKNwsgFa6R_k,10390
|
|
278
|
-
relationalai/semantics/lqp/rewrite/extract_common.py,sha256=
|
|
278
|
+
relationalai/semantics/lqp/rewrite/extract_common.py,sha256=sbihURqk4wtc1ekDWXWltq9LrO42XTLfOHl5D6nT5vw,18371
|
|
279
279
|
relationalai/semantics/lqp/rewrite/extract_keys.py,sha256=dSr5SVkYmrhiR0XPY5eRAnWD66dcZYgXdilXcERv634,18682
|
|
280
280
|
relationalai/semantics/lqp/rewrite/fd_constraints.py,sha256=a4jetchlOfm2rb3QD0Nil0bvermpBx1up7p1uNgcJHg,3503
|
|
281
281
|
relationalai/semantics/lqp/rewrite/quantify_vars.py,sha256=wYMEXzCW_D_Y_1rSLvuAAqw9KN1oIOn_vIMxELzRVb4,11568
|
|
@@ -284,7 +284,7 @@ relationalai/semantics/metamodel/__init__.py,sha256=I-XqQAGycD0nKkKYvnF3G9d0QK_1
|
|
|
284
284
|
relationalai/semantics/metamodel/builtins.py,sha256=vGzNGTqD1q4Yn2K1Rx6D47t_CUZDbp-16ndE56RiT70,35965
|
|
285
285
|
relationalai/semantics/metamodel/compiler.py,sha256=XBsAnbFwgZ_TcRry6yXGWLyw_MaO2WJDp1EnC_ubhps,4525
|
|
286
286
|
relationalai/semantics/metamodel/dataflow.py,sha256=wfj1tARrR4yEAaTwUTrAcxEcz81VkUal4U_AX1esovk,3929
|
|
287
|
-
relationalai/semantics/metamodel/dependency.py,sha256=
|
|
287
|
+
relationalai/semantics/metamodel/dependency.py,sha256=iJLx-w_zqde7CtbGcXxLxZBdUKZYl7AUykezPI9ccck,33926
|
|
288
288
|
relationalai/semantics/metamodel/executor.py,sha256=_pm--QNvAjd-GgiMKMrpkPZ2eE0H1IM5ZB0l1DGkRs0,2614
|
|
289
289
|
relationalai/semantics/metamodel/factory.py,sha256=Vk3ASwWan08mfGehoOOwMixuU_mEbG2vNl8JLSCJ2OU,12581
|
|
290
290
|
relationalai/semantics/metamodel/helpers.py,sha256=aeXWkS-iKfLqqXtlMjQZyqIIiIsG9dqP4cQA3cUmM08,15403
|
|
@@ -296,14 +296,14 @@ relationalai/semantics/metamodel/rewrite/__init__.py,sha256=aPgZuRGpULwPVWtENUEH
|
|
|
296
296
|
relationalai/semantics/metamodel/rewrite/discharge_constraints.py,sha256=0v613BqCLlo4sgWuZjcLSxxakp3d34mYWbG4ldhzGno,1949
|
|
297
297
|
relationalai/semantics/metamodel/rewrite/dnf_union_splitter.py,sha256=dlyD868Pg424BLowY4A0gOmSziSy1U-dYGXZEE3SW8E,7956
|
|
298
298
|
relationalai/semantics/metamodel/rewrite/extract_nested_logicals.py,sha256=0aKMoniIzq-uAm1iItRQEUxGGDYVnT8pWZxFXWlOk50,3366
|
|
299
|
-
relationalai/semantics/metamodel/rewrite/flatten.py,sha256=
|
|
299
|
+
relationalai/semantics/metamodel/rewrite/flatten.py,sha256=W3zoUWY2UVOO_d-vyLZYb4y61wcM99ceJJFCuz_Rz-Q,27891
|
|
300
300
|
relationalai/semantics/metamodel/typer/__init__.py,sha256=E3ydmhWRdm-cAqWsNR24_Qd3NcwiHx8ElO2tzNysAXc,143
|
|
301
301
|
relationalai/semantics/metamodel/typer/checker.py,sha256=frY0gilDO6skbDiYFiIpDUOWyt9s9jAJsRBs848DcG0,19184
|
|
302
302
|
relationalai/semantics/metamodel/typer/typer.py,sha256=jBo0CwY6G0qKBzqoAmQ09dPtcPDHvCiutolnwbV4gus,62294
|
|
303
303
|
relationalai/semantics/reasoners/__init__.py,sha256=Tu4U26rrkBIzAk3a4tXRJaeD5mAtK9Z7JXh2c6VJ-J4,249
|
|
304
304
|
relationalai/semantics/reasoners/graph/README.md,sha256=QgKEXTllp5PO-yK8oDfMx1PNTYF2uVoneMRKsWTY5GU,23953
|
|
305
305
|
relationalai/semantics/reasoners/graph/__init__.py,sha256=jSXR6J05SQZdjxQ5Y-ovqFqGTAXCOWeQDcvpfoBYgDA,1282
|
|
306
|
-
relationalai/semantics/reasoners/graph/core.py,sha256=
|
|
306
|
+
relationalai/semantics/reasoners/graph/core.py,sha256=8-YVdxz20zj3pzTD14sWanif5zRbe9PbNuXRhGXeSXI,351965
|
|
307
307
|
relationalai/semantics/reasoners/graph/design/beyond_demand_transform.md,sha256=Givh0W6B6Hlow6TpmK-8adpEYd8b3O_WmdgMOQIyKs0,55749
|
|
308
308
|
relationalai/semantics/reasoners/graph/paths/README.md,sha256=ydm6CzMN_vOOgq7a6_hBCiyGi3D6g5gxAbf4OXbQbDE,3433
|
|
309
309
|
relationalai/semantics/reasoners/graph/paths/__init__.py,sha256=oCoDJVg4moGe0tNMNeRn7XQqa1AIEZGlxAiaD5bBahg,211
|
|
@@ -376,7 +376,7 @@ relationalai/semantics/reasoners/optimization/solvers_pb.py,sha256=ryNARpyph3uvr
|
|
|
376
376
|
relationalai/semantics/rel/__init__.py,sha256=pMlVTC_TbQ45mP1LpzwFBBgPxpKc0H3uJDvvDXEWzvs,55
|
|
377
377
|
relationalai/semantics/rel/builtins.py,sha256=qu4yZvLovn4Vn2x44D4XugqGD6Qo5xLxj_RKA34cpF4,1527
|
|
378
378
|
relationalai/semantics/rel/compiler.py,sha256=hiLIaZzhVU5VMtU6rdo3tH2pmKMEtXvh6AHh94CvRAg,42203
|
|
379
|
-
relationalai/semantics/rel/executor.py,sha256=
|
|
379
|
+
relationalai/semantics/rel/executor.py,sha256=PfCm9tsY_HVYnw7Zo7RMR7GisQ9-5W-Hy6kqK4jFfpU,16133
|
|
380
380
|
relationalai/semantics/rel/rel.py,sha256=9I_V6dQ83QRaLzq04Tt-KjBWhmNxNO3tFzeornBK4zc,15738
|
|
381
381
|
relationalai/semantics/rel/rel_utils.py,sha256=F14Ch8mn45J8SmM7HZnIHUNqDnb3WQLnkEGLme04iBk,9386
|
|
382
382
|
relationalai/semantics/snowflake/__init__.py,sha256=BW_zvPQBWGTAtY6cluG6tDDG-QmU_jRb-F7PeCpDhIU,134
|
|
@@ -386,7 +386,7 @@ relationalai/semantics/sql/sql.py,sha256=7nUnm0RbHlOGSGQbnFrgzPYdmnoppifQ5jylR5W
|
|
|
386
386
|
relationalai/semantics/sql/executor/__init__.py,sha256=F3HqQPJVP9wgV3rkwI5jy1_QBCD_3qj2IGxbdT_pX9k,120
|
|
387
387
|
relationalai/semantics/sql/executor/duck_db.py,sha256=laI0jquMNNhj1pcFlaqxYAvvnCmSuvzzkibfjMz7liY,1909
|
|
388
388
|
relationalai/semantics/sql/executor/result_helpers.py,sha256=kVfspHHuzyq4SNklrtvhYte1wqRFzct-dAKin_lOmR4,3215
|
|
389
|
-
relationalai/semantics/sql/executor/snowflake.py,sha256=
|
|
389
|
+
relationalai/semantics/sql/executor/snowflake.py,sha256=p7jLfDtqGgM8GEjHBGqdm_PbxEIQn0cmI_msEj9F0Xo,5861
|
|
390
390
|
relationalai/semantics/sql/rewrite/__init__.py,sha256=AT1WR0rqQHQ7E06NLoVym0zrZpBVPqK85uRFNJUTDp4,254
|
|
391
391
|
relationalai/semantics/sql/rewrite/denormalize.py,sha256=gPF0s1edthoHfuBIdS9-hnEaX5RFYBdtOlK5iAfB5CA,9040
|
|
392
392
|
relationalai/semantics/sql/rewrite/double_negation.py,sha256=QXynhjwfPW52Hpdo78VWPNk2TXe_mMTWYN470in51z8,1710
|
|
@@ -453,8 +453,8 @@ frontend/debugger/dist/index.html,sha256=0wIQ1Pm7BclVV1wna6Mj8OmgU73B9rSEGPVX-Wo
|
|
|
453
453
|
frontend/debugger/dist/assets/favicon-Dy0ZgA6N.png,sha256=tPXOEhOrM4tJyZVJQVBO_yFgNAlgooY38ZsjyrFstgg,620
|
|
454
454
|
frontend/debugger/dist/assets/index-Cssla-O7.js,sha256=MxgIGfdKQyBWgufck1xYggQNhW5nj6BPjCF6Wleo-f0,298886
|
|
455
455
|
frontend/debugger/dist/assets/index-DlHsYx1V.css,sha256=21pZtAjKCcHLFjbjfBQTF6y7QmOic-4FYaKNmwdNZVE,60141
|
|
456
|
-
relationalai-0.12.
|
|
457
|
-
relationalai-0.12.
|
|
458
|
-
relationalai-0.12.
|
|
459
|
-
relationalai-0.12.
|
|
460
|
-
relationalai-0.12.
|
|
456
|
+
relationalai-0.12.4.dist-info/METADATA,sha256=SRoHgqRMSESO_6_I1ZTQOyI3iGl-3gtN64rQcupY7Ww,2562
|
|
457
|
+
relationalai-0.12.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
458
|
+
relationalai-0.12.4.dist-info/entry_points.txt,sha256=fo_oLFJih3PUgYuHXsk7RnCjBm9cqRNR--ab6DgI6-0,88
|
|
459
|
+
relationalai-0.12.4.dist-info/licenses/LICENSE,sha256=pPyTVXFYhirkEW9VsnHIgUjT0Vg8_xsE6olrF5SIgpc,11343
|
|
460
|
+
relationalai-0.12.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|