tracdap-runtime 0.6.2__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracdap_runtime-0.6.2/tracdap_runtime.egg-info → tracdap_runtime-0.6.4}/PKG-INFO +4 -4
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/setup.cfg +4 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/actors.py +87 -10
- tracdap_runtime-0.6.4/src/tracdap/rt/_exec/context.py +426 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/dev_mode.py +52 -20
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/engine.py +79 -14
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/functions.py +14 -17
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/runtime.py +83 -40
- tracdap_runtime-0.6.4/src/tracdap/rt/_exec/server.py +345 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/config_parser.py +219 -49
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/data.py +70 -5
- tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/codec.py +99 -0
- tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py +51 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.pyi +11 -9
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2_grpc.py +25 -25
- tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +51 -0
- tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py +63 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.pyi +37 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py +8 -3
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi +13 -2
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/guard_rails.py +21 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/models.py +25 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/static_api.py +43 -13
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/type_system.py +17 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/validation.py +47 -4
- tracdap_runtime-0.6.4/src/tracdap/rt/_plugins/config_local.py +49 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_version.py +1 -1
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/hook.py +6 -5
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/model_api.py +50 -7
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/static_api.py +81 -23
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/__init__.py +4 -4
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/common.py +25 -15
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/job.py +2 -2
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/platform.py +25 -35
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/result.py +2 -2
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/runtime.py +4 -2
- tracdap_runtime-0.6.4/src/tracdap/rt/ext/config.py +34 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/embed.py +1 -3
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/plugins.py +47 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/cli.py +11 -4
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/launch.py +53 -12
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/common.py +2 -2
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/custom.py +3 -3
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/data.py +12 -12
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/file.py +6 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/flow.py +6 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/job.py +8 -8
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/model.py +21 -11
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/object.py +3 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/object_id.py +8 -8
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/search.py +5 -5
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/stoarge.py +6 -6
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/tag.py +1 -1
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/tag_update.py +1 -1
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/type.py +4 -4
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4/tracdap_runtime.egg-info}/PKG-INFO +4 -4
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/SOURCES.txt +2 -11
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/requires.txt +3 -3
- tracdap_runtime-0.6.2/src/tracdap/rt/_exec/context.py +0 -319
- tracdap_runtime-0.6.2/src/tracdap/rt/_exec/server.py +0 -68
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/codec.py +0 -44
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py +0 -51
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/common_pb2.py +0 -55
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/common_pb2.pyi +0 -103
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/job_pb2.py +0 -42
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/job_pb2.pyi +0 -44
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/platform_pb2.py +0 -71
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/platform_pb2.pyi +0 -197
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/result_pb2.py +0 -37
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/result_pb2.pyi +0 -35
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/runtime_pb2.py +0 -42
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/runtime_pb2.pyi +0 -46
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +0 -51
- tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py +0 -51
- tracdap_runtime-0.6.2/src/tracdap/rt/ext/_guard.py +0 -37
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/LICENSE +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/README.md +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/pyproject.toml +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/graph.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/graph_builder.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/custom_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/custom_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/flow_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/flow_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/stoarge_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/stoarge_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.pyi +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/repos.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/schemas.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/shim.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/storage.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/util.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/_helpers.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_arrow.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_csv.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_parquet.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_git.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_local.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_pypi.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_aws.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_azure.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_gcp.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_local.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/exceptions.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/repos.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/storage.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/__init__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/__main__.py +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/__init__.py +17 -17
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/dependency_links.txt +0 -0
- {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: tracdap-runtime
|
3
|
-
Version: 0.6.
|
3
|
+
Version: 0.6.4
|
4
4
|
Summary: Runtime package for building models on the TRAC Data & Analytics Platform
|
5
5
|
Home-page: https://tracdap.finos.org/
|
6
6
|
Author: Martin Traverse
|
@@ -16,9 +16,9 @@ Classifier: Operating System :: OS Independent
|
|
16
16
|
Requires-Python: <3.13,>=3.8
|
17
17
|
Description-Content-Type: text/markdown
|
18
18
|
License-File: LICENSE
|
19
|
-
Requires-Dist: protobuf==5.
|
20
|
-
Requires-Dist: pyarrow==16.
|
21
|
-
Requires-Dist: pyyaml==6.0.
|
19
|
+
Requires-Dist: protobuf==5.28.2
|
20
|
+
Requires-Dist: pyarrow==16.1.0
|
21
|
+
Requires-Dist: pyyaml==6.0.2
|
22
22
|
Requires-Dist: dulwich==0.22.1
|
23
23
|
Requires-Dist: requests==2.32.3
|
24
24
|
Requires-Dist: pandas<2.3.0,>=1.2.0
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[metadata]
|
2
2
|
name = tracdap-runtime
|
3
|
-
version = 0.6.
|
3
|
+
version = 0.6.4
|
4
4
|
description = Runtime package for building models on the TRAC Data & Analytics Platform
|
5
5
|
long_description = file: README.md
|
6
6
|
long_description_content_type = text/markdown
|
@@ -31,8 +31,6 @@ packages =
|
|
31
31
|
tracdap.rt._impl.grpc
|
32
32
|
tracdap.rt._impl.grpc.tracdap
|
33
33
|
tracdap.rt._impl.grpc.tracdap.metadata
|
34
|
-
tracdap.rt._impl.grpc.tracdap.config
|
35
|
-
tracdap.rt._impl.grpc.tracdap.api
|
36
34
|
tracdap.rt._impl.grpc.tracdap.api.internal
|
37
35
|
tracdap.rt._plugins
|
38
36
|
package_dir =
|
@@ -40,9 +38,9 @@ package_dir =
|
|
40
38
|
tracdap.rt = src/tracdap/rt
|
41
39
|
python_requires = >= 3.8, < 3.13
|
42
40
|
install_requires =
|
43
|
-
protobuf == 5.
|
44
|
-
pyarrow == 16.
|
45
|
-
pyyaml == 6.0.
|
41
|
+
protobuf == 5.28.2
|
42
|
+
pyarrow == 16.1.0
|
43
|
+
pyyaml == 6.0.2
|
46
44
|
dulwich == 0.22.1
|
47
45
|
requests == 2.32.3
|
48
46
|
|
@@ -25,6 +25,7 @@ import queue
|
|
25
25
|
import time
|
26
26
|
|
27
27
|
import tracdap.rt._impl.util as util # noqa
|
28
|
+
import tracdap.rt._impl.validation as _val # noqa
|
28
29
|
import tracdap.rt.exceptions as _ex
|
29
30
|
|
30
31
|
|
@@ -180,6 +181,49 @@ class ActorContext:
|
|
180
181
|
return self.__error or self.__node.error
|
181
182
|
|
182
183
|
|
184
|
+
class ThreadsafeActor(Actor):
|
185
|
+
|
186
|
+
def __init__(self):
|
187
|
+
super().__init__()
|
188
|
+
self.__threadsafe: tp.Optional[ThreadsafeContext] = None
|
189
|
+
|
190
|
+
def threadsafe(self) -> ThreadsafeContext:
|
191
|
+
return self.__threadsafe
|
192
|
+
|
193
|
+
|
194
|
+
class ThreadsafeContext:
|
195
|
+
|
196
|
+
def __init__(self, node: ActorNode):
|
197
|
+
self.__node = node
|
198
|
+
self.__id = node.actor_id
|
199
|
+
self.__parent = node.parent.actor_id if node.parent is not None else None
|
200
|
+
|
201
|
+
def spawn(self, actor: Actor):
|
202
|
+
self.__node.event_loop.post_message(
|
203
|
+
None, lambda _:
|
204
|
+
self.__node.spawn(actor) and None)
|
205
|
+
|
206
|
+
def send(self, target_id: ActorId, message: str, *args, **kwargs):
|
207
|
+
self.__node.event_loop.post_message(
|
208
|
+
None, lambda _:
|
209
|
+
self.__node.send_message(self.__id, target_id, message, args, kwargs))
|
210
|
+
|
211
|
+
def send_parent(self, message: str, *args, **kwargs):
|
212
|
+
self.__node.event_loop.post_message(
|
213
|
+
None, lambda _:
|
214
|
+
self.__node.send_message(self.__id, self.__parent, message, args, kwargs))
|
215
|
+
|
216
|
+
def stop(self):
|
217
|
+
self.__node.event_loop.post_message(
|
218
|
+
None, lambda _:
|
219
|
+
self.__node.send_signal(self.__id, self.__id, SignalNames.STOP))
|
220
|
+
|
221
|
+
def fail(self, error: Exception):
|
222
|
+
self.__node.event_loop.post_message(
|
223
|
+
None, lambda _:
|
224
|
+
self.__node.send_signal(self.__id, self.__id, SignalNames.STOP, error))
|
225
|
+
|
226
|
+
|
183
227
|
class EventLoop:
|
184
228
|
|
185
229
|
_T_MSG = tp.TypeVar("_T_MSG")
|
@@ -340,7 +384,7 @@ class ActorNode:
|
|
340
384
|
self.state: ActorState = ActorState.NOT_STARTED
|
341
385
|
self.error: tp.Optional[Exception] = None
|
342
386
|
|
343
|
-
def spawn(self, child_actor: Actor):
|
387
|
+
def spawn(self, child_actor: Actor) -> ActorId:
|
344
388
|
|
345
389
|
if self._log.isEnabledFor(logging.DEBUG):
|
346
390
|
self._log.debug(f"spawn [{self.actor_id}]: [{type(child_actor)}]")
|
@@ -355,6 +399,11 @@ class ActorNode:
|
|
355
399
|
child_node = ActorNode(child_id, child_actor, self, self.system, event_loop)
|
356
400
|
self.children[child_id] = child_node
|
357
401
|
|
402
|
+
# If this is a threadsafe actor, set up the threadsafe context
|
403
|
+
if isinstance(child_actor, ThreadsafeActor):
|
404
|
+
threadsafe = ThreadsafeContext(child_node)
|
405
|
+
child_actor._ThreadsafeActor__threadsafe = threadsafe
|
406
|
+
|
358
407
|
child_node.send_signal(self.actor_id, child_id, SignalNames.START)
|
359
408
|
|
360
409
|
return child_id
|
@@ -542,6 +591,12 @@ class ActorNode:
|
|
542
591
|
if not self._check_message_target(signal):
|
543
592
|
return
|
544
593
|
|
594
|
+
# Do not process signals after the actor has stopped
|
595
|
+
# This is common with e.g. STOP signals that propagate up and down the tree
|
596
|
+
|
597
|
+
if self.state in [ActorState.STOPPED, ActorState.FAILED]:
|
598
|
+
return
|
599
|
+
|
545
600
|
# Call the signal receiver function
|
546
601
|
# This gives the actor a chance to respond to the signal
|
547
602
|
|
@@ -768,10 +823,12 @@ class ActorNode:
|
|
768
823
|
# Positional arg types
|
769
824
|
for pos_param, pos_arg in zip(pos_params, args):
|
770
825
|
|
826
|
+
# If no type hint is available, allow anything through
|
827
|
+
# Otherwise, reuse the validator logic to type check individual args
|
771
828
|
type_hint = type_hints.get(pos_param.name)
|
829
|
+
type_check = type_hint is None or _val.check_type(type_hint, pos_arg)
|
772
830
|
|
773
|
-
|
774
|
-
if type_hint is not None and not isinstance(pos_arg, type_hint):
|
831
|
+
if not type_check:
|
775
832
|
error = f"Invalid message: [{message}] -> {target_id} (wrong parameter type for '{pos_param.name}')"
|
776
833
|
self._log.error(error)
|
777
834
|
raise EBadActor(error)
|
@@ -780,20 +837,20 @@ class ActorNode:
|
|
780
837
|
for kw_param in kw_params:
|
781
838
|
|
782
839
|
kw_arg = kwargs.get(kw_param.name)
|
783
|
-
type_hint = type_hints.get(kw_param.name)
|
784
840
|
|
785
841
|
# If param has taken a default value, no type check is needed
|
786
842
|
if kw_arg is None:
|
787
843
|
continue
|
788
844
|
|
789
|
-
#
|
790
|
-
|
845
|
+
# Otherwise use the same type-validation logic as positional args
|
846
|
+
type_hint = type_hints.get(kw_param.name)
|
847
|
+
type_check = type_hint is None or _val.check_type(type_hint, kw_arg)
|
848
|
+
|
849
|
+
if not type_check:
|
791
850
|
error = f"Invalid message: [{message}] -> {target_id} (wrong parameter type for '{kw_param.name}')"
|
792
851
|
self._log.error(error)
|
793
852
|
raise EBadActor(error)
|
794
853
|
|
795
|
-
# TODO: Verify generics for both args and kwargs
|
796
|
-
|
797
854
|
|
798
855
|
class RootActor(Actor):
|
799
856
|
|
@@ -864,11 +921,17 @@ class ActorSystem:
|
|
864
921
|
|
865
922
|
self.__root_started = threading.Event()
|
866
923
|
self.__root_stopped = threading.Event()
|
924
|
+
|
867
925
|
self.__root_actor = RootActor(main_actor, self.__root_started, self.__root_stopped)
|
868
926
|
self.__root_node = ActorNode(self.ROOT_ID, self.__root_actor, None, self, self.__system_event_loop)
|
869
927
|
|
870
928
|
# Public API
|
871
929
|
|
930
|
+
def main_id(self) -> ActorId:
|
931
|
+
if not self.__root_started.is_set():
|
932
|
+
raise EBadActor("System has not started yet")
|
933
|
+
return self.__root_actor.main_id
|
934
|
+
|
872
935
|
def start(self, wait=True):
|
873
936
|
|
874
937
|
self.__system_thread.start()
|
@@ -913,12 +976,26 @@ class ActorSystem:
|
|
913
976
|
|
914
977
|
return self.__root_node.error
|
915
978
|
|
916
|
-
def
|
979
|
+
def spawn_agent(self, agent: Actor) -> ActorId:
|
980
|
+
|
981
|
+
if not self.__root_started.is_set():
|
982
|
+
raise EBadActor("System has not started yet")
|
983
|
+
|
984
|
+
return self.__root_node.spawn(agent)
|
985
|
+
|
986
|
+
def send_main(self, message: str, *args, **kwargs):
|
917
987
|
|
918
988
|
if self.__root_actor.main_id is None:
|
919
989
|
raise EBadActor("System has not started yet")
|
920
990
|
|
921
|
-
self.__root_node.send_message("/external", self.__root_actor.main_id, message, args, kwargs)
|
991
|
+
self.__root_node.send_message("/external", self.__root_actor.main_id, message, args, kwargs) # TODO
|
992
|
+
|
993
|
+
def send(self, actor_id: ActorId, message: str, *args, **kwargs):
|
994
|
+
|
995
|
+
if not self.__root_started.is_set():
|
996
|
+
raise EBadActor("System has not started yet")
|
997
|
+
|
998
|
+
self.__root_node.send_message("/external", actor_id, message, args, kwargs)
|
922
999
|
|
923
1000
|
def _setup_event_loops(self, thread_pools: tp.Dict[str, int]):
|
924
1001
|
|
@@ -0,0 +1,426 @@
|
|
1
|
+
# Copyright 2022 Accenture Global Solutions Limited
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
import copy
|
16
|
+
import logging
|
17
|
+
import pathlib
|
18
|
+
import typing as tp
|
19
|
+
import re
|
20
|
+
import traceback
|
21
|
+
|
22
|
+
import pandas as pd
|
23
|
+
|
24
|
+
import tracdap.rt.api as _api
|
25
|
+
import tracdap.rt.metadata as _meta
|
26
|
+
import tracdap.rt.exceptions as _ex
|
27
|
+
import tracdap.rt._impl.type_system as _types # noqa
|
28
|
+
import tracdap.rt._impl.data as _data # noqa
|
29
|
+
import tracdap.rt._impl.util as _util # noqa
|
30
|
+
import tracdap.rt._impl.validation as _val # noqa
|
31
|
+
|
32
|
+
|
33
|
+
class TracContextImpl(_api.TracContext):
|
34
|
+
|
35
|
+
"""
|
36
|
+
TracContextImpl is the main implementation of the API class TracContext (from tracdap.rt.api).
|
37
|
+
It provides get/put operations on the inputs, outputs and parameters of a model according to the model definition,
|
38
|
+
as well as exposing other information needed by the model at runtime and offering a few utility functions.
|
39
|
+
|
40
|
+
An instance of TracContextImpl is constructed by the runtime engine for each model node in the execution graph.
|
41
|
+
Parameters and schemas will be pre-populated from the job definition. Normally, input data will be created by parent
|
42
|
+
nodes in the execution graph and passed in when the context is constructed, so they are available when required. In
|
43
|
+
the simplest case, a get call picks a dataset from the map and a put call puts a dataset into the map. Under this
|
44
|
+
mechanism, no outputs are passed downstream until the model finishes executing.
|
45
|
+
|
46
|
+
Optimizations for lazy loading and eager saving require the context to call back into the runtime engine. For lazy
|
47
|
+
load, the graph node to prepare an input is injected when the data is requested and the model thread blocks until
|
48
|
+
it is available; for eager save outputs are sent to child actors as soon as they are produced. In both cases this
|
49
|
+
complexity is hidden from the model, which only sees one thread with synchronous get/put calls.
|
50
|
+
|
51
|
+
:param model_def: Definition object for the model that will run in this context
|
52
|
+
:param model_class: Type for the model that will run in this context
|
53
|
+
:param local_ctx: Dictionary of all parameters, inputs and outputs that will be available to the model
|
54
|
+
Parameters are supplied as python native types.
|
55
|
+
Output views will contain schemas but no data.
|
56
|
+
"""
|
57
|
+
|
58
|
+
__DEFAULT_TEMPORAL_OBJECTS = False
|
59
|
+
|
60
|
+
def __init__(self,
|
61
|
+
model_def: _meta.ModelDefinition,
|
62
|
+
model_class: _api.TracModel.__class__,
|
63
|
+
local_ctx: tp.Dict[str, tp.Any],
|
64
|
+
checkout_directory: pathlib.Path = None):
|
65
|
+
|
66
|
+
self.__ctx_log = _util.logger_for_object(self)
|
67
|
+
self.__model_log = _util.logger_for_class(model_class)
|
68
|
+
|
69
|
+
self.__model_def = model_def
|
70
|
+
self.__model_class = model_class
|
71
|
+
self.__local_ctx = local_ctx or {}
|
72
|
+
|
73
|
+
self.__val = TracContextValidator(
|
74
|
+
self.__ctx_log,
|
75
|
+
self.__model_def,
|
76
|
+
self.__local_ctx,
|
77
|
+
checkout_directory)
|
78
|
+
|
79
|
+
def get_parameter(self, parameter_name: str) -> tp.Any:
|
80
|
+
|
81
|
+
_val.validate_signature(self.get_parameter, parameter_name)
|
82
|
+
|
83
|
+
self.__val.check_param_valid_identifier(parameter_name)
|
84
|
+
self.__val.check_param_defined_in_model(parameter_name)
|
85
|
+
self.__val.check_param_available_in_context(parameter_name)
|
86
|
+
|
87
|
+
value: _meta.Value = self.__local_ctx.get(parameter_name)
|
88
|
+
|
89
|
+
self.__val.check_context_object_type(parameter_name, value, _meta.Value)
|
90
|
+
|
91
|
+
return _types.MetadataCodec.decode_value(value)
|
92
|
+
|
93
|
+
def has_dataset(self, dataset_name: str) -> bool:
|
94
|
+
|
95
|
+
_val.validate_signature(self.has_dataset, dataset_name)
|
96
|
+
|
97
|
+
self.__val.check_dataset_valid_identifier(dataset_name)
|
98
|
+
self.__val.check_dataset_defined_in_model(dataset_name)
|
99
|
+
|
100
|
+
data_view: _data.DataView = self.__local_ctx.get(dataset_name)
|
101
|
+
|
102
|
+
if data_view is None:
|
103
|
+
return False
|
104
|
+
|
105
|
+
self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
|
106
|
+
|
107
|
+
return not data_view.is_empty()
|
108
|
+
|
109
|
+
def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:
|
110
|
+
|
111
|
+
_val.validate_signature(self.get_schema, dataset_name)
|
112
|
+
|
113
|
+
self.__val.check_dataset_valid_identifier(dataset_name)
|
114
|
+
self.__val.check_dataset_defined_in_model(dataset_name)
|
115
|
+
self.__val.check_dataset_available_in_context(dataset_name)
|
116
|
+
|
117
|
+
static_schema = self.__get_static_schema(self.__model_def, dataset_name)
|
118
|
+
data_view: _data.DataView = self.__local_ctx.get(dataset_name)
|
119
|
+
|
120
|
+
# Check the data view has a well-defined schema even if a static schema exists in the model
|
121
|
+
# This ensures errors are always reported and is consistent with get_pandas_table()
|
122
|
+
|
123
|
+
self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
|
124
|
+
self.__val.check_dataset_schema_defined(dataset_name, data_view)
|
125
|
+
|
126
|
+
# If a static schema exists, that takes priority
|
127
|
+
# Return deep copies, do not allow model code to change schemas provided by the engine
|
128
|
+
|
129
|
+
if static_schema is not None:
|
130
|
+
return copy.deepcopy(static_schema)
|
131
|
+
else:
|
132
|
+
return copy.deepcopy(data_view.trac_schema)
|
133
|
+
|
134
|
+
def get_pandas_table(self, dataset_name: str, use_temporal_objects: tp.Optional[bool] = None) -> pd.DataFrame:
|
135
|
+
|
136
|
+
_val.validate_signature(self.get_pandas_table, dataset_name, use_temporal_objects)
|
137
|
+
|
138
|
+
self.__val.check_dataset_valid_identifier(dataset_name)
|
139
|
+
self.__val.check_dataset_defined_in_model(dataset_name)
|
140
|
+
self.__val.check_dataset_available_in_context(dataset_name)
|
141
|
+
|
142
|
+
static_schema = self.__get_static_schema(self.__model_def, dataset_name)
|
143
|
+
data_view = self.__local_ctx.get(dataset_name)
|
144
|
+
part_key = _data.DataPartKey.for_root()
|
145
|
+
|
146
|
+
self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
|
147
|
+
self.__val.check_dataset_schema_defined(dataset_name, data_view)
|
148
|
+
self.__val.check_dataset_part_present(dataset_name, data_view, part_key)
|
149
|
+
|
150
|
+
# If the model defines a static input schema, use that for schema conformance
|
151
|
+
# Otherwise, take what is in the incoming dataset (schema is dynamic)
|
152
|
+
|
153
|
+
if static_schema is not None:
|
154
|
+
schema = _data.DataMapping.trac_to_arrow_schema(static_schema)
|
155
|
+
else:
|
156
|
+
schema = data_view.arrow_schema
|
157
|
+
|
158
|
+
if use_temporal_objects is None:
|
159
|
+
use_temporal_objects = self.__DEFAULT_TEMPORAL_OBJECTS
|
160
|
+
|
161
|
+
return _data.DataMapping.view_to_pandas(data_view, part_key, schema, use_temporal_objects)
|
162
|
+
|
163
|
+
def put_schema(self, dataset_name: str, schema: _meta.SchemaDefinition):
|
164
|
+
|
165
|
+
_val.validate_signature(self.get_schema, dataset_name, schema)
|
166
|
+
|
167
|
+
# Copy the schema - schema cannot be changed in model code after put_schema
|
168
|
+
# If field ordering is not assigned by the model, assign it here (model code will not see the numbers)
|
169
|
+
schema_copy = self.__assign_field_order(copy.deepcopy(schema))
|
170
|
+
|
171
|
+
self.__val.check_dataset_valid_identifier(dataset_name)
|
172
|
+
self.__val.check_dataset_is_dynamic_output(dataset_name)
|
173
|
+
self.__val.check_provided_schema_is_valid(dataset_name, schema_copy)
|
174
|
+
|
175
|
+
static_schema = self.__get_static_schema(self.__model_def, dataset_name)
|
176
|
+
data_view = self.__local_ctx.get(dataset_name)
|
177
|
+
|
178
|
+
if data_view is None:
|
179
|
+
if static_schema is not None:
|
180
|
+
data_view = _data.DataView.for_trac_schema(static_schema)
|
181
|
+
else:
|
182
|
+
data_view = _data.DataView.create_empty()
|
183
|
+
|
184
|
+
# If there is a prior view it must contain nothing and will be replaced
|
185
|
+
self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
|
186
|
+
self.__val.check_dataset_schema_not_defined(dataset_name, data_view)
|
187
|
+
self.__val.check_dataset_is_empty(dataset_name, data_view)
|
188
|
+
|
189
|
+
updated_view = data_view.with_trac_schema(schema_copy)
|
190
|
+
|
191
|
+
self.__local_ctx[dataset_name] = updated_view
|
192
|
+
|
193
|
+
def put_pandas_table(self, dataset_name: str, dataset: pd.DataFrame):
|
194
|
+
|
195
|
+
_val.validate_signature(self.put_pandas_table, dataset_name, dataset)
|
196
|
+
|
197
|
+
self.__val.check_dataset_valid_identifier(dataset_name)
|
198
|
+
self.__val.check_dataset_is_model_output(dataset_name)
|
199
|
+
self.__val.check_provided_dataset_type(dataset, pd.DataFrame)
|
200
|
+
|
201
|
+
static_schema = self.__get_static_schema(self.__model_def, dataset_name)
|
202
|
+
data_view = self.__local_ctx.get(dataset_name)
|
203
|
+
part_key = _data.DataPartKey.for_root()
|
204
|
+
|
205
|
+
if data_view is None:
|
206
|
+
if static_schema is not None:
|
207
|
+
data_view = _data.DataView.for_trac_schema(static_schema)
|
208
|
+
else:
|
209
|
+
data_view = _data.DataView.create_empty()
|
210
|
+
|
211
|
+
self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
|
212
|
+
self.__val.check_dataset_schema_defined(dataset_name, data_view)
|
213
|
+
self.__val.check_dataset_part_not_present(dataset_name, data_view, part_key)
|
214
|
+
|
215
|
+
# Prefer static schemas for data conformance
|
216
|
+
|
217
|
+
if static_schema is not None:
|
218
|
+
schema = _data.DataMapping.trac_to_arrow_schema(static_schema)
|
219
|
+
else:
|
220
|
+
schema = data_view.arrow_schema
|
221
|
+
|
222
|
+
# Data conformance is applied inside these conversion functions
|
223
|
+
|
224
|
+
updated_item = _data.DataMapping.pandas_to_item(dataset, schema)
|
225
|
+
updated_view = _data.DataMapping.add_item_to_view(data_view, part_key, updated_item)
|
226
|
+
|
227
|
+
self.__local_ctx[dataset_name] = updated_view
|
228
|
+
|
229
|
+
def log(self) -> logging.Logger:
|
230
|
+
|
231
|
+
_val.validate_signature(self.log)
|
232
|
+
|
233
|
+
return self.__model_log
|
234
|
+
|
235
|
+
@staticmethod
|
236
|
+
def __get_static_schema(model_def: _meta.ModelDefinition, dataset_name: str):
|
237
|
+
|
238
|
+
input_schema = model_def.inputs.get(dataset_name)
|
239
|
+
|
240
|
+
if input_schema is not None and not input_schema.dynamic:
|
241
|
+
return input_schema.schema
|
242
|
+
|
243
|
+
output_schema = model_def.outputs.get(dataset_name)
|
244
|
+
|
245
|
+
if output_schema is not None and not output_schema.dynamic:
|
246
|
+
return output_schema.schema
|
247
|
+
|
248
|
+
return None
|
249
|
+
|
250
|
+
@staticmethod
|
251
|
+
def __assign_field_order(schema_def: _meta.SchemaDefinition):
|
252
|
+
|
253
|
+
if schema_def is None or schema_def.table is None or schema_def.table.fields is None:
|
254
|
+
return schema_def
|
255
|
+
|
256
|
+
if all(map(lambda f: f.fieldOrder is None, schema_def.table.fields)):
|
257
|
+
for index, field in enumerate(schema_def.table.fields):
|
258
|
+
field.fieldOrder = index
|
259
|
+
|
260
|
+
return schema_def
|
261
|
+
|
262
|
+
|
263
|
+
class TracContextValidator:
|
264
|
+
|
265
|
+
__VALID_IDENTIFIER = re.compile("^[a-zA-Z_]\\w*$",)
|
266
|
+
__RESERVED_IDENTIFIER = re.compile("^(trac_|_)\\w*")
|
267
|
+
|
268
|
+
def __init__(
|
269
|
+
self, log: logging.Logger,
|
270
|
+
model_def: _meta.ModelDefinition,
|
271
|
+
local_ctx: tp.Dict[str, tp.Any],
|
272
|
+
checkout_directory: pathlib.Path):
|
273
|
+
|
274
|
+
self.__log = log
|
275
|
+
self.__model_def = model_def
|
276
|
+
self.__local_ctx = local_ctx
|
277
|
+
self.__checkout_directory = checkout_directory
|
278
|
+
|
279
|
+
def _report_error(self, message, cause: Exception = None):
|
280
|
+
|
281
|
+
full_stack = traceback.extract_stack()
|
282
|
+
model_stack = _util.filter_model_stack_trace(full_stack, self.__checkout_directory)
|
283
|
+
model_stack_str = ''.join(traceback.format_list(list(reversed(model_stack))))
|
284
|
+
details = _util.error_details_from_trace(model_stack)
|
285
|
+
message = f"{message} {details}"
|
286
|
+
|
287
|
+
self.__log.error(message)
|
288
|
+
self.__log.error(f"Model stack trace:\n{model_stack_str}")
|
289
|
+
|
290
|
+
if cause:
|
291
|
+
raise _ex.ERuntimeValidation(message) from cause
|
292
|
+
else:
|
293
|
+
raise _ex.ERuntimeValidation(message)
|
294
|
+
|
295
|
+
def check_param_valid_identifier(self, param_name: str):
|
296
|
+
|
297
|
+
if param_name is None:
|
298
|
+
self._report_error(f"Parameter name is null")
|
299
|
+
|
300
|
+
if not self.__VALID_IDENTIFIER.match(param_name):
|
301
|
+
self._report_error(f"Parameter name {param_name} is not a valid identifier")
|
302
|
+
|
303
|
+
def check_param_defined_in_model(self, param_name: str):
|
304
|
+
|
305
|
+
if param_name not in self.__model_def.parameters:
|
306
|
+
self._report_error(f"Parameter {param_name} is not defined in the model")
|
307
|
+
|
308
|
+
def check_param_available_in_context(self, param_name: str):
|
309
|
+
|
310
|
+
if param_name not in self.__local_ctx:
|
311
|
+
self._report_error(f"Parameter {param_name} is not available in the current context")
|
312
|
+
|
313
|
+
def check_dataset_valid_identifier(self, dataset_name: str):
|
314
|
+
|
315
|
+
if dataset_name is None:
|
316
|
+
self._report_error(f"Dataset name is null")
|
317
|
+
|
318
|
+
if not self.__VALID_IDENTIFIER.match(dataset_name):
|
319
|
+
self._report_error(f"Dataset name {dataset_name} is not a valid identifier")
|
320
|
+
|
321
|
+
def check_dataset_defined_in_model(self, dataset_name: str):
|
322
|
+
|
323
|
+
if dataset_name not in self.__model_def.inputs and dataset_name not in self.__model_def.outputs:
|
324
|
+
self._report_error(f"Dataset {dataset_name} is not defined in the model")
|
325
|
+
|
326
|
+
def check_dataset_is_model_output(self, dataset_name: str):
|
327
|
+
|
328
|
+
if dataset_name not in self.__model_def.outputs:
|
329
|
+
self._report_error(f"Dataset {dataset_name} is not defined as a model output")
|
330
|
+
|
331
|
+
def check_dataset_is_dynamic_output(self, dataset_name: str):
|
332
|
+
|
333
|
+
model_output: _meta.ModelOutputSchema = self.__model_def.outputs.get(dataset_name)
|
334
|
+
|
335
|
+
if model_output is None:
|
336
|
+
self._report_error(f"Dataset {dataset_name} is not defined as a model output")
|
337
|
+
|
338
|
+
if not model_output.dynamic:
|
339
|
+
self._report_error(f"Model output {dataset_name} is not a dynamic output")
|
340
|
+
|
341
|
+
def check_dataset_available_in_context(self, item_name: str):
|
342
|
+
|
343
|
+
if item_name not in self.__local_ctx:
|
344
|
+
self._report_error(f"Dataset {item_name} is not available in the current context")
|
345
|
+
|
346
|
+
def check_dataset_schema_defined(self, dataset_name: str, data_view: _data.DataView):
|
347
|
+
|
348
|
+
schema = data_view.trac_schema if data_view is not None else None
|
349
|
+
|
350
|
+
if schema is None or schema.table is None or not schema.table.fields:
|
351
|
+
self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
|
352
|
+
|
353
|
+
def check_dataset_schema_not_defined(self, dataset_name: str, data_view: _data.DataView):
|
354
|
+
|
355
|
+
schema = data_view.trac_schema if data_view is not None else None
|
356
|
+
|
357
|
+
if schema is not None and (schema.table or schema.schemaType != _meta.SchemaType.SCHEMA_TYPE_NOT_SET):
|
358
|
+
self._report_error(f"Schema already defined for dataset {dataset_name} in the current context")
|
359
|
+
|
360
|
+
def check_dataset_part_present(self, dataset_name: str, data_view: _data.DataView, part_key: _data.DataPartKey):
|
361
|
+
|
362
|
+
part = data_view.parts.get(part_key) if data_view.parts is not None else None
|
363
|
+
|
364
|
+
if part is None or len(part) == 0:
|
365
|
+
self._report_error(f"No data present for {dataset_name} ({part_key}) in the current context")
|
366
|
+
|
367
|
+
def check_dataset_part_not_present(self, dataset_name: str, data_view: _data.DataView, part_key: _data.DataPartKey):
|
368
|
+
|
369
|
+
part = data_view.parts.get(part_key) if data_view.parts is not None else None
|
370
|
+
|
371
|
+
if part is not None and len(part) > 0:
|
372
|
+
self._report_error(f"Data already present for {dataset_name} ({part_key}) in the current context")
|
373
|
+
|
374
|
+
def check_dataset_is_empty(self, dataset_name: str, data_view: _data.DataView):
|
375
|
+
|
376
|
+
if not data_view.is_empty():
|
377
|
+
self._report_error(f"Dataset {dataset_name} is not empty")
|
378
|
+
|
379
|
+
def check_provided_schema_is_valid(self, dataset_name: str, schema: _meta.SchemaDefinition):
|
380
|
+
|
381
|
+
if schema is None:
|
382
|
+
self._report_error(f"The schema provided for [{dataset_name}] is null")
|
383
|
+
|
384
|
+
if not isinstance(schema, _meta.SchemaDefinition):
|
385
|
+
schema_type_name = self._type_name(type(schema))
|
386
|
+
self._report_error(f"The object provided for [{dataset_name}] is not a schema (got {schema_type_name})")
|
387
|
+
|
388
|
+
try:
|
389
|
+
_val.StaticValidator.quick_validate_schema(schema)
|
390
|
+
except _ex.EModelValidation as e:
|
391
|
+
self._report_error(f"The schema provided for [{dataset_name}] failed validation: {str(e)}", e)
|
392
|
+
|
393
|
+
def check_provided_dataset_type(self, dataset: tp.Any, expected_type: type):
|
394
|
+
|
395
|
+
if dataset is None:
|
396
|
+
self._report_error(f"Provided dataset is null")
|
397
|
+
|
398
|
+
if not isinstance(dataset, expected_type):
|
399
|
+
|
400
|
+
expected_type_name = self._type_name(expected_type)
|
401
|
+
actual_type_name = self._type_name(type(dataset))
|
402
|
+
|
403
|
+
self._report_error(
|
404
|
+
f"Provided dataset is the wrong type" +
|
405
|
+
f" (expected {expected_type_name}, got {actual_type_name})")
|
406
|
+
|
407
|
+
def check_context_object_type(self, item_name: str, item: tp.Any, expected_type: type):
|
408
|
+
|
409
|
+
if not isinstance(item, expected_type):
|
410
|
+
|
411
|
+
expected_type_name = self._type_name(expected_type)
|
412
|
+
actual_type_name = self._type_name(type(item))
|
413
|
+
|
414
|
+
self._report_error(
|
415
|
+
f"The object referenced by [{item_name}] in the current context has the wrong type" +
|
416
|
+
f" (expected {expected_type_name}, got {actual_type_name})")
|
417
|
+
|
418
|
+
@staticmethod
|
419
|
+
def _type_name(type_: type):
|
420
|
+
|
421
|
+
module = type_.__module__
|
422
|
+
|
423
|
+
if module is None or module == str.__class__.__module__:
|
424
|
+
return type_.__qualname__
|
425
|
+
|
426
|
+
return module + '.' + type_.__name__
|