tracdap-runtime 0.6.2__tar.gz → 0.6.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. {tracdap_runtime-0.6.2/tracdap_runtime.egg-info → tracdap_runtime-0.6.4}/PKG-INFO +4 -4
  2. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/setup.cfg +4 -6
  3. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/actors.py +87 -10
  4. tracdap_runtime-0.6.4/src/tracdap/rt/_exec/context.py +426 -0
  5. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/dev_mode.py +52 -20
  6. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/engine.py +79 -14
  7. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/functions.py +14 -17
  8. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/runtime.py +83 -40
  9. tracdap_runtime-0.6.4/src/tracdap/rt/_exec/server.py +345 -0
  10. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/config_parser.py +219 -49
  11. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/data.py +70 -5
  12. tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/codec.py +99 -0
  13. tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py +51 -0
  14. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.pyi +11 -9
  15. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2_grpc.py +25 -25
  16. tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +51 -0
  17. tracdap_runtime-0.6.4/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py +63 -0
  18. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.pyi +37 -6
  19. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py +8 -3
  20. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi +13 -2
  21. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/guard_rails.py +21 -0
  22. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/models.py +25 -0
  23. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/static_api.py +43 -13
  24. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/type_system.py +17 -0
  25. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/validation.py +47 -4
  26. tracdap_runtime-0.6.4/src/tracdap/rt/_plugins/config_local.py +49 -0
  27. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_version.py +1 -1
  28. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/hook.py +6 -5
  29. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/model_api.py +50 -7
  30. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/static_api.py +81 -23
  31. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/__init__.py +4 -4
  32. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/common.py +25 -15
  33. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/job.py +2 -2
  34. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/platform.py +25 -35
  35. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/result.py +2 -2
  36. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/config/runtime.py +4 -2
  37. tracdap_runtime-0.6.4/src/tracdap/rt/ext/config.py +34 -0
  38. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/embed.py +1 -3
  39. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/plugins.py +47 -6
  40. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/cli.py +11 -4
  41. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/launch.py +53 -12
  42. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/common.py +2 -2
  43. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/custom.py +3 -3
  44. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/data.py +12 -12
  45. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/file.py +6 -6
  46. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/flow.py +6 -6
  47. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/job.py +8 -8
  48. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/model.py +21 -11
  49. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/object.py +3 -0
  50. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/object_id.py +8 -8
  51. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/search.py +5 -5
  52. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/stoarge.py +6 -6
  53. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/tag.py +1 -1
  54. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/tag_update.py +1 -1
  55. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/type.py +4 -4
  56. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4/tracdap_runtime.egg-info}/PKG-INFO +4 -4
  57. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/SOURCES.txt +2 -11
  58. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/requires.txt +3 -3
  59. tracdap_runtime-0.6.2/src/tracdap/rt/_exec/context.py +0 -319
  60. tracdap_runtime-0.6.2/src/tracdap/rt/_exec/server.py +0 -68
  61. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/codec.py +0 -44
  62. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py +0 -51
  63. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/common_pb2.py +0 -55
  64. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/common_pb2.pyi +0 -103
  65. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/job_pb2.py +0 -42
  66. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/job_pb2.pyi +0 -44
  67. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/platform_pb2.py +0 -71
  68. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/platform_pb2.pyi +0 -197
  69. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/result_pb2.py +0 -37
  70. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/result_pb2.pyi +0 -35
  71. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/runtime_pb2.py +0 -42
  72. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/config/runtime_pb2.pyi +0 -46
  73. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +0 -51
  74. tracdap_runtime-0.6.2/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py +0 -51
  75. tracdap_runtime-0.6.2/src/tracdap/rt/ext/_guard.py +0 -37
  76. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/LICENSE +0 -0
  77. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/README.md +0 -0
  78. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/pyproject.toml +0 -0
  79. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/__init__.py +0 -0
  80. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/__init__.py +0 -0
  81. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/graph.py +0 -0
  82. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_exec/graph_builder.py +0 -0
  83. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/__init__.py +0 -0
  84. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/__init__.py +0 -0
  85. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.py +0 -0
  86. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.pyi +0 -0
  87. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/custom_pb2.py +0 -0
  88. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/custom_pb2.pyi +0 -0
  89. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi +0 -0
  90. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.py +0 -0
  91. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.pyi +0 -0
  92. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/flow_pb2.py +0 -0
  93. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/flow_pb2.pyi +0 -0
  94. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +0 -0
  95. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi +0 -0
  96. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.py +0 -0
  97. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.pyi +0 -0
  98. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.py +0 -0
  99. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.pyi +0 -0
  100. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/stoarge_pb2.py +0 -0
  101. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/stoarge_pb2.pyi +0 -0
  102. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.py +0 -0
  103. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.pyi +0 -0
  104. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.py +0 -0
  105. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.pyi +0 -0
  106. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py +0 -0
  107. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.pyi +0 -0
  108. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/repos.py +0 -0
  109. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/schemas.py +0 -0
  110. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/shim.py +0 -0
  111. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/storage.py +0 -0
  112. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_impl/util.py +0 -0
  113. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/__init__.py +0 -0
  114. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/_helpers.py +0 -0
  115. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_arrow.py +0 -0
  116. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_csv.py +0 -0
  117. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/format_parquet.py +0 -0
  118. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_git.py +0 -0
  119. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_local.py +0 -0
  120. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/repo_pypi.py +0 -0
  121. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_aws.py +0 -0
  122. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_azure.py +0 -0
  123. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_gcp.py +0 -0
  124. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/_plugins/storage_local.py +0 -0
  125. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/api/__init__.py +0 -0
  126. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/exceptions.py +0 -0
  127. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/__init__.py +0 -0
  128. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/repos.py +0 -0
  129. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/ext/storage.py +0 -0
  130. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/__init__.py +0 -0
  131. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/launch/__main__.py +0 -0
  132. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/src/tracdap/rt/metadata/__init__.py +17 -17
  133. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/dependency_links.txt +0 -0
  134. {tracdap_runtime-0.6.2 → tracdap_runtime-0.6.4}/tracdap_runtime.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tracdap-runtime
3
- Version: 0.6.2
3
+ Version: 0.6.4
4
4
  Summary: Runtime package for building models on the TRAC Data & Analytics Platform
5
5
  Home-page: https://tracdap.finos.org/
6
6
  Author: Martin Traverse
@@ -16,9 +16,9 @@ Classifier: Operating System :: OS Independent
16
16
  Requires-Python: <3.13,>=3.8
17
17
  Description-Content-Type: text/markdown
18
18
  License-File: LICENSE
19
- Requires-Dist: protobuf==5.27.4
20
- Requires-Dist: pyarrow==16.0.0
21
- Requires-Dist: pyyaml==6.0.1
19
+ Requires-Dist: protobuf==5.28.2
20
+ Requires-Dist: pyarrow==16.1.0
21
+ Requires-Dist: pyyaml==6.0.2
22
22
  Requires-Dist: dulwich==0.22.1
23
23
  Requires-Dist: requests==2.32.3
24
24
  Requires-Dist: pandas<2.3.0,>=1.2.0
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = tracdap-runtime
3
- version = 0.6.2
3
+ version = 0.6.4
4
4
  description = Runtime package for building models on the TRAC Data & Analytics Platform
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -31,8 +31,6 @@ packages =
31
31
  tracdap.rt._impl.grpc
32
32
  tracdap.rt._impl.grpc.tracdap
33
33
  tracdap.rt._impl.grpc.tracdap.metadata
34
- tracdap.rt._impl.grpc.tracdap.config
35
- tracdap.rt._impl.grpc.tracdap.api
36
34
  tracdap.rt._impl.grpc.tracdap.api.internal
37
35
  tracdap.rt._plugins
38
36
  package_dir =
@@ -40,9 +38,9 @@ package_dir =
40
38
  tracdap.rt = src/tracdap/rt
41
39
  python_requires = >= 3.8, < 3.13
42
40
  install_requires =
43
- protobuf == 5.27.4
44
- pyarrow == 16.0.0
45
- pyyaml == 6.0.1
41
+ protobuf == 5.28.2
42
+ pyarrow == 16.1.0
43
+ pyyaml == 6.0.2
46
44
  dulwich == 0.22.1
47
45
  requests == 2.32.3
48
46
 
@@ -25,6 +25,7 @@ import queue
25
25
  import time
26
26
 
27
27
  import tracdap.rt._impl.util as util # noqa
28
+ import tracdap.rt._impl.validation as _val # noqa
28
29
  import tracdap.rt.exceptions as _ex
29
30
 
30
31
 
@@ -180,6 +181,49 @@ class ActorContext:
180
181
  return self.__error or self.__node.error
181
182
 
182
183
 
184
+ class ThreadsafeActor(Actor):
185
+
186
+ def __init__(self):
187
+ super().__init__()
188
+ self.__threadsafe: tp.Optional[ThreadsafeContext] = None
189
+
190
+ def threadsafe(self) -> ThreadsafeContext:
191
+ return self.__threadsafe
192
+
193
+
194
+ class ThreadsafeContext:
195
+
196
+ def __init__(self, node: ActorNode):
197
+ self.__node = node
198
+ self.__id = node.actor_id
199
+ self.__parent = node.parent.actor_id if node.parent is not None else None
200
+
201
+ def spawn(self, actor: Actor):
202
+ self.__node.event_loop.post_message(
203
+ None, lambda _:
204
+ self.__node.spawn(actor) and None)
205
+
206
+ def send(self, target_id: ActorId, message: str, *args, **kwargs):
207
+ self.__node.event_loop.post_message(
208
+ None, lambda _:
209
+ self.__node.send_message(self.__id, target_id, message, args, kwargs))
210
+
211
+ def send_parent(self, message: str, *args, **kwargs):
212
+ self.__node.event_loop.post_message(
213
+ None, lambda _:
214
+ self.__node.send_message(self.__id, self.__parent, message, args, kwargs))
215
+
216
+ def stop(self):
217
+ self.__node.event_loop.post_message(
218
+ None, lambda _:
219
+ self.__node.send_signal(self.__id, self.__id, SignalNames.STOP))
220
+
221
+ def fail(self, error: Exception):
222
+ self.__node.event_loop.post_message(
223
+ None, lambda _:
224
+ self.__node.send_signal(self.__id, self.__id, SignalNames.STOP, error))
225
+
226
+
183
227
  class EventLoop:
184
228
 
185
229
  _T_MSG = tp.TypeVar("_T_MSG")
@@ -340,7 +384,7 @@ class ActorNode:
340
384
  self.state: ActorState = ActorState.NOT_STARTED
341
385
  self.error: tp.Optional[Exception] = None
342
386
 
343
- def spawn(self, child_actor: Actor):
387
+ def spawn(self, child_actor: Actor) -> ActorId:
344
388
 
345
389
  if self._log.isEnabledFor(logging.DEBUG):
346
390
  self._log.debug(f"spawn [{self.actor_id}]: [{type(child_actor)}]")
@@ -355,6 +399,11 @@ class ActorNode:
355
399
  child_node = ActorNode(child_id, child_actor, self, self.system, event_loop)
356
400
  self.children[child_id] = child_node
357
401
 
402
+ # If this is a threadsafe actor, set up the threadsafe context
403
+ if isinstance(child_actor, ThreadsafeActor):
404
+ threadsafe = ThreadsafeContext(child_node)
405
+ child_actor._ThreadsafeActor__threadsafe = threadsafe
406
+
358
407
  child_node.send_signal(self.actor_id, child_id, SignalNames.START)
359
408
 
360
409
  return child_id
@@ -542,6 +591,12 @@ class ActorNode:
542
591
  if not self._check_message_target(signal):
543
592
  return
544
593
 
594
+ # Do not process signals after the actor has stopped
595
+ # This is common with e.g. STOP signals that propagate up and down the tree
596
+
597
+ if self.state in [ActorState.STOPPED, ActorState.FAILED]:
598
+ return
599
+
545
600
  # Call the signal receiver function
546
601
  # This gives the actor a chance to respond to the signal
547
602
 
@@ -768,10 +823,12 @@ class ActorNode:
768
823
  # Positional arg types
769
824
  for pos_param, pos_arg in zip(pos_params, args):
770
825
 
826
+ # If no type hint is available, allow anything through
827
+ # Otherwise, reuse the validator logic to type check individual args
771
828
  type_hint = type_hints.get(pos_param.name)
829
+ type_check = type_hint is None or _val.check_type(type_hint, pos_arg)
772
830
 
773
- # If no type hint is available, allow anything through
774
- if type_hint is not None and not isinstance(pos_arg, type_hint):
831
+ if not type_check:
775
832
  error = f"Invalid message: [{message}] -> {target_id} (wrong parameter type for '{pos_param.name}')"
776
833
  self._log.error(error)
777
834
  raise EBadActor(error)
@@ -780,20 +837,20 @@ class ActorNode:
780
837
  for kw_param in kw_params:
781
838
 
782
839
  kw_arg = kwargs.get(kw_param.name)
783
- type_hint = type_hints.get(kw_param.name)
784
840
 
785
841
  # If param has taken a default value, no type check is needed
786
842
  if kw_arg is None:
787
843
  continue
788
844
 
789
- # If no type hint is available, allow anything through
790
- if type_hint is not None and not isinstance(kw_arg, type_hint):
845
+ # Otherwise use the same type-validation logic as positional args
846
+ type_hint = type_hints.get(kw_param.name)
847
+ type_check = type_hint is None or _val.check_type(type_hint, kw_arg)
848
+
849
+ if not type_check:
791
850
  error = f"Invalid message: [{message}] -> {target_id} (wrong parameter type for '{kw_param.name}')"
792
851
  self._log.error(error)
793
852
  raise EBadActor(error)
794
853
 
795
- # TODO: Verify generics for both args and kwargs
796
-
797
854
 
798
855
  class RootActor(Actor):
799
856
 
@@ -864,11 +921,17 @@ class ActorSystem:
864
921
 
865
922
  self.__root_started = threading.Event()
866
923
  self.__root_stopped = threading.Event()
924
+
867
925
  self.__root_actor = RootActor(main_actor, self.__root_started, self.__root_stopped)
868
926
  self.__root_node = ActorNode(self.ROOT_ID, self.__root_actor, None, self, self.__system_event_loop)
869
927
 
870
928
  # Public API
871
929
 
930
+ def main_id(self) -> ActorId:
931
+ if not self.__root_started.is_set():
932
+ raise EBadActor("System has not started yet")
933
+ return self.__root_actor.main_id
934
+
872
935
  def start(self, wait=True):
873
936
 
874
937
  self.__system_thread.start()
@@ -913,12 +976,26 @@ class ActorSystem:
913
976
 
914
977
  return self.__root_node.error
915
978
 
916
- def send(self, message: str, *args, **kwargs):
979
+ def spawn_agent(self, agent: Actor) -> ActorId:
980
+
981
+ if not self.__root_started.is_set():
982
+ raise EBadActor("System has not started yet")
983
+
984
+ return self.__root_node.spawn(agent)
985
+
986
+ def send_main(self, message: str, *args, **kwargs):
917
987
 
918
988
  if self.__root_actor.main_id is None:
919
989
  raise EBadActor("System has not started yet")
920
990
 
921
- self.__root_node.send_message("/external", self.__root_actor.main_id, message, args, kwargs)
991
+ self.__root_node.send_message("/external", self.__root_actor.main_id, message, args, kwargs) # TODO
992
+
993
+ def send(self, actor_id: ActorId, message: str, *args, **kwargs):
994
+
995
+ if not self.__root_started.is_set():
996
+ raise EBadActor("System has not started yet")
997
+
998
+ self.__root_node.send_message("/external", actor_id, message, args, kwargs)
922
999
 
923
1000
  def _setup_event_loops(self, thread_pools: tp.Dict[str, int]):
924
1001
 
@@ -0,0 +1,426 @@
1
+ # Copyright 2022 Accenture Global Solutions Limited
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import copy
16
+ import logging
17
+ import pathlib
18
+ import typing as tp
19
+ import re
20
+ import traceback
21
+
22
+ import pandas as pd
23
+
24
+ import tracdap.rt.api as _api
25
+ import tracdap.rt.metadata as _meta
26
+ import tracdap.rt.exceptions as _ex
27
+ import tracdap.rt._impl.type_system as _types # noqa
28
+ import tracdap.rt._impl.data as _data # noqa
29
+ import tracdap.rt._impl.util as _util # noqa
30
+ import tracdap.rt._impl.validation as _val # noqa
31
+
32
+
33
+ class TracContextImpl(_api.TracContext):
34
+
35
+ """
36
+ TracContextImpl is the main implementation of the API class TracContext (from tracdap.rt.api).
37
+ It provides get/put operations on the inputs, outputs and parameters of a model according to the model definition,
38
+ as well as exposing other information needed by the model at runtime and offering a few utility functions.
39
+
40
+ An instance of TracContextImpl is constructed by the runtime engine for each model node in the execution graph.
41
+ Parameters and schemas will be pre-populated from the job definition. Normally, input data will be created by parent
42
+ nodes in the execution graph and passed in when the context is constructed, so they are available when required. In
43
+ the simplest case, a get call picks a dataset from the map and a put call puts a dataset into the map. Under this
44
+ mechanism, no outputs are passed downstream until the model finishes executing.
45
+
46
+ Optimizations for lazy loading and eager saving require the context to call back into the runtime engine. For lazy
47
+ load, the graph node to prepare an input is injected when the data is requested and the model thread blocks until
48
+ it is available; for eager save outputs are sent to child actors as soon as they are produced. In both cases this
49
+ complexity is hidden from the model, which only sees one thread with synchronous get/put calls.
50
+
51
+ :param model_def: Definition object for the model that will run in this context
52
+ :param model_class: Type for the model that will run in this context
53
+ :param local_ctx: Dictionary of all parameters, inputs and outputs that will be available to the model
54
+ Parameters are supplied as python native types.
55
+ Output views will contain schemas but no data.
56
+ """
57
+
58
+ __DEFAULT_TEMPORAL_OBJECTS = False
59
+
60
+ def __init__(self,
61
+ model_def: _meta.ModelDefinition,
62
+ model_class: _api.TracModel.__class__,
63
+ local_ctx: tp.Dict[str, tp.Any],
64
+ checkout_directory: pathlib.Path = None):
65
+
66
+ self.__ctx_log = _util.logger_for_object(self)
67
+ self.__model_log = _util.logger_for_class(model_class)
68
+
69
+ self.__model_def = model_def
70
+ self.__model_class = model_class
71
+ self.__local_ctx = local_ctx or {}
72
+
73
+ self.__val = TracContextValidator(
74
+ self.__ctx_log,
75
+ self.__model_def,
76
+ self.__local_ctx,
77
+ checkout_directory)
78
+
79
+ def get_parameter(self, parameter_name: str) -> tp.Any:
80
+
81
+ _val.validate_signature(self.get_parameter, parameter_name)
82
+
83
+ self.__val.check_param_valid_identifier(parameter_name)
84
+ self.__val.check_param_defined_in_model(parameter_name)
85
+ self.__val.check_param_available_in_context(parameter_name)
86
+
87
+ value: _meta.Value = self.__local_ctx.get(parameter_name)
88
+
89
+ self.__val.check_context_object_type(parameter_name, value, _meta.Value)
90
+
91
+ return _types.MetadataCodec.decode_value(value)
92
+
93
+ def has_dataset(self, dataset_name: str) -> bool:
94
+
95
+ _val.validate_signature(self.has_dataset, dataset_name)
96
+
97
+ self.__val.check_dataset_valid_identifier(dataset_name)
98
+ self.__val.check_dataset_defined_in_model(dataset_name)
99
+
100
+ data_view: _data.DataView = self.__local_ctx.get(dataset_name)
101
+
102
+ if data_view is None:
103
+ return False
104
+
105
+ self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
106
+
107
+ return not data_view.is_empty()
108
+
109
+ def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:
110
+
111
+ _val.validate_signature(self.get_schema, dataset_name)
112
+
113
+ self.__val.check_dataset_valid_identifier(dataset_name)
114
+ self.__val.check_dataset_defined_in_model(dataset_name)
115
+ self.__val.check_dataset_available_in_context(dataset_name)
116
+
117
+ static_schema = self.__get_static_schema(self.__model_def, dataset_name)
118
+ data_view: _data.DataView = self.__local_ctx.get(dataset_name)
119
+
120
+ # Check the data view has a well-defined schema even if a static schema exists in the model
121
+ # This ensures errors are always reported and is consistent with get_pandas_table()
122
+
123
+ self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
124
+ self.__val.check_dataset_schema_defined(dataset_name, data_view)
125
+
126
+ # If a static schema exists, that takes priority
127
+ # Return deep copies, do not allow model code to change schemas provided by the engine
128
+
129
+ if static_schema is not None:
130
+ return copy.deepcopy(static_schema)
131
+ else:
132
+ return copy.deepcopy(data_view.trac_schema)
133
+
134
+ def get_pandas_table(self, dataset_name: str, use_temporal_objects: tp.Optional[bool] = None) -> pd.DataFrame:
135
+
136
+ _val.validate_signature(self.get_pandas_table, dataset_name, use_temporal_objects)
137
+
138
+ self.__val.check_dataset_valid_identifier(dataset_name)
139
+ self.__val.check_dataset_defined_in_model(dataset_name)
140
+ self.__val.check_dataset_available_in_context(dataset_name)
141
+
142
+ static_schema = self.__get_static_schema(self.__model_def, dataset_name)
143
+ data_view = self.__local_ctx.get(dataset_name)
144
+ part_key = _data.DataPartKey.for_root()
145
+
146
+ self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
147
+ self.__val.check_dataset_schema_defined(dataset_name, data_view)
148
+ self.__val.check_dataset_part_present(dataset_name, data_view, part_key)
149
+
150
+ # If the model defines a static input schema, use that for schema conformance
151
+ # Otherwise, take what is in the incoming dataset (schema is dynamic)
152
+
153
+ if static_schema is not None:
154
+ schema = _data.DataMapping.trac_to_arrow_schema(static_schema)
155
+ else:
156
+ schema = data_view.arrow_schema
157
+
158
+ if use_temporal_objects is None:
159
+ use_temporal_objects = self.__DEFAULT_TEMPORAL_OBJECTS
160
+
161
+ return _data.DataMapping.view_to_pandas(data_view, part_key, schema, use_temporal_objects)
162
+
163
+ def put_schema(self, dataset_name: str, schema: _meta.SchemaDefinition):
164
+
165
+ _val.validate_signature(self.get_schema, dataset_name, schema)
166
+
167
+ # Copy the schema - schema cannot be changed in model code after put_schema
168
+ # If field ordering is not assigned by the model, assign it here (model code will not see the numbers)
169
+ schema_copy = self.__assign_field_order(copy.deepcopy(schema))
170
+
171
+ self.__val.check_dataset_valid_identifier(dataset_name)
172
+ self.__val.check_dataset_is_dynamic_output(dataset_name)
173
+ self.__val.check_provided_schema_is_valid(dataset_name, schema_copy)
174
+
175
+ static_schema = self.__get_static_schema(self.__model_def, dataset_name)
176
+ data_view = self.__local_ctx.get(dataset_name)
177
+
178
+ if data_view is None:
179
+ if static_schema is not None:
180
+ data_view = _data.DataView.for_trac_schema(static_schema)
181
+ else:
182
+ data_view = _data.DataView.create_empty()
183
+
184
+ # If there is a prior view it must contain nothing and will be replaced
185
+ self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
186
+ self.__val.check_dataset_schema_not_defined(dataset_name, data_view)
187
+ self.__val.check_dataset_is_empty(dataset_name, data_view)
188
+
189
+ updated_view = data_view.with_trac_schema(schema_copy)
190
+
191
+ self.__local_ctx[dataset_name] = updated_view
192
+
193
+ def put_pandas_table(self, dataset_name: str, dataset: pd.DataFrame):
194
+
195
+ _val.validate_signature(self.put_pandas_table, dataset_name, dataset)
196
+
197
+ self.__val.check_dataset_valid_identifier(dataset_name)
198
+ self.__val.check_dataset_is_model_output(dataset_name)
199
+ self.__val.check_provided_dataset_type(dataset, pd.DataFrame)
200
+
201
+ static_schema = self.__get_static_schema(self.__model_def, dataset_name)
202
+ data_view = self.__local_ctx.get(dataset_name)
203
+ part_key = _data.DataPartKey.for_root()
204
+
205
+ if data_view is None:
206
+ if static_schema is not None:
207
+ data_view = _data.DataView.for_trac_schema(static_schema)
208
+ else:
209
+ data_view = _data.DataView.create_empty()
210
+
211
+ self.__val.check_context_object_type(dataset_name, data_view, _data.DataView)
212
+ self.__val.check_dataset_schema_defined(dataset_name, data_view)
213
+ self.__val.check_dataset_part_not_present(dataset_name, data_view, part_key)
214
+
215
+ # Prefer static schemas for data conformance
216
+
217
+ if static_schema is not None:
218
+ schema = _data.DataMapping.trac_to_arrow_schema(static_schema)
219
+ else:
220
+ schema = data_view.arrow_schema
221
+
222
+ # Data conformance is applied inside these conversion functions
223
+
224
+ updated_item = _data.DataMapping.pandas_to_item(dataset, schema)
225
+ updated_view = _data.DataMapping.add_item_to_view(data_view, part_key, updated_item)
226
+
227
+ self.__local_ctx[dataset_name] = updated_view
228
+
229
+ def log(self) -> logging.Logger:
230
+
231
+ _val.validate_signature(self.log)
232
+
233
+ return self.__model_log
234
+
235
+ @staticmethod
236
+ def __get_static_schema(model_def: _meta.ModelDefinition, dataset_name: str):
237
+
238
+ input_schema = model_def.inputs.get(dataset_name)
239
+
240
+ if input_schema is not None and not input_schema.dynamic:
241
+ return input_schema.schema
242
+
243
+ output_schema = model_def.outputs.get(dataset_name)
244
+
245
+ if output_schema is not None and not output_schema.dynamic:
246
+ return output_schema.schema
247
+
248
+ return None
249
+
250
+ @staticmethod
251
+ def __assign_field_order(schema_def: _meta.SchemaDefinition):
252
+
253
+ if schema_def is None or schema_def.table is None or schema_def.table.fields is None:
254
+ return schema_def
255
+
256
+ if all(map(lambda f: f.fieldOrder is None, schema_def.table.fields)):
257
+ for index, field in enumerate(schema_def.table.fields):
258
+ field.fieldOrder = index
259
+
260
+ return schema_def
261
+
262
+
263
+ class TracContextValidator:
264
+
265
+ __VALID_IDENTIFIER = re.compile("^[a-zA-Z_]\\w*$",)
266
+ __RESERVED_IDENTIFIER = re.compile("^(trac_|_)\\w*")
267
+
268
+ def __init__(
269
+ self, log: logging.Logger,
270
+ model_def: _meta.ModelDefinition,
271
+ local_ctx: tp.Dict[str, tp.Any],
272
+ checkout_directory: pathlib.Path):
273
+
274
+ self.__log = log
275
+ self.__model_def = model_def
276
+ self.__local_ctx = local_ctx
277
+ self.__checkout_directory = checkout_directory
278
+
279
+ def _report_error(self, message, cause: Exception = None):
280
+
281
+ full_stack = traceback.extract_stack()
282
+ model_stack = _util.filter_model_stack_trace(full_stack, self.__checkout_directory)
283
+ model_stack_str = ''.join(traceback.format_list(list(reversed(model_stack))))
284
+ details = _util.error_details_from_trace(model_stack)
285
+ message = f"{message} {details}"
286
+
287
+ self.__log.error(message)
288
+ self.__log.error(f"Model stack trace:\n{model_stack_str}")
289
+
290
+ if cause:
291
+ raise _ex.ERuntimeValidation(message) from cause
292
+ else:
293
+ raise _ex.ERuntimeValidation(message)
294
+
295
+ def check_param_valid_identifier(self, param_name: str):
296
+
297
+ if param_name is None:
298
+ self._report_error(f"Parameter name is null")
299
+
300
+ if not self.__VALID_IDENTIFIER.match(param_name):
301
+ self._report_error(f"Parameter name {param_name} is not a valid identifier")
302
+
303
+ def check_param_defined_in_model(self, param_name: str):
304
+
305
+ if param_name not in self.__model_def.parameters:
306
+ self._report_error(f"Parameter {param_name} is not defined in the model")
307
+
308
+ def check_param_available_in_context(self, param_name: str):
309
+
310
+ if param_name not in self.__local_ctx:
311
+ self._report_error(f"Parameter {param_name} is not available in the current context")
312
+
313
+ def check_dataset_valid_identifier(self, dataset_name: str):
314
+
315
+ if dataset_name is None:
316
+ self._report_error(f"Dataset name is null")
317
+
318
+ if not self.__VALID_IDENTIFIER.match(dataset_name):
319
+ self._report_error(f"Dataset name {dataset_name} is not a valid identifier")
320
+
321
+ def check_dataset_defined_in_model(self, dataset_name: str):
322
+
323
+ if dataset_name not in self.__model_def.inputs and dataset_name not in self.__model_def.outputs:
324
+ self._report_error(f"Dataset {dataset_name} is not defined in the model")
325
+
326
+ def check_dataset_is_model_output(self, dataset_name: str):
327
+
328
+ if dataset_name not in self.__model_def.outputs:
329
+ self._report_error(f"Dataset {dataset_name} is not defined as a model output")
330
+
331
+ def check_dataset_is_dynamic_output(self, dataset_name: str):
332
+
333
+ model_output: _meta.ModelOutputSchema = self.__model_def.outputs.get(dataset_name)
334
+
335
+ if model_output is None:
336
+ self._report_error(f"Dataset {dataset_name} is not defined as a model output")
337
+
338
+ if not model_output.dynamic:
339
+ self._report_error(f"Model output {dataset_name} is not a dynamic output")
340
+
341
+ def check_dataset_available_in_context(self, item_name: str):
342
+
343
+ if item_name not in self.__local_ctx:
344
+ self._report_error(f"Dataset {item_name} is not available in the current context")
345
+
346
+ def check_dataset_schema_defined(self, dataset_name: str, data_view: _data.DataView):
347
+
348
+ schema = data_view.trac_schema if data_view is not None else None
349
+
350
+ if schema is None or schema.table is None or not schema.table.fields:
351
+ self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
352
+
353
+ def check_dataset_schema_not_defined(self, dataset_name: str, data_view: _data.DataView):
354
+
355
+ schema = data_view.trac_schema if data_view is not None else None
356
+
357
+ if schema is not None and (schema.table or schema.schemaType != _meta.SchemaType.SCHEMA_TYPE_NOT_SET):
358
+ self._report_error(f"Schema already defined for dataset {dataset_name} in the current context")
359
+
360
+ def check_dataset_part_present(self, dataset_name: str, data_view: _data.DataView, part_key: _data.DataPartKey):
361
+
362
+ part = data_view.parts.get(part_key) if data_view.parts is not None else None
363
+
364
+ if part is None or len(part) == 0:
365
+ self._report_error(f"No data present for {dataset_name} ({part_key}) in the current context")
366
+
367
+ def check_dataset_part_not_present(self, dataset_name: str, data_view: _data.DataView, part_key: _data.DataPartKey):
368
+
369
+ part = data_view.parts.get(part_key) if data_view.parts is not None else None
370
+
371
+ if part is not None and len(part) > 0:
372
+ self._report_error(f"Data already present for {dataset_name} ({part_key}) in the current context")
373
+
374
+ def check_dataset_is_empty(self, dataset_name: str, data_view: _data.DataView):
375
+
376
+ if not data_view.is_empty():
377
+ self._report_error(f"Dataset {dataset_name} is not empty")
378
+
379
+ def check_provided_schema_is_valid(self, dataset_name: str, schema: _meta.SchemaDefinition):
380
+
381
+ if schema is None:
382
+ self._report_error(f"The schema provided for [{dataset_name}] is null")
383
+
384
+ if not isinstance(schema, _meta.SchemaDefinition):
385
+ schema_type_name = self._type_name(type(schema))
386
+ self._report_error(f"The object provided for [{dataset_name}] is not a schema (got {schema_type_name})")
387
+
388
+ try:
389
+ _val.StaticValidator.quick_validate_schema(schema)
390
+ except _ex.EModelValidation as e:
391
+ self._report_error(f"The schema provided for [{dataset_name}] failed validation: {str(e)}", e)
392
+
393
+ def check_provided_dataset_type(self, dataset: tp.Any, expected_type: type):
394
+
395
+ if dataset is None:
396
+ self._report_error(f"Provided dataset is null")
397
+
398
+ if not isinstance(dataset, expected_type):
399
+
400
+ expected_type_name = self._type_name(expected_type)
401
+ actual_type_name = self._type_name(type(dataset))
402
+
403
+ self._report_error(
404
+ f"Provided dataset is the wrong type" +
405
+ f" (expected {expected_type_name}, got {actual_type_name})")
406
+
407
+ def check_context_object_type(self, item_name: str, item: tp.Any, expected_type: type):
408
+
409
+ if not isinstance(item, expected_type):
410
+
411
+ expected_type_name = self._type_name(expected_type)
412
+ actual_type_name = self._type_name(type(item))
413
+
414
+ self._report_error(
415
+ f"The object referenced by [{item_name}] in the current context has the wrong type" +
416
+ f" (expected {expected_type_name}, got {actual_type_name})")
417
+
418
+ @staticmethod
419
+ def _type_name(type_: type):
420
+
421
+ module = type_.__module__
422
+
423
+ if module is None or module == str.__class__.__module__:
424
+ return type_.__qualname__
425
+
426
+ return module + '.' + type_.__name__