tracdap-runtime 0.6.5__py3-none-any.whl → 0.7.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracdap/rt/__init__.py +6 -5
- tracdap/rt/_exec/actors.py +6 -5
- tracdap/rt/_exec/context.py +278 -110
- tracdap/rt/_exec/dev_mode.py +237 -143
- tracdap/rt/_exec/engine.py +223 -64
- tracdap/rt/_exec/functions.py +31 -6
- tracdap/rt/_exec/graph.py +15 -5
- tracdap/rt/_exec/graph_builder.py +301 -203
- tracdap/rt/_exec/runtime.py +13 -10
- tracdap/rt/_exec/server.py +6 -5
- tracdap/rt/_impl/__init__.py +6 -5
- tracdap/rt/_impl/config_parser.py +17 -9
- tracdap/rt/_impl/data.py +284 -172
- tracdap/rt/_impl/ext/__init__.py +14 -0
- tracdap/rt/_impl/ext/sql.py +117 -0
- tracdap/rt/_impl/ext/storage.py +58 -0
- tracdap/rt/_impl/grpc/__init__.py +6 -5
- tracdap/rt/_impl/grpc/codec.py +6 -5
- tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +62 -54
- tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi +37 -2
- tracdap/rt/_impl/guard_rails.py +6 -5
- tracdap/rt/_impl/models.py +6 -5
- tracdap/rt/_impl/repos.py +6 -5
- tracdap/rt/_impl/schemas.py +6 -5
- tracdap/rt/_impl/shim.py +6 -5
- tracdap/rt/_impl/static_api.py +30 -16
- tracdap/rt/_impl/storage.py +8 -7
- tracdap/rt/_impl/type_system.py +6 -5
- tracdap/rt/_impl/util.py +16 -5
- tracdap/rt/_impl/validation.py +72 -18
- tracdap/rt/_plugins/__init__.py +6 -5
- tracdap/rt/_plugins/_helpers.py +6 -5
- tracdap/rt/_plugins/config_local.py +6 -5
- tracdap/rt/_plugins/format_arrow.py +6 -5
- tracdap/rt/_plugins/format_csv.py +6 -5
- tracdap/rt/_plugins/format_parquet.py +6 -5
- tracdap/rt/_plugins/repo_git.py +6 -5
- tracdap/rt/_plugins/repo_local.py +6 -5
- tracdap/rt/_plugins/repo_pypi.py +6 -5
- tracdap/rt/_plugins/storage_aws.py +6 -5
- tracdap/rt/_plugins/storage_azure.py +6 -5
- tracdap/rt/_plugins/storage_gcp.py +6 -5
- tracdap/rt/_plugins/storage_local.py +6 -5
- tracdap/rt/_plugins/storage_sql.py +418 -0
- tracdap/rt/_plugins/storage_sql_dialects.py +118 -0
- tracdap/rt/_version.py +7 -6
- tracdap/rt/api/__init__.py +23 -5
- tracdap/rt/api/experimental.py +85 -37
- tracdap/rt/api/hook.py +16 -5
- tracdap/rt/api/model_api.py +110 -90
- tracdap/rt/api/static_api.py +142 -100
- tracdap/rt/config/common.py +26 -27
- tracdap/rt/config/job.py +5 -6
- tracdap/rt/config/platform.py +41 -42
- tracdap/rt/config/result.py +5 -6
- tracdap/rt/config/runtime.py +6 -7
- tracdap/rt/exceptions.py +13 -7
- tracdap/rt/ext/__init__.py +6 -5
- tracdap/rt/ext/config.py +6 -5
- tracdap/rt/ext/embed.py +6 -5
- tracdap/rt/ext/plugins.py +6 -5
- tracdap/rt/ext/repos.py +6 -5
- tracdap/rt/ext/storage.py +6 -5
- tracdap/rt/launch/__init__.py +10 -5
- tracdap/rt/launch/__main__.py +6 -5
- tracdap/rt/launch/cli.py +6 -5
- tracdap/rt/launch/launch.py +38 -15
- tracdap/rt/metadata/__init__.py +4 -0
- tracdap/rt/metadata/common.py +2 -3
- tracdap/rt/metadata/custom.py +3 -4
- tracdap/rt/metadata/data.py +30 -31
- tracdap/rt/metadata/file.py +6 -7
- tracdap/rt/metadata/flow.py +22 -23
- tracdap/rt/metadata/job.py +89 -45
- tracdap/rt/metadata/model.py +26 -27
- tracdap/rt/metadata/object.py +11 -12
- tracdap/rt/metadata/object_id.py +23 -24
- tracdap/rt/metadata/resource.py +0 -1
- tracdap/rt/metadata/search.py +15 -16
- tracdap/rt/metadata/stoarge.py +22 -23
- tracdap/rt/metadata/tag.py +8 -9
- tracdap/rt/metadata/tag_update.py +11 -12
- tracdap/rt/metadata/type.py +38 -38
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/LICENSE +1 -1
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/METADATA +4 -2
- tracdap_runtime-0.7.0rc1.dist-info/RECORD +121 -0
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/WHEEL +1 -1
- tracdap_runtime-0.6.5.dist-info/RECORD +0 -116
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/top_level.txt +0 -0
tracdap/rt/_exec/runtime.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -154,7 +155,6 @@ class TracRuntime:
|
|
154
155
|
_plugins.PluginManager.register_plugin_package(plugin_package)
|
155
156
|
|
156
157
|
_static_api.StaticApiImpl.register_impl()
|
157
|
-
_guard.PythonGuardRails.protect_dangerous_functions()
|
158
158
|
|
159
159
|
# Load sys config (or use embedded), config errors are detected before start()
|
160
160
|
# Job config can also be checked before start() by using load_job_config()
|
@@ -201,6 +201,11 @@ class TracRuntime:
|
|
201
201
|
self._models = _models.ModelLoader(self._sys_config, self._scratch_dir)
|
202
202
|
self._storage = _storage.StorageManager(self._sys_config)
|
203
203
|
|
204
|
+
# Enable protection after the initial setup of the runtime is complete
|
205
|
+
# Storage plugins in particular are likely to tigger protected imports
|
206
|
+
# Once the runtime is up, no more plugins should be loaded
|
207
|
+
_guard.PythonGuardRails.protect_dangerous_functions()
|
208
|
+
|
204
209
|
self._engine = _engine.TracEngine(
|
205
210
|
self._sys_config, self._models, self._storage,
|
206
211
|
notify_callback=self._engine_callback)
|
@@ -329,10 +334,8 @@ class TracRuntime:
|
|
329
334
|
config_file_name="job")
|
330
335
|
|
331
336
|
if self._dev_mode:
|
332
|
-
|
333
|
-
|
334
|
-
self._scratch_dir, self._config_mgr,
|
335
|
-
model_class)
|
337
|
+
translator = _dev_mode.DevModeTranslator(self._sys_config, self._config_mgr, self._scratch_dir)
|
338
|
+
job_config = translator.translate_job_config(job_config, model_class)
|
336
339
|
|
337
340
|
return job_config
|
338
341
|
|
tracdap/rt/_exec/server.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
tracdap/rt/_impl/__init__.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -341,10 +342,17 @@ class ConfigParser(tp.Generic[_T]):
|
|
341
342
|
|
342
343
|
if isinstance(raw_value, tp.Dict):
|
343
344
|
return self._parse_simple_class(location, raw_value, annotation)
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
345
|
+
|
346
|
+
if self._is_dev_mode_location(location):
|
347
|
+
if type(raw_value) in ConfigParser.__primitive_types:
|
348
|
+
return self._parse_primitive(location, raw_value, type(raw_value))
|
349
|
+
if isinstance(raw_value, list):
|
350
|
+
if len(raw_value) == 0:
|
351
|
+
return []
|
352
|
+
list_type = type(raw_value[0])
|
353
|
+
return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
|
354
|
+
|
355
|
+
return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
|
348
356
|
|
349
357
|
if isinstance(annotation, self.__generic_metaclass):
|
350
358
|
return self._parse_generic_class(location, raw_value, annotation) # noqa
|