tracdap-runtime 0.6.5__py3-none-any.whl → 0.7.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. tracdap/rt/__init__.py +6 -5
  2. tracdap/rt/_exec/actors.py +6 -5
  3. tracdap/rt/_exec/context.py +278 -110
  4. tracdap/rt/_exec/dev_mode.py +237 -143
  5. tracdap/rt/_exec/engine.py +223 -64
  6. tracdap/rt/_exec/functions.py +31 -6
  7. tracdap/rt/_exec/graph.py +15 -5
  8. tracdap/rt/_exec/graph_builder.py +301 -203
  9. tracdap/rt/_exec/runtime.py +13 -10
  10. tracdap/rt/_exec/server.py +6 -5
  11. tracdap/rt/_impl/__init__.py +6 -5
  12. tracdap/rt/_impl/config_parser.py +17 -9
  13. tracdap/rt/_impl/data.py +284 -172
  14. tracdap/rt/_impl/ext/__init__.py +14 -0
  15. tracdap/rt/_impl/ext/sql.py +117 -0
  16. tracdap/rt/_impl/ext/storage.py +58 -0
  17. tracdap/rt/_impl/grpc/__init__.py +6 -5
  18. tracdap/rt/_impl/grpc/codec.py +6 -5
  19. tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +62 -54
  20. tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi +37 -2
  21. tracdap/rt/_impl/guard_rails.py +6 -5
  22. tracdap/rt/_impl/models.py +6 -5
  23. tracdap/rt/_impl/repos.py +6 -5
  24. tracdap/rt/_impl/schemas.py +6 -5
  25. tracdap/rt/_impl/shim.py +6 -5
  26. tracdap/rt/_impl/static_api.py +30 -16
  27. tracdap/rt/_impl/storage.py +8 -7
  28. tracdap/rt/_impl/type_system.py +6 -5
  29. tracdap/rt/_impl/util.py +16 -5
  30. tracdap/rt/_impl/validation.py +72 -18
  31. tracdap/rt/_plugins/__init__.py +6 -5
  32. tracdap/rt/_plugins/_helpers.py +6 -5
  33. tracdap/rt/_plugins/config_local.py +6 -5
  34. tracdap/rt/_plugins/format_arrow.py +6 -5
  35. tracdap/rt/_plugins/format_csv.py +6 -5
  36. tracdap/rt/_plugins/format_parquet.py +6 -5
  37. tracdap/rt/_plugins/repo_git.py +6 -5
  38. tracdap/rt/_plugins/repo_local.py +6 -5
  39. tracdap/rt/_plugins/repo_pypi.py +6 -5
  40. tracdap/rt/_plugins/storage_aws.py +6 -5
  41. tracdap/rt/_plugins/storage_azure.py +6 -5
  42. tracdap/rt/_plugins/storage_gcp.py +6 -5
  43. tracdap/rt/_plugins/storage_local.py +6 -5
  44. tracdap/rt/_plugins/storage_sql.py +418 -0
  45. tracdap/rt/_plugins/storage_sql_dialects.py +118 -0
  46. tracdap/rt/_version.py +7 -6
  47. tracdap/rt/api/__init__.py +23 -5
  48. tracdap/rt/api/experimental.py +85 -37
  49. tracdap/rt/api/hook.py +16 -5
  50. tracdap/rt/api/model_api.py +110 -90
  51. tracdap/rt/api/static_api.py +142 -100
  52. tracdap/rt/config/common.py +26 -27
  53. tracdap/rt/config/job.py +5 -6
  54. tracdap/rt/config/platform.py +41 -42
  55. tracdap/rt/config/result.py +5 -6
  56. tracdap/rt/config/runtime.py +6 -7
  57. tracdap/rt/exceptions.py +13 -7
  58. tracdap/rt/ext/__init__.py +6 -5
  59. tracdap/rt/ext/config.py +6 -5
  60. tracdap/rt/ext/embed.py +6 -5
  61. tracdap/rt/ext/plugins.py +6 -5
  62. tracdap/rt/ext/repos.py +6 -5
  63. tracdap/rt/ext/storage.py +6 -5
  64. tracdap/rt/launch/__init__.py +10 -5
  65. tracdap/rt/launch/__main__.py +6 -5
  66. tracdap/rt/launch/cli.py +6 -5
  67. tracdap/rt/launch/launch.py +38 -15
  68. tracdap/rt/metadata/__init__.py +4 -0
  69. tracdap/rt/metadata/common.py +2 -3
  70. tracdap/rt/metadata/custom.py +3 -4
  71. tracdap/rt/metadata/data.py +30 -31
  72. tracdap/rt/metadata/file.py +6 -7
  73. tracdap/rt/metadata/flow.py +22 -23
  74. tracdap/rt/metadata/job.py +89 -45
  75. tracdap/rt/metadata/model.py +26 -27
  76. tracdap/rt/metadata/object.py +11 -12
  77. tracdap/rt/metadata/object_id.py +23 -24
  78. tracdap/rt/metadata/resource.py +0 -1
  79. tracdap/rt/metadata/search.py +15 -16
  80. tracdap/rt/metadata/stoarge.py +22 -23
  81. tracdap/rt/metadata/tag.py +8 -9
  82. tracdap/rt/metadata/tag_update.py +11 -12
  83. tracdap/rt/metadata/type.py +38 -38
  84. {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/LICENSE +1 -1
  85. {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/METADATA +4 -2
  86. tracdap_runtime-0.7.0rc1.dist-info/RECORD +121 -0
  87. {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/WHEEL +1 -1
  88. tracdap_runtime-0.6.5.dist-info/RECORD +0 -116
  89. {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,9 @@
1
- # Copyright 2022 Accenture Global Solutions Limited
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
1
+ # Licensed to the Fintech Open Source Foundation (FINOS) under one or
2
+ # more contributor license agreements. See the NOTICE file distributed
3
+ # with this work for additional information regarding copyright ownership.
4
+ # FINOS licenses this file to you under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
6
7
  #
7
8
  # http://www.apache.org/licenses/LICENSE-2.0
8
9
  #
@@ -154,7 +155,6 @@ class TracRuntime:
154
155
  _plugins.PluginManager.register_plugin_package(plugin_package)
155
156
 
156
157
  _static_api.StaticApiImpl.register_impl()
157
- _guard.PythonGuardRails.protect_dangerous_functions()
158
158
 
159
159
  # Load sys config (or use embedded), config errors are detected before start()
160
160
  # Job config can also be checked before start() by using load_job_config()
@@ -201,6 +201,11 @@ class TracRuntime:
201
201
  self._models = _models.ModelLoader(self._sys_config, self._scratch_dir)
202
202
  self._storage = _storage.StorageManager(self._sys_config)
203
203
 
204
+ # Enable protection after the initial setup of the runtime is complete
205
+ # Storage plugins in particular are likely to tigger protected imports
206
+ # Once the runtime is up, no more plugins should be loaded
207
+ _guard.PythonGuardRails.protect_dangerous_functions()
208
+
204
209
  self._engine = _engine.TracEngine(
205
210
  self._sys_config, self._models, self._storage,
206
211
  notify_callback=self._engine_callback)
@@ -329,10 +334,8 @@ class TracRuntime:
329
334
  config_file_name="job")
330
335
 
331
336
  if self._dev_mode:
332
- job_config = _dev_mode.DevModeTranslator.translate_job_config(
333
- self._sys_config, job_config,
334
- self._scratch_dir, self._config_mgr,
335
- model_class)
337
+ translator = _dev_mode.DevModeTranslator(self._sys_config, self._config_mgr, self._scratch_dir)
338
+ job_config = translator.translate_job_config(job_config, model_class)
336
339
 
337
340
  return job_config
338
341
 
@@ -1,8 +1,9 @@
1
- # Copyright 2024 Accenture Global Solutions Limited
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
1
+ # Licensed to the Fintech Open Source Foundation (FINOS) under one or
2
+ # more contributor license agreements. See the NOTICE file distributed
3
+ # with this work for additional information regarding copyright ownership.
4
+ # FINOS licenses this file to you under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
6
7
  #
7
8
  # http://www.apache.org/licenses/LICENSE-2.0
8
9
  #
@@ -1,8 +1,9 @@
1
- # Copyright 2021 Accenture Global Solutions Limited
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
1
+ # Licensed to the Fintech Open Source Foundation (FINOS) under one or
2
+ # more contributor license agreements. See the NOTICE file distributed
3
+ # with this work for additional information regarding copyright ownership.
4
+ # FINOS licenses this file to you under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
6
7
  #
7
8
  # http://www.apache.org/licenses/LICENSE-2.0
8
9
  #
@@ -1,8 +1,9 @@
1
- # Copyright 2022 Accenture Global Solutions Limited
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
1
+ # Licensed to the Fintech Open Source Foundation (FINOS) under one or
2
+ # more contributor license agreements. See the NOTICE file distributed
3
+ # with this work for additional information regarding copyright ownership.
4
+ # FINOS licenses this file to you under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
6
7
  #
7
8
  # http://www.apache.org/licenses/LICENSE-2.0
8
9
  #
@@ -341,10 +342,17 @@ class ConfigParser(tp.Generic[_T]):
341
342
 
342
343
  if isinstance(raw_value, tp.Dict):
343
344
  return self._parse_simple_class(location, raw_value, annotation)
344
- elif self._is_dev_mode_location(location) and type(raw_value) in ConfigParser.__primitive_types:
345
- return self._parse_primitive(location, raw_value, type(raw_value))
346
- else:
347
- return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
345
+
346
+ if self._is_dev_mode_location(location):
347
+ if type(raw_value) in ConfigParser.__primitive_types:
348
+ return self._parse_primitive(location, raw_value, type(raw_value))
349
+ if isinstance(raw_value, list):
350
+ if len(raw_value) == 0:
351
+ return []
352
+ list_type = type(raw_value[0])
353
+ return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
354
+
355
+ return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
348
356
 
349
357
  if isinstance(annotation, self.__generic_metaclass):
350
358
  return self._parse_generic_class(location, raw_value, annotation) # noqa