tracdap-runtime 0.6.5__py3-none-any.whl → 0.7.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracdap/rt/__init__.py +6 -5
- tracdap/rt/_exec/actors.py +6 -5
- tracdap/rt/_exec/context.py +278 -110
- tracdap/rt/_exec/dev_mode.py +237 -143
- tracdap/rt/_exec/engine.py +223 -64
- tracdap/rt/_exec/functions.py +31 -6
- tracdap/rt/_exec/graph.py +15 -5
- tracdap/rt/_exec/graph_builder.py +301 -203
- tracdap/rt/_exec/runtime.py +13 -10
- tracdap/rt/_exec/server.py +6 -5
- tracdap/rt/_impl/__init__.py +6 -5
- tracdap/rt/_impl/config_parser.py +17 -9
- tracdap/rt/_impl/data.py +284 -172
- tracdap/rt/_impl/ext/__init__.py +14 -0
- tracdap/rt/_impl/ext/sql.py +117 -0
- tracdap/rt/_impl/ext/storage.py +58 -0
- tracdap/rt/_impl/grpc/__init__.py +6 -5
- tracdap/rt/_impl/grpc/codec.py +6 -5
- tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +62 -54
- tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi +37 -2
- tracdap/rt/_impl/guard_rails.py +6 -5
- tracdap/rt/_impl/models.py +6 -5
- tracdap/rt/_impl/repos.py +6 -5
- tracdap/rt/_impl/schemas.py +6 -5
- tracdap/rt/_impl/shim.py +6 -5
- tracdap/rt/_impl/static_api.py +30 -16
- tracdap/rt/_impl/storage.py +8 -7
- tracdap/rt/_impl/type_system.py +6 -5
- tracdap/rt/_impl/util.py +16 -5
- tracdap/rt/_impl/validation.py +72 -18
- tracdap/rt/_plugins/__init__.py +6 -5
- tracdap/rt/_plugins/_helpers.py +6 -5
- tracdap/rt/_plugins/config_local.py +6 -5
- tracdap/rt/_plugins/format_arrow.py +6 -5
- tracdap/rt/_plugins/format_csv.py +6 -5
- tracdap/rt/_plugins/format_parquet.py +6 -5
- tracdap/rt/_plugins/repo_git.py +6 -5
- tracdap/rt/_plugins/repo_local.py +6 -5
- tracdap/rt/_plugins/repo_pypi.py +6 -5
- tracdap/rt/_plugins/storage_aws.py +6 -5
- tracdap/rt/_plugins/storage_azure.py +6 -5
- tracdap/rt/_plugins/storage_gcp.py +6 -5
- tracdap/rt/_plugins/storage_local.py +6 -5
- tracdap/rt/_plugins/storage_sql.py +418 -0
- tracdap/rt/_plugins/storage_sql_dialects.py +118 -0
- tracdap/rt/_version.py +7 -6
- tracdap/rt/api/__init__.py +23 -5
- tracdap/rt/api/experimental.py +85 -37
- tracdap/rt/api/hook.py +16 -5
- tracdap/rt/api/model_api.py +110 -90
- tracdap/rt/api/static_api.py +142 -100
- tracdap/rt/config/common.py +26 -27
- tracdap/rt/config/job.py +5 -6
- tracdap/rt/config/platform.py +41 -42
- tracdap/rt/config/result.py +5 -6
- tracdap/rt/config/runtime.py +6 -7
- tracdap/rt/exceptions.py +13 -7
- tracdap/rt/ext/__init__.py +6 -5
- tracdap/rt/ext/config.py +6 -5
- tracdap/rt/ext/embed.py +6 -5
- tracdap/rt/ext/plugins.py +6 -5
- tracdap/rt/ext/repos.py +6 -5
- tracdap/rt/ext/storage.py +6 -5
- tracdap/rt/launch/__init__.py +10 -5
- tracdap/rt/launch/__main__.py +6 -5
- tracdap/rt/launch/cli.py +6 -5
- tracdap/rt/launch/launch.py +38 -15
- tracdap/rt/metadata/__init__.py +4 -0
- tracdap/rt/metadata/common.py +2 -3
- tracdap/rt/metadata/custom.py +3 -4
- tracdap/rt/metadata/data.py +30 -31
- tracdap/rt/metadata/file.py +6 -7
- tracdap/rt/metadata/flow.py +22 -23
- tracdap/rt/metadata/job.py +89 -45
- tracdap/rt/metadata/model.py +26 -27
- tracdap/rt/metadata/object.py +11 -12
- tracdap/rt/metadata/object_id.py +23 -24
- tracdap/rt/metadata/resource.py +0 -1
- tracdap/rt/metadata/search.py +15 -16
- tracdap/rt/metadata/stoarge.py +22 -23
- tracdap/rt/metadata/tag.py +8 -9
- tracdap/rt/metadata/tag_update.py +11 -12
- tracdap/rt/metadata/type.py +38 -38
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/LICENSE +1 -1
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/METADATA +4 -2
- tracdap_runtime-0.7.0rc1.dist-info/RECORD +121 -0
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/WHEEL +1 -1
- tracdap_runtime-0.6.5.dist-info/RECORD +0 -116
- {tracdap_runtime-0.6.5.dist-info → tracdap_runtime-0.7.0rc1.dist-info}/top_level.txt +0 -0
tracdap/rt/api/experimental.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -22,70 +23,91 @@ from tracdap.rt.api import *
|
|
22
23
|
from .hook import _StaticApiHook
|
23
24
|
|
24
25
|
|
25
|
-
|
26
|
+
_PROTOCOL = _tp.TypeVar('_PROTOCOL')
|
27
|
+
|
28
|
+
@_dc.dataclass(frozen=True)
|
29
|
+
class _Protocol(_tp.Generic[_PROTOCOL]):
|
26
30
|
|
31
|
+
protocol_name: str
|
32
|
+
api_type: _tp.Type[_PROTOCOL]
|
27
33
|
|
28
|
-
|
34
|
+
def __str__(self):
|
35
|
+
return self.protocol_name
|
29
36
|
|
30
|
-
PANDAS: "_DataFramework"
|
31
|
-
POLARS: "_DataFramework"
|
32
37
|
|
33
|
-
|
34
|
-
|
35
|
-
|
38
|
+
def __pandas_api_type() -> "_tp.Type[pandas.DataFrame]":
|
39
|
+
try:
|
40
|
+
import pandas
|
41
|
+
return pandas.DataFrame
|
42
|
+
except ModuleNotFoundError:
|
43
|
+
return None # noqa
|
36
44
|
|
37
|
-
|
38
|
-
|
45
|
+
def __polars_api_type() -> "_tp.Type[polars.DataFrame]":
|
46
|
+
try:
|
47
|
+
import polars
|
48
|
+
return polars.DataFrame
|
49
|
+
except ModuleNotFoundError:
|
50
|
+
return None # noqa
|
39
51
|
|
52
|
+
DATA_API = _tp.TypeVar('DATA_API', __pandas_api_type(), __polars_api_type())
|
40
53
|
|
41
|
-
|
54
|
+
class DataFramework(_Protocol[DATA_API]):
|
42
55
|
|
43
|
-
|
44
|
-
|
45
|
-
"""The original Python dataframe library, most widely used"""
|
46
|
-
else:
|
47
|
-
_DataFramework.PANDAS = _DataFramework('pandas', None)
|
48
|
-
"""Pandas data framework is not installed"""
|
56
|
+
def __init__(self, protocol_name: str, api_type: _tp.Type[DATA_API]):
|
57
|
+
super().__init__(protocol_name, api_type)
|
49
58
|
|
50
|
-
|
51
|
-
|
52
|
-
""
|
53
|
-
else:
|
54
|
-
_DataFramework.POLARS = _DataFramework('polars', None)
|
55
|
-
"""Polars data framework is not installed"""
|
59
|
+
@classmethod
|
60
|
+
def pandas(cls) -> "DataFramework[pandas.DataFrame]":
|
61
|
+
return DataFramework("pandas", DATA_API.__constraints__[0])
|
56
62
|
|
57
|
-
|
63
|
+
@classmethod
|
64
|
+
def polars(cls) -> "DataFramework[polars.DataFrame]":
|
65
|
+
return DataFramework("polars", DATA_API.__constraints__[1])
|
58
66
|
|
59
|
-
|
60
|
-
|
67
|
+
PANDAS = DataFramework.pandas()
|
68
|
+
"""Data framework constant for the Pandas data library"""
|
61
69
|
|
62
|
-
|
63
|
-
|
70
|
+
POLARS = DataFramework.polars()
|
71
|
+
"""Data framework constant for the Polars data library"""
|
64
72
|
|
65
73
|
|
66
74
|
class TracContext(TracContext):
|
67
75
|
|
68
76
|
@_abc.abstractmethod
|
69
|
-
def get_table(self, dataset_name: str, framework:
|
77
|
+
def get_table(self, dataset_name: str, framework: DataFramework[DATA_API]) -> DATA_API:
|
70
78
|
|
71
79
|
pass
|
72
80
|
|
73
81
|
@_abc.abstractmethod
|
74
|
-
def put_table(self, dataset_name: str, dataset:
|
82
|
+
def put_table(self, dataset_name: str, dataset: DATA_API):
|
75
83
|
|
76
84
|
pass
|
77
85
|
|
78
86
|
|
79
87
|
def init_static():
|
88
|
+
"""Ensure TRAC's static model API is available to use (for static definitions at module or class scope)"""
|
80
89
|
import tracdap.rt._impl.static_api as _static_impl # noqa
|
81
90
|
_static_impl.StaticApiImpl.register_impl()
|
82
91
|
|
83
92
|
|
84
|
-
def infer_schema(dataset:
|
93
|
+
def infer_schema(dataset: DATA_API) -> SchemaDefinition:
|
94
|
+
"""Infer the full TRAC schema of an existing dataset"""
|
85
95
|
sa = _StaticApiHook.get_instance()
|
86
96
|
return sa.infer_schema(dataset)
|
87
97
|
|
88
98
|
|
99
|
+
def array_type(item_type: BasicType) -> TypeDescriptor:
|
100
|
+
"""Build a type descriptor for an ARRAY type"""
|
101
|
+
sa = _StaticApiHook.get_instance()
|
102
|
+
return sa.array_type(item_type)
|
103
|
+
|
104
|
+
|
105
|
+
def map_type(entry_type: BasicType) -> TypeDescriptor:
|
106
|
+
"""Build a type descriptor for a MAP type"""
|
107
|
+
sa = _StaticApiHook.get_instance()
|
108
|
+
return sa.map_type(entry_type)
|
109
|
+
|
110
|
+
|
89
111
|
class FileType(_enum.Enum):
|
90
112
|
|
91
113
|
FILE = 1
|
@@ -172,6 +194,32 @@ class TracFileStorage:
|
|
172
194
|
stream.write(data)
|
173
195
|
|
174
196
|
|
197
|
+
class TracDataStorage(_tp.Generic[DATA_API]):
|
198
|
+
|
199
|
+
@_abc.abstractmethod
|
200
|
+
def has_table(self, table_name: str) -> bool:
|
201
|
+
pass
|
202
|
+
|
203
|
+
@_abc.abstractmethod
|
204
|
+
def list_tables(self) -> _tp.List[str]:
|
205
|
+
pass
|
206
|
+
|
207
|
+
@_abc.abstractmethod
|
208
|
+
def create_table(self, table_name: str, schema: SchemaDefinition):
|
209
|
+
pass
|
210
|
+
|
211
|
+
@_abc.abstractmethod
|
212
|
+
def read_table(self, table_name: str) -> DATA_API:
|
213
|
+
pass
|
214
|
+
|
215
|
+
@_abc.abstractmethod
|
216
|
+
def write_table(self, table_name: str, dataset: DATA_API):
|
217
|
+
pass
|
218
|
+
|
219
|
+
@_abc.abstractmethod
|
220
|
+
def native_read_query(self, query: str, **parameters) -> DATA_API:
|
221
|
+
pass
|
222
|
+
|
175
223
|
|
176
224
|
class TracDataContext(TracContext):
|
177
225
|
|
@@ -180,7 +228,7 @@ class TracDataContext(TracContext):
|
|
180
228
|
pass
|
181
229
|
|
182
230
|
@_abc.abstractmethod
|
183
|
-
def get_data_storage(self, storage_key: str) ->
|
231
|
+
def get_data_storage(self, storage_key: str, framework: DataFramework[DATA_API], **framework_args) -> TracDataStorage[DATA_API]:
|
184
232
|
pass
|
185
233
|
|
186
234
|
@_abc.abstractmethod
|
@@ -188,7 +236,7 @@ class TracDataContext(TracContext):
|
|
188
236
|
pass
|
189
237
|
|
190
238
|
@_abc.abstractmethod
|
191
|
-
def set_source_metadata(self, dataset_key: str, storage_key: str, source_info: FileStat):
|
239
|
+
def set_source_metadata(self, dataset_key: str, storage_key: str, source_info: _tp.Union[FileStat, str]):
|
192
240
|
pass
|
193
241
|
|
194
242
|
@_abc.abstractmethod
|
tracdap/rt/api/hook.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -61,6 +62,16 @@ class _StaticApiHook:
|
|
61
62
|
|
62
63
|
return cls.__static_api_hook
|
63
64
|
|
65
|
+
@_abc.abstractmethod
|
66
|
+
def array_type(self, item_type: _meta.BasicType) -> _meta.TypeDescriptor:
|
67
|
+
|
68
|
+
pass
|
69
|
+
|
70
|
+
@_abc.abstractmethod
|
71
|
+
def map_type(self, entry_type: _meta.BasicType) -> _meta.TypeDescriptor:
|
72
|
+
|
73
|
+
pass
|
74
|
+
|
64
75
|
@_abc.abstractmethod
|
65
76
|
def define_attributes(
|
66
77
|
self, *attrs: _tp.Union[_meta.TagUpdate, _tp.List[_meta.TagUpdate]]) \
|
tracdap/rt/api/model_api.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
1
|
+
# Licensed to the Fintech Open Source Foundation (FINOS) under one or
|
2
|
+
# more contributor license agreements. See the NOTICE file distributed
|
3
|
+
# with this work for additional information regarding copyright ownership.
|
4
|
+
# FINOS licenses this file to you under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
6
7
|
#
|
7
8
|
# http://www.apache.org/licenses/LICENSE-2.0
|
8
9
|
#
|
@@ -34,7 +35,7 @@ if _tp.TYPE_CHECKING:
|
|
34
35
|
pass
|
35
36
|
|
36
37
|
|
37
|
-
class TracContext:
|
38
|
+
class TracContext(metaclass=_abc.ABCMeta):
|
38
39
|
|
39
40
|
"""
|
40
41
|
Interface that allows model components to interact with the platform at runtime
|
@@ -58,33 +59,33 @@ class TracContext:
|
|
58
59
|
All the context API methods are validated at runtime and will raise ERuntimeValidation if a model
|
59
60
|
tries to access an unknown identifier or perform some other invalid operation.
|
60
61
|
|
61
|
-
.. seealso:: :py:class:`TracModel
|
62
|
+
.. seealso:: :py:class:`TracModel <tracdap.rt.api.TracModel>`
|
62
63
|
"""
|
63
64
|
|
64
|
-
@_abc.abstractmethod
|
65
65
|
def get_parameter(self, parameter_name: str) -> _tp.Any:
|
66
66
|
|
67
67
|
"""
|
68
|
-
Get the value of a model parameter
|
68
|
+
Get the value of a model parameter.
|
69
69
|
|
70
|
-
Model parameters defined
|
71
|
-
by this method. Values are returned as native Python types. Parameter names
|
70
|
+
Model parameters defined using :py:meth:`define_parameters() <tracdap.rt.api.TracModel.define_parameters>`
|
71
|
+
can be retrieved at runtime by this method. Values are returned as native Python types. Parameter names
|
72
|
+
are case-sensitive.
|
72
73
|
|
73
74
|
Attempting to retrieve parameters not defined by the model will result in a runtime validation
|
74
75
|
error, even if those parameters are supplied in the job config and used by other models.
|
75
76
|
|
76
77
|
:param parameter_name: The name of the parameter to get
|
77
78
|
:return: The parameter value, as a native Python data type
|
79
|
+
:type parameter_name: str
|
78
80
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
79
81
|
"""
|
80
82
|
|
81
83
|
pass
|
82
84
|
|
83
|
-
@_abc.abstractmethod
|
84
85
|
def has_dataset(self, dataset_name: str) -> bool:
|
85
86
|
|
86
87
|
"""
|
87
|
-
Check whether a dataset is available in the current context
|
88
|
+
Check whether a dataset is available in the current context.
|
88
89
|
|
89
90
|
This method can be used to check whether optional model inputs have been supplied or not.
|
90
91
|
Models should use this method before calling get methods on optional inputs.
|
@@ -96,22 +97,20 @@ class TracContext:
|
|
96
97
|
|
97
98
|
:param dataset_name: The name of the dataset to check
|
98
99
|
:return: True if the dataset exists in the current context, False otherwise
|
99
|
-
:
|
100
|
+
:type dataset_name: str
|
100
101
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
101
102
|
"""
|
102
103
|
|
103
104
|
pass
|
104
105
|
|
105
|
-
@_abc.abstractmethod
|
106
106
|
def get_schema(self, dataset_name: str) -> SchemaDefinition:
|
107
107
|
|
108
108
|
"""
|
109
|
-
Get the schema of a model input or output
|
109
|
+
Get the schema of a model input or output.
|
110
110
|
|
111
111
|
Use this method to get the :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
|
112
112
|
for any input or output of the current model.
|
113
|
-
For datasets with static schemas,
|
114
|
-
:py:class:`TracModel <tracdap.rt.api.TracModel>` methods
|
113
|
+
For datasets with static schemas, these will be the same schemas that were defined using
|
115
114
|
:py:meth:`define_inputs() <tracdap.rt.api.TracModel.define_inputs>` and
|
116
115
|
:py:meth:`define_outputs() <tracdap.rt.api.TracModel.define_outputs>`.
|
117
116
|
|
@@ -124,9 +123,9 @@ class TracContext:
|
|
124
123
|
For optional inputs, use :py:meth:`has_dataset() <tracdap.rt.api.TracContext.has_dataset>`
|
125
124
|
to check whether the input was provided. Calling :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`
|
126
125
|
for an optional input that was not provided will always result in a validation error,
|
127
|
-
regardless of whether the input
|
128
|
-
:py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` can be called,
|
129
|
-
|
126
|
+
regardless of whether the input has a static or dynamic schema. For optional outputs
|
127
|
+
:py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` can be called, however if an
|
128
|
+
output is both optional and dynamic then the schema must first be set by calling
|
130
129
|
:py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`.
|
131
130
|
|
132
131
|
Attempting to retrieve the schema for a dataset that is not defined as a model input or output
|
@@ -135,165 +134,183 @@ class TracContext:
|
|
135
134
|
|
136
135
|
:param dataset_name: The name of the input or output to get the schema for
|
137
136
|
:return: The schema definition for the named dataset
|
137
|
+
:type dataset_name: str
|
138
138
|
:rtype: :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
|
139
139
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
140
140
|
"""
|
141
141
|
|
142
142
|
pass
|
143
143
|
|
144
|
-
@_abc.abstractmethod
|
145
144
|
def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) \
|
146
145
|
-> "pandas.DataFrame":
|
147
146
|
|
148
147
|
"""
|
149
|
-
Get the data for a model input or output as a Pandas dataframe
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
Attempting to retrieve a dataset that is not defined as a model input or
|
167
|
-
in a runtime validation error, even if that dataset
|
168
|
-
other models.
|
148
|
+
Get the data for a model input or output as a Pandas dataframe.
|
149
|
+
|
150
|
+
Model inputs can be accessed as Pandas dataframes using this method.
|
151
|
+
The TRAC runtime will handle fetching data from storage and apply any necessary
|
152
|
+
format conversions (to improve performance, data may be preloaded).
|
153
|
+
Only defined inputs can be accessed, use
|
154
|
+
:py:meth:`define_inputs() <tracdap.rt.api.TracModel.define_inputs>`
|
155
|
+
to define the inputs of a model. Input names are case-sensitive.
|
156
|
+
|
157
|
+
Model inputs are always available and can be accessed at any time inside
|
158
|
+
:py:meth:`run_model() <tracdap.rt.api.TracModel.run_model>`.
|
159
|
+
Model outputs can also be retrieved using this method, however they are
|
160
|
+
only available after they have been saved using
|
161
|
+
:py:meth:`put_pandas_table() <tracdap.rt.api.TracContext.put_pandas_table>`
|
162
|
+
(or another put method). Calling this method will simply return the
|
163
|
+
saved dataset.
|
164
|
+
|
165
|
+
Attempting to retrieve a dataset that is not defined as a model input or
|
166
|
+
output will result in a runtime validation error, even if that dataset
|
167
|
+
exists in the job config and is used by other models. Attempting to retrieve
|
168
|
+
an output before it has been saved will also cause a validation error.
|
169
169
|
|
170
170
|
:param dataset_name: The name of the model input or output to get data for
|
171
171
|
:param use_temporal_objects: Use Python objects for date/time fields instead of the NumPy *datetime64* type
|
172
172
|
:return: A pandas dataframe containing the data for the named dataset
|
173
|
+
:type dataset_name: str
|
174
|
+
:type use_temporal_objects: bool | None
|
175
|
+
:rtype: :py:class:`pandas.DataFrame`
|
173
176
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
174
177
|
"""
|
175
178
|
pass
|
176
179
|
|
177
|
-
@_abc.abstractmethod
|
178
180
|
def get_polars_table(self, dataset_name: str) -> "polars.DataFrame":
|
179
181
|
|
180
182
|
"""
|
181
|
-
Get the data for a model input or output as a Polars dataframe
|
183
|
+
Get the data for a model input or output as a Polars dataframe.
|
182
184
|
|
183
185
|
This method has equivalent semantics to :py:meth:`get_pandas_table`, but returns
|
184
186
|
a Polars dataframe.
|
185
187
|
|
186
188
|
:param dataset_name: The name of the model input or output to get data for
|
187
189
|
:return: A polars dataframe containing the data for the named dataset
|
190
|
+
:type dataset_name: str
|
191
|
+
:rtype: :py:class:`polars.DataFrame`
|
188
192
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
189
193
|
"""
|
190
194
|
|
191
195
|
pass
|
192
196
|
|
193
|
-
@_abc.abstractmethod
|
194
197
|
def put_schema(self, dataset_name: str, schema: SchemaDefinition):
|
195
198
|
|
196
199
|
"""
|
197
|
-
Set the schema of a dynamic model output
|
200
|
+
Set the schema of a dynamic model output.
|
198
201
|
|
199
|
-
For outputs marked as dynamic
|
200
|
-
|
201
|
-
|
202
|
+
For outputs marked as dynamic in :py:meth:`define_outputs() <tracdap.rt.api.TracModel.define_outputs>`,
|
203
|
+
a :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>` must be supplied using this
|
204
|
+
method before attempting to save the data. Once a schema has been set, it can be retrieved by calling
|
205
|
+
:py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` and data can be saved using
|
206
|
+
:py:meth:`put_pandas_table() <tracdap.rt.api.TracContext.put_pandas_table>` or another put method.
|
207
|
+
|
208
|
+
TRAC API functions are available to help with building schemas, such as
|
209
|
+
:py:func:`trac.F() <tracdap.rt.api.F>` to define individual fields or
|
202
210
|
:py:func:`load_schema() <tracdap.rt.api.load_schema>` to load predefined schemas.
|
203
|
-
|
204
|
-
|
205
|
-
If :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>` is called for an optional
|
206
|
-
output the model must also supply data for that output, otherwise TRAC will report a runtime
|
207
|
-
validation error after the model completes.
|
211
|
+
See the :py:mod:`tracdap.rt.api` package for a full list of functions that can be used
|
212
|
+
to build and manipulate schemas.
|
208
213
|
|
209
214
|
Each schema can only be set once and the schema will be validated using the normal
|
210
|
-
validation rules. If
|
211
|
-
|
215
|
+
validation rules. If :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>` is called for
|
216
|
+
an optional output the model must supply data for that output, otherwise TRAC will report a
|
217
|
+
validation error after the model completes.
|
218
|
+
|
212
219
|
Attempting to set the schema for a dataset that is not defined as a dynamic model output
|
213
|
-
for the current model will result in a runtime validation error.
|
220
|
+
for the current model will result in a runtime validation error. Supplying a schema that
|
221
|
+
fails validation will also result in a validation error.
|
222
|
+
|
214
223
|
|
215
224
|
:param dataset_name: The name of the output to set the schema for
|
216
225
|
:param schema: A TRAC schema definition to use for the named output
|
226
|
+
:type dataset_name: str
|
217
227
|
:type schema: :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
|
218
228
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
|
219
229
|
"""
|
220
230
|
|
221
231
|
pass
|
222
232
|
|
223
|
-
@_abc.abstractmethod
|
224
233
|
def put_pandas_table(self, dataset_name: str, dataset: "pandas.DataFrame"):
|
225
234
|
|
226
235
|
"""
|
227
|
-
Save the data for a model output as a Pandas dataframe
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
236
|
+
Save the data for a model output as a Pandas dataframe.
|
237
|
+
|
238
|
+
Model outputs can then be saved as Pandas dataframes using this method.
|
239
|
+
The TRAC runtime will validate the supplied data and send it to storage,
|
240
|
+
applying any necessary format conversions. Only defined outputs can be
|
241
|
+
saved, use :py:meth:`define_outputs() <tracdap.rt.api.TracModel.define_outputs>`
|
242
|
+
to define the outputs of a model. Output names are case-sensitive. Once
|
243
|
+
an output has been saved it can be retrieved by calling
|
244
|
+
:py:meth:`get_pandas_table() <tracdap.rt.api.TracContext.get_pandas_table>`
|
245
|
+
(or another get method).
|
246
|
+
|
247
|
+
Each model output can only be saved once and the supplied data must match the schema of
|
248
|
+
the named output. Missing fields or fields of the wrong type will result in a data
|
249
|
+
conformance error. Extra fields will be discarded with a warning. The schema of an output
|
250
|
+
dataset can be checked using :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`.
|
251
|
+
For dynamic outputs, the schema must first be set using
|
252
|
+
:py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`
|
253
|
+
|
254
|
+
Attempting to save a dataset that is not defined as a model output will cause a runtime
|
255
|
+
validation error. Attempting to save an output twice, or save a dynamic output before its
|
256
|
+
schema is set will also cause a validation error.
|
241
257
|
|
242
258
|
:param dataset_name: The name of the model output to save data for
|
243
259
|
:param dataset: A pandas dataframe containing the data for the named dataset
|
260
|
+
:type dataset_name: str
|
261
|
+
:type dataset: :py:class:`pandas.Dataframe`
|
244
262
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
|
245
263
|
:py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
|
246
264
|
"""
|
247
265
|
|
248
266
|
pass
|
249
267
|
|
250
|
-
@_abc.abstractmethod
|
251
268
|
def put_polars_table(self, dataset_name: str, dataset: "polars.DataFrame"):
|
252
269
|
|
253
270
|
"""
|
254
|
-
Save the data for a model output as a Polars dataframe
|
271
|
+
Save the data for a model output as a Polars dataframe.
|
255
272
|
|
256
273
|
This method has equivalent semantics to :py:meth:`put_pandas_table`, but accepts
|
257
274
|
a Polars dataframe.
|
258
275
|
|
259
276
|
:param dataset_name: The name of the model output to save data for
|
260
277
|
:param dataset: A polars dataframe containing the data for the named dataset
|
278
|
+
:type dataset_name: str
|
279
|
+
:type dataset: :py:class:`polars.DataFrame`
|
261
280
|
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
|
262
281
|
:py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
|
263
282
|
"""
|
264
283
|
|
265
284
|
pass
|
266
285
|
|
267
|
-
@_abc.abstractmethod
|
268
286
|
def log(self) -> _logging.Logger:
|
269
287
|
|
270
288
|
"""
|
271
|
-
Get a Python logger that can be used for writing model logs
|
289
|
+
Get a Python logger that can be used for writing model logs.
|
272
290
|
|
273
291
|
Logs written to this logger are recorded by TRAC. When models are run on the platform,
|
274
292
|
these logs are assembled and saved with the job outputs as a dataset, that can be queried
|
275
293
|
through the regular TRAC data and metadata APIs.
|
276
294
|
|
277
295
|
:return: A Python logger that can be used for writing model logs
|
296
|
+
:rtype: :py:class:`logging.Logger`
|
278
297
|
"""
|
279
298
|
|
280
299
|
pass
|
281
300
|
|
282
301
|
|
283
|
-
class TracModel:
|
302
|
+
class TracModel(metaclass=_abc.ABCMeta):
|
284
303
|
|
285
304
|
"""
|
286
305
|
Base class that model components inherit from to be recognised by the platform
|
287
306
|
|
288
307
|
The modelling API is designed to be as simple and un-opinionated as possible.
|
289
|
-
Models inherit from :py:class:`TracModel` and implement the :py:meth:`run_model` method to provide their
|
290
|
-
model logic. :py:meth:`run_model` has one parameter, a :class:`TracContext` object which is supplied to
|
308
|
+
Models inherit from :py:class:`TracModel` and implement the :py:meth:`run_model()` method to provide their
|
309
|
+
model logic. :py:meth:`run_model()` has one parameter, a :class:`TracContext` object which is supplied to
|
291
310
|
the model at runtime, allowing it to access parameters, inputs and outputs.
|
292
311
|
|
293
312
|
Models must also as a minimum implement three methods to define the model schema,
|
294
|
-
:py:meth:`define_parameters()
|
295
|
-
:py:meth:`define_inputs()<TracModel.define_inputs>` and
|
296
|
-
:py:meth:`define_outputs()<TracModel.define_outputs>`.
|
313
|
+
:py:meth:`define_parameters()`, :py:meth:`define_inputs()` and :py:meth:`define_outputs()`.
|
297
314
|
The parameters, inputs and outputs that are defined will be available in the context at runtime.
|
298
315
|
The :py:mod:`tracdap.rt.api` package includes a number of helper functions to implement these methods in
|
299
316
|
a clear and robust way.
|
@@ -311,13 +328,13 @@ class TracModel:
|
|
311
328
|
|
312
329
|
Models should also avoid making system calls, or using the Python builtins exec() or eval().
|
313
330
|
|
314
|
-
.. seealso:: :py:class:`TracContext
|
331
|
+
.. seealso:: :py:class:`TracContext <tracdap.rt.api.TracContext>`
|
315
332
|
"""
|
316
333
|
|
317
334
|
def define_attributes(self) -> _tp.Dict[str, Value]: # noqa
|
318
335
|
|
319
336
|
"""
|
320
|
-
Define attributes that will be associated with the model when it is loaded into the TRAC platform
|
337
|
+
Define attributes that will be associated with the model when it is loaded into the TRAC platform.
|
321
338
|
|
322
339
|
.. note::
|
323
340
|
This is an experimental API that is not yet stabilised, expect changes in future versions of TRAC
|
@@ -342,7 +359,7 @@ class TracModel:
|
|
342
359
|
def define_parameters(self) -> _tp.Dict[str, ModelParameter]:
|
343
360
|
|
344
361
|
"""
|
345
|
-
Define parameters that will be available to the model at runtime
|
362
|
+
Define parameters that will be available to the model at runtime.
|
346
363
|
|
347
364
|
Implement this method to define the model's parameters, every parameter that the
|
348
365
|
model uses must be defined. Models may choose to ignore some parameters,
|
@@ -363,7 +380,7 @@ class TracModel:
|
|
363
380
|
def define_inputs(self) -> _tp.Dict[str, ModelInputSchema]:
|
364
381
|
|
365
382
|
"""
|
366
|
-
Define data inputs that will be available to the model at runtime
|
383
|
+
Define data inputs that will be available to the model at runtime.
|
367
384
|
|
368
385
|
Implement this method to define the model's inputs, every data input that the
|
369
386
|
model uses must be defined. Models may choose to ignore some inputs,
|
@@ -384,7 +401,7 @@ class TracModel:
|
|
384
401
|
def define_outputs(self) -> _tp.Dict[str, ModelOutputSchema]:
|
385
402
|
|
386
403
|
"""
|
387
|
-
Define data outputs that will be produced by the model at runtime
|
404
|
+
Define data outputs that will be produced by the model at runtime.
|
388
405
|
|
389
406
|
Implement this method to define the model's outputs, every data output that the
|
390
407
|
model produces must be defined and every output that is defined must be
|
@@ -406,20 +423,23 @@ class TracModel:
|
|
406
423
|
def run_model(self, ctx: TracContext):
|
407
424
|
|
408
425
|
"""
|
409
|
-
Entry point for running model code
|
426
|
+
Entry point for running model code.
|
410
427
|
|
411
|
-
Implement this method to provide the model logic. A
|
428
|
+
Implement this method to provide the model logic. A
|
429
|
+
:py:class:`TracContext <tracdap.rt.api.TracContext>` is provided
|
412
430
|
at runtime, which makes parameters and inputs available and provides a means to save outputs.
|
413
431
|
All the outputs defined in :py:meth:`define_outputs` must be saved before this method returns,
|
414
432
|
otherwise a runtime validation error will be raised.
|
415
433
|
|
416
434
|
Model code can raise exceptions, either in a controlled way by detecting error conditions and raising
|
417
435
|
errors explicitly, or in an uncontrolled way as a result of bugs in the model code. Exceptions may also
|
418
|
-
originate inside libraries the model code is using. If an exception escapes from
|
436
|
+
originate inside libraries the model code is using. If an exception escapes from
|
437
|
+
:py:meth:`run_model() <tracdap.rt.api.TracModel.run_model>`
|
419
438
|
TRAC will mark the model as failed, the job that contains the model will also fail.
|
420
439
|
|
421
440
|
:param ctx: A context use to access model inputs, outputs and parameters
|
422
441
|
and communicate with the TRAC platform
|
442
|
+
:type ctx: :py:class:`TracContext <tracdap.rt.api.TracContext>`
|
423
443
|
"""
|
424
444
|
|
425
445
|
pass
|