tracdap-runtime 0.7.0__tar.gz → 0.7.1__tar.gz

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: tracdap-runtime
-Version: 0.7.0
+Version: 0.7.1
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tracdap-runtime
-version = 0.7.0
+version = 0.7.1
 description = Runtime package for building models on the TRAC Data & Analytics Platform
 long_description = file: README.md
 long_description_content_type = text/markdown

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_exec/dev_mode.py

@@ -37,12 +37,12 @@ DEV_MODE_JOB_CONFIG = [
     re.compile(r"job\.\w+\.model"),
     re.compile(r"job\.\w+\.flow"),
 
-    re.compile(r".*\.jobs\.\d+\.\w+\.parameters\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.inputs\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.outputs\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.models\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.model"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.flow")
+    re.compile(r".*\.jobs\[\d+]\.\w+\.parameters\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.inputs\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.outputs\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.models\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.model"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.flow")
 ]
 
 DEV_MODE_SYS_CONFIG = []
@@ -764,10 +764,15 @@ class DevModeTranslator:
             else:
                 p_spec = param_specs[p_name]
 
-                cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType}")
+                try:
+                    cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType}")
+                    encoded_value = _types.MetadataCodec.convert_value(p_value, p_spec.paramType)
+                    encoded_values[p_name] = encoded_value
 
-                encoded_value = _types.MetadataCodec.convert_value(p_value, p_spec.paramType)
-                encoded_values[p_name] = encoded_value
+                except Exception as e:
+                    msg = f"Failed to encode parameter [{p_name}]: {str(e)}"
+                    cls._log.error(msg)
+                    raise _ex.EConfigParse(msg) from e
 
         return encoded_values
 

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_impl/config_parser.py

@@ -321,6 +321,23 @@ class ConfigParser(tp.Generic[_T]):
 
     def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
 
+        if self._is_dev_mode_location(location):
+
+            if type(raw_value) in ConfigParser.__primitive_types:
+                return self._parse_primitive(location, raw_value, type(raw_value))
+
+            if isinstance(raw_value, list):
+                if len(raw_value) == 0:
+                    return []
+                items = iter((self._child_location(location, i), x) for i, x in enumerate(raw_value))
+                return list(self._parse_value(loc, x, tp.Any) for loc, x in items)
+
+            if isinstance(raw_value, dict):
+                if len(raw_value) == 0:
+                    return {}
+                items = iter((self._child_location(location, k), k, v) for k, v in raw_value.items())
+                return dict((k, self._parse_value(loc, v, tp.Any)) for loc, k, v in items)
+
         if raw_value is None:
             return None
 
@@ -339,24 +356,13 @@ class ConfigParser(tp.Generic[_T]):
             return self._parse_enum(location, raw_value, annotation)
 
         if _dc.is_dataclass(annotation):
-
-            if isinstance(raw_value, tp.Dict):
-                return self._parse_simple_class(location, raw_value, annotation)
-
-            if self._is_dev_mode_location(location):
-                if type(raw_value) in ConfigParser.__primitive_types:
-                    return self._parse_primitive(location, raw_value, type(raw_value))
-                if isinstance(raw_value, list):
-                    if len(raw_value) == 0:
-                        return []
-                    list_type = type(raw_value[0])
-                    return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
-
-            return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
+            return self._parse_simple_class(location, raw_value, annotation)
 
         if isinstance(annotation, self.__generic_metaclass):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
+        return self._error(location, f"Cannot parse value of type {annotation.__name__}")
+
     def _is_dev_mode_location(self, location):
 
         return any(map(lambda pattern: re.match(pattern, location), self._dev_mode_locations))
@@ -416,7 +422,7 @@ class ConfigParser(tp.Generic[_T]):
     def _parse_simple_class(self, location: str, raw_dict: tp.Any, metaclass: type) -> object:
 
         if raw_dict is not None and not isinstance(raw_dict, dict):
-            pass
+            return self._error(location, f"Expected type {metaclass.__name__}, got '{str(raw_dict)}'")
 
         obj = metaclass.__new__(metaclass, object())  # noqa
 
@@ -510,7 +516,7 @@ class ConfigParser(tp.Generic[_T]):
                 return self._error(location, f"Expected a list, got {type(raw_value)}")
 
             return [
-                self._parse_value(self._child_location(location, str(idx)), item, list_type)
+                self._parse_value(self._child_location(location, idx), item, list_type)
                 for (idx, item) in enumerate(raw_value)]
 
         if origin == tp.Dict or origin == dict:
@@ -541,12 +547,14 @@ class ConfigParser(tp.Generic[_T]):
         return None
 
     @staticmethod
-    def _child_location(parent_location: str, item: str):
+    def _child_location(parent_location: str, item: tp.Union[str, int]):
 
         if parent_location is None or parent_location == "":
             return item
+        elif isinstance(item, int):
+            return f"{parent_location}[{item}]"
         else:
-            return parent_location + "." + item
+            return f"{parent_location}.{item}"
 
 
 class ConfigQuoter:

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_impl/static_api.py

@@ -55,7 +55,7 @@ class StaticApiImpl(_StaticApiHook):
         if not _val.is_primitive_type(entry_type):
             raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
 
-        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, mapType=_meta.TypeDescriptor(entry_type))
 
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_impl/type_system.py

@@ -133,9 +133,13 @@ class MetadataCodec:
 
         if basic_type == _meta.BasicType.ARRAY:
             items = value.arrayValue.items
-            return list(map(lambda x: MetadataCodec._decode_value_for_type(x, type_desc.arrayType), items))
+            return list(MetadataCodec._decode_value_for_type(x, type_desc.arrayType) for x in items)
 
-        raise _ex.ETracInternal(f"Decoding value type [{basic_type}] is not supported yet")
+        if basic_type == _meta.BasicType.MAP:
+            items = value.mapValue.entries.items()
+            return dict((k, MetadataCodec._decode_value_for_type(v, type_desc.mapType)) for k, v in items)
+
+        raise _ex.ETracInternal(f"Cannot decode value of type [{basic_type}]")
 
     @classmethod
     def encode_value(cls, value: tp.Any) -> _meta.Value:
@@ -183,19 +187,36 @@ class MetadataCodec:
             if any(map(lambda x: type(x) != array_raw_type, value)):
                 raise _ex.ETracInternal("Cannot encode a list with values of different types")
 
-            encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type), value))
+            encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type, True), value))
 
             return _meta.Value(
                 _meta.TypeDescriptor(_meta.BasicType.ARRAY, arrayType=array_trac_type),
                 arrayValue=_meta.ArrayValue(encoded_items))
 
-        raise _ex.ETracInternal(f"Value type [{type(value)}] is not supported yet")
+        if isinstance(value, dict):
+
+            if len(value) == 0:
+                raise _ex.ETracInternal("Cannot encode an empty dict")
+
+            map_raw_type = type(next(iter(value.values())))
+            map_trac_type = TypeMapping.python_to_trac(map_raw_type)
+
+            if any(map(lambda x: type(x) != array_raw_type, value.values())):
+                raise _ex.ETracInternal("Cannot encode a dict with values of different types")
+
+            encoded_entries = dict(map(lambda kv: (kv[0], cls.convert_value(kv[1], map_trac_type, True)), value.items()))
+
+            return _meta.Value(
+                _meta.TypeDescriptor(_meta.BasicType.ARRAY, mapType=map_trac_type),
+                mapValue=_meta.MapValue(encoded_entries))
+
+        raise _ex.ETracInternal(f"Cannot encode value of type [{type(value).__name__}]")
 
     @classmethod
-    def convert_value(cls, raw_value: tp.Any, type_desc: _meta.TypeDescriptor):
+    def convert_value(cls, raw_value: tp.Any, type_desc: _meta.TypeDescriptor, nested: bool = False):
 
         if type_desc.basicType == _meta.BasicType.BOOLEAN:
-            return cls.convert_boolean_value(raw_value)
+            return cls.convert_boolean_value(raw_value, nested)
 
         if type_desc.basicType == _meta.BasicType.INTEGER:
             return cls.convert_integer_value(raw_value)
@@ -218,78 +239,97 @@ class MetadataCodec:
         if type_desc.basicType == _meta.BasicType.ARRAY:
             return cls.convert_array_value(raw_value, type_desc.arrayType)
 
+        if type_desc.basicType == _meta.BasicType.MAP:
+            return cls.convert_map_value(raw_value, type_desc.mapType)
+
         raise _ex.ETracInternal(f"Conversion to value type [{type_desc.basicType.name}] is not supported yet")
 
     @staticmethod
     def convert_array_value(raw_value: tp.List[tp.Any], array_type: _meta.TypeDescriptor) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.ARRAY, array_type)
+        type_desc = _meta.TypeDescriptor(basicType=_meta.BasicType.ARRAY, arrayType=array_type)
 
         if not isinstance(raw_value, list):
-            msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.ARRAY.name}"
+            msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.ARRAY.name}"
             raise _ex.ETracInternal(msg)
 
-        items = list(map(lambda x: MetadataCodec.convert_value(x, array_type), raw_value))
+        items = list(map(lambda x: MetadataCodec.convert_value(x, array_type, True), raw_value))
 
         return _meta.Value(type_desc, arrayValue=_meta.ArrayValue(items))
 
     @staticmethod
-    def convert_boolean_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_map_value(raw_value: tp.Dict[str, tp.Any], map_type: _meta.TypeDescriptor) -> _meta.Value:
+
+        type_desc = _meta.TypeDescriptor(basicType=_meta.BasicType.MAP, mapType=map_type)
+
+        if not isinstance(raw_value, dict):
+            msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.MAP.name}"
+            raise _ex.ETracInternal(msg)
+
+        entries = dict(map(lambda kv: (kv[0], MetadataCodec.convert_value(kv[1], map_type, True)), raw_value.items()))
+
+        return _meta.Value(type_desc, mapValue=_meta.MapValue(entries))
+
+    @staticmethod
+    def convert_boolean_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.BOOLEAN)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.BOOLEAN) if not nested else None
 
         if isinstance(raw_value, bool):
             return _meta.Value(type_desc, booleanValue=raw_value)
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.BOOLEAN.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.BOOLEAN.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_integer_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_integer_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.INTEGER)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.INTEGER) if not nested else None
 
-        if isinstance(raw_value, int):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, integerValue=raw_value)
 
         if isinstance(raw_value, float) and raw_value.is_integer():
             return _meta.Value(type_desc, integerValue=int(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.INTEGER.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.INTEGER.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_float_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_float_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.FLOAT)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.FLOAT) if not nested else None
 
         if isinstance(raw_value, float):
             return _meta.Value(type_desc, floatValue=raw_value)
 
-        if isinstance(raw_value, int):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, floatValue=float(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.FLOAT.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.FLOAT.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_decimal_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_decimal_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DECIMAL)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DECIMAL) if not nested else None
 
         if isinstance(raw_value, decimal.Decimal):
             return _meta.Value(type_desc, decimalValue=_meta.DecimalValue(str(raw_value)))
 
-        if isinstance(raw_value, int) or isinstance(raw_value, float):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) or isinstance(raw_value, float) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, decimalValue=_meta.DecimalValue(str(raw_value)))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DECIMAL.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DECIMAL.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_string_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_string_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.STRING)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.STRING) if not nested else None
 
         if isinstance(raw_value, str):
             return _meta.Value(type_desc, stringValue=raw_value)
@@ -301,13 +341,13 @@ class MetadataCodec:
 
             return _meta.Value(type_desc, stringValue=str(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.STRING.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.STRING.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_date_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_date_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE) if not nested else None
 
         if isinstance(raw_value, dt.date):
             return _meta.Value(type_desc, dateValue=_meta.DateValue(isoDate=raw_value.isoformat()))
@@ -316,13 +356,13 @@ class MetadataCodec:
             date_value = dt.date.fromisoformat(raw_value)
             return _meta.Value(type_desc, dateValue=_meta.DateValue(isoDate=date_value.isoformat()))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DATE.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DATE.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_datetime_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_datetime_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATETIME)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATETIME) if not nested else None
 
         if isinstance(raw_value, dt.datetime):
             return _meta.Value(type_desc, datetimeValue=_meta.DatetimeValue(isoDatetime=raw_value.isoformat()))
@@ -331,5 +371,5 @@ class MetadataCodec:
             datetime_value = dt.datetime.fromisoformat(raw_value)
             return _meta.Value(type_desc, datetimeValue=_meta.DatetimeValue(isoDatetime=datetime_value.isoformat()))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DATETIME.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DATETIME.name}"
         raise _ex.ETracInternal(msg)

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_plugins/storage_sql.py

@@ -268,10 +268,18 @@ plugins.PluginManager.register_plugin(IStorageProvider, SqlStorageProvider, ["SQ
 
 
 try:
-
     import sqlalchemy as sqla  # noqa
     import sqlalchemy.exc as sqla_exc  # noqa
 
+    # Only 2.x versions of SQL Alchemy are currently supported
+    sqla_supported = sqla.__version__.startswith("2.")
+
+except ModuleNotFoundError:
+    sqla = None
+    sqla_supported = False
+
+if sqla_supported:
+
     class SqlAlchemyDriver(ISqlDriver):
 
         def __init__(self, properties: tp.Dict[str, str]):
@@ -336,7 +344,7 @@ try:
 
         class ConnectionWrapper(DbApiWrapper.Connection):
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
 
             def close(self):
@@ -355,7 +363,7 @@ try:
 
             arraysize: int = 1000
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
                 self.__result: tp.Optional[sqla.CursorResult] = None
 
@@ -414,5 +422,4 @@ try:
 
     plugins.PluginManager.register_plugin(ISqlDriver, SqlAlchemyDriver, ["alchemy"])
 
-except ModuleNotFoundError:
-    pass
+

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.7.0"
+__version__ = "0.7.1"

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/tracdap_runtime.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: tracdap-runtime
-Version: 0.7.0
+Version: 0.7.1
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse

{tracdap_runtime-0.7.0 → tracdap_runtime-0.7.1}/src/tracdap/rt/metadata/__init__.py

@@ -9,6 +9,8 @@ from .type import Value
 from .type import ArrayValue
 from .type import MapValue
 
+from .custom import CustomDefinition
+
 from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
@@ -21,6 +23,13 @@ from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
+from .stoarge import CopyStatus
+from .stoarge import IncarnationStatus
+from .stoarge import StorageCopy
+from .stoarge import StorageIncarnation
+from .stoarge import StorageItem
+from .stoarge import StorageDefinition
+
 from .model import ModelType
 from .model import ModelParameter
 from .model import ModelInputSchema
@@ -56,23 +65,14 @@ from .job import JobGroup
 from .job import SequentialJobGroup
 from .job import ParallelJobGroup
 
-from .file import FileDefinition
-
-from .custom import CustomDefinition
-
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
-
-from .object import ObjectDefinition
-
-from .resource import ResourceType
-
 from .common import MetadataFormat
 from .common import MetadataVersion
 from .common import TenantInfo
 
+from .file import FileDefinition
+
+from .object import ObjectDefinition
+
 from .tag import Tag
+
+from .resource import ResourceType