tracdap-runtime 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

tracdap/rt/_exec/graph_builder.py

@@ -12,8 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import tracdap.rt.config as config
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.data as _data  # noqa
@@ -42,6 +40,9 @@ class GraphBuilder:
         if job_config.job.jobType == meta.JobType.RUN_FLOW:
             return cls.build_standard_job(job_config, result_spec, cls.build_run_flow_job)
 
+        if job_config.job.jobType in [meta.JobType.IMPORT_DATA, meta.JobType.EXPORT_DATA]:
+            return cls.build_standard_job(job_config, result_spec, cls.build_import_export_data_job)
+
         raise _ex.EConfigParse(f"Job type [{job_config.job.jobType}] is not supported yet")
 
     @classmethod
@@ -114,6 +115,28 @@ class GraphBuilder:
 
         return cls._join_sections(main_section, result_section)
 
+    @classmethod
+    def build_import_export_data_job(
+            cls, job_config: config.JobConfig, result_spec: JobResultSpec,
+            job_namespace: NodeNamespace, job_push_id: NodeId) \
+            -> GraphSection:
+
+        # TODO: These are processed as regular calculation jobs for now
+        # That might be ok, but is worth reviewing
+
+        if job_config.job.jobType == meta.JobType.IMPORT_DATA:
+            job_def = job_config.job.importData
+        else:
+            job_def = job_config.job.exportData
+
+        target_selector = job_def.model
+        target_obj = _util.get_job_resource(target_selector, job_config)
+        target_def = target_obj.model
+
+        return cls.build_calculation_job(
+            job_config, result_spec, job_namespace, job_push_id,
+            target_selector, target_def, job_def)
+
     @classmethod
     def build_run_model_job(
             cls, job_config: config.JobConfig, result_spec: JobResultSpec,
@@ -380,6 +403,65 @@ class GraphBuilder:
 
         return GraphSection(nodes, inputs=inputs)
 
+    @classmethod
+    def build_runtime_outputs(cls, output_names: tp.List[str], job_namespace: NodeNamespace):
+
+        # TODO: Factor out common logic with regular job outputs (including static / dynamic)
+
+        nodes = {}
+        inputs = set()
+        outputs = list()
+
+        for output_name in output_names:
+
+            # Output data view must already exist in the namespace
+            data_view_id = NodeId.of(output_name, job_namespace, _data.DataView)
+            data_spec_id = NodeId.of(f"{output_name}:SPEC", job_namespace, _data.DataSpec)
+
+            data_key = output_name + ":DATA"
+            data_id = _util.new_object_id(meta.ObjectType.DATA)
+            storage_key = output_name + ":STORAGE"
+            storage_id = _util.new_object_id(meta.ObjectType.STORAGE)
+
+            data_spec_node = DynamicDataSpecNode(
+                data_spec_id, data_view_id,
+                data_id, storage_id,
+                prior_data_spec=None)
+
+            output_data_key = _util.object_key(data_id)
+            output_storage_key = _util.object_key(storage_id)
+
+            # Map one data item from each view, since outputs are single part/delta
+            data_item_id = NodeId(f"{output_name}:ITEM", job_namespace, _data.DataItem)
+            data_item_node = DataItemNode(data_item_id, data_view_id)
+
+            # Create a physical save operation for the data item
+            data_save_id = NodeId.of(f"{output_name}:SAVE", job_namespace, None)
+            data_save_node = SaveDataNode(data_save_id, data_spec_id, data_item_id)
+
+            data_result_id = NodeId.of(f"{output_name}:RESULT", job_namespace, ObjectBundle)
+            data_result_node = DataResultNode(
+                data_result_id, output_name,
+                data_item_id, data_spec_id, data_save_id,
+                output_data_key, output_storage_key)
+
+            nodes[data_spec_id] = data_spec_node
+            nodes[data_item_id] = data_item_node
+            nodes[data_save_id] = data_save_node
+            nodes[data_result_id] = data_result_node
+
+            # Job-level data view is an input to the save operation
+            inputs.add(data_view_id)
+            outputs.append(data_result_id)
+
+        runtime_outputs = JobOutputs(bundles=outputs)
+        runtime_outputs_id = NodeId.of("trac_runtime_outputs", job_namespace, JobOutputs)
+        runtime_outputs_node = RuntimeOutputsNode(runtime_outputs_id, runtime_outputs)
+
+        nodes[runtime_outputs_id] = runtime_outputs_node
+
+        return GraphSection(nodes, inputs=inputs, outputs={runtime_outputs_id})
+
     @classmethod
     def build_job_results(
             cls, job_config: cfg.JobConfig, job_namespace: NodeNamespace, result_spec: JobResultSpec,
@@ -396,7 +478,8 @@ class GraphBuilder:
 
             build_result_node = BuildJobResultNode(
                 build_result_id, job_config.jobId,
-                objects=objects, explicit_deps=explicit_deps)
+                outputs = JobOutputs(objects=objects),
+                explicit_deps=explicit_deps)
 
         elif bundles is not None:
 
@@ -404,7 +487,8 @@ class GraphBuilder:
 
             build_result_node = BuildJobResultNode(
                 build_result_id, job_config.jobId,
-                bundles=bundles, explicit_deps=explicit_deps)
+                outputs = JobOutputs(bundles=bundles),
+                explicit_deps=explicit_deps)
 
         else:
             raise _ex.EUnexpected()
@@ -459,7 +543,7 @@ class GraphBuilder:
             -> GraphSection:
 
         if model_or_flow.objectType == meta.ObjectType.MODEL:
-            return cls.build_model(namespace, model_or_flow.model, explicit_deps)
+            return cls.build_model(job_config, namespace, model_or_flow.model, explicit_deps)
 
         elif model_or_flow.objectType == meta.ObjectType.FLOW:
             return cls.build_flow(job_config, namespace, model_or_flow.flow)
@@ -469,11 +553,13 @@ class GraphBuilder:
 
     @classmethod
     def build_model(
-            cls, namespace: NodeNamespace,
+            cls, job_config: config.JobConfig, namespace: NodeNamespace,
             model_def: meta.ModelDefinition,
             explicit_deps: tp.Optional[tp.List[NodeId]] = None) \
             -> GraphSection:
 
+        cls.check_model_type(job_config, model_def)
+
         def param_id(node_name):
             return NodeId(node_name, namespace, meta.Value)
 
@@ -485,6 +571,14 @@ class GraphBuilder:
         input_ids = set(map(data_id, model_def.inputs))
         output_ids = set(map(data_id, model_def.outputs))
 
+        # Set up storage access for import / export data jobs
+        if job_config.job.jobType == meta.JobType.IMPORT_DATA:
+            storage_access = job_config.job.importData.storageAccess
+        elif job_config.job.jobType == meta.JobType.EXPORT_DATA:
+            storage_access = job_config.job.exportData.storageAccess
+        else:
+            storage_access = None
+
         # Create the model node
         # Always add the prior graph root ID as a dependency
         # This is to ensure dependencies are still pulled in for models with no inputs!
@@ -500,7 +594,8 @@ class GraphBuilder:
         model_node = RunModelNode(
             model_id, model_scope, model_def,
             frozenset(parameter_ids), frozenset(input_ids),
-            explicit_deps=explicit_deps, bundle=model_id.namespace)
+            explicit_deps=explicit_deps, bundle=model_id.namespace,
+            storage_access=storage_access)
 
         model_result_id = NodeId(f"{model_name}:RESULT", namespace)
         model_result_node = RunModelResultNode(model_result_id, model_id)
@@ -637,6 +732,7 @@ class GraphBuilder:
 
             # Explicit check for model compatibility - report an error now, do not try build_model()
             cls.check_model_compatibility(model_selector, model_obj.model, node_name, node)
+            cls.check_model_type(job_config, model_obj.model)
 
             return cls.build_model_or_flow_with_context(
                 job_config, namespace, node_name, model_obj,
@@ -647,8 +743,8 @@ class GraphBuilder:
 
     @classmethod
     def check_model_compatibility(
-            cls, model_selector: meta.TagSelector, model_def: meta.ModelDefinition,
-            node_name: str, flow_node: meta.FlowNode):
+            cls, model_selector: meta.TagSelector,
+            model_def: meta.ModelDefinition, node_name: str, flow_node: meta.FlowNode):
 
         model_params = list(sorted(model_def.parameters.keys()))
         model_inputs = list(sorted(model_def.inputs.keys()))
@@ -662,6 +758,21 @@ class GraphBuilder:
             model_key = _util.object_key(model_selector)
             raise _ex.EJobValidation(f"Incompatible model for flow node [{node_name}] (Model: [{model_key}])")
 
+    @classmethod
+    def check_model_type(cls, job_config: config.JobConfig, model_def: meta.ModelDefinition):
+
+        if job_config.job.jobType == meta.JobType.IMPORT_DATA:
+            allowed_model_types = [meta.ModelType.DATA_IMPORT_MODEL]
+        elif job_config.job.jobType == meta.JobType.EXPORT_DATA:
+            allowed_model_types = [meta.ModelType.DATA_EXPORT_MODEL]
+        else:
+            allowed_model_types = [meta.ModelType.STANDARD_MODEL]
+
+        if model_def.modelType not in allowed_model_types:
+            job_type = job_config.job.jobType.name
+            model_type = model_def.modelType.name
+            raise _ex.EJobValidation(f"Job type [{job_type}] cannot use model type [{model_type}]")
+
     @staticmethod
     def build_context_push(
             namespace: NodeNamespace, input_mapping: tp.Dict[str, NodeId],

tracdap/rt/_impl/data.py

@@ -12,8 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import dataclasses as dc
 import typing as tp
 import datetime as dt
@@ -22,7 +20,16 @@ import platform
 
 import pyarrow as pa
 import pyarrow.compute as pc
-import pandas as pd
+
+try:
+    import pandas  # noqa
+except ModuleNotFoundError:
+    pandas = None
+
+try:
+    import polars  # noqa
+except ModuleNotFoundError:
+    polars = None
 
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
@@ -42,7 +49,7 @@ class DataSpec:
 class DataPartKey:
 
     @classmethod
-    def for_root(cls) -> DataPartKey:
+    def for_root(cls) -> "DataPartKey":
         return DataPartKey(opaque_key='part_root')
 
     opaque_key: str
@@ -55,14 +62,14 @@ class DataItem:
     table: tp.Optional[pa.Table] = None
     batches: tp.Optional[tp.List[pa.RecordBatch]] = None
 
-    pandas: tp.Optional[pd.DataFrame] = None
+    pandas: "tp.Optional[pandas.DataFrame]" = None
     pyspark: tp.Any = None
 
     def is_empty(self) -> bool:
         return self.table is None and (self.batches is None or len(self.batches) == 0)
 
     @staticmethod
-    def create_empty() -> DataItem:
+    def create_empty() -> "DataItem":
         return DataItem(pa.schema([]))
 
 
@@ -74,17 +81,21 @@ class DataView:
 
     parts: tp.Dict[DataPartKey, tp.List[DataItem]]
 
+    @staticmethod
+    def create_empty() -> "DataView":
+        return DataView(_meta.SchemaDefinition(), pa.schema([]), dict())
+
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
         return DataView(trac_schema, arrow_schema, dict())
 
-    def is_empty(self) -> bool:
-        return self.parts is None or len(self.parts) == 0
+    def with_trac_schema(self, trac_schema: _meta.SchemaDefinition):
+        arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
+        return DataView(trac_schema, arrow_schema, self.parts)
 
-    @staticmethod
-    def create_empty() -> DataView:
-        return DataView(_meta.SchemaDefinition(), pa.schema([]), dict())
+    def is_empty(self) -> bool:
+        return self.parts is None or not any(self.parts.values())
 
 
 class _DataInternal:
@@ -121,14 +132,14 @@ class DataMapping:
     }
 
     # Check the Pandas dtypes for handling floats are available before setting up the type mapping
-    __PANDAS_VERSION_ELEMENTS = pd.__version__.split(".")
+    __PANDAS_VERSION_ELEMENTS = pandas.__version__.split(".")
     __PANDAS_MAJOR_VERSION = int(__PANDAS_VERSION_ELEMENTS[0])
     __PANDAS_MINOR_VERSION = int(__PANDAS_VERSION_ELEMENTS[1])
 
     if __PANDAS_MAJOR_VERSION == 2:
 
-        __PANDAS_DATE_TYPE = pd.to_datetime([dt.date(2000, 1, 1)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
-        __PANDAS_DATETIME_TYPE = pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
+        __PANDAS_DATE_TYPE = pandas.to_datetime([dt.date(2000, 1, 1)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
+        __PANDAS_DATETIME_TYPE = pandas.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
 
         @classmethod
         def __pandas_datetime_type(cls, tz, unit):
@@ -136,41 +147,61 @@ class DataMapping:
                 return cls.__PANDAS_DATETIME_TYPE
             _unit = unit if unit is not None else cls.__TRAC_TIMESTAMP_UNIT
             if tz is None:
-                return pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(_unit).dtype
+                return pandas.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(_unit).dtype
             else:
-                return pd.DatetimeTZDtype(tz=tz, unit=_unit)
+                return pandas.DatetimeTZDtype(tz=tz, unit=_unit)
 
-    # Minimum supported version for Pandas is 1.2, when pd.Float64Dtype was introduced
+    # Minimum supported version for Pandas is 1.2, when pandas.Float64Dtype was introduced
     elif __PANDAS_MAJOR_VERSION == 1 and __PANDAS_MINOR_VERSION >= 2:
 
-        __PANDAS_DATE_TYPE = pd.to_datetime([dt.date(2000, 1, 1)]).dtype
-        __PANDAS_DATETIME_TYPE = pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).dtype
+        __PANDAS_DATE_TYPE = pandas.to_datetime([dt.date(2000, 1, 1)]).dtype
+        __PANDAS_DATETIME_TYPE = pandas.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).dtype
 
         @classmethod
         def __pandas_datetime_type(cls, tz, unit):  # noqa
             if tz is None:
                 return cls.__PANDAS_DATETIME_TYPE
             else:
-                return pd.DatetimeTZDtype(tz=tz)
+                return pandas.DatetimeTZDtype(tz=tz)
 
     else:
-        raise _ex.EStartup(f"Pandas version not supported: [{pd.__version__}]")
+        raise _ex.EStartup(f"Pandas version not supported: [{pandas.__version__}]")
 
     # Only partial mapping is possible, decimal and temporal dtypes cannot be mapped this way
     __ARROW_TO_PANDAS_TYPE_MAPPING = {
-        pa.bool_(): pd.BooleanDtype(),
-        pa.int8(): pd.Int8Dtype(),
-        pa.int16(): pd.Int16Dtype(),
-        pa.int32(): pd.Int32Dtype(),
-        pa.int64(): pd.Int64Dtype(),
-        pa.uint8(): pd.UInt8Dtype(),
-        pa.uint16(): pd.UInt16Dtype(),
-        pa.uint32(): pd.UInt32Dtype(),
-        pa.uint64(): pd.UInt64Dtype(),
-        pa.float16(): pd.Float32Dtype(),
-        pa.float32(): pd.Float32Dtype(),
-        pa.float64(): pd.Float64Dtype(),
-        pa.utf8(): pd.StringDtype()
+        pa.bool_(): pandas.BooleanDtype(),
+        pa.int8(): pandas.Int8Dtype(),
+        pa.int16(): pandas.Int16Dtype(),
+        pa.int32(): pandas.Int32Dtype(),
+        pa.int64(): pandas.Int64Dtype(),
+        pa.uint8(): pandas.UInt8Dtype(),
+        pa.uint16(): pandas.UInt16Dtype(),
+        pa.uint32(): pandas.UInt32Dtype(),
+        pa.uint64(): pandas.UInt64Dtype(),
+        pa.float16(): pandas.Float32Dtype(),
+        pa.float32(): pandas.Float32Dtype(),
+        pa.float64(): pandas.Float64Dtype(),
+        pa.string(): pandas.StringDtype(),
+        pa.utf8(): pandas.StringDtype()
+    }
+
+    __ARROW_TO_TRAC_BASIC_TYPE_MAPPING = {
+        pa.bool_(): _meta.BasicType.BOOLEAN,
+        pa.int8(): _meta.BasicType.INTEGER,
+        pa.int16(): _meta.BasicType.INTEGER,
+        pa.int32(): _meta.BasicType.INTEGER,
+        pa.int64():_meta.BasicType.INTEGER,
+        pa.uint8(): _meta.BasicType.INTEGER,
+        pa.uint16(): _meta.BasicType.INTEGER,
+        pa.uint32(): _meta.BasicType.INTEGER,
+        pa.uint64(): _meta.BasicType.INTEGER,
+        pa.float16(): _meta.BasicType.FLOAT,
+        pa.float32(): _meta.BasicType.FLOAT,
+        pa.float64(): _meta.BasicType.FLOAT,
+        pa.string(): _meta.BasicType.STRING,
+        pa.utf8(): _meta.BasicType.STRING,
+        pa.date32(): _meta.BasicType.DATE,
+        pa.date64(): _meta.BasicType.DATE
     }
 
     @staticmethod
@@ -265,6 +296,47 @@ class DataMapping:
             cls.__TRAC_DECIMAL_PRECISION,
             cls.__TRAC_DECIMAL_SCALE)
 
+    @classmethod
+    def arrow_to_trac_schema(cls, arrow_schema: pa.Schema) -> _meta.SchemaDefinition:
+
+        trac_fields = list(
+            cls.arrow_to_trac_field(i, arrow_schema.field(i))
+            for (i, f) in enumerate(arrow_schema.names))
+
+        return _meta.SchemaDefinition(
+            schemaType=_meta.SchemaType.TABLE,
+            partType=_meta.PartType.PART_ROOT,
+            table=_meta.TableSchema(trac_fields))
+
+    @classmethod
+    def arrow_to_trac_field(cls, field_index: int, field: pa.Field) -> _meta.FieldSchema:
+
+        field_type = cls.arrow_to_trac_type(field.type)
+        label = field.metadata["label"] if field.metadata and "label" in field.metadata else field.name
+
+        return _meta.FieldSchema(
+            field.name, field_index, field_type,
+            label=label,
+            businessKey=False,
+            notNull=not field.nullable,
+            categorical=False)
+
+    @classmethod
+    def arrow_to_trac_type(cls, arrow_type: pa.DataType) -> _meta.BasicType:
+
+        mapped_basic_type = cls.__ARROW_TO_TRAC_BASIC_TYPE_MAPPING.get(arrow_type)  # noqa
+
+        if mapped_basic_type is not None:
+            return mapped_basic_type
+
+        if pa.types.is_decimal(arrow_type):
+            return _meta.BasicType.DECIMAL
+
+        if pa.types.is_timestamp(arrow_type):
+            return _meta.BasicType.DATETIME
+
+        raise _ex.ETracInternal(f"No data type mapping available for Arrow type [{arrow_type}]")
+
     @classmethod
     def pandas_date_type(cls):
         return cls.__PANDAS_DATE_TYPE
@@ -275,18 +347,31 @@ class DataMapping:
 
     @classmethod
     def view_to_pandas(
-            cls, view: DataView, part: DataPartKey, schema: tp.Optional[pa.Schema],
-            temporal_objects_flag: bool) -> pd.DataFrame:
+            cls, view:  DataView,  part: DataPartKey, schema: tp.Optional[pa.Schema],
+            temporal_objects_flag: bool) -> "pandas.DataFrame":
 
         table = cls.view_to_arrow(view, part)
         return cls.arrow_to_pandas(table, schema, temporal_objects_flag)
 
     @classmethod
-    def pandas_to_item(cls, df: pd.DataFrame, schema: tp.Optional[pa.Schema]) -> DataItem:
+    def view_to_polars(
+            cls, view:  DataView, part: DataPartKey, schema: tp.Optional[pa.Schema]):
+
+        table = cls.view_to_arrow(view, part)
+        return cls.arrow_to_polars(table, schema)
+
+    @classmethod
+    def pandas_to_item(cls, df: "pandas.DataFrame", schema: tp.Optional[pa.Schema]) -> DataItem:
 
         table = cls.pandas_to_arrow(df, schema)
         return DataItem(table.schema, table)
 
+    @classmethod
+    def polars_to_item(cls, df: "polars.DataFrame", schema: tp.Optional[pa.Schema]) -> DataItem:
+
+        table = cls.polars_to_arrow(df, schema)
+        return DataItem(table.schema, table)
+
     @classmethod
     def add_item_to_view(cls, view: DataView, part: DataPartKey, item: DataItem) -> DataView:
 
@@ -336,7 +421,7 @@ class DataMapping:
     @classmethod
     def arrow_to_pandas(
             cls, table: pa.Table, schema: tp.Optional[pa.Schema] = None,
-            temporal_objects_flag: bool = False) -> pd.DataFrame:
+            temporal_objects_flag: bool = False) -> "pandas.DataFrame":
 
         if schema is not None:
             table = DataConformance.conform_to_schema(table, schema, warn_extra_columns=False)
@@ -361,7 +446,18 @@ class DataMapping:
             split_blocks=True)  # noqa
 
     @classmethod
-    def pandas_to_arrow(cls, df: pd.DataFrame, schema: tp.Optional[pa.Schema] = None) -> pa.Table:
+    def arrow_to_polars(
+            cls, table: pa.Table, schema: tp.Optional[pa.Schema] = None) -> "polars.DataFrame":
+
+        if schema is not None:
+            table = DataConformance.conform_to_schema(table, schema, warn_extra_columns=False)
+        else:
+            DataConformance.check_duplicate_fields(table.schema.names, False)
+
+        return polars.from_arrow(table)
+
+    @classmethod
+    def pandas_to_arrow(cls, df: "pandas.DataFrame", schema: tp.Optional[pa.Schema] = None) -> pa.Table:
 
         # Converting pandas -> arrow needs care to ensure type coercion is applied correctly
         # Calling Table.from_pandas with the supplied schema will very often reject data
@@ -403,6 +499,30 @@ class DataMapping:
             df_types = df.dtypes.filter(column_filter) if column_filter else df.dtypes
             return DataConformance.conform_to_schema(table, schema, df_types)
 
+    @classmethod
+    def pandas_to_arrow_schema(cls, df: "pandas.DataFrame") -> pa.Schema:
+
+        return pa.Schema.from_pandas(df, preserve_index=False)  # noqa
+
+    @classmethod
+    def polars_to_arrow(cls, df: "polars.DataFrame", schema: tp.Optional[pa.Schema] = None) -> pa.Table:
+
+        column_filter = DataConformance.column_filter(df.columns, schema)
+
+        filtered_df = df.select(polars.col(*column_filter)) if column_filter else df
+        table = filtered_df.to_arrow()
+
+        if schema is None:
+            DataConformance.check_duplicate_fields(table.schema.names, False)
+            return table
+        else:
+            return DataConformance.conform_to_schema(table, schema, None)
+
+    @classmethod
+    def polars_to_arrow_schema(cls, df: "polars.DataFrame") -> pa.Schema:
+
+        return df.top_k(1).to_arrow().schema
+
 
 class DataConformance:
 
@@ -719,21 +839,32 @@ class DataConformance:
     @classmethod
     def _coerce_string(cls, vector: pa.Array, field: pa.Field) -> pa.Array:
 
-        if pa.types.is_string(field.type):
-            if pa.types.is_string(vector.type):
-                return vector
+        try:
 
-        if pa.types.is_large_string(field.type):
-            if pa.types.is_large_string(vector.type):
-                return vector
-            # Allow up-casting string -> large_string
-            if pa.types.is_string(vector.type):
-                return pc.cast(vector, field.type)
+            if pa.types.is_string(field.type):
+                if pa.types.is_string(vector.type):
+                    return vector
+                # Try to down-cast large string -> string, will raise ArrowInvalid if data does not fit
+                if pa.types.is_large_string(vector.type):
+                    return pc.cast(vector, field.type, safe=True)
+
+            if pa.types.is_large_string(field.type):
+                if pa.types.is_large_string(vector.type):
+                    return vector
+                # Allow up-casting string -> large_string
+                if pa.types.is_string(vector.type):
+                    return pc.cast(vector, field.type)
 
-        error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)
-        cls.__log.error(error_message)
+            error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)
+            cls.__log.error(error_message)
+            raise _ex.EDataConformance(error_message)
+
+        except pa.ArrowInvalid as e:
+
+            error_message = cls._format_error(cls.__E_DATA_LOSS_DID_OCCUR, vector, field, e)
+            cls.__log.error(error_message)
+            raise _ex.EDataConformance(error_message) from e
 
-        raise _ex.EDataConformance(error_message)
 
     @classmethod
     def _coerce_date(cls, vector: pa.Array, field: pa.Field, pandas_type=None) -> pa.Array:
@@ -751,7 +882,7 @@ class DataConformance:
         # For Pandas 2.x dates are still np.datetime64 but can be in s, ms, us or ns
         # This conversion will not apply to dates held in Pandas using the Python date object types
         if pandas_type is not None:
-            if pa.types.is_timestamp(vector.type) and pd.api.types.is_datetime64_any_dtype(pandas_type):
+            if pa.types.is_timestamp(vector.type) and pandas.api.types.is_datetime64_any_dtype(pandas_type):
                 return pc.cast(vector, field.type)
 
         error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)

tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py

@@ -16,7 +16,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_do
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\xba\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x42\x16\n\x14schemaTypeDefinition\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*0\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\xb3\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x42\x0f\n\rschemaDetails\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*0\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -26,26 +26,26 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
   _globals['_DATADEFINITION_PARTSENTRY']._options = None
   _globals['_DATADEFINITION_PARTSENTRY']._serialized_options = b'8\001'
-  _globals['_SCHEMATYPE']._serialized_start=1496
-  _globals['_SCHEMATYPE']._serialized_end=1544
-  _globals['_PARTTYPE']._serialized_start=1546
-  _globals['_PARTTYPE']._serialized_end=1609
+  _globals['_SCHEMATYPE']._serialized_start=1489
+  _globals['_SCHEMATYPE']._serialized_end=1537
+  _globals['_PARTTYPE']._serialized_start=1539
+  _globals['_PARTTYPE']._serialized_end=1602
   _globals['_FIELDSCHEMA']._serialized_start=179
   _globals['_FIELDSCHEMA']._serialized_end=410
   _globals['_TABLESCHEMA']._serialized_start=412
   _globals['_TABLESCHEMA']._serialized_end=472
   _globals['_SCHEMADEFINITION']._serialized_start=475
-  _globals['_SCHEMADEFINITION']._serialized_end=661
-  _globals['_PARTKEY']._serialized_start=664
-  _globals['_PARTKEY']._serialized_end=921
-  _globals['_DATADEFINITION']._serialized_start=924
-  _globals['_DATADEFINITION']._serialized_end=1494
-  _globals['_DATADEFINITION_DELTA']._serialized_start=1157
-  _globals['_DATADEFINITION_DELTA']._serialized_end=1202
-  _globals['_DATADEFINITION_SNAP']._serialized_start=1204
-  _globals['_DATADEFINITION_SNAP']._serialized_end=1285
-  _globals['_DATADEFINITION_PART']._serialized_start=1287
-  _globals['_DATADEFINITION_PART']._serialized_end=1390
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=1392
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=1475
+  _globals['_SCHEMADEFINITION']._serialized_end=654
+  _globals['_PARTKEY']._serialized_start=657
+  _globals['_PARTKEY']._serialized_end=914
+  _globals['_DATADEFINITION']._serialized_start=917
+  _globals['_DATADEFINITION']._serialized_end=1487
+  _globals['_DATADEFINITION_DELTA']._serialized_start=1150
+  _globals['_DATADEFINITION_DELTA']._serialized_end=1195
+  _globals['_DATADEFINITION_SNAP']._serialized_start=1197
+  _globals['_DATADEFINITION_SNAP']._serialized_end=1278
+  _globals['_DATADEFINITION_PART']._serialized_start=1280
+  _globals['_DATADEFINITION_PART']._serialized_end=1383
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=1385
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=1468
 # @@protoc_insertion_point(module_scope)