tracdap-runtime 0.5.28__py3-none-any.whl → 0.5.30__py3-none-any.whl

tracdap/rt/_impl/data.py

@@ -74,11 +74,7 @@ class DataView:
 
 
 class _DataInternal:
-
-    @staticmethod
-    def float_dtype_check():
-        if "Float64Dtype" not in pd.__dict__:
-            raise _ex.EStartup("TRAC D.A.P. requires Pandas >= 1.2")
+    pass
 
 
 class DataMapping:
@@ -111,8 +107,40 @@ class DataMapping:
     }
 
     # Check the Pandas dtypes for handling floats are available before setting up the type mapping
-    __PANDAS_FLOAT_DTYPE_CHECK = _DataInternal.float_dtype_check()
-    __PANDAS_DATETIME_TYPE = pd.to_datetime([]).dtype
+    __PANDAS_VERSION_ELEMENTS = pd.__version__.split(".")
+    __PANDAS_MAJOR_VERSION = int(__PANDAS_VERSION_ELEMENTS[0])
+    __PANDAS_MINOR_VERSION = int(__PANDAS_VERSION_ELEMENTS[1])
+
+    if __PANDAS_MAJOR_VERSION == 2:
+
+        __PANDAS_DATE_TYPE = pd.to_datetime([dt.date(2000, 1, 1)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
+        __PANDAS_DATETIME_TYPE = pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(__TRAC_TIMESTAMP_UNIT).dtype
+
+        @classmethod
+        def __pandas_datetime_type(cls, tz, unit):
+            if tz is None and unit is None:
+                return cls.__PANDAS_DATETIME_TYPE
+            _unit = unit if unit is not None else cls.__TRAC_TIMESTAMP_UNIT
+            if tz is None:
+                return pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).as_unit(_unit).dtype
+            else:
+                return pd.DatetimeTZDtype(tz=tz, unit=_unit)
+
+    # Minimum supported version for Pandas is 1.2, when pd.Float64Dtype was introduced
+    elif __PANDAS_MAJOR_VERSION == 1 and __PANDAS_MINOR_VERSION >= 2:
+
+        __PANDAS_DATE_TYPE = pd.to_datetime([dt.date(2000, 1, 1)]).dtype
+        __PANDAS_DATETIME_TYPE = pd.to_datetime([dt.datetime(2000, 1, 1, 0, 0, 0)]).dtype
+
+        @classmethod
+        def __pandas_datetime_type(cls, tz, unit):  # noqa
+            if tz is None:
+                return cls.__PANDAS_DATETIME_TYPE
+            else:
+                return pd.DatetimeTZDtype(tz=tz)
+
+    else:
+        raise _ex.EStartup(f"Pandas version not supported: [{pd.__version__}]")
 
     # Only partial mapping is possible, decimal and temporal dtypes cannot be mapped this way
     __ARROW_TO_PANDAS_TYPE_MAPPING = {
@@ -224,8 +252,12 @@ class DataMapping:
             cls.__TRAC_DECIMAL_SCALE)
 
     @classmethod
-    def pandas_datetime_type(cls):
-        return cls.__PANDAS_DATETIME_TYPE
+    def pandas_date_type(cls):
+        return cls.__PANDAS_DATE_TYPE
+
+    @classmethod
+    def pandas_datetime_type(cls, tz=None, unit=None):
+        return cls.__pandas_datetime_type(tz, unit)
 
     @classmethod
     def view_to_pandas(
@@ -297,7 +329,8 @@ class DataMapping:
         else:
             DataConformance.check_duplicate_fields(table.schema.names, False)
 
-        return table.to_pandas(
+        # Use Arrow's built-in function to convert to Pandas
+        df_from_arrow = table.to_pandas(
 
             # Mapping for arrow -> pandas types for core types
             types_mapper=cls.__ARROW_TO_PANDAS_TYPE_MAPPING.get,
@@ -313,6 +346,33 @@ class DataMapping:
             # This is a significant performance win for very wide datasets
             split_blocks=True)  # noqa
 
+        # Arrow 12 doesn't support the new precision handling for datetime values supported in Pandas 2
+        # However Arrow 13 dropped support for Python 3.7, which is a requirement for the TRAC 0.5.x series
+        # So to backport Pandas 2 support, special handling is needed for datetime fields when using Pandas 2
+        # This is not needed from TRAC 0.6 onward, which upgrades to Arrow 13 and drops Python 3.7 support
+        # Also it is not needed if the temporal objects flag is set, since it only affects NumPy datetime64
+
+        if cls.__PANDAS_MAJOR_VERSION == 2 and not temporal_objects_flag:
+            # Use table.schema, it is always present and has been normalized if a separate schema was supplied
+            return cls._fix_pandas_2_datetime_precision(df_from_arrow, table.schema)
+        else:
+            return df_from_arrow
+
+    @classmethod
+    def _fix_pandas_2_datetime_precision(cls, df: pd.DataFrame, schema: pa.Schema) -> pd.DataFrame:
+
+        for field in schema:
+            if pa.types.is_date(field.type):
+                dtype = cls.__PANDAS_DATE_TYPE
+                if df[field.name].dtype != dtype:
+                    df[field.name] = df[field.name].astype(dtype)
+            if pa.types.is_timestamp(field.type):
+                dtype = cls.__pandas_datetime_type(field.type.tz, field.type.unit)
+                if df[field.name].dtype != dtype:
+                    df[field.name] = df[field.name].astype(dtype)
+
+        return df
+
     @classmethod
     def pandas_to_arrow(cls, df: pd.DataFrame, schema: tp.Optional[pa.Schema] = None) -> pa.Table:
 
@@ -463,7 +523,7 @@ class DataConformance:
 
                 table_column: pa.Array = table.column(table_index)
 
-                pandas_type = pandas_types[table_index] \
+                pandas_type = pandas_types.iloc[table_index] \
                     if pandas_types is not None \
                     else None
 
@@ -691,16 +751,20 @@ class DataConformance:
     @classmethod
     def _coerce_date(cls, vector: pa.Array, field: pa.Field, pandas_type=None) -> pa.Array:
 
-        # Allow casting date32 -> date64, both range and precision are greater so there is no data loss
+        # The bit-width restriction could be removed here
+        # For date types there is never loss of precision and pa.cast will raise an error on overflow
+        # Impact to client code is unlikely, still this change should happen with a TRAC minor version update
         if pa.types.is_date(vector.type):
             if field.type.bit_width >= vector.type.bit_width:
                 return pc.cast(vector, field.type)
 
-        # Special handling for Pandas/NumPy date values
-        # These are encoded as np.datetime64[ns] in Pandas -> pa.timestamp64[ns] in Arrow
-        # Only allow this conversion if the vector is coming from Pandas with datetime type
-        if pandas_type == DataMapping.pandas_datetime_type():
-            if pa.types.is_timestamp(vector.type) and vector.type.unit == "ns":
+        # Special handling for date values coming from Pandas/NumPy
+        # Only allow these conversions if the vector is supplied with Pandas type info
+        # For Pandas 1.x, dates are always encoded as np.datetime64[ns]
+        # For Pandas 2.x dates are still np.datetime64 but can be in s, ms, us or ns
+        # This conversion will not apply to dates held in Pandas using the Python date object types
+        if pandas_type is not None:
+            if pa.types.is_timestamp(vector.type) and pd.api.types.is_datetime64_any_dtype(pandas_type):
                 return pc.cast(vector, field.type)
 
         error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)

tracdap/rt/_version.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.5.28"
+__version__ = "0.5.30"

tracdap/rt/config/__init__.py

@@ -1,10 +1,5 @@
 # Code generated by TRAC
 
-from .result import TagUpdateList
-from .result import JobResult
-
-from .job import JobConfig
-
 from .common import _ConfigFile
 from .common import PluginConfig
 from .common import PlatformInfo
@@ -22,9 +17,6 @@ from .platform import InstanceConfig
 from .platform import ServiceMap
 from .platform import ServiceConfig
 
-from .runtime import RuntimeConfig
-from .runtime import SparkSettings
-
 from .gateway import GwProtocol
 from .gateway import GwRestMapping
 from .gateway import GatewayConfig
@@ -33,3 +25,11 @@ from .gateway import GwMatch
 from .gateway import GwTarget
 from .gateway import GwServiceMap
 from .gateway import GwService
+
+from .result import TagUpdateList
+from .result import JobResult
+
+from .runtime import RuntimeConfig
+from .runtime import SparkSettings
+
+from .job import JobConfig

tracdap/rt/metadata/__init__.py

@@ -13,13 +13,17 @@ from .type import Value
 from .type import ArrayValue
 from .type import MapValue
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
 from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
+from .search import SearchOperator
+from .search import LogicalOperator
+from .search import SearchTerm
+from .search import LogicalExpression
+from .search import SearchExpression
+from .search import SearchParameters
+
 from .data import SchemaType
 from .data import PartType
 from .data import FieldSchema
@@ -28,18 +32,23 @@ from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
+from .file import FileDefinition
+
+from .stoarge import CopyStatus
+from .stoarge import IncarnationStatus
+from .stoarge import StorageCopy
+from .stoarge import StorageIncarnation
+from .stoarge import StorageItem
+from .stoarge import StorageDefinition
+
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
+
 from .model import ModelParameter
 from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .search import SearchOperator
-from .search import LogicalOperator
-from .search import SearchTerm
-from .search import LogicalExpression
-from .search import SearchExpression
-from .search import SearchParameters
-
 from .flow import FlowNodeType
 from .flow import FlowNode
 from .flow import FlowSocket
@@ -53,17 +62,8 @@ from .job import RunModelJob
 from .job import RunFlowJob
 from .job import ImportModelJob
 
-from .file import FileDefinition
-
 from .custom import CustomDefinition
 
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
-
 from .object import ObjectDefinition
 
 from .tag import Tag

{tracdap_runtime-0.5.28.dist-info → tracdap_runtime-0.5.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracdap-runtime
-Version: 0.5.28
+Version: 0.5.30
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -16,16 +16,17 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <3.12,>=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf (==4.21.10)
-Requires-Dist: pyarrow (==10.0.1)
-Requires-Dist: pyyaml (==6.0.0)
-Requires-Dist: dulwich (==0.21.2)
-Requires-Dist: requests (==2.28.1)
-Requires-Dist: pandas (<1.6.0,>=1.2.0)
+Requires-Dist: protobuf ==4.23.2
+Requires-Dist: pyarrow ==12.0.1
+Requires-Dist: pyyaml ==6.0.0
+Requires-Dist: dulwich ==0.21.2
+Requires-Dist: requests ==2.31.0
+Requires-Dist: pandas <2.2.0,>=1.2.0
+Requires-Dist: numpy <2.0.0
 Provides-Extra: aws
-Requires-Dist: boto3 (==1.26.22) ; extra == 'aws'
+Requires-Dist: boto3 ==1.26.22 ; extra == 'aws'
 Provides-Extra: spark
-Requires-Dist: pyspark (<3.4.0,>=2.4.0) ; extra == 'spark'
+Requires-Dist: pyspark <3.4.0,>=2.4.0 ; extra == 'spark'
 
 # TRAC Model Runtime for Python
 
@@ -46,7 +47,7 @@ Documentation for the TRAC platform is available on our website at
 The TRAC runtime for Python has these requirements:
 
 * Python: 3.7 up to 3.11.x
-* Pandas: 1.2 up to 1.5.x
+* Pandas: 1.2 up to 2.1.x
 * PySpark 2.4.x, or 3.0 up to 3.3.x
 
 Not every combination of versions will work, e.g. PySpark 3 requires Python 3.8.

{tracdap_runtime-0.5.28.dist-info → tracdap_runtime-0.5.30.dist-info}/RECORD

@@ -1,5 +1,5 @@
 tracdap/rt/__init__.py,sha256=rz9ERpKMlnR4LFZNGLtdNE26B_Y2V168bdd8hRmasKk,643
-tracdap/rt/_version.py,sha256=WibObUtJ9RYv5ay-pWo3yOkMVZvwdPKOOw1e_OoUKbQ,632
+tracdap/rt/_version.py,sha256=krI77g3AnXMgUfWuNKj9uat-Gv5TDR8WCP6P9Sb8y4E,632
 tracdap/rt/exceptions.py,sha256=7nLYOOquROJwMBHJAoPHUbmrwp9jkT7LuR6lCsc7Am8,7730
 tracdap/rt/_exec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tracdap/rt/_exec/actors.py,sha256=faZ7hCxsONLbjQKrFRZv-wL_aXaKjsgbLOHDI0uUXDc,32242
@@ -12,7 +12,7 @@ tracdap/rt/_exec/graph_builder.py,sha256=5vDjgLuzt8HrZp0c-U5CQ8dm_cwebEgeYfgR2ht
 tracdap/rt/_exec/runtime.py,sha256=MCfpEBwbh1vs5Ir4vOOK_R2z6XVaK-teAm2Iz3pAmLo,13708
 tracdap/rt/_impl/__init__.py,sha256=eBZMpgFx9uHJxMA-yMWHCQMvShTKxJRAizdz4vy5eWg,609
 tracdap/rt/_impl/config_parser.py,sha256=udnYRafOcP3aQSVXSfjCc70gOEbh7MxJpR1v0wqnBUw,15263
-tracdap/rt/_impl/data.py,sha256=GXe1CSe30G8S4TBOz76oEQVTxrSgUr5AjrnhZVULWYI,30498
+tracdap/rt/_impl/data.py,sha256=I88rzp__hmrae288-oRd8Ozo5Y9BOGODBEWYD3ShWyQ,33763
 tracdap/rt/_impl/guard_rails.py,sha256=eZbhmBvAY-6SP-Y1CM33bMOQu4xLmAjDVctSR-yocZk,10724
 tracdap/rt/_impl/models.py,sha256=BWTL7y1fcSWCOxBn7f4NLpzcNrPABzP9hC7bWTVT2wc,9020
 tracdap/rt/_impl/repos.py,sha256=bdpUkBbNOIQOKJQGOhCDdXUMz2bY91YNIYlKRKTtM4Y,2063
@@ -37,7 +37,7 @@ tracdap/rt/api/__init__.py,sha256=rOiUwK6sav9QTXohpZAFXJ9MgsL0KBfUyKA7dYspfGQ,11
 tracdap/rt/api/hook.py,sha256=3gc90J2mzp27koDysHvl8HwiSePw0hjg1BED3VhZVR4,4033
 tracdap/rt/api/model_api.py,sha256=S2Dz7cre5pFIK3jw8Qlv6D90bv6KSULdbPkMBurEkAM,15402
 tracdap/rt/api/static_api.py,sha256=6wwsF6nQG56NPocAs8VUUN5zL1fVFLDDOvcqW-dbhvo,20725
-tracdap/rt/config/__init__.py,sha256=7O8x6B3InoUk_Ujwy3c6pk50QUAf3hra0yX4zQCMKBs,991
+tracdap/rt/config/__init__.py,sha256=doDFsgsVexgXHsGYtigdIwf_0UfyqZuD3QZOqQGY3GI,991
 tracdap/rt/config/common.py,sha256=p5K6W6PVfTB9Ag3VXsM0HAhk4lW-_owaFmmELsoPkSI,1026
 tracdap/rt/config/common_pb2.py,sha256=2x6--pFrp1pwJkZNwCiBik6kTJ32CdzFD7DjZDmnrBY,3732
 tracdap/rt/config/gateway.py,sha256=Uz1ihOTHGLU0GBBIHAaZCV0ufyyHQJtQnAHeqD1dyOM,1551
@@ -60,7 +60,7 @@ tracdap/rt/launch/__init__.py,sha256=Zz_4f_ODsmweCxRmG2Dq1Slpb927jSugYclfF_Wgfws
 tracdap/rt/launch/__main__.py,sha256=9UVYYSsqvvMVOqjjBBeLNdzV_6IeIa_97KWOMXIpXY4,654
 tracdap/rt/launch/cli.py,sha256=giC30Dffz9eYd-7fDQqTZ65OXDDf0OPmKb6NV_meePE,2196
 tracdap/rt/launch/launch.py,sha256=ap6PWdIS4eg43aCK2A-bdAoXS75ZUIT3Kt-6Uc-4eNw,4497
-tracdap/rt/metadata/__init__.py,sha256=jYz1JNLb3WtIqUjUUjHbt-dhE0Jl_AvsxZzFy-tut-Y,1779
+tracdap/rt/metadata/__init__.py,sha256=hm_6z9Dr8lCyOLzcqo69xtWTg0KkiCKH0I4Mnb1Umlc,1779
 tracdap/rt/metadata/common.py,sha256=HIOphjbX_y7gRpERThb0Hrgiij1qaSc9XmvcWiuE3nI,1453
 tracdap/rt/metadata/common_pb2.py,sha256=l3PJBc6jOoPD8o6KFJMoDYHMMQ5S3HyuTH2lFTfGPF4,1710
 tracdap/rt/metadata/custom.py,sha256=bNG2Fy8uYz1qE8idgN1h4EHkecV95BG0ruEl7XUWvRU,295
@@ -89,8 +89,8 @@ tracdap/rt/metadata/tag_update.py,sha256=gSNG1Nv4mIhcmBcuMa5P6pmuj_R4DwHc9R7riSw
 tracdap/rt/metadata/tag_update_pb2.py,sha256=Fcp59cwjzlPWnjoOxIBxVLA6frEVHhqwg-_NntqLu-Y,1816
 tracdap/rt/metadata/type.py,sha256=a11dBdO8Cp8fkXfS9Xom1IDqz3BQZenzvGY20V855QE,9402
 tracdap/rt/metadata/type_pb2.py,sha256=RooCizEbIXEA_UJ0i1qd2mtBY0SjcA_aJWuJ8qNhNV8,3920
-tracdap_runtime-0.5.28.dist-info/LICENSE,sha256=Q5Gh9SdMNa_F2ehQRShh7dJBz6qW_EQFtWzLukOWFWY,11365
-tracdap_runtime-0.5.28.dist-info/METADATA,sha256=oerdnj0vHdvY3ownivvq8oAPOqdxojfyHS2RAYQmW0g,4129
-tracdap_runtime-0.5.28.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-tracdap_runtime-0.5.28.dist-info/top_level.txt,sha256=Uv0JfaE1Lp4JnCzqW8lqXNJAEcsAFpAUGOghJolVNdM,8
-tracdap_runtime-0.5.28.dist-info/RECORD,,
+tracdap_runtime-0.5.30.dist-info/LICENSE,sha256=Q5Gh9SdMNa_F2ehQRShh7dJBz6qW_EQFtWzLukOWFWY,11365
+tracdap_runtime-0.5.30.dist-info/METADATA,sha256=z5roQQrBYaWoOIAVxt8luqd1KqZ81AnHUiuquhhb7VY,4140
+tracdap_runtime-0.5.30.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+tracdap_runtime-0.5.30.dist-info/top_level.txt,sha256=Uv0JfaE1Lp4JnCzqW8lqXNJAEcsAFpAUGOghJolVNdM,8
+tracdap_runtime-0.5.30.dist-info/RECORD,,

{tracdap_runtime-0.5.28.dist-info → tracdap_runtime-0.5.30.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any