tracdap-runtime 0.5.29__tar.gz → 0.6.0__tar.gz

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracdap-runtime
-Version: 0.5.29
+Version: 0.6.0
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -13,19 +13,31 @@ Platform: any
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.12,>=3.7
+Requires-Python: <3.13,>=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf==4.23.2
-Requires-Dist: pyarrow==12.0.1
-Requires-Dist: pyyaml==6.0.0
-Requires-Dist: dulwich==0.21.2
-Requires-Dist: requests==2.31.0
-Requires-Dist: pandas<2.2.0,>=1.2.0
+Requires-Dist: protobuf==4.25.3
+Requires-Dist: pyarrow==16.0.0
+Requires-Dist: pyyaml==6.0.1
+Requires-Dist: dulwich==0.22.1
+Requires-Dist: requests==2.32.3
+Requires-Dist: pandas<2.3.0,>=1.2.0
+Requires-Dist: numpy<2.0.0
 Provides-Extra: spark
-Requires-Dist: pyspark<3.4.0,>=2.4.0; extra == "spark"
+Requires-Dist: pyspark<3.6.0,>=3.0.0; extra == "spark"
 Provides-Extra: aws
-Requires-Dist: boto3==1.26.22; extra == "aws"
+Requires-Dist: botocore==1.34.93; extra == "aws"
+Requires-Dist: boto3==1.34.93; extra == "aws"
+Provides-Extra: gcp
+Requires-Dist: google-auth==2.29.0; extra == "gcp"
+Requires-Dist: google-cloud-core==2.4.1; extra == "gcp"
+Requires-Dist: google-cloud-storage==2.16.0; extra == "gcp"
+Requires-Dist: gcsfs==2024.3.1; extra == "gcp"
+Provides-Extra: azure
+Requires-Dist: azure-core==1.30.1; extra == "azure"
+Requires-Dist: azure-identity==1.16.1; extra == "azure"
+Requires-Dist: azure-storage-blob==12.19.1; extra == "azure"
+Requires-Dist: adlfs==2024.4.1; extra == "azure"
 
 # TRAC Model Runtime for Python
 
@@ -45,12 +57,12 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.7 up to 3.11.x
-* Pandas: 1.2 up to 2.1.x
-* PySpark 2.4.x, or 3.0 up to 3.3.x
-
-Not every combination of versions will work, e.g. PySpark 3 requires Python 3.8.
+* Python: 3.8 up to 3.12
+* Pandas: 1.2 up to 2.2
+* PySpark 3.0 up to 3.5 (optional)
 
+3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
+As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
 
 ## Installing the runtime
 

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/README.md

@@ -16,12 +16,12 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.7 up to 3.11.x
-* Pandas: 1.2 up to 2.1.x
-* PySpark 2.4.x, or 3.0 up to 3.3.x
-
-Not every combination of versions will work, e.g. PySpark 3 requires Python 3.8.
+* Python: 3.8 up to 3.12
+* Pandas: 1.2 up to 2.2
+* PySpark 3.0 up to 3.5 (optional)
 
+3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
+As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
 
 ## Installing the runtime
 

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tracdap-runtime
-version = 0.5.29
+version = 0.6.0
 description = Runtime package for building models on the TRAC Data & Analytics Platform
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -32,21 +32,33 @@ packages =
 package_dir = 
 	tracdap = src/tracdap
 	tracdap.rt = src/tracdap/rt
-python_requires = >= 3.7, < 3.12
+python_requires = >= 3.8, < 3.13
 install_requires = 
-	protobuf == 4.23.2
-	pyarrow == 12.0.1
-	pyyaml == 6.0.0
-	dulwich == 0.21.2
-	requests == 2.31.0
+	protobuf == 4.25.3
+	pyarrow == 16.0.0
+	pyyaml == 6.0.1
+	dulwich == 0.22.1
+	requests == 2.32.3
 	
-	pandas >= 1.2.0, < 2.2.0
+	pandas >= 1.2.0, < 2.3.0
+	numpy < 2.0.0
 
 [options.extras_require]
 spark = 
-	pyspark >= 2.4.0, < 3.4.0
+	pyspark >= 3.0.0, < 3.6.0
 aws = 
-	boto3 == 1.26.22
+	botocore == 1.34.93
+	boto3 == 1.34.93
+gcp = 
+	google-auth == 2.29.0
+	google-cloud-core == 2.4.1
+	google-cloud-storage == 2.16.0
+	gcsfs == 2024.3.1
+azure = 
+	azure-core == 1.30.1
+	azure-identity == 1.16.1
+	azure-storage-blob == 12.19.1
+	adlfs == 2024.4.1
 
 [egg_info]
 tag_build = 

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/src/tracdap/rt/_exec/dev_mode.py

@@ -648,7 +648,8 @@ class DevModeTranslator:
             x_name = x_orig_path.name
 
             if x_storage.exists(str(x_orig_path.parent)):
-                existing_files = x_storage.ls(str(x_orig_path.parent))
+                listing = x_storage.ls(str(x_orig_path.parent))
+                existing_files = list(map(lambda stat: stat.file_name, listing))
             else:
                 existing_files = []
 

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/src/tracdap/rt/_impl/data.py

@@ -330,7 +330,7 @@ class DataMapping:
             DataConformance.check_duplicate_fields(table.schema.names, False)
 
         # Use Arrow's built-in function to convert to Pandas
-        df_from_arrow = table.to_pandas(
+        return table.to_pandas(
 
             # Mapping for arrow -> pandas types for core types
             types_mapper=cls.__ARROW_TO_PANDAS_TYPE_MAPPING.get,
@@ -346,33 +346,6 @@ class DataMapping:
             # This is a significant performance win for very wide datasets
             split_blocks=True)  # noqa
 
-        # Arrow 12 doesn't support the new precision handling for datetime values supported in Pandas 2
-        # However Arrow 13 dropped support for Python 3.7, which is a requirement for the TRAC 0.5.x series
-        # So to backport Pandas 2 support, special handling is needed for datetime fields when using Pandas 2
-        # This is not needed from TRAC 0.6 onward, which upgrades to Arrow 13 and drops Python 3.7 support
-        # Also it is not needed if the temporal objects flag is set, since it only affects NumPy datetime64
-
-        if cls.__PANDAS_MAJOR_VERSION == 2 and not temporal_objects_flag:
-            # Use table.schema, it is always present and has been normalized if a separate schema was supplied
-            return cls._fix_pandas_2_datetime_precision(df_from_arrow, table.schema)
-        else:
-            return df_from_arrow
-
-    @classmethod
-    def _fix_pandas_2_datetime_precision(cls, df: pd.DataFrame, schema: pa.Schema) -> pd.DataFrame:
-
-        for field in schema:
-            if pa.types.is_date(field.type):
-                dtype = cls.__PANDAS_DATE_TYPE
-                if df[field.name].dtype != dtype:
-                    df[field.name] = df[field.name].astype(dtype)
-            if pa.types.is_timestamp(field.type):
-                dtype = cls.__pandas_datetime_type(field.type.tz, field.type.unit)
-                if df[field.name].dtype != dtype:
-                    df[field.name] = df[field.name].astype(dtype)
-
-        return df
-
     @classmethod
     def pandas_to_arrow(cls, df: pd.DataFrame, schema: tp.Optional[pa.Schema] = None) -> pa.Table:
 

{tracdap-runtime-0.5.29 → tracdap_runtime-0.6.0}/src/tracdap/rt/_impl/static_api.py

@@ -82,7 +82,11 @@ class StaticApiImpl(_StaticApiHook):
             param_type_descriptor = _meta.TypeDescriptor(param_type, None, None)
 
         if default_value is not None and not isinstance(default_value, _meta.Value):
-            default_value = _type_system.MetadataCodec.encode_value(default_value)
+            try:
+                default_value = _type_system.MetadataCodec.convert_value(default_value, param_type_descriptor)
+            except _ex.ETrac as e:
+                msg = f"Default value for parameter [{param_name}] does not match the declared type"
+                raise _ex.EModelValidation(msg) from e
 
         return _Named(param_name, _meta.ModelParameter(param_type_descriptor, label, default_value))