finbourne-sdk-utils 0.0.33__py3-none-any.whl → 0.0.35__py3-none-any.whl

finbourne_sdk_utils/cocoon/properties.py

@@ -203,15 +203,50 @@ def create_property_definitions_from_file(
         # Make the column name LUSID friendly
         lusid_friendly_code = cocoon.utilities.make_code_lusid_friendly(column_name)
 
-        # If there is no data Pandas infers a type of float, would prefer to infer object
+        # Check if column is all nulls
         if missing_property_data_frame[column_name].isnull().all():
             logging.warning(
-                f"{column_name} is null, no type can be inferred it will be treated as a string"
+                f"{column_name} is entirely null, no type can be inferred - will be treated as a string"
             )
             data_type = "object"
             data_frame[column_name] = data_frame[column_name].astype(
                 "object", copy=False
             )
+        else:
+            # Find the first non-null value to infer the proper type
+            first_valid_value = missing_property_data_frame[column_name].dropna().iloc[0]
+            
+            # Infer the type from the first non-null value
+            inferred_type = type(first_valid_value).__name__
+            
+            # Map Python types to pandas dtypes
+            type_mapping = {
+                'str': 'object',
+                'int': 'int64',
+                'float': 'float64',
+                'bool': 'bool',
+                'Timestamp': 'datetime64[ns]',
+            }
+            
+            # Get the appropriate pandas dtype
+            inferred_dtype = type_mapping.get(inferred_type, 'object')
+            
+            # Only override if the current dtype is object and we found something more specific
+            if str(data_type) == 'object' and inferred_dtype != 'object':
+                logging.info(
+                    f"{column_name} has leading nulls - inferring type '{inferred_dtype}' from first non-null value"
+                )
+                data_type = inferred_dtype
+                # Optionally convert the column to the inferred type
+                try:
+                    data_frame[column_name] = data_frame[column_name].astype(
+                        inferred_dtype, copy=False
+                    )
+                except (ValueError, TypeError) as e:
+                    logging.warning(
+                        f"Could not convert {column_name} to {inferred_dtype}, keeping as object. Error: {e}"
+                    )
+                    data_type = 'object'
 
         # Create a request to define the property, assumes value_required is false for all
         property_request = lusid.models.CreatePropertyDefinitionRequest(

{finbourne_sdk_utils-0.0.33.dist-info → finbourne_sdk_utils-0.0.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: finbourne-sdk-utils
-Version: 0.0.33
+Version: 0.0.35
 Summary: Python utilities for LUSID V2 SDK
 Home-page: https://github.com/finbourne/finbourne-sdk-utils
 Author: FINBOURNE Technology
@@ -14,7 +14,7 @@ Requires-Dist: requests>=2.27.1
 Requires-Dist: coloredlogs>=14.0
 Requires-Dist: detect-delimiter>=0.1
 Requires-Dist: flatten-json>=0.1.7
-Requires-Dist: pandas>=1.1.4
+Requires-Dist: pandas<=2.3.3,>=1.1.4
 Requires-Dist: PyYAML>=5.4
 Requires-Dist: tqdm>=4.52.0
 Requires-Dist: openpyxl>=3.0.7

{finbourne_sdk_utils-0.0.33.dist-info → finbourne_sdk_utils-0.0.35.dist-info}/RECORD

@@ -7,7 +7,7 @@ finbourne_sdk_utils/cocoon/cocoon.py,sha256=eecSS8hkF6LyjZpoWhgZqnklrRkJBg4F9_3z
 finbourne_sdk_utils/cocoon/cocoon_printer.py,sha256=bDKHRBJV9s_ivPQ8WvNALwm0Z4lUidW34U--tfppCrY,14046
 finbourne_sdk_utils/cocoon/dateorcutlabel.py,sha256=02MSDQGiSFMycb0SAfHLwMORRiXcXhh39lmaL3TkgMc,5793
 finbourne_sdk_utils/cocoon/instruments.py,sha256=r92uUME_Er0pYiChg2hivW5qQ8oMzPB_E4bong1TeIw,17399
-finbourne_sdk_utils/cocoon/properties.py,sha256=wxGZDopWSg_hruCwADYRuWI9ZiXBvG3q-TAtIjH9aKY,14938
+finbourne_sdk_utils/cocoon/properties.py,sha256=Ggl8C69A8DLpGeRQARMAD-IwnRUPog4TW3R0Is_gCHs,16511
 finbourne_sdk_utils/cocoon/seed_sample_data.py,sha256=SYvuq08j8BbzE3uQMoTmS-xrNtbJ2IRg6KJ1PBm-uVI,4390
 finbourne_sdk_utils/cocoon/systemConfiguration.py,sha256=qheX-jVTaKuD34DSJn57RyTu4jlLMyXjkSYGVe8goIM,3284
 finbourne_sdk_utils/cocoon/transaction_type_upload.py,sha256=xASAzh7ycOzksW--azya-QMtgci48w4phfhyxw7v-gg,4439
@@ -45,8 +45,8 @@ finbourne_sdk_utils/lpt/txn_config.py,sha256=4WIEzypXXuoHdbEDJLl90iZj7dYfV-KaE8d
 finbourne_sdk_utils/lpt/txn_config_yaml.py,sha256=F5kHG-7l5khJvnaLEndHurgmannCi-Pt3U6MMJRuJ9Y,17717
 finbourne_sdk_utils/pandas_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 finbourne_sdk_utils/pandas_utils/lusid_pandas.py,sha256=rJENs5lIYEKcwvYaagKc-ei_3sOnbNZT9oFLuO8PU6Y,4900
-finbourne_sdk_utils-0.0.33.dist-info/LICENSE,sha256=pT32QD9c76jv4OMjXhZrYpjqlZTuiex52qmKUsLKRQc,1066
-finbourne_sdk_utils-0.0.33.dist-info/METADATA,sha256=z1o4qEZd8G3F3zajULuZaXEy0IXmmrzJe0Of7qCPNZ4,731
-finbourne_sdk_utils-0.0.33.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-finbourne_sdk_utils-0.0.33.dist-info/top_level.txt,sha256=DiS6bVuYtHoXKSgWOlYhiXKPVFCJqhdtmQgMCGNZxxY,29
-finbourne_sdk_utils-0.0.33.dist-info/RECORD,,
+finbourne_sdk_utils-0.0.35.dist-info/LICENSE,sha256=pT32QD9c76jv4OMjXhZrYpjqlZTuiex52qmKUsLKRQc,1066
+finbourne_sdk_utils-0.0.35.dist-info/METADATA,sha256=eU1GIrPS7Kgyzeb5RfSGwOvIsLYIO4RZlnBZJuylD7M,739
+finbourne_sdk_utils-0.0.35.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+finbourne_sdk_utils-0.0.35.dist-info/top_level.txt,sha256=DiS6bVuYtHoXKSgWOlYhiXKPVFCJqhdtmQgMCGNZxxY,29
+finbourne_sdk_utils-0.0.35.dist-info/RECORD,,