canopy 8.83__tar.gz → 9.0.dev1__tar.gz

{canopy-8.83 → canopy-9.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: canopy
-Version: 8.83
+Version: 9.0.dev1
 Summary: Python Client for the Canopy Simulation API
 License: MIT
 Project-URL: Repository, https://github.com/canopysimulations/canopy-python/
@@ -19,6 +19,8 @@ Requires-Dist: urllib3>=1.23
 Requires-Dist: pandas>=0.25.1
 Requires-Dist: aiohttp
 Requires-Dist: munch
+Requires-Dist: polars
+Dynamic: license-file
 
 # Installation
 
@@ -215,7 +217,7 @@ Proceed? (y/n):
 
 Note: The `openapi/configuration.py` file will need to be manually modified to add the default API host URL.  
 Note: The `openapi_asyncio/rest.py` file will need to be manually modified to support proxy servers after generation.  
-Note: The `openapi_asyncio/client_api.py` and `openapi/client_api.py` files will need to be manually modified to support numpy array serialization after generation.  
+Note: The `openapi_asyncio/api_client.py` and `openapi/api_client.py` files will need to be manually modified to support numpy array serialization after generation.  
 Note: The `availability_api.py`, `membership_api.py` and `study_api.py` files will need reverting to specify 'Bearer' in AuthSettings  
 
 ## Using a local version of the API as source

{canopy-8.83 → canopy-9.0.dev1}/README.md

@@ -193,7 +193,7 @@ Proceed? (y/n):
 
 Note: The `openapi/configuration.py` file will need to be manually modified to add the default API host URL.  
 Note: The `openapi_asyncio/rest.py` file will need to be manually modified to support proxy servers after generation.  
-Note: The `openapi_asyncio/client_api.py` and `openapi/client_api.py` files will need to be manually modified to support numpy array serialization after generation.  
+Note: The `openapi_asyncio/api_client.py` and `openapi/api_client.py` files will need to be manually modified to support numpy array serialization after generation.  
 Note: The `availability_api.py`, `membership_api.py` and `study_api.py` files will need reverting to specify 'Bearer' in AuthSettings  
 
 ## Using a local version of the API as source

{canopy-8.83 → canopy-9.0.dev1}/canopy/__init__.py

@@ -42,7 +42,7 @@ from canopy.load_config import load_config
 from canopy.update_config import update_config
 from canopy.load_study import load_study
 from canopy.load_study_job import load_study_job
-from canopy.load_channel import load_channel
+from canopy.load_channels import load_channels
 from canopy.load_vector_metadata import load_vector_metadata
 from canopy.get_study_document import get_study_document
 from canopy.job_count_to_simulation_count import job_count_to_simulation_count

canopy-9.0.dev1/canopy/load_channels.py

@@ -0,0 +1,135 @@
+import polars as pl
+from typing import List, Optional
+import numpy as np
+import pandas as pd
+import canopy
+import logging
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+
+async def load_channels(
+        session: canopy.Session,
+        job_access_information: canopy.openapi.BlobAccessInformation,
+        sim_type: str,
+        channel_names: List[str],
+        vector_metadata: Optional[pd.DataFrame] = None,
+        semaphore: Optional[asyncio.Semaphore] = None) -> List[Optional[canopy.LoadedChannel]]:
+    sim_type = canopy.ensure_sim_type_string(sim_type)
+
+    if semaphore is None:
+        semaphore = asyncio.Semaphore(session.default_blob_storage_concurrency)
+
+    if vector_metadata is None:
+        vector_metadata = await canopy.load_vector_metadata(session, job_access_information, sim_type)
+
+    if vector_metadata is None:
+        return [None] * len(channel_names)
+
+    # First attempt to load from parquet if available
+    # We group channels by their x-domain to load from the correct parquet files.
+    parquet_results = {}
+    
+    channels_by_x_domain = {}
+    for name in channel_names:
+        if name in vector_metadata.index:
+            x_domain = vector_metadata.at[name, 'xDomainName']
+            if pd.isna(x_domain) or not x_domain:
+                continue
+            
+            if x_domain not in channels_by_x_domain:
+                channels_by_x_domain[x_domain] = []
+            channels_by_x_domain[x_domain].append(name)
+
+    # Try loading from parquet for each x-domain
+    for x_domain, domain_channels in channels_by_x_domain.items():
+        loaded_from_parquet = await _try_load_channels_from_parquet(
+            job_access_information,
+            sim_type,
+            x_domain,
+            domain_channels,
+            vector_metadata)
+        
+        if loaded_from_parquet:
+            for channel in loaded_from_parquet:
+                if channel is not None:
+                    parquet_results[channel.name] = channel
+
+    async def _load_channel(channel_name: str) -> Optional[canopy.LoadedChannel]:
+        if channel_name in parquet_results:
+            return parquet_results[channel_name]
+
+        async with semaphore:
+            if channel_name not in vector_metadata.index:
+                logger.debug('Channel not found: %s', channel_name)
+                return None
+
+            channel_metadata = vector_metadata.xs(channel_name)
+
+            points_count: int = channel_metadata['NPtsInChannel']
+            units: str = channel_metadata['units']
+
+            file_name = ''.join([sim_type, '_', channel_name, '.bin'])
+            channel_url = ''.join([job_access_information.url, file_name, job_access_information.access_signature])
+
+            channel_bytes: Optional[bytes] = await session.try_load_bytes(
+                channel_url,
+                f'"{file_name}" from "{job_access_information.url}"')
+
+            if channel_bytes is None:
+                return None
+
+            if points_count * 4 == len(channel_bytes):
+                data_type = np.float32
+            else:
+                data_type = np.float64
+            channel_data: np.array = np.frombuffer(channel_bytes, data_type)
+
+            loaded_channel = canopy.LoadedChannel(channel_name, units, channel_data)
+            return loaded_channel
+
+    tasks = [asyncio.ensure_future(_load_channel(name)) for name in channel_names]
+    return await asyncio.gather(*tasks)
+
+async def _try_load_channels_from_parquet(
+        job_access_information: canopy.openapi.BlobAccessInformation,
+        sim_type: str,
+        x_domain: str,
+        channel_names: List[str],
+        vector_metadata: pd.DataFrame) -> Optional[List[Optional[canopy.LoadedChannel]]]:
+    file_name = f'{sim_type}_{x_domain}_VectorResults.parquet'
+    url = f'{job_access_information.url}{file_name}{job_access_information.access_signature}'
+
+    try:
+        # We'll use a single scan for all requested channels that exist in metadata.
+        valid_channels = [name for name in channel_names if name in vector_metadata.index]
+        if not valid_channels:
+            return None
+
+        lf = pl.scan_parquet(url, storage_options={ "max_retries": 1, "retry_timeout": 100 })
+        
+        # Check which columns actually exist in the parquet
+        schema = await asyncio.to_thread(lf.collect_schema)
+        available_columns = schema.names()
+        requested_available = [name for name in valid_channels if name in available_columns]
+        
+        if not requested_available:
+            return None
+
+        # Fetch all required columns in one go
+        df: pl.DataFrame = await lf.select(requested_available).collect_async()
+        
+        results: List[Optional[canopy.LoadedChannel]] = []
+        for name in channel_names:
+            if name in requested_available:
+                data: np.ndarray = df.get_column(name).to_numpy()
+                units: str = str(vector_metadata.at[name, 'units'])
+                results.append(canopy.LoadedChannel(name, units, data))
+            else:
+                results.append(None)
+        
+        return results
+    except Exception as e:
+        logger.debug(f"Failed to load channels from parquet {file_name}: {e}")
+        return None

{canopy-8.83 → canopy-9.0.dev1}/canopy/load_study_job.py

@@ -84,20 +84,15 @@ async def load_study_job(
 
             if vector_metadata is not None and channel_names is not None:
                 channels_semaphore = asyncio.Semaphore(session.default_blob_storage_concurrency)
-                tasks: List[Future[Optional[canopy.LoadedChannel]]] = []
-                for channel_name in channel_names:
-                    loaded_channel_task = asyncio.ensure_future(canopy.load_channel(
-                        session,
-                        job_access_information,
-                        sim_type,
-                        channel_name,
-                        vector_metadata=vector_metadata,
-                        semaphore=channels_semaphore))
-
-                    tasks.append(loaded_channel_task)
-
-                for channel_name, task in zip(channel_names, tasks):
-                    loaded_channel = await task
+                loaded_channels = await canopy.load_channels(
+                    session,
+                    job_access_information,
+                    sim_type,
+                    channel_names,
+                    vector_metadata=vector_metadata,
+                    semaphore=channels_semaphore)
+
+                for channel_name, loaded_channel in zip(channel_names, loaded_channels):
                     if loaded_channel is not None:
                         # Convert to a series, which allows the DataFrame to pad with NaNs if
                         # channels happen to be different lengths.

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/canopy_document.py

@@ -36,12 +36,12 @@ class CanopyDocument(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'document_id': 'object',
+        'document_id': 'str',
         'tenant_id': 'str',
         'user_id': 'str',
         'name': 'str',
         'type': 'str',
-        'sub_type': 'object',
+        'sub_type': 'str',
         'sim_version': 'str',
         'creation_date': 'datetime',
         'modified_date': 'datetime',
@@ -50,7 +50,7 @@ class CanopyDocument(object):
         'support_session': 'CanopyDocumentSupportSession',
         'notes': 'str',
         'delete_requested': 'bool',
-        'parent_worksheet_id': 'object',
+        'parent_worksheet_id': 'str',
         'lower_name': 'str'
     }
 
@@ -124,7 +124,7 @@ class CanopyDocument(object):
 
 
         :return: The document_id of this CanopyDocument.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._document_id
 
@@ -134,7 +134,7 @@ class CanopyDocument(object):
 
 
         :param document_id: The document_id of this CanopyDocument.  # noqa: E501
-        :type document_id: object
+        :type document_id: str
         """
 
         self._document_id = document_id
@@ -229,7 +229,7 @@ class CanopyDocument(object):
 
 
         :return: The sub_type of this CanopyDocument.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._sub_type
 
@@ -239,7 +239,7 @@ class CanopyDocument(object):
 
 
         :param sub_type: The sub_type of this CanopyDocument.  # noqa: E501
-        :type sub_type: object
+        :type sub_type: str
         """
 
         self._sub_type = sub_type
@@ -418,7 +418,7 @@ class CanopyDocument(object):
 
 
         :return: The parent_worksheet_id of this CanopyDocument.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._parent_worksheet_id
 
@@ -428,7 +428,7 @@ class CanopyDocument(object):
 
 
         :param parent_worksheet_id: The parent_worksheet_id of this CanopyDocument.  # noqa: E501
-        :type parent_worksheet_id: object
+        :type parent_worksheet_id: str
         """
 
         self._parent_worksheet_id = parent_worksheet_id

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/chart_settings.py

@@ -36,7 +36,7 @@ class ChartSettings(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'chart_type': 'object',
+        'chart_type': 'str',
         'chart_id': 'str',
         'preferred_chart_id': 'str',
         'preferred_chart_user_id': 'str'
@@ -72,7 +72,7 @@ class ChartSettings(object):
 
 
         :return: The chart_type of this ChartSettings.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._chart_type
 
@@ -82,7 +82,7 @@ class ChartSettings(object):
 
 
         :param chart_type: The chart_type of this ChartSettings.  # noqa: E501
-        :type chart_type: object
+        :type chart_type: str
         """
 
         self._chart_type = chart_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_column_label_definitions.py

@@ -36,7 +36,7 @@ class ConfigColumnLabelDefinitions(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'config_type': 'object',
+        'config_type': 'str',
         'labels': 'list[LabelDefinition]'
     }
 
@@ -64,7 +64,7 @@ class ConfigColumnLabelDefinitions(object):
 
 
         :return: The config_type of this ConfigColumnLabelDefinitions.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -74,7 +74,7 @@ class ConfigColumnLabelDefinitions(object):
 
 
         :param config_type: The config_type of this ConfigColumnLabelDefinitions.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_post_config_request.py

@@ -37,12 +37,12 @@ class ConfigPostConfigRequest(object):
     """
     openapi_types = {
         'name': 'str',
-        'config_type': 'object',
+        'config_type': 'str',
         'properties': 'list[DocumentCustomPropertyData]',
         'config': 'object',
         'notes': 'str',
         'sim_version': 'str',
-        'parent_worksheet_id': 'object'
+        'parent_worksheet_id': 'str'
     }
 
     attribute_map = {
@@ -108,7 +108,7 @@ class ConfigPostConfigRequest(object):
         The type of the config.  # noqa: E501
 
         :return: The config_type of this ConfigPostConfigRequest.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -119,7 +119,7 @@ class ConfigPostConfigRequest(object):
         The type of the config.  # noqa: E501
 
         :param config_type: The config_type of this ConfigPostConfigRequest.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type
@@ -223,7 +223,7 @@ class ConfigPostConfigRequest(object):
         The ID of the parent worksheet for the config.  # noqa: E501
 
         :return: The parent_worksheet_id of this ConfigPostConfigRequest.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._parent_worksheet_id
 
@@ -234,7 +234,7 @@ class ConfigPostConfigRequest(object):
         The ID of the parent worksheet for the config.  # noqa: E501
 
         :param parent_worksheet_id: The parent_worksheet_id of this ConfigPostConfigRequest.  # noqa: E501
-        :type parent_worksheet_id: object
+        :type parent_worksheet_id: str
         """
 
         self._parent_worksheet_id = parent_worksheet_id

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_put_config_request.py

@@ -37,7 +37,7 @@ class ConfigPutConfigRequest(object):
     """
     openapi_types = {
         'name': 'str',
-        'config_type': 'object',
+        'config_type': 'str',
         'properties': 'list[DocumentCustomPropertyData]',
         'config': 'object',
         'notes': 'str',
@@ -104,7 +104,7 @@ class ConfigPutConfigRequest(object):
         The config type.  # noqa: E501
 
         :return: The config_type of this ConfigPutConfigRequest.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -115,7 +115,7 @@ class ConfigPutConfigRequest(object):
         The config type.  # noqa: E501
 
         :param config_type: The config_type of this ConfigPutConfigRequest.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_reference_tenant.py

@@ -37,7 +37,7 @@ class ConfigReferenceTenant(object):
     """
     openapi_types = {
         'tenant_id': 'str',
-        'target_id': 'object',
+        'target_id': 'str',
         'job_index': 'int'
     }
 
@@ -89,7 +89,7 @@ class ConfigReferenceTenant(object):
 
 
         :return: The target_id of this ConfigReferenceTenant.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._target_id
 
@@ -99,7 +99,7 @@ class ConfigReferenceTenant(object):
 
 
         :param target_id: The target_id of this ConfigReferenceTenant.  # noqa: E501
-        :type target_id: object
+        :type target_id: str
         """
 
         self._target_id = target_id

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_resolved_reference_data.py

@@ -39,7 +39,7 @@ class ConfigResolvedReferenceData(object):
         'modified_date': 'datetime',
         'user_id': 'str',
         'name': 'str',
-        'config_type': 'object',
+        'config_type': 'str',
         'hashes': 'list[ConfigHash]',
         'is_support_session_open': 'bool'
     }
@@ -144,7 +144,7 @@ class ConfigResolvedReferenceData(object):
 
 
         :return: The config_type of this ConfigResolvedReferenceData.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -154,7 +154,7 @@ class ConfigResolvedReferenceData(object):
 
 
         :param config_type: The config_type of this ConfigResolvedReferenceData.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_type_definition.py

@@ -36,7 +36,7 @@ class ConfigTypeDefinition(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'config_type': 'object',
+        'config_type': 'str',
         'state': 'ConfigTypeState'
     }
 
@@ -64,7 +64,7 @@ class ConfigTypeDefinition(object):
 
 
         :return: The config_type of this ConfigTypeDefinition.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -74,7 +74,7 @@ class ConfigTypeDefinition(object):
 
 
         :param config_type: The config_type of this ConfigTypeDefinition.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_type_metadata.py

@@ -36,7 +36,7 @@ class ConfigTypeMetadata(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'singular_key': 'object',
+        'singular_key': 'str',
         'plural_key': 'str',
         'name': 'str',
         'title_name': 'str',
@@ -76,7 +76,7 @@ class ConfigTypeMetadata(object):
 
 
         :return: The singular_key of this ConfigTypeMetadata.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._singular_key
 
@@ -86,7 +86,7 @@ class ConfigTypeMetadata(object):
 
 
         :param singular_key: The singular_key of this ConfigTypeMetadata.  # noqa: E501
-        :type singular_key: object
+        :type singular_key: str
         """
 
         self._singular_key = singular_key

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/config_upgrade_config_deprecated_request.py

@@ -36,7 +36,7 @@ class ConfigUpgradeConfigDeprecatedRequest(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'config_type': 'object',
+        'config_type': 'str',
         'config': 'object',
         'sim_version': 'str'
     }
@@ -69,7 +69,7 @@ class ConfigUpgradeConfigDeprecatedRequest(object):
         The type of the config document.  # noqa: E501
 
         :return: The config_type of this ConfigUpgradeConfigDeprecatedRequest.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -80,7 +80,7 @@ class ConfigUpgradeConfigDeprecatedRequest(object):
         The type of the config document.  # noqa: E501
 
         :param config_type: The config_type of this ConfigUpgradeConfigDeprecatedRequest.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/default_config_id.py

@@ -36,7 +36,7 @@ class DefaultConfigId(object):
                             and the value is json key in definition.
     """
     openapi_types = {
-        'config_type': 'object',
+        'config_type': 'str',
         'name': 'str'
     }
 
@@ -64,7 +64,7 @@ class DefaultConfigId(object):
 
 
         :return: The config_type of this DefaultConfigId.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_type
 
@@ -74,7 +74,7 @@ class DefaultConfigId(object):
 
 
         :param config_type: The config_type of this DefaultConfigId.  # noqa: E501
-        :type config_type: object
+        :type config_type: str
         """
 
         self._config_type = config_type

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/document_name_result.py

@@ -37,7 +37,7 @@ class DocumentNameResult(object):
     """
     openapi_types = {
         'name': 'str',
-        'config_id': 'object',
+        'config_id': 'str',
         'user_id': 'str'
     }
 
@@ -89,7 +89,7 @@ class DocumentNameResult(object):
 
 
         :return: The config_id of this DocumentNameResult.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._config_id
 
@@ -99,7 +99,7 @@ class DocumentNameResult(object):
 
 
         :param config_id: The config_id of this DocumentNameResult.  # noqa: E501
-        :type config_id: object
+        :type config_id: str
         """
 
         self._config_id = config_id

{canopy-8.83 → canopy-9.0.dev1}/canopy/openapi/models/duplicate_configs_data.py

@@ -37,8 +37,8 @@ class DuplicateConfigsData(object):
     """
     openapi_types = {
         'source_tenant_id': 'str',
-        'source_worksheet_id': 'object',
-        'source_config_ids': 'list[object]',
+        'source_worksheet_id': 'str',
+        'source_config_ids': 'list[str]',
         'source_default_config_ids': 'list[DefaultConfigId]'
     }
 
@@ -96,7 +96,7 @@ class DuplicateConfigsData(object):
         The ID of the source worksheet.  # noqa: E501
 
         :return: The source_worksheet_id of this DuplicateConfigsData.  # noqa: E501
-        :rtype: object
+        :rtype: str
         """
         return self._source_worksheet_id
 
@@ -107,7 +107,7 @@ class DuplicateConfigsData(object):
         The ID of the source worksheet.  # noqa: E501
 
         :param source_worksheet_id: The source_worksheet_id of this DuplicateConfigsData.  # noqa: E501
-        :type source_worksheet_id: object
+        :type source_worksheet_id: str
         """
 
         self._source_worksheet_id = source_worksheet_id
@@ -119,7 +119,7 @@ class DuplicateConfigsData(object):
         The IDs of the source configs.  # noqa: E501
 
         :return: The source_config_ids of this DuplicateConfigsData.  # noqa: E501
-        :rtype: list[object]
+        :rtype: list[str]
         """
         return self._source_config_ids
 
@@ -130,7 +130,7 @@ class DuplicateConfigsData(object):
         The IDs of the source configs.  # noqa: E501
 
         :param source_config_ids: The source_config_ids of this DuplicateConfigsData.  # noqa: E501
-        :type source_config_ids: list[object]
+        :type source_config_ids: list[str]
         """
 
         self._source_config_ids = source_config_ids