canopy 8.86__tar.gz → 9.0.dev1__tar.gz

{canopy-8.86 → canopy-9.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: canopy
-Version: 8.86
+Version: 9.0.dev1
 Summary: Python Client for the Canopy Simulation API
 License: MIT
 Project-URL: Repository, https://github.com/canopysimulations/canopy-python/
@@ -19,6 +19,8 @@ Requires-Dist: urllib3>=1.23
 Requires-Dist: pandas>=0.25.1
 Requires-Dist: aiohttp
 Requires-Dist: munch
+Requires-Dist: polars
+Dynamic: license-file
 
 # Installation
 

{canopy-8.86 → canopy-9.0.dev1}/canopy/__init__.py

@@ -42,7 +42,7 @@ from canopy.load_config import load_config
 from canopy.update_config import update_config
 from canopy.load_study import load_study
 from canopy.load_study_job import load_study_job
-from canopy.load_channel import load_channel
+from canopy.load_channels import load_channels
 from canopy.load_vector_metadata import load_vector_metadata
 from canopy.get_study_document import get_study_document
 from canopy.job_count_to_simulation_count import job_count_to_simulation_count

canopy-9.0.dev1/canopy/load_channels.py

@@ -0,0 +1,135 @@
+import polars as pl
+from typing import List, Optional
+import numpy as np
+import pandas as pd
+import canopy
+import logging
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+
+async def load_channels(
+        session: canopy.Session,
+        job_access_information: canopy.openapi.BlobAccessInformation,
+        sim_type: str,
+        channel_names: List[str],
+        vector_metadata: Optional[pd.DataFrame] = None,
+        semaphore: Optional[asyncio.Semaphore] = None) -> List[Optional[canopy.LoadedChannel]]:
+    sim_type = canopy.ensure_sim_type_string(sim_type)
+
+    if semaphore is None:
+        semaphore = asyncio.Semaphore(session.default_blob_storage_concurrency)
+
+    if vector_metadata is None:
+        vector_metadata = await canopy.load_vector_metadata(session, job_access_information, sim_type)
+
+    if vector_metadata is None:
+        return [None] * len(channel_names)
+
+    # First attempt to load from parquet if available
+    # We group channels by their x-domain to load from the correct parquet files.
+    parquet_results = {}
+    
+    channels_by_x_domain = {}
+    for name in channel_names:
+        if name in vector_metadata.index:
+            x_domain = vector_metadata.at[name, 'xDomainName']
+            if pd.isna(x_domain) or not x_domain:
+                continue
+            
+            if x_domain not in channels_by_x_domain:
+                channels_by_x_domain[x_domain] = []
+            channels_by_x_domain[x_domain].append(name)
+
+    # Try loading from parquet for each x-domain
+    for x_domain, domain_channels in channels_by_x_domain.items():
+        loaded_from_parquet = await _try_load_channels_from_parquet(
+            job_access_information,
+            sim_type,
+            x_domain,
+            domain_channels,
+            vector_metadata)
+        
+        if loaded_from_parquet:
+            for channel in loaded_from_parquet:
+                if channel is not None:
+                    parquet_results[channel.name] = channel
+
+    async def _load_channel(channel_name: str) -> Optional[canopy.LoadedChannel]:
+        if channel_name in parquet_results:
+            return parquet_results[channel_name]
+
+        async with semaphore:
+            if channel_name not in vector_metadata.index:
+                logger.debug('Channel not found: %s', channel_name)
+                return None
+
+            channel_metadata = vector_metadata.xs(channel_name)
+
+            points_count: int = channel_metadata['NPtsInChannel']
+            units: str = channel_metadata['units']
+
+            file_name = ''.join([sim_type, '_', channel_name, '.bin'])
+            channel_url = ''.join([job_access_information.url, file_name, job_access_information.access_signature])
+
+            channel_bytes: Optional[bytes] = await session.try_load_bytes(
+                channel_url,
+                f'"{file_name}" from "{job_access_information.url}"')
+
+            if channel_bytes is None:
+                return None
+
+            if points_count * 4 == len(channel_bytes):
+                data_type = np.float32
+            else:
+                data_type = np.float64
+            channel_data: np.array = np.frombuffer(channel_bytes, data_type)
+
+            loaded_channel = canopy.LoadedChannel(channel_name, units, channel_data)
+            return loaded_channel
+
+    tasks = [asyncio.ensure_future(_load_channel(name)) for name in channel_names]
+    return await asyncio.gather(*tasks)
+
+async def _try_load_channels_from_parquet(
+        job_access_information: canopy.openapi.BlobAccessInformation,
+        sim_type: str,
+        x_domain: str,
+        channel_names: List[str],
+        vector_metadata: pd.DataFrame) -> Optional[List[Optional[canopy.LoadedChannel]]]:
+    file_name = f'{sim_type}_{x_domain}_VectorResults.parquet'
+    url = f'{job_access_information.url}{file_name}{job_access_information.access_signature}'
+
+    try:
+        # We'll use a single scan for all requested channels that exist in metadata.
+        valid_channels = [name for name in channel_names if name in vector_metadata.index]
+        if not valid_channels:
+            return None
+
+        lf = pl.scan_parquet(url, storage_options={ "max_retries": 1, "retry_timeout": 100 })
+        
+        # Check which columns actually exist in the parquet
+        schema = await asyncio.to_thread(lf.collect_schema)
+        available_columns = schema.names()
+        requested_available = [name for name in valid_channels if name in available_columns]
+        
+        if not requested_available:
+            return None
+
+        # Fetch all required columns in one go
+        df: pl.DataFrame = await lf.select(requested_available).collect_async()
+        
+        results: List[Optional[canopy.LoadedChannel]] = []
+        for name in channel_names:
+            if name in requested_available:
+                data: np.ndarray = df.get_column(name).to_numpy()
+                units: str = str(vector_metadata.at[name, 'units'])
+                results.append(canopy.LoadedChannel(name, units, data))
+            else:
+                results.append(None)
+        
+        return results
+    except Exception as e:
+        logger.debug(f"Failed to load channels from parquet {file_name}: {e}")
+        return None

{canopy-8.86 → canopy-9.0.dev1}/canopy/load_study_job.py

@@ -84,20 +84,15 @@ async def load_study_job(
 
             if vector_metadata is not None and channel_names is not None:
                 channels_semaphore = asyncio.Semaphore(session.default_blob_storage_concurrency)
-                tasks: List[Future[Optional[canopy.LoadedChannel]]] = []
-                for channel_name in channel_names:
-                    loaded_channel_task = asyncio.ensure_future(canopy.load_channel(
-                        session,
-                        job_access_information,
-                        sim_type,
-                        channel_name,
-                        vector_metadata=vector_metadata,
-                        semaphore=channels_semaphore))
-
-                    tasks.append(loaded_channel_task)
-
-                for channel_name, task in zip(channel_names, tasks):
-                    loaded_channel = await task
+                loaded_channels = await canopy.load_channels(
+                    session,
+                    job_access_information,
+                    sim_type,
+                    channel_names,
+                    vector_metadata=vector_metadata,
+                    semaphore=channels_semaphore)
+
+                for channel_name, loaded_channel in zip(channel_names, loaded_channels):
                     if loaded_channel is not None:
                         # Convert to a series, which allows the DataFrame to pad with NaNs if
                         # channels happen to be different lengths.

{canopy-8.86 → canopy-9.0.dev1}/canopy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: canopy
-Version: 8.86
+Version: 9.0.dev1
 Summary: Python Client for the Canopy Simulation API
 License: MIT
 Project-URL: Repository, https://github.com/canopysimulations/canopy-python/
@@ -19,6 +19,8 @@ Requires-Dist: urllib3>=1.23
 Requires-Dist: pandas>=0.25.1
 Requires-Dist: aiohttp
 Requires-Dist: munch
+Requires-Dist: polars
+Dynamic: license-file
 
 # Installation
 

{canopy-8.86 → canopy-9.0.dev1}/canopy.egg-info/SOURCES.txt

@@ -28,7 +28,7 @@ canopy/get_study_document.py
 canopy/get_study_type_definition_for_sim_version.py
 canopy/get_study_type_definition_for_sim_version_test.py
 canopy/job_count_to_simulation_count.py
-canopy/load_channel.py
+canopy/load_channels.py
 canopy/load_config.py
 canopy/load_default_config.py
 canopy/load_study.py

{canopy-8.86 → canopy-9.0.dev1}/canopy.egg-info/requires.txt

@@ -6,3 +6,4 @@ urllib3>=1.23
 pandas>=0.25.1
 aiohttp
 munch
+polars

{canopy-8.86 → canopy-9.0.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "canopy"
-version = "8.86"
+version = "9.0.dev1"
 description = "Python Client for the Canopy Simulation API"
 readme = "README.md"
 license = {text = "MIT"}
@@ -23,7 +23,8 @@ dependencies = [
     "urllib3>=1.23",
     "pandas>=0.25.1",
     "aiohttp",
-    "munch"
+    "munch",
+    "polars"
 ]
 
 [tool.setuptools]

canopy-8.86/canopy/load_channel.py

@@ -1,67 +0,0 @@
-from threading import Thread
-from typing import Optional, Union
-from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError, ServerTimeoutError, ClientError
-
-import numpy as np
-import pandas as pd
-import canopy
-import logging
-import asyncio
-
-logger = logging.getLogger(__name__)
-
-
-async def load_channel(
-        session: canopy.Session,
-        job_access_information: canopy.openapi.BlobAccessInformation,
-        sim_type: str,
-        channel_name: str,
-        vector_metadata: Optional[pd.DataFrame] = None,
-        semaphore: Optional[asyncio.Semaphore] = None) -> Optional[canopy.LoadedChannel]:
-    sim_type = canopy.ensure_sim_type_string(sim_type)
-
-    if semaphore is None:
-        semaphore = asyncio.Semaphore(1)
-
-    async with semaphore:
-        if vector_metadata is None:
-            vector_metadata = await canopy.load_vector_metadata(session, job_access_information, sim_type)
-
-        if vector_metadata is None:
-            return None
-
-        if channel_name not in vector_metadata.index:
-            logger.debug('Channel not found: %s', channel_name)
-            return None
-
-        channel_metadata = vector_metadata.xs(channel_name)
-
-        points_count: int = channel_metadata['NPtsInChannel']
-        units: str = channel_metadata['units']
-
-        file_name = ''.join([sim_type, '_', channel_name, '.bin'])
-        channel_url = ''.join([job_access_information.url, file_name, job_access_information.access_signature])
-
-        channel_bytes: Optional[bytes] = await session.try_load_bytes(
-            channel_url,
-            f'"{file_name}" from "{job_access_information.url}"')
-
-        if channel_bytes is None:
-            return None
-
-        if points_count * 4 == len(channel_bytes):
-            data_type = np.float32
-        else:
-            data_type = np.float64
-        channel_data: np.array = np.frombuffer(channel_bytes, data_type)
-
-        loaded_channel = canopy.LoadedChannel(channel_name, units, channel_data)
-        return loaded_channel
-
-
-async def read_channel_data(channel_url: str, session: canopy.Session) -> bytes:
-    async with session.async_client_session.get(
-            channel_url,
-            raise_for_status=True,
-            proxy=session.configuration.proxy) as response:
-        return await response.read()