arraymorph 0.2.0b2.dev0__cp314-cp314-macosx_15_0_arm64.whl

arraymorph/__init__.py

@@ -0,0 +1,161 @@
+"""
+ArrayMorph - HDF5 VOL connector for cloud object storage.
+
+Supports AWS S3 and Azure Blob Storage via HDF5's Virtual Object Layer.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+__version__ = "0.2.0"
+
+# The compiled VOL plugin lives next to this file after installation
+_PLUGIN_DIR = str(Path(__file__).parent / "lib")
+
+
+def get_plugin_path() -> str:
+    """Return the directory containing the ArrayMorph VOL plugin (.so/.dylib).
+
+    Use this to set HDF5_PLUGIN_PATH:
+        >>> import arraymorph
+        >>> os.environ["HDF5_PLUGIN_PATH"] = arraymorph.get_plugin_path()
+    """
+    return _PLUGIN_DIR
+
+
+def enable() -> None:
+    """Configure HDF5 environment variables to use ArrayMorph.
+
+    Sets HDF5_PLUGIN_PATH and HDF5_VOL_CONNECTOR so that any
+    subsequent h5py calls route through the ArrayMorph VOL connector.
+
+    Usage:
+        >>> import arraymorph
+        >>> arraymorph.enable()
+        >>> import h5py
+        >>> f = h5py.File("s3://bucket/data.h5", "r")
+    """
+    os.environ["HDF5_PLUGIN_PATH"] = _PLUGIN_DIR
+    os.environ["HDF5_VOL_CONNECTOR"] = "arraymorph"
+
+
+def configure_s3(
+    bucket: str,
+    access_key: str = "",
+    secret_key: str = "",
+    endpoint: str | None = None,
+    region: str = "us-east-2",
+    use_tls: bool = False,
+    addressing_style: bool = False,
+    use_signed_payloads: bool = False,
+) -> None:
+    """Configure AWS S3 credentials and client behavior for ArrayMorph.
+
+    Sets the environment variables read by the VOL connector's S3 client
+    at initialization time. Call this before any h5py file operations.
+
+    Args:
+        bucket: Name of the S3 bucket where HDF5 files are stored.
+            Maps to: BUCKET_NAME
+        access_key: Access key ID for authentication with the S3 service.
+            Maps to: AWS_ACCESS_KEY_ID
+        secret_key: Secret access key paired with access_key for authentication.
+            Maps to: AWS_SECRET_ACCESS_KEY
+        endpoint: Custom S3-compatible endpoint URL (e.g. 'http://localhost:3900').
+            When None, the S3 client targets the default AWS endpoint. Required
+            for any non-AWS S3-compatible object store (MinIO, Ceph, etc.).
+            Maps to: AWS_ENDPOINT_URL_S3
+        region: Region label used in SigV4 request signing. Must match the region
+            your bucket or S3-compatible store is configured with — a mismatch
+            produces signature validation errors. Defaults to 'us-east-2'.
+            Maps to: AWS_REGION
+        use_tls: Whether to use HTTPS (True) or HTTP (False) for S3 connections.
+            Set to False for object stores that do not have TLS configured.
+            Defaults to False.
+            Maps to: AWS_USE_TLS
+        addressing_style: URL addressing style for the S3 client. When True,
+            uses path-style ('endpoint/bucket/key'). When False, uses
+            virtual-hosted style ('bucket.endpoint/key'), which can cause the
+            S3 client to misinterpret the HDF5 filename as the bucket name.
+            Most S3-compatible stores require path-style addressing.
+            Defaults to False.
+            Maps to: AWS_S3_ADDRESSING_STYLE
+        use_signed_payloads: Whether to include the request body in the SigV4
+            signature (PayloadSigningPolicy::Always). Some S3-compatible stores
+            require signed payloads and will reject requests with signature
+            validation errors if this is disabled. Defaults to False.
+            Maps to: AWS_SIGNED_PAYLOADS
+
+    Example:
+        >>> import arraymorph
+        >>> arraymorph.configure_s3(
+        ...     bucket="my-bucket",
+        ...     access_key="my-access-key",
+        ...     secret_key="my-secret-key",
+        ...     endpoint="http://localhost:3900",
+        ...     region="us-east-1",
+        ...     use_tls=False,
+        ...     addressing_style=True,
+        ...     use_signed_payloads=True,
+        ... )
+        >>> arraymorph.enable()
+    """
+    if not (access_key and secret_key):
+        raise ValueError(
+            "configure_s3() requires both 'access_key' and 'secret_key'. "
+            "Set them explicitly or export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY "
+            "before calling this function."
+        )
+
+    os.environ["AWS_ACCESS_KEY_ID"] = access_key
+    os.environ["AWS_SECRET_ACCESS_KEY"] = secret_key
+    os.environ["STORAGE_PLATFORM"] = "S3"
+    os.environ["BUCKET_NAME"] = bucket
+    os.environ["AWS_REGION"] = region
+
+    if endpoint:
+        os.environ["AWS_ENDPOINT_URL_S3"] = endpoint
+
+    os.environ["AWS_USE_TLS"] = str(use_tls).lower()
+    os.environ["AWS_S3_ADDRESSING_STYLE"] = "path" if addressing_style else "virtual"
+    os.environ["AWS_SIGNED_PAYLOADS"] = str(use_signed_payloads).lower()
+
+
+def configure_azure(
+    container: str,
+    connection_string: str | None = None,
+) -> None:
+    """Configure Azure Blob Storage credentials for ArrayMorph.
+
+    Sets the environment variables read by the VOL connector's Azure client
+    at initialization time. Call this before any h5py file operations.
+
+    Args:
+        container: Name of the Azure Blob Storage container where HDF5 files
+            are stored. Maps to: BUCKET_NAME
+        connection_string: Azure Storage connection string used to authenticate
+            and locate the storage account. If None, the connector will fall back
+            to the existing AZURE_STORAGE_CONNECTION_STRING environment variable.
+            Maps to: AZURE_STORAGE_CONNECTION_STRING
+
+    Example:
+        >>> import arraymorph
+        >>> arraymorph.configure_azure(
+        ...     container="my-container",
+        ...     connection_string="DefaultEndpointsProtocol=https;AccountName=...",
+        ... )
+        >>> arraymorph.enable()
+    """
+    if not connection_string and not os.environ.get("AZURE_STORAGE_CONNECTION_STRING"):
+        raise ValueError(
+            "configure_azure() requires a 'connection_string'. "
+            "Set it explicitly or export AZURE_STORAGE_CONNECTION_STRING "
+            "before calling this function."
+        )
+
+    os.environ["STORAGE_PLATFORM"] = "Azure"
+    os.environ["BUCKET_NAME"] = container
+    if connection_string:
+        os.environ["AZURE_STORAGE_CONNECTION_STRING"] = connection_string

arraymorph-0.2.0b2.dev0.dist-info/METADATA

@@ -0,0 +1,142 @@
+Metadata-Version: 2.2
+Name: arraymorph
+Version: 0.2.0b2.dev0
+Summary: HDF5 VOL connector for cloud object storage (AWS S3, Azure Blob)
+Author: ruochenj123, wangtg2013
+License: MIT
+Project-URL: Homepage, https://github.com/ICICLE-ai/ArrayMorph
+Project-URL: Repository, https://github.com/ICICLE-ai/ArrayMorph
+Project-URL: Issues, https://github.com/ICICLE-ai/ArrayMorph/issues
+Requires-Python: >=3.9
+Requires-Dist: h5py>=3.11.0
+Description-Content-Type: text/markdown
+
+# ArrayMorph
+
+[![Build Status](https://github.com/ICICLE-ai/arraymorph/actions/workflows/build.yml/badge.svg)](https://github.com/ICICLE-ai/arraymorph/actions/workflows/build.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+ArrayMorph is a software to manage array data stored on cloud object storage efficiently. It supports both HDF5 C++ API and h5py API. The data returned by h5py API is numpy arrays. By using h5py API, users can access array data stored on the cloud and feed the read data into machine learning pipelines seamlessly.
+
+**Tag**: CI4AI
+
+---
+
+# How-To Guides
+
+## Install dependencies
+
+It is recommended to use Conda (and conda-forge) for managing dependencies.
+
+1. Install [Miniconda](https://docs.anaconda.com/miniconda/)  
+2. Install [conda-build](https://docs.conda.io/projects/conda-build/en/stable/install-conda-build.html) for installing local conda packages
+3. Create and activate environment with dependencies:
+   ```bash
+   conda create -n arraymorph conda-forge::gxx=9
+   conda activate arraymorph
+   conda install -n arraymorph cmake conda-forge::hdf5=1.14.2 conda-forge::aws-sdk-cpp conda-forge::azure-storage-blobs-cpp conda-forge::h5py
+   ```
+
+## Install ArrayMorph via ArrayMorph local conda package
+   ```bash
+   git clone https://github.com/ICICLE-ai/arraymorph.git
+   cd arraymorph/arraymorph_channel
+   conda index .
+   conda install -n arraymorph arraymorph -c file://$(pwd) -c conda-forge
+   ```
+
+## Install ArryMorph from source code
+
+### Build ArrayMorph
+```bash
+git clone https://github.com/ICICLE-ai/arraymorph.git
+cd arraymorph/arraymorph
+cmake -B ./build -S . -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
+cd build
+make
+```
+
+### Enable VOL plugin:
+```bash
+export HDF5_PLUGIN_PATH=/path/to/arraymorph/arraymorph/build/src
+export HDF5_VOL_CONNECTOR=arraymorph
+```
+
+## Configure Environment for Cloud Access
+
+### AWS Configuration:
+```bash
+export STORAGE_PLATFORM=S3
+export BUCKET_NAME=XXXXXX
+export AWS_ACCESS_KEY_ID=XXXXXX
+export AWS_SECRET_ACCESS_KEY=XXXXXX
+export AWS_REGION=us-east-2  # or your bucket's region
+```
+
+### Azure Configuration:
+```bash
+export STORAGE_PLATFORM=Azure
+export BUCKET_NAME=XXXXXX
+export AZURE_STORAGE_CONNECTION_STRING=XXXXXX
+```
+
+---
+
+# Tutorials
+
+## Run a simple example: Writing and Reading HDF5 files from Cloud
+
+### Prerequisites:
+- AWS or Azure cloud account with credentials
+- S3 bucket or Azure container
+- ArrayMorph dependencies installed
+
+### Steps:
+1. Activate conda environment  
+   ```bash
+   conda activate arraymorph
+   ```
+
+2. Write sample HDF5 data to the cloud  
+   ```bash
+   cd examples/python
+   python3 write.py
+   ```
+
+3. Read data back from cloud HDF5 file  
+   ```bash
+   cd examples/python
+   python3 read.py
+   ```
+---
+
+# Explanation
+
+### How ArrayMorph Works
+
+ArrayMorph plugs into the HDF5 stack using a VOL (Virtual Object Layer) plugin that intercepts file operations and routes them to cloud object storage instead of local files. This allows existing HDF5 APIs (both C++ and h5py in Python) to operate on cloud-based data seamlessly, enabling transparent cloud access for scientific or ML pipelines.
+
+It supports:
+- Cloud backends: AWS S3 and Azure Blob
+- File formats: Current binary data stream (we plan to extend to other formats like jpg in the future)
+- Languages: C++ and Python (via h5py compatibility)
+
+The system is designed to be efficient in latency-sensitive scenarios and aims to integrate well with large-scale distributed training and inference.
+
+---
+
+## References
+
+- [HDF5 VOL connectors](https://docs.hdfgroup.org/hdf5/develop/_v_o_l.html)
+- [AWS SDK for C++](https://github.com/aws/aws-sdk-cpp)
+- [Azure SDK for C++](https://github.com/Azure/azure-sdk-for-cpp)
+- [h5py documentation](https://docs.h5py.org/en/stable/)
+- [conda-forge](https://conda-forge.org/)
+
+---
+
+## Acknowledgements
+
+This project is supported by:
+
+*National Science Foundation (NSF) funded AI institute for Intelligent Cyberinfrastructure with Computational Learning in the Environment (ICICLE) (OAC 2112606)*

arraymorph-0.2.0b2.dev0.dist-info/RECORD

@@ -0,0 +1,6 @@
+arraymorph/__init__.py,sha256=Gz0-ZYGOoSpLbsLBlPIiF5buuizi-Q9iq-HHbsWlI44,6257
+arraymorph/lib/lib_arraymorph.dylib,sha256=ujM3DRZPhmlbzC5PIxcOuBkHQayxhezywk6hrYjgevM,14191016
+arraymorph-0.2.0b2.dev0.dist-info/RECORD,,
+arraymorph-0.2.0b2.dev0.dist-info/WHEEL,sha256=nEGB58_ldWTXGbP8ePRzOy9ovTHE0yL0yPMzHJ-Wakw,141
+arraymorph-0.2.0b2.dev0.dist-info/METADATA,sha256=Br1w8Hol-GWPFZLlDrtSuBRA-KoLiz9e198JNo3MPyg,4526
+arraymorph-0.2.0b2.dev0.dist-info/licenses/LICENSE,sha256=dfxSWfn7Ool0X832DV85QnXw3rUwggqXZ1vVDGyxeEA,1145

arraymorph-0.2.0b2.dev0.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 0.12.0
+Root-Is-Purelib: false
+Tag: cp314-cp314-macosx_15_0_arm64
+Generator: delocate 0.13.0
+

arraymorph-0.2.0b2.dev0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Intelligent Cyberinfrastructure with Computational Learning in the Environment -- ICICLE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.