tracdap-runtime 0.5.30__py3-none-any.whl → 0.6.0.dev1__py3-none-any.whl

tracdap/rt/_impl/util.py

@@ -145,6 +145,28 @@ def logger_for_namespace(namespace: str) -> logging.Logger:
     return logging.getLogger(namespace)
 
 
+def format_file_size(size: int) -> str:
+
+    if size < 1024:
+        if size == 0:
+            return "0 bytes"
+        elif size == 1:
+            return "1 byte"
+        else:
+            return f"{size} bytes"
+
+    if size < 1024 ** 2:
+        kb = size / 1024
+        return f"{kb:.1f} KB"
+
+    if size < 1024 ** 3:
+        mb = size / (1024 ** 2)
+        return f"{mb:.1f} MB"
+
+    gb = size / (1024 ** 3)
+    return f"{gb:.1f} GB"
+
+
 def new_object_id(object_type: meta.ObjectType) -> meta.TagHeader:
 
     timestamp = dt.datetime.utcnow()
@@ -296,7 +318,7 @@ def error_details_from_exception(error: Exception):
 
 def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pathlib.Path):
 
-    frame_names = list(map(lambda frame: frame.name, full_stack))
+    frame_names = list(map(lambda frame_: frame_.name, full_stack))
 
     if __FIRST_MODEL_FRAME_NAME in frame_names:
         first_model_frame = frame_names.index(__FIRST_MODEL_FRAME_NAME)
@@ -309,7 +331,7 @@ def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pa
 
     for frame_index, frame in enumerate(full_stack[first_model_frame:]):
         module_path = pathlib.Path(frame.filename)
-        if ("tracdap" in module_path.parts):
+        if "tracdap" in module_path.parts:
             tracdap_index = len(module_path.parts) - 1 - list(reversed(module_path.parts)).index("tracdap")
             if tracdap_index < len(module_path.parts)-1:
                 if module_path.parts[tracdap_index+1] == "rt":
@@ -322,4 +344,3 @@ def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pa
         last_model_frame = first_model_frame + frame_index
 
     return full_stack[first_model_frame:last_model_frame+1]
-

tracdap/rt/_plugins/_helpers.py

@@ -22,6 +22,8 @@ import platform
 import urllib.parse
 import typing as tp
 
+import tracdap.rt.exceptions as _ex
+
 
 def get_plugin_property(properties: tp.Dict[str, str], property_name: str):
 
@@ -40,6 +42,30 @@ def get_plugin_property(properties: tp.Dict[str, str], property_name: str):
     return None
 
 
+def get_plugin_property_boolean(properties: tp.Dict[str, str], property_name: str, property_default: bool = False):
+
+    property_value = get_plugin_property(properties, property_name)
+
+    if property_value is None:
+        return property_default
+
+    if isinstance(property_value, bool):
+        return property_value
+
+    if isinstance(property_value, str):
+
+        if len(property_value.strip()) == 0:
+            return property_default
+
+        if property_value.strip().lower() == "true":
+            return True
+
+        if property_value.strip().lower() == "false":
+            return False
+
+    raise _ex.EConfigParse(f"Invalid value for [{property_name}]: Expected a boolean value, got [{property_value}]")
+
+
 # Handling for credentials supplied via HTTP(S) URLs
 
 __HTTP_TOKEN_KEY = "token"
@@ -98,31 +124,6 @@ def apply_http_credentials(url: urllib.parse.ParseResult, credentials: str) -> u
 
 # Logging helpers
 
-_T = tp.TypeVar("_T")
-
-
-class _LogClose(tp.Generic[_T]):
-
-    def __init__(self, ctx_mgr: _T, log, msg):
-        self.__ctx_mgr = ctx_mgr
-        self.__log = log
-        self.__msg = msg
-
-    def __getitem__(self, item):
-        return self.__ctx_mgr.__getitem__(item)
-
-    def __enter__(self):
-        return self.__ctx_mgr.__enter__()
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.__ctx_mgr.__exit__(exc_type, exc_val, exc_tb)
-        self.__log.info(self.__msg)
-
-
-def log_close(ctx_mgg: _T, log: logging.Logger, msg: str) -> _T:
-
-    return _LogClose(ctx_mgg, log, msg)
-
 
 def log_safe(param: tp.Any):
 

tracdap/rt/_plugins/storage_aws.py

@@ -23,31 +23,26 @@ import tracdap.rt.exceptions as ex
 import tracdap.rt.ext.plugins as plugins
 from tracdap.rt.ext.storage import *
 
+from pyarrow import fs as afs
+
 # Set of common helpers across the core plugins (do not reference rt._impl)
 from . import _helpers
 
-# TODO: Remove dependencies on internal implementation details
-import tracdap.rt._impl.storage as _storage
-
-# AWS SDK
-import boto3
-import botocore.response  # noqa
-import botocore.exceptions as aws_ex  # noqa
-
-
-class S3ObjectStorage(IFileStorage):
-
-    # This is a quick implementation of IFileStorage on S3 using the boto3 AWS SDK
 
-    # TODO: Migrate IFileStorage interface to use object storage as the primary concept
-    # It is much easier to express objects on a file system than vice versa
-    # This change must also be made in the Java code
+try:
+    # AWS SDK
+    import boto3
+    import botocore.response
+    import botocore.exceptions as aws_ex
+    __aws_available = True
+except ImportError:
+    boto3 = None
+    botocore = None
+    aws_ex = None
+    __aws_available = False
 
-    # TODO: Switch to using Apache Arrow file system interface
-    # Arrow already has implementations for AWS, GCP, HDFS and local files
-    # The arrow interface also allows extension with fsspec, to support Azure blob storage or custom implementations
 
-    # https://arrow.apache.org/docs/python/filesystems.html
+class AwsStorageProvider(IStorageProvider):
 
     BUCKET_PROPERTY = "bucket"
     PREFIX_PROPERTY = "prefix"
@@ -61,38 +56,106 @@ class S3ObjectStorage(IFileStorage):
     ACCESS_KEY_ID_PROPERTY = "accessKeyId"
     SECRET_ACCESS_KEY_PROPERTY = "secretAccessKey"
 
-    def __init__(self, config: cfg.PluginConfig, options: dict = None):
+    RUNTIME_FS_PROPERTY = "runtimeFs"
+    RUNTIME_FS_AUTO = "auto"
+    RUNTIME_FS_ARROW = "arrow"
+    RUNTIME_FS_BOTO3 = "boto3"
+    RUNTIME_FS_DEFAULT = RUNTIME_FS_AUTO
+
+    ARROW_CLIENT_ARGS = {
+        REGION_PROPERTY: "region",
+        ENDPOINT_PROPERTY: "endpoint_override",
+        ACCESS_KEY_ID_PROPERTY: "access_key",
+        SECRET_ACCESS_KEY_PROPERTY: "secret_key"
+    }
+
+    BOTO_CLIENT_ARGS = {
+        REGION_PROPERTY: "region_name",
+        ENDPOINT_PROPERTY: "endpoint_url",
+        ACCESS_KEY_ID_PROPERTY: "aws_access_key_id",
+        SECRET_ACCESS_KEY_PROPERTY: "aws_secret_access_key"
+    }
+
+    def __init__(self, properties: tp.Dict[str, str]):
 
         self._log = _helpers.logger_for_object(self)
+        self._properties = properties
 
-        self._properties = config.properties
+        self._runtime_fs = _helpers.get_plugin_property(
+            properties, self.RUNTIME_FS_PROPERTY) \
+            or self.RUNTIME_FS_DEFAULT
+
+    def has_arrow_native(self) -> bool:
+        if self._runtime_fs == self.RUNTIME_FS_ARROW:
+            return True
+        elif self._runtime_fs == self.RUNTIME_FS_AUTO:
+            return afs.S3FileSystem is not None
+        else:
+            return False
+
+    def has_file_storage(self) -> bool:
+        if self._runtime_fs == self.RUNTIME_FS_BOTO3:
+            return True
+        elif self._runtime_fs == self.RUNTIME_FS_AUTO:
+            return afs.S3FileSystem is None
+        else:
+            return False
 
-        self._bucket = _helpers.get_plugin_property(self._properties, self.BUCKET_PROPERTY)
-        self._prefix = _helpers.get_plugin_property(self._properties, self.PREFIX_PROPERTY) or ""
-        self._region = _helpers.get_plugin_property(self._properties, self.REGION_PROPERTY)
-        self._endpoint = _helpers.get_plugin_property(self._properties, self.ENDPOINT_PROPERTY)
+    def get_arrow_native(self) -> afs.SubTreeFileSystem:
 
-        credentials_params = self.setup_credentials()
+        s3fs_args = self.setup_client_args(self.ARROW_CLIENT_ARGS)
+        s3fs = afs.S3FileSystem(**s3fs_args)
 
-        client_args = {
-            "service_name": "s3",
-            **credentials_params}
+        bucket = _helpers.get_plugin_property(self._properties, self.BUCKET_PROPERTY)
+        prefix = _helpers.get_plugin_property(self._properties, self.PREFIX_PROPERTY)
 
-        if self._region is not None:
-            client_args["region_name"] = self._region
+        if bucket is None or len(bucket.strip()) == 0:
+            message = f"Missing required config property [{self.BUCKET_PROPERTY}] for S3 storage"
+            self._log.error(message)
+            raise ex.EConfigParse(message)
 
-        if self._endpoint is not None:
-            client_args["endpoint_url"] = self._endpoint
+        root_path = f"{bucket}/{prefix}" if prefix else bucket
 
-        self._client = boto3.client(**client_args)
+        return afs.SubTreeFileSystem(root_path, s3fs)
+
+    def get_file_storage(self) -> IFileStorage:
+
+        client_args = self.setup_client_args(self.BOTO_CLIENT_ARGS)
+        client_args["service_name"] = "s3"
+
+        config = cfg.PluginConfig()
+        config.protocol = "S3"
+        config.properties = self._properties
+
+        return S3ObjectStorage(config, client_args)
+
+    def setup_client_args(self, key_mapping: tp.Dict[str, str]) -> tp.Dict[str, tp.Any]:
 
-    def setup_credentials(self):
+        client_args = dict()
 
-        mechanism = _helpers.get_plugin_property(self._properties, self.CREDENTIALS_PROPERTY) or self.CREDENTIALS_DEFAULT
+        region = _helpers.get_plugin_property(self._properties, self.REGION_PROPERTY)
+        endpoint = _helpers.get_plugin_property(self._properties, self.ENDPOINT_PROPERTY)
 
-        if mechanism.lower() == self.CREDENTIALS_DEFAULT:
+        if region is not None:
+            region_key = key_mapping[self.REGION_PROPERTY]
+            client_args[region_key] = region
+
+        if endpoint is not None:
+            endpoint_key = key_mapping[self.ENDPOINT_PROPERTY]
+            client_args[endpoint_key] = endpoint
+
+        credentials = self.setup_credentials(key_mapping)
+        client_args.update(credentials)
+
+        return client_args
+
+    def setup_credentials(self, key_mapping: tp.Dict[str, str]):
+
+        mechanism = _helpers.get_plugin_property(self._properties, self.CREDENTIALS_PROPERTY)
+
+        if mechanism is None or len(mechanism) == 0 or mechanism.lower() == self.CREDENTIALS_DEFAULT:
             self._log.info(f"Using [{self.CREDENTIALS_DEFAULT}] credentials mechanism")
-            return {}
+            return dict()
 
         if mechanism.lower() == self.CREDENTIALS_STATIC:
 
@@ -103,12 +166,49 @@ class S3ObjectStorage(IFileStorage):
                 f"Using [{self.CREDENTIALS_STATIC}] credentials mechanism, " +
                 f"access key id = [{access_key_id}]")
 
-            return {"aws_access_key_id": access_key_id, "aws_secret_access_key": secret_access_key}
+            access_key_id_arg = key_mapping[self.ACCESS_KEY_ID_PROPERTY]
+            secret_access_key_arg = key_mapping[self.SECRET_ACCESS_KEY_PROPERTY]
+
+            return {
+                access_key_id_arg: access_key_id,
+                secret_access_key_arg: secret_access_key}
 
         message = f"Unrecognised credentials mechanism: [{mechanism}]"
         self._log.error(message)
         raise ex.EStartup(message)
 
+
+if __aws_available:
+    plugins.PluginManager.register_plugin(IStorageProvider, AwsStorageProvider, ["S3"])
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# CUSTOM IMPLEMENTATION FOR S3 STORAGE
+# ----------------------------------------------------------------------------------------------------------------------
+
+# This is the old implementation that was used before Arrow native was made available
+# It is likely to be removed in a future release
+
+
+class S3ObjectStorage(IFileStorage):
+
+    # This is a quick implementation of IFileStorage on S3 using the boto3 AWS SDK
+
+    def __init__(self, config: cfg.PluginConfig, client_args: dict):
+
+        self._log = _helpers.logger_for_object(self)
+
+        self._properties = config.properties
+        self._bucket = _helpers.get_plugin_property(self._properties, AwsStorageProvider.BUCKET_PROPERTY)
+        self._prefix = _helpers.get_plugin_property(self._properties, AwsStorageProvider.PREFIX_PROPERTY) or ""
+
+        if self._bucket is None or len(self._bucket.strip()) == 0:
+            message = f"Missing required config property [{AwsStorageProvider.BUCKET_PROPERTY}] for S3 storage"
+            self._log.error(message)
+            raise ex.EConfigParse(message)
+
+        self._client = boto3.client(**client_args)
+
     def exists(self, storage_path: str) -> bool:
 
         try:
@@ -140,19 +240,21 @@ class S3ObjectStorage(IFileStorage):
 
         self._log.info(f"STAT [{storage_path}]")
 
+        name = storage_path.split("/")[-1]
+
         if self.exists(storage_path):
 
             # Only OBJECTS can support stat atm
             # Handling for directories needs to be changed, as part of refactor onto object storage
             size = self.size(storage_path)
-            return FileStat(FileType.FILE, size)
+            return FileStat(name, FileType.FILE, storage_path, size)
 
         else:
 
             self.ls(storage_path)
-            return FileStat(FileType.DIRECTORY, 0)
+            return FileStat(name, FileType.DIRECTORY, storage_path, 0)
 
-    def ls(self, storage_path: str) -> tp.List[str]:
+    def ls(self, storage_path: str, recursive: bool = False) -> tp.List[FileStat]:
 
         self._log.info(f"LS [{storage_path}]")
 
@@ -174,36 +276,41 @@ class S3ObjectStorage(IFileStorage):
                 if raw_key == prefix:
                     continue
                 key = raw_key.replace(prefix, "")
-                keys.append(key)
+                size = entry["Size"]
+                mtime = entry["LastModified "]
+                stat = FileStat(key, FileType.FILE, raw_key, size, mtime=mtime)
+                keys.append(stat)
 
         if "CommonPrefixes" in response:
             for raw_prefix in response["CommonPrefixes"]:
                 common_prefix = raw_prefix.replace(prefix, "")
-                keys.append(common_prefix)
+                stat = FileStat(common_prefix, FileType.DIRECTORY, raw_prefix, 0)
+                keys.append(stat)
 
         return keys
 
-    def mkdir(self, storage_path: str, recursive: bool = False, exists_ok: bool = False):
+    def mkdir(self, storage_path: str, recursive: bool = False):
 
         self._log.info(f"MKDIR [{storage_path}]")
 
         # No-op in object storage
         pass
 
-    def rm(self, storage_path: str, recursive: bool = False):
+    def rm(self, storage_path: str):
 
         try:
             self._log.info(f"RM [{storage_path}]")
 
-            if recursive:
-                raise RuntimeError("RM (recursive) not available for S3 storage")
-
             object_key = self._resolve_path(storage_path)
             self._client.delete_object(Bucket=self._bucket, Key=object_key)
 
         except aws_ex.ClientError as error:
             raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
+    def rmdir(self, storage_path: str):
+
+        raise RuntimeError("RMDIR (recursive) not available for S3 storage")
+
     def read_bytes(self, storage_path: str) -> bytes:
 
         self._log.info(f"READ BYTES [{storage_path}]")
@@ -218,7 +325,7 @@ class S3ObjectStorage(IFileStorage):
         data = self.read_bytes(storage_path)
         return io.BytesIO(data)
 
-    def _read_impl(self, storage_path: str) -> botocore.response.StreamingBody:
+    def _read_impl(self, storage_path: str):
 
         try:
 
@@ -229,7 +336,7 @@ class S3ObjectStorage(IFileStorage):
         except aws_ex.ClientError as error:
             raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-    def write_bytes(self, storage_path: str, data: bytes, overwrite: bool = False):
+    def write_bytes(self, storage_path: str, data: bytes):
 
         try:
             self._log.info(f"WRITE BYTES [{storage_path}]")
@@ -244,43 +351,27 @@ class S3ObjectStorage(IFileStorage):
         except aws_ex.ClientError as error:
             raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-    def write_byte_stream(self, storage_path: str, overwrite: bool = False) -> tp.BinaryIO:
+    def write_byte_stream(self, storage_path: str) -> tp.BinaryIO:
 
         self._log.info(f"WRITE BYTE STREAM [{storage_path}]")
 
-        return self._AwsWriteBuf(self, storage_path, overwrite)
+        return self._AwsWriteBuf(self, storage_path)
 
     class _AwsWriteBuf(io.BytesIO):
 
-        def __init__(self, storage, storage_path, overwrite: bool):
+        def __init__(self, storage, storage_path):
             super().__init__()
             self._storage = storage
             self._storage_path = storage_path
-            self._overwrite = overwrite
             self._written = False
 
         def close(self):
             if not self._written:
                 self.seek(0)
                 data = self.read()
-                self._storage.write_bytes(self._storage_path, data, self._overwrite)
+                self._storage.write_bytes(self._storage_path, data)
                 self._written = True
 
-    # TODO: These methods can be removed from the interface, they are not needed
-    # (storage layer only needs to work in binary mode)
-
-    def read_text(self, storage_path: str, encoding: str = 'utf-8') -> str:
-        raise RuntimeError("READ (text mode) not available for S3 storage")
-
-    def read_text_stream(self, storage_path: str, encoding: str = 'utf-8') -> tp.TextIO:
-        raise RuntimeError("READ (text mode) not available for S3 storage")
-
-    def write_text(self, storage_path: str, data: str, encoding: str = 'utf-8', overwrite: bool = False):
-        raise RuntimeError("WRITE (text mode) not available for S3 storage")
-
-    def write_text_stream(self, storage_path: str, encoding: str = 'utf-8', overwrite: bool = False) -> tp.TextIO:
-        raise RuntimeError("WRITE (text mode) not available for S3 storage")
-
     def _resolve_path(self, storage_path: str) -> str:
 
         if self._prefix is None or self._prefix.strip() == "":
@@ -290,8 +381,3 @@ class S3ObjectStorage(IFileStorage):
         full_path = self._prefix + separator + storage_path
 
         return full_path[1:] if full_path.startswith("/") else full_path
-
-
-# Register the S3 storage plugin
-
-_storage.StorageManager.register_storage_type("S3", S3ObjectStorage, _storage.CommonDataStorage)

tracdap/rt/_plugins/storage_azure.py

@@ -0,0 +1,155 @@
+#  Copyright 2023 Accenture Global Solutions Limited
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import logging
+import typing as tp
+
+# TRAC interfaces
+import tracdap.rt.exceptions as ex
+import tracdap.rt.ext.plugins as plugins
+from tracdap.rt.ext.storage import *
+
+import pyarrow.fs as afs
+
+try:
+    # These dependencies are provided by the optional [azure] feature
+    # For local development, pip install -r requirements_plugins.txt
+    import azure.storage.blob as az_blob  # noqa
+    import adlfs  # noqa
+    __azure_available = True
+except ImportError:
+    adlfs = None
+    __azure_available = False
+
+# Set of common helpers across the core plugins (do not reference rt._impl)
+from . import _helpers
+
+
+class AzureBlobStorageProvider(IStorageProvider):
+
+    # This client depends on the Azure fsspec implementation, since there is no native implementation from Arrow
+    # To enable it, the tracdap package must be installed with the optional [azure] feature
+
+    # Current supported authentication mechanisms are "default" and "access_key"
+    # Client always uses location mode = primary, version aware = False
+
+    STORAGE_ACCOUNT_PROPERTY = "storageAccount"
+    CONTAINER_PROPERTY = "container"
+    PREFIX_PROPERTY = "prefix"
+
+    CREDENTIALS_PROPERTY = "credentials"
+    CREDENTIALS_DEFAULT = "default"
+    CREDENTIALS_ACCESS_KEY = "access_key"
+
+    ACCESS_KEY_PROPERTY = "accessKey"
+
+    RUNTIME_FS_PROPERTY = "runtimeFs"
+    RUNTIME_FS_AUTO = "auto"
+    RUNTIME_FS_FSSPEC = "fsspec"
+    RUNTIME_FS_DEFAULT = RUNTIME_FS_AUTO
+
+    def __init__(self, properties: tp.Dict[str, str]):
+
+        self._log = _helpers.logger_for_object(self)
+        self._properties = properties
+
+        self._runtime_fs = _helpers.get_plugin_property(
+            properties, self.RUNTIME_FS_PROPERTY) \
+            or self.RUNTIME_FS_DEFAULT
+
+        # The Azure SDK is very verbose with logging
+        # Avoid log noise by raising the log level for the Azure namespace
+        azure_log = _helpers.logger_for_namespace("azure.core")
+        azure_log.level = logging.WARNING
+
+    def has_arrow_native(self) -> bool:
+        return True
+
+    def get_arrow_native(self) -> afs.SubTreeFileSystem:
+
+        if self._runtime_fs == self.RUNTIME_FS_AUTO or self._runtime_fs == self.RUNTIME_FS_FSSPEC:
+            azure_fs = self.create_fsspec()
+        else:
+            message = f"Requested runtime FS [{self._runtime_fs}] is not available for Azure storage"
+            self._log.error(message)
+            raise ex.EStartup(message)
+
+        container = _helpers.get_plugin_property(self._properties, self.CONTAINER_PROPERTY)
+        prefix = _helpers.get_plugin_property(self._properties, self.PREFIX_PROPERTY)
+
+        if container is None or container.strip() == "":
+            message = f"Missing required config property [{self.CONTAINER_PROPERTY}] for Azure blob storage"
+            self._log.error(message)
+            raise ex.EConfigParse(message)
+
+        root_path = f"{container}/{prefix}" if prefix else container
+
+        return afs.SubTreeFileSystem(root_path, azure_fs)
+
+    def create_fsspec(self) -> afs.FileSystem:
+
+        azure_fsspec_args = self.setup_client_args()
+        azure_fsspec = adlfs.AzureBlobFileSystem(**azure_fsspec_args)
+
+        return afs.PyFileSystem(afs.FSSpecHandler(azure_fsspec))
+
+    def setup_client_args(self) -> tp.Dict[str, tp.Any]:
+
+        client_args = dict()
+
+        storage_account = _helpers.get_plugin_property(self._properties, self.STORAGE_ACCOUNT_PROPERTY)
+
+        if storage_account is None or len(storage_account.strip()) == 0:
+            message = f"Missing required config property [{self.STORAGE_ACCOUNT_PROPERTY}] for Azure blob storage"
+            self._log.error(message)
+            raise ex.EConfigParse(message)
+
+        client_args["account_name"] = storage_account
+
+        credentials = self.setup_credentials()
+        client_args.update(credentials)
+
+        return client_args
+
+    def setup_credentials(self):
+
+        # Only default (Google ADC) mechanism is supported
+        # Arrow GCP FS does also support access tokens, but ADC is probably all we ever need
+
+        mechanism = _helpers.get_plugin_property(self._properties, self.CREDENTIALS_PROPERTY)
+
+        if mechanism is None or len(mechanism) == 0 or mechanism.lower() == self.CREDENTIALS_DEFAULT:
+            self._log.info(f"Using [{self.CREDENTIALS_DEFAULT}] credentials mechanism")
+            return {"anon": False}
+
+        if mechanism == self.CREDENTIALS_ACCESS_KEY:
+
+            self._log.info(f"Using [{self.CREDENTIALS_ACCESS_KEY}] credentials mechanism")
+
+            access_key = _helpers.get_plugin_property(self._properties, self.ACCESS_KEY_PROPERTY)
+
+            if access_key is None or len(access_key.strip()) == 0:
+                message = f"Missing required config property [{self.ACCESS_KEY_PROPERTY}] for Azure blob storage"
+                raise ex.EConfigParse(message)
+
+            return {"account_key": access_key}
+
+        message = f"Unrecognised credentials mechanism: [{mechanism}]"
+        self._log.error(message)
+        raise ex.EStartup(message)
+
+
+# Only register the plugin if the [azure] feature is available
+if __azure_available:
+    plugins.PluginManager.register_plugin(IStorageProvider, AzureBlobStorageProvider, ["BLOB"])