tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_plugins/repo_git.py

@@ -23,6 +23,7 @@ import time
 import dulwich.repo as git_repo
 import dulwich.client as git_client
 import dulwich.index as git_index
+import urllib3.exceptions  # noqa
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -75,20 +76,45 @@ class GitRepository(IModelRepository):
 
     def do_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
-        self._log.info(
-            f"Git checkout: repo = [{model_def.repository}], " +
-            f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
+        try:
 
-        self._log.info(f"Checkout location: [{checkout_dir}]")
+            self._log.info(
+                f"Git checkout: repo = [{model_def.repository}], " +
+                f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
 
-        if self._native_git:
-            package_path = self._do_native_checkout(model_def, checkout_dir)
-        else:
-            package_path = self._do_python_checkout(model_def, checkout_dir)
+            self._log.info(f"Checkout location: [{checkout_dir}]")
+
+            if self._native_git:
+                package_path = self._do_native_checkout(model_def, checkout_dir)
+            else:
+                package_path = self._do_python_checkout(model_def, checkout_dir)
+
+            self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+
+            return package_path
+
+        except Exception as e:
+
+            error = e
+
+            # For retry failures, try to find the original cause
+            while e.__cause__ is not None:
+                if isinstance(e, urllib3.exceptions.MaxRetryError):
+                    error = e.__cause__
+                    break
+                else:
+                    e = e.__cause__
+
+            # Try to sanitize error messages from urllib3
+            if isinstance(error, urllib3.exceptions.HTTPError):
+                detail = self._clean_urllib3_error(error)
+            else:
+                detail = str(error)
 
-        self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+            message = f"Failed to check out [{model_def.repository}]: {detail}"
 
-        return package_path
+            self._log.error(message)
+            raise ex.EModelRepo(message) from error
 
     def _do_native_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
@@ -150,10 +176,15 @@ class GitRepository(IModelRepository):
                 for line in cmd_err:
                     self._log.info(line)
 
-            else:
+            elif cmd_err:
+
                 for line in cmd_err:
                     self._log.error(line)
 
+                raise ex.EModelRepo(cmd_err[-1])
+
+            else:
+
                 error_msg = f"Git checkout failed for {model_def.package} {model_def.version}"
                 self._log.error(error_msg)
                 raise ex.EModelRepo(error_msg)
@@ -265,6 +296,20 @@ class GitRepository(IModelRepository):
     def _ref_key(key):
         return bytes(key, "ascii")
 
+    @classmethod
+    def _clean_urllib3_error(cls, error: urllib3.exceptions.HTTPError):
+
+        match = cls._URLLIB3_ERROR_PATTERN.match(str(error))
+
+        # Best efforts to clean up the message, fall back on str(error)
+        if match:
+            return match.group(1)
+        else:
+            return str(error)
+
+    # Error message format is like this:
+    # <pkg.ClassName object at 0xXXXXXXX>: Message
+    _URLLIB3_ERROR_PATTERN = re.compile(r"<[^>]*>: (.*)")
 
 # Register plugin
 plugins.PluginManager.register_plugin(IModelRepository, GitRepository, ["git"])

tracdap/rt/_plugins/storage_aws.py

@@ -30,17 +30,24 @@ from pyarrow import fs as afs
 from . import _helpers
 
 
-try:
-    # AWS SDK
-    import boto3
-    import botocore.response
-    import botocore.exceptions as aws_ex
-    __aws_available = True
-except ImportError:
-    boto3 = None
-    botocore = None
-    aws_ex = None
-    __aws_available = False
+def _aws_arrow_available():
+    try:
+        # Shipped as part of PyArrow, but may not be available on all platforms
+        return afs.S3FileSystem is not None
+    except ImportError:
+        return False
+
+def _aws_boto3_available():
+    try:
+        # AWS SDK
+        # These dependencies are provided by the optional [aws] feature
+        # For local development, pip install -r requirements_plugins.txt
+        import boto3   # noqa
+        import botocore.response  # noqa
+        import botocore.exceptions as aws_ex  # noqa
+        return True
+    except ImportError:
+        return False
 
 
 class AwsStorageProvider(IStorageProvider):
@@ -87,23 +94,22 @@ class AwsStorageProvider(IStorageProvider):
             or self.RUNTIME_FS_DEFAULT
 
     def has_arrow_native(self) -> bool:
-        if self._runtime_fs == self.RUNTIME_FS_ARROW:
-            return True
-        elif self._runtime_fs == self.RUNTIME_FS_AUTO:
-            return afs.S3FileSystem is not None
-        else:
-            return False
+
+        return _aws_arrow_available()
 
     def has_file_storage(self) -> bool:
-        if self._runtime_fs == self.RUNTIME_FS_BOTO3:
-            return True
-        elif self._runtime_fs == self.RUNTIME_FS_AUTO:
-            return afs.S3FileSystem is None
-        else:
+
+        # Do not advertise the custom storage implementation if arrow native is available
+        if _aws_arrow_available():
             return False
 
+        return _aws_boto3_available()
+
     def get_arrow_native(self) -> afs.SubTreeFileSystem:
 
+        if not _aws_arrow_available():
+            raise ex.EStorage(f"S3 storage setup failed: Plugin for [{self.RUNTIME_FS_ARROW}] is not available")
+
         s3fs_args = self.setup_client_args(self.ARROW_CLIENT_ARGS)
         s3fs = afs.S3FileSystem(**s3fs_args)
 
@@ -121,6 +127,9 @@ class AwsStorageProvider(IStorageProvider):
 
     def get_file_storage(self) -> IFileStorage:
 
+        if not _aws_boto3_available():
+            raise ex.EStorage(f"S3 storage setup failed: Plugin for [{self.RUNTIME_FS_BOTO3}] is not available")
+
         client_args = self.setup_client_args(self.BOTO_CLIENT_ARGS)
         client_args["service_name"] = "s3"
 
@@ -179,7 +188,7 @@ class AwsStorageProvider(IStorageProvider):
         raise ex.EStartup(message)
 
 
-if __aws_available:
+if _aws_arrow_available() or _aws_boto3_available():
     plugins.PluginManager.register_plugin(IStorageProvider, AwsStorageProvider, ["S3"])
 
 
@@ -190,195 +199,201 @@ if __aws_available:
 # This is the old implementation that was used before Arrow native was made available
 # It is likely to be removed in a future release
 
+if _aws_boto3_available():
 
-class S3ObjectStorage(IFileStorage):
+    # These dependencies are provided by the optional [aws] feature
+    # For local development, pip install -r requirements_plugins.txt
+    import boto3  # noqa
+    import botocore.exceptions as aws_ex # noqa
 
-    # This is a quick implementation of IFileStorage on S3 using the boto3 AWS SDK
+    class S3ObjectStorage(IFileStorage):
 
-    def __init__(self, config: cfg.PluginConfig, client_args: dict):
+        # This is a quick implementation of IFileStorage on S3 using the boto3 AWS SDK
 
-        self._log = _helpers.logger_for_object(self)
+        def __init__(self, config: cfg.PluginConfig, client_args: dict):
 
-        self._properties = config.properties
-        self._bucket = _helpers.get_plugin_property(self._properties, AwsStorageProvider.BUCKET_PROPERTY)
-        self._prefix = _helpers.get_plugin_property(self._properties, AwsStorageProvider.PREFIX_PROPERTY) or ""
+            self._log = _helpers.logger_for_object(self)
 
-        if self._bucket is None or len(self._bucket.strip()) == 0:
-            message = f"Missing required config property [{AwsStorageProvider.BUCKET_PROPERTY}] for S3 storage"
-            self._log.error(message)
-            raise ex.EConfigParse(message)
+            self._properties = config.properties
+            self._bucket = _helpers.get_plugin_property(self._properties, AwsStorageProvider.BUCKET_PROPERTY)
+            self._prefix = _helpers.get_plugin_property(self._properties, AwsStorageProvider.PREFIX_PROPERTY) or ""
 
-        self._client = boto3.client(**client_args)
+            if self._bucket is None or len(self._bucket.strip()) == 0:
+                message = f"Missing required config property [{AwsStorageProvider.BUCKET_PROPERTY}] for S3 storage"
+                self._log.error(message)
+                raise ex.EConfigParse(message)
 
-    def exists(self, storage_path: str) -> bool:
+            self._client = boto3.client(**client_args)
 
-        try:
-            self._log.info(f"EXISTS [{storage_path}]")
+        def exists(self, storage_path: str) -> bool:
 
-            object_key = self._resolve_path(storage_path)
-            self._client.head_object(Bucket=self._bucket, Key=object_key)
-            return True
+            try:
+                self._log.info(f"EXISTS [{storage_path}]")
 
-        except aws_ex.ClientError as error:
-            aws_code = error.response['Error']['Code']
-            if aws_code == str(http.HTTPStatus.NOT_FOUND.value):  # noqa
-                return False
-            raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
+                object_key = self._resolve_path(storage_path)
+                self._client.head_object(Bucket=self._bucket, Key=object_key)
+                return True
 
-    def size(self, storage_path: str) -> int:
+            except aws_ex.ClientError as error:
+                aws_code = error.response['Error']['Code']
+                if aws_code == str(http.HTTPStatus.NOT_FOUND.value):  # noqa
+                    return False
+                raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-        try:
-            self._log.info(f"SIZE [{storage_path}]")
+        def size(self, storage_path: str) -> int:
 
-            object_key = self._resolve_path(storage_path)
-            response = self._client.head_object(Bucket=self._bucket, Key=object_key)
-            return response['ContentLength']
+            try:
+                self._log.info(f"SIZE [{storage_path}]")
 
-        except aws_ex.ClientError as error:
-            raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
+                object_key = self._resolve_path(storage_path)
+                response = self._client.head_object(Bucket=self._bucket, Key=object_key)
+                return response['ContentLength']
 
-    def stat(self, storage_path: str) -> FileStat:
+            except aws_ex.ClientError as error:
+                raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-        self._log.info(f"STAT [{storage_path}]")
+        def stat(self, storage_path: str) -> FileStat:
 
-        name = storage_path.split("/")[-1]
+            self._log.info(f"STAT [{storage_path}]")
 
-        if self.exists(storage_path):
+            name = storage_path.split("/")[-1]
 
-            # Only OBJECTS can support stat atm
-            # Handling for directories needs to be changed, as part of refactor onto object storage
-            size = self.size(storage_path)
-            return FileStat(name, FileType.FILE, storage_path, size)
+            if self.exists(storage_path):
 
-        else:
+                # Only OBJECTS can support stat atm
+                # Handling for directories needs to be changed, as part of refactor onto object storage
+                size = self.size(storage_path)
+                return FileStat(name, FileType.FILE, storage_path, size)
 
-            self.ls(storage_path)
-            return FileStat(name, FileType.DIRECTORY, storage_path, 0)
+            else:
 
-    def ls(self, storage_path: str, recursive: bool = False) -> tp.List[FileStat]:
+                self.ls(storage_path)
+                return FileStat(name, FileType.DIRECTORY, storage_path, 0)
 
-        self._log.info(f"LS [{storage_path}]")
+        def ls(self, storage_path: str, recursive: bool = False) -> tp.List[FileStat]:
 
-        prefix = self._resolve_path(storage_path) + "/"
+            self._log.info(f"LS [{storage_path}]")
 
-        response = self._client.list_objects_v2(
-            Bucket=self._bucket,
-            Prefix=prefix,
-            Delimiter="/")
+            prefix = self._resolve_path(storage_path) + "/"
 
-        keys = []
+            response = self._client.list_objects_v2(
+                Bucket=self._bucket,
+                Prefix=prefix,
+                Delimiter="/")
 
-        if "Contents" not in response and "CommonPrefixes" not in response:
-            raise ex.EStorageRequest(f"Storage prefix not found: [{storage_path}]")
+            keys = []
 
-        if "Contents" in response:
-            for entry in response["Contents"]:
-                raw_key = entry["Key"]
-                if raw_key == prefix:
-                    continue
-                key = raw_key.replace(prefix, "")
-                size = entry["Size"]
-                mtime = entry["LastModified "]
-                stat = FileStat(key, FileType.FILE, raw_key, size, mtime=mtime)
-                keys.append(stat)
+            if "Contents" not in response and "CommonPrefixes" not in response:
+                raise ex.EStorageRequest(f"Storage prefix not found: [{storage_path}]")
 
-        if "CommonPrefixes" in response:
-            for raw_prefix in response["CommonPrefixes"]:
-                common_prefix = raw_prefix.replace(prefix, "")
-                stat = FileStat(common_prefix, FileType.DIRECTORY, raw_prefix, 0)
-                keys.append(stat)
+            if "Contents" in response:
+                for entry in response["Contents"]:
+                    raw_key = entry["Key"]
+                    if raw_key == prefix:
+                        continue
+                    key = raw_key.replace(prefix, "")
+                    size = entry["Size"]
+                    mtime = entry["LastModified "]
+                    stat = FileStat(key, FileType.FILE, raw_key, size, mtime=mtime)
+                    keys.append(stat)
 
-        return keys
+            if "CommonPrefixes" in response:
+                for raw_prefix in response["CommonPrefixes"]:
+                    common_prefix = raw_prefix.replace(prefix, "")
+                    stat = FileStat(common_prefix, FileType.DIRECTORY, raw_prefix, 0)
+                    keys.append(stat)
 
-    def mkdir(self, storage_path: str, recursive: bool = False):
+            return keys
 
-        self._log.info(f"MKDIR [{storage_path}]")
+        def mkdir(self, storage_path: str, recursive: bool = False):
 
-        # No-op in object storage
-        pass
+            self._log.info(f"MKDIR [{storage_path}]")
 
-    def rm(self, storage_path: str):
+            # No-op in object storage
+            pass
 
-        try:
-            self._log.info(f"RM [{storage_path}]")
+        def rm(self, storage_path: str):
 
-            object_key = self._resolve_path(storage_path)
-            self._client.delete_object(Bucket=self._bucket, Key=object_key)
+            try:
+                self._log.info(f"RM [{storage_path}]")
 
-        except aws_ex.ClientError as error:
-            raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
+                object_key = self._resolve_path(storage_path)
+                self._client.delete_object(Bucket=self._bucket, Key=object_key)
 
-    def rmdir(self, storage_path: str):
+            except aws_ex.ClientError as error:
+                raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-        raise RuntimeError("RMDIR (recursive) not available for S3 storage")
+        def rmdir(self, storage_path: str):
 
-    def read_bytes(self, storage_path: str) -> bytes:
+            raise RuntimeError("RMDIR (recursive) not available for S3 storage")
 
-        self._log.info(f"READ BYTES [{storage_path}]")
+        def read_bytes(self, storage_path: str) -> bytes:
 
-        body = self._read_impl(storage_path)
-        return body.read()
+            self._log.info(f"READ BYTES [{storage_path}]")
 
-    def read_byte_stream(self, storage_path: str) -> tp.BinaryIO:
+            body = self._read_impl(storage_path)
+            return body.read()
 
-        self._log.info(f"READ BYTE STREAM [{storage_path}]")
+        def read_byte_stream(self, storage_path: str) -> tp.BinaryIO:
 
-        data = self.read_bytes(storage_path)
-        return io.BytesIO(data)
+            self._log.info(f"READ BYTE STREAM [{storage_path}]")
 
-    def _read_impl(self, storage_path: str):
+            data = self.read_bytes(storage_path)
+            return io.BytesIO(data)
 
-        try:
+        def _read_impl(self, storage_path: str):
 
-            object_key = self._resolve_path(storage_path)
-            response = self._client.get_object(Bucket=self._bucket, Key=object_key)
-            return response['Body']
+            try:
 
-        except aws_ex.ClientError as error:
-            raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
+                object_key = self._resolve_path(storage_path)
+                response = self._client.get_object(Bucket=self._bucket, Key=object_key)
+                return response['Body']
 
-    def write_bytes(self, storage_path: str, data: bytes):
+            except aws_ex.ClientError as error:
+                raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-        try:
-            self._log.info(f"WRITE BYTES [{storage_path}]")
+        def write_bytes(self, storage_path: str, data: bytes):
 
-            object_key = self._resolve_path(storage_path)
+            try:
+                self._log.info(f"WRITE BYTES [{storage_path}]")
 
-            self._client.put_object(
-                Bucket=self._bucket,
-                Key=object_key,
-                Body=data)
+                object_key = self._resolve_path(storage_path)
+
+                self._client.put_object(
+                    Bucket=self._bucket,
+                    Key=object_key,
+                    Body=data)
 
-        except aws_ex.ClientError as error:
-            raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
+            except aws_ex.ClientError as error:
+                raise ex.EStorageRequest(f"Storage error: {str(error)}") from error
 
-    def write_byte_stream(self, storage_path: str) -> tp.BinaryIO:
+        def write_byte_stream(self, storage_path: str) -> tp.BinaryIO:
 
-        self._log.info(f"WRITE BYTE STREAM [{storage_path}]")
+            self._log.info(f"WRITE BYTE STREAM [{storage_path}]")
 
-        return self._AwsWriteBuf(self, storage_path)
+            return self._AwsWriteBuf(self, storage_path)
 
-    class _AwsWriteBuf(io.BytesIO):
+        class _AwsWriteBuf(io.BytesIO):
 
-        def __init__(self, storage, storage_path):
-            super().__init__()
-            self._storage = storage
-            self._storage_path = storage_path
-            self._written = False
+            def __init__(self, storage, storage_path):
+                super().__init__()
+                self._storage = storage
+                self._storage_path = storage_path
+                self._written = False
 
-        def close(self):
-            if not self._written:
-                self.seek(0)
-                data = self.read()
-                self._storage.write_bytes(self._storage_path, data)
-                self._written = True
+            def close(self):
+                if not self._written:
+                    self.seek(0)
+                    data = self.read()
+                    self._storage.write_bytes(self._storage_path, data)
+                    self._written = True
 
-    def _resolve_path(self, storage_path: str) -> str:
+        def _resolve_path(self, storage_path: str) -> str:
 
-        if self._prefix is None or self._prefix.strip() == "":
-            return storage_path
+            if self._prefix is None or self._prefix.strip() == "":
+                return storage_path
 
-        separator = "" if self._prefix.endswith("/") else "/"
-        full_path = self._prefix + separator + storage_path
+            separator = "" if self._prefix.endswith("/") else "/"
+            full_path = self._prefix + separator + storage_path
 
-        return full_path[1:] if full_path.startswith("/") else full_path
+            return full_path[1:] if full_path.startswith("/") else full_path

tracdap/rt/_plugins/storage_azure.py

@@ -23,20 +23,21 @@ from tracdap.rt.ext.storage import *
 
 import pyarrow.fs as afs
 
-try:
-    # These dependencies are provided by the optional [azure] feature
-    # For local development, pip install -r requirements_plugins.txt
-    import azure.storage.blob as az_blob  # noqa
-    import adlfs  # noqa
-    __azure_available = True
-except ImportError:
-    adlfs = None
-    __azure_available = False
-
 # Set of common helpers across the core plugins (do not reference rt._impl)
 from . import _helpers
 
 
+def _azure_fsspec_available():
+    try:
+        # These dependencies are provided by the optional [azure] feature
+        # For local development, pip install -r requirements_plugins.txt
+        import azure.storage.blob as az_blob  # noqa
+        import adlfs  # noqa
+        return True
+    except ImportError:
+        return False
+
+
 class AzureBlobStorageProvider(IStorageProvider):
 
     # This client depends on the Azure fsspec implementation, since there is no native implementation from Arrow
@@ -100,6 +101,11 @@ class AzureBlobStorageProvider(IStorageProvider):
 
     def create_fsspec(self) -> afs.FileSystem:
 
+        if not _azure_fsspec_available():
+            raise ex.EStorage(f"BLOB storage setup failed: Plugin for [{self.RUNTIME_FS_FSSPEC}] is not available")
+
+        import adlfs  # noqa
+
         azure_fsspec_args = self.setup_client_args()
         azure_fsspec = adlfs.AzureBlobFileSystem(**azure_fsspec_args)
 
@@ -152,5 +158,5 @@ class AzureBlobStorageProvider(IStorageProvider):
 
 
 # Only register the plugin if the [azure] feature is available
-if __azure_available:
+if _azure_fsspec_available():
     plugins.PluginManager.register_plugin(IStorageProvider, AzureBlobStorageProvider, ["BLOB"])