plexus-python-common 1.0.34__tar.gz → 1.0.35__tar.gz

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.34
+Version: 1.0.35
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -8,8 +8,7 @@ Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.12
 Requires-Dist: aiobotocore<2.27,>=2.26
 Requires-Dist: boto3<1.42,>=1.41
-Requires-Dist: boto3-stubs<1.42,>=1.41
-Requires-Dist: cloudpathlib>=0.21
+Requires-Dist: boto3-stubs[s3]<1.42,>=1.41
 Requires-Dist: fsspec>=2025.10
 Requires-Dist: lxml>=6.0
 Requires-Dist: numpy>=2.3

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/pyproject.toml

@@ -10,8 +10,7 @@ build-backend = "setuptools.build_meta"
 dev = [
     "aiobotocore>=2.26,<2.27",
     "boto3>=1.41,<1.42",
-    "boto3-stubs>=1.41,<1.42",
-    "cloudpathlib>=0.21",
+    "boto3-stubs[s3]>=1.41,<1.42",
     "fsspec>=2025.10",
     "lxml>=6.0",
     "numpy>=2.3",
@@ -53,8 +52,7 @@ classifiers = [
 dependencies = [
     "aiobotocore>=2.26,<2.27",
     "boto3>=1.41,<1.42",
-    "boto3-stubs>=1.41,<1.42",
-    "cloudpathlib>=0.21",
+    "boto3-stubs[s3]>=1.41,<1.42",
     "fsspec>=2025.10",
     "lxml>=6.0",
     "numpy>=2.3",

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/src/plexus/common/utils/s3utils.py

@@ -13,21 +13,21 @@ import typing
 import zipfile
 import zlib
 from collections.abc import Callable, Generator
-from pathlib import Path
 from typing import Any, Literal
 
 import boto3
 import fsspec
 import fsspec.utils
-from cloudpathlib import CloudPath, S3Client, S3Path
 from iker.common.utils.sequtils import chunk_between, head, last
 from iker.common.utils.shutils import glob_match, listfile, path_depth
 from iker.common.utils.strutils import is_empty, trim_to_none
-from rich.progress import BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TransferSpeedColumn
+from mypy_boto3_s3 import S3Client
+from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TransferSpeedColumn
 
 __all__ = [
     "S3ObjectMeta",
     "s3_make_client",
+    "s3_head_object",
     "s3_list_objects",
     "s3_listfile",
     "s3_cp_download",
@@ -36,10 +36,8 @@ __all__ = [
     "s3_sync_upload",
     "s3_pull_text",
     "s3_push_text",
-    "S3TransferCallbackClient",
-    "ArchiveMemberChunk",
-    "s3_make_progress_callback",
     "s3_make_progressed_client",
+    "ArchiveMemberChunk",
     "s3_archive_member_tree",
     "s3_archive_listfile",
     "s3_archive_open_member",
@@ -84,12 +82,28 @@ def s3_make_client(
     session = boto3.Session(aws_access_key_id=trim_to_none(access_key_id),
                             aws_secret_access_key=trim_to_none(secret_access_key),
                             region_name=trim_to_none(region_name))
-    client = S3Client(boto3_session=session, endpoint_url=trim_to_none(endpoint_url))
+    client = session.client("s3", endpoint_url=trim_to_none(endpoint_url))
     try:
         yield client
     finally:
-        if hasattr(client, "close"):
-            client.close()
+        client.close()
+
+
+def s3_head_object(
+    client: S3Client,
+    bucket: str,
+    key: str,
+) -> S3ObjectMeta:
+    """
+    Retrieves metadata of an object from the given S3 ``bucket`` and ``key``.
+
+    :param client: An instance of ``S3Client``.
+    :param bucket: Bucket name.
+    :param key: Object key.
+    :return: An ``S3ObjectMeta`` object representing the S3 object.
+    """
+    response = client.head_object(Bucket=bucket, Key=key)
+    return S3ObjectMeta(key=key, last_modified=response["LastModified"], size=response["ContentLength"])
 
 
 def s3_list_objects(
@@ -111,12 +125,12 @@ def s3_list_objects(
     count = 0
     while True:
         if is_empty(continuation_token):
-            response = client.client.list_objects_v2(MaxKeys=1000, Bucket=bucket, Prefix=prefix)
+            response = client.list_objects_v2(MaxKeys=1000, Bucket=bucket, Prefix=prefix)
         else:
-            response = client.client.list_objects_v2(MaxKeys=1000,
-                                                     Bucket=bucket,
-                                                     Prefix=prefix,
-                                                     ContinuationToken=continuation_token)
+            response = client.list_objects_v2(MaxKeys=1000,
+                                              Bucket=bucket,
+                                              Prefix=prefix,
+                                              ContinuationToken=continuation_token)
 
         contents = response.get("Contents", [])
         count += len(contents)
@@ -175,7 +189,7 @@ def s3_cp_download(client: S3Client, bucket: str, key: str, file_path: str | os.
     :param key: Object key.
     :param file_path: Local file path to save the object.
     """
-    client.client.download_file(bucket, key, file_path)
+    client.download_file(bucket, key, file_path)
 
 
 def s3_cp_upload(client: S3Client, file_path: str | os.PathLike[str], bucket: str, key: str):
@@ -188,10 +202,10 @@ def s3_cp_upload(client: S3Client, file_path: str | os.PathLike[str], bucket: st
     :param key: Object key for the uploaded file.
     """
     t, _ = mimetypes.MimeTypes().guess_type(file_path)
-    client.client.upload_file(file_path,
-                              bucket,
-                              key,
-                              ExtraArgs={"ContentType": "binary/octet-stream" if t is None else t})
+    client.upload_file(file_path,
+                       bucket,
+                       key,
+                       ExtraArgs={"ContentType": "binary/octet-stream" if t is None else t})
 
 
 def s3_sync_download(
@@ -311,7 +325,7 @@ def s3_pull_text(client: S3Client, bucket: str, key: str, encoding: str = None)
     :return: The decoded text content.
     """
     with tempfile.TemporaryFile() as fp:
-        client.client.download_fileobj(bucket, key, fp)
+        client.download_fileobj(bucket, key, fp)
         fp.seek(0)
         return fp.read().decode(encoding or "utf-8")
 
@@ -329,95 +343,111 @@ def s3_push_text(client: S3Client, text: str, bucket: str, key: str, encoding: s
     with tempfile.TemporaryFile() as fp:
         fp.write(text.encode(encoding or "utf-8"))
         fp.seek(0)
-        client.client.upload_fileobj(fp, bucket, key)
+        client.upload_fileobj(fp, bucket, key)
 
 
-TransferDirection = Literal["download", "upload"]
-TransferState = Literal["start", "update", "stop"]
+class S3ClientProgressProxy(object):
+    def __init__(self, client: S3Client, progress: Progress):
+        self.client = client
+        self.progress = progress
 
+    @contextlib.contextmanager
+    def make_transfer_callback(self, key: str, bytes_total: int, direction: Literal["download", "upload"]):
+        task_id = self.progress.add_task(direction, total=bytes_total, key=key)
+        try:
+            yield lambda bytes_sent: self.progress.update(task_id, advance=bytes_sent)
+        finally:
+            self.progress.remove_task(task_id)
 
-@contextlib.contextmanager
-def make_transfer_callback(
-    callback: Callable[[CloudPath, TransferDirection, TransferState, int], None],
-    path: Path | CloudPath,
-    direction: TransferDirection,
-):
-    if callback is None:
-        yield None
-        return
-
-    callback(path, direction, "start", 0)
-    try:
-        yield functools.partial(callback, path, direction, "update")
-    finally:
-        callback(path, direction, "stop", 0)
+    def __getattr__(self, name):
+        return getattr(self.client, name)
 
-
-class S3TransferCallbackClient(S3Client):
-    def __init__(
+    def download_file(
         self,
-        *args,
-        transfer_callback: Callable[[Path | CloudPath, TransferDirection, TransferState, int], None],
-        **kwargs,
+        Bucket,
+        Key,
+        Filename,
+        ExtraArgs=None,
+        Callback=None,
+        Config=None,
     ):
-        super().__init__(*args, **kwargs)
-        self.transfer_callback = transfer_callback
-
-    def _download_file(self, cloud_path: S3Path, local_path: str | os.PathLike[str]) -> Path:
-        local_path = Path(local_path)
-
-        obj = self.s3.Object(cloud_path.bucket, cloud_path.key)
-
-        with make_transfer_callback(self.transfer_callback, cloud_path, "download") as callback:
-            obj.download_file(
-                str(local_path),
-                Config=self.boto3_transfer_config,
-                ExtraArgs=self.boto3_dl_extra_args,
-                Callback=callback,
-            )
-        return local_path
-
-    def _upload_file(self, local_path: str | os.PathLike[str], cloud_path: S3Path) -> S3Path:
-        local_path = Path(local_path)
-
-        obj = self.s3.Object(cloud_path.bucket, cloud_path.key)
-
-        extra_args = self.boto3_ul_extra_args.copy()
-
-        if self.content_type_method is not None:
-            content_type, content_encoding = self.content_type_method(str(local_path))
-            if content_type is not None:
-                extra_args["ContentType"] = content_type
-            if content_encoding is not None:
-                extra_args["ContentEncoding"] = content_encoding
-
-        with make_transfer_callback(self.transfer_callback, local_path, "upload") as callback:
-            obj.upload_file(
-                str(local_path),
-                Config=self.boto3_transfer_config,
-                ExtraArgs=extra_args,
-                Callback=callback,
-            )
-        return cloud_path
-
-
-def s3_make_progress_callback(
-    progress: Progress,
-) -> Callable[[Path | CloudPath, TransferDirection, TransferState, int], None]:
-    task_ids: dict[Path | CloudPath, TaskID] = {}
-
-    def progress_callback(path: Path | CloudPath, direction: TransferDirection, state: TransferState, bytes_sent: int):
-        if state == "start":
-            size = path.stat().st_size
-            task_ids[path] = progress.add_task(direction, total=size, filename=path.name)
-        elif state == "stop":
-            if path in task_ids:
-                progress.remove_task(task_ids[path])
-                del task_ids[path]
-        else:
-            progress.update(task_ids[path], advance=bytes_sent)
-
-    return progress_callback
+        object_meta = s3_head_object(self.client, Bucket, Key)
+        with (
+            contextlib.nullcontext(Callback) if Callback is not None
+            else self.make_transfer_callback(Key, object_meta.size, "download")
+        ) as callback:
+            return self.client.download_file(Bucket,
+                                             Key,
+                                             Filename,
+                                             ExtraArgs=ExtraArgs,
+                                             Callback=callback,
+                                             Config=Config)
+
+    def download_fileobj(
+        self,
+        Bucket,
+        Key,
+        Fileobj,
+        ExtraArgs=None,
+        Callback=None,
+        Config=None,
+    ):
+        object_meta = s3_head_object(self.client, Bucket, Key)
+        with (
+            contextlib.nullcontext(Callback) if Callback is not None
+            else self.make_transfer_callback(Key, object_meta.size, "download")
+        ) as callback:
+            return self.client.download_fileobj(Bucket,
+                                                Key,
+                                                Fileobj,
+                                                ExtraArgs=ExtraArgs,
+                                                Callback=callback,
+                                                Config=Config)
+
+    def upload_file(
+        self,
+        Filename,
+        Bucket,
+        Key,
+        ExtraArgs=None,
+        Callback=None,
+        Config=None,
+    ):
+        bytes_total = os.path.getsize(Filename)
+        with (
+            contextlib.nullcontext(Callback) if Callback is not None
+            else self.make_transfer_callback(Key, bytes_total, "upload")
+        ) as callback:
+            return self.client.upload_file(Filename,
+                                           Bucket,
+                                           Key,
+                                           ExtraArgs=ExtraArgs,
+                                           Callback=callback,
+                                           Config=Config)
+
+    def upload_fileobj(
+        self,
+        Fileobj,
+        Bucket,
+        Key,
+        ExtraArgs=None,
+        Callback=None,
+        Config=None,
+    ):
+        current_pos = Fileobj.tell()
+        Fileobj.seek(0, os.SEEK_END)
+        bytes_total = Fileobj.tell() - current_pos
+        Fileobj.seek(current_pos)
+        with (
+            contextlib.nullcontext(Callback) if Callback is not None
+            else self.make_transfer_callback(Key, bytes_total, "upload")
+        ) as callback:
+            return self.client.upload_fileobj(Fileobj,
+                                              Bucket,
+                                              Key,
+                                              ExtraArgs=ExtraArgs,
+                                              Callback=callback,
+                                              Config=Config)
 
 
 if typing.TYPE_CHECKING:
@@ -435,18 +465,18 @@ def s3_make_progressed_client(
     secret_access_key: str = None,
     region_name: str = None,
     endpoint_url: str = None,
-) -> Generator[S3Client, None, None]:
+) -> Generator[S3ClientProgressProxy, None, None]:
     """
-    Creates an S3 client with progress callback as a context manager for safe resource handling.
+    Creates an S3 client with progress reporting as a context manager.
 
     :param access_key_id: AWS access key ID.
     :param secret_access_key: AWS secret access key.
     :param region_name: AWS service region name.
     :param endpoint_url: AWS service endpoint URL.
-    :return: An instance of ``S3TransferCallbackClient``.
+    :return: An instance of ``S3Client`` with progress reporting.
     """
     with Progress(
-        TextColumn("[blue]{task.fields[filename]}"),
+        TextColumn("[blue]{task.fields[key]}"),
         BarColumn(),
         DownloadColumn(),
         TransferSpeedColumn(),
@@ -454,9 +484,12 @@ def s3_make_progressed_client(
         session = boto3.Session(aws_access_key_id=trim_to_none(access_key_id),
                                 aws_secret_access_key=trim_to_none(secret_access_key),
                                 region_name=trim_to_none(region_name))
-        yield S3TransferCallbackClient(boto3_session=session,
-                                       endpoint_url=trim_to_none(endpoint_url),
-                                       transfer_callback=s3_make_progress_callback(progress))
+        client = session.client("s3", endpoint_url=trim_to_none(endpoint_url))
+        proxy = S3ClientProgressProxy(client, progress)
+        try:
+            yield proxy
+        finally:
+            proxy.close()
 
 
 def s3_options_from_s3_client(client: S3Client) -> dict[str, Any]:
@@ -466,12 +499,12 @@ def s3_options_from_s3_client(client: S3Client) -> dict[str, Any]:
     :param client: An instance of ``S3Client``.
     :return: A dictionary of S3 connection options.
     """
-    if client.sess is None:
+    if client is None:
         return {}
 
     s3_options: dict[str, Any] = {}
 
-    credentials = client.sess.get_credentials()
+    credentials = client._request_signer._credentials
     if credentials is not None:
         if credentials.access_key:
             s3_options["key"] = credentials.access_key
@@ -481,10 +514,10 @@ def s3_options_from_s3_client(client: S3Client) -> dict[str, Any]:
             s3_options["token"] = credentials.token
 
     client_kwargs = {}
-    if client.sess.region_name or client.client.meta.region_name:
-        client_kwargs["region_name"] = client.sess.region_name or client.client.meta.region_name
-    if client.client.meta.endpoint_url:
-        client_kwargs["endpoint_url"] = client.client.meta.endpoint_url
+    if client.meta.region_name:
+        client_kwargs["region_name"] = client.meta.region_name
+    if client.meta.endpoint_url:
+        client_kwargs["endpoint_url"] = client.meta.endpoint_url
 
     if client_kwargs:
         s3_options["client_kwargs"] = client_kwargs

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/src/plexus_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.34
+Version: 1.0.35
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -8,8 +8,7 @@ Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.12
 Requires-Dist: aiobotocore<2.27,>=2.26
 Requires-Dist: boto3<1.42,>=1.41
-Requires-Dist: boto3-stubs<1.42,>=1.41
-Requires-Dist: cloudpathlib>=0.21
+Requires-Dist: boto3-stubs[s3]<1.42,>=1.41
 Requires-Dist: fsspec>=2025.10
 Requires-Dist: lxml>=6.0
 Requires-Dist: numpy>=2.3

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/src/plexus_python_common.egg-info/requires.txt

@@ -1,7 +1,6 @@
 aiobotocore<2.27,>=2.26
 boto3<1.42,>=1.41
-boto3-stubs<1.42,>=1.41
-cloudpathlib>=0.21
+boto3-stubs[s3]<1.42,>=1.41
 fsspec>=2025.10
 lxml>=6.0
 numpy>=2.3

{plexus_python_common-1.0.34 → plexus_python_common-1.0.35}/test/plexus_tests/common/utils/s3utils_test.py

@@ -31,7 +31,7 @@ class S3UtilsTest(unittest.TestCase):
 
     def test_s3_list_object__random_text_files(self):
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             data = []
             for i in range(0, 2000):
@@ -49,7 +49,7 @@ class S3UtilsTest(unittest.TestCase):
                 self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
         with moto.mock_aws(), s3_make_progressed_client() as client:
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             data = []
             for i in range(0, 2000):
@@ -58,17 +58,17 @@ class S3UtilsTest(unittest.TestCase):
                 data.append((key, text))
 
             for key, text in data:
-                client.CloudPath("s3://dummy-bucket/", key).write_text(text)
+                s3_push_text(client, text, "dummy-bucket", key)
 
             result = list(s3_list_objects(client, "dummy-bucket", "dummy_prefix/"))
 
             for key, text in data:
                 self.assertTrue(any(o.key == key for o in result))
-                self.assertEqual(client.CloudPath("s3://dummy-bucket", key).read_text(), text)
+                self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
     def test_s3_listfile__random_text_files(self):
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             data = []
             for i in range(0, 2000):
@@ -86,7 +86,7 @@ class S3UtilsTest(unittest.TestCase):
                 self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
         with moto.mock_aws(), s3_make_progressed_client() as client:
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             data = []
             for i in range(0, 2000):
@@ -95,13 +95,13 @@ class S3UtilsTest(unittest.TestCase):
                 data.append((key, text))
 
             for key, text in data:
-                client.CloudPath("s3://dummy-bucket/", key).write_text(text)
+                s3_push_text(client, text, "dummy-bucket", key)
 
             result = list(s3_listfile(client, "dummy-bucket", "dummy_prefix/"))
 
             for key, text in data:
                 self.assertTrue(any(o.key == key for o in result))
-                self.assertEqual(client.CloudPath("s3://dummy-bucket", key).read_text(), text)
+                self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
     data_s3_listfile = [
         (
@@ -236,7 +236,7 @@ class S3UtilsTest(unittest.TestCase):
     @ddt.unpack
     def test_s3_listfile(self, src, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket=bucket)
+            client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, src),
@@ -394,7 +394,7 @@ class S3UtilsTest(unittest.TestCase):
     @ddt.unpack
     def test_s3_sync(self, src, dst, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket=bucket)
+            client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, src),
@@ -414,7 +414,7 @@ class S3UtilsTest(unittest.TestCase):
                                     set(map(lambda x: norm_path(os.path.join(temp_directory, x)), expect)))
 
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket=bucket)
+            client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, src),
@@ -434,7 +434,7 @@ class S3UtilsTest(unittest.TestCase):
                                     set(map(lambda x: norm_path(os.path.join(temp_directory, x)), expect)))
 
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket=bucket)
+            client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, src),
@@ -467,7 +467,7 @@ class S3UtilsTest(unittest.TestCase):
     @ddt.unpack
     def test_s3_text(self, bucket, key, text, encoding):
         with moto.mock_aws(), s3_make_client() as client:
-            client.client.create_bucket(Bucket=bucket)
+            client.create_bucket(Bucket=bucket)
 
             s3_push_text(client, text, bucket, key, encoding=encoding)
             self.assertEqual(s3_pull_text(client, bucket, key, encoding=encoding), text)
@@ -478,7 +478,7 @@ class S3UtilsTest(unittest.TestCase):
             moto_server_endpoint_url() as endpoint_url,
             s3_make_client(endpoint_url=endpoint_url) as client,
         ):
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, "unittest", "s3utils_archive"),
@@ -538,7 +538,7 @@ class S3UtilsTest(unittest.TestCase):
             moto_server_endpoint_url() as endpoint_url,
             s3_make_client(endpoint_url=endpoint_url) as client,
         ):
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, "unittest", "s3utils_archive"),
@@ -565,7 +565,7 @@ class S3UtilsTest(unittest.TestCase):
             moto_server_endpoint_url() as endpoint_url,
             s3_make_client(endpoint_url=endpoint_url) as client,
         ):
-            client.client.create_bucket(Bucket="dummy-bucket")
+            client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
                            os.path.join(resources_directory, "unittest", "s3utils_archive"),