iker-python-common 1.0.58__py3-none-any.whl → 1.0.59__py3-none-any.whl

iker/common/utils/shutils.py

@@ -95,7 +95,11 @@ def path_depth(root: str, child: str) -> int:
     return child_expanded[len(root_expanded):].count(os.sep)
 
 
-def glob_match(names: list[str], include_patterns: list[str] = None, exclude_patterns: list[str] = None) -> list[str]:
+def glob_match(
+    names: list[str],
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
+) -> list[str]:
     """
     Applies the given inclusive and exclusive glob patterns to the given ``names`` and returns the filtered result.
 
@@ -121,8 +125,8 @@ class CopyFuncProtocol(Protocol):
 def listfile(
     path: str,
     *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
     depth: int = 0,
 ) -> list[str]:
     """
@@ -153,8 +157,8 @@ def copy(
     src: str,
     dst: str,
     *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
     depth: int = 0,
     follow_symlinks: bool = False,
     ignore_dangling_symlinks: bool = False,

{iker_python_common-1.0.58.dist-info → iker_python_common-1.0.59.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: iker-python-common
-Version: 1.0.58
+Version: 1.0.59
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.12
-Requires-Dist: boto3>=1.35
 Requires-Dist: docker>=7.1
 Requires-Dist: numpy>=2.3
 Requires-Dist: psycopg>=3.2
@@ -16,7 +15,6 @@ Provides-Extra: all
 Requires-Dist: iker-python-common; extra == "all"
 Provides-Extra: test
 Requires-Dist: ddt>=1.7; extra == "test"
-Requires-Dist: moto[all,ec2,s3]>=5.0; extra == "test"
 Requires-Dist: pytest-cov>=5.0; extra == "test"
 Requires-Dist: pytest-mysql>=3.0; extra == "test"
 Requires-Dist: pytest-order>=1.3; extra == "test"

{iker_python_common-1.0.58.dist-info → iker_python_common-1.0.59.dist-info}/RECORD

@@ -12,14 +12,13 @@ iker/common/utils/logger.py,sha256=FJaai6Sbchy4wKHcUMUCrrkBcXvIxq4qByERZ_TJBps,3
 iker/common/utils/numutils.py,sha256=p6Rz1qyCcUru3v1zDy2PM-nds2NWJdL5A_vLmG-kswk,4294
 iker/common/utils/randutils.py,sha256=Sxf852B18CJ-MfrEDsv1ROO_brmz79dRZ4jpJiH65v4,12843
 iker/common/utils/retry.py,sha256=H9lR6pp_jzgOwKTM-dOWIddjTlQbK-ijcwuDmVvurZM,8938
-iker/common/utils/s3utils.py,sha256=rb-JVCJuIbmVn4ml7MQ7qKD8Z25t8xnU_u4oY1-APe4,9368
 iker/common/utils/sequtils.py,sha256=Wc8RcbNjVYSJYZv_07SOKWfYjhmGWz9_RXWbG2-tE1o,25060
-iker/common/utils/shutils.py,sha256=44_Qkzkhrs9LsfDflsaY_4Va0IpVLU3o8K_NvqCB04w,7859
+iker/common/utils/shutils.py,sha256=dUm1Y7m8u1Ri_R5598oQJsxwgQaBnVzhtpcsL7_Vzp0,7916
 iker/common/utils/span.py,sha256=u_KuWi2U7QDMUotl4AeW2_57ItL3YhVDSeCwaOiFDvs,5963
 iker/common/utils/strutils.py,sha256=Tu_qFeH3K-SfwvMxdrZAc9iLPV8ZmtX4ntyyFGNslf8,5094
 iker/common/utils/testutils.py,sha256=2VieV5yeCDntSKQSpIeyqRT8BZmZYE_ArMeQz3g7fXY,5568
 iker/common/utils/typeutils.py,sha256=RVkYkFRgDrx77OHFH7PavMV0AIB0S8ly40rs4g7JWE4,8220
-iker_python_common-1.0.58.dist-info/METADATA,sha256=dI9O2Xjd_YKTQkPvtA_xf9ANw2-vdsJDFC1k8FO0m7A,894
-iker_python_common-1.0.58.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-iker_python_common-1.0.58.dist-info/top_level.txt,sha256=4_B8Prfc_lxFafFYTQThIU1ZqOYQ4pHHHnJ_fQ_oHs8,5
-iker_python_common-1.0.58.dist-info/RECORD,,
+iker_python_common-1.0.59.dist-info/METADATA,sha256=C3LhE0abnoE8e5KeGHrKj2DmN8RCL8fP7M9aWY6KVE0,813
+iker_python_common-1.0.59.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+iker_python_common-1.0.59.dist-info/top_level.txt,sha256=4_B8Prfc_lxFafFYTQThIU1ZqOYQ4pHHHnJ_fQ_oHs8,5
+iker_python_common-1.0.59.dist-info/RECORD,,

iker/common/utils/s3utils.py

@@ -1,270 +0,0 @@
-import concurrent.futures
-import contextlib
-import dataclasses
-import datetime
-import mimetypes
-import os
-import tempfile
-
-import boto3
-from botocore.client import BaseClient
-
-from iker.common.utils.shutils import glob_match, listfile, path_depth
-from iker.common.utils.strutils import is_empty, trim_to_none
-
-__all__ = [
-    "S3ObjectMeta",
-    "s3_make_client",
-    "s3_list_objects",
-    "s3_listfile",
-    "s3_cp_download",
-    "s3_cp_upload",
-    "s3_sync_download",
-    "s3_sync_upload",
-    "s3_pull_text",
-    "s3_push_text",
-]
-
-
-@dataclasses.dataclass
-class S3ObjectMeta(object):
-    key: str
-    last_modified: datetime.datetime
-    size: int
-
-
-def s3_make_client(
-    access_key_id: str = None,
-    secret_access_key: str = None,
-    region_name: str = None,
-    endpoint_url: str = None,
-) -> contextlib.AbstractContextManager[BaseClient]:
-    """
-    Creates an AWS S3 client as a context manager for safe resource handling.
-
-    :param access_key_id: AWS access key ID.
-    :param secret_access_key: AWS secret access key.
-    :param region_name: AWS service region name.
-    :param endpoint_url: AWS service endpoint URL.
-    :return: A context manager yielding an S3 client instance.
-    """
-    client = boto3.client("s3",
-                          region_name=trim_to_none(region_name),
-                          endpoint_url=trim_to_none(endpoint_url),
-                          aws_access_key_id=trim_to_none(access_key_id),
-                          aws_secret_access_key=trim_to_none(secret_access_key))
-    return contextlib.closing(client)
-
-
-def s3_list_objects(client: BaseClient, bucket: str, prefix: str, limit: int = None) -> list[S3ObjectMeta]:
-    """
-    Lists all objects from the given S3 ``bucket`` and ``prefix``.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param limit: Maximum number of objects to return (``None`` for all).
-    :return: List of ``S3ObjectMeta`` items.
-    """
-    entries = []
-
-    next_marker = None
-    while True:
-        if is_empty(next_marker):
-            response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix)
-        else:
-            response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix, Marker=next_marker)
-
-        entries.extend(response.get("Contents", []))
-
-        if limit is not None and len(entries) >= limit:
-            entries = entries[:limit]
-
-        if not response.get("IsTruncated"):
-            break
-
-        next_marker = response.get("NextMarker")
-        if is_empty(next_marker):
-            next_marker = entries[-1]["Key"]
-
-    return [S3ObjectMeta(key=e["Key"], last_modified=e["LastModified"], size=e["Size"]) for e in entries]
-
-
-def s3_listfile(
-    client: BaseClient,
-    bucket: str,
-    prefix: str,
-    *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-) -> list[S3ObjectMeta]:
-    """
-    Lists all objects from the given S3 ``bucket`` and ``prefix``, filtered by patterns and directory depth.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    :return: List of ``S3ObjectMeta`` items.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    objects = s3_list_objects(client, bucket, prefix)
-
-    def filter_object_meta(object_meta: S3ObjectMeta) -> bool:
-        if 0 < depth <= path_depth(prefix, os.path.dirname(object_meta.key)):
-            return False
-        if len(glob_match([os.path.basename(object_meta.key)], include_patterns, exclude_patterns)) == 0:
-            return False
-        return True
-
-    return list(filter(filter_object_meta, objects))
-
-
-def s3_cp_download(client: BaseClient, bucket: str, key: str, file_path: str):
-    """
-    Downloads an object from the given S3 ``bucket`` and ``key`` to a local file path.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param key: Object key.
-    :param file_path: Local file path to save the object.
-    """
-    client.download_file(bucket, key, file_path)
-
-
-def s3_cp_upload(client: BaseClient, file_path: str, bucket: str, key: str):
-    """
-    Uploads a local file to the given S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param file_path: Local file path to upload.
-    :param bucket: Bucket name.
-    :param key: Object key for the uploaded file.
-    """
-    t, _ = mimetypes.MimeTypes().guess_type(file_path)
-    client.upload_file(file_path, bucket, key, ExtraArgs={"ContentType": "binary/octet-stream" if t is None else t})
-
-
-def s3_sync_download(
-    client: BaseClient,
-    bucket: str,
-    prefix: str,
-    dir_path: str,
-    *,
-    max_workers: int = None,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-):
-    """
-    Recursively downloads all objects from the given S3 ``bucket`` and ``prefix`` to a local directory path, using a thread pool.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param dir_path: Local directory path to save objects.
-    :param max_workers: Maximum number of worker threads.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    objects = s3_listfile(client,
-                          bucket,
-                          prefix,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
-
-    def download_file(key: str):
-        file_path = os.path.join(dir_path, key[len(prefix):])
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        s3_cp_download(client, bucket, key, file_path)
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        concurrent.futures.wait([executor.submit(download_file, obj.key) for obj in objects],
-                                return_when=concurrent.futures.FIRST_EXCEPTION)
-
-
-def s3_sync_upload(
-    client: BaseClient,
-    dir_path: str,
-    bucket: str,
-    prefix: str,
-    *,
-    max_workers: int = None,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-):
-    """
-    Recursively uploads all files from a local directory to the given S3 ``bucket`` and ``prefix``, using a thread pool.
-
-    :param client: AWS S3 client instance.
-    :param dir_path: Local directory path to upload from.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix for uploaded files.
-    :param max_workers: Maximum number of worker threads.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    file_paths = listfile(dir_path,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
-
-    def upload_file(file_path: str):
-        s3_cp_upload(client, file_path, bucket, prefix + os.path.relpath(file_path, dir_path))
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        concurrent.futures.wait([executor.submit(upload_file, file_path) for file_path in file_paths],
-                                return_when=concurrent.futures.FIRST_EXCEPTION)
-
-
-def s3_pull_text(client: BaseClient, bucket: str, key: str, encoding: str = None) -> str:
-    """
-    Downloads and decodes text content stored as an object in the given S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param key: Object key storing the text.
-    :param encoding: String encoding to use (defaults to UTF-8).
-    :return: The decoded text content.
-    """
-    with tempfile.TemporaryFile() as fp:
-        client.download_fileobj(bucket, key, fp)
-        fp.seek(0)
-        return fp.read().decode(encoding or "utf-8")
-
-
-def s3_push_text(client: BaseClient, text: str, bucket: str, key: str, encoding: str = None):
-    """
-    Uploads the given text as an object to the specified S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param text: Text content to upload.
-    :param bucket: Bucket name.
-    :param key: Object key to store the text.
-    :param encoding: String encoding to use (defaults to UTF-8).
-    """
-    with tempfile.TemporaryFile() as fp:
-        fp.write(text.encode(encoding or "utf-8"))
-        fp.seek(0)
-        client.upload_fileobj(fp, bucket, key)