iker-python-common 1.0.58__tar.gz → 1.0.59__tar.gz

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: iker-python-common
-Version: 1.0.58
+Version: 1.0.59
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.12
-Requires-Dist: boto3>=1.35
 Requires-Dist: docker>=7.1
 Requires-Dist: numpy>=2.3
 Requires-Dist: psycopg>=3.2
@@ -16,7 +15,6 @@ Provides-Extra: all
 Requires-Dist: iker-python-common; extra == "all"
 Provides-Extra: test
 Requires-Dist: ddt>=1.7; extra == "test"
-Requires-Dist: moto[all,ec2,s3]>=5.0; extra == "test"
 Requires-Dist: pytest-cov>=5.0; extra == "test"
 Requires-Dist: pytest-mysql>=3.0; extra == "test"
 Requires-Dist: pytest-order>=1.3; extra == "test"

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/pyproject.toml

@@ -8,7 +8,6 @@ build-backend = "setuptools.build_meta"
 
 [dependency-groups]
 dev = [
-    "boto3>=1.35",
     "docker>=7.1",
     "numpy>=2.3",
     "psycopg>=3.2",
@@ -17,7 +16,6 @@ dev = [
 ]
 test = [
     "ddt>=1.7",
-    "moto[ec2,s3,all]>=5.0",
     "pytest-cov>=5.0",
     "pytest-mysql>=3.0",
     "pytest-order>=1.3",
@@ -36,7 +34,6 @@ classifiers = [
     "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
-    "boto3>=1.35",
     "docker>=7.1",
     "numpy>=2.3",
     "psycopg>=3.2",
@@ -50,7 +47,6 @@ all = [
 ]
 test = [
     "ddt>=1.7",
-    "moto[ec2,s3,all]>=5.0",
     "pytest-cov>=5.0",
     "pytest-mysql>=3.0",
     "pytest-order>=1.3",

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/src/iker/common/utils/shutils.py

@@ -95,7 +95,11 @@ def path_depth(root: str, child: str) -> int:
     return child_expanded[len(root_expanded):].count(os.sep)
 
 
-def glob_match(names: list[str], include_patterns: list[str] = None, exclude_patterns: list[str] = None) -> list[str]:
+def glob_match(
+    names: list[str],
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
+) -> list[str]:
     """
     Applies the given inclusive and exclusive glob patterns to the given ``names`` and returns the filtered result.
 
@@ -121,8 +125,8 @@ class CopyFuncProtocol(Protocol):
 def listfile(
     path: str,
     *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
     depth: int = 0,
 ) -> list[str]:
     """
@@ -153,8 +157,8 @@ def copy(
     src: str,
     dst: str,
     *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
+    include_patterns: list[str] | None = None,
+    exclude_patterns: list[str] | None = None,
     depth: int = 0,
     follow_symlinks: bool = False,
     ignore_dangling_symlinks: bool = False,

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/src/iker_python_common.egg-info/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: iker-python-common
-Version: 1.0.58
+Version: 1.0.59
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.12
-Requires-Dist: boto3>=1.35
 Requires-Dist: docker>=7.1
 Requires-Dist: numpy>=2.3
 Requires-Dist: psycopg>=3.2
@@ -16,7 +15,6 @@ Provides-Extra: all
 Requires-Dist: iker-python-common; extra == "all"
 Provides-Extra: test
 Requires-Dist: ddt>=1.7; extra == "test"
-Requires-Dist: moto[all,ec2,s3]>=5.0; extra == "test"
 Requires-Dist: pytest-cov>=5.0; extra == "test"
 Requires-Dist: pytest-mysql>=3.0; extra == "test"
 Requires-Dist: pytest-order>=1.3; extra == "test"

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/src/iker_python_common.egg-info/SOURCES.txt

@@ -10,16 +10,6 @@ setup.py
 resources/unittest/config/config.cfg
 resources/unittest/csv/data.csv
 resources/unittest/csv/data.tsv
-resources/unittest/s3utils/dir.baz/file.bar.baz
-resources/unittest/s3utils/dir.baz/file.foo.bar
-resources/unittest/s3utils/dir.baz/file.foo.baz
-resources/unittest/s3utils/dir.foo/file.bar
-resources/unittest/s3utils/dir.foo/file.baz
-resources/unittest/s3utils/dir.foo/file.foo
-resources/unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz
-resources/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar
-resources/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz
-resources/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz
 resources/unittest/shutils/dir.baz/file.bar.baz
 resources/unittest/shutils/dir.baz/file.foo.bar
 resources/unittest/shutils/dir.baz/file.foo.baz
@@ -44,7 +34,6 @@ src/iker/common/utils/logger.py
 src/iker/common/utils/numutils.py
 src/iker/common/utils/randutils.py
 src/iker/common/utils/retry.py
-src/iker/common/utils/s3utils.py
 src/iker/common/utils/sequtils.py
 src/iker/common/utils/shutils.py
 src/iker/common/utils/span.py
@@ -72,7 +61,6 @@ test/iker_tests/common/utils/logger_test.py
 test/iker_tests/common/utils/numutils_test.py
 test/iker_tests/common/utils/randutils_test.py
 test/iker_tests/common/utils/retry_test.py
-test/iker_tests/common/utils/s3utils_test.py
 test/iker_tests/common/utils/sequtils_test.py
 test/iker_tests/common/utils/shutils_test.py
 test/iker_tests/common/utils/span_test.py

{iker_python_common-1.0.58 → iker_python_common-1.0.59}/src/iker_python_common.egg-info/requires.txt

@@ -1,4 +1,3 @@
-boto3>=1.35
 docker>=7.1
 numpy>=2.3
 psycopg>=3.2
@@ -10,7 +9,6 @@ iker-python-common
 
 [test]
 ddt>=1.7
-moto[all,ec2,s3]>=5.0
 pytest-cov>=5.0
 pytest-mysql>=3.0
 pytest-order>=1.3

iker_python_common-1.0.58/src/iker/common/utils/s3utils.py

@@ -1,270 +0,0 @@
-import concurrent.futures
-import contextlib
-import dataclasses
-import datetime
-import mimetypes
-import os
-import tempfile
-
-import boto3
-from botocore.client import BaseClient
-
-from iker.common.utils.shutils import glob_match, listfile, path_depth
-from iker.common.utils.strutils import is_empty, trim_to_none
-
-__all__ = [
-    "S3ObjectMeta",
-    "s3_make_client",
-    "s3_list_objects",
-    "s3_listfile",
-    "s3_cp_download",
-    "s3_cp_upload",
-    "s3_sync_download",
-    "s3_sync_upload",
-    "s3_pull_text",
-    "s3_push_text",
-]
-
-
-@dataclasses.dataclass
-class S3ObjectMeta(object):
-    key: str
-    last_modified: datetime.datetime
-    size: int
-
-
-def s3_make_client(
-    access_key_id: str = None,
-    secret_access_key: str = None,
-    region_name: str = None,
-    endpoint_url: str = None,
-) -> contextlib.AbstractContextManager[BaseClient]:
-    """
-    Creates an AWS S3 client as a context manager for safe resource handling.
-
-    :param access_key_id: AWS access key ID.
-    :param secret_access_key: AWS secret access key.
-    :param region_name: AWS service region name.
-    :param endpoint_url: AWS service endpoint URL.
-    :return: A context manager yielding an S3 client instance.
-    """
-    client = boto3.client("s3",
-                          region_name=trim_to_none(region_name),
-                          endpoint_url=trim_to_none(endpoint_url),
-                          aws_access_key_id=trim_to_none(access_key_id),
-                          aws_secret_access_key=trim_to_none(secret_access_key))
-    return contextlib.closing(client)
-
-
-def s3_list_objects(client: BaseClient, bucket: str, prefix: str, limit: int = None) -> list[S3ObjectMeta]:
-    """
-    Lists all objects from the given S3 ``bucket`` and ``prefix``.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param limit: Maximum number of objects to return (``None`` for all).
-    :return: List of ``S3ObjectMeta`` items.
-    """
-    entries = []
-
-    next_marker = None
-    while True:
-        if is_empty(next_marker):
-            response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix)
-        else:
-            response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix, Marker=next_marker)
-
-        entries.extend(response.get("Contents", []))
-
-        if limit is not None and len(entries) >= limit:
-            entries = entries[:limit]
-
-        if not response.get("IsTruncated"):
-            break
-
-        next_marker = response.get("NextMarker")
-        if is_empty(next_marker):
-            next_marker = entries[-1]["Key"]
-
-    return [S3ObjectMeta(key=e["Key"], last_modified=e["LastModified"], size=e["Size"]) for e in entries]
-
-
-def s3_listfile(
-    client: BaseClient,
-    bucket: str,
-    prefix: str,
-    *,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-) -> list[S3ObjectMeta]:
-    """
-    Lists all objects from the given S3 ``bucket`` and ``prefix``, filtered by patterns and directory depth.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    :return: List of ``S3ObjectMeta`` items.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    objects = s3_list_objects(client, bucket, prefix)
-
-    def filter_object_meta(object_meta: S3ObjectMeta) -> bool:
-        if 0 < depth <= path_depth(prefix, os.path.dirname(object_meta.key)):
-            return False
-        if len(glob_match([os.path.basename(object_meta.key)], include_patterns, exclude_patterns)) == 0:
-            return False
-        return True
-
-    return list(filter(filter_object_meta, objects))
-
-
-def s3_cp_download(client: BaseClient, bucket: str, key: str, file_path: str):
-    """
-    Downloads an object from the given S3 ``bucket`` and ``key`` to a local file path.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param key: Object key.
-    :param file_path: Local file path to save the object.
-    """
-    client.download_file(bucket, key, file_path)
-
-
-def s3_cp_upload(client: BaseClient, file_path: str, bucket: str, key: str):
-    """
-    Uploads a local file to the given S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param file_path: Local file path to upload.
-    :param bucket: Bucket name.
-    :param key: Object key for the uploaded file.
-    """
-    t, _ = mimetypes.MimeTypes().guess_type(file_path)
-    client.upload_file(file_path, bucket, key, ExtraArgs={"ContentType": "binary/octet-stream" if t is None else t})
-
-
-def s3_sync_download(
-    client: BaseClient,
-    bucket: str,
-    prefix: str,
-    dir_path: str,
-    *,
-    max_workers: int = None,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-):
-    """
-    Recursively downloads all objects from the given S3 ``bucket`` and ``prefix`` to a local directory path, using a thread pool.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix.
-    :param dir_path: Local directory path to save objects.
-    :param max_workers: Maximum number of worker threads.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    objects = s3_listfile(client,
-                          bucket,
-                          prefix,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
-
-    def download_file(key: str):
-        file_path = os.path.join(dir_path, key[len(prefix):])
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        s3_cp_download(client, bucket, key, file_path)
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        concurrent.futures.wait([executor.submit(download_file, obj.key) for obj in objects],
-                                return_when=concurrent.futures.FIRST_EXCEPTION)
-
-
-def s3_sync_upload(
-    client: BaseClient,
-    dir_path: str,
-    bucket: str,
-    prefix: str,
-    *,
-    max_workers: int = None,
-    include_patterns: list[str] = None,
-    exclude_patterns: list[str] = None,
-    depth: int = 0,
-):
-    """
-    Recursively uploads all files from a local directory to the given S3 ``bucket`` and ``prefix``, using a thread pool.
-
-    :param client: AWS S3 client instance.
-    :param dir_path: Local directory path to upload from.
-    :param bucket: Bucket name.
-    :param prefix: Object keys prefix for uploaded files.
-    :param max_workers: Maximum number of worker threads.
-    :param include_patterns: Inclusive glob patterns applied to filenames.
-    :param exclude_patterns: Exclusive glob patterns applied to filenames.
-    :param depth: Maximum depth of subdirectories to include in the scan.
-    """
-
-    # We add trailing slash "/" to the prefix if it is absent
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
-
-    file_paths = listfile(dir_path,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
-
-    def upload_file(file_path: str):
-        s3_cp_upload(client, file_path, bucket, prefix + os.path.relpath(file_path, dir_path))
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        concurrent.futures.wait([executor.submit(upload_file, file_path) for file_path in file_paths],
-                                return_when=concurrent.futures.FIRST_EXCEPTION)
-
-
-def s3_pull_text(client: BaseClient, bucket: str, key: str, encoding: str = None) -> str:
-    """
-    Downloads and decodes text content stored as an object in the given S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param bucket: Bucket name.
-    :param key: Object key storing the text.
-    :param encoding: String encoding to use (defaults to UTF-8).
-    :return: The decoded text content.
-    """
-    with tempfile.TemporaryFile() as fp:
-        client.download_fileobj(bucket, key, fp)
-        fp.seek(0)
-        return fp.read().decode(encoding or "utf-8")
-
-
-def s3_push_text(client: BaseClient, text: str, bucket: str, key: str, encoding: str = None):
-    """
-    Uploads the given text as an object to the specified S3 ``bucket`` and ``key``.
-
-    :param client: AWS S3 client instance.
-    :param text: Text content to upload.
-    :param bucket: Bucket name.
-    :param key: Object key to store the text.
-    :param encoding: String encoding to use (defaults to UTF-8).
-    """
-    with tempfile.TemporaryFile() as fp:
-        fp.write(text.encode(encoding or "utf-8"))
-        fp.seek(0)
-        client.upload_fileobj(fp, bucket, key)

iker_python_common-1.0.58/test/iker_tests/common/utils/s3utils_test.py

@@ -1,422 +0,0 @@
-import tempfile
-import unittest
-
-import ddt
-import moto
-
-from iker.common.utils.randutils import randomizer
-from iker.common.utils.s3utils import *
-from iker.common.utils.shutils import listfile
-from iker.common.utils.testutils import norm_path
-from iker_tests import resources_directory
-
-
-@ddt.ddt
-class S3UtilsTest(unittest.TestCase):
-
-    def test_s3_list_object__random_text_files(self):
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket="dummy-bucket")
-
-            data = []
-            for i in range(0, 1000):
-                text = randomizer().random_alphanumeric(randomizer().next_int(1000, 2000))
-                key = "dummy_prefix/%03d" % i
-                data.append((key, text))
-
-            for key, text in data:
-                s3_push_text(client, text, "dummy-bucket", key)
-
-            result = s3_list_objects(client, "dummy-bucket", "dummy_prefix/")
-
-            for key, text in data:
-                self.assertTrue(any(o.key == key for o in result))
-                self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
-
-    def test_s3_listfile__random_text_files(self):
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket="dummy-bucket")
-
-            data = []
-            for i in range(0, 1000):
-                text = randomizer().random_alphanumeric(randomizer().next_int(1000, 2000))
-                key = "dummy_prefix/%03d" % i
-                data.append((key, text))
-
-            for key, text in data:
-                s3_push_text(client, text, "dummy-bucket", key)
-
-            result = s3_listfile(client, "dummy-bucket", "dummy_prefix/")
-
-            for key, text in data:
-                self.assertTrue(any(o.key == key for o in result))
-                self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
-
-    data_s3_listfile = [
-        (
-            "{res_dir}/unittest/s3utils/",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            [],
-            0,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.baz/file.foo.baz",
-                "unittest/s3utils/dir.baz/file.bar.baz",
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.baz",
-                "unittest/s3utils/dir.foo/file.foo",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            [],
-            0,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.baz/file.foo.baz",
-                "unittest/s3utils/dir.baz/file.bar.baz",
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.baz",
-                "unittest/s3utils/dir.foo/file.foo",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils/dir.foo",
-            "dummy-bucket",
-            "unittest/s3utils/dir.foo",
-            [],
-            [],
-            0,
-            [
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.baz",
-                "unittest/s3utils/dir.foo/file.foo",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils/dir.baz",
-            "dummy-bucket",
-            "unittest/s3utils/dir.baz",
-            [],
-            [],
-            0,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.baz/file.foo.baz",
-                "unittest/s3utils/dir.baz/file.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            ["*.foo", "*.bar"],
-            ["*.foo.bar"],
-            0,
-            [
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.foo",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            ["*.foo", "*.bar"],
-            [],
-            0,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.foo",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            ["*.baz"],
-            0,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.foo",
-                "unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils",
-            [],
-            [],
-            2,
-            [
-                "unittest/s3utils/dir.baz/file.foo.bar",
-                "unittest/s3utils/dir.baz/file.foo.baz",
-                "unittest/s3utils/dir.baz/file.bar.baz",
-                "unittest/s3utils/dir.foo/file.bar",
-                "unittest/s3utils/dir.foo/file.baz",
-                "unittest/s3utils/dir.foo/file.foo",
-            ],
-        ),
-    ]
-
-    @ddt.idata(data_s3_listfile)
-    @ddt.unpack
-    def test_s3_listfile(self, src, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket=bucket)
-
-            s3_sync_upload(client,
-                           src.format(res_dir=resources_directory),
-                           bucket,
-                           prefix)
-
-            object_metas = s3_listfile(client,
-                                       bucket,
-                                       prefix,
-                                       include_patterns=include_patterns,
-                                       exclude_patterns=exclude_patterns,
-                                       depth=depth)
-
-            self.assertSetEqual(set(map(lambda x: norm_path(x.key), object_metas)),
-                                set(map(lambda x: norm_path(x), expect)))
-
-    data_s3_sync = [
-        (
-            "{res_dir}/unittest/s3utils/",
-            "{tmp_dir}/unittest/s3utils/",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            [],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "{tmp_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            [],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils/dir.foo",
-            "{tmp_dir}/unittest/s3utils/dir.foo",
-            "dummy-bucket",
-            "unittest/s3utils/dir.foo",
-            [],
-            [],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils/dir.baz",
-            "{tmp_dir}/unittest/s3utils/dir.baz",
-            "dummy-bucket",
-            "unittest/s3utils/dir.baz",
-            [],
-            [],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.bar.baz",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "{tmp_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            ["*.foo", "*.bar"],
-            ["*.foo.bar"],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "{tmp_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            ["*.foo", "*.bar"],
-            [],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "{tmp_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils/",
-            [],
-            ["*.baz"],
-            0,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-                "{tmp_dir}/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar",
-            ],
-        ),
-        (
-            "{res_dir}/unittest/s3utils",
-            "{tmp_dir}/unittest/s3utils",
-            "dummy-bucket",
-            "unittest/s3utils",
-            [],
-            [],
-            2,
-            [
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.bar",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.foo.baz",
-                "{tmp_dir}/unittest/s3utils/dir.baz/file.bar.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.bar",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.baz",
-                "{tmp_dir}/unittest/s3utils/dir.foo/file.foo",
-            ],
-        ),
-    ]
-
-    @ddt.idata(data_s3_sync)
-    @ddt.unpack
-    def test_s3_sync(self, src, dst, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket=bucket)
-
-            s3_sync_upload(client,
-                           src.format(res_dir=resources_directory),
-                           bucket,
-                           prefix,
-                           include_patterns=include_patterns,
-                           exclude_patterns=exclude_patterns,
-                           depth=depth)
-
-            with tempfile.TemporaryDirectory() as temp_directory:
-                s3_sync_download(client,
-                                 bucket,
-                                 prefix,
-                                 dst.format(tmp_dir=temp_directory))
-
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(dst.format(tmp_dir=temp_directory)))),
-                                    set(map(lambda x: norm_path(x.format(tmp_dir=temp_directory)), expect)))
-
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket=bucket)
-
-            s3_sync_upload(client,
-                           src.format(res_dir=resources_directory),
-                           bucket,
-                           prefix)
-
-            with tempfile.TemporaryDirectory() as temp_directory:
-                s3_sync_download(client,
-                                 bucket,
-                                 prefix,
-                                 dst.format(tmp_dir=temp_directory),
-                                 include_patterns=include_patterns,
-                                 exclude_patterns=exclude_patterns,
-                                 depth=depth)
-
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(dst.format(tmp_dir=temp_directory)))),
-                                    set(map(lambda x: norm_path(x.format(tmp_dir=temp_directory)), expect)))
-
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket=bucket)
-
-            s3_sync_upload(client,
-                           src.format(res_dir=resources_directory),
-                           bucket,
-                           prefix,
-                           include_patterns=include_patterns,
-                           depth=depth)
-
-            with tempfile.TemporaryDirectory() as temp_directory:
-                s3_sync_download(client,
-                                 bucket,
-                                 prefix,
-                                 dst.format(tmp_dir=temp_directory),
-                                 exclude_patterns=exclude_patterns,
-                                 depth=depth)
-
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(dst.format(tmp_dir=temp_directory)))),
-                                    set(map(lambda x: norm_path(x.format(tmp_dir=temp_directory)), expect)))
-
-    data_s3_text = [
-        ("dummy-bucket", "dummy/key", "dummy content", None),
-        ("dummy-bucket", "dummy/key.alpha", "Old MacDonald had a farm", None),
-        ("dummy-bucket", "dummy/key.beta", "Ee-i-ee-i-o", None),
-        ("dummy-bucket", "dummy/key", "dummy content", "ascii"),
-        ("dummy-bucket", "dummy/key.alpha", "Old MacDonald had a farm", "ascii"),
-        ("dummy-bucket", "dummy/key.beta", "Ee-i-ee-i-o", "ascii"),
-    ]
-
-    @ddt.idata(data_s3_text)
-    @ddt.unpack
-    def test_s3_text(self, bucket, key, text, encoding):
-        with moto.mock_aws(), s3_make_client(region_name="us-east-1") as client:
-            client.create_bucket(Bucket=bucket)
-
-            s3_push_text(client, text, bucket, key, encoding=encoding)
-            self.assertEqual(s3_pull_text(client, bucket, key, encoding=encoding), text)