megfile 2.2.1.post1__py3-none-any.whl → 2.2.3__py3-none-any.whl

megfile/cli.py

@@ -141,10 +141,18 @@ def cp(
                                 2) as executor:
             if progress_bar:
                 smart_sync_with_progress(
-                    src_path, dst_path, followlinks=True, map_func=executor.map)
+                    src_path,
+                    dst_path,
+                    followlinks=True,
+                    map_func=executor.map,
+                    force=True)
             else:
                 smart_sync(
-                    src_path, dst_path, followlinks=True, map_func=executor.map)
+                    src_path,
+                    dst_path,
+                    followlinks=True,
+                    map_func=executor.map,
+                    force=True)
     else:
         if progress_bar:
             file_size = smart_stat(src_path).size
@@ -251,7 +259,14 @@ def rm(path: str, recursive: bool):
     type=click.INT,
     default=8,
     help='Number of concurrent workers.')
-def sync(src_path: str, dst_path: str, progress_bar: bool, worker):
+@click.option(
+    '-f',
+    '--force',
+    is_flag=True,
+    help='Copy files forcely, ignore same files.')
+def sync(
+        src_path: str, dst_path: str, progress_bar: bool, worker: int,
+        force: bool):
     with ThreadPoolExecutor(max_workers=worker) as executor:
         if has_magic(src_path):
             root_dir = get_non_glob_dir(src_path)
@@ -278,6 +293,7 @@ def sync(src_path: str, dst_path: str, progress_bar: bool, worker):
                     callback_after_copy_file=callback_after_copy_file,
                     src_file_stats=path_stats,
                     map_func=executor.map,
+                    force=force,
                 )
 
                 tbar.close()
@@ -288,14 +304,23 @@ def sync(src_path: str, dst_path: str, progress_bar: bool, worker):
                     dst_path,
                     src_file_stats=path_stats,
                     map_func=executor.map,
+                    force=force,
                 )
         else:
             if progress_bar:
                 smart_sync_with_progress(
-                    src_path, dst_path, followlinks=True, map_func=executor.map)
+                    src_path,
+                    dst_path,
+                    followlinks=True,
+                    map_func=executor.map,
+                    force=force)
             else:
                 smart_sync(
-                    src_path, dst_path, followlinks=True, map_func=executor.map)
+                    src_path,
+                    dst_path,
+                    followlinks=True,
+                    map_func=executor.map,
+                    force=force)
 
 
 @cli.command(short_help="Make the path if it doesn't already exist.")

megfile/errors.py

@@ -2,7 +2,8 @@ import time
 from contextlib import contextmanager
 from functools import wraps
 from logging import getLogger
-from typing import Callable, List, Optional
+from shutil import SameFileError
+from typing import Callable, Optional
 
 import botocore.exceptions
 import requests.exceptions
@@ -27,6 +28,7 @@ __all__ = [
     'ProtocolExistsError',
     'ProtocolNotFoundError',
     'S3UnknownError',
+    'SameFileError',
     'translate_http_error',
     'translate_s3_error',
     'patch_method',
@@ -305,7 +307,9 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
     elif isinstance(s3_error, ClientError):
         code = client_error_code(s3_error)
         if code in ('NoSuchBucket'):
-            return S3BucketNotFoundError('No such bucket: %r' % s3_url)
+            return S3BucketNotFoundError(
+                'No such bucket: %r' %
+                s3_error.response.get('Error', {}).get('BucketName') or s3_url)  # pytype: disable=attribute-error
         if code in ('404', 'NoSuchKey'):
             return S3FileNotFoundError('No such file: %r' % s3_url)
         if code in ('401', '403', 'AccessDenied'):

megfile/fs.py

@@ -345,13 +345,19 @@ def fs_copy(
     return FSPath(src_path).copy(dst_path, callback, followlinks)
 
 
-def fs_sync(src_path: PathLike, dst_path: PathLike, followlinks: bool = False):
+def fs_sync(
+        src_path: PathLike,
+        dst_path: PathLike,
+        followlinks: bool = False,
+        force: bool = False) -> None:
     '''Force write of everything to disk.
 
     :param src_path: Given path
     :param dst_path: Target file path
+    :param followlinks: False if regard symlink as file, else True
+    :param force: Sync file forcely, do not ignore same files
     '''
-    return FSPath(src_path).sync(dst_path, followlinks)
+    return FSPath(src_path).sync(dst_path, followlinks, force)
 
 
 def fs_symlink(src_path: PathLike, dst_path: PathLike) -> None:

megfile/fs_path.py

@@ -765,10 +765,16 @@ class FSPath(URIPath):
             else:
                 raise
 
-    def sync(self, dst_path: PathLike, followlinks: bool = False):
+    def sync(
+            self,
+            dst_path: PathLike,
+            followlinks: bool = False,
+            force: bool = False) -> None:
         '''Force write of everything to disk.
 
         :param dst_path: Target file path
+        :param followlinks: False if regard symlink as file, else True
+        :param force: Sync file forcely, do not ignore same files
         '''
         if self.is_dir(followlinks=followlinks):
 
@@ -776,7 +782,7 @@ class FSPath(URIPath):
                 ignore_files = []
                 for name in names:
                     dst_obj = self.from_path(dst_path).joinpath(name)
-                    if dst_obj.exists() and is_same_file(
+                    if not force and dst_obj.exists() and is_same_file(
                             self.joinpath(name).stat(), dst_obj.stat(), 'copy'):
                         ignore_files.append(name)
                 return ignore_files
@@ -852,7 +858,7 @@ class FSPath(URIPath):
 
     def open(
             self,
-            mode: str,
+            mode: str = 'r',
             buffering=-1,
             encoding=None,
             errors=None,

megfile/pathlike.py

@@ -317,7 +317,7 @@ class BasePath:
         """Remove (delete) the directory."""
 
     @method_not_implemented
-    def open(self, mode: str, **kwargs) -> IO[AnyStr]:  # type: ignore
+    def open(self, mode: str = 'r', **kwargs) -> IO[AnyStr]:  # type: ignore
         """Open the file with mode."""
 
     @method_not_implemented

megfile/s3.py

@@ -64,7 +64,6 @@ def s3_access(
         followlinks: bool = False) -> bool:
     '''
     Test if path has access permission described by mode
-    Using head_bucket(), now READ/WRITE are same.
 
     :param path: Given path
     :param mode: access mode
@@ -325,15 +324,19 @@ def s3_copy(
 
 
 def s3_sync(
-        src_url: PathLike, dst_url: PathLike,
-        followlinks: bool = False) -> None:
+        src_url: PathLike,
+        dst_url: PathLike,
+        followlinks: bool = False,
+        force: bool = False) -> None:
     '''
     Copy file/directory on src_url to dst_url
 
     :param src_url: Given path
     :param dst_url: Given destination path
+    :param followlinks: False if regard symlink as file, else True
+    :param force: Sync file forcely, do not ignore same files
     '''
-    return S3Path(src_url).sync(dst_url, followlinks)
+    return S3Path(src_url).sync(dst_url, followlinks, force)
 
 
 def s3_symlink(src_path: PathLike, dst_path: PathLike) -> None:

megfile/s3_path.py

@@ -12,7 +12,7 @@ import boto3
 import botocore
 from botocore.awsrequest import AWSResponse
 
-from megfile.errors import S3BucketNotFoundError, S3ConfigError, S3FileExistsError, S3FileNotFoundError, S3IsADirectoryError, S3NameTooLongError, S3NotADirectoryError, S3NotALinkError, S3PermissionError, S3UnknownError, UnsupportedError, _create_missing_ok_generator
+from megfile.errors import S3BucketNotFoundError, S3ConfigError, S3FileExistsError, S3FileNotFoundError, S3IsADirectoryError, S3NameTooLongError, S3NotADirectoryError, S3NotALinkError, S3PermissionError, S3UnknownError, SameFileError, UnsupportedError, _create_missing_ok_generator
 from megfile.errors import _logger as error_logger
 from megfile.errors import patch_method, raise_s3_error, s3_error_code_should_retry, s3_should_retry, translate_fs_error, translate_s3_error
 from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, PathLike, StatResult, URIPath
@@ -189,6 +189,20 @@ def get_s3_client(
 
     :returns: S3 client
     '''
+    addressing_style_env_key = 'AWS_S3_ADDRESSING_STYLE'
+    if profile_name:
+        addressing_style_env_key = f'{profile_name}__AWS_S3_ADDRESSING_STYLE'.upper(
+        )
+    addressing_style = os.environ.get(addressing_style_env_key)
+    if addressing_style:
+        if config:
+            config = config.merge(
+                botocore.config.Config(
+                    s3={'addressing_style': addressing_style}))
+        else:
+            config = botocore.config.Config(
+                s3={'addressing_style': addressing_style})
+
     if cache_key is not None:
         return thread_local(
             cache_key, get_s3_client, config=config, profile_name=profile_name)
@@ -518,10 +532,6 @@ def _s3_binary_mode(s3_open_func):
                 raise S3FileExistsError('File exists: %r' % s3_url)
             mode = mode.replace('x', 'w')
 
-        if 'w' in mode or 'a' in mode:
-            if not S3Path(s3_url).hasbucket():
-                raise S3BucketNotFoundError('No such bucket: %r' % s3_url)
-
         fileobj = s3_open_func(s3_url, get_binary_mode(mode), **kwargs)
         if 'b' not in mode:
             fileobj = io.TextIOWrapper(
@@ -1260,7 +1270,6 @@ class S3Path(URIPath):
             followlinks: bool = False) -> bool:
         '''
         Test if path has access permission described by mode
-        Using head_bucket(), now READ/WRITE are same.
 
         :param mode: access mode
         :returns: bool, if the bucket of s3_url has read/write access.
@@ -1271,7 +1280,7 @@ class S3Path(URIPath):
                 s3_url = self.readlink().path_with_protocol
             except S3NotALinkError:
                 pass
-        bucket, _ = parse_s3_url(s3_url)  # only check bucket accessibility
+        bucket, key = parse_s3_url(s3_url)  # only check bucket accessibility
         if not bucket:
             raise Exception("No available bucket")
         if not isinstance(mode, Access):
@@ -1280,15 +1289,38 @@ class S3Path(URIPath):
                 .format(mode, ', '.join([str(a) for a in Access])))
         if mode not in (Access.READ, Access.WRITE):
             raise TypeError('Unsupported mode: {}'.format(mode))
+
         try:
-            self._client.head_bucket(Bucket=bucket)
+            if not self.exists():
+                return False
         except Exception as error:
             error = translate_s3_error(error, s3_url)
-            if isinstance(error, (S3PermissionError, S3FileNotFoundError,
-                                  S3BucketNotFoundError)):
+            if isinstance(error, S3PermissionError):
+                return False
+            raise error
+
+        if mode == Access.READ:
+            return True
+        try:
+            if not key:
+                key = 'test'
+            elif key.endswith('/'):
+                key = key[:-1]
+            upload_id = self._client.create_multipart_upload(
+                Bucket=bucket,
+                Key=key,
+            )['UploadId']
+            self._client.abort_multipart_upload(
+                Bucket=bucket,
+                Key=key,
+                UploadId=upload_id,
+            )
+            return True
+        except Exception as error:
+            error = translate_s3_error(error, s3_url)
+            if isinstance(error, S3PermissionError):
                 return False
             raise error
-        return True
 
     def exists(self, followlinks: bool = False) -> bool:
         '''
@@ -1302,7 +1334,7 @@ class S3Path(URIPath):
         if not bucket:  # s3:// => True, s3:///key => False
             return not key
 
-        return self.is_dir() or self.is_file(followlinks)
+        return self.is_file(followlinks) or self.is_dir()
 
     def getmtime(self, follow_symlinks: bool = False) -> float:
         '''
@@ -1531,10 +1563,21 @@ class S3Path(URIPath):
             self._client.head_bucket(Bucket=bucket)
         except Exception as error:
             error = translate_s3_error(error, self.path_with_protocol)
-            if isinstance(error,
-                          (S3UnknownError, S3ConfigError, S3PermissionError)):
+            if isinstance(error, S3PermissionError):
+                # Aliyun OSS doesn't give bucket api permission when you only have read and write permission
+                try:
+                    self._client.list_objects_v2(Bucket=bucket, MaxKeys=1)
+                    return True
+                except Exception as error2:
+                    error2 = translate_s3_error(error2, self.path_with_protocol)
+                    if isinstance(
+                            error2,
+                        (S3UnknownError, S3ConfigError, S3PermissionError)):
+                        raise error2
+                    return False
+            elif isinstance(error, (S3UnknownError, S3ConfigError)):
                 raise error
-            if isinstance(error, S3FileNotFoundError):
+            elif isinstance(error, S3FileNotFoundError):
                 return False
 
         return True
@@ -1978,6 +2021,10 @@ class S3Path(URIPath):
         src_url = self.path_with_protocol
         src_bucket, src_key = parse_s3_url(src_url)
         dst_bucket, dst_key = parse_s3_url(dst_url)
+        if dst_bucket == src_bucket and src_key.rstrip('/') == dst_key.rstrip(
+                '/'):
+            raise SameFileError(
+                f"'{src_url}' and '{dst_url}' are the same file")
 
         if not src_bucket:
             raise S3BucketNotFoundError('Empty bucket name: %r' % src_url)
@@ -1996,7 +2043,7 @@ class S3Path(URIPath):
             except S3NotALinkError:
                 pass
 
-        try:
+        with raise_s3_error(f"'{src_url}' or '{dst_url}'"):
             self._client.copy(
                 {
                     'Bucket': src_bucket,
@@ -2005,28 +2052,22 @@ class S3Path(URIPath):
                 Bucket=dst_bucket,
                 Key=dst_key,
                 Callback=callback)
-        except Exception as error:
-            error = translate_s3_error(error, dst_url)
-            # Error can't help tell which is problematic
-            if isinstance(error, S3BucketNotFoundError):
-                if not self.hasbucket():
-                    raise S3BucketNotFoundError('No such bucket: %r' % src_url)
-            elif isinstance(error, S3FileNotFoundError):
-                if not self.is_file():
-                    raise S3FileNotFoundError('No such file: %r' % src_url)
-            raise error
 
-    def sync(self, dst_url: PathLike, followlinks: bool = False) -> None:
+    def sync(
+            self, dst_url: PathLike, followlinks: bool = False,
+            force: bool = False) -> None:
         '''
         Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
+        :param followlinks: False if regard symlink as file, else True
+        :param force: Sync file forcely, do not ignore same files
         '''
         for src_file_path, dst_file_path in _s3_scan_pairs(
                 self.path_with_protocol, dst_url):
             src_file_path = self.from_path(src_file_path)
             dst_file_path = self.from_path(dst_file_path)
-            if dst_file_path.exists() and is_same_file(
+            if not force and dst_file_path.exists() and is_same_file(
                     src_file_path.stat(), dst_file_path.stat(), 'copy'):
                 continue
             src_file_path.copy(dst_file_path, followlinks=followlinks)

megfile/sftp.py

@@ -395,10 +395,15 @@ def sftp_copy(
 
 
 def sftp_sync(
-        src_path: PathLike, dst_path: PathLike, followlinks: bool = False):
+        src_path: PathLike,
+        dst_path: PathLike,
+        followlinks: bool = False,
+        force: bool = False):
     '''Copy file/directory on src_url to dst_url
 
     :param src_path: Given path
     :param dst_url: Given destination path
+    :param followlinks: False if regard symlink as file, else True
+    :param force: Sync file forcely, do not ignore same files
     '''
-    return SftpPath(src_path).sync(dst_path, followlinks)
+    return SftpPath(src_path).sync(dst_path, followlinks, force)

megfile/sftp_path.py

@@ -3,6 +3,7 @@ import hashlib
 import io
 import os
 import shlex
+import socket
 import subprocess
 from logging import getLogger as get_logger
 from stat import S_ISDIR, S_ISLNK, S_ISREG
@@ -11,7 +12,7 @@ from urllib.parse import urlsplit, urlunsplit
 
 import paramiko
 
-from megfile.errors import _create_missing_ok_generator, patch_method
+from megfile.errors import SameFileError, _create_missing_ok_generator, patch_method
 from megfile.interfaces import ContextIterator, FileEntry, PathLike, StatResult
 from megfile.lib.compare import is_same_file
 from megfile.lib.glob import FSFunc, iglob
@@ -45,6 +46,7 @@ SFTP_PRIVATE_KEY_TYPE = "SFTP_PRIVATE_KEY_TYPE"
 SFTP_PRIVATE_KEY_PASSWORD = "SFTP_PRIVATE_KEY_PASSWORD"
 MAX_RETRIES = 10
 DEFAULT_SSH_CONNECT_TIMEOUT = 5
+DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
 
 
 def _make_stat(stat: paramiko.SFTPAttributes) -> StatResult:
@@ -94,8 +96,11 @@ def provide_connect_info(
 def sftp_should_retry(error: Exception) -> bool:
     if type(error) is EOFError:
         return False
-    elif isinstance(error,
-                    (paramiko.ssh_exception.SSHException, ConnectionError)):
+    elif isinstance(error, (
+            paramiko.ssh_exception.SSHException,
+            ConnectionError,
+            socket.timeout,
+    )):
         return True
     elif isinstance(error, OSError) and str(error) == 'Socket is closed':
         return True
@@ -148,8 +153,14 @@ def _get_sftp_client(
 
     :returns: sftp client
     '''
-    ssh_client = get_ssh_client(hostname, port, username, password)
-    sftp_client = ssh_client.open_sftp()
+    session = get_ssh_session(
+        hostname=hostname,
+        port=port,
+        username=username,
+        password=password,
+    )
+    session.invoke_subsystem("sftp")
+    sftp_client = paramiko.SFTPClient(session)
     _patch_sftp_client_request(sftp_client, hostname, port, username, password)
     return sftp_client
 
@@ -189,6 +200,9 @@ def _get_ssh_client(
         username=username,
         password=password,
         pkey=private_key,
+        timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
+        auth_timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
+        banner_timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
     )
     atexit.register(ssh_client.close)
     return ssh_client
@@ -211,10 +225,9 @@ def get_ssh_session(
         username: Optional[str] = None,
         password: Optional[str] = None,
 ) -> paramiko.Channel:
-    ssh_client = get_ssh_client(hostname, port, username, password)
-    try:
-        return _open_session(ssh_client)
-    except paramiko.SSHException:
+
+    def retry_callback(error, *args, **kwargs):
+        ssh_client = get_ssh_client(hostname, port, username, password)
         ssh_client.close()
         atexit.unregister(ssh_client.close)
         ssh_key = f'ssh_client:{hostname},{port},{username},{password}'
@@ -223,20 +236,34 @@ def get_ssh_session(
         sftp_key = f'sftp_client:{hostname},{port},{username},{password}'
         if thread_local.get(sftp_key):
             del thread_local[sftp_key]
-        return _open_session(
-            get_ssh_client(
-                hostname=hostname,
-                port=port,
-                username=username,
-                password=password,
-            ))
+
+    return patch_method(
+        _open_session,  # pytype: disable=attribute-error
+        max_retries=MAX_RETRIES,
+        should_retry=sftp_should_retry,
+        retry_callback=retry_callback)(
+            hostname,
+            port,
+            username,
+            password,
+        )
 
 
-def _open_session(ssh_client: paramiko.SSHClient) -> paramiko.Channel:
+def _open_session(
+        hostname: str,
+        port: Optional[int] = None,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+) -> paramiko.Channel:
+    ssh_client = get_ssh_client(hostname, port, username, password)
     transport = ssh_client.get_transport()
     if not transport:
-        raise paramiko.SSHException()
+        raise paramiko.SSHException('Get transport error')
+    transport.set_keepalive(DEFAULT_SSH_KEEPALIVE_INTERVAL)
     session = transport.open_session(timeout=DEFAULT_SSH_CONNECT_TIMEOUT)
+    if not session:
+        raise paramiko.SSHException('Create session error')
+    session.settimeout(DEFAULT_SSH_CONNECT_TIMEOUT)
     return session
 
 
@@ -1167,6 +1194,9 @@ class SftpPath(URIPath):
         self.from_path(os.path.dirname(dst_path)).makedirs(exist_ok=True)
         dst_path = self.from_path(dst_path)
         if self._is_same_backend(dst_path):
+            if self._real_path == dst_path._real_path:
+                raise SameFileError(
+                    f"'{self.path}' and '{dst_path.path}' are the same file")
             exec_result = self._exec_command(
                 ["cp", self._real_path, dst_path._real_path])
             if exec_result.returncode != 0:
@@ -1189,10 +1219,16 @@ class SftpPath(URIPath):
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
         dst_path._client.chmod(dst_path._real_path, src_stat.st_mode)
 
-    def sync(self, dst_path: PathLike, followlinks: bool = False):
+    def sync(
+            self,
+            dst_path: PathLike,
+            followlinks: bool = False,
+            force: bool = False):
         '''Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
+        :param followlinks: False if regard symlink as file, else True
+        :param force: Sync file forcely, do not ignore same files
         '''
         if not self._is_same_protocol(dst_path):
             raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
@@ -1201,8 +1237,8 @@ class SftpPath(URIPath):
                 self.path_with_protocol, dst_path):
             dst_path = self.from_path(dst_file_path)
             src_path = self.from_path(src_file_path)
-            if dst_path.exists() and is_same_file(src_path.stat(),
-                                                  dst_path.stat(), 'copy'):
+            if not force and dst_path.exists() and is_same_file(
+                    src_path.stat(), dst_path.stat(), 'copy'):
                 continue
             self.from_path(os.path.dirname(dst_file_path)).mkdir(
                 parents=True, exist_ok=True)

megfile/smart.py

@@ -7,6 +7,7 @@ from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterable, Iterator, List
 
 from tqdm import tqdm
 
+from megfile.errors import SameFileError
 from megfile.fs import fs_copy, is_fs
 from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, NullCacher, PathLike, StatResult
 from megfile.lib.combine_reader import CombineReader
@@ -342,6 +343,7 @@ def _smart_sync_single_file(items: dict):
     callback = items['callback']
     followlinks = items['followlinks']
     callback_after_copy_file = items['callback_after_copy_file']
+    force = items['force']
 
     content_path = os.path.relpath(src_file_path, start=src_root_path)
     if len(content_path) and content_path != '.':
@@ -355,7 +357,7 @@ def _smart_sync_single_file(items: dict):
     dst_protocol, _ = SmartPath._extract_protocol(dst_abs_file_path)
     should_sync = True
     try:
-        if smart_exists(dst_abs_file_path) and is_same_file(
+        if not force and smart_exists(dst_abs_file_path) and is_same_file(
                 smart_stat(src_file_path), smart_stat(dst_abs_file_path),
                 get_sync_type(src_protocol, dst_protocol)):
             should_sync = False
@@ -380,7 +382,8 @@ def smart_sync(
         followlinks: bool = False,
         callback_after_copy_file: Optional[Callable[[str, str], None]] = None,
         src_file_stats: Optional[Iterable[FileEntry]] = None,
-        map_func: Callable[[Callable, Iterable], Any] = map) -> None:
+        map_func: Callable[[Callable, Iterable], Any] = map,
+        force: bool = False) -> None:
     '''
     Sync file or directory on s3 and fs
 
@@ -443,6 +446,7 @@ def smart_sync(
                     callback=callback,
                     followlinks=followlinks,
                     callback_after_copy_file=callback_after_copy_file,
+                    force=force,
                 )
 
     for _ in map_func(_smart_sync_single_file, create_generator()):
@@ -454,7 +458,8 @@ def smart_sync_with_progress(
         dst_path,
         callback: Optional[Callable[[str, int], None]] = None,
         followlinks: bool = False,
-        map_func: Callable[[Callable, Iterable], Iterator] = map):
+        map_func: Callable[[Callable, Iterable], Iterator] = map,
+        force: bool = False):
     src_path, dst_path = get_traditional_path(src_path), get_traditional_path(
         dst_path)
     file_stats = list(
@@ -478,6 +483,7 @@ def smart_sync_with_progress(
         callback_after_copy_file=callback_after_copy_file,
         src_file_stats=file_stats,
         map_func=map_func,
+        force=force,
     )
     tbar.close()
     sbar.close()

megfile/stdio.py

@@ -11,7 +11,7 @@ __all__ = [
 
 def stdio_open(
         path: PathLike,
-        mode: str,
+        mode: str = 'rb',
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
         **kwargs) -> IO[AnyStr]:  # pytype: disable=signature-mismatch

megfile/stdio_path.py

@@ -74,7 +74,7 @@ class StdioPath(BaseURIPath):
 
     def open(
             self,
-            mode: str,
+            mode: str = 'rb',
             encoding: Optional[str] = None,
             errors: Optional[str] = None,
             **kwargs) -> IO[AnyStr]:  # pytype: disable=signature-mismatch

megfile/version.py

@@ -1 +1 @@
-VERSION = "2.2.1.post1"
+VERSION = "2.2.3"

{megfile-2.2.1.post1.dist-info → megfile-2.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 2.2.1.post1
+Version: 2.2.3
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii
@@ -69,7 +69,7 @@ Here's an example of writing a file to s3 / sftp / fs, syncing to local, reading
         - `/data/test.txt`
         - `test.txt`
         - 1
-- oss
+- s3
     - `s3[+profile_name]://bucket/key`
 - sftp
     - `sftp://[username[:password]@]hostname[:port]//absolute_file_path`
@@ -170,7 +170,7 @@ pip3 install -r requirements.txt -r requirements-dev.txt
 
 ## Configuration
 
-Before using `megfile` to access files on s3, you need to set up authentication credentials for your s3 account using the [AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/configure/index.html) or editing the file `~/.aws/config` directly, see also: [boto3 configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) & [boto3 credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html). You can set your endpoint url with environments `OSS_ENDPOINT`.
+Before using `megfile` to access files on s3, you need to set up authentication credentials for your s3 account using the [AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/configure/index.html) or editing the file `~/.aws/config` directly, see also: [boto3 configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) & [boto3 credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html). Megfile also support environments for s3, like `OSS_ENDPOINT`, `AWS_ACCESS_KEY_ID` , `AWS_SECRET_ACCESS_KEY` and `AWS_S3_ADDRESSING_STYLE`.
 
 ```
 $ aws configure
@@ -193,7 +193,7 @@ s3 =
     endpoint_url = http://oss-cn-hangzhou.aliyuncs.com
 ```
 
-You also can operate s3 files with different endpoint urls, access keys and secret keys. You can set config for different profiles by environment(`PROFILE_NAME__AWS_ACCESS_KEY_ID`, `PROFILE_NAME__AWS_SECRET_ACCESS_KEY`, `PROFILE_NAME__OSS_ENDPOINT`) or `~/.aws/config`. Then you can operate files with path `s3+profile_name://bucket/key`.
+You also can operate s3 files with different endpoint urls, access keys and secret keys. You can set config for different profiles by environment(`PROFILE_NAME__AWS_ACCESS_KEY_ID`, `PROFILE_NAME__AWS_SECRET_ACCESS_KEY`, `PROFILE_NAME__OSS_ENDPOINT`, `PROFILE_NAME__AWS_S3_ADDRESSING_STYLE`) or `~/.aws/config`. Then you can operate files with path `s3+profile_name://bucket/key`.
 For example:
 ```
 # set config with environment
@@ -217,6 +217,7 @@ s3 =
 aws_secret_access_key = profile2-accesskey
 aws_access_key_id = profile2-secretkey
 s3 =
+    addressing_style = virtual
     endpoint_url = https://profile2.s3.custom.com
 
 

{megfile-2.2.1.post1.dist-info → megfile-2.2.3.dist-info}/RECORD

@@ -1,21 +1,21 @@
 megfile/__init__.py,sha256=Qsi3XNP_0XYoSol-1AGutZqo0rfBnzaiZ-HVXll4fB0,5721
-megfile/cli.py,sha256=MMhdYDiArEF5WkOX6B59NH5URPt5inz4WbO8Jb1kB2o,13045
-megfile/errors.py,sha256=y0PJ4lzkm-uqgVD1EqakmN0IHBMt1CCzDbdbq84sVb4,11491
-megfile/fs.py,sha256=sjkm_LsvNCw8hj9Ee-1HSNzvg7bRHTPem8KTDDBQlCw,11621
-megfile/fs_path.py,sha256=Lz-u8XL2RMViHl_39yapAwX5XSE_ZpQsuhAvVve6rk8,38532
+megfile/cli.py,sha256=ZRS2Ityuqk51T-BDvU-q1EDAS66Vw8gk817Wg_OWwSk,13625
+megfile/errors.py,sha256=BIZ8XlQLRo0oZsU38Jhv8eooCFEjzsg3RSb1nTuLc5g,11663
+megfile/fs.py,sha256=LtrzQsyZgogTJeoRFz4L52gxx0jByzRBLkpWYpvkp5I,11819
+megfile/fs_path.py,sha256=JkY8qGIIboK5MK2rSagYEvnu5FTzmk9OHXIhTO7BjeY,38767
 megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
 megfile/http_path.py,sha256=pbIlle-scbKC88XO0uV9y3sIm0HME6E_R061VTBg8w8,8966
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
-megfile/pathlike.py,sha256=aU6j-Z3Sr6XH_REuJ37_vNv6XucEW6qpASWQyzGbL9g,29258
-megfile/s3.py,sha256=6d9bBVkVdXIvmJLfpOgJkK4nVAsLR3lirrB6x1-P2y8,12437
-megfile/s3_path.py,sha256=c6lzX-dinvJw1jRCpqctRy5JdjVzGhREnyY3_8MAafI,85697
-megfile/sftp.py,sha256=h2cy_XnQQOlCtHskW7rSQKG1S7Uoq7uq7X_Pyt2V16M,12590
-megfile/sftp_path.py,sha256=ELxJVsbZoOhkZJVbk9gxcM_BReiDsYC5NTdhSpEe2rk,47982
-megfile/smart.py,sha256=SFrHDMbglOTo-g03OXh8sz5zdhtPq1VS7_cFxVZYPk8,33010
+megfile/pathlike.py,sha256=WpP8zWSOAcAfYrD65hZS08UEi4_iCoEMs2xvfFMwZvY,29264
+megfile/s3.py,sha256=7XZSWjcSY-hoLhLH9dtfyRpokfYH9raTO_Mf69RjpEs,12560
+megfile/s3_path.py,sha256=fGMrV0_cRWmjqRcbziuhC7vCLXnDvbEEHbGDmW4ZHGQ,87172
+megfile/sftp.py,sha256=CZYv1WKL0d_vuJ5aPgMhm0W8uskzaO5zbYGhGwt_mQs,12771
+megfile/sftp_path.py,sha256=rwEfw93fZkNEf7TjZ1-8bGiaVU0oxwb3H7ub6x9iUsA,49153
+megfile/smart.py,sha256=PAg8i56kFn94a6LGlH1v2qamCojYM9qKb-Ryw6rX3ks,33204
 megfile/smart_path.py,sha256=Rwb1McXsshi9-F6miTRqE6j8FO2j1edjmSxZF32YZ6E,6708
-megfile/stdio.py,sha256=28AAQ7YLBZ51nrQDtK2th5lWhDJO3sZFKdQYNXQ_eNM,700
-megfile/stdio_path.py,sha256=QqZ5XJw4VFdP2s14MP7ooe9TgYIZtdNtu5vny5cUW5Y,2866
-megfile/version.py,sha256=J75j1o3zUHTJ6fEnNz_Qg0RLb_Uul_oUSapamVffVZU,25
+megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
+megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
+megfile/version.py,sha256=NUfAFE_kB_uAzWs7eyqhwlH9YxsrTSBEHoHyhugYN4Y,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=SjrBffHVgvJnYtr8HNqiOozP9OJRYS37Eu1KQcZu1Z8,13221
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
@@ -38,10 +38,10 @@ megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-2.2.1.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-2.2.1.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-2.2.1.post1.dist-info/METADATA,sha256=EVFMWdWi2j42YXbfnD4kELPbMh-B1gBsTxHvakEiOPA,10601
-megfile-2.2.1.post1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-megfile-2.2.1.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-2.2.1.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-2.2.1.post1.dist-info/RECORD,,
+megfile-2.2.3.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-2.2.3.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-2.2.3.dist-info/METADATA,sha256=VL8zMVQHM7NRnGr4X3A6uCXzRN220PvKQR47pjJC46g,10742
+megfile-2.2.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+megfile-2.2.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-2.2.3.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-2.2.3.dist-info/RECORD,,