megfile 4.0.3__py3-none-any.whl → 4.1.0__py3-none-any.whl

megfile/sftp_path.py

@@ -116,10 +116,12 @@ def _patch_sftp_client_request(
         ssh_client = get_ssh_client(hostname, port, username, password, default_policy)
         ssh_client.close()
         atexit.unregister(ssh_client.close)
-        ssh_key = f"ssh_client:{hostname},{port},{username},{password}"
+        ssh_key = f"ssh_client:{hostname},{port},{username},{password},{default_policy}"
         if thread_local.get(ssh_key):
             del thread_local[ssh_key]
-        sftp_key = f"sftp_client:{hostname},{port},{username},{password}"
+        sftp_key = (
+            f"sftp_client:{hostname},{port},{username},{password},{default_policy}"
+        )
         if thread_local.get(sftp_key):
             del thread_local[sftp_key]
 
@@ -179,7 +181,7 @@ def get_sftp_client(
     :returns: sftp client
     """
     return thread_local(
-        f"sftp_client:{hostname},{port},{username},{password}",
+        f"sftp_client:{hostname},{port},{username},{password},{default_policy}",
         _get_sftp_client,
         hostname,
         port,
@@ -219,7 +221,7 @@ def _get_ssh_client(
             ),
             os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
         )
-    except Exception:
+    except Exception:  # pragma: no cover
         _logger.warning(
             "Can't create file lock in '/tmp', "
             "please control the SFTP concurrency count by yourself."
@@ -252,7 +254,7 @@ def get_ssh_client(
     default_policy: Type[paramiko.MissingHostKeyPolicy] = paramiko.RejectPolicy,
 ) -> paramiko.SSHClient:
     return thread_local(
-        f"ssh_client:{hostname},{port},{username},{password}",
+        f"ssh_client:{hostname},{port},{username},{password},{default_policy}",
         _get_ssh_client,
         hostname,
         port,
@@ -273,10 +275,12 @@ def get_ssh_session(
         ssh_client = get_ssh_client(hostname, port, username, password, default_policy)
         ssh_client.close()
         atexit.unregister(ssh_client.close)
-        ssh_key = f"ssh_client:{hostname},{port},{username},{password}"
+        ssh_key = f"ssh_client:{hostname},{port},{username},{password},{default_policy}"
         if thread_local.get(ssh_key):
             del thread_local[ssh_key]
-        sftp_key = f"sftp_client:{hostname},{port},{username},{password}"
+        sftp_key = (
+            f"sftp_client:{hostname},{port},{username},{password},{default_policy}"
+        )
         if thread_local.get(sftp_key):
             del thread_local[sftp_key]
 
@@ -530,8 +534,11 @@ class SftpPath(URIPath):
             glob_path = self.joinpath(pattern).path_with_protocol
 
         def _scandir(dirname: str) -> Iterator[Tuple[str, bool]]:
+            result = []
             for entry in self.from_path(dirname).scandir():
-                yield entry.name, entry.is_dir()
+                result.append((entry.name, entry.is_dir()))
+            for name, is_dir in sorted(result):
+                yield name, is_dir
 
         def _exist(path: PathLike, followlinks: bool = False):
             return self.from_path(path).exists(followlinks=followlinks)
@@ -596,21 +603,18 @@ class SftpPath(URIPath):
 
         :returns: All contents have in the path in ascending alphabetical order
         """
-        if not self.is_dir():
-            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
-        return sorted(self._client.listdir(self._real_path))
+        with self.scandir() as entries:
+            return sorted([entry.name for entry in entries])
 
     def iterdir(self) -> Iterator["SftpPath"]:
         """
-        Get all contents of given sftp path.
-        The result is in ascending alphabetical order.
+        Get all contents of given sftp path. The order of result is in arbitrary order.
 
-        :returns: All contents have in the path in ascending alphabetical order
+        :returns: All contents have in the path.
         """
-        if not self.is_dir():
-            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
-        for path in self.listdir():
-            yield self.joinpath(path)
+        with self.scandir() as entries:
+            for entry in entries:
+                yield self.joinpath(entry.name)
 
     def load(self) -> BinaryIO:
         """Read all content on specified path and write into memory
@@ -807,20 +811,21 @@ class SftpPath(URIPath):
             FileNotFoundError("No match any file in: %r" % self.path_with_protocol),
         )
 
-    def scandir(self) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
         Get all content of given file path.
 
         :returns: An iterator contains all contents have prefix path
         """
-        if not self.exists():
-            raise FileNotFoundError("No such directory: %r" % self.path_with_protocol)
-
-        if not self.is_dir():
-            raise NotADirectoryError("Not a directory: %r" % self.path_with_protocol)
+        real_path = self._real_path
+        stat = self.stat(follow_symlinks=False)
+        if stat.is_symlink():
+            real_path = self.readlink()._real_path
+        elif not stat.is_dir():
+            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
 
         def create_generator():
-            for name in self.listdir():
+            for name in self._client.listdir(real_path):
                 current_path = self.joinpath(name)
                 yield FileEntry(
                     current_path.name,
@@ -917,7 +922,7 @@ class SftpPath(URIPath):
         path = self._client.normalize(self._real_path)
         return self._generate_path_object(path, resolve=True)
 
-    def md5(self, recalculate: bool = False, followlinks: bool = True):
+    def md5(self, recalculate: bool = False, followlinks: bool = False):
         """
         Calculate the md5 value of the file
 
@@ -992,6 +997,7 @@ class SftpPath(URIPath):
     def open(
         self,
         mode: str = "r",
+        *,
         buffering=-1,
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
@@ -1022,7 +1028,7 @@ class SftpPath(URIPath):
             )  # pytype: disable=wrong-arg-types
         return fileobj  # pytype: disable=bad-return-type
 
-    def chmod(self, mode: int, follow_symlinks: bool = True):
+    def chmod(self, mode: int, *, follow_symlinks: bool = True):
         """
         Change the file mode and permissions, like os.chmod().
 
@@ -1173,7 +1179,11 @@ class SftpPath(URIPath):
             ):
                 continue
 
-            self.from_path(src_file_path).copy(dst_file_path, followlinks=followlinks)
+            src_path.copy(
+                dst_file_path,
+                followlinks=followlinks,
+                overwrite=True,
+            )
 
     def utime(self, atime: Union[float, int], mtime: Union[float, int]) -> None:
         """

megfile/smart.py

@@ -285,8 +285,6 @@ def register_copy_func(
         dst_dict = _copy_funcs.get(src_protocol, {})
         dst_dict[dst_protocol] = copy_func
         _copy_funcs[src_protocol] = dst_dict
-    except Exception as error:
-        raise error
     else:
         raise ValueError(
             "Copy Function has already existed: {}->{}".format(
@@ -1081,13 +1079,14 @@ def smart_touch(path: PathLike):
         pass
 
 
-def smart_getmd5(path: PathLike, recalculate: bool = False):
+def smart_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
     """Get md5 value of file
 
     param path: File path
     param recalculate: calculate md5 in real-time or not return s3 etag when path is s3
+    param followlinks: If is True, calculate md5 for real file
     """
-    return SmartPath(path).md5(recalculate=recalculate)
+    return SmartPath(path).md5(recalculate=recalculate, followlinks=followlinks)
 
 
 _concat_funcs = {"s3": s3_concat, "sftp": sftp_concat}

megfile/stdio.py

@@ -12,6 +12,7 @@ __all__ = [
 def stdio_open(
     path: PathLike,
     mode: str = "rb",
+    *,
     encoding: Optional[str] = None,
     errors: Optional[str] = None,
     **kwargs,
@@ -26,4 +27,4 @@ def stdio_open(
     :param mode: Only supports 'rb' and 'wb' now
     :return: STDReader, STDWriter
     """
-    return StdioPath(path).open(mode, encoding, errors)
+    return StdioPath(path).open(mode, encoding=encoding, errors=errors)

megfile/stdio_path.py

@@ -72,6 +72,7 @@ class StdioPath(BasePath):
     def open(
         self,
         mode: str = "rb",
+        *,
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
         **kwargs,

megfile/utils/__init__.py

@@ -280,24 +280,8 @@ class classproperty(property):
         # apply the __get__ on the class
         return super(classproperty, self).__get__(cls)
 
-    def __set__(self, cls_or_obj, value: object) -> None:
-        """
-        This method gets called when a property value should be set.
-        @param cls_or_obj: The class or instance of which the property should be
-            changed.
-        @param value: The new value.
-        """
-        # call this method only on the class, not the instance
-        super(classproperty, self).__set__(_get_class(cls_or_obj), value)
-
-    def __delete__(self, cls_or_obj) -> None:
-        """
-        This method gets called when a property should be deleted.
-        @param cls_or_obj: The class or instance of which the property should be
-            deleted.
-        """
-        # call this method only on the class, not the instance
-        super(classproperty, self).__delete__(_get_class(cls_or_obj))
+    # __set__ and __delete__ only work for instance
+    # classproperty only support __get__
 
 
 class cached_classproperty(cached_property):
@@ -333,7 +317,7 @@ class cached_classproperty(cached_property):
         @return: The value of the property.
         """
         if self.attrname is None:
-            raise TypeError(
+            raise TypeError(  # pragma: no cover
                 "Cannot use cached_classproperty instance without calling "
                 "__set_name__ on it."
             )

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.0.3"
+VERSION = "4.1.0"

{megfile-4.0.3.dist-info → megfile-4.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: megfile
-Version: 4.0.3
+Version: 4.1.0
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.0.3.dist-info → megfile-4.1.0.dist-info}/RECORD

@@ -1,47 +1,47 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=e3VVr8oe8iR7L_PtpNtyqAvQL_WgJzzEz8oewSAlgX4,24887
-megfile/config.py,sha256=_6HiGeXEyk6RjPdjA0eEj1unq9iLJV_vQJBzQ-eHNvs,4185
-megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
-megfile/fs.py,sha256=bPGbapv41FzME87X3MhSNQRjNmHrI23FuFnjPT0ukQs,18239
-megfile/fs_path.py,sha256=ZK-po1xqhHocMb9Vrxf5K9tDx3zxQmGxNIHY3Z7Akp8,39085
-megfile/hdfs.py,sha256=JfqTTvMzTeDFAJ-uq5BU0sjFaE3qshlVZ0_5h9oImfA,13361
-megfile/hdfs_path.py,sha256=_ir7En78K0nZudVOL7yYcKZgBQRZwlMDvWTRmYXEfOE,25637
+megfile/cli.py,sha256=Y7ZyWsYmzMUjUrJ1Vz-WMWJQcpK-4og_5xyRzfKI91U,24957
+megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
+megfile/errors.py,sha256=su85hEFEk18gwUJW_GNdgCe4FeXuuy7Vf3b1pmNkHIk,14347
+megfile/fs.py,sha256=TJ0ifJ57IEL-gFgQhJD_WVEqmf28zC5kAvmH6c1QzbU,18240
+megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
+megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
+megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http.py,sha256=1nuGe-JbnwMFyV3s35CJxByED3uoRoS9y8Y8cSGP9Kw,3865
-megfile/http_path.py,sha256=c-xAu5wDxcTevmIUmrNEy-m-QiCfDJToaVI7y8SVIUI,14492
+megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
-megfile/pathlike.py,sha256=vfuTBqSTIciRxkkqMfLfnBxWTEl9yns1yR8zgK4Raw0,31268
-megfile/s3.py,sha256=zqAegH5tijcztEKcfHXmOYhAR880nTxaAzc2O0JJnjc,16661
-megfile/s3_path.py,sha256=lpUKy4n5DTf6hK6TvPhMjt_ZgdIXO4vcyK_VLaGkvhg,93395
-megfile/sftp.py,sha256=0ZnQlmhgvs7pYjFTcvvOyxTo2IUurE-hp1GN0hnIrdQ,26473
-megfile/sftp_path.py,sha256=4rcbn3wqcOEs71W6qWu1efcj6MZUgrZm6U0Jan-eB70,43604
-megfile/smart.py,sha256=Ps8acPx6jeG1UJnRD8xL2aQjRp7IMW8sV6VFkMF0TQk,36910
+megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
+megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
+megfile/s3_path.py,sha256=zqXrS4aclIE0454AjMsD64Lxy3BlXlh4kAdi_5Bj19E,93558
+megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
+megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
+megfile/smart.py,sha256=xkNVEa3DAThM4zJf3aoHJg_t8PQeVv9yz66GhGIdhpA,36974
 megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
-megfile/stdio.py,sha256=C_cGID_npthpwoPcsJMMEqqbVUPUnDxxJV9jLY2_D7c,635
-megfile/stdio_path.py,sha256=L8ODNIwO79UIv13YYc2OTr6f4XTv4ZPyvBeRk83-AjA,2700
-megfile/version.py,sha256=aJJATI8dDtIUcu59WVztQ3az97BkUrCKd86VVa6W_aM,19
+megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
+megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
+megfile/version.py,sha256=5UUfPKQaA2LgHzk7jZgvPthpTs-MIguSBfg3cfykU3k,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=6Dy2ZwlowqAvyUUa7bpQLCKOclmmUDhqEF-_CDDp0Og,13100
-megfile/lib/combine_reader.py,sha256=nKGAug29lOpNIZuLKu7_qVrJJRpXL_J4jxLglWbGJ1w,4808
+megfile/lib/base_prefetch_reader.py,sha256=oSiQq6wlV9DADQapkf3Up3-86zl6QrtusVu93MAt1KE,12848
+megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
 megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=aTK2aFyFobUs0W4deB4iASgtuq6DAE1jILmcAiP8Cb4,9840
+megfile/lib/glob.py,sha256=1u0j_DoqT7VmTUsDAdcKhPbr4XOwIBpOkCnyAFEGw1I,9633
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OjP5pdWK_e1QiFIt2xmflceLnrz3S7B7ePBZFK-OwQE,4558
 megfile/lib/joinpath.py,sha256=gaPNtBi8fzd5LZNyZp5zrHzaybcqKJ1xlntGmVNyFEM,929
 megfile/lib/lazy_handler.py,sha256=bE7RGt1x_xYWMgGAvHr7dwEt52qy-D3z90X3oyCvE6g,1875
 megfile/lib/s3_buffered_writer.py,sha256=kXvz1bdoaVIxjnEQeg4dxEzAXMYNrk-5uIMww86ty00,7860
-megfile/lib/s3_cached_handler.py,sha256=X8PdeRC-BY6eSmOO5f2BeyjTPxyEwNtHgmAm9VgmDR4,1426
-megfile/lib/s3_limited_seekable_writer.py,sha256=mUeoTS98LHluwDN7zxdCVcsjOGBT1bOYV8nRvi9QMGE,6212
-megfile/lib/s3_memory_handler.py,sha256=4uzBzz2jfRI_u6jl0CpOGAhpNJhDQo18FSAweauCUFs,4136
-megfile/lib/s3_pipe_handler.py,sha256=dm7NnZd1Ym5ABS1GvOQtoCJEO_CB8e6p4sUhLiid0go,3622
+megfile/lib/s3_cached_handler.py,sha256=MkNt6AAapd5x8BH2gnW5_S0cLofN-mshEdb0qSoLho8,1426
+megfile/lib/s3_limited_seekable_writer.py,sha256=joVcjoTx48jataXAwEqOI3toRxxO3-XQRvyWHtJ23lQ,6232
+megfile/lib/s3_memory_handler.py,sha256=_2JCLbbPUuIGGCeUEuSRZ-mdDRtGow2ofgtAQwzru4s,4136
+megfile/lib/s3_pipe_handler.py,sha256=QKhcjXWkedM4-eX0BhZVmkmCjawfyBJfYzmGFUclO8w,3642
 megfile/lib/s3_prefetch_reader.py,sha256=R37-y_L9l8IKJhpT8HwBrZEbo2X72vCqEV6fvqPCBug,4437
 megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/utils/__init__.py,sha256=sATf_NlsSTYIMEiA8-gM6K1M-Q1K6_7rx2VM31hrqaA,10838
+megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
 scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
 scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
@@ -53,10 +53,10 @@ scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMf
 scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
 scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
 scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.0.3.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.0.3.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.0.3.dist-info/METADATA,sha256=At2cDDQKTUSj0QmjmPAjPRpqG_M5TZdefe-_2xMNjX8,9573
-megfile-4.0.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-megfile-4.0.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.0.3.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
-megfile-4.0.3.dist-info/RECORD,,
+megfile-4.1.0.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.0.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.0.dist-info/METADATA,sha256=Q6vaVV6xtJSdmFIOdBaPHRyvhcVPhI0NUlUpwhUUZfA,9573
+megfile-4.1.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+megfile-4.1.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.0.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-4.1.0.dist-info/RECORD,,

{megfile-4.0.3.dist-info → megfile-4.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any