megfile 4.0.4__py3-none-any.whl → 4.1.0.post1__py3-none-any.whl

megfile/sftp_path.py

@@ -116,10 +116,12 @@ def _patch_sftp_client_request(
         ssh_client = get_ssh_client(hostname, port, username, password, default_policy)
         ssh_client.close()
         atexit.unregister(ssh_client.close)
-        ssh_key = f"ssh_client:{hostname},{port},{username},{password}"
+        ssh_key = f"ssh_client:{hostname},{port},{username},{password},{default_policy}"
         if thread_local.get(ssh_key):
             del thread_local[ssh_key]
-        sftp_key = f"sftp_client:{hostname},{port},{username},{password}"
+        sftp_key = (
+            f"sftp_client:{hostname},{port},{username},{password},{default_policy}"
+        )
         if thread_local.get(sftp_key):
             del thread_local[sftp_key]
 
@@ -179,7 +181,7 @@ def get_sftp_client(
     :returns: sftp client
     """
     return thread_local(
-        f"sftp_client:{hostname},{port},{username},{password}",
+        f"sftp_client:{hostname},{port},{username},{password},{default_policy}",
         _get_sftp_client,
         hostname,
         port,
@@ -219,7 +221,7 @@ def _get_ssh_client(
             ),
             os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
         )
-    except Exception:
+    except Exception:  # pragma: no cover
         _logger.warning(
             "Can't create file lock in '/tmp', "
             "please control the SFTP concurrency count by yourself."
@@ -252,7 +254,7 @@ def get_ssh_client(
     default_policy: Type[paramiko.MissingHostKeyPolicy] = paramiko.RejectPolicy,
 ) -> paramiko.SSHClient:
     return thread_local(
-        f"ssh_client:{hostname},{port},{username},{password}",
+        f"ssh_client:{hostname},{port},{username},{password},{default_policy}",
         _get_ssh_client,
         hostname,
         port,
@@ -273,10 +275,12 @@ def get_ssh_session(
         ssh_client = get_ssh_client(hostname, port, username, password, default_policy)
         ssh_client.close()
         atexit.unregister(ssh_client.close)
-        ssh_key = f"ssh_client:{hostname},{port},{username},{password}"
+        ssh_key = f"ssh_client:{hostname},{port},{username},{password},{default_policy}"
         if thread_local.get(ssh_key):
             del thread_local[ssh_key]
-        sftp_key = f"sftp_client:{hostname},{port},{username},{password}"
+        sftp_key = (
+            f"sftp_client:{hostname},{port},{username},{password},{default_policy}"
+        )
         if thread_local.get(sftp_key):
             del thread_local[sftp_key]
 
@@ -530,8 +534,11 @@ class SftpPath(URIPath):
             glob_path = self.joinpath(pattern).path_with_protocol
 
         def _scandir(dirname: str) -> Iterator[Tuple[str, bool]]:
+            result = []
             for entry in self.from_path(dirname).scandir():
-                yield entry.name, entry.is_dir()
+                result.append((entry.name, entry.is_dir()))
+            for name, is_dir in sorted(result):
+                yield name, is_dir
 
         def _exist(path: PathLike, followlinks: bool = False):
             return self.from_path(path).exists(followlinks=followlinks)
@@ -596,21 +603,18 @@ class SftpPath(URIPath):
 
         :returns: All contents have in the path in ascending alphabetical order
         """
-        if not self.is_dir():
-            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
-        return sorted(self._client.listdir(self._real_path))
+        with self.scandir() as entries:
+            return sorted([entry.name for entry in entries])
 
     def iterdir(self) -> Iterator["SftpPath"]:
         """
-        Get all contents of given sftp path.
-        The result is in ascending alphabetical order.
+        Get all contents of given sftp path. The order of result is in arbitrary order.
 
-        :returns: All contents have in the path in ascending alphabetical order
+        :returns: All contents have in the path.
         """
-        if not self.is_dir():
-            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
-        for path in self.listdir():
-            yield self.joinpath(path)
+        with self.scandir() as entries:
+            for entry in entries:
+                yield self.joinpath(entry.name)
 
     def load(self) -> BinaryIO:
         """Read all content on specified path and write into memory
@@ -807,20 +811,21 @@ class SftpPath(URIPath):
             FileNotFoundError("No match any file in: %r" % self.path_with_protocol),
         )
 
-    def scandir(self) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
         Get all content of given file path.
 
         :returns: An iterator contains all contents have prefix path
         """
-        if not self.exists():
-            raise FileNotFoundError("No such directory: %r" % self.path_with_protocol)
-
-        if not self.is_dir():
-            raise NotADirectoryError("Not a directory: %r" % self.path_with_protocol)
+        real_path = self._real_path
+        stat = self.stat(follow_symlinks=False)
+        if stat.is_symlink():
+            real_path = self.readlink()._real_path
+        elif not stat.is_dir():
+            raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
 
         def create_generator():
-            for name in self.listdir():
+            for name in self._client.listdir(real_path):
                 current_path = self.joinpath(name)
                 yield FileEntry(
                     current_path.name,
@@ -917,7 +922,7 @@ class SftpPath(URIPath):
         path = self._client.normalize(self._real_path)
         return self._generate_path_object(path, resolve=True)
 
-    def md5(self, recalculate: bool = False, followlinks: bool = True):
+    def md5(self, recalculate: bool = False, followlinks: bool = False):
         """
         Calculate the md5 value of the file
 
@@ -992,6 +997,7 @@ class SftpPath(URIPath):
     def open(
         self,
         mode: str = "r",
+        *,
         buffering=-1,
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
@@ -1022,7 +1028,7 @@ class SftpPath(URIPath):
             )  # pytype: disable=wrong-arg-types
         return fileobj  # pytype: disable=bad-return-type
 
-    def chmod(self, mode: int, follow_symlinks: bool = True):
+    def chmod(self, mode: int, *, follow_symlinks: bool = True):
         """
         Change the file mode and permissions, like os.chmod().
 

megfile/smart.py

@@ -285,8 +285,6 @@ def register_copy_func(
         dst_dict = _copy_funcs.get(src_protocol, {})
         dst_dict[dst_protocol] = copy_func
         _copy_funcs[src_protocol] = dst_dict
-    except Exception as error:
-        raise error
     else:
         raise ValueError(
             "Copy Function has already existed: {}->{}".format(
@@ -1081,13 +1079,14 @@ def smart_touch(path: PathLike):
         pass
 
 
-def smart_getmd5(path: PathLike, recalculate: bool = False):
+def smart_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
     """Get md5 value of file
 
     param path: File path
     param recalculate: calculate md5 in real-time or not return s3 etag when path is s3
+    param followlinks: If is True, calculate md5 for real file
     """
-    return SmartPath(path).md5(recalculate=recalculate)
+    return SmartPath(path).md5(recalculate=recalculate, followlinks=followlinks)
 
 
 _concat_funcs = {"s3": s3_concat, "sftp": sftp_concat}

megfile/stdio.py

@@ -12,6 +12,7 @@ __all__ = [
 def stdio_open(
     path: PathLike,
     mode: str = "rb",
+    *,
     encoding: Optional[str] = None,
     errors: Optional[str] = None,
     **kwargs,
@@ -26,4 +27,4 @@ def stdio_open(
     :param mode: Only supports 'rb' and 'wb' now
     :return: STDReader, STDWriter
     """
-    return StdioPath(path).open(mode, encoding, errors)
+    return StdioPath(path).open(mode, encoding=encoding, errors=errors)

megfile/stdio_path.py

@@ -72,6 +72,7 @@ class StdioPath(BasePath):
     def open(
         self,
         mode: str = "rb",
+        *,
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
         **kwargs,

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.0.4"
+VERSION = "4.1.0.post1"

{megfile-4.0.4.dist-info → megfile-4.1.0.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: megfile
-Version: 4.0.4
+Version: 4.1.0.post1
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.0.4.dist-info → megfile-4.1.0.post1.dist-info}/RECORD

@@ -1,41 +1,41 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
 megfile/cli.py,sha256=Y7ZyWsYmzMUjUrJ1Vz-WMWJQcpK-4og_5xyRzfKI91U,24957
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=KrzJFBgX7awoSJBADswCo4u6qbj-rcVQMN5It209Dvk,14333
-megfile/fs.py,sha256=bPGbapv41FzME87X3MhSNQRjNmHrI23FuFnjPT0ukQs,18239
-megfile/fs_path.py,sha256=K485rResPqL56m9oIkbBBCTrUkoKBWIP5EjhjbpTkIY,39141
-megfile/hdfs.py,sha256=JfqTTvMzTeDFAJ-uq5BU0sjFaE3qshlVZ0_5h9oImfA,13361
-megfile/hdfs_path.py,sha256=DbFuROHqlLo9o94N0HF1X83DMr7wKhZ9SR2mnKNVjyc,25739
+megfile/errors.py,sha256=qSIiqiJmDLb2k1qKiTLdNQzj4VNgfM9bHkSClyCp1Kg,14385
+megfile/fs.py,sha256=TJ0ifJ57IEL-gFgQhJD_WVEqmf28zC5kAvmH6c1QzbU,18240
+megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
+megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
+megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http.py,sha256=1nuGe-JbnwMFyV3s35CJxByED3uoRoS9y8Y8cSGP9Kw,3865
 megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
-megfile/pathlike.py,sha256=vfuTBqSTIciRxkkqMfLfnBxWTEl9yns1yR8zgK4Raw0,31268
-megfile/s3.py,sha256=aVse1PsYchTkkMPUUW7GWfKOApvkl_Qh5pwNYdk7Ehc,16747
-megfile/s3_path.py,sha256=hhp0zkLfxe1MnKGlLnvPLi7mSYQirt81yGU8ToWl6r8,93576
-megfile/sftp.py,sha256=0ZnQlmhgvs7pYjFTcvvOyxTo2IUurE-hp1GN0hnIrdQ,26473
-megfile/sftp_path.py,sha256=LttyxwoDMb2nhY4b9qUAHTichrG6Rec8Q_xUTqC6VWI,43662
-megfile/smart.py,sha256=Ps8acPx6jeG1UJnRD8xL2aQjRp7IMW8sV6VFkMF0TQk,36910
+megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
+megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
+megfile/s3_path.py,sha256=w9_I3BMbLTmu4WCa0W0O0NnFkOHS0CQ-9fvtViL-2iU,93565
+megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
+megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
+megfile/smart.py,sha256=xkNVEa3DAThM4zJf3aoHJg_t8PQeVv9yz66GhGIdhpA,36974
 megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
-megfile/stdio.py,sha256=C_cGID_npthpwoPcsJMMEqqbVUPUnDxxJV9jLY2_D7c,635
-megfile/stdio_path.py,sha256=L8ODNIwO79UIv13YYc2OTr6f4XTv4ZPyvBeRk83-AjA,2700
-megfile/version.py,sha256=hJ-Qyt0gNnGNOAUMy6dMT2dsePu73CM3Gc3e0kxG4hQ,19
+megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
+megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
+megfile/version.py,sha256=0T8ZzQ_dcDIRsR27VgeThkcTcJi4fE4WvZ8Me4aXAwc,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=6Dy2ZwlowqAvyUUa7bpQLCKOclmmUDhqEF-_CDDp0Og,13100
-megfile/lib/combine_reader.py,sha256=nKGAug29lOpNIZuLKu7_qVrJJRpXL_J4jxLglWbGJ1w,4808
+megfile/lib/base_prefetch_reader.py,sha256=oSiQq6wlV9DADQapkf3Up3-86zl6QrtusVu93MAt1KE,12848
+megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
 megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=aTK2aFyFobUs0W4deB4iASgtuq6DAE1jILmcAiP8Cb4,9840
+megfile/lib/glob.py,sha256=1u0j_DoqT7VmTUsDAdcKhPbr4XOwIBpOkCnyAFEGw1I,9633
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OjP5pdWK_e1QiFIt2xmflceLnrz3S7B7ePBZFK-OwQE,4558
 megfile/lib/joinpath.py,sha256=gaPNtBi8fzd5LZNyZp5zrHzaybcqKJ1xlntGmVNyFEM,929
 megfile/lib/lazy_handler.py,sha256=bE7RGt1x_xYWMgGAvHr7dwEt52qy-D3z90X3oyCvE6g,1875
 megfile/lib/s3_buffered_writer.py,sha256=kXvz1bdoaVIxjnEQeg4dxEzAXMYNrk-5uIMww86ty00,7860
-megfile/lib/s3_cached_handler.py,sha256=X8PdeRC-BY6eSmOO5f2BeyjTPxyEwNtHgmAm9VgmDR4,1426
-megfile/lib/s3_limited_seekable_writer.py,sha256=mUeoTS98LHluwDN7zxdCVcsjOGBT1bOYV8nRvi9QMGE,6212
-megfile/lib/s3_memory_handler.py,sha256=4uzBzz2jfRI_u6jl0CpOGAhpNJhDQo18FSAweauCUFs,4136
-megfile/lib/s3_pipe_handler.py,sha256=dm7NnZd1Ym5ABS1GvOQtoCJEO_CB8e6p4sUhLiid0go,3622
+megfile/lib/s3_cached_handler.py,sha256=MkNt6AAapd5x8BH2gnW5_S0cLofN-mshEdb0qSoLho8,1426
+megfile/lib/s3_limited_seekable_writer.py,sha256=joVcjoTx48jataXAwEqOI3toRxxO3-XQRvyWHtJ23lQ,6232
+megfile/lib/s3_memory_handler.py,sha256=_2JCLbbPUuIGGCeUEuSRZ-mdDRtGow2ofgtAQwzru4s,4136
+megfile/lib/s3_pipe_handler.py,sha256=QKhcjXWkedM4-eX0BhZVmkmCjawfyBJfYzmGFUclO8w,3642
 megfile/lib/s3_prefetch_reader.py,sha256=R37-y_L9l8IKJhpT8HwBrZEbo2X72vCqEV6fvqPCBug,4437
 megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
@@ -53,10 +53,10 @@ scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMf
 scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
 scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
 scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.0.4.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.0.4.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.0.4.dist-info/METADATA,sha256=ZOXOJNvkRh_LaCknY7VU_qS2KM6LNJf3MZVN6Pygsg0,9573
-megfile-4.0.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-megfile-4.0.4.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.0.4.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
-megfile-4.0.4.dist-info/RECORD,,
+megfile-4.1.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.0.post1.dist-info/METADATA,sha256=McvjueS05XMnfNKJMjzNz6w5Oj5KK27DgJkYRm8q6hE,9579
+megfile-4.1.0.post1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+megfile-4.1.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.0.post1.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-4.1.0.post1.dist-info/RECORD,,

{megfile-4.0.4.dist-info → megfile-4.1.0.post1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any