megfile 3.1.6.post1__py3-none-any.whl → 4.0.0__py3-none-any.whl

megfile/sftp.py

@@ -1,21 +1,17 @@
+import os
+from logging import getLogger as get_logger
 from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
 
 from megfile.interfaces import FileEntry, PathLike, StatResult
+from megfile.lib.compat import fspath
+from megfile.lib.joinpath import uri_join
 from megfile.sftp_path import (
     SftpPath,
     is_sftp,
-    sftp_concat,
-    sftp_download,
-    sftp_glob,
-    sftp_glob_stat,
-    sftp_iglob,
-    sftp_lstat,
-    sftp_path_join,
-    sftp_readlink,
-    sftp_resolve,
-    sftp_upload,
 )
 
+_logger = get_logger(__name__)
+
 __all__ = [
     "is_sftp",
     "sftp_readlink",
@@ -59,6 +55,300 @@ __all__ = [
 ]
 
 
+def sftp_readlink(path: PathLike) -> "str":
+    """
+    Return a SftpPath instance representing the path to which the symbolic link points.
+
+    :param path: Given path
+    :returns: Return a SftpPath instance representing the path to
+        which the symbolic link points.
+    """
+    return SftpPath(path).readlink().path_with_protocol
+
+
+def sftp_glob(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> List[str]:
+    """Return path list in ascending alphabetical order,
+    in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+       Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+       when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+       fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+       Assume there exists a path `/a/b/c/b/d.txt`
+       use path pattern like `/**/b/**/*.txt` to glob,
+       the path above will be returned twice
+    3. `**` will match any matched file, directory, symlink and '' by default,
+       when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True)
+       in ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param path: Given path
+    :param pattern: Glob the given relative pattern in the directory represented
+        by this path
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: A list contains paths match `pathname`
+    """
+    return list(sftp_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
+
+
+def sftp_glob_stat(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> Iterator[FileEntry]:
+    """Return a list contains tuples of path and file stat, in ascending alphabetical
+    order, in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+       Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+       when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+       sftp_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+       Assume there exists a path `/a/b/c/b/d.txt`
+       use path pattern like `/**/b/**/*.txt` to glob,
+       the path above will be returned twice
+    3. `**` will match any matched file, directory, symlink and '' by default,
+       when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True) in
+       ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param path: Given path
+    :param pattern: Glob the given relative pattern in the directory represented
+        by this path
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: A list contains tuples of path and file stat,
+        in which paths match `pathname`
+    """
+    for path in sftp_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
+        path_object = SftpPath(path)
+        yield FileEntry(
+            path_object.name, path_object.path_with_protocol, path_object.lstat()
+        )
+
+
+def sftp_iglob(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> Iterator[str]:
+    """Return path iterator in ascending alphabetical order,
+    in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+       Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+       when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+       fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+       Assume there exists a path `/a/b/c/b/d.txt`
+       use path pattern like `/**/b/**/*.txt` to glob,
+       the path above will be returned twice
+    3. `**` will match any matched file, directory, symlink and '' by default,
+       when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True) in
+       ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param path: Given path
+    :param pattern: Glob the given relative pattern in the directory represented
+        by this path
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: An iterator contains paths match `pathname`
+    """
+
+    for path in SftpPath(path).iglob(
+        pattern="", recursive=recursive, missing_ok=missing_ok
+    ):
+        yield path.path_with_protocol
+
+
+def sftp_resolve(path: PathLike, strict=False) -> "str":
+    """Equal to fs_realpath
+
+    :param path: Given path
+    :param strict: Ignore this parameter, just for compatibility
+    :return: Return the canonical path of the specified filename,
+        eliminating any symbolic links encountered in the path.
+    :rtype: SftpPath
+    """
+    return SftpPath(path).resolve(strict).path_with_protocol
+
+
+def sftp_download(
+    src_url: PathLike,
+    dst_url: PathLike,
+    callback: Optional[Callable[[int], None]] = None,
+    followlinks: bool = False,
+    overwrite: bool = True,
+):
+    """
+    Downloads a file from sftp to local filesystem.
+
+    :param src_url: source sftp path
+    :param dst_url: target fs path
+    :param callback: Called periodically during copy, and the input parameter is
+        the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
+    """
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
+    if not is_fs(dst_url):
+        raise OSError(f"dst_url is not fs path: {dst_url}")
+    if not is_sftp(src_url) and not isinstance(src_url, SftpPath):
+        raise OSError(f"src_url is not sftp path: {src_url}")
+
+    dst_path = FSPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
+    if isinstance(src_url, SftpPath):
+        src_path: SftpPath = src_url
+    else:
+        src_path: SftpPath = SftpPath(src_url)
+
+    if followlinks and src_path.is_symlink():
+        src_path = src_path.readlink()
+    if src_path.is_dir():
+        raise IsADirectoryError("Is a directory: %r" % src_url)
+    if str(dst_url).endswith("/"):
+        raise IsADirectoryError("Is a directory: %r" % dst_url)
+
+    dst_path.parent.makedirs(exist_ok=True)
+
+    sftp_callback = None
+    if callback:
+        bytes_transferred_before = 0
+
+        def sftp_callback(bytes_transferred: int, _total_bytes: int):
+            nonlocal bytes_transferred_before
+            callback(bytes_transferred - bytes_transferred_before)  # pyre-ignore[29]
+            bytes_transferred_before = bytes_transferred
+
+    src_path._client.get(
+        src_path._real_path, dst_path.path_without_protocol, callback=sftp_callback
+    )
+
+    src_stat = src_path.stat()
+    dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+    dst_path.chmod(src_stat.st_mode)
+
+
+def sftp_upload(
+    src_url: PathLike,
+    dst_url: PathLike,
+    callback: Optional[Callable[[int], None]] = None,
+    followlinks: bool = False,
+    overwrite: bool = True,
+):
+    """
+    Uploads a file from local filesystem to sftp server.
+
+    :param src_url: source fs path
+    :param dst_url: target sftp path
+    :param callback: Called periodically during copy, and the input parameter is
+        the data size (in bytes) of copy since the last call
+    :param overwrite: whether or not overwrite file when exists, default is True
+    """
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
+    if not is_fs(src_url):
+        raise OSError(f"src_url is not fs path: {src_url}")
+    if not is_sftp(dst_url) and not isinstance(dst_url, SftpPath):
+        raise OSError(f"dst_url is not sftp path: {dst_url}")
+
+    if followlinks and os.path.islink(src_url):
+        src_url = os.readlink(src_url)
+    if os.path.isdir(src_url):
+        raise IsADirectoryError("Is a directory: %r" % src_url)
+    if str(dst_url).endswith("/"):
+        raise IsADirectoryError("Is a directory: %r" % dst_url)
+
+    src_path = FSPath(src_url)
+    if isinstance(dst_url, SftpPath):
+        dst_path: SftpPath = dst_url
+    else:
+        dst_path: SftpPath = SftpPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
+    dst_path.parent.makedirs(exist_ok=True)
+
+    sftp_callback = None
+    if callback:
+        bytes_transferred_before = 0
+
+        def sftp_callback(bytes_transferred: int, _total_bytes: int):
+            nonlocal bytes_transferred_before
+            callback(bytes_transferred - bytes_transferred_before)  # pyre-ignore[29]
+            bytes_transferred_before = bytes_transferred
+
+    dst_path._client.put(
+        src_path.path_without_protocol, dst_path._real_path, callback=sftp_callback
+    )
+
+    src_stat = src_path.stat()
+    dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+    dst_path.chmod(src_stat.st_mode)
+
+
+def sftp_path_join(path: PathLike, *other_paths: PathLike) -> str:
+    """
+    Concat 2 or more path to a complete path
+
+    :param path: Given path
+    :param other_paths: Paths to be concatenated
+    :returns: Concatenated complete path
+
+    .. note ::
+
+        The difference between this function and ``os.path.join`` is that this function
+        ignores left side slash (which indicates absolute path) in ``other_paths``
+        and will directly concat.
+
+        e.g. os.path.join('/path', 'to', '/file') => '/file',
+        but sftp_path_join('/path', 'to', '/file') => '/path/to/file'
+    """
+    return uri_join(fspath(path), *map(fspath, other_paths))
+
+
+def sftp_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
+    """Concatenate sftp files to one file.
+
+    :param src_paths: Given source paths
+    :param dst_path: Given destination path
+    """
+    dst_path_obj = SftpPath(dst_path)
+
+    def get_real_path(path: PathLike) -> str:
+        return SftpPath(path)._real_path
+
+    command = ["cat", *map(get_real_path, src_paths), ">", get_real_path(dst_path)]
+    exec_result = dst_path_obj._exec_command(command)
+    if exec_result.returncode != 0:
+        _logger.error(exec_result.stderr)
+        raise OSError(f"Failed to concat {src_paths} to {dst_path}")
+
+
+def sftp_lstat(path: PathLike) -> StatResult:
+    """
+    Get StatResult of file on sftp, including file size and mtime,
+    referring to fs_getsize and fs_getmtime
+
+    :param path: Given path
+    :returns: StatResult
+    """
+    return SftpPath(path).lstat()
+
+
 def sftp_exists(path: PathLike, followlinks: bool = False) -> bool:
     """
     Test if the path exists