megfile 2.1.3.post1__py3-none-any.whl → 2.2.0__py3-none-any.whl

megfile/__init__.py

@@ -1,5 +1,6 @@
 from megfile.fs import fs_abspath, fs_access, fs_cwd, fs_exists, fs_expanduser, fs_getmd5, fs_getmtime, fs_getsize, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_isabs, fs_isdir, fs_isfile, fs_islink, fs_ismount, fs_listdir, fs_load_from, fs_lstat, fs_makedirs, fs_move, fs_readlink, fs_realpath, fs_relpath, fs_remove, fs_rename, fs_resolve, fs_save_as, fs_scan, fs_scan_stat, fs_scandir, fs_stat, fs_symlink, fs_sync, fs_unlink, fs_walk, is_fs
 from megfile.fs_path import FSPath
+from megfile.http import http_exists, http_getmtime, http_getsize, http_open, http_stat, is_http
 from megfile.http_path import HttpPath, HttpsPath
 from megfile.s3 import is_s3, s3_access, s3_buffered_open, s3_cached_open, s3_concat, s3_copy, s3_download, s3_exists, s3_getmd5, s3_getmtime, s3_getsize, s3_glob, s3_glob_stat, s3_hasbucket, s3_iglob, s3_isdir, s3_isfile, s3_listdir, s3_load_content, s3_load_from, s3_lstat, s3_makedirs, s3_memory_open, s3_move, s3_open, s3_path_join, s3_pipe_open, s3_prefetch_open, s3_readlink, s3_remove, s3_rename, s3_save_as, s3_scan, s3_scan_stat, s3_scandir, s3_stat, s3_symlink, s3_sync, s3_unlink, s3_upload, s3_walk
 from megfile.s3_path import S3Path
@@ -7,12 +8,11 @@ from megfile.sftp import is_sftp, sftp_absolute, sftp_chmod, sftp_concat, sftp_c
 from megfile.sftp_path import SftpPath
 from megfile.smart import smart_access, smart_cache, smart_combine_open, smart_concat, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob, smart_glob_stat, smart_iglob, smart_isdir, smart_isfile, smart_islink, smart_listdir, smart_load_content, smart_load_from, smart_load_text, smart_lstat, smart_makedirs, smart_move, smart_open, smart_path_join, smart_readlink, smart_realpath, smart_remove, smart_rename, smart_save_as, smart_save_content, smart_save_text, smart_scan, smart_scan_stat, smart_scandir, smart_stat, smart_symlink, smart_sync, smart_touch, smart_unlink, smart_walk
 from megfile.smart_path import SmartPath
+from megfile.stdio import is_stdio, stdio_open
 from megfile.stdio_path import StdioPath
 from megfile.version import VERSION as __version__
 
 __all__ = [
-    'is_fs',
-    'is_s3',
     'smart_access',
     'smart_cache',
     'smart_combine_open',
@@ -54,6 +54,7 @@ __all__ = [
     'smart_readlink',
     'smart_lstat',
     'smart_concat',
+    'is_s3',
     's3_access',
     's3_buffered_open',
     's3_cached_open',
@@ -94,6 +95,7 @@ __all__ = [
     's3_symlink',
     's3_readlink',
     's3_concat',
+    'is_fs',
     'fs_abspath',
     'fs_access',
     'fs_exists',
@@ -131,6 +133,14 @@ __all__ = [
     'fs_getmd5',
     'fs_symlink',
     'fs_readlink',
+    'is_http',
+    'http_open',
+    'http_stat',
+    'http_getsize',
+    'http_getmtime',
+    'http_exists',
+    'is_stdio',
+    'stdio_open',
     'is_sftp',
     'sftp_readlink',
     'sftp_absolute',

megfile/cli.py

@@ -17,11 +17,12 @@ from megfile.version import VERSION
 
 logging.basicConfig(level=logging.ERROR)
 logging.getLogger('megfile').setLevel(level=logging.INFO)
+DEFAULT_BLOCK_SIZE = 8 * 2**20  # 8MB
 
 
 @click.group()
 def cli():
-    """Megfile Client"""
+    """Client"""
 
 
 def safe_cli():  # pragma: no cover
@@ -306,11 +307,87 @@ def cat(path: str):
         shutil.copyfileobj(file, sys.stdout.buffer)
 
 
+@cli.command(
+    short_help='Concatenate any files and send first n lines of them to stdout.'
+)
+@click.argument('path')
+@click.option(
+    '-n',
+    '--lines',
+    type=click.INT,
+    default=10,
+    help='print the first NUM lines')
+def head(path: str, lines: int):
+    with smart_open(path, 'rb') as f:
+        f.seek(0, os.SEEK_END)
+        file_size = f.tell()
+        f.seek(0, os.SEEK_SET)
+        for _ in range(lines):
+            click.echo(f.readline().strip(b'\n'))
+            if f.tell() >= file_size:
+                break
+
+
+@cli.command(
+    short_help='Concatenate any files and send last n lines of them to stdout.')
+@click.argument('path')
+@click.option(
+    '-n',
+    '--lines',
+    type=click.INT,
+    default=10,
+    help='print the last NUM lines')
+def tail(path: str, lines: int):
+    line_list = []
+    with smart_open(path, 'rb') as f:
+        f.seek(0, os.SEEK_END)
+        file_size = f.tell()
+        f.seek(0, os.SEEK_SET)
+
+        for current_offset in range(file_size - DEFAULT_BLOCK_SIZE,
+                                    0 - DEFAULT_BLOCK_SIZE,
+                                    -DEFAULT_BLOCK_SIZE):
+            current_offset = max(0, current_offset)
+            f.seek(current_offset)
+            block_lines = f.read(DEFAULT_BLOCK_SIZE).split(b'\n')
+            if len(line_list) > 0:
+                block_lines[-1] += line_list[0]
+                block_lines.extend(line_list[1:])
+            if len(block_lines) > lines:
+                line_list = block_lines[-lines:]
+                break
+            else:
+                line_list = block_lines
+    for line in line_list:
+        click.echo(line)
+
+
+@cli.command(short_help='Write bytes from stdin to file.')
+@click.argument('path')
+@click.option('-a', '--append', is_flag=True, help='Append to the given file')
+@click.option(
+    '-o', '--stdout', is_flag=True, help='File content to standard output')
+def to(path: str, append: bool, stdout: bool):
+    mode = 'wb'
+    if append:
+        mode = 'ab'
+    with smart_open('stdio://0', 'rb') as stdin, smart_open(
+            path, mode) as f, smart_open('stdio://1', 'wb') as stdout_fd:
+        length = 16 * 1024
+        while True:
+            buf = stdin.read(length)
+            if not buf:
+                break
+            f.write(buf)
+            if stdout:
+                stdout_fd.write(buf)
+
+
 @cli.command(
     short_help='Produce an md5sum file for all the objects in the path.')
 @click.argument('path')
 def md5sum(path: str):
-    click.echo(smart_getmd5(path))
+    click.echo(smart_getmd5(path, recalculate=True))
 
 
 @cli.command(

megfile/lib/compat.py

@@ -19,7 +19,7 @@ import sys
 
 if sys.version_info.major == 3 and sys.version_info.minor >= 8:
     from shutil import copytree
-else:
+else:  # pragma: no cover
     from shutil import Error, copy2, copystat
 
     def copytree(

megfile/sftp_path.py

@@ -475,8 +475,11 @@ def sftp_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
 class SftpPath(URIPath):
     """sftp protocol
 
-    uri format: sftp://[username[:password]@]hostname[:port]/file_path
-    e.g. sftp://username:password@127.0.0.1:22/data/test/
+    uri format: 
+    - absolute path
+        - sftp://[username[:password]@]hostname[:port]//file_path
+    - relative path
+        - - sftp://[username[:password]@]hostname[:port]/file_path
     """
 
     protocol = "sftp"
@@ -484,10 +487,13 @@ class SftpPath(URIPath):
     def __init__(self, path: "PathLike", *other_paths: "PathLike"):
         super().__init__(path, *other_paths)
         parts = urlsplit(self.path)
-        self._real_path = parts.path
-        if not self._real_path.startswith('/'):
-            self._real_path = f"/{self._real_path}"
         self._urlsplit_parts = parts
+        self._real_path = parts.path
+        if parts.path.startswith('//'):
+            self._root_dir = '/'
+        else:
+            self._root_dir = self._client.normalize('.')
+        self._real_path = os.path.join(self._root_dir, parts.path.lstrip('/'))
 
     @property
     def _client(self):
@@ -497,9 +503,14 @@ class SftpPath(URIPath):
             username=self._urlsplit_parts.username,
             password=self._urlsplit_parts.password)
 
-    def _generate_path_object(self, sftp_local_path: str):
-        new_parts = self._urlsplit_parts._replace(
-            path=sftp_local_path.lstrip('/'))
+    def _generate_path_object(
+            self, sftp_local_path: str, resolve: bool = False):
+        if resolve or self._root_dir == '/':
+            sftp_local_path = f"//{sftp_local_path.lstrip('/')}"
+        else:
+            sftp_local_path = os.path.relpath(
+                sftp_local_path, start=self._root_dir)
+        new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
         return self.from_path(urlunsplit(new_parts))
 
     def exists(self, followlinks: bool = False) -> bool:
@@ -971,7 +982,7 @@ class SftpPath(URIPath):
         :rtype: SftpPath
         '''
         path = self._client.normalize(self._real_path)
-        return self._generate_path_object(path)
+        return self._generate_path_object(path, resolve=True)
 
     def md5(self, recalculate: bool = False, followlinks: bool = True):
         '''

megfile/smart.py

@@ -7,16 +7,16 @@ from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterable, Iterator, List
 
 from tqdm import tqdm
 
-from megfile.fs import fs_copy
-from megfile.interfaces import Access, ContextIterator, FileEntry, NullCacher, PathLike, StatResult
+from megfile.fs import fs_copy, is_fs
+from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, NullCacher, PathLike, StatResult
 from megfile.lib.combine_reader import CombineReader
 from megfile.lib.compare import get_sync_type, is_same_file
 from megfile.lib.compat import fspath
 from megfile.lib.glob import globlize, ungloblize
-from megfile.s3 import S3Cacher, is_s3, s3_concat, s3_copy, s3_download, s3_load_content, s3_open, s3_upload
+from megfile.s3 import is_s3, s3_concat, s3_copy, s3_download, s3_load_content, s3_open, s3_upload
 from megfile.sftp import sftp_concat, sftp_copy, sftp_download, sftp_upload
 from megfile.smart_path import SmartPath, get_traditional_path
-from megfile.utils import combine
+from megfile.utils import combine, generate_cache_path
 
 __all__ = [
     'smart_access',
@@ -64,6 +64,7 @@ __all__ = [
     'smart_readlink',
     'register_copy_func',
     'smart_concat',
+    'SmartCacher',
 ]
 
 
@@ -883,15 +884,38 @@ def smart_save_text(path: PathLike, text: str) -> None:
         fd.write(text)
 
 
-def smart_cache(path, s3_cacher=S3Cacher, **options):
+class SmartCacher(FileCacher):
+    cache_path = None
+
+    def __init__(
+            self, path: str, cache_path: Optional[str] = None, mode: str = 'r'):
+        if mode not in ('r', 'w', 'a'):
+            raise ValueError('unacceptable mode: %r' % mode)
+        if mode in ('r', 'a'):
+            if cache_path is None:
+                cache_path = generate_cache_path(path)
+            smart_copy(path, cache_path)
+        self.name = path
+        self.mode = mode
+        self.cache_path = cache_path
+
+    def _close(self):
+        if self.cache_path is not None and \
+            os.path.exists(self.cache_path):
+            if self.mode in ('w', 'a'):
+                smart_copy(self.cache_path, self.name)
+            os.unlink(self.cache_path)
+
+
+def smart_cache(path, cacher=SmartCacher, **options):
     '''Return a path to Posixpath Interface
 
     param path: Path to cache
     param s3_cacher: Cacher for s3 path
     param options: Optional arguments for s3_cacher
     '''
-    if is_s3(path):
-        return s3_cacher(path, **options)
+    if not is_fs(path):
+        return cacher(path, **options)
     return NullCacher(path)
 
 

megfile/version.py

@@ -1 +1 @@
-VERSION = "2.1.3.post1"
+VERSION = "2.2.0"

{megfile-2.1.3.post1.dist-info → megfile-2.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 2.1.3.post1
+Version: 2.2.0
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii
@@ -62,6 +62,29 @@ megfile - Megvii FILE library
 
 Here's an example of writing a file to s3 / sftp / fs, syncing to local, reading and finally deleting it.
 
+### Path Format
+- local file
+    - unix filesystem path
+    - examples:
+        - `/data/test.txt`
+        - `test.txt`
+        - 1
+- oss
+    - `s3[+profile_name]://bucket/key`
+- sftp
+    - `sftp://[username[:password]@]hostname[:port]//absolute_file_path`
+    - `sftp://[username[:password]@]hostname[:port]/relative_file_path`
+- http
+    - http / https url
+    - examples:
+        - `http://hostname/test`
+        - `https://hostname/test`
+- stdio
+    - `stdio://-`
+    - `stdio://0`
+    - `stdio://1`
+    - `stdio://2`
+
 ### Functional Interface
 ```python
 from megfile import smart_open, smart_exists, smart_sync, smart_remove, smart_glob
@@ -201,7 +224,7 @@ s3 =
 megfile.smart_copy('s3+profile1://bucket/key', 's3+profile2://bucket/key')
 ```
 
-sftp path format is `sftp://[username[:password]@]hostname[:port]/file_path`, and sftp support some environments:
+sftp support some environments:
 ```
 # If you are not set username or password in path, you can set them in environments
 $ export SFTP_USERNAME=user

{megfile-2.1.3.post1.dist-info → megfile-2.2.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
-megfile/__init__.py,sha256=wlijLJS6F8Vdm1xjpsjd9fKBeDGpgIMrVsXYSeOQ4Xw,5434
-megfile/cli.py,sha256=5WnLtC2PBXfNDB4h9i4idjMEwu334MGXz-VdbfHoed8,10396
+megfile/__init__.py,sha256=Qsi3XNP_0XYoSol-1AGutZqo0rfBnzaiZ-HVXll4fB0,5721
+megfile/cli.py,sha256=zB6a1QjvIXqmd4nQR9YSTy3RH61pTX14E63uICjIRG8,12797
 megfile/errors.py,sha256=H5wCiVBvOtQo5cu3JLhAUCpQJsCkPNm1KNw9o8y4Las,11349
 megfile/fs.py,sha256=sjkm_LsvNCw8hj9Ee-1HSNzvg7bRHTPem8KTDDBQlCw,11621
 megfile/fs_path.py,sha256=Lz-u8XL2RMViHl_39yapAwX5XSE_ZpQsuhAvVve6rk8,38532
@@ -10,16 +10,16 @@ megfile/pathlike.py,sha256=52e6POtMfUCC3Hb9XNwBLXM3Gkz3aGP8H8AagGKlDSg,29307
 megfile/s3.py,sha256=6d9bBVkVdXIvmJLfpOgJkK4nVAsLR3lirrB6x1-P2y8,12437
 megfile/s3_path.py,sha256=2AY_Rcn0ZR2TpPSqWxBcbUuQ3JuS7XerZ-wBK4wXyMo,84754
 megfile/sftp.py,sha256=qzfgI-MGzNsr7adUQaiVKS7zRHGuK-Hs6o45wNIAcww,11943
-megfile/sftp_path.py,sha256=Gb-x0H3ziX5N7pMxj768-ootIfn5ZTyATi_gT4FWqMw,46668
-megfile/smart.py,sha256=Q-vfhUL06F4FDOmjqJDupP6ZFOF9y9IdUG7XY1AWIjA,31892
+megfile/sftp_path.py,sha256=h_NeZtkKJ44hBIuhvBbLUlADm02uEkRT2T4PBU44uP4,47103
+megfile/smart.py,sha256=xjRRujYAPzXlrYxV7ZCAPXl0biUNn4730ATKc_tzIzE,32694
 megfile/smart_path.py,sha256=Rwb1McXsshi9-F6miTRqE6j8FO2j1edjmSxZF32YZ6E,6708
 megfile/stdio.py,sha256=qmi1c7VTll6Y6ya9MCd-dSKffocX2GafA-YUncZRU1Y,559
 megfile/stdio_path.py,sha256=ULMzGOj7SBMn3c7fYVSDepT_1nEUiVetc-xRt2pR5o4,2682
-megfile/version.py,sha256=_FTDnhkRJDhpJvVJsl1dT-5bZzNKJvdbZf5nQuT1Er0,25
+megfile/version.py,sha256=SX94SfG1HNW6IdIYW_iqO17A0qZmHZUfyXnyAzilMUs,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
-megfile/lib/compat.py,sha256=6m4eIwCndB8CbIl0FRx0Y0tg8VAcjdRLHma2bdWPO0k,3013
+megfile/lib/compat.py,sha256=rYjfzQ3svuY7pB37W1JGyWH1kxd9aT4RtIe90npPtXI,3033
 megfile/lib/fnmatch.py,sha256=HgdlnEWBsdFUOZqnW_v1kj1jeH_9lMcCqW85pyMu4vM,4054
 megfile/lib/glob.py,sha256=7i9dIput9rI9JIPyTZX-JDmFS7IP_THlX1k-35foAfw,9732
 megfile/lib/joinpath.py,sha256=D4Px6-lnDDpYs1LMUHkTIGqMPJQ0oCBGfTzREs373iU,929
@@ -36,10 +36,10 @@ megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=DEVSwUQ1-1rkHBIuXBxgEYiepq8W25JIWYQrC1gcFrc,9005
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-2.1.3.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-2.1.3.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-2.1.3.post1.dist-info/METADATA,sha256=M2qXA2n9vE2mTZqq0-uhCcYhTb5fz7J0BGKr0lQHCGk,10160
-megfile-2.1.3.post1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-megfile-2.1.3.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-2.1.3.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-2.1.3.post1.dist-info/RECORD,,
+megfile-2.2.0.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-2.2.0.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-2.2.0.dist-info/METADATA,sha256=5hJWi8RWnKtRNaoubN8ZWAtJ-x62XVGsDUEjN9XVI-4,10595
+megfile-2.2.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+megfile-2.2.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-2.2.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-2.2.0.dist-info/RECORD,,