megfile 3.0.6.post1__py3-none-any.whl → 3.1.0.post1__py3-none-any.whl

megfile/lib/s3_cached_handler.py

@@ -1,6 +1,5 @@
 import os
-from io import UnsupportedOperation
-from typing import Iterable, List, Optional
+from typing import Optional
 
 from megfile.errors import translate_fs_error, translate_s3_error
 from megfile.lib.s3_memory_handler import S3MemoryHandler
@@ -36,13 +35,13 @@ class S3CachedHandler(S3MemoryHandler):
         self._download_fileobj()
 
         if remove_cache_when_open:
-            os.unlink(self._cache_path)
+            os.unlink(self._cache_path)  # pyre-ignore[6]
 
     def fileno(self) -> int:
         # allow numpy.array to create a memmaped ndarray
         return self._fileobj.fileno()
 
     def _translate_error(self, error: Exception):
-        error = translate_fs_error(error, self._cache_path)
+        error = translate_fs_error(error, self._cache_path)  # pyre-ignore[6]
         error = translate_s3_error(error, self.name)
         return error

megfile/lib/s3_limited_seekable_writer.py

@@ -3,7 +3,7 @@ from io import BytesIO
 from logging import getLogger as get_logger
 from typing import Optional
 
-from megfile.config import DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE
+from megfile.config import DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, DEFAULT_MIN_BLOCK_SIZE
 from megfile.errors import raise_s3_error
 from megfile.interfaces import Seekable
 from megfile.lib.s3_buffered_writer import S3BufferedWriter
@@ -11,7 +11,7 @@ from megfile.lib.s3_buffered_writer import S3BufferedWriter
 _logger = get_logger(__name__)
 
 
-class S3LimitedSeekableWriter(Seekable, S3BufferedWriter):
+class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
     ''' For file format like msgpack and mp4, it's a pain that you need to write
     header before writing the data. So it's kind of hard to make streaming write
     to unseekable file system like s3. In this case, we will try to keep the first
@@ -25,7 +25,7 @@ class S3LimitedSeekableWriter(Seekable, S3BufferedWriter):
             key: str,
             *,
             s3_client,
-            block_size: int = DEFAULT_BLOCK_SIZE,
+            block_size: int = DEFAULT_MIN_BLOCK_SIZE,
             head_block_size: Optional[int] = None,
             tail_block_size: Optional[int] = None,
             max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
@@ -69,6 +69,8 @@ class S3LimitedSeekableWriter(Seekable, S3BufferedWriter):
             target_offset = self._offset + offset
         elif whence == os.SEEK_END:
             target_offset = self._content_size + offset
+        else:
+            raise OSError('Unsupported whence value: %d' % whence)
 
         if target_offset < self._head_block_size:
             self._head_buffer.seek(target_offset)

megfile/lib/s3_memory_handler.py

@@ -2,11 +2,11 @@ import os
 from io import BytesIO, UnsupportedOperation
 from typing import Iterable, List, Optional
 
-from megfile.errors import S3ConfigError, UnknownError, raise_s3_error, translate_fs_error, translate_s3_error
+from megfile.errors import S3ConfigError, UnknownError, raise_s3_error, translate_s3_error
 from megfile.interfaces import Readable, Seekable, Writable
 
 
-class S3MemoryHandler(Readable, Seekable, Writable):
+class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
 
     def __init__(
             self,
@@ -41,8 +41,8 @@ class S3MemoryHandler(Readable, Seekable, Writable):
     def tell(self) -> int:
         return self._fileobj.tell()
 
-    def seek(self, cookie: int, whence: int = os.SEEK_SET) -> int:
-        return self._fileobj.seek(cookie, whence)
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        return self._fileobj.seek(offset, whence)
 
     def readable(self) -> bool:
         return self._mode[0] == 'r' or self._mode[-1] == '+'
@@ -55,12 +55,16 @@ class S3MemoryHandler(Readable, Seekable, Writable):
     def readline(self, size: Optional[int] = None) -> bytes:
         if not self.readable():
             raise UnsupportedOperation('not readable')
+        if size is None:
+            size = -1
         return self._fileobj.readline(size)
 
-    def readlines(self) -> List[bytes]:
+    def readlines(self, hint: Optional[int] = None) -> List[bytes]:
         if not self.readable():
             raise UnsupportedOperation('not readable')
-        return self._fileobj.readlines()
+        if hint is None:
+            hint = -1
+        return self._fileobj.readlines(hint)
 
     def writable(self) -> bool:
         return self._mode[0] == 'w' or \

megfile/lib/s3_pipe_handler.py

@@ -24,7 +24,7 @@ def _close_s3_pipes():  # pragma: no cover
         try_close_pipe(w)
 
 
-class S3PipeHandler(Readable, Writable):
+class S3PipeHandler(Readable[bytes], Writable[bytes]):
 
     def __init__(
             self,

megfile/lib/s3_prefetch_reader.py

@@ -1,4 +1,3 @@
-import os
 from concurrent.futures import Future
 from io import BytesIO
 from typing import Optional
@@ -42,6 +41,8 @@ class S3PrefetchReader(BasePrefetchReader):
         self._key = key
         self._client = s3_client
         self._profile_name = profile_name
+        self._content_etag = None
+        self._content_info = None
 
         super().__init__(
             block_size=block_size,
@@ -58,7 +59,7 @@ class S3PrefetchReader(BasePrefetchReader):
                 first_index_response['ContentRange'].split('/')[-1])
         except S3InvalidRangeError:
             # usually when read a empty file
-            # TODO: use minio test empty file: https://hub.docker.com/r/minio/minio
+            # can use minio test empty file: https://hub.docker.com/r/minio/minio
             first_index_response = self._fetch_response()
             content_size = int(first_index_response['ContentLength'])
 
@@ -76,7 +77,8 @@ class S3PrefetchReader(BasePrefetchReader):
             self._bucket, self._key)
 
     def _fetch_response(
-            self, start: Optional[int] = None,
+            self,
+            start: Optional[int] = None,
             end: Optional[int] = None) -> dict:
 
         def fetch_response() -> dict:
@@ -103,9 +105,9 @@ class S3PrefetchReader(BasePrefetchReader):
             index + 1) * self._block_size - 1
         response = self._fetch_response(start=start, end=end)
         etag = response.get('ETag', None)
-        if etag is not None and etag != self._content_etag:  # pytype: disable=attribute-error
+        if etag is not None and etag != self._content_etag:
             raise S3FileChangedError(
                 'File changed: %r, etag before: %s, after: %s' %
-                (self.name, self._content_info, response))  # pytype: disable=attribute-error
+                (self.name, self._content_info, response))
 
         return response['Body']

megfile/lib/s3_share_cache_reader.py

@@ -44,14 +44,14 @@ class S3ShareCacheReader(S3PrefetchReader):
             profile_name=profile_name,
         )
 
-    def _get_futures(self):
+    def _get_futures(self) -> 'ShareCacheFutureManager':
         futures = thread_local(
             'S3ShareCacheReader.' + self._cache_key, ShareCacheFutureManager)
         futures.register(self.name)
         return futures
 
     def _seek_buffer(self, index: int, offset: int = 0):
-        # The corresponding block is probably not downloaded when seeked to a new position
+        # The corresponding block is probably not downloaded when sought to a new position
         # So record the offset first, set it when it is accessed
         self._cached_offset = offset
         self._block_index = index

megfile/lib/shadow_handler.py

@@ -58,18 +58,18 @@ class ShadowHandler(Readable, Seekable, Writable, BaseShadowHandler):
     def readable(self) -> bool:
         return is_readable(self._file_object)
 
-    def read(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def read(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         with self._ensure_offset():
-            return self._file_object.read(size)
+            return self._file_object.read(size)  # pyre-ignore[6]
 
-    def readline(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         with self._ensure_offset():
-            return self._file_object.readline(size)
+            return self._file_object.readline(size)  # pyre-ignore[6]
 
     def writable(self) -> bool:
         return is_writable(self._file_object)
 
-    def write(self, data: AnyStr):  # pytype: disable=signature-mismatch
+    def write(self, data: AnyStr):
         with self._ensure_offset():
             return self._file_object.write(data)
 

megfile/lib/stdio_handler.py

@@ -40,10 +40,10 @@ class STDReader(STDHandler, Readable):
     def name(self) -> str:
         return "stdin"
 
-    def read(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def read(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         return self._handler.read(size)
 
-    def readline(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         return self._handler.readline()
 
 
@@ -79,5 +79,5 @@ class STDWriter(STDHandler, Writable):
     def name(self) -> str:
         return self._name
 
-    def write(self, data: AnyStr) -> int:  # pytype: disable=signature-mismatch
+    def write(self, data: AnyStr) -> int:
         return self._handler.write(data)