maxframe 0.1.0b5__cp38-cp38-macosx_11_0_arm64.whl

maxframe/lib/aio/lru.py

@@ -0,0 +1,242 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The MIT License
+#
+# Copyright (c) 2018 aio-libs team https://github.com/aio-libs/
+# Copyright (c) 2017 Ocean S. A. https://ocean.io/
+# Copyright (c) 2016-2017 WikiBusiness Corporation http://wikibusiness.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import asyncio
+import os
+import weakref
+from collections import OrderedDict
+from functools import _CacheInfo, _make_key, partial, wraps
+
+__version__ = "1.0.2"
+
+__all__ = ("alru_cache", "clear_all_alru_caches")
+
+_is_ci = (os.environ.get("CI") or "0").lower() in ("1", "true")
+_all_wrapped = weakref.WeakSet()
+
+
+def clear_all_alru_caches():
+    for wrapped in _all_wrapped:
+        wrapped.cache_clear()
+
+
+def unpartial(fn):
+    while hasattr(fn, "func"):
+        fn = fn.func
+
+    return fn
+
+
+def _done_callback(fut, task):
+    if task.cancelled():
+        fut.cancel()
+        return
+
+    exc = task.exception()
+    if exc is not None:
+        fut.set_exception(exc)
+        return
+
+    fut.set_result(task.result())
+
+
+def _cache_invalidate(wrapped, typed, *args, **kwargs):
+    key = _make_key(args, kwargs, typed)
+
+    exists = key in wrapped._cache
+
+    if exists:
+        wrapped._cache.pop(key)
+
+    return exists
+
+
+def _cache_clear(wrapped):
+    wrapped.hits = wrapped.misses = 0
+    wrapped._cache = OrderedDict()
+    wrapped.tasks = set()
+
+
+def _open(wrapped):
+    if not wrapped.closed:
+        raise RuntimeError("alru_cache is not closed")
+
+    was_closed = (
+        wrapped.hits == wrapped.misses == len(wrapped.tasks) == len(wrapped._cache) == 0
+    )
+
+    if not was_closed:
+        raise RuntimeError("alru_cache was not closed correctly")
+
+    wrapped.closed = False
+
+
+def _close(wrapped, *, cancel=False, return_exceptions=True):
+    if wrapped.closed:
+        raise RuntimeError("alru_cache is closed")
+
+    wrapped.closed = True
+
+    if cancel:
+        for task in wrapped.tasks:
+            if not task.done():  # not sure is it possible
+                task.cancel()
+
+    return _wait_closed(wrapped, return_exceptions=return_exceptions)
+
+
+async def _wait_closed(wrapped, *, return_exceptions):
+    wait_closed = asyncio.gather(*wrapped.tasks, return_exceptions=return_exceptions)
+
+    wait_closed.add_done_callback(partial(_close_waited, wrapped))
+
+    ret = await wait_closed
+
+    # hack to get _close_waited callback to be executed
+    await asyncio.sleep(0)
+
+    return ret
+
+
+def _close_waited(wrapped, _):
+    wrapped.cache_clear()
+
+
+def _cache_info(wrapped, maxsize):
+    return _CacheInfo(
+        wrapped.hits,
+        wrapped.misses,
+        maxsize,
+        len(wrapped._cache),
+    )
+
+
+def __cache_touch(wrapped, key):
+    try:
+        wrapped._cache.move_to_end(key)
+    except KeyError:  # not sure is it possible
+        pass
+
+
+def _cache_hit(wrapped, key):
+    wrapped.hits += 1
+    __cache_touch(wrapped, key)
+
+
+def _cache_miss(wrapped, key):
+    wrapped.misses += 1
+    __cache_touch(wrapped, key)
+
+
+def alru_cache(
+    fn=None,
+    maxsize=128,
+    typed=False,
+    *,
+    cache_exceptions=True,
+):
+    def wrapper(fn):
+        _origin = unpartial(fn)
+
+        if not asyncio.iscoroutinefunction(_origin):
+            raise RuntimeError("Coroutine function is required, got {}".format(fn))
+
+        # functools.partialmethod support
+        if hasattr(fn, "_make_unbound_method"):
+            fn = fn._make_unbound_method()
+
+        @wraps(fn)
+        async def wrapped(*fn_args, **fn_kwargs):
+            if wrapped.closed:
+                raise RuntimeError("alru_cache is closed for {}".format(wrapped))
+
+            loop = asyncio.get_event_loop()
+
+            key = _make_key(fn_args, fn_kwargs, typed)
+
+            fut = wrapped._cache.get(key)
+
+            if fut is not None:
+                if not fut.done():
+                    _cache_hit(wrapped, key)
+                    return await asyncio.shield(fut)
+
+                exc = fut._exception
+
+                if exc is None or cache_exceptions:
+                    _cache_hit(wrapped, key)
+                    return fut.result()
+
+                # exception here and cache_exceptions == False
+                wrapped._cache.pop(key)
+
+            fut = loop.create_future()
+            task = loop.create_task(fn(*fn_args, **fn_kwargs))
+            task.add_done_callback(partial(_done_callback, fut))
+
+            wrapped.tasks.add(task)
+            task.add_done_callback(wrapped.tasks.remove)
+
+            wrapped._cache[key] = fut
+
+            if maxsize is not None and len(wrapped._cache) > maxsize:
+                wrapped._cache.popitem(last=False)
+
+            _cache_miss(wrapped, key)
+            return await asyncio.shield(fut)
+
+        _cache_clear(wrapped)
+        wrapped._origin = _origin
+        wrapped.closed = False
+        wrapped.cache_info = partial(_cache_info, wrapped, maxsize)
+        wrapped.cache_clear = partial(_cache_clear, wrapped)
+        wrapped.invalidate = partial(_cache_invalidate, wrapped, typed)
+        wrapped.close = partial(_close, wrapped)
+        wrapped.open = partial(_open, wrapped)
+
+        if _is_ci:
+            _all_wrapped.add(wrapped)
+        return wrapped
+
+    if fn is None:
+        return wrapper
+
+    if callable(fn) or hasattr(fn, "_make_unbound_method"):
+        return wrapper(fn)
+
+    raise NotImplementedError("{} decorating is not supported".format(fn))

maxframe/lib/aio/parallelism.py

@@ -0,0 +1,37 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import multiprocessing
+import threading
+from concurrent.futures import Executor
+from typing import Union
+
+from .base import AioBase, delegate_to_executor, proxy_method_directly
+
+event_types = Union[threading.Event, multiprocessing.Event]
+
+
+@delegate_to_executor("wait")
+@proxy_method_directly("set", "is_set", "clear")
+class AioEvent(AioBase):
+    def __init__(
+        self,
+        event: event_types = None,
+        loop: asyncio.BaseEventLoop = None,
+        executor: Executor = None,
+    ):
+        if event is None:
+            event = threading.Event()
+        super().__init__(event, loop=loop, executor=executor)

maxframe/lib/aio/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/lib/aio/tests/test_aio_file.py

@@ -0,0 +1,55 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+
+import pytest
+
+from ...filesystem import LocalFileSystem
+from .. import AioFileObject, AioFilesystem
+
+
+@pytest.mark.asyncio
+async def test_aio_filesystem():
+    local_fs = LocalFileSystem.get_instance()
+    aio_fs = AioFilesystem(local_fs)
+
+    assert aio_fs.pathsep == local_fs.pathsep
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        file_path = os.path.join(tempdir, "test")
+
+        with open(file_path, "wb") as f:
+            f.write(b"text for test")
+
+        stat = await aio_fs.stat(tempdir)
+        assert stat["type"] == "directory"
+
+
+@pytest.mark.asyncio
+async def test_aio_file_object():
+    with tempfile.TemporaryDirectory() as tempdir:
+        file_path = os.path.join(tempdir, "test")
+
+        f = AioFileObject(open(file_path, "w"))
+        async with f:
+            assert f.readable() is False
+            assert f.mode == "w"
+            await f.write("text for test")
+
+        f2 = AioFileObject(open(file_path))
+        async with f2:
+            async for l in f2:
+                assert len(l) > 0

maxframe/lib/compression.py

@@ -0,0 +1,55 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gzip import GzipFile
+from typing import BinaryIO
+
+try:
+    import lz4
+    import lz4.frame
+except ImportError:  # pragma: no cover
+    lz4 = None
+
+
+_compressions = {"gzip": lambda f: GzipFile(fileobj=f)}
+
+if lz4:
+    _compressions["lz4"] = lz4.frame.open
+
+
+def compress(file: BinaryIO, compress_type: str) -> BinaryIO:
+    """
+    Return a compressed file object.
+
+    Parameters
+    ----------
+    file:
+        file object.
+    compress_type: str
+       compression type.
+
+    Returns
+    -------
+    compressed_file:
+        compressed file object.
+    """
+    try:
+        compress_ = _compressions[compress_type]
+    except KeyError:  # pragma: no cover
+        raise ValueError(
+            f"Unknown compress type: {compress_type}, "
+            f'available include: {", ".join(_compressions)}'
+        )
+
+    return compress_(file)

maxframe/lib/cython/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/lib/cython/libcpp.pxd

@@ -0,0 +1,30 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# complementary header for C++ STL libs not included in Cython
+
+from libc.stdint cimport uint_fast64_t
+
+
+cdef extern from "<random>" namespace "std" nogil:
+    cdef cppclass mt19937_64:
+        ctypedef uint_fast64_t result_type
+
+        mt19937_64() except +
+        mt19937_64(result_type seed) except +
+        result_type operator()() except +
+        result_type min() except +
+        result_type max() except +
+        void discard(size_t z) except +
+        void seed(result_type seed) except +

maxframe/lib/filesystem/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .base import FileSystem
+from .core import file_size, get_fs, glob, open_file, register_filesystem
+from .fsmap import FSMap
+
+# noinspection PyUnresolvedReferences
+from .hdfs import HadoopFileSystem
+from .local import LocalFileSystem

maxframe/lib/filesystem/_glob.py

@@ -0,0 +1,173 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+import os
+import re
+
+from .core import FileSystem
+
+magic_check = re.compile("([*?[])")
+magic_check_bytes = re.compile(b"([*?[])")
+
+
+def has_magic(s):
+    if isinstance(s, bytes):  # pragma: no cover
+        match = magic_check_bytes.search(s)
+    else:
+        match = magic_check.search(s)
+    return match is not None
+
+
+def _ishidden(path):
+    return path[0] in (".", b"."[0])
+
+
+def _isrecursive(pattern):
+    if isinstance(pattern, bytes):  # pragma: no cover
+        return pattern == b"**"
+    else:
+        return pattern == "**"
+
+
+class FileSystemGlob:
+    def __init__(self, fs: FileSystem):
+        self._fs = fs
+
+    def glob(self, pathname, recursive=False):
+        """Return a list of paths matching a pathname pattern.
+
+        The pattern may contain simple shell-style wildcards a la
+        fnmatch. However, unlike fnmatch, filenames starting with a
+        dot are special cases that are not matched by '*' and '?'
+        patterns.
+
+        If recursive is true, the pattern '**' will match any files and
+        zero or more directories and subdirectories.
+        """
+        return list(self.iglob(pathname, recursive=recursive))
+
+    def iglob(self, pathname, recursive=False):
+        """Return an iterator which yields the paths matching a pathname pattern.
+
+        The pattern may contain simple shell-style wildcards a la
+        fnmatch. However, unlike fnmatch, filenames starting with a
+        dot are special cases that are not matched by '*' and '?'
+        patterns.
+
+        If recursive is true, the pattern '**' will match any files and
+        zero or more directories and subdirectories.
+        """
+        it = self._iglob(pathname, recursive, False)
+        if recursive and _isrecursive(pathname):  # pragma: no cover
+            s = next(it)  # skip empty string
+            assert not s
+        return it
+
+    def _iglob(self, pathname, recursive, dironly):
+        dirname, basename = self._fs.path_split(pathname.replace(os.path.sep, "/"))
+        if not has_magic(pathname):
+            assert not dironly
+            if basename:
+                if self._fs.exists(pathname):
+                    yield pathname
+            else:  # pragma: no cover
+                # Patterns ending with a slash should match only directories
+                if self._fs.isdir(dirname):
+                    yield pathname
+            return
+        if not dirname:  # pragma: no cover
+            if recursive and _isrecursive(basename):
+                yield from self._glob2(dirname, basename, dironly)
+            else:
+                yield from self._glob1(dirname, basename, dironly)
+            return
+        # `os.path.split()` returns the argument itself as a dirname if it is a
+        # drive or UNC path.  Prevent an infinite recursion if a drive or UNC path
+        # contains magic characters (i.e. r'\\?\C:').
+        if dirname != pathname and has_magic(dirname):
+            dirs = self._iglob(dirname, recursive, True)
+        else:
+            dirs = [dirname]
+        if has_magic(basename):
+            if recursive and _isrecursive(basename):
+                glob_in_dir = self._glob2
+            else:
+                glob_in_dir = self._glob1
+        else:
+            glob_in_dir = self._glob0
+        for dirname in dirs:
+            for name in glob_in_dir(dirname, basename, dironly):
+                if dirname:
+                    yield self._fs.path_join(dirname, name)
+                else:
+                    yield name
+
+    # These 2 helper functions non-recursively glob inside a literal directory.
+    # They return a list of basenames.  _glob1 accepts a pattern while _glob0
+    # takes a literal basename (so it only has to check for its existence).
+
+    def _glob1(self, dirname, pattern, dironly):
+        names = list(self._iterdir(dirname, dironly))
+        if not _ishidden(pattern):
+            names = (x for x in names if not _ishidden(x))
+        return fnmatch.filter(names, pattern)
+
+    def _glob0(self, dirname, basename, dironly):  # pragma: no cover
+        if not basename:
+            # `os.path.split()` returns an empty basename for paths ending with a
+            # directory separator.  'q*x/' should match only directories.
+            if self._fs.isdir(dirname):
+                return [basename]
+        else:
+            if self._fs.exists(self._fs.path_join(dirname, basename)):
+                return [basename]
+        return []
+
+    # Following functions are not public but can be used by third-party code.
+
+    def glob0(self, dirname, pattern):  # pragma: no cover
+        return self._glob0(dirname, pattern, False)
+
+    def glob1(self, dirname, pattern):  # pragma: no cover
+        return self._glob1(dirname, pattern, False)
+
+    # This helper function recursively yields relative pathnames inside a literal
+    # directory.
+
+    def _glob2(self, dirname, pattern, dironly):  # pragma: no cover
+        assert _isrecursive(pattern)
+        yield pattern[:0]
+        yield from self._rlistdir(dirname, dironly)
+
+    # If dironly is false, yields all file names inside a directory.
+    # If dironly is true, yields only directory names.
+    def _iterdir(self, dirname, dironly):
+        if not dirname:  # pragma: no cover
+            dirname = ""
+        if not self._fs.isdir(dirname):
+            return iter(())
+        for entry in self._fs.ls(dirname):
+            if not dironly or self._fs.isdir(entry):
+                yield self._fs.path_split(entry)[-1]
+
+    # Recursively yields relative pathnames inside a literal directory.
+    def _rlistdir(self, dirname, dironly):  # pragma: no cover
+        names = list(self._iterdir(dirname, dironly))
+        for x in names:
+            if not _ishidden(x):
+                yield x
+                path = self._fs.path_join(dirname, x) if dirname else x
+                for y in self._rlistdir(path, dironly):
+                    yield self._fs.path_join(x, y)

maxframe/lib/filesystem/_oss_lib/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.