maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/lib/filesystem/tests/test_oss.py

@@ -0,0 +1,182 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from io import BytesIO
+
+import mock
+import pytest
+
+from .. import oss
+from .._oss_lib import glob as og
+from .._oss_lib.common import OSSFileEntry
+from ..oss import build_oss_path
+
+
+class OSSObjInfo:
+    def __init__(self, name, content):
+        self.key = name
+        # Use the current time as "Last-Modified" in the test.
+        self.last_modified = int(time.time())
+        self.size = len(content.encode("utf8"))
+
+
+class ObjectMeta:
+    def __init__(self, key, obj_dict):
+        self.headers = {}
+        self.headers["Last-Modified"] = int(time.time())
+        self.headers["Content-Length"] = len(obj_dict[key].encode("utf8"))
+
+
+class MockObject:
+    def __init__(self, obj_dict, key, byte_range):
+        self._stream = BytesIO(obj_dict[key].encode("utf8"))
+        self._byte_range = byte_range
+
+    def read(self):
+        self._stream.seek(self._byte_range[0])
+        if self._byte_range[1] is None:
+            return self._stream.read()
+        else:
+            size = self._byte_range[1] - self._byte_range[0] + 1
+            return self._stream.read(size)
+
+
+class SideEffectBucket:
+    def __init__(self, *_, **__):
+        self.obj_dict = {
+            "file.csv": "id1,id2,id3\n1,2,3\n",
+            "dir/": "",
+            "dir/file1.csv": "2",
+            "dir/file2.csv": "3",
+            "dir/subdir/": "",
+            "dir/subdir/file3.csv": "s4",
+            "dir/subdir/file4.csv": "s5",
+            "dir2/": "",
+            "dir2/file6.csv": "6",
+            "dir2/file7.csv": "7",
+        }
+
+    def get_object_meta(self, key):
+        return ObjectMeta(key, self.obj_dict)
+
+    def object_exists(self, key):
+        return key in self.obj_dict.keys()
+
+    def get_object(self, key, byte_range):
+        return MockObject(self.obj_dict, key, byte_range)
+
+
+class SideEffectObjIter:
+    def __init__(self, *args, **kwargs):
+        self.bucket = args[0]
+        self.prefix = kwargs["prefix"]
+
+    def __iter__(self):
+        for name, content in self.bucket.obj_dict.items():
+            if name.startswith(self.prefix):
+                yield OSSObjInfo(name, content)
+
+
+@mock.patch("oss2.Bucket", side_effect=SideEffectBucket)
+@mock.patch("oss2.ObjectIteratorV2", side_effect=SideEffectObjIter)
+def test_oss_filesystem(fake_obj_iter, fake_oss_bucket):
+    access_key_id = "your_access_key_id"
+    access_key_secret = "your_access_key_secret"
+    end_point = "your_endpoint"
+
+    file_path = f"oss://bucket/file.csv"
+    dir_path = f"oss://bucket/dir/"
+    dir_path_content_magic = f"oss://bucket/dir*/"
+    other_scheme_path = f"scheme://netloc/path"
+    not_exist_file_path = f"oss://bucket/not_exist.csv"
+
+    fake_file_path = build_oss_path(
+        file_path, access_key_id, access_key_secret, end_point
+    )
+    fake_dir_path = build_oss_path(
+        dir_path, access_key_id, access_key_secret, end_point
+    )
+    fake_dir_path_contains_magic = build_oss_path(
+        dir_path_content_magic, access_key_id, access_key_secret, end_point
+    )
+    fake_other_scheme_path = build_oss_path(
+        other_scheme_path, access_key_id, access_key_secret, end_point
+    )
+    fake_not_exist_file_path = build_oss_path(
+        not_exist_file_path, access_key_id, access_key_secret, end_point
+    )
+    fs = oss.OSSFileSystem.get_instance()
+
+    # Test OSSFileSystem.
+    assert len(fs.ls(fake_dir_path)) == 4
+    assert not fs.isfile(fake_dir_path)
+    assert fs.isdir(fake_dir_path)
+    assert not fs.isdir(fake_file_path)
+    assert fs.isfile(fake_file_path)
+    assert fs.exists(fake_file_path)
+    assert not fs.exists(fake_not_exist_file_path)
+    assert fs.stat(fake_file_path)["type"] == "file"
+    assert fs.stat(fake_dir_path)["type"] == "directory"
+    assert fs.glob(fake_dir_path) == [fake_dir_path]
+
+    with pytest.raises(ValueError) as e:
+        fs.exists(fake_other_scheme_path)
+    msg1 = e.value.args[0]
+    assert (
+        msg1 == f"Except scheme oss, but got scheme: "
+        f"scheme in path: {fake_other_scheme_path}"
+    )
+
+    with pytest.raises(RuntimeError) as e:
+        fs.exists(file_path)
+    msg2 = e.value.args[0]
+    assert msg2 == "Please use build_oss_path to add OSS info"
+
+    with pytest.raises(OSError):
+        print(fs.ls(fake_file_path))
+
+    assert len(fs.glob(fake_file_path)) == 1
+    assert len(fs.glob(fake_dir_path + "*", recursive=True)) == 4
+    assert len(fs.glob(fake_dir_path_contains_magic)) == 2
+
+    # Test the specific functions of glob.
+    assert og.has_magic(b"*")
+    assert og.escape(b"*") == b"[*]"
+    assert og.escape("*") == "[*]"
+
+    # test OSSIOBase
+    with fs.open(fake_file_path) as f:
+        assert f.readline() == b"id1,id2,id3\n"
+        assert f.readline() == b"1,2,3\n"
+        f.seek(-1, 2)
+        assert f.readline() == b"\n"
+        with pytest.raises(AttributeError):
+            f.fileno()
+        with pytest.raises(OSError):
+            f.seek(-1)
+        with pytest.raises(OSError):
+            f.seek(-100, 1)
+        with pytest.raises(ValueError):
+            f.seek(1, 3)
+        f.seek(0)
+        assert f.read() == b"id1,id2,id3\n1,2,3\n"
+        f.seek(0)
+        assert f.readline(2) == b"id"
+        f.seek(0)
+        with pytest.raises(TypeError):
+            f.readline("2")
+
+    fe = OSSFileEntry(fake_file_path)
+    assert fe.path == fake_file_path

maxframe/lib/functools_compat.py

@@ -0,0 +1,81 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Backported functools APIs from Python 3.8
+"""
+
+from typing import List
+
+__all__ = ["cached_property"]
+
+# Avoid importing types, so we can speedup import time
+GenericAlias = type(List[int])
+
+_NOT_FOUND = object()
+
+
+class cached_property:
+    def __init__(self, func):
+        self.func = func
+        self.attrname = None
+        self.__doc__ = func.__doc__
+        self.__module__ = func.__module__
+
+    def __set_name__(self, owner, name):
+        if self.attrname is None:
+            self.attrname = name
+        elif name != self.attrname:
+            raise TypeError(
+                "Cannot assign the same cached_property to two different names "
+                f"({self.attrname!r} and {name!r})."
+            )
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        if self.attrname is None:
+            raise TypeError(
+                "Cannot use cached_property instance without calling __set_name__ on it."
+            )
+        try:
+            cache = instance.__dict__
+        except (
+            AttributeError
+        ):  # not all objects have __dict__ (e.g. class defines slots)
+            msg = (
+                f"No '__dict__' attribute on {type(instance).__name__!r} "
+                f"instance to cache {self.attrname!r} property."
+            )
+            raise TypeError(msg) from None
+        val = cache.get(self.attrname, _NOT_FOUND)
+        if val is _NOT_FOUND:
+            val = self.func(instance)
+            try:
+                cache[self.attrname] = val
+            except TypeError:
+                msg = (
+                    f"The '__dict__' attribute on {type(instance).__name__!r} instance "
+                    f"does not support item assignment for caching {self.attrname!r} property."
+                )
+                raise TypeError(msg) from None
+        return val
+
+    __class_getitem__ = classmethod(GenericAlias)
+
+
+try:
+    from functools import cached_property  # noqa: F811
+except ImportError:
+    pass

maxframe/lib/mmh3_src/MurmurHash3.cpp

@@ -0,0 +1,339 @@
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+// Note - The x86 and x64 versions do _not_ produce the same results, as the
+// algorithms are optimized for their respective platforms. You can still
+// compile and run any of them on any platform, but your performance with the
+// non-native version will be less than optimal.
+
+#include "MurmurHash3.h"
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+
+#define FORCE_INLINE	__forceinline
+
+#include <stdlib.h>
+
+#define ROTL32(x,y)	_rotl(x,y)
+#define ROTL64(x,y)	_rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x)
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && GNUC_MINOR >= 4))
+/* gcc version >= 4.4 4.1 = RHEL 5, 4.4 = RHEL 6. Don't inline for RHEL 5 gcc which is 4.1*/
+#define FORCE_INLINE inline __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+inline uint32_t rotl32 ( uint32_t x, int8_t r )
+{
+  return (x << r) | (x >> (32 - r));
+}
+
+inline uint64_t rotl64 ( uint64_t x, int8_t r )
+{
+  return (x << r) | (x >> (64 - r));
+}
+
+#define	ROTL32(x,y)	rotl32(x,y)
+#define ROTL64(x,y)	rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+// Block read - if your platform needs to do endian-swapping or can only
+// handle aligned reads, do the conversion here
+
+FORCE_INLINE uint32_t getblock ( const uint32_t * p, int i )
+{
+  return p[i];
+}
+
+FORCE_INLINE uint64_t getblock ( const uint64_t * p, int i )
+{
+  return p[i];
+}
+
+//-----------------------------------------------------------------------------
+// Finalization mix - force all bits of a hash block to avalanche
+
+FORCE_INLINE uint32_t fmix ( uint32_t h )
+{
+  h ^= h >> 16;
+  h *= 0x85ebca6b;
+  h ^= h >> 13;
+  h *= 0xc2b2ae35;
+  h ^= h >> 16;
+
+  return h;
+}
+
+//----------
+
+FORCE_INLINE uint64_t fmix ( uint64_t k )
+{
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xff51afd7ed558ccd);
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
+  k ^= k >> 33;
+
+  return k;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_32 ( const void * key, Py_ssize_t len,
+                          uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const Py_ssize_t nblocks = len / 4;
+
+  uint32_t h1 = seed;
+
+  const uint32_t c1 = 0xcc9e2d51;
+  const uint32_t c2 = 0x1b873593;
+
+  //----------
+  // body
+
+  const uint32_t * blocks = (const uint32_t *)(data + nblocks*4);
+
+  for(Py_ssize_t i = -nblocks; i; i++)
+  {
+    uint32_t k1 = getblock(blocks,i);
+
+    k1 *= c1;
+    k1 = ROTL32(k1,15);
+    k1 *= c2;
+
+    h1 ^= k1;
+    h1 = ROTL32(h1,13);
+    h1 = h1*5+0xe6546b64;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*4);
+
+  uint32_t k1 = 0;
+
+  switch(len & 3)
+  {
+  case 3: k1 ^= tail[2] << 16;
+  case 2: k1 ^= tail[1] << 8;
+  case 1: k1 ^= tail[0];
+          k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len;
+
+  h1 = fmix(h1);
+
+  *(uint32_t*)out = h1;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_128 ( const void * key, const Py_ssize_t len,
+                           uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const Py_ssize_t nblocks = len / 16;
+
+  uint32_t h1 = seed;
+  uint32_t h2 = seed;
+  uint32_t h3 = seed;
+  uint32_t h4 = seed;
+
+  const uint32_t c1 = 0x239b961b;
+  const uint32_t c2 = 0xab0e9789;
+  const uint32_t c3 = 0x38b34ae5;
+  const uint32_t c4 = 0xa1e38b93;
+
+  //----------
+  // body
+
+  const uint32_t * blocks = (const uint32_t *)(data + nblocks*16);
+
+  for(Py_ssize_t i = -nblocks; i; i++)
+  {
+    uint32_t k1 = getblock(blocks,i*4+0);
+    uint32_t k2 = getblock(blocks,i*4+1);
+    uint32_t k3 = getblock(blocks,i*4+2);
+    uint32_t k4 = getblock(blocks,i*4+3);
+
+    k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+
+    h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;
+
+    k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+
+    h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;
+
+    k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+
+    h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;
+
+    k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+
+    h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+
+  uint32_t k1 = 0;
+  uint32_t k2 = 0;
+  uint32_t k3 = 0;
+  uint32_t k4 = 0;
+
+  switch(len & 15)
+  {
+  case 15: k4 ^= tail[14] << 16;
+  case 14: k4 ^= tail[13] << 8;
+  case 13: k4 ^= tail[12] << 0;
+           k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+
+  case 12: k3 ^= tail[11] << 24;
+  case 11: k3 ^= tail[10] << 16;
+  case 10: k3 ^= tail[ 9] << 8;
+  case  9: k3 ^= tail[ 8] << 0;
+           k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+
+  case  8: k2 ^= tail[ 7] << 24;
+  case  7: k2 ^= tail[ 6] << 16;
+  case  6: k2 ^= tail[ 5] << 8;
+  case  5: k2 ^= tail[ 4] << 0;
+           k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+
+  case  4: k1 ^= tail[ 3] << 24;
+  case  3: k1 ^= tail[ 2] << 16;
+  case  2: k1 ^= tail[ 1] << 8;
+  case  1: k1 ^= tail[ 0] << 0;
+           k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+
+  h1 += h2; h1 += h3; h1 += h4;
+  h2 += h1; h3 += h1; h4 += h1;
+
+  h1 = fmix(h1);
+  h2 = fmix(h2);
+  h3 = fmix(h3);
+  h4 = fmix(h4);
+
+  h1 += h2; h1 += h3; h1 += h4;
+  h2 += h1; h3 += h1; h4 += h1;
+
+  ((uint32_t*)out)[0] = h1;
+  ((uint32_t*)out)[1] = h2;
+  ((uint32_t*)out)[2] = h3;
+  ((uint32_t*)out)[3] = h4;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x64_128 ( const void * key, const Py_ssize_t len,
+                           const uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const Py_ssize_t nblocks = len / 16;
+
+  uint64_t h1 = seed;
+  uint64_t h2 = seed;
+
+  const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
+  const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
+
+  //----------
+  // body
+
+  const uint64_t * blocks = (const uint64_t *)(data);
+
+  for(Py_ssize_t i = 0; i < nblocks; i++)
+  {
+    uint64_t k1 = getblock(blocks,i*2+0);
+    uint64_t k2 = getblock(blocks,i*2+1);
+
+    k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+
+    h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
+
+    k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+    h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+
+  uint64_t k1 = 0;
+  uint64_t k2 = 0;
+
+  switch(len & 15)
+  {
+  case 15: k2 ^= uint64_t(tail[14]) << 48;
+  case 14: k2 ^= uint64_t(tail[13]) << 40;
+  case 13: k2 ^= uint64_t(tail[12]) << 32;
+  case 12: k2 ^= uint64_t(tail[11]) << 24;
+  case 11: k2 ^= uint64_t(tail[10]) << 16;
+  case 10: k2 ^= uint64_t(tail[ 9]) << 8;
+  case  9: k2 ^= uint64_t(tail[ 8]) << 0;
+           k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+  case  8: k1 ^= uint64_t(tail[ 7]) << 56;
+  case  7: k1 ^= uint64_t(tail[ 6]) << 48;
+  case  6: k1 ^= uint64_t(tail[ 5]) << 40;
+  case  5: k1 ^= uint64_t(tail[ 4]) << 32;
+  case  4: k1 ^= uint64_t(tail[ 3]) << 24;
+  case  3: k1 ^= uint64_t(tail[ 2]) << 16;
+  case  2: k1 ^= uint64_t(tail[ 1]) << 8;
+  case  1: k1 ^= uint64_t(tail[ 0]) << 0;
+           k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len; h2 ^= len;
+
+  h1 += h2;
+  h2 += h1;
+
+  h1 = fmix(h1);
+  h2 = fmix(h2);
+
+  h1 += h2;
+  h2 += h1;
+
+  ((uint64_t*)out)[0] = h1;
+  ((uint64_t*)out)[1] = h2;
+}
+
+//-----------------------------------------------------------------------------

maxframe/lib/mmh3_src/MurmurHash3.h

@@ -0,0 +1,43 @@
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+#ifndef _MURMURHASH3_H_
+#define _MURMURHASH3_H_
+
+
+// To handle 64-bit data; see https://docs.python.org/2.7/c-api/arg.html
+#ifndef PY_SSIZE_T_CLEAN
+#define PY_SSIZE_T_CLEAN
+#endif
+#include <Python.h>
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#include <stdint.h>
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_32  ( const void * key, Py_ssize_t len, uint32_t seed, void * out );
+
+void MurmurHash3_x86_128 ( const void * key, Py_ssize_t len, uint32_t seed, void * out );
+
+void MurmurHash3_x64_128 ( const void * key, Py_ssize_t len, uint32_t seed, void * out );
+
+//-----------------------------------------------------------------------------
+
+#endif // _MURMURHASH3_H_