maque 0.2.1__py3-none-any.whl

maque/algorithms/bktree.py

@@ -0,0 +1,117 @@
+imap = lambda *args, **kwargs: list(map(*args, **kwargs))
+ifilter = lambda *args, **kwargs: list(filter(*args, **kwargs))
+
+
+class BKTree:
+    def __init__(self, distfn, words):
+        """
+        Create a new BK-tree from the given distance function and
+        words.
+
+        Arguments:
+
+        distfn: a binary function that returns the distance between
+        two words.  Return value is a non-negative integer.  the
+        distance function must be a metric space.
+
+        words: an iterable.  produces values that can be passed to
+        distfn
+
+        """
+        self.distfn = distfn
+
+        it = iter(words)
+        root = it.next()
+        self.tree = (root, {})
+
+        for i in it:
+            self._add_word(self.tree, i)
+
+    def _add_word(self, parent, word):
+        pword, children = parent
+        d = self.distfn(word, pword)
+        if d in children:
+            self._add_word(children[d], word)
+        else:
+            children[d] = (word, {})
+
+    def query(self, word, n):
+        """
+        Return all words in the tree that are within a distance of `n'
+        from `word`.
+
+        Arguments:
+
+        word: a word to query on
+
+        n: a non-negative integer that specifies the allowed distance
+        from the query word.
+
+        Return value is a list of tuples (distance, word), sorted in
+        ascending order of distance.
+
+        """
+
+        def rec(parent):
+            pword, children = parent
+            d = self.distfn(word, pword)
+            results = []
+            if d <= n:
+                results.append((d, pword))
+
+            for i in range(d - n, d + n + 1):
+                child = children.get(i)
+                if child is not None:
+                    results.extend(rec(child))
+            return results
+
+        # sort by distance
+        return sorted(rec(self.tree))
+
+
+def brute_query(word, words, distfn, n):
+    """A brute force distance query
+
+    Arguments:
+
+    word: the word to query for
+
+    words: a iterable that produces words to test
+
+    distfn: a binary function that returns the distance between a
+    `word' and an item in `words'.
+
+    n: an integer that specifies the distance of a matching word
+
+    """
+    return [i for i in words
+            if distfn(i, word) <= n]
+
+
+# def maxdepth(tree, count=0):
+#     _, children = t
+#     if len(children):
+#         return max(maxdepth(i, c + 1) for i in children.values())
+#     else:
+#         return c
+
+
+# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance#Python
+def levenshtein(s, t):
+    m, n = len(s), len(t)
+    d = [range(n + 1)]
+    d += [[i] for i in range(1, m + 1)]
+    for i in range(0, m):
+        for j in range(0, n):
+            cost = 1
+            if s[i] == t[j]: cost = 0
+
+            d[i + 1].append(min(d[i][j + 1] + 1,  # deletion
+                                d[i + 1][j] + 1,  # insertion
+                                d[i][j] + cost)  # substitution
+                            )
+    return d[m][n]
+
+
+if __name__ == "__main__":
+    ...

maque/algorithms/core.py

@@ -0,0 +1,104 @@
+from functools import partial, wraps, reduce
+import inspect
+import numpy as np
+import operator as op
+import random
+import time
+from pandas import DataFrame
+from .utils import exists
+
+__all__ = ["topk", "dict_topk", "random_idx", "clamp", "get_num_args", "get_parameters"]
+
+
+def dict_topk(a: dict, k: int, reverse=False):
+    df = DataFrame({'key': a.keys(), 'value': a.values()})
+    if not reverse:
+        return df.nlargest(k, 'value')
+    else:
+        return df.nsmallest(k, 'value')
+
+
+def topk(a, k, axis=-1, largest=True, sort=True):
+    """Series top K"""
+    a = np.asanyarray(a)
+    if axis is None:
+        axis_size = a.size
+    else:
+        axis_size = a.shape[axis]
+    assert 1 <= k <= axis_size
+
+    if largest:
+        index_array = np.argpartition(a, axis_size - k, axis=axis)
+        topk_indices = np.take(index_array, -np.arange(k) - 1, axis=axis)
+    else:
+        index_array = np.argpartition(a, k - 1, axis=axis)
+        topk_indices = np.take(index_array, np.arange(k), axis=axis)
+    topk_values = np.take_along_axis(a, topk_indices, axis=axis)
+    if sort:
+        sorted_indices_in_topk = np.argsort(topk_values, axis=axis)
+        if largest:
+            sorted_indices_in_topk = np.flip(sorted_indices_in_topk, axis=axis)
+        sorted_topk_values = np.take_along_axis(
+            topk_values, sorted_indices_in_topk, axis=axis
+        )
+        sorted_topk_indices = np.take_along_axis(
+            topk_indices, sorted_indices_in_topk, axis=axis
+        )
+        return sorted_topk_values, sorted_topk_indices
+    return topk_values, topk_indices
+
+
+def random_idx(idx_range, exclude_idx=None):
+    random.seed(time.time())
+    rand_idx = random.randint(*idx_range)
+    if rand_idx == exclude_idx:
+        return random_idx(idx_range, exclude_idx)
+    else:
+        return rand_idx
+
+
+def clamp(x, x_min=None, x_max=None):
+    """Clamp a number to same range.
+    Examples:
+        >>> clamp(-1, 0, 1)
+        >>> 0
+        >>> clamp([-1, 2, 3], [0, 0, 0], [1, 1, 1])
+        >>> [0, 1, 1]
+    """
+    assert exists(x_min) or exists(x_max)
+    if exists(x_min):
+        x = np.maximum(x, x_min)
+    if exists(x_max):
+        x = np.minimum(x, x_max)
+    return x
+
+
+CHOOSE_CACHE = {}
+
+
+def choose_using_cache(n, r):
+    if n not in CHOOSE_CACHE:
+        CHOOSE_CACHE[n] = {}
+    if r not in CHOOSE_CACHE[n]:
+        CHOOSE_CACHE[n][r] = choose(n, r, use_cache=False)
+    return CHOOSE_CACHE[n][r]
+
+
+def choose(n, r, use_cache=True):
+    if use_cache:
+        return choose_using_cache(n, r)
+    if n < r:
+        return 0
+    if r == 0:
+        return 1
+    denom = reduce(op.mul, range(1, r + 1), 1)
+    numer = reduce(op.mul, range(n, n - r, -1), 1)
+    return numer // denom
+
+
+def get_num_args(function):
+    return len(get_parameters(function))
+
+
+def get_parameters(function):
+    return inspect.signature(function).parameters

maque/algorithms/hilbert.py

@@ -0,0 +1,16 @@
+import numpy as np
+
+
+def get_hilbert_1d_array(image: np.ndarray):
+    from hilbert import decode, encode  # $ pip install numpy-hilbert-curve
+    num_dims = 2
+    num_bits = np.log2(image.shape[0] * image.shape[1]) / num_dims
+    num_bits = int(num_bits)
+    max_hil = 2 ** (num_bits * num_dims)
+    hilberts = np.arange(max_hil)
+    locs = decode(hilberts, num_dims, num_bits)
+
+    image1d = []
+    for coord in locs:
+        image1d.append(image[coord[0], coord[1]])
+    return np.array(image1d)[None, ...]

maque/algorithms/rate_function.py

@@ -0,0 +1,92 @@
+import numpy as np
+from .bezier import bezier
+
+
+def linear(t):
+    return t
+
+
+def smooth(t):
+    # Zero first and second derivatives at t=0 and t=1.
+    # Equivalent to bezier([0, 0, 0, 1, 1, 1])
+    s = 1 - t
+    return (t**3) * (10 * s * s + 5 * s * t + t * t)
+
+
+def rush_into(t):
+    return 2 * smooth(0.5 * t)
+
+
+def rush_from(t):
+    return 2 * smooth(0.5 * (t + 1)) - 1
+
+
+def slow_into(t):
+    return np.sqrt(1 - (1 - t) * (1 - t))
+
+
+def double_smooth(t):
+    if t < 0.5:
+        return 0.5 * smooth(2 * t)
+    else:
+        return 0.5 * (1 + smooth(2 * t - 1))
+
+
+def there_and_back(t):
+    new_t = 2 * t if t < 0.5 else 2 * (1 - t)
+    return smooth(new_t)
+
+
+def there_and_back_with_pause(t, pause_ratio=1.0 / 3):
+    a = 1.0 / pause_ratio
+    if t < 0.5 - pause_ratio / 2:
+        return smooth(a * t)
+    elif t < 0.5 + pause_ratio / 2:
+        return 1
+    else:
+        return smooth(a - a * t)
+
+
+def running_start(t, pull_factor=-0.5):
+    return bezier([0, 0, pull_factor, pull_factor, 1, 1, 1])(t)
+
+
+def not_quite_there(func=smooth, proportion=0.7):
+    def result(t):
+        return proportion * func(t)
+
+    return result
+
+
+def wiggle(t, wiggles=2):
+    return there_and_back(t) * np.sin(wiggles * np.pi * t)
+
+
+def squish_rate_func(func, a=0.4, b=0.6):
+    def result(t):
+        if a == b:
+            return a
+        elif t < a:
+            return func(0)
+        elif t > b:
+            return func(1)
+        else:
+            return func((t - a) / (b - a))
+
+    return result
+
+
+# Stylistically, should this take parameters (with default values)?
+# Ultimately, the functionality is entirely subsumed by squish_rate_func,
+# but it may be useful to have a nice name for with nice default params for
+# "lingering", different from squish_rate_func's default params
+
+
+def lingering(t):
+    return squish_rate_func(lambda t: t, 0, 0.8)(t)
+
+
+def exponential_decay(t, half_life=0.1):
+    # The half-life should be rather small to minimize
+    # the cut-off error at the end
+    return 1 - np.exp(-t / half_life)

maque/algorithms/transform.py

@@ -0,0 +1,27 @@
+# from einops import rearrange, reduce
+import einops
+
+
+def repeat(tensor, n, axis=-1):
+    shape = tensor.shape
+    dims = len(shape)
+    if dims == 1:
+        if axis == 0:
+            res = einops.repeat(tensor, "w -> h w", h=n)
+        else:
+            res = einops.repeat(tensor, "h -> h w", w=n)
+    elif dims == 2:
+        if axis == 0:
+            res = einops.repeat(tensor, "h w -> c h w", c=n)
+        else:
+            res = einops.repeat(tensor, "h w -> h w c", c=n)
+    elif dims == 3:
+        if axis == 0:
+            res = einops.repeat(tensor, "h w c -> b h w c", b=n)
+        elif axis == 1:
+            res = einops.repeat(tensor, "h w c -> h b w c", b=n)
+        else:
+            res = einops.repeat(tensor, "h w c -> h w c b", b=n)
+    else:
+        raise ValueError
+    return res

maque/algorithms/trie.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+from typing import List, Dict, Union
+from abc import ABCMeta, abstractmethod
+from pathlib import Path
+
+
+class Trie(metaclass=ABCMeta):
+    trie = None
+    rtrie = None
+
+    def matches(self, word: str) -> list:
+        matched_list = self.startwith(word)
+        matched_list.extend(self.prefixes(word))
+        return matched_list
+
+    def __contains__(self, item):
+        return item in self.trie
+
+    @abstractmethod
+    def prefixes(self, word: str) -> list:
+        """在trie中匹配出所有可以作为word的前缀的keys
+
+        Parameters
+        ----------
+        word : str
+            待匹配词语
+            
+        Returns
+        -------
+        list
+            Trie中所有匹配到的元素数组
+
+        Example
+        -------
+        >>> trie = Trie(["ab", "bc", "bcd", "bcde"])
+        >>> trie.prefixes("bcdf")
+        ["bc", "bcd"]
+        """
+
+    def full_prefixes(self, word: str) -> list:
+        full_matches = []
+        for idx in range(len(word)):
+            cur_word = word[idx:]
+            lst = self.prefixes(cur_word)
+            lst.sort(key=lambda x: len(x), reverse=True)
+            full_matches.extend(lst)
+        return full_matches
+
+    @abstractmethod
+    def startwith(self, word: str) -> list:
+        """在前缀树中匹配出所有以word为前缀的keys
+
+        Parameters
+        ----------
+        word : str
+            待匹配词语
+            
+        Returns
+        -------
+        list
+            Trie中所有匹配到的元素数组
+        
+        Example
+        -------
+        >>> trie = Trie(["ab", "bc", "bcd", "bcde"])
+        >>> trie.startwith("bc")
+        ["bc", "bcd", "bcde"]
+        """
+
+    @abstractmethod
+    def endwith(self, word: str) -> list:
+        """在前缀树中匹配出所有以word为后缀的keys
+
+        Parameters
+        ----------
+        word : str
+            待匹配词语
+
+        Returns
+        -------
+        list
+            Trie中所有匹配到的元素数组
+
+        Example
+        -------
+        >>> trie = Trie(["bc", "abc", "bcd", "edbc"])
+        >>> trie.endwith("bc")
+        ["bc", "abc", "edbc"]
+        """
+
+    def has_keys_with_prefix(self, word):
+        ...
+
+    def save(self):
+        ...
+
+    def load(self, file_name: Union[str, Path]):
+        ...
+
+
+class PyTrie:
+    def __init__(self, words: List[str]):
+        self.trie = {}
+        self._end = "eos"
+        for word in words:
+            self.add(word)
+
+    def add(self, word):
+        word = word.strip()
+        assert word
+        cur_node = self.trie
+        for char in word:
+            cur_node = cur_node.setdefault(char, {})
+        # cur_node[self._end] = ""
+        cur_node[self._end] = word
+
+    def find(self, word: str, full=True) -> bool:
+        """
+        Parameters
+        ----------
+        full: bool
+            True: 完全匹配时返回True
+            False: 前缀匹配到则返回True
+        """
+        trie = self.trie
+        for c in word:
+            if c not in trie:
+                return False
+            trie = trie[c]
+        if full:
+            return self._end in trie
+        else:
+            return True
+
+    def extract_longest_item(self, word: str):
+        """从一个文本的开头开始匹配字典中最长的词，返回最长词和长度"""
+        curr_dict, longest, offset = self.trie, None, 0
+
+        if not word:
+            return longest, offset
+
+        for i, c in enumerate(word):
+            if c not in curr_dict:
+                return longest, offset
+            curr_dict = curr_dict[c]
+            if 'end' in curr_dict:
+                longest, offset = curr_dict['end'], i + 1
+        return longest, offset
+
+    def __str__(self):
+        return self.trie
+
+
+class HatTrie(Trie):
+    ...
+
+
+class MarisaTrie(Trie):
+    """Static memory-efficient Trie-like structures for Python
+    based on marisa-trie C++ library.
+    String data in a MARISA-trie may take up to 50x-100x less memory than in a standard Python dict;
+    the raw lookup speed is comparable;
+    trie also provides fast advanced methods like prefix search.
+    """
+
+    def __init__(self, key_list: list):
+        import marisa_trie
+        self.trie = marisa_trie.Trie(key_list)
+        rkey_list = [i[::-1] for i in key_list]
+        self.rtrie = marisa_trie.Trie(rkey_list)
+
+    def prefixes(self, word: str):
+        return self.trie.prefixes(word)
+
+    def rprefixes(self, word: str):
+        rword = word[::-1]
+        return self.rtrie.prefixes(rword)
+
+    def startwith(self, word: str):
+        return self.trie.keys(word)
+
+    def endwith(self, word: str):
+        rword = word[::-1]
+        return [i[::-1] for i in self.rtrie.keys(rword)]
+
+    def get_longest(self, text):
+        """匹配整个字典中最长的词，返回最长词"""
+        if not text:
+            return None
+
+        longest = ''
+        for idx, item in enumerate(text):
+            items = self.trie.prefixes(text[idx:])
+
+            for item in items:
+                if len(item) > len(longest):
+                    longest = item
+        if longest == '':
+            return None
+        return longest
+
+    def has_keys_with_prefix(self, word):
+        return self.trie.has_keys_with_prefix(word)
+
+
+class DaTrie(Trie):
+    def __init__(self, word_list: list):
+        import datrie
+        self.trie = datrie.BaseTrie(word_list)
+        rkey_list = [i[::-1] for i in word_list]
+        self.rtrie = datrie.BaseTrie(rkey_list)
+
+    def prefixes(self, name: str):
+        return self.trie.prefixes(name)
+
+    def startwith(self, name: str):
+        return self.trie.startwith(name)
+
+    def longest_prefix(self, word: str):
+        return self.trie.longest_prefix(word)
+
+    def has_keys_with_prefix(self, word):
+        return self.trie.has_keys_with_prefix(word)
+
+
+class AutomatonTrie:
+    def __init__(self, key_list: list):
+        import ahocorasick
+        self.trie = ahocorasick.Automaton()
+        for idx, key in enumerate(key_list):
+            self.trie.add_word(key, (idx, key))
+
+    def __contains__(self, item):
+        return item in self.trie
+
+
+class Benchmark:
+    def __init__(self):
+        from maque.performance import MeasureTime
+        self.ms = MeasureTime()
+        n = 2000000
+        word_list = [str(i) for i in range(n)]
+        self.ms.start()
+        pytrie = PyTrie(word_list)
+        self.ms.show_interval(f"pytrie build")
+        mtrie = MarisaTrie(word_list)
+        self.ms.show_interval(f"marisa trie build")
+
+    def run(self):
+        self.ms.start()
+
+
+if __name__ == "__main__":
+    from rich import print
+
+    key_list = ['as', 'asdf', "basdfg", 'casd']
+    t = MarisaTrie(key_list)
+    print(t.startwith("as"))
+    print(t.endwith("sd"))
+    print('asd' in t)
+    t2 = AutomatonTrie(key_list)
+    print('asd' in t2)
+
+    # print(t.rkeys("asd"))
+    # t = AutomatonTrie(['as', '1asdf'])
+    # print("a" in t.trie)
+    # t = PyTrie(["a", "ab", "abc", "bc",
+    #             "abcd"
+    #             ])
+    # print(t.find("abc"))
+    # print(t.find("bcd", ))
+    # Benchmark()

maque/algorithms/utils.py

@@ -0,0 +1,63 @@
+from functools import partial, wraps, reduce
+from contextlib import contextmanager
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+def cast_tuple(val, length=1):
+    return val if isinstance(val, tuple) else ((val,) * length)
+
+
+@contextmanager
+def null_context(*args, **kwargs):
+    yield
+
+
+def pick_and_pop(keys, d):
+    values = list(map(lambda key: d.pop(key), keys))
+    return dict(zip(keys, values))
+
+
+def group_dict_by_key(cond, d):
+    return_val = [dict(), dict()]
+    for key in d.keys():
+        match = bool(cond(key))
+        ind = int(not match)
+        return_val[ind][key] = d[key]
+    return (*return_val,)
+
+
+def string_begins_with(prefix, str):
+    return str.startswith(prefix)
+
+
+def group_by_key_prefix(prefix, d):
+    return group_dict_by_key(partial(string_begins_with, prefix), d)
+
+
+def groupby_prefix_and_trim(prefix, d):
+    kwargs_with_prefix, kwargs = group_dict_by_key(partial(string_begins_with, prefix), d)
+    kwargs_without_prefix = dict(map(lambda x: (x[0][len(prefix):], x[1]), tuple(kwargs_with_prefix.items())))
+    return kwargs_without_prefix, kwargs
+
+
+def num_to_groups(num, divisor):
+    groups = num // divisor
+    remainder = num % divisor
+    arr = [divisor] * groups
+    if remainder > 0:
+        arr.append(remainder)
+    return arr
+
+
+def find_first(cond, arr):
+    for el in arr:
+        if cond(el):
+            return el
+    return None