python-wml 3.0.0__py3-none-any.whl

wml/threadtoolkit.py

@@ -0,0 +1,50 @@
+#coding=utf-8
+from multiprocessing import cpu_count
+from multiprocessing import Process,Queue,Pool
+from functools import partial
+import traceback
+import time
+import os
+import wml.wml_utils as wmlu
+
+DEFAULT_THREAD_NR=cpu_count() if cpu_count()<4 else cpu_count()-1
+
+def fn_wraper(datas,fn,is_factory_fn=False):
+    if is_factory_fn:
+        fn = fn()
+    res_queue = []
+    print(f"Process {os.getpid()}: data nr {len(datas)}.")
+    for data,i in datas:
+        try:
+            res = fn(data)
+            res_queue.append((i,res))
+        except:
+            traceback.print_exc()
+    print(f"Process {os.getpid()} is finished.")
+    return res_queue
+
+def par_for_each(data,fn,thread_nr=DEFAULT_THREAD_NR,is_factory_fn=False,timeout=None):
+    if len(data) == 0:
+        return []
+    thread_nr = min(len(data),thread_nr)
+    pool = Pool(thread_nr)
+    data = list(zip(data,range(len(data))))
+    datas = wmlu.list_to_2dlistv2(data,thread_nr)
+    raw_res = list(pool.map(partial(fn_wraper,fn=fn,is_factory_fn=is_factory_fn),datas))
+    pool.close()
+    pool.join()
+
+    res_data = []
+    for res in raw_res:
+        res_data.extend(res)
+    res_data = sorted(res_data,key=lambda x:x[0])
+    _,res_data = zip(*res_data)
+    return res_data
+
+def par_for_each_no_return(data,fn,thread_nr=DEFAULT_THREAD_NR):
+    thread_nr = min(len(data),thread_nr)
+    pool = Pool(thread_nr)
+    datas = wmlu.list_to_2dlistv2(data,thread_nr)
+    pool.map(fn,datas)
+    pool.close()
+    pool.join()

wml/walgorithm.py

@@ -0,0 +1,228 @@
+#coding=utf-8
+from multiprocessing import Pool
+import numpy as np
+import math
+import cv2
+
+def _edit_distance(v0,v1):
+    if v0 == v1:
+        return 0
+    if (len(v0)==0) or (len(v1)==0):
+        return max(len(v0),len(v1))
+    c0 = _edit_distance(v0[:-1],v1)+1
+    c1 = _edit_distance(v0,v1[:-1])+1
+    cr = 0
+    if v0[-1] != v1[-1]:
+        cr = 1
+    c2 = _edit_distance(v0[:-1],v1[:-1])+cr
+    return min(min(c0,c1),c2)
+
+def mt_edit_distance(v0,v1,pool):
+    if v0 == v1:
+        return 0
+    if (len(v0)==0) or (len(v1)==0):
+        return max(len(v0),len(v1))
+    c0 = edit_distance(v0[:-1],v1)+1
+    c1 = edit_distance(v0,v1[:-1])+1
+    cr = 0
+    if v0[-1] != v1[-1]:
+        cr = 1
+    c2 = edit_distance(v0[:-1],v1[:-1])+cr
+    return min(min(c0,c1),c2)
+
+def edit_distance(sm, sn):
+    m, n = len(sm) + 1, len(sn) + 1
+
+    matrix = np.ndarray(shape=[m,n],dtype=np.int32)
+
+    matrix[0][0] = 0
+    for i in range(1, m):
+        matrix[i][0] = matrix[i - 1][0] + 1
+
+    for j in range(1, n):
+        matrix[0][j] = matrix[0][j - 1] + 1
+
+    for i in range(1, m):
+        for j in range(1, n):
+            if sm[i - 1] == sn[j - 1]:
+                cost = 0
+            else:
+                cost = 1
+
+            matrix[i][j] = min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost)
+
+    return matrix[m - 1][n - 1]
+
+
+def pearsonr(x,y):
+    #Pearson_correlation coefficient [-1,1]
+    if not isinstance(x,np.ndarray):
+        x = np.array(x)
+
+    if not isinstance(y, np.ndarray):
+        y = np.array(y)
+
+    x_mean = np.mean(x)
+    y_mean = np.mean(y)
+    x_ba = x-x_mean
+    y_ba = y-y_mean
+    v = np.sum(x_ba*y_ba)
+    dx = np.sum((x-x_mean)**2)
+    dy = np.sum((y-y_mean)**2)
+    sv = np.sqrt(dx*dy)+1e-8
+
+    return v/sv
+
+def points_to_polygon(points):
+    '''
+
+    Args:
+        points: [N,2],(x,y)
+
+    Returns:
+        idxs,[N],sorted points[N,2]
+    '''
+
+    points = np.array(points)
+    base_point = 0
+    if points.shape[0]<=3:
+        return list(range(points.shape[0])),points
+    for i in range(points.shape[0]):
+        if points[i,1]<points[base_point,1]:
+            base_point = i
+        elif points[i, 1] == points[base_point, 1] and points[i,0]<points[base_point,0]:
+            base_point = i
+
+    angles = np.zeros([points.shape[0]],dtype=np.float32)
+
+    for i in range(points.shape[0]):
+        y = points[i,1]-points[base_point,1]
+        x = points[i,0]-points[base_point,0]
+        angles[i] = math.atan2(y,x)
+        if angles[i]<0:
+            angles[i] += math.pi
+    angles[base_point] = -1e-8
+    idxs = np.argsort(angles)
+    return idxs,points[idxs]
+
+def left_shift_array(array,size=1):
+    '''
+
+    Args:
+        array: [N]
+        size: 1->N-1
+    example:
+        array = [1,2,3,4]
+        size=1
+        return:
+        [2,3,4,1]
+    Returns:
+        [N]
+    '''
+    first_part = array[size:]
+    second_part = array[:size]
+    return np.concatenate([first_part,second_part],axis=0)
+
+def right_shift_array(array, size=1):
+    '''
+
+    Args:
+        array: [N]
+        size: 1->N-1
+    example:
+        array = [1,2,3,4]
+        size=1
+        return:
+        [4,1,2,3,]
+    Returns:
+        [N]
+    '''
+    first_part = array[-size:]
+    second_part = array[:-size]
+    return np.concatenate([first_part, second_part], axis=0)
+
+
+def sign_point_line(point,line):
+    '''
+
+    Args:
+        point: [2] x,y
+        line: np.array([2,2]) [(x0,y0),(x1,y1)]
+
+    Returns:
+        True or False
+    '''
+    line = np.array(line)
+    p0 = line[0]
+    vec0 = line[1]-p0
+    vec1 = point-p0
+    return vec0[0]*vec1[1]-vec0[1]*vec1[0]<0
+
+def in_range(v,*kargs):
+    if len(kargs)==1:
+        min_v = kargs[0][0]
+        max_v = kargs[0][1]
+    elif len(kargs)==2:
+        min_v = kargs[0]
+        max_v = kargs[1]
+    else:
+        raise RuntimeError(f"in_range: ERROR args {kargs}")
+
+    return v>=min_v and v<=max_v
+
+def points_on_circle(center=None,r=None,points_nr=100):
+    '''
+    将圆离散化为散点
+    '''
+    points = []
+    for i in range(points_nr):
+        angle = math.pi*2*i/points_nr
+        x = math.cos(angle)
+        y = math.sin(angle)
+        points.append([x,y])
+    if r is not None:
+        points = np.array(points)*r
+    if center is not None:
+        center = np.reshape(np.array(center),[1,2])
+        points = points + center
+    
+    return points
+
+def getRotationMatrix2D(center, angle, scale,out_offset=None):
+    if out_offset is None:
+        '''
+        cv2为先平移-center,scale,rotate,再平移center
+        M(center)*M(rotate)*M(scale)*M(-center)*X
+        '''
+        return cv2.getRotationMatrix2D(center=center,angle=angle,scale=scale)
+    offset_in = np.array([[1,0,-center[0]],[0,1,-center[1]]],dtype=np.float32)
+    rotate_m = cv2.getRotationMatrix2D(center=[0,0],angle=angle,scale=scale)
+    offset_out = np.array([[1,0,out_offset[0]],[0,1,out_offset[1]]],dtype=np.float32)
+    line3 = np.array([[0,0,1]],dtype=np.float32)
+    offset_in = np.concatenate([offset_in,line3],axis=0)
+    rotate_m = np.concatenate([rotate_m,line3],axis=0)
+    offset_out = np.concatenate([offset_out,line3],axis=0)
+    m = np.dot(rotate_m,offset_in)
+    m = np.dot(offset_out,m)
+    return m[:2]
+
+def lower_bound(datas, target):
+    """
+    对于升序数组，找到第一个大于等于（或不小于）给定值的目标元素的位置
+    """
+    if datas[-1]<target:
+        return -1
+    if datas[0]>=target:
+        return 0
+    left, right = 0, len(datas) - 1  # 闭区间[left, right]
+    while left <= right:  # 区间不为空
+        mid = (left + right) // 2
+        if datas[mid] < target:
+            left = mid + 1  # [mid + 1, right]
+        else:
+            right = mid - 1  # [left, mid - 1]
+    return left
+
+
+def remove_non_ascii(s):
+    return ''.join(filter(str.isascii, s))

wml/wcollections.py

@@ -0,0 +1,212 @@
+import numpy as np
+import random
+import copy
+from collections import abc
+
+
+class ExperienceBuffer():
+    def __init__(self, buffer_size = 100000):
+        self.buffer = []
+        self.buffer_size = buffer_size
+
+    def add(self, experience):
+        if len(self.buffer) + len(experience) >= self.buffer_size:
+            self.buffer[0: (len(experience) + len(self.buffer)) - self.buffer_size] = []
+        self.buffer.extend(experience)
+
+    def sample(self, size):
+        if not isinstance(self.buffer[0],np.ndarray):
+            data = random.sample(self.buffer,size)
+            data = list(zip(*data))
+            return [np.array(list(x)) for x in data]
+        else:
+            return np.reshape(np.array(random.sample(self.buffer, size)), [size]+list(self.buffer[0].shape))
+
+
+class CycleBuffer:
+    def __init__(self,cap=5):
+        self.cap = cap
+        self.buffer = []
+    def append(self,v):
+        self.buffer.append(v)
+        l = len(self.buffer)
+        if l>self.cap:
+            self.buffer = self.buffer[l-self.cap:]
+
+    def __getitem__(self, slice):
+        return self.buffer[slice]
+
+    def __len__(self):
+        return len(self.buffer)
+
+class AlwaysNullObj(object):
+    def __init__(self,*args,**kwargs):
+        print(f"Construct a always null object")
+        pass
+
+    def __getattr__(self, item):
+        return self
+
+    def __setattr__(self, key, value):
+        pass
+
+    def __delattr__(self, item):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        return self
+
+
+class MDict(dict):
+    def __init__(self, *args, **kwargs):
+        '''
+
+        Args:
+            *args:
+            **kwargs:
+
+            example 1:
+            x = MDict(dtype=list)
+            x[1].append('a')
+            x[2].append('b')
+            x[1].append('c')
+            print(x)
+            output:
+            {1: ['a', 'c'], 2: ['b']}
+
+            example 2:
+            x = MDict(dvalue=[])
+            x[1].append('a')
+            x[2].append('b')
+            x[1].append('c')
+            print(x)
+            output:
+            {1: ['a', 'c'], 2: ['b']}
+        '''
+        self.default_type = None
+        self.default_value = None
+        if "dtype" in kwargs:
+            self.default_type = kwargs.pop("dtype")
+        elif "dvalue" in kwargs:
+            self.default_value = kwargs.pop("dvalue")
+        super().__init__(*args,**kwargs)
+
+    @classmethod
+    def from_dict(cls,data:dict,auto_dtype=True):
+        x = data.values()
+        assert len(x)>0, "error dict data"
+        if auto_dtype:
+            dtype = type(list(x)[0])
+            ret = cls(dtype=dtype)
+        else:
+            ret = cls(dvalue=None)
+        for k,v in data.items():
+            ret[k] = v
+        return ret
+
+    def __getattr__(self, key):
+        if key in self.__dict__:
+            return self.__dict__[key]
+        return self.__getitem__(key)
+
+    def __call__(self,key):
+        return self.__getitem__(key)
+
+    def __getitem__(self, key):
+        if key in self.__dict__:
+            return self.__dict__[key]
+        if key in self:
+            return super().__getitem__(key)
+        elif self.default_type is not None:
+            super().__setitem__(key,self.default_type())
+            return super().__getitem__(key)
+        elif self.default_value is not None:
+            super().__setitem__(key,self.default_value)
+        return None
+
+    def __delattr__(self, key):
+        del self[key]
+
+
+class Counter(dict):
+    '''
+    Example:
+    counter = Counter()
+    counter.add("a")
+    counter.add("a")
+    counter.add("b")
+    print(counter):
+    {
+        a: 2
+        b: 1
+    }
+    '''
+    def add(self,key,nr=1):
+        if key in self:
+            self[key] += nr
+        else:
+            self[key] = nr
+        return self[key]
+    
+    def total_size(self):
+        return np.sum(list(self.values()))
+
+class EDict(dict):
+    '''
+    只能添加键值，不允许更新
+    '''
+    def __setitem__(self,item,value):
+        if item in self:
+            raise RuntimeError(f"ERROR: key {item} already exists.")
+        super().__setitem__(item,value)
+
+def safe_update_dict(target_dict,src_dict,do_raise=True):
+    duplicate_keys = []
+    for k in src_dict.keys():
+        if k in target_dict and target_dict[k] != src_dict[k]:
+            duplicate_keys.append(k)
+    
+
+    if len(duplicate_keys)>0:
+        if do_raise:
+            raise RuntimeError(f"key {duplicate_keys} already in target dict, target dict keys {list(target_dict.keys())}")
+        else:
+            print(f"ERROR: key {duplicate_keys} already in target dict, target dict keys {list(target_dict.keys())}")
+
+    target_dict.update(src_dict)
+
+def trans_dict_key2lower(data):
+    res = type(data)()
+    for k,v in data.items():
+        res[k.lower()] = v
+    return res
+
+def is_seq_of(seq, expected_type, seq_type=None):
+    """Check whether it is a sequence of some type.
+
+    Args:
+        seq (Sequence): The sequence to be checked.
+        expected_type (type): Expected type of sequence items.
+        seq_type (type, optional): Expected sequence type.
+
+    Returns:
+        bool: Whether the sequence is valid.
+    """
+    if seq_type is None:
+        exp_seq_type = abc.Sequence
+    else:
+        assert isinstance(seq_type, type)
+        exp_seq_type = seq_type
+    if not isinstance(seq, exp_seq_type):
+        return False
+    for item in seq:
+        if not isinstance(item, expected_type):
+            return False
+    return True
+
+def is_list_of(seq, expected_type):
+    """Check whether it is a list of some type.
+
+    A partial method of :func:`is_seq_of`.
+    """
+    return is_seq_of(seq, expected_type, seq_type=list)