feasytools 0.0.2__tar.gz

feasytools-0.0.2/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.1
+Name: feasytools
+Version: 0.0.2
+Summary: A set of tools for data processing, including Time Function, Table, Priority Queue, Range List, etc.
+Home-page: UNKNOWN
+Author: fmy_xfk
+License: UNKNOWN
+Description: # FEasyTools: Some useful components
+        - dprov: Data provider as time functions
+        - argchk: Argument parser with type checking
+        - table: Table helper for the CSV, SDT & SDT.GZ format, and their reader and writer.
+        - pq: Heap, priority queue & buffered priority queue
+        - rangelist: An xml reader for a time range list
+        
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown

feasytools-0.0.2/README.md

@@ -0,0 +1,6 @@
+# FEasyTools: Some useful components
+- dprov: Data provider as time functions
+- argchk: Argument parser with type checking
+- table: Table helper for the CSV, SDT & SDT.GZ format, and their reader and writer.
+- pq: Heap, priority queue & buffered priority queue
+- rangelist: An xml reader for a time range list

feasytools-0.0.2/feasytools/__init__.py

@@ -0,0 +1,12 @@
+from .tfunc import *
+from .argchk import *
+from .table import *
+from .pq import *
+from .rangelist import *
+
+def time2str(tspan:float):
+    tspan=round(tspan)
+    s=tspan%60
+    m=tspan//60%60
+    h=tspan//3600
+    return f"{h:02}:{m:02}:{s:02}"

feasytools-0.0.2/feasytools/argchk.py

@@ -0,0 +1,67 @@
+# 具有类型注释功能的ArgParse: ArgChecker
+import sys
+from typing import Any, Optional, Union
+    
+class ArgChecker:
+    @staticmethod
+    def __cast(v:str)->'Union[None,bool,int,float,str]':
+        if v=="True": return True
+        if v=="False": return False
+        if v=="None": return None
+        try:
+            return int(v)
+        except:
+            pass
+        try:
+            return float(v)
+        except:
+            return v.strip('"')
+    @staticmethod
+    def get_dict()->'dict[str,Union[str,Any]]':
+        '''将输入参数以字典的形式返回'''
+        cur_key=None
+        ret={}
+        for v in sys.argv[1:]:                
+            if v.startswith('-'):
+                if cur_key!=None:
+                    ret[cur_key]=True
+                cur_key=v.strip('-')
+            elif cur_key!=None:
+                ret[cur_key]=ArgChecker.__cast(v)
+                cur_key=None
+            else:
+                raise ValueError(f"无效参数'{v}'")
+        if cur_key!=None: ret[cur_key]=True
+        return ret
+    
+    def __init__(self,pars: 'Optional[dict[str,Any]]'=None):
+        self.__args=self.get_dict() if pars is None else pars
+    
+    def pop_bool(self, key: str) -> bool:
+        if self.__args.pop(key, False): return True
+        return False
+    
+    def pop_int(self, key: str, default: Optional[int] = None) -> int:
+        val = self.__args.pop(key, default)
+        if val is None: raise ValueError(f"必须指定'{key}'参数")
+        return int(val)
+    
+    def pop_str(self, key: str, default: Optional[str]=None) -> str:
+        val = self.__args.pop(key, default)
+        if val is None: raise ValueError(f"必须指定'{key}'参数")
+        return str(val)
+   
+    def pop_float(self, key: str, default: Optional[float] = None) -> float:
+        val = self.__args.pop(key, default)
+        if val is None: raise ValueError(f"必须指定'{key}'参数")
+        return float(val)
+    
+    def empty(self) -> bool:
+        return len(self.__args) == 0
+    
+    def keys(self): return self.__args.keys()
+    def values(self): return self.__args.values()
+    def items(self): return self.__args.items()
+
+    def __str__(self):
+        return str(self.__args)

feasytools-0.0.2/feasytools/pq.py

@@ -0,0 +1,185 @@
+import heapq
+from collections import deque
+from typing import Generic, TypeVar
+
+QItem=TypeVar("QItem")
+
+class Heap(Generic[QItem]):
+    '''小根堆数据结构'''
+    def __init__(self):
+        self._q=[]
+    
+    def push(self,item:QItem):
+        '''入堆, O(log n)'''
+        heapq.heappush(self._q,item)
+    
+    def pop(self)->QItem:
+        '''弹出堆顶元素, O(log n)'''
+        return heapq.heappop(self._q)
+    
+    def remove(self,item)->bool:
+        '''弹出item元素, O(n)'''
+        try:
+            idx=self._q.index(item)
+        except ValueError:
+            return False
+        self._q.pop(idx)
+        heapq.heapify(self._q)
+        return True
+    
+    @property
+    def top(self)->QItem:
+        '''获取堆顶元素，但不弹出, O(1)'''
+        return self._q[0]
+    
+    def __len__(self)->int:
+        return len(self._q)
+    
+    def __contains__(self,obj)->bool:
+        '''检查元素是否存在, O(n)'''
+        return obj in self._q
+    
+    def empty(self)->bool:
+        '''检测堆是否为空, O(1)'''
+        return len(self._q)==0
+    
+class PQueue(Generic[QItem]):
+    '''优先级队列（小根堆数据结构）'''
+    def __init__(self):
+        self._q:'list[tuple[int,QItem]]'=[]
+    
+    def push(self,pri:int,item:QItem)->None:
+        '''
+        入队, O(log n)
+            pri: 优先级
+            item: 元素
+        '''
+        heapq.heappush(self._q,(pri,item))
+    
+    def pop(self)->'tuple[int,QItem]':
+        '''获取队首元素并出队, O(log n)'''
+        return heapq.heappop(self._q)
+    
+    def remove(self,item:QItem)->bool:
+        '''弹出第一个item元素, O(n), 返回是否成功'''
+        idx=-1
+        for i,(_,data) in enumerate(self._q):
+            if data == item:
+                idx = i
+                break
+        if idx==-1: return False
+        self._q.pop(idx)
+        heapq.heapify(self._q)
+        return True
+    
+    @property
+    def top(self)->'tuple[int,QItem]':
+        '''获取队首元素，但不出队, O(1)'''
+        return self._q[0]
+    
+    def __len__(self)->int:
+        return len(self._q)
+    
+    def __contains__(self,obj)->bool:
+        return obj in self._q
+    
+    def empty(self)->bool:
+        '''检测队列是否为空, O(1)'''
+        return len(self._q)==0
+    
+    def __str__(self)->str:
+        '''转为字符串, O(nlogn)'''
+        q2=self._q.copy()
+        q2.sort()
+        return str(q2)
+
+class BufferedPQ(Generic[QItem]):
+    '''
+    含有缓冲的优先级队列。
+    该数据结构分为两部分，一个普通队列(记为q)和一个优先级队列(记为p)。
+    优先级队列大小恒定(设大小为n)，普通队列大小可变(设大小为m)。
+    数据进入普通队列，而后进入优先级队列，最后离开本结构。
+    '''
+    def __init__(self,p_size:int):
+        self._sz=p_size
+        self._P:'PQueue[QItem]'=PQueue()
+        self._Q:'deque[tuple[int,QItem]]'=deque()
+        self._sP=set()
+        self._sQ=set()
+
+    def __len__(self)->int:
+        return len(self._P)+len(self._Q)
+    
+    @property
+    def p_size(self)->int:
+        '''获取优先级区大小'''
+        return self._sz
+    
+    @property
+    def p_len(self)->int:
+        '''检查优先级区长度, O(1)'''
+        return len(self._P)
+    
+    @property
+    def q_len(self)->int:
+        '''检查缓冲区长度, O(1)'''
+        return len(self._Q)
+    
+    def __contains__(self,obj)->bool:
+        '''检查元素是否存在, O(1)'''
+        return obj in self._sP or obj in self._sQ
+    
+    def push(self,pri:int,obj:QItem)->bool:
+        '''
+        入队, O(log n)
+            pri: 优先级
+            item: 元素
+        '''
+        if self.__contains__(obj): return False
+        if len(self._P)<self._sz:
+            self._P.push(pri,obj)
+            self._sP.add(obj)
+        else:
+            self._Q.append((pri,obj))
+            self._sQ.add(obj)
+        return True
+    
+    def top(self)->'tuple[int,QItem]':
+        '''获取队首元素，但不出队, O(1)'''
+        return self._P.top
+    
+    def pop(self)->'tuple[int,QItem]':
+        '''获取队首元素并出队, O(log n)'''
+        ret=self._P.pop()
+        self._sP.remove(ret[1])
+        if len(self._Q)>0:
+            pri,obj=self._Q.popleft()
+            self._sQ.remove(obj)
+            self._P.push(pri,obj)
+            self._sP.add(obj)
+        return ret
+    
+    def empty(self)->bool:
+        '''检测队列是否为空, O(1)'''
+        return self._P.empty()
+    
+    def remove(self,obj)->bool:
+        '''弹出第一个obj元素, O(n+m), 返回是否成功'''
+        if self._P.remove(obj): return True
+        try:
+            self._Q.remove(obj)
+            return True
+        except:
+            return False
+        
+    def p_has(self,obj)->bool:
+        '''检查元素是否存在于优先级区, O(1)'''
+        return obj in self._sP
+    
+    def q_has(self,obj)->bool:
+        '''检查元素是否存在于缓冲区, O(1)'''
+        return obj in self._sQ
+    
+    def __str__(self)->str:
+        '''转为字符串, O(nlogn+m)'''
+        return f"BufferedPQ[P={self._P},Q={self._Q}]"

feasytools-0.0.2/feasytools/rangelist.py

@@ -0,0 +1,46 @@
+import xml.etree.ElementTree as ET
+from typing import Union, overload
+
+class RangeList:
+    '''时间范围列表，用于表示一系列时间段'''
+    @staticmethod
+    def parse_time(s:str)->int:
+        '''将时间字符串转换为秒数, 支持格式为hh:mm:ss或者一个可转化成int的字符串'''
+        try:
+            return int(s)
+        except:
+            h,m,s = s.split(":")
+            return int(h)*3600+int(m)*60+int(s)
+    
+    @overload
+    def __init__(self, data: ET.Element): '''从xml节点初始化'''
+    @overload
+    def __init__(self, data: 'list[tuple[int,int]]'): '''从列表初始化'''
+
+    def __init__(self, data: 'Union[list[tuple[int,int]], ET.Element]'):
+        if isinstance(data,ET.Element):
+            loop_period = int(data.attrib.get("loop_period", 0))
+            loop_times = int(data.attrib.get("loop_times", 1))
+            assert loop_times >= 1
+            data = [(self.parse_time(itm.attrib['btime']),self.parse_time(itm.attrib["etime"])) for itm in data]
+            if loop_times > 1 and len(data) > 1:
+                assert loop_period > data[-1][1]
+                n = len(data)
+                for j in range(1, loop_times):
+                    for i in range(n):
+                        if data[i][0]>=data[i][1]: raise ValueError(f"起始时间{data[i][0]}晚于终止时间{data[i][1]}")
+                        data.append((data[i][0]+loop_period*j,data[i][1]+loop_period*j))
+        self._d:'list[tuple[int,int]]' = data
+    
+    def __contains__(self, t:int):
+        for (l,r) in self._d:
+            if l<=t and t<r: return True
+        return False
+    
+    def __len__(self)->int: return self._d.__len__()
+
+    def __getitem__(self, indices): return self._d.__getitem__(indices)
+    
+    def __str__(self): return str(self._d)
+
+    def __iter__(self): return iter(self._d)

feasytools-0.0.2/feasytools/table.py

@@ -0,0 +1,490 @@
+import struct, gzip
+from typing import IO, BinaryIO, Generic, Optional, TextIO, Type, TypeVar, Union, Iterable, overload
+import numpy as np
+from abc import abstractmethod, ABCMeta
+
+Table_DType = Type[Union[np.int32,np.float32]]
+_Table_DType = TypeVar("_Table_DType",np.int32,np.float32)
+
+def _Lchk(L):
+    assert L=='i' or L=='f'
+
+def _dtypechk(dtype):
+    assert dtype==np.int32 or dtype==np.float32
+
+def _L2dtype(L)->Type:
+    return np.int32 if L=='i' else np.float32
+
+def _dtype2L(dtype)->str:
+    return 'i' if dtype==np.int32 else 'f'
+
+class TableWriter(metaclass = ABCMeta):
+    '''数据表写入器(抽象类)'''
+    @abstractmethod
+    def __init__(self,col_names:'list[str]',dtype:Type)->None:
+        _dtypechk(dtype)
+        self._col_names:'list[str]' = col_names
+        self._dtype:Table_DType = dtype
+    @property
+    def col_num(self)->int: 
+        '''数据表的列数'''
+        return len(self._col_names)
+    @property
+    def dtype(self)->Type:
+        '''数据表的数据类型'''
+        return self._dtype
+    @abstractmethod
+    def write(self,data:list)->None: '''写入一条数据, 数据长度必须与列数相等'''
+    @abstractmethod
+    def write_all(self,data:np.ndarray)->None: '''写入多条数据, 数据长度(参数ndarray的列数)必须与本表的列数相等'''
+    @abstractmethod
+    def close(self)->None: '''关闭写入器'''
+
+class FileTableWriter(TableWriter):
+    '''文件数据表写入器(抽象类)'''
+    @abstractmethod
+    def __init__(self,fh:IO,col_names:'list[str]',dtype:Type)->None:
+        super().__init__(col_names,dtype)
+        self._fh=fh
+
+class MemoryTableWriter(TableWriter):
+    '''内存数据表写入器'''
+    _data:np.ndarray
+    def __init__(self,col_names:'list[str]',dtype:Type)->None:
+        '''
+        初始化
+            col_names: 列名
+            dtype: 数据类型, 可以是np.int32或np.float32
+        '''
+        super().__init__(col_names,dtype)
+        self._data=np.zeros((0,len(col_names)),dtype=dtype)
+    
+    def write(self,data:list)->None:
+        self._data=np.vstack([self._data,np.array(data,dtype=self._dtype)])
+    
+    def write_all(self,data:np.ndarray)->None:
+        self._data=np.vstack([self._data,data])
+    
+    def close(self): pass
+
+    @property
+    def data(self)->np.ndarray: 
+        '''获取写入的所有数据'''
+        return self._data
+        
+class CsvTableWriter(FileTableWriter):
+    '''CSV数据表写入器'''
+    _fh: TextIO
+    def __init__(self,fname:str,col_names:'list[str]',dtype:Type)->None:
+        '''
+        初始化
+            fname: 文件名
+            col_names: 列名
+            dtype: 数据类型, 可以是np.int32或np.float32
+        '''
+        super().__init__(open(fname,"w"),col_names,dtype)
+        self._fh.write(','.join(self._col_names)+"\n")
+
+    def write(self,data:list)->None:
+        self._fh.write(','.join(map(str,data))+"\n")
+    
+    def write_all(self, data: np.ndarray)->None:
+        for ln in data: self.write(ln)
+    
+    def close(self): self._fh.close()
+
+class BinTableWriter(FileTableWriter):
+    '''二进制数据表写入器(抽象类)'''
+    _fh: IO
+    @abstractmethod
+    def __init__(self,fh,col_names:'list[str]',dtype:Type,buf_sz:int=1024):
+        super().__init__(fh,col_names,dtype)
+        header=_dtype2L(dtype)+('|'.join(self._col_names))
+        header+=" "*((4-len(header)%4)%4)
+        header=header.encode()
+        self._fh.write(struct.pack("<I",len(header)))
+        self._fh.write(header)
+        self._buf=[]
+        self._buf_sz=buf_sz
+        self._dcnt=0
+    
+    def __wbuf(self):
+        self._fh.write(np.stack(self._buf,dtype=self._dtype).tobytes())
+        self._buf=[]
+
+    def write(self,data:list):
+        self._buf.append(data)
+        self._dcnt+=1
+        if len(data)>=self._buf_sz: self.__wbuf()
+    
+    def write_all(self, data: np.ndarray):
+        if data.dtype!=self._dtype: data.astype(self._dtype)
+        self._fh.write(data.tobytes())
+
+    def close(self):
+        if len(self._buf)>0: self.__wbuf()
+        self._fh.close()
+
+class SdtTableWriter(BinTableWriter):
+    '''SDT数据表写入器'''
+    def __init__(self,fname:str,col_names:'list[str]',dtype:Type,buf_sz:int=1024):
+        '''
+        初始化
+            fname: 文件名
+            col_names: 列名
+            dtype: 数据类型, 可以是np.int32或np.float32
+            buf_sz: 缓冲区大小, 默认为1024
+        '''
+        super().__init__(open(fname,"wb"),col_names,dtype,buf_sz)
+
+class SdtGzTableWriter(BinTableWriter):
+    '''SDT.GZ数据表写入器'''
+    def __init__(self,fname:str,col_names:'list[str]',dtype:Type,buf_sz:int=1024):
+        '''
+        初始化
+            fname: 文件名
+            col_names: 列名
+            dtype: 数据类型, 可以是np.int32或np.float32
+            buf_sz: 缓冲区大小, 默认为1024
+        '''
+        super().__init__(gzip.open(fname,"wb"),col_names,dtype,buf_sz)
+
+class TableReader(metaclass = ABCMeta):
+    '''数据表读取器(抽象类)'''
+    @abstractmethod
+    def __init__(self,col_names:'list[str]',dtype:Type)->None:
+        _dtypechk(dtype)
+        self._col_names:'list[str]'=col_names
+        self._col_cnt=len(self._col_names)
+        self._cmap = {cn:i for i,cn in enumerate(self._col_names)}
+        self._dtype:Type=dtype
+    @property
+    def head(self)->'list[str]': 
+        '''表头'''
+        return self._col_names
+    @property
+    def dtype(self)->Type: 
+        '''数据类型, 可以是np.int32或np.float32'''
+        return self._dtype
+    @abstractmethod
+    def read(self,cnt:int)->np.ndarray: '''从当前位置开始, 读取1行内容'''
+    @abstractmethod
+    def read_all(self)->np.ndarray: '''从头开始, 读取所有内容'''
+    @abstractmethod
+    def close(self)->None: '''关闭TableReader'''
+
+class FileTableReader(TableReader):
+    '''文件数据表读取器(抽象类)'''
+    @abstractmethod
+    def __init__(self,f:IO,col_names:'list[str]',dtype:Type)->None:
+        super().__init__(col_names,dtype)
+        self._fh:IO=f
+    def close(self)->None:
+        self._fh.close()
+    
+class MemoryTableReader(TableReader):
+    '''内存数据表读取器'''
+    _data:np.ndarray
+    def __init__(self,col_names:'list[str]',data:np.ndarray):
+        '''
+        初始化
+            col_names: 列名
+            data: 数据
+        '''
+        super().__init__(col_names,data.dtype)
+        self._data=data
+        self.__pos = 0
+
+    def read(self,cnt:int)->np.ndarray:
+        assert cnt > 0, "读取行数必须大于0"
+        ed = cnt + self.__pos
+        if ed > self._data.shape[0]: ed = self._data.shape[0]
+        return self._data[self.__pos:ed]
+    
+    def read_all(self)->np.ndarray:
+        return self._data
+    
+    def close(self): pass
+    
+class BinTableReader(FileTableReader):
+    '''二进制数据表读取器(抽象类)'''
+    _fh:BinaryIO
+    @abstractmethod
+    def __init__(self,fname:str,openFunc,allowNegativeSeek:bool=False)->None:
+        self.__fn=fname
+        self._openF=openFunc
+        fh:BinaryIO=self._openF(self.__fn,"rb")
+        hlen=struct.unpack("<I",fh.read(4))[0]
+        header=fh.read(hlen).decode()
+        super().__init__(fh,header[1:].strip().split('|'),_L2dtype(header[0]))
+        self._dstart=hlen+4
+        self._itmsz=self._col_cnt*4
+        self._neg=allowNegativeSeek
+
+    def seek(self,row:int,col:int=0)->int:
+        '''定位到第row行的第col个数据, row和col均从0开始编号'''
+        dst=self._dstart+self._itmsz*row+4*col
+        if self._neg or self._fh.tell()<=dst:
+            return self._fh.seek(dst)
+        else:
+            self._fh.close()
+            self._fh=self._openF(self.__fn,"rb")
+            return self._fh.seek(dst)
+
+    def read(self,cnt:int)->Optional[np.ndarray]:
+        data=self._fh.read(self._itmsz*cnt)
+        if data==b'': return None
+        return np.frombuffer(data,self._dtype).reshape(-1,self._col_cnt)
+    
+    def read_all(self)->np.ndarray:
+        self.seek(0,0)
+        return np.frombuffer(self._fh.read(),self._dtype).reshape(-1,self._col_cnt)
+
+    def read_col(self,col_id:Union[int,str])->np.ndarray:
+        '''读取col_id列. col_id可以是下标, 也可以是列名'''
+        if isinstance(col_id,str): col_id=self._cmap[col_id]
+        i=0; dat=[]
+        while True:
+            self.seek(i,col_id)
+            d=self._fh.read(4)
+            if d==b'': break
+            dat.append(d)
+            i+=1
+        return np.frombuffer(b''.join(dat),dtype=self._dtype)
+
+    def read_at(self,start:int,cnt:int)->Optional[np.ndarray]:
+        '''读取range(start,start+cnt)范围的行'''
+        self.seek(start)
+        return self.read(cnt)
+
+class SdtTableReader(BinTableReader):
+    '''SDT表格读取器'''
+    def __init__(self,fname:str):
+        '''
+        初始化
+            fname: 文件名
+        '''
+        super().__init__(fname,open,True)
+
+class SdtGzTableReader(BinTableReader):
+    '''SDT.GZ表格读取器'''
+    def __init__(self,fname:str):
+        '''
+        初始化
+            fname: 文件名
+        '''
+        super().__init__(fname,gzip.open,False)
+    
+class CsvTableReader(FileTableReader):
+    '''CSV表格读取器'''
+    _fh:TextIO
+    def __init__(self,fname:str,dtype:Type)->None:
+        '''
+        初始化
+            fname: 文件名
+            dtype: 数据类型, 可以是np.int32或np.float32
+        '''
+        self.__fn=fname
+        fh=open(fname,"r")
+        super().__init__(fh,fh.readline().strip().split(","),dtype)
+
+    def __parseline(self,ln:str)->np.ndarray:
+        strs=map(lambda x: x.strip(), ln.split(','))
+        if self._dtype==np.int32:
+            return np.array(list(map(int,strs)),dtype=np.int32)
+        else:
+            return np.array(list(map(float,strs)),dtype=np.float32)
+    
+    def __read1(self)->Optional[np.ndarray]:
+        '''从当前位置开始, 读取1行'''
+        ln = self._fh.readline().strip()
+        if ln == "": return None
+        return self.__parseline(ln)
+    
+    def read(self,cnt:int)->Optional[np.ndarray]:
+        if cnt == 1: return self.__read1()
+        ret = []
+        for _ in range(cnt):
+            dat = self.__read1()
+            if dat is None: break
+            ret.append(dat)
+        if len(ret) == 0: return None
+        return np.stack(ret)
+    
+    def read_all(self)->np.ndarray:
+        self._fh.close()
+        self._fh = open(self.__fn,"r")
+        self._fh.readline()
+        return np.stack([self.__parseline(x) for x in self._fh.readlines()])
+
+def createTableReader(fname:str,dtype:Optional[Type]=None)->TableReader:
+    '''从文件名创建TableReader, 对于CSV文件需要指定dtype'''
+    fn=fname.lower()
+    if fn.endswith(".csv"):
+        assert dtype is not None
+        return CsvTableReader(fname,dtype)
+    elif fn.endswith(".sdt"):
+        return SdtTableReader(fname)
+    elif fn.endswith(".sdt.gz"):
+        return SdtGzTableReader(fname)
+    else:
+        raise ValueError("不支持的文件类型")
+
+def createTableWriter(fname:str,cols:'list[str]',dtype:Type)->TableWriter:
+    '''从文件名创建TableWriter'''
+    fn=fname.lower()
+    if fn.endswith(".csv"):
+        return CsvTableWriter(fname,cols,dtype)
+    elif fn.endswith(".sdt"):
+        return SdtTableWriter(fname,cols,dtype)
+    elif fn.endswith(".sdt.gz"):
+        return SdtGzTableWriter(fname,cols,dtype)
+    elif fn.endswith("<mem>"):
+        return MemoryTableWriter(cols,dtype)
+    else:
+        raise ValueError("不支持的文件类型")
+
+def _convbinfile(r:Union[BinaryIO,gzip.GzipFile],w:Union[BinaryIO,gzip.GzipFile],bufsz:int=1024*1024*64):
+    while True:
+        data=r.read(bufsz)
+        if data==b'': break
+        w.write(data)
+    r.close()
+    w.close()
+
+def convertTableFile(rfile:str,wfile:str,dtype:Optional[Type]=None,bufsz:int=1024)->None:
+    '''将一种文件类型的Table转化成另一文件类型的Table, 如果输入文件为CSV文件, 还需要指定dtype'''
+    if rfile.lower().endswith(".sdt.gz") and wfile.lower().endswith(".sdt"):
+        _convbinfile(gzip.open(rfile,"rb"),open(wfile,"wb"))
+        return
+    elif rfile.lower().endswith(".gz") and wfile.lower().endswith(".sdt.gz"):
+        _convbinfile(open(rfile,"rb"),gzip.open(wfile,"wb"))
+        return
+    r=createTableReader(rfile,dtype)
+    w=createTableWriter(wfile,r.head,r.dtype)
+    while True:
+        data=r.read(bufsz)
+        if data is None: break
+        w.write_all(data)
+    r.close()
+    w.close()
+
+class ReadOnlyTable(Generic[_Table_DType]): 
+    '''
+    **只读**数据表, 表中所有数据必须是同一类型, 必须是32位int或float格式.
+    某一行的内容直接用下标获取; 某一列的内容用col方法获取.
+    由于是只读列表，因此必须从文件加载. 支持的文件类型包含csv, sdt和sdt.gz.
+    '''
+    _d:'Optional[np.ndarray[tuple[int,int],np.dtype[_Table_DType]]]'
+    _btr:TableReader
+
+    @overload
+    def __init__(self,source:MemoryTableReader):
+        '''使用MemoryTableReader初始化'''
+    @overload
+    def __init__(self,source:MemoryTableWriter): 
+        '''使用MemoryTableWriter初始化'''
+    @overload
+    def __init__(self,source:str,dtype:Optional[Table_DType]=None,preload:bool=False):
+        '''
+        初始化
+            fname: 数据表文件名, 仅支持csv, sdt和sdt.gz文件.
+            dtype: 表格数据类型, 只有csv文件需要提供此项.
+            preload: 是否在初始化时预加载全部数据.
+        
+        注意: 
+            预加载使得初始化速度大幅降低, 但大幅提升了运行时性能.
+            csv强制开启预加载功能, sdt和sdt.gz自选(推荐sdt.gz开启, sdt关闭).
+            如果数据过大导致内存不足, 请勿使用预加载功能.
+        '''
+    
+    def __init__(self,source:Union[str,MemoryTableWriter,MemoryTableReader],dtype:Optional[Table_DType]=None,preload:bool=False):
+        if isinstance(source,str):
+            self._btr=createTableReader(source,dtype)
+            fn=source.lower()
+            if preload or fn.lower().endswith(".csv"):
+                self._d=self._btr.read_all()
+            else:
+                self._d=None
+        elif isinstance(source,MemoryTableReader):
+            self._btr=source
+            self._d=source._data
+        elif isinstance(source,MemoryTableWriter):
+            self._btr=MemoryTableReader(source._col_names,source._data)
+            self._d=source._data
+
+    @property
+    def head(self)->'list[str]': 
+        '''表头'''
+        return self._btr._col_names
+
+    @property
+    def dtype(self)->Table_DType: 
+        '''
+        表中数据类型, 可取np.int32或np.float32
+        '''
+        return self._btr._dtype
+    
+    @property
+    def data(self)->'np.ndarray[(int,int),np.dtype[_Table_DType]]':
+        '''
+        表格数据. 如果没有预加载全部数据, 则调用此项时会加载所有数据.
+        '''
+        if self._d is None: self._d=self._btr.read_all()
+        return self._d
+    
+    def force_load_all(self): self._d=self._btr.read_all()
+    
+    @overload
+    def col(self,c:str): '''获取列名为c的列'''
+    @overload
+    def col(self,c:int): '''获取第c列'''
+    @overload
+    def col(self,c:Iterable[Union[str,int]]): '''获取多列'''
+
+    def col(self,c:Union[Iterable[Union[str,int]],str,int])->'np.ndarray[int,np.dtype[_Table_DType]]':
+        if isinstance(c,int) or isinstance(c,str):
+            if self._d is None:
+                assert isinstance(self._btr,BinTableReader)
+                return self._btr.read_col(c)
+            else:
+                if isinstance(c,str): c=self._btr._cmap[c]
+                return self._d[:,c]
+        elif isinstance(c,Iterable):
+            nc:list[int]=[self._btr._cmap[x] if isinstance(x,str) else x for x in c]
+            return self.data[:,nc]
+        else:
+            raise TypeError("不支持的索引类型")
+            
+    def row(self,row_id:int)->'Optional[np.ndarray[int,np.dtype[_Table_DType]]]':
+        '''获取一行数据'''
+        if self._d is not None: return self._d[row_id]
+        assert isinstance(self._btr,BinTableReader)
+        return self._btr.read_at(row_id,1)
+    
+    def at(self,col_name:str,row_id:int)->_Table_DType:
+        '''获取列名为col_name,行为row_id的数据'''
+        return self.data[self._btr._cmap[col_name],row_id]
+    
+    def __getitem__(self,indices)->'Union[np.ndarray[tuple[int,...],np.dtype[_Table_DType]],_Table_DType]':
+        '''仅限数字下标'''
+        return self.data[indices]
+    
+    def save(self,path:str):
+        '''
+        保存到.csv或.bin.gz文件
+            path: 文件路径
+        '''
+        if self._d is None: self._d=self._btr.read_all()
+        fn=path.lower()
+        if fn.endswith(".csv"):
+            CsvTableWriter(path,self.head,self.dtype).write_all(self._d)
+        elif fn.endswith(".sdt"):
+            SdtTableWriter(path,self.head,self.dtype).write_all(self._d)
+        elif fn.endswith(".sdt.gz"):
+            SdtGzTableWriter(path,self.head,self.dtype).write_all(self._d)
+        else:
+            raise ValueError("不支持的文件名")
+        
+    def __str__(self):
+        return ','.join(self.head)+"\n"+super().__str__()

feasytools-0.0.2/feasytools/tfunc.py

@@ -0,0 +1,203 @@
+from abc import ABCMeta, abstractmethod
+from functools import reduce
+from operator import add, sub, mul, truediv, floordiv
+from typing import Callable, Iterable, Optional, Union
+import bisect
+
+class TimeFunc(metaclass=ABCMeta):
+    '''时变函数'''
+    @abstractmethod
+    def __call__(self,time:int)->float: ...
+    @abstractmethod
+    def __str__(self)->str: ...
+    def __add__(self,other:'FloatLike')->'TimeFunc':
+        return calcFunc(self,other,'+')
+    def __sub__(self,other:'TimeFunc')->'TimeFunc':
+        return calcFunc(self,other,'-')
+    def __mul__(self,other)->'TimeFunc':
+        return calcFunc(self,other,'*')
+    def __truediv__(self,other)->'TimeFunc':
+        return calcFunc(self,other,'/')
+    def __floordiv__(self,other)->'TimeFunc':
+        return calcFunc(self,other,'//')
+
+_oper_trans = {
+    '+': add,
+    '-': sub,
+    '*': mul,
+    '/': truediv,
+    '//': floordiv
+}
+_Oper = Callable[[float,float],float]
+FloatLike = Union[TimeFunc,float,int]
+
+class OverrideFunc(TimeFunc):
+    '''重载函数. 正常情况下返回原始时间函数的值, 当设置了重载值时返回重载值'''
+    def __init__(self,default:TimeFunc,override_val:Optional[float] = None):
+        self._val:TimeFunc=default
+        self._override:Optional[float] = override_val
+    
+    def __call__(self,t:int)->float:
+        if self._override is not None: return self._override
+        return self._val(t)
+    
+    def setOverride(self,overval:float):
+        '''设置重载值'''
+        self._override=overval
+    
+    def clearOverride(self):
+        '''清除重载值'''
+        self._override=None
+    
+    def __str__(self):
+        if self._override is None: return f"OverrideF<{self._val}>"
+        return f"OverrideF<O={self._override}>"
+
+    def __repr__(self): return str(self)
+
+class PlusFunc(TimeFunc):
+    '''和函数'''
+    def __init__(self,f1:TimeFunc,f2:TimeFunc): self._f1=f1; self._f2=f2
+    def __call__(self,_t:int)->float: return self._f1(_t)+self._f2(_t)
+    def __str__(self)->str: return f"F<{self._f1}+{self._f2}>"
+    def __repr__(self)->str: return str(self)
+
+class QuickSumFunc(TimeFunc):
+    '''快速求和函数'''
+    def __init__(self,funcs:'list[TimeFunc]'): self._fs=funcs
+    def __call__(self,_t:int)->float: return sum(f(_t) for f in self._fs)
+    def __str__(self)->str: return f"FSum<{len(self._fs)} funcs>"
+    def __repr__(self)->str: return str(self)
+
+class MinusFunc(TimeFunc):
+    '''差函数'''
+    def __init__(self,f1:TimeFunc,f2:TimeFunc): self._f1=f1; self._f2=f2
+    def __call__(self,_t:int)->float: return self._f1(_t)-self._f2(_t)
+    def __str__(self)->str: return f"F<{self._f1}-{self._f2}>"
+    def __repr__(self)->str: return str(self)
+
+class MulFunc(TimeFunc):
+    '''积函数'''
+    def __init__(self,f1:TimeFunc,f2:TimeFunc): self._f1=f1; self._f2=f2
+    def __call__(self,_t:int)->float: return self._f1(_t)*self._f2(_t)
+    def __str__(self)->str: return f"F<{self._f1}*{self._f2}>"
+    def __repr__(self)->str: return str(self)
+
+class QuickMulFunc(TimeFunc):
+    '''快速求积函数'''
+    def __init__(self,funcs:'list[TimeFunc]'): self._fs=funcs
+    def __call__(self,_t:int)->float: return reduce(mul, (f(_t) for f in self._fs))
+    def __str__(self)->str: return f"FMul<{len(self._fs)} funcs>"
+    def __repr__(self)->str: return str(self)
+
+class TrueDivFunc(TimeFunc):
+    '''商函数'''
+    def __init__(self,f1:TimeFunc,f2:TimeFunc): self._f1=f1; self._f2=f2
+    def __call__(self,_t:int)->float: return self._f1(_t)/self._f2(_t)
+    def __str__(self)->str: return f"F<{self._f1}/{self._f2}>"
+    def __repr__(self)->str: return str(self)
+
+class FloorDivFunc(TimeFunc):
+    '''整除函数'''
+    def __init__(self,f1:TimeFunc,f2:TimeFunc): self._f1=f1; self._f2=f2
+    def __call__(self,_t:int)->float: return self._f1(_t)//self._f2(_t)
+    def __str__(self)->str: return f"F<{self._f1}//{self._f2}>"
+    def __repr__(self)->str: return str(self)
+
+class ConstFunc(TimeFunc):
+    '''常数函数'''
+    def __init__(self,const:float): self._val:float=const
+    def __call__(self,time:int)->float: return self._val
+    def __str__(self)->str: return f"ConstF<{self._val}>"
+    def __repr__(self)->str: return str(self)
+
+class SegFunc(TimeFunc):
+    '''分段常数函数'''
+    def __init__(self,time_line:'list[int]',data:'list[float]'):
+        if len(time_line) != len(data): raise ValueError(f"时间线长度{len(time_line)}和数据长度{len(data)}不一致")
+        for i in range(1,len(time_line)):
+            if time_line[i]<=time_line[i-1]:
+                raise ValueError(f"时间必须严格递增: [{i}]={time_line[i]}<=[{i-1}]={time_line[i-1]}")
+        self._tl = time_line
+        self._d = data   
+    
+    def __call__(self,time:int)->float:
+        if time < self._tl[0]: raise ValueError(f"时间{time}必须在开始时间{self._tl[0]}之后")
+        return self._d[bisect.bisect_right(self._tl, time) - 1]
+    def __str__(self)->str: return f"SegF<{len(self._d)} segs>"
+    def __repr__(self)->str: return str(self)
+
+class TimeImplictFunc(TimeFunc):
+    '''隐式时变函数, 即调用的时机决定了时间, 而无需在__call__时指定。__call__的参数time无效'''
+    def __init__(self,func:'Callable[[],float]'):self._f=func
+    def __call__(self,time:int)->float:return self._f()
+    def __str__(self)->str:return f"TImpF<{self._f}>"
+    def __repr__(self)->str: return str(self)
+
+class ComFunc(TimeFunc):
+    '''将普通Python函数包装成可运算函数'''
+    def __init__(self,func:'Callable[[int],float]'):self._f=func
+    def __call__(self,time:int)->float:return self._f(time)
+    def __str__(self)->str:return f"TImpF<{self._f}>"
+    def __repr__(self)->str: return str(self)
+
+class ManualFunc(TimeFunc):
+    '''手动指定常数函数'''
+    def __init__(self,init_val:float):self._v=init_val
+    def setManual(self,val:float):self._v=val
+    def __call__(self,time:int)->float:return self._v
+    def __str__(self)->str:return f"ManF<{self._v}>"
+    def __repr__(self)->str: return str(self)
+
+def __calc_c0(f1:Union[ConstFunc,float],f2:float,op:_Oper)->float:
+    if isinstance(f1,float): return op(f1,f2)
+    elif isinstance(f1,ConstFunc): return op(f1._val,f2)
+    else: raise TypeError(f1)
+
+def __calc_c1(f1:SegFunc,f2:float,op:_Oper)->SegFunc:
+    return SegFunc(f1._tl,[op(d,f2) for d in f1._d])
+
+def __calc_c2(f1:TimeImplictFunc,f2:float,op:_Oper)->TimeImplictFunc:
+    return TimeImplictFunc(lambda: op(f1._f(),f2))
+
+def quicksum(funcs:Iterable[TimeFunc])->TimeFunc:
+    '''快速求和'''
+    ret = list(funcs)
+    if len(ret)==0: return ConstFunc(0)
+    if len(ret)==1: return ret[0]
+    if len(ret)==2: return PlusFunc(ret[0],ret[1])
+    return QuickSumFunc(ret)
+
+def quickmul(funcs:Iterable[TimeFunc])->TimeFunc:
+    '''快速求积'''
+    ret = list(funcs)
+    if len(ret)==0: return ConstFunc(1)
+    if len(ret)==1: return ret[0]
+    if len(ret)==2: return MulFunc(ret[0],ret[1])
+    return QuickMulFunc(ret)
+
+def calcFunc(f1:FloatLike,f2:FloatLike,op:str)->TimeFunc:
+    _op = _oper_trans[op]
+    if isinstance(f2,ConstFunc): f2=f2._val
+    if isinstance(f2,(float,int)):
+        if isinstance(f1,(ConstFunc,float,int)): return ConstFunc(__calc_c0(f1,f2,_op))
+        elif isinstance(f1,SegFunc): return __calc_c1(f1,f2,_op)
+        elif isinstance(f1,TimeImplictFunc): return __calc_c2(f1,f2,_op)
+        else: f2 = ConstFunc(f2)
+    if isinstance(f1,(float,int)): f1 = ConstFunc(f1)
+    if isinstance(f1,ConstFunc) and op in ['+','*']:
+        if isinstance(f2,SegFunc): return __calc_c1(f2,f1._val,_op)
+        elif isinstance(f2,TimeImplictFunc): return __calc_c2(f2,f1._val,_op)
+    assert isinstance(f1,TimeFunc) and isinstance(f2,TimeFunc)
+    if op=='+': return PlusFunc(f1,f2)
+    elif op=='-': return MinusFunc(f1,f2)
+    elif op=='*': return MulFunc(f1,f2)
+    elif op=='/': return TrueDivFunc(f1,f2)
+    elif op=='//': return FloorDivFunc(f1,f2)
+    else: raise ValueError(op)
+
+def makeFunc(time_line:'list[int]',data:list)->TimeFunc:
+    '''生成分段常数函数或常数函数'''
+    if len(time_line)!=len(data): raise ValueError(f"时间线长度{len(time_line)}和数据长度{len(data)}不一致")
+    if len(data)==1: return ConstFunc(data[0])
+    else: return SegFunc(time_line,data)

feasytools-0.0.2/feasytools.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.1
+Name: feasytools
+Version: 0.0.2
+Summary: A set of tools for data processing, including Time Function, Table, Priority Queue, Range List, etc.
+Home-page: UNKNOWN
+Author: fmy_xfk
+License: UNKNOWN
+Description: # FEasyTools: Some useful components
+        - dprov: Data provider as time functions
+        - argchk: Argument parser with type checking
+        - table: Table helper for the CSV, SDT & SDT.GZ format, and their reader and writer.
+        - pq: Heap, priority queue & buffered priority queue
+        - rangelist: An xml reader for a time range list
+        
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown

feasytools-0.0.2/feasytools.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+setup.py
+feasytools/__init__.py
+feasytools/argchk.py
+feasytools/pq.py
+feasytools/rangelist.py
+feasytools/table.py
+feasytools/tfunc.py
+feasytools.egg-info/PKG-INFO
+feasytools.egg-info/SOURCES.txt
+feasytools.egg-info/dependency_links.txt
+feasytools.egg-info/requires.txt
+feasytools.egg-info/top_level.txt
+test/test.py

feasytools-0.0.2/feasytools.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

feasytools-0.0.2/feasytools.egg-info/requires.txt

@@ -0,0 +1 @@
+numpy>=1.19.0

feasytools-0.0.2/feasytools.egg-info/top_level.txt

@@ -0,0 +1 @@
+feasytools

feasytools-0.0.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

feasytools-0.0.2/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r") as f:
+    long_description = f.read()
+
+setup(
+    name="feasytools",
+    version="0.0.2",
+    author="fmy_xfk",
+    packages=find_packages(),
+    description="A set of tools for data processing, including Time Function, Table, Priority Queue, Range List, etc.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.6",
+    install_requires=["numpy>=1.19.0"],
+)

feasytools-0.0.2/test/test.py

@@ -0,0 +1,61 @@
+import sys
+from pathlib import Path
+print(__file__)
+sys.path.append(str(Path(__file__).parent.parent))
+print(sys.path)
+from feasytools import *
+
+print("ArgChecker test:")
+print(ArgChecker())
+
+print("PQ test:")
+hp = Heap()
+hp.push("h")
+hp.push("e")
+assert hp.empty() == False
+assert len(hp) == 2
+assert hp.remove("h") == True
+hp.push("f")
+assert hp.pop() == "e"
+assert hp.pop() == "f"
+assert hp.empty() == True
+
+q = PQueue()
+q.push(3,"henks")
+q.push(1,"shjd")
+assert q.pop() == (1,"shjd")
+assert q.remove("henks") == True
+
+print("Rangelist test:")
+rl = RangeList([(0,1),(2,3),(4,5)])
+assert 0 in rl
+assert not 1 in rl
+
+# More general tests will be added in the future
+
+print("TimeFunc test:")
+f1 = ConstFunc(1)
+f2 = TimeImplictFunc(lambda: 1+2+3)
+f3 = ComFunc(lambda t: t)
+f4 = ManualFunc(1)
+f4.setManual(2)
+print(f1(0))
+print(f2(0))
+print(f3(0))
+print(f4(0))
+print(f1)
+print(f2)
+print(f3)
+print(f4)
+f5 = quicksum([f1,f2,f3,f4])
+print(f5)
+f6 = quickmul([f1,f2,f3,f4])
+print(f6)
+f7 = f5 + 1
+print(f7)
+f8 = f6 * 2
+print(f8)
+f9 = f5 + f6
+print(f9)
+f10 = f5 * f6
+print(f10)