buildz 0.9.8__tar.gz → 0.9.10__tar.gz

{buildz-0.9.8/buildz.egg-info → buildz-0.9.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.9.8
+Version: 0.9.10
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.9.8 → buildz-0.9.10}/buildz/_dz/conf.py

@@ -12,7 +12,49 @@ def dzkeys(key, spt):
     if type(key) not in (list, tuple):
         key = [key]
     return key
+
+class BindKey(Base):
+    def sub(self, key):
+        return BindKey(key, False, self, self.obj)
+    def init(self, key, abs=True, up=None, conf=None):
+        self.key = key
+        self.abs = abs
+        self.spt = spt
+        self.up = up
+        self.obj = conf
+    def bind(self, conf):
+        self.obj=conf
+    def get(self, conf=None, default=None):
+        conf = self.conf(conf)
+        if self.up:
+            conf = self.up(conf)
+        elif self.abs:
+            conf = conf()
+        return conf.get(self.key, default)
+    def conf(self, obj):
+        obj = obj or self.obj
+        assert obj is not None
+        return obj
+    def set(self, val, conf=None):
+        conf = self.conf(conf)
+        if self.up:
+            conf = self.up(conf)
+        elif self.abs:
+            conf = conf()
+        return conf.set(self.key, val)
+    def call(self, conf):
+        conf = self.conf(conf)
+        if self.up:
+            conf = self.up(conf)
+        elif self.abs:
+            conf = conf()
+        return conf(self.key)
+
+pass
 class Conf(Base):
+    @staticmethod
+    def bind_key(key, abs=True, up = None):
+        return BindKey(key, abs, up)
     def val(self):
         return self.get_conf()
     def get_type(self):
@@ -292,4 +334,15 @@ class Conf(Base):
         for key in keys:
             if self.has(key, link):
                 return True
-        return False
+        return False
+
+pass
+
+class ArgsConf(Base):
+    def init(self, conf, args):
+        self.conf =conf
+        self.args =args
+    def __getattr__(self, key):
+        return getattr(self.conf,key)
+
+pass

buildz-0.9.10/buildz/_dz/oconf.py

@@ -0,0 +1,89 @@
+class ObjConf:
+    def __str__(self):
+        return str(self._maps)
+    def __repr__(self):
+        return self.__str__()
+    def __init__(self):
+        self._maps = {}
+    def __getitem__(self, key):
+        return self._maps[key]
+    def __getattr__(self, key):
+        if key in self._maps:
+            return self._maps[key][0]
+        return self.__getattribute__(key)
+    def __setattr__(self, key, val):
+        if key != '_maps':
+            self(key, val)
+        else:
+            object.__setattr__(self,key,val)
+    def __call__(self, *args, **maps):
+        '''
+            call(key,val)
+            or
+            call(key0=val0, key1=val1)
+        '''
+        if len(args)==2:
+            key,val = args
+            self._maps[key] = val
+        else:
+            for key,val in maps.items():
+                self._maps[key] = val
+    def __delattr__(self, key):
+        if key in self._maps:
+            del self._maps[key]
+        return super().__delattr__(key)
+    def __contains__(self, key):
+        return self._maps.__contains__(key)
+
+pass
+
+class ObjTypeConf(ObjConf):
+    def __init__(self, vtype=0):
+        super().__init__()
+        self._vtypes = {}
+        self._vtype = vtype
+    def __getitem__(self, vtype=None):
+        if vtype is None:
+            return self._maps
+        maps = {}
+        for key in self._maps:
+            _vtype = self._vtypes[key]
+            if _vtype==vtype:
+                maps[key]=self._maps[key]
+        return maps
+    def __setattr__(self, key, val):
+        if key not in {'_maps','_vtype','_vtypes'}:
+            self(key, val)
+        else:
+            object.__setattr__(self,key,val)
+    def __call__(self, *args, **maps):
+        '''
+            call(key,val)
+            call(key,val,vtype)
+            or
+            call(key0=val0, key1=val1)
+        '''
+        if len(maps)==0:
+            if len(args)<=1:
+                return self.__getitem__(*args)
+        if len(args) in {2,3}:
+            key = args[0]
+            val = args[1]
+            vtype = None
+            if len(args)==3:
+                vtype = args[2]
+            vtype = vtype or self._vtype
+            self._maps[key] = val
+            self._vtypes[key] = vtype
+        else:
+            for key,val in maps.items():
+                self._maps[key] = val
+                self._vtypes[key] = self._vtype
+    def __delattr__(self, key):
+        if key in self._maps:
+            del self._maps[key]
+            del self._vtypes[key]
+            return
+        return super().__delattr__(key)
+
+pass

{buildz-0.9.8 → buildz-0.9.10}/buildz/argx.py

@@ -132,6 +132,7 @@ def fetch(argv = None):
         v = argv[i]
         make_plus = 0
         if v in ["-", "--", "+"]or v[0] not in "+-":
+            v = to_val(v)
             lists.append(v)
             i+=1
             continue

{buildz-0.9.8 → buildz-0.9.10}/buildz/confz.py

@@ -10,6 +10,7 @@ def load_conf(conf, dp=None, dp_key = 'dp', src_key = 'src.conf'):
     conf_first,replace,flush,visit_list = conf.gets('conf_first,replace,flush,visit_list',1, 1,1,0)
     spt, spts = conf.gets('spt, spts','.',',')
     dp = conf.get(dp_key, dp)
+    conf.set(dp_key, dp)
     path = pathz.Path()
     path.set("dp", dp)
     rst = Conf(spt, spts)
@@ -51,7 +52,7 @@ def calls(conf):
     if not local:
         obj = root
     for key in calls:
-        assert obj.has(key), f"not has key: '{key}'"
+        #assert obj.has(key), f"not has key: '{key}'"
         simple(obj.l(key))
     return conf
 fn_key = "confz.fns"
@@ -104,7 +105,11 @@ def simple(conf):
         conf = conf.ltop(conf.domain)
     deep_link(conf)
     deep_copy(conf)
-    fc = get_fc(conf)
+    fc = get_fc(conf, default_fn = "calls")
+    if fc == calls and not conf.has("calls"):
+        fc = None
+    if fc is None :
+        fc = pyz.load(conf.domain)
     assert fc is not None, f"conf has not setted deal fc: {conf}"
     return fc(conf)
 def get_sys_conf(conf = []):

{buildz-0.9.8 → buildz-0.9.10}/buildz/db/dv/orm.py

@@ -13,10 +13,14 @@ def dict2list(data):
     return rst
 def deal_item(item, conf):
     key = item[0]
-    if key in "sql_before,sql_after,sql_delete_before,sql_delete_after".split(","):
-        conf[key].append(item[1])
+    if len(item)==2:
+        obj = item[1]
     else:
-        conf[key] = item[1]
+        obj = item[1:]
+    if key in "sql_before,sql_after,sql_delete_before,sql_delete_after,index.unique".split(","):
+        conf[key].append(obj)
+    else:
+        conf[key] = obj
 def deal_key(it, conf):
     data_keys = xf.g(conf, data_keys=[])
     tmp = {}
@@ -78,7 +82,7 @@ def make(data, table=None, data_keys = "sql_key,sql_def,py_key".split(","), py_n
     elif type(data) == dict:
         data = dict2list(data)
     conf = {}
-    xf.s(conf, sql_before = [], sql_after = [], sql_delete_before=[], sql_delete_after=[],auto_translate=False,table=table,keys=[],py2sqls={},query_keys=None, data_keys = data_keys, vars=[], py_name = py_name)
+    xf.s(conf, sql_before = [], sql_after = [], sql_delete_before=[], sql_delete_after=[],auto_translate=False,table=table,keys=[],py2sqls={},query_keys=None, data_keys = data_keys, vars=[], py_name = py_name, index=[], unique=[])
     for it in data:
         if len(it)==1 and type(it[0]) in (list, tuple):
             deal_item(it[0], conf)
@@ -93,9 +97,22 @@ def make(data, table=None, data_keys = "sql_key,sql_def,py_key".split(","), py_n
     assert table is not None
     sql = f"create table if not exists {table}({vars})"
     sql_del = f"drop table if exists {table}"
+    indexes,uniques =xf.g(conf, index=[], unique=[])
+    i_u = len(indexes)
+    indexes+=uniques
+    idx_sqls =[]
+    for i in range(len(indexes)):
+        keys = indexes[i]
+        unique = i>=i_u
+        s_u = "unique" if unique else ""
+        if type(keys) not in (list,tuple):
+            keys = [keys]
+        index_name = f"idx_{table}_idx_{'_'.join(keys)}"
+        idx_sql = f"create {s_u} index if not exists {index_name} on {table}({','.join(keys)});"
+        idx_sqls.append(idx_sql)
     sqls = xf.g(conf, sql_before=[], sql_after=[], sql_delete_before=[], sql_delete_after=[])
     #sqls = [";".join(k) for k in sqls]
-    sqls_crt = sqls[0]+[sql]+sqls[1]
+    sqls_crt = sqls[0]+[sql]+idx_sqls+sqls[1]
     sqls_del = sqls[2]+[sql_del]+sqls[3]
     sqls = [sqls_crt, sqls_del]
     sqls = [(";\n".join(k)+";").replace(";;",";") for k in sqls]
@@ -202,6 +219,34 @@ class TableObject(Base):
     def query_all(self, dv=None, sql2py=False):
         sql = f"select * from {self.table};"
         return self.query(sql, dv, sql2py)
+    def sql_where(self, maps, py2sql=True):
+        rst = []
+        for key,val in maps:
+            if py2sql:
+                key = self.py2sqls[key]
+            opt = "="
+            if type(val) in (list, tuple):
+                opt = val[0]
+                val = val[1]
+            if type(val)==str:
+                val = f"'{val}'"
+            if type(val) in (list, tuple):
+                val = [k if type(k)!=str else f"'{k}'" for k in val]
+                val = f"({','.join(val)})"
+            rst.append(f"{key}{opt}{val}")
+        return " and ".join(rst)
+    def find(self, **maps):
+        rst = self.find_all(**maps)
+        if len(rst)==0:
+            return None
+        return rst[0]
+    def find_all(self, **maps):
+        sql_where = self.sql_where(maps)
+        sql = f"find * from {self.table}"
+        if sql_where!="":
+            sql += f"where {sql_where}"
+        sql+=";"
+        return self.query(sql)
     def execute(self, sql, dv=None):
         dv=self.rdv(dv)
         return dv.execute(sql)

buildz-0.9.10/buildz/dz.py

@@ -0,0 +1,5 @@
+
+from ._dz.mapz import *
+from ._dz.omapz import Mapz
+from ._dz.conf import Conf, BindKey
+from ._dz.oconf import ObjConf, ObjTypeConf

{buildz-0.9.8 → buildz-0.9.10}/buildz/fz/__init__.py

@@ -4,5 +4,6 @@ from .dirz import dirs, FileDeal
 from .lsf import lists, search, searchd,searchs,_search
 from .fio import fread, freads, fwrite, fwrites, read,reads,write,writes, makefdir, dirpath,makedir,removes, cover, fcover, sread, swrite, is_abs, xors, fxors, coverx, covers, fcovers
 from .fhs import fhash, fhashs
+from .csv import to_csv, from_csv
 
 __author__ = "Zzz, emails: 1174534295@qq.com, 1309458652@qq.com"

buildz-0.9.10/buildz/fz/csv.py

@@ -0,0 +1,36 @@
+#
+from . import fio
+def to_csv(keys, cols, fp):
+    assert len(keys)==len(cols)
+    fio.makefdir(fp)
+    keys = ", ".join(keys)
+    rst = [keys]
+    for tmp in zip(*cols):
+        tmp = [str(k) for k in tmp]
+        tmp = ", ".join(tmp)
+        rst.append(tmp)
+    rs = "\n".join(rst)
+    fio.write(rs.encode("utf-8"), fp)
+
+pass
+
+def from_csv(fp, col=True, as_map = False):
+    # 好像还没测试
+    s = fio.read(fp).decode("utf-8")
+    arr = s.split("\n")
+    arr = [[v.strip() for v in k.split(",")] for k in arr if k.strip()!=""]
+    keys = arr[0]
+    datas = arr[1:]
+    if col:
+        datas = []
+        for i in range(len(keys)):
+            tmp = [k[i] for k in datas]
+            datas.append(tmp)
+        if as_map:
+            datas = {k:arr for k,arr in zip(keys, datas)}
+    elif as_map:
+        datas = [{k:v for k,v in zip(keys, dt)} for dt in datas]
+    return keys, datas
+
+pass
+

{buildz-0.9.8 → buildz-0.9.10}/buildz/gpuz/test/dm1.py

@@ -44,6 +44,7 @@ cache = MiddleCache(models, 3)#, cal_dv = "cpu", cal_nets = cuda_models, cache_n
 # 训练:
 def train():
     cache.train()
+    _loss = 0
     for inputs,targets in dataloader:
         targets = targets.cuda()
         #inputs,targets = inputs.cuda(),targets.cuda()
@@ -55,17 +56,19 @@ def train():
             loss.backward()
         torch.nn.utils.clip_grad_norm_(real_model.parameters(), max_norm=1.0)
         opt.step()
+        _loss+=loss.cpu().item()
         #cache.do_backward(lambda: loss.backward())
         # opt.step()在do_backward里会自动调用
         # print(loss.item())
         # break
         #break
+    return _loss / (len(dataloader))
 pass
 import time
 print("start")
 start = time.time()
 for i in range(50):
-    train()
+    print(train())
 sec = time.time()-start
 print("time cost:", sec)
 

{buildz-0.9.8 → buildz-0.9.10}/buildz/gpuz/torch/__init__.py

@@ -11,3 +11,12 @@ from .middle_cache import MiddleCache
 from .dict_middle import DictCache, Fcs
 #from .dict_middle import *
 Dict = DictCache
+
+'''
+20250917
+写好的是MiddleCache
+暂时别用DictCache，没测过
+
+20251016
+测试发现，数据在内存和显存间传输消耗太多，显存不足用此代码未必能提升多少性能，还不如加显存，或者把模型改小，不建议用
+'''

{buildz-0.9.8 → buildz-0.9.10}/buildz/gpuz/torch/middle_base.py

@@ -2,7 +2,18 @@
 '''
 model.register_forward_pre_hook 会在模型forward之前调用
 model.register_forward_hook 会在模型forward之后调用
+
+-------------
+之前写的不准确的：
 model.register_full_backward_hook 会在模型反向梯度计算之后调用，注，模型列表第一个模型会在计算梯度前调用，该hook参数是(model, grad_ins, grad_outs),model是当前模型，grad_outs是后面的模型传来的，grad_ins是grad_outs经过该model计算后往前传的梯度，好像是第一个模型因为不用往前传梯度，提前调用了
+
+修改后：
+model.register_full_backward_hook 会在模型反向梯度计算之后调用，注，当前模型如果不需要再往前传送梯度，会在反向梯度计算前调用该方法，该hook参数是(model, grad_ins, grad_outs),model是当前模型，grad_outs是后面的模型传来的，grad_ins是grad_outs经过该model计算后往前传的梯度
+
+导致的后果是调用该方法的时候，计算数据还没存到显卡，需要延后调用
+-------------
+
+
 torch.autograd.graph.saved_tensors_hooks(hook_pack, hook_unpack):
     hook_pack会在Tensor计算完成后把之后反向梯度计算要用的tensor进行存储
     hook_unpack是在Tensor反向梯度计算前取回之前存储的tensor，注：调用loss.backward不用在with ...saved_tensors_hooks(..)中
@@ -37,16 +48,17 @@ class MiddleBase(Base):
     # 下面是不应修改的部分
     def hook(self, nets):
         net = nets[0]
-        self.start_id = id(net)
-        self.start_net = net
+        # self.start_id = id(net)
+        # self.start_net = net
         hooks = []
         hooks += [net.register_forward_pre_hook(self.hook_forward_before) for net in nets]
         hooks+=[net.register_forward_hook(self.native_hook_forward_after) for net in nets]
-        if hasattr(self.start_net, "register_full_backward_hook"):
+        if hasattr(nets[0], "register_full_backward_hook"):
             hooks+=[net.register_full_backward_hook(self.native_hook_backward_after) for net in nets]
         else:
             hooks+=[net.register_backward_hook(self.native_hook_backward_after) for net in nets]
         self.hooks = hooks
+        self.wait_backup_models = []
     def unhook(self):
         [hook.remove() for hook in self.hooks]
     def do_forward(self, fc):
@@ -55,11 +67,17 @@ class MiddleBase(Base):
     def do_backward(self, fc):
         with self.wrap_backward():
             return fc()
+    def backupable(self, model):
+        return True
     def wrap_backward(self):
         def wrap_enter():
+            self.wait_backup_models = []
             self.before_backward()
         def wrap_out(exc_type, exc_val, exc_tb):
-            self.hook_backward_after(self.start_net)
+            #self.hook_backward_after(self.start_net)
+            #print(f"BACK OUT MODELS:", self.wait_backup_models)
+            for model in self.wait_backup_models:
+                self.hook_backward_after(model)
             self.after_backward()
         return pyz.With(wrap_enter, wrap_out, True)
     def wrap_forward(self):
@@ -72,7 +90,10 @@ class MiddleBase(Base):
             self.after_forward()
         return pyz.With(wrap_enter, wrap_out, True)
     def native_hook_backward_after(self, model, grad_up, grad_src):
-        if id(model)==self.start_id:
+        # if id(model)==self.start_id:
+        #     return
+        if not self.backupable(model):
+            self.wait_backup_models.append(model)
             return
         self.hook_backward_after(model)
     def native_hook_forward_after(self, model, ins, outs):

{buildz-0.9.8 → buildz-0.9.10}/buildz/gpuz/torch/middle_cache.py

@@ -2,6 +2,9 @@
 
 from .middle_base import MiddleBase
 import torch
+from buildz import logz
+log = logz.simple("middle_cache.log")("GPUZ_CACHE")
+log.show("debug", 0)
 class MiddleCache(MiddleBase):
     '''
         nets: 模型列表
@@ -43,6 +46,7 @@ class MiddleCache(MiddleBase):
         self.nexts = {}
         self.prevs = {}
         self.curr = -1
+        #self.caches = set()
         self.mark_static = None
         self.win_size = win_size
         self.preload = preload
@@ -57,28 +61,56 @@ class MiddleCache(MiddleBase):
         # assert datas is not None, "inputs datas is null"
         inputs = tuple([k.to(dv) for k in datas])
         return inputs
+    @staticmethod
+    def deep_inputs_to(datas, dv):
+        '''
+            数据比较复杂用这个，会递归搜索list，tuple和dict病做转换
+        '''
+        log.debug(f"deep_inputs_to")
+        if isinstance(datas, torch.Tensor):
+            return datas.to(dv)
+        if type(datas) in (list, tuple):
+            rst = [MiddleCache.deep_inputs_to(k, dv) for k in datas]
+            if type(datas)==tuple:
+                rst = tuple(rst)
+            return rst
+        if type(datas) == dict:
+            rst = {k:MiddleCache.deep_inputs_to(v, dv) for k,v in datas.items()}
+            return rst
+        assert 0, f"undealable datas type: {type(datas)}"
     def before_forward(self):
+        log.debug(f"before_forward")
         self.curr = -1
+        #self.caches = set()
         for mid in self.datas:
             self.datas[mid] = []
     def after_forward(self):
         pass
     def before_backward(self):
+        log.debug(f"before_backward")
         self.curr = -1
+    def backupable(self, model):
+        mid = id(model)
+        return mid in self.cal_ids or mid in self.static_dv
+        #return mid not in self.caches
     def after_backward(self):
         pass
     def to_dv(self, mid, dv):
         self.nets[mid].to(dv)
         self.datas[mid] = [self.inputs_to(k, dv) if k is not None else k for k in self.datas[mid]]
     def to_cal(self, mid, pop = 0):
+        log.debug(f"to_cal")
         if mid in self.cal_ids:
             return 0
         if len(self.cal_ids)>=self.win_size:
             xid = self.cal_ids.pop(pop)
             #print(f"model_{self.indexes[xid]} to CPU device")
             self.to_dv(xid, self.cache_dv)
+            #self.caches.add(xid)
         #print(f"model_{self.indexes[mid]} to CUDA device")
         self.to_dv(mid, self.cal_dv)
+        # if mid in self.caches:
+        #     self.caches.remove(mid)
         if pop==-1:
             self.cal_ids = [mid]+self.cal_ids
         else:
@@ -86,6 +118,7 @@ class MiddleCache(MiddleBase):
         return 1
     def hook_forward_before(self, model, ins):
         mid = id(model)
+        log.debug(f"hook_forward_before", model, id(model))
         self.mark_static = mid in self.static_dv
         if self.mark_static:
             # assert ins is not None, "hook_forward_before A"
@@ -100,8 +133,12 @@ class MiddleCache(MiddleBase):
         # assert ins is not None, "hook_forward_before B"
         return self.inputs_to(ins, self.cal_dv)
     def hook_forward_after(self, model):
+        log.debug(f"hook_forward_after", model, id(model))
         pass
     def hook_backward_after(self, model):
+        log.debug(f"hook_backward_after", model, id(model))
+        #print(f"hook_backward_after", model, id(model))
+        assert self.backupable(model)
         mid = id(model)
         if self.deal_after_backup is not None:
             self.deal_after_backup(model)
@@ -112,12 +149,15 @@ class MiddleCache(MiddleBase):
             self.prevs[after] = mid
         self.curr = mid
     def tensor_save(self, tuple_data):
+        log.debug(f"tensor_save")
         if self.mark_static:
             return -1, tuple_data
         index = len(self.datas[self.curr])
         self.datas[self.curr].append(tuple_data)
+        log.debug(f"done tensor_save", self.curr, index)
         return self.curr, index
     def tensor_load(self, obj):
+        log.debug(f"tensor_load", obj)
         if obj[0]<0:
             return obj[1]
         curr, index = obj
@@ -127,6 +167,7 @@ class MiddleCache(MiddleBase):
         self.datas[curr][index] = None
         return tuple_data
     def loop_to_cal(self, mid, pop, nexts):
+        log.debug(f"loop_to_cal")
         if mid in self.cal_ids:
             return
         self.to_cal(mid, pop)

{buildz-0.9.8 → buildz-0.9.10}/buildz/logz.py

@@ -10,16 +10,18 @@ ns = wrap.ns("buildz.logz")
 @ns.obj(id="baseLog")
 @ns.obj_args("ref, buildz.logz.shows, null", "ref, buildz.logz.tag, null", "ref, buildz.logz.base, null")
 class Log(Base):
-    def show(self, type):
+    def show(self, type, on=True):
+        if not on:
+            return self.unshow(type)
         if type not in self.shows:
-            self.show.append(type)
+            self.shows.append(type)
     def unshow(self, type):
         if type in self.shows:
             self.shows.remove(type)
     def call(self, _tag):
         return self.tag(_tag)
     def tag(self, _tag):
-        if type(_tag) != str:
+        if _tag is not None and type(_tag) != str:
             try:
                 if not hasattr(_tag, "__name__"):
                     _tag = _tag.__class__
@@ -56,22 +58,28 @@ class Log(Base):
         if self.base is not None:
             return self.base.log(level, tag, *args)
         raise Exception("unimpl")
-    def info(self, *args):
-        if "info" not in self.shows:
-            return
-        self.log("info", self._tag, *args)
-    def warn(self, *args):
-        if "warn" not in self.shows:
-            return
-        self.log("warn", self._tag, *args)
-    def debug(self, *args):
-        if "debug" not in self.shows:
-            return
-        self.log("debug", self._tag, *args)
-    def error(self, *args):
-        if "error" not in self.shows:
-            return
-        self.log("error", self._tag, *args)
+    def __getattr__(self, key):
+        def fc(*args):
+            if key not in self.shows:
+                return
+            self.log(key, self._tag, *args)
+        return fc
+    # def info(self, *args):
+    #     if "info" not in self.shows:
+    #         return
+    #     self.log("info", self._tag, *args)
+    # def warn(self, *args):
+    #     if "warn" not in self.shows:
+    #         return
+    #     self.log("warn", self._tag, *args)
+    # def debug(self, *args):
+    #     if "debug" not in self.shows:
+    #         return
+    #     self.log("debug", self._tag, *args)
+    # def error(self, *args):
+    #     if "error" not in self.shows:
+    #         return
+    #     self.log("error", self._tag, *args)
     def clean(self):
         pass