buildz 0.9.3__tar.gz → 0.9.5__tar.gz

{buildz-0.9.3/buildz.egg-info → buildz-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.9.3
+Version: 0.9.5
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.9.3 → buildz-0.9.5}/buildz/_dz/conf.py

@@ -15,11 +15,18 @@ def dzkeys(key, spt):
 class Conf(Base):
     def val(self):
         return self.get_conf()
-    def top(self, domain = None):
+    def get_type(self):
+        return type(self.val())
+    def has_val(self):
+        obj = self.root or self
+        return obj.has(self.domain)
+    def top(self, domain = None, loop=0, link=0):
         root = self.root or self
         if domain is not None:
-            root = root(domain)
+            root = root(domain, loop, link)
         return root
+    def ltop(self, domain=None, loop=-1,link=0):
+        return self.top(domain,loop,link)
         # return self.root or self
     def get_conf(self):
         if self.domain:
@@ -30,12 +37,20 @@ class Conf(Base):
         return obj.conf
     def str(self):
         return str(self.get_conf())
-    def call(self, domain=None):
+    def l(self, domain=None, loop=-1,link=0):
+        return self(domain,loop,link)
+    def call(self, domain=None, loop=0, link=0):
         if domain is None:
             return self.top()
         if self.domain:
             domain = self.domain+self.spt+domain
         obj = self.root or self
+        if loop!=0:
+            val, find = obj.hget(domain,link=link)
+            while loop!=0 and find and type(val)==str:
+                domain = val
+                val,find = obj.hget(domain,link=link)
+                loop-=1
         return Conf(self.spt, self.spts, domain, obj)
     def init(self, spt='.', spts=',', domain=None, root = None):
         self.spt = spt
@@ -44,10 +59,12 @@ class Conf(Base):
         self.root = root
         if root is None:
             self.conf = {}
-            self.stacks = {}
-            self._links = [{},None]
+            self.history = {}
+            self._links = [{},None,0]
         self.dr_bind('_get', 'get')
         self.dr_bind('_hget', 'hget')
+        self.dr_bind('_lget', 'lget')
+        self.dr_bind('_lhget', 'lhget')
         self.dr_bind('_set', 'set')
         self.dr_bind('_has', 'has')
         self.dr_bind('_remove', 'remove')
@@ -58,10 +75,11 @@ class Conf(Base):
         self.dr_bind('_stack_set', 'stack_set')
         self.dr_bind('_stack_unset', 'stack_unset')
         self.fcs_bind('get', 'gets', False, True)
+        self.fcs_bind('lget', 'lgets', False, True)
         self.fcs_bind('set', 'sets', True)
         self.fcs_bind('remove', 'removes')
         self.fcs_bind('push', 'pushs', True)
-        self.fcs_bind('pop', 'pops')
+        #self.fcs_bind('pop', 'pops')
         self.fcs_bind('stack_set', 'stack_sets', True)
         self.fcs_bind('stack_unset', 'stack_unsets')
         self.fcs_bind('link', 'links', True)
@@ -72,8 +90,8 @@ class Conf(Base):
     def clean(self):
         obj = self.root or self
         obj.conf = {}
-        obj.stacks = {}
-        obj._links = [{}, None]
+        obj.history = {}
+        obj._links = [{}, None, 0]
         return self
     def dkey(self, key):
         if self.domain:
@@ -116,84 +134,131 @@ class Conf(Base):
                 rst.append(val)
             return rst
         setattr(self, wfn, wfc)
-    def _stack_set(self, key, val):
-        if key not in self.stacks:
-            self.stacks[key] = []
-        self.stacks[key] = [val]
-    def _stack_unset(self, key, val):
-        if key not in self.stacks:
-            return False
-        del self.stacks[key]
-        return True
-    def _push(self, key, val):
-        if key not in self.stacks:
-            self.stacks[key] = []
-        self.stacks[key].append(val)
-    def _pop(self, key):
-        if key not in self.stacks:
+    def _stack_set(self, key, value, flush = 1, update=0):
+        return self._push(key, value, flush, update, 1)
+    def _stack_unset(self, key):
+        return self._pop(key, 1)
+    def _push(self, key, value, flush = 1, update=0, clean_history = 0):
+        keys = dzkeys(key, self.spt)
+        val, find = mapz.dget(self.conf, keys)
+        val = mapz.deep_clone(val)
+        if clean_history or key not in self.history:
+            self.history[key] = []
+        self.history[key].append([val, find, update])
+        if flush and type(value)==dict:
+            value = xf.flush_maps(value, lambda x:x.split(self.spt) if type(x)==str else [x], 0)
+        if type(value)==dict and update:
+            self(key).update(value,flush=0)
+        else:
+            self._set(key, value)
+    def _pop(self, key, clean_history = 0):
+        if key not in self.history:
             return False
-        stk = self.stacks[key]
-        if len(stk)==0:
+        lst = self.history[key]
+        if len(lst)==0:
             return False
-        stk.pop(-1)
+        rst = lst.pop(-1)
+        if clean_history:
+            self.history[key] = []
+        if not rst[1]:
+            self._remove(key)
+            return True
+        self._set(key, rst[0])
         return True
-    def _link(self, src, target):
-        keys = dzkeys(src, self.spt)
-        links = self._links
+    def pops(self, keys, *a, **b):
+        keys = self.spts_ks(keys)
+        keys.reverse()
         for key in keys:
-            if key not in links[0]:
-                links[0][key] = [{},None]
-            links = links[0][key]
-        links[1] = target
-    def _unlink(self, key):
+            self.pop(key)
+    def with_push(self, key, *a, **b):
+        self.push(key, *a, **b)
+        def out():
+            self.pop(key)
+        return pyz.with_out(out)
+    def with_pushs(self, keys, *a, **b):
+        self.pushs(keys, *a, **b)
+        def out():
+            self.pops(keys)
+        return pyz.with_out(out)
+    def ld_get(self, obj, key, fc_init=None):
         keys = dzkeys(key, self.spt)
-        links = self._links
         for key in keys:
-            if key not in links[0]:
-                return False
-            links = links[0][key]
-        links[1] = None
-        return True
-    def link_match(self, keys):
-        obj = self.root or self
-        links = obj._links
+            if key not in obj[0]:
+                if fc_init is None:
+                    return None
+                obj[0][key] = fc_init()
+            obj = obj[0][key]
+        return obj
+    def ld_visit(self, obj, key, fc):
+        keys = dzkeys(key, self.spt)
         deep = 0
+        fc(obj, deep, len(keys))
         for key in keys:
-            if key not in links[0]:
+            if key not in obj[0]:
                 break
             deep+=1
-            links = links[0][key]
-        return links[1], deep
+            obj = obj[0][key]
+            fc(obj, deep, len(keys))
+    def _link(self, src, target):
+        links = self.ld_get(self._links, src, lambda :[{}, None, 0])
+        links[1] = target
+        links[2] =1
+    def _unlink(self, key):
+        links = self.ld_get(self._links, key)
+        if links is None:
+            return False
+        links[1] = None
+        links[2] = 0
+        return True
+    def link_match(self, keys):
+        obj = self.root or self
+        rst = [[None,0,0]]
+        def fc_match(val, deep, size):
+            if not val[2]:
+                return
+            rst[0] = val[1], val[2], deep
+        self.ld_visit(obj._links, keys, fc_match)
+        return rst[0]
     def _set(self, key, val):
         keys = dzkeys(key, self.spt)
         mapz.dset(self.conf, keys, val)
-    def _hget(self, key, default=None, loop=-1):
-        #print(f"_hget: {key}, {default}, {loop}")
-        stk = mapz.get(self.stacks, key, [])
-        if len(stk)>0:
-            return stk[-1],1
+    def _lhget(self, key, default=None, loop=-1, link=-1):
+        a,b = self._hget(key, default, link)
+        bak = a, b
+        while b and type(a)==str and loop!=0:
+            a,b = self._hget(a, default, link)
+            if b:
+                bak = a,b
+            loop-=1
+        return bak
+    def _lget(self, key, default=None, loop=-1, link=-1):
+        return self._lhget(key, default, loop, link)[0]
+    def _hget(self, key, default=None, link=-1):
+        #print(f"_hget: {key}, {default}, {link}")
         keys = dzkeys(key, self.spt)
         val, find = mapz.dget(self.conf, keys, default)
-        if find or loop==0:
+        if find or link==0:
             return val, find
-        lnk, deep = self.link_match(keys)
-        if lnk is None:
+        lnk, has_lnk, deep = self.link_match(keys)
+        if not has_lnk:
             return val, find
         keys = keys[deep:]
-        keys = self.spt.join(keys)
-        key = lnk+self.spt+keys
-        if loop>0:
-            loop-=1
-        return self._hget(key, default, loop)
-    def _get(self, key, default=None, loop=-1):
-        return self._hget(key, default,loop)[0]
+        key = self.spt.join(keys)
+        if lnk is not None:
+            key = lnk+self.spt+key
+        if link>0:
+            link-=1
+        return self._hget(key, default, link)
+    def _get(self, key, default=None, link=-1):
+        return self._hget(key, default,link)[0]
         keys = dzkeys(key, self.spt)
         return mapz.dget(self.conf, keys, default)[0]
     def _remove(self, key):
+        # TODO
         keys = dzkeys(key, self.spt)
         return mapz.dremove(self.conf, keys)
-    def _has(self, key, loop=0):
-        return self._hget(key, None, loop)[1]
+    def _has(self, key, link=0):
+        return self._hget(key, None, link)[1]
         keys = dzkeys(key, self.spt)
         return mapz.dhas(self.conf, keys)
     def spts_ks(self, keys):
@@ -207,13 +272,13 @@ class Conf(Base):
         return rst
     def s(self, **maps):
         [self.set(k,v) for k,v in maps.items()]
-    def has_all(self, keys, loop = 0):
+    def has_all(self, keys, link = 0):
         keys = self.spts_ks(keys)
-        rst = [1-self.has(key, loop) for key in keys]
+        rst = [1-self.has(key, link) for key in keys]
         return sum(rst)==0
-    def has_any(self, keys, loop=0):
+    def has_any(self, keys, link=0):
         keys = self.spts_ks(keys)
         for key in keys:
-            if self.has(key, loop):
+            if self.has(key, link):
                 return True
         return False

buildz-0.9.5/buildz/confz.py

@@ -0,0 +1,166 @@
+from .dz import Conf
+from . import xf, pathz, pyz, argx, dz
+import sys, os
+def load_conf(conf, dp=None, dp_key = 'dp', src_key = 'src.conf'):
+    if type(conf)==dict:
+        conf = Conf().update(conf)
+    fps, base = conf.gets('fps, conf', [], {})
+    if type(fps)==str:
+        fps = [fps]
+    conf_first,replace,flush,visit_list = conf.gets('conf_first,replace,flush,visit_list',1, 1,1,0)
+    spt, spts = conf.gets('spt, spts','.',',')
+    dp = conf.get(dp_key, dp)
+    path = pathz.Path()
+    path.set("dp", dp)
+    rst = Conf(spt, spts)
+    if src_key is not None:
+        rst.set(src_key, conf)
+    if conf_first:
+        rst.update(base, flush, replace, visit_list)
+    for fp in fps:
+        tmp = xf.loadf(path.dp(fp))
+        rst.update(tmp, flush, replace, visit_list)
+    if not conf_first:
+        rst.update(base, flush, replace, visit_list)
+    return rst
+# using
+def calls(conf):
+    calls = conf.get("calls", [])
+    local = conf.get("local", False)
+    root = conf.top()
+    if type(calls)==dict:
+        target = dz.g(calls, target='run')
+        calls = dz.get(calls, target, [])
+        # if target in calls:
+        #     calls = dz.get(calls, target, [])
+        # else:
+        #     calls = root.get(target, [])
+    if type(calls)==dict:
+        dm, init, calls, init_cover = dz.g(calls, domain=None, init = {},calls=[], init_cover=False)
+        if len(init)>0:
+            init_conf = conf.top("confz.init")
+            if dm is not None:
+                init_conf = init_conf(dm)
+            init_conf.update(init, replace=init_cover)
+            root.update(init_conf.val())
+    if type(calls)==str:
+        calls = [calls]
+    obj = conf
+    if not local:
+        obj = root
+    for key in calls:
+        assert obj.has(key), f"not has key: '{key}'"
+        simple(obj.l(key))
+    return conf
+fn_key = "confz.fns"
+fn_cache_key = "confz.fn.caches"
+default_fn_key = "confz.fn.default"
+def fn2fc(conf, key = 'fn', default_fn = None, default_fc = None):
+    fn = conf.get(key)
+    conf = conf()
+    if fn is None:
+        if default_fc is not None:
+            return default_fc
+        fn = default_fn or conf.get(default_fn_key, 'calls')
+    fc = conf(fn_cache_key).get(fn)
+    if fc is not None:
+        return fc
+    path =conf(fn_key).get(fn)
+    if path is None:
+        return None
+    fc = pyz.load(path)
+    conf(fn_cache_key).set(fn,fc)
+    return fc
+def get_fc(conf, fc_key='fc', fn_key='fn', default_fn = None, default_fc = None):
+    fc = conf.get(fc_key)
+    if fc is None:
+        fc = fn2fc(conf, fn_key, default_fn, default_fc)
+    else:
+        fc = pyz.load(fc)
+    return fc
+def conf_update(conf):
+    if conf.get_type()==str:
+        conf = conf.ltop(conf.domain)
+    up = conf.get('up', link=0)
+    return conf
+def simple(conf):
+    if conf.get_type()==str:
+        conf = conf.ltop(conf.domain)
+    up = conf.get('up', link=0)
+    if up:
+        conf().link(conf.domain, up)
+    fc = get_fc(conf)
+    assert fc is not None, f"conf has not setted deal fc: {conf}"
+    return fc(conf)
+def get_sys_conf(conf = []):
+    if type(conf) == str:
+        conf = xf.loadf(conf)
+    if conf is None:
+        conf = []
+    fetch = argx.Fetch(*conf)
+    return fetch()
+def fc_set(conf):
+    """
+        fn: set
+        // domain: default=None
+        // replace: default=1
+        // flush: default=1
+        conf: {
+
+        }
+    """
+    maps,domain,flush,replace = conf.gets("conf, domain,flush,replace",None,None,1,1)
+    if maps is None:
+        return conf
+    conf.top(domain).update(maps, flush=flush, replace=replace)
+    return conf
+def judge_fc(conf):
+    fc = get_fc(conf, "judge_fc", "judge")
+    rst = fc(conf)
+    if rst:
+        if not conf.has("yes"):
+            return None
+        conf = conf.l("yes")
+    else:
+        if not conf.has("no"):
+            return None
+        conf = conf.l("no")
+    return simple(conf)
+def switch_fc(conf):
+    fc = get_fc(conf, "cal_fc", "cal", "get")
+    val = fc(conf)
+    deal_conf = conf("vals")(val)
+    deal = get_fc(conf, "deal_fc", "deal", "mset")
+    return deal(deal_conf)
+def init_fn(conf):
+    maps = {
+        "calls": calls,
+        "set": fc_set,
+        "judge": judge_fc,
+        "switch": switch_fc,
+        "has_set": lambda conf:conf.top().has(conf.get("key")),
+        "equal": lambda conf:conf.top().get(conf.get("key"))==conf.get("val"),
+        "get": lambda conf: conf.top().get(conf.get("key")),
+        "mset": lambda conf:conf.top().update(conf.val()) if conf.has_val() else conf
+    }
+    conf(fn_cache_key).update(maps, replace=False)
+def run(dp = None, fp = None, init_conf = {}):
+    if dp is None:
+        dp = os.path.dirname(__file__)
+    path = pathz.Path()
+    path.set('dp', dp)
+    conf = {}
+    if fp is not None:
+        conf = xf.loadf(path.dp(fp))
+    #sys_conf = get_sys_conf()
+    xf.fill(init_conf, conf, 1)
+    conf = Conf().update(conf)
+    init = conf.get(conf.get("key.init", "init"), {})
+    conf = load_conf(conf, dp).update(init)
+    conf.set("confz.init", init)
+    init_fn(conf)
+    return simple(conf)
+    #return conf
+def test():
+    run()
+pyz.lc(locals(), test)

{buildz-0.9.3 → buildz-0.9.5}/buildz/gpuz/test/demo.py

@@ -13,7 +13,13 @@ class TestDataset(Dataset):
         return self.num
     def __getitem__(self, i):
         return self.datas[i], self.targets[i]
-dims = 12
+dims = 3
+"""
+cd D:\rootz\python\gits\buildz_upd\buildz\gpuz\test
+D:
+python -m buildz.gpuz.test.demo > demo.txt
+
+"""
 dataset = TestDataset(30, dims)
 dataloader = DataLoader(dataset, 10)
 class MiniModel(nn.Module):
@@ -25,7 +31,7 @@ class MiniModel(nn.Module):
         self.nets = nn.Sequential(*nets)
     def forward(self, inputs):
         return self.nets(inputs)
-models = [MiniModel(dims, 32, 3) for i in range(10)]
+models = [MiniModel(dims, 6, 3) for i in range(6)]
 opts = [optim.Adam(model.parameters(), lr=0.001) for model in models]
 #可以指定哪些模型全部放cuda或者全部放cpu
 cuda_models = [models[1],models[2]]
@@ -38,7 +44,7 @@ def opt_step(net, opt):
     # 另外，opt不一定就是优化函数，可以是任何数据，其只取决于创建DictCache的时候传入的opts是什么
     torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
     opt.step()
-cache = DictCache([torch.device('cuda'), torch.device('cpu')],models,opts,3,opt_step, [cuda_models,cpu_models])
+cache = DictCache([torch.device('cuda'), torch.device('cpu')],models,opts,3,opt_step)#, [cuda_models,cpu_models])
 
 # 训练:
 [md.train() for md in models]
@@ -50,6 +56,9 @@ for inputs,targets in dataloader:
     cache.do_backward(lambda: loss.backward())
     # opt.step()在do_backward里会自动调用
     print(loss.item())
+    break
+
+exit()
 
 # 测试:
 inputs = torch.rand(1, dims).cuda()

buildz-0.9.5/buildz/gpuz/test/dm.py

@@ -0,0 +1,83 @@
+
+from buildz.gpuz.torch import DictCache
+import torch
+from torch import nn,optim
+from torch.utils.data import DataLoader, Dataset
+class TestDataset(Dataset):
+    def __init__(self, num, dims):
+        self.num = num
+        self.dims = dims
+        self.datas = torch.rand(num, dims)
+        self.targets = torch.rand(num, dims)
+    def __len__(self):
+        return self.num
+    def __getitem__(self, i):
+        return self.datas[i], self.targets[i]
+dims = 3
+"""
+cd D:\rootz\python\gits\buildz_upd\buildz\gpuz\test
+D:
+python -m buildz.gpuz.test.demo > demo.txt
+
+"""
+dataset = TestDataset(30, dims)
+dataloader = DataLoader(dataset, 10)
+class MiniModel(nn.Module):
+    def __init__(self, dims, mdims, num):
+        super().__init__()
+        nets = [nn.Linear(dims, mdims)]
+        nets += [nn.Linear(mdims,mdims) for i in range(num)]
+        nets.append(nn.Linear(mdims,dims))
+        self.nets = nn.Sequential(*nets)
+    def forward(self, inputs):
+        return self.nets(inputs)
+models = [MiniModel(dims, 6, 3) for i in range(6)]
+opts = [optim.Adam(model.parameters(), lr=0.001) for model in models]
+#可以指定哪些模型全部放cuda或者全部放cpu
+cuda_models = [models[1],models[2]]
+cpu_models = [models[-1]]
+real_model = nn.Sequential(*models)
+loss_fn = torch.nn.MSELoss()
+def opt_step(net, opt):
+    # 如果模型只是用来测试，不做训练，可以不传该函数，同时opts传入空就可以
+    # 对模型的一些其他优化，可以写可以不写，主要是调用opt.step()进行当前小模型的模型训练
+    # 另外，opt不一定就是优化函数，可以是任何数据，其只取决于创建DictCache的时候传入的opts是什么
+    torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
+    opt.step()
+cache = DictCache([torch.device('cuda'), torch.device('cpu')],models,opts,3,opt_step)#, [cuda_models,cpu_models])
+
+# 训练:
+def train():
+    [md.train() for md in models]
+    for inputs,targets in dataloader:
+        inputs,targets = inputs.cuda(),targets.cuda()
+        [opt.zero_grad() for opt in opts]
+        outs = cache.do_forward(lambda:real_model(inputs))
+        loss = loss_fn(outs, targets)
+        cache.do_backward(lambda: loss.backward())
+        # opt.step()在do_backward里会自动调用
+        print(loss.item())
+        #break
+pass
+
+"""
+from buildz.gpuz.test import dm
+
+ws = []
+for md in dm.models:
+    for net in md.nets:
+        ws.append(net.weight.cpu().detach().clone())
+
+pass
+
+dm.train()
+
+nws = []
+for md in dm.models:
+    for net in md.nets:
+        nws.append(net.weight.cpu().detach().clone())
+
+pass
+subs = [(a-b).sum() for a,b in zip(ws, nws)]
+
+"""