buildz 0.9.5__tar.gz → 0.9.7__tar.gz

{buildz-0.9.5/buildz.egg-info → buildz-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.9.5
+Version: 0.9.7
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.9.5 → buildz-0.9.7}/buildz/_dz/conf.py

@@ -52,11 +52,12 @@ class Conf(Base):
                 val,find = obj.hget(domain,link=link)
                 loop-=1
         return Conf(self.spt, self.spts, domain, obj)
-    def init(self, spt='.', spts=',', domain=None, root = None):
+    def init(self, spt='.', spts=',', domain=None, root = None, src = None):
         self.spt = spt
         self.spts = spts
         self.domain = domain
         self.root = root
+        self.src = src
         if root is None:
             self.conf = {}
             self.history = {}
@@ -231,6 +232,10 @@ class Conf(Base):
                 bak = a,b
             loop-=1
         return bak
+    def src_hget(self, key, default, link=-1):
+        if self.src is None:
+            return default, 0
+        return self.src.hget(key, default, loop, link)
     def _lget(self, key, default=None, loop=-1, link=-1):
         return self._lhget(key, default, loop, link)[0]
     def _hget(self, key, default=None, link=-1):
@@ -238,10 +243,13 @@ class Conf(Base):
         keys = dzkeys(key, self.spt)
         val, find = mapz.dget(self.conf, keys, default)
         if find or link==0:
+            if not find:
+                val, find = self.src_hget(key, default, link)
             return val, find
         lnk, has_lnk, deep = self.link_match(keys)
         if not has_lnk:
-            return val, find
+            return self.src_hget(key, default, link)
+            #return val, find
         keys = keys[deep:]
         key = self.spt.join(keys)
         if lnk is not None:
@@ -257,7 +265,7 @@ class Conf(Base):
         # TODO
         keys = dzkeys(key, self.spt)
         return mapz.dremove(self.conf, keys)
-    def _has(self, key, link=0):
+    def _has(self, key, link=-1):
         return self._hget(key, None, link)[1]
         keys = dzkeys(key, self.spt)
         return mapz.dhas(self.conf, keys)
@@ -272,7 +280,7 @@ class Conf(Base):
         return rst
     def s(self, **maps):
         [self.set(k,v) for k,v in maps.items()]
-    def has_all(self, keys, link = 0):
+    def has_all(self, keys, link = -1):
         keys = self.spts_ks(keys)
         rst = [1-self.has(key, link) for key in keys]
         return sum(rst)==0

{buildz-0.9.5 → buildz-0.9.7}/buildz/confz.py

@@ -19,6 +19,8 @@ def load_conf(conf, dp=None, dp_key = 'dp', src_key = 'src.conf'):
         rst.update(base, flush, replace, visit_list)
     for fp in fps:
         tmp = xf.loadf(path.dp(fp))
+        if type(tmp)!=dict:
+            continue
         rst.update(tmp, flush, replace, visit_list)
     if not conf_first:
         rst.update(base, flush, replace, visit_list)

{buildz-0.9.5 → buildz-0.9.7}/buildz/gpuz/test/dm.py

@@ -13,15 +13,15 @@ class TestDataset(Dataset):
         return self.num
     def __getitem__(self, i):
         return self.datas[i], self.targets[i]
-dims = 3
+dims = 64
 """
 cd D:\rootz\python\gits\buildz_upd\buildz\gpuz\test
 D:
 python -m buildz.gpuz.test.demo > demo.txt
 
 """
-dataset = TestDataset(30, dims)
-dataloader = DataLoader(dataset, 10)
+dataset = TestDataset(90, dims)
+dataloader = DataLoader(dataset, 30)
 class MiniModel(nn.Module):
     def __init__(self, dims, mdims, num):
         super().__init__()
@@ -31,7 +31,7 @@ class MiniModel(nn.Module):
         self.nets = nn.Sequential(*nets)
     def forward(self, inputs):
         return self.nets(inputs)
-models = [MiniModel(dims, 6, 3) for i in range(6)]
+models = [MiniModel(dims, 32, 3) for i in range(6)]
 opts = [optim.Adam(model.parameters(), lr=0.001) for model in models]
 #可以指定哪些模型全部放cuda或者全部放cpu
 cuda_models = [models[1],models[2]]
@@ -56,10 +56,16 @@ def train():
         loss = loss_fn(outs, targets)
         cache.do_backward(lambda: loss.backward())
         # opt.step()在do_backward里会自动调用
-        print(loss.item())
+        #print(loss.item())
         #break
 pass
-
+import time
+print("start")
+start = time.time()
+for i in range(50):
+    train()
+sec = time.time()-start
+print("time cost:", sec)
 """
 from buildz.gpuz.test import dm
 

{buildz-0.9.5 → buildz-0.9.7}/buildz/gpuz/test/dm1.py

@@ -39,12 +39,13 @@ opt = optim.Adam(real_model.parameters(), lr=0.001)
 cuda_models = [models[1],models[2]]
 cpu_models = [models[-1]]
 loss_fn = torch.nn.MSELoss()
-cache = MiddleCache(models, 2, cal_dv = "cpu", cal_nets = cuda_models, cache_nets = cpu_models)#, [torch.device('cuda'), torch.device('cpu')],models,opts,3,opt_step)#, [cuda_models,cpu_models])
+cache = MiddleCache(models, 3)#, cal_dv = "cpu", cal_nets = cuda_models, cache_nets = cpu_models)#, [torch.device('cuda'), torch.device('cpu')],models,opts,3,opt_step)#, [cuda_models,cpu_models])
 
 # 训练:
 def train():
     cache.train()
     for inputs,targets in dataloader:
+        targets = targets.cuda()
         #inputs,targets = inputs.cuda(),targets.cuda()
         opt.zero_grad()
         with cache.wrap_forward():
@@ -56,13 +57,17 @@ def train():
         opt.step()
         #cache.do_backward(lambda: loss.backward())
         # opt.step()在do_backward里会自动调用
-        print(loss.item())
-        break
+        # print(loss.item())
+        # break
         #break
 pass
-for i in range(5):
+import time
+print("start")
+start = time.time()
+for i in range(50):
     train()
-pass
+sec = time.time()-start
+print("time cost:", sec)
 
 """
 python -m buildz.gpuz.test.dm1

{buildz-0.9.5 → buildz-0.9.7}/buildz/gpuz/torch/dict_middle.py

@@ -227,7 +227,7 @@ class DictCache(Base):
     def hook_forward_pre(self, model, ins):
         self.no_caches = False
         nid = id(model)
-        print(f"hook_forward_pre {nid}")
+        #print(f"hook_forward_pre {nid}")
         if nid not in self.pools:
             self.copy_forward(nid, model)
         if self.curr>=0:
@@ -235,12 +235,12 @@ class DictCache(Base):
         self.curr = nid
         return self.fc_inputs_to(ins, self.gdv)
     def do_forward(self, fc):
-        print("FWD")
+        #print("FWD")
         self.no_caches = True
         self.ctxs = {k:[] for k in self.nets}
         with torch.autograd.graph.saved_tensors_hooks(self.hook_pack, self.hook_unpack):
             rst = fc()
-        print("DFWD")
+        #print("DFWD")
         return rst.to(self.gdv)
     def wrap_backward_deal(self, net_id):
         if self.backward_deal is None:
@@ -248,10 +248,10 @@ class DictCache(Base):
         self.backward_deal(self.nets[net_id], self.opts[net_id])
     def hook_backward(self, model, grad_ins, grad_outs):
         nid = id(model)
-        print(f"hook_backward {nid}")
+        #print(f"hook_backward {nid}")
         self.wrap_backward_deal(nid)
     def hook_pack(self, dt):
-        print(f"pack")
+        #print(f"pack")
         if self.no_caches:
             # 不做缓存，数据不处理
             return -1, dt
@@ -261,7 +261,7 @@ class DictCache(Base):
         return index, len(self.ctxs[index])-1
     def hook_unpack(self, x):
         nid = x[0]
-        print(f"unpack {nid}")
+        #print(f"unpack {nid}")
         if nid<0:
             return x[1]
         if nid not in self.pools:

{buildz-0.9.5 → buildz-0.9.7}/buildz/gpuz/torch/middle_base.py

@@ -50,27 +50,27 @@ class MiddleBase(Base):
     def unhook(self):
         [hook.remove() for hook in self.hooks]
     def do_forward(self, fc):
-        with self.warp_forward():
+        with self.wrap_forward():
             return fc()
     def do_backward(self, fc):
-        with self.warp_backward():
+        with self.wrap_backward():
             return fc()
     def wrap_backward(self):
-        def warp_enter():
+        def wrap_enter():
             self.before_backward()
-        def warp_out(exc_type, exc_val, exc_tb):
+        def wrap_out(exc_type, exc_val, exc_tb):
             self.hook_backward_after(self.start_net)
             self.after_backward()
-        return pyz.With(warp_enter, warp_out, True)
+        return pyz.With(wrap_enter, wrap_out, True)
     def wrap_forward(self):
         obj = torch.autograd.graph.saved_tensors_hooks(self.tensor_save, self.tensor_load)
-        def warp_enter():
+        def wrap_enter():
             self.before_forward()
             obj.__enter__()
-        def warp_out(exc_type, exc_val, exc_tb):
+        def wrap_out(exc_type, exc_val, exc_tb):
             obj.__exit__(exc_type, exc_val, exc_tb)
             self.after_forward()
-        return pyz.With(warp_enter, warp_out, True)
+        return pyz.With(wrap_enter, wrap_out, True)
     def native_hook_backward_after(self, model, grad_up, grad_src):
         if id(model)==self.start_id:
             return

buildz-0.9.7/buildz/path.py

@@ -0,0 +1,79 @@
+from . import Base
+import os,sys,json,time
+class Key:
+    first='first'
+    ifirst = 0
+    last= 'last'
+    ilast = -1
+    @staticmethod
+    def check_first(val):
+        return val in (Key.first, Key.ifirst)
+    @staticmethod
+    def check_last(val):
+        return val in (Key.last, Key.ilast)
+class Path(Base):
+    Key = Key
+    def init(self, **maps):
+        self.paths = {}
+        self.fcs = {}
+        self.lasts = {}
+        k = "check_abs"
+        self.check_abs = True
+        if k in maps:
+            self.check_abs = maps[k]
+            del maps[k]
+        for k, v in maps.items():
+            self.set(k, v)
+    @staticmethod
+    def dir(fp):
+        return os.path.dirname(fp)
+    @staticmethod
+    def join(*a):
+        return os.path.join(*a)
+    @staticmethod
+    def rjoin(path, *a):
+        a = [k.strip() for k in a if k is not None and k.strip()!=""]
+        if len(a)==0:
+            return None
+        if path is None:
+            return os.path.join(*a)
+        return os.path.join(path, *a)
+    @staticmethod
+    def rfp(paths, *a, last=-1, check_abs=False):
+        a = [k.strip() for k in a if k is not None and k.strip()!=""]
+        if len(a)==0:
+            return None
+        if check_abs and len(a)>0:
+            f = a[0]
+            if f[:1]=="/" or f.find(":")>0:
+                return Path.join(*a)
+        #print(f"[TESTZ] a: {a}, paths: {paths}")
+        #fp = os.path.join(*a)
+        for path in paths:
+            _fp = Path.rjoin(path, *a)
+            if os.path.exists(_fp):
+                return _fp
+        return Path.rjoin(paths[last], *a)
+    def add(self, path, name=None, index=0):
+        self.paths[name].insert(index, path)
+    def set(self, paths, name=None, last = -1, curr = None):
+        if type(paths) not in (list, tuple):
+            paths = [paths]
+        if curr is not None and None not in paths:
+            if Key.check_first(curr):
+                paths = [None]+paths
+            elif Key.check_last(curr):
+                paths.append(None)
+            else:
+                assert False
+        self.paths[name] = paths
+        self.lasts[name] = last
+        def fc(*a):
+            return self.rfp(paths, *a, last=last, check_abs=self.check_abs)
+        self.fcs[name] = fc
+    def __getattr__(self, name):
+        return self.fcs[name]
+    def call(self, *obj):
+        return self.fcs[None](*obj)
+
+pass

{buildz-0.9.5 → buildz-0.9.7/buildz.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.9.5
+Version: 0.9.7
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.9.5 → buildz-0.9.7}/buildz.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ buildz/cmd.py
 buildz/confz.py
 buildz/dz.py
 buildz/logz.py
+buildz/path.py
 buildz/pathz.py
 buildz/pyz.py
 buildz/tls.py

{buildz-0.9.5 → buildz-0.9.7}/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup, find_packages
 
 setup(
     name = 'buildz',
-    version = '0.9.5',
+    version = '0.9.7',
     keywords='buildz',
     long_description=open('README.md', 'r', encoding="utf-8").read(),
     long_description_content_type='text/markdown',