buildz 0.6.60__tar.gz → 0.7.0__tar.gz

{buildz-0.6.60/buildz.egg-info → buildz-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.6.60
+Version: 0.7.0
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.6.60 → buildz-0.7.0}/buildz/__init__.py

@@ -1,6 +1,6 @@
 #coding=utf-8
 
-__version__="0.6.60"
+__version__="0.7.0"
 
 # 小号多
 __author__ = "Zzz, emails: 1174534295@qq.com, 1309458652@qq.com"

buildz-0.7.0/buildz/gpuz/test/analyze.py

@@ -0,0 +1,129 @@
+#
+
+import sys
+from buildz.gpuz.torch import DictCache
+import torch,time
+from torch import nn,optim
+cpu = torch.device('cpu')
+cuda = cpu
+if torch.cuda.is_available():
+    cuda = torch.device('cuda')
+def sz(tensor):
+    return tensor.element_size()*tensor.nelement()
+def unit_sz(net):
+    rst = 0
+    for key in "weight,bias".split(","):
+        if hasattr(net, key):
+            rst+=sz(getattr(net,key))
+    return rst
+
+pass
+dvs = [cpu,cuda]
+def show_size(sz):
+    u = None
+    for unit in "B,KB,MB,GB".split(","):
+        u=unit
+        if sz<256:
+            break
+        sz = sz/1024
+    return sz, unit
+
+def default_fc_opt(net, opt):
+    torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
+    opt.step()
+def analyze(use_cache, use_cuda, mark_train, loop, fc_gen, dataloader, loss_fn, fc_opt, win_size):
+    mds, gmodel, opts, gopt = fc_gen()
+    dv = cuda
+    if not use_cuda:
+        use_cache = False
+    cache = None
+    if use_cache:
+        cache = DictCache([cuda, cpu], mds, opts, win_size, fc_opt)
+    if not use_cuda:
+        dv = cpu
+    gmodel = gmodel.to(dv)
+    s_val = "mean loss"
+    if mark_train:
+        if not use_cache:
+            gmodel.train()
+        else:
+            [md.train() for md in mds]
+    else:
+        s_val = "mean value"
+        if not use_cache:
+            gmodel.eval()
+        else:
+            [md.eval() for md in mds]
+    times = []
+    for i in range(loop):
+        total_loss = 0
+        curr=time.time()
+        if mark_train:
+            for dt in dataloader:
+                dt=dt.to(dv)
+                if not use_cache:
+                    gopt.zero_grad()
+                    out = gmodel(dt)
+                    loss = loss_fn(out, dt)
+                    loss.backward()
+                    fc_opt(gmodel, gopt)
+                else:
+                    [opt.zero_grad() for opt in opts] #写gopt.zero_grad()应该也可以，只是删掉之前计算的梯度
+                    out = cache.do_forward(lambda :gmodel(dt)) # 其实只是加了勾子函数，实际的计算还是模型计算
+                    loss = loss_fn(out, dt)
+                    cache.do_backward(lambda : loss.backward()) # 加勾子函数
+                total_loss+=loss.item()
+        else:
+            with torch.no_grad():
+                for dt in dataloader:
+                    dt=dt.to(dv)
+                    if not use_cache:
+                        out = gmodel(dt)
+                    else:
+                        out = cache.do_forward(lambda :gmodel(dt)) # 其实只是加了勾子函数，实际的计算还是模型
+                    total_loss+=out.mean().item()
+        sec = time.time()-curr
+        print(" train:", i, s_val+":", total_loss/len(dataloader), "time:", sec)
+        times.append(sec)
+    # 清理显存
+    del gmodel, gopt, mds, opts
+    torch.cuda.empty_cache()
+    return times
+        
+def analyzes(mark_train, loop, fc_gen, dataloader, loss_fn = None, fc_opt = None, win_size=3, modes = ['cuda', 'cache','cpu']):
+    '''
+        mds, gmodel, opts, gopt = fc_gen()
+    '''
+    if type(modes)==str:
+        modes = modes.split(",")
+    if loss_fn is None:
+        loss_fn = torch.nn.MSELoss()
+    if fc_opt is None:
+        fc_opt = default_fc_opt
+    if 'gpu' in modes:
+        modes = list(modes)+['cuda']
+    # 正常做法：只用显卡
+    if 'cuda' in modes:
+        print("No Used DictCache")
+        gtimes = analyze(False, True, mark_train, loop, fc_gen, dataloader, loss_fn, fc_opt, win_size)
+        gtimes = gtimes[1:] # 第一次计算因为加载cuda一些东西会比较慢，不计算在内
+        torch.cuda.empty_cache()
+    # 显卡不足的时候：模型只在计算的时候存入显卡显存
+    if 'cache' in modes:
+        print("Using DictCache:")
+        dc_times = analyze(True, True, mark_train, loop, fc_gen, dataloader, loss_fn, fc_opt, win_size)
+        dc_times = dc_times[1:]
+        torch.cuda.empty_cache()
+    # 不用显卡，只用CPU
+    if 'cpu' in modes:
+        print("Using CPU:")
+        cpu_times = analyze(False, False, mark_train, loop, fc_gen, dataloader, loss_fn, fc_opt, win_size)
+        cpu_times = cpu_times[1:]
+        torch.cuda.empty_cache()
+    print(f"\nAnalyze")
+    if 'cuda' in modes:
+        print(f"    mean time cost not used DictCache: {sum(gtimes)/len(gtimes)} sec")
+    if 'cache' in modes:
+        print(f"    mean time cost using DictCache: {sum(dc_times)/len(dc_times)} sec")
+    if 'cpu' in modes:
+        print(f"    mean time cost using CPU: {sum(cpu_times)/len(cpu_times)} sec")

buildz-0.7.0/buildz/gpuz/test/report.txt

@@ -0,0 +1,52 @@
+
+测试结果: 
+结论是：大概能达到纯显卡（75W，更高的没测）二分之一或三分之一的性能，显存不够的可以用用，主要是卷积层，卷积层cpu计算耗时很大，并且神经网络参数相对较少，线性层感觉cpu和显卡本来就差不太多
+
+环境: 
+    笔记本（游戏本）
+    显卡: RTX4060 8GB 75W残血版
+    CPU: i7-13700H
+    内存: 16GB
+
+1）线性层：
+训练模式：
+python -m buildz.gpuz.test.test_linear_demo train cuda,cache,cpu
+结果：
+data size: 0.457763671875 MB
+Model Size: 1.7890334129333496 GB
+Analyze
+    mean time cost not used DictCache: 4.274392604827881 sec
+    mean time cost using DictCache: 4.48450231552124 sec
+    mean time cost using CPU: 24.141416311264038 sec
+
+用不用DictCache耗时都是4秒左右，猜测是因为显存足够，显存里自己做了缓存
+先占掉4GB显存再测试：
+占用4GB显存（开另一个命令行窗口执行）
+python -m buildz.gpuz.test.take_gpu_mem 20
+时间测试（去掉cuda是因为显存不够，全部用cuda会直接报错显存不足，根本用不了）：
+python -m buildz.gpuz.test.test_linear_demo train cache,cpu
+Analyze
+    mean time cost using DictCache: 12.55063670873642 sec
+    mean time cost using CPU: 23.80371403694153 sec
+
+用DictCache要12.6秒左右，性能推测是全部用显存的1/3，但起码比CPU快
+
+测试模式：
+python -m buildz.gpuz.test.test_linear_demo eval cuda,cache,cpu
+Analyze
+    mean time cost not used DictCache: 0.022499561309814453 sec
+    mean time cost using DictCache: 1.3205116987228394 sec
+    mean time cost using CPU: 0.3452974557876587 sec
+
+测试模式DictCache比CPU还慢，没啥好说的，以测试结果为准，结论是测试模式线性层有显存用显存，没显存用CPU，尽量别用DictCache，猜测是因为测试模式下每个小模型的计算量很小，速度很快，这时候就显得显存和内存之间的传输很消耗时间
+
+2）卷积层：
+python -m buildz.gpuz.test.test_resnet_demo train cuda,cache,cpu
+结果：
+data size: 60.0 MB
+Model Size: 2.75390625 MB
+Analyze
+    mean time cost not used DictCache: 4.0497170554267035 sec
+    mean time cost using DictCache: 8.11503267288208 sec
+    mean time cost using CPU: 60.47528860304091 sec
+CPU模式下卷积一直很慢，另外别看模型参数很小，全放显存的时候8GB显卡直接占满了

buildz-0.7.0/buildz/gpuz/test/take_gpu_mem.py

@@ -0,0 +1,45 @@
+#
+import sys
+from buildz.gpuz.test.test_linear_demo import *
+def test():
+    '''
+        占显存用的，直接从test_linear_demo import代码过来完事,调用：
+        python -m buildz.gpuz.test.take_gpu_mem
+        或
+        python -m buildz.gpuz.test.take_gpu_mem num
+        num默认12，占用3GB显存
+    '''
+    nets=10
+    dims=2000
+    trains = 5
+    datas = 60
+    batch=30
+    lr=0.0001
+    win_size=3
+    num = 12
+    args = sys.argv[1:]
+    if len(args)>0:
+        num = int(args[0])
+    print(f"num: {num}")
+    def fc_gen():
+        mds = [Model(dims, nets) for i in range(num)]
+        mds_sz = [md.size() for md in mds]
+        sz, unit = analyze.show_size(sum(mds_sz))
+        print(f"Model Size: {sz} {unit}")
+        opts =[optim.Adam(md.parameters(), lr=lr) for md in mds]
+        gmodel = nn.Sequential(*mds)
+        gopt = optim.Adam(gmodel.parameters(), lr=lr)
+        return mds, gmodel, opts, gopt
+    mds,gmodel,opts,gop=fc_gen()
+    ds = TestDataset(datas, dims)
+    dl = DataLoader(ds, batch)
+    dt = list(dl)[0]
+    gmodel = gmodel.cuda()
+    with torch.no_grad():
+        dt=dt.cuda()
+        while True:
+            out = gmodel(dt)
+            print(out.mean())
+            time.sleep(1)
+
+pyz.lc(locals(),test)

buildz-0.7.0/buildz/gpuz/test/test_linear_demo.py

@@ -0,0 +1,86 @@
+#
+
+import sys
+from buildz.gpuz.torch import DictCache
+from buildz.gpuz.test import analyze
+from buildz import pyz
+import torch,time
+from torch import nn,optim
+from torch.utils.data import DataLoader, Dataset
+cpu,cuda = analyze.dvs
+class Model(nn.Module):
+    def __init__(self, dims, num):
+        super().__init__()
+        nets=[]
+        for i in range(num):
+            nets.append(nn.Linear(dims,dims))
+            nets.append(nn.LeakyReLU())
+        self.nets = nn.Sequential(*nets)
+    def forward(self, inputs):
+        return self.nets(inputs)
+    def size(self):
+        return sum([analyze.unit_sz(net) for net in self.nets])
+
+pass
+class TestDataset(Dataset):
+    def __init__(self, n, dims):
+        self.n = n
+        self.dims = dims
+        self.datas = torch.rand(n, dims)
+        sz = analyze.sz(self.datas)
+        sz, unit = analyze.show_size(sz)
+        print(f"data size: {sz} {unit}")
+    def __len__(self):
+        return self.n
+    def __getitem__(self, i):
+        return self.datas[i]
+
+pass
+
+def test():
+    '''
+        这些参数是不用DictCache的时候8GB显存刚好能跑
+        测试显存不够用的情况，要么用其他程序占显存，要么改参数，num改大一些显存就不够用了
+        也提供了个占用显存的简单代码:
+            占用3GB显存：
+                python -m buildz.gpuz.test.take_gpu_mem 12
+            占用4GB显存：
+                python -m buildz.gpuz.test.take_gpu_mem 20
+
+    '''
+    nets=10
+    dims=2000
+    loop = 5
+    datas = 60
+    batch=30
+    lr=0.0001
+    win_size=3
+    args = sys.argv[1:]
+    mark_train = True
+    if len(args)>0:
+        mark_train = args.pop(0).lower()=='train'
+    modes = 'cuda,cache,cpu'
+    if len(args)>0:
+        modes = args.pop(0)
+    num = 12
+    if len(args)>0:
+        num = int(args.pop(0))
+    print(f"num: {num}")
+    def fc_gen():
+        mds = [Model(dims, nets) for i in range(num)]
+        mds_sz = [md.size() for md in mds]
+        sz, unit = analyze.show_size(sum(mds_sz))
+        print(f"Model Size: {sz} {unit}")
+        opts =[optim.Adam(md.parameters(), lr=lr) for md in mds]
+        gmodel = nn.Sequential(*mds)
+        gopt = optim.Adam(gmodel.parameters(), lr=lr)
+        return mds, gmodel, opts, gopt
+    def fc_opt(net, opt):
+        torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
+        opt.step()
+    ds = TestDataset(datas, dims)
+    dl = DataLoader(ds, batch)
+    loss_fn = torch.nn.MSELoss()
+    analyze.analyzes(mark_train, loop, fc_gen, dl, loss_fn, fc_opt, win_size, modes)
+
+pyz.lc(locals(),test)

buildz-0.7.0/buildz/gpuz/test/test_resnet.py

@@ -0,0 +1,179 @@
+#
+
+
+'''
+测试用DictCache和不用DictCache的时候，反向梯度是否一样，代码比较杂，不建议看，测试结果是效果一致
+'''
+import sys
+from buildz.gpuz.torch import DictCache
+import torch,time
+from torch import nn,optim
+from torch.utils.data import DataLoader, Dataset
+cpu = torch.device('cpu')
+cuda = cpu
+if torch.cuda.is_available():
+    cuda = torch.device('cuda')
+def sz(tensor):
+    return tensor.element_size()*tensor.nelement()
+class ConvModel(nn.Module):
+    def __init__(self, dims, num, ins_channels, middle_channels):
+        super().__init__()
+        nets=[]
+        for i in range(num):
+            nets.append(nn.Conv2d(ins_channels, middle_channels, 5, padding=2))
+            nets.append(nn.Conv2d(middle_channels, ins_channels, 5, padding=2))
+            nets.append(nn.LeakyReLU())
+        self.nets = nn.Sequential(*nets)
+    def dv(self):
+        return self.nets[0].bias.device
+    def forward(self, inputs):
+        return self.nets(inputs)
+    def size(self):
+        total = 0.0
+        for net in self.nets:
+            if not hasattr(net, "weight"):
+                continue
+            w = net.weight
+            b = net.bias
+            total+=sz(w)+sz(b)
+        return total
+
+pass
+class ResNet(nn.Module):
+    def __init__(self, net):
+        super().__init__()
+        self.net = net
+    def forward(self, inputs):
+        return inputs+self.net(inputs)
+class TestDataset(Dataset):
+    def __init__(self, n, dims, channels):
+        self.n = n
+        self.dims = dims
+        self.datas = torch.rand(n, channels, dims,dims)
+        print(f"data size: {sz(self.datas)/1024/1024/1024} GB")
+    def __len__(self):
+        return self.n
+    def __getitem__(self, i):
+        return self.datas[i]
+        return torch.rand(self.dims)
+
+pass
+def gen(dims, nets_num, channels, middle_channels, num_conv, lr):
+    mds = []
+    base = None
+    for i in range(num_conv):
+        nets = [ConvModel(dims, nets_num, channels, middle_channels) for i in range(2)]
+        mds+=nets
+        if base is not None:
+            nets = [nets[0], base, nets[1]]
+        nets = nn.Sequential(*nets)
+        base = ResNet(nets)
+    fullnet = base
+    opts =[optim.Adam(md.parameters(), lr=lr) for md in mds]
+    gopt = optim.Adam(fullnet.parameters(), lr=lr)
+    return mds, fullnet, opts, gopt
+def do_save(mds, gmodel, opts, gopt, fp):
+    save = SaveModel(gmodel=gmodel, gopt=gopt)
+    save.set_list("md", mds)
+    save.set_list("opt", opts)
+    save.save(fp)
+def do_load(mds, gmodel, opts, gopt, fp):
+    save = SaveModel(gmodel=gmodel, gopt=gopt)
+    save.set_list("md", mds)
+    save.set_list("opt", opts)
+    save.load(fp)
+    mds = save.get_list("md")
+    opts = save.get_list("opt")
+    gmodel = save.gmodel
+    gopt = save.gopt
+    return mds, gmodel, opts, gopt
+
+pass
+    
+
+
+from pyz.nn.dv import SaveModel
+def fc_opt(net, opt):
+    torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
+    opt.step()
+def test():
+    nets=2
+    channels=10
+    middle_channels = 30
+    dims=64
+    trains = 5
+    datas = 6
+    batch=2
+    lr=0.0001
+    win_size=3
+    num_conv = 6
+    num_ln = 3
+    mds, gmodel, opts, gopt = gen(dims, nets, channels, middle_channels, num_conv, lr)
+    fp = "tmp.dt"
+    print(mds[0].nets[0].weight.mean(), mds[0].nets[0].bias.grad)
+    #do_save(mds, gmodel, opts, gopt, fp)
+    mds, gmodel, opts, gopt = do_load(mds, gmodel, opts, gopt, fp)
+    print(mds[0].nets[0].weight.mean(), mds[0].nets[0].bias.grad)
+    mds_sz = [md.size() for md in mds]
+    print(f"Model Size: {sum(mds_sz)/1024/1024/1024} GB")
+    #cuda=cpu
+    ds = TestDataset(datas, dims, channels)
+    dl = DataLoader(ds, batch)
+    #return
+    loss_fn = torch.nn.MSELoss()
+    dt = list(dl)[0].to(cuda)
+    dt[:] = 0.1
+    print("start train")
+    gmodel = gmodel.to(cuda)
+    gmodel.train()
+    i_mds = 3
+    i_nets = 1
+    #
+    for i in range(3):
+        out = gmodel(dt)
+        loss = loss_fn(out, dt)
+        print(f"loss: {loss, type(loss)}")
+        loss.backward()
+        print("grad:",mds[i_mds].nets[i_nets].weight.mean(), mds[i_mds].nets[i_nets].bias.grad)
+        gopt.zero_grad()
+        mds, gmodel, opts, gopt = do_load(mds, gmodel, opts, gopt, fp)
+        print("grad empty:", mds[i_mds].nets[i_nets].weight.mean(), mds[i_mds].nets[i_nets].bias.grad)
+    #return
+    gmodel.cpu()
+    torch.cuda.empty_cache()
+    dt[:]=0.9
+    input("start middle:")
+    md = DictCache([cuda, cpu], mds, opts, win_size, fc_opt)
+    [k.train() for k in mds]
+    [opt.zero_grad() for opt in opts]
+    for i in range(3):
+        out = md.do_forward(lambda :gmodel(dt))
+        loss = loss_fn(out, dt)
+        print(f"loss: {loss, type(loss)}")
+        md.do_backward(lambda : loss.backward())
+        print("grad:",mds[i_mds].nets[i_nets].weight.mean(), mds[i_mds].nets[i_nets].bias.grad)
+        [opt.zero_grad() for opt in opts]
+        mds, gmodel, opts, gopt = do_load(mds, gmodel, opts, gopt, fp)
+        print("grad empty:", mds[i_mds].nets[i_nets].weight.mean(), mds[i_mds].nets[i_nets].bias.grad)
+    return
+    seq = nn.Sequential(*mds)
+    [k.train() for k in mds]
+    #with torch.no_grad():
+    for i in range(trains):
+        total_loss = 0
+        curr=time.time()
+        for dt in dl:
+            dt=dt.to(cuda)
+            [opt.zero_grad() for opt in opts] #这一步不必也扔gpu里计算吧，直接这样写了
+            out = md.do_forward(lambda :seq(dt))
+            loss = loss_fn(out, dt)
+            print(f"loss: {loss, type(loss)}")
+            md.do_backward(lambda : loss.backward())
+            total_loss+=loss.item()
+        sec = time.time()-curr
+        print("train:", i, "loss:", total_loss/len(dl), "time:", sec)
+
+    
+
+pass
+test()

buildz-0.7.0/buildz/gpuz/test/test_resnet_demo.py

@@ -0,0 +1,104 @@
+#
+
+import sys
+from buildz.gpuz.torch import DictCache
+from buildz.gpuz.test import analyze
+from buildz import pyz
+import torch,time
+from torch import nn,optim
+from torch.utils.data import DataLoader, Dataset
+cpu,cuda = analyze.dvs
+class ConvModel(nn.Module):
+    def __init__(self, dims, num, ins_channels, middle_channels):
+        super().__init__()
+        nets=[]
+        for i in range(num):
+            nets.append(nn.Conv2d(ins_channels, middle_channels, 5, padding=2))
+            nets.append(nn.Conv2d(middle_channels, ins_channels, 5, padding=2))
+            nets.append(nn.LeakyReLU())
+        self.nets = nn.Sequential(*nets)
+    def forward(self, inputs):
+        return self.nets(inputs)
+    def size(self):
+        return sum([analyze.unit_sz(net) for net in self.nets])
+
+pass
+class ResNet(nn.Module):
+    def __init__(self, net):
+        super().__init__()
+        self.net = net
+    def forward(self, inputs):
+        return inputs+self.net(inputs)
+class TestDataset(Dataset):
+    def __init__(self, n, dims, channels):
+        self.n = n
+        self.dims = dims
+        self.datas = torch.rand(n, channels, dims,dims)
+        sz = analyze.sz(self.datas)
+        sz, unit = analyze.show_size(sz)
+        print(f"data size: {sz} {unit}")
+    def __len__(self):
+        return self.n
+    def __getitem__(self, i):
+        return self.datas[i]
+
+pass
+def gen(dims, nets_num, channels, middle_channels, num_conv, lr):
+    """
+        类似UNet的模型
+    """
+    mds = []
+    base = None
+    for i in range(num_conv):
+        nets = [ConvModel(dims, nets_num, channels, middle_channels) for i in range(2)]
+        mds+=nets
+        if base is not None:
+            nets = [nets[0], base, nets[1]]
+        nets = nn.Sequential(*nets)
+        base = ResNet(nets)
+    fullnet = base
+    opts =[optim.Adam(md.parameters(), lr=lr) for md in mds]
+    gopt = optim.Adam(fullnet.parameters(), lr=lr)
+    mds_sz = [md.size() for md in mds]
+    sz, unit = analyze.show_size(sum(mds_sz))
+    print(f"Model Size: {sz} {unit}")
+    return mds, fullnet, opts, gopt
+    
+
+
+def fc_opt(net, opt):
+    torch.nn.utils.clip_grad_norm_(net.parameters(), max_norm=1.0)
+    opt.step()
+def test():
+    nets=2
+    channels=10
+    middle_channels = 30
+    dims=512
+    loop = 10
+    datas = 6
+    batch=2
+    '''
+        如果batch改成6，不用DictCache的时候8GB显存的显卡直接报错显存不够；用DictCache并且win_size=3时需要5GB显存，训练一次9秒；用cpu则要跑很久很久很久
+    '''
+    # batch=6
+    lr=0.0001
+    win_size=3
+    num_conv = 12
+    args = sys.argv[1:]
+    mark_train = True
+    if len(args)>0:
+        mark_train = args.pop(0).lower()=='train'
+    modes = 'cuda,cache,cpu'
+    if len(args)>0:
+        modes = args.pop(0)
+    if len(args)>0:
+        num_conv = int(args.pop(0))
+    print(f"num_conv: {num_conv}")
+    ds = TestDataset(datas, dims, channels)
+    dl = DataLoader(ds, batch)
+    loss_fn = torch.nn.MSELoss()
+    def fc_gen():
+        return gen(dims, nets, channels, middle_channels, num_conv, lr)
+    analyze.analyzes(mark_train, loop, fc_gen, dl, loss_fn, fc_opt, win_size, 'cuda,cache,cpu')
+
+pyz.lc(locals(),test)

{buildz-0.6.60 → buildz-0.7.0}/buildz/gpuz/torch/__init__.py

@@ -2,7 +2,8 @@
 
 __version__="0.0.1"
 
-# 小号多
 __author__ = "Zzz, emails: 1174534295@qq.com, 1309458652@qq.com"
+# 小号多
 
-from .middlez import CacheModel
+from .dict_middle import DictCache
+Dict = DictCache