buildz 0.7.2__tar.gz → 0.7.4__tar.gz

{buildz-0.7.2/buildz.egg-info → buildz-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.7.2
+Version: 0.7.4
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.7.2 → buildz-0.7.4}/buildz/__init__.py

@@ -22,4 +22,4 @@ buildz.pathz: 简化文件路径相关的代码编写
 '''
 
 from .argx import fetch as args
-from .base import Base, WBase
+from .base import Base, WBase,Args

{buildz-0.7.2 → buildz-0.7.4}/buildz/base.py

@@ -81,3 +81,32 @@ class With(Base):
         return cls
 
 pass
+
+class Args(Base):
+    def size(self):
+        return len(self.args)+len(self.maps)
+    def str(self):
+        return f"<Args args={self.args}, maps={self.maps}>"
+    @property
+    def lists(self):
+        return self.args
+    @lists.setter
+    def lists(self, val):
+        self.args=val
+    @lists.deleter
+    def lists(self):
+        del self.args
+    @property
+    def dicts(self):
+        return self.maps
+    @dicts.setter
+    def dicts(self, val):
+        self.maps=val
+    @dicts.deleter
+    def dicts(self):
+        del self.maps
+    def init(self, args, maps):
+        self.args = args
+        self.maps = maps
+
+pass

buildz-0.7.4/buildz/gpuz/torch/multi_dict_middle.py

@@ -0,0 +1,191 @@
+#
+import torch
+from torch import nn
+import threading as th
+from ... import Base
+import numpy as np
+'''
+注：
+没写好，还不能用
+'''
+class MultiDictCache(Base):
+    def init(self, nets, win_sizes=None, backward_deal=None, fc_inputs_to=None, default_dv = None, default_win_size=1):
+        '''
+            nets: [
+                net
+                (net, run_dv)
+                (net, run_dv, opt)
+                (net, (run_dv, cache_dv), opt)
+            ]
+            win_sizes: {
+                run_dv: size
+            }
+        '''
+        if default_dv is None:
+            default_dv = torch.device("cpu")
+        if fc_inputs_to is None:
+            fc_inputs_to = self.inputs_to
+        self.fc_inputs_to = fc_inputs_to
+        self.default_dv = default_dv
+        if win_sizes is None:
+            win_sizes = {}
+        self.default_win_size = default_win_size
+        self.win_sizes = win_sizes
+        #run_dvs = {}
+        rst = {}
+        nears = {}
+        for it in nets:
+            if type(it) not in (list,tuple):
+                it = [it]
+            if len(it)==1:
+                it.append(default_dv)
+            x = it[1]
+            if type(x) not in (list,tuple):
+                it[1] = [it[1]]
+            run_dvs.add(it[1][0])
+            net = it[0]
+            net_id = id(net0)
+            rst[net_id] = it
+            run_dv = it[1][0]
+            if run_dv not in nears:
+                nears[run_dv] = {}
+            nears[run_dv][net_id] = [-1,-1]
+            run_dvs[run_dv] = it
+        nets = rst
+        self.nets = nets
+        self.nears = nears
+        if type(dvs) not in (list,tuple):
+            dvs = [dvs]
+        self.gdv = dvs[0]
+        self.cdv = dvs[-1]
+        self.dvs = dvs
+        if dvs_nets is None:
+            dvs_nets = [[]]
+        if type(dvs_nets) not in (list, tuple):
+            dvs_nets = [dvs_nets]
+        dvs_nets = [k if k is not None else [] for k in dvs_nets]
+        ids_nocache = set()
+        fcs = []
+        for i in range(len(dvs_nets)):
+            for net in dvs_nets[i]:
+                fc = net.register_forward_pre_hook(self.hook_forward_pre_tensor(i))
+                fcs.append(fc)
+                ids_nocache.add(id(net))
+        self.ids_nocache = ids_nocache
+        self.dvs_nets = dvs_nets
+        fcs += [net.register_forward_pre_hook(self.hook_forward_pre) for net in nets if id(net) not in ids_nocache]
+        fcs+=[net.register_forward_hook(self.hook_forward) for net in nets if id(net) not in ids_nocache]
+        net = nets[0]
+        if hasattr(net, "register_full_backward_hook"):
+            fcs+=[net.register_full_backward_hook(self.hook_backward) for net in nets]
+        else:
+            fcs+=[net.register_backward_hook(self.hook_backward) for net in nets]
+        self.src_nets = nets
+        self.fcs = fcs
+        self.nets = {id(net):net for net in nets}
+        self.ctxs = {id(net):[] for net in nets}
+        if opts is not None:
+            opts = {id(net): opt for net,opt in zip(nets, opts)}
+        self.opts = opts
+        self.pools = []
+        self.win_size = win_size
+        self.backward_deal = backward_deal
+        self.nears = {id(net):[-1,-1] for net in nets}
+        self.curr = -1
+        self.done_backward = -1
+        self.no_caches = True
+    def remove(self):
+        [fc.remove() for fc in self.fcs]
+    def hook_forward_pre_tensor(self, i):
+        def hook(model, ins):
+            model.to(self.dvs[i])
+            ins = self.fc_inputs_to(ins, self.dvs[i])
+            return ins
+        return hook
+    @staticmethod
+    def inputs_to(inputs, dv):
+        if isinstance(inputs, torch.Tensor):
+            return inputs.to(dv)
+        inputs = tuple([k.to(dv) for k in inputs])
+        return inputs
+    def ctxs_to(self, i, dv):
+        if dv is None:
+            self.ctxs[i] = []
+        else:
+            self.ctxs[i] = [k.to(dv) for k in self.ctxs[i]]
+    def copy_backward(self, nid):
+        for c_id in self.pools:
+            self.nets[c_id].to(self.cdv)
+            self.ctxs_to(c_id, None)
+        self.pools = []
+        self.nets[nid].to(self.gdv)
+        self.ctxs_to(nid, self.gdv)
+        self.pools.append(nid)
+        next_id = nid
+        for i in range(self.win_size-1):
+            next_id = self.nears[next_id][1]
+            if next_id<0:
+                break
+            self.nets[next_id].to(self.gdv)
+            self.ctxs_to(next_id, self.gdv)
+            self.pools.append(next_id)
+    def copy_forward(self, nid, model):
+        for c_id in self.pools:
+            self.nets[c_id].to(self.cdv)
+            self.ctxs_to(c_id, self.cdv)
+        self.pools = []
+        model.to(self.gdv)
+        self.pools.append(nid)
+        next_id = nid
+        for i in range(self.win_size-1):
+            next_id = self.nears[next_id][0]
+            if next_id<0:
+                break
+            self.nets[next_id].to(self.gdv)
+            self.pools.append(next_id)
+    def hook_forward(self, model, ins, outs):
+        self.no_caches = True
+    def hook_forward_pre(self, model, ins):
+        self.no_caches = False
+        nid = id(model)
+        if nid not in self.pools:
+            self.copy_forward(nid, model)
+        if self.curr>=0:
+            self.nears[self.curr][0] = nid
+        self.curr = nid
+        return self.fc_inputs_to(ins, self.gdv)
+    def do_forward(self, fc):
+        self.no_caches = True
+        self.ctxs = {k:[] for k in self.nets}
+        with torch.autograd.graph.saved_tensors_hooks(self.hook_pack, self.hook_unpack):
+            rst = fc()
+        return rst.to(self.gdv)
+    def wrap_backward_deal(self, net_id):
+        if self.backward_deal is None:
+            return
+        self.backward_deal(self.nets[net_id], self.opts[net_id])
+    def hook_backward(self, model, grad_ins, grad_outs):
+        nid = id(model)
+        self.wrap_backward_deal(nid)
+    def hook_pack(self, dt):
+        if self.no_caches:
+            # 不做缓存，数据不处理
+            return -1, dt
+        index = self.curr
+        # forward时候为了后面计算梯度存的缓存，放到列表里方便转cpu和gpu
+        self.ctxs[index].append(dt)
+        return index, len(self.ctxs[index])-1
+    def hook_unpack(self, x):
+        nid = x[0]
+        if nid<0:
+            return x[1]
+        if nid not in self.pools:
+            self.copy_backward(nid)
+        dt = self.ctxs[nid][x[1]]
+        if self.curr>=0:
+            self.nears[self.curr][1] = nid
+        self.curr = nid
+        return dt
+    def do_backward(self, fc):
+        self.curr = -1
+        fc()

buildz-0.7.4/buildz/xf/__init__.py

@@ -0,0 +1,21 @@
+
+
+#print("xf.__init__")
+from .readz import loads, load, loadf, loads_args, loadx, is_args, loadxf
+from .writez import dumps, dump, dumpf, dumpx, dumpxf
+from .file import *
+from .mapz import *
+from .xargs import fetch as args
+
+try:
+    # C++加速代码
+    #from .cpp import pcxf
+    #print(f"pcxf:", pcxf)
+    #print(f"pcxf.loads:", dir(pcxf))
+    from buildz.xf.cpp.pcxf import loads, loadx
+    loads_args = loadx
+except Exception as exp:
+    #print("init not cpp:",exp)
+    pass
+pass
+__author__ = "Zzz, emails: 1174534295@qq.com, 1309458652@qq.com"

buildz-0.7.4/buildz/xf/base.py

@@ -0,0 +1,30 @@
+
+from buildz import Base, Args
+# class Args(Base):
+#     def size(self):
+#         return len(self.args)+len(self.maps)
+#     def str(self):
+#         return f"<Args args={self.args}, maps={self.maps}>"
+#     @property
+#     def lists(self):
+#         return self.args
+#     @lists.setter
+#     def lists(self, val):
+#         self.args=val
+#     @lists.deleter
+#     def lists(self):
+#         del self.args
+#     @property
+#     def dicts(self):
+#         return self.maps
+#     @dicts.setter
+#     def dicts(self, val):
+#         self.maps=val
+#     @dicts.deleter
+#     def dicts(self):
+#         del self.maps
+#     def init(self, args, maps):
+#         self.args = args
+#         self.maps = maps
+
+# pass

buildz-0.7.4/buildz/xf/cpp/Makefile

@@ -0,0 +1,24 @@
+.PHONY: cpp_depend ALL
+TARGETLIB = libcxf.a
+TARGETLIB_CPP = cxf
+TARGETDYN=cxf.so
+SRC=$(wildcard loaderz/*.cpp) $(wildcard loaderz/**/*.cpp)
+OBJ=$(SRC:.cpp=.o)
+ALL:$(TARGETLIB) $(TARGETDYN)
+	echo "$(SRC)"
+
+$(TARGETDYN):$(OBJ)
+	g++ -O3 -fPIC -shared -o $@ $^ -static-libstdc++ -static-libgcc
+$(TARGETLIB): $(OBJ)
+	ar -rc $@ $^
+test: $(OBJ)
+	g++ -O3 $^ -o $@
+%.o:%.cpp
+	g++ -O3 -fPIC -Wall -g -c $< -o $@
+
+include cpp_depend
+cpp_depend:
+	g++ -MM $(SRC) > cpp_depend
+
+clean:
+	rm $(OBJ)

buildz-0.7.4/buildz/xf/cpp/README.txt

@@ -0,0 +1,29 @@
+1，简介：
+用C++重写了一版xf.loads和xf.loadx，
+mingw32-x64编译的C++版速度大概是python版的9倍
+linux gnu编译的C++版速度大概是python版的15倍
+windows下msvc没试过，因为没装
+但不管怎么说，都比Python的C版json慢，linux gnu编译的C++版性能是C版json的三分之一
+
+2，编译：
+
+需要先安装Cython：
+pip install Cython
+
+然后进行编译：
+
+linux下：
+python setup.py
+
+windows + mingw32-x64:
+python setup_mingw32.py
+
+windows下的msvc/vs(没试过，感觉有可能会报错，要自己改setup.py，不用msvc的原因是嫌vs太大了，不想装):
+python setup.py
+
+mac没试过，苹果电脑没用过
+
+3，使用：
+编译好之后就自动引用了，编译好之后运行的python，使用的buildz.xf.load*会自动替换成C++版
+也可以手动引用：
+from buildz.xf.cpp.pcxf import loads, loadx

buildz-0.7.4/buildz/xf/cpp/clean.py

@@ -0,0 +1,6 @@
+#
+from buildz import fz
+fz.removes("./build")
+fz.removes("./pcxf.c")
+fz.removes("./pcxf.cpp")
+fz.removes("./pcxf.o")

buildz-0.7.4/buildz/xf/cpp/loaderz/base.h

@@ -0,0 +1,36 @@
+
+
+#ifndef XF_BASE
+#define XF_BASE
+#include<string>
+#include<sstream>
+#include<stdio.h>
+//#include<iostream>
+#include<cstring>
+#include<stdlib.h>
+#include <vector>
+#include <exception>
+#include <set>
+#include <algorithm>
+#include <map>
+
+#define DEFAULT_DEAL 0
+typedef unsigned char byte;
+
+#define BUILD_TYPE_LIST 1
+#define BUILD_TYPE_STR 2
+#define BUILD_TYPE_KEYVAL 3
+#define BUILD_TYPE_SPT 4
+
+#define TYPE_NULL 0
+#define TYPE_BOOL 1
+#define TYPE_BOOL_TRUE 101
+#define TYPE_BOOL_FALSE 102
+#define TYPE_INT 2
+#define TYPE_FLOAT 3
+#define TYPE_STR 4
+#define TYPE_STR_TRANSLATE 401
+#define TYPE_LIST 5
+#define TYPE_DICT 6
+#define TYPE_LISTDICT 7
+#endif

buildz-0.7.4/buildz/xf/cpp/loaderz/base_deal.h

@@ -0,0 +1,28 @@
+#ifndef XF_BASE_DEAL
+#define XF_BASE_DEAL
+#include"base.h"
+#include "buffer.h"
+#include"item.h"
+#include <cctype>
+struct Manager;
+struct BaseDeal{
+    virtual bool deal(BufferBase& buffer, ItemList& rst, Manager& mg){return false;}
+    // if do build, do delete obj
+    virtual Item* build(Item* obj, Manager& mg) {return NULL;}
+    virtual int label(){return -1;}
+    virtual int type(){return -1;}
+    virtual void regist(Manager& mg){}
+    virtual ~BaseDeal(){}
+};
+
+// std::string strip(const std::string &inpt)
+// {
+//     auto start_it = inpt.begin();
+//     auto end_it = inpt.rbegin();
+//     while (std::isspace(*start_it))
+//         ++start_it;
+//     while (std::isspace(*end_it))
+//         ++end_it;
+//     return std::string(start_it, end_it.base());
+// }
+#endif

buildz-0.7.4/buildz/xf/cpp/loaderz/buffer.cpp

@@ -0,0 +1,88 @@
+
+#include "buffer.h"
+#include <cctype>
+    std::string StrBuffer::pos2str(Int2& pos){
+        int size = pos.last-pos.first;
+        char chars[size+1];
+        std::memcpy(chars, str+pos.first, size);
+        chars[size]=0;
+        return std::string(chars);
+    }
+    // int StrBuffer::read(char* chars, int size){
+    //     size = std::min(size, str_size+1-read_base);
+    //     std::memcpy(chars, str+read_base, size);
+    //     chars[size]=0;
+    //     return size;
+    // }
+    // int StrBuffer::read_cache(char* chars, int size){
+    //     size = read(chars, size);
+    //     buffer_size+=size;
+    //     read_base+=size;
+    //     return size;
+    // }
+    char StrBuffer::read(){
+        return str[read_base];
+    }
+    char StrBuffer::read_cache(){
+        char c = read();
+        if (c==0)return c;
+        buffer_size++;
+        read_base++;
+        return c;
+    }
+    bool StrBuffer::check_read(const char* chars, int size){
+        if (size>(str_size-read_base))return false;
+        for(int i=0;i<size;++i){
+            if (chars[i]!=str[read_base+i])return false;
+        }
+        return true;
+    }
+    int StrBuffer::clean(int read_size){
+        read_base+=read_size;
+        buffer_base=read_base;
+        buffer_size=0;
+        return 0;
+    }
+    int StrBuffer::size(){return buffer_size;}
+    int StrBuffer::full(char*& chars, bool strip) {
+        int base = buffer_base;
+        int last = buffer_base+buffer_size;
+        if (strip){
+            while(base<last&&std::isspace(str[base]))++base;
+            while(last>base&&std::isspace(str[last-1]))--last;
+        }
+        int size = last-base;
+        if (size>0){
+            if (chars==NULL){
+                chars = new char[size+1];
+            }
+            std::memcpy(chars, str+base,size);
+            chars[size]=0;
+        }
+        return size;
+    }
+    int StrBuffer::rget(char* chars, int size){
+        size = std::min(size, buffer_size);
+        std::memcpy(chars, str+(buffer_base+buffer_size-size), size);
+        return size;
+    }
+    int StrBuffer::get(char* chars, int size){
+        size = std::min(size, buffer_size);
+        std::memcpy(chars, str+buffer_base, size);
+        return size;
+    }
+    bool StrBuffer::_check_get(const char* chars, int size, bool right){
+        if (buffer_size<size)return false;
+        int base = buffer_base;
+        if (right)base += buffer_size-size;
+        for(int i=0;i<size;++i){
+            if (chars[i]!=str[base+i])return false;
+        }
+        return true;
+    }
+    bool StrBuffer::check_get(const char* chars, int size){
+        return _check_get(chars, size);
+    }
+    bool StrBuffer::check_rget(const char* chars, int size){
+        return _check_get(chars, size, true);
+    }

buildz-0.7.4/buildz/xf/cpp/loaderz/buffer.h

@@ -0,0 +1,57 @@
+#ifndef XF_BUFFER
+#define XF_BUFFER
+#include "base.h"
+#include <cctype>
+//#include <cstring>
+struct Int2{
+    int first,last;
+    Int2(){}
+    Int2(int a,int b):first(a),last(b){}
+    Int2(const Int2& a){first=a.first;last=a.last;}
+    void reset(int a, int b) { first=a; last=b; }
+    void reset(Int2& a){first=a.first;last=a.last;}
+    std::string str(){
+        std::stringstream ss;
+        ss<<"("<<first<<", "<<last<<")";
+        return ss.str();
+    }
+};
+struct BufferBase {
+    int buffer_base;
+    int read_base;
+    BufferBase():buffer_base(0),read_base(0){}
+    void pos(Int2& i2){i2.reset(buffer_base, buffer_base+size());}
+    void offsets(Int2& i2){ i2.reset(buffer_base, read_base); }
+    int clean2read(int size=1) { return clean(size); }
+    // virtual
+    virtual std::string pos2str(Int2& pos)=0;
+    virtual char read_cache() =0;
+    virtual bool check_read(const char* chars, int size)=0;
+    virtual char read() = 0;
+    virtual int clean(int read_size=0)=0;
+    virtual int size()=0;
+    virtual int full(char*& chars, bool strip=false)=0;
+    virtual int rget(char* chars, int size=1)=0;
+    virtual int get(char* chars, int size=1)=0;
+    virtual bool check_rget(const char* chars, int size=1)=0;
+    virtual bool check_get(const char* chars, int size=1)=0;
+};
+struct StrBuffer: public BufferBase {
+    const char* str;
+    int buffer_size;
+    int str_size;
+    StrBuffer(const char* str, int size):BufferBase(),str(str),buffer_size(0),str_size(size){}
+    std::string pos2str(Int2& pos);
+    char read_cache();
+    bool check_read(const char* chars, int size);
+    char read();
+    int clean(int read_size=0);
+    int size();
+    int full(char*& chars, bool strip=false);
+    int rget(char* chars, int size=1);
+    int get(char* chars, int size=1);
+    inline bool _check_get(const char* chars, int size=1, bool right=false);
+    bool check_get(const char* chars, int size=1);
+    bool check_rget(const char* chars, int size=1);
+};
+#endif

buildz-0.7.4/buildz/xf/cpp/loaderz/callback.h

@@ -0,0 +1,11 @@
+
+#ifndef XF_CALLBACK
+#define XF_CALLBACK
+#include"base.h"
+struct Callback{
+    virtual void* create(int type, void* val=NULL, int ival=0)=0;
+    virtual void dict_set(void* dict, void* key, void *val)=0;
+    virtual void list_add(void* list, void* val)=0;
+    virtual void* exp(const char* s)=0;
+};
+#endif