nodev 1.0.0__py3-none-any.whl

nodev/operations_conv.py

@@ -0,0 +1,474 @@
+
+from nodev.utils import get_conv_outsize,pair
+from nodev import cuda, as_datafield
+import numpy as np
+
+from nodev.core import Operation
+from nodev.operations import broadcast_to
+
+def im2col(x, kernel_h, kernel_w, stride=1, pad=0,to_matrix=False):    #x 输入 kenrnel_size 卷积核大小 stride 步幅 pad 填充 to_matrix 能否变形为矩阵
+    xp = cuda.get_array_module(x)
+    N, C, H, W = x.shape
+
+    out_h = get_conv_outsize(H, kernel_h, stride, pad)
+    out_w = get_conv_outsize(W, kernel_w, stride, pad)
+
+    img = xp.pad(x,
+                 [(0,0),(0,0),(pad,pad),(pad,pad)],
+                 'constant')
+
+    col = xp.zeros((N, C, kernel_h, kernel_w, out_h, out_w))
+
+    for y in range(kernel_h):
+        y_max = y + stride*out_h
+        for x in range(kernel_w):
+            x_max = x + stride*out_w
+
+            col[:, :, y, x, :, :] = img[:, :, y:y_max:stride, x:x_max:stride]
+
+    col = col.transpose(0,4,5,1,2,3).reshape(N*out_h*out_w, -1)
+
+    return col
+"""def conv2d_simple(x,W,b=None,stride=1,pad=0):
+    x,W=as_variable(x),as_variable(W)
+    weight=W
+    N,C,H,W=x.shape
+    OC,C,KH,KW=weight.shape
+    SH,SW=pair(stride)
+    PH,PW=pair(pad)
+    OH=get_conv_outsize(H,KH,SH,PH)
+    OW=get_conv_outsize(W,KW,SW,PW)
+    col=im2col(x,(KH,KW),stride,pad,to_matrix=True)
+    weight=weight.reshape(OC,-1).transpose()
+    t=linear(col,weight,b)
+    y=t.reshape(N,OH,OW,OC).transpose(0,3,1,2)
+    return y"""
+
+
+class Conv2d(Operation):
+    def __init__(self, stride=1, pad=0):
+        super().__init__()
+        self.stride = pair(stride)
+        self.pad = pair(pad)
+
+    def forward(self, x, W, b):
+        xp = cuda.get_array_module(x)
+
+        KH, KW = W.shape[2:]
+        col = im2col_array(x, (KH, KW), self.stride, self.pad, to_matrix=False)
+        #col = im2col_array(x, (KH, KW), self.stride, self.pad)
+        y = xp.tensordot(col, W, ((1, 2, 3), (1, 2, 3)))
+        if b is not None:
+            y += b
+        y = xp.rollaxis(y, 3, 1)
+        # y = np.transpose(y, (0, 3, 1, 2))
+        return y
+
+    def backward(self, gy):
+        x, W, b = self.inputs
+        # ==== gx ====
+        gx = deconv2d(gy, W, b=None, stride=self.stride, pad=self.pad,
+                      outsize=(x.shape[2], x.shape[3]))
+        # ==== gW ====
+        gW = Conv2DGradW(self)(x, gy)
+        # ==== gb ====
+        gb = None
+        if b.data is not None:
+            gb = gy.sum(axis=(0, 2, 3))
+        return gx, gW, gb
+
+
+def conv2d(x, W, b=None, stride=1, pad=0):
+    return Conv2d(stride, pad)(x, W, b)
+
+
+class Deconv2d(Operation):
+    def __init__(self, stride=1, pad=0, outsize=None):
+        super().__init__()
+        self.stride = pair(stride)
+        self.pad = pair(pad)
+        self.outsize = outsize
+
+    def forward(self, x, W, b):
+        xp = cuda.get_array_module(x)
+
+        Weight = W
+        SH, SW = self.stride
+        PH, PW = self.pad
+        C, OC, KH, KW = Weight.shape
+        N, C, H, W = x.shape
+        if self.outsize is None:
+            out_h = get_deconv_outsize(H, KH, SH, PH)
+            out_w = get_deconv_outsize(W, KW, SW, PW)
+        else:
+            out_h, out_w = pair(self.outsize)
+        img_shape = (N, OC, out_h, out_w)
+
+        gcol = xp.tensordot(Weight, x, (0, 1))
+        gcol = xp.rollaxis(gcol, 3)
+        y = col2im_array(gcol, img_shape, (KH, KW), self.stride, self.pad,
+                         to_matrix=False)
+        # b, k, h, w
+        if b is not None:
+            self.no_bias = True
+            y += b.reshape((1, b.size, 1, 1))
+        return y
+
+    def backward(self, gy):
+        x, W, b = self.inputs
+
+        # ==== gx ====
+        gx = conv2d(gy, W, b=None, stride=self.stride, pad=self.pad)
+        # ==== gW ====
+        f = Conv2DGradW(self)
+        gW = f(gy, x)
+        # ==== gb ====
+        gb = None
+        if b.data is not None:
+            gb = gy.sum(axis=(0, 2, 3))
+        return gx, gW, gb
+
+
+def deconv2d(x, W, b=None, stride=1, pad=0, outsize=None):
+    return Deconv2d(stride, pad, outsize)(x, W, b)
+
+
+class Conv2DGradW(Operation):
+    def __init__(self, conv2d):
+        W = conv2d.inputs[1]
+        kh, kw = W.shape[2:]
+        self.kernel_size = (kh, kw)
+        self.stride = conv2d.stride
+        self.pad = conv2d.pad
+
+    def forward(self, x, gy):
+        xp = cuda.get_array_module(x)
+
+        col = im2col_array(x, self.kernel_size, self.stride, self.pad,
+                           to_matrix=False)
+        gW = xp.tensordot(gy, col, ((0, 2, 3), (0, 4, 5)))
+        return gW
+
+    def backward(self, gys):
+        x, gy = self.inputs
+        gW, = self.outputs
+
+        xh, xw = x.shape[2:]
+        gx = deconv2d(gy, gW, stride=self.stride, pad=self.pad,
+                      outsize=(xh, xw))
+        ggy = conv2d(x, gW, stride=self.stride, pad=self.pad)
+        return gx, ggy
+
+
+# =============================================================================
+#  pooling(max-pooling) / average_pooling
+# =============================================================================
+class Pooling(Operation):
+    def __init__(self, kernel_size, stride=1, pad=0):
+        super().__init__()
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.pad = pad
+
+    def forward(self, x):
+        col = im2col_array(x, self.kernel_size, self.stride, self.pad,
+                           to_matrix=False)
+
+        N, C, KH, KW, OH, OW = col.shape
+        col = col.reshape(N, C, KH * KW, OH, OW)
+        self.indexes = col.argmax(axis=2)
+        y = col.max(axis=2)
+        return y
+
+    def backward(self, gy):
+        return Pooling2DGrad(self)(gy)
+
+
+class Pooling2DGrad(Operation):
+    def __init__(self, mpool2d):
+        self.mpool2d = mpool2d
+        self.kernel_size = mpool2d.kernel_size
+        self.stride = mpool2d.stride
+        self.pad = mpool2d.pad
+        self.input_shape = mpool2d.inputs[0].shape
+        self.dtype = mpool2d.inputs[0].dtype
+        self.indexes = mpool2d.indexes
+
+    def forward(self, gy):
+        xp = cuda.get_array_module(gy)
+
+        N, C, OH, OW = gy.shape
+        N, C, H, W = self.input_shape
+        KH, KW = pair(self.kernel_size)
+
+        gcol = xp.zeros((N * C * OH * OW * KH * KW), dtype=self.dtype)
+
+        indexes = (self.indexes.ravel()
+                   + xp.arange(0, self.indexes.size * KH * KW, KH * KW))
+
+        gcol[indexes] = gy.ravel()
+        gcol = gcol.reshape(N, C, OH, OW, KH, KW)
+        gcol = xp.swapaxes(gcol, 2, 4)
+        gcol = xp.swapaxes(gcol, 3, 5)
+
+        gx = col2im_array(gcol, (N, C, H, W), self.kernel_size, self.stride,
+                          self.pad, to_matrix=False)
+        return gx
+
+    def backward(self, ggx):
+        f = Pooling2DWithIndexes(self.mpool2d)
+        return f(ggx)
+
+
+class Pooling2DWithIndexes(Operation):
+    def __init__(self, mpool2d):
+        self.kernel_size = mpool2d.kernel_size
+        self.stride = mpool2d.stride
+        self.pad = mpool2d.pad
+        self.input_shpae = mpool2d.inputs[0].shape
+        self.dtype = mpool2d.inputs[0].dtype
+        self.indexes = mpool2d.indexes
+
+    def forward(self, x):
+        col = im2col_array(x, self.kernel_size, self.stride, self.pad,
+                           to_matrix=False)
+        N, C, KH, KW, OH, OW = col.shape
+        col = col.reshape(N, C, KH * KW, OH, OW)
+        col = col.transpose(0, 1, 3, 4, 2).reshape(-1, KH * KW)
+        indexes = self.indexes.ravel()
+        col = col[np.arange(len(indexes)), indexes]
+        return col.reshape(N, C, OH, OW)
+
+
+def pooling(x, kernel_size, stride=1, pad=0):
+    return Pooling(kernel_size, stride, pad)(x)
+
+
+class AveragePooling(Operation):
+    def __init__(self, kernel_size, stride=1, pad=0):
+        super().__init__()
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.pad = pad
+        self.input_shape = None
+
+    def forward(self, x):
+        self.input_shape = x.shape
+        col = im2col_array(x, self.kernel_size, self.stride, self.pad,
+                           to_matrix=False)
+        y = col.mean(axis=(2, 3))
+        return y
+
+    def backward(self, gy):
+        # TODO(Koki): This is simple implementation
+        N, C, OH, OW = gy.shape
+        KW, KH = pair(self.kernel_size)
+        gy /= (KW * KH)
+        gcol = broadcast_to(gy.reshape(-1), (KH, KW, N * C * OH * OW))
+        gcol = gcol.reshape(KH, KW, N, C, OH, OW).transpose(2, 3, 0, 1, 4, 5)
+        gx = col2im(gcol, self.input_shape, self.kernel_size, self.stride,
+                    self.pad, to_matrix=False)
+        return gx
+
+
+def average_pooling(x, kernel_size, stride=1, pad=0):
+    return AveragePooling(kernel_size, stride, pad)(x)
+
+
+# =============================================================================
+#  im2col / col2im
+# =============================================================================
+class Im2col(Operation):
+    def __init__(self, kernel_size, stride, pad, to_matrix):
+        super().__init__()
+        self.input_shape = None
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.pad = pad
+        self.to_matrix = to_matrix
+
+    def forward(self, x):
+        self.input_shape = x.shape
+        y = im2col_array(x, self.kernel_size, self.stride, self.pad,
+                         self.to_matrix)
+        return y
+
+    def backward(self, gy):
+        gx = col2im(gy, self.input_shape, self.kernel_size, self.stride,
+                    self.pad, self.to_matrix)
+        return gx
+
+
+
+class Col2im(Operation):
+    def __init__(self, input_shape, kernel_size, stride, pad, to_matrix):
+        super().__init__()
+        self.input_shape = input_shape
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.pad = pad
+        self.to_matrix = to_matrix
+
+    def forward(self, x):
+        y = col2im_array(x, self.input_shape, self.kernel_size, self.stride,
+                         self.pad, self.to_matrix)
+        return y
+
+    def backward(self, gy):
+        gx = im2col(gy, self.kernel_size, self.stride, self.pad,
+                    self.to_matrix)
+        return gx
+
+
+def col2im(x, input_shape, kernel_size, stride=1, pad=0, to_matrix=True):
+    return Col2im(input_shape, kernel_size, stride, pad, to_matrix)(x)
+
+
+# =============================================================================
+#  numpy im2col
+# =============================================================================
+def im2col_array(img, kernel_size, stride, pad, to_matrix=True):
+    N, C, H, W = img.shape
+    KH, KW = pair(kernel_size)
+    SH, SW = pair(stride)
+    PH, PW = pair(pad)
+    OH = get_conv_outsize(H, KH, SH, PH)
+    OW = get_conv_outsize(W, KW, SW, PW)
+
+    xp = cuda.get_array_module(img)
+    if xp != np:
+        col = _im2col_gpu(img, kernel_size, stride, pad)
+    else:
+        img = np.pad(img,
+                     ((0, 0), (0, 0), (PH, PH + SH - 1), (PW, PW + SW - 1)),
+                     mode='constant', constant_values=(0,))
+        col = np.ndarray((N, C, KH, KW, OH, OW), dtype=img.dtype)
+
+        for j in range(KH):
+            j_lim = j + SH * OH
+            for i in range(KW):
+                i_lim = i + SW * OW
+                col[:, :, j, i, :, :] = img[:, :, j:j_lim:SH, i:i_lim:SW]
+
+    if to_matrix:
+        col = col.transpose((0, 4, 5, 1, 2, 3)).reshape((N * OH * OW, -1))
+
+    return col
+
+
+def col2im_array(col, img_shape, kernel_size, stride, pad, to_matrix=True):
+    N, C, H, W = img_shape
+    KH, KW = pair(kernel_size)
+    SH, SW = pair(stride)
+    PH, PW = pair(pad)
+    OH = get_conv_outsize(H, KH, SH, PH)
+    OW = get_conv_outsize(W, KW, SW, PW)
+
+    if to_matrix:
+        col = col.reshape(N, OH, OW, C, KH, KW).transpose(0, 3, 4, 5, 1, 2)
+
+    xp = cuda.get_array_module(col)
+    if xp != np:
+        img = _col2im_gpu(col, SH, SW, PH, PW, H, W)
+        return img
+    else:
+        img = np.zeros((N, C, H + 2 * PH + SH - 1, W + 2 * PW + SW - 1),
+                       dtype=col.dtype)
+        for j in range(KH):
+            j_lim = j + SH * OH
+            for i in range(KW):
+                i_lim = i + SW * OW
+                img[:, :, j:j_lim:SH, i:i_lim:SW] += col[:, :, j, i, :, :]
+        return img[:, :, PH:H + PH, PW:W + PW]
+
+
+def _im2col_gpu(img, kernel_size, stride, pad):
+    """im2col function for GPU.
+    This code is ported from Chainer:
+    https://github.com/chainer/chainer/blob/v6.4.0/chainer/utils/conv.py
+    """
+    n, c, h, w = img.shape
+    kh, kw = pair(kernel_size)
+    sy, sx = pair(stride)
+    ph, pw = pair(pad)
+    out_h = get_conv_outsize(h, kh, sy, ph)
+    out_w = get_conv_outsize(w, kw, sx, pw)
+    dy, dx = 1, 1
+    col = cuda.cupy.empty((n, c, kh, kw, out_h, out_w), dtype=img.dtype)
+
+    cuda.cupy.ElementwiseKernel(
+        'raw T img, int32 h, int32 w, int32 out_h, int32 out_w,'
+        'int32 kh, int32 kw, int32 sy, int32 sx, int32 ph, int32 pw,'
+        'int32 dy, int32 dx',
+        'T col',
+        '''
+           int c0 = i / (kh * kw * out_h * out_w);
+           int ky = i / (kw * out_h * out_w) % kh;
+           int kx = i / (out_h * out_w) % kw;
+           int out_y = i / out_w % out_h;
+           int out_x = i % out_w;
+           int in_y = ky * dy + out_y * sy - ph;
+           int in_x = kx * dx + out_x * sx - pw;
+           if (in_y >= 0 && in_y < h && in_x >= 0 && in_x < w) {
+             col = img[in_x + w * (in_y + h * c0)];
+           } else {
+             col = 0;
+           }
+        ''',
+        'im2col')(img.reduced_view(),
+                  h, w, out_h, out_w, kh, kw, sy, sx, ph, pw, dy, dx, col)
+
+    return col
+def get_deconv_outsize(size, k, stride, pad):
+    return stride * (size - 1) + k - 2 * pad
+def _col2im_gpu(col, sy, sx, ph, pw, h, w):
+    """col2im function for GPU.
+    This code is ported from Chainer:
+    https://github.com/chainer/chainer/blob/v6.4.0/chainer/utils/conv.py
+    """
+    n, c, kh, kw, out_h, out_w = col.shape
+    dx, dy = 1, 1
+    img = cuda.cupy.empty((n, c, h, w), dtype=col.dtype)
+
+    cuda.cupy.ElementwiseKernel(
+        'raw T col, int32 h, int32 w, int32 out_h, int32 out_w,'
+        'int32 kh, int32 kw, int32 sy, int32 sx, int32 ph, int32 pw,'
+        'int32 dx, int32 dy',
+        'T img',
+        '''
+           int c0 = i / (h * w);
+           int y  = i / w % h;
+           int x  = i % w;
+           T val = 0;
+           for (int ky = 0; ky < kh; ++ky) {
+             int out_y = (y + ph - ky * dy);
+             if (0 > out_y || out_y >= out_h * sy) continue;
+             if (out_y % sy != 0) continue;
+             out_y /= sy;
+             for (int kx = 0; kx < kw; ++kx) {
+               int out_x = (x + pw - kx * dx);
+               if (0 > out_x || out_x >= out_w * sx) continue;
+               if (out_x % sx != 0) continue;
+               out_x /= sx;
+               int k = out_y + out_h * (kx + kw * (ky + kh * c0));
+               val = val + col[out_x + out_w * k];
+             }
+           }
+           img = val;
+        ''',
+        'col2im')(col.reduced_view(),
+                  h, w, out_h, out_w, kh, kw, sy, sx, ph, pw, dx, dy, img)
+    return img
+def pooling_simple(x,kernel_size,stride=1,pad=0):
+    x=as_datafield(x)
+    N,C,H,W=x.shape
+    KH,KW=pair(kernel_size)
+    PH,PW=pair(pad)
+    SH,SW=pair(stride)
+    OH=get_conv_outsize(H,KH,SH,PH)
+    OW=get_conv_outsize(W,KW,SW,PW)
+    col=im2col(x,kernel_size,stride,pad,to_matrix=True)
+    col=col.reshape(-1,KH*KW)
+    y=col.max(axis=1)
+    y=y.reshape((N,OH,OW,C)).transpose(0,3,1,2)
+    return y

nodev/optimizer.py

@@ -0,0 +1,72 @@
+
+import math
+from nodev import cuda
+from nodev.cuda import get_array_module
+
+class Optimizer:
+    def __init__(self):
+        self.target=None
+        self.hooks=[]
+    def setup(self,target):
+        self.target=target
+        return self
+    def update(self):
+        params=[p for p in self.target.params() if p.grad is not None]
+        #预处理
+        for f in self.hooks:
+            f(params)
+        #更新参数
+        for param in params:
+            self.update_one(param)
+    def update_one(self,param):
+        raise NotImplementedError
+    def add_hook(self,f):
+        self.hooks.append(f)
+
+class SGD(Optimizer):
+    def __init__(self,lr=0.01):
+        super().__init__()
+        self.lr=lr
+
+    def update_one(self,param):
+        param.data-=self.lr*param.grad.data
+
+class MomentumSGD(Optimizer):
+    def __init__(self,lr=0.01,momentum=0.9):
+        super().__init__()
+        self.lr=lr
+        self.momentum=momentum
+        self.vs={}
+    def update_one(self,param):
+        v_key=id(param)
+        if v_key not in self.vs:
+            xp = get_array_module(param.data)
+            self.vs[v_key] = xp.zeros_like(param.data)
+
+        v=self.vs[v_key]
+        v*=self.momentum
+        v-=self.lr*param.grad.data
+        param.data += v
+
+class AdaGrad(Optimizer):
+    def __init__(self, lr=0.001, eps=1e-8):
+        super().__init__()
+        self.lr = lr
+        self.eps = eps
+        self.hs = {}
+
+    def update_one(self, param):
+        xp = cuda.get_array_module(param.data)
+
+        h_key = id(param)
+        if h_key not in self.hs:
+            self.hs[h_key] = xp.zeros_like(param.data)
+
+        lr = self.lr
+        eps = self.eps
+        grad = param.grad.data
+        h = self.hs[h_key]
+
+        h += grad * grad
+        param.data -= lr * grad / (xp.sqrt(h) + eps)
+

nodev/transforms.py

@@ -0,0 +1,65 @@
+import numpy as np
+
+
+class Compose:
+    """Compose multiple transforms."""
+
+    def __init__(self, transforms=[]):
+        self.transforms = transforms
+
+    def __call__(self, x):
+        if not self.transforms:
+            return x
+        for t in self.transforms:
+            x = t(x)
+        return x
+
+
+class Flatten:
+    """Flatten ndarray."""
+
+    def __call__(self, x):
+        return x.reshape(-1)
+
+
+class ToFloat:
+    """Convert ndarray dtype to float32."""
+
+    def __call__(self, x):
+        return x.astype(np.float32)
+
+
+class Normalize:
+    """Normalize ndarray."""
+
+    def __init__(self, mean=0., std=1.):
+        self.mean = mean
+        self.std = std
+
+    def __call__(self, x):
+        mean = self.mean
+        std = self.std
+
+        if not np.isscalar(mean):
+            mean = np.array(mean, dtype=x.dtype)
+        if not np.isscalar(std):
+            std = np.array(std, dtype=x.dtype)
+
+        return (x - mean) / std
+
+
+class AsType:
+    """Convert ndarray dtype."""
+
+    def __init__(self, dtype=np.float32):
+        self.dtype = dtype
+
+    def __call__(self, x):
+        return x.astype(self.dtype)
+
+
+class ToInt:
+    """Convert ndarray dtype to int."""
+
+    def __call__(self, x):
+        return x.astype(np.int32)