pyxllib 0.0.43__py3-none-any.whl → 0.3.197__py3-none-any.whl

pyxllib/torch/trainlib.py

@@ -1,344 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# @Author : 陈坤泽
-# @Email  : 877362867@qq.com
-# @Data   : 2020/10/18 16:06
-
-
-"""
-
-常见的训练操作的代码封装
-
-"""
-from abc import ABC
-
-from pyxllib.debug import *
-
-import torch
-from torch import nn, optim
-import torch.utils.data
-
-import torchvision
-from torchvision import transforms
-
-# 可视化工具
-try:
-    import visdom
-except ModuleNotFoundError:
-    subprocess.run(['pip', 'install', 'visdom'])
-    import visdom
-
-
-class Visdom(visdom.Visdom, metaclass=SingletonForEveryInitArgs):
-    """
-
-    visdom文档： https://www.yuque.com/code4101/pytorch/visdom
-    """
-
-    def __init__(
-            self,
-            server='http://localhost',
-            endpoint='events',
-            port=8097,
-            base_url='/',
-            ipv6=True,
-            http_proxy_host=None,
-            http_proxy_port=None,
-            env='main',
-            send=True,
-            raise_exceptions=None,
-            use_incoming_socket=True,
-            log_to_filename=None):
-        self.is_connection = is_url_connect(f'{server}:{port}')
-
-        if self.is_connection:
-            super().__init__(server, endpoint, port, base_url, ipv6,
-                             http_proxy_host, http_proxy_port, env, send,
-                             raise_exceptions, use_incoming_socket, log_to_filename)
-        else:
-            get_xllog().info('visdom server not support')
-
-        self.plot_windows = set()
-
-    def __bool__(self):
-        return self.is_connection
-
-    def one_batch_images(self, imgs, targets, title='one_batch_image', *, nrow=8, padding=2):
-        self.images(imgs, nrow=nrow, padding=padding,
-                    win=title, opts={'title': title, 'caption': str(targets)})
-
-    def _check_plot_win(self, win, update=None):
-        # 记录窗口是否为本次执行程序时第一次初始化，并且据此推导update是首次None，还是复用append
-        if update is None:
-            if win in self.plot_windows:
-                update = 'append'
-            else:
-                update = None
-        self.plot_windows.add(win)
-        return update
-
-    def _refine_opts(self, opts=None, *, title=None, legend=None, **kwargs):
-        if opts is None:
-            opts = {}
-        if title and 'title' not in opts: opts['title'] = title
-        if legend and 'legend' not in opts: opts['legend'] = legend
-        for k, v in kwargs.items():
-            if k not in opts:
-                opts[k] = v
-        return opts
-
-    def loss_line(self, loss_values, epoch, win='loss', *, title=None, update=None):
-        """ 损失函数曲线
-
-        横坐标是epoch
-        """
-        # 1 记录窗口是否为本次执行程序时第一次初始化
-        if title is None: title = win
-        update = self._check_plot_win(win, update)
-
-        # 2 画线
-        xs = np.linspace(epoch - 1, epoch, num=len(loss_values) + 1)
-        self.line(loss_values, xs[1:], win=win, opts={'title': title, 'xlabel': 'epoch'},
-                  update=update)
-
-    def plot_line(self, y, x, win, *, opts=None,
-                  title=None, legend=None, update=None):
-        # 1 记录窗口是否为本次执行程序时第一次初始化
-        if title is None: title = win
-        update = self._check_plot_win(win, update)
-
-        # 2 画线
-        self.line(y, x, win=win, update=update,
-                  opts=self._refine_opts(opts, title=title, legend=legend, xlabel='epoch'))
-
-
-def get_device():
-    return torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-
-class TinyDataset(torch.utils.data.Dataset):
-    def __init__(self, labelfile, label_transform):
-        """ 超轻量级的Dataset类，一般由外部ProjectData类指定每行label的转换规则 """
-        self.labels = labelfile.read().splitlines()
-        self.label_transform = label_transform
-
-    def __len__(self):
-        return len(self.labels)
-
-    def __getitem__(self, idx):
-        return self.label_transform(self.labels[idx])
-
-
-class TrainerBase:
-    def __init__(self, model, datasets, *,
-                 save_dir=None,
-                 batch_size=None,
-                 optimizer=None, loss_func=None):
-        self.log = get_xllog()
-        self.device = get_device()
-        self.save_dir = Path(save_dir) if save_dir else Path()  # 没有指定数据路径则以当前工作目录为准
-        self.model = model
-        self.datasets = datasets
-
-    @classmethod
-    def loss_values_stat(cls, loss_vales):
-        """ 一组loss损失的统计分析
-
-        :param loss_vales: 一次info中，多份batch产生的误差数据
-        """
-        if not loss_vales:
-            raise ValueError
-
-        data = np.array(loss_vales, dtype=float)
-        n, sum_ = len(data), data.sum()
-        mean, std = data.mean(), data.std()
-        msg = f'total_loss={sum_:.3f}, mean±std={mean:.3f}±{std:.3f}({max(data):.3f}->{min(data):.3f})'
-        return msg
-
-    @classmethod
-    def sample_size(cls, data):
-        """ 单个样本占用的空间大小，返回字节数 """
-        x, label = data.dataset[0]  # 取第0个样本作为参考
-        return getasizeof(x.numpy()) + getasizeof(label)
-
-    def save_model_state(self, file):
-        """ 保存模型参数值
-        一般存储model.state_dict，而不是直接存储model，确保灵活性
-
-        # TODO 和path结合，增加if_exists参数
-        """
-        p = Path(file, root=self.save_dir)
-        p.ensure_dir(pathtype='file')
-        torch.save(self.model.state_dict(), str(p))
-
-    def load_model_state(self, file):
-        """ 读取模型参数值 """
-        p = Path(file, root=self.save_dir)
-        self.model.load_state_dict(torch.load(str(p), map_location=self.device))
-
-    def get_train_data(self):
-        train_loader = torch.utils.data.DataLoader(
-            ImageDirectionDataset(self.data_dir, mode='train'),
-            batch_size=self.batch_size, shuffle=True, num_workers=8)
-        return train_loader
-
-    def get_val_data(self):
-        val_loader = torch.utils.data.DataLoader(
-            self.datasets(self.data_dir, mode='val'),
-            batch_size=self.batch_size, shuffle=True, num_workers=8)
-        return val_loader
-
-
-class ClassificationTrainer(TrainerBase):
-    """ 对pytorch（分类）模型的训练、测试等操作的进一步封装
-
-    # TODO log变成可选项，可以关掉
-    """
-
-    def __init__(self, model, *, data_dir=None, save_dir=None,
-                 batch_size=None, optimizer=None, loss_func=None):
-
-        super().__init__(save_dir=save_dir)
-        self.log.info(f'initialize. use_device={self.device}.')
-
-        self.model = model.to(self.device)
-        self.optimizer = optimizer if optimizer else optim.Adam(model.parameters(), lr=0.01)
-        self.loss_func = loss_func if loss_func else nn.CrossEntropyLoss().to(self.device)
-        self.log.info('model parameters size: ' + str(sum(map(lambda p: p.numel(), self.model.parameters()))))
-
-        self.data_dir = Path(data_dir) if data_dir else Path()  # 没有指定数据路径则以当前工作目录为准
-        self.log.info(f'data_dir={self.data_dir}, save_dir={self.save_dir}')
-
-        self.batch_size = batch_size if batch_size else 500
-        self.train_data = self.get_train_data()
-        self.val_data = self.get_val_data()
-        self.train_data_number, self.test_data_number = len(self.train_data.dataset), len(self.val_data.dataset)
-        self.log.info(f'get data, train_data_number={self.train_data_number}(batch={len(self.train_data)}), '
-                      f'test_data_number={self.test_data_number}(batch={len(self.val_data)}), batch_size={self.batch_size}')
-
-    def viz_data(self):
-        """ 用visdom显示样本数据
-
-        TODO 增加一些自定义格式参数
-        TODO 不能使用\n、\r\n、<br/>实现文本换行，有时间可以研究下，结合nrow、图片宽度，自动推算，怎么美化展示效果
-        """
-        viz = Visdom()
-        if not viz: return
-
-        x, label = next(iter(self.train_data))
-        viz.one_batch_images(x, label, 'train data')
-
-        x, label = next(iter(self.val_data))
-        viz.one_batch_images(x, label, 'val data')
-
-    def training_one_epoch(self):
-        # 1 检查模式
-        if not self.model.training:
-            self.model.train(True)
-
-        # 2 训练一轮
-        loss_values = []
-        for x, label in self.train_data:
-            # 每个batch可能很大，所以每个batch依次放到cuda，而不是一次性全放入
-            x, label = x.to(self.device), label.to(self.device)
-
-            logits = self.model(x)
-            if isinstance(logits, tuple):
-                logits = logits[0]  # 如果返回是多个值，一般是RNN等层有其他信息，先只取第一个参数值就行了
-            loss = self.loss_func(logits, label)
-            loss_values.append(float(loss))
-
-            self.optimizer.zero_grad()
-            loss.backward()
-            self.optimizer.step()
-
-        # 3 训练阶段只看loss，不看实际预测准确度，默认每个epoch都会输出
-        return loss_values
-
-    def calculate_accuracy(self, data, prefix=''):
-        """ 测试验证集等数据上的精度 """
-        # 1 eval模式
-        if self.model.training:
-            self.model.train(False)
-
-        # 2 关闭梯度，可以节省显存和加速
-        with torch.no_grad():
-            tt = TicToc()
-
-            # 预测结果，计算正确率
-            loss, correct, number = [], 0, len(data.dataset)
-            for x, label in data:
-                x, label = x.to(self.device), label.to(self.device)
-                logits = self.model(x)
-                if isinstance(logits, tuple):
-                    logits = logits[0]
-                loss.append(self.loss_func(logits, label))
-                correct += logits.argmax(dim=1).eq(label).sum().item()  # 预测正确的数量
-            elapsed_time, mean_loss = tt.tocvalue(), np.mean(loss, dtype=float)
-            accuracy = correct / number
-            info = f'{prefix} accuracy={correct}/{number} ({accuracy:.2%})\t' \
-                   f'mean_loss={mean_loss:.3f}\telapsed_time={elapsed_time:.0f}s'
-            self.log.info(info)
-            return accuracy
-
-    def training(self, epochs=20, *, start_epoch=0,
-                 log_interval=1,
-                 test_interval=0, save_interval=0):
-        """ 主要训练接口
-
-        :param epochs: 训练代数，输出时从1开始编号
-        :param start_epoch: 直接从现有的第几个epoch的模型读取参数
-            使用该参数，需要在self.save_dir有对应名称的model文件
-        :param log_interval: 每隔几个epoch输出当前epoch的训练情况，损失值
-        :param test_interval: 每隔几个epoch进行一次正确率测试（训练阶段只能看到每轮epoch中多个batch的平均损失）
-        :param save_interval: 每隔几个epoch保存一次模型
-        :return:
-        """
-        # 1 参数
-        tag = self.model.__class__.__name__
-        epoch_time_tag = f'elapsed_time' if log_interval == 1 else f'{log_interval}*epoch_time'
-        viz = Visdom()
-        if test_interval == 0 and save_interval: test_interval = save_interval
-
-        # 2 加载之前的模型继续训练
-        if start_epoch:
-            self.load_model_state(f'{tag} epoch={start_epoch}.pth')
-
-        # 3 训练
-        tt = TicToc()
-        for epoch in range(start_epoch + 1, epochs + 1):
-            loss_values = self.training_one_epoch()
-            if viz: viz.loss_line(loss_values, epoch, 'train_loss')
-            if log_interval and epoch % log_interval == 0:
-                msg = self.loss_values_stat(loss_values)
-                elapsed_time = tt.tocvalue(restart=True)
-                self.log.info(f'epoch={epoch}, {epoch_time_tag}={elapsed_time:.0f}s\t{msg}')
-            if test_interval and epoch % test_interval == 0:
-                accuracy1 = self.calculate_accuracy(self.train_data, 'train_data')
-                accuracy2 = self.calculate_accuracy(self.val_data, '  val_data')
-                if viz: viz.plot_line([[accuracy1, accuracy2]], [epoch], 'accuracy', legend=['train', 'val'])
-            if save_interval and epoch % save_interval == 0:
-                self.save_model_state(f'{tag} epoch={epoch}.pth')
-
-
-def get_classification_func(model, state_file, func):
-    """ 工厂函数，生成一个分类器函数
-
-    用这个函数做过渡的一个重要目的，也是避免重复加载模型
-
-    :param model: 模型结构
-    :param state_file: 存储参数的文件
-    :param func: 模型结果的处理器，默认
-    :return: 返回的函数结构见下述cls_func
-    """
-    model.load_state_dict(torch.load(str(state_file), map_location=get_device()))
-
-    def cls_func(x):
-        """
-        :param x: 输入可以是路径、np.ndarray、PIL图片等，都为转为batch结构的tensor
-        :return: 输入如果只有一张图片，则返回一个结果
-            否则会存在list，返回一个batch的多个结果
-        """
-        pass
-
-    return cls_func

pyxllib/util/__init__.py

@@ -1,20 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# @Author : 陈坤泽
-# @Email  : 877362867@qq.com
-# @Data   : 2018/09/19 19:41
-
-"""
-一、一些通用的扩展组件
-
-二、对标准库或一些第三方库，进行的功能扩展
-    也有可能对一些bug进行了修改
-
-有些是小的库，直接把源码搬过来了
-    有些是较大的库，仍然要（会自动在需要使用时 pip install）安装
-
-zipfile: py3.6在windows处理zip，解压中文文件会乱码，要改一个编码
-    这个在py3.8中也没有修复，但是py3.8的zipfile更新了不少内容，有时间我要重新整理过来
-onepy: 做了些中文注解，其他修改了啥我也忘了~~可能是有改源码功能的
-pyautogui: 封装扩展了自己的一个 AutoGui 类
-"""

pyxllib/util/aip_.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# @Author : 陈坤泽
-# @Email  : 877362867@qq.com
-# @Data   : 2020/05/30 21:14
-
-"""
-百度人工智能API接口
-"""
-
-import subprocess
-import pandas as pd
-
-try:
-    import aip
-except ModuleNotFoundError:
-    subprocess.run(['pip', 'install', 'baidu-aip'])
-    import aip
-
-from pyxllib.basic import Path
-from pyxllib.image import get_img_content
-
-
-AIP_OCR_ACCOUNT_FILE = Path(__file__).parent / 'aipocraccount.pkl'
-
-
-def create_account_df(file='aipocraccount.pkl'):
-    """请在这里设置您个人的账户密码，并在运行完后，销毁明文信息"""
-    df = pd.DataFrame.from_records([
-        ['坤泽小号', '16936214', 'aaaaaa', '123456'],
-        ['陈坤泽', '16913345', 'bbbbbb', '123456'],
-        ['欧龙', '16933485', 'cccccc', '123456'],
-        ['韩锦锦', '16933339', 'dddddd', '123456'],
-    ], columns=['user', 'APP_ID', 'API_KEY', 'SECRET_KEY'])
-    Path(file).write(df)
-
-
-class AipOcr:
-    """
-    封装该类
-        目的1：合并输入文件和url的识别
-        目的2：带透明底的png百度api识别不了，要先转成RGB格式
-    """
-    client = None
-    client_id = 0
-    account_df = None
-
-    @classmethod
-    def init(cls, next_client=False, account_file_path=None):
-        # 1 账号信息
-        if cls.account_df is None:
-            if not account_file_path:
-                cls.account_df = Path(AIP_OCR_ACCOUNT_FILE).read()
-
-        # 2 初始化client
-        if cls.client is None or next_client:
-            t = cls.client_id + next_client
-            if t > len(cls.account_df):
-                raise ValueError('今天账号份额都用完啦！Open api daily request limit reached')
-            row = cls.account_df.loc[t]
-            AipOcr.client = aip.AipOcr(row.APP_ID, row.API_KEY, row.SECRET_KEY)
-            AipOcr.client_id = t
-        return AipOcr.client
-
-    @classmethod
-    def text(cls, in_, options=None):
-        """ 调用baidu的普通文本识别
-        这个函数你们随便调用，每天5万次用不完
-
-        :param in_: 可以是图片路径，也可以是网页上的url，也可以是Image对象
-        :param options: 可选参数
-            详见：https://cloud.baidu.com/doc/OCR/s/pjwvxzmtc
-        :return: 返回识别出的dict字典
-
-        >> baidu_accurate_ocr('0.png')
-        >> baidu_accurate_ocr(r'http://ksrc2.gaosiedu.com//...',
-                                 {'language_type': 'ENG'})
-        """
-        client = cls.init()
-        content = get_img_content(in_)
-        return client.basicGeneral(content, options)
-
-    @classmethod
-    def accurate_text(cls, in_, options=None):
-        """ 调用baidu的高精度文本识别
-
-        :param in_: 可以是图片路径，也可以是url
-        :param options: 可选参数
-            详见：https://cloud.baidu.com/doc/OCR/s/pjwvxzmtc
-        :return: 返回识别出的dict字典
-
-        >> baidu_accurate_ocr('0.png')
-        >> baidu_accurate_ocr(r'http://ksrc2.gaosiedu.com//...',
-                                 {'language_type': 'ENG'})
-        """
-        client = cls.init()
-        content = get_img_content(in_)
-        # 会自动转base64
-        while True:
-            t = client.basicAccurate(content, options)
-            # dprint(t)
-            if t.get('error_code', None) == 17:
-                client = AipOcr.init(next_client=True)
-            elif t.get('error_code', None) == 18:
-                # {'error_code': 18, 'error_msg': 'Open api qps request limit reached'}，继续尝试
-                continue
-            else:
-                 break
-        return t
-
-
-def demo_aipocr():
-    client = AipOcr()
-    d = client.text("http://i1.fuimg.com/582188/7b0f9cb22c1770a0.png", {'language_type': 'ENG'})
-    print(d)
-    # d = {'log_id': 8013455108426397566, 'words_result_num': 64,
-    # 'words_result': [{'words': '1 . 4 . cre ated'}, {'words': 'B . shook'},
-    # {'words': 'C . entered'}, {'words': 'D'}, {'words': 'C ,'}, {'words': 'D , until'},
-    # {'words': '3 . A . break up'}, {'words': 'B . hold up'}, {'words': 'C . keep up'},
-    # {'words': 'D . show up'}, {'words': '4 . A . whispered'}, {'words': 'B , fought'},
-    # {'words': 'C . talked'}, {'words': 'D'}, {'words': '5 . A . throughout'}, {'words': 'D . after'},
-    # {'words': '6 . A . where'}, {'words': 'B . although'}, {'words': 'C , whle'}, {'words': 'D . that'},
-    # {'words': '7 . A . visitor'}, {'words': 'B . relative'}, {'words': 'C . nei gabor'},
-    # {'words': 'D . stranger'}, {'words': 'B , interest'}, {'words': 'D . anger'}, {'words': 'B . differenc'},
-    # {'words': 'C . point'}, {'words': '10 . A . forgot'}, {'words': 'B . supported'},
-    # {'words': 'C , resi sted'}, {'words': 'D , valued'}, {'words': '11 . A . serious'},
-    # {'words': 'B , nice'}, {'words': 'C , bad'}, {'words': 'D . generous'}, {'words': '12 . A . Gradually'},
-    # {'words': 'B . Imm ediately C . Usuall'}, {'words': 'D , Real ar'}, {'words': '13 . 4 . mind'},
-    # {'words': 'B , trouble'}, {'words': 'D , , order'}, {'words': 'C , lost'}, {'words': 'D , saved'},
-    # {'words': '15 . A . experi'}, {'words': 'B . inform ation C . impression D . advice'},
-    # {'words': '16 . A . However'}, {'words': 'B . Besides'}, {'words': 'C . Eventually D . Occasionally'},
-    # {'words': '17 . A . wrong'}, {'words': 'B . confident'}, {'words': 'C . gulty'},
-    # {'words': '18 . A . rem arned'}, {'words': 'B . retumed'}, {'words': 'C . changed'},
-    # {'words': 'D , left'}, {'words': '19 . A . method'}, {'words': 'B , truth'}, {'words': 'C , skill'},
-    # {'words': 'D , word'}, {'words': '20 . A . exist'}, {'words': 'B , remain'}, {'words': 'C , continue'},
-    # {'words': 'D . happen'}]}
-
-
-if __name__ == '__main__':
-    # create_account_df()
-    demo_aipocr()

pyxllib/util/casiadb.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# @Author : 陈坤泽
-# @Email  : 877362867@qq.com
-# @Data   : 2020/11/08 09:30
-
-"""
-
-Database Home: http://www.nlpr.ia.ac.cn/databases/handwriting/Home.html
-
-CASIA 在线和离线中文手写数据库的一些数据读取功能
-
-"""
-
-from pyxllib.basic import *
-import numpy as np
-
-
-def read_from_dgrl(dgrl):
-    """ 解析中科院的DGRL格式数据
-
-    参考代码：https://blog.csdn.net/DaGongJiGuoMaLu09/article/details/107050519
-        有做了大量简化、工程封装
-
-    TODO 可以考虑做一个返回类似labelme格式的接口，会更通用
-        因为有时候会需要取整张原图
-        而且如果有整个原图，那么每个文本行用shape形状标记即可，不需要取出子图
-
-    :param dgrl: dgrl 格式的文件，或者对应的二进制数据流
-    :return: [(img0, label0), (img1, label1), ...]
-    """
-    # 输入参数可以是bytes，也可以是文件
-    f = XlBytesIO(dgrl)
-    # 表头尺寸
-    header_size = f.unpack('I')
-    # 表头剩下内容，提取 code_length
-    header = f.read(header_size - 4)
-    code_length = struct_unpack(header[-4:-2], 'H')  # 每个字符存储的字节数，一般都是用gbk编码，2个字节
-    # 读取图像尺寸信息，文本行数量
-    height, width, line_num = f.unpack('I' * 3)
-
-    # 读取每一行的信息
-    res = []
-    for k in range(line_num):
-        # 读取该行的字符数量
-        char_num = f.unpack('I')
-        label = f.readtext(char_num, code_length=code_length)
-        label = label.replace('\x00', '')  # 去掉不可见字符 \x00，这一步不加的话后面保存的内容会出现看不见的问题
-
-        # 读取该行的位置和尺寸
-        y, x, h, w = f.unpack('I' * 4)
-
-        # 读取该行的图片
-        bitmap = f.unpack('B' * (h * w))
-        bitmap = np.array(bitmap).reshape(h, w)
-
-        res.append((bitmap, label))
-
-    return res