pyxllib 0.3.122__tar.gz → 0.3.124__tar.gz

{pyxllib-0.3.122/pyxllib.egg-info → pyxllib-0.3.124}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.122
+Version: 0.3.124
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/algo/pupil.py

@@ -7,7 +7,6 @@
 from bisect import bisect_right
 from collections import defaultdict, Counter
 import datetime
-import math
 import re
 from statistics import quantiles
 import sys
@@ -482,26 +481,6 @@ def matchpairs(xs, ys, cmp_func, least_score=sys.float_info.epsilon, *,
     return pairs
 
 
-def get_number_width(n):
-    """ 判断数值n的长度
-
-    参考资料：https://jstrieb.github.io/posts/digit-length/
-
-    >>> get_number_width(0)
-    1
-    >>> get_number_width(9)
-    1
-    >>> get_number_width(10)
-    2
-    >>> get_number_width(97)
-    2
-    """
-    # assert n > 0
-    # return math.ceil(math.log10(n + 1))
-
-    return 1 if n == 0 else (math.floor(math.log10(n)) + 1)
-
-
 class SearchBase:
     """ 一个dfs、bfs模板类 """
 

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/data/echarts.py

@@ -75,28 +75,71 @@ class XlChart(Chart):
 inject_members(XlChart, Chart)
 
 
+class XlLine(Line):
+    @classmethod
+    def from_dict(cls, yaxis, xaxis=None, *, title=None,
+                  xaxis_name=None, yaxis_name=None, **kwargs):
+        """ 查看一个数据的折线图
+
+        :param list|dict yaxis: 列表数据，或者字典name: values表示的多组数据
+        """
+        c = cls()
+
+        if isinstance(yaxis, (list, tuple)):
+            yaxis = {'value': yaxis}
+        if xaxis is None:
+            xaxis = list(range(max([len(v) for v in yaxis.values()])))
+        c.add_xaxis(xaxis)
+        for k, v in yaxis.items():
+            c.add_yaxis(k, v)
+
+        configs = {
+            'tooltip_opts': opts.TooltipOpts(trigger="axis"),
+            **kwargs,
+        }
+        c.set_global_opts(tooltip_opts=opts.TooltipOpts(trigger="axis"))
+        if title:
+            configs['title_opts'] = opts.TitleOpts(title=title)
+        if xaxis_name:
+            configs['xaxis_opts'] = opts.AxisOpts(name=xaxis_name, axislabel_opts=opts.LabelOpts(rotate=45, interval=0))
+        if yaxis_name:
+            configs['yaxis_opts'] = opts.AxisOpts(name=yaxis_name)
+        c.set_global_opts(**configs)
+        return c
+
+
 class XlBar(Bar):
 
     @classmethod
-    def from_dict(cls, yaxis, xaxis=None, *, title=None):
+    def from_dict(cls, yaxis, xaxis=None, *, title=None,
+                  xaxis_name=None, yaxis_name=None, **kwargs):
         """ 查看一个数据的条形图
 
-        :param dict yaxis: 列表数据，或者字典name: values表示的多组数据
+        :param list|dict yaxis: 列表数据，或者字典name: values表示的多组数据
         """
-        b = cls()
+        c = cls()
 
+        if isinstance(yaxis, (list, tuple)):
+            yaxis = {'value': yaxis}
         if xaxis is None:
             xaxis = list(range(max([len(v) for v in yaxis.values()])))
-
-        b.add_xaxis(xaxis)
-
+        c.add_xaxis(xaxis)
         for k, v in yaxis.items():
-            b.add_yaxis(k, v)
+            c.add_yaxis(k, v)
 
+        configs = {
+            'tooltip_opts': opts.TooltipOpts(trigger="axis"),
+            **kwargs,
+        }
+        c.set_global_opts(tooltip_opts=opts.TooltipOpts(trigger="axis"))
         if title:
-            b.set_title(title)
-
-        return b
+            configs['title_opts'] = opts.TitleOpts(title=title)
+        if xaxis_name:
+            configs['xaxis_opts'] = opts.AxisOpts(name=xaxis_name)
+        if yaxis_name:
+            configs['yaxis_opts'] = opts.AxisOpts(name=yaxis_name)
+        c.set_global_opts(**configs)
+        return c
 
     @classmethod
     def from_list(cls, yaxis, xaxis=None, *, title=None):
@@ -128,11 +171,13 @@ class XlBar(Bar):
         for i in range(groups):
             labels.append(f"{min_val + interval * i:.2f}-{min_val + interval * (i + 1):.2f}")
         # t = cls.from_dict({'value': group_counts}, xaxis=labels, title=title)
-        
+
         return cls.from_dict({'value': group_counts}, xaxis=labels, title=title)
 
 
 class XlRadar(Radar):
+    """ 雷达图 """
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.color_idx = 0
@@ -193,4 +238,3 @@ def draw_pareto_chart(data, accuracy=0.1, *, title='帕累托累积权重', valu
 if __name__ == '__main__':
     with TicToc(__name__):
         pass
-

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/data/pglib.py

@@ -58,13 +58,17 @@ class Connection(psycopg.Connection, SqlBase):
     def __1_库(self):
         pass
 
-    def get_db_activities(self):
+    def get_db_activities(self, datname=None):
         """
         检索当前数据库的活动信息。
+
+        :param datname: 这个字段理论上应该要能自动检测出来才对，但这会急着没空钻研，先手动输入吧
         """
         sql = SqlBuilder('pg_stat_activity')
         sql.select('pid', 'datname', 'usename', 'state', 'query', 'age(now(), query_start) AS "query_age"')
         sql.where("state = 'active'")
+        if datname:
+            sql.where(f"datname = '{datname}'")
         return self.exec2dict(sql.build_select()).fetchall()
 
     def __2_表格(self):

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/data/sqlite.py

@@ -59,6 +59,16 @@ class SqlBuilder:
 
         return self
 
+    def where_dict_match(self, items):
+        """ 输入一个字典，要求每个key的字段对应上具体的某个取值value """
+        for k, v in items.items():
+            if isinstance(v, str):
+                self._where.append(f"{k} = '{v}'")
+            elif isinstance(v, (int, float)):
+                self._where.append(f"{k} = {v}")
+            else:
+                raise TypeError
+
     def where_in(self, column, values):
         if values is None:
             return self

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/file/pdflib.py

@@ -19,7 +19,7 @@ from pyxllib.prog.newbie import round_int, decode_bitflags
 from pyxllib.prog.pupil import DictTool, inject_members, dprint
 from pyxllib.prog.specialist import browser
 from pyxllib.algo.newbie import round_unit
-from pyxllib.algo.pupil import get_number_width
+from pyxllib.prog.pupil import get_number_width
 from pyxllib.file.specialist import XlPath, writefile, get_etag
 from pyxllib.cv.expert import xlcv, xlpil
 from pyxlpr.data.labelme import LabelmeDict

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/file/specialist/filelib.py

@@ -1112,7 +1112,7 @@ class XlPath(type(pathlib.Path())):
             s = str(best_match)
             encoding = best_match.encoding
         else:
-            with open(self, 'r', encoding=encoding) as f:
+            with open(self, 'r', encoding=encoding, errors=errors) as f:
                 s = f.read()
 
         # 如果用\r\n作为换行符会有一些意外不好处理
@@ -1212,7 +1212,7 @@ class XlPath(type(pathlib.Path())):
         :param int max_items: 限制读取的条目数，默认为None，表示读取所有条目
         :param int batch_size:
             默认为None，表示一次性读取所有数据
-            如果设置了数值，则会流式读取，常用语太大，超过内存大小等的jsonl文件读取
+            如果设置了数值，则会流式读取，常用于太大，超过内存大小等的jsonl文件读取
                 注意如果设置了大小，只是底层每次一批读取的大小，但返回的data仍然是一维的数据格式迭代器
         :return: 返回读取到的数据列表，如果return_mode为True，则同时返回文件编码格式
 
@@ -1248,10 +1248,10 @@ class XlPath(type(pathlib.Path())):
         else:
             return data
 
-    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w'):
+    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w', errors=None):
         """ 由于这种格式主要是跟商汤这边对接，就尽量跟它们的格式进行兼容 """
         content = '\n'.join([json.dumps(x, ensure_ascii=ensure_ascii, default=default) for x in list_data])
-        self.write_text_unix(content + '\n', mode=mode)
+        self.write_text_unix(content + '\n', mode=mode, errors=errors)
 
     def add_json_line(self, data, ensure_ascii=False, default=None, mode='a'):
         """ 在文件末尾添加一行JSON数据 """

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/file/xlsxlib.py

@@ -121,7 +121,7 @@ def is_valid_excel_address(address):
         return is_valid_excel_cell(address)
 
 
-@run_once('str', debug=True)
+@run_once('str')
 def xlfmt2pyfmt_date(xl_fmt):
     """ 日期的渲染操作
 
@@ -2478,7 +2478,11 @@ def extract_workbook_summary2(file_path, *,
     res['fileName'] = file_path.name
     start_time = time.time()
     wb, suffix = load_as_xlsx_file(file_path, keep_links=keep_links, keep_vba=keep_vba)
-    res['fileType'] = suffix or 'error'
+    if suffix:
+        res['fileType'] = suffix
+    else:
+        res['error'] = 'fileType error'
+
     load_time = time.time() - start_time
     if wb is None:  # 不支持的文件类型，不报错，只是返回最基本的文件名信息
         if return_mode == 1:
@@ -2959,6 +2963,9 @@ class WorkbookSummary3:
             return y
 
         x = summary2
+        if 'error' in x:
+            return x
+
         y = {
             'fileName': x['fileName'],
             'fileType': x['fileType'],

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/prog/pupil.py

@@ -1129,6 +1129,7 @@ def support_multi_processes(default_processes=1):
         def wrapper(*args, **kwargs):
             process_count = int(kwargs.pop('process_count', default_processes))
             process_id = kwargs.pop('process_id', None)
+            shell = kwargs.pop('shell', False)
 
             if process_count == 1 or process_id is not None:
                 if process_id is None:
@@ -1150,7 +1151,7 @@ def support_multi_processes(default_processes=1):
                     for k, v in kwargs.items():
                         cmds.append(f'--{k}')  # 添加关键字参数的键
                         cmds.append(str(v))  # 添加关键字参数的值
-                    mpl.add_process_python(sys.argv[0], cmds)
+                    mpl.add_process_python(sys.argv[0], cmds, shell=shell)
                 mpl.run_endless()
 
         return wrapper
@@ -1313,3 +1314,53 @@ def tprint(*args, **kwargs):
 def is_valid_identifier(name):
     """ 判断是否是合法的标识符 """
     return re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name)
+
+
+def get_number_width(n):
+    """ 判断数值n的长度
+
+    参考资料：https://jstrieb.github.io/posts/digit-length/
+
+    >>> get_number_width(0)
+    1
+    >>> get_number_width(9)
+    1
+    >>> get_number_width(10)
+    2
+    >>> get_number_width(97)
+    2
+    """
+    # assert n > 0
+    # return math.ceil(math.log10(n + 1))
+
+    return 1 if n == 0 else (math.floor(math.log10(n)) + 1)
+
+
+def aligned_range(start, stop=None, step=1):
+    """ 返回按照域宽对齐的数字序列 """
+    if stop is None:
+        start, stop = 0, start
+
+    max_width = get_number_width(stop - step)
+    format_str = '{:0' + str(max_width) + 'd}'
+
+    return [format_str.format(i) for i in range(start, stop, step)]
+
+
+def percentage_and_value(numbers, precision=2, *, total=None, sep='.'):
+    """ 对输入的一串数值，转换成一种特殊的表达格式 "百分比.次数"
+
+    :param list numbers: 数值列表
+    :param int precision: 百分比的精度（小数点后的位数），默认为 2
+    :param int total: 总数，如果不输入，则默认为输入数值的和
+    :param str sep: 分隔符
+    :return: 整数部分是比例，小数部分是原始数值的整数部分
+    """
+    if total is None:
+        total = sum(numbers)
+    width = get_number_width(total)
+    result = []
+    for num in numbers:
+        percent = safe_div(num, total) * 10 ** precision
+        result.append(f"{percent:.0f}{sep}{num:0{width}d}")
+    return result

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/prog/specialist/__init__.py

@@ -25,7 +25,7 @@ import requests
 from humanfriendly import parse_size
 
 from pyxllib.prog.newbie import human_readable_size
-from pyxllib.prog.pupil import get_installed_packages
+from pyxllib.prog.pupil import get_installed_packages, aligned_range, percentage_and_value
 from pyxllib.prog.xlosenv import XlOsEnv
 from pyxllib.file.specialist import cache_file
 
@@ -215,3 +215,130 @@ class ProgressBar:
                 time.sleep(1)
             pbar.n = self.progress
             pbar.refresh()
+
+
+class BitMaskTool:
+    """ 二进制位掩码工具
+
+    概念术语
+        bitval，每一个位上的具体取值，0或1
+        bitsum，所有位上的取值的和，即二进制数的十进制表示
+    """
+
+    def __init__(self, bit_names=None, bitsum_counter=None):
+        """ 初始化 BitMaskTool 对象
+
+        :param list bit_names: 每一位功能开启时，显示的标签名。未输入时，填充0,1,2,3...注意总数，要按域宽对齐
+        :param dict/list bitsum_counter: 各种值出现的次数，可以是一个字典或者一个包含出现次数的列表
+        """
+        # 1 每一位功能开启时，显示的标签名。未输入时，填充0,1,2,3...注意总数，要按域宽对齐
+        if bit_names is None:
+            bit_names = list(aligned_range(len(bit_names)))
+        self.bit_names = bit_names
+        # 2 各种值出现的次数
+        if isinstance(bitsum_counter, (list, tuple)):
+            bitsum_counter = Counter(bitsum_counter)
+        self.bitsum_counter = bitsum_counter or {}
+
+    def get_bitsum_names(self, bitsum):
+        """ 从bitsum得到names的拼接
+
+        >> get_bitsum_names(3)
+        '语义定位,图表'
+        """
+        if not isinstance(bitsum, int):
+            try:
+                bitsum = int(bitsum)
+            except ValueError:
+                bitsum = 0
+
+        tags = []
+        for i, k in enumerate(self.bit_names):
+            if (1 << i) & bitsum:
+                tags.append(k)
+        return ','.join(tags)
+
+    def count_bitsum_relations(self, target_bitsum, relation='='):
+        """ 计算特定关系的 bitsum 数量
+
+        :param int target_bitsum: 目标 bitsum
+        :param str relation: 关系类型，可以是 '=', '⊂', '⊃'
+            假设bitval对应的bit_names为n1,n2,n3,n4。
+            那么bitsum相当于是bitval的一个集合
+            比如a={n1,n3,n4}，b={n1,n3}，因为a完全包含b，所以认为a⊃b，或者a⊋b、b⊂a、b⊊a
+        :return int: 符合条件的 bitsum 数量
+        """
+        count = 0
+        if relation == '=':
+            # 直接计算等于 target_bitsum 的数量
+            count = self.bitsum_counter.get(target_bitsum, 0)
+        elif relation == '⊂':
+            # 计算所有被 target_bitsum 包含的 bitsum 的数量
+            for bitsum, num in self.bitsum_counter.items():
+                if bitsum and bitsum & target_bitsum == bitsum:
+                    count += num
+        elif relation == '⊃':
+            # 计算所有包含 target_bitsum 的 bitsum 的数量
+            for bitsum, num in self.bitsum_counter.items():
+                if bitsum & target_bitsum == target_bitsum:
+                    count += num
+        return count
+
+    def check_bitflag(self, max_bitsum_len=None, reletion='=',
+                      filter_zero=False, sort_by=None, *,
+                      min_bitsum_len=0):
+        """ 检查并返回 bitsum 关系的 DataFrame
+
+        :param int max_bitsum_len: 最大 bitsum 长度
+        :param str reletion: 关系类型，可以是 '=', '⊂', '⊃'
+            支持输入多个字符，表示要同时计算多种关系
+        :param bool filter_zero: 是否过滤掉零值
+        :param None|str sort_by: 排序字段
+            None, 默认排序
+            count, 按照数量从大到小排序
+            bitsum, 按照 bitsum 从小到大排序
+        :param int min_bitsum_len: 最小 bitsum 长度
+        :return: 包含 bitsum 关系的 DataFrame
+        """
+        from itertools import combinations
+
+        total = sum(self.bitsum_counter.values())
+        rows, columns = [], ['类型', '名称', '百分比.次数']
+        rows.append([-1, '总计', total])
+
+        if max_bitsum_len is None:
+            max_bitsum_len = len(self.bit_names)
+
+        bitvals = [(1 << i) for i in range(len(self.bit_names))]
+        for m in range(min_bitsum_len, max_bitsum_len + 1):
+            for comb in combinations(bitvals, m):
+                bitsum = sum(comb)
+                count = self.count_bitsum_relations(bitsum, relation=reletion)
+                if filter_zero and count == 0:
+                    continue
+                rows.append([f'{reletion}{bitsum}',
+                             self.get_bitsum_names(bitsum),
+                             count])
+
+        if sort_by == 'count':
+            rows.sort(key=lambda x: x[2], reverse=True)
+        elif sort_by == 'bitsum':
+            rows.sort(key=lambda x: int(x[0][1:]) if isinstance(x[0], str) else x[0])
+
+        df = pd.DataFrame.from_records(rows, columns=columns)
+        df['百分比.次数'] = percentage_and_value(df['百分比.次数'], 2, total=total)
+        return df
+
+    def report(self):
+        """ 生成统计报告 """
+        html_content = []
+
+        html_content.append('<h1>1 包含每一位bitval特征的数量</h1>')
+        df1 = self.check_bitflag(1, '⊃')
+        html_content.append(df1.to_html())
+
+        html_content.append('<h1>2 每一种具体bitsum组合的数量</h1>')
+        df2 = self.check_bitflag(reletion='=', filter_zero=True, sort_by='bitsum')
+        html_content.append(df2.to_html())
+
+        return '\n'.join(html_content)

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxllib/text/jiebalib.py

@@ -250,7 +250,7 @@ class TextClassifier:
         return sims[:maxn]
 
     def refine_vecs(self):
-        """ 优化向量数据 """
+        """ 优化向量数据，去掉权重小余0.0001的维度 """
         # 1 计算每个向量的长度
         vecs = []
         for vec in tqdm(self.vecs, '优化向量'):

{pyxllib-0.3.122 → pyxllib-0.3.124/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.122
+Version: 0.3.124
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxlpr/ai/xlpaddle.py

@@ -22,6 +22,8 @@ import humanfriendly
 
 import paddle
 import paddle.inference as paddle_infer
+import paddle.io
+# from paddle.io import DataLoader, Dataset
 
 from pyxllib.algo.pupil import natural_sort
 from pyxllib.xl import XlPath, browser
@@ -53,18 +55,13 @@ class SequenceDataset(paddle.io.Dataset):
         if self.labels is not None:
             return x, self.labels[index]
         else:
-            return x
+            return x 
 
 
 def build_testdata_loader(samples, *, labels=None, transform=None, **kwargs):
-    """ 简化的一个创建paddle的DataLoader的函数。主要用于简化部署阶段的推理。
-
-    :param samples: list类型的输入格式
     """
-    import paddle.fluid.dataloader.fetcher
-    # 暂时不知道怎么关闭这个警告，先用暴力方法
-    paddle.fluid.dataloader.fetcher._WARNING_TO_LOG = False
-
+    简化的一个创建paddle的DataLoader的函数。主要用于简化部署阶段的推理。
+    """
     if isinstance(samples, paddle.io.DataLoader):
         return samples
     elif isinstance(samples, paddle.io.Dataset):

{pyxllib-0.3.122 → pyxllib-0.3.124}/pyxlpr/openai2/_core/chat.py

@@ -1,4 +1,5 @@
 import os
+import json
 from json import dumps as jsonDumps
 from json import loads as jsonLoads
 from pathlib import Path
@@ -423,6 +424,10 @@ class Chat:
         })
         answer: str = completion.choices[0].message.content
         self._messages.add_many(*messages, {"role": "assistant", "content": answer})
+
+        if response_json:
+            answer = json.loads(answer)
+
         return answer
 
     def query_image(self, prompt,

{pyxllib-0.3.122 → pyxllib-0.3.124}/setup.py

@@ -45,7 +45,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.122',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.124',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',