pyxllib 0.3.108__tar.gz → 0.3.109__tar.gz

{pyxllib-0.3.108/pyxllib.egg-info → pyxllib-0.3.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.108
+Version: 0.3.109
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/algo/pupil.py

@@ -4,11 +4,12 @@
 # @Email  : 877362867@qq.com
 # @Date   : 2021/06/03 14:22
 
-import textwrap
+from bisect import bisect_right
 from collections import defaultdict, Counter
 import math
 import re
 import sys
+import textwrap
 
 from pyxllib.prog.newbie import typename, human_readable_number
 from pyxllib.text.pupil import listalign, int2myalphaenum
@@ -156,13 +157,17 @@ class ValuesStat2:
     """ 240509周四17:33，第2代统计器
     """
 
-    def __init__(self, values=None, raw_values=None):
+    def __init__(self, values=None, raw_values=None, data_type=None):
         from statistics import pstdev, mean
 
         # 支持输入可能带有非数值类型的raw_values
         if raw_values:
-            values = [x for x in raw_values if isinstance(x, (int, float))]  # todo 可能需要更泛用的判断数值的方法
+            if 'timestamp' in data_type:
+                values = [x.timestamp() for x in raw_values if hasattr(x, 'timestamp')]
+            else:
+                values = [x for x in raw_values if isinstance(x, (int, float))]  # todo 可能需要更泛用的判断数值的方法
 
+        self.date_type = data_type
         self.raw_values = raw_values
         values = values or []
         self.values = sorted(values)
@@ -171,49 +176,86 @@ class ValuesStat2:
         else:
             self.raw_n = 0
         self.n = len(values)
-        self.sum = sum(values)
+
+        if 'timestamp' in data_type:
+            self.sum = None
+        else:
+            self.sum = sum(values)
+
         if self.n:
             self.mean = mean(self.values)
             self.std = pstdev(self.values)
             self.min, self.max = self.values[0], self.values[-1]
         else:
-            self.mean = self.std = self.min = self.max = float('nan')
+            self.mean = self.std = self.min = self.max = None
 
-        self.dist = []
+        self.dist = None
 
     def __len__(self):
         return self.n
 
-    def summary(self, unit='万', precision=4, percentile=5):
+    def summary(self, unit=None, precision=4, percentile_count=5):
         """ 文本汇总性的报告
 
-        :param percentile: 包括两个极值端点的切分点数，
+        :param percentile_count: 包括两个极值端点的切分点数，
             设置2，就是不设置分位数，就是只展示最小、最大值
             如果设置了3，就表示"中位数、二分位数"，在展示的时候，会显示50%位置的分位数值
             如果设置了5，就相当于"四分位数"，会显示25%、50%、75%位置的分位数值
         :param unit: 展示数值时使用的单位
         :param precision: 展示数值时的精度
         """
+        import datetime
         from statistics import quantiles
 
+        # 1 各种细分的格式化方法
         def fmt0(v):
+            # 数量类整数的格式
             return human_readable_number(v, '万')
 
-        def fmt(v):
-            return human_readable_number(v, unit, precision)
+        def fmt1(v):
+            if isinstance(v, str):
+                return v
+            return human_readable_number(v, unit or 'K', precision)
+
+        def fmt2(v):
+            # 日期类数据的格式化
+            # todo 这个应该数据的具体格式来设置的，但是这个现在有点难写，先写死
+            if isinstance(v, str):
+                return v
+            elif isinstance(v, (int, float)):
+                v = datetime.datetime.fromtimestamp(v)
+
+            return v.strftime(unit or '%Y-%m-%d %H:%M:%S')
 
+        def fmt2b(v):
+            # 时间长度类数据的格式化
+            return human_readable_number(v, '秒')
+
+        if 'timestamp' in self.date_type:
+            fmt = fmt2
+            fmtb = fmt2b
+        else:
+            fmt = fmtb = fmt1
+
+        # 2 生成统计报告
         desc = []
         if self.raw_n and self.raw_n > self.n:
             desc.append(f"非数值数量：{fmt0(self.raw_n - self.n)}")
 
         desc.append(f"总数：{fmt0(self.n)}")
-        desc.append(f"总和：{fmt(self.sum)}")
-        desc.append(f'均值±标准差：{fmt(self.mean)}±{fmt(self.std)}')
+        if self.sum is not None:
+            desc.append(f"总和：{fmt(self.sum)}")
+        if self.mean is not None and self.std is not None:
+            desc.append(f'均值±标准差：{fmt(self.mean)}±{fmtb(self.std)}')
+        elif self.mean is not None:
+            desc.append(f'均值：{fmt(self.mean)}')
+        elif self.std is not None:
+            desc.append(f'标准差：{fmtb(self.std)}')
 
         if self.values:
             dist = [self.values[0]]
-            if percentile > 2:
-                quartiles = quantiles(self.values, n=percentile - 1)
+            if percentile_count > 2:
+                quartiles = quantiles(self.values, n=percentile_count - 1)
                 dist += quartiles
             dist.append(self.values[-1])
 
@@ -223,6 +265,27 @@ class ValuesStat2:
 
         return '\t'.join(desc)
 
+    def calculate_ratios(self, x_values, fmt=False):
+        """ 计算并返回一个字典，其中包含每个 x_values 中的值与其小于等于该值的元素的比例
+
+        :param x_values: 一个数值列表，用来计算每个数值小于等于它的元素的比例
+        :param fmt: 直接将值格式化好
+        :return: 一个字典，键为输入的数值，值为对应的比例（百分比）
+        """
+        ratio_dict = {}
+        for x in x_values:
+            position = bisect_right(self.values, x)
+            if self.n > 0:
+                ratio = (position / self.n)
+            else:
+                ratio = 0
+            ratio_dict[x] = ratio
+
+        if fmt:
+            ratio_dict = {x: f'{ratio:.2%}' for x, ratio in ratio_dict.items()}
+
+        return ratio_dict
+
 
 class Groups:
     def __init__(self, data):

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/data/pglib.py

@@ -46,7 +46,7 @@ from pyxllib.prog.pupil import utc_now, utc_timestamp
 from pyxllib.prog.specialist import XlOsEnv
 from pyxllib.algo.pupil import ValuesStat2
 from pyxllib.file.specialist import get_etag
-from pyxllib.data.sqlite import SqlBase
+from pyxllib.data.sqlite import SqlBase, SQLBuilder
 
 
 class Connection(psycopg.Connection, SqlBase):
@@ -147,6 +147,28 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
         # cur.close()
         return data
 
+    def exec2dict_batch(self, sql, batch_size=1000, **kwargs):
+        """ 分批返回数据的版本
+
+        :return:
+            第1个值，是一个迭代器，看起来仍然能一条一条返回，实际后台是按照batch_size打包获取的
+            第2个值，是数据总数
+        """
+        if not isinstance(sql, SQLBuilder):
+            raise ValueError('暂时只能搭配SQLBuilder使用')
+
+        num = self.exec2one(sql.build_count())
+        cur = self.exec2dict(sql.build_select(), **kwargs)
+
+        def yield_row():
+            while True:
+                rows = cur.fetchmany(batch_size)
+                if not rows:
+                    break
+                yield from rows
+
+        return yield_row(), num
+
     exec_dict = exec2dict
 
     def __4_数据类型(self):
@@ -235,7 +257,7 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
         pass
 
     def get_field_valuesstat(self, table_name, field_name, percentile_count=5,
-                             cvt_numeric=True, by_data=False):
+                             filter_condition=None, by_data=False):
         """ 获得指定表格的某个字段的统计特征ValuesStat2对象
 
         :param table_name: 表名
@@ -246,60 +268,71 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
         """
 
         def init_from_db_data():
-            values = self.exec2col(f'SELECT {field_name} FROM {table_name}')
-            if cvt_numeric:  # 这个是numeric格式字段，是"字符串"，要做个转换
+            sql = SQLBuilder(table_name)
+            if filter_condition:
+                sql.where(filter_condition)
+            values = self.exec2col(sql.build_select(field_name))
+            if data_type == 'numeric':
                 values = [x and float(x) for x in values]
-
-            return ValuesStat2(raw_values=values)
+            return ValuesStat2(raw_values=values, data_type=data_type)
 
         def init_from_db():
-            # 构建基础的 SQL 查询
-            sql_query = f"""
-            SELECT 
-                COUNT(*) AS total_count,
-                COUNT({field_name}) AS non_null_count,
-                SUM({field_name}) AS total_sum,
-                AVG({field_name}) AS average,
-                STDDEV({field_name}) AS standard_deviation,
-                MIN({field_name}) AS min_value,
-                MAX({field_name}) AS max_value
-            """
+            # 1 构建基础的 SQL 查询
+            sql = SQLBuilder(table_name)
+            sql.select("COUNT(*) AS total_count")
+            sql.select(f"COUNT({field_name}) AS non_null_count")
+            sql.select(f"MIN({field_name}) AS min_value")
+            sql.select(f"MAX({field_name}) AS max_value")
+            if 'timestamp' in data_type:
+                percentile_type = 'PERCENTILE_DISC'
+                # todo 其实时间类也可以"泛化"一种平均值、标准差算法的，这需要获取全量数据，然后自己计算
+            elif data_type == 'text':
+                percentile_type = 'PERCENTILE_DISC'
+            else:  # 默认是正常的数值类型
+                sql.select(f"SUM({field_name}) AS total_sum")
+                sql.select(f"AVG({field_name}) AS average")
+                sql.select(f"STDDEV({field_name}) AS standard_deviation")
+                percentile_type = 'PERCENTILE_CONT'
 
             percentiles = []
             # 根据分位点的数量动态添加分位数计算
             if percentile_count > 2:
                 step = 1 / (percentile_count - 1)
                 percentiles = [(i * step) for i in range(1, percentile_count - 1)]
-                percentiles_query = ", ".join(
-                    f"PERCENTILE_CONT({p:.2f}) WITHIN GROUP (ORDER BY {field_name}) AS percentile_{int(p * 100)}"
-                    for p in percentiles
-                )
-                sql_query += ", " + percentiles_query
+                for p in percentiles:
+                    sql.select(f"{percentile_type}({p:.2f}) WITHIN GROUP (ORDER BY {field_name}) "
+                               f"AS percentile_{int(p * 100)}")
 
-            sql_query += f" FROM {table_name};"
+            if filter_condition:
+                sql.where(filter_condition)
 
-            # 执行 SQL 查询
-            row = self.exec2dict(sql_query).fetchone()
+            row = self.exec2dict(sql.build_select()).fetchone()
 
-            # 创建一个新实例并将从数据库获得的数据填充到相应的属性
-            x = ValuesStat2()
+            # 2 统计展示
+            x = ValuesStat2(data_type=data_type)
             x.raw_n = row['total_count']
             x.n = row['non_null_count']
-            x.sum = row['total_sum']
-            x.mean = row['average']
-            x.std = row['standard_deviation']
+            if not x.n:
+                return x
+
+            x.sum = row.get('total_sum', None)
+            x.mean = row.get('average', None)
+            x.std = row.get('standard_deviation', None)
 
             # 如果计算了分位数，填充相应属性
             x.dist = [row['min_value']] + [row[f"percentile_{int(p * 100)}"] for p in percentiles] + [row['max_value']]
-            if cvt_numeric:
+            if data_type == 'numeric':
                 x.dist = [float(x) for x in x.dist]
 
             return x
 
+        data_type = self.get_column_data_type(table_name, field_name)
         if by_data:
-            return init_from_db_data()
+            vs = init_from_db_data()
         else:
-            return init_from_db()
+            vs = init_from_db()
+
+        return vs
 
 
 """

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/data/sqlite.py

@@ -4,6 +4,7 @@
 # @Email  : 877362867@qq.com
 # @Date   : 2022/04/12 08:59
 
+import copy
 import json
 import re
 import sqlite3
@@ -18,6 +19,99 @@ warnings.filterwarnings('ignore', message="pandas only support SQLAlchemy connec
 warnings.filterwarnings('ignore', message="pandas only supports SQLAlchemy connectable")
 
 
+class SQLBuilder:
+    def __init__(self, table=''):
+        self.table = table
+        self._select = []
+        self._set = []
+        self._where = []
+        self._order_by = []
+        self._group_by = []
+        self._limit = None  # 限制最多读多少条数据
+        self._offset = None  # 从第几条数据开始读
+
+    def copy(self):
+        # 拷贝一个当前状态的副本sql
+        return copy.deepcopy(self)
+
+    def __1_组件(self):
+        pass
+
+    def from_table(self, table):
+        self.table = table
+        return self
+
+    def select(self, *columns):
+        self._select.extend(columns)
+        return self
+
+    def set(self, *columns):
+        self._set.extend(columns)
+        return self
+
+    def where(self, condition):
+        self._where.append(condition)
+        return self
+
+    def group_by(self, *columns):
+        self._group_by.extend(columns)
+        return self
+
+    def order_by(self, *columns):
+        self._order_by.extend(columns)
+        return self
+
+    def limit(self, limit, offset=None):
+        self._limit = limit
+        self._offset = offset
+        return self
+
+    def __2_build_命令(self):
+        pass
+
+    def build_select(self, *columns):
+        if columns:
+            columns = self._select + list(columns)
+        else:
+            columns = self._select
+
+        sql = [f"SELECT {', '.join(columns) or '*'}", f"FROM {self.table}"]
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        if self._group_by:
+            sql.append(f"GROUP BY {', '.join(self._group_by)}")
+        if self._order_by:
+            sql.append(f"ORDER BY {', '.join(self._order_by)}")
+        if self._limit is not None:
+            limit_clause = f"LIMIT {self._limit}"
+            if self._offset is not None:
+                limit_clause += f" OFFSET {self._offset}"
+            sql.append(limit_clause)
+        return '\n'.join(sql)
+
+    def build_count(self):
+        sql = [f"SELECT COUNT(*)", f"FROM {self.table}"]
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        if self._group_by:
+            sql.append(f"GROUP BY {', '.join(self._group_by)}")
+        return '\n'.join(sql)
+
+    def build_update(self):
+        sql = [f"UPDATE {self.table}"]
+        if self._set:
+            sql.append(f"SET {', '.join(self._set)}")
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        return '\n'.join(sql)
+
+    def build_check_data_type(self, column):
+        sql = SQLBuilder('information_schema.columns')
+        sql.select(f"data_type")
+        sql.where(f"table_name='{self.table}' AND column_name='{column}'")
+        return sql.build_select()
+
+
 class SqlBase:
     """ Sql语法通用的功能 """
 
@@ -285,6 +379,10 @@ class SqlBase:
             print("No 'FROM' keyword found in the data query.")
             return 0
 
+    def get_column_data_type(self, table_name, col_name):
+        """ 获取表格中某一列的数据类型 """
+        return self.exec2one(SQLBuilder(table_name).build_check_data_type(col_name))
+
 
 class Connection(sqlite3.Connection, SqlBase):
     """

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/file/libreoffice.py

@@ -51,6 +51,7 @@ def infer_file_format(file_path):
     elif ext in ['.ppt', '.pptx']:
         fmt = 'pptx'
     elif ext in ['.xls', '.xlsx']:
+        # todo 其实excel还有xlsm的可能，但是用libreoffice可能没有微软365那样可以判断出宏的情况保存为xlsm
         fmt = 'xlsx'
     else:
         raise ValueError("不支持的文件格式")

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/file/specialist/filelib.py

@@ -1465,7 +1465,9 @@ class XlPath(type(pathlib.Path())):
         if not self.exists():
             return self
 
-        if '/' in new_name:
+        if not isinstance(new_name, str):
+            raise ValueError(f'rename2只能做重命名操作，目标路径必须是一个str')
+        elif '/' in new_name:
             raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"/"')
         elif '\\' in new_name:
             raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"\\"')
@@ -1485,6 +1487,14 @@ class XlPath(type(pathlib.Path())):
             self.rename(dst)
         return dst
 
+    def rename_stem(self, stem, if_exists=None):
+        """ 重命名文件的stem """
+        return self.rename2(stem + self.suffix, if_exists)
+
+    def rename_suffix(self, suffix, if_exists=None):
+        """ 重命名文件的suffix """
+        return self.rename2(self.stem + suffix, if_exists)
+
     def delete(self):
         if self.is_file():
             os.remove(self)
@@ -2504,34 +2514,55 @@ class DirsFileFinder:
         for d in dirs:
             self.add_dir(d)
 
-    def add_dir(self, p):
+    def add_dir(self, p, cvt_name_func=None):
         """ 添加备用检索目录
         当前面的目录找不到匹配项的时候，会使用备用目录的文件
         备用目录可以一直添加，有多个，优先级逐渐降低
+
+        :param cvt_name_func: 对名称做个转换再匹配
         """
         files = list(XlPath(p).rglob_files())
-        for f in files:
-            self.names[f.name].append(f)
-            self.stems[f.stem].append(f)
+        if cvt_name_func:
+            for f in files:
+                self.names[cvt_name_func(f.name)].append(f)
+                self.stems[cvt_name_func(f.stem)].append(f)
+        else:
+            for f in files:
+                self.names[f.name].append(f)
+                self.stems[f.stem].append(f)
 
     def find_name(self, name):
         """ 返回第一个匹配的结果 """
-        names = self.find_names(name)
-        if names:
-            return names[0]
+        files = self.find_names(name)
+        if files:
+            return files[0]
 
     def find_names(self, name):
         """ 返回所有匹配的结果 """
         return self.names[name]
 
     def find_stem(self, stem):
-        stems = self.find_stems(stem)
-        if stems:
-            return stems[0]
+        files = self.find_stems(stem)
+        if files:
+            return files[0]
 
     def find_stems(self, stem):
         return self.stems[stem]
 
+    def find_prefix_name(self, prefix_name, suffix=None):
+        files = self.find_prefix_names(prefix_name, suffix=suffix)
+        if files:
+            return files[0]
+
+    def find_prefix_names(self, prefix_name, suffix=None):
+        """ name中前缀为prefix_name """
+        filess = [files for name, files in self.names.items() if name.startswith(prefix_name)]
+        # 将嵌套的list展平
+        files = [file for files in filess for file in files]
+        if suffix:
+            files = [file for file in files if file.suffix == suffix]
+        return files
+
 
 class TwinDirs:
     def __init__(self, src_dir, dst_dir):

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib/prog/newbie.py

@@ -259,6 +259,9 @@ def human_readable_number(value, base_type='K', precision=4):
     :param str base_type: 进制类型，'K'为1000进制, 'KB'为1024进制（KiB同理）, '万'为中文万进制
     :return: 美化后的字符串
     """
+    if value is None:
+        return ''
+
     if abs(value) < 1:
         return f'{value:.{precision}g}'
 

{pyxllib-0.3.108 → pyxllib-0.3.109/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.108
+Version: 0.3.109
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.108 → pyxllib-0.3.109}/pyxllib.egg-info/SOURCES.txt

@@ -129,6 +129,7 @@ pyxlpr/data/imtextline.py
 pyxlpr/data/labelme.py
 pyxlpr/data/removeline.py
 pyxlpr/data/specialist.py
+pyxlpr/data/tiktokenlib.py
 pyxlpr/data/icdar/__init__.py
 pyxlpr/data/icdar/deteval.py
 pyxlpr/data/icdar/icdar2013.py

pyxllib-0.3.109/pyxlpr/data/tiktokenlib.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Author : 陈坤泽
+# @Email  : 877362867@qq.com
+# @Date   : 2024/05/10
+
+from pyxllib.prog.pupil import check_install_package
+
+check_install_package('tiktoken')
+
+import re
+
+import tiktoken
+
+from pyxllib.prog.newbie import SingletonForEveryInitArgs
+
+
+class TicToken(metaclass=SingletonForEveryInitArgs):
+    def __init__(self, for_model='gpt-4'):
+        """ gpt4，默认使用的就是cl100k_base """
+        self.for_model = for_model
+        self.enc = tiktoken.encoding_for_model(for_model)
+
+    def encode(self, text):
+        """ 将文本转换为token """
+        return self.enc.encode(text)
+
+    def count_tokens(self, text, max_length=20000, num_segments=4, *, delete_space=False):
+        """ 统计文本的token数量，对于超过max_length的文本，采用估算方法 """
+        if delete_space:
+            text = re.sub(r'\s+', '', text)
+
+        if len(text) <= max_length:
+            return len(self.encode(text))
+
+        # 对长文本进行采样估计
+        total_length = len(text)
+        sample_size = max_length // num_segments
+        interval = (total_length - sample_size) // (num_segments - 1)
+        samples = [text[i * interval: i * interval + sample_size] for i in range(num_segments)]
+        sample_tokens = sum(len(self.encode(sample)) for sample in samples)
+        estimated_tokens = sample_tokens * total_length // max_length
+        return estimated_tokens

{pyxllib-0.3.108 → pyxllib-0.3.109}/setup.py

@@ -30,7 +30,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.108',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.109',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',