pyxllib 0.3.106__tar.gz → 0.3.108__tar.gz

{pyxllib-0.3.106/pyxllib.egg-info → pyxllib-0.3.108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.106
+Version: 0.3.108
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/algo/pupil.py

@@ -152,6 +152,78 @@ class ValuesStat:
             raise ValueError("无效的数据数量")
 
 
+class ValuesStat2:
+    """ 240509周四17:33，第2代统计器
+    """
+
+    def __init__(self, values=None, raw_values=None):
+        from statistics import pstdev, mean
+
+        # 支持输入可能带有非数值类型的raw_values
+        if raw_values:
+            values = [x for x in raw_values if isinstance(x, (int, float))]  # todo 可能需要更泛用的判断数值的方法
+
+        self.raw_values = raw_values
+        values = values or []
+        self.values = sorted(values)
+        if self.raw_values:
+            self.raw_n = len(self.raw_values)
+        else:
+            self.raw_n = 0
+        self.n = len(values)
+        self.sum = sum(values)
+        if self.n:
+            self.mean = mean(self.values)
+            self.std = pstdev(self.values)
+            self.min, self.max = self.values[0], self.values[-1]
+        else:
+            self.mean = self.std = self.min = self.max = float('nan')
+
+        self.dist = []
+
+    def __len__(self):
+        return self.n
+
+    def summary(self, unit='万', precision=4, percentile=5):
+        """ 文本汇总性的报告
+
+        :param percentile: 包括两个极值端点的切分点数，
+            设置2，就是不设置分位数，就是只展示最小、最大值
+            如果设置了3，就表示"中位数、二分位数"，在展示的时候，会显示50%位置的分位数值
+            如果设置了5，就相当于"四分位数"，会显示25%、50%、75%位置的分位数值
+        :param unit: 展示数值时使用的单位
+        :param precision: 展示数值时的精度
+        """
+        from statistics import quantiles
+
+        def fmt0(v):
+            return human_readable_number(v, '万')
+
+        def fmt(v):
+            return human_readable_number(v, unit, precision)
+
+        desc = []
+        if self.raw_n and self.raw_n > self.n:
+            desc.append(f"非数值数量：{fmt0(self.raw_n - self.n)}")
+
+        desc.append(f"总数：{fmt0(self.n)}")
+        desc.append(f"总和：{fmt(self.sum)}")
+        desc.append(f'均值±标准差：{fmt(self.mean)}±{fmt(self.std)}')
+
+        if self.values:
+            dist = [self.values[0]]
+            if percentile > 2:
+                quartiles = quantiles(self.values, n=percentile - 1)
+                dist += quartiles
+            dist.append(self.values[-1])
+
+            desc.append(f"分布：{'/'.join([fmt(v) for v in dist])}")
+        elif self.dist:  # 此时当前函数的percentile设置失效，以dist中存储的情况为准
+            desc.append(f"分布：{'/'.join([fmt(v) for v in self.dist])}")
+
+        return '\t'.join(desc)
+
+
 class Groups:
     def __init__(self, data):
         """ 分组

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/data/pglib.py

@@ -44,6 +44,7 @@ import psycopg.rows
 from pyxllib.prog.newbie import round_int
 from pyxllib.prog.pupil import utc_now, utc_timestamp
 from pyxllib.prog.specialist import XlOsEnv
+from pyxllib.algo.pupil import ValuesStat2
 from pyxllib.file.specialist import get_etag
 from pyxllib.data.sqlite import SqlBase
 
@@ -230,6 +231,76 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
 
         self.commit_base(commit, query, params)
 
+    def __6_高级统计(self):
+        pass
+
+    def get_field_valuesstat(self, table_name, field_name, percentile_count=5,
+                             cvt_numeric=True, by_data=False):
+        """ 获得指定表格的某个字段的统计特征ValuesStat2对象
+
+        :param table_name: 表名
+        :param field_name: 用于计算统计数据的字段名
+        :param percentile_count: 分位数的数量，例如 3 表示只计算中位数
+        :param by_data: 是否获得原始数据
+            默认只获得统计特征，不获得原始数据
+        """
+
+        def init_from_db_data():
+            values = self.exec2col(f'SELECT {field_name} FROM {table_name}')
+            if cvt_numeric:  # 这个是numeric格式字段，是"字符串"，要做个转换
+                values = [x and float(x) for x in values]
+
+            return ValuesStat2(raw_values=values)
+
+        def init_from_db():
+            # 构建基础的 SQL 查询
+            sql_query = f"""
+            SELECT 
+                COUNT(*) AS total_count,
+                COUNT({field_name}) AS non_null_count,
+                SUM({field_name}) AS total_sum,
+                AVG({field_name}) AS average,
+                STDDEV({field_name}) AS standard_deviation,
+                MIN({field_name}) AS min_value,
+                MAX({field_name}) AS max_value
+            """
+
+            percentiles = []
+            # 根据分位点的数量动态添加分位数计算
+            if percentile_count > 2:
+                step = 1 / (percentile_count - 1)
+                percentiles = [(i * step) for i in range(1, percentile_count - 1)]
+                percentiles_query = ", ".join(
+                    f"PERCENTILE_CONT({p:.2f}) WITHIN GROUP (ORDER BY {field_name}) AS percentile_{int(p * 100)}"
+                    for p in percentiles
+                )
+                sql_query += ", " + percentiles_query
+
+            sql_query += f" FROM {table_name};"
+
+            # 执行 SQL 查询
+            row = self.exec2dict(sql_query).fetchone()
+
+            # 创建一个新实例并将从数据库获得的数据填充到相应的属性
+            x = ValuesStat2()
+            x.raw_n = row['total_count']
+            x.n = row['non_null_count']
+            x.sum = row['total_sum']
+            x.mean = row['average']
+            x.std = row['standard_deviation']
+
+            # 如果计算了分位数，填充相应属性
+            x.dist = [row['min_value']] + [row[f"percentile_{int(p * 100)}"] for p in percentiles] + [row['max_value']]
+            if cvt_numeric:
+                x.dist = [float(x) for x in x.dist]
+
+            return x
+
+        if by_data:
+            return init_from_db_data()
+        else:
+            return init_from_db()
+
 
 """
 【关于为什么XlprDb要和pglib合一个文件】

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/ext/kq5034lib.py

@@ -135,13 +135,14 @@ class Xiaoetong:
 
 
 class 网课考勤:
-    def __init__(self, today=None):
+    def __init__(self, today=None, *, 表格路径=None):
         self.返款标题 = ''
-        self.表格路径 = r'考勤.xlsx'
+        self.表格路径 = 表格路径 or r'考勤.xlsx'
         self.在线表格 = 'https://docs.qq.com/sheet/DUlF1UnRackJ2Vm5U'  # 生成日报用
         self.开课日期 = '2022-01-08'
         self.视频返款 = [20, 15, 10, 5, 0, 0]  # 直播(当堂)/第1天（当天）/第2天/第3天/第4天/第5天，完成观看的依次返款额。
         self.打卡返款 = {5: 100, 10: 150, 15: 200}  # 打卡满5/10/15次的返款额
+        self.课程链接 = []
         self._init(today)
 
         self.driver = None  # 浏览器脚本

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/file/libreoffice.py

@@ -59,7 +59,7 @@ def infer_file_format(file_path):
 
 class UpgradeOfficeFile:
     @classmethod
-    def to_dir(cls, file_path, out_dir=None, fmt=None):
+    def to_dir(cls, file_path, out_dir=None, fmt=None, timeout=10):
         """ 将doc文件转换为docx文件
 
         :param file_path: 待升级的文件路径
@@ -86,20 +86,26 @@ class UpgradeOfficeFile:
             executor,
             '--headless',  # 无界面模式
             '--convert-to', fmt,  # 转换为docx格式
-            '--outdir', out_dir,  # 输出目录
+            '--outdir', str(out_dir),  # 输出目录
             file_path  # 输入文件路径
         ]
 
-        subprocess.run(command, check=True)
+        subprocess.run(command, timeout=timeout, check=True)
 
         # 返回转换后的文件路径
         base_name = os.path.basename(file_path)
         name, _ = os.path.splitext(base_name)
         new_file_path = os.path.join(out_dir, f"{name}.{fmt}")
+
+        # todo 以目标文件是否存在判断转换是否成功也是有一定bug的，可能目标文件本来就存在
+        #   但如果严谨判断，就要分析subprocess.run的输出结果了，那个太麻烦，先用简便方法处理
+        if not Path(new_file_path).exists():
+            raise ValueError(f"升级文档失败")
+
         return new_file_path
 
     @classmethod
-    def to_file(cls, in_file, out_file=None, fmt=None):
+    def to_file(cls, in_file, out_file=None, fmt=None, timeout=10):
         """ 可以指定转换出的文件名的版本
 
         :param in_file: 待转换的文件路径
@@ -126,7 +132,7 @@ class UpgradeOfficeFile:
         out_file.parent.mkdir(parents=True, exist_ok=True)
 
         # 调用upgrade_office_file函数进行转换
-        temp_file = cls.to_dir(in_file, out_dir=out_file.parent, fmt=fmt)
+        temp_file = cls.to_dir(in_file, out_dir=out_file.parent, fmt=fmt, timeout=timeout)
 
         # 将生成的临时文件重命名为out_file
         os.rename(temp_file, out_file)
@@ -134,7 +140,7 @@ class UpgradeOfficeFile:
         return out_file
 
     @classmethod
-    def to_tempfile(cls, in_file, fmt=None, *, timestamp_stem=False, create_subdir=False):
+    def to_tempfile(cls, in_file, fmt=None, *, timestamp_stem=False, create_subdir=False, timeout=10):
         """ 将文件转换为临时文件
 
         :param timestamp_stem: 时间戳文件名
@@ -151,8 +157,8 @@ class UpgradeOfficeFile:
 
         if timestamp_stem:
             stem = datetime.now().strftime('%Y%m%d.%H%M%S.%f')
-            out_file = cls.to_file(in_file, out_file=root / f"{stem}.{fmt}", fmt=fmt)
+            out_file = cls.to_file(in_file, out_file=root / f"{stem}.{fmt}", fmt=fmt, timeout=timeout)
         else:
-            out_file = cls.to_dir(in_file, out_dir=root, fmt=fmt)
+            out_file = cls.to_dir(in_file, out_dir=root, fmt=fmt, timeout=timeout)
 
         return out_file

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/file/movielib.py

@@ -10,6 +10,7 @@ check_install_package('moviepy')
 
 import cv2
 from moviepy.editor import VideoFileClip
+from moviepy.editor import cvsecs
 import numpy as np
 from tqdm import tqdm
 
@@ -137,3 +138,11 @@ class XlVideoFileClip(VideoFileClip):
 
 
 inject_members(XlVideoFileClip, VideoFileClip)
+
+
+def clip_video(input_file, output_file, start_time, end_time):
+    start_time = cvsecs(start_time)
+    end_time = cvsecs(end_time)
+    clip = VideoFileClip(input_file).subclip(start_time, end_time)
+    clip.write_videofile(output_file)
+    clip.reader.close()

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/file/xlsxlib.py

@@ -2650,30 +2650,39 @@ class WorkbookSummary3:
                     for addr, _ in row:
                         new_cells[addr] = cells[addr]
 
-            new_cells = {}
-            for rows in rows_groups:
+            total_new_cells = []
+            for rows in reversed(rows_groups):
+                new_cells = {}
                 if len(rows) < 10:
                     extract_cells_from_rows(rows)
                 else:  # 压缩中间的数据
                     # 如果评估到最终摘要可能太小，要收敛下删除的范围
                     n, m = len(rows), len(rows[0])
                     target_n = int(target_reduce_cells_num / m + 0.5)  # 本来应该删除多少行才行
-                    cur_n = n - 4 if target_n > n - 4 else target_n  # 实际删除多少行
-                    left_n = n - cur_n  # 剩余多少行
-                    b = left_n // 2
-                    a = left_n - b
-
-                    extract_cells_from_rows(rows[:a])
-                    addr = combine_addresses(rows[a][0][0], rows[-b - 1][-1][0])
-                    # new_cells[addr] = '这块区域的内容跟前面几行、后面几行的内容结构是一致的，省略显示'
-                    new_cells[addr] = '...'
-                    extract_cells_from_rows(rows[-b:])
-
-                    target_reduce_cells_num -= cur_n * m
-                    if target_reduce_cells_num <= 0:
-                        break
-
-            sheet['cells'] = new_cells
+                    if target_n <= 0:  # 如果删除的行数太少，那么就不压缩了
+                        extract_cells_from_rows(rows)
+                    else:
+                        cur_n = n - 4 if target_n > n - 4 else target_n  # 实际删除多少行
+                        left_n = n - cur_n  # 剩余多少行
+                        b = left_n // 2
+                        a = left_n - b
+
+                        extract_cells_from_rows(rows[:a])
+                        addr = combine_addresses(rows[a][0][0], rows[-b - 1][-1][0])
+                        # new_cells[addr] = '这块区域的内容跟前面几行、后面几行的内容结构是一致的，省略显示'
+                        new_cells[addr] = '...'
+                        extract_cells_from_rows(rows[-b:])
+
+                        target_reduce_cells_num -= cur_n * m
+                        # 240429周一21:57，这两行不能开，否则会过渡精简。如果压缩够了，那么后面的单元格需要全量补上。
+                        # if target_reduce_cells_num <= 0:  # 满足以后不是直接break，而是要把后续的内容都保留
+                        #     break
+                total_new_cells.append(new_cells)
+
+            new_cells2 = {}
+            for rows in reversed(total_new_cells):
+                new_cells2.update(rows)
+            sheet['cells'] = new_cells2
 
     @classmethod
     def reduce4_truncate_cells(cls, y, summary_limit_len, *, cur_summary_len=None):
@@ -2805,6 +2814,7 @@ class WorkbookSummary3:
         if cur_summary_len is None:
             cur_summary_len = cls.count_length(y)
 
+        cur_summary_len0 = cur_summary_len
         active_sheet = y['ActiveSheet']
 
         # 1 预计要删除单元格数
@@ -2836,7 +2846,8 @@ class WorkbookSummary3:
             return cls.count_length(y)
 
         # 4 否则每张表按照比例删单元格，只保留前面部分的单元格
-        left_rate = 1 - r  # 原始保留比例
+        # todo 这里应该有更好的筛选机制，后续可以思考思考
+        left_rate = min((summary_limit_len + cur_summary_len) / 2 * cur_summary_len, 0.9)  # 首轮减小一点调整幅度
         while True:
             for i, st in enumerate(y['sheets']):
                 if i == active_sheet_index:
@@ -2848,10 +2859,10 @@ class WorkbookSummary3:
             cur_summary_len = cls.count_length(y)
             if cur_summary_len <= summary_limit_len:
                 return cur_summary_len
-            if left_rate * total_cells_num < 1:
+            if left_rate * total_cells_num < 1:  # 都没有单元格，别删了
                 break
-            else:
-                left_rate *= 0.8  # 缩小保留比例，再试
+            else:  # 更新保留比率，再试
+                left_rate *= min(summary_limit_len / cur_summary_len, 0.9)
 
         return cur_summary_len
 

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/prog/newbie.py

@@ -267,13 +267,17 @@ def human_readable_number(value, base_type='K', precision=4):
         'K': (['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], 1000),
         'KB': (['', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'], 1024),
         'KiB': (['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'], 1024),
-        '万': (['', '万', '亿', '万亿', '亿亿'], 10000)
+        '万': (['', '万', '亿', '万亿', '亿亿'], 10000),
+        '秒': (['秒', [60, '分'], [60, '时'], [24, '天'], [7, '周'], [4.345, '月'], [12, '年']], 60),
     }.get(base_type, ([''], 1))  # 默认为空单位和基数1
 
     x, i = abs(value), 0
     while x >= base and i < len(units) - 1:
         x /= base
         i += 1
+        if isinstance(units[i], list):
+            base = units[i][0]
+            units[i] = units[i][1]
 
     x = f'{x:.{precision}g}'  # 四舍五入到指定精度
     prefix = '-' if value < 0 else ''  # 负数处理

{pyxllib-0.3.106 → pyxllib-0.3.108}/pyxllib/prog/specialist/tictoc.py

@@ -70,7 +70,7 @@ class TicToc:
 
     def tocvalue(self, restart=False):
         """
-        Return time elapsed since last call to tic().
+        Return time elapsed seconds since last call to tic().
 
         Optional argument:
             restart - Boolean specifying whether to restart the timer

{pyxllib-0.3.106 → pyxllib-0.3.108/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.106
+Version: 0.3.108
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.106 → pyxllib-0.3.108}/setup.py

@@ -30,7 +30,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.106',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.108',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',