pyxllib 0.3.108__tar.gz → 0.3.110__tar.gz

{pyxllib-0.3.108/pyxllib.egg-info → pyxllib-0.3.110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.108
+Version: 0.3.110
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/algo/pupil.py

@@ -4,11 +4,14 @@
 # @Email  : 877362867@qq.com
 # @Date   : 2021/06/03 14:22
 
-import textwrap
+from bisect import bisect_right
 from collections import defaultdict, Counter
+import datetime
 import math
 import re
+from statistics import quantiles
 import sys
+import textwrap
 
 from pyxllib.prog.newbie import typename, human_readable_number
 from pyxllib.text.pupil import listalign, int2myalphaenum
@@ -156,13 +159,17 @@ class ValuesStat2:
     """ 240509周四17:33，第2代统计器
     """
 
-    def __init__(self, values=None, raw_values=None):
+    def __init__(self, values=None, raw_values=None, data_type=None):
         from statistics import pstdev, mean
 
         # 支持输入可能带有非数值类型的raw_values
         if raw_values:
-            values = [x for x in raw_values if isinstance(x, (int, float))]  # todo 可能需要更泛用的判断数值的方法
+            if 'timestamp' in data_type:
+                values = [x.timestamp() for x in raw_values if hasattr(x, 'timestamp')]
+            else:
+                values = [x for x in raw_values if isinstance(x, (int, float))]  # todo 可能需要更泛用的判断数值的方法
 
+        self.date_type = data_type
         self.raw_values = raw_values
         values = values or []
         self.values = sorted(values)
@@ -171,57 +178,128 @@ class ValuesStat2:
         else:
             self.raw_n = 0
         self.n = len(values)
-        self.sum = sum(values)
+
+        if 'timestamp' in data_type:
+            self.sum = None
+        else:
+            self.sum = sum(values)
+
         if self.n:
             self.mean = mean(self.values)
             self.std = pstdev(self.values)
             self.min, self.max = self.values[0], self.values[-1]
         else:
-            self.mean = self.std = self.min = self.max = float('nan')
+            self.mean = self.std = self.min = self.max = None
 
-        self.dist = []
+        self.dist = None
 
     def __len__(self):
         return self.n
 
-    def summary(self, unit='万', precision=4, percentile=5):
+    def _summary(self, unit=None, precision=4, percentile_count=5):
+        """ 返回字典结构的总结 """
         """ 文本汇总性的报告
 
-        :param percentile: 包括两个极值端点的切分点数，
+        :param percentile_count: 包括两个极值端点的切分点数，
             设置2，就是不设置分位数，就是只展示最小、最大值
             如果设置了3，就表示"中位数、二分位数"，在展示的时候，会显示50%位置的分位数值
             如果设置了5，就相当于"四分位数"，会显示25%、50%、75%位置的分位数值
         :param unit: 展示数值时使用的单位
         :param precision: 展示数值时的精度
         """
-        from statistics import quantiles
 
+        # 1 各种细分的格式化方法
         def fmt0(v):
+            # 数量类整数的格式
             return human_readable_number(v, '万')
 
-        def fmt(v):
-            return human_readable_number(v, unit, precision)
+        def fmt1(v):
+            if isinstance(v, str):
+                return v
+            return human_readable_number(v, unit or 'K', precision)
 
-        desc = []
+        def fmt2(v):
+            # 日期类数据的格式化
+            # todo 这个应该数据的具体格式来设置的，但是这个现在有点难写，先写死
+            if isinstance(v, str):
+                return v
+            elif isinstance(v, (int, float)):
+                v = datetime.datetime.fromtimestamp(v)
+
+            return v.strftime(unit or '%Y-%m-%d %H:%M:%S')
+
+        def fmt2b(v):
+            # 时间长度类数据的格式化
+            return human_readable_number(v, '秒')
+
+        if 'timestamp' in self.date_type:
+            fmt = fmt2
+            fmtb = fmt2b
+        else:
+            fmt = fmtb = fmt1
+
+        # 2 生成统计报告
+        desc = {}
         if self.raw_n and self.raw_n > self.n:
-            desc.append(f"非数值数量：{fmt0(self.raw_n - self.n)}")
+            desc["总数"] = f"{fmt0(self.n)}/{fmt0(self.raw_n)}≈{self.n / self.raw_n:.2%}"
+        else:
+            desc["总数"] = f"{fmt0(self.n)}"
 
-        desc.append(f"总数：{fmt0(self.n)}")
-        desc.append(f"总和：{fmt(self.sum)}")
-        desc.append(f'均值±标准差：{fmt(self.mean)}±{fmt(self.std)}')
+        if self.sum is not None:
+            desc["总和"] = f"{fmt(self.sum)}"
+        if self.mean is not None and self.std is not None:
+            desc["均值±标准差"] = f"{fmt(self.mean)}±{fmtb(self.std)}"
+        elif self.mean is not None:
+            desc["均值"] = f"{fmt(self.mean)}"
+        elif self.std is not None:
+            desc["标准差"] = f"{fmtb(self.std)}"
 
         if self.values:
             dist = [self.values[0]]
-            if percentile > 2:
-                quartiles = quantiles(self.values, n=percentile - 1)
+            if percentile_count > 2:
+                quartiles = quantiles(self.values, n=percentile_count - 1)
                 dist += quartiles
             dist.append(self.values[-1])
 
-            desc.append(f"分布：{'/'.join([fmt(v) for v in dist])}")
-        elif self.dist:  # 此时当前函数的percentile设置失效，以dist中存储的情况为准
-            desc.append(f"分布：{'/'.join([fmt(v) for v in self.dist])}")
+            desc["分布"] = '/'.join([fmt(v) for v in dist])
+        elif self.dist:
+            desc["分布"] = '/'.join([fmt(v) for v in self.dist])
+
+        return desc
+
+    def summary(self, unit=None, precision=4, percentile_count=5):
+        """ 文本汇总性的报告
+
+        :param percentile_count: 包括两个极值端点的切分点数，
+            设置2，就是不设置分位数，就是只展示最小、最大值
+            如果设置了3，就表示"中位数、二分位数"，在展示的时候，会显示50%位置的分位数值
+            如果设置了5，就相当于"四分位数"，会显示25%、50%、75%位置的分位数值
+        :param unit: 展示数值时使用的单位
+        :param precision: 展示数值时的精度
+        """
+        desc = self._summary(unit, precision, percentile_count)
+        return '\t'.join([f"{key}: {value}" for key, value in desc.items()])
+
+    def calculate_ratios(self, x_values, fmt=False):
+        """ 计算并返回一个字典，其中包含每个 x_values 中的值与其小于等于该值的元素的比例
+
+        :param x_values: 一个数值列表，用来计算每个数值小于等于它的元素的比例
+        :param fmt: 直接将值格式化好
+        :return: 一个字典，键为输入的数值，值为对应的比例（百分比）
+        """
+        ratio_dict = {}
+        for x in x_values:
+            position = bisect_right(self.values, x)
+            if self.n > 0:
+                ratio = (position / self.n)
+            else:
+                ratio = 0
+            ratio_dict[x] = ratio
+
+        if fmt:
+            ratio_dict = {x: f'{ratio:.2%}' for x, ratio in ratio_dict.items()}
 
-        return '\t'.join(desc)
+        return ratio_dict
 
 
 class Groups:

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/data/pglib.py

@@ -46,7 +46,7 @@ from pyxllib.prog.pupil import utc_now, utc_timestamp
 from pyxllib.prog.specialist import XlOsEnv
 from pyxllib.algo.pupil import ValuesStat2
 from pyxllib.file.specialist import get_etag
-from pyxllib.data.sqlite import SqlBase
+from pyxllib.data.sqlite import SqlBase, SqlBuilder
 
 
 class Connection(psycopg.Connection, SqlBase):
@@ -147,6 +147,28 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
         # cur.close()
         return data
 
+    def exec2dict_batch(self, sql, batch_size=1000, **kwargs):
+        """ 分批返回数据的版本
+
+        :return:
+            第1个值，是一个迭代器，看起来仍然能一条一条返回，实际后台是按照batch_size打包获取的
+            第2个值，是数据总数
+        """
+        if not isinstance(sql, SqlBuilder):
+            raise ValueError('暂时只能搭配SQLBuilder使用')
+
+        num = self.exec2one(sql.build_count())
+        cur = self.exec2dict(sql.build_select(), **kwargs)
+
+        def yield_row():
+            while True:
+                rows = cur.fetchmany(batch_size)
+                if not rows:
+                    break
+                yield from rows
+
+        return yield_row(), num
+
     exec_dict = exec2dict
 
     def __4_数据类型(self):
@@ -234,72 +256,83 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
     def __6_高级统计(self):
         pass
 
-    def get_field_valuesstat(self, table_name, field_name, percentile_count=5,
-                             cvt_numeric=True, by_data=False):
+    def get_column_valuesstat(self, table_name, column, percentile_count=5,
+                              filter_condition=None, by_data=False):
         """ 获得指定表格的某个字段的统计特征ValuesStat2对象
 
         :param table_name: 表名
-        :param field_name: 用于计算统计数据的字段名
+        :param column: 用于计算统计数据的字段名
         :param percentile_count: 分位数的数量，例如 3 表示只计算中位数
         :param by_data: 是否获得原始数据
             默认只获得统计特征，不获得原始数据
         """
 
         def init_from_db_data():
-            values = self.exec2col(f'SELECT {field_name} FROM {table_name}')
-            if cvt_numeric:  # 这个是numeric格式字段，是"字符串"，要做个转换
+            sql = SqlBuilder(table_name)
+            if filter_condition:
+                sql.where(filter_condition)
+            values = self.exec2col(sql.build_select(column))
+            if data_type == 'numeric':
                 values = [x and float(x) for x in values]
-
-            return ValuesStat2(raw_values=values)
+            return ValuesStat2(raw_values=values, data_type=data_type)
 
         def init_from_db():
-            # 构建基础的 SQL 查询
-            sql_query = f"""
-            SELECT 
-                COUNT(*) AS total_count,
-                COUNT({field_name}) AS non_null_count,
-                SUM({field_name}) AS total_sum,
-                AVG({field_name}) AS average,
-                STDDEV({field_name}) AS standard_deviation,
-                MIN({field_name}) AS min_value,
-                MAX({field_name}) AS max_value
-            """
+            # 1 构建基础的 SQL 查询
+            sql = SqlBuilder(table_name)
+            sql.select("COUNT(*) AS total_count")
+            sql.select(f"COUNT({column}) AS non_null_count")
+            sql.select(f"MIN({column}) AS min_value")
+            sql.select(f"MAX({column}) AS max_value")
+            if data_type and 'timestamp' in data_type:
+                percentile_type = 'PERCENTILE_DISC'
+                # todo 其实时间类也可以"泛化"一种平均值、标准差算法的，这需要获取全量数据，然后自己计算
+            elif data_type == 'text':
+                percentile_type = 'PERCENTILE_DISC'
+            else:  # 默认是正常的数值类型
+                sql.select(f"SUM({column}) AS total_sum")
+                sql.select(f"AVG({column}) AS average")
+                sql.select(f"STDDEV({column}) AS standard_deviation")
+                percentile_type = 'PERCENTILE_CONT'
 
             percentiles = []
             # 根据分位点的数量动态添加分位数计算
             if percentile_count > 2:
                 step = 1 / (percentile_count - 1)
                 percentiles = [(i * step) for i in range(1, percentile_count - 1)]
-                percentiles_query = ", ".join(
-                    f"PERCENTILE_CONT({p:.2f}) WITHIN GROUP (ORDER BY {field_name}) AS percentile_{int(p * 100)}"
-                    for p in percentiles
-                )
-                sql_query += ", " + percentiles_query
+                for p in percentiles:
+                    sql.select(f"{percentile_type}({p:.2f}) WITHIN GROUP (ORDER BY {column}) "
+                               f"AS percentile_{int(p * 100)}")
 
-            sql_query += f" FROM {table_name};"
+            if filter_condition:
+                sql.where(filter_condition)
 
-            # 执行 SQL 查询
-            row = self.exec2dict(sql_query).fetchone()
+            row = self.exec2dict(sql.build_select()).fetchone()
 
-            # 创建一个新实例并将从数据库获得的数据填充到相应的属性
-            x = ValuesStat2()
+            # 2 统计展示
+            x = ValuesStat2(data_type=data_type)
             x.raw_n = row['total_count']
             x.n = row['non_null_count']
-            x.sum = row['total_sum']
-            x.mean = row['average']
-            x.std = row['standard_deviation']
+            if not x.n:
+                return x
+
+            x.sum = row.get('total_sum', None)
+            x.mean = row.get('average', None)
+            x.std = row.get('standard_deviation', None)
 
             # 如果计算了分位数，填充相应属性
             x.dist = [row['min_value']] + [row[f"percentile_{int(p * 100)}"] for p in percentiles] + [row['max_value']]
-            if cvt_numeric:
+            if data_type == 'numeric':
                 x.dist = [float(x) for x in x.dist]
 
             return x
 
+        data_type = self.get_column_data_type(table_name, column)
         if by_data:
-            return init_from_db_data()
+            vs = init_from_db_data()
         else:
-            return init_from_db()
+            vs = init_from_db()
+
+        return vs
 
 
 """

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/data/sqlite.py

@@ -4,6 +4,7 @@
 # @Email  : 877362867@qq.com
 # @Date   : 2022/04/12 08:59
 
+import copy
 import json
 import re
 import sqlite3
@@ -18,6 +19,118 @@ warnings.filterwarnings('ignore', message="pandas only support SQLAlchemy connec
 warnings.filterwarnings('ignore', message="pandas only supports SQLAlchemy connectable")
 
 
+class SqlBuilder:
+    def __init__(self, table=''):
+        self.table = table
+        self._select = []
+        self._set = []
+        self._where = []
+        self._order_by = []
+        self._group_by = []
+        self._limit = None  # 限制最多读多少条数据
+        self._offset = None  # 从第几条数据开始读
+
+    def copy(self):
+        # 拷贝一个当前状态的副本sql
+        return copy.deepcopy(self)
+
+    def __1_组件(self):
+        pass
+
+    def from_table(self, table):
+        self.table = table
+        return self
+
+    def select(self, *columns):
+        self._select.extend(columns)
+        return self
+
+    def set(self, *columns):
+        self._set.extend(columns)
+        return self
+
+    def where(self, condition):
+        self._where.append(condition)
+        return self
+
+    def where_in(self, column, values):
+        if values is None:
+            return self
+
+        if isinstance(values, str):
+            values = [values]
+        values_str = ', '.join(f"'{str(value)}'" for value in values)
+        if len(values) == 1:
+            self._where.append(f"{column} = {values_str[0]}")
+        else:
+            self._where.append(f"{column} IN ({values_str})")
+        return self
+
+    def where_or(self, *conditions):
+        """ 输入的这一批条件，作为OR组合后成为一个整体条件
+        """
+        self._where.append(f"({' OR '.join(conditions)})")
+        return self
+
+    def group_by(self, *columns):
+        self._group_by.extend(columns)
+        return self
+
+    def order_by(self, *columns):
+        self._order_by.extend(columns)
+        return self
+
+    def limit(self, limit, offset=None):
+        self._limit = limit
+        self._offset = offset
+        return self
+
+    def __2_build_命令(self):
+        pass
+
+    def build_select(self, *columns):
+        if columns:
+            columns = self._select + list(columns)
+        else:
+            columns = self._select
+
+        sql = [f"SELECT {', '.join(columns) or '*'}", f"FROM {self.table}"]
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        if self._group_by:
+            sql.append(f"GROUP BY {', '.join(self._group_by)}")
+        if self._order_by:
+            sql.append(f"ORDER BY {', '.join(self._order_by)}")
+        if self._limit is not None:
+            limit_clause = f"LIMIT {self._limit}"
+            if self._offset is not None:
+                limit_clause += f" OFFSET {self._offset}"
+            sql.append(limit_clause)
+        return '\n'.join(sql)
+
+    def build_count(self):
+        sql = [f"SELECT COUNT(*)", f"FROM {self.table}"]
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        if self._group_by:
+            sql.append(f"GROUP BY {', '.join(self._group_by)}")
+        return '\n'.join(sql)
+
+    def build_update(self):
+        sql = [f"UPDATE {self.table}"]
+        if self._set:
+            sql.append(f"SET {', '.join(self._set)}")
+        if self._where:
+            sql.append(f"WHERE {' AND '.join(self._where)}")
+        return '\n'.join(sql)
+
+    def build_check_data_type(self, column):
+        sql = SqlBuilder('information_schema.columns')
+        sql.select(f"data_type")
+        sql.where(f"table_name='{self.table}' AND column_name='{column}'")
+        return sql.build_select()
+
+
 class SqlBase:
     """ Sql语法通用的功能 """
 
@@ -285,6 +398,10 @@ class SqlBase:
             print("No 'FROM' keyword found in the data query.")
             return 0
 
+    def get_column_data_type(self, table_name, col_name):
+        """ 获取表格中某一列的数据类型 """
+        return self.exec2one(SqlBuilder(table_name).build_check_data_type(col_name))
+
 
 class Connection(sqlite3.Connection, SqlBase):
     """

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/ext/kq5034lib.py

@@ -28,7 +28,7 @@ import fire
 import pandas as pd
 from tqdm import tqdm
 import requests
-import requests_cache
+# import requests_cache
 
 from pyxllib.text.pupil import chinese2digits, grp_chinese_char
 from pyxllib.file.xlsxlib import openpyxl

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/file/libreoffice.py

@@ -51,6 +51,7 @@ def infer_file_format(file_path):
     elif ext in ['.ppt', '.pptx']:
         fmt = 'pptx'
     elif ext in ['.xls', '.xlsx']:
+        # todo 其实excel还有xlsm的可能，但是用libreoffice可能没有微软365那样可以判断出宏的情况保存为xlsm
         fmt = 'xlsx'
     else:
         raise ValueError("不支持的文件格式")

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/file/specialist/filelib.py

@@ -1437,9 +1437,9 @@ class XlPath(type(pathlib.Path())):
         dst = XlPath(dst)
         if dst.exist_preprcs(if_exists):
             if self.is_file():
-                shutil.copy2(self, dst)
+                return shutil.copy2(self, dst)
             else:
-                shutil.copytree(self, dst)
+                return shutil.copytree(self, dst)
 
     def move(self, dst, if_exists=None):
         if not self.exists():
@@ -1465,7 +1465,9 @@ class XlPath(type(pathlib.Path())):
         if not self.exists():
             return self
 
-        if '/' in new_name:
+        if not isinstance(new_name, str):
+            raise ValueError(f'rename2只能做重命名操作，目标路径必须是一个str')
+        elif '/' in new_name:
             raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"/"')
         elif '\\' in new_name:
             raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"\\"')
@@ -1485,6 +1487,14 @@ class XlPath(type(pathlib.Path())):
             self.rename(dst)
         return dst
 
+    def rename_stem(self, stem, if_exists=None):
+        """ 重命名文件的stem """
+        return self.rename2(stem + self.suffix, if_exists)
+
+    def rename_suffix(self, suffix, if_exists=None):
+        """ 重命名文件的suffix """
+        return self.rename2(self.stem + suffix, if_exists)
+
     def delete(self):
         if self.is_file():
             os.remove(self)
@@ -2504,34 +2514,55 @@ class DirsFileFinder:
         for d in dirs:
             self.add_dir(d)
 
-    def add_dir(self, p):
+    def add_dir(self, p, cvt_name_func=None):
         """ 添加备用检索目录
         当前面的目录找不到匹配项的时候，会使用备用目录的文件
         备用目录可以一直添加，有多个，优先级逐渐降低
+
+        :param cvt_name_func: 对名称做个转换再匹配
         """
         files = list(XlPath(p).rglob_files())
-        for f in files:
-            self.names[f.name].append(f)
-            self.stems[f.stem].append(f)
+        if cvt_name_func:
+            for f in files:
+                self.names[cvt_name_func(f.name)].append(f)
+                self.stems[cvt_name_func(f.stem)].append(f)
+        else:
+            for f in files:
+                self.names[f.name].append(f)
+                self.stems[f.stem].append(f)
 
     def find_name(self, name):
         """ 返回第一个匹配的结果 """
-        names = self.find_names(name)
-        if names:
-            return names[0]
+        files = self.find_names(name)
+        if files:
+            return files[0]
 
     def find_names(self, name):
         """ 返回所有匹配的结果 """
         return self.names[name]
 
     def find_stem(self, stem):
-        stems = self.find_stems(stem)
-        if stems:
-            return stems[0]
+        files = self.find_stems(stem)
+        if files:
+            return files[0]
 
     def find_stems(self, stem):
         return self.stems[stem]
 
+    def find_prefix_name(self, prefix_name, suffix=None):
+        files = self.find_prefix_names(prefix_name, suffix=suffix)
+        if files:
+            return files[0]
+
+    def find_prefix_names(self, prefix_name, suffix=None):
+        """ name中前缀为prefix_name """
+        filess = [files for name, files in self.names.items() if name.startswith(prefix_name)]
+        # 将嵌套的list展平
+        files = [file for files in filess for file in files]
+        if suffix:
+            files = [file for file in files if file.suffix == suffix]
+        return files
+
 
 class TwinDirs:
     def __init__(self, src_dir, dst_dir):

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/prog/newbie.py

@@ -259,6 +259,9 @@ def human_readable_number(value, base_type='K', precision=4):
     :param str base_type: 进制类型，'K'为1000进制, 'KB'为1024进制（KiB同理）, '万'为中文万进制
     :return: 美化后的字符串
     """
+    if value is None:
+        return ''
+
     if abs(value) < 1:
         return f'{value:.{precision}g}'
 

{pyxllib-0.3.108 → pyxllib-0.3.110}/pyxllib/prog/specialist/bc.py

@@ -16,7 +16,10 @@ from pyxllib.file.specialist import File, Dir, filesmatch, get_encoding, XlPath
 # BCompare.exe， bcompare函数要用
 
 class BCompare(Explorer):
-    def __init__(self, app='BCompare', shell=False):
+    def __init__(self, app='bcomp', shell=False):
+        """
+        240512周日20:06，本来写的是BCompare，但是友鑫mac电脑上发现似乎有问题，所以改成bcomp，这种在windows上也能用
+        """
         super().__init__(app, shell)
 
     @classmethod
@@ -48,7 +51,7 @@ class BCompare(Explorer):
         default_suffix = None
         for i, arg in enumerate(args):
             f = XlPath.safe_init(arg)
-            if f.is_file():  # 是文件对象，且存在
+            if f is not None and f.is_file():  # 是文件对象，且存在
                 new_args.append(f)
                 if not default_suffix:
                     default_suffix = f.suffix