pyxllib 0.3.88__tar.gz → 0.3.89__tar.gz

{pyxllib-0.3.88/pyxllib.egg-info → pyxllib-0.3.89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.88
+Version: 0.3.89
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.88 → pyxllib-0.3.89}/pyxllib/ext/unixlib.py

@@ -295,7 +295,7 @@ class XlSSHClient(paramiko.SSHClient):
                     return (XlPath(k) / relpath).parent
                 except ValueError:
                     pass
-            raise ValueError('找不到对应的map_path路径映射规则')
+            raise ValueError(f'找不到对应的map_path路径映射规则 {remote_path}')
 
     def __remote_dir(self, local_path, remote_dir):
         if remote_dir is not None:
@@ -415,7 +415,7 @@ class XlSSHClient(paramiko.SSHClient):
             remote_path = self.__remote_dir(local_path, None) / local_path.name
             local_dir = local_path.parent
         else:
-            remote_path = self.Path(remote_path)
+            remote_path = self.Path(remote_path.as_posix())
             local_dir = self.__local_dir(remote_path, local_dir)
             local_path = local_dir / remote_path.name
 

{pyxllib-0.3.88 → pyxllib-0.3.89}/pyxllib/file/packlib/__init__.py

@@ -276,14 +276,18 @@ def smart_compress_zip(root, paths, check_func=None):
 
     root = XlPath(root)
     num = len(paths)
+    outfile = None
     if num > 1:
-        zipf = XlZipFile(root.name + '.zip', 'w', zipfile.ZIP_DEFLATED)
+        outfile = root.parent / XlPath(root.name + '.zip')
     elif num == 1:
-        zipf = XlZipFile(XlPath(paths[0]).name + '.zip', 'w', zipfile.ZIP_DEFLATED)
+        outfile = root.parent / (XlPath(paths[0]).name + '.zip')
     else:
         return
+    zipf = XlZipFile(outfile, 'w', zipfile.ZIP_DEFLATED)
 
     for subroot in paths:
         add_path(XlPath(subroot))
 
     zipf.close()
+
+    return outfile

{pyxllib-0.3.88 → pyxllib-0.3.89}/pyxllib/file/xlsxlib.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # @Author : 陈坤泽
 # @Email  : 877362867@qq.com
-# @Date   : 2020/06/02
+# @Date   : 2024/01/07
 
 """
 扩展了些自己的openpyxl工具
@@ -48,7 +48,7 @@ from pyxllib.file.specialist import XlPath
 
 
 def __1_basic():
-    """ 基础组件 """
+    """ 表格的组件功能 """
 
 
 def excel_addr(n, m) -> str:
@@ -92,8 +92,8 @@ def is_valid_excel_range(range):
     if ':' in range:
         start, end = range.split(':')
         return (is_valid_excel_cell(start) or start.isdigit() or re.fullmatch(r'[A-Z]+', start)) and \
-               (is_valid_excel_cell(end) or end.isdigit() or re.fullmatch(r'[A-Z]+', end)) and \
-               start <= end
+            (is_valid_excel_cell(end) or end.isdigit() or re.fullmatch(r'[A-Z]+', end)) and \
+            start <= end
     else:
         return is_valid_excel_cell(range)
 
@@ -279,6 +279,19 @@ def sort_excel_files(file_paths):
     return file_paths2
 
 
+def excel2md5(file, reduction_degree=1):
+    try:
+        wb = openpyxl.load_workbook(file)
+    except (ValueError, TypeError) as e:
+        # 有些表格直接读取会失败，但使用read_only就能读了
+        wb = openpyxl.load_workbook(file, read_only=True)
+    except Exception as e:  # 还有其他zipfile.BadZipFile等错误
+        print(file, str(e))
+        return ''
+
+    return wb.to_md5(reduction_degree)
+
+
 def convert_csv_to_xlsx(csv_file):
     """ 将 csv 文件转换为 xlsx 文件 """
     wb = Workbook()
@@ -317,8 +330,8 @@ def convert_xls_to_xlsx(xls_file):
     return wb
 
 
-def __2_extend_methods():
-    """ 给各种类扩展的成员方法 """
+def __2_openpyxl_class():
+    """ 对openpyxl已有类的功能的增强 """
 
 
 class XlCell(openpyxl.cell.cell.Cell):  # 适用于 openpyxl.cell.cell.MergedCell，但这里不能多重继承
@@ -1632,80 +1645,11 @@ class XlWorkbook(openpyxl.Workbook):
 
         return workbook_summary
 
-    def extract_summary3(self):
-        """ 另一套按照单元格提取摘要的程序 """
-        wb = self
-
-        all_sheets_summary = []
-
-        for ws in wb._sheets:  # 非数据表，也要遍历出来，所以使用了_sheets
-            # 如果是标准工作表（Worksheet），使用现有的摘要提取机制
-            if isinstance(ws, openpyxl.worksheet.worksheet.Worksheet):
-                # 找到使用范围和表头范围
-                raw_used_range = ws.get_raw_usedrange()
-                if raw_used_range:
-                    used_range = ws.get_usedrange()
-                    summary = ({
-                        "sheetName": ws.title,
-                        "sheetType": "Worksheet",
-                        "rawUsedRange": raw_used_range,
-                        "usedRange": used_range,
-
-                    })
-                    summary['tables'] = ws.find_tables(used_range)
-
-                    # 'cells': extract_cells_content(ws)  # todo 改进点
-
-                    # todo 231213周三21:36，好久没看，我都不确定这种类型问题了，后面要再核对一遍
-                    if not summary['cells']:  # 如果没有数据，则大概率是数据透视表，是计算出来的，读取不到~
-                        summary['sheetType'] = 'PivotTable'
-                        del summary['cells']
-                else:
-                    summary = ({
-                        "sheetName": ws.title,
-                        "sheetType": "DialogOrMacroSheet",
-                        "usedRange": None,
-                    })
-
-            # 如果是其他类型的工作表，提供基础摘要
-            else:
-                summary = ({
-                    "sheetName": ws.title,
-                    "sheetType": ws.__class__.__name__  # 使用工作表的类名作为类型
-                })
-
-            all_sheets_summary.append(summary)
-
-        workbook_summary = {
-            "fileName": Path(self.path).name if self.path else None,
-            "sheetNames": wb.sheetnames,
-            "sheets": all_sheets_summary,
-        }
-
-        return workbook_summary
-
 
 inject_members(XlWorkbook, openpyxl.Workbook)
 
 
-def __3_hash():
-    """ 表格的哈希表达 """
-
-
-def excel2md5(file, reduction_degree=1):
-    try:
-        wb = openpyxl.load_workbook(file)
-    except (ValueError, TypeError) as e:
-        # 有些表格直接读取会失败，但使用read_only就能读了
-        wb = openpyxl.load_workbook(file, read_only=True)
-    except Exception as e:  # 还有其他zipfile.BadZipFile等错误
-        print(file, str(e))
-        return ''
-
-    return wb.to_md5(reduction_degree)
-
-
-def __4_extract_summary():
+def __3_extract_summary():
     """ 提取表格摘要 """
 
 
@@ -1911,6 +1855,52 @@ def extract_header_structure(ws, header_range):
     return header_structure
 
 
+def determine_field_type_and_summary(ws, col, start_row, end_row, rows):
+    """ 根据指定的列范围确定字段的摘要信息
+
+    :param rows: 由外部传入要抽样的数据编号
+    """
+    # 1 需要全量读取数据，获知主要格式，和数值范围
+    data = defaultdict(list)
+    for i in range(start_row, end_row + 1):
+        cell = ws.cell(i, col)
+        k, v = cell.get_number_format(), cell.value
+        data[k].append(v)
+
+    data2 = sorted(data.items(), key=lambda item: len(item[1]), reverse=True)
+    number_formats = [x[0] for x in data2]
+
+    # 2 获得要展示的样本值
+    sample_values = []
+    for i in rows:
+        cell = ws.cell(i, col)
+        value = cell.get_render_value()
+        if isinstance(value, str) and len(value) > 20:
+            value = value[:17] + '...'
+        sample_values.append(value)
+
+    # 3 数值范围（只要判断主类型的数值范围就行了）
+    numeric_range = None
+    for x in data2:
+        try:
+            fmt, values = x
+            values = [v for v in values if (v is not None and not isinstance(v, str))]
+            numeric_range = [min(values), max(values)]
+            numeric_range[0] = xl_render_value(numeric_range[0], fmt)
+            numeric_range[1] = xl_render_value(numeric_range[1], fmt)
+            break
+        except (TypeError, ValueError) as e:
+            pass
+
+    summary = {
+        "number_formats": number_formats,
+        "numeric_range": numeric_range,
+        "sample_values": sample_values,
+    }
+
+    return summary
+
+
 def extract_cells_content(ws):
     """ 提取一个工作表中的所有单元格内容 """
     cells = {}
@@ -1965,52 +1955,6 @@ def extract_field_summaries(ws, header_range, data_range, samples_num=5):
     return field_summaries
 
 
-def determine_field_type_and_summary(ws, col, start_row, end_row, rows):
-    """ 根据指定的列范围确定字段的摘要信息
-
-    :param rows: 由外部传入要抽样的数据编号
-    """
-    # 1 需要全量读取数据，获知主要格式，和数值范围
-    data = defaultdict(list)
-    for i in range(start_row, end_row + 1):
-        cell = ws.cell(i, col)
-        k, v = cell.get_number_format(), cell.value
-        data[k].append(v)
-
-    data2 = sorted(data.items(), key=lambda item: len(item[1]), reverse=True)
-    number_formats = [x[0] for x in data2]
-
-    # 2 获得要展示的样本值
-    sample_values = []
-    for i in rows:
-        cell = ws.cell(i, col)
-        value = cell.get_render_value()
-        if isinstance(value, str) and len(value) > 20:
-            value = value[:17] + '...'
-        sample_values.append(value)
-
-    # 3 数值范围（只要判断主类型的数值范围就行了）
-    numeric_range = None
-    for x in data2:
-        try:
-            fmt, values = x
-            values = [v for v in values if (v is not None and not isinstance(v, str))]
-            numeric_range = [min(values), max(values)]
-            numeric_range[0] = xl_render_value(numeric_range[0], fmt)
-            numeric_range[1] = xl_render_value(numeric_range[1], fmt)
-            break
-        except (TypeError, ValueError) as e:
-            pass
-
-    summary = {
-        "number_formats": number_formats,
-        "numeric_range": numeric_range,
-        "sample_values": sample_values,
-    }
-
-    return summary
-
-
 class WorkbookSummary:
     """ 工作薄摘要相关处理功能 """
 
@@ -2233,8 +2177,6 @@ def extract_workbook_summary2(file_path, *,
         # todo py好像没办法提取Selection。但jsa、vba应该要尽力取出这些相关的特征，尤其在操作等场景很有用
         # res['SelectionAddress'] = ...
 
-    # todo 摘要精简？
-
     return res
 
 
@@ -2541,10 +2483,91 @@ class WorkbookSummary3:
         return y
 
     @classmethod
-    def reduce4b(cls, y, summary_limit_len, *, cur_summary_len=None):
+    def reduce4b(cls, y, summary_limit_len, *, cur_summary_len=None, active_sheet_weight=0.5):
         """
+        :param active_sheet_weight: 当前活动表格被删除的权重，0.5表示按比例被删除的量只有其他表格的一半
         """
-        pass
+        if cur_summary_len is None:
+            cur_summary_len = len(json.dumps(y, ensure_ascii=False))
+
+        active_sheet = y['ActiveSheet']
+
+        # 1 预计要删除单元格数
+        sheet_cells_num = [len(st['cells']) for st in y['sheets']]
+        # 每个sheet本身其他摘要，按照5个单元格估算
+        total_cells_num = sum(sheet_cells_num) + len(sheet_cells_num) * 5
+        avg_cell_len = cur_summary_len / total_cells_num
+        # 目标删除单元格数量，向上取整
+        target_reduce_cells_num = int((cur_summary_len - summary_limit_len) / avg_cell_len + 0.5)
+
+        # 2 对当前活动表格，会减小删除权重
+        # 标记当前活动表格的单元格数
+        active_sheet_index = [i for i, st in enumerate(y['sheets']) if st['sheetName'] == active_sheet][0]
+        active_cells_num = sheet_cells_num[active_sheet_index]
+
+        # 计算权重系数
+        w = active_sheet_weight  # 当前激活表的权重系数
+        m = active_cells_num
+        n = total_cells_num
+        r = target_reduce_cells_num / n
+
+        # 计算非活动表格的额外权重系数
+        w2 = 1 + m * (1 - w) / (n - m)
+
+        # 3 所有的单元格如果都不够删，那就先把所有cells删了再说
+        if total_cells_num < target_reduce_cells_num:
+            for st in y['sheets']:
+                st['cells'] = {}
+            return len(json.dumps(y, ensure_ascii=False))
+
+        # 4 否则每张表按照比例删单元格，只保留前面部分的单元格
+        left_rate = 1 - r  # 原始保留比例
+        while True:
+            for i, st in enumerate(y['sheets']):
+                if i == active_sheet_index:
+                    # 当前激活的sheet保留更多单元格
+                    st['cells'] = dict(islice(st['cells'].items(), int(left_rate * w * sheet_cells_num[i])))
+                else:
+                    # 其他sheet按照w2权重删除单元格
+                    st['cells'] = dict(islice(st['cells'].items(), int(left_rate * w2 * sheet_cells_num[i])))
+            cur_summary_len = len(json.dumps(y, ensure_ascii=False))
+            if cur_summary_len <= summary_limit_len:
+                return cur_summary_len
+            if left_rate * total_cells_num < 1:
+                break
+            else:
+                left_rate *= 0.8  # 缩小保留比例，再试
+
+        return cur_summary_len
+
+    @classmethod
+    def reduce5b(cls, y, summary_limit_len, *, cur_summary_len=None):
+        """ 计算平均每张表的长度，保留前面部分的表格 """
+        if cur_summary_len is None:
+            cur_summary_len = len(json.dumps(y, ensure_ascii=False))
+
+        n = len(y['sheets'])
+        active_sheet_name = y['ActiveSheet']
+
+        avg_sheet_len = cur_summary_len / n
+        # target_reduce_sheet_num = int((cur_summary_len - summary_limit_len) / avg_sheet_len + 0.5)
+        # y['sheets'] = y['sheets'][:n - target_reduce_sheet_num]
+
+        while y['sheets']:
+            cur_summary_len = len(json.dumps(y, ensure_ascii=False))
+            if cur_summary_len <= summary_limit_len:
+                return cur_summary_len
+
+            # 如果最后一张表格是激活的表格，尝试删除前一张
+            if y['sheets'][-1]['sheetName'] == active_sheet_name:
+                if len(y['sheets']) > 1:
+                    y['sheets'] = y['sheets'][:-2] + [y['sheets'][-1]]
+                else:
+                    y['sheets'] = []
+            else:
+                y['sheets'] = y['sheets'][:-1]  # 删除最后一张表格的详细信息
+
+        return cur_summary_len
 
     @classmethod
     def summary2_to_summary3b(cls, summary2, summary_limit_len=4000):
@@ -2585,13 +2608,13 @@ class WorkbookSummary3:
                 return y
 
             # 4 每张表都按比例删除后面部分的单元格
-            cur_summary_len = cls.reduce4_truncate_cells(y, summary_limit_len, cur_summary_len=cur_summary_len)
+            cur_summary_len = cls.reduce4b(y, summary_limit_len, cur_summary_len=cur_summary_len)
             if cur_summary_len <= summary_limit_len:
                 y['mode'] = ', '.join(mode_tags[:4])
                 return y
 
             # 5 从后往前删每张表格的详细信息
-            cls.reduce5_truncate_sheets(y, summary_limit_len, cur_summary_len=cur_summary_len)
+            cls.reduce5b(y, summary_limit_len, cur_summary_len=cur_summary_len)
             y['mode'] = ', '.join(mode_tags[:5])
             return y
 
@@ -2599,8 +2622,8 @@ class WorkbookSummary3:
         y = {
             'fileName': x['fileName'],
             'sheetNames': x['sheetNames'],
-            'ActiveSheet': '',  # 当期激活的工作表
-            'SelectionAddress': '',
+            'ActiveSheet': x['ActiveSheet'],  # 当期激活的工作表
+            # 'SelectionAddress': '',
             'sheets': x['sheets'],
             'mode': 'Complete information',
         }
@@ -2622,7 +2645,7 @@ class WorkbookSummary3:
 
 def extract_workbook_summary3(file_path, summary_limit_len=4000, **kwargs):
     """ 增加了全局ratio的计算 """
-    data = extract_workbook_summary2plus(file_path, **kwargs)
+    data = extract_workbook_summary2(file_path, **kwargs)
     if not data:
         return data
     data = WorkbookSummary3.summary2_to_summary3(data, summary_limit_len)
@@ -2631,7 +2654,7 @@ def extract_workbook_summary3(file_path, summary_limit_len=4000, **kwargs):
 
 def extract_workbook_summary3b(file_path, summary_limit_len=4000, **kwargs):
     """ 增加了全局ratio的计算 """
-    data = extract_workbook_summary2plus(file_path, mode=1, **kwargs)
+    data = extract_workbook_summary2(file_path, mode=1, **kwargs)
     if not data:
         return data
     data = WorkbookSummary3.summary2_to_summary3b(data, summary_limit_len)

{pyxllib-0.3.88 → pyxllib-0.3.89}/pyxllib/prog/pupil.py

@@ -1000,7 +1000,17 @@ def xlmd5(content):
 
 @run_once()
 def get_hostname():
-    return socket.getfqdn()
+    hostname = socket.getfqdn()
+    return hostname
+
+
+@run_once()
+def get_hostname2():
+    """ 更加定制化的操作 """
+    hostname = socket.getfqdn()
+    hostname = hostname.replace('-', '_')
+    hostname = hostname.split('.')[0]
+    return hostname
 
 
 @run_once()

{pyxllib-0.3.88 → pyxllib-0.3.89/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.88
+Version: 0.3.89
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.88 → pyxllib-0.3.89}/setup.py

@@ -30,7 +30,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.88',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.89',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',