pyxllib 0.3.129__tar.gz → 0.3.131__tar.gz

pyxllib-0.3.131/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: pyxllib
+Version: 0.3.131
+Summary: 厦门理工模式识别团队通用python代码工具库
+Home-page: https://github.com/XLPRUtils/pyxllib
+Author: code4101
+Author-email: 877362867@qq.com
+License: Apache License 2.0
+Keywords: pyxllib,pyxlpr
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: arrow
+Requires-Dist: charset-normalizer
+Requires-Dist: requests
+Requires-Dist: qiniu
+Requires-Dist: pyyaml
+Requires-Dist: loguru
+Requires-Dist: humanfriendly
+Requires-Dist: tqdm
+Requires-Dist: ujson
+Requires-Dist: Deprecated
+Requires-Dist: pyperclip
+Requires-Dist: envariable
+Requires-Dist: decorator
+Requires-Dist: more-itertools
+Requires-Dist: fire
+Requires-Dist: xlrd
+Requires-Dist: pympler
+Requires-Dist: lxml
+Requires-Dist: beautifulsoup4
+Requires-Dist: numpy
+Requires-Dist: Jinja2
+Requires-Dist: pandas>=0.23.4
+Requires-Dist: pillow
+Provides-Extra: xlcv
+Requires-Dist: opsdroid-get-image-size; extra == "xlcv"
+Requires-Dist: opencv-python; extra == "xlcv"
+Requires-Dist: pillow; extra == "xlcv"
+Provides-Extra: xlai
+Requires-Dist: pynvml; extra == "xlai"
+Requires-Dist: visdom; extra == "xlai"
+Requires-Dist: opencv-python; extra == "xlai"
+Requires-Dist: pillow; extra == "xlai"
+Requires-Dist: opsdroid-get-image-size; extra == "xlai"
+Provides-Extra: xlall
+Requires-Dist: flask-cors; extra == "xlall"
+Requires-Dist: arts; extra == "xlall"
+Requires-Dist: uvicorn; extra == "xlall"
+Requires-Dist: openai2; extra == "xlall"
+Requires-Dist: opencv-python; extra == "xlall"
+Requires-Dist: pillow; extra == "xlall"
+Requires-Dist: sentencepiece; extra == "xlall"
+Requires-Dist: xlrd; extra == "xlall"
+Requires-Dist: html2text; extra == "xlall"
+Requires-Dist: flask; extra == "xlall"
+Requires-Dist: flask-wtf; extra == "xlall"
+Requires-Dist: fastapi; extra == "xlall"
+Requires-Dist: zhconv; extra == "xlall"
+Requires-Dist: opsdroid-get-image-size; extra == "xlall"
+Requires-Dist: ujson; extra == "xlall"
+Requires-Dist: flask-jwt-extended; extra == "xlall"
+Requires-Dist: flask-restful; extra == "xlall"
+
+# 1 install
+
+```
+pip install pyxllib
+```
+
+more details: https://www.yuque.com/xlpr/pyxllib/install
+
+# 2 document
+
+document: https://www.yuque.com/xlpr/pyxllib.
+Work hard to improve, welcome more assistant partners.
+
+# 3 Acknowledge
+
+This project uses PyCharm to develop.
+Thanks to [JetBrains'](https://www.jetbrains.com/?from=pyxllib) support
+of [free license for open sourse project development](https://pycharm.iswbm.com/c01/c01_05.html).

{pyxllib-0.3.129 → pyxllib-0.3.131}/pyxllib/algo/intervals.py

@@ -883,7 +883,7 @@ class StrIdxBack:
         self.keystr = s
 
     def delchars(self, pattern, flags=0):
-        """模仿正则的替换语法
+        r""" 模仿正则的替换语法
         但是不用输入替换目标s，以及目标格式，因为都是删除操作
 
         利用正则可以知道被删除的是哪个区间范围

{pyxllib-0.3.129 → pyxllib-0.3.131}/pyxllib/algo/newbie.py

@@ -164,3 +164,57 @@ def cvsecs(time):
         return time
 
     return sum(mult * part for mult, part in zip(factors, reversed(time)))
+
+
+def find_optimal_page_size(total_items, page_sizes, seen_items=0):
+    """
+    找到最优的分页大小，使得最后一页的新样本数尽可能多，已见样本数尽可能少。
+    本函数由gpt-o1生成。
+
+    :param int total_items: 总的条目数（样本数）。
+    :param List[int] page_sizes: 可供选择的分页大小列表。
+    :param int seen_items: 已经看过的样本数，默认为0。
+    :return: Tuple[int, int, int], 包含最优的分页大小、最后一页的新样本数、最后一页的已见样本数。
+
+    >>> find_optimal_page_size(64, [2, 5, 10, 14, 20, 50, 100])
+    (100, 64, 0)
+    >>> find_optimal_page_size(64, [2, 5, 10, 14, 20, 50, 100], seen_items=55)
+    (50, 9, 5)
+    """
+    page_sizes = sorted(set(page_sizes), reverse=True)  # 去重并从大到小排序
+    max_new_items = -1
+    min_seen_items_in_page = total_items + 1  # 初始化为一个大于可能的最大值
+    optimal_page_size = None
+    total_new_items = total_items - seen_items  # 总的新样本数
+
+    for page_size in page_sizes:
+        if page_size <= 0:
+            continue  # 跳过非正整数的分页大小
+
+        total_pages = -(-total_items // page_size)  # 等价于 math.ceil(total_items / page_size)
+        start_index = (total_pages - 1) * page_size + 1
+        end_index = min(total_pages * page_size, total_items)
+        last_page_indices = range(start_index, end_index + 1)
+
+        # 计算新样本数和已见样本数
+        new_items_in_page = len([i for i in last_page_indices if i > seen_items])
+        seen_items_in_page = len([i for i in last_page_indices if i <= seen_items])
+
+        # 更新最优分页大小
+        if new_items_in_page > max_new_items:
+            max_new_items = new_items_in_page
+            min_seen_items_in_page = seen_items_in_page
+            optimal_page_size = page_size
+            # 如果已达到最大可能的新样本数且已见样本数为 0，提前退出
+            if max_new_items == total_new_items and min_seen_items_in_page == 0:
+                break
+        elif new_items_in_page == max_new_items and seen_items_in_page < min_seen_items_in_page:
+            min_seen_items_in_page = seen_items_in_page
+            optimal_page_size = page_size
+            if min_seen_items_in_page == 0:
+                break
+        # 如果当前的 page_size 已经小于或等于最大新样本数，后续的 page_size 不会有更好的结果，可以退出循环
+        elif page_size <= max_new_items:
+            break
+
+    return optimal_page_size, max_new_items, min_seen_items_in_page

{pyxllib-0.3.129 → pyxllib-0.3.131}/pyxllib/file/pupil.py

@@ -140,7 +140,7 @@ def gen_file_filter(s):
 
 
 def getfiles(root, filter_rule=None):
-    """对os.walk进一步封装，返回所有匹配的文件
+    r""" 对os.walk进一步封装，返回所有匹配的文件
 
     可以这样遍历一个目录下的所有文件：
     for f in getfiles(r'C:\pycode\code4101py', r'.py'):

{pyxllib-0.3.129 → pyxllib-0.3.131}/pyxllib/file/specialist/filelib.py

@@ -923,7 +923,7 @@ class XlPath(type(pathlib.Path())):
                 res.add(p.relative_to(self).as_posix())
         return res
 
-    def relpath(self, ref_dir) -> str:
+    def relpath(self, ref_dir) -> 'XlPath':
         r""" 当前路径，相对于ref_dir的路径位置
 
         >>> File('C:/a/b/c.txt').relpath('C:/a/')

{pyxllib-0.3.129 → pyxllib-0.3.131}/pyxllib/file/xlsxlib.py

@@ -766,7 +766,7 @@ class XlCell(openpyxl.cell.cell.Cell):  # 适用于 openpyxl.cell.cell.MergedCel
         # openpyxl的机制，如果没有配置日期格式，读取到的是默认的'mm-dd-yy'，其实在中文场景，默认格式应该是后者
         if fmt == 'mm-dd-yy':
             return 'yyyy/m/d'  # 中文的默认日期格式
-        elif fmt == 'yyyy\-mm\-dd':  # 不知道为什么会有提取到这种\的情况，先暴力替换了
+        elif fmt == r'yyyy\-mm\-dd':  # 不知道为什么会有提取到这种\的情况，先暴力替换了
             fmt = 'yyyy-mm-dd'
         return fmt
 
@@ -2870,7 +2870,7 @@ class WorkbookSummary3:
 
         # 4 否则每张表按照比例删单元格，只保留前面部分的单元格
         # todo 这里应该有更好的筛选机制，后续可以思考思考
-        left_rate = min((summary_limit_len + cur_summary_len) / 2 * cur_summary_len, 0.9)  # 首轮减小一点调整幅度
+        left_rate = min((summary_limit_len + cur_summary_len) / (2 * cur_summary_len), 0.9)  # 首轮减小一点调整幅度
         while True:
             for i, st in enumerate(y['sheets']):
                 if i == active_sheet_index:

pyxllib-0.3.131/pyxllib/file/xlsyncfile.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Author : 陈坤泽
+# @Email  : 877362867@qq.com
+# @Date   : 2024/10/12
+
+"""
+我们自己的一套利用后端服务进行文件同步的工具
+"""
+
+import os
+
+from tqdm import tqdm
+import requests
+
+from pyxllib.file.specialist import XlPath, GetEtag
+
+
+class SyncFileClient:
+    """
+
+    注意：该类填写的路径，一律是相对local_root、remote_root的路径，不是当前实际工作目录下的路径！
+        可以通过把local_host设为空的来实现一些特殊精细化的操作
+        但一般推荐还是都把local_root、remote_root设置上
+    """
+
+    def __1_basic(self):
+        pass
+
+    def __init__(self, host, token=None):
+        """
+        :param host: 可以只写主机名
+        """
+        token = token or os.environ.get("XL_COMMON_SERVER_TOKEN")
+        self.headers = {
+            'Authorization': f'Bearer {token}',
+        }
+        self.local_root = None
+        self.remote_root = None
+        self.host = self.link_host(host)
+
+    def link_host(self, hostname):
+        from requests.exceptions import Timeout
+
+        host = f'https://xmutpriu.com/{hostname}'
+        resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
+        ip = resp.json()['ip']
+
+        if ip:
+            try:
+                resp = requests.get(f'{ip}/{hostname}/healthy', timeout=5)
+                if resp.status_code == 200 and resp.json() == 'OK':
+                    host = f'{ip}/{hostname}'
+            except Timeout:
+                pass
+            except Exception as e:
+                raise e
+
+        remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
+        self.remote_root = XlPath(remote_root)
+
+        return host
+
+    def get_abs_local_path(self, local_path, remote_path=None):
+        # 1 如果需要，从远程路径推测本地路径
+        if local_path is None and remote_path is not None:
+            local_path = remote_path
+
+        # 2 本地存储的文件路径
+        if self.local_root:
+            return self.local_root / local_path
+        else:
+            return XlPath(local_path)
+
+    def get_abs_remote_path(self, remote_path, local_path=None):
+        # 1 如果需要，从本地路径推测远程路径
+        if remote_path is None and local_path is not None:
+            remote_path = local_path
+
+        # 2 远程存储的文件路径
+        if self.remote_root:
+            return self.remote_root / remote_path
+        else:
+            return XlPath(remote_path)
+
+    def __2_sync(self):
+        pass
+
+    def upload_file(self, local_file=None, remote_file=None, relpath=True):
+        """
+        :param str remote_file: 要存储的远程文件路径，默认可以不指定，自动通过local_file存储到对称位置
+
+        使用示例：
+        sfc = SyncFileClient('codepc_mi15')
+        sfc.upload_file('abc/README.md')
+        sfc.upload_file('abc/README.md', 'bcd/1.md')
+        """
+        file = [local_file]
+        if remote_file:
+            file.append(remote_file)
+        return self.upload_files([file], relpath=relpath)
+
+    def upload_files(self, files, relpath=True):
+        """
+        :param list[local_file<, remote_path>] files:
+            remote_path可选，未写的时候会自动填充
+        :return:
+
+        todo 加进度条？不过问了下gpt，这个实现好像有点复杂，先不折腾了。本来其实就推荐单个单个文件上传，在外部进行进度展示管控。
+
+        使用示例：
+        sfc = SyncFileClient('codepc_mi15')
+        print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
+        """
+        # 1 映射出目标路径
+        files2 = []
+        for file in files:
+            if not isinstance(file, (list, tuple)):
+                local_file, remote_file = file, None
+            else:
+                local_file, remote_file = file
+
+            if relpath:
+                local_file = self.get_abs_local_path(local_file, remote_file)
+                remote_file = self.get_abs_remote_path(remote_file, local_file)
+
+            files2.append([local_file, remote_file.as_posix()])
+
+        files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
+
+        # 4 发送请求
+        resp = requests.post(f'{self.host}/common/upload_files',
+                             headers=self.headers,
+                             files=files3)
+        return resp.json()
+
+    def download_file(self, remote_file=None, local_file=None, relpath=True):
+        """
+        :param str remote_file: 要下载的远程文件路径，相对get_wkdir下的路径
+        :param str local_file: 下载到本地的文件路径，默认可以不指定，自动通过remote_file存储到对称位置
+
+        使用示例：
+        sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
+        sfc.download_file('.vscode/launch.json', 'launch.json')
+        """
+        if relpath:
+            local_file = self.get_abs_local_path(local_file, remote_file)
+            remote_file = self.get_abs_remote_path(remote_file, local_file)
+
+        data = {'file': remote_file.as_posix()}
+        # 使用 stream=True 开启流式处理
+        with requests.post(f'{self.host}/common/download_file',
+                           headers=self.headers, json=data, stream=True) as resp:
+            if resp.status_code == 200:
+                # 以二进制方式写入文件
+                with open(local_file, 'wb') as f:
+                    # 分块写入文件，每次 1MB (1024 * 1024 bytes)
+                    for chunk in resp.iter_content(chunk_size=1024 * 1024):
+                        if chunk:  # 忽略keep-alive的空块
+                            f.write(chunk)
+                return local_file
+            else:
+                # 处理其他状态码，抛出异常或进行错误处理
+                print(f"Error: Unable to download file, status code {resp.status_code}")
+                return None
+
+    def verify_etag(self, local_file=None, remote_file=None, relpath=True):
+        """
+        :param relpath: 是否使用的是相对路径
+            当不使用relpath的时候，注意local_file、remote_file都是手动必填的XlPath类型字段
+        :return:
+        """
+        if relpath:
+            local_file = self.get_abs_local_path(local_file, remote_file)
+            remote_file = self.get_abs_remote_path(remote_file, local_file)
+        local_etag = GetEtag.from_file(local_file)
+        json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
+        resp = requests.post(f'{self.host}/common/check_file_etag',
+                             headers=self.headers,
+                             json=json_data)
+        return resp.json()
+
+    def upload_dir(self,
+                   local_dir=None,
+                   remote_dir=None,
+                   *,
+                   delete_local_file=False,
+                   verify_etag=False,
+                   relpath=True):
+        """
+        :param local_dir: 本地目录
+        :param remote_dir: 远程目录
+        :param delete_local_file: 每个文件上传成功后，是否删除本地文件
+            默认不删除，一般不删除是配合etag校验使用的
+        :param verify_etag: 上传前是否校验etag，如果服务器上对应位置已经有对应etag的文件，其实就不用上传了
+
+        该方法已修改以支持上传子目录和空目录
+        """
+        if relpath:
+            # D:\home\chenkunze\data\temp2，举个具体的例子案例值，供参考理解
+            local_dir = self.get_abs_local_path(local_dir, remote_dir)
+            # /home/chenkunze/data/temp2
+            remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
+
+        for root, dirs, files in os.walk(local_dir):
+            # root拿到的是这样的绝对路径：D:\home\chenkunze\data\temp2\数值问题
+            # '数值问题'
+            subdir = XlPath(root).relpath(local_dir)
+            if not files and not dirs:
+                # 如果该目录为空，则在远程服务器上创建空目录
+                self.create_remote_dir(remote_dir / subdir, relpath=False)
+
+            for file in tqdm(files, desc=f'上传目录：{root}'):
+                local_path = local_dir / subdir / file
+                remote_path = remote_dir / subdir / file
+                if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
+                    pass  # 如果etag校验通过，跳过上传
+                else:
+                    self.upload_file(local_path, remote_path, relpath=False)
+
+                if delete_local_file:
+                    os.remove(local_path)
+
+    def create_remote_dir(self, remote_dir, relpath=True):
+        """
+        创建远程目录的辅助方法
+        """
+        if relpath:
+            remote_dir = self.get_abs_remote_path(remote_dir)
+        json_data = {'dir': remote_dir.as_posix()}
+        resp = requests.post(f'{self.host}/common/create_dir',
+                             headers=self.headers,
+                             json=json_data)
+        return resp.json()
+
+    def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
+        """
+        下载远程目录到本地
+        :param remote_dir: 远程目录路径，相对路径或绝对路径都支持
+        :param local_dir: 本地保存目录，默认会自动映射到对应的远程位置
+        :param verify_etag: 是否进行etag校验，跳过已经在本地存在并且未修改的文件
+        :return:
+
+        示例：
+        sfc = SyncFileClient('codepc_mi15')
+        sfc.download_dir('data/temp2')
+        """
+        # 1. 获取绝对路径
+        if relpath:
+            local_dir = self.get_abs_local_path(local_dir, remote_dir)
+            remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
+
+        # 2. 从服务器获取远程目录的文件和子目录结构
+        json_data = {'dir': remote_dir.as_posix()}
+        resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
+        if resp.status_code != 200:
+            raise Exception(f"Failed to list remote directory: {resp.text}")
+
+        dir_structure = resp.json()  # 期望返回格式为 {'dirs': [...], 'files': [...]}
+
+        # 3. 创建本地目录
+        if not local_dir.exists():
+            local_dir.mkdir(parents=True)
+
+        # 4. 先处理子目录
+        for subdir in dir_structure.get('dirs', []):
+            local_subdir = local_dir / subdir
+            if not local_subdir.exists():
+                local_subdir.mkdir(parents=True)
+
+            # 递归下载子目录中的文件
+            self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
+
+        # 5. 下载文件
+        for file in tqdm(dir_structure.get('files', []), desc=f'下载目录：{remote_dir}'):
+            remote_file = remote_dir / file
+            local_file = local_dir / file
+
+            # 校验etag，如果verify_etag为True并且校验成功，则跳过下载
+            if verify_etag and self.verify_etag(local_file, remote_file, relpath=False):
+                continue
+
+            # 下载文件
+            self.download_file(remote_file, local_file, relpath=False)
+
+    def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
+        """
+        下载远程文件或目录到本地
+        :param remote_path: 远程路径，可以是文件或者目录
+        :param local_path: 本地保存路径，默认会自动映射到对应的远程位置
+        :param verify_etag: 是否进行etag校验，跳过已经在本地存在并且未修改的文件
+        :return:
+        """
+        if relpath:
+            # 获取远程路径的绝对路径
+            remote_path = self.get_abs_remote_path(remote_path, local_path)
+            local_path = self.get_abs_local_path(local_path, remote_path)
+
+        # 判断是文件还是目录
+        json_data = {'path': remote_path.as_posix()}
+        resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
+
+        if resp.status_code != 200:
+            raise Exception(f"Failed to check remote path type: {resp.text}")
+
+        path_info = resp.json()
+
+        if path_info['type'] == 'file':
+            # 如果是文件，调用 download_file
+            self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
+        elif path_info['type'] == 'dir':
+            # 如果是目录，调用 download_dir
+            self.download_dir(remote_dir=remote_path, local_dir=local_path,
+                              verify_etag=verify_etag, relpath=False)
+        else:
+            raise ValueError(f"Unknown path type: {path_info['type']}")