pyxllib 0.3.126__tar.gz → 0.3.128__tar.gz

{pyxllib-0.3.126/pyxllib.egg-info → pyxllib-0.3.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.126
+Version: 0.3.128
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/data/pglib.py

@@ -194,7 +194,7 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
     @classmethod
     def cvt_type(cls, val):
         if isinstance(val, (dict, list)):
-            val = json.dumps(val, ensure_ascii=False)
+            val = json.dumps(val, ensure_ascii=False, default=str)
         # 注意list数组类型读、写都会自动适配py
         return val
 

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/ext/drissionlib.py

@@ -74,7 +74,11 @@ def get_dp_tab(dp_page=None) -> 'XlPage':
     elif dp_page is True:
         return ChromiumPage().new_tab()
     elif isinstance(dp_page, str):
-        return ChromiumPage().new_tab(dp_page)
+        def dp_page2(tab):  # 默认开启页面复用。 todo 以后可以改进成可选参数
+            if tab.url.startswith(dp_page):
+                return tab
+
+        return get_dp_page(dp_page2)
     else:
         return ChromiumPage().latest_tab
 

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/ext/yuquelib.py

@@ -197,7 +197,7 @@ class Yuque:
         elif return_mode == 'md':
             return res['data']['body']
         elif return_mode == 'title_and_md':
-            return f'文章标题：{res["data"]["title"]}\n\n{res["data"]["body"]}'
+            return res["data"]["title"], res["data"]["body"]
 
     def export_markdown(self, url, output_dir=None, post_mode=1):
         """ 导出md格式文件

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/file/xlsxlib.py

@@ -3018,12 +3018,51 @@ def extract_workbook_summary3(file_path, summary_limit_len=4000, **kwargs):
     return data
 
 
+def summary2_add_enums(summary2, enum_values):
+    # 1 预备
+    if enum_values is True:
+        enum_values = (20, 10)
+    max_len, max_num = enum_values
+
+    # 2 枚举值
+    for sheet in summary2['sheets']:
+        # 2.1 遍历计数
+        cols = defaultdict(Counter)
+        for addr, val in sheet['cells'].items():
+            n = len(str(val))
+            if not n or n > max_len:
+                continue
+            col = re.match(r'[A-Z]+', addr).group()
+            cols[col][val] += 1
+
+        # 2.2 添加枚举值列
+        enums = {}
+        keys = sorted(cols.keys(), key=column_index_from_string)
+        for k in keys:
+            ct = cols[k]
+            if len(ct) > max_num:
+                continue
+            vals = ct.most_common()
+            if vals[0][1] == 1:  # 都只出现了一次，也不认为是枚举值，跳过。或者是小数据表，一般也能全量展示。
+                continue
+            enums[k] = [v for v, _ in vals]
+
+        # 2.3 保存
+        if enums:
+            sheet['enums'] = enums
+            # enums2 = json.dumps(enums, ensure_ascii=False, default=str)
+            # sheet['enums'] = json.loads(enums2)
+
+    return summary2
+
+
 def extract_workbook_summary3b(file_path,
                                summary_limit_len=4000,
                                timeout_seconds=10,
                                return_mode=0,
                                debug=False,
                                len_mode=0,
+                               enum_values=False,
                                **kwargs):
     """
 
@@ -3036,6 +3075,9 @@ def extract_workbook_summary3b(file_path,
     :param len_mode:
         0, 使用len作为token长度评估
         1, 使用模型评估实际token长度
+    :param enum_values: 是否展示每列枚举值
+        False, 默认不展示
+        True, 展示，并且默认参数 (20, 10) 表示长度超过20的丢弃，只保留枚举类型不超过10种值的列
     :param kwargs: 其他是summary2读取文件的时候的参数，其实都不太关键，一般不用特地设置
     """
     res = {}
@@ -3056,7 +3098,10 @@ def extract_workbook_summary3b(file_path,
             res, load_time = extract_workbook_summary2(file_path, mode=1, return_mode=1, **kwargs)
             # res = convert_to_json_compatible(res)
             summary2_res = copy.deepcopy(res)
+            if enum_values:
+                res = summary2_add_enums(res, enum_values)
             summary2_time = time.time() - start_time - load_time
+
             start_time = time.time()
             if len_mode == 1:
                 res = WorkbookSummary3plus.summary2_to_summary3b(res, summary_limit_len)

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/prog/pupil.py

@@ -603,6 +603,7 @@ def inject_members(from_obj, to_obj, member_list=None, *,
     # 把XlDocxTable的成员方法绑定到docx.table.Table里
     >> inject_members(XlDocxTable, docx.table.Table)
 
+    240826周一，其他可参考学习的三方现成工具库：from fastcore.foundation import patch
     """
     # 1 整理需要注入的方法清单
     dst = set(dir(to_obj))
@@ -1029,10 +1030,13 @@ class MultiProcessLauncher:
         if name is None:
             cmd2 = cmd.split() if isinstance(cmd, str) else cmd
             name = cmd2[0]
-
-        proc = subprocess.Popen(cmd,
-                                shell=shell,
-                                preexec_fn=self._set_pdeathsig())
+        if sys.platform == 'darwin':
+            proc = subprocess.Popen(cmd,
+                                    shell=shell)
+        else:
+            proc = subprocess.Popen(cmd,
+                                    shell=shell,
+                                    preexec_fn=self._set_pdeathsig())
 
         worker = ProcessWorker(name, proc, raw_cmd=cmd, **kwargs)
         self.workers.append(worker)
@@ -1118,6 +1122,73 @@ class MultiProcessLauncher:
                 print_full_dataframe(df)
 
 
+def support_multi_processes_hyx(default_processes=1):
+    """ 对函数进行扩展，支持并发多进程运行
+    增加重跑
+    注意被装饰的函数，需要支持 process_count、process_id 两个参数，来获得总进程数，当前进程id的信息
+    """
+
+    def decorator(func):
+
+        def wrapper(*args, **kwargs):
+            process_count = int(kwargs.pop('process_count', default_processes))
+            process_id = kwargs.pop('process_id', None)
+            shell = kwargs.pop('shell', False)
+
+            if process_count == 1 or process_id is not None:
+                if process_id is None:
+                    return func(*args, **kwargs)
+                else:
+                    return func(*args, **kwargs, process_count=process_count, process_id=int(process_id))
+            else:
+                mpl = MultiProcessLauncher()
+                for i in range(int(process_count)):
+                    if isinstance(process_id, int) and i != process_id:
+                        continue
+
+                    '''
+                    sys.argv[0] 为 /Users/youx/NutstoreCloudBridge/slns/xlproject/xlproject/m2404ragdata/b清洗/hyx240806统计图表数.py
+                    将其转化为 xlproject.m2404ragdata.b清洗.hyx240806统计图表数 
+                    '''
+                    header = 'xlproject.code4101'
+
+                    root_directory_name = 'xlproject'
+                    occurrence = 2
+                    path = sys.argv[0]
+
+                    # 查找第 occurrence 次出现的 root_directory_name 位置
+                    positions = [i for i in range(len(path)) if path.startswith(root_directory_name, i)]
+                    if len(positions) < occurrence:
+                        raise ValueError(
+                            f"Path does not contain {occurrence} occurrences of the root directory name: {root_directory_name}")
+
+                    # 提取并转换为模块名称
+                    index = positions[occurrence - 1]
+                    module_name = os.path.splitext(path[index:])[0].replace(os.path.sep, '.')
+
+                    # todo 这样使用有个坑，process_count、process_id都是以str类型传入的，开发者下游使用容易出问题
+                    cmds = [
+                            'run_python_module',
+                            '--wait_mode',
+                            '60',
+                            module_name,
+                            func.__name__,
+                            '--process_count', str(process_count),
+                            '--process_id', str(i)
+                            ]
+                    cmds.extend(map(str, args))  # 添加位置参数
+                    for k, v in kwargs.items():
+                        cmds.append(f'--{k}')  # 添加关键字参数的键
+                        cmds.append(str(v))  # 添加关键字参数的值
+
+                    mpl.add_process_python_module(header, cmds, shell=shell)
+                mpl.run_endless()
+
+        return wrapper
+
+    return decorator
+
+
 def support_multi_processes(default_processes=1):
     """ 对函数进行扩展，支持并发多进程运行
 
@@ -1151,6 +1222,7 @@ def support_multi_processes(default_processes=1):
                     for k, v in kwargs.items():
                         cmds.append(f'--{k}')  # 添加关键字参数的键
                         cmds.append(str(v))  # 添加关键字参数的值
+
                     mpl.add_process_python(sys.argv[0], cmds, shell=shell)
                 mpl.run_endless()
 

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib/prog/specialist/xllog.py

@@ -53,8 +53,7 @@ def get_xllog(name='xllog', *, log_file=None):
             xllog = logging.getLogger('pyxllib.xllog')
             fmt = '%(asctime)s %(message)s'
             if log_file:
-                # TODO 这里的格式设置是否能统一、精简？
-                file_handler = logging.FileHandler(f'{log_file}', 'w')
+                file_handler = logging.FileHandler(f'{log_file}', 'a')
                 file_handler.setLevel(logging.DEBUG)
                 file_handler.setFormatter(logging.Formatter(fmt))
                 xllog.addHandler(file_handler)

{pyxllib-0.3.126 → pyxllib-0.3.128/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.126
+Version: 0.3.128
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxllib.egg-info/requires.txt

@@ -3,7 +3,7 @@ charset-normalizer
 requests
 qiniu
 pyyaml
-coloredlogs
+loguru
 humanfriendly
 tqdm
 ujson
@@ -13,39 +13,41 @@ envariable
 decorator
 more-itertools
 fire
+xlrd
 pympler
 lxml
 beautifulsoup4
 numpy
 Jinja2
 pandas>=0.23.4
+pillow
 
 [xlai]
-pynvml
+pillow
 visdom
 opsdroid-get-image-size
 opencv-python
-pillow
+pynvml
 
 [xlall]
-arts
-flask-cors
+pillow
+fastapi
+xlrd
+flask-wtf
+sentencepiece
+flask
 zhconv
 ujson
-html2text
-uvicorn
-opsdroid-get-image-size
 flask-jwt-extended
-xlrd
-fastapi
+arts
+uvicorn
 flask-restful
+opsdroid-get-image-size
+flask-cors
 opencv-python
-flask-wtf
-flask
-pillow
-sentencepiece
+html2text
 
 [xlcv]
-opencv-python
 pillow
+opencv-python
 opsdroid-get-image-size

{pyxllib-0.3.126 → pyxllib-0.3.128}/pyxlpr/openai2/_core/chat.py

@@ -16,6 +16,8 @@ from unittest.mock import MagicMock
 import math
 from types import SimpleNamespace
 
+import requests
+from loguru import logger as loguru_logger
 from tqdm import tqdm
 from joblib import Parallel, delayed
 from openai import OpenAI, AsyncOpenAI
@@ -143,7 +145,7 @@ class Chat:
                  model: Literal["gpt-4-1106-preview", "gpt-4-vision-preview",
                  "gpt-4", "gpt-4-0314", "gpt-4-0613",
                  "gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613", "gpt-3.5-turbo",
-                 "gpt-4o", "gpt-4-turbo"] = "gpt-3.5-turbo",
+                 "gpt-4o", "gpt-4-turbo"] = "gpt-4o-mini",
                  # Chat
                  msg_max_count: int = None,
                  # kwargs
@@ -289,20 +291,35 @@ class Chat:
         self._messages.add_many(*messages)
 
     def __getattr__(self, name):
-        match name:  # 兼容旧代码
-            case 'asy_request':
-                return self.async_request
-            case 'forge':
-                return self.add_dialogs
-            case 'pin':
-                return self.pin_messages
-            case 'unpin':
-                return self.unpin_messages
-            case 'dump':
-                return self._dump
-            case 'load':
-                return self._load
-        raise AttributeError(name)
+        # match name:  # 兼容旧代码
+        #     case 'asy_request':
+        #         return self.async_request
+        #     case 'forge':
+        #         return self.add_dialogs
+        #     case 'pin':
+        #         return self.pin_messages
+        #     case 'unpin':
+        #         return self.unpin_messages
+        #     case 'dump':
+        #         return self._dump
+        #     case 'load':
+        #         return self._load
+        # raise AttributeError(name)
+
+        if name == 'asy_request':
+            return self.async_request
+        elif name == 'forge':
+            return self.add_dialogs
+        elif name == 'pin':
+            return self.pin_messages
+        elif name == 'unpin':
+            return self.unpin_messages
+        elif name == 'dump':
+            return self._dump
+        elif name == 'load':
+            return self._load
+        else:
+            raise AttributeError(name)
 
     def _dump(self, fpath: str):
         """ 存档 """
@@ -480,6 +497,106 @@ class Chat:
         return image_url
 
 
+class Chat2:
+    default_api_key = None
+    default_base_url = 'https://beta.gpt4api.plus'
+
+    def __init__(self, api_key=None,
+                 gizmo_id=None,
+                 *,
+                 base_url=None,
+                 model='gpt-4o-mini'):
+        """
+        :param gizmo_id: gpts的id，比如 https://chatgpt.com/g/g-ABCD1234-ce-shi。则id就是"g-ABCD1234"
+            后面的gpts的名称变了没关系，前面前缀的id不要变就行
+        """
+        self.api_key = api_key or self.default_api_key
+        self.base_url = base_url or self.default_base_url
+        self.gizmo_id = gizmo_id  # 是否有action模块需要点击确认
+
+        self.headers = {
+            'Authorization': f'Bearer {self.api_key}'
+        }
+        self.session_state = {
+            'stream': False,
+            'model': model,
+        }
+
+        self.messages = list()  # 历史记录
+
+    def _upload_file(self, file_path, file_type='my_files', parent_payload=None):
+        """ 上传文件
+
+        :param payload: 如果传入外部主payload自动，会自动添加已上传的文件的状态内容信息
+        """
+        # 1 请求
+        url = f"{self.base_url}/chat/uploaded"
+        payload = {'type': file_type}
+        if 'conversation_id' in self.session_state:
+            payload['conversation_id'] = self.session_state['conversation_id']
+        files = [
+            ('files', (Path(file_path).name, open(file_path, 'rb')))
+        ]
+        resp = requests.post(url, headers=self.headers, data=payload, files=files)
+        resp_json = resp.json()
+
+        # 2 保存状态
+        self.session_state['conversation_id'] = resp_json['conversation_id']
+        self.messages.append(resp_json)
+        if parent_payload:
+            key_name = 'attachments' if file_type == 'my_files' else 'parts'
+            if key_name not in parent_payload:
+                parent_payload[key_name] = []
+            parent_payload[key_name].append(resp_json[key_name[:-1]])
+
+        return resp_json
+
+    def _upload_image(self, image_path, parent_payload=None):
+        """ 上传图片 """
+        return self._upload_file(image_path, 'multimodal', parent_payload=parent_payload)
+
+    def query(self, message, *, files=None, images=None):
+        # 1 预备
+        if self.gizmo_id:
+            url = f"{self.base_url}/chat/action"
+        else:
+            url = f"{self.base_url}/chat/all-tools"
+
+        headers = self.headers.copy()
+        headers['Content-Type'] = 'application/json'
+
+        # 2 api初步配置
+        payload = self.session_state.copy()
+        payload['message'] = message
+        if self.gizmo_id:
+            payload['gizmo_id'] = self.gizmo_id
+
+        # 3 添加文件
+        if files:
+            if isinstance(files, str):
+                files = [files]
+            for file in files:
+                self._upload_file(file, parent_payload=payload)
+
+        # 4 添加图片
+        if images:
+            if isinstance(images, str):
+                images = [images]
+            for image in images:
+                self._upload_image(image, parent_payload=payload)
+
+        # 5 请求
+        resp = requests.post(url, headers=headers, data=json.dumps(payload))
+        resp_json = resp.json()
+
+        # 6 保存状态
+        self.session_state['conversation_id'] = resp_json['conversation_id']
+        self.session_state['parent_message_id'] = resp_json['message_id']
+        self.messages.append(resp_json)
+
+        return '\n\n'.join(resp_json['contents'][-1]['message']['content']['parts'])
+
+
 def __2_信息摘要():
     pass
 
@@ -580,8 +697,7 @@ class CompressContent:
               expect_len=None,
               max_round=-1,
               text_spliter=None,
-              style=None,
-              model='gpt-3.5-turbo',
+              model='gpt-4o-mini',
               soft_merge_prompt=None,
               calc_len=None,
               logger=None,
@@ -606,7 +722,6 @@ class CompressContent:
         :param func text_spliter: 文本分割器，默认使用均切
             均匀切分，在不超过segment_len的情况下，尽量保持每段文本长度相同
             后续要支持langchain里分割算法，觉得那个切法更泛用，虽然效率估计更低
-        :param str style: 提取风格。暂未实装，涉及到如何跟已有prompts配合。
         :param str model: 使用的模型
         :param soft_merge_prompt: 最后一轮合并后，如果已经满足要求，是否要加一段提示词，再优化下描述，避免暴力合并效果过于生硬
 
@@ -621,6 +736,9 @@ class CompressContent:
 
         expect_len = expect_len or segment_len // 2
         logger = logger or MagicMock()
+        if logger is True:
+            logger = loguru_logger
+
         calc_len = calc_len or len
 
         if text_spliter is None:
@@ -641,22 +759,21 @@ class CompressContent:
                 contents2.extend(text_spliter(content))
 
             # 3 单个提取的接口
-            def extract_info(content, prompt):
+            def extract_info(i, content, prompt):
                 chat = Chat(model=model)
                 chat.add_system_prompt(prompt)
                 summary = chat.query(content)
-                logger.info(f'>>> 原始内容：{content}\n\n>>> 提取信息：{summary}\n\n')
+                logger.debug(f'>>> 第{i}块原始内容：\n{content}\n\n>>> 提取信息：{summary}\n\n')
                 return summary
 
             # 4 (并发)提取信息
             prompt_idx = min(round_num, len(extra_prompts) - 1)
             backend = 'threading' if max_workers != 1 else 'sequential'
             parallel = Parallel(n_jobs=max_workers, backend=backend, return_as='generator')
-            tasks = [delayed(extract_info)(c, extra_prompts[prompt_idx]) for c in contents2]
-            disable = False if ((not isinstance(logger, MagicMock)) and logger.level <= logging.INFO) else True
-            # disable = False
+            logger.debug(f'>>> 提取信息轮次：{round_num}，使用提示词：\n{extra_prompts[prompt_idx]}')
+            tasks = [delayed(extract_info)(i, c, extra_prompts[prompt_idx]) for i, c in enumerate(contents2, start=1)]
             contents = list(tqdm(parallel(tasks), total=len(contents2),
-                                 desc=f'round {round_num} 提取信息', disable=disable))
+                                 desc=f'round {round_num} 提取信息', disable=isinstance(logger, MagicMock)))
 
             # 5 合并数据
             contents = ['\n\n'.join(contents)]
@@ -682,8 +799,13 @@ class CompressContent:
                          **kwargs)
 
     @classmethod
-    def with_query(cls, content, query, extra_prompts=None, **kwargs):
-        """ 通用的信息压缩，根据用户的问题，进行信息压缩 """
+    def with_query(cls, content, query=None, extra_prompts=None, **kwargs):
+        """ 通用的信息压缩，根据用户的问题，进行信息压缩
+        但如果没传入query参数，也会自动改回common机制处理
+        """
+        if not query:
+            return cls.common(content, extra_prompts, **kwargs)
+
         return cls.basic(content,
                          extra_prompts or xlprompts.compress_content_with_query.render(query=query),
                          **kwargs)

{pyxllib-0.3.126 → pyxllib-0.3.128}/requirements.txt

@@ -3,7 +3,7 @@ charset-normalizer
 requests
 qiniu
 pyyaml
-coloredlogs
+loguru
 humanfriendly
 tqdm
 ujson
@@ -13,10 +13,11 @@ envariable
 decorator
 more-itertools
 fire
-
+xlrd
 pympler
 lxml
 beautifulsoup4
 numpy
 Jinja2
-pandas>=0.23.4
+pandas>=0.23.4
+pillow

{pyxllib-0.3.126 → pyxllib-0.3.128}/setup.py

@@ -48,7 +48,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.126',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.128',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',