pyxllib 0.3.121__tar.gz → 0.3.123__tar.gz

{pyxllib-0.3.121/pyxllib.egg-info → pyxllib-0.3.123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.121
+Version: 0.3.123
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/data/pglib.py

@@ -58,13 +58,17 @@ class Connection(psycopg.Connection, SqlBase):
     def __1_库(self):
         pass
 
-    def get_db_activities(self):
+    def get_db_activities(self, datname=None):
         """
         检索当前数据库的活动信息。
+
+        :param datname: 这个字段理论上应该要能自动检测出来才对，但这会急着没空钻研，先手动输入吧
         """
         sql = SqlBuilder('pg_stat_activity')
         sql.select('pid', 'datname', 'usename', 'state', 'query', 'age(now(), query_start) AS "query_age"')
         sql.where("state = 'active'")
+        if datname:
+            sql.where(f"datname = '{datname}'")
         return self.exec2dict(sql.build_select()).fetchall()
 
     def __2_表格(self):

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/file/specialist/download.py

@@ -17,13 +17,20 @@ from pyxllib.file.specialist import File, Dir
 
 
 def download_file(url, fn=None, *, encoding=None, if_exists=None, ext=None, temp=False):
-    """ 类似writefile，只是源数据是从url里下载
+    r""" 类似writefile，只是源数据是从url里下载
 
     :param url: 数据下载链接
     :param fn: 保存位置，会从url智能提取文件名
     :param if_exists: 详见 File.write 参数解释
     :para temp: 将文件写到临时文件夹
     :return:
+
+    >> download_file(image_url)  # 保存在当前工作目录下
+    D:/home/chenkunze/slns/xlproject/xlsln/ckz2024/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
+    >> download_file(image_url, fn=r"D:/home/chenkunze/slns/xlproject/xlsln/ckz2024/a.png")  # 指定路径
+    D:/home/chenkunze/slns/xlproject/xlsln/ckz2024/a.png
+    >> download_file(image_url, fn=r"D:/home/chenkunze/slns/xlproject/xlsln/ckz2024")  # 暂不支持目录
+    ValueError: 不能用目录初始化一个File对象 D:\home\chenkunze\slns\xlproject\xlsln\ckz2023
     """
     if not fn: fn = url.split('/')[-1]
     root = Dir.TEMP if temp else None

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/file/specialist/filelib.py

@@ -1112,7 +1112,7 @@ class XlPath(type(pathlib.Path())):
             s = str(best_match)
             encoding = best_match.encoding
         else:
-            with open(self, 'r', encoding=encoding) as f:
+            with open(self, 'r', encoding=encoding, errors=errors) as f:
                 s = f.read()
 
         # 如果用\r\n作为换行符会有一些意外不好处理
@@ -1212,7 +1212,7 @@ class XlPath(type(pathlib.Path())):
         :param int max_items: 限制读取的条目数，默认为None，表示读取所有条目
         :param int batch_size:
             默认为None，表示一次性读取所有数据
-            如果设置了数值，则会流式读取，常用语太大，超过内存大小等的jsonl文件读取
+            如果设置了数值，则会流式读取，常用于太大，超过内存大小等的jsonl文件读取
                 注意如果设置了大小，只是底层每次一批读取的大小，但返回的data仍然是一维的数据格式迭代器
         :return: 返回读取到的数据列表，如果return_mode为True，则同时返回文件编码格式
 
@@ -1248,10 +1248,10 @@ class XlPath(type(pathlib.Path())):
         else:
             return data
 
-    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w'):
+    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w', errors=None):
         """ 由于这种格式主要是跟商汤这边对接，就尽量跟它们的格式进行兼容 """
         content = '\n'.join([json.dumps(x, ensure_ascii=ensure_ascii, default=default) for x in list_data])
-        self.write_text_unix(content + '\n', mode=mode)
+        self.write_text_unix(content + '\n', mode=mode, errors=errors)
 
     def add_json_line(self, data, ensure_ascii=False, default=None, mode='a'):
         """ 在文件末尾添加一行JSON数据 """

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/file/xlsxlib.py

@@ -121,7 +121,7 @@ def is_valid_excel_address(address):
         return is_valid_excel_cell(address)
 
 
-@run_once('str', debug=True)
+@run_once('str')
 def xlfmt2pyfmt_date(xl_fmt):
     """ 日期的渲染操作
 
@@ -2478,7 +2478,11 @@ def extract_workbook_summary2(file_path, *,
     res['fileName'] = file_path.name
     start_time = time.time()
     wb, suffix = load_as_xlsx_file(file_path, keep_links=keep_links, keep_vba=keep_vba)
-    res['fileType'] = suffix or 'error'
+    if suffix:
+        res['fileType'] = suffix
+    else:
+        res['error'] = 'fileType error'
+
     load_time = time.time() - start_time
     if wb is None:  # 不支持的文件类型，不报错，只是返回最基本的文件名信息
         if return_mode == 1:
@@ -2959,6 +2963,9 @@ class WorkbookSummary3:
             return y
 
         x = summary2
+        if 'error' in x:
+            return x
+
         y = {
             'fileName': x['fileName'],
             'fileType': x['fileType'],

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/prog/pupil.py

@@ -1129,6 +1129,7 @@ def support_multi_processes(default_processes=1):
         def wrapper(*args, **kwargs):
             process_count = int(kwargs.pop('process_count', default_processes))
             process_id = kwargs.pop('process_id', None)
+            shell = kwargs.pop('shell', False)
 
             if process_count == 1 or process_id is not None:
                 if process_id is None:
@@ -1150,7 +1151,7 @@ def support_multi_processes(default_processes=1):
                     for k, v in kwargs.items():
                         cmds.append(f'--{k}')  # 添加关键字参数的键
                         cmds.append(str(v))  # 添加关键字参数的值
-                    mpl.add_process_python(sys.argv[0], cmds)
+                    mpl.add_process_python(sys.argv[0], cmds, shell=shell)
                 mpl.run_endless()
 
         return wrapper

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxllib/text/jiebalib.py

@@ -250,7 +250,7 @@ class TextClassifier:
         return sims[:maxn]
 
     def refine_vecs(self):
-        """ 优化向量数据 """
+        """ 优化向量数据，去掉权重小余0.0001的维度 """
         # 1 计算每个向量的长度
         vecs = []
         for vec in tqdm(self.vecs, '优化向量'):

{pyxllib-0.3.121 → pyxllib-0.3.123/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.121
+Version: 0.3.123
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.121 → pyxllib-0.3.123}/pyxlpr/openai2/_core/chat.py

@@ -1,9 +1,14 @@
+import os
 from json import dumps as jsonDumps
 from json import loads as jsonLoads
 from pathlib import Path
 from typing import List, Literal
+import base64
+
 from openai import OpenAI, AsyncOpenAI
 
+from pyxllib.file.specialist import download_file
+
 
 class AKPool:
     """ 轮询获取api_key """
@@ -106,24 +111,35 @@ class Chat:
     """
 
     recently_request_data: dict  # 最近一次请求所用的参数
+    default_api_key = None
+    default_base_url = "https://console.chatdata.online/v1"
 
     def __init__(self,
                  # kwargs
-                 api_key: str|AKPool,
+                 api_key: str | AKPool = None,
                  base_url: str = None,  # base_url 参数用于修改基础URL
                  timeout=None,
                  max_retries=None,
                  http_client=None,
                  # request_kwargs
-                 model: Literal["gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4", "gpt-4-0314", "gpt-4-0613",
-                                "gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613", "gpt-3.5-turbo"] = "gpt-3.5-turbo",
+                 model: Literal["gpt-4-1106-preview", "gpt-4-vision-preview",
+                 "gpt-4", "gpt-4-0314", "gpt-4-0613",
+                 "gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613", "gpt-3.5-turbo",
+                 "gpt-4o", "gpt-4-turbo"] = "gpt-3.5-turbo",
                  # Chat
-                 msg_max_count: int=None,
+                 msg_max_count: int = None,
                  # kwargs
                  **kwargs,
                  ):
+        """
+
+        :param kwargs: 主要是供OpenAI初始化时使用的参数
+
+        """
+        api_key = api_key or self.default_api_key or os.environ.get("OPENAI_API_KEY")
+
         api_base = kwargs.pop('api_base', None)
-        base_url = base_url or api_base
+        base_url = base_url or api_base or self.default_base_url or os.environ.get("OPENAI_BASE_URL")
         MsgMaxCount = kwargs.pop('MsgMaxCount', None)
         msg_max_count = msg_max_count or MsgMaxCount
 
@@ -134,16 +150,22 @@ class Chat:
 
         self.reset_api_key(api_key)
         self._kwargs = kwargs
-        self._request_kwargs = {'model':model}
+        self._request_kwargs = {'model': model}
         self._messages = Temque(maxlen=msg_max_count)
 
-    def reset_api_key(self, api_key: str|AKPool):
+    def __1_原作者提供功能(self):
+        pass
+
+    def reset_api_key(self, api_key: str | AKPool):
         if isinstance(api_key, AKPool):
             self._akpool = api_key
         else:
             self._akpool = AKPool([api_key])
 
-    def request(self, text: str=None, **kwargs):
+    def request(self, text: str = None, **kwargs):
+        """
+        :param kwargs: 支持本次请求的参数覆盖全局 self._request_kwargs 的参数
+        """
         messages = [{"role": "user", "content": text}]
         messages += (kwargs.pop('messages', None) or [])  # 兼容官方包[openai]用户, 使其代码可以无缝切换到[openai2]
         assert messages
@@ -160,7 +182,7 @@ class Chat:
         self._messages.add_many(*messages, {"role": "assistant", "content": answer})
         return answer
 
-    def stream_request(self, text: str=None, **kwargs):
+    def stream_request(self, text: str = None, **kwargs):
         messages = [{"role": "user", "content": text}]
         messages += (kwargs.pop('messages', None) or [])  # 兼容官方包[openai]用户, 使其代码可以无缝切换到[openai2]
         assert messages
@@ -180,7 +202,7 @@ class Chat:
                 yield content
         self._messages.add_many(*messages, {"role": "assistant", "content": answer})
 
-    async def async_request(self, text: str=None, **kwargs):
+    async def async_request(self, text: str = None, **kwargs):
         messages = [{"role": "user", "content": text}]
         messages += (kwargs.pop('messages', None) or [])  # 兼容官方包[openai]用户, 使其代码可以无缝切换到[openai2]
         assert messages
@@ -197,7 +219,7 @@ class Chat:
         self._messages.add_many(*messages, {"role": "assistant", "content": answer})
         return answer
 
-    async def async_stream_request(self, text: str=None, **kwargs):
+    async def async_stream_request(self, text: str = None, **kwargs):
         messages = [{"role": "user", "content": text}]
         messages += (kwargs.pop('messages', None) or [])  # 兼容官方包[openai]用户, 使其代码可以无缝切换到[openai2]
         assert messages
@@ -221,7 +243,7 @@ class Chat:
         '''
         回滚对话
         '''
-        self._messages.core[-2 * n :] = []
+        self._messages.core[-2 * n:] = []
         for x in self._messages.core[-2:]:
             x = x["obj"]
             print(f"[{x['role']}]:{x['content']}")
@@ -241,7 +263,7 @@ class Chat:
     def fetch_messages(self):
         return list(self._messages)
 
-    def add_dialogs(self, *ms: dict|system_msg|user_msg|assistant_msg):
+    def add_dialogs(self, *ms: dict | system_msg | user_msg | assistant_msg):
         '''
         添加历史对话
         '''
@@ -274,4 +296,160 @@ class Chat:
         """ 载入存档 """
         jt = Path(fpath).read_text(encoding="utf8")
         self._messages.add_many(*jsonLoads(jt))
-        return True
+        return True
+
+    def __2_扩展功能(self):
+        pass
+
+    def add_message(self, message):
+        """ 添加单条历史消息记录 """
+        self._messages.add_many(message)
+
+    def add_messages(self, messages: list[dict]):
+        """ 添加多条历史消息记录 """
+        self._messages.add_many(*messages)
+
+    def parse_cont_to_conts(self, in_cont) -> list[dict]:
+        """ 自定义的一套内容设置格式，用来简化openai源生的配置逻辑
+
+        :return: 注意虽然看似输入是一个cont，实际可能解析出多个conts，所以返回的是list类型
+        """
+
+        def encode_image(image_path):
+            """ 读取本地图片文件 """
+            with open(image_path, "rb") as image_file:
+                return base64.b64encode(image_file.read()).decode('utf-8')
+
+        def parse_image_file(image_file):
+            suffix = Path(image_file).suffix[1:].lower()
+            if suffix == 'jpg':
+                suffix = 'jpeg'
+            base64_image = encode_image(image_file)
+            image_url = {'url': f"data:image/{suffix};base64,{base64_image}"}
+            return image_url
+
+        if isinstance(in_cont, str):
+            return [{"type": "text", "text": in_cont}]
+
+        if isinstance(in_cont, dict):
+            if 'image_url' in in_cont and isinstance(in_cont['image_url'], str):
+                url = in_cont['image_url']
+                return [{"type": "image_url", "image_url": {'url': url}}]
+            if 'image_file' in in_cont:
+                image_url = parse_image_file(in_cont['image_file'])
+                in_cont.pop('image_file')
+                if in_cont:
+                    image_url.update(in_cont)
+                return [{"type": "image_url", "image_url": image_url}]
+
+            if 'image_urls' in in_cont:
+                return [{"type": "image_url", "image_url": {'url': x}} for x in in_cont['image_urls']]
+            if 'image_files' in in_cont:
+                return [{"type": "image_url", "image_url": parse_image_file(x)} for x in in_cont['image_files']]
+
+            return [in_cont]
+        else:
+            raise ValueError
+
+    def parse_conts_to_message(self, conts, role='user'):
+        """ 可以用一种简化的格式，来配置用户要提问的信息内容
+
+        :param conts:
+            str, 单条文本提问
+            list,
+                类似这样的数据，可以完全兼容openai的格式
+                    [
+                    {"type": "text", "text": "What’s in this image?"},
+                    {
+                      "type": "image_url",
+                      "image_url": {
+                        "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
+                      },
+                    },
+                    ]
+              也可以简化：用我这里自定义的简化格式
+                'text'    文本格式直接输入
+                {'image_url': 'https://...'}
+                {'image_file': 'path/to/file'}
+        """
+        message = {"role": role}
+
+        if isinstance(conts, str):
+            message["content"] = conts
+        else:
+            conts = conts if isinstance(conts, (list, tuple)) else [conts]
+            message["content"] = []
+            for x in conts:
+                message["content"] += self.parse_cont_to_conts(x)
+
+        return message
+
+    def add_message_from_conts(self, conts, role='user'):
+        """ 添加content版本的message """
+        message = self.parse_conts_to_message(conts, role=role)
+        self._messages.add_many(message)
+
+    def query(self, conts=None, response_json=None, **kwargs):
+        """ 提交一个请求，并获得回复结果
+
+        :param conts:
+            None, content可以不输入，这种一般是外部已经提前配置好各种add_message，这里只要直接请求即可
+            str, 常规的文本提问使用
+            list, 一般是搭配图片的提问使用
+        :param response_json: 是否返回json格式
+            注意就算这个参数为True，提示里也要显示说明需要json格式。官方示例是在初始的role=system中配置
+        """
+        # 1 total_messages
+        if conts:
+            messages = [self.parse_conts_to_message(conts)]
+            messages += (kwargs.pop('messages', None) or [])
+        else:
+            messages = []
+        total_messages = list(self._messages + messages)
+        assert total_messages
+
+        if response_json:
+            kwargs['response_format'] = {"type": "json_object"}
+
+        # 2 请求结果
+        self.recently_request_data = {
+            'api_key': (api_key := self._akpool.fetch_key()),
+        }
+        completion = OpenAI(api_key=api_key, **self._kwargs).chat.completions.create(**{
+            **self._request_kwargs,  # 全局参数
+            **kwargs,  # 单次请求的参数覆盖全局参数
+            "messages": total_messages,
+            "stream": False,
+        })
+        answer: str = completion.choices[0].message.content
+        self._messages.add_many(*messages, {"role": "assistant", "content": answer})
+        return answer
+
+    def query_image(self, prompt,
+                    size='1024x1024', model='dall-e-3', quality='standard', n=1,
+                    save_file=None,
+                    **kwargs):
+        """ 生成图片，注意，此功能不支持引用旧历史记录，而是一个独立的记录来请求model获得一张图
+
+        :param save_file: 指定保存到本地某个位置
+        """
+        self.recently_request_data = {
+            'api_key': (api_key := self._akpool.fetch_key()),
+        }
+        client = OpenAI(api_key=api_key, **self._kwargs)
+        response = client.images.generate(
+            model=model,
+            prompt=prompt,
+            size=size,
+            quality=quality,
+            n=n,
+            **kwargs
+        )
+        image_url = response.data[0].url
+        self.add_message_from_conts(prompt, 'user')
+        self.add_message_from_conts([{'image_url': image_url}], 'assistant')
+
+        if save_file:
+            download_file(image_url, save_file)
+
+        return image_url

{pyxllib-0.3.121 → pyxllib-0.3.123}/setup.py

@@ -45,7 +45,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.121',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.123',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',