MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/oss/minio_oss.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : minio_oss
+# @Time         : 2024/3/14 17:54
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import datetime
+import mimetypes
+
+from meutils.pipe import *
+from minio import Minio as _Minio
+from openai.types.file_object import FileObject
+from fastapi import APIRouter, File, UploadFile, Query, Form, BackgroundTasks, Depends, HTTPException, Request, status
+from asgiref.sync import sync_to_async
+
+
+class Minio(_Minio):
+
+    def __init__(self, endpoint: Optional[str] = None,
+                 access_key: Optional[str] = None,
+                 secret_key: Optional[str] = None,
+                 **kwargs):
+        self.endpoint = endpoint or os.getenv('MINIO_ENDPOINT', 'oss.ffire.cc')  # 默认国内 oss.ffire.cn
+        access_key = access_key or os.getenv('MINIO_ACCESS_KEY', 'minio')
+        secret_key = secret_key or os.getenv('MINIO_SECRET_KEY')
+
+        secure = False if ":" in self.endpoint else True
+
+        super().__init__(endpoint=self.endpoint, access_key=access_key, secret_key=secret_key, secure=secure, **kwargs)
+
+    # def list_bucket_objects(self):
+    #     super().list_buckets()
+    #     super().list_objects('中职职教高考政策解读.pdf')
+    #     return super().list_buckets()
+
+    async def put_object_for_openai(
+            self,
+            file: Union[str, bytes, UploadFile],
+
+            bucket_name: str = "cdn",
+            purpose: str = "oss",
+
+            filename: Optional[str] = None,  # 预生成：提前生成链接返回 适用于异步任务
+
+            headers: Optional[dict] = None,
+            follow_redirects: bool = False,
+
+            content_type: Optional[str] = "application/octet-stream",
+
+    ):
+        """
+        FileObject(id='ZDcQjHb9nsmwnYXaw8eKem.docx', bytes=55563, created_at=1710729984, filename='ZDcQjHb9nsmwnYXaw8eKem.docx', object='file', purpose='file-upload', status='processed', status_details=None)
+
+        """
+        # self.make_bucket(bucket_name, object_lock=True)
+
+        # file 标准化为 UploadFile
+        if isinstance(file, str) and file.startswith("http"):  # todo: url2UploadFile 从网络下载
+            async with httpx.AsyncClient(timeout=300, headers=headers, follow_redirects=follow_redirects) as client:
+                for i in range(3):
+                    response = await client.get(url=file)
+                    if response.is_success: break
+
+                file = UploadFile(
+                    file=io.BytesIO(response.content),
+                    filename=filename or Path(file).name,
+                    size=len(response.content),
+                    headers=response.headers,  # content_type
+                )
+                logger.debug(file)
+
+        elif isinstance(file, bytes):
+            file = UploadFile(
+                file=io.BytesIO(file),
+                size=len(file),
+
+                filename=filename,
+            )
+        file_name = filename or file.filename or shortuuid.random()
+
+        content_type = (
+                file.content_type
+                or mimetypes.guess_type(file_name)[0]
+                or content_type
+                or "application/octet-stream"
+        )
+
+        logger.debug(f"content_type: {content_type}")
+        object_name = f"""{datetime.datetime.now().strftime("%Y-%m-%d")}/{file_name}"""
+        _ = await self.aput_object(
+            bucket_name,
+            object_name=object_name,
+            data=file.file,
+            length=file.size,  # 不能为None
+            content_type=content_type
+        )
+
+        # construct 创建实例，跳过验证
+        file_object = FileObject.construct(
+            id=object_name,
+            bytes=file.size,
+            created_at=int(time.time()),
+            filename=f"https://{self.endpoint}/{bucket_name}/{object_name}",
+            # file_url: oss.chatfire.cn/files/{file_id}
+
+            object='file',
+
+            purpose=purpose,
+
+            status='processed' if _ else "error",  # todo: 抛错处理
+
+        )
+
+        logger.debug(file_object)
+
+        return file_object
+
+    @sync_to_async(thread_sensitive=False)
+    def aget_object(self, *args, **kwargs):
+        return self.get_object(*args, **kwargs)
+
+    # async def aget_object(self, *args, **kwargs):
+    #     return sync_to_async(thread_sensitive=False)(self.get_object)(*args, **kwargs)
+
+    @sync_to_async(thread_sensitive=False)
+    def aput_object(self, *args, **kwargs):
+        return self.put_object(*args, **kwargs)
+
+    def get_file_url(self, filename, bucket_name='files'):
+        return f"https://{self.endpoint}/{bucket_name}/{filename}"
+
+
+# minio_client.put_object(OPENAI_BUCKET, f"{api_key}/{file.filename}", data=file.file, length=file.size)
+
+# # Make a bucket with the make_bucket API call.
+# bucket_name = 'bname'
+# # minioClient.make_bucket(bucket_name)
+#
+# print(client.list_buckets())
+
+if __name__ == '__main__':
+    client = Minio()
+    # bucket_name = 'test'
+    # prefix = 'prefix'
+    # filename = Path('minio_oss.py').name
+    # data = Path('minio_oss.py').read_bytes()
+    #
+    # extension = Path("xx.x").suffix
+    #
+    # file_url = Path(f"{bucket_name}/{prefix}/xxxxxxxx{extension}")  # base url
+    #
+    # print(file_url)
+    #
+    # obj = client.put_object(bucket_name, f"{prefix}/{filename}", data=io.BytesIO(data), length=len(data),
+    #                         metadata={"url": str(file_url), "content_type": "application/octet-stream"}
+    #                         )
+    #
+    # print(obj)
+
+    # _ = client.put_object_for_openai(
+    #     "https://sfile.chatglm.cn/chatglm4/82834747-0fcf-4ecb-94b0-92e5e749798b.docx",
+    #     bucket_name="files",
+    #     file_id='xx.docx'
+    # )
+
+    url = "https://sfile.chatglm.cn/testpath/4f520e7f-c2f5-5555-8d1f-4fda0fec0e9d_0.png"
+    # url = "https://cdn1.suno.ai/bb10a2e9-3543-4ddc-aad1-2c7ca95bfa7c.mp3"
+    # url = "https://p1.a.kwimgs.com/bs2/upload-ylab-stunt/special-effect/output/HB1_PROD_ai_web_31199871/685408188778288330/output_ffmpeg.mp4"
+    # url = 'https://klingai.com/api/works/batch_download?workIds=58740791&fwm=false'
+    # _ = client.put_object_for_openai(
+    #     url,
+    #     bucket_name="caches",
+    #     filename='cff.png'
+    # )
+    # print(arun(_, debug=True))
+
+    _ = client.put_object_for_openai(
+        url,
+        filename='cff.png'
+    )
+    arun(_)

meutils/oss/minio_utils.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : minio_utils
+# @Time         : 2024/1/3 14:42
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : https://www.cnblogs.com/fanpiao/p/17603553.html
+from minio import Minio
+from meutils.pipe import *
+
+minioClient = Minio(
+    endpoint=os.getenv('MINIO_ENDPOINT'),
+    access_key=os.getenv('MINIO_ACCESS_KEY'),
+    secret_key=os.getenv('MINIO_SECRET_KEY'),
+    # secure=False
+)
+
+# Make a bucket with the make_bucket API call.
+bucket_name = 'bname'
+# minioClient.make_bucket(bucket_name)
+
+print(minioClient.list_buckets())
+# Put an object 'pumaserver_debug.log' with contents from 'pumaserver_debug.log'.
+
+# minioClient.fput_object(bucket_name, 'object_name', 'x.yml')
+#
+# minioClient.fput_object(bucket_name, 'img.png', 'img.png')
+#
+# minioClient.fput_object(bucket_name, 'file.py', 'file.py')
+
+
+minioClient.fput_object(bucket_name, 'x/img','img.png', content_type='image/png')
+
+
+# url = minioClient.get_presigned_url(
+#     "GET",
+#     bucket_name,
+#     'img.png',
+# )
+# print(url)
+
+# url = minioClient.presigned_get_object(bucket_name, 'img.png', )
+# print(url)
+#
+# response = minioClient.get_object(bucket_name, 'img.png', )
+# print(len(response.read()))

meutils/other/__demo.py

@@ -14,15 +14,17 @@ from aiohttp import request
 from aiomultiprocess import Pool  # todo
 
 
-async def get(url):
-    async with request("GET", url) as response:
-        return await response.text("utf-8")
+async def func(url):
+    await asyncio.sleep(1)
+    return url
 
 
 async def mainmain():
     urls = ["https://www.baidu.com"] * 10  # todo: openai
     async with Pool() as pool:
-        results = await pool.map(get, urls)
+        results = await pool.map(func, urls)
+
+        logger.debug(results)
 
         return results
 

meutils/pandas_utils/__init__.py

@@ -9,3 +9,4 @@
 # @Description  : 
 
 
+from meutils.pandas_utils.common import *

meutils/pandas_utils/common.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : common
+# @Time         : 2024/9/21 14:02
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import pandas as pd
+
+from meutils.pipe import *
+
+
+def to_html_plus(df, title='Title', subtitle='Subtitle', return_filename=False):
+    from jinja2 import Environment, PackageLoader
+    env = Environment(loader=PackageLoader('meutils'))
+    template = env.get_template('df_html.j2')
+
+    html_content = template.render(df_to_html=df.to_html(), title=title, subtitle=subtitle)
+    if return_filename:
+        with tempfile.NamedTemporaryFile(mode='w+', suffix='.html', delete=False) as file:
+            # logger.debug(file.name)
+            file.write(html_content)
+            file.seek(0)
+            return file.name
+    return html_content
+
+
+if __name__ == '__main__':
+    to_html_plus(pd.DataFrame(range(10)))

meutils/pandas_utils/pd_utils.py

@@ -144,15 +144,19 @@ def reduce_mem_usage(df):
     return df
 
 
-def to_html_plus(df, title='Title', subtitle='Subtitle'):
+def to_html_plus(df, title='Title', subtitle='Subtitle', return_filename=False):
     from jinja2 import Environment, PackageLoader
     env = Environment(loader=PackageLoader('meutils'))
     template = env.get_template('df_html.j2')
 
-    content = template.render(df_to_html=df.to_html(), title=title, subtitle=subtitle)
-    # with open('./df.html', 'w') as fp:
-    #     fp.write(content)
-    return content
+    html_content = template.render(df_to_html=df.to_html(), title=title, subtitle=subtitle)
+    if return_filename:
+        with tempfile.NamedTemporaryFile(mode='w+', suffix='.html', delete=False) as file:
+            # logger.debug(file.name)
+            file.write(html_content)
+            file.seek(0)
+            return file.name
+    return html_content
 
 
 def to_compress(df: pd.DataFrame, filename, method='zip', archive_name='data.csv', index=False, **to_csv_kwargs):
@@ -191,7 +195,7 @@ if __name__ == '__main__':
     template = env.get_template('df_html.j2')
 
     df = pd.DataFrame(range(10))
-    content = template.render(df_to_html=df.to_html(), title="title", subtitle='subtitle')
+    content = template.render(df_to_html=df.to_html(), title="title", subtitle='')
 
     print(content)
 

meutils/parsers/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2025/1/10 15:45
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+

meutils/parsers/file_parsers.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : fileparser
+# @Time         : 2025/1/7 17:48
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : https://bigmodel.cn/dev/activities/freebie/fileextracion
+import asyncio
+
+from meutils.pipe import *
+from meutils.io.files_utils import to_bytes, guess_mime_type
+from meutils.llm.clients import moonshot_client, zhipuai_client, APIStatusError
+from meutils.notice.feishu import send_message as _send_message, FILES
+from meutils.caches.redis_cache import cache
+
+send_message = partial(
+    _send_message,
+    title=__name__,
+    url=FILES
+)
+"""
+
+# 智谱
+# 格式限制：.PDF .DOCX .DOC .XLS .XLSX .PPT .PPTX .PNG .JPG .JPEG .CSV .PY .TXT .MD .BMP .GIF
+
+# kimi todo: 定期删除文件
+文件接口与 Kimi 智能助手中上传文件功能所使用的相同，支持相同的文件格式，它们包括 
+.pdf .txt .csv .doc .docx .xls .xlsx .ppt .pptx .md .jpeg .png .bmp .gif .svg .svgz .webp .ico .xbm .dib .pjp .tif 
+.pjpeg .avif .dot .apng .epub .tiff .jfif .html .json .mobi .log .go .h .c .cpp .cxx .cc .cs .java .js .css .jsp .php 
+.py .py3 .asp .yaml .yml .ini .conf .ts .tsx 等格式。
+
+# todo: 
++ .sh
+"""
+
+
+@cache(ttl=24 * 3600)
+async def file_extract(
+        file,
+        provider: Union[str, Literal['kimi', 'moonshot', 'zhipu']] = 'moonshot'
+):
+    """todo 定时删除文件
+    todo: 兼容下 url bytes path
+
+    # moonshot_client.files.
+
+# len(moonshot_client.files.list().data)
+
+    """
+    mime_type = guess_mime_type(file)
+    if mime_type == "application/octet-stream": return  # 不解析
+
+    logger.debug(f"file_extract: {mime_type}")
+
+    if provider == "zhipu":
+        client = zhipuai_client
+    else:
+        client = moonshot_client  # 默认
+
+    filename = Path(file).name
+    file_bytes: bytes = await to_bytes(file)
+
+    # todo: zhipu兜底
+    try:
+        file_object = await client.files.create(
+            # file=file,
+            # file=("filename.pdf", file),
+            file=(filename, file_bytes, mime_type),
+            purpose="file-extract"
+        )
+
+        if isinstance(file, str) and file.startswith('http'):
+            file_object.url = file
+        send_message(file_object)
+
+        response = await client.files.content(file_id=file_object.id)  # 抛错处理
+        return response.json()
+    except APIStatusError as e:
+        logger.debug(e)
+        _ = e.response.json()
+        if isinstance(file, str) and file.startswith('http'):
+            _['url'] = file
+        send_message(_)
+        return _
+
+
+if __name__ == '__main__':
+    # file = "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf"
+    file = "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf"
+    "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？"
+    # file = "https://oss.ffire.cc/files/百炼系列手机产品介绍.docx"
+    # file = Path("/Users/betterme/PycharmProjects/AI/MeUtils/meutils/llm/completions/rag/百炼系列手机产品介绍.docx")
+
+    # file = "/Users/betterme/PycharmProjects/AI/MeUtils/meutils/io/img_1.png"
+
+    # openai.BadRequestError: Error code: 400 - {'error': {'message': 'text extract error: 没有解析出内容', 'type': 'invalid_request_error'}}
+    # file = "https://oss.ffire.cc/files/kling_watermark.png"
+
+    # with timer():
+    #     r = arun(file_extract(file, moonshot_client))
+
+    # with timer():
+    #     r = arun(file_extract(file, provider='kimi'))
+
+    # with timer():
+    #     arun(file_extract(file))
+
+    print(guess_mime_type('x.sh'))

meutils/parsers/fileparser/demo.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : demo
+# @Time         : 2024/6/5 09:00
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import pandas as pd
+
+from meutils.pipe import *
+import pandas as pd
+
+
+import  pdfplumber
+with pdfplumber.open('银行间市场债券交易结算情况（按投资者）.pdf') as pdf:
+    for page in pdf.pages:
+        data = page.extract_table()
+
+
+        cols = pd.MultiIndex.from_tuples(zip(*data[:2]))
+        print(cols)
+        df = pd.DataFrame(data[2:], columns=cols)
+
+        print(df.fillna(method='ffill', axis=1))
+
+
+#
+# import camelot
+# import pandas as pd
+# # 使用Camelot读取PDF文件中的表格
+# tables = camelot.read_pdf('银行间市场债券交易结算情况（按投资者）.pdf', pages='all', flavor='lattice')
+#
+# # # 将所有表格转换为 DataFrame 并合并
+# # all_data = pd.concat([table.df for table in tables], ignore_index=True)
+# #
+# # all_data.to_excel('all_data.xlsx',index=False)
+#
+#
+# print(tables[0].df)

meutils/parsers/fileparser/filetype.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : filetype
+# @Time         : 2023/7/15 15:05
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 文件或扩展名被更改
+
+from meutils.pipe import *
+
+import filetype
+import magic
+import mimetypes
+
+import mimetypes
+
+# mimetype, _ = mimetypes.guess_type('example.txt')
+# print(mimetype)
+#
+# import magic
+#
+# file_type = magic.from_file('example.pdf', mime=True)
+# print(file_type)
+#
+# import filetype
+#
+# kind = filetype.guess('example.jpg')
+# if kind is None:
+#     print('Cannot guess file type!')
+# else:
+#     print('File extension: %s' % kind.extension)
+#     print('File MIME type: %s' % kind.mime)
+
+if __name__ == '__main__':
+    # mimetype, _ = mimetypes.guess_type('example.mp3')
+    mimetype, _ = mimetypes.guess_type('https://www.baidu.com/img/flexible/logo/pc/result@2.png')
+
+    print(mimetype)
+    print(_)

meutils/pay.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : pay
+# @Time         : 2024/4/29 16:39
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+
+
+headers = {
+    "Cookie": os.getenv("PAY_COOKIE"),
+    "Content-Type": "application/x-www-form-urlencoded"
+}
+
+payload = {
+    # "sitename": "",
+    "trade_no": 2024042915220175845,
+}
+
+# print(httpx.post("https://zf.96ym.cn/User/Order/Callback", headers=headers, data=payload).text)
+
+
+# df = pd.read_html("https://zf.96ym.cn/User/Order/Index", storage_options=headers)[0]
+# df = df.query(f"`状态`!='已完成' and `商品名称`!='测试通道' and `创建时间 支付时间`>'{str(datetime.datetime.now())[:10]} 00:00:00'")
+#
+# print(df)
+
+# """
+# import httpx; httpx.post("https://zf.96ym.cn/User/Order/Callback", headers=headers, data=payload).text
+#
+# """
+

meutils/pipe.py

@@ -6,10 +6,11 @@
 # @Author       : yuanjie
 # @Email        : meutils@qq.com
 # @Software     : PyCharm
-# @Description  : 
+# @Description  :
+import json
 
 from meutils.common import *
-from meutils.async_utils.common import arun, aclose, alru_cache, async2sync_generator, close_event_loop
+from meutils.async_utils.common import arun, aclose, alru_cache, async2sync_generator, close_event_loop, achain
 
 
 class Pipe(object):
@@ -303,6 +304,22 @@ def xAsyncio(tasks, return_exceptions=False):
     return loop.run_until_complete(_)
 
 
+@Pipe
+def xsse_parser(iterable, prefix: str = 'data:', skip_strings: Optional[list] = None):
+    l = []
+    skip_strings = skip_strings or []
+    for line in iterable:
+        if (line := line.lstrip()) and line.startswith(prefix):
+            if any(s in line for s in skip_strings):
+                continue
+            try:
+                _ = json.loads(line[len(prefix):])
+                l.append(_)
+            except Exception as e:
+                logger.debug(f'error: {e}\nline: {line}')
+    return l
+
+
 if __name__ == '__main__':
     # @Pipe
     # def xfunc1(x):
@@ -356,5 +373,21 @@ if __name__ == '__main__':
     # print(range(10) | xThreadPoolExecutor(print, 1) | xlist)
     # print(range(10) | xThreadPoolExecutor(print, 2) | xlist)
 
-    print(range(10) | xProcessPoolExecutor(print, 1) | xlist)
-    print(range(10) | xProcessPoolExecutor(print, 2) | xlist)
+    # print(range(10) | xProcessPoolExecutor(print, 1) | xlist)
+    # print(range(10) | xProcessPoolExecutor(print, 2) | xlist)
+
+    sse = """
+    data: {"type":"step","msg":"努力生成中...","code":200}
+data: {"type":"progress","value":0.08333333333333333,"code":200}
+data: {"type":"progress","value":0.16666666666666666,"code":200}
+data: {"type":"progress","value":0.25,"code":200}
+data: {"type":"progress","value":0.3333333333333333,"code":200}
+data: {"type":"progress","value":0.4166666666666667,"code":200}
+data: {"type":"progress","value":0.5,"code":200}
+data: {"type":"removewatermark","id":"c0d1e312-b374-11ef-a22c-6a452a83f9ec","imageUrl":"https://hunyuan-prod-1258344703.cos.ap-guangzhou.myqcloud.com/removewatermark/f55df3ef029c13eba0832a11f77d03ad/20241206095236h0_d589b0620869d87ad340b4b4ed68b4ec.png?q-sign-algorithm=sha1\u0026q-ak=AKID0qSq0xJRL7h3A4nIYJFrFOJ1VlnbIm26\u0026q-sign-time=1733449957;1764985957\u0026q-key-time=1733449957;1764985957\u0026q-header-list=host\u0026q-url-param-list=\u0026q-signature=7bbec24fe87917896f662f65b80320a4a3fb8569","source":"inspiration","code":200,"thumbnailUrl":"https://hunyuan-prod-1258344703.cos.ap-guangzhou.myqcloud.com/removewatermark/f55df3ef029c13eba0832a11f77d03ad/20241206095236h0_d589b0620869d87ad340b4b4ed68b4ec.png?q-sign-algorithm=sha1\u0026q-ak=AKID0qSq0xJRL7h3A4nIYJFrFOJ1VlnbIm26\u0026q-sign-time=1733449957;1764985957\u0026q-key-time=1733449957;1764985957\u0026q-header-list=host\u0026q-url-param-list=\u0026q-signature=7bbec24fe87917896f662f65b80320a4a3fb8569\u0026imageMogr2/strip/format/jpg/size-limit/500k!/ignore-error/1"}
+data: [TRACEID:8a50e3687e26e03097e507c076fe7145]
+data: [DONE]"""
+
+    # print()
+
+    sse.splitlines() | xsse_parser(skip_strings=['DONE', 'TRACEID']) | xprint