MeUtils 2025.1.6.14.2.0__py3-none-any.whl → 2025.1.10.20.14.42__py3-none-any.whl

meutils/io/image.py

@@ -13,7 +13,31 @@ import mimetypes
 from PIL import Image, ImageDraw
 
 from meutils.pipe import *
-from meutils.io.files_utils import to_url
+from meutils.io.files_utils import to_url, to_bytes
+
+
+async def describe_image(image: Union[str, bytes]):
+    """
+    图片转base64
+    :param image: 图片路径或图片对象
+    :return: base64字符串
+    """
+    _ = await to_bytes(image)
+    img = Image.open(io.BytesIO(_))
+
+    info = {
+        'format': img.format,  # 图片格式
+        'mode': img.mode,  # 颜色模式
+
+        'width': img.width,  # 宽度
+        'height': img.height,  # 高度
+        'size': img.size,  # (宽度, 高度)
+
+        'is_animated': getattr(img, 'is_animated', False),  # 是否是动图
+        'n_frames': getattr(img, 'n_frames', 1),  # 帧数
+        'info': img.info,  # 图片附加信息
+    }
+    return info
 
 
 def crop_polygon(image: Union[str, bytes], outline_points, inline_points):
@@ -199,4 +223,6 @@ if __name__ == '__main__':
 
     url = "https://s22-def.ap4r.com/bs2/upload-ylab-stunt-sgp/se/ai_portal_sgp_queue_mmu_img2img_aiweb/7de17faa-cc11-48a6-b94a-e80fe7a34841/1.png"
 
-    arun(image2nowatermark_image(url, picinfo=PICINFO2_RIGHT_BOTTOM))
+    # arun(image2nowatermark_image(url, picinfo=PICINFO2_RIGHT_BOTTOM))
+    with timer():
+        arun(describe_image("https://oss.ffire.cc/files/kling_watermark.png"))

meutils/llm/clients.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : chat
+# @Time         : 2024/8/19 14:23
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+from openai import Client, AsyncClient, AsyncStream, APIStatusError
+
+from meutils.pipe import *
+
+OpenAI = lru_cache(Client)
+AsyncOpenAI = lru_cache(AsyncClient)
+
+moonshot_client = AsyncOpenAI(
+    api_key=os.getenv("MOONSHOT_API_KEY"),
+    base_url=os.getenv("MOONSHOT_BASE_URL")
+)
+zhipuai_client = AsyncOpenAI(
+    api_key=os.getenv("ZHIPUAI_API_KEY"),
+    base_url=os.getenv("ZHIPUAI_BASE_URL")
+)
+
+if __name__ == '__main__':
+    from meutils.pipe import *
+
+    # OpenAI().chat.completions.create(messages=[{"role": "user", "content": "hi"}], model='glm-4-flash')
+
+    # arun(zhipuai_client.chat.completions.create(messages=[{"role": "user", "content": "hi"}], model='glm-4-flash'))
+
+    # web-search-pro
+
+    arun(zhipuai_client.chat.completions.create(
+        messages=[{"role": "user", "content": "中国队奥运会拿了多少奖牌"}],
+        model='web-search-pro')
+    )

meutils/llm/completions/agents/file.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : files
+# @Time         : 2025/1/3 15:38
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 支持文档、图片、音频、视频问答
+"""单一智能体
+任意模型支持文档、图片、音频、视频问答
+api形式
+- /agents/v1
+- /v1 前缀区分 agents-{model}【底层调用 /agents/v1】
+
+"""
+
+from meutils.pipe import *
+from meutils.parsers.file_parsers import file_extract
+
+from meutils.llm.clients import AsyncOpenAI
+from meutils.llm.openai_utils import to_openai_params
+
+from meutils.str_utils.regular_expression import parse_url
+
+from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, ChatCompletionRequest, CompletionUsage
+
+
+class Completions(object):
+
+    def __init__(self, api_key: Optional[str] = None):
+        self.api_key = api_key
+
+    async def create(self, request: ChatCompletionRequest):
+        """[{'role': 'user', 'content': 'hi'}]
+        {"type": "file_url", "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}}"""
+
+        if urls := parse_url(str(request.messages)):
+            logger.debug(urls)
+
+            file_reponse = await file_extract(urls[-1])
+            if file_reponse:  # 仅关注最后一个
+                request.system_messages.append({
+                    "role": "system",
+                    "content": json.dumps(file_reponse, ensure_ascii=False),
+                })
+
+            request.messages = request.system_messages + request.messages
+
+        logger.debug(request)
+
+        data = to_openai_params(request)
+        return await AsyncOpenAI(api_key=self.api_key).chat.completions.create(**data)
+
+
+# data: {"event": "message", "task_id": "900bbd43-dc0b-4383-a372-aa6e6c414227", "id": "663c5084-a254-4040-8ad3-51f2a3c1a77c", "answer": "Hi", "created_at": 1705398420}\n\n
+if __name__ == '__main__':
+    c = Completions()
+
+    request = ChatCompletionRequest(
+        # model="qwen-turbo-2024-11-01",
+        # model="claude-3-5-sonnet-20241022",
+        model="gpt-4o-mini",
+
+        messages=[
+            {
+                'role': 'system',
+                'content': '你是一个文件问答助手'
+            },
+            # {
+            #     'role': 'user',
+            #     # 'content': {
+            #     #     "type": "file_url",
+            #     #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #     # },
+            #     'content': [
+            #         {
+            #             "type": "text",
+            #             "text": "这个文件讲了什么？"
+            #         },
+            #         # 多轮的时候要剔除
+            #         {
+            #             "type": "file_url",
+            #             "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #         }
+            #     ]
+            # },
+
+            {
+                'role': 'user',
+                # 'content': {
+                #     "type": "file_url",
+                #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+                # },
+                # 'content': "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？",
+                # 'content': "https://translate.google.com/?sl=zh-CN&tl=en&text=%E6%8F%90%E4%BE%9B%E6%96%B9&op=tr1anslate 这个文件讲了什么？",
+
+                "content": "总结下 https://oss.ffire.cc/files/百炼系列手机产品介绍.docx"
+
+            },
+
+            # {'role': 'assistant', 'content': "好的"},
+            # {
+            #     'role': 'user',
+            #     # 'content': {
+            #     #     "type": "file_url",
+            #     #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #     # },
+            #     'content': [
+            #         {
+            #             "type": "text",
+            #             "text": "错了 继续回答"
+            #         },
+            #         # {
+            #         #     "type": "file_url",
+            #         #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #         # }
+            #     ]
+            # }
+        ]
+
+    )
+
+    arun(c.create(request))

meutils/llm/openai_utils/common.py

@@ -7,11 +7,11 @@
 # @WeChat       : meutils
 # @Software     : PyCharm
 # @Description  :
-import time
 
 import tiktoken
 from contextlib import asynccontextmanager
 from openai import AsyncOpenAI, OpenAI, AsyncStream
+from meutils.llm.clients import AsyncOpenAI, OpenAI, AsyncStream
 
 from meutils.pipe import *
 from meutils.async_utils import achain, async_to_sync
@@ -266,7 +266,7 @@ if __name__ == '__main__':
         with timer():
             try:
                 async with ppu_flow(api_key="sk-OYK4YxtTlWauT2TdGR5FTAJpkRmSnDwPly4cve0cAvMcrBkZ", post="api-oss",
-                                    n=100):
+                                    n=1):
                     logger.debug("消费了哦")
 
             except Exception as e:

meutils/llm/utils.py

@@ -8,7 +8,66 @@
 # @Software     : PyCharm
 # @Description  :
 
+from contextlib import asynccontextmanager
+
 from meutils.pipe import *
+from meutils.llm.clients import AsyncOpenAI, OpenAI, AsyncStream
+from meutils.schemas.oneapi import MODEL_PRICE
+from meutils.notice.feishu import send_message
+from meutils.apis.oneapi.utils import get_user_quota
+
+
+async def ppu(
+        model: str = 'ppu',
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+):
+    if model not in MODEL_PRICE:
+        send_message(f"模型未找到{model}，ppu-1默认", title=__name__)
+        model = "ppu-1"
+
+    client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+    r = await client.chat.completions.create(messages=[{'role': 'user', 'content': 'hi'}], model=model)
+
+
+@asynccontextmanager
+async def ppu_flow(
+        api_key: str,
+        base_url: Optional[str] = None,
+
+        model: str = "ppu-1",  # 后计费
+
+        n: float = 1,  # 计费次数
+
+        **kwargs
+):
+    """
+    查余额
+    失败，先扣费
+    成功，充足，后扣费
+    成功，不足，报错
+    """
+    n = int(np.ceil(n))  # 0 不计费
+
+    if n:  # 计费
+        try:
+            money = await get_user_quota(api_key)
+            logger.debug(f"PREPAY: USER 余额 {money}")
+        except Exception as e:
+            logger.error(e)
+            money = None
+
+        if money and money > MODEL_PRICE.get(model, 0.1):
+            yield  # 先执行
+
+        # 执行计费逻辑
+        await asyncio.gather(*[ppu(model, api_key=api_key, base_url=base_url) for _ in range(n)])
+
+        if money is None:
+            yield  # 后执行
+
+    else:  # 不计费
+        yield
 
 
 def oneturn2multiturn(messages, template: Optional[str] = None):
@@ -52,3 +111,23 @@ def oneturn2multiturn(messages, template: Optional[str] = None):
         context += f"{role}:\n{content}\n\n"
     context += "assistant:\n"
     return context
+
+
+if __name__ == '__main__':
+    async def main():
+        with timer():
+            try:
+                async with ppu_flow(
+                        api_key="sk-OYK4YxtTlWauT2TdGR5FTAJpkRmSnDwPly4cve0cAvMcrBkZ",
+                        model="api-oss",
+                        n=1):
+                    logger.debug("消费了哦")
+
+            except Exception as e:
+                pass
+                logger.error(e)
+                # logger.debug(e.response.status_code)
+                # logger.debug(e.response.text)
+
+
+    arun(main())

meutils/llm/x.py

@@ -0,0 +1,75 @@
+import os
+
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : x
+# @Time         : 2025/1/7 17:31
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+from meutils.llm.clients import zhipuai_client
+from meutils.schemas.openai_types import _ChatCompletion
+
+import requests
+import uuid
+
+api_key = os.getenv("ZHIPUAI_API_KEY")
+
+msg = [
+    {
+        "role": "user",
+        "content": "中国队奥运会拿了多少奖牌"
+    }
+]
+
+tool = "web-search-pro"
+
+
+async def run_v4_sync():
+    url = "https://open.bigmodel.cn/api/paas/v4/tools"
+    request_id = str(uuid.uuid4())
+    data = {
+        "request_id": request_id,
+        "tool": tool,
+        "stream": False,
+        "messages": msg
+    }
+
+    # resp = requests.post(
+    #     url,
+    #     json=data,
+    #     headers={'Authorization': api_key},
+    #     timeout=300
+    # )
+    # print(resp.content.decode())
+
+    resp = await zhipuai_client.post(
+        "/tools",
+        body={
+            "request_id": str(uuid.uuid4()),
+            "tool": tool,
+            "stream": False,
+            "messages": msg
+        },
+        cast_to=object
+    )
+    return resp
+
+
+file_object = zhipuai_client.files.create(file=Path("/Users/betterme/PycharmProjects/AI/MeUtils/meutils/llm/completions/rag/百炼系列手机产品介绍.docx"), purpose="file-extract")
+
+
+
+if __name__ == '__main__':
+    # r = arun(run_v4_sync())
+    # arun(file_object)
+
+    file_id = "1736243045_3771f3dfb394424885f24c3dc0583741"
+
+    file_content = json.loads(zhipuai_client.files.content(file_id=file_id).content)["content"]
+
+    file_content = json.loads(r.content)["content"]

meutils/notice/feishu.py

@@ -19,6 +19,7 @@ LLM_REVERSE = "https://open.feishu.cn/open-apis/bot/v2/hook/cd5c9126-a882-4a24-9
 Vison = ""
 # AUDIOS_TTS = "https://open.feishu.cn/open-apis/bot/v2/hook/ff7d4b86-d238-436c-9447-f88cf603454d"
 AUDIO = "https://open.feishu.cn/open-apis/bot/v2/hook/80c2a700-adfa-4b9b-8e3f-00b78f2f5c8b"
+FILES = "https://open.feishu.cn/open-apis/bot/v2/hook/075fb2fa-a559-4a7e-89ac-3ab9934ff15c"
 
 
 @background_task

meutils/parsers/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2025/1/10 15:45
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+

meutils/parsers/file_parsers.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : fileparser
+# @Time         : 2025/1/7 17:48
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : https://bigmodel.cn/dev/activities/freebie/fileextracion
+import asyncio
+
+from meutils.pipe import *
+from meutils.io.files_utils import to_bytes, guess_mime_type
+from meutils.llm.clients import moonshot_client, zhipuai_client, APIStatusError
+from meutils.notice.feishu import send_message as _send_message, FILES
+from meutils.caches.redis_cache import cache
+
+send_message = partial(
+    _send_message,
+    title=__name__,
+    url=FILES
+)
+"""
+
+# 智谱
+# 格式限制：.PDF .DOCX .DOC .XLS .XLSX .PPT .PPTX .PNG .JPG .JPEG .CSV .PY .TXT .MD .BMP .GIF
+
+# kimi todo: 定期删除文件
+文件接口与 Kimi 智能助手中上传文件功能所使用的相同，支持相同的文件格式，它们包括 
+.pdf .txt .csv .doc .docx .xls .xlsx .ppt .pptx .md .jpeg .png .bmp .gif .svg .svgz .webp .ico .xbm .dib .pjp .tif 
+.pjpeg .avif .dot .apng .epub .tiff .jfif .html .json .mobi .log .go .h .c .cpp .cxx .cc .cs .java .js .css .jsp .php 
+.py .py3 .asp .yaml .yml .ini .conf .ts .tsx 等格式。
+
+"""
+
+
+@cache(ttl=24 * 3600)
+async def file_extract(
+        file,
+        provider: Union[str, Literal['kimi', 'moonshot', 'zhipu']] = 'moonshot'
+):
+    """todo 定时删除文件
+    todo: 兼容下 url bytes path
+
+    # moonshot_client.files.
+
+# len(moonshot_client.files.list().data)
+
+    """
+    mime_type = guess_mime_type(file)
+    if mime_type == "application/octet-stream": return  # 不解析
+
+    logger.debug(f"file_extract: {mime_type}")
+
+    if provider == "zhipu":
+        client = zhipuai_client
+    else:
+        client = moonshot_client  # 默认
+
+    filename = Path(file).name
+    file_bytes: bytes = await to_bytes(file)
+
+    # todo: zhipu兜底
+    try:
+        file_object = await client.files.create(
+            # file=file,
+            # file=("filename.pdf", file),
+            file=(filename, file_bytes, mime_type),
+            purpose="file-extract"
+        )
+
+        if file_object.status and file_object.status != "ok":
+            if isinstance(file, str) and file.startswith('http'):
+                file_object.url = file
+            send_message(file_object)
+
+        response = await client.files.content(file_id=file_object.id)  # 抛错处理
+        return response.json()
+    except APIStatusError as e:
+        logger.debug(e)
+        _ = e.response.json()
+        if isinstance(file, str) and file.startswith('http'):
+            _['url'] = file
+        send_message(_)
+        return _
+
+
+if __name__ == '__main__':
+    # file = "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf"
+    file = "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf"
+    "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？"
+    # file = "https://oss.ffire.cc/files/百炼系列手机产品介绍.docx"
+    # file = Path("/Users/betterme/PycharmProjects/AI/MeUtils/meutils/llm/completions/rag/百炼系列手机产品介绍.docx")
+
+    # file = "/Users/betterme/PycharmProjects/AI/MeUtils/meutils/io/img_1.png"
+
+    # openai.BadRequestError: Error code: 400 - {'error': {'message': 'text extract error: 没有解析出内容', 'type': 'invalid_request_error'}}
+    # file = "https://oss.ffire.cc/files/kling_watermark.png"
+
+    # with timer():
+    #     r = arun(file_extract(file, moonshot_client))
+
+    # with timer():
+    #     r = arun(file_extract(file, provider='kimi'))
+
+    with timer():
+        arun(file_extract(file))

meutils/schemas/db/oneapi_types.py

@@ -101,7 +101,10 @@ class OneapiToken(SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
     user_id: Optional[int] = Field(default=None)
     key: Optional[str] = Field(default=None)
+
     used_quota: Optional[int] = Field(default=None)
+    remain_quota: Optional[int] = Field(default=None)
+    unlimited_quota: Optional[bool] = Field(default=False)
 
     class Config:
         arbitrary_types_allowed = True

meutils/schemas/image_types.py

@@ -10,6 +10,7 @@
 
 from meutils.pipe import *
 
+from pydantic import constr
 from openai.types import ImagesResponse as _ImagesResponse, Image
 
 ASPECT_RATIOS = {
@@ -58,7 +59,7 @@ class ImagesResponse(_ImagesResponse):
 
 
 class ImageRequest(BaseModel):  # openai
-    prompt: str = ""
+    prompt: constr(min_length=1, max_length=1000) = ""
     negative_prompt: Optional[str] = None
 
     model: str = ""
@@ -215,6 +216,31 @@ class KlingImageRequest(ImageRequest):
         self.size = "1:1" if self.size not in {"1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9"} else self.size
 
 
+class KolorsRequest(ImageRequest):
+    model: Literal["kolors-1.0", "kolors-1.5"] = "kolors-1.0"
+    n: Optional[int] = 4
+
+    """
+    1:1(1024*1024)
+    4:3(1152*896)
+    3:4(768*1024)
+    16:9(1024 576)
+    9:16(576 1024)
+    3:2(1024*640)
+    2:3(640*1024)
+    """
+    size: Literal[
+        "1024x1024", "1152x896", "768x1024", "1024x576", "576x1024", "1024x640", "640x1024"
+    ] = "1024x1024"
+
+    # 图生图
+    image: Optional[str] = None
+    image_fidelity: Optional[float] = None
+
+    def __init__(self, /, **data: Any):
+        super().__init__(**data)
+
+
 class CogviewImageRequest(ImageRequest):
     model: str = "cogview-3"
     size: Optional[Literal["1024x1024", "768x1344", "864x1152", "1344x768", "1152x864", "1440x720", "720x1440"]] = None
@@ -440,4 +466,4 @@ if __name__ == '__main__':
 
     # print(ImagesResponse(data=[{'url': 1}]))
 
-    print(Recraft(prompt="一条猫").model_dump_json())
+    print(RecraftImageRequest(prompt="").model_dump_json())

meutils/schemas/kuaishou_types.py

@@ -113,7 +113,7 @@ class KlingaiImageRequest(BaseModel):
             {'name': 'imageCount', 'value': self.n},
             {"name": "fidelity", "value": self.image_fidelity or 0.5},
             {'name': 'biz', 'value': 'klingai'},
-            {"name": "kolors_version", "value": "1.5"}
+            {"name": "kolors_version", "value": "1.0"} ####### 1.5
         ]
         self.payload = {
             'arguments': arguments,