MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/init/evn.py

@@ -35,7 +35,7 @@ os.environ['PROMPT_TEMPLATE'] = """
 # 如果无法从中得到答案，请回答“根据已知信息无法回答该问题”或“没有提供足够的信息”。请使用中文回答，不允许添加编造内容。
 # """
 
-# MODELSCOPE_CACHE
+# modelscope_cache
 # PPNLP
 if __name__ == '__main__':
     from pprint import pprint

meutils/io/files_utils.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : MeUtils.
+# @File         : file
+# @Time         : 2022/7/5 下午3:31
+# @Author       : yuanjie
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+import mimetypes
+from meutils.pipe import *
+from meutils.decorators.retry import retrying
+from meutils.caches.redis_cache import cache
+
+# from fastapi import UploadFile 有点区别
+from starlette.datastructures import UploadFile
+from contextlib import asynccontextmanager
+from httpx import AsyncClient
+
+
+def guess_mime_type(file):
+    mime_type = None
+    if isinstance(file, str):
+        mime_type, _ = mimetypes.guess_type(file)
+    return mime_type or "application/octet-stream"
+
+
+def file_append_firstline(line):
+    with open('untitled.txt', "r+") as f:
+        old = f.read()
+
+        f.seek(0)
+        f.write(line)
+        f.write(old)
+
+
+def base64_to_bytes(base64_image_string):
+    """
+    # 将字节数据写入图片文件
+    image_data = base64_to_bytes(...)
+    with open(filename, 'wb') as file:
+        file.write(image_data)
+    """
+    return base64.b64decode(base64_image_string.split(",", 1)[-1])
+
+
+async def to_bytes(
+        file: Union[UploadFile, str, bytes],
+        headers: Optional[dict] = None
+):
+    """
+
+    :param file: 文件对象、路径、base64、url
+    :param headers: httpx
+
+    :return: todo: bytes、filepath、io.BytesIO
+    """
+    # assert file
+
+    logger.debug(type(file))
+
+    if isinstance(file, bytes): return file
+
+    file_bytes = None
+    if isinstance(file, UploadFile):
+        file_bytes = await file.read()
+
+    elif isinstance(file, str) and file.startswith('http'):  # url
+        async with AsyncClient(headers=headers or {}, timeout=60) as cilent:
+            resp = await cilent.get(file)
+            file_bytes = resp.content
+
+    elif isinstance(file, str) and len(file) > 256:  # base64
+        file_bytes = base64_to_bytes(file)
+
+    elif isinstance(file, str) and len(file) < 256 and Path(file).is_file():  # file
+        file_bytes = Path(file).read_bytes()
+
+    return file_bytes
+
+
+@asynccontextmanager
+async def to_tempfile(file: Union[UploadFile, str]):
+    """
+
+    :param file: 文件对象、路径、base64、url
+    :return: todo: bytes、filepath、io.BytesIO
+    """
+    file_bytes = await to_bytes(file)
+
+    with tempfile.NamedTemporaryFile(mode='wb+') as temp:
+        temp.write(file_bytes)
+        temp.seek(0)
+
+        logger.debug(temp.name)
+
+        yield temp.name
+
+
+@cache(ttl=7 * 24 * 3600)
+async def to_url_fal(
+        file: Union[str, bytes, List],
+        filename: Optional[str] = None,
+        headers: Optional[dict] = None,
+        content_type: str = "application/octet-stream",
+):
+    """对象存储"""
+    if isinstance(file, list):
+        tasks = [to_url_fal(_, filename, headers, content_type) for _ in file]
+        urls = await asyncio.gather(*tasks)
+        return urls
+
+    if not file: return
+
+    if isinstance(file, str) and file.startswith("http"):  # 转存： todo: base64
+        content_type = mimetypes.guess_type(file)[0] or content_type
+
+    file = await to_bytes(file, headers=headers)
+    content_type = (
+            mimetypes.guess_type(filename or '')[0]
+            or mimetypes.guess_type(f"x.{content_type}")[0]  # format: image/png
+            or content_type
+    )
+
+    import fal_client
+
+    url = await fal_client.upload_async(data=file, content_type=content_type, file_name=filename)
+    return url
+
+
+async def to_url(
+        file: Union[UploadFile, str, bytes],
+        filename: Optional[str] = None,
+        headers: Optional[dict] = None,
+        content_type: Optional[str] = "image/*"  # todo: 适配 mime_type = "application/octet-stream"
+):
+    """对象存储"""
+    if not file: return
+
+    from openai.types.file_object import FileObject
+    from meutils.oss.minio_oss import Minio
+    uploads = [partial(Minio().put_object_for_openai, filename=filename)]
+
+    if len(file) < 1024 and isinstance(file, str) or filename:  # 不指定就自行判断 url
+        content_type = mimetypes.guess_type(filename)[0] or mimetypes.guess_type(file)[0] or content_type
+
+    if content_type and content_type.startswith("image"):
+        from meutils.apis.chatglm.glm_video import upload_task as glm_upload
+
+        uploads = [glm_upload, ] + uploads
+
+    for upload in uploads:
+        try:
+            file = await to_bytes(file, headers=headers)
+            url = await upload(file)
+            if isinstance(url, FileObject):
+                url = url.filename
+            return url
+        except Exception as e:
+            logger.error(e)
+
+
+async def to_base64(file: Union[UploadFile, str, bytes]):
+    _ = base64.b64encode(await to_bytes(file)).decode('utf-8')
+    return _
+
+
+def base64_to_file(base64_image_string, filename):
+    image_data = base64_to_bytes(base64_image_string)
+    with open(filename, 'wb') as file:
+        file.write(image_data)
+
+
+def file_to_base64(file):
+    _ = base64.b64encode(Path(file).read_bytes()).decode('utf-8')
+    content_type = mimetypes.guess_type('img.webp')[0] or "data:image/jpeg"
+
+    return f"data:{content_type},{_}"
+
+
+async def to_file(file: Union[UploadFile, str, bytes], filename: Optional[str] = None):
+    file = await to_bytes(file)
+    if not filename:  # 临时文件
+        with tempfile.NamedTemporaryFile(mode='wb+', delete=False) as temp:
+            temp.write(file)
+            temp.seek(0)
+            return temp.name
+    else:
+        Path(filename).write_bytes(file)
+        return Path(filename).resolve()
+
+
+if __name__ == '__main__':
+    # import tempfile
+    #
+    # # 使用上下文管理器自动处理文件的关闭和删除
+    # with tempfile.NamedTemporaryFile(mode='wb+') as temp:
+    #     temp.write(b"This is a temporary file.")
+    #     temp.seek(0)
+    #     print(f"文件内容: {temp.read()}")
+    #     print(f"临时文件名: {temp.name}")
+    # 文件在这里自动关闭和删除
+
+    # arun(to_bytes(''))
+
+    # arun(to_url("x.png"))
+
+    # url = "https://storage.googleapis.com/bsp-remini-image-in-web-us-central1-autodelete/54a2d77f-3070-4a95-8a1b-906ac1c74d44/64439c90-c628-47e0-9e1b-36dbbbd06664/ba502f58/input.jpg?X-Goog-Algorithm=GOOG4-HMAC-SHA256&X-Goog-Credential=GOOG1ETQDJI557KBP4YD5TQG6FMZHVKCP3S53FHI6XLBYYMT24W3PZAZNZZWQ%2F20241014%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241014T060335Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=b7745e3639a1cb81465409bfa7ed20801c5b997945739810c32cb4af02304893"
+
+    # arun(to_base64(url))[:100]
+
+    url = "https://oss.ffire.cc/files/kling_watermark.png"
+
+    # arun(to_file(url, filename='x.jpg'))
+
+    file = Path("/Users/betterme/PycharmProjects/AI/MeUtils/meutils/io/x.py").read_bytes()
+
+    arun(to_url_fal([file] * 10))
+
+    # print(mimetypes.guess_type("x.jpg"))
+    # print(mimetypes.guess_type("x.png"))
+    # print(mimetypes.guess_type("x.jpg"))
+
+    # print(mimetypes.guess_extension("x.mp4", False))
+
+    # arun(to_url(
+    #     "https://cdn.hailuoai.video/moss/prod/2024-11-11-09/video/1731287464150180347-video_raw_8ba15c5c206f8d393a9248f4f9215ed8_312186282087260162.mp4",
+    #     content_type=None))
+
+    # arun(to_url_fal(url))
+    # print(guess_mime_type("base64xxxxxxxxxxxxxxxxxx.png"))

meutils/io/image.py

@@ -9,10 +9,65 @@
 # @Description  : https://blog.csdn.net/Q_452086192/article/details/125014538
 # https://www.jb51.net/article/207138.htm
 
-import cv2
-from PIL import Image
+import mimetypes
+from PIL import Image, ImageDraw
 
 from meutils.pipe import *
+from meutils.io.files_utils import to_url, to_bytes
+
+
+async def describe_image(image: Union[str, bytes]):
+    """
+    图片转base64  webp会报错
+    :param image: 图片路径或图片对象
+    :return: base64字符串
+    """
+    _ = await to_bytes(image)
+    img = Image.open(io.BytesIO(_))
+
+    info = {
+        'format': img.format,  # 图片格式
+        'mode': img.mode,  # 颜色模式
+
+        'width': img.width,  # 宽度
+        'height': img.height,  # 高度
+        'size': img.size,  # (宽度, 高度)
+
+        'is_animated': getattr(img, 'is_animated', False),  # 是否是动图
+        'n_frames': getattr(img, 'n_frames', 1),  # 帧数
+        'info': img.info,  # 图片附加信息
+    }
+    return info
+
+
+def crop_polygon(image: Union[str, bytes], outline_points, inline_points):
+    if isinstance(image, bytes):
+        image = io.BytesIO(image)
+
+    # 打开图像
+    img = Image.open(image)
+
+    # 创建一个与原图大小相同的黑色遮罩
+    mask = Image.new('L', img.size, 0)
+
+    # 在遮罩上绘制白色多边形
+    for points in outline_points:
+        ImageDraw.Draw(mask).polygon(points, outline=1, fill=255)
+
+    for points in inline_points:
+        ImageDraw.Draw(mask).polygon(points, outline=1, fill=0)
+
+    # 将遮罩应用到原图
+    output = Image.new('RGBA', img.size, (0, 0, 0, 0))
+    output.paste(img, (0, 0), mask)
+
+    # # 将 PIL Image 转换为字节
+    # buffer = io.BytesIO()
+    # output.save(buffer, format="PNG")
+    # byte_data = buffer.getvalue()
+    #
+    # return byte_data
+    return output
 
 
 def img2bytes(img, format=None):
@@ -22,16 +77,21 @@ def img2bytes(img, format=None):
     @param format:
     @return:
     """
-    if isinstance(img, Image):
-        a = np.asarray(img)
+    # cv2 = try_import("cv2", pip_name="opencv-python")
+    import cv2
+
+    # if isinstance(img, Image):
+    #     img = np.asarray(img)
 
     # a = cv2.cvtColor(a, cv2.COLOR_RGB2BGR)
-    _, img_encode = cv2.imencode(format, a)
+    _, img_encode = cv2.imencode(format, img)
 
     return img_encode.tobytes()
 
 
 def bytes2img(_bytes):
+    import cv2
+
     np_arr = np.frombuffer(_bytes, dtype=np.uint8)
     # np.asarray(bytearray(bs), dtype=np.uint8)
     img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
@@ -39,10 +99,12 @@ def bytes2img(_bytes):
 
 
 def image_read(filename):
+    import cv2
+
     filename = str(filename)
     _bytes = b''
     if filename.startswith('http'):
-        _bytes = requests.get(url, stream=True).content
+        _bytes = requests.get(filename, stream=True).content
 
     elif Path(filename).exists():
         _bytes = Path(filename).read_bytes()
@@ -74,17 +136,93 @@ def base64_to_image(base64_str):
     return img_array
 
 
-def image_to_base64(image_path, for_image_url: bool = True):
-    _ = base64.b64encode(Path(image_path).read_bytes()).decode('utf-8')
+def image_to_base64(image_path_or_url, for_image_url: bool = True):
+    content_type = mimetypes.guess_type(image_path_or_url)[0] or "image/jpeg"
+    logger.debug(content_type)
+
+    if image_path_or_url.startswith('http'):
+        data = httpx.get(image_path_or_url, follow_redirects=True, timeout=100).content
+    else:
+        data = Path(image_path_or_url).read_bytes()
+
+    _ = base64.b64encode(data).decode('utf-8')
     if for_image_url:
-        _ = f"data:image/jpeg;base64,{_}"
+        _ = f"data:{content_type};base64,{_}"
     return _
 
 
+def base64_to_bytes(base64_image_string):
+    """
+    # 将字节数据写入图片文件
+    image_data = base64_to_bytes(...)
+    with open(filename, 'wb') as file:
+        file.write(image_data)
+    """
+    return base64.b64decode(base64_image_string.split(",", 1)[-1])
+
+
+@alru_cache()
+async def image2nowatermark_image(url, picinfo: str = '', oss: str = 'glm', token: Optional[str] = None):
+    if not url: return
+
+    from meutils.apis.baidu.bdaitpzs import create_task, BDAITPZSRequest
+
+    # response = await httpx.AsyncClient(timeout=100, headers=headers, follow_redirects=True).get(url)
+    # response.raise_for_status()
+
+    # 去水印
+    # request = BDAITPZSRequest(original_url=url, thumb_url=url) # 网络错误
+    request = BDAITPZSRequest(picInfo=image_to_base64(url), picInfo2=picinfo)
+    if picinfo:
+        request.type = 2
+        request.picInfo2 = picinfo
+
+    data = await create_task(request, is_async=False, token=token)
+    base64_string = data['picArr'][-1]['src']
+    file = base64_to_bytes(base64_string)
+
+    if oss == 'kling':  # kuaishou
+        from meutils.apis.kuaishou import klingai
+
+        file_task = await klingai.upload(file, cookie=token)
+        return file_task and file_task.url
+
+    else:
+        url = await to_url(file) or url
+        return url
+
+
 # alias
 base64_to_img = base64_to_image
 
 if __name__ == '__main__':
     url = "https://i1.mifile.cn/f/i/mioffice/img/slogan_5.png?1604383825042"
+    #
+    # print(image_read(url))
+    # base64_image_string = image_to_base64("img.png").split(",")[1]
+    # # print(base64_to_bytes(image_to_base64("img.png")))
+    # image_data = base64.b64decode(base64_image_string)
+    #
+    # with open("demo1.png", 'wb') as file:
+    #     file.write(base64_to_bytes(image_to_base64("img.png")))
+
+    # import mimetypes
+
+    # print(mimetypes.guess_type('img.webp')[0])
+    # image_to_base64(url)
+    s = """
+    """
+    # base64_to_file(s, 'x.png')
+
+    # url = "https://sfile.chatglm.cn/chatglm-videoserver/image/50/5049bae9.png"
+    url = "https://oss.ffire.cc/files/kling_watermark.png"
+    # arun(image2nowatermark_image(url))
+    # arun(url2url(url))
+
+    from meutils.schemas.baidu_types import PICINFO2_RIGHT_BOTTOM
+
+    url = "https://s22-def.ap4r.com/bs2/upload-ylab-stunt-sgp/se/ai_portal_sgp_queue_mmu_img2img_aiweb/7de17faa-cc11-48a6-b94a-e80fe7a34841/1.png"
 
-    print(image_read(url))
+    # arun(image2nowatermark_image(url, picinfo=PICINFO2_RIGHT_BOTTOM))
+    with timer():
+        arun(describe_image("https://oss.ffire.cc/files/kling_watermark.png"))

meutils/io/x.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : x
+# @Time         : 2024/9/25 15:27
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+# from minio import Minio
+from meutils.oss.minio_oss import Minio
+
+# MinIO client setup
+# minio_client = Minio(
+#     "minio-server-url:9000",
+#     access_key="your-access-key",
+#     secret_key="your-secret-key",
+#     secure=True  # set to False if not using HTTPS
+# )
+minio_client = Minio()
+
+
+async def download_and_upload(video_url, bucket_name, object_name):
+    buffer_size = 5 * 1024 * 1024  # 5MB buffer to meet MinIO's minimum part size
+
+    async with httpx.AsyncClient() as client:
+        try:
+            async with client.stream("GET", video_url) as response:
+                response.raise_for_status()
+
+                total_size = int(response.headers.get('content-length', 0))
+                buffer = io.BytesIO()
+                uploaded = 0
+
+                async for chunk in response.aiter_bytes(chunk_size=buffer_size):
+                    buffer.write(chunk)
+                    buffer_size = buffer.tell()
+                    buffer.seek(0)
+
+                    if buffer_size >= 5 * 1024 * 1024 or response.is_closed:
+                        try:
+                            minio_client.put_object(
+                                bucket_name,
+                                object_name,
+                                buffer,
+                                length=buffer_size,
+                                part_size=5 * 1024 * 1024,
+                                content_type='video/mp4'
+                            )
+                            uploaded += buffer_size
+                            print(f"Uploaded {uploaded}/{total_size} bytes")
+
+                        except Exception as upload_error:
+                            print(f"Unexpected upload error: {upload_error}")
+                            raise
+
+                        buffer = io.BytesIO()  # Reset buffer after upload
+
+                print("Upload completed")
+        except httpx.HTTPStatusError as http_error:
+            print(f"HTTP error occurred: {http_error}")
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
+
+
+if __name__ == '__main__':
+    # Usage
+    url = "https://s22-def.ap4r.com/bs2/upload-ylab-stunt-sgp/se/ai_portal_sgp_queue_m2v_txt2video_camera/b7eded0c-452c-4282-ad0a-02d96bd97f3e/0.mp4"
+    bucket_name = "videos"
+    object_name = "video11.mp4"
+
+    with timer():
+        arun(download_and_upload(url, bucket_name, object_name))

meutils/llm/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2024/4/24 13:35
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+

meutils/llm/check_api.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : check_api_key
+# @Time         : 2024/6/17 08:42
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+
+
+@alru_cache()
+async def check(api_key_or_token, check_url):
+    headers = {
+        "Authorization": f"Bearer {api_key_or_token}",
+        "Accept": "application/json"
+    }
+    payload = {
+        "token": api_key_or_token
+    }
+    try:
+        async with httpx.AsyncClient(headers=headers, timeout=60) as client:
+
+            if api_key_or_token.startswith("ey"):
+
+                response: httpx.Response = await client.post(check_url, json=payload)  # free-api: POST /token/check
+            else:
+                response: httpx.Response = await client.get(check_url)
+
+            # todo: 可扩展
+
+            logger.debug(response.text)
+            logger.debug(response.status_code)
+
+            return (
+                    response.is_success
+                    and response.json().get("is_available")  # deepseek
+                    or response.json().get("status", False)  # moonshot、siliconflow
+                    or response.json().get("live", False)  # free-api: POST /token/check
+            )
+
+    except httpx.RequestError as exc:
+        logger.error(exc)
+        return False
+
+
+async def check_api_key_or_token(
+        api_keys: Union[str, List[str]],
+        check_url: Optional[str] = "https://api.deepseek.com/user/balance",
+        return_api_keys: bool = True
+):
+    """
+        {
+        "is_available": false,
+        "balance_infos": [
+            {
+                "currency": "CNY",
+                "total_balance": "-0.00",
+                "granted_balance": "0.00",
+                "topped_up_balance": "-0.00"
+            }
+        ]
+        }
+    """
+    if isinstance(api_keys, str):
+        api_keys = [api_keys]
+
+    if not check_url: return api_keys
+
+    tasks = map(partial(check, check_url=check_url), tqdm(api_keys))
+    responses = await asyncio.gather(*tasks, return_exceptions=True)
+    # responses = await asyncio.gather(*tasks)
+    if return_api_keys:
+        responses = [k for k, v in zip(api_keys, responses) if v is True]
+
+    return responses
+
+
+
+
+if __name__ == '__main__':
+    from meutils.config_utils.lark_utils import get_spreadsheet_values, get_next_token_for_polling
+
+    #
+    # api_keys = get_spreadsheet_values(feishu_url="https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=lVghgx", to_dataframe=True)[0].tolist()
+    # print(arun(check_deepseek_api_key(api_keys)))
+
+    # api_keys = os.getenv("DEEPSEEK_API_KEY")
+    #
+    # # arun(check_api_key_or_token(api_keys))
+    #
+    # # print(arun(check_deepseek_api_key(api_keys)))
+    # # print(arun(check('sk-f42b2fdc036247e79cd471ab63b1142d', "https://api.deepseek.com/user/balance")))
+    #
+    # api_keys = os.getenv("SILICONFLOW_API_KEY").split()
+    # for api_key in api_keys:
+    #     arun(check(api_key, "https://api.siliconflow.cn/v1/user/info"))
+
+    # api_keys = get_spreadsheet_values(feishu_url="https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=EOZuBW", to_dataframe=True)[0].tolist()
+
+    # kimi
+    api_keys = get_spreadsheet_values(feishu_url="https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=Y7HVfo", to_dataframe=True)[0].tolist()
+    l = []
+    for api_key in api_keys:
+        if api_key:
+            if arun(check(api_key, "https://all.chatfire.cn/kimi/token/check")):
+                l.append(api_key)