MeUtils 2024.12.13.15.39.7__py3-none-any.whl → 2024.12.19.11.21.54__py3-none-any.whl

meutils/apis/jimeng/utils.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : utils
+# @Time         : 2024/12/18 11:03
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import httpx
+
+from meutils.pipe import *
+import random
+import base64
+import time
+import uuid
+import secrets
+import string
+import hmac
+import hashlib
+
+# 常量定义
+MODEL_NAME = "doubao"
+DEFAULT_ASSISTANT_ID = "497858"
+VERSION_CODE = "20800"
+DEVICE_ID = random.random() * 999999999999999999 + 7000000000000000000
+WEB_ID = random.random() * 999999999999999999 + 7000000000000000000
+USER_ID = str(uuid.uuid4()).replace('-', '')
+MAX_RETRY_COUNT = 3
+RETRY_DELAY = 5000
+FILE_MAX_SIZE = 100 * 1024 * 1024
+
+# 伪装headers
+FAKE_HEADERS = {
+    "Accept": "*/*",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    "Accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+    "Cache-control": "no-cache",
+    "Last-event-id": "undefined",
+    "Origin": "https://www.doubao.com",
+    "Pragma": "no-cache",
+    "Priority": "u=1, i",
+    "Referer": "https://www.doubao.com",
+    "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Platform": '"Windows"',
+    "Sec-Fetch-Dest": "empty",
+    "Sec-Fetch-Mode": "cors",
+    "Sec-Fetch-Site": "same-origin",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+}
+
+
+async def acquire_token(refresh_token: str) -> str:
+    """
+    获取缓存中的access_token
+    目前doubao的access_token是固定的，暂无刷新功能
+
+    Args:
+        refresh_token: 用于刷新access_token的refresh_token
+
+    Returns:
+        str: access_token
+    """
+    return refresh_token
+
+
+def generate_fake_ms_token() -> str:
+    """
+    生成伪msToken
+    """
+    # 生成96字节的随机数据
+    random_bytes = secrets.token_bytes(96)
+    # 转换为base64，并替换特殊字符
+    token = base64.b64encode(random_bytes).decode('utf-8')
+    return token.replace('+', '-').replace('/', '_').rstrip('=')
+
+
+def generate_random_string(length: int) -> str:
+    """
+    生成指定长度的随机字符串
+    """
+    chars = string.ascii_letters + string.digits
+    return ''.join(random.choice(chars) for _ in range(length))
+
+
+def generate_fake_a_bogus() -> str:
+    """
+    生成伪a_bogus
+    """
+    return f"mf-{generate_random_string(34)}-{generate_random_string(6)}"
+
+
+def generate_cookie(refresh_token: str, ms_token: Optional[str] = None) -> str:
+    """
+    生成cookie
+    """
+    ms_token = ms_token or generate_fake_ms_token()
+
+    current_timestamp = int(time.time())
+    cookie_parts = [
+        f"is_staff_user=false",
+        f"store-region=cn-gd",
+        f"store-region-src=uid",
+        f"sid_guard={refresh_token}%7C{current_timestamp}%7C5184000%7CSun%2C+02-Feb-2025+04%3A17%3A20+GMT",
+        f"uid_tt={USER_ID}",
+        f"uid_tt_ss={USER_ID}",
+        f"sid_tt={refresh_token}",
+        f"sessionid={refresh_token}",
+        f"sessionid_ss={refresh_token}",
+        f"msToken={ms_token}",
+    ]
+    return "; ".join(cookie_parts)
+
+
+async def get_upload_token():  # 3600过期
+    """
+
+    {'code': 0,
+     'data': {'auth': {'access_key_id': 'AKTPYzNkMjJlNTNjMWE1NDJiN2E5MWFkOTYxMWViYzQxYTM',
+                       'current_time': '2024-12-18T11:17:22+08:00',
+                       'expired_time': '2024-12-18T12:17:22+08:00',
+                       'secret_access_key': 'HFFTkEFKf+0DVpUrYy2yMzvgnkxLMU6+qydnGUSaDmd0vSRedpIi0qmeWSVElyOU',
+                       'session_token': 'STS2eyJMVEFjY2Vzc0tleUlkIjoiQUtMVFlUZGhPR0ptWVRNNFl6ZG1OR1JoWVRoaE0yWTJPVFl5TW1SbU0yRmhNREEiLCJBY2Nlc3NLZXlJZCI6IkFLVFBZek5rTWpKbE5UTmpNV0UxTkRKaU4yRTVNV0ZrT1RZeE1XVmlZelF4WVRNIiwiU2lnbmVkU2VjcmV0QWNjZXNzS2V5IjoiTC9WZTVSMmt4N3dsY1kvS0E5alp1WVlpSlVFM0ZjdHMzQ2Q5QjJZMVE3NlRnUDVONWViMmpKQkRQMUdyUEtqeXNYNXRKVkJPdExvVjVNOGFyY24wQ2ZtdUZRRWMxMG8xMSs3UHdKdGY0LzQ9IiwiRXhwaXJlZFRpbWUiOjE3MzQ0OTU0NDIsIlBvbGljeVN0cmluZyI6IntcIlN0YXRlbWVudFwiOlt7XCJFZmZlY3RcIjpcIkFsbG93XCIsXCJBY3Rpb25cIjpbXCJJbWFnZVg6QXBwbHlJbWFnZVVwbG9hZFwiLFwiSW1hZ2VYOkNvbW1pdEltYWdlVXBsb2FkXCJdLFwiUmVzb3VyY2VcIjpbXCJ0cm46SW1hZ2VYOio6KjpTZXJ2aWNlSWQvYTlybnMycmw5OFwiXX0se1wiRWZmZWN0XCI6XCJBbGxvd1wiLFwiQWN0aW9uXCI6W1wiUFNNXCJdLFwiUmVzb3VyY2VcIjpbXCJmbG93LmFsaWNlLnJlc291cmNlX2FwaVwiXX1dfSIsIlNpZ25hdHVyZSI6ImI2MGUxNDZkZTU0Njg2NTdlYzVlZmFjZjJlOWNlOWE5YTdhY2UwNTFlZTdkYTJjZTRmNjdiYmRiM2U4MDQ3N2IifQ=='},
+              'service_id': 'a9rns2rl98',
+              'upload_host': 'imagex.bytedanceapi.com',
+              'upload_path_prefix': 'bot-chat-image'},
+     'msg': ''}
+
+    :return:
+    """
+    cookie = generate_cookie('de2215a7bb8e442774cf388f03fac84c')
+    url = "https://www.doubao.com/alice/upload/auth_token"
+
+    headers = {
+        'priority': 'u=1, i',
+        'Cookie': cookie,
+        'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
+    }
+    payload = {
+        "scene": "bot_chat",
+        "data_type": "image"
+    }
+
+    async with httpx.AsyncClient() as client:
+        response = await client.post(url, headers=headers, json=payload)
+        response.raise_for_status()
+
+        return response.json()
+
+
+def hmac_hash256(key, msg):
+    if type(key) == str:
+        return hmac.new(key.encode('utf-8'), msg.encode('utf-8'), hashlib.sha256)
+    elif type(key) == hmac.HMAC:
+        return hmac.new(key.digest(), msg.encode('utf-8'), hashlib.sha256)
+
+
+def get_signing_key(secret_access_key, r="cn-north-1", n="imagex"):
+    dt = str(datetime.datetime.now())[:10].replace('-', '')
+    o = hmac_hash256("AWS4" + secret_access_key, dt)
+    i = hmac_hash256(o, str(r))
+    s = hmac_hash256(i, str(n))
+    return hmac_hash256(s, "aws4_request")
+
+def signature(secret_access_key):
+    r = get_signing_key(secret_access_key)
+    return hmac_hash256(r, self.stringToSign()).hexdigest()
+
+# ccd4fef2cca1a114e776badad7f4b6e73f305a4dbb09e68f336759ddb6ac0025
+
+if __name__ == '__main__':
+    # generate_cookie("")
+
+    # arun(get_upload_token())
+
+    print(get_signing_key('xW9YbDhTlWsXdaN7O2g1lfcyePxf5kJyg/r2mwSZG/iuSmbvVgToO6LVCLmUjVJ3'))

meutils/apis/siliconflow/videos.py

@@ -7,13 +7,16 @@
 # @WeChat       : meutils
 # @Software     : PyCharm
 # @Description  :
+import asyncio
 
 from meutils.pipe import *
 from meutils.caches.redis_cache import cache
+from meutils.io.files_utils import to_url, to_url_fal
 from meutils.llm.check_utils import check_token_for_siliconflow
 from meutils.schemas.task_types import TaskResponse
 from meutils.schemas.siliconflow_types import BASE_URL, VideoRequest
 from meutils.config_utils.lark_utils import get_next_token_for_polling
+from meutils.apis.translator import deeplx
 
 from openai import OpenAI, AsyncOpenAI
 
@@ -21,38 +24,54 @@ FEISHU_URL_FREE = "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf
 
 check_token = partial(check_token_for_siliconflow, threshold=0.01)
 
+MODELS_MAP = {
+    "hunyuan-video": "tencent/HunyuanVideo",
+    "hunyuanvideo": "tencent/HunyuanVideo",
+    "mochi-1-preview": "genmo/mochi-1-preview",
+    "ltx-video": "Lightricks/LTX-Video",
+}
+
 
 @cache(ttl=7 * 24 * 3600)
 async def create_task(request: VideoRequest, token: Optional[str] = None):
     token = token or await get_next_token_for_polling(FEISHU_URL_FREE, check_token=check_token, from_redis=True)
 
     payload = request.model_dump(exclude_none=True)
-    payload["model"] = "tencent/HunyuanVideo"
+    payload["model"] = MODELS_MAP.get(request.model, "Lightricks/LTX-Video")
+
+    if payload["model"] in {"genmo/mochi-1-preview", "Lightricks/LTX-Video"}:  # 中文不友好
+        payload['prompt'] = (
+            await deeplx.translate(deeplx.DeeplxRequest(text=request.prompt, target_lang="EN"))).get("data")
 
-    client = OpenAI(
+    client = AsyncOpenAI(
         base_url=BASE_URL,
         api_key=token
     )
 
-    response = client.post("/video/submit", cast_to=object, body=payload)
+    response = await client.post("/video/submit", body=payload, cast_to=object)
     logger.debug(response)
-    task_id = response.get('requestId')
 
+    task_id = response.get('requestId')
     return TaskResponse(task_id=task_id, system_fingerprint=token)
 
 
 async def get_task(task_id, token: str):
-    client = OpenAI(
+    client = AsyncOpenAI(
         base_url=BASE_URL,
         api_key=token
     )
     payload = {"requestId": task_id}
-    response = client.post(f"/video/status", cast_to=object, body=payload)
+    response = await client.post(f"/video/status", cast_to=object, body=payload)
     logger.debug(response)
 
+    data = response.get("results") or {}
+
+    for video in data.get("videos", []):
+        video["url"] = await to_url_fal(video.get("url"))  # 异步执行
+
     return TaskResponse(
         task_id=task_id,
-        data=response.get("results"),
+        data=data,
         status=response.get("status"),
         message=response.get("reason"),
     )
@@ -60,12 +79,22 @@ async def get_task(task_id, token: str):
 
 if __name__ == '__main__':
     token = None
+    token = "sk-raapiguffsnsxgkfiwfusjmbpcyqoxhcohhxaybflrnvpqjw"
+
+    request = VideoRequest(
+        model="Lightricks/LTX-Video",
+        prompt="这个女人笑起来",
+        image='https://oss.ffire.cc/files/kling_watermark.png'  # 1148f2e4-0a62-4208-84de-0bf2c88f740d
+    )
     # tokens = config_manager.text.split()
 
     # tokens_ = arun(check_token_for_siliconflow(tokens, threshold=0.01))
 
-    token = "sk-raapiguffsnsxgkfiwfusjmbpcyqoxhcohhxaybflrnvpqjw"
-    arun(get_task("73a15e11-310f-4464-a9af-3ab84b201fff", token))
+    # arun(create_task(request, token=token))
+
+    # arun(get_task("fa248aac-00ec-4b00-a2f8-3d6bf1cea6d3", token))
+    # arun(get_task("c716a328-438e-4612-aff2-a669034499cb", token))
+    arun(get_task("1148f2e4-0a62-4208-84de-0bf2c88f740d", token))
 
     # token = "sk-oeptckzkhfzeidbtsqvbrvyrfdtyaaehubfwsxjytszbgohd"
     # arun(get_task("5ea22f57-45f0-425c-9d1e-bf3dae7e1e81", token))

meutils/data/VERSION

@@ -1 +1 @@
-2024.12.13.15.39.07
+2024.12.19.11.21.54

meutils/data/oneapi/NOTICE.md

@@ -2,7 +2,7 @@
 
 ---
 <details markdown="1">
-  <summary><b>业务范围</b></summary>
+  <summary><b>🔥业务经营范围</b></summary>
 
 - api服务（没有的找企微客服增加）
     - 提供主流大模型服务，gpt/claude/gemini/llama/国产大模型等等
@@ -31,6 +31,10 @@
 
 </details>
 
+## 2024-12-13
+- 新增模型 混元视频（支持高并发，非逆向可商用，限时特价1毛）[接口文档](https://api.chatfire.cn/docs/api-244309840)
+  HunyuanVideo 是腾讯推出的开源视频生成基础模型，拥有超过 130 亿参数，是目前最大的开源视频生成模型。该模型采用统一的图像和视频生成架构，集成了数据整理、图像-视频联合模型训练和高效基础设施等关键技术。模型使用多模态大语言模型作为文本编码器，通过 3D VAE 进行空间-时间压缩，并提供提示词重写功能。根据专业人工评估结果，HunyuanVideo 在文本对齐、运动质量和视觉质量等方面的表现优于现有最先进的模型
+
 ## 2024-12-09
 - 新增模型
   - meta-llama/Llama-3.3-70B-Instruct: Llama 3.3 是 Llama 系列最先进的多语言开源大型语言模型，以极低成本体验媲美 405B 模型的性能。基于 Transformer 结构，并通过监督微调（SFT）和人类反馈强化学习（RLHF）提升有用性和安全性。其指令调优版本专为多语言对话优化，在多项行业基准上表现优于众多开源和封闭聊天模型。知识截止日期为 2023 年 12 月。
@@ -43,6 +47,13 @@
   - o1-pro: （官网 200刀 plus 版本 `逆向工程`，有思考过程显示）o1-pro 是OpenAI针对复杂任务的新推理模型，该任务需要广泛的常识。该模型具有 200k 上下文，目前全球最强模型，支持图片识别
 
 
+
+
+---
+
+<details markdown="1">
+  <summary><b>历史更新</b></summary>
+
 ## 2024-12-05
 - 新增模型gpt-4-plus/gpt-4o-plus按倍率计算
   > OpenAI-plus会员 逆向工程
@@ -59,32 +70,4 @@
 
 
 
----
-
-<details markdown="1">
-  <summary><b>历史更新</b></summary>
-## 2024-11-20 
-
-### 支持ReplicateAPI，兼容goamz系统
-
-- [支持的模型列表](https://api.chatfire.cn/docs/doc-5518412)
-
-    | 模型                               | 每张图片单价 |
-    |------------------------------------------|----------------|
-    | black-forest-labs/flux-1.1-pro            | $0.040         |
-    | black-forest-labs/flux-1.1-pro-ultra      | $0.060         |
-    | black-forest-labs/flux-dev                | $0.025         |
-    | black-forest-labs/flux-pro                | $0.055         |
-    | black-forest-labs/flux-schnell            | $0.003         |
-    | ideogram-ai/ideogram-v2                   | $0.080         |
-    | ideogram-ai/ideogram-v2-turbo             | $0.050         |
-    | recraft-ai/recraft-v3                     | $0.040         |
-    | recraft-ai/recraft-v3-svg                 | $0.080         |
-    | stability-ai/stable-diffusion-3           | $0.035         |
-    | stability-ai/stable-diffusion-3.5-large   | $0.065         |
-    | stability-ai/stable-diffusion-3.5-large-turbo | $0.040         |
-    | stability-ai/stable-diffusion-3.5-medium  | $0.035         |
-
-- [接口文档](https://api.chatfire.cn/docs/api-235361039)
-
 </details>

meutils/io/files_utils.py

@@ -11,6 +11,8 @@
 import mimetypes
 from meutils.pipe import *
 from meutils.decorators.retry import retrying
+from meutils.caches.redis_cache import cache
+
 # from fastapi import UploadFile 有点区别
 from starlette.datastructures import UploadFile
 from contextlib import asynccontextmanager
@@ -85,6 +87,33 @@ async def to_tempfile(file: Union[UploadFile, str]):
         yield temp.name
 
 
+@cache(ttl=7 * 24 * 3600)
+async def to_url_fal(
+        file: Union[str, bytes],
+        filename: Optional[str] = None,
+        headers: Optional[dict] = None,
+        content_type: str = "application/octet-stream",
+):
+    """对象存储"""
+    if not file: return
+
+    if file.startswith("http"):  # 转存： todo: base64
+        content_type = mimetypes.guess_type(file)[0] or content_type
+
+        file = await to_bytes(file, headers=headers)
+
+    content_type = (
+            mimetypes.guess_type(filename or '')[0]
+            or mimetypes.guess_type(f"x.{content_type}")[0]  # format: image/png
+            or content_type
+    )
+
+    import fal_client
+
+    url = await fal_client.upload_async(data=file, content_type=content_type, file_name=filename)
+    return url
+
+
 async def to_url(
         file: Union[UploadFile, str, bytes],
         filename: Optional[str] = None,
@@ -179,12 +208,14 @@ if __name__ == '__main__':
 
     # arun(to_url(file, content_type=None))
 
-    print(mimetypes.guess_type("x.jpg"))
+    # print(mimetypes.guess_type("x.jpg"))
     # print(mimetypes.guess_type("x.png"))
-    print(mimetypes.guess_type("x"))
+    # print(mimetypes.guess_type("x.jpg"))
 
-    print(mimetypes.guess_extension("x.mp4", False))
+    # print(mimetypes.guess_extension("x.mp4", False))
 
     # arun(to_url(
     #     "https://cdn.hailuoai.video/moss/prod/2024-11-11-09/video/1731287464150180347-video_raw_8ba15c5c206f8d393a9248f4f9215ed8_312186282087260162.mp4",
     #     content_type=None))
+
+    arun(to_url_fal(url))

meutils/llm/completions/rag/fire.py

@@ -77,7 +77,8 @@ if __name__ == '__main__':
     c = Completions()
 
     request = ChatCompletionRequest(
-        model="qwen-turbo-2024-11-01",
+        # model="qwen-turbo-2024-11-01",
+        model="claude-3-5-sonnet-20241022",
         messages=[
             {
                 'role': 'system',

meutils/llm/mappers.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : mappers
+# @Time         : 2024/12/16 14:49
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+
+def llm_mapper(model):
+    return

meutils/office_automation/pdf.py

@@ -58,4 +58,8 @@ def doc2text(filename):
 
 if __name__ == '__main__':
     with timer():
-        extract_text('5.1-SNZ413量锐指数增强9号私募证券投资基金私募基金合同--套印版.pdf')
+       r = extract_text('x.pdf')
+
+
+    # import tiktoken
+    # print(tiktoken.encoding_for_model('gpt-3.5-turbo'))

meutils/schemas/jimeng_types.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : jimeng_types
+# @Time         : 2024/12/16 18:20
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import uuid
+
+from meutils.pipe import *
+
+BASE_URL = "https://jimeng.jianying.com"
+FEISHU_URL = "https://xchatllm.feishu.cn/sheets/GYCHsvI4qhnDPNtI4VPcdw2knEd?sheet=zkPAHw"
+
+MODELS_MAP = {
+    "jimeng-2.1": "high_aes_general_v21_L:general_v2.1_L",
+
+    "jimeng-2.0-pro": "high_aes_general_v20_L:general_v2.0_L",
+    "high_aes_general_v20_L:general_v2.0_L": "high_aes_general_v20_L:general_v2.0_L",
+
+    "jimeng-2.0": "high_aes_general_v20:general_v2.0",
+    "jimeng-1.4": "high_aes_general_v14:general_v1.4",
+    "jimeng-xl-pro": "text2img_xl_sft",
+
+    "default": "high_aes_general_v21_L:general_v2.1_L",
+}

meutils/schemas/oneapi/common.py

@@ -63,6 +63,10 @@ MODEL_PRICE = {
     "api-images-flux.1.1-pro": 0.1,
     "api-images-recraftv3": 0.03,
 
+    "api-images-seededit": 0.1,
+    "seededit": 0.1,
+    "chat-seededit": 0.1,
+
     "api-tripo3d": 0.1,
 
     # 图片 音频 视频
@@ -83,6 +87,8 @@ MODEL_PRICE = {
 
     "ideogram": 0.3,
 
+    "api-hunyuan-video": 0.1,
+
     # replicate
     "api-replicate-flux-1.1-pro": 0.040,
     "api-replicate-flux-1.1-pro-ultra": 0.060,
@@ -197,7 +203,12 @@ MODEL_PRICE = {
     "cogview-3-plus": 0.02,
 
     "glm-4-all": 0.01,
+
     "kimi-all": 0.01,
+    "kimi-math": 0.01,
+    "kimi-k1": 0.05,
+    "kimi-search": 0.01,
+    "kimi-research": 0.05,
 
     "spark-all": 0.01,
     "step-1-all": 0.01,
@@ -293,6 +304,7 @@ MODEL_RATIO = {
     "glm-4v": 2.5,
     "glm-4v-plus": 1,  # 2.5
     "glm-4v-flash": 0.1,
+    "glm-4-plus": 25,
 
     # 月之暗面 https://platform.moonshot.cn/docs/price/chat#%E4%BA%A7%E5%93%81%E5%AE%9A%E4%BB%B7
     "moonshot-v1-8k": 6 / 2,  # 特价
@@ -500,6 +512,7 @@ MODEL_RATIO = {
     "gpt-4o-2024-08-06": 1.25,
     "gpt-4o-2024-11-20": 1.25,
 
+    "o1": 7.5,
     "o1-mini": 1.5,
     "o1-preview": 7.5,
     "o1-mini-2024-09-12": 1.5,

meutils/schemas/task_types.py

@@ -63,7 +63,7 @@ class TaskResponse(BaseModel):
 
     def __init__(self, /, **data: Any):
         super().__init__(**data)
-        self.status = STATUSES.get(self.status.lower(), "UNKNOWN")
+        self.status = STATUSES.get((self.status or '').lower(), "UNKNOWN")
 
     class Config:
         # 允许额外字段，增加灵活性