MeUtils 2025.6.6.18.22.44__py3-none-any.whl → 2025.6.9.9.17.14__py3-none-any.whl

meutils/apis/jimeng_global/doubao_images.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : images
+# @Time         : 2024/12/16 17:46
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from openai import AsyncClient
+from meutils.pipe import *
+from meutils.apis.jimeng.doubao_utils import generate_cookie, generate_params
+
+from meutils.schemas.image_types import ImageRequest
+
+from meutils.schemas.jimeng_types import BASE_URL, MODELS_MAP, FEISHU_URL
+from meutils.config_utils.lark_utils import get_next_token_for_polling
+
+
+async def create(token: Optional[str] = None):
+    token = token or "712a47e7eec7c03b4cc7229775e06841"
+    cookie = generate_cookie(token)
+    params = generate_params()
+
+    headers = {
+        'Cookie': cookie,
+        'agw-js-conv': 'str',
+        'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
+        'content-type': 'application/json'
+    }
+    payload = {
+        "messages": [
+            {
+                "content": "{\"text\":\"一只猫\"}",
+                "content_type": 2009,
+                "attachments": [
+
+                ]
+            }
+        ],
+        "completion_option": {
+            "is_regen": False,
+            "with_suggest": False,
+            "need_create_conversation": False,
+            "launch_stage": 1,
+            "is_replace": False,
+            "is_delete": False,
+            "message_from": 0,
+            "event_id": "0"
+        },
+        "section_id": "6287920686327298",
+        "conversation_id": "6287920686327042",
+        "local_message_id": "936eee40-354d-11f0-83df-6b1810ffef8a"
+
+        # "local_message_id": str(uuid.uuid4())
+
+    }
+
+    client = AsyncClient(base_url="https://www.doubao.com/samantha", default_headers=headers, api_key='xx',
+                         )
+    response = await client.post("/chat/completion", body=payload, cast_to=object, stream=True,
+                                 options=dict(params=params))
+    async for i in response:
+        print(i)
+    # return response
+
+
+if __name__ == '__main__':
+    arun(create())

meutils/apis/jimeng_global/doubao_utils.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : utils
+# @Time         : 2024/12/18 11:03
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+import random
+import base64
+import time
+import uuid
+import secrets
+import string
+
+# 常量定义
+MODEL_NAME = "doubao"
+DEFAULT_ASSISTANT_ID = "497858"
+VERSION_CODE = "20800"
+DEVICE_ID = random.random() * 999999999999999999 + 7000000000000000000
+WEB_ID = random.random() * 999999999999999999 + 7000000000000000000
+USER_ID = str(uuid.uuid4()).replace('-', '')
+MAX_RETRY_COUNT = 3
+RETRY_DELAY = 5000
+FILE_MAX_SIZE = 100 * 1024 * 1024
+
+# 伪装headers
+FAKE_HEADERS = {
+    "Accept": "*/*",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    "Accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+    "Cache-control": "no-cache",
+    "Last-event-id": "undefined",
+    "Origin": "https://www.doubao.com",
+    "Pragma": "no-cache",
+    "Priority": "u=1, i",
+    "Referer": "https://www.doubao.com",
+    "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Platform": '"Windows"',
+    "Sec-Fetch-Dest": "empty",
+    "Sec-Fetch-Mode": "cors",
+    "Sec-Fetch-Site": "same-origin",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+}
+
+
+async def acquire_token(refresh_token: str) -> str:
+    """
+    获取缓存中的access_token
+    目前doubao的access_token是固定的，暂无刷新功能
+
+    Args:
+        refresh_token: 用于刷新access_token的refresh_token
+
+    Returns:
+        str: access_token
+    """
+    return refresh_token
+
+
+def generate_fake_ms_token() -> str:
+    """
+    生成伪msToken
+    """
+    # 生成96字节的随机数据
+    random_bytes = secrets.token_bytes(96)
+    # 转换为base64，并替换特殊字符
+    token = base64.b64encode(random_bytes).decode('utf-8')
+    return token.replace('+', '-').replace('/', '_').rstrip('=')
+
+
+def generate_random_string(length: int) -> str:
+    """
+    生成指定长度的随机字符串
+    """
+    chars = string.ascii_letters + string.digits
+    return ''.join(random.choice(chars) for _ in range(length))
+
+
+def generate_fake_a_bogus() -> str:
+    """
+    生成伪a_bogus
+    """
+    return f"mf-{generate_random_string(34)}-{generate_random_string(6)}"
+
+
+def generate_params():
+    params = {
+        "aid": "497858",
+        "device_id": int(DEVICE_ID),
+        "device_platform": "web",
+        "language": "zh",
+        "pkg_type": "release_version",
+        "real_aid": "497858",
+        "region": "CN",
+        "samantha_web": "1",
+        "sys_region": "CN",
+        "tea_uuid": int(WEB_ID),
+        "use-olympus-account": "1",
+        "version_code": "20800",
+        "web_id": int(WEB_ID),
+        "msToken": generate_fake_ms_token(),
+        "a_bogus": generate_fake_a_bogus()
+    }
+
+    return params
+
+
+def generate_cookie(refresh_token: str, ms_token: Optional[str] = None) -> str:
+    """
+    生成cookie
+    """
+    ms_token = ms_token or generate_fake_ms_token()
+
+    current_timestamp = int(time.time())
+    cookie_parts = [
+        f"is_staff_user=false",
+        f"store-region=cn-gd",
+        f"store-region-src=uid",
+        f"sid_guard={refresh_token}%7C{current_timestamp}%7C5184000%7CSun%2C+02-Feb-2025+04%3A17%3A20+GMT",
+        f"uid_tt={USER_ID}",
+        f"uid_tt_ss={USER_ID}",
+        f"sid_tt={refresh_token}",
+        f"sessionid={refresh_token}",
+        f"sessionid_ss={refresh_token}",
+        f"msToken={ms_token}",
+    ]
+    return "; ".join(cookie_parts)
+
+
+async def get_upload_token(refresh_token):  # 3600过期
+    """
+
+    {'code': 0,
+     'data': {'auth': {'access_key_id': 'AKTPYzNkMjJlNTNjMWE1NDJiN2E5MWFkOTYxMWViYzQxYTM',
+                       'current_time': '2024-12-18T11:17:22+08:00',
+                       'expired_time': '2024-12-18T12:17:22+08:00',
+                       'secret_access_key': 'HFFTkEFKf+0DVpUrYy2yMzvgnkxLMU6+qydnGUSaDmd0vSRedpIi0qmeWSVElyOU',
+                       'session_token': 'STS2eyJMVEFjY2Vzc0tleUlkIjoiQUtMVFlUZGhPR0ptWVRNNFl6ZG1OR1JoWVRoaE0yWTJPVFl5TW1SbU0yRmhNREEiLCJBY2Nlc3NLZXlJZCI6IkFLVFBZek5rTWpKbE5UTmpNV0UxTkRKaU4yRTVNV0ZrT1RZeE1XVmlZelF4WVRNIiwiU2lnbmVkU2VjcmV0QWNjZXNzS2V5IjoiTC9WZTVSMmt4N3dsY1kvS0E5alp1WVlpSlVFM0ZjdHMzQ2Q5QjJZMVE3NlRnUDVONWViMmpKQkRQMUdyUEtqeXNYNXRKVkJPdExvVjVNOGFyY24wQ2ZtdUZRRWMxMG8xMSs3UHdKdGY0LzQ9IiwiRXhwaXJlZFRpbWUiOjE3MzQ0OTU0NDIsIlBvbGljeVN0cmluZyI6IntcIlN0YXRlbWVudFwiOlt7XCJFZmZlY3RcIjpcIkFsbG93XCIsXCJBY3Rpb25cIjpbXCJJbWFnZVg6QXBwbHlJbWFnZVVwbG9hZFwiLFwiSW1hZ2VYOkNvbW1pdEltYWdlVXBsb2FkXCJdLFwiUmVzb3VyY2VcIjpbXCJ0cm46SW1hZ2VYOio6KjpTZXJ2aWNlSWQvYTlybnMycmw5OFwiXX0se1wiRWZmZWN0XCI6XCJBbGxvd1wiLFwiQWN0aW9uXCI6W1wiUFNNXCJdLFwiUmVzb3VyY2VcIjpbXCJmbG93LmFsaWNlLnJlc291cmNlX2FwaVwiXX1dfSIsIlNpZ25hdHVyZSI6ImI2MGUxNDZkZTU0Njg2NTdlYzVlZmFjZjJlOWNlOWE5YTdhY2UwNTFlZTdkYTJjZTRmNjdiYmRiM2U4MDQ3N2IifQ=='},
+              'service_id': 'a9rns2rl98',
+              'upload_host': 'imagex.bytedanceapi.com',
+              'upload_path_prefix': 'bot-chat-image'},
+     'msg': ''}
+
+    :return:
+    """
+    cookie = generate_cookie(refresh_token)
+    url = "https://www.doubao.com/alice/upload/auth_token"
+
+    headers = {
+        'Cookie': cookie,
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    }
+    payload = {
+        "scene": "bot_chat",
+        "data_type": "image"
+    }
+
+    async with httpx.AsyncClient(timeout=60) as client:
+        response = await client.post(url, headers=headers, json=payload)
+        response.raise_for_status()
+
+        return response.json()
+
+
+if __name__ == '__main__':
+    # generate_cookie("")
+
+    arun(get_upload_token("712a47e7eec7c03b4cc7229775e06841"))
+    # print(generate_params())
+    # print(get_signing_key('xW9YbDhTlWsXdaN7O2g1lfcyePxf5kJyg/r2mwSZG/iuSmbvVgToO6LVCLmUjVJ3'))

meutils/apis/jimeng_global/files.py

@@ -0,0 +1,368 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : kaiyuan
+# @Time         : 2024/12/18 16:03
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+import hmac
+import zlib
+import hashlib
+
+from meutils.pipe import *
+from meutils.caches.redis_cache import cache
+from meutils.io.files_utils import to_bytes
+from meutils.apis.jimeng.common import get_upload_token, get_headers, BASE_URL, FEISHU_URL, check_token
+from meutils.notice.feishu import send_message_for_images
+from meutils.config_utils.lark_utils import get_next_token_for_polling
+
+
+def random_str(n):
+    return ''.join(random.sample('zyxwvutsrqponmlkjihgfedcba0123456789', n))
+
+
+def hash256(msg):
+    return hashlib.sha256(msg.encode('utf-8')).hexdigest()
+
+
+def hmac_hash256(key, msg):
+    if type(key) == str:
+        return hmac.new(key.encode('utf-8'), msg.encode('utf-8'), hashlib.sha256)
+    elif type(key) == hmac.HMAC:
+        return hmac.new(key.digest(), msg.encode('utf-8'), hashlib.sha256)
+
+
+def fileCRC32(file_buffer):
+    return hex(zlib.crc32(file_buffer) & 0xFFFFFFFF)[2:]
+
+
+def u(params):
+    new_params = sorted(params.items(), key=lambda x: x[0])
+    new_params = [f"{k}={v}" for k, v in new_params]
+    return "&".join(new_params)
+
+
+class JJRequest:
+    def __init__(self, e, t, api, method="GET", params=None, data=None, serviceName="imagex"):
+        self.t = t
+        self.e = e
+        self.api = api
+        self.method = method
+        self.params = params
+        self.data = data
+
+        self.serviceName = serviceName
+
+    def getAuthorization(self):
+        return f"AWS4-HMAC-SHA256 Credential={self.e['access_key_id']}/{self.t[:8]}/cn-north-1/{self.serviceName}/aws4_request, SignedHeaders=x-amz-date;x-amz-security-token, Signature={self.signature()}"
+
+    def signature(self):
+        r = self.getSigningKey()
+        return hmac_hash256(r, self.stringToSign()).hexdigest()
+
+    def getSigningKey(self, r="cn-north-1"):
+        n = self.serviceName
+
+        o = hmac_hash256("AWS4" + self.e['secret_access_key'], str(self.t[0:8]))
+        i = hmac_hash256(o, str(r))
+        s = hmac_hash256(i, str(n))
+        return hmac_hash256(s, "aws4_request")
+
+    def stringToSign(self):
+        t = []
+        t.append("AWS4-HMAC-SHA256")
+        t.append(self.t)
+        t.append(self.credentialString())
+        t.append(hash256(self.canonicalString()))
+        return "\n".join(t)
+
+    def credentialString(self, region="cn-north-1"):
+        return "/".join([self.t[0:8], region, self.serviceName, "aws4_request"])
+
+    def canonicalString(self):
+        e = []
+        e.append(self.method)
+        e.append("/")
+        e.append(u(self.params))
+        e.append(self.canonicalHeaders())
+        e.append(self.signedHeaders())
+        e.append(self.hexEncodedBodyHash())
+        return "\n".join(e)
+
+    def canonicalHeaders(self):
+        return f"x-amz-date:{self.t}\nx-amz-security-token:{self.e['session_token']}\n"
+
+    def signedHeaders(self):
+        return "x-amz-date;x-amz-security-token"
+
+    def hexEncodedBodyHash(self):
+        return hash256("")
+
+
+# @cache(ttl=15 * 60)
+async def upload(image: bytes, upload_token: dict):  # oss 跨账号不知道是否可以使用
+    # e = auth = upload_token['data']['auth'] # 豆包
+    data = upload_token['data']
+    # service_id = data.get('service_id', '3jr8j4ixpe')  # 即梦 3jr8j4ixpe 豆包 a9rns2rl98
+
+    service_id = data.get('space_name') or "3jr8j4ixpe"  # 即梦视频 tb4s082cfz
+
+    if 'auth' in data:
+        data = data['auth']
+
+    session_token = data['session_token']
+
+    t = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
+
+    # "ServiceId": "tb4s082cfz",
+    params = {
+        "Action": "ApplyImageUpload",
+        "Version": "2018-08-01",
+        "ServiceId": service_id,
+        "s": random_str(10),
+
+        # "FileExtension": ".png",  #####
+        "FileSize": len(image),
+    }
+
+    r = JJRequest(data, t, "https://imagex.bytedanceapi.com/", method="GET", params=params)
+    headers = {
+        'authorization': r.getAuthorization(),
+        'x-amz-date': t,
+        'x-amz-security-token': session_token,
+    }
+    # logger.debug(headers)
+    response = requests.get(r.api, params=params, headers=headers)
+    response.raise_for_status()
+    logger.debug(response.status_code)
+    response = response.json()
+
+    logger.debug(bjson(response))
+    if "Result" not in response:
+        return
+
+    store_info = response['Result']['UploadAddress']['StoreInfos'][0]
+    logger.debug(bjson(store_info))
+
+    oss_uri = store_info['StoreUri']
+    oss_token = store_info['Auth']
+
+    headers = {
+        "authorization": oss_token,
+        "content-length": str(len(image)),
+        "content-Type": "image/jpeg",
+        "content-crc32": fileCRC32(image),
+    }
+
+    # oss_url = f"https://{resp['Result']['UploadAddress']['UploadHosts'][0]}/{oss_uri}"
+
+    # image_resp = requests.put(  # post
+    #     oss_url,
+    #     headers=headers,
+    #     data=image,
+    # )
+    # logger.debug(image_resp.json())
+
+    # return get_url(StoreUri)
+
+    # upload_url = f"https://tos-hl-x.snssdk.com/upload/v1/{oss_uri}"
+    upload_url = f"https://{response['Result']['UploadAddress']['UploadHosts'][0]}/upload/v1/{oss_uri}"
+
+    # response = requests.post(upload_url, headers=headers, data=image)
+    # response.raise_for_status()
+    # response = response.json()
+    # logger.debug(response)
+
+    async with httpx.AsyncClient(headers=headers, timeout=60) as client:
+        response = await client.post(upload_url, content=image)
+        response.raise_for_status()
+        data = response.json()
+        logger.debug(bjson(data))
+
+    return oss_uri
+
+
+async def upload_for_vod(image: bytes, upload_token: dict):  # oss 跨账号不知道是否可以使用
+    """
+                    "UploadHost": "tos-lf-x.snssdk.com",
+                    "UploadHost": "tos-hl-x.snssdk.com",
+
+    :param image:
+    :param upload_token:
+    :return:
+    """
+    # e = auth = upload_token['data']['auth'] # 豆包
+    data = upload_token['data']
+    service_id = data.get('space_name', 'tb4s082cfz')  # 即梦 3jr8j4ixpe 豆包 a9rns2rl98
+
+    if 'auth' in data:
+        data = data['auth']
+
+    session_token = data['session_token']
+
+    t = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
+
+    params = {
+        "Action": "ApplyUploadInner",
+        "Version": "2020-11-19",
+
+        "ServiceId": service_id,
+        # "SpaceName": "artist_op",
+        # "SpaceName": "dreamina",
+
+        "IsInner": "1",
+
+        "FileType": "video",
+        "FileSize": len(image),
+
+        "s": random_str(10),
+
+    }
+
+    r = JJRequest(data, t, "https://vod.bytedanceapi.com/", method="GET", params=params, serviceName="vod")
+    headers = {
+        'authorization': r.getAuthorization(),
+        'x-amz-date': t,
+        'x-amz-security-token': session_token,
+    }
+    # logger.debug(headers)
+    # response = requests.get(r.api, params=params, headers=headers)
+    # response.raise_for_status()
+    # logger.debug(response.status_code)
+    # response = response.json()
+    # logger.debug(bjson(response))
+
+    async with httpx.AsyncClient(headers=headers, params=params, timeout=120) as client:
+        response = await client.get(r.api)
+        response.raise_for_status()
+        response = response.json()
+        logger.debug(bjson(response))
+
+    if "Result" not in response:
+        return
+
+    upload_node = response['Result']['InnerUploadAddress']['UploadNodes'][0]
+    store_info = upload_node['StoreInfos'][0]
+    logger.debug(bjson(store_info))
+
+    vid = upload_node['Vid']
+    upload_host = upload_node['UploadHost']
+
+    oss_uri = store_info['StoreUri']
+    oss_token = store_info['Auth']
+
+    headers = {
+        "authorization": oss_token,
+        "content-length": str(len(image)),
+        "content-Type": "application/octet-stream",
+        "content-crc32": fileCRC32(image),
+    }
+
+    # upload_url = f"https://tos-hl-x.snssdk.com/upload/v1/{oss_uri}"
+    upload_url = f"https://{upload_host}/upload/v1/{oss_uri}"
+
+    # response = requests.post(upload_url, headers=headers, data=image)
+    # response.raise_for_status()
+    # response = response.json()
+    # logger.debug(response)
+
+    async with httpx.AsyncClient(headers=headers, timeout=120) as client:
+        response = await client.post(upload_url, content=image)
+        response.raise_for_status()
+        data = response.json()
+        logger.debug(bjson(data))
+
+    return vid, oss_uri
+
+
+async def upload_for_image(image, token, biz: Optional[str] = None):  # todo: 跨账号token
+    """image url base64 bytes"""
+    if not image: return
+
+    upload_token = await get_upload_token(token, biz)
+    image_uri = await upload(await to_bytes(image), upload_token)
+    return image_uri
+
+
+async def upload_for_video(video, token):  # 跨账号token
+    """video url base64 bytes
+    """
+    if not video: return
+
+    upload_token = await get_upload_token(token)
+    vid, uri = await upload_for_vod(await to_bytes(video), upload_token)
+    return vid, uri
+
+
+async def face_recognize(image, token: Optional[str] = None):
+    """
+    图片识别
+    """
+    # token = "1c21a9fe6a4230609d7ff13e5cec41ec"  # 跨账号测试
+
+    token = token or await get_next_token_for_polling(FEISHU_URL, check_token)
+
+    url = "/mweb/v1/face_recognize"
+    headers = get_headers(url, token)
+
+    image_uri = await upload_for_image(image, token)
+    payload = {
+        "image_uri_list": [
+            image_uri
+        ]
+    }
+
+    async with httpx.AsyncClient(base_url=BASE_URL, headers=headers, timeout=120) as client:
+        response = await client.post(url, json=payload)
+        response.raise_for_status()
+        data = response.json()
+
+        try:
+            data['data']['image_uri'] = image_uri
+            data['data']['face_recognize_list'][0][0]['type'] = ""
+            data['data']['face_recognize_list'][0][0]['id'] = str(uuid.uuid4())
+            data['data']['face_recognize_list'][0][0]['is_selected'] = True
+        except Exception as e:
+            logger.error(e)
+            send_message_for_images(f"图片识别失败 {bjson(data)}", __name__)
+
+        return data
+
+
+if __name__ == "__main__":
+    # 豆包
+    token = "de2215a7bb8e442774cf388f03fac84c"
+
+    # jimeng
+    token = "ffeee346fbd19eceebb79a7bfbca4bfe"
+    token = "34438eb03d165737122180caf62a8058"
+    #
+    # upload_token = arun(get_upload_token(token))
+
+    #
+    # with open("test.jpg", "rb") as f:
+    #     file = image = f.read()
+    #
+    #     print(upload(image, upload_token))
+    # print(upload_for_vod(image, upload_token))
+    #
+    # with timer():
+    #     arun(upload_for_video("https://fal.media/files/koala/8teUPbRRMtAUTORDvqy0l.mp4", token))
+
+    # with timer():
+    #     arun(upload_for_image("https://oss.ffire.cc/files/kling_watermark.png", token))
+
+    # with timer():
+    #     url = "https://oss.ffire.cc/files/lipsync.mp3"
+    #     # arun(upload_for_video("https://oss.ffire.cc/files/lipsync.mp3", token))
+    #     arun(upload_for_video(url, token))
+    image_url = "https://oss.ffire.cc/files/kling_watermark.png"
+    # arun(face_recognize(image_url, token))
+    # arun(face_recognize(image_url))
+
+    token = "ed16bb360a4744696f88a7b52b7c10a3"
+
+    arun(upload_for_image(image_url, token, "video"))