MeUtils 2024.12.26.22.14.1__tar.gz → 2024.12.31.17.51.53__tar.gz

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53/MeUtils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MeUtils
-Version: 2024.12.26.22.14.1
+Version: 2024.12.31.17.51.53
 Summary: description
 Home-page: https://github.com/yuanjie-ai/MeUtils
 Author: yuanjie
@@ -108,42 +108,42 @@ Requires-Dist: pandas-profiling[notebook]; extra == "pd"
 Requires-Dist: pandas_summary; extra == "pd"
 Requires-Dist: polars; extra == "pd"
 Provides-Extra: all
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: jmespath; extra == "all"
+Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: simplejson; extra == "all"
+Requires-Dist: schedule; extra == "all"
 Requires-Dist: thefuck; extra == "all"
-Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: pymilvus; extra == "all"
 Requires-Dist: filetype; extra == "all"
-Requires-Dist: pymupd; extra == "all"
-Requires-Dist: jieba; extra == "all"
 Requires-Dist: pandas_summary; extra == "all"
-Requires-Dist: gunicorn; extra == "all"
-Requires-Dist: reportlab; extra == "all"
+Requires-Dist: geopy; extra == "all"
+Requires-Dist: polars; extra == "all"
+Requires-Dist: seaborn; extra == "all"
+Requires-Dist: cachetools; extra == "all"
 Requires-Dist: faiss-cpu; extra == "all"
-Requires-Dist: missingno; extra == "all"
-Requires-Dist: schedule; extra == "all"
-Requires-Dist: simplejson; extra == "all"
-Requires-Dist: openai; extra == "all"
+Requires-Dist: pandas-profiling[notebook]; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: uvicorn; extra == "all"
-Requires-Dist: fastapi[all]; extra == "all"
+Requires-Dist: pymysql; extra == "all"
 Requires-Dist: langchain; extra == "all"
-Requires-Dist: pymongo; extra == "all"
-Requires-Dist: seaborn; extra == "all"
-Requires-Dist: redis-py-cluster; extra == "all"
-Requires-Dist: sqlalchemy; extra == "all"
-Requires-Dist: pymilvus; extra == "all"
-Requires-Dist: pandas-profiling[notebook]; extra == "all"
-Requires-Dist: polars; extra == "all"
-Requires-Dist: jmespath; extra == "all"
-Requires-Dist: geopy; extra == "all"
-Requires-Dist: iteration_utilities; extra == "all"
 Requires-Dist: pretty_errors; extra == "all"
-Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
+Requires-Dist: missingno; extra == "all"
+Requires-Dist: gunicorn; extra == "all"
+Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: jieba; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: pymysql; extra == "all"
 Requires-Dist: streamlit; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: fastapi; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: fastapi[all]; extra == "all"
+Requires-Dist: reportlab; extra == "all"
+Requires-Dist: redis-py-cluster; extra == "all"
 Requires-Dist: dataframe_image; extra == "all"
 Requires-Dist: asyncmy; extra == "all"
+Requires-Dist: iteration_utilities; extra == "all"
+Requires-Dist: pymupd; extra == "all"
+Requires-Dist: fastapi; extra == "all"
 
 [![Downloads](http://pepy.tech/badge/meutils)](http://pepy.tech/project/meutils)
 

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/MeUtils.egg-info/SOURCES.txt

@@ -349,8 +349,6 @@ meutils/io/image.py
 meutils/io/minio_utils.py
 meutils/io/tf_io.py
 meutils/io/x.py
-meutils/jwt_utils/__init__.py
-meutils/jwt_utils/common.py
 meutils/llm/__init__.py
 meutils/llm/check_api.py
 meutils/llm/check_utils.py
@@ -436,14 +434,18 @@ meutils/queues/demo.py
 meutils/queues/smooth_queue.py
 meutils/queues/uniform_queue.py
 meutils/request_utils/__init__.py
+meutils/request_utils/ark.py
 meutils/request_utils/a爬虫.py
 meutils/request_utils/crawler.py
 meutils/request_utils/download.py
 meutils/request_utils/results.py
 meutils/request_utils/tiangong.py
+meutils/request_utils/volc.py
 meutils/request_utils/wechat.py
 meutils/request_utils/公网ip.py
 meutils/request_utils/爬虫.py
+meutils/request_utils/jwt_utils/__init__.py
+meutils/request_utils/jwt_utils/common.py
 meutils/schemas/__init__.py
 meutils/schemas/baidu_api.py
 meutils/schemas/baidu_types.py

{meutils-2024.12.26.22.14.1/MeUtils.egg-info → meutils-2024.12.31.17.51.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MeUtils
-Version: 2024.12.26.22.14.1
+Version: 2024.12.31.17.51.53
 Summary: description
 Home-page: https://github.com/yuanjie-ai/MeUtils
 Author: yuanjie
@@ -108,42 +108,42 @@ Requires-Dist: pandas-profiling[notebook]; extra == "pd"
 Requires-Dist: pandas_summary; extra == "pd"
 Requires-Dist: polars; extra == "pd"
 Provides-Extra: all
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: jmespath; extra == "all"
+Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: simplejson; extra == "all"
+Requires-Dist: schedule; extra == "all"
 Requires-Dist: thefuck; extra == "all"
-Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: pymilvus; extra == "all"
 Requires-Dist: filetype; extra == "all"
-Requires-Dist: pymupd; extra == "all"
-Requires-Dist: jieba; extra == "all"
 Requires-Dist: pandas_summary; extra == "all"
-Requires-Dist: gunicorn; extra == "all"
-Requires-Dist: reportlab; extra == "all"
+Requires-Dist: geopy; extra == "all"
+Requires-Dist: polars; extra == "all"
+Requires-Dist: seaborn; extra == "all"
+Requires-Dist: cachetools; extra == "all"
 Requires-Dist: faiss-cpu; extra == "all"
-Requires-Dist: missingno; extra == "all"
-Requires-Dist: schedule; extra == "all"
-Requires-Dist: simplejson; extra == "all"
-Requires-Dist: openai; extra == "all"
+Requires-Dist: pandas-profiling[notebook]; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: uvicorn; extra == "all"
-Requires-Dist: fastapi[all]; extra == "all"
+Requires-Dist: pymysql; extra == "all"
 Requires-Dist: langchain; extra == "all"
-Requires-Dist: pymongo; extra == "all"
-Requires-Dist: seaborn; extra == "all"
-Requires-Dist: redis-py-cluster; extra == "all"
-Requires-Dist: sqlalchemy; extra == "all"
-Requires-Dist: pymilvus; extra == "all"
-Requires-Dist: pandas-profiling[notebook]; extra == "all"
-Requires-Dist: polars; extra == "all"
-Requires-Dist: jmespath; extra == "all"
-Requires-Dist: geopy; extra == "all"
-Requires-Dist: iteration_utilities; extra == "all"
 Requires-Dist: pretty_errors; extra == "all"
-Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
+Requires-Dist: missingno; extra == "all"
+Requires-Dist: gunicorn; extra == "all"
+Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: jieba; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: pymysql; extra == "all"
 Requires-Dist: streamlit; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: fastapi; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: fastapi[all]; extra == "all"
+Requires-Dist: reportlab; extra == "all"
+Requires-Dist: redis-py-cluster; extra == "all"
 Requires-Dist: dataframe_image; extra == "all"
 Requires-Dist: asyncmy; extra == "all"
+Requires-Dist: iteration_utilities; extra == "all"
+Requires-Dist: pymupd; extra == "all"
+Requires-Dist: fastapi; extra == "all"
 
 [![Downloads](http://pepy.tech/badge/meutils)](http://pepy.tech/project/meutils)
 

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/_utils.py

@@ -10,6 +10,7 @@
 
 import json
 import base64
+from pydantic import BaseModel
 
 
 # bytes/string
@@ -39,8 +40,21 @@ def list_intersection(l1, l2):
 
 list4log = lambda ls: "\n\t" + "\n\t".join(ls)
 
+
 # 字典
-dict2json = bjson = lambda dic: json.dumps(dic, indent=4, ensure_ascii=False)
+
+
+def bjson(json_data):
+    if isinstance(json_data, dict):
+        return json.dumps(json_data, indent=4, ensure_ascii=False)
+    elif isinstance(json_data, BaseModel):
+        return json_data.model_dump_json(indent=4)
+
+    else:
+        return json_data
+
+
+dict2json = bjson
 json2dict = lambda s: json.loads(s.replace("'", '"'))
 
 

meutils-2024.12.31.17.51.53/meutils/apis/images/recraft.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : ecraft
+# @Time         : 2024/10/31 16:30
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from meutils.io.files_utils import to_url
+from meutils.schemas.image_types import ImageRequest, ImagesResponse, RecraftImageRequest
+from meutils.notice.feishu import IMAGES, send_message as _send_message
+from meutils.config_utils.lark_utils import get_next_token_for_polling, aget_spreadsheet_values
+from meutils.decorators.retry import retrying
+
+BASE_URL = "https://api.recraft.ai"
+
+FEISHU_URL = "https://xchatllm.feishu.cn/sheets/GYCHsvI4qhnDPNtI4VPcdw2knEd?sheet=Lrhtf2"
+
+DEFAULT_MODEL = "recraftv3"
+MODELS = {}
+
+send_message = partial(
+    _send_message,
+    title=__name__,
+    url=IMAGES
+)
+
+
+@alru_cache(ttl=10 * 60)
+@retrying()
+async def get_access_token(token: Optional[str] = None):
+    token = token or await get_next_token_for_polling(feishu_url=FEISHU_URL, check_token=check_token)
+    headers = {"cookie": token}
+
+    async with httpx.AsyncClient(base_url="https://www.recraft.ai", headers=headers, timeout=60) as client:
+        response = await client.get("/api/auth/session")
+        response.raise_for_status()
+        logger.debug(response.json())
+        return response.json()["accessToken"]
+
+
+# @retrying()
+async def generate(request: RecraftImageRequest, token: Optional[str] = None):
+    token = await get_access_token(token)
+    headers = {"Authorization": f"Bearer {token}"}
+    # params = {"project_id": "26016b99-3ad0-413b-821b-5f884bd9454e"}  # project_id 是否是必要的
+    params = {}  # project_id 是否是必要的
+    # params = {"project_id": "47ba6825-0fde-4cea-a17e-ed7398c0a298"}
+    payload = request.model_dump(exclude_none=True)
+    logger.debug(payload)
+
+    async with httpx.AsyncClient(base_url=BASE_URL, headers=headers, timeout=60) as client:
+        response = await client.post(f"/queue_recraft/prompt_to_image", params=params, json=payload)
+        response.raise_for_status()
+        params = {
+            "operation_id": response.json()["operationId"]
+        }
+        logger.debug(params)
+
+        response = await client.get("/poll_recraft", params=params)
+        response.raise_for_status()
+        metadata = response.json()
+        logger.debug(metadata)
+
+        # {'credits': 1,
+        #  'height': 1024,
+        #  'images': [{'image_id': 'f9d8e7dd-c31f-4208-abe4-f44cdff050c2',
+        #              'image_invariants': {'preset': 'any'},
+        #              'transparent': False,
+        #              'vector_image': False}],
+        #  'random_seed': 1423697946,
+        #  'request_id': '77bd917d-0960-4921-916f-038c773a41fd',
+        #  'transform_model': 'recraftv3',
+        #  'width': 1024}
+
+        params = {"raster_image_content_type": "image/webp"}  #####
+        params = {"raster_image_content_type": "image/png"}
+
+        images = []
+        for image in response.json()["images"]:
+            response = await client.get(f"""/image/{image["image_id"]}""", params=params)
+            url = await to_url(response.content)
+            images.append(url)
+
+        return ImagesResponse(image=images, metadata=metadata)
+
+
+async def check_token(token, threshold: float = 1):
+    if not isinstance(token, str):
+        tokens = token
+        r = []
+        for batch in tqdm(tokens | xgroup(32)):
+            bools = await asyncio.gather(*map(check_token, batch))
+            r += list(itertools.compress(batch, bools))
+        return r
+    try:
+        access_token = await get_access_token(token)
+        headers = {"Authorization": f"Bearer {access_token}"}
+
+        async with httpx.AsyncClient(base_url=BASE_URL, headers=headers, timeout=60) as client:
+            response = await client.get("/users/me")
+            response.raise_for_status()
+            data = response.json()
+            logger.debug(data["credits"])
+            return data["credits"] >= threshold
+    except Exception as e:
+        logger.error(e)
+        logger.debug(token)
+        return False
+
+
+if __name__ == '__main__':
+    token = None
+    # arun(get_access_token())
+    request = RecraftImageRequest(
+        prompt='一条猫'
+    )
+    # arun(generate(request, token=token))
+
+    tokens = list(arun(aget_spreadsheet_values(feishu_url=FEISHU_URL, to_dataframe=True))[0]) | xfilter_
+
+
+    r = arun(check_token(tokens))

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/apis/jimeng/images.py

@@ -72,7 +72,17 @@ async def get_task(task_id, token):
         data = response.json()
         logger.debug(bjson(data))
         # {'ret': '1015', 'errmsg': 'login error', 'systime': '1734524280', 'logid': '20241218201800AC3267447B287E9E6C46', 'data': None}
-        item_list = (data.get("data") or {}).get(task_id, {}).get("item_list")
+        task_info = (data.get("data") or {}).get(task_id, {})
+        item_list = task_info.get("item_list")  # "status": 30,
+
+        status_code = task_info.get("status")
+        logger.debug(f"status: {status_code}")
+
+        """
+        "status": 30, # 内容审核
+        "status": 50,
+        """
+
         image_data = map(lambda x: x.get("image", {}).get("large_images"), item_list)
 
         task_data = sum(image_data, []) | xmap_(lambda x: {"url": x.get("image_url")})
@@ -80,8 +90,9 @@ async def get_task(task_id, token):
         return TaskResponse(
             task_id=task_id,
             data=task_data,
-            message=data.get("errmsg"),
-            status="success" if task_data else None,
+            message=data.get("errmsg") if status_code!=30 else "触发内容审核451",
+            status="success" if item_list else 'processing',
+            code=status_code,
         )
 
 
@@ -117,4 +128,13 @@ if __name__ == '__main__':
 
     # arun(get_task(task.task_id, task.system_fingerprint))
 
-    arun(generate(ImageRequest(prompt="做一个圣诞节的海报")))
+    task_id = "10184295086338"
+    system_fingerprint = "eb4d120829cfd3ee957943f63d6152ed"
+
+    t1 = ("10184295086338", "eb4d120829cfd3ee957943f63d6152ed")
+    t2 = ("10184877310722", "dcf7bbc31faed9740b0bf748cd4d2c74")
+    t3 = ("10186352959490", "eb4d120829cfd3ee957943f63d6152ed")
+
+    arun(get_task(*t3))
+
+    # arun(generate(ImageRequest(prompt="做一个圣诞节的海报")))

meutils-2024.12.31.17.51.53/meutils/data/VERSION

@@ -0,0 +1 @@
+2024.12.31.17.51.53

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/data/oneapi/NOTICE.md

@@ -31,6 +31,11 @@
 
 </details>
 
+## 2024-12-27
+  - deepseek-v3
+  - deepseek-r1：deepseek-v3的思考模型
+  - deepseek-search：deepseek-v3的联网模型
+
 ## 2024-12-24
 - 上线新模型
   - doubao-pro-256k：相比Doubao-pro-128k/240628，长文任务效果显著提升10%以上，要点提取、字数遵循、多轮对话上文记忆等能力大幅提升

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/init/evn.py

@@ -35,7 +35,7 @@ os.environ['PROMPT_TEMPLATE'] = """
 # 如果无法从中得到答案，请回答“根据已知信息无法回答该问题”或“没有提供足够的信息”。请使用中文回答，不允许添加编造内容。
 # """
 
-# MODELSCOPE_CACHE
+# modelscope_cache
 # PPNLP
 if __name__ == '__main__':
     from pprint import pprint

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/llm/completions/rag/fire.py

@@ -12,6 +12,7 @@ import mimetypes
 from meutils.pipe import *
 from meutils.io.files_utils import to_bytes
 from meutils.llm.openai_utils import to_openai_params
+from meutils.str_utils.regular_expression import parse_url
 
 from openai import OpenAI, AsyncOpenAI
 from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, ChatCompletionRequest, CompletionUsage
@@ -66,6 +67,22 @@ class Completions(object):
 
             request.messages = request.system_messages + [{'role': 'user', 'content': request.last_content}]
 
+        elif isinstance(request.last_content, str) and (urls := parse_url(request.last_content)):
+            logger.debug(urls)
+
+            url = urls[-1]
+            request.last_content = request.last_content.replace(url, '')
+
+            files = await asyncio.gather(*map(self.file_extract, urls))
+
+            for file_content in files:
+                request.system_messages.append({
+                    "role": "system",
+                    "content": file_content,
+                })
+
+            request.messages = request.system_messages + [{'role': 'user', 'content': request.last_content}]
+
         logger.debug(request)
 
         data = to_openai_params(request)
@@ -78,30 +95,43 @@ if __name__ == '__main__':
 
     request = ChatCompletionRequest(
         # model="qwen-turbo-2024-11-01",
-        model="claude-3-5-sonnet-20241022",
+        # model="claude-3-5-sonnet-20241022",
+        model="gpt-4o-mini",
+
         messages=[
             {
                 'role': 'system',
                 'content': '你是一个文件问答助手'
             },
+            # {
+            #     'role': 'user',
+            #     # 'content': {
+            #     #     "type": "file_url",
+            #     #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #     # },
+            #     'content': [
+            #         {
+            #             "type": "text",
+            #             "text": "这个文件讲了什么？"
+            #         },
+            #         # 多轮的时候要剔除
+            #         {
+            #             "type": "file_url",
+            #             "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
+            #         }
+            #     ]
+            # },
+
             {
                 'role': 'user',
                 # 'content': {
                 #     "type": "file_url",
                 #     "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
                 # },
-                'content': [
-                    {
-                        "type": "text",
-                        "text": "这个文件讲了什么？"
-                    },
-                    # 多轮的时候要剔除
-                    {
-                        "type": "file_url",
-                        "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}
-                    }
-                ]
+                'content': "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？",
+
             },
+
             # {'role': 'assistant', 'content': "好的"},
             # {
             #     'role': 'user',

{meutils-2024.12.26.22.14.1 → meutils-2024.12.31.17.51.53}/meutils/llm/openai_utils/common.py

@@ -266,7 +266,7 @@ if __name__ == '__main__':
         with timer():
             try:
                 async with ppu_flow(api_key="sk-OYK4YxtTlWauT2TdGR5FTAJpkRmSnDwPly4cve0cAvMcrBkZ", post="api-oss",
-                                    n=10000):
+                                    n=100):
                     logger.debug("消费了哦")
 
             except Exception as e:

meutils-2024.12.31.17.51.53/meutils/request_utils/ark.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : ark
+# @Time         : 2024/12/31 13:44
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import json
+import os
+
+from meutils.pipe import *
+import volcenginesdkcore
+from volcenginesdkcore.signv4 import SignerV4
+import volcenginesdkark
+
+# sign(path, method, headers, body, query, ak, sk, region, service)
+
+headers = {}
+payload = {
+    "Prompt": "关于星空的歌",
+    "Genre": "R&B/Soul",
+    "Mood": "Dynamic/Energetic",
+    "Gender": "Male",
+    "ModelVersion": "v4.0"
+}
+params = {"Action": "GenLyrics", "Version": "2024-08-12"}
+SignerV4.sign(
+    path="/",
+    method="POST",
+    query=params,
+    body=json.dumps(payload),
+    headers=headers,
+    ak=os.getenv("ARK_ACCESS_KEY"),
+    sk=os.getenv("ARK_SECRET_ACCESS_KEY"),
+    region="cn-beijing",
+    service="imagination"
+)
+
+print(headers)
+
+s = ''
+for k, v in headers.items():
+    s += f"{k}:{v}\n"
+
+print(s)

{meutils-2024.12.26.22.14.1/meutils → meutils-2024.12.31.17.51.53/meutils/request_utils}/jwt_utils/common.py

@@ -33,6 +33,7 @@ payload = {
 # Your secret key
 secret = ""
 
+
 # Create the JWT
 token = jwt.encode(payload, secret, algorithm="HS256", headers=header)