MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/async_task/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2024/11/28 15:28
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+# from meutils.async_task.common import worker, shared_task, AsyncResult, states
+from meutils.async_task.common import *

meutils/async_task/celery_config.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : config
+# @Time         : 2024/11/28 14:47
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from datetime import timedelta
+from kombu import Exchange, Queue
+
+# 基础配置
+broker_url = 'redis://localhost:6379/0'  # 消息代理（推荐使用Redis）
+broker_url = os.getenv('REDIS_URL', broker_url)
+
+# result_backend = f"redis://localhost:6379/1"  # 结果存储
+result_backend = f"{broker_url.replace('/0', '')}/1"  # 结果存储
+
+# logger.debug(result_backend)
+
+# 添加以下配置增加可靠性
+broker_connection_retry = True
+broker_connection_max_retries = 5
+broker_connection_retry_on_startup = True
+
+# 时区设置
+enable_utc = False
+timezone = 'Asia/Shanghai'
+
+# 任务序列化与反序列化配置
+task_serializer = 'json'
+result_serializer = 'json'
+accept_content = ['json']
+
+# 并发配置
+# worker_pool = 'solo'  # 工作进程池类型，可选：'prefork'、'eventlet'、'gevent'
+worker_pool_restarts = True
+worker_concurrency = 8  # worker并发数，一般设置为CPU核心数
+worker_prefetch_multiplier = 3  # 预取任务数
+worker_max_tasks_per_child = 256  # 每个worker执行多少个任务后自动重启
+
+# 任务队列配置
+task_default_queue = 'default'  # 默认队列
+task_queues = (
+    Queue('default', Exchange('default'), routing_key='default'),
+    Queue('high_priority', Exchange('high_priority'), routing_key='high_priority'),
+    Queue('low_priority', Exchange('low_priority'), routing_key='low_priority'),
+)
+
+# 任务执行设置
+task_soft_time_limit = 10 * 60  # 任务软超时时间（秒）
+task_time_limit = 2 * task_soft_time_limit  # 任务硬超时时间（秒）
+
+# 任务重试配置
+task_publish_retry = True  # 发布任务失败时重试
+task_publish_retry_policy = {
+    'max_retries': 5,  # 最大重试次数
+    'interval_start': 0,  # 初始重试等待时间
+    'interval_step': 2,  # 重试间隔递增步长
+    'interval_max': 16,  # 最大重试间隔
+}
+
+# # 任务路由配置
+# task_routes = {
+#     'project.tasks.high_priority_task': {'queue': 'high_priority'},
+#     'project.tasks.low_priority_task': {'queue': 'low_priority'},
+# }
+#
+# # 定时任务配置
+# beat_schedule = {
+#     'task-every-30-seconds': {
+#         'task': 'project.tasks.periodic_task',
+#         'schedule': timedelta(seconds=30),
+#     },
+# }
+
+# 日志配置
+worker_log_format = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s'
+worker_task_log_format = '[%(asctime)s: %(levelname)s/%(processName)s] [%(task_name)s(%(task_id)s)] %(message)s'
+
+# 性能优化
+task_acks_late = True  # 任务执行完成后再确认
+task_reject_on_worker_lost = True  # worker异常关闭时任务会被重新分配
+worker_disable_rate_limits = True  # 禁用任务频率限制
+
+# 结果后端配置
+result_expires = 30 * 24 * 3600  # 结果过期时间（秒）
+result_persistent = True  # 结果持久化存储
+
+# 任务执行配置
+# task_always_eager = False  # 是否立即执行任务（调试用）
+
+# 添加任务持久化配置
+celery_task_store_errors_even_if_ignored = True
+celery_task_ignore_result = False  # 不忽略结果
+celery_task_track_started = True  # 跟踪任务状态
+
+# Redis 持久化配置
+broker_transport_options = {
+    'visibility_timeout': 43200,  # 12小时
+    'fanout_prefix': True,
+    'fanout_patterns': True,
+}

meutils/async_task/common.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : common
+# @Time         : 2024/11/28 15:28
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+from meutils.decorators.retry import retrying
+
+from meutils.async_task import celery_config
+
+from celery import Celery, Task, shared_task, states
+
+
+
+worker = Celery()
+
+worker.config_from_object(celery_config)
+
+worker.conf.update(
+    # result_expires=30 * 24 * 60 * 60,
+    # enable_utc=False,
+    # timezone='Asia/Shanghai',
+    task_track_started=True,
+)
+
+if __name__ == '__main__':
+    print(worker.conf.humanize(with_defaults=False))
+
+    print(worker.conf.broker_url)
+    print(worker.conf.result_backend)
+
+    print(arun(get_task("59ddf636-3f27-4110-948e-5977c8cbe1b3")))

meutils/async_task/demo_create_tasks.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : x
+# @Time         : 2024/11/28 15:42
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+from meutils.async_task import worker
+from meutils.async_task.tasks import test, kling, hailuo
+from celery.result import AsyncResult
+
+# print(worker.conf)
+
+# @shared_task(
+#     autoretry_for=(Exception,),      # 自动重试的异常类型
+#     retry_kwargs={
+#         'max_retries': 3,            # 最大重试次数
+#         'countdown': 60              # 重试等待时间（秒）
+#     },
+#     retry_backoff=True,              # 启用指数退避
+#     retry_backoff_max=600,           # 最大退避时间（秒）
+#     retry_jitter=True                # 添加随机抖动
+# )
+# def my_task():
+#     try:
+#         # 任务逻辑
+#         result = some_operation()
+#         return result
+#     except Exception as exc:
+#         logger.error(f"Task failed: {exc}")
+#         raise  # 触发自动重试
+
+
+if __name__ == '__main__':
+    # r = test.do_sync_task.apply_async(kwargs={'a': 1, 'b': 2})
+    # logger.debug(r.backend)
+    # test.ado_sync_task.apply_async(kwargs={'a': 1, 'b': 2})
+    # test.ado_sync_task.apply_async(kwargs={'a': 1, 'b': 3})
+    # print(AsyncResult("42e191a0-6099-419e-b61d-07bf7e2df2fc").result)
+    # result.backend
+    # AsyncResult("42e191a0-6099-419e-b61d-07bf7e2df2fc").get()
+    # AsyncResult(r.id).result
+
+    # test.do_pydantic_task.apply_async(kwargs={'request': test.Request()})
+
+    # test.do_pydantic_task.apply_async(kwargs={"request": {"method": "POST", "url": "测试"}})
+
+    # print(type(test.do_pydantic_task))
+
+    # test.do_pydantic_task.apply_async(kwargs={"request": {"method": "POST", "url": "测试"}})
+    # test.do_pydantic_task.apply_async(kwargs={"request": {"method": "xx"}})
+    # test.do_pydantic_task.apply_async(args=({"method": "xxxxxxx"},))
+
+    #
+    # test.do_task_retrying.apply_async(kwargs={'name': "###do_task_retrying"})
+    # test.do_task_retry.apply_async(kwargs={'name': "###do_task_retry"})
+    # test.do_task_retry_backoff.apply_async(kwargs={'name': "###do_task_retry_backoff"})
+    # test.ado_pydantic_task.apply_async(kwargs={'name': "###do_task_retry_backoff"})
+
+    test.Request()
+    # test.ado_pydantic_task.apply_async(kwargs={"request": None})
+
+    # test.ado_pydantic_task.apply_async(kwargs={"request": test.Request().model_dump()})
+    # test.ado_pydantic_task_2.apply_async(kwargs={"request": test.Request().model_dump()})
+
+    # kling.create_task.apply_async(kwargs={"request": kling.kolors_virtual_try_on.TryOnRequest().model_dump()})
+
+    hailuo.create_task.apply_async(kwargs={"request": hailuo.VideoRequest(prompt='a dog').model_dump()})

meutils/async_task/tasks/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2024/11/28 16:03
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+

meutils/async_task/tasks/_all.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : all_tasks
+# @Time         : 2024/11/28 17:50
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+# 所有异步任务
+from meutils.pipe import *
+from meutils.async_task import worker
+
+for p in get_resolve_path('.', __file__).glob('*'):
+
+    if (not p.name.startswith('_')) and p.name.endswith('.py'):
+        logger.debug(p)
+        _ = try_import(f"meutils.async_task.tasks.{p.stem}")

meutils/async_task/tasks/hailuo.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : hailuo
+# @Time         : 2024/12/6 09:05
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+from meutils.async_utils import async_to_sync_pro
+from meutils.async_task import worker, shared_task
+from meutils.apis.hailuoai.videos import VideoRequest, create_task as remote_create_task, get_task
+
+
+@shared_task(pydantic=True, retry_kwargs={'max_retries': 10, 'countdown': 10})
+@async_to_sync_pro
+async def create_task(request, **kwargs):
+    if isinstance(request, dict):
+        request = VideoRequest(**request)
+
+    response = await remote_create_task(request)
+    return response.model_dump()

meutils/async_task/tasks/kling.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : kling
+# @Time         : 2024/11/28 16:18
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 注册任务
+
+from meutils.async_utils import async_to_sync_pro
+from meutils.async_task import worker, shared_task
+
+from meutils.apis.kling import kolors_virtual_try_on
+
+
+@shared_task(pydantic=True, retry_kwargs={'max_retries': 5, 'countdown': 10})
+@async_to_sync_pro
+async def create_task(request: kolors_virtual_try_on.TryOnRequest, **kwargs):
+    if isinstance(request, dict):
+        request = kolors_virtual_try_on.TryOnRequest(**request)
+
+    response = await kolors_virtual_try_on.create_task(request)
+    return response.model_dump()
+
+
+if __name__ == '__main__':
+    pass
+    # create_task.apply_async()
+    create_task.apply_async(kwargs={"request": kolors_virtual_try_on.TryOnRequest().model_dump()})

meutils/async_task/tasks/replicateai.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : replicateai
+# @Time         : 2024/11/29 19:08
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+from meutils.pipe import *
+from meutils.async_utils import async_to_sync_pro
+from meutils.decorators.retry import retrying
+from meutils.async_task import worker, shared_task
+
+from meutils.apis.replicateai import raw as replicate
+
+
+@shared_task(pydantic=True, retry_kwargs={'max_retries': 5, 'countdown': 10})
+@async_to_sync_pro
+async def create_task(request: replicate.ReplicateRequest, **kwargs):
+    response = await replicate.create_task(request)
+    return response

meutils/async_task/tasks/test.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : kling
+# @Time         : 2024/11/28 16:18
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+from meutils.decorators.retry import retrying
+from meutils.async_utils import async_to_sync_pro
+from meutils.async_task import worker, shared_task
+
+
+class Request(BaseModel):
+    method: str = "GET"
+    url: str = "https://api.chatfire.cn/"
+
+    class Config:
+        frozen = True
+
+
+@shared_task
+def do_sync_task(sleep=10, **kwargs):
+    logger.debug("同步任务")
+    time.sleep(sleep)
+
+    return kwargs
+
+
+# AttributeError: 'AsyncToSync' object has no attribute '__name__'. Did you mean: '__ne__'?
+
+
+@alru_cache()
+async def create_task(request: Request, **kwargs):
+    logger.debug(request)
+    await asyncio.sleep(10)
+    return request
+
+
+worker.task()
+
+
+@shared_task(pydantic=True)
+@async_to_sync_pro
+async def ado_pydantic_task(request: Request):
+    logger.debug("同步任务+协程+结构体+缓存")
+    logger.debug(request)
+    if isinstance(request, dict):
+        request = Request(**request)
+
+    return await create_task(request)
+
+
+@worker.task(pydantic=True)
+@async_to_sync_pro
+async def ado_pydantic_task_2(request: Request):
+    logger.debug("同步任务+协程+结构体+缓存")
+    logger.debug(request)
+
+    return await create_task(request)
+
+
+@shared_task(pydantic=True)
+def do_pydantic_task(request: Request) -> Request:
+    logger.debug(request)
+    return request
+
+
+@shared_task(retry_kwargs={'max_retries': 10})
+def do_task_retry(*args, **kwargs):
+    logger.debug("do_task_retry")
+
+    return 1 / 0
+
+
+@shared_task(
+    default_retry_delay=3,
+    retry_backoff=True,
+    retry_kwargs={'max_retries': 10}
+)
+def do_task_retry_backoff_noautoretry(*args, **kwargs):
+    logger.debug("do_task_retry_backoff")
+
+    return 1 / 0
+
+
+@shared_task(
+    autoretry_for=(Exception,),
+    default_retry_delay=3,
+    retry_backoff=True,
+    retry_kwargs={'max_retries': 10}
+)
+def do_task_retry_backoff(*args, **kwargs):
+    logger.debug("do_task_retry_backoff autoretry")
+
+    return 1 / 0
+
+
+@shared_task(retry_kwargs={'max_retries': 3})
+@retrying(10)
+def do_task_retrying(*args, **kwargs):
+    logger.debug("do_task_retrying")
+    return 1 / 0
+
+
+@shared_task
+@retrying(3)
+def proxy_task(**kwargs):
+    method = kwargs.pop('method', '')
+    url = kwargs.pop('url', '')
+
+    logger.debug(kwargs)
+
+    response = requests.request(method, url, **kwargs).json()
+
+    return response
+
+
+if __name__ == '__main__':
+    # print(do_sync_task.apply_async(kwargs={'a': 1, 'b': 2}))
+    print(AsyncToSync)

meutils/async_task/tasks/vidu.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : kling
+# @Time         : 2024/11/28 16:18
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+from meutils.decorators.retry import retrying
+from meutils.async_task import worker, shared_task
+
+from meutils.apis.vidu.vidu_video import create_task, get_task, ViduRequest
+
+
+@shared_task
+async def do_task(**kwargs):
+    return create_task(ViduRequest(**kwargs))
+
+
+@shared_task
+@retrying(3)
+async def proxy_task(**kwargs):
+    pass
+
+    return response

meutils/async_task/utils.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : utils
+# @Time         : 2024/11/29 16:22
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 通用异步设计，兼容oneapi监控
+
+from celery.result import AsyncResult
+
+from meutils.pipe import *
+from meutils.schemas.task_types import TaskResponse
+
+from meutils.db.orm import update_or_insert
+from meutils.schemas.db.oneapi_types import OneapiTask
+
+from meutils.db.redis_db import redis_aclient
+
+from fastapi import APIRouter, File, UploadFile, Query, Form, Depends, Request, HTTPException, status, BackgroundTasks
+
+
+async def create_task(async_task, request: Union[BaseModel, dict]):
+    if not isinstance(request, dict):
+        request = request.model_dump()
+
+    result = async_task.apply_async(kwargs={"request": request})
+    task_id = result.id
+
+    return TaskResponse(task_id=task_id, request_id=task_id)
+
+
+# {
+#   "code": 0,
+#   "message": "SUCCEED",
+#   "request_id": "CjMkWmdJhuIAAAAAAAS0mA",
+#   "data": {
+#     "task_id": "CjMkWmdJhuIAAAAAAAS0mA",
+#     "task_status": "submitted",
+#     "created_at": 1732881630997,
+#     "updated_at": 1732881630997
+#   }
+# }
+
+
+async def get_task(
+        task_id: str,
+        remote_get_task: Optional[Callable] = None,
+        filter_kwargs: Optional[dict] = None,
+        background_tasks: Optional[BackgroundTasks] = None,
+):
+    """
+
+    :param task_id:
+    :param remote_get_task:
+    :param filter_kwargs: #######参数可以舍弃
+         filter_kwargs = {
+            "task_id": task_id, #########理论上只需这个
+            "user_id": user_id,
+            "platform": "replicate",
+            "action": "replicate",  # 模型
+        }
+    :return:
+    """
+    filter_kwargs = filter_kwargs or {}
+    filter_kwargs["task_id"] = task_id
+
+    result = AsyncResult(id=task_id)
+    logger.debug(bjson(result._get_task_meta()))
+    # logger.debug(bjson(result.get(timeout=30)))
+    logger.debug(result)
+    logger.debug(result.ready())
+    logger.debug(result.state)
+
+    # if result.status=="PENDING": # worker可能还未启动，会阻塞
+    # if await redis_aclient.select(1) and not await redis_aclient.exists(f"celery-task-meta-{task_id}"):
+    #     raise HTTPException(status_code=404, detail="TaskID not found")
+
+    if result.ready():
+        if result.successful():
+            data = result.get(timeout=30).copy()  # 创建任务时：remote task的返回结果 ####### copy避免丢失字段
+            logger.debug(bjson(data))
+            token = data.pop("system_fingerprint", None)  # 远程任务 token/apikey
+
+            remote_task_id = (
+                    data.get("task_id")  # 提前写
+                    or data.get("data", {}).get("task_id")
+                    or data.get("data", {}).get("id")
+                    or data.get("id")
+                    or data.get("request_id")
+            )
+
+            response = TaskResponse(
+                task_id=task_id,
+
+                message="Task completed successfully",
+                status=result.status,
+                data=data
+            )
+
+            if remote_get_task:
+                if inspect.iscoroutinefunction(remote_get_task):
+                    remote_task_response = await remote_get_task(remote_task_id, token)
+                else:
+                    remote_task_response = remote_get_task(remote_task_id, token)
+
+                if not isinstance(remote_task_response, dict):
+                    remote_task_response = remote_task_response.model_dump()
+
+                # logger.debug(response.model_dump_json(indent=4))
+                # logger.debug(bjson(remote_task_response))
+
+                _response = {**response.model_dump(), **remote_task_response}
+                response = response.construct(**_response)
+
+                # logger.debug(response.model_dump_json(indent=4))
+
+        else:
+            response = TaskResponse(
+                task_id=task_id,
+
+                code=1,
+                message=str(result.result),
+                status=result.status,
+            )
+    else:
+        response = TaskResponse(
+            task_id=task_id,
+
+            message="Task is still running",
+            status=result.status,
+        )
+
+    # 更新到数据库：异步任务
+    update_fn = partial(update_oneapi_from_response, task_response=response)
+    if background_tasks:
+        background_tasks.add_task(update_or_insert, OneapiTask, filter_kwargs, update_fn)
+    else:
+        await update_or_insert(OneapiTask, filter_kwargs, update_fn)
+
+    return response
+
+
+# todo: 可以设计更通用的
+async def update_oneapi_from_response(task: OneapiTask, task_response: TaskResponse):
+    """
+
+    filter_kwargs = {
+            "task_id": task_id,
+            "user_id": user_id,
+            "platform": "replicate",
+            "action": "replicate",  # 模型
+        }
+
+    需要获取这几个信息 user_id
+
+    """
+    # if task.status in {"SUCCESS", "FAILURE"}: return False  # 跳出轮询，不再更新
+
+    task.data = task_response.model_dump(exclude={"system_fingerprint"})
+    task.status = task_response.status
+    task.progress = time.time() // 10 % 100
+
+    if task.status == "SUCCESS":  ###### todo: 状态对齐
+        task.progress = "100%"
+    elif task.status == "FAILURE":
+        task.fail_reason = "查看详情"
+
+    task.updated_at = int(time.time())
+    task.finish_time = int(time.time())  # 不是实际时间
+
+
+if __name__ == '__main__':
+    from meutils.async_task import worker, shared_task  #######重要 需要识别到worker
+
+    from meutils.apis.kling import kolors_virtual_try_on
+    from meutils.async_task.tasks import hailuo
+
+    task_id = "7d4fbaf3-f641-482b-b02c-ccd30e61195a"
+    # filter_kwargs = {
+    #     "task_id": task_id,  #########理论上只需这个
+    #     "user_id": 1,
+    #     "platform": "kling",
+    #     "action": "kling",  # 模型
+    # }
+
+    # arun(get_task(task_id, hailuo.get_task))
+    #
+    # task_id = "e3f76be8-f7eb-4562-a7af-029928d074d5"
+    # arun(get_task(task_id, kolors_virtual_try_on.get_task))