MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/async_task//351/200/232/347/224/250/350/256/276/350/256/241.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : 通用设计
+# @Time         : 2024/11/25 18:13
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : https://github.com/GregaVrbancic/fastapi-celery/blob/master/app/main.py#L31
+
+from meutils.pipe import *
+from typing import Any, Dict, Optional
+from celery.result import AsyncResult
+
+from fastapi import FastAPI, APIRouter, Request, HTTPException
+from enum import Enum
+from pydantic import BaseModel
+
+
+# 1. 定义标准响应模型
+class TaskStatus(str, Enum):
+    PENDING = "PENDING"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+    RUNNING = "RUNNING"
+
+
+class TaskResponse(BaseModel):
+    code: int
+    message: str
+    task_id: str
+    status: TaskStatus
+    data: Optional[Any] = None
+
+
+# 2. 定义任务管理器
+class TaskManager:
+    @staticmethod
+    async def create_task(task_func, **kwargs) -> TaskResponse:
+        try:
+            task = task_func.delay(**kwargs)
+            return TaskResponse(
+                code=0,
+                message="Task created successfully",
+                task_id=task.id,
+                status=TaskStatus.PENDING
+            )
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+
+    @staticmethod
+    async def get_task_result(task_id: str) -> TaskResponse:
+        result = AsyncResult(id=task_id)
+
+        if result.ready():
+            if result.successful():
+                return TaskResponse(
+                    code=0,
+                    message="Task completed successfully",
+                    task_id=task_id,
+                    status=TaskStatus.SUCCESS,
+                    data=result.get()
+                )
+            else:
+                return TaskResponse(
+                    code=1,
+                    message=str(result.result),
+                    task_id=task_id,
+                    status=TaskStatus.FAILURE
+                )
+        else:
+            return TaskResponse(
+                code=0,
+                message="Task is still running",
+                task_id=task_id,
+                status=TaskStatus.RUNNING
+            )
+
+
+# 3. 路由实现
+router = APIRouter(prefix="/celery")
+
+
+@router.get("/{task_id}", response_model=TaskResponse)
+async def get_result(task_id: str):
+    return await TaskManager.get_task_result(task_id)
+
+
+@router.post("/task", response_model=TaskResponse)
+async def create_task(request: Request):
+    try:
+        kwargs = await request.json()
+        return await TaskManager.create_task(proxy_task, **kwargs)
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+# 4. 错误处理装饰器
+def handle_task_errors(func):
+    async def wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except Exception as e:
+            return TaskResponse(
+                code=1,
+                message=str(e),
+                task_id="",
+                status=TaskStatus.FAILURE
+            )
+
+    return wrapper
+
+
+if __name__ == '__main__':
+    from meutils.serving.fastapi import App
+
+    app = App()
+    app.include_router(router)
+    app.run(port=8899)

meutils/async_utils/asyncer_.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : x
+# @Time         : 2024/11/28 19:30
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : https://github.com/fastapi/asyncer
+
+from meutils.pipe import *
+import time
+
+import anyio
+from asyncer import asyncify, syncify
+
+@asyncify
+def do_sync_work(name: str):
+    time.sleep(1)
+    return f"Hello, {name}"
+
+
+@syncify
+async def do_async_work(name: str):
+    await asyncio.sleep(1)
+    return f"Hello, a{name}"
+
+async def main():
+    message = await asyncify(do_sync_work)(name="World")
+    print(message)
+
+
+if __name__ == '__main__':
+
+
+    # arun(do_sync_work('xx'))
+    arun(do_async_work('xxx'))

meutils/async_utils/background.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : background
+# @Time         : 2024/6/12 11:00
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.decorators import background_task
+
+from asgiref.sync import async_to_sync
+from starlette.background import BackgroundTask as _BackgroundTask, BackgroundTasks as _BackgroundTasks
+
+
+class BackgroundTask(_BackgroundTask):
+    """
+    async def f(x):
+        for i in range(100):
+            logger.debug(i)
+            await asyncio.sleep(1)
+    BackgroundTask(f, x='xxxxxxxxxx').call()
+
+    bt = BackgroundTasks()
+    bt.add_task(f, x='xxxxxxxxxx')
+    bt.call()
+    """
+
+    @background_task
+    def call(self):
+        async_to_sync(super().__call__)()
+
+
+class BackgroundTasks(_BackgroundTasks):
+    """
+    async def f(x):
+        for i in range(100):
+            logger.debug(i)
+            await asyncio.sleep(1)
+    BackgroundTask(f, x='xxxxxxxxxx').call()
+
+    bt = BackgroundTasks()
+    bt.add_task(f, x='xxxxxxxxxx')
+    bt.call()
+    """
+
+    @background_task
+    def call(self):
+        async_to_sync(super().__call__)()
+
+
+if __name__ == '__main__':
+    from meutils.pipe import *
+
+
+    async def f(x):
+        logger.debug(x)
+
+        for i in range(100):
+            logger.debug(i)
+            await asyncio.sleep(1)
+
+
+    bt = BackgroundTasks()
+    bt.add_task(f, x='xxxxxxxxxx')
+    bt.call()
+

meutils/async_utils/common.py

@@ -10,14 +10,94 @@
 
 import inspect
 import asyncio
-from typing import Coroutine
+from typing import Union, Any, Coroutine, AsyncIterator, Iterator, Iterable, AsyncIterable, Optional, Callable
 
+import numpy as np
+import pandas as pd
 from async_lru import alru_cache
-from asgiref.sync import sync_to_async
+from asgiref.sync import sync_to_async, async_to_sync
+from httpx import Client, AsyncClient
+from loguru import logger
+from pprint import pprint
+from aiostream import stream
+from functools import wraps
 
 
-def arun(coroutine: Coroutine, debug=None):
-    return asyncio.run(coroutine, debug=debug)
+def async_to_sync_pro(func):
+    @wraps(func)  # 添加这行来保留原函数的属性
+    def wrapper(*args, **kwargs):
+        return async_to_sync(func)(*args, **kwargs)
+
+    return wrapper
+
+
+async def achain(*iterables: Union[Iterator[Any], AsyncIterator[Any]], delay: int = 0) -> AsyncIterator[Any]:
+    """
+        async def async_generator_1() -> AsyncIterator[str]:
+            for i in range(1, 6):
+                await asyncio.sleep(1)
+                yield f'gen1-{i}'
+
+
+        async def async_generator_2() -> AsyncIterator[str]:
+            for i in range(1, 6):
+                await asyncio.sleep(1.5)
+                yield f'gen2-{i}'
+
+
+        async def main():
+            async for value in achain(async_generator_1(), async_generator_2()):
+                print(value)
+
+
+        asyncio.run(main())
+    :param iterables:
+    :return:
+    """
+
+    for iterable in iterables or []:
+
+        # logger.debug(f"{iterable}: {type(iterable)}")
+
+        if isinstance(iterable, AsyncIterator):
+            async for item in iterable:
+                yield item
+                delay and await asyncio.sleep(delay)
+
+        else:
+            # await asyncio.sleep(0)  # 神奇的代码：不起作用
+
+            for item in iterable or []:
+                # logger.debug(item)
+                yield item
+                await asyncio.sleep(delay)  # 神奇的代码：同步转异步
+
+
+def arun(awaitable_object: Union[Coroutine, AsyncIterator], debug: bool = True):
+    # asyncio.iscoroutine(coroutine)
+    # logger.debug(type(awaitable_object))
+    # logger.debug(isinstance(awaitable_object, Coroutine))
+    # logger.debug(isinstance(awaitable_object, AsyncIterator))
+    #
+    # logger.debug(inspect.isawaitable(awaitable_object))
+    # logger.debug(inspect.isasyncgen(awaitable_object))
+    # logger.debug(inspect.isasyncgenfunction(awaitable_object))
+
+    # asyncio.iscoroutine(awaitable_object)
+    if inspect.isasyncgen(awaitable_object):
+        async def main():
+            async for i in awaitable_object:
+                print(i, end='')
+            # return await stream.list(awaitable_object)
+
+        _awaitable_object = main()
+    else:
+        _awaitable_object = awaitable_object
+
+    _ = asyncio.run(_awaitable_object, debug=debug)
+    if debug:
+        isinstance(_, (np.ndarray, pd.Series, pd.DataFrame)) or _ and pprint(_)
+    return _
 
 
 def aclose():
@@ -25,9 +105,7 @@ def aclose():
     nest_asyncio.apply()
 
 
-def close_event_loop():
-    import nest_asyncio
-    nest_asyncio.apply()
+close_event_loop = aclose
 
 
 def async2sync_generator(generator):
@@ -57,18 +135,60 @@ def async2sync_generator(generator):
         yield from generator
 
 
-# from asgiref.sync import sync_to_async
-# anyio.to_thread.run_sync
+async def arequest(url, method='get', payload=None, **client_params):
+    if payload:
+        if method.lower() == 'get':
+            payload = {"params": payload}
+        else:
+            payload = {"json": payload}
+    async with AsyncClient(**client_params) as client:
+        resp = await client.request(method=method, url=url, **payload)
+        return resp
 
 
-if __name__ == '__main__':
-    from meutils.pipe import *
+async def poll(fn, n: int = 30, sleep_fn: Optional[Callable] = None):
+    if sleep_fn is None:
+        sleep_fn = lambda i: max(n / (i + 1), 1)  # 预估大概 2~3 *n
 
+    for i in range(n):
+        await asyncio.sleep(sleep_fn(i))
+        if _ := await fn():  # 跳出
+            return _
 
-    async def async_generator():
-        for i in range(10):
-            await asyncio.sleep(1)
-            yield i
+
+if __name__ == '__main__':
+    from meutils.pipe import *
 
 
-    async_generator() | xprint
+    # async def async_generator():
+    #     for i in range(10):
+    #         await asyncio.sleep(1)
+    #         yield i
+    #
+    #
+    # async_generator() | xprint
+
+    # async def async_generator_1() -> AsyncIterator[str]:
+    #     for i in range(1, 6):
+    #         await asyncio.sleep(1)
+    #         yield f'gen1-{i}'
+    #
+    #
+    # def async_generator_2():
+    #     for i in range(1, 6):
+    #         yield f'gen2-{i}'
+    #
+    #
+    # async def main():
+    #     async for value in achain(async_generator_1(), async_generator_2()):
+    #         print(value)
+    #
+    #
+    # asyncio.run(main())
+
+    async def main():
+        async for value in achain():
+            print(value)
+
+
+    asyncio.run(main())

meutils/async_utils/test.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : test
+# @Time         : 2024/6/12 10:47
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : todo； 并没有啥用
+
+from meutils.pipe import *
+from starlette.background import BackgroundTask as _BackgroundTask, BackgroundTasks as _BackgroundTasks
+from asgiref.sync import sync_to_async, async_to_sync
+
+
+class BackgroundTask(_BackgroundTask):
+    """
+    async def f(x):
+        for i in range(100):
+            logger.debug(i)
+            await asyncio.sleep(1)
+    BackgroundTask(f, x='xxxxxxxxxx').call()
+
+    bt = BackgroundTasks()
+    bt.add_task(f, x='xxxxxxxxxx')
+    bt.call()
+    """
+
+    def call(self):
+        async_to_sync(super().__call__)()
+
+
+class BackgroundTasks(_BackgroundTasks):
+    """
+    async def f(x):
+        for i in range(100):
+            logger.debug(i)
+            await asyncio.sleep(1)
+    BackgroundTask(f, x='xxxxxxxxxx').call()
+
+    bt = BackgroundTasks()
+    bt.add_task(f, x='xxxxxxxxxx')
+    bt.call()
+    """
+
+    def call(self):
+        async_to_sync(super().__call__)()

meutils/cache_utils.py

@@ -7,6 +7,8 @@
 # @WeChat       : 313303303
 # @Software     : PyCharm
 # @Description  : https://cachetools.readthedocs.io/en/stable/
+
+# https://github.com/awolverp/cachebox 缓存 todo
 """
 异步缓存 from aiocache import cached
 
@@ -23,6 +25,7 @@ RR：Random Replacement，就是随机替换。
 TTL：time-to-live 的简称，也就是说，Cache 中的每个元素都是有过期时间的，如果超过了这个时间，那这个元素就会被自动销毁。
 如果都没过期并且 Cache 已经满了的话，那就会采用 LRU 置换算法来替换掉最久不用的，以此来保证数量。
 """
+import time
 import sys
 import math
 import pickle
@@ -32,8 +35,8 @@ import pandas as pd
 import inspect
 import joblib
 
-from typing import Iterable
-from functools import lru_cache
+from typing import Iterable, Optional
+from functools import lru_cache as _lru_cache
 from pathlib import Path
 from joblib import Memory
 from joblib.func_inspect import filter_args
@@ -49,7 +52,8 @@ from cachetools import cached, cachedmethod, LRUCache, RRCache, TTLCache as _TTL
 from meutils.decorators import decorator, singleton, background_task
 from meutils.hash_utils import md5
 
-TTLCache = lru_cache()(_TTLCache)
+TTLCache = _lru_cache()(_TTLCache)
+
 
 # todo: cache_clear 很重要
 def map_cache():
@@ -83,7 +87,7 @@ def ttl_cache(ttl=np.inf, maxsize=1024, key=key_fn):
 def redis_cache(func, rc=None, ex=3, ignore=None, verbose=1, *args, **kwargs):
     """redis 缓存"""
     import redis
-    rc = rc or redis.Redis(host='localhost', port=6379, db=0)
+    rc = rc or redis.Redis()
 
     ##############################################################################
     fn_name = func.__name__ if hasattr(func, '__name__') else 'fn'
@@ -130,7 +134,7 @@ def joblib_cache(func, location='cachedir', ignore=None, verbose=1, *args, **kwa
 
 
 @decorator
-def diskcache(func, location='cachedir', ttl=None, ignore=None, verbose=1, tag=None, *args, **kwargs):
+def diskcache(func, location='cachedir', ttl=None, ignore: Optional[list] = None, verbose=1, tag=None, *args, **kwargs):
     """
     https://zhuanlan.zhihu.com/p/356447502
     https://grantjenks.com/docs/diskcache/
@@ -145,11 +149,12 @@ def diskcache(func, location='cachedir', ttl=None, ignore=None, verbose=1, tag=N
     raw_key = {**raw_key.pop('**', {}), **raw_key, '__tag__': tag or func.__name__}
 
     # self cls 看场景选择
-    ignore = (ignore or []) + [
-        'api_base', 'api_key', 'openai_api_base', 'openai_api_key',
-        'organization', 'openai_organization',
-        'request_timeout'
-    ]
+    ignore = ignore or []
+    # ignore += [
+    #     'api_base', 'api_key', 'openai_api_base', 'openai_api_key',
+    #     'organization', 'openai_organization',
+    #     'request_timeout'
+    # ]
     for arg in ignore:
         raw_key.pop(arg, None)
 
@@ -261,16 +266,17 @@ def cache(func, cache_name='cachedir', ttl=None, ignore=None, verbose=1, tag=Non
 
 
 if __name__ == '__main__':
-
-    import time
-
-
-    @redis_cache(ex=15 * 60)
-    def f(x):
-        time.sleep(1)
-        return x
-
-
-    for i in range(110):
-        print(f(i))
-        print(f(i))
+    # import time
+    #
+    #
+    # @redis_cache(ex=15 * 60)
+    # def f(x):
+    #     time.sleep(1)
+    #     return x
+    #
+    #
+    # for i in range(110):
+    #     print(f(i))
+    #     print(f(i))
+
+    pass

meutils/caches/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2024/6/14 10:16
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :

meutils/caches/acache.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : acache
+# @Time         : 2025/1/14 09:45
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import os
+
+from meutils.pipe import *
+
+from aiocache import cached, Cache, RedisCache
+from aiocache import multi_cached
+
+# @multi_cached(ttl=60, caches=[Cache.MEMORY, Cache.MEMORY])
+# async def complex_function(user_id, **kwargs):
+#     logger.debug(user_id)
+#     return False
+
+
+# Cache.MEMORY
+
+# Cache.REDIS
+# mcache = cached(ttl=60, cache=Cache.REDIS)(cached)
+
+
+rcache = Cache.from_url("redis://:chatfirechatfire@110.42.51.201:6379/11")
+print(rcache)
+
+
+# @cached(ttl=60)
+@cached(ttl=15, cache=rcache)
+async def complex_function(user_id, **kwargs):
+    logger.debug(user_id)
+    return False
+
+
+class A(BaseModel):
+    a: Any = 1
+
+
+if __name__ == '__main__':
+    arun(complex_function(A(a={})))

meutils/caches/redis_cache.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : redis_cache
+# @Time         : 2024/12/6 10:46
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :  https://hermescache.readthedocs.io/en/latest/
+# https://mp.weixin.qq.com/s/-T2UmkinmtQoNQo4DVpnfw
+
+
+import hermes.backend.redis
+import hermes.backend.inprocess
+
+from meutils.db.redis_db import pool
+
+cache = hermes.Hermes(
+    backend=hermes.backend.redis.Backend,
+    connection_pool=pool
+)
+
+cache_inmemory = hermes.Hermes(
+    backend=hermes.backend.inprocess.Backend,
+)
+
+acache_inmemory = hermes.Hermes(
+    backend=hermes.backend.inprocess.AsyncBackend,
+)
+
+
+from aiocache import cached, Cache, RedisCache
+
+
+if __name__ == '__main__':
+    from meutils.pipe import *
+
+    @cached(ttl=100)
+    @cache(tags=('test',))
+    async def foo(a, b):
+        time.sleep(3)
+        logger.debug('没有缓存')
+
+        return a * b
+
+
+    # @cache(tags=('test',), key=lambda fn, a, b: f'avg:{a}')  # 这样可以忽略b
+    # async def func_ignore(a, b):
+    #     await asyncio.sleep(5)
+    #     logger.debug('异步函数')
+    #     return a, b
+
+
+    # arun(func_ignore(1, 1))
+    # ttl: Optional[int] = None,
+    # tags: Sequence[str] = (),
+    # key: Optional[Callable] = None,
+    # @cache(ttl=100, key=lambda fn, a, b: (a, b))
+    # def func(x, y):
+    #     logger.debug('xxxxxxx')
+    #     return x
+
+    arun(foo(1, 2))