MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/serving/fastapi/dependencies/auth.py

@@ -8,8 +8,61 @@
 # @Software     : PyCharm
 # @Description  : 
 
-from meutils.pipe import *
+import numpy as np
+from typing import Optional, Union
 
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from fastapi import Depends, HTTPException, status
 
-get_bearer_token = HTTPBearer()
+http_bearer = HTTPBearer()
+
+
+# 定义获取token的函数
+
+async def get_bearer_token(
+        auth: Optional[HTTPAuthorizationCredentials] = Depends(http_bearer)
+) -> Optional[str]:
+    """
+    获取Bearer token
+    :param auth: HTTP认证凭证
+    :return: token字符串
+    """
+    if auth is None:
+        return None
+
+    token = auth.credentials
+    if token.startswith('redis:'):  # 初始化吧，太长？
+        pass
+        if ',' in token:  # 初始化redis
+            pass
+    elif ',' in token:  # 分隔符
+        token = np.random.choice(token.split(','))
+
+    return token
+
+
+# async def get_next_token(redis_key):
+#     """轮询"""
+#     if api_key := await redis_aclient.lpop(redis_key):
+#         await redis_aclient.rpush(redis_key, api_key)
+#         return api_key
+
+
+async def get_bearer_token_for_oneapi(
+        auth: Optional[HTTPAuthorizationCredentials] = Depends(http_bearer)
+) -> Optional[str]:
+    """
+    # todo: oneapi userinfo apikey info
+    """
+    if auth is None:
+        return None
+
+    token = auth.credentials
+
+    return token
+
+
+if __name__ == '__main__':
+    from meutils.pipe import *
+
+    arun(get_bearer_token(HTTPAuthorizationCredentials(scheme="Bearer", credentials="123")))

meutils/serving/fastapi/exceptions/http_error.py

@@ -1,11 +1,35 @@
-from fastapi import HTTPException, status
+import json
+import traceback
+from functools import partial
+from httpx import HTTPStatusError
+from openai import APIStatusError
+
+from fastapi import status
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
+from fastapi.exceptions import RequestValidationError, HTTPException
+
+from meutils.notice.feishu import send_message as _send_message
+
+send_message = partial(
+    _send_message,
+    title=__name__,
+    url="https://open.feishu.cn/open-apis/bot/v2/hook/d1c7b67d-b0f8-4067-a2f5-109f20eeb696"
+)
+
 
 async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse:
+    # print(exc)
+    content = {
+        "error":
+            {
+                "message": f"{exc.detail}",
+                "type": "http-error",
+            }
+    }
     return JSONResponse(
-        {"errors": [exc.detail]},
+        content=content,
         status_code=exc.status_code
     )
 
@@ -14,5 +38,46 @@ async def general_exception_handler(request: Request, exc: Exception):
     return JSONResponse(
         content={"message": str(exc)},
         status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+    )
+
+
+async def chatfire_api_exception_handler(request: Request, exc: Exception):
+    content = {
+        "error":
+            {
+                "message": f"{exc}",
+                "type": "cf-api-error",
+            },
+
+        "code": status.HTTP_500_INTERNAL_SERVER_ERROR
+    }
 
+    # 默认值
+    reps = None
+    request_json = {"body": await request.body()}
+    if isinstance(exc, (HTTPStatusError, APIStatusError)):
+        status_code = exc.response.status_code or 500
+
+        content['code'] = status_code
+        content['error']['message'] = f"{exc.response.text}"
+
+        reps = JSONResponse(
+            content=content,
+            status_code=status_code,
+        )
+
+    # send_message
+    content_detail = f"{traceback.format_exc()}"
+    if any(code in content_detail for code in {'451', }):
+        content_detail = ""
+
+    send_message([request_json, content, content_detail])
+
+    return reps or JSONResponse(
+        content=content,
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
     )
+
+
+if __name__ == '__main__':
+    pass

meutils/serving/fastapi/exceptions/validation_error.py

@@ -1,5 +1,6 @@
 from typing import Union
 
+from fastapi import status
 from fastapi.exceptions import RequestValidationError
 from fastapi.openapi.constants import REF_PREFIX
 from fastapi.openapi.utils import validation_error_response_definition
@@ -10,8 +11,8 @@ from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 
 
 async def http422_error_handler(
-    _: Request,
-    exc: Union[RequestValidationError, ValidationError],
+        _: Request,
+        exc: Union[RequestValidationError, ValidationError],
 ) -> JSONResponse:
     return JSONResponse(
         {"errors": exc.errors()},
@@ -26,3 +27,18 @@ validation_error_response_definition["properties"] = {
         "items": {"$ref": "{0}ValidationError".format(REF_PREFIX)},
     },
 }
+
+
+async def validation_exception_handler(_: Request, exc: RequestValidationError) -> JSONResponse:
+    # print(exc)
+    content = {
+        "error":
+            {
+                "message": f"{exc}",
+                "type": "RequestValidationError",
+            }
+    }
+    return JSONResponse(
+        content=content,
+        status_code=status.HTTP_400_BAD_REQUEST
+    )

meutils/serving/fastapi/lifespans.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : lifespans
+# @Time         : 2024/12/13 10:50
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+from contextlib import asynccontextmanager
+
+from meutils.pipe import *
+from meutils.config_utils.manager import ConfigManager
+
+
+class Resources(BaseModel):
+    config_manager: Optional[ConfigManager] = None
+
+    class Config:
+        # 允许额外字段，增加灵活性
+        extra = 'allow'
+        arbitrary_types_allowed = True
+
+
+# def fake_answer_to_everything_ml_model(x: float):
+#     return x * 42
+#
+#
+# ml_models = {}
+#
+#
+# @asynccontextmanager
+# async def lifespan(app):
+#     # Load the ML model
+#     ml_models["answer_to_everything"] = fake_answer_to_everything_ml_model
+#     yield
+#     # Clean up the ML models and release the resources
+#     ml_models.clear()
+
+
+resources = Resources()
+
+
+@asynccontextmanager
+async def nacos_lifespan(app):
+    import gc
+
+    logger.debug("Init lifespans")
+
+    config_manager = ConfigManager("test", "testdata")
+
+    resources.config_manager = config_manager
+
+    yield
+
+    # release the resources
+    logger.debug("Release the resources")
+    del config_manager
+    gc.collect()
+
+
+if __name__ == '__main__':
+    from fastapi import FastAPI
+
+    app = FastAPI(lifespan=lifespan)
+
+    app.state.config_manager = ConfigManager("test", "testdata")
+
+
+    @app.get("/predict")
+    async def predict(x: float):
+        result = ml_models["answer_to_everything"](x)
+        return {"result": result}

meutils/serving/fastapi/routers/scheduler.py

@@ -49,3 +49,15 @@ async def start_scheduler():
 #     scheduler = Scheduler(timezone="Asia/Shanghai")
 #     scheduler.add_job(tick, 'interval', seconds=3)
 #     scheduler.start()
+
+
+# from meutils.pipe import *
+# from meutils.serving.fastapi import App
+# from meutils.serving.fastapi.routers import scheduler
+#
+# if __name__ == '__main__':
+#     app = App()
+#     scheduler.scheduler.add_job(lambda: print("hello"), 'interval', seconds=3)
+#
+#     app.include_router(scheduler.router)
+#     app.run(port=8899)

meutils/serving/fastapi/routers/screenshot.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : screenshot
+# @Time         : 2024/9/18 16:02
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from meutils.io.files_utils import to_url
+from meutils.playwright_utils import get_new_page
+from meutils.schemas.playwright_types import ScreenshotRequest
+
+from fastapi import APIRouter, File, UploadFile, Query, Form, Response, Request
+
+router = APIRouter()
+
+HTML_PARSER = re.compile(r'```html(.*?)```', re.DOTALL)
+
+
+@router.post("/screenshot")  # todo: 多个url
+async def capture_screenshot(
+        request: ScreenshotRequest,
+):
+    htmls = HTML_PARSER.findall(request.html)
+    if htmls:
+        request.html = htmls[0]
+
+    page = await get_new_page()
+    await page.set_content(request.html)
+    await page.set_viewport_size({"width": request.width, "height": request.height})
+
+    with tempfile.NamedTemporaryFile(suffix='.png') as file:
+        await page.screenshot(path=file.name)
+        url = await to_url(file.name)
+        page.close()
+        return {"url": url}
+
+
+if __name__ == '__main__':
+    from meutils.serving.fastapi import App
+
+    app = App()
+    app.include_router(router)
+    app.run(port=39666)

meutils/serving/fastapi/routers/spider.py

@@ -17,7 +17,7 @@ router = APIRouter()
 
 
 @router.get("/playwright")  # todo: 多个url
-@alru_cache(maxsize=10240)
+@alru_cache(maxsize=10240, ttl=3 * 24 * 3600)
 async def get_content(
         url: str,
         # page=Depends(get_new_page) # Depends get_new_page 里面的参数也会暴露出来
@@ -31,8 +31,13 @@ async def get_content(
         content = await page.inner_text('//*[@id="page-content"]')  # 使用page.query_selector_all获取a节点列表
     else:
         # todo: xpath
+        # wait
+        await page.wait_for_load_state(state='load')
+        await page.wait_for_load_state(state='networkidle')
+        await page.wait_for_load_state(state='domcontentloaded')
+        await page.wait_for_selector('body')
+
         content = await page.inner_text('body')
-        # page.wait_for_load_state("networkidle")
 
     return {"content": content}
 
@@ -42,4 +47,4 @@ if __name__ == '__main__':
 
     app = App()
     app.include_router(router)
-    app.run()
+    app.run(port=39666)

meutils/serving/fastapi/routers/task.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : task
+# @Time         : 2024/5/9 15:41
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from meutils.db.redis_db import redis_aclient
+from fastapi import status
+from fastapi import APIRouter, File, UploadFile, Query, Form, Response, Request, FastAPI
+from fastapi.responses import JSONResponse
+
+router = APIRouter()
+
+
+@router.get("/redis/{task_id}")
+@alru_cache(ttl=60)  # 缓存
+async def get_task_from_redis(task_id: str):
+    task_info = await redis_aclient.get(task_id)
+    return task_info
+
+
+class RedisRequest(BaseModel):
+    task_id: str
+    task_info: str
+    ttl: Optional[int] = None
+
+
+@router.post("/redis")
+async def set_task_for_redis(requst: RedisRequest):
+    await redis_aclient.set(requst.task_id, requst.task_info, ex=requst.ttl)
+    # return JSONResponse(content="", status_code=status.HTTP_200_OK)
+
+
+
+redis_aclient.response_callbacks
+
+
+if __name__ == '__main__':
+    from meutils.serving.fastapi import App
+
+    app = App()
+    app.include_router(router)
+    app.run()

meutils/serving/fastapi/utils.py

@@ -8,9 +8,33 @@
 # @Software     : PyCharm
 # @Description  : 
 
-from starlette.requests import Request
 from meutils.pipe import *
 
+from starlette.requests import Request
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+from fastapi import APIRouter, File, UploadFile, Query, Form, BackgroundTasks, Depends, HTTPException as _HTTPException, \
+    Request, status
+
+
+class HTTPException(_HTTPException):
+    message: str = ""
+    type: str = "error"
+    param: Optional[Any] = None
+    code: Optional[Any] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.detail = self.detail or {
+            "error": {
+                "message": self.message,
+                "type": self.type,
+                "param": self.param,
+                "code": self.code or self.status_code,
+            }
+        }
+
 
 def get_ipaddr(request: Request) -> str:
     """
@@ -51,3 +75,26 @@ def limit(limit_value='3/second', error_message: Optional[str] = None, **kwargs)
 
     limiter = Limiter(key_func=get_remote_address)
     return limiter.limit(limit_value=limit_value, error_message=error_message, **kwargs)
+
+
+def check_api_key(auth: HTTPAuthorizationCredentials):
+    api_key = auth
+    if api_key is None:
+        detail = {
+            "error": {
+                "message": "invalid_api_key",
+                "type": "invalid_request_error",
+                "param": None,
+                "code": None,
+            }
+        }
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=detail)
+
+
+if __name__ == '__main__':
+    d = {
+        "error": {
+            "message": "当前分组 chatfire 下对于模型 ERNIE-Bot 无可用渠道 (request id: 20240314180638675032749TVFiqyF1)",
+            "type": "new_api_error"
+        }
+    }

meutils/serving/streamlit/common.py

@@ -232,7 +232,7 @@ def st_chat_message(
 
     with st.chat_message(name=message.name, avatar=message.avatar):
         written_content = write(message.generator, unsafe_allow_html=unsafe_allow_html, **kwargs)
-        message.generator = written_content | xfilter | xjoin('')
+        message.generator = written_content | xfilter(lambda x: x is None) | xjoin('')
 
     if bhook: bhook()
 

meutils/smooth_utils.py

@@ -59,3 +59,6 @@ def walson_ctr(num_click, num_pv, z=1.96):
     return ctr
 
 
+if __name__ == '__main__':
+    for i in range(10): # SLEEP
+        print(exponential_decay(i, 10, 10, 3))

meutils/str_utils/__init__.py

@@ -33,6 +33,9 @@ def remove_punctuation(sentence: str, punctuation='!"#$%&\'()*+,-./:;<=>?@[\\]^_
 
 def chinese_convert(s, config='t2s'):
     """https://github.com/BYVoid/OpenCC
+    pip install opencc
+    pip install opencc-python-reimplemented
+
     s2t.json Simplified Chinese to Traditional Chinese 簡體到繁體
     t2s.json Traditional Chinese to Simplified Chinese 繁體到簡體
     s2tw.json Simplified Chinese to Traditional Chinese (Taiwan Standard) 簡體到臺灣正體
@@ -49,7 +52,7 @@ def chinese_convert(s, config='t2s'):
     tw2t.json Traditional Chinese (Taiwan standard) to Traditional Chinese 臺灣正體到繁體（OpenCC 標準）
     @return:
     """
-    import opencc # pip install opencc opencc-python-reimplemented
+    opencc = try_import("opencc", pip_name="opencc-python-reimplemented")
     converter = opencc.OpenCC(config)
     return converter.convert(s)
 
@@ -89,7 +92,7 @@ def half2all(ustring):
 
 def all2half(all_string):
     """全角转半角"""
-    from idna import unichr
+    from idna import chr
 
     half_string = ""
     for char in all_string:
@@ -179,6 +182,11 @@ class Encrypt(object):
         return b.decode("utf-8")
 
 
+def unicode_normalize(s):
+    import unicodedata
+    return unicodedata.normalize('NFKC', s)
+
+
 if __name__ == '__main__':
     # print(str_replace('abcd', {'a': '8', 'd': '88'}))
     # print(unquote())
@@ -188,4 +196,15 @@ if __name__ == '__main__':
 
     # print(json_loads("{1: 1}"))
 
-    print(Encrypt().encrypt('123'*100))
+    # print(Encrypt().encrypt('123' * 100))
+
+    # print(query_params2json("act=get&uid=134597&vkey=07A1D82FDDE1E96DB5CEF4EF12C8125F&num=1&time=30&plat=1&re=1&type=0&so=1&ow=1&spl=1&addr=&db=1"))
+
+    print(half2all('⽉月'))
+    print(all2half('⽉月'))
+
+# import chardet
+#
+# def detect_encoding(byte_content):
+#     result = chardet.detect(byte_content)
+#     return result['encoding']

meutils/str_utils/json_utils.py

@@ -7,6 +7,9 @@
 # @WeChat       : 313303303
 # @Software     : PyCharm
 # @Description  : pd.io.json.json_normalize
+import jsonpath
+
+# json https://blog.csdn.net/freeking101/article/details/103048514
 # https://github.com/ijl/orjson#quickstart
 # https://jmespath.org/tutorial.html
 # https://goessner.net/articles/JsonPath/
@@ -41,3 +44,7 @@ def json_loads(s):
         logger.warning(e)
 
         return eval(s)
+
+
+if __name__ == '__main__':
+    print(jsonpath.jsonpath({"a": 1}, expr='$.a'))