MeUtils 2024.3.18.15.56.48__py3-none-any.whl → 2025.1.17.17.55.21__py3-none-any.whl

meutils/serving/fastapi/dependencies/auth.py

@@ -8,8 +8,61 @@
 # @Software     : PyCharm
 # @Description  : 
 
-from meutils.pipe import *
+import numpy as np
+from typing import Optional, Union
 
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from fastapi import Depends, HTTPException, status
 
-get_bearer_token = HTTPBearer()
+http_bearer = HTTPBearer()
+
+
+# 定义获取token的函数
+
+async def get_bearer_token(
+        auth: Optional[HTTPAuthorizationCredentials] = Depends(http_bearer)
+) -> Optional[str]:
+    """
+    获取Bearer token
+    :param auth: HTTP认证凭证
+    :return: token字符串
+    """
+    if auth is None:
+        return None
+
+    token = auth.credentials
+    if token.startswith('redis:'):  # 初始化吧，太长？
+        pass
+        if ',' in token:  # 初始化redis
+            pass
+    elif ',' in token:  # 分隔符
+        token = np.random.choice(token.split(','))
+
+    return token
+
+
+# async def get_next_token(redis_key):
+#     """轮询"""
+#     if api_key := await redis_aclient.lpop(redis_key):
+#         await redis_aclient.rpush(redis_key, api_key)
+#         return api_key
+
+
+async def get_bearer_token_for_oneapi(
+        auth: Optional[HTTPAuthorizationCredentials] = Depends(http_bearer)
+) -> Optional[str]:
+    """
+    # todo: oneapi userinfo apikey info
+    """
+    if auth is None:
+        return None
+
+    token = auth.credentials
+
+    return token
+
+
+if __name__ == '__main__':
+    from meutils.pipe import *
+
+    arun(get_bearer_token(HTTPAuthorizationCredentials(scheme="Bearer", credentials="123")))

meutils/serving/fastapi/exceptions/http_error.py

@@ -1,11 +1,35 @@
-from fastapi import HTTPException, status
+import json
+import traceback
+from functools import partial
+from httpx import HTTPStatusError
+from openai import APIStatusError
+
+from fastapi import status
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
+from fastapi.exceptions import RequestValidationError, HTTPException
+
+from meutils.notice.feishu import send_message as _send_message
+
+send_message = partial(
+    _send_message,
+    title=__name__,
+    url="https://open.feishu.cn/open-apis/bot/v2/hook/d1c7b67d-b0f8-4067-a2f5-109f20eeb696"
+)
+
 
 async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse:
+    # print(exc)
+    content = {
+        "error":
+            {
+                "message": f"{exc.detail}",
+                "type": "http-error",
+            }
+    }
     return JSONResponse(
-        {"errors": [exc.detail]},
+        content=content,
         status_code=exc.status_code
     )
 
@@ -15,3 +39,45 @@ async def general_exception_handler(request: Request, exc: Exception):
         content={"message": str(exc)},
         status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
     )
+
+
+async def chatfire_api_exception_handler(request: Request, exc: Exception):
+    content = {
+        "error":
+            {
+                "message": f"{exc}",
+                "type": "cf-api-error",
+            },
+
+        "code": status.HTTP_500_INTERNAL_SERVER_ERROR
+    }
+
+    # 默认值
+    reps = None
+    request_json = {"body": await request.body()}
+    if isinstance(exc, (HTTPStatusError, APIStatusError)):
+        status_code = exc.response.status_code or 500
+
+        content['code'] = status_code
+        content['error']['message'] = f"{exc.response.text}"
+
+        reps = JSONResponse(
+            content=content,
+            status_code=status_code,
+        )
+
+    # send_message
+    content_detail = f"{traceback.format_exc()}"
+    if any(code in content_detail for code in {'451', }):
+        content_detail = ""
+
+    send_message([request_json, content, content_detail])
+
+    return reps or JSONResponse(
+        content=content,
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+    )
+
+
+if __name__ == '__main__':
+    pass

meutils/serving/fastapi/exceptions/validation_error.py

@@ -1,5 +1,6 @@
 from typing import Union
 
+from fastapi import status
 from fastapi.exceptions import RequestValidationError
 from fastapi.openapi.constants import REF_PREFIX
 from fastapi.openapi.utils import validation_error_response_definition
@@ -10,8 +11,8 @@ from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 
 
 async def http422_error_handler(
-    _: Request,
-    exc: Union[RequestValidationError, ValidationError],
+        _: Request,
+        exc: Union[RequestValidationError, ValidationError],
 ) -> JSONResponse:
     return JSONResponse(
         {"errors": exc.errors()},
@@ -26,3 +27,18 @@ validation_error_response_definition["properties"] = {
         "items": {"$ref": "{0}ValidationError".format(REF_PREFIX)},
     },
 }
+
+
+async def validation_exception_handler(_: Request, exc: RequestValidationError) -> JSONResponse:
+    # print(exc)
+    content = {
+        "error":
+            {
+                "message": f"{exc}",
+                "type": "RequestValidationError",
+            }
+    }
+    return JSONResponse(
+        content=content,
+        status_code=status.HTTP_400_BAD_REQUEST
+    )

meutils/serving/fastapi/lifespans.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : lifespans
+# @Time         : 2024/12/13 10:50
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+from contextlib import asynccontextmanager
+
+from meutils.pipe import *
+from meutils.config_utils.manager import ConfigManager
+
+
+class Resources(BaseModel):
+    config_manager: Optional[ConfigManager] = None
+
+    class Config:
+        # 允许额外字段，增加灵活性
+        extra = 'allow'
+        arbitrary_types_allowed = True
+
+
+# def fake_answer_to_everything_ml_model(x: float):
+#     return x * 42
+#
+#
+# ml_models = {}
+#
+#
+# @asynccontextmanager
+# async def lifespan(app):
+#     # Load the ML model
+#     ml_models["answer_to_everything"] = fake_answer_to_everything_ml_model
+#     yield
+#     # Clean up the ML models and release the resources
+#     ml_models.clear()
+
+
+resources = Resources()
+
+
+@asynccontextmanager
+async def nacos_lifespan(app):
+    import gc
+
+    logger.debug("Init lifespans")
+
+    config_manager = ConfigManager("test", "testdata")
+
+    resources.config_manager = config_manager
+
+    yield
+
+    # release the resources
+    logger.debug("Release the resources")
+    del config_manager
+    gc.collect()
+
+
+if __name__ == '__main__':
+    from fastapi import FastAPI
+
+    app = FastAPI(lifespan=lifespan)
+
+    app.state.config_manager = ConfigManager("test", "testdata")
+
+
+    @app.get("/predict")
+    async def predict(x: float):
+        result = ml_models["answer_to_everything"](x)
+        return {"result": result}

meutils/serving/fastapi/routers/scheduler.py

@@ -49,3 +49,15 @@ async def start_scheduler():
 #     scheduler = Scheduler(timezone="Asia/Shanghai")
 #     scheduler.add_job(tick, 'interval', seconds=3)
 #     scheduler.start()
+
+
+# from meutils.pipe import *
+# from meutils.serving.fastapi import App
+# from meutils.serving.fastapi.routers import scheduler
+#
+# if __name__ == '__main__':
+#     app = App()
+#     scheduler.scheduler.add_job(lambda: print("hello"), 'interval', seconds=3)
+#
+#     app.include_router(scheduler.router)
+#     app.run(port=8899)

meutils/serving/fastapi/routers/screenshot.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : screenshot
+# @Time         : 2024/9/18 16:02
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from meutils.io.files_utils import to_url
+from meutils.playwright_utils import get_new_page
+from meutils.schemas.playwright_types import ScreenshotRequest
+
+from fastapi import APIRouter, File, UploadFile, Query, Form, Response, Request
+
+router = APIRouter()
+
+HTML_PARSER = re.compile(r'```html(.*?)```', re.DOTALL)
+
+
+@router.post("/screenshot")  # todo: 多个url
+async def capture_screenshot(
+        request: ScreenshotRequest,
+):
+    htmls = HTML_PARSER.findall(request.html)
+    if htmls:
+        request.html = htmls[0]
+
+    page = await get_new_page()
+    await page.set_content(request.html)
+    await page.set_viewport_size({"width": request.width, "height": request.height})
+
+    with tempfile.NamedTemporaryFile(suffix='.png') as file:
+        await page.screenshot(path=file.name)
+        url = await to_url(file.name)
+        page.close()
+        return {"url": url}
+
+
+if __name__ == '__main__':
+    from meutils.serving.fastapi import App
+
+    app = App()
+    app.include_router(router)
+    app.run(port=39666)

meutils/serving/fastapi/routers/spider.py

@@ -17,7 +17,7 @@ router = APIRouter()
 
 
 @router.get("/playwright")  # todo: 多个url
-@alru_cache(maxsize=10240)
+@alru_cache(maxsize=10240, ttl=3 * 24 * 3600)
 async def get_content(
         url: str,
         # page=Depends(get_new_page) # Depends get_new_page 里面的参数也会暴露出来
@@ -31,8 +31,13 @@ async def get_content(
         content = await page.inner_text('//*[@id="page-content"]')  # 使用page.query_selector_all获取a节点列表
     else:
         # todo: xpath
+        # wait
+        await page.wait_for_load_state(state='load')
+        await page.wait_for_load_state(state='networkidle')
+        await page.wait_for_load_state(state='domcontentloaded')
+        await page.wait_for_selector('body')
+
         content = await page.inner_text('body')
-        # page.wait_for_load_state("networkidle")
 
     return {"content": content}
 
@@ -42,4 +47,4 @@ if __name__ == '__main__':
 
     app = App()
     app.include_router(router)
-    app.run()
+    app.run(port=39666)

meutils/serving/fastapi/routers/task.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : task
+# @Time         : 2024/5/9 15:41
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+
+from meutils.pipe import *
+from meutils.db.redis_db import redis_aclient
+from fastapi import status
+from fastapi import APIRouter, File, UploadFile, Query, Form, Response, Request, FastAPI
+from fastapi.responses import JSONResponse
+
+router = APIRouter()
+
+
+@router.get("/redis/{task_id}")
+@alru_cache(ttl=60)  # 缓存
+async def get_task_from_redis(task_id: str):
+    task_info = await redis_aclient.get(task_id)
+    return task_info
+
+
+class RedisRequest(BaseModel):
+    task_id: str
+    task_info: str
+    ttl: Optional[int] = None
+
+
+@router.post("/redis")
+async def set_task_for_redis(requst: RedisRequest):
+    await redis_aclient.set(requst.task_id, requst.task_info, ex=requst.ttl)
+    # return JSONResponse(content="", status_code=status.HTTP_200_OK)
+
+
+
+redis_aclient.response_callbacks
+
+
+if __name__ == '__main__':
+    from meutils.serving.fastapi import App
+
+    app = App()
+    app.include_router(router)
+    app.run()

meutils/serving/fastapi/utils.py

@@ -78,7 +78,7 @@ def limit(limit_value='3/second', error_message: Optional[str] = None, **kwargs)
 
 
 def check_api_key(auth: HTTPAuthorizationCredentials):
-    api_key = auth and auth.credentials or None
+    api_key = auth
     if api_key is None:
         detail = {
             "error": {

meutils/serving/streamlit/common.py

@@ -232,7 +232,7 @@ def st_chat_message(
 
     with st.chat_message(name=message.name, avatar=message.avatar):
         written_content = write(message.generator, unsafe_allow_html=unsafe_allow_html, **kwargs)
-        message.generator = written_content | xfilter | xjoin('')
+        message.generator = written_content | xfilter(lambda x: x is None) | xjoin('')
 
     if bhook: bhook()
 

meutils/smooth_utils.py

@@ -59,3 +59,6 @@ def walson_ctr(num_click, num_pv, z=1.96):
     return ctr
 
 
+if __name__ == '__main__':
+    for i in range(10): # SLEEP
+        print(exponential_decay(i, 10, 10, 3))

meutils/str_utils/__init__.py

@@ -33,6 +33,9 @@ def remove_punctuation(sentence: str, punctuation='!"#$%&\'()*+,-./:;<=>?@[\\]^_
 
 def chinese_convert(s, config='t2s'):
     """https://github.com/BYVoid/OpenCC
+    pip install opencc
+    pip install opencc-python-reimplemented
+
     s2t.json Simplified Chinese to Traditional Chinese 簡體到繁體
     t2s.json Traditional Chinese to Simplified Chinese 繁體到簡體
     s2tw.json Simplified Chinese to Traditional Chinese (Taiwan Standard) 簡體到臺灣正體
@@ -49,7 +52,7 @@ def chinese_convert(s, config='t2s'):
     tw2t.json Traditional Chinese (Taiwan standard) to Traditional Chinese 臺灣正體到繁體（OpenCC 標準）
     @return:
     """
-    import opencc # pip install opencc opencc-python-reimplemented
+    opencc = try_import("opencc", pip_name="opencc-python-reimplemented")
     converter = opencc.OpenCC(config)
     return converter.convert(s)
 
@@ -89,7 +92,7 @@ def half2all(ustring):
 
 def all2half(all_string):
     """全角转半角"""
-    from idna import unichr
+    from idna import chr
 
     half_string = ""
     for char in all_string:
@@ -179,6 +182,11 @@ class Encrypt(object):
         return b.decode("utf-8")
 
 
+def unicode_normalize(s):
+    import unicodedata
+    return unicodedata.normalize('NFKC', s)
+
+
 if __name__ == '__main__':
     # print(str_replace('abcd', {'a': '8', 'd': '88'}))
     # print(unquote())
@@ -188,4 +196,15 @@ if __name__ == '__main__':
 
     # print(json_loads("{1: 1}"))
 
-    print(Encrypt().encrypt('123'*100))
+    # print(Encrypt().encrypt('123' * 100))
+
+    # print(query_params2json("act=get&uid=134597&vkey=07A1D82FDDE1E96DB5CEF4EF12C8125F&num=1&time=30&plat=1&re=1&type=0&so=1&ow=1&spl=1&addr=&db=1"))
+
+    print(half2all('⽉月'))
+    print(all2half('⽉月'))
+
+# import chardet
+#
+# def detect_encoding(byte_content):
+#     result = chardet.detect(byte_content)
+#     return result['encoding']

meutils/str_utils/json_utils.py

@@ -7,6 +7,9 @@
 # @WeChat       : 313303303
 # @Software     : PyCharm
 # @Description  : pd.io.json.json_normalize
+import jsonpath
+
+# json https://blog.csdn.net/freeking101/article/details/103048514
 # https://github.com/ijl/orjson#quickstart
 # https://jmespath.org/tutorial.html
 # https://goessner.net/articles/JsonPath/
@@ -41,3 +44,7 @@ def json_loads(s):
         logger.warning(e)
 
         return eval(s)
+
+
+if __name__ == '__main__':
+    print(jsonpath.jsonpath({"a": 1}, expr='$.a'))

meutils/str_utils/regular_expression.py

@@ -6,16 +6,41 @@
 # @Author       : yuanjie
 # @WeChat       : meutils
 # @Software     : PyCharm
-# @Description  : 
+# @Description  :
+import json
 
 from meutils.pipe import *
+from urllib.parse import unquote, unquote_plus
 
 zh = re.compile('[a-zA-Z\u4e00-\u9fa5]+')  # 中文 + 字母
 nozh = re.compile('[^a-zA-Z\u4e00-\u9fa5]+')  # 中文 + 字母
 
+HTML_PARSER = re.compile(r'```html(.*?)```', re.DOTALL)
+
 
 # re.sub(r'=(.+)', r'=123','s=xxxxx')
 
+@lru_cache()
+def remove_date_suffix(filename):
+    """
+    # 测试示例
+    filenames = [
+        "claude-3-5-haiku-20241022",
+        "o1-mini-2024-09-12",
+        "gpt-3.5-turbo-0125"
+    ]
+
+    # 输出结果
+    for fname in filenames:
+        print(remove_date_suffix(fname))
+    :param filename:
+    :return:
+    """
+    # 匹配日期格式（YYYYMMDD 或 YYYY-MM-DD）
+    pattern = r'(-\d{8}|-\d{4}-\d{2}-\d{2}|-\d+)$'
+    # 使用正则表达式替换日期后缀
+    return re.sub(pattern, '', filename)
+
 
 def get_parse_and_index(text, pattern):
     """
@@ -33,17 +58,58 @@ def get_parse_and_index(text, pattern):
         yield match.start(), match.end(), match.group()
 
 
-@lru_cache
-def parse_url(text: str):
+@lru_cache()
+def parse_url(text: str, for_image=False):
     # url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
     url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+|#]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
-    # url_pattern = 'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'
-    url = re.findall(url_pattern, str(text))
 
-    return url
+    # url_pattern = r"((https?|ftp|www\\.)?:\\/\\/)?([a-zA-Z0-9.-]+\\.[a-zA-Z]{2,})(:[0-9]+)?(\\/[^\\s]*)?"
+
+    if for_image:
+        text = unquote_plus(text)
+        # suffix = [
+        #     ".jpg",
+        #     ".jpeg",
+        #     ".png",
+        #     ".gif",
+        #     ".bmp",
+        #     ".tiff",
+        #     ".psd",
+        #     ".ai",
+        #     ".svg",
+        #     ".webp",
+        #     ".ico",
+        #     ".raw",
+        #     ".dng"
+        # ]
+        # url_pattern = r'https?://[\w\-\.]+/\S+\.(?:png|jpg|jpeg|gif)'
+        url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[#]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\.(?:jpg|jpeg|png|gif|svg|webp)'
+
+    urls = re.findall(url_pattern, text)
+
+    valid_urls = []
+    for url in urls:
+        url = url.strip(r"\n")
+        valid_urls.append(url)
+
+    return valid_urls
+
+
+def parse_url_from_json():
+    pass
 
 
 if __name__ == '__main__':
+    # from urllib.parse import urlparse
+    #
+    #
+    # def is_valid_url(url):
+    #     try:
+    #         result = urlparse(url)
+    #         return all([result.scheme, result.netloc])
+    #     except:
+    #         return False
+
     text = """7个正规url
     这是一段包含URL的文本，https://www.google.com 是一个URL，另一个URL是http://www.baidu.com
     解读这个文本https://www.url1.com
@@ -51,12 +117,38 @@ if __name__ == '__main__':
     http://www.url2.com 解读这个文本
 
     https://www.url2.com解读这个文本
-    
+
     总结 waptianqi.2345.com/wea_history/58238.html
-    
+
     总结 https://waptianqi.2345.com/wea_history/58238.htm
+    解释下这张照片 https://img-home.csdnimg.cn/images/20201124032511.png
+        解释下这张https://img-home.csdnimg.cn/images/x.png
+
+        img-home.csdnimg.cn/images/20201124032511.png
+
+
+    https://oss.ffire.cc/files/百炼系列手机产品介绍.docx
+    
+    https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf\n\n文档里说了什么？
+
+    
+        https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？
+
     """
+    # https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 正则匹配会卡死
+    # from urllib3.util import parse_url
+    # text = "@firebot /换衣 https://oss.ffire.cc/files/try-on.png"
+    # text = "@firebot /换衣 https://oss.ffire.cc/files/try-on.pn"
+
+    # print(parse_url(text))
+    # print(parse_url(text, for_image=True))
+    print(parse_url(text, for_image=False))
+
+    d = {"url": "https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf\n\n总结下"}
+    print(parse_url(str(d)))
+
+    print('https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf\\n\\n'.strip(r"\n"))
 
-    print(parse_url(text))
+    # print(parse_url("http://154.3.0.117:39666/docs#/default/get_content_preview_spider_playwright_get"))
 
-    print(parse_url("http://154.3.0.117:39666/docs#/default/get_content_preview_spider_playwright_get"))
+    # print(parse_url(text, True))

meutils/templates/xx.html

@@ -0,0 +1,21 @@
+<style> h1 {
+    text-align: center;
+    font-size: 24px;
+    margin-bottom: 10px;
+}
+
+table {
+    border-collapse: collapse;
+    width: 100%;
+}
+
+th, td {
+    border: 1px solid black;
+    padding: 8px;
+    text-align: center;
+}
+
+th {
+    background-color: #f2f2f2;
+}  </style>
+<head><title>2023年胡润百富榜</title></head><h1>2023年胡润百富榜</h1>