MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {MeUtils-2024.3.4.13.4.45.dist-info → MeUtils-2025.1.16.17.15.52.dist-info}/METADATA +38 -32
- MeUtils-2025.1.16.17.15.52.dist-info/RECORD +864 -0
- {MeUtils-2024.3.4.13.4.45.dist-info → MeUtils-2025.1.16.17.15.52.dist-info}/WHEEL +1 -1
- {MeUtils-2024.3.4.13.4.45.dist-info → MeUtils-2025.1.16.17.15.52.dist-info}/entry_points.txt +1 -0
- apps/spider.py +24 -8
- examples/_openaisdk/4v.py +110 -0
- examples/_openaisdk/__init__.py +11 -0
- examples/_openaisdk/baichuan.py +38 -0
- examples/_openaisdk/bpo.py +138 -0
- examples/_openaisdk/chat_latex.py +95 -0
- examples/_openaisdk/chattts.py +85 -0
- examples/_openaisdk/copilot.py +48 -0
- examples/_openaisdk/dalle3.py +48 -0
- examples/_openaisdk/deeplx.py +31 -0
- examples/_openaisdk/demo.py +77 -0
- examples/_openaisdk/embeddings.py +34 -0
- examples/_openaisdk/gpt4all.py +69 -0
- examples/_openaisdk/gpt_fc.py +23 -0
- examples/_openaisdk/gr_vl.py +46 -0
- examples/_openaisdk/json_mode.py +12 -0
- examples/_openaisdk/kimi.py +91 -0
- examples/_openaisdk/kimi_qa.py +57 -0
- examples/_openaisdk/minimax.py +75 -0
- examples/_openaisdk/open_router.py +48 -0
- examples/_openaisdk/openai_aiplus.py +54 -0
- examples/_openaisdk/openai_audio.py +20 -0
- examples/_openaisdk/openai_baichuan.py +59 -0
- examples/_openaisdk/openai_cache.py +37 -0
- examples/_openaisdk/openai_chatfire.py +228 -0
- examples/_openaisdk/openai_chatfire_all.py +166 -0
- examples/_openaisdk/openai_deepinfra.py +55 -0
- examples/_openaisdk/openai_deepseek.py +29 -0
- examples/_openaisdk/openai_doubao.py +43 -0
- examples/_openaisdk/openai_embeddings.py +36 -0
- examples/_openaisdk/openai_files.py +57 -0
- examples/_openaisdk/openai_gitee.py +33 -0
- examples/_openaisdk/openai_god.py +45 -0
- examples/_openaisdk/openai_groq.py +240 -0
- examples/_openaisdk/openai_images.py +203 -0
- examples/_openaisdk/openai_json.py +78 -0
- examples/_openaisdk/openai_lingyi.py +59 -0
- examples/_openaisdk/openai_modelscope.py +54 -0
- examples/_openaisdk/openai_moon.py +55 -0
- examples/_openaisdk/openai_oi.py +61 -0
- examples/_openaisdk/openai_ppu.py +47 -0
- examples/_openaisdk/openai_qwen.py +58 -0
- examples/_openaisdk/openai_search.py +42 -0
- examples/_openaisdk/openai_sensenova.py +81 -0
- examples/_openaisdk/openai_siliconflow.py +52 -0
- examples/_openaisdk/openai_step.py +45 -0
- examples/_openaisdk/openai_test.py +66 -0
- examples/_openaisdk/openai_together.py +57 -0
- examples/_openaisdk/openai_tune.py +38 -0
- examples/_openaisdk/openai_zhipu.py +59 -0
- examples/_openaisdk/ppu.py +28 -0
- examples/_openaisdk/rag.py +54 -0
- examples/_openaisdk/rag_.py +26 -0
- examples/_openaisdk/test.py +52 -0
- examples/_openaisdk/x.py +32 -0
- examples/_openaisdk/xx.py +29 -0
- examples/_openaisdk/zhipu_files.py +47 -0
- examples/_openaisdk/zhipu_/346/231/272/350/203/275/344/275/223.py +45 -0
- examples/_openaisdk//345/205/234/345/272/225/346/265/213/350/257/225.py +50 -0
- examples/_openaisdk//345/244/232/346/250/241/346/200/201/346/265/213/350/257/225.py +76 -0
- examples/_openaisdk//345/244/232/346/250/241/346/200/201/346/265/213/350/257/225_.py +56 -0
- examples/_openaisdk//346/226/207/344/273/266/351/227/256/347/255/224.py +36 -0
- examples/_openaisdk//346/226/207/346/241/243/350/247/243/346/236/220.py +34 -0
- examples/_openaisdk//346/250/241/345/236/213/346/265/213/350/257/225.py +53 -0
- examples/_openaisdk//351/230/277/351/207/214.py +80 -0
- {meutils/serving/jina/__demo → examples/ann}/__init__.py +1 -1
- examples/ann/main.py +31 -0
- examples/apis/kl.py +28 -0
- examples/apis/x.py +17 -0
- examples/apis/xx.py +17 -0
- examples/arq_demo/demo.py +3 -0
- examples/backgroundtasks.py +25 -0
- examples/bserver.py +513 -21
- examples/cache_demo/HermesCache_demo.py +81 -0
- examples/cache_demo/acacge.py +26 -0
- examples/cache_demo/x.py +31 -0
- {meutils/docarray_utils → examples/caches}/__init__.py +1 -1
- examples/caches/llmcache.py +18 -0
- examples/celery_demo/product_task.py +2 -0
- examples/demo.py +17 -1
- examples/fastapi_caching.py +59 -0
- {meutils/dependencies → examples/gr}/__init__.py +1 -1
- examples/gr/d.py +22 -0
- examples/gr/demo.py +30 -0
- examples/ip2/345/234/260/345/214/272.py +16 -0
- examples/jinja2_demo/j2_demo.py +20 -1
- examples/json/346/240/207/345/207/206/345/214/226.py +54 -0
- examples/md.py +29 -0
- {meutils/serving/jina → examples/nesc}/__init__.py +1 -1
- examples/nesc/main.py +76 -0
- examples/orm/mysql_orm.py +113 -0
- examples/orm/sql_creater.py +57 -0
- examples/orm/sqlm.py +134 -0
- examples/rq_demo/fns.py +18 -0
- examples/rq_demo/redis/351/230/237/345/210/227.py +14 -7
- examples/rq_demo/redis/351/230/237/345/210/227_add_chatfire.py +30 -0
- examples/size_map.py +43 -0
- examples/test.py +59 -0
- examples/webs/__init__.py +11 -0
- examples/webs/main.py +34 -0
- examples/x.py +13 -0
- examples//345/216/273/346/260/264/345/215/260.py +20 -0
- examples//346/226/207/346/241/243/346/231/272/350/203/275/__init__.py +11 -0
- meutils/_utils.py +15 -6
- meutils/ai_audio/asr/__init__.py +3 -2
- meutils/ai_audio/asr/cf_asr.py +53 -0
- meutils/ai_audio/asr/de.py +11 -0
- meutils/ai_audio/asr/fast_asr.py +15 -7
- meutils/ai_audio/asr/openai_asr.py +83 -6
- meutils/ai_audio/fast_asr.py +8 -4
- meutils/ai_audio/tts/EdgeTTS.py +33 -7
- meutils/ai_audio/tts/openai_tts.py +24 -20
- meutils/ai_audio/tts/tts_ui.py +1 -0
- meutils/ai_audio/utils.py +9 -0
- meutils/ai_cv/__init__.py +0 -1
- meutils/ai_cv/ocr.py +3 -2
- meutils/ai_cv/utils.py +154 -0
- meutils/ai_video/avmerge.py +6 -0
- meutils/ai_video/video.py +11 -2
- meutils/{api → apis}/__init__.py +1 -1
- meutils/apis/ali_apis.py +60 -0
- meutils/apis/audio/__init__.py +10 -0
- meutils/apis/audio/deepinfra.py +59 -0
- meutils/apis/audio/fish.py +248 -0
- meutils/apis/baidu/__init__.py +9 -0
- meutils/apis/baidu/bdaitpzs.py +229 -0
- meutils/apis/baidu/test.py +78 -0
- meutils/apis/chatglm/__init__.py +11 -0
- meutils/apis/chatglm/glm_video.py +273 -0
- meutils/apis/chatglm/glm_video_api.py +116 -0
- meutils/apis/chatglm/images.py +63 -0
- meutils/apis/chatglm/temp.py +259 -0
- meutils/apis/chatglm/x.py +31 -0
- meutils/{api → apis}/common.py +10 -6
- meutils/apis/fal/__init__.py +11 -0
- meutils/apis/fal/files.py +53 -0
- meutils/apis/fal/images.py +57 -0
- meutils/apis/fal/images_.py +72 -0
- meutils/apis/fal/videos.py +77 -0
- meutils/apis/firecrawl.py +45 -0
- meutils/apis/gitee/__init__.py +11 -0
- meutils/apis/gitee/images/__init__.py +9 -0
- meutils/apis/gitee/images/kolors.py +99 -0
- meutils/apis/hailuoai/__init__.py +11 -0
- meutils/apis/hailuoai/demo.py +34 -0
- meutils/apis/hailuoai/hasha_new.py +248 -0
- meutils/apis/hailuoai/music.py +11 -0
- meutils/apis/hailuoai/upload.py +116 -0
- meutils/apis/hailuoai/videos.py +460 -0
- meutils/apis/hailuoai/yy.py +242 -0
- meutils/apis/hf/__init__.py +11 -0
- meutils/apis/hf/got_ocr.py +64 -0
- meutils/apis/hf/gradio.py +34 -0
- meutils/apis/hf/hivisionidphotos.py +80 -0
- meutils/apis/hf/kolors.py +68 -0
- meutils/apis/hf/kolors_virtual_try_on.py +107 -0
- meutils/apis/hf/r.py +53 -0
- meutils/apis/hf/x.py +26 -0
- meutils/apis/hf//350/257/201/344/273/266/347/205/247.py +41 -0
- meutils/apis/hunyuan/__init__.py +11 -0
- meutils/apis/hunyuan/image_tools.py +84 -0
- meutils/apis/images/__init__.py +11 -0
- meutils/apis/images/deepinfra.py +92 -0
- meutils/apis/images/demo.py +150 -0
- meutils/apis/images/eidt.py +36 -0
- meutils/apis/images/flux/__init__.py +11 -0
- meutils/apis/images/flux/fluxpro.py +108 -0
- meutils/apis/images/flux/mystic.py +116 -0
- meutils/apis/images/ideogram/__init__.py +10 -0
- meutils/apis/images/ideogram/ideogram_images.py +193 -0
- meutils/apis/images/prodia/__init__.py +12 -0
- meutils/apis/images/prodia/faceswap.py +76 -0
- meutils/apis/images/recraft.py +152 -0
- meutils/apis/images/virtual_try_on/__init__.py +11 -0
- meutils/apis/images/virtual_try_on/images.py +65 -0
- meutils/apis/jiema/24mail.py +96 -0
- meutils/apis/jiema/__init__.py +11 -0
- meutils/apis/jiema/yezi.py +97 -0
- meutils/apis/jimeng/__init__.py +11 -0
- meutils/apis/jimeng/common.py +328 -0
- meutils/apis/jimeng/doubao.py +68 -0
- meutils/apis/jimeng/doubao_utils.py +175 -0
- meutils/apis/jimeng/files.py +263 -0
- meutils/apis/jimeng/images.py +140 -0
- meutils/apis/jimeng/lip_sync.py +11 -0
- meutils/apis/jina.py +55 -0
- meutils/apis/kling/__init__.py +11 -0
- meutils/apis/kling/api.py +60 -0
- meutils/apis/kling/images.py +174 -0
- meutils/apis/kling/kolors_virtual_try_on.py +111 -0
- meutils/apis/kling/kolors_virtual_try_on_web.py +126 -0
- meutils/apis/kling/videos.py +67 -0
- meutils/apis/kling//351/211/264/346/235/203.py +34 -0
- meutils/apis/kuaidi.py +32 -0
- meutils/apis/kuaishou/__init__.py +10 -0
- meutils/apis/kuaishou/klingai.py +523 -0
- meutils/apis/kuaishou/klingai_video.py +197 -0
- meutils/apis/kuaishou/kolors.py +189 -0
- meutils/apis/llm_qa.py +55 -0
- meutils/apis/luma/__init__.py +11 -0
- meutils/apis/luma/luma.py +123 -0
- meutils/apis/minicpm/__init__.py +9 -0
- meutils/apis/minicpm/luca.py +137 -0
- meutils/apis/monica/__init__.py +11 -0
- meutils/apis/monica/llm.py +11 -0
- meutils/apis/napkin/__init__.py +11 -0
- meutils/apis/napkin/icons.py +42 -0
- meutils/apis/niutrans.py +73 -0
- meutils/apis/oneapi/__init__.py +11 -0
- meutils/apis/oneapi/channel.py +68 -0
- meutils/apis/oneapi/common.py +135 -0
- meutils/apis/oneapi/log.py +47 -0
- meutils/apis/oneapi/token.py +48 -0
- meutils/apis/oneapi/token_.py +112 -0
- meutils/apis/oneapi/user.py +100 -0
- meutils/apis/oneapi/utils.py +47 -0
- meutils/apis/pixverse/__init__.py +11 -0
- meutils/apis/pixverse/pixverse.py +150 -0
- meutils/apis/proxy/__init__.py +11 -0
- meutils/apis/proxy/ips.py +178 -0
- meutils/apis/remini/__init__.py +11 -0
- meutils/apis/remini/remini.py +89 -0
- meutils/apis/replicateai/__init__.py +11 -0
- meutils/apis/replicateai/images.py +79 -0
- meutils/apis/replicateai/raw.py +53 -0
- meutils/apis/runwayml/__init__.py +10 -0
- meutils/apis/runwayml/gen.py +143 -0
- meutils/apis/search/__init__.py +11 -0
- meutils/apis/search/baichuan.py +11 -0
- meutils/apis/search/metaso.py +218 -0
- meutils/apis/search/metaso_.py +77 -0
- meutils/apis/search/n.py +99 -0
- meutils/apis/search/searxng.py +42 -0
- meutils/apis/search_music.py +39 -0
- meutils/apis/siliconflow/__init__.py +9 -0
- meutils/apis/siliconflow/audio.py +63 -0
- meutils/apis/siliconflow/image_to_image.py +116 -0
- meutils/apis/siliconflow/images.py +154 -0
- meutils/apis/siliconflow/rerankers.py +40 -0
- meutils/apis/siliconflow/text_to_image.py +132 -0
- meutils/apis/siliconflow/utils.py +66 -0
- meutils/apis/siliconflow/videos.py +102 -0
- meutils/apis/sunoai/__init__.py +10 -0
- meutils/apis/sunoai/haimian.py +135 -0
- meutils/apis/sunoai/suno.py +373 -0
- meutils/apis/textcard/__init__.py +11 -0
- meutils/apis/textcard/demo.py +25 -0
- meutils/apis/textcard/hanyuxinjie.py +81 -0
- meutils/apis/textin.py +159 -0
- meutils/apis/to_image/__init__.py +11 -0
- meutils/apis/to_image/html2image.py +29 -0
- meutils/apis/to_image/md.py +29 -0
- meutils/apis/to_image/url2image.py +41 -0
- meutils/apis/together/__init__.py +11 -0
- meutils/apis/together/images.py +80 -0
- meutils/apis/translator/__init__.py +9 -0
- meutils/apis/translator/deeplx.py +55 -0
- meutils/apis/tripo3d/__init__.py +11 -0
- meutils/apis/tripo3d/images.py +106 -0
- meutils/apis/ts.py +60 -0
- meutils/apis/uptime_kuma/__init__.py +11 -0
- meutils/apis/uptime_kuma/common.py +56 -0
- meutils/apis/uptime_kuma//345/233/275/344/272/247/345/210/206/347/273/204.py +68 -0
- meutils/apis/utils.py +47 -0
- meutils/apis/videos/__init__.py +11 -0
- meutils/apis/videos/sora.py +16 -0
- meutils/apis/vidu/__init__.py +9 -0
- meutils/apis/vidu/vidu_video.py +254 -0
- meutils/apis/vidu/x.py +14 -0
- meutils/apis/voice_clone/__init__.py +10 -0
- meutils/apis/voice_clone/fish.py +236 -0
- meutils/apis/voice_clone/fish_api.py +16 -0
- meutils/apis/web_search.py +31 -0
- meutils/apis/yezi.py +97 -0
- meutils/async_task/__init__.py +13 -0
- meutils/async_task/celery_config.py +106 -0
- meutils/async_task/common.py +37 -0
- meutils/async_task/demo_create_tasks.py +73 -0
- meutils/async_task/tasks/__init__.py +11 -0
- meutils/async_task/tasks/_all.py +20 -0
- meutils/async_task/tasks/hailuo.py +24 -0
- meutils/async_task/tasks/kling.py +30 -0
- meutils/async_task/tasks/replicateai.py +24 -0
- meutils/async_task/tasks/test.py +124 -0
- meutils/async_task/tasks/vidu.py +28 -0
- meutils/async_task/utils.py +191 -0
- meutils/async_task//351/200/232/347/224/250/350/256/276/350/256/241.py +119 -0
- meutils/async_utils/asyncer_.py +37 -0
- meutils/async_utils/background.py +68 -0
- meutils/async_utils/common.py +136 -16
- meutils/async_utils/test.py +47 -0
- meutils/cache_utils.py +29 -23
- meutils/caches/__init__.py +9 -0
- meutils/caches/acache.py +45 -0
- meutils/caches/redis_cache.py +63 -0
- meutils/clis/check_api.py +66 -0
- meutils/clis/cli.py +1 -1
- meutils/common.py +56 -17
- meutils/config_utils/__init__.py +11 -0
- meutils/config_utils/lark_utils/__init__.py +11 -0
- meutils/config_utils/lark_utils/common.py +385 -0
- meutils/config_utils/lark_utils/demo.py +13 -0
- meutils/config_utils/lark_utils/x.py +50 -0
- meutils/config_utils/manager.py +108 -0
- meutils/crawlers/__init__.py +11 -0
- meutils/data/VERSION +1 -1
- meutils/data/cowboy-hat-face.webp +0 -0
- meutils/data/oneapi/FOOTER.md +7 -0
- meutils/data/oneapi/NOTICE.md +138 -0
- meutils/data/oneapi/__init__.py +15 -0
- meutils/db/orm.py +179 -0
- meutils/db/redis_db.py +87 -0
- meutils/decorators/cache.py +1 -1
- meutils/decorators/common.py +84 -5
- meutils/decorators/contextmanagers.py +17 -6
- meutils/decorators/fastapi_decorator.py +77 -3
- meutils/decorators/polling.py +46 -0
- meutils/decorators/retry.py +150 -26
- meutils/fastapi_utils/__init__.py +11 -0
- meutils/fastapi_utils/exceptions/http_error.py +72 -0
- meutils/fastapi_utils/exceptions/validation_error.py +44 -0
- meutils/hash_utils.py +9 -4
- meutils/hooks/__init__.py +11 -0
- meutils/hooks/hook_test.py +174 -0
- meutils/hooks/wechat.py +162 -0
- meutils/hooks/wechat_channel.py +303 -0
- meutils/init/evn.py +1 -1
- meutils/io/files_utils.py +232 -0
- meutils/io/image.py +148 -10
- meutils/io/x.py +75 -0
- meutils/llm/__init__.py +10 -0
- meutils/llm/check_api.py +109 -0
- meutils/llm/check_utils.py +106 -0
- meutils/llm/clients.py +38 -0
- meutils/llm/completions/__init__.py +11 -0
- meutils/llm/completions/agents/__init__.py +11 -0
- meutils/llm/completions/agents/file.py +125 -0
- meutils/llm/completions/cp.py +112 -0
- meutils/llm/completions/delilegal.py +135 -0
- meutils/llm/completions/dify.py +81 -0
- meutils/llm/completions/kimi.py +47 -0
- meutils/llm/completions/modelscope.py +11 -0
- meutils/{fileparser/filetype.py → llm/completions/oi.py} +5 -3
- meutils/llm/completions/rag/__init__.py +11 -0
- meutils/llm/completions/rag/fire.py +157 -0
- meutils/llm/completions/rag/qwen.py +11 -0
- meutils/llm/completions/rag/rag.py +41 -0
- meutils/llm/completions/rag.py +38 -0
- meutils/llm/completions/tryblend.py +201 -0
- meutils/llm/completions/tune.py +284 -0
- meutils/llm/completions/x.py +26 -0
- meutils/llm/completions/xx.py +61 -0
- meutils/llm/completions/yuanbao.py +176 -0
- meutils/llm/demo.py +114 -0
- meutils/llm/functions/__init__.py +11 -0
- meutils/llm/mappers.py +15 -0
- meutils/llm/openai_utils/__init__.py +11 -0
- meutils/llm/openai_utils/common.py +284 -0
- meutils/llm/openai_utils/tool_outputs.py +45 -0
- meutils/llm/output_parsers/__init__.py +80 -0
- meutils/llm/prompts/__init__.py +244 -0
- meutils/llm/prompts/demo.py +155 -0
- meutils/llm/prompts/html2image_test.py +19 -0
- meutils/llm/utils.py +133 -0
- meutils/llm/x.py +75 -0
- meutils/notice/feishu.py +40 -9
- meutils/notice/wechat.py +23 -21
- meutils/np_utils.py +10 -1
- meutils/office_automation/pdf.py +6 -1
- meutils/oss/__init__.py +20 -0
- meutils/oss/minio_oss.py +184 -0
- meutils/oss/minio_utils.py +48 -0
- meutils/other/__demo.py +6 -4
- meutils/pandas_utils/__init__.py +1 -0
- meutils/pandas_utils/common.py +31 -0
- meutils/pandas_utils/pd_utils.py +10 -6
- meutils/parsers/__init__.py +10 -0
- meutils/parsers/file_parsers.py +110 -0
- meutils/parsers/fileparser/demo.py +41 -0
- meutils/parsers/fileparser/filetype.py +41 -0
- meutils/pay.py +37 -0
- meutils/pipe.py +37 -4
- meutils/playwright_utils/common.py +20 -12
- meutils/plots/common.py +35 -34
- meutils/queues/demo.py +56 -0
- meutils/queues/smooth_queue.py +120 -0
- meutils/queues/uniform_queue.py +3 -1
- meutils/request_utils/__init__.py +26 -2
- meutils/request_utils/ark.py +47 -0
- meutils/request_utils/crawler.py +34 -5
- meutils/request_utils/jwt_utils/__init__.py +11 -0
- meutils/request_utils/jwt_utils/common.py +42 -0
- meutils/request_utils/volc.py +160 -0
- meutils/schemas/__init__.py +0 -1
- meutils/schemas/baidu_types.py +70 -0
- meutils/schemas/batch_types.py +450 -0
- meutils/schemas/celery_types.py +64 -0
- meutils/schemas/chatfire_types.py +15 -0
- meutils/schemas/chatglm_types.py +197 -0
- meutils/schemas/db/__init__.py +11 -0
- meutils/schemas/db/oneapi_types.py +117 -0
- meutils/schemas/dify_types.py +40 -0
- meutils/schemas/embedding.py +31 -0
- meutils/schemas/fal_types.py +13 -0
- meutils/schemas/fish_types.py +11 -0
- meutils/schemas/hailuo_types.py +208 -0
- meutils/schemas/haimian_types.py +51 -0
- meutils/schemas/idphoto_types.py +43 -0
- meutils/schemas/image_types.py +476 -0
- meutils/schemas/jimeng_types.py +28 -0
- meutils/schemas/jina_types.py +67 -0
- meutils/schemas/kimi_types.py +86 -0
- meutils/schemas/kling_types.py +235 -0
- meutils/schemas/kuaishou_types.py +328 -0
- meutils/schemas/luma_types.py +59 -0
- meutils/schemas/message_types.py +165 -0
- meutils/schemas/message_types_.py +219 -0
- meutils/schemas/metaso_types.py +64 -0
- meutils/schemas/napkin_types.py +85 -0
- meutils/schemas/ocr_types.py +37 -0
- meutils/schemas/oneapi/__init__.py +11 -0
- meutils/schemas/oneapi/_types.py +49 -0
- meutils/schemas/oneapi/common.py +883 -0
- meutils/schemas/oneapi/icons.py +30 -0
- meutils/schemas/oneapi/model_group_info.py +48 -0
- meutils/schemas/oneapi/model_info.py +34 -0
- meutils/schemas/oneapi/models.py +26 -0
- meutils/schemas/oneapi/x.py +26 -0
- meutils/schemas/oneapi//351/207/215/345/256/232/345/220/221.py +132 -0
- meutils/schemas/openai_api_protocol.py +411 -0
- meutils/schemas/openai_types.py +366 -0
- meutils/schemas/pixverse_types.py +88 -0
- meutils/schemas/playwright_types.py +57 -0
- meutils/schemas/prodia_types.py +19 -0
- meutils/schemas/replicate_types.py +112 -0
- meutils/schemas/request_types.py +20 -0
- meutils/schemas/runwayml_types.py +190 -0
- meutils/schemas/siliconflow_types.py +80 -0
- meutils/schemas/step_types.py +19 -0
- meutils/schemas/suno_types.py +319 -0
- meutils/schemas/task_types.py +192 -0
- meutils/schemas/translator_types.py +29 -0
- meutils/schemas/tripo3d_types.py +57 -0
- meutils/schemas/tryblend_types.py +51 -0
- meutils/schemas/video_types.py +62 -0
- meutils/schemas/vidu_types.py +350 -0
- meutils/schemas/wechat_types.py +316 -0
- meutils/schemas/yuanbao_types.py +260 -0
- meutils/serving/celery/__init__.py +8 -0
- meutils/serving/celery/config.py +115 -0
- meutils/serving/celery/router.py +4 -6
- meutils/serving/celery/tasks.py +6 -4
- meutils/serving/celery//351/200/232/347/224/250/350/256/276/350/256/241.py +119 -0
- meutils/serving/fastapi/common.py +27 -31
- meutils/serving/fastapi/dependencies/__init__.py +0 -1
- meutils/serving/fastapi/dependencies/auth.py +55 -2
- meutils/serving/fastapi/exceptions/http_error.py +67 -2
- meutils/serving/fastapi/exceptions/validation_error.py +18 -2
- meutils/serving/fastapi/lifespans.py +73 -0
- meutils/serving/fastapi/routers/scheduler.py +12 -0
- meutils/serving/fastapi/routers/screenshot.py +47 -0
- meutils/serving/fastapi/routers/spider.py +8 -3
- meutils/serving/fastapi/routers/task.py +48 -0
- meutils/serving/fastapi/utils.py +48 -1
- meutils/serving/streamlit/common.py +1 -1
- meutils/smooth_utils.py +3 -0
- meutils/str_utils/__init__.py +22 -3
- meutils/str_utils/json_utils.py +7 -0
- meutils/str_utils/regular_expression.py +102 -10
- meutils/templates/xx.html +21 -0
- meutils/templates/xxx.html +117 -0
- meutils/todo.py +12 -0
- meutils/tools/token_monitor.py +33 -0
- MeUtils-2024.3.4.13.4.45.dist-info/RECORD +0 -540
- meutils/ai_audio/asr/subtitle.srt +0 -40
- meutils/ai_audio/demo.ipynb +0 -1215
- meutils/ai_audio/example.srt +0 -348
- meutils/ai_audio/new.srt +0 -179
- meutils/ai_audio/subtitles.srt +0 -696
- meutils/ai_audio/tts/new.srt +0 -179
- meutils/ai_audio//350/247/206/351/242/221/345/220/210/345/271/266.sh +0 -32
- meutils/ai_cv/1.jpg +0 -0
- meutils/ai_cv/197.jpg +0 -0
- meutils/ai_cv/2.jpg +0 -0
- meutils/ai_cv/img.png +0 -0
- meutils/ai_cv/invoice.jpg +0 -0
- meutils/ai_cv/tbl.png +0 -0
- meutils/ai_cv/test.png +0 -0
- meutils/ann/README.md +0 -33
- meutils/ann/README_gensim.md +0 -47
- meutils/ann/examples/client.py +0 -59
- meutils/ann/examples/demo.py +0 -24
- meutils/api/deeplx.py +0 -29
- meutils/api/qr.png +0 -0
- meutils/clis/README.md +0 -29
- meutils/clis/__test.sh +0 -17
- meutils/clis/deepseek.txt +0 -8
- meutils/clis/deepseek_13003330042.json +0 -1
- meutils/clis/deepseek_13003872192.json +0 -1
- meutils/clis/deepseek_13852263862.json +0 -1
- meutils/clis/deepseek_13913898681.json +0 -1
- meutils/clis/deepseek_13962978617.json +0 -1
- meutils/clis/deepseek_15251801790.json +0 -1
- meutils/clis/deepseek_15720826383.json +0 -1
- meutils/clis/deepseek_18395563611.json +0 -1
- meutils/clis/deepseek_313303303@qq.com.json +0 -1
- meutils/clis/kimi_state.json +0 -1
- meutils/cmds/README.md +0 -55
- meutils/coding/__init__.py +0 -11
- meutils/coding/find132.py +0 -40
- meutils/db/README.md +0 -51
- meutils/decorators/README.md +0 -17
- meutils/docarray_utils/demo_es.py +0 -34
- meutils/docarray_utils/demo_hnsw.py +0 -55
- meutils/docarray_utils/in_memory.py +0 -38
- meutils/docarray_utils//346/224/271/351/200/240/344/270/213hnsw.py +0 -43
- meutils/io/file.py +0 -20
- meutils/io/img.png +0 -0
- meutils/io/x.yml +0 -1
- meutils/notice/img.png +0 -0
- meutils/notice/todo.md +0 -10
- meutils/office_automation//346/212/225/350/265/204/347/256/241/347/220/206/347/263/273/347/273/237O3.2_/344/272/244/346/230/223/347/273/204.pdm +0 -22469
- meutils/playwright_utils/__test.sh +0 -2
- meutils/playwright_utils/kimi1_cookies.json +0 -1
- meutils/playwright_utils/kimi2_cookies.json +0 -1
- meutils/playwright_utils/kimi_cookies.json +0 -93
- meutils/serving/README.md +0 -1
- meutils/serving/celery/_run.sh +0 -10
- meutils/serving/gui/run.sh +0 -9
- meutils/serving/jina/__demo/client.py +0 -42
- meutils/serving/jina/__demo/flow.svg +0 -1
- meutils/serving/jina/__demo/s.py +0 -34
- meutils/serving/jina/__demo/s2.py +0 -37
- meutils/serving/jina/__demo/server.py +0 -83
- meutils/serving/jina/__demo/test.py +0 -40
- meutils/serving/jina/executors/SentenceEncoder.py +0 -62
- meutils/serving/jina/executors/SentenceEncoder_.py +0 -63
- meutils/serving/jina/executors/__init__.py +0 -46
- meutils/serving/jina/executors/base.py +0 -40
- meutils/serving/jina/nlp_serving/__init__.py +0 -11
- meutils/serving/jina/nlp_serving/word_segmentation.py +0 -40
- meutils/serving/streamlit/conf.yaml +0 -5
- meutils/serving/streamlit/ocr.png +0 -0
- meutils/serving/streamlit/run.sh +0 -17
- meutils/serving/webui/.streamlit/_config.toml +0 -186
- meutils/serving/webui/.streamlit/config.toml +0 -26
- meutils/serving/webui/pages/_1_/345/210/206/350/257/215.py +0 -56
- meutils/serving/webui/pages/_2_/350/257/215/346/200/247/346/240/207/346/263/250/344/270/216/345/256/236/344/275/223/350/257/206/345/210/253.py +0 -54
- meutils/serving/webui/pages/_3_/346/226/207/346/234/254/345/214/271/351/205/215.py +0 -64
- meutils/serving/webui/run.sh +0 -9
- meutils/spark/__init__.py +0 -26
- meutils/tools/monitor.yml +0 -29
- {MeUtils-2024.3.4.13.4.45.dist-info → MeUtils-2025.1.16.17.15.52.dist-info}/LICENSE +0 -0
- {MeUtils-2024.3.4.13.4.45.dist-info → MeUtils-2025.1.16.17.15.52.dist-info}/top_level.txt +0 -0
- {meutils → examples}/comp_utils/__init__.py +0 -0
- {meutils → examples}/comp_utils/reverse_metric.py +0 -0
- /meutils/{fileparser/README.md → fastapi_utils/exceptions/__init__.py} +0 -0
- /meutils/{fileparser → parsers/fileparser}/PDF/346/212/275/345/217/226.py" +0 -0
- /meutils/{fileparser → parsers/fileparser}/__init__.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/common.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/__init__.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/__main__.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/filetype.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/helpers.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/match.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/__init__.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/application.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/archive.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/audio.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/base.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/document.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/font.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/image.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/isobmff.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/types/video.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/filetype/utils.py +0 -0
- /meutils/{fileparser → parsers/fileparser}/pdf.py +0 -0
- /meutils/{fileparser → parsers/fileparser}//350/241/250/346/240/274/346/212/275/345/217/226.py" +0 -0
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Project : AI. @by PyCharm
|
4
|
+
# @File : check_api_key
|
5
|
+
# @Time : 2024/6/17 14:56
|
6
|
+
# @Author : betterme
|
7
|
+
# @WeChat : meutils
|
8
|
+
# @Software : PyCharm
|
9
|
+
# @Description :
|
10
|
+
|
11
|
+
from meutils.pipe import *
|
12
|
+
from meutils.async_utils import async_to_sync
|
13
|
+
|
14
|
+
from meutils.config_utils.lark_utils import get_spreadsheet_values
|
15
|
+
from meutils.notice.feishu import send_message as _send_message
|
16
|
+
|
17
|
+
from meutils.db.redis_db import redis_client
|
18
|
+
from meutils.llm.check_api import check_api_key_or_token
|
19
|
+
|
20
|
+
send_message = partial(
|
21
|
+
_send_message,
|
22
|
+
url="https://open.feishu.cn/open-apis/bot/v2/hook/e0db85db-0daf-4250-9131-a98d19b909a9",
|
23
|
+
)
|
24
|
+
|
25
|
+
|
26
|
+
@cli.command()
|
27
|
+
def check_and_update_api_keys(check_url: Optional[str], feishu_url, ttl: Optional[int] = None):
|
28
|
+
"""
|
29
|
+
python check_api.py https://api.deepseek.com/user/balance https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=X0ZN3H
|
30
|
+
|
31
|
+
|
32
|
+
"""
|
33
|
+
|
34
|
+
api_keys = get_spreadsheet_values(feishu_url=feishu_url, to_dataframe=True)[0].tolist()
|
35
|
+
|
36
|
+
if feishu_url in redis_client:
|
37
|
+
api_keys += redis_client.lrange(feishu_url, 0, -1) | xmap(lambda x: x.decode())
|
38
|
+
|
39
|
+
# 有效 api_keys
|
40
|
+
api_keys = async_to_sync(check_api_key_or_token)(set(api_keys), check_url)
|
41
|
+
|
42
|
+
if feishu_url in redis_client:
|
43
|
+
lastest_api_keys = redis_client.lrange(feishu_url, 0, -1) | xmap_(lambda x: x.decode())
|
44
|
+
|
45
|
+
# 新增的
|
46
|
+
to_update_api_keys = list(set(api_keys) - set(lastest_api_keys))
|
47
|
+
to_update_api_keys += lastest_api_keys | xfilter_(lambda api_key: api_key in api_keys) # 存量有效的&按当前顺序
|
48
|
+
|
49
|
+
else:
|
50
|
+
to_update_api_keys = api_keys
|
51
|
+
|
52
|
+
num = 0
|
53
|
+
if to_update_api_keys: # 更新 redis
|
54
|
+
to_update_api_keys = to_update_api_keys | xUnique
|
55
|
+
|
56
|
+
redis_client.delete(feishu_url)
|
57
|
+
num = redis_client.rpush(feishu_url, *to_update_api_keys)
|
58
|
+
|
59
|
+
if ttl: redis_client.expire(feishu_url, ttl) # Redis 的 RPUSH 命令本身不支持直接设置过期时间
|
60
|
+
|
61
|
+
api_keys_str = (to_update_api_keys and to_update_api_keys[:20]) | xjoin('\n')
|
62
|
+
send_message(f"有效轮询 {num} 个\n\n{feishu_url}\n\n{api_keys_str}", title="更新api-keys")
|
63
|
+
|
64
|
+
|
65
|
+
if __name__ == '__main__':
|
66
|
+
cli() # https://api.deepseek.com/user/balance https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf\?sheet\=X0ZN3H
|
meutils/clis/cli.py
CHANGED
meutils/common.py
CHANGED
@@ -25,11 +25,14 @@ import inspect
|
|
25
25
|
import textwrap
|
26
26
|
import socket
|
27
27
|
import logging
|
28
|
+
import tempfile
|
28
29
|
import warnings
|
29
30
|
|
30
31
|
warnings.filterwarnings("ignore")
|
31
32
|
|
32
33
|
import functools
|
34
|
+
from functools import wraps
|
35
|
+
|
33
36
|
import argparse
|
34
37
|
import traceback
|
35
38
|
import threading
|
@@ -48,12 +51,14 @@ import toml
|
|
48
51
|
import yaml
|
49
52
|
import typer
|
50
53
|
import json
|
54
|
+
import mimetypes
|
51
55
|
import joblib
|
52
56
|
from joblib.hashing import hash
|
53
57
|
import httpx
|
54
58
|
import requests
|
55
59
|
import wrapt
|
56
|
-
import
|
60
|
+
import shortuuid
|
61
|
+
# import sklearn #######
|
57
62
|
import numpy as np
|
58
63
|
import pandas as pd
|
59
64
|
|
@@ -68,11 +73,13 @@ from functools import reduce, lru_cache, partial
|
|
68
73
|
from collections import Counter, OrderedDict
|
69
74
|
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
|
70
75
|
|
71
|
-
import matplotlib.pyplot as plt
|
72
|
-
from matplotlib.font_manager import FontProperties
|
76
|
+
# import matplotlib.pyplot as plt #######
|
77
|
+
# from matplotlib.font_manager import FontProperties #######
|
78
|
+
#
|
79
|
+
# FONT = FontProperties(fname=get_resolve_path('./data/SimHei.ttf', __file__))
|
73
80
|
|
74
81
|
# matplotlib.use('Agg'): matplotlib 这个库,这个库在使用时会创建额外的线程。如果你的应用不需要使用到 matplotlib,你可以考虑移除这个库,这可能能够解决问题。如果你的应用需要使用 matplotlib,但是并不需要它的图形界面功能,你可以尝试在导入 matplotlib 之前设置 matplotlib 的后端为 'Agg'
|
75
|
-
plt.rcParams['axes.unicode_minus'] = False
|
82
|
+
# plt.rcParams['axes.unicode_minus'] = False
|
76
83
|
|
77
84
|
from loguru import logger
|
78
85
|
# logger.remove()
|
@@ -91,6 +98,9 @@ fake_zh = Faker(locale='zh_CN')
|
|
91
98
|
# from PIL import Image, ImageGrab
|
92
99
|
# image
|
93
100
|
# im = ImageGrab.grabclipboard() 获取剪切板的图片
|
101
|
+
# json
|
102
|
+
import jsonpath
|
103
|
+
import json_repair
|
94
104
|
|
95
105
|
# 第三方
|
96
106
|
from meutils.other.crontab import CronTab
|
@@ -122,7 +132,6 @@ cli = typer.Typer(name="MeUtils CLI")
|
|
122
132
|
|
123
133
|
# 常量
|
124
134
|
CPU_NUM = os.cpu_count()
|
125
|
-
FONT = FontProperties(fname=get_resolve_path('./data/SimHei.ttf', __file__))
|
126
135
|
|
127
136
|
HOST_NAME = DOMAIN_NAME = LOCAL_HOST = LOCAL = HOST = PORT = ''
|
128
137
|
|
@@ -161,20 +170,18 @@ try:
|
|
161
170
|
|
162
171
|
ic.configureOutput(includeContext=True)
|
163
172
|
|
164
|
-
except
|
173
|
+
except Exception as e:
|
165
174
|
pass
|
166
175
|
|
167
176
|
|
168
|
-
def
|
169
|
-
|
170
|
-
|
171
|
-
i = int(current / total * 100)
|
172
|
-
s = f"""\r{i}%|{"█" * (i // 5)}|"""
|
173
|
-
print(s, end="", flush=True)
|
174
|
-
# sys.stdout.flush()
|
177
|
+
def download(url, filename: Optional[str] = None):
|
178
|
+
filename = filename or Path(url).name
|
175
179
|
|
176
|
-
|
177
|
-
|
180
|
+
with httpx.Client(follow_redirects=True, timeout=100) as client:
|
181
|
+
response = client.get(url)
|
182
|
+
with open(filename, 'wb') as f:
|
183
|
+
f.write(response.content)
|
184
|
+
return filename
|
178
185
|
|
179
186
|
|
180
187
|
class BaseConfig(BaseModel):
|
@@ -315,7 +322,7 @@ def get_current_fn():
|
|
315
322
|
return f_name
|
316
323
|
|
317
324
|
|
318
|
-
@lru_cache
|
325
|
+
@lru_cache()
|
319
326
|
def get_function_params(fn: Optional[Callable] = None):
|
320
327
|
if fn is None:
|
321
328
|
import openai
|
@@ -544,7 +551,35 @@ def try_tasks(tasks: List[Callable], handle_error: Optional[Callable] = None, *a
|
|
544
551
|
return handle_error(*args, **kwargs)
|
545
552
|
|
546
553
|
|
554
|
+
def ttl_fn(ttl: int = 60):
|
555
|
+
return time.time() // ttl # 缓存时间
|
556
|
+
|
557
|
+
|
558
|
+
def storage_to_cookie(storage: Union[str, Path, dict]):
|
559
|
+
if isinstance(storage, Path) or (isinstance(storage, str) and len(storage) < 256 and Path(storage).is_file()):
|
560
|
+
storage = json.loads(Path(storage).read_bytes())
|
561
|
+
elif isinstance(storage, str):
|
562
|
+
storage = json.loads(storage)
|
563
|
+
|
564
|
+
storage = storage['cookies']
|
565
|
+
|
566
|
+
import jsonpath
|
567
|
+
cookies = [f"{n}={v}" for n, v in
|
568
|
+
zip(jsonpath.jsonpath(storage, "$..name"), jsonpath.jsonpath(storage, "$..value"))]
|
569
|
+
return '; '.join(cookies)
|
570
|
+
|
571
|
+
|
572
|
+
@lru_cache()
|
573
|
+
def url2fileview(url):
|
574
|
+
encoded_url = base64.b64encode(url.encode()).decode()
|
575
|
+
# import urllib.parse
|
576
|
+
# encoded_url = urllib.parse.quote(encoded_url)
|
577
|
+
|
578
|
+
return f"https://v.ffire.cc/onlinePreview?url={encoded_url}"
|
579
|
+
|
580
|
+
|
547
581
|
if __name__ == '__main__':
|
582
|
+
pass
|
548
583
|
# s = "import pandas as pd; output = pd.__version__"
|
549
584
|
# s = "import os; output = os.popen(cmd).read().split()"
|
550
585
|
# print(exec_callback(s, cmd='ls'))
|
@@ -562,4 +597,8 @@ if __name__ == '__main__':
|
|
562
597
|
# print(get_var_name(s))
|
563
598
|
|
564
599
|
# print(try_tasks([lambda x: 1 / x, lambda x: 2 / x], handle_error=lambda x: 3 / (x + 0.000001), x=0))
|
565
|
-
print(get_function_params(hot_reload))
|
600
|
+
# print(get_function_params(hot_reload))
|
601
|
+
# print(url2view("https://sf-maas-uat-prod.oss-cn-shanghai.aliyuncs.com/outputs/dd25c8e0-fc6b-4d9b-aede-f6a489b04dee_00001_.png"))
|
602
|
+
# print(get_function_params())
|
603
|
+
|
604
|
+
pass
|
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Project : AI. @by PyCharm
|
4
|
+
# @File : __init__.py
|
5
|
+
# @Time : 2024/4/30 16:46
|
6
|
+
# @Author : betterme
|
7
|
+
# @WeChat : meutils
|
8
|
+
# @Software : PyCharm
|
9
|
+
# @Description :
|
10
|
+
|
11
|
+
from meutils.config_utils.lark_utils.common import *
|
@@ -0,0 +1,385 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Project : AI. @by PyCharm
|
4
|
+
# @File : common
|
5
|
+
# @Time : 2024/5/6 08:52
|
6
|
+
# @Author : betterme
|
7
|
+
# @WeChat : meutils
|
8
|
+
# @Software : PyCharm
|
9
|
+
# @Description : httpx重试 transport = httpx.AsyncHTTPTransport(retries=3) # response.raise_for_status()
|
10
|
+
|
11
|
+
from meutils.pipe import *
|
12
|
+
from meutils.decorators.retry import retrying
|
13
|
+
from meutils.decorators.contextmanagers import try_catcher
|
14
|
+
from meutils.notice.feishu import send_message
|
15
|
+
from meutils.db.redis_db import redis_client, redis_aclient
|
16
|
+
|
17
|
+
from urllib.parse import urlparse, parse_qs
|
18
|
+
|
19
|
+
FEISHU_BASE_URL = "https://open.feishu.cn/open-apis/"
|
20
|
+
|
21
|
+
|
22
|
+
def get_app_access_token():
|
23
|
+
payload = {
|
24
|
+
"app_id": os.getenv("FEISHU_APP_ID", "cli_a60cfd9cad76100b"),
|
25
|
+
"app_secret": os.getenv("FEISHU_APP_SECRET")
|
26
|
+
}
|
27
|
+
response = httpx.post(
|
28
|
+
"https://open.feishu.cn/open-apis/auth/v3/app_access_token/internal",
|
29
|
+
json=payload,
|
30
|
+
timeout=30,
|
31
|
+
)
|
32
|
+
|
33
|
+
# logger.debug(response.json())
|
34
|
+
|
35
|
+
return response.json().get("app_access_token")
|
36
|
+
|
37
|
+
|
38
|
+
@retrying(max_retries=3, predicate=lambda x: not x)
|
39
|
+
async def aget_app_access_token():
|
40
|
+
"""不加缓存,access_token会提前失效
|
41
|
+
该报错是因为开发者本次调用过程中使用的 Tenant Access Token 已经失效或有误,飞书开放平台无法判断当前请求是否来自一个可信的用户,因此拦截了情况。
|
42
|
+
:return:
|
43
|
+
"""
|
44
|
+
payload = {
|
45
|
+
"app_id": os.getenv("FEISHU_APP_ID"),
|
46
|
+
"app_secret": os.getenv("FEISHU_APP_SECRET")
|
47
|
+
}
|
48
|
+
async with httpx.AsyncClient(base_url=FEISHU_BASE_URL, timeout=30) as client:
|
49
|
+
response = await client.post("/auth/v3/app_access_token/internal", json=payload)
|
50
|
+
|
51
|
+
return response.is_success and response.json().get("app_access_token") # False / None
|
52
|
+
|
53
|
+
|
54
|
+
def get_spreadsheet_values(
|
55
|
+
spreadsheet_token: Optional[str] = None,
|
56
|
+
sheet_id: Optional[str] = None,
|
57
|
+
feishu_url=None,
|
58
|
+
to_dataframe: Optional[bool] = False
|
59
|
+
):
|
60
|
+
if feishu_url and feishu_url.startswith("http"):
|
61
|
+
parsed_url = urlparse(feishu_url)
|
62
|
+
spreadsheet_token = parsed_url.path.split('/')[-1]
|
63
|
+
sheet_id = parsed_url.query.split('=')[-1]
|
64
|
+
|
65
|
+
url = f"https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/{spreadsheet_token}/values/{sheet_id}"
|
66
|
+
|
67
|
+
headers = {
|
68
|
+
"Authorization": f"Bearer {get_app_access_token()}"
|
69
|
+
}
|
70
|
+
response = httpx.get(url, headers=headers, timeout=30)
|
71
|
+
_ = response.json()
|
72
|
+
return _ if not to_dataframe else pd.DataFrame(_.get('data').get('valueRange').get('values'))
|
73
|
+
|
74
|
+
|
75
|
+
@alru_cache(ttl=600)
|
76
|
+
async def aget_spreadsheet_values(
|
77
|
+
spreadsheet_token: Optional[str] = None,
|
78
|
+
sheet_id: Optional[str] = None,
|
79
|
+
feishu_url: Optional[str] = None,
|
80
|
+
to_dataframe: Optional[bool] = False
|
81
|
+
):
|
82
|
+
if feishu_url and feishu_url.startswith("http"):
|
83
|
+
parsed_url = urlparse(feishu_url)
|
84
|
+
spreadsheet_token = parsed_url.path.split('/')[-1]
|
85
|
+
sheet_id = parsed_url.query.split('=')[-1]
|
86
|
+
|
87
|
+
access_token = await aget_app_access_token()
|
88
|
+
headers = {
|
89
|
+
"Authorization": f"Bearer {access_token}"
|
90
|
+
}
|
91
|
+
async with httpx.AsyncClient(base_url=FEISHU_BASE_URL, timeout=30, headers=headers) as client:
|
92
|
+
response = await client.get(f"/sheets/v2/spreadsheets/{spreadsheet_token}/values/{sheet_id}")
|
93
|
+
if response.is_success:
|
94
|
+
data = response.json()
|
95
|
+
if to_dataframe:
|
96
|
+
# values = data.get('data').get('valueRange').get('values')
|
97
|
+
# [value | xmap(lambda x: x['text']) | xjoin('') for value in values]
|
98
|
+
|
99
|
+
return pd.DataFrame(data.get('data').get('valueRange').get('values'))
|
100
|
+
return data
|
101
|
+
else:
|
102
|
+
send_message(
|
103
|
+
f"{response.status_code}\n\n{access_token}\n\n{response.text}",
|
104
|
+
'飞书为啥为none: 已经去掉缓存了'
|
105
|
+
)
|
106
|
+
return get_spreadsheet_values(spreadsheet_token, sheet_id, feishu_url, to_dataframe)
|
107
|
+
|
108
|
+
|
109
|
+
async def spreadsheet_values_append(
|
110
|
+
spreadsheet_token: Optional[str] = None,
|
111
|
+
sheet_id: Optional[str] = None,
|
112
|
+
feishu_url=None,
|
113
|
+
range: Optional[str] = None,
|
114
|
+
values: Optional[list] = None
|
115
|
+
):
|
116
|
+
""" https://open.feishu.cn/document/server-docs/docs/sheets-v3/data-operation/append-data
|
117
|
+
|
118
|
+
https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/:spreadsheetToken/values_prepend
|
119
|
+
https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/:spreadsheetToken/values_append
|
120
|
+
https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/:spreadsheetToken/values_image
|
121
|
+
|
122
|
+
:param spreadsheet_token:
|
123
|
+
:param sheet_id:
|
124
|
+
:param feishu_url:
|
125
|
+
:param values:
|
126
|
+
:return:
|
127
|
+
"""
|
128
|
+
|
129
|
+
if feishu_url and feishu_url.startswith("http"):
|
130
|
+
parsed_url = urlparse(feishu_url)
|
131
|
+
spreadsheet_token = parsed_url.path.split('/')[-1]
|
132
|
+
sheet_id = parsed_url.query.split('=')[-1]
|
133
|
+
|
134
|
+
access_token = await aget_app_access_token()
|
135
|
+
headers = {
|
136
|
+
"Authorization": f"Bearer {access_token}",
|
137
|
+
"Content-Type": "application/json; charset=utf-8",
|
138
|
+
|
139
|
+
}
|
140
|
+
payload = {
|
141
|
+
"valueRange": {
|
142
|
+
# https://open.feishu.cn/document/server-docs/docs/sheets-v3/overview
|
143
|
+
# "range": f"{sheet_id}!A:B", # <sheetId>!<开始列>:<结束列>
|
144
|
+
# "range": f"{sheet_id}!A1:E3", # <sheetId>!<开始单元格>:<结束列>
|
145
|
+
# "range": f"{sheet_id}", # 非空的最大行列范围内的数据
|
146
|
+
|
147
|
+
"range": f"{sheet_id}!{range}" if range else sheet_id,
|
148
|
+
"values": values
|
149
|
+
}
|
150
|
+
}
|
151
|
+
|
152
|
+
async with httpx.AsyncClient(base_url=FEISHU_BASE_URL, timeout=30, headers=headers) as client:
|
153
|
+
# 默认覆盖 ?insertDataOption=OVERWRITE
|
154
|
+
response = await client.post(f"/sheets/v2/spreadsheets/{spreadsheet_token}/values_append", json=payload)
|
155
|
+
return response.is_success and response.json() or response.text
|
156
|
+
|
157
|
+
|
158
|
+
def create_document(title: str = "一篇新文档🔥", folder_token: Optional[str] = None):
|
159
|
+
payload = {
|
160
|
+
"title": title,
|
161
|
+
"folder_token": folder_token,
|
162
|
+
}
|
163
|
+
|
164
|
+
url = "https://open.feishu.cn/open-apis/docx/v1/documents"
|
165
|
+
headers = {
|
166
|
+
"Authorization": f"Bearer {get_app_access_token()}"
|
167
|
+
}
|
168
|
+
response = httpx.post(url, headers=headers, timeout=30, json=payload)
|
169
|
+
return response.json()
|
170
|
+
|
171
|
+
|
172
|
+
def get_doc_raw_content(document_id: str = "BxlwdZhbyoyftZx7xFbcGCZ8nah"):
|
173
|
+
url = f"https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/raw_content"
|
174
|
+
headers = {
|
175
|
+
"Authorization": f"Bearer {get_app_access_token()}"
|
176
|
+
}
|
177
|
+
response = httpx.get(url, headers=headers, timeout=30)
|
178
|
+
return response.json()
|
179
|
+
|
180
|
+
|
181
|
+
async def get_next_from_redis(redis_key):
|
182
|
+
if api_key := await redis_aclient.lpop(redis_key):
|
183
|
+
await redis_aclient.rpush(redis_key, api_key)
|
184
|
+
return api_key
|
185
|
+
|
186
|
+
|
187
|
+
async def get_next_token(
|
188
|
+
feishu_url,
|
189
|
+
check_token: Optional[Callable] = None,
|
190
|
+
min_points: float = 0, # vidu min_points=4
|
191
|
+
ttl: Optional[int] = None,
|
192
|
+
):
|
193
|
+
for i in range(10): # 避免死循环
|
194
|
+
if i > 5:
|
195
|
+
send_message(f"重复同步了{i}次\n\n{feishu_url}", title="更新tokens")
|
196
|
+
|
197
|
+
if token := await redis_aclient.lpop(feishu_url):
|
198
|
+
token = token.decode()
|
199
|
+
|
200
|
+
if check_token is None:
|
201
|
+
logger.info("写回队列「跳过核验」")
|
202
|
+
await redis_aclient.rpush(feishu_url, token)
|
203
|
+
return token
|
204
|
+
elif await check_token(token, threshold=min_points):
|
205
|
+
logger.info("写回队列「大于最小消耗」")
|
206
|
+
await redis_aclient.rpush(feishu_url, token)
|
207
|
+
return token
|
208
|
+
elif await check_token(token):
|
209
|
+
logger.info("不写回队列「最后一次消耗」")
|
210
|
+
return token
|
211
|
+
|
212
|
+
logger.info("不写回队列「积分不够」")
|
213
|
+
|
214
|
+
else: # 更新tokens到redis
|
215
|
+
df = await aget_spreadsheet_values(feishu_url=feishu_url, to_dataframe=True)
|
216
|
+
# api_keys = [k for k in df[0] if isinstance(k, str)] # 过滤空值, 避免 list: 一般超链接造成的
|
217
|
+
api_keys = []
|
218
|
+
for api_key in df[0]:
|
219
|
+
if api_key:
|
220
|
+
if isinstance(api_key, str):
|
221
|
+
api_keys.append(api_key)
|
222
|
+
else:
|
223
|
+
api_key = "".join(map(lambda x: x.get('text', ""), api_key))
|
224
|
+
api_keys.append(api_key)
|
225
|
+
|
226
|
+
# 初始化
|
227
|
+
if check_token:
|
228
|
+
bools = await asyncio.gather(*map(check_token, api_keys)) # todo: 提前过滤一遍, 减少初始化次数
|
229
|
+
api_keys = list(itertools.compress(api_keys, bools))
|
230
|
+
|
231
|
+
if not api_keys: # 全部失效
|
232
|
+
logger.debug(f"全部无效:{feishu_url}")
|
233
|
+
break
|
234
|
+
|
235
|
+
await redis_aclient.delete(feishu_url)
|
236
|
+
num = await redis_aclient.rpush(feishu_url, *api_keys)
|
237
|
+
|
238
|
+
send_message(f"新增:{num}\n\n{feishu_url}", title="更新tokens")
|
239
|
+
|
240
|
+
if ttl:
|
241
|
+
await redis_aclient.expire(feishu_url, ttl) # Redis 的 RPUSH 命令本身不支持直接设置过期时间
|
242
|
+
|
243
|
+
|
244
|
+
async def get_next_token_for_polling(
|
245
|
+
feishu_url,
|
246
|
+
check_token: Optional[Callable] = None,
|
247
|
+
max_size: Optional[int] = 10,
|
248
|
+
from_redis: Optional[bool] = False,
|
249
|
+
min_points: float = 0
|
250
|
+
):
|
251
|
+
if from_redis:
|
252
|
+
return await get_next_token(feishu_url, check_token, ttl=7 * 24 * 3600, min_points=min_points)
|
253
|
+
|
254
|
+
# 轮询
|
255
|
+
max_size = max_size or 10000
|
256
|
+
df = await aget_spreadsheet_values(feishu_url=feishu_url, to_dataframe=True)
|
257
|
+
|
258
|
+
api_keys = []
|
259
|
+
for api_key in df[0]:
|
260
|
+
if api_key:
|
261
|
+
if isinstance(api_key, str):
|
262
|
+
api_keys.append(api_key)
|
263
|
+
else:
|
264
|
+
logger.debug("feishu表格数据: 非字符串")
|
265
|
+
api_key = "".join(map(lambda x: x.get('text', ""), api_key))
|
266
|
+
api_keys.append(api_key)
|
267
|
+
|
268
|
+
api_keys = np.random.choice(api_keys, size=min(max_size, len(api_keys)))
|
269
|
+
|
270
|
+
num = len(api_keys)
|
271
|
+
logger.debug(f"重复数:{num - len(set(list(api_keys)))}")
|
272
|
+
|
273
|
+
if check_token:
|
274
|
+
with try_catcher("CHECK_TOKEN"):
|
275
|
+
tasks = map(check_token, api_keys)
|
276
|
+
api_keys = [k for k, v in zip(api_keys, await asyncio.gather(*tasks)) if v]
|
277
|
+
|
278
|
+
prob = len(api_keys) / num
|
279
|
+
logger.debug(f"过滤前:{num} 过滤后:{len(api_keys)},存活率:{prob:.2%}")
|
280
|
+
if prob < 0.3:
|
281
|
+
send_message(f"存活率 ≤ {prob},及时更新", title=__name__) # todo: 提醒
|
282
|
+
|
283
|
+
return np.random.choice(api_keys)
|
284
|
+
|
285
|
+
|
286
|
+
@alru_cache(ttl=3600)
|
287
|
+
async def get_dataframe(iloc_tuple: Optional[tuple] = None, feishu_url: Optional[str] = None, ): # 系统配置
|
288
|
+
feishu_url = feishu_url or "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=bl4jYm"
|
289
|
+
df = await aget_spreadsheet_values(feishu_url=feishu_url, to_dataframe=True)
|
290
|
+
|
291
|
+
if iloc_tuple:
|
292
|
+
return df.iloc._getitem_tuple(iloc_tuple) # df.iloc._getitem_tuple((0, 1))
|
293
|
+
return df
|
294
|
+
|
295
|
+
|
296
|
+
async def get_series(feishu_url: str, index: int = 0, duplicated: bool = False): # 系统配置
|
297
|
+
df = await aget_spreadsheet_values(feishu_url=feishu_url, to_dataframe=True)
|
298
|
+
series = df[index]
|
299
|
+
values = [i for i in series if i] # todo: 非标准字符串处理
|
300
|
+
if duplicated: # 有序去重
|
301
|
+
values = values | xUnique
|
302
|
+
|
303
|
+
return values
|
304
|
+
|
305
|
+
|
306
|
+
if __name__ == '__main__':
|
307
|
+
# print(get_app_access_token())
|
308
|
+
# print(get_spreadsheet_values("Qy6OszlkIhwjRatkaOecdZhOnmh", "0f8eb3"))
|
309
|
+
# pprint(get_spreadsheet_values("Bmjtst2f6hfMqFttbhLcdfRJnNf", "Y9oalh"))
|
310
|
+
# pd.DataFrame(
|
311
|
+
# get_spreadsheet_values("Bmjtst2f6hfMqFttbhLcdfRJnNf", "79272d").get('data').get('valueRange').get('values'))
|
312
|
+
|
313
|
+
# print(get_doc_raw_content("TAEFdXmzyobvgUxKM3lcLfc2nxe"))
|
314
|
+
# print(create_document())
|
315
|
+
# "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=79272d"
|
316
|
+
|
317
|
+
# r = get_spreadsheet_values(feishu_url="https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=79272d",
|
318
|
+
# to_dataframe=True)
|
319
|
+
# print(list(filter(None, r[0])))
|
320
|
+
# print(get_spreadsheet_values("Bmjtst2f6hfMqFttbhLcdfRJnNf", "79272d"))
|
321
|
+
|
322
|
+
# print(arun(aget_app_access_token()))
|
323
|
+
# df = arun(aget_spreadsheet_values(
|
324
|
+
# feishu_url="https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=5i64gO",
|
325
|
+
# to_dataframe=True
|
326
|
+
#
|
327
|
+
# ))
|
328
|
+
# print(df)
|
329
|
+
|
330
|
+
from inspect import iscoroutinefunction
|
331
|
+
|
332
|
+
# print(filter(lambda x: x and x.strip(), df[0]) | xlist)
|
333
|
+
|
334
|
+
# func = aget_app_access_token()
|
335
|
+
|
336
|
+
# print(alru_cache(ttl=300)(sync_to_async(aget_app_access_token))())
|
337
|
+
|
338
|
+
# for i in tqdm(range(10)):
|
339
|
+
# # print(aget_app_access_token())
|
340
|
+
# print(arun(aget_app_access_token()))
|
341
|
+
# # print(get_app_access_token())
|
342
|
+
|
343
|
+
# values = [
|
344
|
+
# [
|
345
|
+
# "2023/12/25",
|
346
|
+
# "收入",
|
347
|
+
# "微信",
|
348
|
+
# "100",
|
349
|
+
# "帐号 老表max"
|
350
|
+
# ],
|
351
|
+
# [
|
352
|
+
# "2023/12/25",
|
353
|
+
# "支出",
|
354
|
+
# "支付宝",
|
355
|
+
# "10",
|
356
|
+
# "买东西 老表max"
|
357
|
+
# ],
|
358
|
+
# [
|
359
|
+
# "2023/12/26",
|
360
|
+
# "支出",
|
361
|
+
# "支付宝",
|
362
|
+
# "19.9",
|
363
|
+
# "买东西 老表max"
|
364
|
+
# ],
|
365
|
+
# ]
|
366
|
+
#
|
367
|
+
# _ = spreadsheet_values_append(
|
368
|
+
# feishu_url="https://xchatllm.feishu.cn/sheets/BPxjsmNj7hZr7Ytwk9uccvOKn6Z?sheet=7ce4e3",
|
369
|
+
#
|
370
|
+
# range="A1:E3",
|
371
|
+
# values=values
|
372
|
+
#
|
373
|
+
# )
|
374
|
+
#
|
375
|
+
# print(arun(_))
|
376
|
+
#
|
377
|
+
# print(create_document())
|
378
|
+
# arun(get_dataframe(iloc_tuple=(1, 0)))
|
379
|
+
feishu_url = "https://xchatllm.feishu.cn/sheets/GYCHsvI4qhnDPNtI4VPcdw2knEd?sheet=EYgZ8c"
|
380
|
+
feishu_url = "https://xchatllm.feishu.cn/sheets/GYCHsvI4qhnDPNtI4VPcdw2knEd?sheet=V84y6z"
|
381
|
+
# with timer():
|
382
|
+
# token = arun(get_next_token(feishu_url))
|
383
|
+
|
384
|
+
# FEISHU_URL = "https://xchatllm.feishu.cn/sheets/XfjqszII3hZAEvtTOgecOgv2nye?sheet=c14b34"
|
385
|
+
# print(arun(get_series(FEISHU_URL)))
|
@@ -0,0 +1,13 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
from meutils.config_utils.lark_utils import get_next_token, aget_spreadsheet_values
|
4
|
+
|
5
|
+
feishu_url = "https://xchatllm.feishu.cn/sheets/GYCHsvI4qhnDPNtI4VPcdw2knEd?sheet=Hcr2i8"
|
6
|
+
|
7
|
+
|
8
|
+
if __name__ == '__main__':
|
9
|
+
from meutils.pipe import *
|
10
|
+
|
11
|
+
df = arun(get_next_token(feishu_url=feishu_url))
|
12
|
+
|
13
|
+
|