why-tools 0.2.26__py3-none-any.whl

ytools/__init__.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Date    : 2024/4/15 下午7:28
+@Author  : yintian
+@Desc    : 
+"""
+
+from ytools.log import logger
+from ytools.version import get_version
+from ytools.utils.magic import empty
+
+__version__ = get_version(path=__file__)
+
+G = T = C = None
+
+
+def init_var(name: str):
+    global G, T, C
+    match name:
+        case 'G':
+            from ytools.utils.variable import VariableG
+            G = VariableG()
+        case 'T':
+            from ytools.utils.variable import VariableT
+            T = VariableT()
+        case 'C':
+            from ytools.utils.variable import VariableC
+            C = VariableC()
+        case _:
+            print(f"请输入: G/T/C")
+
+
+if __name__ == '__main__':
+    pass

ytools/alert/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Author  : yintian
+@Date    : 2026/1/8 15:00
+@Software: PyCharm
+@Desc    : 
+"""
+
+if __name__ == '__main__':
+    pass

ytools/alert/wechat.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : wechat.py
+@Author  : yintian
+@Date    : 2026/1/8 15:00
+@Software: PyCharm
+@Desc    : 
+"""
+from typing import Literal
+
+from ytools import logger
+from ytools.utils.magic import require
+
+msg_type_literal = Literal["text", "markdown", "image", "news", "file", "voice", "template_card", ""]
+
+
+async def async_send_wechat(content: str, to: list, at: list = None, msg_type: msg_type_literal = "text", **kwargs):
+    """发送企业微信告警消息
+
+    Args:
+        content: 消息内容（text类型为纯文本，markdown类型为markdown格式）
+        to: 机器人webhook URL列表或key列表
+        at: at 列表
+        msg_type: 消息类型，支持 "text" 和 "markdown"，默认为 "text"
+    """
+
+    require("httpx", action="fix")
+    import httpx  # noqa
+    for webhook in to:
+        # 如果to是key，需要拼接完整URL，否则直接使用
+        if not webhook.startswith("http"):
+            # 这里假设webhook是完整URL，如果是key需要根据实际情况拼接
+            webhook_url = webhook
+        else:
+            webhook_url = webhook
+
+        # 根据消息类型构建payload
+        payload = {
+            "msgtype": msg_type,
+            msg_type: {
+                "content": content,
+                "mentioned_list": at or [],
+                **kwargs
+            }
+        }
+        try:
+            async with httpx.AsyncClient(timeout=10.0) as client:
+                response = await client.post(webhook_url, json=payload)
+                response.raise_for_status()
+                result = response.json()
+
+                if result.get("errcode") == 0:
+                    logger.info(f"[企业微信]发送成功！-> {content[:50]}...")
+                else:
+                    logger.error(f"[企业微信]发送失败！错误码：{result.get('errcode')}，错误信息：{result.get('errmsg')}")
+        except Exception as e:
+            logger.error(f"[企业微信]发送失败！原因：{e}")
+
+
+def send_wechat(content: str, to: list, at: list = None, msg_type: msg_type_literal = "text", **kwargs):
+    """发送企业微信告警消息
+
+    Args:
+        content: 消息内容（text类型为纯文本，markdown类型为markdown格式）
+        to: 机器人webhook URL列表或key列表
+        at: at 列表
+        msg_type: 消息类型，支持 "text" 和 "markdown"，默认为 "text"
+    """
+
+    require("requests", action="fix")
+    import requests  # noqa
+    payload = {
+        "msgtype": msg_type,
+        msg_type: {
+            "content": content,
+            "mentioned_list": at or [],
+            **kwargs
+        }
+    }
+    for webhook in to:
+        # 如果to是key，需要拼接完整URL，否则直接使用
+        if not webhook.startswith("http"):
+            # 这里假设webhook是完整URL，如果是key需要根据实际情况拼接
+            webhook_url = webhook
+        else:
+            webhook_url = webhook
+
+        # 根据消息类型构建payload
+
+        try:
+            with requests.Session() as client:
+                response = client.post(webhook_url, json=payload, timeout=10)
+                response.raise_for_status()
+                result = response.json()
+
+                if result.get("errcode") == 0:
+                    logger.info(f"[企业微信]发送成功！-> {content[:50]}...")
+                else:
+                    logger.error(f"[企业微信]发送失败！错误码：{result.get('errcode')}，错误信息：{result.get('errmsg')}")
+        except Exception as e:
+            logger.error(f"[企业微信]发送失败！原因：{e}")
+
+
+def fmt_msg(*rows, msg: str | list = "", title: str = "微信通知", at: list = None, index=True, level: Literal["comment", "warning", "info"] = "info"):
+    msg = msg if isinstance(msg, list) else [msg]
+    msg = [f"# <font color=\"{level}\">{title}</font>\n", "\n", *msg, "\n"]
+
+    for row in rows:
+        msg.append("\n" +
+                   "\n".join(
+                       [f">{f'{_index + 1}. ' if index else ''}{item[0]}: <font color=\"comment\">{item[1]}</font>" for _index, item in
+                        enumerate(row.items())]
+                   )
+                   + "\n")
+    if at:
+        msg.append(f"\n\n {' '.join([f'<@{_at}>' for _at in at])}")
+    content = ''.join(msg)
+    return content
+
+
+def send_msg(*rows, to: list, msg: str | list = "", title: str = "微信通知", at: list = None, index=True, level: Literal["comment", "warning", "info"] = "info"):
+    content = fmt_msg(*rows, msg=msg, title=title, at=at, index=index, level=level)
+    send_wechat(content=content, to=to, msg_type="markdown")
+
+
+async def async_send_msg(*rows, to: list, msg: str | list = "", title: str = "微信通知", at: list = None, index=True, level: Literal["comment", "warning", "info"] = "info"):
+    content = fmt_msg(*rows, msg=msg, title=title, at=at, index=index, level=level)
+    await async_send_wechat(content=content, to=to, msg_type="markdown")
+
+
+if __name__ == '__main__':
+    pass

ytools/arq/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Author  : yintian
+@Date    : 2025/10/21 10:07
+@Software: PyCharm
+@Desc    : 
+"""
+
+if __name__ == '__main__':
+    pass

ytools/arq/client/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Author  : yintian
+@Date    : 2025/10/21 10:07
+@Software: PyCharm
+@Desc    : 
+"""
+
+if __name__ == '__main__':
+    pass

ytools/arq/client/agent.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : agent.py
+@Author  : yintian
+@Date    : 2025/10/21 11:33
+@Software: PyCharm
+@Desc    : 
+"""
+import asyncio
+import contextlib
+import inspect
+from asyncio import Event
+from typing import Callable, Any
+
+from ytools.arq import setting
+from ytools.arq.client.base import BaseClient
+from ytools.arq.task.task import Task
+from ytools.utils import magic
+from ytools.utils.counter import FastWriteCounter
+
+
+class Agent(BaseClient):
+    def __init__(self, worker: Callable[[Task], Any] | None = None, max_concurrency=None, **kwargs):
+        super().__init__(**kwargs)
+        self.worker = worker or self.run_task
+        self.success_tasks = FastWriteCounter()
+        self.extra = {
+            "success_tasks": self.success_tasks.value,
+        }
+        if max_concurrency:
+            self.context = asyncio.Semaphore(max_concurrency)
+        else:
+            self.context = contextlib.nullcontext()
+
+    async def do(self, task):
+        async with self.context:
+            try:
+                res = await self.run_callable(self.worker, task)
+            except Exception as e:
+                self.log(f"执行任务失败 task_id={task.task_id}: {type(e).__name__}: {e}", level="error")
+                res = f"ERROR::{type(e)}|{str(e)}"
+            await self.put_result(res, task)
+            self.success_tasks.increment()
+            self.extra["success_tasks"] = self.success_tasks.value
+            if task.callback:
+                asyncio.create_task(self.callback(task, res))
+
+    @staticmethod
+    def is_async_callable(func):
+        return inspect.iscoroutinefunction(func) or inspect.iscoroutinefunction(getattr(func, "__call__", None))
+
+    async def run_callable(self, func, *args, **kwargs):
+        pre = magic.prepare(func, *args, **kwargs)
+        if self.is_async_callable(pre.func):
+            return await pre()
+        res = await asyncio.to_thread(pre)
+        if inspect.isawaitable(res):
+            return await res
+        return res
+
+    async def run_task(self, task: Task):
+        payload = task.data
+        if isinstance(payload, (str, bytes)):
+            with contextlib.suppress(Exception):
+                payload = task.decode_data()
+        self.log(f"收到任务 task_id={task.task_id}: {payload}")
+        result = await self.execute_task(task, payload)
+        self.log(f"任务结果 task_id={task.task_id}: {result}")
+        return result
+
+    @staticmethod
+    def normalize_task_payload(payload):
+        if not isinstance(payload, dict):
+            return None
+        func = payload.get("func")
+        if func is None:
+            return None
+        func = magic.load_object(func)
+        args = payload.get("args", ())
+        kwargs = payload.get("kwargs", {})
+        if not isinstance(args, (list, tuple)):
+            args = (args,)
+        if kwargs is None:
+            kwargs = {}
+        if not isinstance(kwargs, dict):
+            raise TypeError("task kwargs 必须为 dict")
+        return func, tuple(args), kwargs
+
+    async def execute_task(self, task: Task, payload):
+        normalized = self.normalize_task_payload(payload)
+        if normalized is None:
+            return payload
+        func, args, kwargs = normalized
+        return await self.run_callable(func, *args, **kwargs, task=task)
+
+    async def callback(self, task: Task, res):
+        try:
+            pre = magic.prepare(task.callback, task=task, res=res, result=res)
+            if pre.is_async:
+                await pre()
+            else:
+                pre()
+
+        except Exception as e:
+            self.log(f"执行 callback 报错: {e}", level="error")
+
+    async def run(self, event: Event = None):
+        while True:
+            try:
+                if event and event.is_set():
+                    await asyncio.sleep(setting.INTERVAL)
+                    continue
+                task: Task = await self.get_task()
+                if not task:
+                    continue
+                self.task_count.increment()
+                asyncio.create_task(self.do(task))
+            finally:
+                await asyncio.sleep(setting.INTERVAL)
+
+    async def get_task(self):
+        task_id = await self.zpop(self.tasks_queue)
+        if not task_id:
+            return None
+        data = await self.redis.get(self.get_queue(task_id, base=self.data_queue))
+        if data is None:
+            self.log(f"task_id:{task_id} 未获取到数据", level="error")
+            return None
+        return Task(
+            data=data,
+            client=self,
+            task_id=task_id,
+            fmt=True
+        )
+
+    async def zpop(self, key: str):
+        result = None
+        if (await self.redis.info("server")).get("redis_version", "0.0.0") >= "5.0.0":
+            result = await self.redis.zpopmin(self.tasks_queue, count=1)
+            result = result and result[0]
+        else:
+            async with self.redis.pipeline(transaction=True) as pipe:
+                # 取出最小分数的任务
+                await pipe.zrange(key, 0, 0, withscores=False)
+                # 删除它
+                await pipe.zremrangebyrank(key, 0, 0)
+                result, _ = await pipe.execute()
+
+        if result:
+            return result[0].decode() if isinstance(result[0], bytes) else result[0]
+        return None
+
+    async def put_result(self, result, task):
+        result_queue = self.get_queue(task.task_id, base=self.result_queue)
+        data = task.encode_data(result)
+        await self.redis.publish(result_queue, data)
+
+
+if __name__ == '__main__':
+    pass

ytools/arq/client/base.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : base.py
+@Author  : yintian
+@Date    : 2025/10/21 10:07
+@Software: PyCharm
+@Desc    : 
+"""
+import asyncio
+import json
+
+from ytools import logger as default_logger
+from ytools.utils.counter import FastWriteCounter
+from ytools.utils.host_ip import get_local_ip
+from ytools.utils.magic import require
+
+require("redis")
+from redis.asyncio import Redis  # noqa
+from ytools.arq import setting
+
+
+class BaseClient:
+    redis: Redis
+    split = ':'
+    queue_name: str
+    tasks_queue: str
+    data_queue: str
+    result_queue: str
+    status_queue: str
+
+    def __init__(
+            self,
+            queue_name=None,
+            redis=None,
+            logger=None,
+            level="info",
+    ):
+        self.task_count = FastWriteCounter()
+        self.extra = {}
+        self.logger = logger or default_logger
+        self.level = (level or "info").lower()
+        self._host_ip = None
+        self.set_queue(queue_name)
+        if isinstance(redis, dict):
+            self.redis = self.make_redis(**redis)
+        elif isinstance(redis, str):
+            self.redis = Redis.from_url(redis)
+        elif redis:
+            self.redis = redis
+        asyncio.create_task(self.heartbeat())
+
+    @property
+    def info(self):
+        return {
+            "host_ip": self.get_host_ip(),
+            "task_count": self.task_count.value,
+            **self.extra
+        }
+
+    def set_queue(self, queue_name):
+        self.queue_name = queue_name or setting.DEFAULT_QUEUE_NAME
+        self.tasks_queue = self.get_queue("tasks")
+        self.data_queue = self.get_queue("data")
+        self.result_queue = self.get_queue("result")
+        self.status_queue = self.get_queue("status")
+
+    def get_queue(self, *queue: str, base=None):
+        return self.split.join([base or self.queue_name, *queue])
+
+    def log(self, message, level=None, *args, **kwargs):
+        level = (level or self.level or "info").lower()
+        logger_func = getattr(self.logger, level, None)
+        if callable(logger_func):
+            return logger_func(message, *args, **kwargs)
+        logger_func = getattr(self.logger, "info", None)
+        if callable(logger_func):
+            return logger_func(message, *args, **kwargs)
+
+    def get_host_ip(self):
+        if self._host_ip:
+            return self._host_ip
+        try:
+            self._host_ip = get_local_ip()
+        except Exception as e:
+            self.log(f"获取内网IP失败: {e}", level="error")
+            self._host_ip = "unknown"
+        return self._host_ip
+
+    @classmethod
+    def make_redis(
+            cls,
+            host: str,
+            port: int,
+            db=0,
+            password=None,
+            **kwargs
+    ):
+        return Redis(host=host, port=port, db=db, password=password, **kwargs)
+
+    @classmethod
+    def from_redis(cls, queue_name=None, **redis_config):
+        return cls(queue_name, redis=redis_config)
+
+    async def set_status(self, status_id, data=None, expire_time=None):
+        status_queue = self.get_queue(status_id, base=self.status_queue)
+        expire_time = expire_time or setting.EXPIRE_TIME
+        if data is not None:
+            await self.redis.set(status_queue, data, ex=expire_time)
+        if await self.redis.exists(status_id):
+            await self.redis.expire(status_queue, expire_time)
+
+    async def get_status(self, status_id):
+        status_queue = self.get_queue(status_id, base=self.status_queue)
+        return await self.redis.get(status_queue)
+
+    async def del_status(self, status_id):
+        status_queue = self.get_queue(status_id, base=self.status_queue)
+        return await self.redis.delete(status_queue)
+
+    async def heartbeat(self):
+        interval = 5
+        while True:
+            h_key = self.get_queue(self.__class__.__name__.lower(), self.get_host_ip(), base=self.queue_name)
+            await self.redis.set(h_key, value=json.dumps(self.info), ex=interval + 1)
+            await asyncio.sleep(interval)
+
+
+if __name__ == '__main__':
+    pass

ytools/arq/client/client.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : Client.py
+@Author  : yintian
+@Date    : 2025/10/21 10:20
+@Software: PyCharm
+@Desc    : 
+"""
+import asyncio
+import inspect
+from typing import Literal
+from uuid import uuid4
+
+from redis.asyncio.client import Pipeline
+
+from ytools.arq import setting
+from ytools.arq.client.base import BaseClient
+from ytools.arq.task.task import Task
+
+
+class Client(BaseClient):
+    max_sleep_time = 5
+
+    def __init__(self, *args, max_task_num: int = None, max_action: Literal["sleep", "break", "raise"] = "sleep", **kwargs):
+        self.max_task_num = max_task_num
+        self.max_action = max_action
+        super().__init__(*args, **kwargs)
+
+    async def put(self, data, task_id=None, **kwargs):
+        task_id = task_id or str(uuid4())
+        task = Task(data=data, client=self, task_id=task_id, result_queue=self.get_queue(task_id, base=self.result_queue), **kwargs)
+        await self.put_task(task, kwargs.get('auto_ensure', False))
+        return task
+
+    async def put_task(self, task: Task, auto_ensure=False):
+        data_queue = self.get_queue(task.task_id, base=self.data_queue)
+
+        auto_ensure and await task.ensure()
+        async with self.redis.pipeline(transaction=True) as pipe:
+            if self.max_task_num and await self.check_max(pipe):
+                return
+            await pipe.zadd(self.tasks_queue, {task.task_id: task.score})
+            await pipe.set(data_queue, task.encode_data(), ex=setting.EXPIRE_TIME)
+            results = await pipe.execute()
+            if not all(results):  # 检查是否有命令失败
+                raise ValueError(f"投放任务至队列失败: {results}")
+            self.task_count.increment()
+
+    async def check_max(self, pipe: Pipeline):
+        while True:
+            now_task_num = await pipe.zcard(self.tasks_queue)
+            if now_task_num >= self.max_task_num:
+                if self.max_action == "sleep":
+                    self.log(f"当前任务数量: {now_task_num} 超出最大任务数: {self.max_task_num}, 等待任务消费, 休眠 {self.max_sleep_time}s...")
+                    await asyncio.sleep(self.max_sleep_time)
+                elif self.max_action == "break":
+                    return True
+                else:
+                    raise ValueError("超出最大任务数")
+            else:
+                return
+
+    @staticmethod
+    async def get_result(task: Task, timeout=None, timeout_back=None):
+        res = await task.get_result(timeout=timeout, timeout_back=timeout_back)
+        return res
+
+    async def get_result_by_id(self, task_id, timeout=None, timeout_back=None):
+        result_queue = self.get_queue(task_id, base=self.result_queue)
+        pubsub = self.redis.pubsub()
+        await pubsub.subscribe(result_queue)
+
+        async def get_result():
+            async for msg in pubsub.listen():
+                if msg["type"] == "message":
+                    return msg["data"]
+
+        try:
+            done_future = asyncio.ensure_future(get_result())
+            try:
+                res = await asyncio.wait_for(done_future, timeout=timeout)
+                return res
+            except asyncio.TimeoutError:
+                if inspect.iscoroutinefunction(timeout_back):
+                    await timeout_back()
+                elif inspect.isawaitable(timeout_back):
+                    await timeout_back
+                elif timeout_back:
+                    timeout_back()
+                raise
+            finally:
+                done_future.cancel()
+        finally:
+            await pubsub.unsubscribe(result_queue)
+            await pubsub.close()
+
+
+if __name__ == '__main__':
+    pass

ytools/arq/setting.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : setting.py
+@Author  : yintian
+@Date    : 2025/10/21 10:30
+@Software: PyCharm
+@Desc    : 
+"""
+# 默认队列名
+DEFAULT_QUEUE_NAME = "ytools:arq"
+# 是否启用加密, 若传入方法则使用传入方法
+ENCRYPT = False
+# 默认编码
+DEFAULT_ENCODING = "utf-8"
+# 轮询休眠
+INTERVAL = 0.01
+# 传入的是否是可序列化数据
+OBJ_DATA = False
+# key 超时删除时间
+EXPIRE_TIME = 300
+
+if __name__ == '__main__':
+    pass

ytools/arq/task/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Author  : yintian
+@Date    : 2025/10/21 10:35
+@Software: PyCharm
+@Desc    : 
+"""
+
+if __name__ == '__main__':
+    pass