onestep 0.5.0__py3-none-any.whl

onestep/cron.py

@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+"""
+Cron DSL/Builder 与宏解析
+
+提供人性化的 API 来生成 cron 表达式，并支持常用别名/宏：
+- Cron.every(minutes=5) / Cron.every(seconds=10) / Cron.every(hours=2) / Cron.every(days=3) / Cron.every(months=1)
+- Cron.daily(at="09:00")
+- Cron.weekly(on="mon" or ["mon","fri"], at="10:30")
+- Cron.monthly(on_day=1, at="00:00")
+- Cron.yearly(on="01-01", at="00:00")
+- 宏别名：@hourly/@daily/@weekly/@monthly/@yearly
+- 扩展宏：@workdays（1-5）/@weekends（0,6）/@every <n><unit>（unit: s/m/h/d/mo）
+
+注意：
+- 标准 5 字段：minute hour day month day_of_week
+- 当包含 seconds（如 `at` 使用 HH:MM:SS 或 `every(seconds=...)`）时输出 6 字段：second minute hour day month day_of_week
+- `@every <n>d` 等按“日/月字段取模”的语义生成表达式，并非滚动间隔。
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Tuple, Union
+
+DOW_MAP = {
+    "sun": 0, "mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6,
+}
+
+ALIAS_MAP = {
+    "@hourly": "0 * * * *",
+    "@daily": "0 0 * * *",
+    "@weekly": "0 0 * * 0",
+    "@monthly": "0 0 1 * *",
+    "@yearly": "0 0 1 1 *",
+}
+
+
+def _parse_at(at: Optional[str]) -> Tuple[Optional[int], int, int]:
+    """解析 at 参数，支持 HH:MM 或 HH:MM:SS。
+    返回 (second, minute, hour)。如果不传 at，默认 minute=0, hour=0。
+    """
+    if not at:
+        return None, 0, 0
+    parts = at.strip().split(":")
+    if len(parts) == 2:
+        h, m = parts
+        return None, int(m), int(h)
+    elif len(parts) == 3:
+        h, m, s = parts
+        return int(s), int(m), int(h)
+    raise ValueError(f"Invalid time format for at='{at}', expected HH:MM or HH:MM:SS")
+
+
+def _to_dow_expr(on: Union[str, int, List[Union[str, int]]]) -> str:
+    """将星期输入转换为 day_of_week 字段表达式。
+    支持：字符串缩写（mon...sun）、数字 0-6、列表。
+    """
+    def _to_num(v: Union[str, int]) -> int:
+        if isinstance(v, int):
+            if v < 0 or v > 6:
+                raise ValueError("day_of_week must be in 0-6")
+            return v
+        v = str(v).lower()
+        if v.isdigit():
+            num = int(v)
+            if num < 0 or num > 6:
+                raise ValueError("day_of_week must be in 0-6")
+            return num
+        if v not in DOW_MAP:
+            raise ValueError(f"Unknown day_of_week '{v}'")
+        return DOW_MAP[v]
+
+    if isinstance(on, (list, tuple, set)):
+        vals = sorted({_to_num(v) for v in on})
+        return ",".join(str(x) for x in vals)
+    else:
+        return str(_to_num(on))
+
+
+def _join_fields(second: Optional[int], minute: Union[int, str], hour: Union[int, str], day: Union[int, str], month: Union[int, str], dow: Union[int, str]) -> str:
+    """组合 5/6 字段。若 second 为 None，输出 5 字段，否则输出 6 字段（秒在最后）。"""
+    if second is None:
+        return f"{minute} {hour} {day} {month} {dow}"
+    return f"{minute} {hour} {day} {month} {dow} {second}"
+
+
+class Cron:
+    @staticmethod
+    def every(*, seconds: Optional[int] = None, minutes: Optional[int] = None, hours: Optional[int] = None, days: Optional[int] = None, months: Optional[int] = None) -> str:
+        """每隔指定单位执行。
+        只能指定一个非 None 的单位。
+        - seconds: 6 字段：*/s * * * * *
+        - minutes: 5 字段：*/m * * * *
+        - hours:   5 字段：0 */h * * *
+        - days:    5 字段：0 0 */d * *
+        - months:  5 字段：0 0 1 */mo *
+        """
+        units = [("seconds", seconds), ("minutes", minutes), ("hours", hours), ("days", days), ("months", months)]
+        chosen = [(k, v) for k, v in units if v is not None]
+        if len(chosen) != 1:
+            raise ValueError("Cron.every() must specify exactly one unit")
+        k, v = chosen[0]
+        if not isinstance(v, int) or v <= 0:
+            raise ValueError("Interval must be a positive integer")
+        if k == "seconds":
+            return f"* * * * * */{v}"
+        if k == "minutes":
+            return "*/%d * * * *" % v
+        if k == "hours":
+            return "0 */%d * * *" % v
+        if k == "days":
+            return "0 0 */%d * *" % v
+        if k == "months":
+            return "0 0 1 */%d *" % v
+        raise RuntimeError("unreachable")
+
+    @staticmethod
+    def daily(*, at: Optional[str] = None) -> str:
+        s, m, h = _parse_at(at)
+        day, month, dow = "*", "*", "*"
+        return _join_fields(s, m, h, day, month, dow)
+
+    @staticmethod
+    def weekly(*, on: Union[str, int, List[Union[str, int]]] = "mon", at: Optional[str] = None) -> str:
+        s, m, h = _parse_at(at)
+        dow = _to_dow_expr(on)
+        day, month = "*", "*"
+        return _join_fields(s, m, h, day, month, dow)
+
+    @staticmethod
+    def monthly(*, on_day: Union[int, List[int]] = 1, at: Optional[str] = None) -> str:
+        s, m, h = _parse_at(at)
+        month, dow = "*", "*"
+        def _to_day(v: Union[int, str]) -> int:
+            iv = int(v)
+            if iv < 1 or iv > 31:
+                raise ValueError("day_of_month must be in 1-31")
+            return iv
+        if isinstance(on_day, (list, tuple, set)):
+            day = ",".join(str(_to_day(v)) for v in sorted({int(x) for x in on_day}))
+        else:
+            day = str(_to_day(on_day))
+        return _join_fields(s, m, h, day, month, dow)
+
+    @staticmethod
+    def yearly(*, on: str = "01-01", at: Optional[str] = None) -> str:
+        s, m, h = _parse_at(at)
+        dow = "*"
+        try:
+            mon_str, day_str = on.split("-")
+            month = str(int(mon_str))
+            day = str(int(day_str))
+        except Exception:
+            raise ValueError("yearly(on=...) expects 'MM-DD', e.g., '01-01'")
+        return _join_fields(s, m, h, day, month, dow)
+
+    @staticmethod
+    def alias(name: str) -> str:
+        """返回内置宏别名对应的表达式（如果已知）。未知别名原样返回，以便 croniter 自行处理。"""
+        return ALIAS_MAP.get(name.strip().lower(), name)
+
+
+def resolve_cron(cron_like: str) -> str:
+    """将类似 cron 的输入（字符串/宏）解析为标准 cron 表达式。
+    - 内置关键字直接映射或保留（croniter 也支持）；
+    - 扩展宏：@workdays/@weekends/@every <n><unit>（unit: s/m/h/d/mo）。
+    """
+    s = str(cron_like).strip()
+    if not s.startswith("@"):
+        return s
+    lower = s.lower()
+    # 内置别名
+    if lower in ALIAS_MAP:
+        return ALIAS_MAP[lower]
+    # 扩展：工作日/周末（默认 00:00 执行）
+    if lower == "@workdays":
+        return "0 0 * * 1-5"
+    if lower == "@weekends":
+        return "0 0 * * 0,6"
+    # 扩展：@every <n><unit>
+    if lower.startswith("@every"):
+        tail = lower.replace("@every", "", 1).strip()
+        if not tail:
+            raise ValueError("@every requires an interval, e.g., '@every 5m'")
+        # 支持类似 "5m" / "10 h" / "2d" / "3mo" / "30s"
+        tail = tail.replace(" ", "")
+        # 处理 'mo' 与单位优先级
+        if tail.endswith("mo"):
+            num = int(tail[:-2])
+            if num <= 0:
+                raise ValueError("@every months must be > 0")
+            return "0 0 1 */%d *" % num
+        unit = tail[-1]
+        try:
+            num = int(tail[:-1])
+        except Exception:
+            raise ValueError("@every expects '<n><unit>', units: s/m/h/d/mo")
+        if num <= 0:
+            raise ValueError("@every interval must be > 0")
+        if unit == "s":
+            return "* * * * * */%d" % num
+        if unit == "m":
+            return "*/%d * * * *" % num
+        if unit == "h":
+            return "0 */%d * * *" % num
+        if unit == "d":
+            return "0 0 */%d * *" % num
+        if unit == "w":
+            # Cron 不支持“每 N 周”，仅建议直接使用 weekly(on=..., at=...)
+            raise ValueError("@every <n>w is not supported; use weekly(on=..., at=...) instead")
+        raise ValueError("Unknown unit for @every; use s/m/h/d/mo")
+    # 未知宏直接返回，交由 croniter 处理（可能抛错）
+    return s

onestep/exception.py

@@ -0,0 +1,35 @@
+class StopMiddleware(Exception):
+    ...
+
+
+class RetryException(Exception):
+    def __init__(self, message=None, times=None, **kwargs):
+        self.message = message
+        self.times = times
+        self.kwargs = kwargs
+
+
+class RetryInQueue(RetryException):
+    """消息重试-通过重试队列
+
+    抛出此异常，消息将被重新放入队列，等待下次消费。
+    """
+
+
+class RetryInLocal(RetryException):
+    """消息重试-本地
+
+    不经过队列，直接在本地重试，直到达到重试次数。
+    """
+
+
+class DropMessage(Exception):
+    """从 Brokers 中 丢弃该消息"""
+
+
+class RejectMessage(DropMessage):
+    """从 Brokers 中 丢弃该消息"""
+
+
+class RequeueMessage(Exception):
+    """重新入队该消息"""

onestep/message.py

@@ -0,0 +1,169 @@
+import json
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+from traceback import TracebackException
+from typing import Optional, Any, Union, Type
+from types import TracebackType
+
+from onestep._utils import catch_error
+
+
+class MessageTracebackException(TracebackException):
+    def __init__(self, exc_type: Type[BaseException], exc_value: BaseException, exc_traceback: Optional[TracebackType], **kwargs):
+        super().__init__(exc_type, exc_value, exc_traceback, **kwargs)
+        self.exc_value = exc_value
+
+
+@dataclass
+class Extra:
+    task_id: Optional[str] = None
+    publish_time: Optional[float] = None
+    failure_count: int = 0
+
+    def __post_init__(self):
+        self.task_id = self.task_id or str(uuid.uuid4())
+        self.publish_time = self.publish_time or round(time.time(), 3)
+
+    def to_dict(self):
+        return {
+            'task_id': self.task_id,
+            'publish_time': self.publish_time,
+            'failure_count': self.failure_count,
+        }
+
+    def __str__(self):
+        return str(self.to_dict())
+
+
+class Message:
+
+    def __init__(
+            self,
+            body: Optional[Union[dict, Any]] = None,
+            extra: Optional[Union[dict, Extra]] = None,
+            message: Optional[Any] = None,
+            broker=None
+    ):
+        """ Message
+
+        :param body: 解析后的消息体
+        :param extra: 额外信息
+        :param message: 原始消息体
+        :param broker: 当前消息所属的 broker
+        """
+        self.body = body
+        self.extra = self._set_extra(extra)
+        self.message = message
+
+        self.broker = broker
+        self._exception = None
+
+    @staticmethod
+    def _set_extra(extra):
+        if isinstance(extra, Extra):
+            return extra
+        elif isinstance(extra, dict):
+            return Extra(**extra)
+        else:
+            return Extra()
+
+    def set_exception(self):
+        """设置异常信息，会自动获取"""
+        exc_type, exc_value, exc_tb = sys.exc_info()
+        if exc_type is None or exc_value is None:
+            exc_type = Exception
+            exc_value = Exception("No exception info")
+        self._exception = MessageTracebackException(exc_type, exc_value, exc_tb)
+        self.failure_count = self.failure_count + 1
+
+    @property
+    def exception(self) -> Optional[MessageTracebackException]:
+        return self._exception
+
+    @property
+    def fail(self):
+        return self.exception is not None
+
+    @property
+    def failure_count(self):
+        return self.extra.failure_count
+
+    @failure_count.setter
+    def failure_count(self, value):
+        self.extra.failure_count = value
+
+    def replace(self, **kwargs):
+        """替换当前message的属性"""
+        for key, value in kwargs.items():
+            if not hasattr(self, key):
+                continue
+            if key == 'extra':
+                value = self._set_extra(value)
+            setattr(self, key, value)
+        return self
+
+    def to_dict(self, include_exception=False) -> dict:
+        data = {'body': self.body, 'extra': self.extra.to_dict()}
+        if include_exception and self.exception:
+            data['exception'] = "".join(self.exception.format(chain=True))  # noqa
+
+        return data
+
+    def to_json(self, include_exception=False) -> str:
+        return json.dumps(self.to_dict(include_exception))
+
+    @catch_error()
+    def confirm(self, **kwargs):
+        """确认消息"""
+        broker = getattr(self, 'broker', None)
+        if broker and hasattr(broker, 'confirm'):
+            if hasattr(broker, 'before_emit'):
+                broker.before_emit('confirm', message=self, step=kwargs.get('step'))
+            broker.confirm(self)
+            if hasattr(broker, 'after_emit'):
+                broker.after_emit('confirm', message=self, step=kwargs.get('step'))
+
+    @catch_error()
+    def reject(self, **kwargs):
+        """拒绝消息"""
+        broker = getattr(self, 'broker', None)
+        if broker and hasattr(broker, 'reject'):
+            if hasattr(broker, 'before_emit'):
+                broker.before_emit('reject', message=self, step=kwargs.get('step'))
+            broker.reject(self)
+            if hasattr(broker, 'after_emit'):
+                broker.after_emit('reject', message=self, step=kwargs.get('step'))
+
+    @catch_error()
+    def requeue(self, is_source=False, **kwargs):
+        """
+        重发消息：先拒绝 再 重入
+        
+        :param is_source: 是否是源消息，True: 使用消息的最新数据重入当前队列，False: 使用消息的最新数据重入当前队列
+        """
+        broker = getattr(self, 'broker', None)
+        if broker and hasattr(broker, 'requeue'):
+            if hasattr(broker, 'before_emit'):
+                broker.before_emit('requeue', message=self, step=kwargs.get('step'))
+            broker.requeue(self, is_source=is_source)
+            if hasattr(broker, 'after_emit'):
+                broker.after_emit('requeue', message=self, step=kwargs.get('step'))
+
+    def __getattr__(self, item):
+        return None
+
+    def __delattr__(self, item):
+        if hasattr(self, item):
+            setattr(self, item, None)
+
+    def __str__(self):
+        return str(self.to_dict())
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__} {self.body}>"
+
+    @classmethod
+    def from_broker(cls, broker_message: Any):
+        return cls(body=broker_message, extra=None, message=broker_message, broker=None)

onestep/middleware/__init__.py

@@ -0,0 +1,7 @@
+from .base import BaseMiddleware
+from .config import (
+    BaseConfigMiddleware, PublishConfigMixin, ConsumeConfigMixin,
+    NacosConfigMixin, NacosPublishConfigMiddleware, NacosConsumeConfigMiddleware,
+    RedisConfigMixin, RedisPublishConfigMiddleware, RedisConsumeConfigMiddleware
+)
+from .unique import UniqueMiddleware, MemoryUniqueMiddleware

onestep/middleware/base.py

@@ -0,0 +1,32 @@
+class DeprecationMeta(type):
+
+    def __new__(cls, name, bases, attrs):
+        if "before_receive" in attrs or "after_receive" in attrs:
+            raise DeprecationWarning(
+                f"`{name}` "
+                "The before_receive and after_receive methods are deprecated, "
+                "please use before_consume and after_consume instead"
+            )
+        return super().__new__(cls, name, bases, attrs)
+
+
+class BaseMiddleware(metaclass=DeprecationMeta):
+
+    def before_send(self, step, message, *args, **kwargs):
+        """消息发送之前"""
+        pass
+
+    def after_send(self, step, message, *args, **kwargs):
+        """消息发送之后"""
+        pass
+
+    def before_consume(self, step, message, *args, **kwargs):
+        """消费消息之前"""
+        pass
+
+    def after_consume(self, step, message, *args, **kwargs):
+        """消费消息之后"""
+        pass
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__}>"

onestep/middleware/config.py

@@ -0,0 +1,77 @@
+from .base import BaseMiddleware
+
+
+class BaseConfigMiddleware(BaseMiddleware):
+
+    def __init__(self, client, *args, **kwargs):
+        self.config_key = kwargs.pop('config_key', None)
+        self.client = client
+
+    def process(self, step, message, *args, **kwargs):
+        """实际获取配置的逻辑"""
+        config = self.get(self.config_key)
+        return config
+
+    def get(self, key):
+        raise NotImplementedError
+
+
+class PublishConfigMixin:
+    def before_send(self, step, message, *args, **kwargs):
+        """消息发送之前，给消息添加配置"""
+        config = self.process(step, message, *args, **kwargs)  # noqa
+        # 持久性，会跟随消息传递到其他 broker
+        message.extra['config'] = config
+
+
+class ConsumeConfigMixin:
+    def before_consume(self, step, message, *args, **kwargs):
+        """消息消费之前，给消息附加配置"""
+        config = self.process(step, message, *args, **kwargs)  # noqa
+        # 一次性，不跟随消息传递到其他 broker
+        message.config = config
+
+
+class NacosConfigMixin:
+
+    def __init__(self, client, *args, **kwargs):
+        super().__init__(client, *args, **kwargs)
+        self.config_group = kwargs.pop("config_group", "DEFAULT_GROUP")
+
+    def get(self, key):
+        pass
+
+    def set(self, key, value):
+        pass
+
+
+class NacosPublishConfigMiddleware(NacosConfigMixin, PublishConfigMixin, BaseConfigMiddleware):
+    """发布前附加来自 Nacos 的配置"""
+
+
+class NacosConsumeConfigMiddleware(NacosConfigMixin, ConsumeConfigMixin, BaseConfigMiddleware):
+    """消费前附加来自 Nacos 的配置"""
+
+
+class RedisConfigMixin:
+
+    def __init__(self, client, *args, **kwargs):
+        super().__init__(client, *args, **kwargs)
+        self.config_group = kwargs.pop('config_group', 'onestep:config')
+
+    def get(self, key):
+        value = self.client.hget(name=self.config_group, key=key)  # noqa
+        if isinstance(value, bytes):
+            value = value.decode()
+        return value
+
+    def set(self, key, value):
+        return self.client.hset(name=self.config_group, key=key, value=value)  # noqa
+
+
+class RedisPublishConfigMiddleware(RedisConfigMixin, PublishConfigMixin, BaseConfigMiddleware):
+    """发布前附加来自 Redis 的配置"""
+
+
+class RedisConsumeConfigMiddleware(RedisConfigMixin, ConsumeConfigMixin, BaseConfigMiddleware):
+    """消费前附加来自 Redis 的配置"""

onestep/middleware/unique.py

@@ -0,0 +1,48 @@
+import hashlib
+import json
+
+from .base import BaseMiddleware
+from ..exception import DropMessage
+
+
+class hash_func:
+    """ Default message to hash function """
+
+    def __call__(self, body):
+        data = json.dumps(body) if isinstance(body, dict) else str(body)
+        return hashlib.sha1(data.encode("utf-8")).hexdigest()
+
+
+class UniqueMiddleware(BaseMiddleware):
+    default_hash_func = hash_func()
+
+    def before_consume(self, step, message, *args, **kwargs):
+        message_hash = self.default_hash_func(message.body)
+        if self.has_seen(message_hash):
+            raise DropMessage(f"Message<{message}> has been seen before")
+
+    def after_consume(self, step, message, *args, **kwargs):
+        if message.fail or message.body is None:
+            return
+        message_hash = self.default_hash_func(message.body)
+        self.mark_seen(message_hash)
+
+    def has_seen(self, message):
+        raise NotImplementedError
+
+    def mark_seen(self, message):
+        raise NotImplementedError
+
+
+class MemoryUniqueMiddleware(UniqueMiddleware):
+    """基于内存的去重中间件"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.seen = set()
+
+    def has_seen(self, hash_value):
+        return hash_value in self.seen
+
+    def mark_seen(self, hash_value):
+        self.seen.add(hash_value)