mopr 0.0.10__py3-none-any.whl

mopr/__init__.py

@@ -0,0 +1,5 @@
+"""MOPR Python package entry point."""
+
+from ._version import __version__
+
+__all__ = ["__version__"]

mopr/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.10'
+__version_tuple__ = version_tuple = (0, 0, 10)
+
+__commit_id__ = commit_id = 'g10dd4e1d7'

mopr/distributed/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""分布式基础设施包 (Python 探索层, 与 C++/Go/Rust 语义对齐)."""

mopr/distributed/id/__init__.py

@@ -0,0 +1,48 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""分布式 ID (Python 探索层).
+
+与 Go/Rust 实现语义对齐: HLC 混合逻辑时钟 + mmap 双槽 checkpoint 持久化
++ 锁字跨进程互斥 + 有界 MPMC 队列.
+"""
+
+from .generator import CanceledError, Generator
+from .hlc import HLC
+from .id import EPOCH_MS, PAYLOAD_BITS, PHYSICAL_BITS, ID, check_epoch
+from .option import (
+    with_clock,
+    with_node_count,
+    with_seq_bits,
+    with_seq_seed,
+    with_state_file,
+    with_state_strict,
+    with_state_sync_every,
+)
+from .queue import Queue, QueueClosedError, QueueEmptyError, QueueFullError
+from .state_store import FileStateStore, PersistentState, StateFileError
+
+__all__ = [
+    "CanceledError",
+    "EPOCH_MS",
+    "FileStateStore",
+    "Generator",
+    "HLC",
+    "ID",
+    "PAYLOAD_BITS",
+    "PHYSICAL_BITS",
+    "PersistentState",
+    "Queue",
+    "QueueClosedError",
+    "QueueEmptyError",
+    "QueueFullError",
+    "StateFileError",
+    "check_epoch",
+    "with_clock",
+    "with_node_count",
+    "with_seq_bits",
+    "with_seq_seed",
+    "with_state_file",
+    "with_state_strict",
+    "with_state_sync_every",
+]

mopr/distributed/id/generator.py

@@ -0,0 +1,74 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""分布式 ID 生成器 (与 Go/Rust 语义对齐).
+
+Next() 组合 HLC 水位: (physical - epoch) << 22 | node_id << seq_bits | seq.
+Serve() 持续生产并入队, 响应 cancel 事件与队列关闭 (对齐 Rust 修复后的语义).
+"""
+
+import threading
+import time
+
+from .id import EPOCH_MS, PAYLOAD_BITS, ID, check_epoch
+from .queue import Queue, QueueClosedError, QueueFullError
+
+
+class CanceledError(Exception):
+    """Serve 被取消 (对应 Go 的 context.Canceled)."""
+
+
+class Generator:
+    """按 node_id + HLC 生成全局唯一 ID."""
+
+    def __init__(self, node_id: int, hlc):
+        if hlc is None:
+            raise ValueError("hlc 不能为 None")
+        self.hlc = hlc
+        self.seq_bits = hlc.seq_bits
+        self.worker_bits = PAYLOAD_BITS - self.seq_bits
+        if node_id < 0 or node_id >= (1 << self.worker_bits):
+            raise ValueError(
+                f"node_id {node_id} 超出 {self.worker_bits} 位 worker 范围 "
+                f"({1 << self.worker_bits} 个节点)"
+            )
+        self.node_id = node_id
+
+    def next(self) -> ID:
+        """生成下一个 ID (对齐 Go 的 Next)."""
+        physical, sequence = self.hlc.now()
+        elapsed = check_epoch(physical - EPOCH_MS)
+        return ID(
+            (elapsed << PAYLOAD_BITS)
+            | (self.node_id << self.seq_bits)
+            | (sequence & ((1 << self.seq_bits) - 1))
+        )
+
+    def serve(self, queue: Queue, cancel: threading.Event = None) -> None:
+        """持续生产并入队, 直到取消或队列关闭 (对齐 Go 的 Serve).
+
+        cancel 为 threading.Event, set 后返回并抛 CanceledError.
+        队列关闭时正常返回 None; 队列满时重试同一 ID, 每轮检查取消/关闭,
+        避免阻塞在满队列上无法响应取消.
+        """
+        if queue is None:
+            raise ValueError("queue 不能为 None")
+        while True:
+            if cancel is not None and cancel.is_set():
+                raise CanceledError()
+            if queue.is_closed():
+                return
+            identifier = self.next()
+            # 队列满时重试同一个 ID, 每轮回到循环顶部检查取消/关闭
+            while True:
+                if cancel is not None and cancel.is_set():
+                    raise CanceledError()
+                if queue.is_closed():
+                    return
+                try:
+                    queue.try_push(identifier)
+                    break
+                except QueueFullError:
+                    time.sleep(0)  # 让出 GIL, 对齐 Rust 的 yield_now
+                except QueueClosedError:
+                    return

mopr/distributed/id/hlc.py

@@ -0,0 +1,109 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""混合逻辑时钟 HLC (与 Go 的 hlc.go / Rust 的 hlc.rs 语义对齐).
+
+约定:
+    now()      推进并返回 (physical_ms, seq)
+    timestamp() 返回 physical_ms (不推进)
+    close()    刷新状态文件 (幂等)
+
+物理时钟回拨时通过递增 seq 保证单调; seq 溢出时物理时钟 +1 并重置 seq.
+"""
+
+import os
+import struct
+import threading
+import time
+
+from .option import default_sync_every_value
+from .state_store import FileStateStore, PersistentState
+
+
+def random_uint16() -> int:
+    """进程级随机种子 (对齐 Go 的 randUint16: crypto/rand, 失败时退化)."""
+    try:
+        return struct.unpack(">H", os.urandom(2))[0]
+    except NotImplementedError:
+        return (int(time.time_ns()) ^ (os.getpid() << 16)) & 0xFFFF
+
+
+class HLC:
+    """混合逻辑时钟: 41 位物理毫秒 + 22 位 payload (worker + seq)."""
+
+    def __init__(self, *opts):
+        self._lock = threading.Lock()
+        self._clock = lambda: int(time.time() * 1000)
+        self.physical = 0
+        self.seq = 0
+        self.seed = random_uint16()
+        self.seq_bits = 11  # payloadBits(22) - 默认 2^11 节点
+        self.sync_every = default_sync_every_value()
+        self.strict = False
+        self.state_file = None
+        self.store = None
+
+        for opt in opts:
+            opt(self)
+
+        if self.state_file is not None:
+            self.store = FileStateStore(self.state_file, self.sync_every, self.strict)
+
+        restored = None
+        ok = False
+        if self.store is not None:
+            restored, ok = self.store.load()
+        if ok:
+            self.physical = restored.physical
+            self.seq = restored.seq
+        else:
+            self.physical = self._clock()
+            self.seq = self.seed & self.seq_mask
+
+    @property
+    def seq_mask(self) -> int:
+        return (1 << self.seq_bits) - 1
+
+    def now(self):
+        """推进并返回 (physical, seq), 对齐 Go 的 Now()."""
+        with self._lock:
+            now = self._clock()
+            if self.store is not None:
+                next_state = self.store.next(
+                    PersistentState(self.physical, self.seq), now, self.seq_bits
+                )
+                self.physical = next_state.physical
+                self.seq = next_state.seq
+                return self.physical, self.seq
+
+            if now > self.physical:
+                self.physical = now
+                self.seq = 0
+            elif self.seq >= self.seq_mask:
+                self.physical += 1
+                self.seq = 0
+            else:
+                self.seq += 1
+            return self.physical, self.seq
+
+    def timestamp(self) -> int:
+        """返回物理毫秒, 对齐 Go 的 Timestamp()."""
+        return self.physical
+
+    def close(self) -> None:
+        """刷新状态文件并释放映射 (幂等, 对齐 Go 的 Close())."""
+        with self._lock:
+            if self.store is None:
+                return
+            first_err = None
+            try:
+                self.store.flush()
+            except Exception as e:
+                first_err = e
+            try:
+                self.store.close()
+            except Exception as e:
+                if first_err is None:
+                    first_err = e
+            if first_err is not None:
+                raise first_err

mopr/distributed/id/id.py

@@ -0,0 +1,83 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""分布式 ID 类型与位布局常量 (与 Go/Rust 语义对齐).
+
+位布局 (64 位, 大端):
+    | 41 位 physical(ms) | 22 位 payload |
+    payload 又分: | worker_bits 位 node_id | seq_bits 位 seq |
+"""
+
+import base64
+
+# 纪元起点: 2026-01-01T00:00:00Z (毫秒), 与 Go/Rust 的 EpochMs 一致
+EPOCH_MS = 1767225600000
+# 22 位 payload: 2^22 = 4194304, 单进程/线程每毫秒容量
+PAYLOAD_BITS = 22
+# 41 位 physical: 2^41 ms ≈ 69.7 年 (自纪元起)
+PHYSICAL_BITS = 41
+# worker 位数上限 (seq 至少保留 4 位)
+MAX_WORKER_BITS = PAYLOAD_BITS - 4
+
+
+class ID(int):
+    """64 位分布式 ID, 语义对齐 Go 的 ID (uint64 包装类型).
+
+    Python int 无符号溢出限制, 但本布局最大 63 位 (physical 41 + payload 22),
+    与 Go 的 uint64 表示一一对应.
+    """
+
+    def __new__(cls, value: int) -> "ID":
+        if value < 0 or value >= (1 << 64):
+            raise ValueError(f"ID 超出 uint64 范围: {value}")
+        return super().__new__(cls, value)
+
+    @classmethod
+    def from_int(cls, value: int) -> "ID":
+        return cls(value)
+
+    def to_int(self) -> int:
+        return int(self)
+
+    def physical(self) -> int:
+        """自纪元起的毫秒数 (高 41 位)."""
+        return int(self) >> PAYLOAD_BITS
+
+    def node_id(self, worker_bits: int) -> int:
+        """节点号 (payload 高 worker_bits 位)."""
+        if worker_bits < 0 or worker_bits > PAYLOAD_BITS:
+            raise ValueError(f"worker_bits 超出范围: {worker_bits}")
+        return (int(self) >> (PAYLOAD_BITS - worker_bits)) & ((1 << worker_bits) - 1)
+
+    def seq(self, worker_bits: int) -> int:
+        """序列号 (payload 低 seq_bits 位)."""
+        if worker_bits < 0 or worker_bits > PAYLOAD_BITS:
+            raise ValueError(f"worker_bits 超出范围: {worker_bits}")
+        return int(self) & ((1 << (PAYLOAD_BITS - worker_bits)) - 1)
+
+    def to_bytes(self) -> bytes:
+        """大端 8 字节编码, 对齐 Go 的 Bytes()."""
+        return int(self).to_bytes(8, byteorder="big")
+
+    def string(self) -> str:
+        """URL-safe Base64 无填充 11 字符表示, 对齐 Go 的 String().
+
+        RawURLEncoding: 无 padding, 字符集 [-_0-9A-Za-z].
+        """
+        return base64.urlsafe_b64encode(self.to_bytes()).rstrip(b"=").decode("ascii")
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> "ID":
+        """解码大端 8 字节, 对齐 Go 的 FromBytes."""
+        if len(data) != 8:
+            raise ValueError(f"ID 字节长度必须为 8, 实际 {len(data)}")
+        return cls(int.from_bytes(data, byteorder="big"))
+
+
+def check_epoch(elapsed: int) -> int:
+    """校验自纪元起的毫秒数落在 physical 41 位可表示范围内, 对齐 Go 的 checkEpoch."""
+    if elapsed < 0 or elapsed >= (1 << PHYSICAL_BITS):
+        raise ValueError(
+            f"自纪元起毫秒数超出 {PHYSICAL_BITS} 位可表示范围: {elapsed}"
+        )
+    return elapsed

mopr/distributed/id/option.py

@@ -0,0 +1,80 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""HLC 构造选项 (与 Go 的 option.go / Rust 的 option 语义对齐).
+
+每个 with_* 返回一个可调用对象, 在 HLC 构造时按其语义应用.
+"""
+
+import os
+
+from .id import PAYLOAD_BITS
+
+# 默认每 N 次 Now 落盘一次 checkpoint
+DEFAULT_SYNC_EVERY = 1000
+# seq 保留位数下限 (worker 位上限 18)
+MIN_SEQ_BITS = 4
+
+
+def default_sync_every_value() -> int:
+    """读取环境变量 QUANT1X_ID64_SYNC_EVERY, 非法/缺失时回退默认值, 对齐 Go."""
+    raw = os.environ.get("QUANT1X_ID64_SYNC_EVERY")
+    if raw is not None:
+        try:
+            value = int(raw)
+            if value > 0:
+                return value
+        except ValueError:
+            pass
+    return DEFAULT_SYNC_EVERY
+
+
+def with_clock(now):
+    """自定义毫秒时钟函数 (默认 time.time() * 1000), 对齐 Go 的 WithClock."""
+    return lambda hlc: setattr(hlc, "_clock", now)
+
+
+def with_seq_seed(seed: int) -> object:
+    """初始化 seq 的种子值, 对齐 Go 的 WithSeqSeed."""
+    if seed < 0 or seed > 0xFFFF:
+        raise ValueError(f"seq seed 必须为 uint16, 实际 {seed}")
+    return lambda hlc: setattr(hlc, "seed", seed)
+
+
+def with_state_file(path) -> object:
+    """启用跨进程持久化状态文件, 对齐 Go 的 WithStateFile."""
+    if not path:
+        raise ValueError("state 文件路径不能为空")
+    return lambda hlc: setattr(hlc, "state_file", path)
+
+
+def with_state_sync_every(every: int) -> object:
+    """设置 checkpoint 落盘间隔, 对齐 Go 的 WithStateSyncEvery."""
+    if every <= 0:
+        raise ValueError(f"sync_every 必须为正数, 实际 {every}")
+    return lambda hlc: setattr(hlc, "sync_every", every)
+
+
+def with_state_strict() -> object:
+    """严格模式: 每次 Now 先抢跨进程锁再 checkpooint, 对齐 Go 的 WithStateStrict."""
+    return lambda hlc: setattr(hlc, "strict", True)
+
+
+def with_node_count(count: int) -> object:
+    """按节点数推导 seq 位数, 对齐 Go 的 WithNodeCount.
+
+    seq_bits = 22 - ceil(log2(count)); count < 2^18 时 seq_bits >= 4.
+    """
+    if count < 1:
+        count = 1
+    seq_bits = PAYLOAD_BITS - count.bit_length()
+    if seq_bits < MIN_SEQ_BITS:
+        raise ValueError(f"节点数 {count} 过大: 需至少保留 {MIN_SEQ_BITS} 位序列号")
+    return lambda hlc: setattr(hlc, "seq_bits", seq_bits)
+
+
+def with_seq_bits(seq_bits: int) -> object:
+    """直接指定 seq 位数 (4..21), 对齐 Go 的 WithSeqBits."""
+    if seq_bits < MIN_SEQ_BITS or seq_bits > PAYLOAD_BITS - 1:
+        raise ValueError(f"seq_bits 必须在 {MIN_SEQ_BITS}..{PAYLOAD_BITS - 1}, 实际 {seq_bits}")
+    return lambda hlc: setattr(hlc, "seq_bits", seq_bits)

mopr/distributed/id/queue.py

@@ -0,0 +1,125 @@
+# -*- coding: UTF-8 -*-
+# Copyright (c) Quant1X <wangfengxy@sina.cn>.
+# Licensed under the MIT License.
+"""有界多生产者多消费者队列 (对齐 Go runtime.RingBuffer 的语义).
+
+API 语义:
+    try_push   非阻塞, 满 -> QueueFullError, 已关闭 -> QueueClosedError
+    try_pop    非阻塞, 空 -> QueueEmptyError, 已关闭且空 -> QueueClosedError
+    push       阻塞直到有空位, 关闭后 -> QueueClosedError
+    pop        阻塞直到有数据, 已关闭且空 -> QueueClosedError
+    wait_for_close  等待所有已入队数据被消费完毕
+
+容量向上取整为 2 的幂 (与 Go/Rust 一致), 幂等 close.
+Python 探索层使用标准库线程原语实现同等语义 (无锁环缓冲为生产层实现).
+"""
+
+import collections
+import threading
+
+
+class QueueError(Exception):
+    """队列错误基类."""
+
+
+class QueueFullError(QueueError):
+    """队列已满 (对应 Go 的 ErrQueueFull)."""
+
+
+class QueueEmptyError(QueueError):
+    """队列已空 (对应 Go 的 ErrQueueEmpty)."""
+
+
+class QueueClosedError(QueueError):
+    """队列已关闭 (对应 Go 的 ErrClosed)."""
+
+
+class Queue:
+    """有界 MPMC 队列, 容量向上取整为 2 的幂."""
+
+    def __init__(self, capacity: int):
+        if capacity < 1:
+            raise ValueError(f"capacity 必须为正数, 实际 {capacity}")
+        cap = 1
+        while cap < capacity:
+            cap <<= 1
+        self._cap = cap
+        self._deque = collections.deque()
+        self._lock = threading.Lock()
+        self._not_full = threading.Condition(self._lock)
+        self._not_empty = threading.Condition(self._lock)
+        self._closed = False
+
+    def try_push(self, value) -> None:
+        """非阻塞入队, 满 -> QueueFullError, 已关闭 -> QueueClosedError."""
+        with self._lock:
+            if self._closed:
+                raise QueueClosedError()
+            if len(self._deque) >= self._cap:
+                raise QueueFullError()
+            self._deque.append(value)
+            self._not_empty.notify()
+
+    def try_pop(self):
+        """非阻塞出队, 空 -> QueueEmptyError, 已关闭且空 -> QueueClosedError."""
+        with self._lock:
+            if self._deque:
+                value = self._deque.popleft()
+                self._not_full.notify()
+                return value
+            if self._closed:
+                raise QueueClosedError()
+            raise QueueEmptyError()
+
+    def push(self, value) -> None:
+        """阻塞入队直到有空位; 队列关闭后 -> QueueClosedError."""
+        with self._not_full:
+            while len(self._deque) >= self._cap:
+                self._not_full.wait()
+            if self._closed:
+                raise QueueClosedError()
+            self._deque.append(value)
+            self._not_empty.notify()
+
+    def pop(self):
+        """阻塞出队直到有数据; 已关闭且空 -> QueueClosedError."""
+        with self._not_empty:
+            while not self._deque and not self._closed:
+                self._not_empty.wait()
+            if self._deque:
+                value = self._deque.popleft()
+                self._not_full.notify()
+                return value
+            raise QueueClosedError()
+
+    def len(self) -> int:
+        with self._lock:
+            return len(self._deque)
+
+    def cap(self) -> int:
+        return self._cap
+
+    def is_empty(self) -> bool:
+        return self.len() == 0
+
+    def is_full(self) -> bool:
+        return self.len() == self._cap
+
+    def close(self) -> None:
+        """关闭队列: 唤醒所有等待者, 幂等 (对齐 Go 的 Close)."""
+        with self._lock:
+            if self._closed:
+                return
+            self._closed = True
+            self._not_full.notify_all()
+            self._not_empty.notify_all()
+
+    def is_closed(self) -> bool:
+        with self._lock:
+            return self._closed
+
+    def wait_for_close(self) -> None:
+        """阻塞直到所有已入队数据被消费 (对齐 Go 的 WaitForClose)."""
+        with self._not_empty:
+            while self._deque:
+                self._not_empty.wait()