sol-parser-sdk-python 0.4.4__py3-none-any.whl

sol_parser/order_buffer.py

@@ -0,0 +1,171 @@
+"""OrderMode buffers for low-latency DEX subscriptions."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Tuple
+
+from .event_types import DexEvent
+from .grpc_types import ClientConfig, OrderMode
+
+
+@dataclass
+class _Batch:
+    slot: int
+    tx_index: int
+    seq: int
+    events: List[DexEvent]
+
+
+def _event_slot_tx(events: List[DexEvent], fallback_slot: int, fallback_tx_index: int) -> Tuple[int, int]:
+    if not events:
+        return fallback_slot, fallback_tx_index
+    meta = getattr(events[0].data, "metadata", None)
+    slot = int(getattr(meta, "slot", fallback_slot) or fallback_slot)
+    tx_index = int(getattr(meta, "tx_index", fallback_tx_index) or fallback_tx_index)
+    return slot, tx_index
+
+
+class OrderDispatcher:
+    def __init__(self, config: ClientConfig):
+        self.mode = config.order_mode
+        self.timeout_s = max(0.001, float(config.order_timeout_ms or 100) / 1000.0)
+        self.micro_batch_s = max(0.000001, float(config.micro_batch_us or 100) / 1_000_000.0)
+        self.slots: Dict[int, List[_Batch]] = {}
+        self.watermarks: Dict[int, int] = {}
+        self.micro_batch: List[_Batch] = []
+        self.micro_start = 0.0
+        self.last_flush = time.monotonic()
+        self.current_slot = 0
+        self.seq = 0
+
+    @property
+    def needs_timer(self) -> bool:
+        return self.mode != OrderMode.UNORDERED
+
+    @property
+    def interval_s(self) -> float:
+        if self.mode == OrderMode.MICRO_BATCH:
+            return self.micro_batch_s
+        return max(0.001, self.timeout_s / 2.0)
+
+    def push_transaction_events(
+        self,
+        events: List[DexEvent],
+        fallback_slot: int,
+        fallback_tx_index: int,
+        emit: Callable[[DexEvent], None],
+    ) -> None:
+        if not events:
+            return
+        slot, tx_index = _event_slot_tx(events, fallback_slot, fallback_tx_index)
+        batch = _Batch(slot=slot, tx_index=tx_index, seq=self.seq, events=events)
+        self.seq += 1
+
+        if self.mode == OrderMode.UNORDERED:
+            self._emit_batch(batch, emit)
+        elif self.mode == OrderMode.ORDERED:
+            self._push_ordered(batch, emit)
+        elif self.mode == OrderMode.STREAMING_ORDERED:
+            self._push_streaming(batch, emit)
+        elif self.mode == OrderMode.MICRO_BATCH:
+            self._push_micro_batch(batch, emit)
+        else:
+            self._emit_batch(batch, emit)
+
+    def flush_due(self, emit: Callable[[DexEvent], None]) -> None:
+        now = time.monotonic()
+        if self.mode in (OrderMode.ORDERED, OrderMode.STREAMING_ORDERED):
+            if self.slots and now - self.last_flush > self.timeout_s:
+                self._flush_all_slots(emit)
+        if self.mode == OrderMode.MICRO_BATCH:
+            if self.micro_batch and now - self.micro_start >= self.micro_batch_s:
+                self._flush_micro_batch(emit)
+
+    def flush_all(self, emit: Callable[[DexEvent], None]) -> None:
+        self._flush_all_slots(emit)
+        self._flush_micro_batch(emit)
+
+    def _push_ordered(self, batch: _Batch, emit: Callable[[DexEvent], None]) -> None:
+        if batch.slot > self.current_slot and self.current_slot > 0:
+            self._flush_before(batch.slot, emit)
+        if batch.slot > self.current_slot:
+            self.current_slot = batch.slot
+        self.slots.setdefault(batch.slot, []).append(batch)
+
+    def _push_streaming(self, batch: _Batch, emit: Callable[[DexEvent], None]) -> None:
+        if batch.slot > self.current_slot and self.current_slot > 0:
+            self._flush_before(batch.slot, emit)
+            for slot in list(self.watermarks):
+                if slot < batch.slot:
+                    self.watermarks.pop(slot, None)
+        if batch.slot > self.current_slot:
+            self.current_slot = batch.slot
+
+        expected = self.watermarks.get(batch.slot, 0)
+        if batch.tx_index == expected:
+            self._emit_batch(batch, emit)
+            watermark = expected + 1
+            buffered = self.slots.get(batch.slot, [])
+            buffered.sort(key=_batch_key)
+            while True:
+                pos = next((i for i, item in enumerate(buffered) if item.tx_index == watermark), -1)
+                if pos < 0:
+                    break
+                self._emit_batch(buffered.pop(pos), emit)
+                watermark += 1
+            if buffered:
+                self.slots[batch.slot] = buffered
+            else:
+                self.slots.pop(batch.slot, None)
+            self.watermarks[batch.slot] = watermark
+            self.last_flush = time.monotonic()
+        elif batch.tx_index > expected:
+            self.slots.setdefault(batch.slot, []).append(batch)
+
+    def _push_micro_batch(self, batch: _Batch, emit: Callable[[DexEvent], None]) -> None:
+        now = time.monotonic()
+        if not self.micro_batch:
+            self.micro_start = now
+        self.micro_batch.append(batch)
+        if now - self.micro_start >= self.micro_batch_s:
+            self._flush_micro_batch(emit)
+
+    def _flush_before(self, slot: int, emit: Callable[[DexEvent], None]) -> None:
+        for s in sorted(k for k in self.slots if k < slot):
+            batches = self.slots.pop(s)
+            batches.sort(key=_batch_key)
+            for batch in batches:
+                self._emit_batch(batch, emit)
+            self.watermarks.pop(s, None)
+        self.last_flush = time.monotonic()
+
+    def _flush_all_slots(self, emit: Callable[[DexEvent], None]) -> None:
+        for s in sorted(self.slots):
+            batches = self.slots[s]
+            batches.sort(key=_batch_key)
+            for batch in batches:
+                self._emit_batch(batch, emit)
+        self.slots.clear()
+        self.watermarks.clear()
+        self.last_flush = time.monotonic()
+
+    def _flush_micro_batch(self, emit: Callable[[DexEvent], None]) -> None:
+        if not self.micro_batch:
+            return
+        self.micro_batch.sort(key=_batch_key)
+        for batch in self.micro_batch:
+            self._emit_batch(batch, emit)
+        self.micro_batch = []
+        self.micro_start = 0.0
+        self.last_flush = time.monotonic()
+
+    @staticmethod
+    def _emit_batch(batch: _Batch, emit: Callable[[DexEvent], None]) -> None:
+        for event in batch.events:
+            emit(event)
+
+
+def _batch_key(batch: _Batch) -> Tuple[int, int, int]:
+    return (batch.slot, batch.tx_index, batch.seq)

sol_parser/parser.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+import base58
+import base64
+import struct
+import time
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+
+from .dex_parsers import (
+    DexEvent,
+    apply_event_type_filter,
+    dispatch_program_data,
+    event_type_for_discriminator,
+    filter_allows_unknown_log_event,
+    parse_trade_from_data,
+)
+
+if TYPE_CHECKING:
+    from .grpc_types import SubscribeUpdateTransactionInfo
+from .pumpfun_fee_enrich import enrich_pumpfun_same_tx_post_merge
+
+
+def _disc8(bs: bytes) -> int:
+    return struct.unpack("<Q", bs)[0]
+
+
+def decode_program_data_line(log: str) -> Optional[bytes]:
+    p = "Program data: "
+    i = log.find(p)
+    if i < 0:
+        return None
+    s = log[i + len(p) :].strip()
+    if len(s) > 2700:
+        return None
+    try:
+        raw = base64.standard_b64decode(s)
+    except Exception:
+        return None
+    if len(raw) < 8 or len(raw) > 2048:
+        return None
+    return raw
+
+
+def _meta(
+    sig: str,
+    slot: int,
+    tx_idx: int,
+    block_us: Optional[int],
+    grpc_us: int,
+    recent_blockhash: str = "",
+) -> dict:
+    m: dict = {
+        "signature": sig,
+        "slot": slot,
+        "tx_index": tx_idx,
+        "block_time_us": 0 if block_us is None else block_us,
+        "grpc_recv_us": grpc_us,
+    }
+    if recent_blockhash:
+        m["recent_blockhash"] = recent_blockhash
+    return m
+
+
+def parse_log_optimized(
+    log: str,
+    signature: str,
+    slot: int,
+    tx_index: int = 0,
+    block_time_us: Optional[int] = None,
+    grpc_recv_us: Optional[int] = None,
+    event_type_filter: Any = None,
+    is_created_buy: bool = False,
+    recent_blockhash: str = "",
+) -> Optional[DexEvent]:
+    """单次 base64 decode 后按 discriminator 做 early filter，再按实际事件类型二次过滤。"""
+    grpc = int(time.time() * 1_000_000) if grpc_recv_us is None else grpc_recv_us
+    buf = decode_program_data_line(log)
+    if not buf:
+        return None
+    disc = _disc8(buf[:8])
+    if event_type_filter is not None:
+        event_type = event_type_for_discriminator(disc)
+        if event_type is not None:
+            if not event_type_filter.should_include(event_type):
+                return None
+        elif not filter_allows_unknown_log_event(event_type_filter):
+            return None
+    data = buf[8:]
+    meta = _meta(signature, slot, tx_index, block_time_us, grpc, recent_blockhash)
+    return apply_event_type_filter(
+        dispatch_program_data(disc, data, buf, meta, is_created_buy),
+        event_type_filter,
+    )
+
+
+def parse_log_unified(
+    log: str,
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int] = None,
+    *,
+    tx_index: int = 0,
+) -> Optional[DexEvent]:
+    grpc = int(time.time() * 1_000_000)
+    return parse_log_optimized(
+        log,
+        signature,
+        slot,
+        tx_index,
+        block_time_us,
+        grpc,
+        None,
+        False,
+        "",
+    )
+
+
+def parse_transaction_events(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int] = None,
+    *,
+    subscribe_tx_info: Optional["SubscribeUpdateTransactionInfo"] = None,
+    tx_index: Optional[int] = None,
+) -> List[DexEvent]:
+    """对齐 Rust `parse_transaction_events` - 解析完整交易并返回所有 DEX 事件"""
+    return parse_logs_only(
+        logs,
+        signature,
+        slot,
+        block_time_us,
+        subscribe_tx_info=subscribe_tx_info,
+        tx_index=tx_index,
+    )
+
+
+def parse_logs_only(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int] = None,
+    *,
+    subscribe_tx_info: Optional["SubscribeUpdateTransactionInfo"] = None,
+    tx_index: Optional[int] = None,
+) -> List[DexEvent]:
+    """解析日志中的 Program data 事件。
+
+    若传入 ``subscribe_tx_info`` 且含 ``transaction_raw`` / ``meta_raw``（Yellowstone 订阅），
+    会在解析后调用 :func:`grpc_instruction_parser.enrich_dex_events_with_subscribe_tx_info`
+    从指令账户补全 bonding_curve、creator_vault 等字段（与 Rust gRPC 路径一致）。
+
+    ``tx_index`` 为区块内交易序号（与 gRPC ``SubscribeUpdateTransactionInfo.index`` 一致）。
+    未显式传入且提供了 ``subscribe_tx_info`` 时，使用 ``subscribe_tx_info.index``。
+    """
+    resolved_tx_index = 0
+    if tx_index is not None:
+        resolved_tx_index = int(tx_index)
+    elif subscribe_tx_info is not None:
+        resolved_tx_index = int(getattr(subscribe_tx_info, "index", 0) or 0)
+    out: List[DexEvent] = []
+    for log in logs:
+        ev = parse_log_unified(log, signature, slot, block_time_us, tx_index=resolved_tx_index)
+        if ev:
+            out.append(ev)
+    enrich_pumpfun_same_tx_post_merge(out)
+    if subscribe_tx_info is not None:
+        from .grpc_instruction_parser import enrich_dex_events_with_subscribe_tx_info
+
+        enrich_dex_events_with_subscribe_tx_info(out, subscribe_tx_info)
+    return out
+
+
+def parse_transaction_events_streaming(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int],
+    callback: Callable[[DexEvent], None],
+    *,
+    tx_index: int = 0,
+) -> None:
+    """对齐 Rust `parse_transaction_events_streaming`"""
+    parse_logs_streaming(logs, signature, slot, block_time_us, callback, tx_index=tx_index)
+
+
+def parse_logs_streaming(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int],
+    callback: Callable[[DexEvent], None],
+    *,
+    tx_index: int = 0,
+) -> None:
+    """对齐 Rust `parse_logs_streaming` - 流式解析，每解析出一个事件立即回调"""
+    for log in logs:
+        ev = parse_log_unified(log, signature, slot, block_time_us, tx_index=tx_index)
+        if ev:
+            callback(ev)
+
+
+class EventListener:
+    """对齐 Rust `EventListener` trait"""
+
+    def on_dex_event(self, event: DexEvent) -> None:
+        raise NotImplementedError
+
+
+def parse_transaction_with_listener(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int],
+    listener: EventListener,
+    *,
+    subscribe_tx_info: Optional["SubscribeUpdateTransactionInfo"] = None,
+    tx_index: Optional[int] = None,
+) -> None:
+    """对齐 Rust `parse_transaction_with_listener`"""
+    events = parse_logs_only(
+        logs,
+        signature,
+        slot,
+        block_time_us,
+        subscribe_tx_info=subscribe_tx_info,
+        tx_index=tx_index,
+    )
+    for ev in events:
+        listener.on_dex_event(ev)
+
+
+class StreamingEventListener:
+    """对齐 Rust `StreamingEventListener` trait"""
+
+    def on_dex_event_streaming(self, event: DexEvent) -> None:
+        raise NotImplementedError
+
+
+def parse_transaction_with_streaming_listener(
+    logs: List[str],
+    signature: str,
+    slot: int,
+    block_time_us: Optional[int],
+    listener: StreamingEventListener,
+) -> None:
+    """对齐 Rust `parse_transaction_with_streaming_listener`"""
+
+    def callback(ev: DexEvent) -> None:
+        listener.on_dex_event_streaming(ev)
+
+    parse_logs_streaming(logs, signature, slot, block_time_us, callback)
+
+
+def parse_log(
+    log: str,
+    signature: str,
+    slot: int,
+    tx_index: int,
+    block_time_us: Optional[int],
+    grpc_recv_us: int,
+    is_created_buy: bool,
+    recent_blockhash: str = "",
+) -> Optional[DexEvent]:
+    """对齐 Rust `parse_log` - 带完整 gRPC 元数据字段的日志解析"""
+    return parse_log_optimized(
+        log,
+        signature,
+        slot,
+        tx_index,
+        block_time_us,
+        grpc_recv_us,
+        None,
+        is_created_buy,
+        recent_blockhash,
+    )
+
+
+def warmup_parser() -> None:
+    decode_program_data_line("Program data: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
+
+
+__all__ = [
+    "DexEvent",
+    "decode_program_data_line",
+    "dispatch_program_data",
+    "parse_log",
+    "parse_log_unified",
+    "parse_log_optimized",
+    "parse_logs_only",
+    "parse_logs_streaming",
+    "parse_transaction_events",
+    "parse_transaction_events_streaming",
+    "parse_transaction_with_listener",
+    "parse_transaction_with_streaming_listener",
+    "EventListener",
+    "StreamingEventListener",
+    "parse_trade_from_data",
+    "warmup_parser",
+]

sol_parser/pumpfun_fee_enrich.py

@@ -0,0 +1,75 @@
+"""同笔交易 PumpFun 后处理（对齐 Rust ``pumpfun_fee_enrich``）。"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Tuple
+
+from .dex_parsers import Z
+from .event_types import DexEvent, PumpFunCreateEvent, PumpFunCreateV2TokenEvent, PumpFunTradeEvent
+from .grpc_types import EventType
+
+
+def _buy_like_mint_fee(ev: DexEvent) -> Optional[Tuple[str, str]]:
+    if not isinstance(ev.data, PumpFunTradeEvent):
+        return None
+    t = ev.data
+    if t.mint == Z or not t.mint:
+        return None
+    if ev.type == EventType.PUMP_FUN_TRADE:
+        if t.is_buy:
+            return (t.mint, t.fee_recipient)
+        return None
+    if ev.type in (EventType.PUMP_FUN_BUY, EventType.PUMP_FUN_BUY_EXACT_SOL_IN):
+        return (t.mint, t.fee_recipient)
+    return None
+
+
+def enrich_create_v2_observed_fee_recipient(events: List[DexEvent]) -> None:
+    mint_to_fee: Dict[str, str] = {}
+    for e in events:
+        p = _buy_like_mint_fee(e)
+        if not p:
+            continue
+        mint, fee = p
+        if fee and fee != Z:
+            mint_to_fee.setdefault(mint, fee)
+    if not mint_to_fee:
+        return
+    for e in events:
+        if e.type != EventType.PUMP_FUN_CREATE_V2:
+            continue
+        if not isinstance(e.data, PumpFunCreateV2TokenEvent):
+            continue
+        c = e.data
+        if not c.observed_fee_recipient and c.mint in mint_to_fee:
+            c.observed_fee_recipient = mint_to_fee[c.mint]
+
+
+def enrich_pumpfun_trades_from_create_instructions(events: List[DexEvent]) -> None:
+    flags: Dict[str, Tuple[bool, bool]] = {}
+    for e in events:
+        if e.type not in (EventType.PUMP_FUN_CREATE, EventType.PUMP_FUN_CREATE_V2):
+            continue
+        if not isinstance(e.data, (PumpFunCreateEvent, PumpFunCreateV2TokenEvent)):
+            continue
+        c = e.data
+        if c.mint and c.mint != Z:
+            flags.setdefault(c.mint, (c.is_cashback_enabled, c.is_mayhem_mode))
+    if not flags:
+        return
+    for e in events:
+        if not isinstance(e.data, PumpFunTradeEvent):
+            continue
+        t = e.data
+        if not t.mint or t.mint == Z or t.mint not in flags:
+            continue
+        cashback_enabled, mayhem_mode = flags[t.mint]
+        t.is_cashback_coin = t.is_cashback_coin or cashback_enabled
+        t.mayhem_mode = t.mayhem_mode or mayhem_mode
+        if cashback_enabled:
+            t.track_volume = True
+
+
+def enrich_pumpfun_same_tx_post_merge(events: List[DexEvent]) -> None:
+    enrich_create_v2_observed_fee_recipient(events)
+    enrich_pumpfun_trades_from_create_instructions(events)