tonutils 2.0.1b3__py3-none-any.whl → 2.0.1b4__py3-none-any.whl

tonutils/tools/block_scanner/dispatcher.py

@@ -0,0 +1,141 @@
+import asyncio
+import inspect
+import traceback
+import typing as t
+
+from tonutils.tools.block_scanner.annotations import (
+    AnyHandler,
+    AnyWhere,
+    Decorator,
+    Handler,
+    HandlerEntry,
+    TEvent,
+    Where,
+)
+from tonutils.tools.block_scanner.events import EventBase
+
+
+class EventDispatcher:
+    """Dispatches events to registered handlers asynchronously."""
+
+    def __init__(self, max_concurrency: int = 1000) -> None:
+        """
+        Initialize EventDispatcher.
+
+        :param max_concurrency: maximum number of concurrent handler tasks.
+        """
+        self._handlers: t.Dict[t.Type[EventBase], t.List[HandlerEntry]] = {}
+        self._sem = asyncio.Semaphore(max(1, max_concurrency))
+        self._tasks: t.Set[asyncio.Task[None]] = set()
+        self._closed = False
+
+    def register(
+        self,
+        event_type: t.Type[TEvent],
+        handler: Handler[TEvent],
+        *,
+        where: t.Optional[Where[TEvent]] = None,
+    ) -> None:
+        """
+        Register a handler for a specific event type.
+
+        :param event_type: subclass of EventBase to handle.
+        :param handler: callable receiving the event.
+        :param where: optional filter predicate. Handler is invoked only if predicate returns True.
+        """
+        if not callable(handler):
+            raise TypeError("handler must be callable")
+
+        entry: HandlerEntry = (
+            t.cast(AnyHandler, handler),
+            t.cast(t.Optional[AnyWhere], where),
+        )
+        self._handlers.setdefault(event_type, []).append(entry)
+
+    def on(
+        self,
+        event_type: t.Type[TEvent],
+        *,
+        where: t.Optional[Where[TEvent]] = None,
+    ) -> Decorator[TEvent]:
+        """
+        Decorator to register a handler for an event type.
+
+        :param event_type: event class to handle.
+        :param where: optional filter predicate.
+        :return: Decorator that registers the handler.
+        """
+
+        def decorator(fn: Handler[TEvent]) -> Handler[TEvent]:
+            self.register(event_type=event_type, handler=fn, where=where)
+            return fn
+
+        return decorator
+
+    def _iter_handlers(self, event: EventBase) -> t.Sequence[HandlerEntry]:
+        """Return all handlers matching the type of `event`."""
+        out: t.List[HandlerEntry] = []
+        for tp in type(event).mro():
+            if tp is EventBase:
+                break
+            entries = self._handlers.get(t.cast(t.Type[EventBase], tp))
+            if entries:
+                out.extend(entries)
+        return out
+
+    def _on_task_done(self, task: asyncio.Task[None]) -> None:
+        """Callback to handle task completion and print exceptions."""
+        self._tasks.discard(task)
+        try:
+            exc = task.exception()
+        except asyncio.CancelledError:
+            return
+        if exc is not None:
+            traceback.print_exception(type(exc), exc, exc.__traceback__)
+
+    async def _run_task(
+        self,
+        handler: AnyHandler,
+        event: EventBase,
+        where: t.Optional[AnyWhere] = None,
+    ) -> None:
+        """
+        Run a single handler task with optional 'where' filtering.
+
+        :param handler: async callable to execute.
+        :param event: event instance to pass.
+        :param where: optional predicate, skip handler if False.
+        """
+        async with self._sem:
+            if where is not None:
+                result = where(event)
+                if inspect.isawaitable(result):
+                    result = await result
+                if not result:
+                    return
+            await handler(event)
+
+    def emit(self, event: EventBase) -> None:
+        """
+        Emit an event to all matching handlers.
+
+        Handlers are executed asynchronously.
+        """
+        if self._closed:
+            return
+
+        for handler, where in self._iter_handlers(event):
+            task = asyncio.create_task(self._run_task(handler, event, where))
+            self._tasks.add(task)
+            task.add_done_callback(self._on_task_done)
+
+    async def aclose(self) -> None:
+        """
+        Close the dispatcher and wait for all running handler tasks.
+
+        After calling, no new events will be dispatched.
+        """
+        self._closed = True
+        if self._tasks:
+            await asyncio.gather(*self._tasks, return_exceptions=True)
+        self._tasks.clear()

tonutils/tools/block_scanner/events.py

@@ -0,0 +1,31 @@
+import typing as t
+from dataclasses import dataclass, field
+
+from pytoniq_core import Transaction
+from pytoniq_core.tl import BlockIdExt
+
+from tonutils.clients import LiteBalancer, LiteClient
+
+
+@dataclass(frozen=True, slots=True)
+class EventBase:
+    client: t.Union[LiteBalancer, LiteClient]
+    mc_block: BlockIdExt
+    context: t.Dict[str, t.Any]
+
+
+@dataclass(frozen=True, slots=True)
+class BlockEvent(EventBase):
+    block: BlockIdExt
+
+
+@dataclass(frozen=True, slots=True)
+class TransactionEvent(EventBase):
+    block: BlockIdExt
+    transaction: Transaction
+
+
+@dataclass(frozen=True, slots=True)
+class TransactionsEvent(EventBase):
+    block: BlockIdExt
+    transactions: t.List[Transaction] = field(default_factory=list)

tonutils/tools/block_scanner/scanner.py

@@ -0,0 +1,315 @@
+import asyncio
+import typing as t
+from dataclasses import dataclass
+
+from pytoniq_core import Transaction
+from pytoniq_core.tl import BlockIdExt
+
+from tonutils.clients import LiteBalancer, LiteClient
+from tonutils.tools.block_scanner.annotations import (
+    BlockWhere,
+    Decorator,
+    Handler,
+    TransactionWhere,
+    TransactionsWhere,
+)
+from tonutils.tools.block_scanner.dispatcher import EventDispatcher
+from tonutils.tools.block_scanner.events import (
+    BlockEvent,
+    TransactionEvent,
+    TransactionsEvent,
+)
+from tonutils.tools.block_scanner.traversal import ShardTraversal
+from tonutils.types import WorkchainID, MASTERCHAIN_SHARD
+
+
+@dataclass(slots=True)
+class _ScanState:
+    """Internal scanner state per masterchain block."""
+
+    mc_block: BlockIdExt
+    shards_seqno: t.Dict[t.Tuple[int, int], int]
+
+
+class BlockScanner:
+    """Asynchronous scanner for TON blockchain."""
+
+    def __init__(
+        self,
+        *,
+        client: t.Union[LiteBalancer, LiteClient],
+        poll_interval: float = 0.1,
+        include_transactions: bool = True,
+        max_concurrency: int = 1000,
+        **context: t.Any,
+    ) -> None:
+        """
+        Initialize a BlockScanner.
+
+        :param client: LiteClient or LiteBalancer instance for blockchain access.
+        :param poll_interval: Interval in seconds to poll for new masterchain blocks.
+        :param include_transactions: If True, emit TransactionEvent and TransactionsEvent.
+        :param max_concurrency: Maximum number of concurrent event handler tasks.
+        :param context: Additional key/value data passed to all emitted events.
+        """
+        self._client = client
+        self._context = dict(context)
+        self._poll_interval = poll_interval
+        self._include_transactions = include_transactions
+
+        self._traversal = ShardTraversal()
+        self._dispatcher = EventDispatcher(max_concurrency)
+
+        self._stop_event = asyncio.Event()
+        self._running = False
+
+    @t.overload
+    def register(
+        self,
+        event_type: t.Type[BlockEvent],
+        handler: Handler[BlockEvent],
+        *,
+        where: t.Optional[BlockWhere] = None,
+    ) -> None: ...
+
+    @t.overload
+    def register(
+        self,
+        event_type: t.Type[TransactionEvent],
+        handler: Handler[TransactionEvent],
+        *,
+        where: t.Optional[TransactionWhere] = None,
+    ) -> None: ...
+
+    @t.overload
+    def register(
+        self,
+        event_type: t.Type[TransactionsEvent],
+        handler: Handler[TransactionsEvent],
+        *,
+        where: t.Optional[TransactionsWhere] = None,
+    ) -> None: ...
+
+    def register(
+        self,
+        event_type: t.Any,
+        handler: t.Any,
+        *,
+        where: t.Any = None,
+    ) -> None:
+        """Register a handler for an event type with optional filter."""
+        self._dispatcher.register(event_type, handler, where=where)
+
+    def on_block(
+        self,
+        where: t.Optional[BlockWhere] = None,
+    ) -> Decorator[BlockEvent]:
+        """Decorator for block event handlers."""
+        return self._dispatcher.on(BlockEvent, where=where)
+
+    def on_transaction(
+        self,
+        where: t.Optional[TransactionWhere] = None,
+    ) -> Decorator[TransactionEvent]:
+        """Decorator for transaction event handlers."""
+        return self._dispatcher.on(TransactionEvent, where=where)
+
+    def on_transactions(
+        self,
+        where: t.Optional[TransactionsWhere] = None,
+    ) -> Decorator[TransactionsEvent]:
+        """Decorator for batch transaction event handlers."""
+        return self._dispatcher.on(TransactionsEvent, where=where)
+
+    def _get_last_mc_block(self) -> BlockIdExt:
+        """Return last masterchain block."""
+        return self._client.provider.last_mc_block
+
+    async def _lookup_mc_block(
+        self,
+        seqno: t.Optional[int] = None,
+        lt: t.Optional[int] = None,
+        utime: t.Optional[int] = None,
+    ) -> BlockIdExt:
+        """Lookup masterchain block by seqno, lt, or utime."""
+        mc_block, _info = await self._client.lookup_block(
+            workchain=WorkchainID.MASTERCHAIN,
+            shard=MASTERCHAIN_SHARD,
+            seqno=seqno,
+            lt=lt,
+            utime=utime,
+        )
+        return mc_block
+
+    async def _init_state(
+        self,
+        seqno: t.Optional[int] = None,
+        lt: t.Optional[int] = None,
+        utime: t.Optional[int] = None,
+    ) -> _ScanState:
+        """Initialize scanning state."""
+        if seqno is None and lt is None and utime is None:
+            mc_block = self._get_last_mc_block()
+        else:
+            mc_block = await self._lookup_mc_block(seqno=seqno, lt=lt, utime=utime)
+
+        if mc_block.seqno > 0:
+            prev_mc = await self._lookup_mc_block(seqno=mc_block.seqno - 1)
+        else:
+            prev_mc = mc_block
+
+        shards_seqno: t.Dict[t.Tuple[int, int], int] = {}
+        for shard in await self._client.get_all_shards_info(prev_mc):
+            shards_seqno[self._traversal.shard_key(shard)] = shard.seqno
+
+        return _ScanState(mc_block=mc_block, shards_seqno=shards_seqno)
+
+    def _ensure_running(self) -> None:
+        """Raise CancelledError if scanner was stopped."""
+        if self._stop_event.is_set():
+            raise asyncio.CancelledError("Block scanner stopped")
+
+    async def _collect_blocks(
+        self,
+        mc_block: BlockIdExt,
+        shards_seqno: t.Dict[t.Tuple[int, int], int],
+    ) -> t.List[BlockIdExt]:
+        """Collect all unseen shard blocks for a masterchain block."""
+        shards = await self._client.get_all_shards_info(mc_block)
+
+        blocks: t.List[BlockIdExt] = []
+        for shard_tip in shards:
+            blocks.extend(
+                await self._traversal.walk_unseen(
+                    root=shard_tip,
+                    seen_seqno=shards_seqno,
+                    get_header=self._client.get_block_header,
+                )
+            )
+
+        blocks.sort(key=lambda b: (b.workchain, b.shard, b.seqno))
+        return blocks
+
+    def _emit_block(self, mc_block: BlockIdExt, block: BlockIdExt) -> None:
+        """Emit block event."""
+        self._dispatcher.emit(
+            BlockEvent(
+                mc_block=mc_block,
+                client=self._client,
+                context=self._context,
+                block=block,
+            )
+        )
+
+    def _emit_transactions(
+        self,
+        mc_block: BlockIdExt,
+        block: BlockIdExt,
+        transactions: t.List[Transaction],
+    ) -> None:
+        """Emit batch transactions event."""
+        self._dispatcher.emit(
+            TransactionsEvent(
+                mc_block=mc_block,
+                client=self._client,
+                context=self._context,
+                block=block,
+                transactions=transactions,
+            )
+        )
+
+    def _emit_transaction(
+        self,
+        mc_block: BlockIdExt,
+        block: BlockIdExt,
+        transaction: Transaction,
+    ) -> None:
+        """Emit single transaction event."""
+        self._dispatcher.emit(
+            TransactionEvent(
+                mc_block=mc_block,
+                client=self._client,
+                context=self._context,
+                block=block,
+                transaction=transaction,
+            )
+        )
+
+    async def _handle_block(
+        self,
+        mc_block: BlockIdExt,
+        block: BlockIdExt,
+        shards_seqno: t.Dict[t.Tuple[int, int], int],
+    ) -> None:
+        """Process shard block and emit events for block + transactions."""
+        self._ensure_running()
+
+        shards_seqno[self._traversal.shard_key(block)] = block.seqno
+        self._emit_block(mc_block, block)
+
+        if not self._include_transactions:
+            return
+
+        transactions = await self._client.get_block_transactions_ext(block)
+        self._emit_transactions(mc_block, block, transactions)
+
+        for transaction in transactions:
+            self._ensure_running()
+            self._emit_transaction(mc_block, block, transaction)
+
+    async def _wait_next_mc_block(self, current: BlockIdExt) -> BlockIdExt:
+        """Wait for next masterchain block, polling until available."""
+        next_seqno = current.seqno + 1
+
+        while True:
+            self._ensure_running()
+            last_mc_block = self._get_last_mc_block()
+
+            if next_seqno <= last_mc_block.seqno:
+                if next_seqno == last_mc_block.seqno:
+                    return last_mc_block
+                return await self._lookup_mc_block(seqno=next_seqno)
+
+            await asyncio.sleep(self._poll_interval)
+
+    async def start(
+        self,
+        from_seqno: t.Optional[int] = None,
+        from_lt: t.Optional[int] = None,
+        from_utime: t.Optional[int] = None,
+    ) -> None:
+        """
+        Start scanning from the specified point.
+
+        :param from_seqno: start from specific masterchain sequence number.
+        :param from_lt: start from specific logical time (LT) of a block.
+        :param from_utime: start from specific Unix timestamp.
+        """
+        if self._running:
+            raise RuntimeError("BlockScanner is already running")
+
+        self._running = True
+        self._stop_event.clear()
+
+        state = await self._init_state(
+            seqno=from_seqno,
+            lt=from_lt,
+            utime=from_utime,
+        )
+
+        try:
+            while not self._stop_event.is_set():
+                blocks = await self._collect_blocks(
+                    mc_block=state.mc_block,
+                    shards_seqno=state.shards_seqno,
+                )
+                for block in blocks:
+                    await self._handle_block(state.mc_block, block, state.shards_seqno)
+                state.mc_block = await self._wait_next_mc_block(state.mc_block)
+        finally:
+            await self._dispatcher.aclose()
+            self._running = False
+
+    async def stop(self) -> None:
+        """Stop scanning."""
+        self._stop_event.set()

tonutils/tools/block_scanner/traversal.py

@@ -0,0 +1,96 @@
+import typing as t
+
+from pytoniq_core.tl import BlockIdExt
+from pytoniq_core.tlb.block import ExtBlkRef
+
+
+class ShardTraversal:
+
+    @staticmethod
+    def shard_key(blk: BlockIdExt) -> t.Tuple[int, int]:
+        return blk.workchain, blk.shard
+
+    @staticmethod
+    def simulate_overflow(x: int) -> int:
+        return (x + 2**63) % 2**64 - 2**63
+
+    @staticmethod
+    def lower_bit64(num: int) -> int:
+        return num & (~num + 1)
+
+    def get_child_shard(self, shard: int, *, left: bool) -> int:
+        x = self.lower_bit64(shard) >> 1
+        if left:
+            return self.simulate_overflow(shard - x)
+        return self.simulate_overflow(shard + x)
+
+    def get_parent_shard(self, shard: int) -> int:
+        x = self.lower_bit64(shard)
+        return self.simulate_overflow((shard - x) | (x << 1))
+
+    async def walk_unseen(
+        self,
+        *,
+        root: BlockIdExt,
+        seen_seqno: t.Dict[t.Tuple[int, int], int],
+        get_header: t.Callable[[BlockIdExt], t.Awaitable[t.Any]],
+    ) -> t.List[BlockIdExt]:
+        out: t.List[BlockIdExt] = []
+        stack: t.List[BlockIdExt] = [root]
+        post: t.List[BlockIdExt] = []
+
+        while stack:
+            blk = stack.pop()
+            key = self.shard_key(blk)
+            if seen_seqno.get(key, -1) >= blk.seqno:
+                continue
+
+            post.append(blk)
+            _, header = await get_header(blk)
+            prev_ref = header.info.prev_ref
+
+            if prev_ref.type_ == "prev_blk_info":
+                prev: ExtBlkRef = prev_ref.prev
+                prev_shard = (
+                    self.get_parent_shard(blk.shard)
+                    if header.info.after_split
+                    else blk.shard
+                )
+                stack.append(
+                    BlockIdExt(
+                        workchain=blk.workchain,
+                        shard=prev_shard,
+                        seqno=prev.seqno,
+                        root_hash=prev.root_hash,
+                        file_hash=prev.file_hash,
+                    )
+                )
+
+            else:
+                prev1, prev2 = prev_ref.prev1, prev_ref.prev2
+                stack.append(
+                    BlockIdExt(
+                        workchain=blk.workchain,
+                        shard=self.get_child_shard(blk.shard, left=True),
+                        seqno=prev1.seqno,
+                        root_hash=prev1.root_hash,
+                        file_hash=prev1.file_hash,
+                    )
+                )
+                stack.append(
+                    BlockIdExt(
+                        workchain=blk.workchain,
+                        shard=self.get_child_shard(blk.shard, left=False),
+                        seqno=prev2.seqno,
+                        root_hash=prev2.root_hash,
+                        file_hash=prev2.file_hash,
+                    )
+                )
+
+        for blk in reversed(post):
+            key = self.shard_key(blk)
+            if seen_seqno.get(key, -1) >= blk.seqno:
+                continue
+            out.append(blk)
+
+        return out