yellowstone-fumarole-client 0.1.0rc2__tar.gz

yellowstone_fumarole_client-0.1.0rc2/PKG-INFO

@@ -0,0 +1,115 @@
+Metadata-Version: 2.1
+Name: yellowstone-fumarole-client
+Version: 0.1.0rc2
+Summary: Yellowstone Fumarole Python Client
+Home-page: https://github.com/rpcpool/yellowstone-fumarole
+Author: Louis-Vincent
+Author-email: louis-vincent@triton.one
+Requires-Python: >=3.13,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: asyncio (>=3.4.3,<4.0.0)
+Requires-Dist: base58 (>=2.1.1,<3.0.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: grpcio (>=1.71.1,<2.0.0)
+Requires-Dist: protobuf (>=5.29.1,<6.0.0)
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: toml (>=0.10.2,<0.11.0)
+Project-URL: Repository, https://github.com/rpcpool/yellowstone-fumarole
+Description-Content-Type: text/markdown
+
+# Fumarole Python SDK
+
+This module contains Fumarole SDK for `python` programming language.
+
+## Configuration
+
+```yaml
+endpoint: <"https://fumarole.endpoint.rpcpool.com">
+x-token: <YOUR X-TOKEN secret here>
+```
+
+## Manage consumer group
+
+Refer to [fume CLI](https://crates.io/crates/yellowstone-fumarole-cli) to manage your consumer groups.
+
+## Examples
+
+```python
+
+from typing import Optional
+import uuid
+import asyncio
+import logging
+from os import environ
+from collections import defaultdict
+from yellowstone_fumarole_client.config import FumaroleConfig
+from yellowstone_fumarole_client import FumaroleClient
+from yellowstone_fumarole_proto.fumarole_v2_pb2 import CreateConsumerGroupRequest
+from yellowstone_fumarole_proto.geyser_pb2 import (
+    SubscribeRequest,
+    SubscribeRequestFilterAccounts,
+    SubscribeRequestFilterTransactions,
+    SubscribeRequestFilterBlocksMeta,
+    SubscribeRequestFilterEntry,
+    SubscribeRequestFilterSlots,
+)
+from yellowstone_fumarole_proto.geyser_pb2 import (
+    SubscribeUpdate,
+    SubscribeUpdateTransaction,
+    SubscribeUpdateBlockMeta,
+    SubscribeUpdateAccount,
+    SubscribeUpdateEntry,
+    SubscribeUpdateSlot,
+)
+
+async def dragonsmouth_like_session(fumarole_config):
+    with open("~/.fumarole/config.yaml") as f:
+        fumarole_config = FumaroleConfig.from_yaml(f)
+    
+    client: FumaroleClient = await FumaroleClient.connect(fumarole_config)
+    await client.delete_all_consumer_groups()
+
+    # --- This is optional ---
+    resp = await client.create_consumer_group(
+        CreateConsumerGroupRequest(
+            consumer_group_name="test",
+        )
+    )
+    assert resp.consumer_group_id, "Failed to create consumer group"
+    # --- END OF OPTIONAL BLOCK ---
+
+    session = await client.dragonsmouth_subscribe(
+        consumer_group_name="test",
+        request=SubscribeRequest(
+            # accounts={"fumarole": SubscribeRequestFilterAccounts()},
+            transactions={"fumarole": SubscribeRequestFilterTransactions()},
+            blocks_meta={"fumarole": SubscribeRequestFilterBlocksMeta()},
+            entry={"fumarole": SubscribeRequestFilterEntry()},
+            slots={"fumarole": SubscribeRequestFilterSlots()},
+        ),
+    )
+    dragonsmouth_source = session.source
+    handle = session.fumarole_handle
+    block_map = defaultdict(BlockConstruction)
+    while True:
+        tasks = [asyncio.create_task(dragonsmouth_source.get()), handle]
+        done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+        for t in done:
+            if tasks[0] == t:
+                result: SubscribeUpdate = t.result()
+                if result.HasField("block_meta"):
+                    block_meta: SubscribeUpdateBlockMeta = result.block_meta
+                elif result.HasField("transaction"):
+                    tx: SubscribeUpdateTransaction = result.transaction
+                elif result.HasField("account"):
+                    account: SubscribeUpdateAccount = result.account
+                elif result.HasField("entry"):
+                    entry: SubscribeUpdateEntry = result.entry
+                elif result.HasField("slot"):
+                    result: SubscribeUpdateSlot = result.slot
+            else:
+                result = t.result()
+                raise RuntimeError("failed to get dragonsmouth source: %s" % result)
+```

yellowstone_fumarole_client-0.1.0rc2/README.md

@@ -0,0 +1,94 @@
+# Fumarole Python SDK
+
+This module contains Fumarole SDK for `python` programming language.
+
+## Configuration
+
+```yaml
+endpoint: <"https://fumarole.endpoint.rpcpool.com">
+x-token: <YOUR X-TOKEN secret here>
+```
+
+## Manage consumer group
+
+Refer to [fume CLI](https://crates.io/crates/yellowstone-fumarole-cli) to manage your consumer groups.
+
+## Examples
+
+```python
+
+from typing import Optional
+import uuid
+import asyncio
+import logging
+from os import environ
+from collections import defaultdict
+from yellowstone_fumarole_client.config import FumaroleConfig
+from yellowstone_fumarole_client import FumaroleClient
+from yellowstone_fumarole_proto.fumarole_v2_pb2 import CreateConsumerGroupRequest
+from yellowstone_fumarole_proto.geyser_pb2 import (
+    SubscribeRequest,
+    SubscribeRequestFilterAccounts,
+    SubscribeRequestFilterTransactions,
+    SubscribeRequestFilterBlocksMeta,
+    SubscribeRequestFilterEntry,
+    SubscribeRequestFilterSlots,
+)
+from yellowstone_fumarole_proto.geyser_pb2 import (
+    SubscribeUpdate,
+    SubscribeUpdateTransaction,
+    SubscribeUpdateBlockMeta,
+    SubscribeUpdateAccount,
+    SubscribeUpdateEntry,
+    SubscribeUpdateSlot,
+)
+
+async def dragonsmouth_like_session(fumarole_config):
+    with open("~/.fumarole/config.yaml") as f:
+        fumarole_config = FumaroleConfig.from_yaml(f)
+    
+    client: FumaroleClient = await FumaroleClient.connect(fumarole_config)
+    await client.delete_all_consumer_groups()
+
+    # --- This is optional ---
+    resp = await client.create_consumer_group(
+        CreateConsumerGroupRequest(
+            consumer_group_name="test",
+        )
+    )
+    assert resp.consumer_group_id, "Failed to create consumer group"
+    # --- END OF OPTIONAL BLOCK ---
+
+    session = await client.dragonsmouth_subscribe(
+        consumer_group_name="test",
+        request=SubscribeRequest(
+            # accounts={"fumarole": SubscribeRequestFilterAccounts()},
+            transactions={"fumarole": SubscribeRequestFilterTransactions()},
+            blocks_meta={"fumarole": SubscribeRequestFilterBlocksMeta()},
+            entry={"fumarole": SubscribeRequestFilterEntry()},
+            slots={"fumarole": SubscribeRequestFilterSlots()},
+        ),
+    )
+    dragonsmouth_source = session.source
+    handle = session.fumarole_handle
+    block_map = defaultdict(BlockConstruction)
+    while True:
+        tasks = [asyncio.create_task(dragonsmouth_source.get()), handle]
+        done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+        for t in done:
+            if tasks[0] == t:
+                result: SubscribeUpdate = t.result()
+                if result.HasField("block_meta"):
+                    block_meta: SubscribeUpdateBlockMeta = result.block_meta
+                elif result.HasField("transaction"):
+                    tx: SubscribeUpdateTransaction = result.transaction
+                elif result.HasField("account"):
+                    account: SubscribeUpdateAccount = result.account
+                elif result.HasField("entry"):
+                    entry: SubscribeUpdateEntry = result.entry
+                elif result.HasField("slot"):
+                    result: SubscribeUpdateSlot = result.slot
+            else:
+                result = t.result()
+                raise RuntimeError("failed to get dragonsmouth source: %s" % result)
+```

yellowstone_fumarole_client-0.1.0rc2/pyproject.toml

@@ -0,0 +1,36 @@
+[tool.poetry]
+name = "yellowstone-fumarole-client"
+version = "0.1.0rc2"
+homepage = "https://github.com/rpcpool/yellowstone-fumarole"
+repository = "https://github.com/rpcpool/yellowstone-fumarole"
+description = "Yellowstone Fumarole Python Client"
+authors = ["Louis-Vincent <louis-vincent@triton.one>", "Triton One <help@triton.one>"]
+readme = "README.md"
+
+packages = [
+    { include = "yellowstone_fumarole_proto" },
+    { include = "yellowstone_fumarole_client" },
+]
+
+[tool.poetry.dependencies]
+python = "^3.13"
+click = "^8.1.7"
+grpcio = "^1.71.1"
+protobuf = "^5.29.1"
+toml = "^0.10.2"
+base58 = "^2.1.1"
+tabulate = "^0.9.0"
+asyncio = "^3.4.3"
+pyyaml = "^6.0.2"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.3.4"
+
+[tool.poetry.group.dev.dependencies]
+grpcio-tools = "^1.68.1"
+black = "^24.10.0"
+pytest-asyncio = "^0.26.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

yellowstone_fumarole_client-0.1.0rc2/yellowstone_fumarole_client/__init__.py

@@ -0,0 +1,297 @@
+import asyncio
+import logging
+from yellowstone_fumarole_client.grpc_connectivity import (
+    FumaroleGrpcConnector,
+)
+from typing import Dict, Optional
+from dataclasses import dataclass
+from . import config
+from yellowstone_fumarole_client.runtime.aio import (
+    AsyncioFumeDragonsmouthRuntime,
+    FumaroleSM,
+    DEFAULT_GC_INTERVAL,
+    DEFAULT_SLOT_MEMORY_RETENTION,
+    GrpcSlotDownloader,
+)
+from yellowstone_fumarole_proto.geyser_pb2 import SubscribeRequest, SubscribeUpdate
+from yellowstone_fumarole_proto.fumarole_v2_pb2 import (
+    ControlResponse,
+    VersionRequest,
+    VersionResponse,
+    JoinControlPlane,
+    ControlCommand,
+    ListConsumerGroupsRequest,
+    ListConsumerGroupsResponse,
+    GetConsumerGroupInfoRequest,
+    ConsumerGroupInfo,
+    DeleteConsumerGroupRequest,
+    DeleteConsumerGroupResponse,
+    CreateConsumerGroupRequest,
+    CreateConsumerGroupResponse,
+)
+from yellowstone_fumarole_proto.fumarole_v2_pb2_grpc import FumaroleStub
+import grpc
+
+__all__ = [
+    "FumaroleClient",
+    "FumaroleConfig",
+    "FumaroleSubscribeConfig",
+    "DragonsmouthAdapterSession",
+    "DEFAULT_DRAGONSMOUTH_CAPACITY",
+    "DEFAULT_COMMIT_INTERVAL",
+    "DEFAULT_MAX_SLOT_DOWNLOAD_ATTEMPT",
+    "DEFAULT_CONCURRENT_DOWNLOAD_LIMIT_PER_TCP",
+]
+
+# Constants
+DEFAULT_DRAGONSMOUTH_CAPACITY = 10000
+DEFAULT_COMMIT_INTERVAL = 5.0  # seconds
+DEFAULT_MAX_SLOT_DOWNLOAD_ATTEMPT = 3
+DEFAULT_CONCURRENT_DOWNLOAD_LIMIT_PER_TCP = 10
+
+# Error classes
+
+
+# FumaroleSubscribeConfig
+@dataclass
+class FumaroleSubscribeConfig:
+    """Configuration for subscribing to a dragonsmouth stream."""
+
+    # The maximum number of concurrent download tasks per TCP connection.
+    concurrent_download_limit: int = DEFAULT_CONCURRENT_DOWNLOAD_LIMIT_PER_TCP
+
+    # The interval at which to commit the slot memory.
+    commit_interval: float = DEFAULT_COMMIT_INTERVAL
+
+    # The maximum number of failed slot download attempts before giving up.
+    max_failed_slot_download_attempt: int = DEFAULT_MAX_SLOT_DOWNLOAD_ATTEMPT
+
+    # The maximum number of slots to download concurrently.
+    data_channel_capacity: int = DEFAULT_DRAGONSMOUTH_CAPACITY
+
+    # The interval at which to perform garbage collection on the slot memory.
+    gc_interval: int = DEFAULT_GC_INTERVAL
+
+    # The retention period for slot memory in seconds.
+    slot_memory_retention: int = DEFAULT_SLOT_MEMORY_RETENTION
+
+
+# DragonsmouthAdapterSession
+@dataclass
+class DragonsmouthAdapterSession:
+    """Session for interacting with the dragonsmouth-like stream."""
+
+    # The queue for sending SubscribeRequest update to the dragonsmouth stream.
+    sink: asyncio.Queue
+
+    # The queue for receiving SubscribeUpdate from the dragonsmouth stream.
+    source: asyncio.Queue
+
+    # The task handle for the fumarole runtime.
+    fumarole_handle: asyncio.Task
+
+
+# FumaroleClient
+class FumaroleClient:
+    """Fumarole client for interacting with the Fumarole server."""
+
+    logger = logging.getLogger(__name__)
+
+    def __init__(self, connector: FumaroleGrpcConnector, stub: FumaroleStub):
+        self.connector = connector
+        self.stub = stub
+
+    @staticmethod
+    async def connect(config: config.FumaroleConfig) -> "FumaroleClient":
+        """Connect to the Fumarole server using the provided configuration.
+        Args:
+            config (FumaroleConfig): Configuration for the Fumarole client.
+        """
+        endpoint = config.endpoint
+        connector = FumaroleGrpcConnector(config=config, endpoint=endpoint)
+        FumaroleClient.logger.debug(f"Connecting to {endpoint}")
+        client = await connector.connect()
+        FumaroleClient.logger.debug(f"Connected to {endpoint}")
+        return FumaroleClient(connector=connector, stub=client)
+
+    async def version(self) -> VersionResponse:
+        """Get the version of the Fumarole server."""
+        request = VersionRequest()
+        response = await self.stub.version(request)
+        return response
+
+    async def dragonsmouth_subscribe(
+        self, consumer_group_name: str, request: SubscribeRequest
+    ) -> DragonsmouthAdapterSession:
+        """Subscribe to a dragonsmouth stream with default configuration.
+
+        Args:
+            consumer_group_name (str): The name of the consumer group.
+            request (SubscribeRequest): The request to subscribe to the dragonsmouth stream.
+        """
+        return await self.dragonsmouth_subscribe_with_config(
+            consumer_group_name, request, FumaroleSubscribeConfig()
+        )
+
+    async def dragonsmouth_subscribe_with_config(
+        self,
+        consumer_group_name: str,
+        request: SubscribeRequest,
+        config: FumaroleSubscribeConfig,
+    ) -> DragonsmouthAdapterSession:
+        """Subscribe to a dragonsmouth stream with custom configuration.
+
+        Args:
+            consumer_group_name (str): The name of the consumer group.
+            request (SubscribeRequest): The request to subscribe to the dragonsmouth stream.
+            config (FumaroleSubscribeConfig): The configuration for the dragonsmouth subscription.
+        """
+        dragonsmouth_outlet = asyncio.Queue(maxsize=config.data_channel_capacity)
+        fume_control_plane_q = asyncio.Queue(maxsize=100)
+
+        initial_join = JoinControlPlane(consumer_group_name=consumer_group_name)
+        initial_join_command = ControlCommand(initial_join=initial_join)
+        await fume_control_plane_q.put(initial_join_command)
+
+        FumaroleClient.logger.debug(
+            f"Sent initial join command: {initial_join_command}"
+        )
+
+        async def control_plane_sink():
+            while True:
+                try:
+                    update = await fume_control_plane_q.get()
+                    yield update
+                except asyncio.QueueShutDown:
+                    break
+
+        fume_control_plane_stream_rx: grpc.aio.StreamStreamMultiCallable = (
+            self.stub.Subscribe(control_plane_sink())
+        )
+
+        control_response: ControlResponse = await fume_control_plane_stream_rx.read()
+        init = control_response.init
+        if init is None:
+            raise ValueError(f"Unexpected initial response: {control_response}")
+
+        # Once we have the initial response, we can spin a task to read from the stream
+        # and put the updates into the queue.
+        # This is a bit of a hack, but we need a Queue not a StreamStreamMultiCallable
+        # because Queue are cancel-safe, while Stream are not, or at least didn't find any docs about it.
+        fume_control_plane_rx_q = asyncio.Queue(maxsize=100)
+
+        async def control_plane_source():
+            while True:
+                try:
+                    async for update in fume_control_plane_stream_rx:
+                        await fume_control_plane_rx_q.put(update)
+                except asyncio.QueueShutDown:
+                    break
+
+        _cp_src_task = asyncio.create_task(control_plane_source())
+
+        FumaroleClient.logger.debug(f"Control response: {control_response}")
+
+        last_committed_offset = init.last_committed_offsets.get(0)
+        if last_committed_offset is None:
+            raise ValueError("No last committed offset")
+
+        sm = FumaroleSM(last_committed_offset, config.slot_memory_retention)
+        subscribe_request_queue = asyncio.Queue(maxsize=100)
+
+        data_plane_client = await self.connector.connect()
+
+        grpc_slot_downloader = GrpcSlotDownloader(
+            client=data_plane_client,
+        )
+
+        rt = AsyncioFumeDragonsmouthRuntime(
+            sm=sm,
+            slot_downloader=grpc_slot_downloader,
+            subscribe_request_update_q=subscribe_request_queue,
+            subscribe_request=request,
+            consumer_group_name=consumer_group_name,
+            control_plane_tx_q=fume_control_plane_q,
+            control_plane_rx_q=fume_control_plane_rx_q,
+            dragonsmouth_outlet=dragonsmouth_outlet,
+            commit_interval=config.commit_interval,
+            gc_interval=config.gc_interval,
+            max_concurrent_download=config.concurrent_download_limit,
+        )
+
+        fumarole_handle = asyncio.create_task(rt.run())
+        FumaroleClient.logger.debug(f"Fumarole handle created: {fumarole_handle}")
+        return DragonsmouthAdapterSession(
+            sink=subscribe_request_queue,
+            source=dragonsmouth_outlet,
+            fumarole_handle=fumarole_handle,
+        )
+
+    async def list_consumer_groups(
+        self,
+    ) -> ListConsumerGroupsResponse:
+        """Lists all consumer groups."""
+        return await self.stub.ListConsumerGroups(ListConsumerGroupsRequest())
+
+    async def get_consumer_group_info(
+        self, consumer_group_name: str
+    ) -> Optional[ConsumerGroupInfo]:
+        """Gets information about a consumer group by name.
+        Returns None if the consumer group does not exist.
+
+        Args:
+            consumer_group_name (str): The name of the consumer group to retrieve information for.
+        """
+        try:
+            return await self.stub.GetConsumerGroupInfo(
+                GetConsumerGroupInfoRequest(consumer_group_name=consumer_group_name)
+            )
+        except grpc.aio.AioRpcError as e:
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                return None
+            else:
+                raise
+
+    async def delete_consumer_group(
+        self, consumer_group_name: str
+    ) -> DeleteConsumerGroupResponse:
+        """Delete a consumer group by name.
+
+        NOTE: this operation is idempotent, meaning that if the consumer group does not exist, it will not raise an error.
+        Args:
+            consumer_group_name (str): The name of the consumer group to delete.
+        """
+        return await self.stub.DeleteConsumerGroup(
+            DeleteConsumerGroupRequest(consumer_group_name=consumer_group_name)
+        )
+
+    async def delete_all_consumer_groups(
+        self,
+    ) -> DeleteConsumerGroupResponse:
+        """Deletes all consumer groups."""
+        consumer_group_list = await self.list_consumer_groups()
+
+        tasks = []
+
+        async with asyncio.TaskGroup() as tg:
+            for group in consumer_group_list.consumer_groups:
+                cg_name = group.consumer_group_name
+                task = tg.create_task(self.delete_consumer_group(cg_name))
+                tasks.append((cg_name, task))
+
+        # Raise an error if any task fails
+        for cg_name, task in tasks:
+            result = task.result()
+            if not result.success:
+                raise RuntimeError(
+                    f"Failed to delete consumer group {cg_name}: {result.error}"
+                )
+
+    async def create_consumer_group(
+        self, request: CreateConsumerGroupRequest
+    ) -> CreateConsumerGroupResponse:
+        """Creates a new consumer group.
+        Args:
+            request (CreateConsumerGroupRequest): The request to create a consumer group.
+        """
+        return await self.stub.CreateConsumerGroup(request)

yellowstone_fumarole_client-0.1.0rc2/yellowstone_fumarole_client/config.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Dict, Optional
+import yaml
+
+
+@dataclass
+class FumaroleConfig:
+    endpoint: str
+    x_token: Optional[str] = None
+    max_decoding_message_size_bytes: int = 512_000_000
+    x_metadata: Dict[str, str] = None
+
+    def __post_init__(self):
+        self.x_metadata = self.x_metadata or {}
+
+    @classmethod
+    def from_yaml(cls, fileobj) -> "FumaroleConfig":
+        data = yaml.safe_load(fileobj)
+        return cls(
+            endpoint=data["endpoint"],
+            x_token=data.get("x-token") or data.get("x_token"),
+            max_decoding_message_size_bytes=data.get(
+                "max_decoding_message_size_bytes", cls.max_decoding_message_size_bytes
+            ),
+            x_metadata=data.get("x-metadata", {}),
+        )