yellowstone-fumarole-client 0.1.0rc2__tar.gz → 0.2.0__tar.gz

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: yellowstone-fumarole-client
-Version: 0.1.0rc2
+Version: 0.2.0
 Summary: Yellowstone Fumarole Python Client
 Home-page: https://github.com/rpcpool/yellowstone-fumarole
 Author: Louis-Vincent
@@ -8,14 +8,9 @@ Author-email: louis-vincent@triton.one
 Requires-Python: >=3.13,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: asyncio (>=3.4.3,<4.0.0)
-Requires-Dist: base58 (>=2.1.1,<3.0.0)
-Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: grpcio (>=1.71.1,<2.0.0)
-Requires-Dist: protobuf (>=5.29.1,<6.0.0)
+Requires-Dist: protobuf (>=6.32.0,<7.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
-Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: toml (>=0.10.2,<0.11.0)
 Project-URL: Repository, https://github.com/rpcpool/yellowstone-fumarole
 Description-Content-Type: text/markdown
 
@@ -83,33 +78,43 @@ async def dragonsmouth_like_session(fumarole_config):
     session = await client.dragonsmouth_subscribe(
         consumer_group_name="test",
         request=SubscribeRequest(
-            # accounts={"fumarole": SubscribeRequestFilterAccounts()},
+            accounts={"fumarole": SubscribeRequestFilterAccounts()},
             transactions={"fumarole": SubscribeRequestFilterTransactions()},
             blocks_meta={"fumarole": SubscribeRequestFilterBlocksMeta()},
             entry={"fumarole": SubscribeRequestFilterEntry()},
             slots={"fumarole": SubscribeRequestFilterSlots()},
         ),
     )
-    dragonsmouth_source = session.source
-    handle = session.fumarole_handle
-    block_map = defaultdict(BlockConstruction)
-    while True:
-        tasks = [asyncio.create_task(dragonsmouth_source.get()), handle]
-        done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
-        for t in done:
-            if tasks[0] == t:
-                result: SubscribeUpdate = t.result()
-                if result.HasField("block_meta"):
-                    block_meta: SubscribeUpdateBlockMeta = result.block_meta
-                elif result.HasField("transaction"):
-                    tx: SubscribeUpdateTransaction = result.transaction
-                elif result.HasField("account"):
-                    account: SubscribeUpdateAccount = result.account
-                elif result.HasField("entry"):
-                    entry: SubscribeUpdateEntry = result.entry
-                elif result.HasField("slot"):
-                    result: SubscribeUpdateSlot = result.slot
-            else:
-                result = t.result()
-                raise RuntimeError("failed to get dragonsmouth source: %s" % result)
+    async with session:
+        dragonsmouth_like_source = session.source
+        # result: SubscribeUpdate
+        async for result in dragonsmouth_like_source:
+            if result.HasField("block_meta"):
+                block_meta: SubscribeUpdateBlockMeta = result.block_meta
+            elif result.HasField("transaction"):
+                tx: SubscribeUpdateTransaction = result.transaction
+            elif result.HasField("account"):
+                account: SubscribeUpdateAccount = result.account
+            elif result.HasField("entry"):
+                entry: SubscribeUpdateEntry = result.entry
+            elif result.HasField("slot"):
+                result: SubscribeUpdateSlot = result.slot
+
+    # OUTSIDE THE SCOPE, YOU SHOULD NEVER USE `session` again.
+```
+
+
+At any point you can get a rough estimate if you are progression through the slot using `DragonsmouthAdapterSession.stats()` call:
+
+```python
+
+async with session:
+    stats: FumaroleSubscribeStats = session.stats()
+    print(f"{stats.log_committed_offset}, {stats.log_committable_offset}, {stats.max_slot_seen}")
 ```
+
+`log_committed_offset` : what have been ACK so for to fumarole remote service.
+`log_committable_offset` : what can be ACK to next commit call.
+`max_slot_seen` : maximum slot seen in the inner fumarole client state -- not yet processed by your code.
+
+

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/README.md

@@ -62,33 +62,42 @@ async def dragonsmouth_like_session(fumarole_config):
     session = await client.dragonsmouth_subscribe(
         consumer_group_name="test",
         request=SubscribeRequest(
-            # accounts={"fumarole": SubscribeRequestFilterAccounts()},
+            accounts={"fumarole": SubscribeRequestFilterAccounts()},
             transactions={"fumarole": SubscribeRequestFilterTransactions()},
             blocks_meta={"fumarole": SubscribeRequestFilterBlocksMeta()},
             entry={"fumarole": SubscribeRequestFilterEntry()},
             slots={"fumarole": SubscribeRequestFilterSlots()},
         ),
     )
-    dragonsmouth_source = session.source
-    handle = session.fumarole_handle
-    block_map = defaultdict(BlockConstruction)
-    while True:
-        tasks = [asyncio.create_task(dragonsmouth_source.get()), handle]
-        done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
-        for t in done:
-            if tasks[0] == t:
-                result: SubscribeUpdate = t.result()
-                if result.HasField("block_meta"):
-                    block_meta: SubscribeUpdateBlockMeta = result.block_meta
-                elif result.HasField("transaction"):
-                    tx: SubscribeUpdateTransaction = result.transaction
-                elif result.HasField("account"):
-                    account: SubscribeUpdateAccount = result.account
-                elif result.HasField("entry"):
-                    entry: SubscribeUpdateEntry = result.entry
-                elif result.HasField("slot"):
-                    result: SubscribeUpdateSlot = result.slot
-            else:
-                result = t.result()
-                raise RuntimeError("failed to get dragonsmouth source: %s" % result)
-```
+    async with session:
+        dragonsmouth_like_source = session.source
+        # result: SubscribeUpdate
+        async for result in dragonsmouth_like_source:
+            if result.HasField("block_meta"):
+                block_meta: SubscribeUpdateBlockMeta = result.block_meta
+            elif result.HasField("transaction"):
+                tx: SubscribeUpdateTransaction = result.transaction
+            elif result.HasField("account"):
+                account: SubscribeUpdateAccount = result.account
+            elif result.HasField("entry"):
+                entry: SubscribeUpdateEntry = result.entry
+            elif result.HasField("slot"):
+                result: SubscribeUpdateSlot = result.slot
+
+    # OUTSIDE THE SCOPE, YOU SHOULD NEVER USE `session` again.
+```
+
+
+At any point you can get a rough estimate if you are progression through the slot using `DragonsmouthAdapterSession.stats()` call:
+
+```python
+
+async with session:
+    stats: FumaroleSubscribeStats = session.stats()
+    print(f"{stats.log_committed_offset}, {stats.log_committable_offset}, {stats.max_slot_seen}")
+```
+
+`log_committed_offset` : what have been ACK so for to fumarole remote service.
+`log_committable_offset` : what can be ACK to next commit call.
+`max_slot_seen` : maximum slot seen in the inner fumarole client state -- not yet processed by your code.
+

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "yellowstone-fumarole-client"
-version = "0.1.0rc2"
+version = "0.2.0"
 homepage = "https://github.com/rpcpool/yellowstone-fumarole"
 repository = "https://github.com/rpcpool/yellowstone-fumarole"
 description = "Yellowstone Fumarole Python Client"
@@ -14,13 +14,8 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.13"
-click = "^8.1.7"
 grpcio = "^1.71.1"
-protobuf = "^5.29.1"
-toml = "^0.10.2"
-base58 = "^2.1.1"
-tabulate = "^0.9.0"
-asyncio = "^3.4.3"
+protobuf = "^6.32.0"
 pyyaml = "^6.0.2"
 
 [tool.poetry.group.test.dependencies]
@@ -30,6 +25,7 @@ pytest = "^8.3.4"
 grpcio-tools = "^1.68.1"
 black = "^24.10.0"
 pytest-asyncio = "^0.26.0"
+deptry = "^0.23.1"
 
 [build-system]
 requires = ["poetry-core"]

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/yellowstone_fumarole_client/__init__.py

@@ -3,18 +3,21 @@ import logging
 from yellowstone_fumarole_client.grpc_connectivity import (
     FumaroleGrpcConnector,
 )
-from typing import Dict, Optional
+from typing import AsyncGenerator, Optional
 from dataclasses import dataclass
-from . import config
+from yellowstone_fumarole_client.config import FumaroleConfig
 from yellowstone_fumarole_client.runtime.aio import (
     AsyncioFumeDragonsmouthRuntime,
-    FumaroleSM,
     DEFAULT_GC_INTERVAL,
     DEFAULT_SLOT_MEMORY_RETENTION,
     GrpcSlotDownloader,
 )
+from yellowstone_fumarole_client.runtime.state_machine import (
+    FumaroleSM,
+    FumeOffset,
+)
 from yellowstone_fumarole_proto.geyser_pb2 import SubscribeRequest, SubscribeUpdate
-from yellowstone_fumarole_proto.fumarole_v2_pb2 import (
+from yellowstone_fumarole_proto.fumarole_pb2 import (
     ControlResponse,
     VersionRequest,
     VersionResponse,
@@ -29,9 +32,11 @@ from yellowstone_fumarole_proto.fumarole_v2_pb2 import (
     CreateConsumerGroupRequest,
     CreateConsumerGroupResponse,
 )
-from yellowstone_fumarole_proto.fumarole_v2_pb2_grpc import FumaroleStub
+from yellowstone_fumarole_proto.fumarole_pb2_grpc import FumaroleStub
 import grpc
 
+from yellowstone_fumarole_client import config
+
 __all__ = [
     "FumaroleClient",
     "FumaroleConfig",
@@ -44,7 +49,7 @@ __all__ = [
 ]
 
 # Constants
-DEFAULT_DRAGONSMOUTH_CAPACITY = 10000
+DEFAULT_DRAGONSMOUTH_CAPACITY = 100000
 DEFAULT_COMMIT_INTERVAL = 5.0  # seconds
 DEFAULT_MAX_SLOT_DOWNLOAD_ATTEMPT = 3
 DEFAULT_CONCURRENT_DOWNLOAD_LIMIT_PER_TCP = 10
@@ -72,10 +77,23 @@ class FumaroleSubscribeConfig:
     # The interval at which to perform garbage collection on the slot memory.
     gc_interval: int = DEFAULT_GC_INTERVAL
 
-    # The retention period for slot memory in seconds.
+    # How many processed slot numbers to retain in memory to avoid duplication.
     slot_memory_retention: int = DEFAULT_SLOT_MEMORY_RETENTION
 
 
+@dataclass
+class FumaroleSubscribeStats:
+    """Commit/slot statistics for the Fumarole subscribe session."""
+
+    # Last committed log offset in Fumarole -- this is a low-level, implementation detail.
+    # NOTE: this should not be part as business logic, can change any time.
+    log_committed_offset: FumeOffset
+    # NOTE:: this is a low-level information, can change any time.
+    log_committable_offset: FumeOffset
+    # Max slot seen by the in the current session - does not mean it has been processed.
+    max_slot_seen: int
+
+
 # DragonsmouthAdapterSession
 @dataclass
 class DragonsmouthAdapterSession:
@@ -85,10 +103,31 @@ class DragonsmouthAdapterSession:
     sink: asyncio.Queue
 
     # The queue for receiving SubscribeUpdate from the dragonsmouth stream.
-    source: asyncio.Queue
+    source: AsyncGenerator[SubscribeUpdate, None]
 
     # The task handle for the fumarole runtime.
-    fumarole_handle: asyncio.Task
+    _fumarole_handle: asyncio.Task
+
+    _sm: FumaroleSM
+
+    async def __aenter__(self):
+        """Enter the session context."""
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        self.sink.shutdown()
+        self._fumarole_handle.cancel()
+
+    def stats(self) -> FumaroleSubscribeStats:
+        """Get low-level statistics of the Fumarole state-machine."""
+        commitable = self._sm.committable_offset
+        committed = self._sm.last_committed_offset
+        max_slot = self._sm.max_slot_detected
+        return FumaroleSubscribeStats(
+            log_committed_offset=committed,
+            log_committable_offset=commitable,
+            max_slot_seen=max_slot,
+        )
 
 
 # FumaroleClient
@@ -117,7 +156,7 @@ class FumaroleClient:
     async def version(self) -> VersionResponse:
         """Get the version of the Fumarole server."""
         request = VersionRequest()
-        response = await self.stub.version(request)
+        response = await self.stub.Version(request)
         return response
 
     async def dragonsmouth_subscribe(
@@ -162,12 +201,12 @@ class FumaroleClient:
                 try:
                     update = await fume_control_plane_q.get()
                     yield update
-                except asyncio.QueueShutDown:
+                except (asyncio.CancelledError, asyncio.QueueShutDown):
                     break
 
-        fume_control_plane_stream_rx: grpc.aio.StreamStreamMultiCallable = (
-            self.stub.Subscribe(control_plane_sink())
-        )
+        fume_control_plane_stream_rx: grpc.aio.StreamStreamCall = self.stub.Subscribe(
+            control_plane_sink()
+        )  # it's actually InterceptedStreamStreamCall, but grpc lib doesn't export it
 
         control_response: ControlResponse = await fume_control_plane_stream_rx.read()
         init = control_response.init
@@ -187,8 +226,12 @@ class FumaroleClient:
                         await fume_control_plane_rx_q.put(update)
                 except asyncio.QueueShutDown:
                     break
+                except asyncio.CancelledError:
+                    break
+                finally:
+                    fume_control_plane_rx_q.shutdown()
 
-        _cp_src_task = asyncio.create_task(control_plane_source())
+        control_plane_src_task = asyncio.create_task(control_plane_source())
 
         FumaroleClient.logger.debug(f"Control response: {control_response}")
 
@@ -219,12 +262,38 @@ class FumaroleClient:
             max_concurrent_download=config.concurrent_download_limit,
         )
 
-        fumarole_handle = asyncio.create_task(rt.run())
-        FumaroleClient.logger.debug(f"Fumarole handle created: {fumarole_handle}")
+        async def rt_run(rt):
+            async with rt as rt:
+                await rt.run()
+
+        rt_task = asyncio.create_task(rt_run(rt))
+
+        async def fumarole_overseer():
+            done, pending = await asyncio.wait(
+                [rt_task, control_plane_src_task], return_when=asyncio.FIRST_COMPLETED
+            )
+            for t in pending:
+                t.cancel()
+
+        fumarole_handle = asyncio.create_task(fumarole_overseer())
+
+        async def source_gen() -> AsyncGenerator[SubscribeUpdate, None]:
+            try:
+                while True:
+                    update = await dragonsmouth_outlet.get()
+                    yield update
+            except asyncio.CancelledError:
+                pass
+            except asyncio.Queue:
+                pass
+            finally:
+                dragonsmouth_outlet.shutdown()
+
         return DragonsmouthAdapterSession(
             sink=subscribe_request_queue,
-            source=dragonsmouth_outlet,
-            fumarole_handle=fumarole_handle,
+            source=source_gen(),
+            _fumarole_handle=fumarole_handle,
+            _sm=sm,
         )
 
     async def list_consumer_groups(

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/yellowstone_fumarole_client/config.py

@@ -1,14 +1,19 @@
 from dataclasses import dataclass
-from typing import Dict, Optional
+from typing import Dict, Literal, Optional
 import yaml
 
 
+SUPPORTED_COMPRESSION = ["gzip"]
+SupportedCompression = Literal["gzip"]
+
+
 @dataclass
 class FumaroleConfig:
     endpoint: str
     x_token: Optional[str] = None
     max_decoding_message_size_bytes: int = 512_000_000
     x_metadata: Dict[str, str] = None
+    response_compression: Optional[SupportedCompression] = None
 
     def __post_init__(self):
         self.x_metadata = self.x_metadata or {}
@@ -16,6 +21,14 @@ class FumaroleConfig:
     @classmethod
     def from_yaml(cls, fileobj) -> "FumaroleConfig":
         data = yaml.safe_load(fileobj)
+        response_compression = data.get(
+            "response_compression", cls.response_compression
+        )
+        if (
+            response_compression is not None
+            and response_compression not in SUPPORTED_COMPRESSION
+        ):
+            raise ValueError(f"response_compression must be in {SUPPORTED_COMPRESSION}")
         return cls(
             endpoint=data["endpoint"],
             x_token=data.get("x-token") or data.get("x_token"),
@@ -23,4 +36,7 @@ class FumaroleConfig:
                 "max_decoding_message_size_bytes", cls.max_decoding_message_size_bytes
             ),
             x_metadata=data.get("x-metadata", {}),
+            response_compression=data.get(
+                "response_compression", cls.response_compression
+            ),
         )

{yellowstone_fumarole_client-0.1.0rc2 → yellowstone_fumarole_client-0.2.0}/yellowstone_fumarole_client/grpc_connectivity.py

@@ -2,7 +2,7 @@ import logging
 from typing import Optional
 import grpc
 from yellowstone_fumarole_client.config import FumaroleConfig
-from yellowstone_fumarole_proto.fumarole_v2_pb2_grpc import FumaroleStub
+from yellowstone_fumarole_proto.fumarole_pb2_grpc import FumaroleStub
 
 X_TOKEN_HEADER = "x-token"
 
@@ -31,34 +31,8 @@ class TritonAuthMetadataPlugin(grpc.AuthMetadataPlugin):
         return _triton_sign_request(callback, self.x_token, None)
 
 
-def grpc_channel(endpoint: str, x_token=None, compression=None, *grpc_options):
-    options = [("grpc.max_receive_message_length", 111111110), *grpc_options]
-    if x_token is not None:
-        auth = TritonAuthMetadataPlugin(x_token)
-        # ssl_creds allow you to use our https endpoint
-        # grpc.ssl_channel_credentials with no arguments will look through your CA trust store.
-        ssl_creds = grpc.ssl_channel_credentials()
-
-        # call credentials will be sent on each request if setup with composite_channel_credentials.
-        call_creds: grpc.CallCredentials = grpc.metadata_call_credentials(auth)
-
-        # Combined creds will store the channel creds aswell as the call credentials
-        combined_creds = grpc.composite_channel_credentials(ssl_creds, call_creds)
-
-        return grpc.secure_channel(
-            endpoint,
-            credentials=combined_creds,
-            compression=compression,
-            options=options,
-        )
-    else:
-        return grpc.insecure_channel(endpoint, compression=compression, options=options)
-
-
 # Because of a bug in grpcio library, multiple inheritance of ClientInterceptor subclasses does not work.
 # You have to create a new class for each type of interceptor you want to use.
-
-
 class MetadataInterceptor(
     grpc.aio.UnaryStreamClientInterceptor,
     grpc.aio.StreamUnaryClientInterceptor,
@@ -166,6 +140,11 @@ class FumaroleGrpcConnector:
     async def connect(self, *grpc_options) -> FumaroleStub:
         options = [("grpc.max_receive_message_length", 111111110), *grpc_options]
         interceptors = MetadataInterceptor(self.config.x_metadata).interceptors()
+        compression = (
+            grpc.Compression.Gzip
+            if self.config.response_compression == "gzip"
+            else None
+        )
         if self.config.x_token is not None:
             auth = TritonAuthMetadataPlugin(self.config.x_token)
             # ssl_creds allow you to use our https endpoint
@@ -184,6 +163,7 @@ class FumaroleGrpcConnector:
                 self.endpoint,
                 credentials=combined_creds,
                 options=options,
+                compression=compression,
                 interceptors=interceptors,
             )
         else:
@@ -191,7 +171,10 @@ class FumaroleGrpcConnector:
                 "Using insecure channel without authentication"
             )
             channel = grpc.aio.insecure_channel(
-                self.endpoint, options=options, interceptors=interceptors
+                self.endpoint,
+                options=options,
+                interceptors=interceptors,
+                compression=compression,
             )
 
         return FumaroleStub(channel)