eth-streams-py 0.1.4__tar.gz

eth_streams_py-0.1.4/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.1
+Name: eth-streams-py
+Version: 0.1.4
+Requires-Python: >=3.10
+Requires-Dist: eth_protocols_py
+Requires-Dist: eth_typeshed_py
+Requires-Dist: eth_rpc_py
+Requires-Dist: tortoise-orm
+Provides-Extra: lint
+Requires-Dist: mypy; extra == "lint"
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: test
+Requires-Dist: pytest==7.4.1; extra == "test"
+Requires-Dist: pytest-cov==4.1.0; extra == "test"
+Requires-Dist: coverage[toml]==7.3.1; extra == "test"
+Provides-Extra: build
+Requires-Dist: build[virtualenv]==1.0.3; extra == "build"
+Provides-Extra: dev
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: eth-streams-py[lint]; extra == "dev"
+Requires-Dist: eth-streams-py[test]; extra == "dev"
+Requires-Dist: eth-streams-py[build]; extra == "dev"

eth_streams_py-0.1.4/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools>=68", "setuptools_scm[toml]>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "eth-streams-py"
+requires-python = ">=3.10"
+dynamic = ["version"]
+dependencies = [
+    "eth_protocols_py",
+    "eth_typeshed_py",
+    "eth_rpc_py",
+    "tortoise-orm",
+]
+
+# Enables the usage of setuptools_scm
+[tool.setuptools_scm]
+root = "../../"
+
+[project.optional-dependencies]
+lint = [
+    "mypy",
+    "ruff",
+]
+test = [
+    "pytest==7.4.1",
+    "pytest-cov==4.1.0",
+    "coverage[toml]==7.3.1",
+]
+build = [
+    "build[virtualenv]==1.0.3",
+]
+dev = [
+    "tox",
+    "eth-streams-py[lint]",
+    "eth-streams-py[test]",
+    "eth-streams-py[build]",
+]

eth_streams_py-0.1.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

eth_streams_py-0.1.4/setup.py

@@ -0,0 +1,13 @@
+import os
+
+from setuptools import find_packages, setup
+
+
+def get_version():
+    return os.getenv("PACKAGE_VERSION", "0.0.0")
+
+
+setup(
+    name="eth-streams-py",
+    version=get_version(),
+)

eth_streams_py-0.1.4/src/eth_streams/__init__.py

@@ -0,0 +1,50 @@
+# flake8: noqa
+from .context import Context
+from .coordinator import Coordinator, CoordinatorContext
+from .types import (
+    Address,
+    Callback,
+    Envelope,
+    FilterTopic,
+    Monitor,
+    Sink,
+    Source,
+    Task,
+    Topic,
+)
+
+Coordinator.model_rebuild()
+
+from .eth import BlockSource, EventBackfillSource, ReorgError
+from .pipeline import Pipeline, Stage, Transformer
+from .utils import init_db
+from .workers import Batch, Batcher, Combinator, Counter, Echo, Skipper, Throttler
+
+__all__ = [
+    "Address",
+    "Batch",
+    "Batcher",
+    "BlockSource",
+    "Callback",
+    "Combinator",
+    "Context",
+    "Coordinator",
+    "CoordinatorContext",
+    "Counter",
+    "Echo",
+    "Envelope",
+    "EventBackfillSource",
+    "FilterTopic",
+    "Monitor",
+    "Pipeline",
+    "ReorgError",
+    "Source",
+    "Sink",
+    "Skipper",
+    "Stage",
+    "Task",
+    "Throttler",
+    "Topic",
+    "Transformer",
+    "init_db",
+]

eth_streams_py-0.1.4/src/eth_streams/context.py

@@ -0,0 +1,20 @@
+from abc import ABC
+from typing import Any
+
+from pydantic import BaseModel
+
+# TODO: Maintain staged events before commiting them
+#       This allows for reorgs to be handled gracefully
+
+
+class Context(ABC, BaseModel):
+    """
+    This should store all state changes
+    """
+
+    @classmethod
+    def load(cls, json_data: str) -> "Context":
+        return cls.model_validate_json(json_data)
+
+    def dump(self) -> dict[str, Any]:
+        return self.model_dump()

eth_streams_py-0.1.4/src/eth_streams/coordinator.py

@@ -0,0 +1,102 @@
+import asyncio
+from collections import deque
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+from tortoise import Tortoise
+
+if TYPE_CHECKING:
+    from .types import Callback, Sink, Task
+
+
+class Coordinator(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+    )
+
+    callbacks: dict[str, "Callback"] = Field(default_factory=dict)
+    tasks: dict[str, "Task"] = Field(default_factory=dict)
+    sinks: dict[str, "Sink"] = Field(default_factory=dict)
+    concurrency: int = Field(default=10)
+    _semaphore: asyncio.Semaphore = PrivateAttr()
+
+    def model_post_init(self, __context):
+        self._semaphore = asyncio.Semaphore(self.concurrency)
+
+    @property
+    def semaphore(self):
+        return self._semaphore
+
+    def add_task(self, task: "Task"):
+        if task.name in self.tasks:
+            raise NameError("task already exists")
+        self.tasks[task.name] = task
+
+    def add_sink(self, sink: "Sink"):
+        if sink.name in self.sinks:
+            if self.sinks[sink.name] != sink:
+                raise NameError(f"Sink already exists: {sink.name}")
+        self.sinks[sink.name] = sink
+
+    def load_callbacks(self):
+        print("Loading all the strategies")
+
+    def load_callback(self, callback: "Callback"):
+        if callback.name in self.callbacks:
+            raise NameError("Callback already exists")
+        self.callbacks[callback.name] = callback
+
+    def on_safe_shutdown(self):
+        # TODO: callbacks are not used yet
+        for callback in self.callbacks.values():
+            asyncio.create_task(callback.shutdown(self.semaphore))
+
+    @property
+    def completed(self):
+        return all([c.stopped for c in self.sinks.values()])
+
+    async def run(self):
+        tasks = []
+
+        for task in self.tasks.values():
+            tasks.append(asyncio.create_task(task.run()))
+        await asyncio.gather(*tasks)
+
+    async def close(self):
+        await Tortoise.close_connections()
+
+    def __enter__(self):
+        CoordinatorContext.push_context_managed_coordinator(self)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        CoordinatorContext.push_context_managed_coordinator(self)
+
+
+class CoordinatorContext:
+    _context_managed_coordinators: deque[Coordinator] = deque()
+    implicits: dict[str, Any] = {}
+
+    @classmethod
+    def set_implicits(cls, **kwargs):
+        cls.implicits.update(kwargs)
+
+    @classmethod
+    def clear_implicits(cls):
+        cls.implicits = {}
+
+    @classmethod
+    def push_context_managed_coordinator(cls, dag: Coordinator):
+        cls._context_managed_coordinators.appendleft(dag)
+
+    @classmethod
+    def pop_context_managed_coordinator(cls) -> Coordinator | None:
+        dag = cls._context_managed_coordinators.popleft()
+        return dag
+
+    @classmethod
+    def get_current_coordinator(cls) -> Coordinator | None:
+        try:
+            return cls._context_managed_coordinators[0]
+        except IndexError:
+            return None

eth_streams_py-0.1.4/src/eth_streams/db.py

@@ -0,0 +1,29 @@
+from importlib.util import find_spec
+from typing import Optional
+
+from tortoise import Tortoise
+
+
+async def init_db(
+    load_schema: bool = False,
+    db_url: str = "sqlite://db.sqlite3",
+    modules: Optional[list[str]] = None,
+):
+    if not modules:
+        spec = find_spec("__main__")
+        if spec:
+            modules = [spec.name.replace("__main__", "models")]
+        else:
+            modules = []
+
+    await Tortoise.init(
+        db_url=db_url,
+        modules={
+            "models": [
+                "eth_streams.models",
+                *modules,
+            ],
+        },
+    )
+    if load_schema:
+        await Tortoise.generate_schemas()

eth_streams_py-0.1.4/src/eth_streams/eth/__init__.py

@@ -0,0 +1,11 @@
+from .blocks.source import BlockSource, ReorgError
+from .logs import ContractEventSink, EventBackfillSource, LogEventVertex, LogSubscriber
+
+__all__ = [
+    "BlockSource",
+    "ContractEventSink",
+    "EventBackfillSource",
+    "LogSubscriber",
+    "LogEventVertex",
+    "ReorgError",
+]

eth_streams_py-0.1.4/src/eth_streams/eth/blocks/__init__.py

@@ -0,0 +1,6 @@
+from .source import BlockSource, ReorgError
+
+__all__ = [
+    "BlockSource",
+    "ReorgError",
+]

eth_streams_py-0.1.4/src/eth_streams/eth/blocks/publisher.py

@@ -0,0 +1,30 @@
+from typing import Generic, TypeVar
+
+from eth_rpc import Block
+from eth_streams.models import Block as BlockModel
+from eth_streams.types import Envelope, Sink
+
+T = TypeVar("T", bound=Block)
+
+
+class BlockSink(Sink[list[Block]], Generic[T]):
+    async def notify(self, envelope: Envelope[list[Block]]):
+        """Converts a log to a contract event and writes it to the database"""
+        events = envelope.message
+        # TODO: add batching logic
+        batch = []
+        for block in events:
+            contract_event = BlockModel(
+                number=block.number,
+                timestamp=block.timestamp,
+                chain_id=block.network.chain_id,
+                hash=block.hash,
+                parent_block_hash=block.parent_hash,
+                hot_block=False,
+            )
+            batch.append(contract_event)
+        if batch:
+            await BlockModel.bulk_create(
+                batch,
+                ignore_conflicts=True,
+            )

eth_streams_py-0.1.4/src/eth_streams/eth/blocks/source.py

@@ -0,0 +1,60 @@
+from collections.abc import AsyncIterator
+
+from eth_rpc import Block, get_current_network
+from eth_rpc.types import BLOCK_STRINGS, Network
+from eth_streams.types import Source, Topic
+from eth_streams.utils import ExpiringDict, get_implicit
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ReorgError(BaseModel):
+    block_number: int
+
+
+class BlockSource(Source[ReorgError | Block], BaseModel):
+    __name__ = "block-source"
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    reorg_topic: Topic[ReorgError]
+
+    network: Network = Field(default_factory=get_current_network)
+    start_block: int | BLOCK_STRINGS = Field(
+        default_factory=lambda: get_implicit("start_block", "earliest")
+    )
+    reorg_distance: int = Field(5)
+    history: ExpiringDict[int, Block] = Field(
+        default_factory=lambda: ExpiringDict(100, 12 * 100)
+    )
+    restart_point: int | None = Field(None)
+
+    @property
+    def block_topic(self):
+        return self.default_topic
+
+    def __class_getitem__(self, network: Network):
+        self.network = network
+
+    async def _run(self) -> AsyncIterator[tuple[Topic, ReorgError | Block]]:
+        if self.start_block == "latest":
+            latest = await Block[self.network].get_number()
+            prev_block = await Block[self.network].load_by_number(
+                block_number=latest - 1
+            )
+        else:
+            prev_block = await Block[self.network].load_by_number(
+                block_number=self.start_block - 1
+            )
+        self.history[prev_block.number] = prev_block
+        current_block: int = prev_block.number + 1
+
+        while True:
+            async for block in Block[self.network].subscribe_from(
+                start_block=current_block
+            ):
+                self.history[block.number] = block
+                if self.history[block.number - 1].hash != block.parent_hash:
+                    # go back the reorg_distance to reindex those blocks
+                    current_block = block.number - self.reorg_distance
+                    yield (self.reorg_topic, ReorgError(block_number=current_block))
+                    break
+                yield (self.block_topic, block)

eth_streams_py-0.1.4/src/eth_streams/eth/eth_call_vertex.py

@@ -0,0 +1,32 @@
+from abc import abstractmethod
+from collections.abc import AsyncIterator
+from typing import Any, Generic, TypeVar, final
+
+from eth_rpc import ContractFunc
+from eth_streams.types import Envelope, Topic, Vertex
+
+ArgType = TypeVar("ArgType")
+ResponseType = TypeVar("ResponseType")
+U = TypeVar("U")
+
+
+class EthCallVertex(Vertex[Any, U], Generic[ArgType, ResponseType, U]):
+    func: ContractFunc[ArgType, ResponseType]
+    args: ArgType
+
+    @final
+    async def transform(
+        self, envelope: Envelope[Any]
+    ) -> AsyncIterator[tuple[Topic[U], U]]:
+        result = await self.func(self.args)
+        if await self.conditions(envelope, result):
+            response = await self.modify(envelope, result)
+            yield self.default_topic, response
+
+    @abstractmethod
+    async def conditions(self, envelope: Envelope[Any], result: ResponseType) -> bool:
+        """Specify what condition needs to be met by the eth_call to alert the user"""
+
+    @abstractmethod
+    async def modify(self, envelope: Envelope[Any], result: ResponseType) -> U:
+        """Specify what to return if the condition is met"""

eth_streams_py-0.1.4/src/eth_streams/eth/logs/__init__.py

@@ -0,0 +1,18 @@
+from .add_block import AddBlockVertex
+from .contract_event_vertex import ContractEventSink
+from .db_loader import DBLoader
+from .signals import AddAddress, RemoveAddress
+from .sources import EventBackfillSource, LogSubscriber
+from .vertex import BlockNumberToLogsVertex, LogEventVertex
+
+__all__ = [
+    "AddAddress",
+    "AddBlockVertex",
+    "BlockNumberToLogsVertex",
+    "ContractEventSink",
+    "DBLoader",
+    "EventBackfillSource",
+    "LogEventVertex",
+    "LogSubscriber",
+    "RemoveAddress",
+]

eth_streams_py-0.1.4/src/eth_streams/eth/logs/add_block.py

@@ -0,0 +1,56 @@
+from abc import abstractmethod
+from collections.abc import AsyncIterator
+from typing import ClassVar, Generic, TypeVar
+
+from eth_rpc import Block, Log
+from eth_rpc.types import Network
+from eth_streams.types import Envelope, Topic, Vertex
+from pydantic import BaseModel, Field
+
+T = TypeVar("T")
+
+
+class BlockWrap(BaseModel, Generic[T]):
+    data: T
+    block: Block
+
+
+class AddBlockVertex(Vertex[Log, BlockWrap[T]]):
+    """
+    A singleton class, depending on whether `with_tx_data` is set to True or False.
+    This wraps the transaction into a BaseModel with the Block data.
+    """
+
+    __instances: ClassVar[dict[bool, "AddBlockVertex"]] = {}
+
+    blocks: dict[tuple[int, Network], Block] = Field(default_factory=dict)
+    with_tx_data: bool = Field(default=True)
+
+    def __new__(cls, with_tx_data=True, **kwargs):
+        if cls.__instances.get(with_tx_data) is None:
+            cls.__instances[with_tx_data] = super().__new__(cls)
+        return cls.__instances[with_tx_data]
+
+    def __init__(self, **kwargs):
+        if not self.__dict__:
+            super().__init__(**kwargs)
+
+    @abstractmethod
+    def get_block_number(self, envelope: Envelope[T]) -> tuple[Block, Network]: ...
+
+    async def transform(
+        self, envelope: Envelope[T]
+    ) -> AsyncIterator[tuple[Topic[BlockWrap[T]], BlockWrap[T]]]:
+        key = self.get_block_number(envelope)
+        block_number, network = key
+
+        if key not in self.blocks:
+            self.blocks[key] = await Block[network].load_by_number(block_number)
+
+        yield (
+            self.default_topic,
+            BlockWrap(
+                data=envelope.message,
+                block=self.blocks[key],
+            ),
+        )

eth_streams_py-0.1.4/src/eth_streams/eth/logs/address_filter.py

@@ -0,0 +1,28 @@
+from collections.abc import AsyncIterator
+from typing import cast
+
+from eth_rpc import Log
+from eth_streams.types import Address, Batch, Envelope, Topic, Vertex
+from eth_typing import HexAddress, HexStr
+from pydantic import Field
+
+
+class AddressFilterVertex(Vertex[Log | Batch[Log] | Address, Log | Batch[Log]]):
+    addresses: set[HexAddress] = Field(default_factory=set)
+
+    async def transform(
+        self, envelope: Envelope[Log | Batch[Log] | Address]
+    ) -> AsyncIterator[tuple[Topic[Log | Batch[Log]], Log | Batch[Log]]]:
+        if isinstance(envelope.message, Address):
+            self.addresses.add(HexAddress(HexStr(envelope.message)))
+        elif isinstance(envelope.message, Log):
+            if envelope.message.address in self.addresses:
+                yield (self.default_topic, envelope.message)
+        elif isinstance(envelope.message, Batch):
+            batch = Batch[Log]()
+            for item in envelope.message:
+                item = cast(Log, item)
+                if item.address in self.addresses:
+                    batch.append(item)
+            if len(batch) > 0:
+                yield (self.default_topic, batch)

eth_streams_py-0.1.4/src/eth_streams/eth/logs/contract_event_vertex.py

@@ -0,0 +1,43 @@
+import asyncio
+from typing import ClassVar, Generic, Optional, TypeVar
+
+from eth_rpc import EventData
+from eth_streams.models import ContractEvent
+from pydantic import BaseModel
+
+from .db_loader import DBLoader
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class ContractEventSink(DBLoader[T, ContractEvent], Generic[T]):
+    __instance: ClassVar[Optional["ContractEventSink"]] = None
+
+    lock: ClassVar[asyncio.Lock] = asyncio.Lock()
+    model: type[ContractEvent] = ContractEvent
+
+    def __new__(cls, **kwargs):
+        # this allows us to create a singleton
+        if cls.__instance is None:
+            cls.__instance = super().__new__(cls)
+        return cls.__instance
+
+    def __init__(self, **kwargs):
+        if not self.__dict__:
+            super().__init__(**kwargs)
+
+    def _convert(self, event_data: EventData[T]) -> ContractEvent:
+        return ContractEvent(
+            chain=event_data.network.chain_id,
+            address=event_data.log.address,
+            block_number=event_data.log.block_number,
+            block_hash=event_data.log.block_hash,
+            transaction_index=event_data.log.transaction_index,
+            transaction_hash=event_data.log.transaction_hash,
+            log_index=event_data.log.log_index,
+            name=event_data.name,
+            topic0=event_data.log.topics[0] if len(event_data.log.topics) > 0 else "",
+            event_type=event_data.log.__class__.__name__,
+            event_data=event_data.event.model_dump(),
+            confirmed=True,
+        )

eth_streams_py-0.1.4/src/eth_streams/eth/logs/db_loader.py

@@ -0,0 +1,45 @@
+import asyncio
+import sqlite3
+from abc import abstractmethod
+from typing import ClassVar, Generic, TypeVar
+
+from eth_rpc import EventData, get_current_network
+from eth_rpc.types import Network
+from eth_streams.logger import logger
+from eth_streams.types import Envelope, Sink
+from eth_streams.workers import Batch
+from pydantic import Field
+from tortoise import Model
+
+T = TypeVar("T", bound=EventData)
+U = TypeVar("U", bound=Model)
+
+
+class DBLoader(Sink[EventData[T] | list[EventData[T]]], Generic[T, U]):
+    network: Network = Field(default_factory=get_current_network)
+    lock: ClassVar[asyncio.Lock] = asyncio.Lock()
+    model: type[U]
+
+    @abstractmethod
+    def _convert(self, event_data: EventData[T]) -> U:
+        """Convert the input type to the model"""
+
+    async def _notify(self, envelope: Envelope[EventData[T] | list[EventData[T]]]):
+        """Converts a log to a contract event and writes it to the database"""
+        events = envelope.message
+        # TODO: add batching logic
+        batch = []
+        if isinstance(events, Batch):
+            for event_data in events:
+                batch.append(self._convert(event_data))
+        elif isinstance(events, EventData):
+            batch.append(self._convert(events))
+        if batch:
+            logger.debug(f"WRITING BATCH: {len(batch)}")
+            try:
+                await self.model.bulk_create(
+                    batch,
+                    ignore_conflicts=True,
+                )
+            except (ValueError, sqlite3.ProgrammingError) as exc:
+                raise exc

eth_streams_py-0.1.4/src/eth_streams/eth/logs/signals.py

@@ -0,0 +1,6 @@
+class AddAddress(str):
+    """Add an address to the event filter"""
+
+
+class RemoveAddress(str):
+    """Remove an address to the event filter"""

eth_streams_py-0.1.4/src/eth_streams/eth/logs/sources/__init__.py

@@ -0,0 +1,7 @@
+from .backfill import EventBackfillSource
+from .subscriber import LogSubscriber
+
+__all__ = [
+    "EventBackfillSource",
+    "LogSubscriber",
+]