pfeed-bybit 0.0.1__tar.gz

pfeed_bybit-0.0.1/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.3
+Name: pfeed-bybit
+Version: 0.0.1
+Summary: Bybit data source plugin for pfeed: historical and live market data from Bybit.
+Author: softwareentrepreneer
+Author-email: softwareentrepreneer <softwareentrepreneer+pfeed-bybit@gmail.com>
+Requires-Dist: pfeed>=0.0.21
+Requires-Dist: pfund-bybit>=0.0.1
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# pfeed-bybit
+Bybit data source plugin for pfeed: historical and live market data from Bybit.

pfeed_bybit-0.0.1/README.md

@@ -0,0 +1,2 @@
+# pfeed-bybit
+Bybit data source plugin for pfeed: historical and live market data from Bybit.

pfeed_bybit-0.0.1/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "pfeed-bybit"
+version = "0.0.1"
+description = "Bybit data source plugin for pfeed: historical and live market data from Bybit."
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "pfeed>=0.0.21",
+    "pfund-bybit>=0.0.1",
+]
+
+[[project.authors]]
+name = "softwareentrepreneer"
+email = "softwareentrepreneer+pfeed-bybit@gmail.com"
+
+[project.entry-points."pfeed.sources"]
+bybit = "pfeed_bybit:Bybit"
+
+[tool.uv]
+exclude-newer = "0 days"
+
+[tool.uv.exclude-newer-package]
+uv_build = false
+pfund-blocks = false
+pfund-kit = false
+pfeed = false
+pfund-bybit = false
+
+[tool.pytest]
+markers = [
+    "network: Run tests that call Bybit's real servers",
+    "rate_limit: Run tests that can be rate-limited",
+]
+strict_markers = true
+addopts = ["--import-mode=importlib"]
+testpaths = ["tests"]
+
+[build-system]
+requires = ["uv_build>=0.13.7,<0.14.0"]
+build-backend = "uv_build"

pfeed_bybit-0.0.1/pyproject.toml.orig

@@ -0,0 +1,34 @@
+[project]
+name = "pfeed-bybit"
+version = "0.0.1"
+description = "Bybit data source plugin for pfeed: historical and live market data from Bybit."
+readme = "README.md"
+authors = [
+    { name = "softwareentrepreneer", email = "softwareentrepreneer+pfeed-bybit@gmail.com" }
+]
+requires-python = ">=3.13"
+dependencies = [
+    "pfeed>=0.0.21",
+    "pfund-bybit>=0.0.1"
+]
+
+[project.entry-points."pfeed.sources"]
+bybit = "pfeed_bybit:Bybit"
+
+[tool.uv]
+# FIXME: set it back to "14 days" when the package is stable
+exclude-newer = "0 days"
+exclude-newer-package = { uv_build = false, pfund-blocks = false, pfund-kit = false, pfeed = false, pfund-bybit = false }
+
+[build-system]
+requires = ["uv_build>=0.13.7,<0.14.0"]
+build-backend = "uv_build"
+
+[tool.pytest]
+markers = [
+    "network: Run tests that call Bybit's real servers",
+    "rate_limit: Run tests that can be rate-limited",
+]
+strict_markers = true
+addopts = ["--import-mode=importlib"]
+testpaths = ["tests"]

pfeed_bybit-0.0.1/src/pfeed_bybit/__init__.py

@@ -0,0 +1,5 @@
+from pfeed_bybit.source import Bybit
+
+__all__ = [
+    "Bybit",
+]

pfeed_bybit-0.0.1/src/pfeed_bybit/batch_api.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+if TYPE_CHECKING:
+    from httpx2 import Response
+    from pfund_blocks.entities.products.product_base import BaseProduct
+    from pfund_blocks.resolution import Resolution
+
+import datetime
+
+import polars as pl
+
+# NOTE: do NOT trust the catalog in https://public.bybit.com/trading/, BTC_USDT_FUTs are missing,
+# e.g. BTCUSDT-22AUG25 is missing in the catalog but its url is valid: https://public.bybit.com/trading/BTCUSDT-22AUG25/
+# the same applies to https://quote-saver.bycsi.com/orderbook/, 'spot' is missing in the catalog but its url is valid: https://quote-saver.bycsi.com/orderbook/spot
+"""
+Bybit's naming conventions for different contracts (non-spot, no options data):
+BTCUSDT = BTC_USDT_PERP
+BTCUSDT-22AUG25 = BTC_USDT_FUT
+BTCPERP = BTC_USDC_PERP
+# BTC-30AUG24 = BTC_USDC_FUT  (DEPRECATED, no USDC futures anymore)
+BTCUSD = BTC_USD_IPERP
+BTCUSDH25 = BTC_USD_IFUT
+"""
+
+
+# url: e.g. https://quote-saver.bycsi.com/orderbook/linear/BTCUSDT/2025-01-01_BTCUSDT_ob500.data.zip
+# NOTE: productId=orderbook/trade, bizType=contract/spot
+# See downloadable symbols:
+# https://www.bybit.com/x-api/quote/public/support/download/list-options?bizType={contract_or_spot}&productId={orderbook_or_trade}
+class BatchAPI:
+    """Custom API for downloading data from Bybit"""
+
+    DATA_NAMING_REGEX_PATTERNS: ClassVar[dict[str, str]] = {
+        "PERPETUAL": r"(USDT\/|PERP\/)$",  # USDT perp or USDC perp;
+        "FUTURE": r"-\d{2}[A-Z]{3}\d{2}\/$",  # USDC futures e.g. BTC-10NOV23/
+        "INVERSE-PERPETUAL": r"USD\/$",  # inverse perps;
+        "INVERSE-FUTURE": r"USD[A-Z]\d{2}\/$",  # inverse futures e.g. BTCUSDH24/
+        "CRYPTO": ".*",  # match everything since everything from https://public.bybit.com/spot is spot
+    }
+
+    @staticmethod
+    def _get_base_url(product: BaseProduct, resolution: Resolution) -> str:
+        if resolution.is_quote():
+            if product.is_crypto_spot():
+                return "https://quote-saver.bycsi.com/orderbook/spot"
+            elif product.is_inverse():
+                return "https://quote-saver.bycsi.com/orderbook/inverse"
+            else:
+                return "https://quote-saver.bycsi.com/orderbook/linear"
+        else:
+            if product.is_crypto_spot():
+                return "https://public.bybit.com/spot"
+            else:
+                return "https://public.bybit.com/trading"
+
+    @staticmethod
+    def _create_filename(
+        product: BaseProduct, resolution: Resolution, date: str | datetime.date
+    ) -> str:
+        if resolution.is_quote():
+            from pfund_kit.utils.temporal import convert_to_date
+
+            orderbook_levels = 200
+            # NOTE: somehow after this date, the orderbook (non-spot) levels are changed from 500 to 200
+            cutoff_date = "2025-08-21"
+            is_before_cutoff = convert_to_date(date) < convert_to_date(cutoff_date)
+            if not product.is_crypto_spot() and is_before_cutoff:
+                orderbook_levels = 500
+            return f"{date}_{product.symbol}_ob{orderbook_levels}.data.zip"
+        else:
+            if product.is_crypto_spot():
+                return f"{product.symbol}_{date}.csv.gz"
+            else:
+                return f"{product.symbol}{date}.csv.gz"
+
+    @staticmethod
+    def _get(url: str, params: dict[str, Any] | None = None) -> bytes | None:
+        """Returns the file content, or None if Bybit has no file at the url (404), e.g. a date with no data.
+
+        Any other error raises; the faucet retries the ones listed in Bybit.retry_policy.
+        """
+        import httpx2
+
+        response: Response = httpx2.get(url, params=params)
+        if response.status_code == httpx2.codes.NOT_FOUND:
+            return None
+        return response.raise_for_status().content
+
+    def _create_url(
+        self, product: BaseProduct, resolution: Resolution, date: str | datetime.date
+    ) -> str:
+        base_url = self._get_base_url(product, resolution)
+        filename = self._create_filename(product, resolution, date)
+        url = f"{base_url}/{product.symbol}/{filename}"
+        return url
+
+    def _convert_orderbook_data_to_df(self, zipped_data: bytes) -> pl.DataFrame:
+        from pfeed.utils.compression import decompress
+
+        data = decompress(zipped_data)
+        df = pl.read_ndjson(data)
+        # flatten nested structs (one level) to mimic pd.json_normalize
+        struct_cols = [
+            name for name, dtype in df.schema.items() if isinstance(dtype, pl.Struct)
+        ]
+        if struct_cols:
+            df = df.unnest(struct_cols)
+        return df
+
+    def get_data(
+        self, product: BaseProduct, resolution: Resolution, date: str | datetime.date
+    ) -> pl.LazyFrame | None:
+        if product.is_option():
+            raise NotImplementedError("Bybit does not provide options data")
+        url = self._create_url(product, resolution, date)
+        zipped_data = self._get(url)
+        if zipped_data is None:
+            return None
+        if not resolution.is_quote():
+            import io
+
+            # pl.read_csv handles gzip via magic bytes — no manual decompress
+            # infer_schema_length=None scans all rows, e.g. inverse products' int 'size'
+            # can contain values in scientific notation (1.099309e+06) deep into the file
+            return pl.read_csv(io.BytesIO(zipped_data), infer_schema_length=None).lazy()
+        else:
+            raise NotImplementedError("orderbook data is not supported yet")
+            # TODO: it's quote_L2 data, need to support converting to quote_L1, converting to different number of levels (e.g. 10quote_L1) etc.
+            # NOTE for the 'data.u' field in data:
+            # Update ID. Is a sequence. Occasionally, you'll receive "u"=1, which is a snapshot data due to the restart of the service.
+            # So please overwrite your local orderbook
+            # df: pl.DataFrame = self._convert_orderbook_data_to_df(zipped_data)
+            # return df
+
+    # def _get_downloadable_symbols(self, ptype: str, epdt: str):
+    #     '''Get external file names (e.g. BTCUSDT2022-10-04.csv.gz)'''
+    #     from bs4 import BeautifulSoup
+    #     url = '/'.join([self.URLS[ptype], epdt])
+    #     if res := self._get(url, frequency=1, num_retry=3):
+    #         soup = BeautifulSoup(res.text, 'html.parser')
+    #         efilenames = [node.get('href') for node in soup.find_all('a')]
+    #         return efilenames
+
+    # def _get_symbols_by_asset_type(self, ptype: str):
+    #     import re
+    #     from bs4 import BeautifulSoup
+    #     ptype = ptype.upper()
+    #     pattern = re.compile(self.DATA_NAMING_REGEX_PATTERNS[ptype])
+    #     url = self.URLS[ptype]
+    #     if res := self._get(url):
+    #         soup = BeautifulSoup(res.text, 'html.parser')
+    #         epdts = [node.get('href').replace('/', '') for node in soup.find_all('a') if pattern.search(node.get('href'))]
+    #         return epdts

pfeed_bybit-0.0.1/src/pfeed_bybit/market/feed.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
+
+if TYPE_CHECKING:
+    from pfeed.market.requests.base_request import MarketFeedBaseRequest
+    from pfund_blocks.typing import Message
+
+    from pfeed_bybit.source import Bybit
+
+import pfeed as pe
+import polars as pl
+from pfeed.market.data_model import MarketDataModel
+
+
+class BybitMarketFeed[PipelineT: bool = Literal[False]](
+    pe.MarketFeed["Bybit[PipelineT]", "MarketFeedBaseRequest", MarketDataModel, PipelineT]
+):
+    downloaded_data_date_cols: ClassVar[list[str]] = ["timestamp"]
+    # Bybit's batch API serves one file per product per day
+    download_dataflow_per_date: ClassVar[bool] = True
+
+    @staticmethod
+    def _normalize_downloaded_data(df: pl.LazyFrame) -> pl.LazyFrame:
+        """Normalize raw Bybit DataFrame into a consistent format.
+
+        Args:
+            df: DataFrame after `_standardize_date_column`
+
+        Returns:
+            Normalized DataFrame with:
+            - 'size' renamed to 'volume'
+            - 'side' mapped from Buy/Sell (case-insensitive) to 1/-1
+            - volume cast to float64
+        """
+        # Bybit spot CSVs ship 'volume' directly (and an 'rpi' column);
+        # non-spot still ships 'size'. strict=False tolerates either schema.
+        RENAMING_COLS: dict[str, str] = {"size": "volume"}
+        MAPPING_COLS: dict[str, int] = {"buy": 1, "sell": -1}
+        return df.rename(RENAMING_COLS, strict=False).with_columns(
+            # some products use "Buy"/"Sell" while some use "buy"/"sell";
+            # replace_strict errors on unexpected values rather than silently producing nulls
+            pl.col("side")
+            .str.to_lowercase()
+            .replace_strict(MAPPING_COLS, return_dtype=pl.Int8),
+            # cast to float64 to pass pandera schema validation — inverse products have int volume
+            pl.col("volume").cast(pl.Float64),
+        )
+
+    @staticmethod
+    def _download_impl(
+        request: MarketFeedBaseRequest, data_model: MarketDataModel
+    ) -> pl.LazyFrame | None:
+        data_source = cast("Bybit", request.data_source)
+        assert data_model.start_date == data_model.end_date, (
+            f"{data_source.name} download() only supports downloading data for a single day"
+        )
+        # data_model.resolution is the target resolution, download the raw data at data_resolution
+        return data_source.get_batch_api().get_data(
+            product=data_model.product,
+            resolution=request.data_resolution,
+            date=data_model.start_date,
+        )
+
+    @staticmethod
+    def _parse_streaming_message(msg: Message) -> Message:
+        from pfund_bybit.ws_api import BybitWebSocketAPI
+
+        # NOTE: already parsed by the websocket API, only unparsed with Ray, where parsing is left to the workers
+        is_parsed_already = msg['data'] is not None
+        if is_parsed_already:
+            return msg
+        channel = cast(str, msg['channel'])
+        message = cast(dict[str, Any], msg['message'])
+        return {**msg, 'data': BybitWebSocketAPI.parser.parse_message(channel, message)}

pfeed_bybit-0.0.1/src/pfeed_bybit/source.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar, Literal
+
+if TYPE_CHECKING:
+    from pfeed.base.feed import BaseFeed
+
+import httpx2
+from pfeed.dataflow.retry_policy import RetryPolicy
+from pfeed.enums import (
+    DataAccessType,
+    DataCategory,
+    DataProviderType,
+    DataType,
+)
+from pfeed.source import BaseSource, SourceMetadata
+from pfund_blocks.enums.env import Environment
+from pfund_bybit.adapter import BybitAdapter
+
+from pfeed_bybit.batch_api import BatchAPI
+from pfeed_bybit.market.feed import BybitMarketFeed
+from pfeed_bybit.stream_api import StreamAPI
+
+
+class Bybit[PipelineT: bool = Literal[False]](BaseSource[PipelineT]):
+    Feeds: ClassVar[dict[DataCategory, type[BaseFeed]]] = {
+        DataCategory.MARKET_DATA: BybitMarketFeed,
+    }
+    metadata: ClassVar[SourceMetadata] = SourceMetadata(
+        name="BYBIT",
+        url="https://www.bybit.com",
+        data_categories={
+            DataCategory.MARKET_DATA: {
+                DataType.TICK: [
+                    "PERPETUAL",
+                    "INVERSE-PERPETUAL",
+                    "FUTURE",
+                    "INVERSE-FUTURE",
+                    "CRYPTO",
+                ],
+            },
+        },
+        provider_type=DataProviderType.VENUE,
+        access_type=DataAccessType.FREE,
+        start_date="2020-01-01",
+    )
+    # timeouts and connection errors; HTTP error responses (e.g. 5xx) are not retried
+    retry_policy: ClassVar[RetryPolicy] = RetryPolicy(retry_on=(httpx2.TransportError,))
+    # Bybit's symbol conventions come from pfund-bybit's adapter, shared with pfund's Bybit venue;
+    # products are created by BaseSource.create_product() with it
+    adapter: ClassVar[BybitAdapter] = BybitAdapter()  # ty: ignore[invalid-mutable-override]
+
+    market_feed: BybitMarketFeed[PipelineT]
+    # NOTE: class-level defaults instead of setting them in __init__, so they exist before BaseSource.__init__() creates the feeds,
+    # and Bybit doesn't override __init__, which would hide BaseSource's overloads that type the pipeline mode
+    _batch_api: BatchAPI | None = None
+    _stream_api: StreamAPI | None = None
+
+    def get_batch_api(self) -> BatchAPI:
+        if self._batch_api is None:
+            self._batch_api = BatchAPI()
+        return self._batch_api
+
+    def get_stream_api(
+        self,
+        env: Literal[
+            Environment.PAPER,
+            Environment.LIVE,
+            "PAPER",
+            "LIVE",
+        ],
+    ) -> StreamAPI:
+        environment = Environment(env.upper())
+        if environment not in (Environment.PAPER, Environment.LIVE):
+            raise ValueError(f"{self.name} streaming only supports PAPER and LIVE, got {environment}")
+        if self._stream_api is None:
+            self._stream_api = StreamAPI(env=environment)
+        elif self._stream_api.env != environment:
+            raise ValueError(
+                f"{self.name} stream API is already in {self._stream_api.env}, cannot stream in {environment}"
+            )
+        return self._stream_api

pfeed_bybit-0.0.1/src/pfeed_bybit/stream_api.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, cast
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from pfeed.base.requests import BaseStreamRequest
+    from pfeed.market.requests import MarketFeedStreamRequest
+    from pfund_blocks.entities.products.product_base import BaseProduct
+    from pfund_blocks.enums.env import Environment
+    from pfund_blocks.resolution import Resolution
+    from pfund_blocks.typing import Message
+
+import asyncio
+import logging
+
+from pfeed.streaming.stream_api import BaseStreamAPI
+from pfund_bybit.venue import Bybit
+from pfund_bybit.ws_api import BybitWebSocketAPI
+
+
+class StreamAPI(BaseStreamAPI):
+    """Simple wrapper of exchange's websocket APIs to connect to public channels,
+    one websocket API per category, since Bybit has one public endpoint per category.
+    """
+
+    def __init__(self, env: Literal[Environment.PAPER, Environment.LIVE]):
+        super().__init__(env)
+        self._logger = logging.getLogger("pfeed.bybit")
+        self._ws_apis: dict[Bybit.Category, BybitWebSocketAPI] = {}
+
+    def _add_ws_api(self, product: BaseProduct) -> BybitWebSocketAPI:
+        category = Bybit.get_category(product)
+        # NOTE: only categories with channels have a websocket API, since connect() raises if there are no channels
+        if category not in self._ws_apis:
+            ws_api = BybitWebSocketAPI(env=self._env, category=category)
+            ws_api._logger = self._logger
+            self._ws_apis[category] = ws_api
+        return self._ws_apis[category]
+
+    def get_supported_resolutions(self, product: BaseProduct) -> list[Resolution]:
+        return BybitWebSocketAPI.get_supported_resolutions(product)
+
+    def add_channel(self, request: BaseStreamRequest) -> None:
+        req = cast("MarketFeedStreamRequest", request)
+        ws_api = self._add_ws_api(req.product)
+        channel = ws_api.create_market_data_channel(req.product, req.data_resolution)
+        ws_api.add_channel(channel, channel_type="public")
+
+    def set_callback(self, callback: Callable[[Message], Awaitable[None] | None], *, skip_parse_message: bool = False) -> None:
+        super().set_callback(callback, skip_parse_message=skip_parse_message)
+        for ws_api in self._ws_apis.values():
+            ws_api.set_callback(callback, skip_parse_message=skip_parse_message)
+
+    async def connect(self) -> None:
+        for ws_api in self._ws_apis.values():
+            await ws_api.connect()
+
+    async def wait_until_disconnected(self) -> None:
+        await asyncio.gather(*[ws_api.wait_until_disconnected() for ws_api in self._ws_apis.values()])
+
+    async def disconnect(self) -> None:
+        await asyncio.gather(*[ws_api.disconnect() for ws_api in self._ws_apis.values()])