onetick-py 1.177.0__py3-none-any.whl

onetick/py/sources/order_book.py

@@ -0,0 +1,271 @@
+from typing import Iterable, Callable
+
+import onetick.py as otp
+from onetick.py.docs.utils import docstring
+
+from ..aggregations.order_book import (
+    OB_SNAPSHOT_DOC_PARAMS,
+    OB_SNAPSHOT_WIDE_DOC_PARAMS,
+    OB_SNAPSHOT_FLAT_DOC_PARAMS,
+    OB_SUMMARY_DOC_PARAMS,
+    OB_SIZE_DOC_PARAMS,
+    OB_VWAP_DOC_PARAMS,
+    OB_NUM_LEVELS_DOC_PARAMS,
+)
+from ..aggregations.functions import (
+    ob_snapshot, ob_snapshot_wide, ob_snapshot_flat, ob_summary, ob_size, ob_vwap, ob_num_levels,
+)
+from .. import utils
+
+from .data_source import DataSource, DATA_SOURCE_DOC_PARAMS
+
+
+class _ObSource(DataSource):
+    OB_AGG_FUNC: Callable
+    OB_AGG_PARAMS: Iterable
+    _PROPERTIES = DataSource._PROPERTIES + ['_ob_agg']
+
+    def __init__(self, db=None, schema=None, **kwargs):
+        if self._try_default_constructor(schema=schema, **kwargs):
+            return
+
+        ob_agg_params = {
+            param.name: kwargs.pop(param.name, param.default)
+            for _, param in self.OB_AGG_PARAMS
+        }
+
+        symbol_param = kwargs.get('symbol')
+        symbols_param = kwargs.get('symbols')
+
+        if symbol_param and symbols_param:
+            raise ValueError(
+                'You have set the `symbol` and `symbols` parameters together, it is not allowed. '
+                'Please, clarify parameters'
+            )
+
+        symbols = symbol_param if symbol_param else symbols_param
+        tmp_otq = None
+
+        # Use bound symbols only in case, if db not passed
+        use_bound_symbols = not db and symbols and symbols is not utils.adaptive
+        if use_bound_symbols:
+            symbols, tmp_otq = self._cross_symbol_convert(symbols, kwargs.get('symbol_date'))
+
+            if symbols_param:
+                del kwargs['symbols']
+
+            kwargs['symbol'] = None
+
+        self._ob_agg = self.__class__.OB_AGG_FUNC(**ob_agg_params)
+
+        if kwargs.get('schema_policy') in [DataSource.POLICY_MANUAL, DataSource.POLICY_MANUAL_STRICT]:
+            self._ob_agg.disable_ob_input_columns_validation()
+
+        if use_bound_symbols:
+            self._ob_agg.set_bound_symbols(symbols)
+
+        super().__init__(db=db, schema=schema, **kwargs)
+
+        ob_agg_output_schema = self._ob_agg._get_output_schema(otp.Empty())
+
+        if getattr(self._ob_agg, 'show_full_detail', None):
+            self.schema.update(**ob_agg_output_schema)
+        else:
+            self.schema.set(**ob_agg_output_schema)
+
+        if tmp_otq:
+            self._tmp_otq.merge(tmp_otq)
+
+    def base_ep(self, *args, **kwargs):
+        src = super().base_ep(*args, **kwargs)
+        return self._ob_agg.apply(src)
+
+    def _base_ep_for_cross_symbol(self, *args, **kwargs):
+        src = super()._base_ep_for_cross_symbol(*args, **kwargs)
+        return self._ob_agg.apply(src)
+
+
+@docstring(parameters=OB_SNAPSHOT_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObSnapshot(_ObSource):
+    """
+    Construct a source providing order book snapshot for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_snapshot`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_snapshot`
+    | :func:`onetick.py.agg.ob_snapshot`
+    | **OB_SNAPSHOT** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObSnapshot(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=1) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  PRICE             UPDATE_TIME  SIZE  LEVEL  BUY_SELL_FLAG
+    0 2003-12-04    2.0 2003-12-01 00:00:00.003     6      1              1
+    1 2003-12-04    5.0 2003-12-01 00:00:00.004     7      1              0
+    """
+    OB_AGG_FUNC = ob_snapshot
+    OB_AGG_PARAMS = OB_SNAPSHOT_DOC_PARAMS
+
+
+@docstring(parameters=OB_SNAPSHOT_WIDE_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObSnapshotWide(_ObSource):
+    """
+    Construct a source providing order book wide snapshot for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_snapshot_wide`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_snapshot_wide`
+    | :func:`onetick.py.agg.ob_snapshot_wide`
+    | **OB_SNAPSHOT_WIDE** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObSnapshotWide(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=1) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  BID_PRICE         BID_UPDATE_TIME  BID_SIZE  ASK_PRICE         ASK_UPDATE_TIME  ASK_SIZE  LEVEL
+    0 2003-12-03        5.0 2003-12-01 00:00:00.004         7        2.0 2003-12-01 00:00:00.003         6      1
+    """
+    OB_AGG_FUNC = ob_snapshot_wide
+    OB_AGG_PARAMS = OB_SNAPSHOT_WIDE_DOC_PARAMS
+
+
+@docstring(parameters=OB_SNAPSHOT_FLAT_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObSnapshotFlat(_ObSource):
+    """
+    Construct a source providing order book flat snapshot for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_snapshot_flat`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_snapshot_flat`
+    | :func:`onetick.py.agg.ob_snapshot_flat`
+    | **OB_SNAPSHOT_FLAT** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObSnapshotFlat(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=1) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  BID_PRICE1        BID_UPDATE_TIME1  BID_SIZE1  ASK_PRICE1        ASK_UPDATE_TIME1  ASK_SIZE1
+    0 2003-12-03         5.0 2003-12-01 00:00:00.004          7         2.0 2003-12-01 00:00:00.003          6
+    """
+    OB_AGG_FUNC = ob_snapshot_flat
+    OB_AGG_PARAMS = OB_SNAPSHOT_FLAT_DOC_PARAMS
+
+
+@docstring(parameters=OB_SUMMARY_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObSummary(_ObSource):
+    """
+    Construct a source providing order book summary for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_summary`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_summary`
+    | :func:`onetick.py.agg.ob_summary`
+    | **OB_SUMMARY** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObSummary(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=1) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  BID_PRICE  BID_SIZE  BID_VWAP  BEST_BID_PRICE  WORST_BID_SIZE  NUM_BID_LEVELS  ASK_SIZE\
+              ASK_VWAP  BEST_ASK_PRICE  WORST_ASK_PRICE  NUM_ASK_LEVELS
+    0 2003-12-04        NaN         7       5.0             5.0             NaN               1         6\
+           2.0             2.0              2.0               1
+    """
+    OB_AGG_FUNC = ob_summary
+    OB_AGG_PARAMS = OB_SUMMARY_DOC_PARAMS
+
+
+@docstring(parameters=OB_SIZE_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObSize(_ObSource):
+    """
+    Construct a source providing number of order book levels for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_size`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_size`
+    | :func:`onetick.py.agg.ob_size`
+    | **OB_SIZE** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObSize(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=10) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  ASK_VALUE  BID_VALUE
+    0 2003-12-01      84800      64500
+    """
+    OB_AGG_FUNC = ob_size
+    OB_AGG_PARAMS = OB_SIZE_DOC_PARAMS
+
+
+@docstring(parameters=OB_VWAP_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObVwap(_ObSource):
+    """
+    Construct a source providing the size-weighted price
+    computed over a specified number of order book levels for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_vwap`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_vwap`
+    | :func:`onetick.py.agg.ob_vwap`
+    | **OB_VWAP** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObVwap(db='SOME_DB', tick_type='PRL', symbols='AA', max_levels=10) # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  ASK_VALUE  BID_VALUE
+    0 2003-12-01     23.313   23.20848
+    """
+    OB_AGG_FUNC = ob_vwap
+    OB_AGG_PARAMS = OB_VWAP_DOC_PARAMS
+
+
+@docstring(parameters=OB_NUM_LEVELS_DOC_PARAMS + DATA_SOURCE_DOC_PARAMS)
+class ObNumLevels(_ObSource):
+    """
+    Construct a source providing the number of levels in the order book for a given ``db``.
+    This is just a shortcut for
+    :class:`~onetick.py.DataSource` + :func:`~onetick.py.agg.ob_num_levels`.
+
+    See also
+    --------
+    | :class:`onetick.py.DataSource`
+    | :meth:`onetick.py.Source.ob_num_levels`
+    | :func:`onetick.py.agg.ob_num_levels`
+    | **OB_NUM_LEVELS** OneTick event processor
+
+    Examples
+    ---------
+
+    >>> data = otp.ObNumLevels(db='SOME_DB', tick_type='PRL', symbols='AA') # doctest: +SKIP
+    >>> otp.run(data) # doctest: +SKIP
+            Time  ASK_VALUE  BID_VALUE
+    0 2003-12-01        248         67
+    """
+    OB_AGG_FUNC = ob_num_levels
+    OB_AGG_PARAMS = OB_NUM_LEVELS_DOC_PARAMS

onetick/py/sources/parquet.py

@@ -0,0 +1,168 @@
+import onetick.py as otp
+from onetick.py.otq import otq
+
+from onetick.py.core.source import Source
+
+from .. import utils
+
+from .common import update_node_tick_type
+
+
+class ReadParquet(Source):
+    def __init__(
+        self,
+        parquet_file_path=None,
+        where=None,
+        time_assignment="end",
+        discard_fields=None,
+        fields=None,
+        symbol_name_field=None,
+        symbol=utils.adaptive,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        schema=None,
+        **kwargs,
+    ):
+        """
+        Read ticks from Parquet file
+
+        Parameters
+        ----------
+        parquet_file_path: str
+            Specifies the path or URL of the Parquet file to read.
+        where: str, None
+            Specifies a criterion for selecting the ticks to propagate.
+        time_assignment: str
+            Timestamps of the ticks created by `ReadParquet` are set to the start/end of the query or to the given
+            tick field depending on the `time_assignment` parameter.
+            Possible values are `start` and `end` (for `_START_TIME` and `_END_TIME`) or a field name.
+            Default: `end`
+        discard_fields: list, str, None
+            A list of fields (`list` or comma-separated string) to be discarded from the output ticks.
+        fields: list, str, None
+            A list of fields (`list` or comma-separated string) to be picked from the output ticks.
+            The opposite to `discard_fields`.
+        symbol_name_field: str, None
+            Field that is expected to contain the symbol name.
+            When this parameter is set and one or more symbols containing time series
+            (i.e. `[dbname]::[time series name]` and not just `[dbname]::`) are bound to the query or to this EP,
+            only rows belonging to those symbols will be propagated.
+        symbol: str, list of str, :class:`Source`, :class:`query`, :py:func:`eval query <onetick.py.eval>`
+            Symbol(s) from which data should be taken.
+        tick_type: str
+            Tick type.
+            Default: ANY.
+        start: :py:class:`otp.datetime <onetick.py.datetime>`
+            Start time for tick generation. By default the start time of the query will be used.
+        end: :py:class:`otp.datetime <onetick.py.datetime>`
+            End time for tick generation. By default the end time of the query will be used.
+        schema: dict
+            Dictionary of columns names with their types.
+            You should set schema manually, if you want to use fields in `onetick-py` query description
+            before its execution.
+        kwargs:
+            Deprecated. Use ``schema`` instead.
+            Dictionary of columns names with their types.
+
+        See also
+        --------
+        | **READ_FROM_PARQUET** OneTick event processor
+        | :py:meth:`onetick.py.Source.write_parquet`
+
+        Examples
+        --------
+        Simple Parquet file read:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file")
+        >>> otp.run(data)  # doctest: +SKIP
+
+        Read Parquet file and filter fields:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file", fields=["some_field", "another_field"])
+        >>> otp.run(data)  # doctest: +SKIP
+
+        Read Parquet file and filter rows:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file", where="PRICE > 20")
+        >>> otp.run(data)  # doctest: +SKIP
+        """
+        if self._try_default_constructor(schema=schema, **kwargs):
+            return
+
+        if parquet_file_path is None:
+            raise ValueError("Missing required parameter `parquet_file_path`")
+
+        if discard_fields and fields:
+            raise ValueError("`discard_fields` and `fields` cannot be specified at the same time")
+
+        if where is None:
+            where = ""
+
+        if discard_fields is None:
+            discard_fields = ""
+        elif isinstance(discard_fields, list):
+            discard_fields = ",".join(discard_fields)
+
+        if fields is None:
+            fields = ""
+        elif isinstance(fields, list):
+            fields = ",".join(fields)
+
+        if time_assignment in {"start", "end"}:
+            time_assignment = f"_{time_assignment}_time".upper()
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=lambda: self.base_ep(
+                db=db,
+                tick_type=tick_type,
+                parquet_file_path=parquet_file_path,
+                where=where,
+                time_assignment=time_assignment,
+                discard_fields=discard_fields,
+                fields=fields,
+                symbol_name_field=symbol_name_field,
+            ),
+            schema=schema,
+            **kwargs,
+        )
+
+    def base_ep(
+        self,
+        parquet_file_path,
+        where=None,
+        time_assignment="end",
+        discard_fields=None,
+        fields=None,
+        symbol_name_field=None,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+    ):
+        if not hasattr(otq, "ReadFromParquet"):
+            raise RuntimeError("Current version of OneTick don't support READ_FROM_PARQUET EP")
+
+        node_kwargs = {}
+        if symbol_name_field:
+            node_kwargs["symbol_name_field"] = symbol_name_field
+
+        src = Source(
+            otq.ReadFromParquet(
+                parquet_file_path=parquet_file_path,
+                time_assignment=time_assignment,
+                where=where,
+                discard_fields=discard_fields,
+                fields=fields,
+                **node_kwargs,
+            )
+        )
+
+        if db and tick_type:
+            update_node_tick_type(src, tick_type, db)
+
+        return src

onetick/py/sources/pit.py

@@ -0,0 +1,191 @@
+from typing import List, Union
+
+import onetick.py as otp
+from onetick.py.backports import Literal
+from onetick.py.core.source import Source
+from onetick.py.otq import otq
+
+from .. import types as ott
+from .. import utils
+from ..compatibility import is_supported_point_in_time
+from .common import update_node_tick_type
+
+
+def PointInTime(  # NOSONAR
+    source: 'Source',
+    times: List[Union[str, ott.datetime]],
+    offsets: List[int],
+    offset_type: Literal['time_msec', 'num_ticks'] = 'time_msec',
+    db=utils.adaptive_to_default,
+    tick_type=utils.adaptive,
+    symbol=utils.adaptive_to_default,
+    start=utils.adaptive,
+    end=utils.adaptive,
+) -> 'Source':
+    """
+    This function propagates ticks from ``source`` that are offset by
+    the specified number of milliseconds or by the specified number of ticks
+    relative to the timestamps specified in ``times``.
+
+    Output tick may be generated for each specified timestamp and offset pair.
+
+    If ``source`` doesn't have a tick with specified timestamp and offset, then output tick is not generated.
+
+    Fields **TICK_TIME** and **OFFSET** are also added to the output ticks,
+    specifying original timestamp of the tick and the offset that was specified for it.
+
+    Note
+    ----
+    In order for this method to have reasonable performance,
+    the set of queried timestamps has to be relatively small.
+
+    In other words, the points in time, which the user is interested in,
+    have to be quite few in order usage of this method to be justified.
+
+    Parameters
+    ----------
+    source:
+        The source from which the data will be queried.
+    times:
+        List of timestamps to get query ticks from.
+    offsets:
+        List of integers specifying offsets for each timestamp.
+    offset_type: 'time_msec' or 'num_ticks'
+        The type of offset: number of milliseconds or the number of ticks.
+    db: str
+        This parameter is used to set database part of a tick type of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    tick_type: str
+        This parameter is used to set tick type part of a tick type of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    symbol: str
+        This parameter is used to set bound symbol of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    start:
+        Can be used to specify custom start time for this source.
+        By default start time of the query will be inherited when running the query.
+    end:
+        Can be used to specify custom start time for this source.
+        By default end time of the query will be inherited when running the query.
+
+    See also
+    --------
+    | **POINT_IN_TIME** OneTick event processor
+    | :meth:`onetick.py.Source.point_in_time`
+    | :func:`onetick.py.join_by_time`
+
+    Examples
+    --------
+
+    Quotes for testing:
+
+    .. testcode::
+
+       qte = otp.Ticks(ASK_PRICE=[20, 21, 22, 23, 24, 25], BID_PRICE=[20, 21, 22, 23, 24, 25])
+       print(otp.run(qte))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE
+       0 2003-12-01 00:00:00.000         20         20
+       1 2003-12-01 00:00:00.001         21         21
+       2 2003-12-01 00:00:00.002         22         22
+       3 2003-12-01 00:00:00.003         23         23
+       4 2003-12-01 00:00:00.004         24         24
+       5 2003-12-01 00:00:00.005         25         25
+
+    Getting quotes exactly at specified timestamps:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 1000), '20031201000000.003'],
+                              offsets=[0])
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.001         21         21 2003-12-01 00:00:00.001       0
+       1 2003-12-01 00:00:00.003         23         23 2003-12-01 00:00:00.003       0
+
+
+    Offset may be positive or negative.
+    If several offsets are specified, several output ticks may be generated for a single timestamp:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 3000)],
+                              offsets=[0, 1])
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.003         23         23 2003-12-01 00:00:00.003       0
+       1 2003-12-01 00:00:00.003         24         24 2003-12-01 00:00:00.004       1
+
+    By default the number of milliseconds is used as an offset.
+    You can also specify the number of ticks as an offset:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 3000)],
+                              offsets=[-1, 1],
+                              offset_type='num_ticks')
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.003         22         22 2003-12-01 00:00:00.002      -1
+       1 2003-12-01 00:00:00.003         24         24 2003-12-01 00:00:00.004       1
+    """
+    if not is_supported_point_in_time():
+        raise RuntimeError('PointInTime event processor is not supported on this OneTick version')
+
+    res = otp.Source(_symbols=symbol, _start=start, _end=end)
+
+    times = [
+        t if isinstance(t, str) else ott._format_datetime(t, '%Y%m%d%H%M%S.%f', add_nano_suffix=True)
+        for t in times or []
+    ]
+
+    if offset_type not in ('time_msec', 'num_ticks'):
+        raise ValueError(f"Wrong value for parameter 'offset_type': {offset_type}")
+
+    query_name = source._store_in_tmp_otq(
+        res._tmp_otq,
+        operation_suffix='point_in_time',
+        # set default symbol, even if it's not set by user, symbol's value doesn't matter in this case
+        symbols=otp.config.get('default_symbol', 'ANY')
+    )
+    otq_query = f'THIS::{query_name}'
+
+    pit_params = dict(
+        otq_query=otq_query,
+        offset_type=offset_type.upper(),
+        offsets=','.join(map(str, offsets)),
+        times=','.join(map(str, times)),
+    )
+    res.source(otq.PointInTime(**pit_params))
+
+    res.schema.set(**source.schema)
+    res.schema.update(
+        **{
+            'TICK_TIME': otp.nsectime,
+            'OFFSET': int,
+        }
+    )
+
+    update_node_tick_type(res, tick_type, db)
+
+    return res