onetick-py 1.162.2__py3-none-any.whl

onetick/py/sources/parquet.py

@@ -0,0 +1,168 @@
+import onetick.py as otp
+from onetick.py.otq import otq
+
+from onetick.py.core.source import Source
+
+from .. import utils
+
+from .common import update_node_tick_type
+
+
+class ReadParquet(Source):
+    def __init__(
+        self,
+        parquet_file_path=None,
+        where=None,
+        time_assignment="end",
+        discard_fields=None,
+        fields=None,
+        symbol_name_field=None,
+        symbol=utils.adaptive,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        schema=None,
+        **kwargs,
+    ):
+        """
+        Read ticks from Parquet file
+
+        Parameters
+        ----------
+        parquet_file_path: str
+            Specifies the path or URL of the Parquet file to read.
+        where: str, None
+            Specifies a criterion for selecting the ticks to propagate.
+        time_assignment: str
+            Timestamps of the ticks created by `ReadParquet` are set to the start/end of the query or to the given
+            tick field depending on the `time_assignment` parameter.
+            Possible values are `start` and `end` (for `_START_TIME` and `_END_TIME`) or a field name.
+            Default: `end`
+        discard_fields: list, str, None
+            A list of fields (`list` or comma-separated string) to be discarded from the output ticks.
+        fields: list, str, None
+            A list of fields (`list` or comma-separated string) to be picked from the output ticks.
+            The opposite to `discard_fields`.
+        symbol_name_field: str, None
+            Field that is expected to contain the symbol name.
+            When this parameter is set and one or more symbols containing time series
+            (i.e. `[dbname]::[time series name]` and not just `[dbname]::`) are bound to the query or to this EP,
+            only rows belonging to those symbols will be propagated.
+        symbol: str, list of str, :class:`Source`, :class:`query`, :py:func:`eval query <onetick.py.eval>`
+            Symbol(s) from which data should be taken.
+        tick_type: str
+            Tick type.
+            Default: ANY.
+        start: :py:class:`otp.datetime <onetick.py.datetime>`
+            Start time for tick generation. By default the start time of the query will be used.
+        end: :py:class:`otp.datetime <onetick.py.datetime>`
+            End time for tick generation. By default the end time of the query will be used.
+        schema: dict
+            Dictionary of columns names with their types.
+            You should set schema manually, if you want to use fields in `onetick-py` query description
+            before its execution.
+        kwargs:
+            Deprecated. Use ``schema`` instead.
+            Dictionary of columns names with their types.
+
+        See also
+        --------
+        | **READ_FROM_PARQUET** OneTick event processor
+        | :py:meth:`onetick.py.Source.write_parquet`
+
+        Examples
+        --------
+        Simple Parquet file read:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file")
+        >>> otp.run(data)  # doctest: +SKIP
+
+        Read Parquet file and filter fields:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file", fields=["some_field", "another_field"])
+        >>> otp.run(data)  # doctest: +SKIP
+
+        Read Parquet file and filter rows:
+
+        >>> data = otp.ReadParquet("/path/to/parquet/file", where="PRICE > 20")
+        >>> otp.run(data)  # doctest: +SKIP
+        """
+        if self._try_default_constructor(schema=schema, **kwargs):
+            return
+
+        if parquet_file_path is None:
+            raise ValueError("Missing required parameter `parquet_file_path`")
+
+        if discard_fields and fields:
+            raise ValueError("`discard_fields` and `fields` cannot be specified at the same time")
+
+        if where is None:
+            where = ""
+
+        if discard_fields is None:
+            discard_fields = ""
+        elif isinstance(discard_fields, list):
+            discard_fields = ",".join(discard_fields)
+
+        if fields is None:
+            fields = ""
+        elif isinstance(fields, list):
+            fields = ",".join(fields)
+
+        if time_assignment in {"start", "end"}:
+            time_assignment = f"_{time_assignment}_time".upper()
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=lambda: self.base_ep(
+                db=db,
+                tick_type=tick_type,
+                parquet_file_path=parquet_file_path,
+                where=where,
+                time_assignment=time_assignment,
+                discard_fields=discard_fields,
+                fields=fields,
+                symbol_name_field=symbol_name_field,
+            ),
+            schema=schema,
+            **kwargs,
+        )
+
+    def base_ep(
+        self,
+        parquet_file_path,
+        where=None,
+        time_assignment="end",
+        discard_fields=None,
+        fields=None,
+        symbol_name_field=None,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+    ):
+        if not hasattr(otq, "ReadFromParquet"):
+            raise RuntimeError("Current version of OneTick don't support READ_FROM_PARQUET EP")
+
+        node_kwargs = {}
+        if symbol_name_field:
+            node_kwargs["symbol_name_field"] = symbol_name_field
+
+        src = Source(
+            otq.ReadFromParquet(
+                parquet_file_path=parquet_file_path,
+                time_assignment=time_assignment,
+                where=where,
+                discard_fields=discard_fields,
+                fields=fields,
+                **node_kwargs,
+            )
+        )
+
+        if db and tick_type:
+            update_node_tick_type(src, tick_type, db)
+
+        return src

onetick/py/sources/pit.py

@@ -0,0 +1,191 @@
+from typing import List, Union
+
+import onetick.py as otp
+from onetick.py.backports import Literal
+from onetick.py.core.source import Source
+from onetick.py.otq import otq
+
+from .. import types as ott
+from .. import utils
+from ..compatibility import is_supported_point_in_time
+from .common import update_node_tick_type
+
+
+def PointInTime(  # NOSONAR
+    source: 'Source',
+    times: List[Union[str, ott.datetime]],
+    offsets: List[int],
+    offset_type: Literal['time_msec', 'num_ticks'] = 'time_msec',
+    db=utils.adaptive_to_default,
+    tick_type=utils.adaptive,
+    symbol=utils.adaptive_to_default,
+    start=utils.adaptive,
+    end=utils.adaptive,
+) -> 'Source':
+    """
+    This function propagates ticks from ``source`` that are offset by
+    the specified number of milliseconds or by the specified number of ticks
+    relative to the timestamps specified in ``times``.
+
+    Output tick may be generated for each specified timestamp and offset pair.
+
+    If ``source`` doesn't have a tick with specified timestamp and offset, then output tick is not generated.
+
+    Fields **TICK_TIME** and **OFFSET** are also added to the output ticks,
+    specifying original timestamp of the tick and the offset that was specified for it.
+
+    Note
+    ----
+    In order for this method to have reasonable performance,
+    the set of queried timestamps has to be relatively small.
+
+    In other words, the points in time, which the user is interested in,
+    have to be quite few in order usage of this method to be justified.
+
+    Parameters
+    ----------
+    source:
+        The source from which the data will be queried.
+    times:
+        List of timestamps to get query ticks from.
+    offsets:
+        List of integers specifying offsets for each timestamp.
+    offset_type: 'time_msec' or 'num_ticks'
+        The type of offset: number of milliseconds or the number of ticks.
+    db: str
+        This parameter is used to set database part of a tick type of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    tick_type: str
+        This parameter is used to set tick type part of a tick type of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    symbol: str
+        This parameter is used to set bound symbol of the OneTick graph node.
+        Should not be specified in most cases,
+        default value will be deduced depending on configuration and query structure.
+    start:
+        Can be used to specify custom start time for this source.
+        By default start time of the query will be inherited when running the query.
+    end:
+        Can be used to specify custom start time for this source.
+        By default end time of the query will be inherited when running the query.
+
+    See also
+    --------
+    | **POINT_IN_TIME** OneTick event processor
+    | :meth:`onetick.py.Source.point_in_time`
+    | :func:`onetick.py.join_by_time`
+
+    Examples
+    --------
+
+    Quotes for testing:
+
+    .. testcode::
+
+       qte = otp.Ticks(ASK_PRICE=[20, 21, 22, 23, 24, 25], BID_PRICE=[20, 21, 22, 23, 24, 25])
+       print(otp.run(qte))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE
+       0 2003-12-01 00:00:00.000         20         20
+       1 2003-12-01 00:00:00.001         21         21
+       2 2003-12-01 00:00:00.002         22         22
+       3 2003-12-01 00:00:00.003         23         23
+       4 2003-12-01 00:00:00.004         24         24
+       5 2003-12-01 00:00:00.005         25         25
+
+    Getting quotes exactly at specified timestamps:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 1000), '20031201000000.003'],
+                              offsets=[0])
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.001         21         21 2003-12-01 00:00:00.001       0
+       1 2003-12-01 00:00:00.003         23         23 2003-12-01 00:00:00.003       0
+
+
+    Offset may be positive or negative.
+    If several offsets are specified, several output ticks may be generated for a single timestamp:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 3000)],
+                              offsets=[0, 1])
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.003         23         23 2003-12-01 00:00:00.003       0
+       1 2003-12-01 00:00:00.003         24         24 2003-12-01 00:00:00.004       1
+
+    By default the number of milliseconds is used as an offset.
+    You can also specify the number of ticks as an offset:
+
+    .. testcode::
+       :skipif: not is_supported_point_in_time()
+
+       data = otp.PointInTime(qte,
+                              times=[otp.dt(2003, 12, 1, 0, 0, 0, 3000)],
+                              offsets=[-1, 1],
+                              offset_type='num_ticks')
+       print(otp.run(data))
+
+    .. testoutput::
+
+                            Time  ASK_PRICE  BID_PRICE               TICK_TIME  OFFSET
+       0 2003-12-01 00:00:00.003         22         22 2003-12-01 00:00:00.002      -1
+       1 2003-12-01 00:00:00.003         24         24 2003-12-01 00:00:00.004       1
+    """
+    if not is_supported_point_in_time():
+        raise RuntimeError('PointInTime event processor is not supported on this OneTick version')
+
+    res = otp.Source(_symbols=symbol, _start=start, _end=end)
+
+    times = [
+        t if isinstance(t, str) else ott._format_datetime(t, '%Y%m%d%H%M%S.%f', add_nano_suffix=True)
+        for t in times or []
+    ]
+
+    if offset_type not in ('time_msec', 'num_ticks'):
+        raise ValueError(f"Wrong value for parameter 'offset_type': {offset_type}")
+
+    query_name = source._store_in_tmp_otq(
+        res._tmp_otq,
+        operation_suffix='point_in_time',
+        # set default symbol, even if it's not set by user, symbol's value doesn't matter in this case
+        symbols=otp.config.get('default_symbol', 'ANY')
+    )
+    otq_query = f'THIS::{query_name}'
+
+    pit_params = dict(
+        otq_query=otq_query,
+        offset_type=offset_type.upper(),
+        offsets=','.join(map(str, offsets)),
+        times=','.join(map(str, times)),
+    )
+    res.source(otq.PointInTime(**pit_params))
+
+    res.schema.set(**source.schema)
+    res.schema.update(
+        **{
+            'TICK_TIME': otp.nsectime,
+            'OFFSET': int,
+        }
+    )
+
+    update_node_tick_type(res, tick_type, db)
+
+    return res