onetick-py 1.177.0__py3-none-any.whl

onetick/py/sources/snapshots.py

@@ -0,0 +1,419 @@
+import onetick.py as otp
+from onetick.py.otq import otq
+
+from onetick.py.core.source import Source
+
+from .. import utils
+
+from .common import update_node_tick_type
+
+
+class ReadSnapshot(Source):
+    def __init__(
+        self,
+        snapshot_name='VALUE',
+        snapshot_storage='memory',
+        allow_snapshot_absence=False,
+        symbol=utils.adaptive,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        schema=None,
+        **kwargs,
+    ):
+        """
+        Reads ticks for a specified symbol name and snapshot name from global memory storage or
+        from a memory mapped file.
+
+        These ticks should be written there by the :py:meth:`onetick.py.Source.save_snapshot` event processor.
+        Ticks with an empty symbol name (for example, those after merging the time series of different symbols)
+        are saved under **CEP_SNAPSHOT::** symbol name, so for reading such ticks a dummy database
+        with the name **CEP_SNAPSHOT** must be configured in the database locator file.
+
+        Parameters
+        ----------
+        snapshot_name: str
+            The name that was specified in :py:meth:`onetick.py.Source.save_snapshot` as a ``snapshot_name``
+            during saving.
+
+            Default: ``VALUE``
+        snapshot_storage: 'memory' or 'memory_mapped_file'
+            This parameter specifies the place of storage of the snapshot. Possible options are:
+
+            * `memory` - the snapshot is stored in the dynamic (heap) memory of the process
+              that ran (or is still running) the :py:meth:`onetick.py.Source.save_snapshot` for the snapshot.
+            * `memory_mapped_file` - the snapshot is stored in a memory mapped file.
+              For each symbol to get the location of the snapshot in the file system, ``ReadSnapshot`` looks at
+              the **SAVE_SNAPSHOT_DIR** parameter value in the locator section for the database of the symbol.
+
+            Default: `memory`
+        allow_snapshot_absence: bool
+            If specified, the EP does not display an error about missing snapshot
+            if the snapshot has not been saved or is still being saved.
+
+            Default: `False`
+        symbol: str, list of str, :class:`Source`, :class:`query`, :py:func:`eval query <onetick.py.eval>`
+            Symbol(s) from which data should be taken.
+        tick_type: str
+            Tick type.
+            Default: ANY.
+        start: :py:class:`otp.datetime <onetick.py.datetime>`
+            Start time for tick generation. By default the start time of the query will be used.
+        end: :py:class:`otp.datetime <onetick.py.datetime>`
+            End time for tick generation. By default the end time of the query will be used.
+        schema: dict
+            Dictionary of columns names with their types.
+
+            .. warning::
+                You should set schema manually, if you want to use fields in `onetick-py` query description
+                before its execution.
+
+        See also
+        --------
+        | **READ_SNAPSHOT** OneTick event processor
+        | :py:class:`onetick.py.ShowSnapshotList`
+        | :py:class:`onetick.py.FindSnapshotSymbols`
+        | :py:meth:`onetick.py.Source.save_snapshot`
+        | :py:meth:`onetick.py.Source.join_with_snapshot`
+
+        Examples
+        --------
+        Read snapshot from memory:
+
+        >>> src = otp.ReadSnapshot(snapshot_name='some_snapshot')
+        >>> otp.run(src)  # doctest: +SKIP
+                Time  PRICE  SIZE               TICK_TIME
+        0 2003-12-01  100.2   500 2003-12-01 00:00:00.000
+        1 2003-12-01   98.3   250 2003-12-01 00:00:00.001
+        2 2003-12-01  102.5   400 2003-12-01 00:00:00.002
+
+        You can specify schema manually in order to reference snapshot fields while constructing query via `onetick-py`:
+
+        >>> src = otp.ReadSnapshot(
+        ...     snapshot_name='some_snapshot', schema={'PRICE': float, 'SIZE': int},
+        ... )
+        >>> src['VOLUME'] = src['PRICE'] * src['SIZE']
+
+        Read snapshot for specified database and symbol name:
+
+        >>> src = otp.ReadSnapshot(snapshot_name='some_snapshot', db='DB', symbol='AAA')
+
+        Read snapshot from memory mapped file:
+
+        >>> src = otp.ReadSnapshot(
+        ...     snapshot_name='some_snapshot', snapshot_storage='memory_mapped_file', db='DB',
+        ... )
+        """
+        if self._try_default_constructor(schema=schema):
+            return
+
+        if not hasattr(otq, "ReadSnapshot"):
+            raise RuntimeError("Current version of OneTick don't support READ_SNAPSHOT EP")
+
+        if schema is None:
+            schema = {}
+
+        schema.update({'TICK_TIME': otp.nsectime})
+
+        if snapshot_storage not in ['memory', 'memory_mapped_file']:
+            raise ValueError('`snapshot_storage` must be one of "memory", "memory_mapped_file"')
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=lambda: self.base_ep(
+                db=db,
+                tick_type=tick_type,
+                snapshot_name=snapshot_name,
+                snapshot_storage=snapshot_storage,
+                allow_snapshot_absence=allow_snapshot_absence,
+            ),
+            schema=schema,
+            **kwargs,
+        )
+
+    def base_ep(
+        self,
+        snapshot_name='VALUE',
+        snapshot_storage='memory',
+        allow_snapshot_absence=False,
+        db=utils.adaptive_to_default,
+        tick_type=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+    ):
+        snapshot_storage = snapshot_storage.upper()
+
+        src = Source(
+            otq.ReadSnapshot(
+                snapshot_name=snapshot_name,
+                snapshot_storage=snapshot_storage,
+                allow_snapshot_absence=allow_snapshot_absence,
+            )
+        )
+
+        if db or tick_type:
+            update_node_tick_type(src, tick_type, db)
+
+        return src
+
+
+class ShowSnapshotList(Source):
+    def __init__(self, snapshot_storage='all', **kwargs):
+        """
+        Outputs all snapshots names for global memory storage and/or for memory mapped files.
+        These snapshots should be written there by the :py:meth:`onetick.py.Source.save_snapshot`.
+        Output ticks have ``SNAPSHOT_NAME``, ``STORAGE_TYPE`` and ``DB_NAME`` fields.
+
+        Parameters
+        ----------
+        snapshot_storage: 'memory' or 'memory_mapped_file'
+            This parameter specifies the place of storage of the snapshot. Possible options are:
+
+            * `memory` - the snapshot is stored in the dynamic (heap) memory of the process
+              that ran (or is still running) the :py:meth:`onetick.py.Source.save_snapshot` for the snapshot.
+            * `memory_mapped_file` - the snapshot is stored in a memory mapped file.
+              For each symbol to get the location of the snapshot in the file system, ``ReadSnapshot`` looks at
+              the **SAVE_SNAPSHOT_DIR** parameter value in the locator section for the database of the symbol.
+            * `all` - shows both, `memory`, `memory_mapped_file` snapshots.
+
+            Default: `all`
+
+        See also
+        --------
+        | **SHOW_SNAPSHOT_LIST** OneTick event processor
+        | :py:class:`onetick.py.ReadSnapshot`
+        | :py:class:`onetick.py.FindSnapshotSymbols`
+        | :py:meth:`onetick.py.Source.save_snapshot`
+        | :py:meth:`onetick.py.Source.join_with_snapshot`
+
+        Examples
+        --------
+        List snapshots from all snapshot storage types:
+
+        >>> src = otp.ShowSnapshotList(snapshot_storage='all')  # doctest: +SKIP
+        >>> otp.run(src)  # doctest: +SKIP
+                Time SNAPSHOT_NAME        STORAGE_TYPE        DB_NAME
+        0 2003-12-01    snapshot_1              MEMORY        DEMO_L1
+        1 2003-12-01    snapshot_2              MEMORY        DEMO_L1
+        2 2003-12-01    snapshot_3  MEMORY_MAPPED_FILE  SNAPSHOT_DEMO
+
+        List snapshots from memory:
+
+        >>> src = otp.ShowSnapshotList(snapshot_storage='memory')  # doctest: +SKIP
+        >>> otp.run(src)  # doctest: +SKIP
+                Time SNAPSHOT_NAME        STORAGE_TYPE        DB_NAME
+        0 2003-12-01    snapshot_1              MEMORY        DEMO_L1
+        1 2003-12-01    snapshot_2              MEMORY        DEMO_L1
+
+        List snapshots from memory mapped files:
+
+        >>> src = otp.ShowSnapshotList(snapshot_storage='memory_mapped_file')  # doctest: +SKIP
+        >>> otp.run(src)  # doctest: +SKIP
+                Time SNAPSHOT_NAME        STORAGE_TYPE        DB_NAME
+        0 2003-12-01    snapshot_3  MEMORY_MAPPED_FILE  SNAPSHOT_DEMO
+        """
+        if 'schema' not in kwargs:
+            kwargs['schema'] = {'SNAPSHOT_NAME': str, 'STORAGE_TYPE': str, 'DB_NAME': str}
+
+        if self._try_default_constructor(**kwargs):
+            return
+
+        if not hasattr(otq, "ShowSnapshotList"):
+            raise RuntimeError("Current version of OneTick don't support SHOW_SNAPSHOT_LIST EP")
+
+        if snapshot_storage not in ['memory', 'memory_mapped_file', 'all']:
+            raise ValueError('`snapshot_storage` must be one of "memory", "memory_mapped_file", "all"')
+
+        super().__init__(
+            _symbols=utils.adaptive,
+            _base_ep_func=lambda: self.base_ep(snapshot_storage=snapshot_storage),
+            **kwargs,
+        )
+
+    def base_ep(self, snapshot_storage='all'):
+        snapshot_storage = snapshot_storage.upper()
+
+        src = Source(otq.ShowSnapshotList(snapshot_storage=snapshot_storage))
+        update_node_tick_type(src, otp.adaptive, otp.config.get('default_db', 'LOCAL'))
+
+        return src
+
+
+class FindSnapshotSymbols(Source):
+    def __init__(
+        self,
+        snapshot_name='VALUE',
+        snapshot_storage='memory',
+        pattern='%',
+        symbology=None,
+        show_original_symbols=False,
+        discard_on_match=False,
+        db=None,
+        tick_type=None,
+        **kwargs,
+    ):
+        """
+        Takes snapshot's name and storage type and produces a union of all symbols in the snapshot.
+        It also optionally translates the symbols into a user-specified symbology and
+        propagates those that match a specified name pattern as a tick with a single field: **SYMBOL_NAME**.
+
+        If symbol name translation is performed, symbol names in native snapshot can optionally be propagated
+        as another tick field: ``ORIGINAL_SYMBOL_NAME``, in which case symbols with missing translations
+        are also propagated. Symbol name translation is performed as of the query symbol date and requires
+        the reference database to be configured.
+
+        The name of the database to query is extracted from the input symbol.
+        For example, if an input symbol is ``TAQ::``, the symbols from the **TAQ** database will be returned.
+
+        The ``pattern`` parameter can contain the special characters ``%`` (matches any number of characters) and
+        ``_`` (matches any character).
+        For example, the pattern ``I_M%`` returns any symbol beginning with I and has M as its third letter.
+
+        If the ``discard_on_match`` parameter is set to ``True``, the names that do not match the pattern
+        will be propagated.
+
+        The timestamps of the ticks created by ``FindSnapshotSymbols`` are set to the start time of the query.
+
+        Parameters
+        ----------
+        snapshot_name: str
+            The name that was specified in :py:meth:`onetick.py.Source.save_snapshot` as a ``snapshot_name``
+            during saving.
+
+            Default: `VALUE`
+        snapshot_storage: 'memory' or 'memory_mapped_file'
+            This parameter specifies the place of storage of the snapshot. Possible options are:
+
+            * `memory` - the snapshot is stored in the dynamic (heap) memory of the process
+              that ran (or is still running) the :py:meth:`onetick.py.Source.save_snapshot` for the snapshot.
+            * `memory_mapped_file` - the snapshot is stored in a memory mapped file.
+              For each symbol to get the location of the snapshot in the file system, ``ReadSnapshot`` looks at
+              the **SAVE_SNAPSHOT_DIR** parameter value in the locator section for the database of the symbol.
+
+            Default: `memory`
+        pattern: str
+            The pattern for symbol selection. It can contain special characters ``%`` (matches any number of characters)
+            and ``_`` (matches any character). To avoid this special interpretation of characters ``%`` and ``_``,
+            the ``\\`` character must be added in front of them. Note that if symbol name translation is required,
+            translated rather than original symbol names are checked to match the name pattern.
+
+             Default: `%`
+        symbology: Optional[str]
+            The destination symbology for a symbol name translation, the latter being performed,
+            if destination symbology is not empty and is different from that of the queried database.
+        show_original_symbols: bool
+            Switches original symbol name propagation as a tick field after symbol name translation is performed.
+            Note that if this parameter is set to true, database symbols with missing translations are also propagated.
+
+            Default: `False`
+        discard_on_match: bool
+            When set to true only ticks that did not match the filter are propagated,
+            otherwise ticks that satisfy the filter condition are propagated.
+
+            Default: `False`
+        db: str, optional
+            Database to use for snapshots search query.
+
+            It's required to fill either this parameter or explicitly specify
+            the ``symbols`` parameter of :py:func:`otp.run <onetick.py.run>`,
+            or have bound symbols on any node of your query
+        tick_type: str, optional
+            Tick type.
+
+        See also
+        --------
+        | **FIND_SNAPSHOT_SYMBOLS** OneTick event processor
+        | :py:class:`onetick.py.ReadSnapshot`
+        | :py:class:`onetick.py.ShowSnapshotList`
+        | :py:meth:`onetick.py.Source.save_snapshot`
+        | :py:meth:`onetick.py.Source.join_with_snapshot`
+
+        Examples
+        --------
+        Find symbols for snapshot `some_snapshot` in database `S1`:
+
+        >>> src = otp.FindSnapshotSymbols(snapshot_name='some_snapshot', db='S1')
+        >>> otp.run(src, symbols='S1::')  # doctest: +SKIP
+                Time SYMBOL_NAME
+        0 2003-12-01    S1::AAPL
+        1 2003-12-01    S1::AAAA
+        2 2003-12-01    S1::MSFT
+
+        Use ``pattern`` parameter to filter symbol names:
+
+        >>> src = otp.FindSnapshotSymbols(snapshot_name='some_snapshot', db='S1', pattern='A%')
+        >>> otp.run(src, symbols='S1::')  # doctest: +SKIP
+                Time SYMBOL_NAME
+        0 2003-12-01    S1::AAPL
+        1 2003-12-01    S1::AAAA
+
+        Select symbol names not matched by pattern:
+
+        >>> src = otp.FindSnapshotSymbols(
+        ...     snapshot_name='some_snapshot', db='S1', pattern='A%', discard_on_match=True,
+        ... )
+        >>> otp.run(src, symbols='S1::')  # doctest: +SKIP
+                Time SYMBOL_NAME
+        0 2003-12-01    S1::MSFT
+        """
+        if 'schema' not in kwargs:
+            kwargs['schema'] = {'SYMBOL_NAME': str}
+
+            if show_original_symbols:
+                kwargs['schema'].update({'ORIGINAL_SYMBOL_NAME': str})
+
+        if self._try_default_constructor(**kwargs):
+            return
+
+        if not hasattr(otq, "FindSnapshotSymbols"):
+            raise RuntimeError("Current version of OneTick don't support SHOW_SNAPSHOT_LIST EP")
+
+        if symbology is None:
+            symbology = ''
+
+        if snapshot_storage not in ['memory', 'memory_mapped_file']:
+            raise ValueError('`snapshot_storage` must be one of "memory", "memory_mapped_file"')
+
+        super().__init__(
+            _base_ep_func=lambda: self.base_ep(
+                snapshot_name=snapshot_name,
+                snapshot_storage=snapshot_storage,
+                pattern=pattern,
+                symbology=symbology,
+                show_original_symbols=show_original_symbols,
+                discard_on_match=discard_on_match,
+                db=db,
+                tick_type=tick_type,
+            ),
+            **kwargs,
+        )
+
+    def base_ep(
+        self,
+        snapshot_name='VALUE',
+        snapshot_storage='memory',
+        pattern='%',
+        symbology='',
+        show_original_symbols=False,
+        discard_on_match=False,
+        db=None,
+        tick_type=None,
+    ):
+        snapshot_storage = snapshot_storage.upper()
+
+        src = Source(
+            otq.FindSnapshotSymbols(
+                snapshot_name=snapshot_name,
+                snapshot_storage=snapshot_storage,
+                pattern=pattern,
+                symbology=symbology,
+                show_original_symbols=show_original_symbols,
+                discard_on_match=discard_on_match,
+            )
+        )
+        update_node_tick_type(src, tick_type, db)
+
+        return src

onetick/py/sources/split_query_output_by_symbol.py

@@ -0,0 +1,198 @@
+import os
+
+from functools import partial
+
+import onetick.py as otp
+from onetick.py.otq import otq
+
+from onetick.py.core.source import Source
+
+from .. import utils
+
+from .common import update_node_tick_type
+
+
+class SplitQueryOutputBySymbol(Source):
+    def __init__(self,
+                 query=None,
+                 symbol_field=None,
+                 single_invocation=False,
+                 db=utils.adaptive_to_default,
+                 tick_type=utils.adaptive,
+                 start=utils.adaptive,
+                 end=utils.adaptive,
+                 symbols=utils.adaptive,
+                 schema=None,
+                 **kwargs):
+        """
+        A data source used to dispatch output ticks,
+        resulting after execution of the specified query,
+        according to the values of the specified field in those ticks.
+
+        Each replica of this EP, corresponding to a particular bound or unbound symbol, thus,
+        propagates resulting ticks of the specified query,
+        with values of the specified field in those ticks equal to that symbol.
+
+        Note, that database name part of symbols is not taken into account.
+
+        Parameters
+        ----------
+        query:
+            Specify query to execute.
+        symbol_field:
+            Specifies the field in the resulting ticks of the underlying query,
+            according to values of which those ticks are dispatched.
+        single_invocation:
+            By default, the underlying query is executed once
+            per symbol batch and per execution thread of the containing query.
+            If this parameter is set to True, the underlying query is executed once
+            regardless of the batch size and the number of CPU cores utilized.
+
+            The former option should be the preferred (hence the default) one, as it reduces memory overhead,
+            while the latter one might be chosen to speed-up the overall execution.
+
+            Note, that if the underlying query is a CEP query, than this option has no effect,
+            as there is a single batch and a single thread anyway.
+        db:
+        tick_type:
+            Tick type to set on the OneTick's graph node.
+            Can be used to specify database name with tick type or tick type only.
+
+            By default setting these parameters is not required, database is usually set
+            with parameter ``symbols`` or in :py:func:`otp.run <onetick.py.run>`.
+        start:
+            Custom start time of the source.
+            If set, will override the value specified in :py:func:`otp.run <onetick.py.run>`.
+        end:
+            Custom end time of the source.
+            If set, will override the value specified in :py:func:`otp.run <onetick.py.run>`.
+        symbols:
+            Symbol(s) from which data should be taken.
+            If set, will override the value specified in :py:func:`otp.run <onetick.py.run>`.
+
+        Examples
+        --------
+
+        Get only the ticks that have needed symbols specified in field ``TICKER``:
+
+        >>> data = otp.Ticks(X=[1, 2, 3, 4], TICKER=['A', 'B', 'A', 'C'])
+        >>> data = otp.SplitQueryOutputBySymbol(data, data['TICKER'])
+        >>> res = otp.run(data, symbols=['A', 'B'])
+        >>> res['A']
+                             Time  X TICKER
+        0 2003-12-01 00:00:00.000  1      A
+        1 2003-12-01 00:00:00.002  3      A
+        >>> res['B']
+                             Time  X TICKER
+        0 2003-12-01 00:00:00.001  2      B
+        """
+
+        if self._try_default_constructor(schema=schema, **kwargs):
+            return
+
+        if isinstance(query, Source):
+            query = query.copy()
+            otq_query = query.to_otq()
+            q_start, q_end, _ = query._set_date_range_and_symbols()
+            if start is utils.adaptive and end is utils.adaptive:
+                start, end = q_start, q_end
+        else:
+            # TODO: support already existing queries
+            raise ValueError('Non supported type of the `query` is specified')
+
+        super().__init__(
+            _symbols=symbols,
+            _start=start,
+            _end=end,
+            _base_ep_func=partial(self.build, db, tick_type, symbol_field, otq_query, single_invocation),
+            schema=schema,
+            **kwargs,
+        )
+
+    def build(self, db, tick_type, symbol_field_name, otq_query, single_invocation):
+        src = Source(otq.SplitQueryOutputBySymbol(otq_query=otq_query,
+                                                  symbol_field_name=str(symbol_field_name),
+                                                  ensure_single_invocation=single_invocation))
+
+        update_node_tick_type(src, tick_type, db)
+
+        return src
+
+
+def by_symbol(src: Source,
+              symbol_field,
+              single_invocation=False,
+              db=utils.adaptive_to_default,
+              tick_type=utils.adaptive,
+              start=utils.adaptive,
+              end=utils.adaptive,
+              ) -> Source:
+    """
+    Create a separate data series for each unique value of ``symbol_field`` in the output of ``src``.
+    ``src`` must specify enough parameters to be run (e.g., symbols, query range). A typical use case is to split a
+    single data series (e.g., from a CSV file) into separate data series by symbol. This method is a source.
+
+    Parameters
+    ----------
+    src: Source
+        a query which output is to be split by ``symbol_field``
+    symbol_field: str
+        the name of the field carrying symbol name in the ``src`` query
+    single_invocation: bool, optional
+        ``True`` means that the ``src`` query is run once and the result stored in memory speeding up the execution.
+        ``False`` means that the ``src`` query is run for every symbol of the query saving memory
+        but slowing down query execution.
+        Default: ``False``
+    db: str, optional
+        Database for running the query. Doesn't affect the ``src`` query. The default value
+        is ``otp.config['default_db']``.
+    tick_type: str, optional
+        Tick type for the query. Doesn't affect the ``src`` query.
+    start: otp.dt, optional
+        By default it is taken from the ``src`` start time
+    end: otp.dt, optional
+        By default it is taken from the ``src`` end time
+
+    See also
+    --------
+    **SPLIT_QUERY_OUTPUT_BY_SYMBOL** OneTick event processor
+
+    Examples
+    --------
+    >>> executions = otp.CSV( # doctest: +SKIP
+    ...     otp.utils.file(os.path.join(cur_dir, 'data', 'example_events.csv')),
+    ...     converters={"time_number": lambda c: c.apply(otp.nsectime)},
+    ...     timestamp_name="time_number",
+    ...     start=otp.dt(2022, 7, 1),
+    ...     end=otp.dt(2022, 7, 2),
+    ...     order_ticks=True
+    ... )[['stock', 'px']]
+    >>> csv = otp.by_symbol(executions, 'stock') # doctest: +SKIP
+    >>> trd = otp.DataSource( # doctest: +SKIP
+    ...     db='US_COMP',
+    ...     tick_type='TRD',
+    ...     start=otp.dt(2022, 7, 1),
+    ...     end=otp.dt(2022, 7, 2)
+    ... )[['PRICE', 'SIZE']]
+    >>> data = otp.join_by_time([csv, trd]) # doctest: +SKIP
+    >>> result = otp.run(data, symbols=executions.distinct(keys='stock')[['stock']], concurrency=8) # doctest: +SKIP
+    >>> result['THG'] # doctest: +SKIP
+                               Time stock      px   PRICE  SIZE
+    0 2022-07-01 11:37:56.432947200   THG  148.02  146.48     1
+    >>> result['TFX'] # doctest: +SKIP
+                               Time stock      px   PRICE  SIZE
+    0 2022-07-01 11:39:45.882808576   TFX  255.61  251.97     1
+    >>> result['BURL'] # doctest: +SKIP
+                               Time stock      px   PRICE  SIZE
+    0 2022-07-01 11:42:35.125718016  BURL  137.53  135.41     2
+    """
+    result = SplitQueryOutputBySymbol(src,
+                                      symbol_field=symbol_field,
+                                      single_invocation=single_invocation,
+                                      db=db,
+                                      tick_type=tick_type,
+                                      start=start,
+                                      end=end)
+
+    result.schema.set(**src.schema)
+    return result