onetick-py 1.177.0__py3-none-any.whl

onetick/py/backports.py

@@ -0,0 +1,26 @@
+# A common place to put backports for specific versions of Python
+
+import sys
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+    from functools import singledispatchmethod, lru_cache
+else:
+    from typing_extensions import Literal
+    from singledispatchmethod import singledispatchmethod
+    from backports.functools_lru_cache import lru_cache
+
+
+if sys.version_info >= (3, 9):
+    import zoneinfo
+    from functools import cached_property, cache
+    import ast
+    astunparse = ast.unparse
+else:
+    from backports.cached_property import cached_property
+    from astunparse import unparse as astunparse
+    from backports import zoneinfo
+
+    def cache(user_function):
+        'Simple lightweight unbounded cache.  Sometimes called "memoize".'
+        return lru_cache(maxsize=None)(user_function)

onetick/py/cache.py

@@ -0,0 +1,374 @@
+from typing import Any, Callable, List, Optional, Tuple, Union
+from types import FunctionType
+from datetime import datetime
+
+import onetick.py as otp
+from onetick.py.otq import otq
+from onetick.py.core.source import Source
+from onetick.py.sources.common import update_node_tick_type
+from onetick.py.sources.cache import process_otq_params
+from onetick.py.types import datetime as otp_datetime
+
+from . import configuration
+
+
+def _check_status(df):
+    if "STATUS" not in df:
+        raise RuntimeError("Empty response returned")
+
+    status = df["STATUS"][0]
+    if status != "SUCCESS":
+        raise RuntimeError(f"Error status returned: {status}")
+
+
+def _convert_dt_to_str(dt: Union[str, datetime, otp_datetime]):
+    if isinstance(dt, str):
+        return dt
+    elif isinstance(dt, (datetime, otp_datetime)):
+        return dt.strftime('%Y%m%d%H%M%S.%f')
+    raise ValueError(f"Unsupported value for 'dt' parameter: {type(dt)}")
+
+
+def _convert_time_intervals(
+    time_intervals_to_cache: List[Tuple[Union[str, datetime, otp_datetime], Union[str, datetime, otp_datetime]]],
+):
+    return "\n".join(
+        [
+            f"{_convert_dt_to_str(start_time)},{_convert_dt_to_str(end_time)}"
+            for start_time, end_time in time_intervals_to_cache
+        ]
+    )
+
+
+def create_cache(
+    cache_name: str,
+    query: Union['Source', Callable, str, None] = None,
+    inheritability: bool = True,
+    otq_params: Union[dict, None] = None,
+    time_granularity: int = 0,
+    time_granularity_units: Optional[str] = None,
+    timezone: str = "",
+    time_intervals_to_cache: Optional[List[tuple]] = None,
+    allow_delete_to_everyone: bool = False,
+    allow_update_to_everyone: bool = False,
+    allow_search_to_everyone: bool = True,
+    cache_expiration_interval: int = 0,
+    tick_type: str = "ANY",
+    symbol: Optional[str] = None,
+    db: Optional[str] = None,
+):
+    """
+    Create cache via CREATE_CACHE EP
+
+    If :py:class:`onetick.py.Source` or callable passed as ``query`` parameter,
+    cache will be created only for current session.
+
+    Also, this case or passing absolute path to .otq as ``query`` parameter are supported
+    only for caching on local OneTick server.
+
+    Parameters ``symbol``, ``db`` and ``tick_type`` could be omitted if you want to use
+    default symbol, default database and ``ANY`` tick type.
+
+    Parameters
+    ----------
+    cache_name: str
+        Name of the cache to be created.
+    query: :py:class:`onetick.py.Source`, callable, str
+        Query to be cached or path to .otq file (in ``filename.otq::QueryName`` format).
+        Only queries residing on the server that runs the caching event processors are currently supported.
+        For local OneTick server you can pass absolute path to local .otq file.
+    inheritability: bool
+        Indicates whether results can be obtained by combining time intervals that were cached with intervals
+        freshly computed to obtain results for larger intervals.
+    otq_params: dict
+        OTQ params of the query to be cached.
+        Setting `otq_params` in :py:class:`onetick.py.ReadCache` may not override this `otq_params`
+        if cache not invalidated.
+    time_granularity: int
+        Value N for seconds/days/months granularity means that start and end time of the query have to be on N
+        second/day/month boundaries relative to start of the day/month/year.
+        This doesn't affect the frequency of data within the cache, just the start and end dates.
+    time_granularity_units: str, None
+        Units used in ``time_granularity`` parameter. Possible values: 'none', 'days', 'months', 'seconds' or None.
+    timezone: str
+        Timezone of the query to be cached.
+    time_intervals_to_cache: List[tuple]
+        List of tuples with start and end times in ``[(<start_time_1>, <end_time_1>), ...]`` format,
+        where ``<start_time>`` and ``<end_time>`` should be one of these:
+
+        * string in ``YYYYMMDDhhmmss[.msec]`` format.
+        * :py:class:`datetime.datetime`
+        * :py:class:`onetick.py.types.datetime`
+
+        If specified only these time intervals can be cached. Ignored if ``inheritability=True``.
+        If you try to make a query outside defined interval, error will be raised.
+    allow_delete_to_everyone: bool
+        When set to ``True`` everyone is allowed to delete the cache.
+    allow_update_to_everyone: bool
+        When set to ``True`` everyone is allowed to update the cache.
+    allow_search_to_everyone: bool
+        When set to ``True`` everyone is allowed to read the cached data.
+    cache_expiration_interval: int
+        If set to a non-zero value determines the periodicity of cache clearing, in seconds.
+        The cache will be cleared every X seconds, triggering new query executions when data is requested.
+    tick_type: str
+        Tick type.
+    symbol: str, list of str, list of otq.Symbol, :py:class:`onetick.py.Source`, :pandas:`pandas.DataFrame`, optional
+        ``symbols`` parameter of ``otp.run()``.
+    db: str
+        Database.
+
+    Note
+    ----
+    This function does NOT populate the cache, it's only reserves the cache name in the OneTick memory.
+    Cache is only populated when an attempt is made to read the data from it via :py:class:`onetick.py.ReadCache`.
+
+    See also
+    --------
+    | **CREATE_CACHE** OneTick event processor
+    | :py:class:`onetick.py.ReadCache`
+    | :py:func:`onetick.py.delete_cache`
+
+    Examples
+    --------
+    Simple cache creation from .otq file on OneTick server under ``OTQ_FILE_PATH``
+
+    >>> otp.create_cache(  # doctest: +SKIP
+    ...    cache_name="some_cache", query="CACHE_EXAMPLE.otq::slowquery",
+    ...    tick_type="TRD", db="LOCAL",
+    ... )
+
+    Cache creation from function
+
+    >>> def query_func():
+    ...    return otp.DataSource("COMMON", tick_type="TRD", symbols="AAPL")
+    >>> otp.create_cache(
+    ...    cache_name="some_cache", query=query_func, tick_type="TRD", db="LOCAL",
+    ... )
+
+    Create cache for time intervals with different datetime types:
+
+    >>> otp.create_cache(  # doctest: +SKIP
+    ...    cache_name="some_cache",
+    ...    query="query_example.otq::query"),
+    ...    inheritability=False,
+    ...    time_intervals_to_cache=[
+    ...        ("20220601123000.000000", "20220601183000.000000"),
+    ...        (datetime(2022, 6, 2, 12, 30), datetime(2003, 1, 2, 18, 30)),
+    ...        (otp.datetime(2022, 6, 3, 12, 30), otp.datetime(2022, 6, 3, 18, 30)),
+    ...    ],
+    ...    timezone="GMT",
+    ...    tick_type="TRD",
+    ...    db="LOCAL",
+    ... )
+
+    Create cache with OTQ params:
+
+    >>> otp.create_cache(  # doctest: +SKIP
+    ...    cache_name="some_cache",
+    ...    query="query_example.otq::query"),
+    ...    otq_params={"some_param": "some_value"},
+    ...    timezone="GMT",
+    ...    tick_type="TRD",
+    ...    db="LOCAL",
+    ... )
+    """
+    if query is None:
+        raise ValueError("Parameter `query` should be set")
+
+    if time_granularity_units is None:
+        time_granularity_units = "none"
+
+    if time_granularity_units not in {'none', 'days', 'months', 'seconds'}:
+        raise ValueError(f"Incorrect `time_granularity_units` param value passed: {time_granularity_units}")
+
+    time_granularity_units = time_granularity_units.upper()
+
+    if symbol is None:
+        symbol = configuration.config.default_symbol
+
+    if db is None:
+        db = configuration.config.default_db
+
+    time_intervals_str = ""
+    if time_intervals_to_cache:
+        time_intervals_str = _convert_time_intervals(time_intervals_to_cache)
+
+    otq_params_str = process_otq_params(otq_params)
+
+    otq_file_path = None
+    if query:
+        if isinstance(query, FunctionType):
+            query = query()
+
+        if isinstance(query, Source):
+            # will create sub-query after source is initialized below
+            otq_file_path = 'THIS::create_cache_query'
+        elif isinstance(query, str):
+            otq_file_path = query
+        else:
+            raise ValueError(f"Passed `query` parameter value with incorrect type: {type(query)}")
+
+    source = Source(
+        otq.CreateCache(
+            cache_name=cache_name,
+            otq_file_path=otq_file_path,
+            inheritability=inheritability,
+            otq_params=otq_params_str,
+            time_granularity=time_granularity,
+            time_granularity_units=time_granularity_units,
+            timezone=timezone,
+            time_intervals_to_cache=time_intervals_str,
+            allow_delete_to_everyone=allow_delete_to_everyone,
+            allow_update_to_everyone=allow_update_to_everyone,
+            allow_search_to_everyone=allow_search_to_everyone,
+            cache_expiration_interval=cache_expiration_interval,
+        ),
+    )
+
+    if isinstance(query, Source):
+        # create temp file with query
+        query._store_in_tmp_otq(source._tmp_otq, name='create_cache_query')
+
+    update_node_tick_type(source, tick_type, db)
+
+    res = otp.run(source, symbols=symbol)
+    _check_status(res)
+
+
+def delete_cache(
+    cache_name: str,
+    apply_to_entire_cache: bool = True,
+    per_cache_otq_params: Union[dict, None] = None,
+    tick_type: str = "ANY",
+    symbol: Optional[str] = None,
+    db: Optional[str] = None,
+):
+    """
+    Delete cache via DELETE_CACHE EP
+
+    Parameters
+    ----------
+    cache_name: str
+        Name of the cache to be deleted.
+    apply_to_entire_cache: bool
+        When set to ``True`` deletes the cache for all symbols and time intervals.
+    per_cache_otq_params: dict
+        Deletes cache that have been associated with this OTQ parameters during its creation.
+        Value of this parameter should be equal to the value of ``otq_params`` of
+        :func:`<onetick.py.create_cache>`.
+    tick_type: str
+        Tick type.
+    symbol: str, list of str, list of otq.Symbol, :py:class:`onetick.py.Source`, :pandas:`pandas.DataFrame`, optional
+        ``symbols`` parameter of ``otp.run()``.
+    db: str
+        Database.
+
+    See also
+    --------
+    | **DELETE_CACHE** OneTick event processor
+    | :py:class:`onetick.py.ReadCache`
+    | :py:func:`onetick.py.create_cache`
+
+    Examples
+    --------
+    Simple cache deletion
+
+    >>> otp.delete_cache(  # doctest: +SKIP
+    ...    cache_name="some_cache", tick_type="TRD", symbol="SYM", db="LOCAL",
+    ... )
+    """
+    if symbol is None:
+        symbol = configuration.config.default_symbol
+
+    if db is None:
+        db = configuration.config.default_db
+
+    source = Source(
+        otq.DeleteCache(
+            cache_name=cache_name,
+            apply_to_entire_cache=apply_to_entire_cache,
+            per_cache_otq_params=process_otq_params(per_cache_otq_params),
+        ),
+    )
+
+    update_node_tick_type(source, tick_type, db)
+
+    res = otp.run(source, symbols=symbol)
+    _check_status(res)
+
+
+def modify_cache_config(
+    cache_name: str,
+    param_name: str,
+    param_value: Any,
+    tick_type: str = "ANY",
+    symbol: Optional[str] = None,
+    db: Optional[str] = None,
+):
+    """
+    Modify cache configuration via MODIFY_CACHE_CONFIG EP
+
+    Parameters
+    ----------
+    cache_name: str
+        Name of the cache to be deleted.
+    param_name: str
+        The name of the configuration parameter to be changed.
+        Supported parameters:
+
+        * ``inheritability``
+        * ``time_granularity``
+        * ``time_granularity_units``
+        * ``timezone``
+        * ``allow_search_to_everyone``
+        * ``allow_delete_to_everyone``
+        * ``allow_update_to_everyone``
+
+    param_value: Any
+        New value of configuration parameter. Will be converted to string.
+    tick_type: str
+        Tick type.
+    symbol: str, list of str, list of otq.Symbol, :py:class:`onetick.py.Source`, :pandas:`pandas.DataFrame`, optional
+        ``symbols`` parameter of ``otp.run()``.
+    db: str
+        Database.
+
+    See also
+    --------
+    | **MODIFY_CACHE_CONFIG** OneTick event processor
+    | :py:class:`onetick.py.ReadCache`
+    | :py:func:`onetick.py.create_cache`
+
+    Examples
+    --------
+    Simple cache config modification
+
+    >>> otp.modify_cache_config(  # doctest: +SKIP
+    ...    cache_name="some_cache",
+    ...    param_name="time_granularity",
+    ...    param_value=3,
+    ...    tick_type="TRD", symbol="SYM", db="LOCAL",
+    ... )
+    """
+    if symbol is None:
+        symbol = configuration.config.default_symbol
+
+    if db is None:
+        db = configuration.config.default_db
+
+    param_name = param_name.upper()
+    param_value = str(param_value)
+
+    source = Source(
+        otq.ModifyCacheConfig(
+            cache_name=cache_name,
+            config_parameter_name=param_name,
+            config_parameter_value=param_value,
+        ),
+    )
+
+    update_node_tick_type(source, tick_type, db)
+
+    res = otp.run(source, symbols=symbol)
+    _check_status(res)

onetick/py/callback/__init__.py

@@ -0,0 +1,5 @@
+from .callback import CallbackBase
+from .callbacks import (
+    LogCallback,
+    ManualDataframeCallback,
+)

onetick/py/callback/callback.py

@@ -0,0 +1,276 @@
+from onetick.py.otq import otq
+
+
+class CallbackBase(otq.CallbackBase):
+    """
+    Base class for user-defined callback classes
+    for :py:func:`onetick.py.run` callback mode.
+
+    Note
+    ----
+    Callbacks are executed sequentially, so make sure
+    they don't take too much time to execute.
+
+    See also
+    --------
+    :py:func:`onetick.py.run`
+
+    Examples
+    --------
+    >>> t = otp.Ticks(A=[1, 2, 3])
+    >>> class NumTicksCallback(otp.CallbackBase):
+    ...     def __init__(self):
+    ...         self.num_ticks = 0
+    ...     def process_tick(self, tick, time):
+    ...         self.num_ticks += 1
+    >>> callback = NumTicksCallback()
+    >>> otp.run(t, callback=callback)
+    >>> callback.num_ticks
+    3
+    """
+
+    # pylint: disable-next=useless-parent-delegation
+    def __init__(self):
+        """
+        Method ``__init__()`` can be used for callback initialization.
+        Can be used to define some variables for future use
+        in callback methods.
+        """
+        super().__init__()
+
+    def replicate(self):
+        """
+        Called to replicate the callback object for each output node.
+        May also be used for internal copying of callback object.
+
+        Returns
+        -------
+            By default reference to this callback object
+        """
+        return self
+
+    def process_callback_label(self, callback_label):
+        """
+        Called immediately before :meth:`process_symbol_name`
+        to supply label assigned to callback.
+
+        Parameters
+        ----------
+        callback_label: str
+            label assigned to this callback object
+        """
+        pass
+
+    def process_symbol_name(self, symbol_name):
+        """
+        Invoked to supply the name of the security that produces
+        all the ticks that will be delivered to this callback object.
+        If these ticks are provided by several securities,
+        the ``symbol_name`` parameter is set to empty string.
+
+        Parameters
+        ----------
+        symbol_name: str
+            name of security
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1)
+        >>> class SymbolNameCallback(otp.CallbackBase):
+        ...     def process_symbol_name(self, symbol_name):
+        ...         self.symbol_name = symbol_name
+        >>> callback = SymbolNameCallback()
+        >>> otp.run(t, callback=callback, symbols='DEMO_L1::X')
+        >>> callback.symbol_name
+        'DEMO_L1::X'
+        """
+        pass
+
+    def process_symbol_group_name(self, symbol_group_name):
+        """
+        Called when a named group of securities, i.e. portfolio, is processed.
+
+        Parameters
+        ----------
+        symbol_group_name: str
+            The name of security group.
+        """
+        pass
+
+    def process_tick_type(self, tick_type):
+        """
+        Reports the type of the security ticks which are processed by this callback object.
+        This method is called before any call to
+        :meth:`process_tick_descriptor` or :meth:`process_tick`.
+        It is called immediately after :meth:`process_symbol_name`.
+
+        Parameters
+        ----------
+        tick_type: str
+            The name of tick type.
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1, symbol='DEMO_L1', tick_type='TT_TT')
+        >>> class TickTypeCallback(otp.CallbackBase):
+        ...     def process_tick_type(self, tick_type):
+        ...         self.tick_type = tick_type
+        >>> callback = TickTypeCallback()
+        >>> otp.run(t, callback=callback)
+        >>> callback.tick_type
+        'DEMO_L1::TT_TT'
+        """
+        pass
+
+    def process_tick_descriptor(self, tick_descriptor):
+        """
+        This method is invoked before the first call to :meth:`process_tick`
+        and every time before tick structure changes.
+
+        Parameters
+        ----------
+        tick_descriptor: list of tuple
+            First element of each tuple is field's name
+            and the second one is a dictionary ``{'type': string_field_type}``.
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1)
+        >>> class TickDescriptorCallback(otp.CallbackBase):
+        ...    def process_tick_descriptor(self, tick_descriptor):
+        ...        self.tick_descriptor = tick_descriptor
+        >>> callback = TickDescriptorCallback()
+        >>> otp.run(t, callback=callback)
+        >>> callback.tick_descriptor
+        [('A', {'type': 'int'})]
+        """
+        pass
+
+    def process_tick(self, tick, time):
+        """
+        Called to deliver each tick.
+
+        Note
+        ----
+        If you are making query through WebAPI mode, use ``process_ticks`` callback method instead.
+
+        Parameters
+        ----------
+        tick: dict
+            mapping of field names to field values
+        time: :py:class:`datetime.datetime`
+            timestamp of the tick in GMT timezone.
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1)
+        >>> class ProcessTickCallback(otp.CallbackBase):
+        ...     def process_tick(self, tick, time):
+        ...         self.result = (tick, time)
+        >>> callback = ProcessTickCallback()
+        >>> otp.run(t, callback=callback)
+        >>> callback.result
+        ({'A': 1}, datetime.datetime(2003, 12, 1, 5, 0))
+        """
+        pass
+
+    def process_ticks(self, ticks):
+        """
+        Called after getting all ticks for WebAPI queries.
+
+        Due to limitation of WebAPI mode ``process_tick`` callback method, which invoked on each tick, isn't supported.
+        Instead, WebAPI supports ``process_ticks`` method which invoked for processing the data
+        after the query is finished.
+        All ticks are returned on the ``ticks`` variable.
+
+        Parameters
+        ----------
+        ticks: dict
+            mapping of field names to field values
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1)
+        >>> class ProcessTicksCallback(otp.CallbackBase):
+        ...     def __init__(self):
+        ...         self.result = {}
+        ...
+        ...     def process_ticks(self, ticks):
+        ...         self.result = ticks
+        >>> callback = ProcessTicksCallback()
+        >>> otp.run(t, callback=callback)  # doctest: +SKIP
+        >>> callback.result  # doctest: +SKIP
+        {'Time': array(['2003-12-01T00:00:00.000000000'], dtype='datetime64[ns]'), 'A': array([1])}
+        """
+        pass
+
+    def process_sorting_order(self, sorted_by_time_flag):
+        """
+        Informs whether the ticks that will be submitted to this callback object
+        will be ordered by time.
+
+        Parameters
+        ----------
+        sorted_by_time_flag: bool
+            Indicates whether the incoming ticks will be sorted by time.
+        """
+        pass
+
+    def process_data_quality_change(self, symbol_name, data_quality, time):
+        """
+        Called to report a data quality change, such as collection outage.
+
+        Parameters
+        ----------
+        symbol_name: str
+            Symbol name for each data quality change is propagated.
+        data_quality: int
+            parameter has following meaning:
+
+            * `QUALITY_UNKNOWN` = -1,
+            * `QUALITY_OK`,
+            * `QUALITY_STALE` = 1,
+            * `QUALITY_MISSING` = 2,
+            * `QUALITY_PATCHED` = 4,
+            * `QUALITY_MOUNT_BAD` = 9,
+            * `QUALITY_DISCONNECT` = 17,
+            * `QUALITY_COLLECTOR_FAILURE` = 33,
+            * `QUALITY_DELAY_STITCHING_WITH_RT` = 64,
+            * `QUALITY_OK_STITCHING_WITH_RT` = 66
+        time: :py:class:`datetime.datetime`
+            Time of the change in GMT timezone.
+        """
+        pass
+
+    def process_error(self, error_code, error_msg):
+        """
+        Called to report a per-security error or per-security warning.
+
+        Parameters
+        ----------
+        error_code: int
+            Values of error code less than 1000 are warnings.
+            Warnings signal issues which might not affect results of the query
+            and thus could be chosen to be ignored
+        error_msg: str
+            Error message
+        """
+        pass
+
+    def done(self):
+        """
+        Invoked when all the raw or computed ticks for a given request
+        were submitted to the callback using the :meth:`process_tick` method.
+
+        Examples
+        --------
+        >>> t = otp.Tick(A=1)
+        >>> class DoneCallback(otp.CallbackBase):
+        ...     def done(self):
+        ...         self.done = True
+        >>> callback = DoneCallback()
+        >>> otp.run(t, callback=callback)
+        >>> callback.done
+        True
+        """
+        pass