onetick-py 1.177.0__py3-none-any.whl

onetick/py/oqd/sources.py

@@ -0,0 +1,325 @@
+from functools import partial
+
+import pandas as pd
+
+import onetick.py as otp
+from onetick.py.otq import otq
+
+from onetick.py import utils
+from onetick.py.sources.data_source import _start_doc, _end_doc, _symbol_doc
+from onetick.py.docs.utils import docstring, param_doc
+
+
+COMMON_SOURCE_DOC_PARAMS = [_start_doc, _end_doc, _symbol_doc]
+OQD_TICK_TYPE = 'OQD::*'
+
+
+def _modify_query_times(src):
+    src.sink(otq.ModifyQueryTimes(
+        start_time=('parse_time("%Y%m%d %H:%M:%S.%q", '
+                    'time_format("%Y%m%d", _START_TIME, _TIMEZONE) + " 00:00:00.000", "GMT")'),
+        end_time=('parse_time("%Y%m%d %H:%M:%S.%q", '
+                  'time_format("%Y%m%d", _END_TIME, _TIMEZONE) + " 24:00:00.000", "GMT")'),
+        output_timestamp='min(max(TIMESTAMP,_START_TIME),_END_TIME)'
+    ))
+
+
+_exch_doc = param_doc(
+    name='exch',
+    desc="""
+    The OneQuantData exchange code for the desired price series. Possible values:
+
+    - 'all'
+        return data for all exchanges;
+
+    - 'main'
+        return data main pricing exchange;
+
+    - any other string value will treated as exchange name to filter data.
+
+    Default: 'all'.
+    """,
+    str_annotation="str, 'all', 'main'",
+    default='all',
+)
+
+
+class OHLCV(otp.Source):
+    @docstring(parameters=[_exch_doc] + COMMON_SOURCE_DOC_PARAMS, add_self=True)
+    def __init__(
+        self,
+        exch='all',
+        symbol=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        **kwargs
+    ):
+        """
+        OneQuantData™ source to retrieve a time series of unadjusted
+        prices for a symbol for one particular pricing exchange of daily OHLCV data.
+        Output ticks have fields: OPEN, HIGH, LOW, CLOSE, VOLUME, CURRENCY, EXCH.
+
+        Examples
+        --------
+        >>> src = otp.oqd.sources.OHLCV(exch="USPRIM")  # doctest: +SKIP
+        >>> otp.run(src,  # doctest: +SKIP
+        ...         symbols='BTKR::::GOOGL US',
+        ...         start=otp.dt(2018, 8, 1),
+        ...         end=otp.dt(2018, 8, 2),
+        ...         symbol_date=otp.dt(2018, 8, 1))
+                         Time    OID    EXCH CURRENCY     OPEN     HIGH      LOW    CLOSE    VOLUME
+        0 2018-08-01 00:00:00  74143  USPRIM      USD  1242.73  1245.72  1225.00  1232.99  605680.0
+        1 2018-08-01 20:00:00  74143  USPRIM      USD  1219.69  1244.25  1218.06  1241.13  596960.0
+        """
+        if self._try_default_constructor(**kwargs):
+            return
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=partial(self.build, exch=exch)
+        )
+
+        self.schema.set(OID=str,
+                        EXCH=str,
+                        CURRENCY=str,
+                        OPEN=float,
+                        HIGH=float,
+                        LOW=float,
+                        CLOSE=float,
+                        VOLUME=float)
+
+    def build(self, exch):
+        ep = None
+        if exch == 'all':
+            ep = otp.oqd.eps.OqdSourceDprcAll()
+        elif exch == 'main':
+            ep = otp.oqd.eps.OqdSourceDprcMain()
+        else:
+            ep = otp.oqd.eps.OqdSourceDprcExch(exch=exch)
+
+        src = otp.Source(ep)
+        src.tick_type(OQD_TICK_TYPE)
+        _modify_query_times(src)
+        return src
+
+
+class CorporateActions(otp.Source):
+    """
+    OneQuantData™ source EP to retrieve a time series of corporate
+    actions for a symbol.
+
+    This source will return all corporate action fields available for a symbol
+    with EX-Dates between the query start time and end time.  The
+    timestamp of the series is equal to the EX-Date of the corporate
+    action with a time of 0:00:00 GMT.
+
+    Examples
+    --------
+    >>> src = otp.oqd.sources.CorporateActions()  # doctest: +SKIP
+    >>> otp.run(src,  # doctest: +SKIP
+    ...         symbols='TDEQ::::AAPL',
+    ...         start=otp.dt(2021, 1, 1),
+    ...         end=otp.dt(2021, 8, 6),
+    ...         symbol_date=otp.dt(2021, 2, 18),
+    ...         timezone='GMT')
+            Time   OID  ACTION_ID    ACTION_TYPE  ACTION_ADJUST ACTION_CURRENCY  ANN_DATE   EX_DATE  PAY_DATE  REC_DATE\
+                       TERM_NOTE TERM_RECORD_TYPE ACTION_STATUS
+    0 2021-02-05  9706   16799540  CASH_DIVIDEND          0.205             USD  20210127  20210205  20210211  20210208\
+          CASH:0.205@USD                         NORMAL
+    1 2021-05-07  9706   17098817  CASH_DIVIDEND          0.220             USD  20210428  20210507  20210513  20210510\
+           CASH:0.22@USD                         NORMAL
+    2 2021-08-06  9706   17331864  CASH_DIVIDEND          0.220             USD  20210727  20210806  20210812  20210809\
+           CASH:0.22@USD                         NORMAL
+
+    """
+
+    @docstring(parameters=COMMON_SOURCE_DOC_PARAMS, add_self=True)
+    def __init__(
+        self,
+        symbol=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        **kwargs
+    ):
+        if self._try_default_constructor(**kwargs):
+            return
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=partial(self.build)
+        )
+
+        self.schema.set(OID=str,
+                        ACTION_ID=int,
+                        ACTION_TYPE=str,
+                        ACTION_ADJUST=float,
+                        ACTION_CURRENCY=str,
+                        ANN_DATE=int,
+                        EX_DATE=int,
+                        PAY_DATE=int,
+                        REC_DATE=int,
+                        TERM_NOTE=str,
+                        TERM_RECORD_TYPE=str,
+                        ACTION_STATUS=str)
+
+    def build(self):
+        ep = otp.oqd.eps.OqdSourceCacs()
+        src = otp.Source(ep)
+        src.tick_type(OQD_TICK_TYPE)
+        _modify_query_times(src)
+        return src
+
+
+class DescriptiveFields(otp.Source):
+    """OneQuantData™ source to retrieve a time series of descriptive fields for a symbol.
+    There will only be ticks on days when some field in the descriptive data changes.
+    Output ticks will have fields:
+    OID, END_DATE, COUNTRY, EXCH, NAME,
+    ISSUE_DESC, ISSUE_CLASS, ISSUE_TYPE, ISSUE_STATUS,
+    SIC_CODE, IDSYM, TICKER, CALENDAR.
+
+    Note: currently actual fields have 9999 year in END_DATE, but it could not fit the
+    nanosecond timestamp, so it is replaced with 2035-01-01 date.
+
+    Examples
+    --------
+    >>> src = otp.oqd.sources.DescriptiveFields()  # doctest: +SKIP
+    >>> otp.run(src,  # doctest: +SKIP
+    ...         symbols='1000001589',
+    ...         start=otp.dt(2020, 3, 1),
+    ...         end=otp.dt(2023, 3, 2),
+    ...         timezone='GMT').iloc[:6]
+            Time         OID    END_DATE COUNTRY  EXCH                NAME                   ISSUE_DESC\
+                 ISSUE_CLASS ISSUE_TYPE ISSUE_STATUS SIC_CODE    IDSYM TICKER CALENDAR
+    0 2020-03-01  1000001589  2020-03-23     LUX  EL^X  INVESTEC GLOBAL ST   EUROPEAN HIGH YLD BD INC 2\
+                FUND                  NORMAL           B2PT4G9
+    1 2020-03-23  1000001589  2020-04-01     LUX  EL^X  NINETY ONE LIMITED   EUROPEAN HIGH YLD BD INC 2\
+                FUND                  NORMAL           B2PT4G9
+    2 2020-04-01  1000001589  2021-01-01     LUX  EL^X  NINETY ONE LUX S.A   EUROPEAN HIGH YLD BD INC 2\
+                FUND                  NORMAL           B2PT4G9
+    3 2021-01-01  1000001589  2021-06-18     LUX  EL^X  NINETY ONE LUX S.A   EUROPEAN HIGH YLD BD INC 2\
+                FUND                  NORMAL           B2PT4G9
+    4 2021-06-18  1000001589  2022-01-01     LUX  EL^X  NINETY ONE LUX S.A  GSF GBL HIGH YLD A2 EUR DIS\
+                FUND                  NORMAL           B2PT4G9
+    5 2022-01-01  1000001589  2022-01-28     LUX  EL^X  NINETY ONE LUX S.A  GSF GBL HIGH YLD A2 EUR DIS\
+                FUND                  NORMAL           B2PT4G9
+    """
+
+    @docstring(parameters=COMMON_SOURCE_DOC_PARAMS, add_self=True)
+    def __init__(
+        self,
+        symbol=utils.adaptive,
+        start=utils.adaptive,
+        end=utils.adaptive,
+        **kwargs
+    ):
+        if self._try_default_constructor(**kwargs):
+            return
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=partial(self.build)
+        )
+
+        self.schema.set(
+            OID=str,
+            END_DATE=otp.nsectime,
+            COUNTRY=str,
+            EXCH=str,
+            NAME=str,
+            ISSUE_DESC=str,
+            ISSUE_CLASS=str,
+            ISSUE_TYPE=str,
+            ISSUE_STATUS=str,
+            SIC_CODE=str,
+            IDSYM=str,
+            TICKER=str,
+            CALENDAR=str,)
+
+    def build(self):
+        ep = otp.oqd.eps.OqdSourceDes()
+        src = otp.Source(ep)
+        src.tick_type(OQD_TICK_TYPE)
+        _modify_query_times(src)
+
+        # work-around to resolve problem with pandas timestamp out of bounds
+        pd_max = pd.Timestamp.max.strftime('%Y%m%d%H%M%S')
+        src.sink(otq.UpdateFields(
+            set='END_DATE=PARSE_NSECTIME("%Y-%m-%d", "2035-01-01", _TIMEZONE)',
+            where=f'AS_YYYYMMDDHHMMSS(END_DATE) > {pd_max}'))
+        return src
+
+
+class SharesOutstanding(otp.Source):
+    """
+    Logic is implemented in OQD_SOURCE_SHO EP to retrieve a time series of shares
+    outstanding for a stock.
+
+    The source retrieves a time series of shares outstanding
+    for a stock. This source only applies to stocks or securities that have
+    published shares outstanding data.
+
+    The series represents total shares outstanding and is not free float
+    adjusted.
+
+    Note: currently actual fields have 9999 year in END_DATE, but it could not fit the
+    nanosecond timestamp, so it is replaced with 2035-01-01 date.
+
+
+    Examples
+    --------
+    >>> src = otp.oqd.sources.SharesOutstanding()  # doctest: +SKIP
+    >>> otp.run(src,  # doctest: +SKIP
+    ...         symbols='TDEQ::::AAPL',
+    ...         start=otp.dt(2021, 1, 1),
+    ...         end=otp.dt(2021, 8, 6),
+    ...         symbol_date=otp.dt(2021, 2, 18),
+    ...         timezone='GMT')
+            Time   OID   END_DATE REPORT_MONTH        SHARES
+    0 2021-01-01  9706 2021-01-06       202009  1.700180e+10
+    1 2021-01-06  9706 2021-01-29       202009  1.682326e+10
+    2 2021-01-29  9706 2021-05-03       202012  1.678810e+10
+    3 2021-05-03  9706 2021-07-30       202103  1.668763e+10
+    4 2021-07-30  9706 2021-10-29       202106  1.653017e+10
+    """
+    @docstring(parameters=COMMON_SOURCE_DOC_PARAMS, add_self=True)
+    def __init__(
+        self,
+        symbol=otp.utils.adaptive,
+        start=otp.utils.adaptive,
+        end=otp.utils.adaptive,
+        **kwargs
+    ):
+        if self._try_default_constructor(**kwargs):
+            return
+
+        super().__init__(
+            _symbols=symbol,
+            _start=start,
+            _end=end,
+            _base_ep_func=partial(self.build)
+        )
+
+        self.schema.set(OID=str,
+                        END_DATE=otp.nsectime,
+                        REPORT_MONTH=int,
+                        SHARES=int)
+
+    def build(self):
+        ep = otp.oqd.eps.OqdSourceSho()
+        src = otp.Source(ep)
+        src.tick_type(OQD_TICK_TYPE)
+        _modify_query_times(src)
+
+        # work-around to resolve problem with pandas timestamp out of bounds
+        pd_max = pd.Timestamp.max.strftime('%Y%m%d%H%M%S')
+        src.sink(otq.UpdateFields(
+            set='END_DATE=PARSE_NSECTIME("%Y-%m-%d", "2035-01-01", _TIMEZONE)',
+            where=f'AS_YYYYMMDDHHMMSS(END_DATE) > {pd_max}'))
+        return src

onetick/py/otq.py

@@ -0,0 +1,216 @@
+# This file is used to import the correct query module:
+# onetick.query or onetick.query_webapi
+# based on the environment variable OTP_WEBAPI
+# Re-use this file in your tests to import the correct onetick.query module
+# Also it is override pyomd and OneTickLib classes with mock classes
+
+import getpass
+import inspect
+import os
+import tempfile
+import warnings
+import onetick.py as otp
+
+
+class OneTickLib:
+    # mock class for OneTickLib, used for webapi and onetick-query-stubs
+    # maybe it is not right to combine this namesake classes (onetick.lib.OneTickLib and pyomd.OneTickLib)
+    # but it is done to simplify the code and it is not affecting anything except tests
+    LOGGING_LEVEL_MIN = 0
+    LOGGING_LEVEL_LOW = 1
+    LOGGING_LEVEL_MEDIUM = 2
+    LOGGING_LEVEL_MAX = 3
+    set_authentication_token = None
+
+    def __init__(self, *args, **kwargs):  # NOSONAR
+        pass
+
+    def set_log_file(self, log_file):  # NOSONAR
+        pass
+
+    def cleanup(self):  # NOSONAR
+        pass
+
+
+if os.getenv("OTP_SKIP_OTQ_VALIDATION"):
+    import onetick_stubs as otq  # noqa: F401
+    import pyomd  # noqa: F401
+
+    class ConfigStub:
+        API_CONFIG: dict = {}
+
+    class otli:
+        OneTickLib = OneTickLib  # NOSONAR
+
+    for key, value in otq.query.__dict__.items():
+        setattr(otq, key, value)
+    setattr(otq, 'webapi', False)
+    setattr(otq, 'config', ConfigStub)
+    setattr(otq, 'graph_components', otq)
+
+elif otp.__webapi__:
+    import onetick.query_webapi as otq  # noqa: F401
+    setattr(otq, 'webapi', True)
+
+    from onetick.py.pyomd_mock import pyomd
+
+    __original_run = otq.run
+
+    def run(*args, **kwargs):
+        from onetick.py import config  # noqa
+        from onetick.py.compatibility import is_max_concurrency_with_webapi_supported
+
+        if not config.http_address and 'http_address' not in kwargs:
+            raise ValueError('otp.run() http_address keyword param, '
+                             'otp.config.http_address or OTP_HTTP_ADDRESS '
+                             'environment variable are required '
+                             'when using WebAPI mode.')
+
+        # TODO WEBAPI review this
+        # if file name is not in single quotes, then put it in single quotes
+        query_name = None
+        query = kwargs.get('query', None)
+        if isinstance(query, str):
+            if query[0] == "'" and query[-1] == "'":
+                query = query.replace("'", "")
+            if '::' in query:
+                query_name = query.split('::')[-1]
+                query = query.replace(f'::{query_name}', '')
+            kwargs['query'] = query
+        kwargs['query_name'] = query_name
+
+        del_params = [
+            'start_time_expression',
+            'end_time_expression',
+            'alternative_username',
+            'batch_size',
+            'treat_byte_arrays_as_strings',
+            'output_matrix_per_field',
+            'return_utc_times',
+            'connection',
+            'svg_path',
+            'use_connection_pool',
+            'time_as_nsec',
+            'max_expected_ticks_per_symbol',
+        ]
+        ignore_deleted_params = [
+            'treat_byte_arrays_as_strings',
+            'time_as_nsec',
+            'alternative_username',
+            'max_expected_ticks_per_symbol'
+        ]
+        for param in del_params:
+            if param in kwargs:
+                if kwargs[param] and param not in ignore_deleted_params:
+                    warnings.warn(f'Parameter {param} is not supported in WebAPI mode and will be ignored.')
+                del kwargs[param]
+
+        from onetick.py import config  # noqa
+        if 'http_address' not in kwargs:
+            kwargs['http_address'] = config.http_address
+
+        if kwargs.get('username'):
+            kwargs['http_username'] = kwargs['username']
+            del kwargs['username']
+        else:
+            kwargs['http_username'] = config.http_username
+
+        if kwargs.get('password'):
+            kwargs['http_password'] = kwargs['password']
+            del kwargs['password']
+        else:
+            kwargs['http_password'] = config.http_password
+
+        if 'access_token' not in kwargs:
+            if config.access_token:
+                kwargs['access_token'] = config.access_token
+        elif not kwargs['access_token']:
+            del kwargs['access_token']
+
+        access_token_url = kwargs.get('access_token_url', config.access_token_url)
+        if access_token_url:
+            if not hasattr(otq, 'get_access_token'):
+                raise RuntimeError('Current `onetick.query_webapi` version doesn\'t have `get_access_token` function')
+
+            if kwargs.get('access_token'):
+                raise ValueError('Both `access_token` and `access_token_url` set, instead of only one of them.')
+
+            if 'client_id' in kwargs:
+                client_id = kwargs.pop('client_id')
+            else:
+                client_id = config.client_id
+
+            if 'client_secret' in kwargs:
+                client_secret = kwargs.pop('client_secret')
+            else:
+                client_secret = config.client_secret
+
+            for param_name, param_value in zip(['client_id', 'client_secret'], [client_id, client_secret]):
+                if not param_value:
+                    raise ValueError(f'`access_token_url` parameter set, however `{param_name}` parameter missing.')
+
+            kwargs['access_token'] = otq.get_access_token(access_token_url, client_id, client_secret)
+
+            if 'access_token_url' in kwargs:
+                del kwargs['access_token_url']
+
+        if 'http_proxy' not in kwargs:
+            kwargs['http_proxy'] = config.http_proxy
+
+        if 'https_proxy' not in kwargs:
+            kwargs['https_proxy'] = config.https_proxy
+
+        webapi_run_parameters = inspect.signature(__original_run).parameters
+
+        trusted_certificate_file_arg = kwargs.pop('trusted_certificates_file',
+                                                  kwargs.pop('trusted_certificate_file', None))
+        trusted_certificate_file_value = (
+            trusted_certificate_file_arg if trusted_certificate_file_arg is not None
+            else config.trusted_certificates_file
+        )
+        if trusted_certificate_file_value is not None:
+            trusted_certificates_supported = set(webapi_run_parameters).intersection({'trusted_certificates_file',
+                                                                                      'trusted_certificate_file'})
+            if not trusted_certificates_supported:
+                raise ValueError(
+                    "Parameter `trusted_certificates_file` was set,"
+                    " however current version of OneTick doesn't support it."
+                )
+            trusted_certificates_supported_param = list(trusted_certificates_supported)[0]
+            kwargs[trusted_certificates_supported_param] = trusted_certificate_file_value
+
+        if 'callback' in kwargs and kwargs['callback'] is not None:
+            kwargs['output_mode'] = otq.QueryOutputMode.callback
+
+        if 'max_concurrency' in kwargs and not is_max_concurrency_with_webapi_supported():
+            kwargs['max_concurrency'] = None
+
+        return __original_run(*args, **kwargs)
+
+    otq.run = run
+
+    otq.OneTickLib = OneTickLib
+
+    class otli:  # type: ignore # noqa: F401
+        OneTickLib = otq.OneTickLib  # NOSONAR
+
+else:
+    import onetick.query as otq  # type: ignore # noqa: F401
+    import pyomd  # type: ignore # noqa: F401
+    import onetick.lib.instance as otli  # type: ignore # noqa: F401
+    setattr(otq, 'webapi', False)
+
+
+def _tmp_otq_path():
+    # copied from onetick.test.fixtures _keep_generated_dir() with replacement to /tmp
+    # required with OTP_WEBAPI_TEST_MODE to separate otq files from shared dbs+config+locator+acl files
+    from onetick.py import utils  # noqa
+    res = os.path.join(utils.TMP_CONFIGS_DIR(), os.environ.get("ONE_TICK_TMP_DIR", "otqs"))
+    res = res.replace(utils.temp.WEBAPI_TEST_MODE_SHARED_CONFIG,
+                      os.path.join(tempfile.gettempdir(), "test_" + getpass.getuser()))
+
+    # % and + is some webapi related path bugs, probably would be fixed someday
+    return res.replace("%", "_").replace("+", "_")
+
+
+__all__ = ['otq', 'pyomd', 'otli', '_tmp_otq_path']

onetick/py/pyomd_mock.py

@@ -0,0 +1,47 @@
+import datetime
+import onetick.py as otp
+
+
+if otp.__webapi__:
+    try:
+        from onetick.query_webapi import QueryProperties  # noqa: E0611
+    except ImportError as e:
+        try:
+            import onetick.query_webapi  # noqa
+            raise RuntimeError("You're trying to use onetick.query_webapi module, "
+                               "that is not compatible with onetick.py. "
+                               "Please, use onetick.query module instead (unset OTP_WEBAPI), "
+                               "or install onetick.query_webapi==1.24.20240715 or newer. "
+                               "Also, check that your PYTHONPATH doesn't have onetick binary path, "
+                               "because onetick distribution could have older onetick.query_webapi, "
+                               "that mirror your pip-installed version.") from e
+        except ImportError as e2:
+            raise ImportError(
+                "OTP_WEBAPI environment variable is set,"
+                " but onetick.query_webapi module is not available."
+                " Please, install onetick.query_webapi to avoid import errors"
+                " or unset OTP_WEBAPI to use onetick.query module instead."
+            ) from e2
+
+    # copied from one_market_data/one_tick/bin/python/onetick/query/_internal_utils.py
+    def quoted(str_object):
+        if len(str_object) <= 1:
+            return str_object
+        if str_object[0] == "'" and str_object[-1] == "'" or str_object[0] == '"' and str_object[-1] == '"':
+            return str_object
+        return f"'{str_object}'"
+
+    class OT_time_nsec_mock:  # NOSONAR
+        def __init__(self, number):
+            self.number = number
+
+        def __int__(self):
+            return int(self.number / 1e9)
+
+    class pyomd:
+        timeval_t = datetime.datetime  # type: ignore
+        QueryProperties = QueryProperties  # type: ignore # NOSONAR
+
+        @staticmethod
+        def OT_time_nsec(number):  # NOSONAR
+            return int(number / 1e9)