onetick-py 1.177.0__py3-none-any.whl

onetick/py/utils/__init__.py

@@ -0,0 +1,70 @@
+from .helpers import (
+    get_symbol_list_from_df,
+    JSONEncoder,
+    json_dumps,
+    query_properties_to_dict,
+    query_properties_from_dict,
+    symbol_date_to_str,
+)
+from .temp import (
+    File,
+    PermanentFile,
+    TmpFile,
+    TmpDir,
+    GeneratedDir,
+    TMP_CONFIGS_DIR,
+    ONE_TICK_TMP_DIR,
+)
+from .default import (
+    get_local_number_of_cores,
+    default_license_dir,
+    default_license_file,
+    default_day_boundary_tz,
+)
+from .acl import (
+    if_db_in_acl,
+    add_user_to_acl,
+    remove_user_from_acl,
+    if_user_in_acl,
+    tmp_acl,
+)
+from .config import (
+    reload_config,
+    modify_config_param,
+    get_config_param,
+    is_param_in_config,
+    tmp_config,
+)
+from .locator import (
+    get_dbs_locations_from_locator,
+    tmp_locator,
+    empty_locator,
+)
+from .script import (
+    write_config_for_tick_server_run,
+    create_file_to_run_config,
+    omd_dist_path,
+)
+from .query import (
+    abspath_to_query_by_otq_path,
+    abspath_to_query_by_name,
+    query_to_path_and_name,
+)
+from .types import (
+    get_type_that_includes,
+    adaptive,
+    adaptive_to_default,
+    default,
+    range,
+)
+from .tz import (
+    get_tzfile_by_name,
+    get_timezone_from_datetime,
+    convert_timezone,
+)
+from .file import (
+    FileBuffer,
+    file,
+)
+from .render import render_otq
+from . import perf

onetick/py/utils/acl.py

@@ -0,0 +1,93 @@
+import getpass
+import os
+
+from locator_parser.io import PrintWriter, FileReader, FileWriter
+from locator_parser.actions import Get, Add, Delete
+from locator_parser.acl import DB, User, Role, parse_acl
+from locator_parser.common import apply_actions
+
+from .temp import TmpFile
+from .types import default
+
+
+def if_db_in_acl(acl, db_name):
+    get_db = Get()
+    get_db.add_where(DB, id=db_name)
+
+    return apply_actions(parse_acl, FileReader(acl), PrintWriter(), [get_db], flush=True)
+
+
+def add_user_to_acl(acl, user):
+    add_user = Add(User(name=user))
+    add_user.add_where(Role, name="Admin")
+
+    assert apply_actions(parse_acl, FileReader(acl), FileWriter(acl), [add_user], flush=True)
+
+
+def remove_user_from_acl(acl, user):
+    remove_user = Delete()
+    remove_user.add_where(Role, name="Admin")
+    remove_user.add_where(User, name=user)
+
+    assert apply_actions(parse_acl, FileReader(acl), FileWriter(acl), [remove_user], flush=True)
+
+
+def if_user_in_acl(acl, user):
+    get_user = Get()
+    get_user.add_where(Role, name="Admin")
+    get_user.add_where(User, name=user)
+
+    return apply_actions(parse_acl, FileReader(acl), PrintWriter(), [get_user], flush=True)
+
+
+def tmp_acl(clean_up=default):
+    data = []
+    data.append("<roles>")
+    data.append('<role name="Admin">')
+    data.append(f'<user name="{getpass.getuser()}" />')
+    data.append('<user name="onetick" />')
+
+    ext_users = os.environ.get("TEST_SESSION_ACL_USERS", None)
+
+    if ext_users:
+        for usr in ext_users.split(","):
+            data.append(f'<user name="{usr}" />')
+
+    data.append("</role>")
+    data.append("</roles>")
+
+    data.append("<databases>")
+    data.append("</databases>")
+
+    data.append("<event_processors>")
+    data.append("")
+    data.append('<ep ID="RELOAD_CONFIG">')
+    data.append('<allow role="Admin" />')
+    data.append("</ep>")
+    data.append("")
+    data.append('<ep ID="WRITE_TEXT">')
+    data.append('<allow role="Admin" />')
+    data.append("</ep>")
+    data.append("")
+    data.append('<ep ID="COMMAND_EXECUTE">')
+    data.append('<allow role="Admin" />')
+    data.append("</ep>")
+    data.append("")
+    data.append('<ep ID="READ_FROM_RAW">')
+    data.append('<allow role="Admin" />')
+    data.append("</ep>")
+    data.append("")
+    data.append('<ep ID="SAVE_SNAPSHOT">')
+    data.append('<allow role="Admin" />')
+    data.append("</ep>")
+    data.append("</event_processors>")
+
+    if os.getenv('OTP_WEBAPI_TEST_MODE'):
+        tmp_file = TmpFile(name="acl.xml", clean_up=clean_up, force=True)
+    else:
+        tmp_file = TmpFile(suffix=".acl", clean_up=clean_up)
+
+    with open(tmp_file, "w") as fout:
+        fout.write("\n".join(data))
+
+    return tmp_file

onetick/py/utils/config.py

@@ -0,0 +1,186 @@
+import difflib
+import os
+import sysconfig
+import warnings
+from collections import OrderedDict
+
+from onetick.py.otq import otq
+
+from .temp import TmpFile, TmpDir, ONE_TICK_TMP_DIR
+from ..core import db_constants
+from . import types
+
+
+def reload_config(db=None, config_type='LOCATOR'):
+    import onetick.py as otp
+
+    if db is not None:
+        warnings.warn("Parameter 'db' is deprecated and has no meaning")
+
+    return otp.run(
+        otq.ReloadConfig(config_type=config_type).tick_type('ANY'),
+        symbols='LOCAL::',
+        # start and end times don't matter for this query, use some constants
+        start=db_constants.DEFAULT_START_DATE,
+        end=db_constants.DEFAULT_END_DATE,
+    )
+
+
+def _remove_quotes(s):
+    return s.replace('"', "").replace("'", "")
+
+
+def _read_config_params(path, _included_paths=None):
+    params = OrderedDict()
+
+    if _included_paths is None:
+        _included_paths = set()
+
+    with open(path, "r") as fin:
+        for line in fin:
+            line = line.strip()
+
+            if line.startswith("#"):
+                continue
+
+            if line.startswith("INCLUDE"):
+                included_path = os.path.expandvars(_remove_quotes(line.split()[1]))
+                if included_path in _included_paths:
+                    raise RecursionError(f"Path '{included_path}' is included more than once")
+                _included_paths.add(included_path)
+                included_params = _read_config_params(included_path, _included_paths)
+                params.update({
+                    k: v for k, v in included_params.items()
+                    if k not in params
+                })
+                continue
+
+            values = line.split("=")
+            key, value = values[0].strip(), os.path.expandvars("".join(values[1:]).strip())
+
+            if key and key not in params:
+                params[key] = value.strip()
+
+    return params
+
+
+def _write_config_params(path, params):
+    with open(path, "w") as fout:
+        for key, value in params.items():
+            fout.write(f"{key}={value}\n")
+
+
+def _check_param_in_params(param, params, throw_exception=True):
+    if param in params:
+        return True
+    if not throw_exception:
+        return False
+    closest = difflib.get_close_matches(param, params.keys())
+    msg = f"Parameter '{param}' is not in the config file."
+    if closest:
+        msg = f"{msg} It might be you supposed '{closest[0]}'?"
+    raise AttributeError(msg)
+
+
+def modify_config_param(path, param, value, throw_on_missing=True):
+    params = _read_config_params(path)
+
+    _check_param_in_params(param, params, throw_on_missing)
+
+    params[param] = f'"{value}"'
+
+    _write_config_params(path, params)
+
+
+def get_config_param(path, param, default=None):
+    params = _read_config_params(path)
+
+    res = _check_param_in_params(param, params, throw_exception=(default is None))
+
+    if res:
+        return _remove_quotes(params[param])
+    return default
+
+
+def is_param_in_config(path, param):
+    params = _read_config_params(path)
+
+    return _check_param_in_params(param, params, throw_exception=False)
+
+
+def tmp_config(
+    locator=None,
+    acl=None,
+    otq_path=None,
+    csv_path=None,
+    clean_up=types.default,
+    license_path=None,
+    license_dir=None,
+    license_servers=None,
+):
+    data = []
+
+    if otq_path is None:
+        otq_path = []
+
+    if ONE_TICK_TMP_DIR():
+        otq_path.append(ONE_TICK_TMP_DIR())
+
+    if csv_path is None:
+        csv_path = []
+
+    otq_path = list(map(os.path.normpath, otq_path))
+    csv_path = list(map(os.path.normpath, csv_path))
+
+    data.append("ONE_TICK_CONFIG.ALLOW_ENV_VARS=Yes")
+    data.append("")
+    config_vars = sysconfig.get_config_vars()
+    data.append(f"PYTHON_VERSION='{config_vars['py_version_short']}{config_vars['abiflags']}'")
+
+    if locator:
+        data.append(f'DB_LOCATOR.DEFAULT="{_remove_quotes(str(locator))}"')
+
+    if acl:
+        data.append(f'ACCESS_CONTROL_FILE="{_remove_quotes(str(acl))}"')
+
+    if license_path:
+        data.append(f'ONE_TICK_LICENSE_FILE="{_remove_quotes(str(license_path))}"')
+
+    if license_dir:
+        data.append(f'LICENSE_REPOSITORY_DIR="{_remove_quotes(str(license_dir))}"')
+
+    if license_servers:
+        data.append(f'LICENSE_TICK_SERVERS="{_remove_quotes(str(license_servers))}"')
+
+    if otq_path:
+        data.append(f'OTQ_FILE_PATH="{",".join(otq_path)}"')
+
+    if csv_path:
+        data.append(f'CSV_FILE_PATH="{",".join(csv_path)}"')
+
+    if os.getenv('OTP_WEBAPI_TEST_MODE'):
+        tmp_dir = TmpDir()
+        data.append(f'TICK_SERVER_CSV_CACHE_DIR="{tmp_dir}"')
+        data.append(f'TICK_SERVER_DATA_CACHE_DIR="{tmp_dir}"')
+        data.append(f'TICK_SERVER_OTQ_CACHE_DIR="{tmp_dir}"')
+
+    data.append("")
+    data.append("ALLOW_REMOTE_CONTROL=Yes")
+    data.append("ALLOW_NO_CERTIFICATE=true")
+    # to be able to run otp.ODBC
+    data.append("LOAD_ODBC_UDF=true")
+
+    # # let's not strip quotes around params by default, this is the old behaviour
+    # new default behaviour breaks backward-compatibility (BEXRTS-1340, BEXRTS-1342)
+    # new behaviour can be achieved by setting STRIP_QUOTES_WHEN_ASSIGNING_SYMBOL_AND_OTQ_PARAMS param in JWQ EP
+    data.append("COMPATIBILITY.JOIN_WITH_QUERY.STRIP_QUOTES_AROUND_SYMBOL_AND_OTQ_PARAMS=FALSE")
+
+    if os.getenv('OTP_WEBAPI_TEST_MODE'):
+        tmp_file = TmpFile(name="onetick.cfg", clean_up=clean_up, force=True)
+    else:
+        tmp_file = TmpFile(suffix=".cfg", clean_up=clean_up)
+
+    with open(tmp_file, "w") as fout:
+        fout.write("\n".join(data))
+
+    return tmp_file

onetick/py/utils/default.py

@@ -0,0 +1,49 @@
+import os
+import sys
+import multiprocessing
+import warnings
+
+
+def get_local_number_of_cores():
+    try:
+        # number of cores allowed for user
+        return len(os.sched_getaffinity(0))
+    except Exception:
+        # number of cores
+        return multiprocessing.cpu_count()
+
+
+def default_license_dir():
+    if sys.platform == "linux":
+        return os.path.join("/", "license")
+    elif sys.platform == "win32":
+        return os.path.join("C:\\", "OMD", "client_data", "config", "license_repository")
+    elif sys.platform == "darwin":
+        return os.path.join("/", "Library", "Application Support", "OneTick")
+    return None
+
+
+def default_license_file():
+    if sys.platform == "linux":
+        return os.path.join("/", "license", "license.dat")
+    elif sys.platform == "win32":
+        return os.path.join("C:\\", "OMD", "client_data", "config", "license.dat")
+    elif sys.platform == "darwin":
+        return os.path.join("/", "Library", "Application Support", "OneTick")
+    return None
+
+
+def default_day_boundary_tz(db_name):
+    import onetick.py as otp
+    if otp.config.tz:
+        return otp.config.tz
+    warnings.warn(
+        "Database property 'day_boundary_tz' can't be set to default timezone "
+        "because default timezone is local and is not known at this moment. "
+        "Default value for this property will be 'GMT'. "
+        f"It may produce unexpected results "
+        f"when reading from or writing to database '{db_name}'. "
+        "Set otp.config.tz property to some known value to avoid this situation.",
+        stacklevel=2,
+    )
+    return None

onetick/py/utils/file.py

@@ -0,0 +1,38 @@
+import os
+import zlib
+
+from pathlib import Path
+from typing import Union
+
+
+class FileBuffer:
+    '''
+    Class holds the file content with goal to delivery
+    it to the execution side in case of remote executions.
+
+    The basic implementation reads file content to a property
+    that allows to transfer file content as pickled object
+    to the server side since the pickling stores all class property
+    values.
+    '''
+
+    def __init__(self, path: Union[str, os.PathLike]):
+        content = Path(path).read_text()
+        self._content = zlib.compress(content.encode('utf-8'), level=9)
+
+    def get(self):
+        ''' Returns file content '''
+        return zlib.decompress(self._content).decode('utf-8')
+
+
+def file(path: Union[str, os.PathLike]) -> FileBuffer:
+    '''
+    Helps to build a file buffer that could be used to
+    delivery on the remote site to be processed there.
+    For example it could be passed as input to the :class:`CSV <onetick.py.CSV>`
+
+    See Also
+    --------
+        :class:`CSV <onetick.py.CSV>`
+    '''
+    return FileBuffer(path)

onetick/py/utils/helpers.py

@@ -0,0 +1,76 @@
+import json
+
+from onetick.py.otq import otq, pyomd
+from datetime import datetime
+
+
+def get_symbol_list_from_df(df, symbol_name_column='SYMBOL_NAME'):
+    """
+    Creates a onetick.query.Symbol object that may be passed as a symbol list from the dataframe
+    with query results. SYMBOL_NAME column is interpreted as symbol names, while other columns are
+    interpreted as symbol params.
+    """
+    if symbol_name_column not in df.columns:
+        raise ValueError(f'Dataframe used as symbol list does not contain a {symbol_name_column} column')
+
+    df = df.drop(columns=['Time'])
+
+    def symbol_from_dict(params):
+        name = params[symbol_name_column]
+        del params[symbol_name_column]
+        return otq.Symbol(name=name, params=params)
+
+    symbols = [symbol_from_dict(row) for row in df.to_dict(orient='records')]
+    return symbols
+
+
+class JSONEncoder(json.JSONEncoder):
+    """
+    onetick.py json encoder that also supports some of the onetick.py objects like otp.adaptive.
+    """
+    def default(self, o):
+        # let's use python str representation by default, all objects in python should have it
+        # maybe we will add better representations for some types later
+        return str(o)
+
+
+def json_dumps(obj, **kwargs) -> str:
+    """
+    Wrapper around json.dumps that also supports some of the onetick.py objects like otp.adaptive.
+    ``kwargs`` arguments are propagated to json.dumps function.
+    By default ``cls`` parameter is set to otp.utils.JSONEncoder.
+    """
+    kwargs.setdefault('cls', JSONEncoder)
+    return json.dumps(obj, **kwargs)
+
+
+def query_properties_to_dict(query_properties: pyomd.QueryProperties) -> dict:  # type: ignore[valid-type]
+    """
+    Convert :py:class:`pyomd.QueryProperties` to dictionary.
+    """
+
+    str_qp = query_properties.convert_to_name_value_pairs_string()  # type: ignore[attr-defined]
+    if not isinstance(str_qp, str):
+        str_qp = str_qp.c_str()
+    pairs = str_qp.split(',') if str_qp else []
+    return dict(pair.split('=', maxsplit=1) for pair in pairs)
+
+
+def query_properties_from_dict(query_properties_dict: dict) -> pyomd.QueryProperties:  # type: ignore[valid-type]
+    """
+    Convert dictionary to :py:class:`pyomd.QueryProperties`.
+    """
+    query_properties = pyomd.QueryProperties()
+    for k, v in query_properties_dict.items():
+        query_properties.set_property_value(k, v)
+    return query_properties
+
+
+def symbol_date_to_str(symbol_date) -> str:
+    if isinstance(symbol_date, int):
+        symbol_date = str(symbol_date)
+    if isinstance(symbol_date, str):
+        symbol_date = datetime.strptime(symbol_date, '%Y%m%d')
+    if hasattr(symbol_date, 'strftime'):
+        symbol_date = symbol_date.strftime('%Y%m%d')
+    return symbol_date

onetick/py/utils/locator.py

@@ -0,0 +1,94 @@
+import os
+from locator_parser.io import PrintWriter, FileReader
+from locator_parser.actions import GetAll
+from locator_parser.locator import Location, parse_locator
+from locator_parser.common import apply_actions
+
+from .temp import TmpFile
+from .types import default
+
+
+def get_dbs_locations_from_locator(locator):
+    action = GetAll()
+    action.add_where(Location)
+
+    apply_actions(parse_locator, FileReader(locator), PrintWriter(), [action])
+
+    return map(lambda x: x.location, action.result)
+
+
+def tmp_locator(clean_up=default, empty=False):
+    import onetick.py as otp
+
+    STUBS = {'COMMON'}
+    default_db = otp.config.get('default_db')
+    if default_db:
+        STUBS.add(default_db)
+
+    data = []
+    data.append('<VERSION_INFO VERSION="2"/>')
+    data.append("<DATABASES>")
+    data.append("")
+
+    if not empty:
+        for stub_name in STUBS:
+            data.append(
+                f'<db ID="{stub_name}" symbology="{otp.config.default_symbology}" time_series_is_composite="YES">'
+            )
+            data.append("<locations>")
+            day_boundary_tz = otp.config.get('tz')
+            day_boundary_tz = f'day_boundary_tz="{day_boundary_tz}"' if day_boundary_tz else ''
+            data.append(
+                '<location access_method="file" location="/opt/one_market_data/one_tick/examples/data/demo_level1" '
+                'start_time="20001201000000" end_time="20301031050000" '
+                f'{day_boundary_tz}/>'
+            )
+            data.append("</locations>")
+            data.append('<feed type="heartbeat_generator">')
+            data.append('<options format="native" />')
+            data.append('</feed>')
+            data.append("</db>")
+            data.append("")
+            data.append("")
+
+    data.append("</DATABASES>")
+
+    data.append("<TICK_SERVERS>")
+    data.append("</TICK_SERVERS>")
+    data.append("<CEP_TICK_SERVERS>")
+    data.append("</CEP_TICK_SERVERS>")
+    data.append("<INCLUDES>")
+    data.append("</INCLUDES>")
+
+    tmp_file = new_tmp_locator(clean_up)
+
+    with open(tmp_file, "w") as fout:
+        fout.write("\n".join(data))
+
+    return tmp_file
+
+
+def new_tmp_locator(clean_up):
+    if os.getenv('OTP_WEBAPI_TEST_MODE'):
+        tmp_file = TmpFile(name="locator.default", clean_up=clean_up, force=True)
+    else:
+        tmp_file = TmpFile(suffix=".locator", clean_up=clean_up)
+    return tmp_file
+
+
+def empty_locator(clean_up=default):
+    data = []
+    data.append('<VERSION_INFO VERSION="2"/>')
+    data.append("<DATABASES>")
+    data.append("</DATABASES>")
+    data.append("<TICK_SERVERS>")
+    data.append("</TICK_SERVERS>")
+    data.append("<CEP_TICK_SERVERS>")
+    data.append("</CEP_TICK_SERVERS>")
+    data.append("<INCLUDES>")
+    data.append("</INCLUDES>")
+
+    tmp_file = new_tmp_locator(clean_up)
+    with open(tmp_file, "w") as fout:
+        fout.write("\n".join(data))
+    return tmp_file