onetick-py 1.177.0__py3-none-any.whl

onetick/py/utils/temp.py

@@ -0,0 +1,471 @@
+import abc
+import datetime
+import errno
+import getpass
+import os
+import shutil
+import tempfile
+import weakref
+from collections import defaultdict
+from typing import Dict, List
+from .types import default
+
+import coolname
+
+
+WEBAPI_TEST_MODE_SHARED_CONFIG = os.getenv('WEBAPI_TEST_MODE_SHARED_CONFIG')
+if os.getenv('OTP_WEBAPI_TEST_MODE') and not WEBAPI_TEST_MODE_SHARED_CONFIG:
+    raise ValueError("WEBAPI_TEST_MODE_SHARED_CONFIG is not set, but it is required with OTP_WEBAPI_TEST_MODE.")
+
+
+def default_clean_up(clean_up):
+    from ..configuration import config
+    if clean_up is default:
+        return config.clean_up_tmp_files
+    return clean_up
+
+
+def get_logger(*args):
+    from .. import log
+    return log.get_logger(__name__, *args)
+
+
+class CleanUpFinalizer:
+    """
+    Class that manages destruction of the object,
+    setting up proper finalizer callback and clean_up boolean flag.
+    """
+
+    def __init__(self, clean_up=default, *args):
+        clean_up = default_clean_up(clean_up)
+        # we need to use reference type here so finalizer can get the latest clean_up value,
+        # even if this value is changed after finalizer is created
+        # (because weakref.finalize can't use object's fields)
+        self._clean_up_ref = [clean_up]
+        self._finalizer = weakref.finalize(self, self._cleanup, self._clean_up_ref, *args)
+
+    @classmethod
+    def _cleanup(cls, clean_up_ref, *args):
+        raise NotImplementedError()
+
+    @property
+    def need_to_cleanup(self):
+        return self._clean_up_ref[0]
+
+    @need_to_cleanup.setter
+    def need_to_cleanup(self, value):
+        self._clean_up_ref[0] = value
+
+
+class File(abc.ABC):
+    def __init__(self, path):
+        self._path = path
+
+    @property
+    def path(self):
+        return self._path
+
+    def __str__(self):
+        return self._path
+
+    def __fspath__(self):
+        """
+        If children inherit os.PathLike, then this method is required and
+        makes files behave compatible with 'os' module
+        """
+        return self._path
+
+
+def __name_generator():
+    """
+    Returns tuple with initialized name generator and
+    the number of unique names this generator will produce
+    before it starts to repeat itself.
+    """
+    def cool_gen():
+        while True:
+            yield coolname.generate_slug(2)
+    return cool_gen(), coolname.get_combinations_count(2)
+
+
+def mktemp(fun, dir, prefix='', suffix='', **kwargs):
+    names, max_unique_values = __name_generator()
+    for _ in range(max_unique_values):
+        name = next(names)
+        filename = os.path.join(dir, prefix + name + suffix)
+        try:
+            return fun(filename, **kwargs), filename
+        except FileExistsError:
+            continue
+        except PermissionError:
+            # This exception is thrown when a directory with the chosen name already exists on Windows
+            if os.name == 'nt' and os.path.isdir(dir) and os.access(dir, os.W_OK):
+                continue
+            else:
+                raise
+    raise FileExistsError(errno.EEXIST, 'No usable temporary file name found')
+
+
+def mkstemp(dir, prefix='', suffix=''):
+    flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
+    if hasattr(os, 'O_NOFOLLOW'):
+        flags |= os.O_NOFOLLOW
+    return mktemp(os.open, dir, prefix, suffix, flags=flags, mode=0o600)
+
+
+def mkdtemp(dir, prefix='', suffix=''):
+    _, filename = mktemp(os.mkdir, dir, prefix, suffix, mode=0o700)
+    return filename
+
+
+_TMP_CONFIGS_DIR_BASE = os.environ.get('OTP_BASE_FOLDER_FOR_GENERATED_RESOURCE',
+                                       os.path.join(tempfile.gettempdir(), "test_" + getpass.getuser()))
+
+
+class TmpFile(File, os.PathLike, CleanUpFinalizer):
+
+    ALL: Dict[str, List['TmpFile']] = defaultdict(list)
+    keep_everything_generated = False
+
+    def __init__(self, suffix="", name="", clean_up=default, force=False, base_dir=None):
+        """
+        Class to create a temporary file.
+        By default, this file will be deleted automatically after all references to it are gone.
+        Base path where temporary files are created could be set using the ``ONE_TICK_TMP_DIR``.
+        By default they are created under the ``tempfile.gettempdir()`` folder.
+
+
+        Parameters
+        ----------
+        name: str
+            name of the temporary file without suffix.
+            By default some random name will be generated.
+        suffix: str
+            suffix of the name of the temporary file.
+        clean_up: bool
+            Controls whether this temporary file will be deleted automatically
+            after all references to it are gone.
+
+            By default,
+            :py:attr:`otp.config.clean_up_tmp_files<onetick.py.configuration.Config.clean_up_tmp_files>` is used.
+        force: bool
+            Rewrite temporary file if it exists and parameter ``name`` is set.
+        base_dir: str
+            Absolute path of the directory where temporary file will be created.
+
+        See also
+        --------
+
+        The testing framework has a ``--keep-generated`` flag that controls clean up for all related instances
+        :ref:`onetick py test features`
+        """
+        clean_up = default_clean_up(clean_up)
+        clean_up = clean_up and not TmpFile.keep_everything_generated
+        fd, self._path = self._create(clean_up, suffix=suffix, name=name, force=force, base_dir=base_dir)
+        # we only needed to create file, so closing opened file descriptor
+        os.close(fd)
+        self._logger = get_logger(self.__class__.__name__)
+        CleanUpFinalizer.__init__(self, clean_up, self.path, self._logger)
+        self._logger.debug(f'created {self._path}, clean_up={self.need_to_cleanup}')
+
+    def __repr__(self):
+        return f"TmpFile({self.path})"
+
+    @classmethod
+    def _cleanup(cls, clean_up_ref, path, logger):
+        if clean_up_ref[0] and os.path.exists(path):
+            logger.debug(f'removing {path}')
+            try:
+                os.remove(path)
+            except Exception:
+                # TODO: remove try-except block when BDS-116 will be fixed
+                pass
+
+    def _create(self, clean_up, suffix="", name="", force=False, base_dir=None):
+
+        self._parent_dir = None
+
+        if ONE_TICK_TMP_DIR() and not base_dir:
+            # allows to set a test name suffix for when we use it in
+            # tests with goal to distinguish tests content (configs, databases, etc)
+            # from each other
+            dir_path = ONE_TICK_TMP_DIR()
+            # save to access them from the teardown
+            TmpFile.ALL[dir_path].append(self)
+
+            if not os.path.exists(dir_path):
+                # If it does not exist, then lets create it. Note that it goes up
+                # recursively until finds an existing folder.
+                # We store it as a field of the instance to guarantee that it will
+                # be destroyed later than a temporary file.
+                self._parent_dir = GeneratedDir(dir_path, clean_up=clean_up)
+            elif dir_path in GeneratedDir.ALL:
+                self._parent_dir = GeneratedDir.ALL[dir_path]
+        else:
+            if base_dir and not os.path.exists(base_dir):
+                # If it does not exist, then lets create it. Note that it goes up
+                # recursively until finds an existing folder.
+                # We store it as a field of the instance to guarantee that it will
+                # be destroyed later than a temporary file.
+                self._parent_dir = GeneratedDir(base_dir, clean_up=clean_up)
+            dir_path = base_dir if base_dir else TMP_CONFIGS_DIR()
+            TmpFile.ALL[dir_path].append(self)
+            if dir_path in GeneratedDir.ALL:
+                self._parent_dir = GeneratedDir.ALL[dir_path]
+
+        # let know the parent generated folder to not destroy itself
+        if self._parent_dir is not None and not clean_up:
+            self._parent_dir.need_to_cleanup = False
+
+        if name:
+            path = os.path.normpath(os.path.join(dir_path, name)) + suffix
+            flags = os.O_RDWR | os.O_CREAT
+            if not force:
+                flags = flags | os.O_EXCL
+            if hasattr(os, 'O_NOFOLLOW'):
+                flags |= os.O_NOFOLLOW
+            mode = 0o600
+            if os.getenv('OTP_WEBAPI_TEST_MODE'):
+                mode = 0o644
+            return os.open(path, flags=flags, mode=mode), path
+        else:
+            return mkstemp(dir=dir_path, suffix=suffix, prefix="")
+
+
+class TmpDir(os.PathLike, CleanUpFinalizer):
+
+    ALL: Dict[str, List['TmpDir']] = defaultdict(list)
+    keep_everything_generated = False
+
+    def __init__(self, rel_path="", *, suffix="", clean_up=default, base_dir=""):
+        """
+        Class to create a temporary directory.
+        By default, this directory will be deleted automatically after all references to it are gone.
+        All files and directories under this one will be deleted too.
+
+        Base path where directories are created could be set using the ``ONE_TICK_TMP_DIR``.
+        By default they are created under the ``tempfile.gettempdir()`` folder.
+
+        Parameters
+        ----------
+        rel_path: str
+            relative path to the temporary directory.
+            If empty, then the name will be auto-generated.
+        suffix: str
+            suffix of the name of the temporary directory.
+        base_dir: str
+            relative path of the directory where temporary directory will be created.
+        clean_up: bool
+            Controls whether this temporary directory will be deleted automatically
+            after all references to it are gone.
+
+            By default,
+            :py:attr:`otp.config.clean_up_tmp_files<onetick.py.configuration.Config.clean_up_tmp_files>` is used.
+
+        See also
+        --------
+
+        The testing framework has a ``--keep-generated`` flag that controls clean up for all related instances
+        :ref:`onetick py test features`
+        """
+        clean_up = default_clean_up(clean_up)
+        clean_up = clean_up and not TmpDir.keep_everything_generated
+        self._parent_dir = None
+
+        if os.path.isabs(rel_path):
+            raise ValueError("Absolute paths are not supported in 'rel_path' parameter.")
+
+        if ONE_TICK_TMP_DIR():
+            # allows to set a test name suffix for when we use it in
+            # tests with goal to distinguish tests content (configs, databases, etc)
+            # from each other
+            dir_path = os.path.normpath(os.path.join(ONE_TICK_TMP_DIR(), base_dir))
+
+            if not os.path.exists(dir_path):
+                # If it does not exist, then lets create it. Note that it goes up
+                # recursively until finds an existing folder.
+                # We store it as a field of the instance to guarantee that it will
+                # be destroyed later than a temporary file.
+                self._parent_dir = GeneratedDir(dir_path, clean_up=clean_up)
+            elif not clean_up:
+                if dir_path in GeneratedDir.ALL:  # NOSONAR
+                    # let know the parent generated folder to not destroy itself
+                    GeneratedDir.ALL[dir_path].need_to_cleanup = False
+
+                    self._parent_dir = GeneratedDir.ALL[dir_path]
+
+        else:
+            dir_path = os.path.normpath(os.path.join(TMP_CONFIGS_DIR(), base_dir))
+
+        if rel_path:
+            path = os.path.normpath(os.path.join(dir_path, rel_path)) + suffix
+            # dir_path should be the parent directory of path
+            dir_path = os.path.dirname(path)
+            for tmp_dir in TmpDir.ALL[dir_path]:
+                # check if path already exists
+                if tmp_dir.path == path:
+                    self.path = tmp_dir.path
+                    if not clean_up:
+                        tmp_dir.need_to_cleanup = clean_up
+                    return
+
+            os.mkdir(path, mode=0o700)
+            self.path = path
+        else:
+            self.path = mkdtemp(dir=dir_path, suffix=suffix, prefix="")
+
+        # save to access them in the teardown
+        TmpDir.ALL[dir_path].append(self)
+
+        self._logger = get_logger(self.__class__.__name__)
+        CleanUpFinalizer.__init__(self, clean_up, self.path, self._logger)
+        self._logger.debug(f'created {self.path}, clean_up={self.need_to_cleanup}')
+
+    @CleanUpFinalizer.need_to_cleanup.setter   # type: ignore
+    def need_to_cleanup(self, value):
+        super(TmpDir, type(self)).need_to_cleanup.fset(self, value)
+        children = self.ALL[self.path]
+        for child in children:
+            child.need_to_cleanup = value
+
+    @classmethod
+    def _cleanup(cls, clean_up_ref, path, logger):
+        if clean_up_ref[0]:
+            logger.debug(f'removing {path}')
+            shutil.rmtree(path, ignore_errors=True)
+
+    def __str__(self):
+        return self.path
+
+    def __repr__(self):
+        return f"TmpDir({self.path})"
+
+    def __fspath__(self):
+        return self.path
+
+
+class GeneratedDir(os.PathLike, CleanUpFinalizer):
+
+    ALL: Dict[str, 'GeneratedDir'] = {}  # store all created dir with goal to set cleanup=False if it requires
+    keep_everything_generated = False
+
+    def __init__(self, dir_path, clean_up=default):
+        """
+        Class to create generated temporary directory.
+        By default, this directory will be deleted automatically after all references to it are gone.
+        All files and directories under this one will be deleted too.
+
+        The main difference from TmpDir class is that also this directory's parent directory will be deleted!
+
+        The exact path to this directory will be: ``dir_path``.
+
+        Parameters
+        ----------
+        dir_path: str
+            path to the temporary directory.
+        clean_up: bool
+            Controls whether this temporary directory will be deleted automatically
+            after all references to it are gone.
+
+            By default,
+            :py:attr:`otp.config.clean_up_tmp_files<onetick.py.configuration.Config.clean_up_tmp_files>` is used.
+        """
+
+        parent = os.path.normpath(os.path.dirname(dir_path))
+
+        clean_up = default_clean_up(clean_up)
+        clean_up = clean_up and not GeneratedDir.keep_everything_generated
+
+        self._parent_dir = None
+        if not os.path.exists(parent):
+            # Do the same thing for the parent folder.
+            # Save it as a field to guarantee that child folder is destroyed
+            # earlier than parent
+            self._parent_dir = GeneratedDir(parent, clean_up=clean_up)
+        elif not clean_up:
+            if parent in GeneratedDir.ALL:  # NOSONAR
+                GeneratedDir.ALL[parent].need_to_cleanup = False
+
+        if self._parent_dir is None and parent in GeneratedDir.ALL:
+            self._parent_dir = GeneratedDir.ALL[parent]
+
+        self.path = dir_path
+
+        if os.path.exists(dir_path):
+            # Do not create and even handle it, because it has been created externally.
+            # It mgiht happen in the concurrent runs.
+            return
+
+        # We need to set exist_ok=True, because it still can be created externally in the concurrent runs
+        os.makedirs(dir_path, exist_ok=True)
+
+        self._logger = get_logger(self.__class__.__name__)
+        CleanUpFinalizer.__init__(self, clean_up, self.path, self._logger)
+        self._logger.debug(f'created {self.path}, clean_up={self.need_to_cleanup}')
+
+        GeneratedDir.ALL[dir_path] = self
+
+    @CleanUpFinalizer.need_to_cleanup.setter   # type: ignore
+    def need_to_cleanup(self, value):
+        super(GeneratedDir, type(self)).need_to_cleanup.fset(self, value)
+
+        if self._parent_dir:
+            self._parent_dir.need_to_cleanup = value
+
+    @classmethod
+    def _cleanup(cls, clean_up_ref, path, logger):
+        if clean_up_ref[0]:
+            logger.debug(f'removing {path}')
+            shutil.rmtree(path, ignore_errors=True)
+
+    def __str__(self):
+        return self.path
+
+    def __repr__(self):
+        return f"GeneratedDir({self.path})"
+
+    def __fspath__(self):
+        return self.path
+
+
+# ----------------------------------------------------
+# We set TMP_CONFIGS_DIR to a function, that returns
+# only the first result.
+# We need to support multiprocessing environment, where
+# every process should use it's own base temporary folder
+# but generate it only when something is required, because
+# otherwise parent process could construct it with its pid
+# and other processes would use the same location.
+
+
+def _gen_root_path_():
+    generated = []
+
+    def _inner_():
+        if not generated:
+            generated.append(
+                GeneratedDir(
+                    os.path.join(
+                        _TMP_CONFIGS_DIR_BASE,
+                        f'run_{datetime.datetime.now().strftime("%Y%m%d_%H%M%S")}_{os.getpid()}',
+                    )
+                ).path
+            )
+        return generated[0]
+
+    return _inner_
+
+
+TMP_CONFIGS_DIR = _gen_root_path_()
+def ONE_TICK_TMP_DIR():  # noqa # NOSONAR
+    if os.getenv('OTP_WEBAPI_TEST_MODE'):
+        return WEBAPI_TEST_MODE_SHARED_CONFIG
+    if 'ONE_TICK_TMP_DIR' not in os.environ:
+        return None
+    return os.path.normpath(os.path.join(TMP_CONFIGS_DIR(), os.environ['ONE_TICK_TMP_DIR']))
+# ----------------------------------------------------
+
+
+class PermanentFile(File):
+    def copy(self):
+        return TmpFile(self._path)

onetick/py/utils/types.py

@@ -0,0 +1,120 @@
+def get_type_that_includes(types):
+    import onetick.py.types as ott
+
+    def merge_two(type1, type2):
+        type_changed = False
+
+        b_type1, b_type2 = ott.get_base_type(type1), ott.get_base_type(type2)
+        if b_type1 != b_type2:
+            if {b_type1, b_type2} == {int, float}:
+                dtype = float
+            elif {b_type1, b_type2} == {ott.decimal, float} or {b_type1, b_type2} == {ott.decimal, int}:
+                dtype = ott.decimal
+            elif {b_type1, b_type2} == {ott.nsectime, ott.msectime}:
+                dtype = ott.nsectime
+            else:
+                raise ValueError(f"Incompatible types: {type1}, {type2}")
+
+            type_changed = True
+        elif issubclass(b_type1, str):
+            t1_length = ott.string.DEFAULT_LENGTH if type1 is str or type1.length is None else type1.length
+            t2_length = ott.string.DEFAULT_LENGTH if type2 is str or type2.length is None else type2.length
+
+            if t1_length is Ellipsis or t2_length is Ellipsis:
+                dtype = ott.varstring
+            else:
+                dtype = type2 if t1_length < t2_length else type1
+
+            if t1_length != t2_length:
+                type_changed = True  # TODO: test
+
+        else:
+            dtype = type1
+
+        return dtype, type_changed
+
+    dtype = types[0]
+    type_changed = False
+
+    for other_dtype in types[1:]:
+        dtype, new_type_change = merge_two(dtype, other_dtype)
+        type_changed |= new_type_change
+
+    return dtype, type_changed
+
+
+class adaptive:
+    """
+    This class is mostly used as the default value for the functions' parameters
+    when the value of ``None`` has some other meaning
+    or when the meaning of the parameter depends on the other parameter's values,
+    :ref:`otp.config <static/configuration/root:configuration>` options or the context.
+
+    Examples
+    --------
+
+    For example, setting :py:class:`~onetick.py.DataSource` ``symbols`` parameter
+    to ``otp.adaptive`` allows to set symbols when running the query later.
+
+    >>> data = otp.DataSource('SOME_DB', tick_type='TT', symbols=otp.adaptive)
+    >>> otp.run(data, symbols='S1')
+                         Time  X
+    0 2003-12-01 00:00:00.000  1
+    1 2003-12-01 00:00:00.001  2
+    2 2003-12-01 00:00:00.002  3
+
+    This is the default value of ``symbols`` parameter, so omitting it also works:
+
+    >>> data = otp.DataSource('SOME_DB', tick_type='TT')
+    >>> otp.run(data, symbols='S1')
+                         Time  X
+    0 2003-12-01 00:00:00.000  1
+    1 2003-12-01 00:00:00.001  2
+    2 2003-12-01 00:00:00.002  3
+    """
+
+
+class adaptive_to_default(adaptive):
+    """
+    If something is not specified and can not be deduced, then use the
+    default one
+    """
+
+
+class default:
+    """
+    Used when you need to specify a default without evaluating it (e.g. default timezone)
+    """
+
+
+class range:
+    """
+    Class that expresses OneTick ranges.
+    For example, if you want to express a range in the .split() method,
+    then you can use this range.
+
+    It has start and stop fields that allow you to define a range.
+
+    See also
+    --------
+    :py:meth:`~onetick.py.Source.split`.
+
+    Examples
+    --------
+    >>> data = otp.Ticks(X=[0.33, -5.1, otp.nan, 9.4])
+    >>> r1, r2, r3 = data.split(data['X'], [otp.nan, otp.range(0, 100)], default=True)
+    >>> otp.run(r1)
+                         Time   X
+    0 2003-12-01 00:00:00.002 NaN
+    >>> otp.run(r2)
+                         Time     X
+    0 2003-12-01 00:00:00.000  0.33
+    1 2003-12-01 00:00:00.003  9.40
+    >>> otp.run(r3)
+                         Time    X
+    0 2003-12-01 00:00:00.001 -5.1
+    """
+
+    def __init__(self, start, stop):
+        self.start = start
+        self.stop = stop

onetick/py/utils/tz.py

@@ -0,0 +1,84 @@
+import datetime
+import sys
+import warnings
+
+from typing import Optional
+from contextlib import suppress
+
+import dateutil.tz
+import tzlocal
+
+import onetick.py as otp
+from onetick.py.backports import zoneinfo
+
+
+def get_tzfile_by_name(timezone):
+    if isinstance(timezone, str):
+        try:
+            timezone = zoneinfo.ZoneInfo(timezone)
+        except zoneinfo.ZoneInfoNotFoundError:
+            timezone = dateutil.tz.gettz(timezone)
+    return timezone
+
+
+def get_local_timezone_name():
+    tz = tzlocal.get_localzone()
+    try:
+        return tz.zone  # type: ignore
+    except AttributeError:
+        return tz.key  # type: ignore
+
+
+def get_timezone_from_datetime(dt) -> Optional[str]:
+    tzinfo = getattr(dt, 'tzinfo', None)
+    if tzinfo is None:
+        return None
+    if tzinfo is datetime.timezone.utc:
+        return 'UTC'
+    with suppress(ModuleNotFoundError):
+        import pytz
+        if isinstance(tzinfo, pytz.BaseTzInfo):
+            return tzinfo.zone
+    if isinstance(tzinfo, zoneinfo.ZoneInfo):
+        return tzinfo.key
+    if isinstance(tzinfo, dateutil.tz.tzlocal):
+        return get_local_timezone_name()
+    if isinstance(tzinfo, dateutil.tz.tzstr) and hasattr(tzinfo, '_s'):
+        return tzinfo._s
+    if isinstance(tzinfo, dateutil.tz.tzfile):
+        if sys.platform == 'win32':
+            warnings.warn(
+                "It's not recommended to use dateutil.tz timezones on Windows platform. "
+                "Function 'get_timezone_from_datetime' can't guarantee correct results in this case. "
+                "Please, use zoneinfo timezones instead."
+            )
+        if hasattr(tzinfo, '_filename'):
+            if tzinfo._filename == '/etc/localtime':
+                return get_local_timezone_name()
+            for timezone in zoneinfo.available_timezones():
+                if tzinfo._filename.endswith(timezone):
+                    return timezone
+    if sys.platform == 'win32':
+        if isinstance(tzinfo, dateutil.tz.win.tzwin) and hasattr(tzinfo, '_name'):
+            return tzinfo._name
+    raise ValueError(f"Can't get timezone name from datetime '{dt}' with tzinfo {tzinfo}")
+
+
+def convert_timezone(dt, src_timezone, dest_timezone) -> datetime.datetime:
+    """
+    Converting timezone-naive ``dt`` object localized in ``src_timezone`` timezone
+    to the specified ``dest_timezone`` and returning also timezone-naive object.
+    """
+    if src_timezone is None:
+        src_timezone = get_local_timezone_name()
+    # using pandas, because stdlib datetime has some bug around epoch on Windows
+    dt = otp.datetime(dt).ts
+    # change timezone-naive to timezone-aware
+    dt = dt.tz_localize(src_timezone)
+    # convert timezone
+    dt = dt.tz_convert(dest_timezone)
+    # change timezone-aware to timezone-naive
+    dt = dt.tz_localize(None)
+    # convert to datetime
+    dt = dt.to_pydatetime()
+    return dt