onetick-py 1.177.0__tar.gz → 1.180.0__tar.gz

{onetick_py-1.177.0/src/onetick_py.egg-info → onetick_py-1.180.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: onetick-py
-Version: 1.177.0
+Version: 1.180.0
 Summary: Python package that allows you to work with OneTick
 Author-email: solutions <solutions@onetick.com>
 License-Expression: MIT
 Project-URL: Documentation, https://docs.pip.distribution.sol.onetick.com
 Project-URL: OneTick, https://onetick.com
 Project-URL: GitHub, https://github.com/onemarketdata/onetick-py
+Project-URL: Issues, https://github.com/onemarketdata/onetick-py/issues
 Classifier: Topic :: Database :: Front-Ends
 Classifier: Topic :: Scientific/Engineering
 Classifier: Programming Language :: Python :: 3.9

{onetick_py-1.177.0 → onetick_py-1.180.0}/pyproject.toml

@@ -39,6 +39,7 @@ classifiers = [
 Documentation = "https://docs.pip.distribution.sol.onetick.com"
 OneTick = "https://onetick.com"
 GitHub = "https://github.com/onemarketdata/onetick-py"
+Issues = "https://github.com/onemarketdata/onetick-py/issues"
 
 [tool.setuptools.packages.find]
 where = ["src"]

{onetick_py-1.177.0 → onetick_py-1.180.0}/setup.py

@@ -31,6 +31,7 @@ setup(name='onetick-py',
       },
       entry_points={
           'console_scripts': [
+              'onetick-render = onetick.py.utils.render_cli:main',
               'jupyter-onetick_snippets = onetick.doc_utilities.snippets:main',
           ],
       })

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/__init__.py

@@ -192,7 +192,7 @@ from onetick.py.sources import (Tick, TTicks, Ticks, Orders, Trades, NBBO, Quote
                                 Custom, query, Symbols, Empty, DataSource, LocalCSVTicks, SymbologyMapping,
                                 ObSnapshot, ObSnapshotWide, ObSnapshotFlat, ObSummary, ObSize, ObVwap, ObNumLevels,
                                 by_symbol, ODBC, SplitQueryOutputBySymbol, DataFile, PointInTime,
-                                ReadSnapshot, ShowSnapshotList, FindSnapshotSymbols)
+                                ReadSnapshot, ShowSnapshotList, FindSnapshotSymbols, ReadFromDataFrame)
 from onetick.py.utils import adaptive, range, perf
 from onetick.py.session import Session, TestSession, Config, Locator, HTTPSession
 from onetick.py.servers import RemoteTS, LoadBalancing, FaultTolerance

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/_version.py

@@ -1,2 +1,2 @@
 # This file was generated automatically. DO NOT CHANGE.
-VERSION = '1.177.0'
+VERSION = '1.180.0'

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/compatibility.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, astuple
 from datetime import datetime
 from typing import Optional
 
+import pandas as pd
 from packaging.version import parse as parse_version
 
 import onetick.py as otp
@@ -103,6 +104,36 @@ def _parse_release_string(release_string: str, build_number: int) -> OnetickVers
     raise ValueError(f"Unknown release type '{release_type}' in release string '{release_string}'")
 
 
+def _get_locator_min_date(db_name, context):
+    graph = otq.GraphQuery(otq.DbShowConfiguredTimeRanges(db_name=db_name).tick_type('ANY')
+                           >> otq.Table(fields='long START_DATE, long END_DATE'))
+    symbols = f'{db_name}::'
+    result = otq.run(graph,
+                     symbols=symbols,
+                     # start and end times don't matter for this query, use some constants
+                     start=datetime(2003, 12, 1),
+                     end=datetime(2003, 12, 1),
+                     # GMT, because start/end timestamp in locator are in GMT
+                     timezone='GMT',
+                     context=context)
+    data = result.output(symbols).data
+    first_date = data['START_DATE'][0]
+    return datetime.fromtimestamp(first_date / 1000)
+
+
+def _get_onetick_version(symbols, context, start, end):
+    node = otq.TickGenerator(bucket_interval=0,
+                             fields='BUILD=GET_ONETICK_VERSION(), RELEASE=GET_ONETICK_RELEASE()')
+    graph = otq.GraphQuery(node.tick_type('DUMMY'))
+    result = otq.run(graph,
+                     symbols=symbols,
+                     start=start,
+                     end=end,
+                     context=context,
+                     timezone='UTC')
+    return result
+
+
 @cache
 def get_onetick_version(db=None, context=None) -> OnetickVersionFromServer:
     """
@@ -132,27 +163,31 @@ def get_onetick_version(db=None, context=None) -> OnetickVersionFromServer:
         s = otp.Session()
     else:
         _ = otli.OneTickLib()
-    p = otq.TickGenerator(bucket_interval=0,
-                          fields='BUILD=GET_ONETICK_VERSION(), RELEASE=GET_ONETICK_RELEASE()')
-    graph = otq.GraphQuery(p.tick_type('DUMMY'))
 
     # if otp.config.default_db is set, then we use it to check compatibility
     # otherwise we use LOCAL database available everywhere
     db = db or otp.config.get('default_db', 'LOCAL')
-    dummy_symbol = f'{db}::'
+    symbols = f'{db}::'
     context = context or otp.config.context
 
-    result = otq.run(graph,
-                     symbols=dummy_symbol,
-                     start=datetime(2003, 12, 1),
-                     end=datetime(2003, 12, 2),
-                     context=context,
-                     timezone='UTC')
-    if s:
-        s.close()
+    try:
+        # let's try some default time range first
+        start = end = datetime(2003, 12, 1)
+        result = _get_onetick_version(symbols, context, start, end)
+    except Exception:
+        if db != 'LOCAL':
+            # for real db we need to set time range correctly
+            # otherwise we may get error "Database locator has a gap"
+            start = end = _get_locator_min_date(db, context)
+            result = _get_onetick_version(symbols, context, start, end)
+        else:
+            raise
+    finally:
+        if s:
+            s.close()
 
-    build_number = result[dummy_symbol]["BUILD"][0]
-    release_string = result[dummy_symbol]["RELEASE"][0]
+    build_number = result[symbols]["BUILD"][0]
+    release_string = result[symbols]["RELEASE"][0]
 
     try:
         onetick_version = _parse_release_string(release_string, build_number=build_number)
@@ -667,7 +702,22 @@ def is_percentile_bug_fixed():
 
 
 def is_limit_ep_supported():
-    return hasattr(otq, 'Limit')
+    # Implemented 0034293: LIMIT ep
+    return (
+        hasattr(otq, 'Limit') and
+        _is_min_build_or_version(1.25, 20241229055942,
+                                 20241018120000, min_update_number=1)
+    )
+
+
+def is_limit_tick_offset_supported():
+    # Implemented OTDEV-37257: LIMIT EP should support TICK_OFFSET parameter
+    return (
+        is_limit_ep_supported() and
+        'tick_offset' in otq.Limit.Parameters.list_parameters() and
+        _is_min_build_or_version(None, None,
+                                 20251010120000, min_update_number=2)
+    )
 
 
 def is_prefer_speed_over_accuracy_supported(**kwargs):
@@ -733,8 +783,11 @@ def is_multi_column_generic_aggregations_supported():
 
 
 def is_max_concurrency_with_webapi_supported():
+    # 0036758: in onetick.query_webapi: max_concurrency is not being saved in otq file when set on otq.Query
+    # 0036759: in onetick.query_webapi:
+    # it's not possible to pass max_concurrency 0 in method otq.run when using otq file
     return _is_min_build_or_version(None, None,
-                                    20250227120000, min_update_number=2)
+                                    20250727120000, min_update_number=3)
 
 
 def is_nanoseconds_fixed_in_run():
@@ -783,12 +836,16 @@ def is_show_db_list_show_description_supported():
 
 def is_symbols_prepend_db_name_supported():
     # 20250924: Implemented 0036753: FIND_DB_SYMBOLS should have EP parameter PREPEND_DB_NAME (true by default)
-    return hasattr(otq.FindDbSymbols.Parameters, 'prepend_db_name')
+    return hasattr(otq.FindDbSymbols.Parameters, 'prepend_db_name') and _is_min_build_or_version(
+        None, None, 20251010120000,
+    )
 
 
 def is_diff_show_all_ticks_supported():
     # 20250919: Implemented 0036784: Add SHOW_ALL_TICKS(false by default) ep parameter to DIFF EP.
-    return hasattr(otq.Diff.Parameters, 'show_all_ticks')
+    return hasattr(otq.Diff.Parameters, 'show_all_ticks') and _is_min_build_or_version(
+        None, None, 20251010120000,
+    )
 
 
 def is_max_spread_supported():
@@ -796,3 +853,15 @@ def is_max_spread_supported():
     # should also support parameter MAX_SPREAD
     return _is_min_build_or_version(None, None,
                                     20251010120000)
+
+
+def is_not_fixed_bds_484():
+    # BDS-484: seems like timezone is ignored in otq.run in some cases
+    return _is_min_build_or_version(None, None,
+                                    20251010120000, min_update_number=2)
+
+
+def is_webapi_access_token_scope_supported():
+    # 20251030: Fixed OTDEV-37063: onetick.query_webapi.get_access_token method must take scope as a parameter
+    return _is_min_build_or_version(None, None,
+                                    20251010120000, min_update_number=2)

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/configuration.py

@@ -644,6 +644,13 @@ class Config:
         env_var_name='OTP_ACCESS_TOKEN_URL',
     )
 
+    access_token_scope = OtpProperty(
+        description='Scope for obtaining SSO access token.',
+        base_default=None,
+        allowed_types=str,
+        env_var_name='OTP_ACCESS_TOKEN_SCOPE',
+    )
+
     trusted_certificates_file = OtpProperty(
         description='Either a boolean, in which case it controls whether we verify the server TLS certificate '
                     'or a string with the path to the file with list of '

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/core/_source/source_methods/fields.py

@@ -8,6 +8,7 @@ from onetick.py.compatibility import is_existing_fields_handling_supported
 from onetick.py.core._internal._state_objects import _StateColumn
 from onetick.py.core.column import _Column, _ColumnAggregation, _LagOperator
 from onetick.py.core.column_operations._methods.methods import is_arithmetical, is_compare
+from onetick.py.core.column_operations._methods.op_types import are_ints_not_time
 from onetick.py.core.column_operations.base import _Operation
 from onetick.py.core.cut_builder import _BaseCutBuilder
 from onetick.py.core.lambda_object import _LambdaIfElse
@@ -280,6 +281,13 @@ def _update_field(self: 'Source', field, value):
         elif issubclass(field.dtype, str) and base_type is int:
             type_changes = True
             convert_to_type = int
+        elif (
+            are_ints_not_time(field.dtype, value_dtype) and
+            not issubclass(field.dtype, bool) and not issubclass(value_dtype, bool) and
+            value_dtype is not field.dtype
+        ):
+            # case for converting values between int based types, like long or byte
+            convert_to_type = value_dtype
         else:
             if issubclass(value_dtype, bool):
                 value_dtype = float

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/core/_source/source_methods/misc.py

@@ -1,9 +1,10 @@
 import functools
 import re
+import warnings
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
-from onetick.py.backports import Literal
 
 import onetick.py as otp
+from onetick.py.backports import Literal
 from onetick.py import types as ott
 from onetick.py import utils
 from onetick.py.core.column import _Column
@@ -1109,16 +1110,26 @@ def book_diff(self: 'Source', include_initial_book: bool = False, inplace=False)
 
 
 @inplace_operation
-def limit(self: 'Source', tick_limit: int, inplace=False) -> Optional['Source']:
+def limit(self: 'Source',
+          tick_limit: int,
+          tick_offset: Optional[int] = None,
+          inplace=False) -> Optional['Source']:
     """
-    Propagates ticks until the count limit is reached. Once the limit is reached,
+    Propagates ticks until the count limit is reached.
+
+    Once the limit is reached,
     hidden ticks will still continue to propagate until the next regular tick appears.
 
     Parameters
     ----------
     tick_limit: int
-        The number of regular ticks to propagate. Must be a non-negative integer or -1, which will mean no limit.
-    inplace : bool
+        The number of regular ticks to propagate.
+        Must be a non-negative integer or -1, which means no limit.
+    tick_offset: int
+        The number of regular ticks to skip before starting to propagate.
+        Must be a non-negative integer.
+        By default no ticks are skipped.
+    inplace: bool
         The flag controls whether operation should be applied inplace or not.
         If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
         object.
@@ -1134,14 +1145,15 @@ def limit(self: 'Source', tick_limit: int, inplace=False) -> Optional['Source']:
     Examples
     --------
 
-    Basic example
+    Simple example, get first 3 ticks:
 
     .. testcode::
        :skipif: not otp.compatibility.is_limit_ep_supported()
 
        data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
-       data = data.limit(tick_limit=3)
-       print(otp.run(data))
+       data = data.limit(3)
+       df = otp.run(data)
+       print(df)
 
     .. testoutput::
 
@@ -1150,14 +1162,15 @@ def limit(self: 'Source', tick_limit: int, inplace=False) -> Optional['Source']:
        1 2003-12-01 00:00:00.001  2
        2 2003-12-01 00:00:00.002  3
 
-    Disable limit
+    Disable limit by setting it to -1:
 
     .. testcode::
        :skipif: not otp.compatibility.is_limit_ep_supported()
 
        data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
-       data = data.limit(tick_limit=-1)
-       print(otp.run(data))
+       data = data.limit(-1)
+       df = otp.run(data)
+       print(df)
 
     .. testoutput::
 
@@ -1168,14 +1181,61 @@ def limit(self: 'Source', tick_limit: int, inplace=False) -> Optional['Source']:
        3 2003-12-01 00:00:00.003  4
        4 2003-12-01 00:00:00.004  5
        5 2003-12-01 00:00:00.005  6
+
+    Setting parameter ``tick_offset`` can be used to skip first ticks before propagating them.
+
+    For example, we can skip first 2 ticks and propagate all other:
+
+    .. testcode::
+       :skipif: not otp.compatibility.is_limit_tick_offset_supported()
+
+       data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
+       data = data.limit(-1, tick_offset=2)
+       df = otp.run(data)
+       print(df)
+
+    .. testoutput::
+
+                            Time  X
+       0 2003-12-01 00:00:00.002  3
+       1 2003-12-01 00:00:00.003  4
+       2 2003-12-01 00:00:00.004  5
+       3 2003-12-01 00:00:00.005  6
+
+    Or we can return ticks from the middle of the stream
+    by skipping first 2 ticks and then returning next 2 ticks like this:
+
+    .. testcode::
+       :skipif: not otp.compatibility.is_limit_tick_offset_supported()
+
+       data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
+       data = data.limit(2, tick_offset=2)
+       df = otp.run(data)
+       print(df)
+
+    .. testoutput::
+
+                            Time  X
+       0 2003-12-01 00:00:00.002  3
+       1 2003-12-01 00:00:00.003  4
     """
-    if not hasattr(otq, 'Limit'):
+    if not otp.compatibility.is_limit_ep_supported():
         raise RuntimeError('LIMIT EP isn\'t supported by the current OneTick version.')
 
     if tick_limit < 0 and tick_limit != -1:
         raise ValueError('Negative values, except -1, not allowed as `tick_limit` in `limit` method.')
 
-    self.sink(otq.Limit(tick_limit=tick_limit))
+    ep_kwargs = {}
+    if tick_offset is not None:
+        if not isinstance(tick_offset, int) or tick_offset < 0:
+            raise ValueError("Parameter 'tick_offset' must be non-negative.")
+
+        if not otp.compatibility.is_limit_tick_offset_supported():
+            warnings.warn("Parameter 'tick_offset' is set, but is not supported on this OneTick version")
+        else:
+            ep_kwargs = {'tick_offset': tick_offset}
+
+    self.sink(otq.Limit(tick_limit=tick_limit, **ep_kwargs))
     return self
 
 

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/core/_source/source_methods/writes.py

@@ -259,16 +259,27 @@ def write(
             raise ValueError('LOAD out_of_range_tick_action cannot be used with start_date+end_date, use date instead')
     elif out_of_range_tick_action.upper() == 'EXCEPTION':
         if start_date and end_date:
+            end = end_date + otp.Day(1)  # end_date is inclusive
+
             # WRITE_TO_ONETICK_DB use DAY_BOUNDARY_TZ and DAY_BOUNDARY_OFFSET
             # to check tick timestamp is out of range or not
             # so we mimic it here with THROW event processor
-            src = otp.Source(otq.DbShowConfig(str(db), 'DB_TIME_INTERVALS'))
+            src = otp.Source(otq.DbShowConfig(str(db), 'DB_TIME_INTERVALS'), schema={
+                'DAY_BOUNDARY_TZ': int, 'DAY_BOUNDARY_OFFSET': int, 'START_DATE': int, 'END_DATE': int,
+            })
+
+            # Filter not relevant locator time intervals
+            src, _ = src[
+                (src['START_DATE'].astype(otp.msectime) <= otp.dt(start_date).to_operation()) &
+                (src['END_DATE'].astype(otp.msectime) > otp.dt(end).to_operation())
+            ]
             src.table(inplace=True, DAY_BOUNDARY_TZ=str, DAY_BOUNDARY_OFFSET=int)
             # DAY_BOUNDARY_OFFSET offset are in seconds
             src['DAY_BOUNDARY_OFFSET'] = src['DAY_BOUNDARY_OFFSET'] * 1000
             src.rename(
                 {'DAY_BOUNDARY_TZ': '__DAY_BOUNDARY_TZ', 'DAY_BOUNDARY_OFFSET': '__DAY_BOUNDARY_OFFSET'}, inplace=True
             )
+
             self = self.join_with_query(src, symbol=f"{str(db)}::DUMMY", caching='per_symbol')
             timezone = self['__DAY_BOUNDARY_TZ']
             offset = self['__DAY_BOUNDARY_OFFSET']
@@ -288,7 +299,6 @@ def write(
                 inplace=True,
             )
 
-            end = end_date + otp.Day(1)  # end_date is inclusive
             end_formatted = end.strftime('%Y-%m-%d')
             end_op = otp.dt(end).to_operation(timezone=timezone) + offset
             self.throw(

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/core/_source/tmp_otq.py

@@ -108,6 +108,14 @@ class TmpOtq:
         res.merge(self)
         return res
 
+    def _get_symbol_dates(self) -> list[str]:
+        # check if any of the saved queries have symbol date set and return them (as strings in %Y%m%d format)
+        return [
+            utils.symbol_date_to_str(query_params['symbol_date'])
+            for _, query_params in self.queries.values()
+            if 'symbol_date' in query_params
+        ]
+
     def save_to_file(self, query=None, query_name="main_query", file_path=None, file_suffix="",
                      start=None, end=None, start_time_expression=None, end_time_expression=None, timezone=None,
                      running_query_flag=None,

{onetick_py-1.177.0 → onetick_py-1.180.0}/src/onetick/py/core/source.py

@@ -432,7 +432,7 @@ class Source:
                end_time_expression=None,
                symbol_date=None):
         """
-        Save data source to .otq file and return path to the saved file.
+        Save :class:`otp.Source <onetick.py.Source>` object to .otq file and return path to the saved file.
 
         Parameters
         ----------
@@ -474,10 +474,17 @@ class Source:
 
         Returns
         -------
-
         result: str
             Relative (if ``file_name`` is relative) or absolute path to the created query
             in the format ``file_name::query_name``
+
+        Examples
+        --------
+        Create the .otq file from a :class:`otp.Source <onetick.py.Source>` object:
+
+        >>> t = otp.Tick(A=1)
+        >>> t.to_otq()  # doctest: +SKIP
+        '/tmp/test_user/run_20251202_181018_11054/impetuous-bullfrog.to_otq.otq::query'
         """
         if raw is not None:
             warnings.warn('The "raw" flag is deprecated and makes no effect', FutureWarning)
@@ -635,7 +642,22 @@ class Source:
                                                  timezone=timezone,
                                                  symbol_date=symbol_date)
 
-        return query_to_run, require_dict, node_name
+        # PY-1423: we should set symbol_date in otp.run always
+        symbol_date_to_run = None
+        if symbol_date is not None:
+            symbol_date_to_run = utils.symbol_date_to_str(symbol_date)
+        else:
+            symbol_dates_in_tmp_otq = obj._tmp_otq._get_symbol_dates()
+            if symbol_dates_in_tmp_otq:
+                symbol_date_to_run = symbol_dates_in_tmp_otq[0]
+                if len(set(symbol_dates_in_tmp_otq)) > 1:
+                    warnings.warn(
+                        f'There are different symbol dates in resulting .otq file: {set(symbol_dates_in_tmp_otq)}.'
+                        'But no symbol date were specified in otp.run.\n'
+                        f'In this case the first symbol_date ({symbol_date_to_run}) will be used automatically.'
+                    )
+
+        return query_to_run, require_dict, node_name, symbol_date_to_run
 
     def __call__(self, *args, **kwargs):
         """