onetick-py 1.162.2__py3-none-any.whl → 1.164.0__py3-none-any.whl

onetick/py/core/column_operations/accessors/dt_accessor.py

@@ -57,10 +57,14 @@ class _DtAccessor(_Accessor):
 
             %S - Seconds (2 digits)
 
+            %q - Milliseconds (3 digits)
+
             %J - Nanoseconds (9 digits)
 
             %p - AM/PM (2 characters)
 
+            %% - % character
+
         Examples
         --------
         >>> t = otp.Ticks(A=[otp.dt(2019, 1, 1, 1, 1, 1), otp.dt(2019, 2, 2, 2, 2, 2)])

onetick/py/core/column_operations/base.py

@@ -880,6 +880,7 @@ class Operation:
                 self._op_str = op_str
                 self._dtype = dtype
             return op_str, dtype
+        return None
 
     def _convert_to(self, to_type):
         return _Operation(_methods.CONVERSIONS[self.dtype, to_type], [self])
@@ -889,16 +890,15 @@ class Operation:
         if dtype is bool:
             return self
         if are_ints_not_time(dtype):
-            self = _Operation(_methods.ne, (self, 0))
+            return _Operation(_methods.ne, (self, 0))
         elif are_time(dtype):
-            self = _Operation(_methods.ne, (self._convert_to(int), 0))
+            return _Operation(_methods.ne, (self._convert_to(int), 0))
         elif are_floats(dtype):
-            self = _Operation(_methods.ne, (self, 0.0))
+            return _Operation(_methods.ne, (self, 0.0))
         elif are_strings(dtype):
-            self = _Operation(_methods.ne, (self, ""))
+            return _Operation(_methods.ne, (self, ""))
         else:
             raise TypeError("Filter expression should return bool, int, float or string")
-        return self
 
     def _replace_parameters(self, operation_cb, return_replace_tuples=False):
         """

onetick/py/core/cut_builder.py

@@ -102,6 +102,7 @@ class _BaseCutBuilder(metaclass=ABCMeta):
         return s
 
 
+# pylint: disable-next=abstract-method
 class _CutBuilder(_BaseCutBuilder):
 
     def compute_bin_variables(self):

onetick/py/core/eval_query.py

@@ -237,6 +237,7 @@ def prepare_params(**kwargs):
     converted_params = {}
     for key, value in kwargs.items():
         dtype = otp.types.get_object_type(value)
+        # pylint: disable-next=unidiomatic-typecheck
         if type(value) is str and len(value) > otp.string.DEFAULT_LENGTH:
             dtype = otp.string[len(value)]
         param = OnetickParameter(key, dtype)

onetick/py/core/lambda_object.py

@@ -253,7 +253,7 @@ class _EmulateObject(_EmulateInputObject):
 
     def __setattr__(self, key, value):
         if key in self.__class__.__dict__:
-            return super().__setattr__(key, value)
+            super().__setattr__(key, value)
         if key not in self.__dict__ or key in _EmulateObject.NEW_VALUES:
             _EmulateObject.NEW_VALUES[key].append(value)
         if key not in self.__dict__:
@@ -381,8 +381,7 @@ class _EmulateObject(_EmulateInputObject):
 
 def _validate_lambda(lambda_f):
     if not (isinstance(lambda_f, types.LambdaType) and lambda_f.__name__ == '<lambda>'
-            or isinstance(lambda_f, types.FunctionType)
-            or isinstance(lambda_f, types.MethodType)):
+            or isinstance(lambda_f, (types.FunctionType, types.MethodType))):
         raise ValueError("It is expected to get a function, method or lambda,"
                          f" but got '{lambda_f}' of type '{type(lambda_f)}'")
 

onetick/py/core/per_tick_script.py

@@ -329,7 +329,7 @@ class Expression:
     @property
     def is_dynamic_tick(self) -> bool:
         try:
-            return type(self.value) is _DynamicTick
+            return isinstance(self.value, _DynamicTick)
         except ValueError:
             return False
 
@@ -585,7 +585,7 @@ class ExpressionParser:
             value = self.fun.emulator if self.fun.emulator is not None else expr.id
             return Expression(value)
 
-        if type(expr.ctx) is not ast.Load:
+        if not isinstance(expr.ctx, ast.Load):
             # local variable, left-hand side
             return Expression(LocalVariable(expr.id))
 
@@ -603,6 +603,7 @@ class ExpressionParser:
             # parameter of the per-tick script function
             return Expression(_Operation(op_str=expr.id, dtype=self.fun.args_annotations[expr.id]))
 
+        # pylint: disable-next=eval-used
         value = eval(expr.id, self.fun.closure_vars.globals, self.fun.closure_vars.nonlocals)  # type: ignore[arg-type]
         return Expression(value)
 
@@ -632,7 +633,7 @@ class ExpressionParser:
         val = self.expression(expr.value)
         item = self.expression(expr.slice)
 
-        if type(expr.ctx) is ast.Load:
+        if isinstance(expr.ctx, ast.Load):
             v = val.value[item.value]
             return Expression(v)
 
@@ -657,7 +658,7 @@ class ExpressionParser:
         val = self.expression(expr.value)
         attr = expr.attr
 
-        if type(expr.ctx) is ast.Load:
+        if isinstance(expr.ctx, ast.Load):
             v = getattr(val.value, attr)
             return Expression(v)
 
@@ -1007,7 +1008,7 @@ class CaseExpressionParser(ExpressionParser):
         def get_if_expr(first, second):
             if isinstance(expr.op, ast.Or):
                 return ast.IfExp(test=first, body=first, orelse=second)
-            if isinstance(expr.op, ast.And):
+            else:
                 return ast.IfExp(test=first, body=second, orelse=first)
 
         first = None

onetick/py/core/query_inspector.py

@@ -101,7 +101,7 @@ def get_query_info(otq_path, query_name="", inspected_graphs=None):
                     stack_info = res.group(1)
             return ep_name, stack_info
 
-    graph = defaultdict(lambda: Node())
+    graph = defaultdict(Node)
 
     if not os.path.exists(otq_path):
         raise FileNotFoundError(f'otq "{otq_path}" is not found')
@@ -176,7 +176,7 @@ def get_query_info(otq_path, query_name="", inspected_graphs=None):
                             graph[num].PARAMETERS[inner_param_name] = inner_param_value
                             continue
 
-                        if param == "SOURCE" or param == "SINK":
+                        if param in ("SOURCE", "SINK"):
                             sources = []
 
                             for v in value.split():
@@ -199,8 +199,6 @@ def get_query_info(otq_path, query_name="", inspected_graphs=None):
                         else:
                             setattr(graph[num], param, value)
 
-        # with
-
         if not found:
             raise QueryNotFoundError(f'Query "{query_name}" is not found in the {otq_path}')
 
@@ -233,6 +231,8 @@ def get_query_info(otq_path, query_name="", inspected_graphs=None):
                         address = node.PARAMETERS["OTQ_NODE"]
                     elif hasattr(node, "PH_PATH"):
                         address = node.PH_PATH
+                    else:
+                        raise ValueError("Can't get NESTED_OTQ address")
 
                     node.EP = "NESTED_OTQ " + address
 
@@ -263,7 +263,7 @@ def get_query_info(otq_path, query_name="", inspected_graphs=None):
 
         def _search_for_bound_sink(self, root, bound_symbol_info):
             if root in bound_symbol_info:
-                return bound_symbol_info[root]
+                return
 
             if not is_commented_out(self.nodes[root]):
                 # A commented node cannot have its own bound symbols, but can have a bound sink
@@ -419,7 +419,7 @@ def add_pins(otq_path, query_name, specification):
                 if res:
                     for inx, v in enumerate(specification):
                         node, pin_flag, pin_name = v
-                        if node.NUM == num:
+                        if node.NUM == num:  # noqa
                             if pin_flag is None:
                                 continue
                             node_name = "NODE_" + str(num)
@@ -432,7 +432,6 @@ def add_pins(otq_path, query_name, specification):
 
                             # don't use this specification item anymore
                             specification[inx] = (node, None, None)
-    # with
 
     with open(otq_path, "w") as fout:
         fout.write(out.getvalue())

onetick/py/core/source.py

@@ -362,9 +362,9 @@ class Source:
                 raise KeyError(f'Column name {name} is not in the schema. Please, check that this column '
                                'is in the schema or add it using the .schema property')
 
-            if name == 0 or name == 1:
+            if name in (0, 1):
                 raise ValueError(f"constant {name} are not supported for indexing for now, please use otp.Empty")
-            if type(name) in (int, float):
+            if isinstance(name, (int, float)):
                 raise ValueError("integer indexes are not supported")
             self.__dict__[name] = _Column(name, dtype, self)
         else:
@@ -775,6 +775,7 @@ class Source:
             )
 
             return mqt_ep
+        return None
 
     def _set_date_range_and_symbols(self, symbols=None, start=None, end=None):
         # will modify self
@@ -1208,7 +1209,7 @@ class Source:
         if not (
             issubclass(type(ep), otq.graph_components.EpBase)
             or issubclass(type(ep), otq.graph_components.EpBase.PinnedEp)
-            or type(ep) is tuple
+            or isinstance(ep, tuple)
         ):
             raise TypeError("sinking is allowed only for EpBase instances")
 
@@ -1217,7 +1218,7 @@ class Source:
         else:
             obj = self.copy()
 
-        if type(ep) is tuple:
+        if isinstance(ep, tuple):
             # for already existed EP fetched from _ProxyNode
             obj.__node.sink(out_pin, *ep)
         else:
@@ -1244,11 +1245,11 @@ class Source:
         if not (
             issubclass(type(ep), otq.graph_components.EpBase)
             or issubclass(type(ep), otq.graph_components.EpBase.PinnedEp)
-            or type(ep) is tuple
+            or isinstance(ep, tuple)
         ):
             raise TypeError("sourcing is allowed only for EpBase instances")
 
-        if type(ep) is tuple:
+        if isinstance(ep, tuple):
             # for already existed EP fetched from _ProxyNode
             return self.__node.source(in_pin, *ep)
         else:
@@ -1259,11 +1260,11 @@ class Source:
         if not (
             issubclass(type(ep), otq.graph_components.EpBase)
             or issubclass(type(ep), otq.graph_components.EpBase.PinnedEp)
-            or type(ep) is tuple
+            or isinstance(ep, tuple)
         ):
             raise TypeError("sourcing is allowed only for EpBase instances")
 
-        if type(ep) is tuple:
+        if isinstance(ep, tuple):
             # for already existed EP fetched from _ProxyNode
             return self.__node.source_by_key(to_key, *ep)
         else:
@@ -1590,6 +1591,7 @@ class Source:
         write,
         write_parquet,
         save_snapshot,
+        write_text,
     )
     from ._source.source_methods.renames import (  # type: ignore[misc]
         _add_prefix_and_suffix,
@@ -1632,6 +1634,7 @@ class Source:
     from ._source.source_methods.misc import (  # type: ignore[misc]
         pause,
         insert_tick,
+        insert_at_end,
         transpose,
         cache,
         pnl_realized,

onetick/py/db/_inspection.py

@@ -278,9 +278,8 @@ class DB:
             tz_gmt = gettz('GMT')
             for inx in range(len(result)):
                 start_date = result['START_DATE'][inx]
-                if start_date < 0:
-                    # On Windows datetime.fromtimestamp throws an OSError for negative values
-                    start_date = 0
+                # On Windows datetime.fromtimestamp throws an OSError for negative values
+                start_date = max(start_date, 0)
                 start = datetime.fromtimestamp(start_date / 1000, tz=tz_gmt)
                 start = start.replace(tzinfo=None)
                 try:
@@ -376,8 +375,7 @@ class DB:
             # future is not loaded yet
             if locator_start > today:
                 continue
-            if locator_end > today:
-                locator_end = today
+            locator_end = min(locator_end, today)
 
             if respect_acl:
                 try:
@@ -627,7 +625,7 @@ class DB:
                            context=self.context)
 
         result = get_schema(use_cache=True)
-        if not len(result):
+        if result.empty:
             # in case cache settings in database are bad (e.g. BEXRTS-1220)
             result = get_schema(use_cache=False)
 
@@ -719,8 +717,6 @@ class DB:
         >>> us_comp_db.symbols(date=otp.dt(2022, 3, 1), tick_type='TRD', pattern='^AAP.*')
         ['AAP', 'AAPL']
         """
-        import onetick.py as otp
-
         if date is None:
             date = self.last_date
         if timezone is None:

onetick/py/db/db.py

@@ -182,7 +182,7 @@ class _DB:
     def _format_params(params):
         res = {}
         for key, value in params.items():
-            if isinstance(value, dt.datetime):
+            if hasattr(value, 'strftime'):
                 res[key] = value.strftime("%Y%m%d%H%M%S")
             else:
                 res[key] = str(value)
@@ -280,6 +280,8 @@ class _DB:
         # because before there was no ability to get written ticks
         if kwargs.get('propagate'):
             return res
+        else:
+            return None
 
     @property
     def properties(self):
@@ -724,105 +726,6 @@ class DB(_DB):
         return self._format_params(feed)
 
 
-"""
-Keep here as example of custom databases that can be defined in client code
-
-
-P_CME = DB(
-    name="P_CME",
-    db_properties={
-        "symbology": "CTA",
-        "archive_compression_type": constants.compression_type.NATIVE_PLUS_GZIP,
-        "tick_search_max_boundary_offset_sec": 1800,
-        "tick_timestamp_type": "NANOS",
-    },
-    db_locations=[
-        {
-            "access_method": constants.access_method.SOCKET,
-            "location": "192.168.5.63:50025",
-            "start_time": datetime(year=2008, month=9, day=1),
-            "end_time": constants.DEFAULT_END_DATE,
-        }
-    ],
-)
-
-MS127 = DB(
-    name="MS127",
-    db_properties={
-        "symbology": "MSGR",
-        "ref_data_db": "REF_DATA_MS127",
-        "archive_compression_type": constants.compression_type.NATIVE_PLUS_GZIP,
-        "ignore_previous_day_corrections_on_reload": "yes",
-    },
-    db_locations=[
-        {
-            "access_method": constants.access_method.SOCKET,
-            "location": "192.168.5.63:50025",
-            "start_time": datetime(year=2010, month=12, day=31),
-            "end_time": constants.DEFAULT_END_DATE,
-        }
-    ],
-)
-
-MS44 = DB(
-    name="MS44",
-    db_properties={
-        "symbology": "MSGR",
-        "ref_data_db": "REF_DATA_MS44",
-        "archive_compression_type": constants.compression_type.NATIVE_PLUS_GZIP,
-        "ignore_previous_day_corrections_on_reload": "yes",
-    },
-    db_locations=[
-        {
-            "access_method": constants.access_method.SOCKET,
-            "location": "192.168.5.63:50025",
-            "start_time": datetime(year=2010, month=12, day=31),
-            "end_time": constants.DEFAULT_END_DATE,
-        }
-    ],
-)
-
-TAQ_NBBO = DB(
-    name="TAQ_NBBO",
-    db_properties={
-        "symbology": "BZX",
-        "price_not_key": True,
-        "memory_data_max_life_hours": 30,
-        "memory_db_dir": "/onetick-tickdata-com/STORAGE_GATEWAY/DEEP_HISTORY/US_TED/NBBO/shmem",
-        "mmap_db_compression_type": constants.compression_type.NATIVE_PLUS_GZIP,
-    },
-    db_locations=[
-        {
-            "access_method": constants.access_method.FILE,
-            "location": "/onetick-tickdata-com/STORAGE_GATEWAY/DEEP_HISTORY/US_TED/NBBO/",
-            "start_time": datetime(year=2001, month=1, day=1),
-            "end_time": constants.DEFAULT_END_DATE,
-        }
-    ],
-)
-
-
-US_COMP = DB(
-    name="US_COMP",
-    db_properties={
-        "symbology": "BZX",
-        "price_not_key": True,
-        "memory_data_max_life_hours": 30,
-        "memory_db_dir": "/onetick-tickdata-com/STORAGE_GATEWAY/DEEP_HISTORY/US_TED/TAQ/shmem",
-        "mmap_db_compression_type": constants.compression_type.NATIVE_PLUS_GZIP,
-    },
-    db_locations=[
-        {
-            "access_method": constants.access_method.FILE,
-            "location": "/onetick-tickdata-com/STORAGE_GATEWAY/DEEP_HISTORY/US_TED/TAQ/",
-            "start_time": datetime(year=2003, month=10, day=1),
-            "end_time": constants.DEFAULT_END_DATE,
-        }
-    ],
-)
-"""
-
-
 class RefDB(DB):
     """ Creates reference database object.
 
@@ -1414,6 +1317,7 @@ class RefDB(DB):
             },
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
+            check=False,
         )
         return p.stdout, p.stderr
 

onetick/py/docs/docstring_parser.py

@@ -8,7 +8,7 @@ from textwrap import dedent
 class Docstring:
 
     def __init__(self, doc: str):
-        if type(doc) is list:
+        if isinstance(doc, list):
             doc = '\n'.join(doc)
         self.doc = doc
         self.indentation = ' ' * (len(doc.lstrip('\n')) - len(doc.lstrip()))

onetick/py/functions.py

@@ -572,6 +572,7 @@ def _add_node_name_prefix_to_columns_in_operation(op, src):
                 raise ValueError('You set to use name for column prefix, but name is empty')
             name = f'{src.node_name()}.{column.name}'
             return Column(name, column.dtype, column.obj_ref, precision=getattr(column, "_precision", None))
+        return None
 
     return op._replace_parameters(fun)
 
@@ -902,8 +903,18 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
     0 2003-12-01 00:00:00.001  1  0
     1 2003-12-01 00:00:00.003  2  1
 
-    In case you willing to add prefix/suffix to all columns in one of the sources you should use
-    :func:`Source.add_prefix` or :func:`Source.add_suffix`
+    Note
+    ----
+    In case different ``sources`` have matching columns, the exception will be raised.
+
+    To fix this error,
+    functions :func:`Source.add_prefix` or :func:`Source.add_suffix` can be used to rename all columns in the source.
+
+    Note that resulting **TIMESTAMP** pseudo-column will be taken from the leading source,
+    and timestamps of ticks from non-leading sources will not be added to the output,
+    so if you need to save them, you need to copy the timestamp to some other column.
+
+    See examples below.
 
     Parameters
     ----------
@@ -1047,7 +1058,8 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
 
     Default joining logic, outer join with the first source is the leader by default:
 
-    >>> otp.run(otp.join_by_time([d1, d2]))
+    >>> data = otp.join_by_time([d1, d2])
+    >>> otp.run(data)
                          Time  A  B
     0 2003-12-01 00:00:00.001  1  0
     1 2003-12-01 00:00:00.002  2  1
@@ -1055,7 +1067,8 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
 
     Leading source can be changed by using parameter ``leading``:
 
-    >>> otp.run(otp.join_by_time([d1, d2], leading=1))
+    >>> data = otp.join_by_time([d1, d2], leading=1)
+    >>> otp.run(data)
                          Time  A  B
     0 2003-12-01 00:00:00.001  1  1
     1 2003-12-01 00:00:00.002  2  2
@@ -1064,7 +1077,8 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
     Note that OneTick's logic is different depending on the order of sources specified,
     so specifying ``leading`` parameter in the previous example is not the same as changing the order of sources here:
 
-    >>> otp.run(otp.join_by_time([d2, d1], leading=0))
+    >>> data = otp.join_by_time([d2, d1], leading=0)
+    >>> otp.run(data)
                          Time  B  A
     0 2003-12-01 00:00:00.001  1  0
     1 2003-12-01 00:00:00.002  2  1
@@ -1073,7 +1087,8 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
     Parameter ``source_fields_order`` can be used to change the order of fields in the output,
     but it also affects the joining logic the same way as changing the order of sources:
 
-    >>> otp.run(otp.join_by_time([d1, d2], leading=1, source_fields_order=[1, 0]))
+    >>> data = otp.join_by_time([d1, d2], leading=1, source_fields_order=[1, 0])
+    >>> otp.run(data)
                          Time  B  A
     0 2003-12-01 00:00:00.001  1  0
     1 2003-12-01 00:00:00.002  2  1
@@ -1082,26 +1097,48 @@ def join_by_time(sources, how="outer", on=None, policy=None, check_schema=True,
     Parameter ``how`` can be set to "inner".
     In this case only ticks that were successfully joined from all sources will be propagated:
 
-    >>> otp.run(otp.join_by_time([d1, d2], how='inner'))
+    >>> data = otp.join_by_time([d1, d2], how='inner')
+    >>> otp.run(data)
                          Time  A  B
     0 2003-12-01 00:00:00.002  2  1
     1 2003-12-01 00:00:00.003  3  2
 
     Set parameter ``match_if_identical_times`` to only join ticks with the same timestamps:
 
-    >>> otp.run(otp.join_by_time([d1, d2], how='inner', match_if_identical_times=True))
+    >>> data = otp.join_by_time([d1, d2], how='inner', match_if_identical_times=True)
+    >>> otp.run(data)
                          Time  A  B
     0 2003-12-01 00:00:00.001  1  1
     1 2003-12-01 00:00:00.002  2  2
 
-    Adding prefix to right source for all columns:
+    In case of conflicting names in different sources, exception will be raised:
+
+    >>> d3 = otp.Ticks({'A': [1, 2, 4], 'offset': [1, 2, 4]})
+    >>> data = otp.join_by_time([d1, d3])
+    Traceback (most recent call last):
+        ...
+    ValueError: There are matched columns between sources: A
+
+    Adding prefix to right source for all columns will fix this problem:
 
-    >>> otp.run(otp.join_by_time([d1, d2.add_prefix('right_')]))
-                         Time  A  right_B
+    >>> data = otp.join_by_time([d1, d3.add_prefix('right_')])
+    >>> otp.run(data)
+                         Time  A  right_A
     0 2003-12-01 00:00:00.001  1        0
     1 2003-12-01 00:00:00.002  2        1
     2 2003-12-01 00:00:00.003  3        2
 
+    Note that timestamps from the non-leading source are not added to the output.
+    You can add them manually in a different field:
+
+    >>> d3['D3_TIMESTAMP'] = d3['TIMESTAMP']
+    >>> data = otp.join_by_time([d1, d3.add_prefix('right_')])
+    >>> otp.run(data)
+                         Time  A  right_A      right_D3_TIMESTAMP
+    0 2003-12-01 00:00:00.001  1        0 1969-12-31 19:00:00.000
+    1 2003-12-01 00:00:00.002  2        1 2003-12-01 00:00:00.001
+    2 2003-12-01 00:00:00.003  3        2 2003-12-01 00:00:00.002
+
     Use parameter ``output_type_index`` to specify which input class to use to create output object.
     It may be useful in case some custom user class was used as input:
 

onetick/py/license.py

@@ -71,6 +71,8 @@ class Custom(_LicenseBase):
     Custom license path
     """
 
+    # it is not useless
+    # pylint: disable=useless-parent-delegation
     def __init__(self, file, directory=None):
         """
         Parameters

onetick/py/math.py

@@ -10,7 +10,7 @@ class _MaxOperator(_Operation):
         super().__init__(dtype=get_type_by_objects(objs))
 
         def _str_max(l_val, r_val):
-            if type(r_val) is list:
+            if isinstance(r_val, list):
                 if len(r_val) > 1:
                     r_val = _str_max(r_val[0], r_val[1:])
                 else:
@@ -55,7 +55,7 @@ class _MinOperator(_Operation):
         super().__init__(dtype=get_type_by_objects(objs))
 
         def _str_min(l_val, r_val):
-            if type(r_val) is list:
+            if isinstance(r_val, list):
                 if len(r_val) > 1:
                     r_val = _str_min(r_val[0], r_val[1:])
                 else:

onetick/py/otq.py

@@ -10,7 +10,6 @@ import os
 import tempfile
 import warnings
 import onetick.py as otp
-from onetick.py import configuration
 
 
 class OneTickLib:
@@ -33,7 +32,7 @@ class OneTickLib:
         pass
 
 
-if os.getenv("OTP_SKIP_OTQ_VALIDATION", False):
+if os.getenv("OTP_SKIP_OTQ_VALIDATION"):
     import onetick_stubs as otq  # noqa: F401
     import pyomd  # noqa: F401