onetick-py 1.177.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- locator_parser/__init__.py +0 -0
- locator_parser/acl.py +73 -0
- locator_parser/actions.py +262 -0
- locator_parser/common.py +368 -0
- locator_parser/io.py +43 -0
- locator_parser/locator.py +150 -0
- onetick/__init__.py +101 -0
- onetick/doc_utilities/__init__.py +3 -0
- onetick/doc_utilities/napoleon.py +40 -0
- onetick/doc_utilities/ot_doctest.py +140 -0
- onetick/doc_utilities/snippets.py +279 -0
- onetick/lib/__init__.py +4 -0
- onetick/lib/instance.py +141 -0
- onetick/py/__init__.py +293 -0
- onetick/py/_stack_info.py +89 -0
- onetick/py/_version.py +2 -0
- onetick/py/aggregations/__init__.py +11 -0
- onetick/py/aggregations/_base.py +648 -0
- onetick/py/aggregations/_docs.py +948 -0
- onetick/py/aggregations/compute.py +286 -0
- onetick/py/aggregations/functions.py +2216 -0
- onetick/py/aggregations/generic.py +104 -0
- onetick/py/aggregations/high_low.py +80 -0
- onetick/py/aggregations/num_distinct.py +83 -0
- onetick/py/aggregations/order_book.py +501 -0
- onetick/py/aggregations/other.py +1014 -0
- onetick/py/backports.py +26 -0
- onetick/py/cache.py +374 -0
- onetick/py/callback/__init__.py +5 -0
- onetick/py/callback/callback.py +276 -0
- onetick/py/callback/callbacks.py +131 -0
- onetick/py/compatibility.py +798 -0
- onetick/py/configuration.py +771 -0
- onetick/py/core/__init__.py +0 -0
- onetick/py/core/_csv_inspector.py +93 -0
- onetick/py/core/_internal/__init__.py +0 -0
- onetick/py/core/_internal/_manually_bound_value.py +6 -0
- onetick/py/core/_internal/_nodes_history.py +250 -0
- onetick/py/core/_internal/_op_utils/__init__.py +0 -0
- onetick/py/core/_internal/_op_utils/every_operand.py +9 -0
- onetick/py/core/_internal/_op_utils/is_const.py +10 -0
- onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script +121 -0
- onetick/py/core/_internal/_proxy_node.py +140 -0
- onetick/py/core/_internal/_state_objects.py +2312 -0
- onetick/py/core/_internal/_state_vars.py +93 -0
- onetick/py/core/_source/__init__.py +0 -0
- onetick/py/core/_source/_symbol_param.py +95 -0
- onetick/py/core/_source/schema.py +97 -0
- onetick/py/core/_source/source_methods/__init__.py +0 -0
- onetick/py/core/_source/source_methods/aggregations.py +809 -0
- onetick/py/core/_source/source_methods/applyers.py +296 -0
- onetick/py/core/_source/source_methods/columns.py +141 -0
- onetick/py/core/_source/source_methods/data_quality.py +301 -0
- onetick/py/core/_source/source_methods/debugs.py +272 -0
- onetick/py/core/_source/source_methods/drops.py +120 -0
- onetick/py/core/_source/source_methods/fields.py +619 -0
- onetick/py/core/_source/source_methods/filters.py +1002 -0
- onetick/py/core/_source/source_methods/joins.py +1413 -0
- onetick/py/core/_source/source_methods/merges.py +605 -0
- onetick/py/core/_source/source_methods/misc.py +1455 -0
- onetick/py/core/_source/source_methods/pandases.py +155 -0
- onetick/py/core/_source/source_methods/renames.py +356 -0
- onetick/py/core/_source/source_methods/sorts.py +183 -0
- onetick/py/core/_source/source_methods/switches.py +142 -0
- onetick/py/core/_source/source_methods/symbols.py +117 -0
- onetick/py/core/_source/source_methods/times.py +627 -0
- onetick/py/core/_source/source_methods/writes.py +986 -0
- onetick/py/core/_source/symbol.py +205 -0
- onetick/py/core/_source/tmp_otq.py +222 -0
- onetick/py/core/column.py +209 -0
- onetick/py/core/column_operations/__init__.py +0 -0
- onetick/py/core/column_operations/_methods/__init__.py +4 -0
- onetick/py/core/column_operations/_methods/_internal.py +28 -0
- onetick/py/core/column_operations/_methods/conversions.py +216 -0
- onetick/py/core/column_operations/_methods/methods.py +292 -0
- onetick/py/core/column_operations/_methods/op_types.py +160 -0
- onetick/py/core/column_operations/accessors/__init__.py +0 -0
- onetick/py/core/column_operations/accessors/_accessor.py +28 -0
- onetick/py/core/column_operations/accessors/decimal_accessor.py +104 -0
- onetick/py/core/column_operations/accessors/dt_accessor.py +537 -0
- onetick/py/core/column_operations/accessors/float_accessor.py +184 -0
- onetick/py/core/column_operations/accessors/str_accessor.py +1367 -0
- onetick/py/core/column_operations/base.py +1121 -0
- onetick/py/core/cut_builder.py +150 -0
- onetick/py/core/db_constants.py +20 -0
- onetick/py/core/eval_query.py +245 -0
- onetick/py/core/lambda_object.py +441 -0
- onetick/py/core/multi_output_source.py +232 -0
- onetick/py/core/per_tick_script.py +2256 -0
- onetick/py/core/query_inspector.py +464 -0
- onetick/py/core/source.py +1744 -0
- onetick/py/db/__init__.py +2 -0
- onetick/py/db/_inspection.py +1128 -0
- onetick/py/db/db.py +1327 -0
- onetick/py/db/utils.py +64 -0
- onetick/py/docs/__init__.py +0 -0
- onetick/py/docs/docstring_parser.py +112 -0
- onetick/py/docs/utils.py +81 -0
- onetick/py/functions.py +2398 -0
- onetick/py/license.py +190 -0
- onetick/py/log.py +88 -0
- onetick/py/math.py +935 -0
- onetick/py/misc.py +470 -0
- onetick/py/oqd/__init__.py +22 -0
- onetick/py/oqd/eps.py +1195 -0
- onetick/py/oqd/sources.py +325 -0
- onetick/py/otq.py +216 -0
- onetick/py/pyomd_mock.py +47 -0
- onetick/py/run.py +916 -0
- onetick/py/servers.py +173 -0
- onetick/py/session.py +1347 -0
- onetick/py/sources/__init__.py +19 -0
- onetick/py/sources/cache.py +167 -0
- onetick/py/sources/common.py +128 -0
- onetick/py/sources/csv.py +642 -0
- onetick/py/sources/custom.py +85 -0
- onetick/py/sources/data_file.py +305 -0
- onetick/py/sources/data_source.py +1045 -0
- onetick/py/sources/empty.py +94 -0
- onetick/py/sources/odbc.py +337 -0
- onetick/py/sources/order_book.py +271 -0
- onetick/py/sources/parquet.py +168 -0
- onetick/py/sources/pit.py +191 -0
- onetick/py/sources/query.py +495 -0
- onetick/py/sources/snapshots.py +419 -0
- onetick/py/sources/split_query_output_by_symbol.py +198 -0
- onetick/py/sources/symbology_mapping.py +123 -0
- onetick/py/sources/symbols.py +374 -0
- onetick/py/sources/ticks.py +825 -0
- onetick/py/sql.py +70 -0
- onetick/py/state.py +251 -0
- onetick/py/types.py +2131 -0
- onetick/py/utils/__init__.py +70 -0
- onetick/py/utils/acl.py +93 -0
- onetick/py/utils/config.py +186 -0
- onetick/py/utils/default.py +49 -0
- onetick/py/utils/file.py +38 -0
- onetick/py/utils/helpers.py +76 -0
- onetick/py/utils/locator.py +94 -0
- onetick/py/utils/perf.py +498 -0
- onetick/py/utils/query.py +49 -0
- onetick/py/utils/render.py +1374 -0
- onetick/py/utils/script.py +244 -0
- onetick/py/utils/temp.py +471 -0
- onetick/py/utils/types.py +120 -0
- onetick/py/utils/tz.py +84 -0
- onetick_py-1.177.0.dist-info/METADATA +137 -0
- onetick_py-1.177.0.dist-info/RECORD +152 -0
- onetick_py-1.177.0.dist-info/WHEEL +5 -0
- onetick_py-1.177.0.dist-info/entry_points.txt +2 -0
- onetick_py-1.177.0.dist-info/licenses/LICENSE +21 -0
- onetick_py-1.177.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
import onetick.py.types as ott
|
|
4
|
+
from onetick.py.core._source._symbol_param import _SymbolParamColumn
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
globals().setdefault("__warningregistry__", {}) # otherwise doctests fails
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SymbolType:
|
|
11
|
+
def __init__(self):
|
|
12
|
+
"""
|
|
13
|
+
You can get symbol name and symbol parameters with this class.
|
|
14
|
+
|
|
15
|
+
Examples
|
|
16
|
+
--------
|
|
17
|
+
>>> symbols = otp.Symbols('SOME_DB')
|
|
18
|
+
>>> symbols['PARAM'] = 'PAM'
|
|
19
|
+
>>> ticks = otp.DataSource('SOME_DB', tick_type='TT')
|
|
20
|
+
>>> ticks['SYMBOL_PARAM'] = ticks.Symbol['PARAM', str]
|
|
21
|
+
>>> ticks['SYMBOL_NAME'] = ticks.Symbol.name
|
|
22
|
+
>>> ticks = otp.merge([ticks], symbols=symbols)
|
|
23
|
+
>>> otp.run(ticks)
|
|
24
|
+
Time X SYMBOL_PARAM SYMBOL_NAME
|
|
25
|
+
0 2003-12-01 00:00:00.000 1 PAM S1
|
|
26
|
+
1 2003-12-01 00:00:00.000 -3 PAM S2
|
|
27
|
+
2 2003-12-01 00:00:00.001 2 PAM S1
|
|
28
|
+
3 2003-12-01 00:00:00.001 -2 PAM S2
|
|
29
|
+
4 2003-12-01 00:00:00.002 3 PAM S1
|
|
30
|
+
5 2003-12-01 00:00:00.002 -1 PAM S2
|
|
31
|
+
|
|
32
|
+
See also
|
|
33
|
+
--------
|
|
34
|
+
| :ref:`Databases, symbols, and tick types <symbols_concept>`
|
|
35
|
+
| :ref:`api/misc/symbol_param:Symbol Parameters Objects`
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
self.__name = _SymbolParamColumn("_SYMBOL_NAME", str)
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def name(self):
|
|
42
|
+
"""
|
|
43
|
+
Get symbol name.
|
|
44
|
+
|
|
45
|
+
Returns
|
|
46
|
+
-------
|
|
47
|
+
_SymbolParamColumn
|
|
48
|
+
|
|
49
|
+
Examples
|
|
50
|
+
--------
|
|
51
|
+
>>> symbols = otp.Symbols('SOME_DB')
|
|
52
|
+
>>> ticks = otp.DataSource('SOME_DB', tick_type='TT')
|
|
53
|
+
>>> ticks['SYMBOL_NAME'] = ticks.Symbol.name
|
|
54
|
+
>>> ticks = otp.merge([ticks], symbols=symbols)
|
|
55
|
+
>>> otp.run(ticks)
|
|
56
|
+
Time X SYMBOL_NAME
|
|
57
|
+
0 2003-12-01 00:00:00.000 1 S1
|
|
58
|
+
1 2003-12-01 00:00:00.000 -3 S2
|
|
59
|
+
2 2003-12-01 00:00:00.001 2 S1
|
|
60
|
+
3 2003-12-01 00:00:00.001 -2 S2
|
|
61
|
+
4 2003-12-01 00:00:00.002 3 S1
|
|
62
|
+
5 2003-12-01 00:00:00.002 -1 S2
|
|
63
|
+
"""
|
|
64
|
+
return self.__name
|
|
65
|
+
|
|
66
|
+
def get(self, name, dtype, default=None):
|
|
67
|
+
"""
|
|
68
|
+
Get symbol parameter by name.
|
|
69
|
+
|
|
70
|
+
Parameters
|
|
71
|
+
----------
|
|
72
|
+
name: str
|
|
73
|
+
The name of the symbol parameter to retrieve.
|
|
74
|
+
dtype: type
|
|
75
|
+
The expected data type of the symbol parameter value.
|
|
76
|
+
default: Any, optional
|
|
77
|
+
The default value to return if the symbol parameter is not found.
|
|
78
|
+
Default is ``None``, which means that default value of dtype will be used.
|
|
79
|
+
|
|
80
|
+
Returns
|
|
81
|
+
-------
|
|
82
|
+
Operation
|
|
83
|
+
|
|
84
|
+
Examples
|
|
85
|
+
--------
|
|
86
|
+
>>> symbols = otp.Symbols('SOME_DB')
|
|
87
|
+
>>> symbols['PARAM'] = 'PAM'
|
|
88
|
+
>>> ticks = otp.DataSource('SOME_DB', tick_type='TT')
|
|
89
|
+
>>> ticks['SYMBOL_PARAM'] = ticks.Symbol.get(name='PARAM', dtype=str, default='default')
|
|
90
|
+
>>> ticks = otp.merge([ticks], symbols=symbols)
|
|
91
|
+
>>> otp.run(ticks)
|
|
92
|
+
Time X SYMBOL_PARAM
|
|
93
|
+
0 2003-12-01 00:00:00.000 1 PAM
|
|
94
|
+
1 2003-12-01 00:00:00.000 -3 PAM
|
|
95
|
+
2 2003-12-01 00:00:00.001 2 PAM
|
|
96
|
+
3 2003-12-01 00:00:00.001 -2 PAM
|
|
97
|
+
4 2003-12-01 00:00:00.002 3 PAM
|
|
98
|
+
5 2003-12-01 00:00:00.002 -1 PAM
|
|
99
|
+
|
|
100
|
+
>>> symbol1 = otq.Symbol('SOME_DB::S1', params={'PARAM': 1})
|
|
101
|
+
>>> symbol2 = otq.Symbol('SOME_DB::S2')
|
|
102
|
+
>>> data = otp.DataSource(tick_type='TT')
|
|
103
|
+
>>> data['P'] = data.Symbol.get(name='PARAM', dtype=int, default=10)
|
|
104
|
+
>>> data = otp.merge([data], symbols=[symbol1, symbol2], identify_input_ts=True)
|
|
105
|
+
>>> otp.run(data)
|
|
106
|
+
Time P X SYMBOL_NAME TICK_TYPE
|
|
107
|
+
0 2003-12-01 00:00:00.000 1 1 SOME_DB::S1 TT
|
|
108
|
+
1 2003-12-01 00:00:00.000 10 -3 SOME_DB::S2 TT
|
|
109
|
+
2 2003-12-01 00:00:00.001 1 2 SOME_DB::S1 TT
|
|
110
|
+
3 2003-12-01 00:00:00.001 10 -2 SOME_DB::S2 TT
|
|
111
|
+
4 2003-12-01 00:00:00.002 1 3 SOME_DB::S1 TT
|
|
112
|
+
5 2003-12-01 00:00:00.002 10 -1 SOME_DB::S2 TT
|
|
113
|
+
|
|
114
|
+
"""
|
|
115
|
+
if default is None:
|
|
116
|
+
default = ott.default_by_type(dtype)
|
|
117
|
+
return self._get(item=name, dtype=dtype, default=default)
|
|
118
|
+
|
|
119
|
+
@staticmethod
|
|
120
|
+
def _get(item, dtype=str, default=None):
|
|
121
|
+
return _SymbolParamColumn(item, dtype, default=default)
|
|
122
|
+
|
|
123
|
+
def __getattr__(self, item):
|
|
124
|
+
"""
|
|
125
|
+
Get symbol parameter by name. Notice, that symbol parameter type will be string.
|
|
126
|
+
|
|
127
|
+
.. deprecated:: 1.74.0
|
|
128
|
+
Please, use :py:meth:`~onetick.py.core._source.symbol.__getitem__` method.
|
|
129
|
+
|
|
130
|
+
Returns
|
|
131
|
+
-------
|
|
132
|
+
_SymbolParamColumn
|
|
133
|
+
"""
|
|
134
|
+
if item == '__objclass__':
|
|
135
|
+
# fix for PY-1399 (some inspect functions try to access this special attribute)
|
|
136
|
+
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{item}'")
|
|
137
|
+
if not item.startswith('_') and item != 'pytest_mock_example_attribute_that_shouldnt_exist':
|
|
138
|
+
warnings.warn("`__getattr__` method is deprecated. Please, use `__getitem__` method instead.",
|
|
139
|
+
FutureWarning, stacklevel=2)
|
|
140
|
+
return self._get(item)
|
|
141
|
+
|
|
142
|
+
def __getitem__(self, item):
|
|
143
|
+
"""
|
|
144
|
+
Get symbol parameter by name.
|
|
145
|
+
|
|
146
|
+
Parameters
|
|
147
|
+
----------
|
|
148
|
+
item: tuple
|
|
149
|
+
The first parameter is string - symbol parameter name. The second one is symbol parameter type.
|
|
150
|
+
|
|
151
|
+
Returns
|
|
152
|
+
-------
|
|
153
|
+
_SymbolParamColumn
|
|
154
|
+
|
|
155
|
+
Examples
|
|
156
|
+
--------
|
|
157
|
+
The second parameter is symbol parameter's type.
|
|
158
|
+
|
|
159
|
+
>>> symbols = otp.Symbols('SOME_DB')
|
|
160
|
+
>>> symbols['PARAM'] = 5
|
|
161
|
+
>>> ticks = otp.DataSource('SOME_DB', tick_type='TT')
|
|
162
|
+
>>> ticks['SYMBOL_PARAM'] = ticks.Symbol['PARAM', int] + 1
|
|
163
|
+
>>> ticks['SYMBOL_PARAM'].dtype
|
|
164
|
+
<class 'int'>
|
|
165
|
+
>>> ticks = otp.merge([ticks], symbols=symbols)
|
|
166
|
+
>>> otp.run(ticks)
|
|
167
|
+
Time X SYMBOL_PARAM
|
|
168
|
+
0 2003-12-01 00:00:00.000 1 6
|
|
169
|
+
1 2003-12-01 00:00:00.000 -3 6
|
|
170
|
+
2 2003-12-01 00:00:00.001 2 6
|
|
171
|
+
3 2003-12-01 00:00:00.001 -2 6
|
|
172
|
+
4 2003-12-01 00:00:00.002 3 6
|
|
173
|
+
5 2003-12-01 00:00:00.002 -1 6
|
|
174
|
+
|
|
175
|
+
It also works with :py:class:`~onetick.py.msectime` and :py:class:`~onetick.py.nsectime` types:
|
|
176
|
+
|
|
177
|
+
>>> symbols = otp.Symbols('SOME_DB')
|
|
178
|
+
>>> symbols['NSECTIME_PARAM'] = symbols['Time'] + otp.Nano(100)
|
|
179
|
+
>>> symbols['MSECTIME_PARAM'] = symbols['Time'] + otp.Milli(1)
|
|
180
|
+
>>> ticks = otp.DataSource('SOME_DB', tick_type='TT')
|
|
181
|
+
>>> ticks['NSECTIME_PARAM'] = ticks.Symbol['NSECTIME_PARAM', otp.nsectime] + otp.Nano(1)
|
|
182
|
+
>>> ticks['MSECTIME_PARAM'] = ticks.Symbol['MSECTIME_PARAM', otp.msectime] + otp.Milli(1)
|
|
183
|
+
>>> ticks['NSECTIME_PARAM'].dtype
|
|
184
|
+
<class 'onetick.py.types.nsectime'>
|
|
185
|
+
>>> ticks['MSECTIME_PARAM'].dtype
|
|
186
|
+
<class 'onetick.py.types.msectime'>
|
|
187
|
+
>>> ticks = otp.merge([ticks], symbols=symbols)
|
|
188
|
+
>>> otp.run(ticks)
|
|
189
|
+
Time X NSECTIME_PARAM MSECTIME_PARAM
|
|
190
|
+
0 2003-12-01 00:00:00.000 1 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
191
|
+
1 2003-12-01 00:00:00.000 -3 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
192
|
+
2 2003-12-01 00:00:00.001 2 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
193
|
+
3 2003-12-01 00:00:00.001 -2 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
194
|
+
4 2003-12-01 00:00:00.002 3 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
195
|
+
5 2003-12-01 00:00:00.002 -1 2003-12-01 00:00:00.000000101 2003-12-01 00:00:00.002
|
|
196
|
+
"""
|
|
197
|
+
if not isinstance(item, tuple):
|
|
198
|
+
raise ValueError(f"tuple should be passed, but {type(item)} was passed")
|
|
199
|
+
if len(item) != 2:
|
|
200
|
+
raise ValueError(f"tuple's length should be 2 but it is {len(item)}")
|
|
201
|
+
item, dtype = item
|
|
202
|
+
return self._get(item, dtype)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
Symbol = SymbolType() # noqa mypy fix
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from onetick.py.otq import otq, _tmp_otq_path, otli, pyomd
|
|
4
|
+
from pandas import Timestamp as pd_Timestamp
|
|
5
|
+
from onetick.py.types import datetime as otp_datetime, datetime2expr
|
|
6
|
+
from datetime import datetime, date
|
|
7
|
+
from onetick.py import utils
|
|
8
|
+
from onetick.py.configuration import config
|
|
9
|
+
from onetick.py.log import get_debug_logger
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def is_datetime_type(dt):
|
|
13
|
+
# TODO: move to types, add docs
|
|
14
|
+
return isinstance(dt, (datetime, date, pd_Timestamp, otp_datetime, pyomd.timeval_t))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def is_millisecond_precision(dt):
|
|
18
|
+
"""
|
|
19
|
+
dt can be a datetime object, a pd.Datetime object or an otp.dt object.
|
|
20
|
+
We check if it has only millisecond precision or not.
|
|
21
|
+
"""
|
|
22
|
+
if (dt.microsecond % 1000) != 0:
|
|
23
|
+
return False
|
|
24
|
+
if isinstance(dt, (pd_Timestamp, otp_datetime)):
|
|
25
|
+
if dt.nanosecond != 0:
|
|
26
|
+
return False
|
|
27
|
+
return True
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TmpOtq:
|
|
31
|
+
"""
|
|
32
|
+
Class that represents a storage of temporary queries
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
class __iter_str:
|
|
36
|
+
"""
|
|
37
|
+
Class that produces unique strings
|
|
38
|
+
"""
|
|
39
|
+
DEFAULT_LENGTH = 6
|
|
40
|
+
FIRST_CODE = 97 # 'a'
|
|
41
|
+
LAST_CODE = 122 # 'z'
|
|
42
|
+
|
|
43
|
+
def __init__(self, length=None):
|
|
44
|
+
if length is None:
|
|
45
|
+
length = self.DEFAULT_LENGTH
|
|
46
|
+
self.__length = length
|
|
47
|
+
self.__n = 0
|
|
48
|
+
|
|
49
|
+
def __num_to_str(self):
|
|
50
|
+
s = ""
|
|
51
|
+
r = self.LAST_CODE - self.FIRST_CODE + 1
|
|
52
|
+
for i in range(0, self.__length):
|
|
53
|
+
s = chr(self.FIRST_CODE + (self.__n // r ** i) % r) + s
|
|
54
|
+
return s
|
|
55
|
+
|
|
56
|
+
def get_str(self):
|
|
57
|
+
s = self.__num_to_str()
|
|
58
|
+
self.__n += 1
|
|
59
|
+
return s
|
|
60
|
+
|
|
61
|
+
name_generator = __iter_str()
|
|
62
|
+
|
|
63
|
+
def __init__(self):
|
|
64
|
+
self.queries = {}
|
|
65
|
+
|
|
66
|
+
def add_query(self, query, suffix="", name=None, params=None):
|
|
67
|
+
"""
|
|
68
|
+
Adds query with a unique generated name to the storage.
|
|
69
|
+
|
|
70
|
+
Parameters
|
|
71
|
+
----------
|
|
72
|
+
query: otq.GraphQuery
|
|
73
|
+
GraphQuery object that is being stored
|
|
74
|
+
suffix: str
|
|
75
|
+
Suffix that is added to the autogenerated string to form name of the query.
|
|
76
|
+
name: str, optional
|
|
77
|
+
If specified, this ``name`` will be used to save query
|
|
78
|
+
and ``suffix`` parameter will be ignored.
|
|
79
|
+
params: dict, optional
|
|
80
|
+
Can specify additional parameters with which this query will be saved to file.
|
|
81
|
+
Currently, only "running_query_flag" and "symbol_date" are supported
|
|
82
|
+
|
|
83
|
+
Returns
|
|
84
|
+
-------
|
|
85
|
+
result: str , name of the query in the file (without THIS:: prefix).
|
|
86
|
+
"""
|
|
87
|
+
name = name or self.name_generator.get_str() + suffix
|
|
88
|
+
if name in self.queries:
|
|
89
|
+
raise ValueError(f"There is already a query with name '{name}' in {self.__class__.__name__} storage")
|
|
90
|
+
if params is None:
|
|
91
|
+
params = {}
|
|
92
|
+
self.queries[name] = (query, params)
|
|
93
|
+
return name
|
|
94
|
+
|
|
95
|
+
def merge(self, tmp_otq):
|
|
96
|
+
"""
|
|
97
|
+
Adds queries from the tmp_otq storage to the current storage.
|
|
98
|
+
As query names are guaranteed to be unique session-wide, no check for collision is necessary.
|
|
99
|
+
Queries with same names will always be the same.
|
|
100
|
+
"""
|
|
101
|
+
self.queries.update(tmp_otq.queries)
|
|
102
|
+
|
|
103
|
+
def copy(self):
|
|
104
|
+
"""
|
|
105
|
+
Creates a copy of the storage
|
|
106
|
+
"""
|
|
107
|
+
res = TmpOtq()
|
|
108
|
+
res.merge(self)
|
|
109
|
+
return res
|
|
110
|
+
|
|
111
|
+
def save_to_file(self, query=None, query_name="main_query", file_path=None, file_suffix="",
|
|
112
|
+
start=None, end=None, start_time_expression=None, end_time_expression=None, timezone=None,
|
|
113
|
+
running_query_flag=None,
|
|
114
|
+
symbol_date=None):
|
|
115
|
+
"""
|
|
116
|
+
Saves all queries from the query dict and one more query (if passed);
|
|
117
|
+
returns absolute path to passed query in file (if passed) or path of resulted file
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
query: otq.GraphQuery or None
|
|
122
|
+
Additional query that is saved with all the queries from the storage. Usually, this will be the main query
|
|
123
|
+
of the Source object that uses this storage
|
|
124
|
+
query_name: str
|
|
125
|
+
Name with which additional query will be saved. As additional query is not stored in any TmpOtq object,
|
|
126
|
+
it can be saved with a name that is specified completely and not generated to be unique
|
|
127
|
+
file_path: str or None
|
|
128
|
+
Path to the file where all queries will be saved. If None, a TmpFile will be created
|
|
129
|
+
file_suffix: str
|
|
130
|
+
Only used if file_path is None. A suffix that is added to the name of a generated file.
|
|
131
|
+
start: :py:class:`otp.datetime <onetick.py.datetime>`
|
|
132
|
+
start time for the resulting .otq file
|
|
133
|
+
end: :py:class:`otp.datetime <onetick.py.datetime>`
|
|
134
|
+
end time for the resulting .otq file
|
|
135
|
+
start_time_expression: str or None
|
|
136
|
+
start time expression for the resulting .otq file
|
|
137
|
+
end_time_expression: str or None
|
|
138
|
+
end time expression for the resulting .otq file
|
|
139
|
+
timezone: str
|
|
140
|
+
timezone for the resulting .otq file
|
|
141
|
+
symbol_date: :py:class:`otp.datetime <onetick.py.datetime>` or :py:class:`datetime.datetime` or int
|
|
142
|
+
Symbol date for the query or integer in the YYYYMMDD format.
|
|
143
|
+
Will be applied only to the query specified in the ``query`` parameter.
|
|
144
|
+
|
|
145
|
+
Returns
|
|
146
|
+
-------
|
|
147
|
+
result: str , contains path to the resulting file with the name of the additional query
|
|
148
|
+
(if "query" parameter is not None) or path to the resulting file otherwise
|
|
149
|
+
|
|
150
|
+
"""
|
|
151
|
+
if file_path is None:
|
|
152
|
+
base_dir = None
|
|
153
|
+
if os.getenv('OTP_WEBAPI_TEST_MODE'):
|
|
154
|
+
base_dir = _tmp_otq_path()
|
|
155
|
+
kwargs = {}
|
|
156
|
+
if config.otq_debug_mode:
|
|
157
|
+
kwargs['clean_up'] = False
|
|
158
|
+
tmp_file = utils.TmpFile(suffix=file_suffix,
|
|
159
|
+
base_dir=base_dir,
|
|
160
|
+
**kwargs)
|
|
161
|
+
file_path = tmp_file.path
|
|
162
|
+
|
|
163
|
+
queries_dict = dict(self.queries)
|
|
164
|
+
|
|
165
|
+
if query is not None:
|
|
166
|
+
if query_name in self.queries.keys():
|
|
167
|
+
query_name = self.name_generator.get_str() + "_" + query_name
|
|
168
|
+
query_params = {}
|
|
169
|
+
if running_query_flag:
|
|
170
|
+
query_params['running_query_flag'] = running_query_flag
|
|
171
|
+
if symbol_date is not None:
|
|
172
|
+
query_params['symbol_date'] = symbol_date
|
|
173
|
+
queries_dict[query_name] = (query, query_params)
|
|
174
|
+
|
|
175
|
+
# defining file-wise start/end times and time expressions
|
|
176
|
+
|
|
177
|
+
# timezone definitions
|
|
178
|
+
# in onetick and in dateutil can differ (e.g. "GMT-10" is +10:00 offset in onetick and -10:00 offset
|
|
179
|
+
# in dateutil), therefore we always use time expressions to force onetick to make time conversions by itself
|
|
180
|
+
# TODO: create a BDS ticket for the onetick team to fix it somehow
|
|
181
|
+
if is_datetime_type(start):
|
|
182
|
+
if not start_time_expression:
|
|
183
|
+
start_time_expression = datetime2expr(start)
|
|
184
|
+
start = None
|
|
185
|
+
if is_datetime_type(end):
|
|
186
|
+
if not end_time_expression:
|
|
187
|
+
end_time_expression = datetime2expr(end)
|
|
188
|
+
end = None
|
|
189
|
+
|
|
190
|
+
# constructing a list of otq.Query objects, which will hold graphs, names, start/end times etc.
|
|
191
|
+
query_list = []
|
|
192
|
+
for stored_query_name in queries_dict.keys(): # noqa
|
|
193
|
+
stored_query = otq.Query(queries_dict[stored_query_name][0])
|
|
194
|
+
stored_query_params = queries_dict[stored_query_name][1]
|
|
195
|
+
if timezone:
|
|
196
|
+
stored_query.set_timezone(timezone)
|
|
197
|
+
stored_query.set_symbols(queries_dict[stored_query_name][0].symbols())
|
|
198
|
+
stored_query.set_query_name(stored_query_name)
|
|
199
|
+
stored_query.set_start_time(start)
|
|
200
|
+
stored_query.set_end_time(end)
|
|
201
|
+
if 'running_query_flag' in stored_query_params.keys():
|
|
202
|
+
stored_query.set_running_query_flag(stored_query_params['running_query_flag'])
|
|
203
|
+
if start_time_expression:
|
|
204
|
+
stored_query.set_start_time_expression(start_time_expression)
|
|
205
|
+
if end_time_expression:
|
|
206
|
+
stored_query.set_end_time_expression(end_time_expression)
|
|
207
|
+
if 'symbol_date' in stored_query_params.keys():
|
|
208
|
+
stored_symbol_date = stored_query_params['symbol_date']
|
|
209
|
+
if stored_symbol_date is not None:
|
|
210
|
+
stored_symbol_date = int(utils.symbol_date_to_str(stored_symbol_date))
|
|
211
|
+
stored_query.set_symbol_date(stored_symbol_date)
|
|
212
|
+
query_list.append(stored_query)
|
|
213
|
+
|
|
214
|
+
_ = otli.OneTickLib()
|
|
215
|
+
otq_file = otq.OtqFile(query_list)
|
|
216
|
+
otq_file.save_to_file(file_path)
|
|
217
|
+
get_debug_logger().debug(f'otq file saved to: {file_path}')
|
|
218
|
+
|
|
219
|
+
if query is not None:
|
|
220
|
+
return file_path + "::" + query_name
|
|
221
|
+
else:
|
|
222
|
+
return file_path
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import string
|
|
3
|
+
|
|
4
|
+
from .column_operations.base import _Operation
|
|
5
|
+
from .. import types as ott
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_allowed_field_name_chars = set(string.ascii_letters) | set(string.digits) | {'_', '.'}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def validate_onetick_field_name(field_name):
|
|
12
|
+
if len(field_name) < 1 or len(field_name) > 127:
|
|
13
|
+
return False
|
|
14
|
+
chars = set(char for char in field_name)
|
|
15
|
+
|
|
16
|
+
return chars.issubset(_allowed_field_name_chars)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def field_name_contains_lowercase(field_name):
|
|
20
|
+
for char in field_name:
|
|
21
|
+
if char.islower():
|
|
22
|
+
return True
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Column(_Operation):
|
|
27
|
+
"""
|
|
28
|
+
:py:class:`~onetick.py.Source` column container.
|
|
29
|
+
|
|
30
|
+
This is the object you get when using :py:meth:`~onetick.py.Source.__getitem__`.
|
|
31
|
+
You can use this object everywhere where :py:class:`~onetick.py.Operation` object can be used.
|
|
32
|
+
|
|
33
|
+
Examples
|
|
34
|
+
--------
|
|
35
|
+
>>> t = otp.Tick(A=1)
|
|
36
|
+
>>> t['A']
|
|
37
|
+
Column(A, <class 'int'>)
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _check_name(name):
|
|
42
|
+
if not validate_onetick_field_name(name):
|
|
43
|
+
raise ValueError(f"Field name '{name}' is not a valid field name. "
|
|
44
|
+
"Onetick field names can contain upper and lower English letters, digits, "
|
|
45
|
+
"symbols '.' and '_'. Also, Onetick field names cannot be longer than 127 characters.")
|
|
46
|
+
|
|
47
|
+
def __init__(self, name, dtype=float, obj_ref=None, precision=None):
|
|
48
|
+
if not dtype or not inspect.isclass(dtype) or not ott.is_type_supported(dtype):
|
|
49
|
+
raise TypeError(f'Column does not support "{dtype}" type')
|
|
50
|
+
|
|
51
|
+
# validating column name
|
|
52
|
+
if name != 'Time': # this is a special value
|
|
53
|
+
self._check_name(name)
|
|
54
|
+
|
|
55
|
+
self.name = name
|
|
56
|
+
super().__init__(dtype=dtype, obj_ref=obj_ref, op_str=name)
|
|
57
|
+
|
|
58
|
+
# optional properties
|
|
59
|
+
if precision is not None:
|
|
60
|
+
if issubclass(dtype, float):
|
|
61
|
+
self._precision = precision
|
|
62
|
+
else:
|
|
63
|
+
raise ValueError("precision is supported only for columns with float or decimal dtypes")
|
|
64
|
+
|
|
65
|
+
def rename(self, new_name, update_parent_object=True):
|
|
66
|
+
self._check_name(new_name)
|
|
67
|
+
if self.obj_ref and update_parent_object:
|
|
68
|
+
self.obj_ref.rename({self.name: new_name}, inplace=True)
|
|
69
|
+
|
|
70
|
+
self.name = new_name
|
|
71
|
+
|
|
72
|
+
def __len__(self):
|
|
73
|
+
if issubclass(self.dtype, str):
|
|
74
|
+
if issubclass(self.dtype, ott.string):
|
|
75
|
+
return self.dtype.length
|
|
76
|
+
else:
|
|
77
|
+
return ott.string.DEFAULT_LENGTH
|
|
78
|
+
else:
|
|
79
|
+
raise TypeError(f'It is not applicable for the column with type {self.dtype}') # TODO: test
|
|
80
|
+
|
|
81
|
+
def __hash__(self):
|
|
82
|
+
return hash(self.name)
|
|
83
|
+
|
|
84
|
+
def __str__(self):
|
|
85
|
+
return self.name
|
|
86
|
+
|
|
87
|
+
def __repr__(self):
|
|
88
|
+
return f"Column({str(self)}, {self.dtype})"
|
|
89
|
+
|
|
90
|
+
def copy(self, obj_ref=None):
|
|
91
|
+
return _Column(self.name, self.dtype, obj_ref)
|
|
92
|
+
|
|
93
|
+
def __bool__(self):
|
|
94
|
+
if _Column.emulation_enabled:
|
|
95
|
+
if issubclass(self.dtype, int):
|
|
96
|
+
return (self != 0).__bool__()
|
|
97
|
+
if issubclass(self.dtype, float):
|
|
98
|
+
return (self != 0).__bool__()
|
|
99
|
+
if issubclass(self.dtype, str):
|
|
100
|
+
return (self != "").__bool__()
|
|
101
|
+
|
|
102
|
+
raise TypeError("It is not allowed to use columns in if-else and while clauses")
|
|
103
|
+
|
|
104
|
+
def __getitem__(self, item):
|
|
105
|
+
|
|
106
|
+
"""
|
|
107
|
+
Provides an ability to get values from future or past ticks.
|
|
108
|
+
|
|
109
|
+
- Negative values refer to past ticks
|
|
110
|
+
|
|
111
|
+
- Zero to current tick
|
|
112
|
+
|
|
113
|
+
- Positive - future ticks
|
|
114
|
+
|
|
115
|
+
Boundary values will be defaulted. For instance for ``item=-1`` first tick value will be defaulted
|
|
116
|
+
(there is no tick before first tick)
|
|
117
|
+
|
|
118
|
+
Parameters
|
|
119
|
+
----------
|
|
120
|
+
item: int
|
|
121
|
+
number of ticks to look back/forward
|
|
122
|
+
|
|
123
|
+
Returns
|
|
124
|
+
-------
|
|
125
|
+
Operation
|
|
126
|
+
|
|
127
|
+
Examples
|
|
128
|
+
--------
|
|
129
|
+
>>> data = otp.Ticks({'A': [1, 2, 3]})
|
|
130
|
+
>>> data['PAST1'] = data['A'][-1]
|
|
131
|
+
>>> data['PAST2'] = data['A'][-2]
|
|
132
|
+
>>> data['FUTURE1'] = data['A'][1]
|
|
133
|
+
>>> data['FUTURE2'] = data['A'][2]
|
|
134
|
+
>>> otp.run(data)
|
|
135
|
+
Time A PAST1 PAST2 FUTURE1 FUTURE2
|
|
136
|
+
0 2003-12-01 00:00:00.000 1 0 0 2 3
|
|
137
|
+
1 2003-12-01 00:00:00.001 2 1 0 3 0
|
|
138
|
+
2 2003-12-01 00:00:00.002 3 2 1 0 0
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
if not isinstance(item, int):
|
|
142
|
+
raise TypeError(
|
|
143
|
+
f"Lag operation supports only integer const values, but passed value of type '{type(item)}'"
|
|
144
|
+
)
|
|
145
|
+
if item == 0:
|
|
146
|
+
return self
|
|
147
|
+
|
|
148
|
+
return _LagOperator(self, item)
|
|
149
|
+
|
|
150
|
+
def cumsum(self):
|
|
151
|
+
"""
|
|
152
|
+
Cumulative sum of the column.
|
|
153
|
+
|
|
154
|
+
Can only be used when creating or updating column.
|
|
155
|
+
|
|
156
|
+
Examples
|
|
157
|
+
--------
|
|
158
|
+
>>> t = otp.Ticks({'A': [1, 2, 3]})
|
|
159
|
+
>>> t['X'] = t['A'].cumsum()
|
|
160
|
+
>>> otp.run(t)
|
|
161
|
+
Time A X
|
|
162
|
+
0 2003-12-01 00:00:00.000 1 1
|
|
163
|
+
1 2003-12-01 00:00:00.001 2 3
|
|
164
|
+
2 2003-12-01 00:00:00.002 3 6
|
|
165
|
+
"""
|
|
166
|
+
import onetick.py as otp
|
|
167
|
+
|
|
168
|
+
return _ColumnAggregation(
|
|
169
|
+
otp.agg.sum(self.name, running=True, all_fields=True, overwrite_output_field=True)
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
def __iter__(self):
|
|
173
|
+
raise TypeError("It is not allowed to use columns in for-clauses")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class _LagOperator(_Operation):
|
|
177
|
+
"""
|
|
178
|
+
Implements referencing to the prior tick
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
def __init__(self, base_column, inx):
|
|
182
|
+
self._inx = inx
|
|
183
|
+
op_str = f"{str(base_column)}[{self.index}]"
|
|
184
|
+
super().__init__(op_params=[base_column], dtype=base_column.dtype,
|
|
185
|
+
op_str=op_str, obj_ref=base_column.obj_ref)
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def index(self):
|
|
189
|
+
return self._inx
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _ColumnAggregation:
|
|
193
|
+
"""
|
|
194
|
+
Object to specify how column will be aggregated.
|
|
195
|
+
"""
|
|
196
|
+
def __init__(self, aggregation):
|
|
197
|
+
from ..aggregations._base import _Aggregation
|
|
198
|
+
if not isinstance(aggregation, _Aggregation):
|
|
199
|
+
raise ValueError(f'Expected aggregation object, got {type(aggregation)}')
|
|
200
|
+
if not aggregation.running or not aggregation.all_fields or not aggregation.overwrite_output_field:
|
|
201
|
+
raise ValueError("Column aggregations only support 'running' aggregations"
|
|
202
|
+
" with 'all_fields' and 'overwrite_output_field' parameters set")
|
|
203
|
+
self.aggregation = aggregation
|
|
204
|
+
|
|
205
|
+
def apply(self, src, name):
|
|
206
|
+
return self.aggregation.apply(src, name=name, inplace=True)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
_Column = Column # alias for backward compatibility
|
|
File without changes
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from collections import namedtuple
|
|
2
|
+
|
|
3
|
+
from onetick.py import types as ott
|
|
4
|
+
from onetick.py.core.column_operations._methods.op_types import are_strings
|
|
5
|
+
from onetick.py.types import value2str
|
|
6
|
+
|
|
7
|
+
MethodResult = namedtuple("MethodResult", ("op_str", "dtype"))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _wrap_object(o):
|
|
11
|
+
if isinstance(o, str) or are_strings(getattr(o, "dtype", None)):
|
|
12
|
+
# In PER_TICK_SCRIPT: parenthesis are not allowed in string expressions.
|
|
13
|
+
return value2str(o)
|
|
14
|
+
return f"({value2str(o)})"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _type_error_for_op(op, types):
|
|
18
|
+
return TypeError(f"Unsupported operand type(s) for {op} operation: {types}")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _init_binary_op(prev_op, other):
|
|
22
|
+
left = _wrap_object(prev_op)
|
|
23
|
+
right = _wrap_object(other)
|
|
24
|
+
left_t = ott.get_object_type(prev_op)
|
|
25
|
+
right_t = ott.get_object_type(other)
|
|
26
|
+
dtype = None
|
|
27
|
+
op_str = None
|
|
28
|
+
return left, right, left_t, right_t, op_str, dtype
|