onetick-py 1.177.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- locator_parser/__init__.py +0 -0
- locator_parser/acl.py +73 -0
- locator_parser/actions.py +262 -0
- locator_parser/common.py +368 -0
- locator_parser/io.py +43 -0
- locator_parser/locator.py +150 -0
- onetick/__init__.py +101 -0
- onetick/doc_utilities/__init__.py +3 -0
- onetick/doc_utilities/napoleon.py +40 -0
- onetick/doc_utilities/ot_doctest.py +140 -0
- onetick/doc_utilities/snippets.py +279 -0
- onetick/lib/__init__.py +4 -0
- onetick/lib/instance.py +141 -0
- onetick/py/__init__.py +293 -0
- onetick/py/_stack_info.py +89 -0
- onetick/py/_version.py +2 -0
- onetick/py/aggregations/__init__.py +11 -0
- onetick/py/aggregations/_base.py +648 -0
- onetick/py/aggregations/_docs.py +948 -0
- onetick/py/aggregations/compute.py +286 -0
- onetick/py/aggregations/functions.py +2216 -0
- onetick/py/aggregations/generic.py +104 -0
- onetick/py/aggregations/high_low.py +80 -0
- onetick/py/aggregations/num_distinct.py +83 -0
- onetick/py/aggregations/order_book.py +501 -0
- onetick/py/aggregations/other.py +1014 -0
- onetick/py/backports.py +26 -0
- onetick/py/cache.py +374 -0
- onetick/py/callback/__init__.py +5 -0
- onetick/py/callback/callback.py +276 -0
- onetick/py/callback/callbacks.py +131 -0
- onetick/py/compatibility.py +798 -0
- onetick/py/configuration.py +771 -0
- onetick/py/core/__init__.py +0 -0
- onetick/py/core/_csv_inspector.py +93 -0
- onetick/py/core/_internal/__init__.py +0 -0
- onetick/py/core/_internal/_manually_bound_value.py +6 -0
- onetick/py/core/_internal/_nodes_history.py +250 -0
- onetick/py/core/_internal/_op_utils/__init__.py +0 -0
- onetick/py/core/_internal/_op_utils/every_operand.py +9 -0
- onetick/py/core/_internal/_op_utils/is_const.py +10 -0
- onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script +121 -0
- onetick/py/core/_internal/_proxy_node.py +140 -0
- onetick/py/core/_internal/_state_objects.py +2312 -0
- onetick/py/core/_internal/_state_vars.py +93 -0
- onetick/py/core/_source/__init__.py +0 -0
- onetick/py/core/_source/_symbol_param.py +95 -0
- onetick/py/core/_source/schema.py +97 -0
- onetick/py/core/_source/source_methods/__init__.py +0 -0
- onetick/py/core/_source/source_methods/aggregations.py +809 -0
- onetick/py/core/_source/source_methods/applyers.py +296 -0
- onetick/py/core/_source/source_methods/columns.py +141 -0
- onetick/py/core/_source/source_methods/data_quality.py +301 -0
- onetick/py/core/_source/source_methods/debugs.py +272 -0
- onetick/py/core/_source/source_methods/drops.py +120 -0
- onetick/py/core/_source/source_methods/fields.py +619 -0
- onetick/py/core/_source/source_methods/filters.py +1002 -0
- onetick/py/core/_source/source_methods/joins.py +1413 -0
- onetick/py/core/_source/source_methods/merges.py +605 -0
- onetick/py/core/_source/source_methods/misc.py +1455 -0
- onetick/py/core/_source/source_methods/pandases.py +155 -0
- onetick/py/core/_source/source_methods/renames.py +356 -0
- onetick/py/core/_source/source_methods/sorts.py +183 -0
- onetick/py/core/_source/source_methods/switches.py +142 -0
- onetick/py/core/_source/source_methods/symbols.py +117 -0
- onetick/py/core/_source/source_methods/times.py +627 -0
- onetick/py/core/_source/source_methods/writes.py +986 -0
- onetick/py/core/_source/symbol.py +205 -0
- onetick/py/core/_source/tmp_otq.py +222 -0
- onetick/py/core/column.py +209 -0
- onetick/py/core/column_operations/__init__.py +0 -0
- onetick/py/core/column_operations/_methods/__init__.py +4 -0
- onetick/py/core/column_operations/_methods/_internal.py +28 -0
- onetick/py/core/column_operations/_methods/conversions.py +216 -0
- onetick/py/core/column_operations/_methods/methods.py +292 -0
- onetick/py/core/column_operations/_methods/op_types.py +160 -0
- onetick/py/core/column_operations/accessors/__init__.py +0 -0
- onetick/py/core/column_operations/accessors/_accessor.py +28 -0
- onetick/py/core/column_operations/accessors/decimal_accessor.py +104 -0
- onetick/py/core/column_operations/accessors/dt_accessor.py +537 -0
- onetick/py/core/column_operations/accessors/float_accessor.py +184 -0
- onetick/py/core/column_operations/accessors/str_accessor.py +1367 -0
- onetick/py/core/column_operations/base.py +1121 -0
- onetick/py/core/cut_builder.py +150 -0
- onetick/py/core/db_constants.py +20 -0
- onetick/py/core/eval_query.py +245 -0
- onetick/py/core/lambda_object.py +441 -0
- onetick/py/core/multi_output_source.py +232 -0
- onetick/py/core/per_tick_script.py +2256 -0
- onetick/py/core/query_inspector.py +464 -0
- onetick/py/core/source.py +1744 -0
- onetick/py/db/__init__.py +2 -0
- onetick/py/db/_inspection.py +1128 -0
- onetick/py/db/db.py +1327 -0
- onetick/py/db/utils.py +64 -0
- onetick/py/docs/__init__.py +0 -0
- onetick/py/docs/docstring_parser.py +112 -0
- onetick/py/docs/utils.py +81 -0
- onetick/py/functions.py +2398 -0
- onetick/py/license.py +190 -0
- onetick/py/log.py +88 -0
- onetick/py/math.py +935 -0
- onetick/py/misc.py +470 -0
- onetick/py/oqd/__init__.py +22 -0
- onetick/py/oqd/eps.py +1195 -0
- onetick/py/oqd/sources.py +325 -0
- onetick/py/otq.py +216 -0
- onetick/py/pyomd_mock.py +47 -0
- onetick/py/run.py +916 -0
- onetick/py/servers.py +173 -0
- onetick/py/session.py +1347 -0
- onetick/py/sources/__init__.py +19 -0
- onetick/py/sources/cache.py +167 -0
- onetick/py/sources/common.py +128 -0
- onetick/py/sources/csv.py +642 -0
- onetick/py/sources/custom.py +85 -0
- onetick/py/sources/data_file.py +305 -0
- onetick/py/sources/data_source.py +1045 -0
- onetick/py/sources/empty.py +94 -0
- onetick/py/sources/odbc.py +337 -0
- onetick/py/sources/order_book.py +271 -0
- onetick/py/sources/parquet.py +168 -0
- onetick/py/sources/pit.py +191 -0
- onetick/py/sources/query.py +495 -0
- onetick/py/sources/snapshots.py +419 -0
- onetick/py/sources/split_query_output_by_symbol.py +198 -0
- onetick/py/sources/symbology_mapping.py +123 -0
- onetick/py/sources/symbols.py +374 -0
- onetick/py/sources/ticks.py +825 -0
- onetick/py/sql.py +70 -0
- onetick/py/state.py +251 -0
- onetick/py/types.py +2131 -0
- onetick/py/utils/__init__.py +70 -0
- onetick/py/utils/acl.py +93 -0
- onetick/py/utils/config.py +186 -0
- onetick/py/utils/default.py +49 -0
- onetick/py/utils/file.py +38 -0
- onetick/py/utils/helpers.py +76 -0
- onetick/py/utils/locator.py +94 -0
- onetick/py/utils/perf.py +498 -0
- onetick/py/utils/query.py +49 -0
- onetick/py/utils/render.py +1374 -0
- onetick/py/utils/script.py +244 -0
- onetick/py/utils/temp.py +471 -0
- onetick/py/utils/types.py +120 -0
- onetick/py/utils/tz.py +84 -0
- onetick_py-1.177.0.dist-info/METADATA +137 -0
- onetick_py-1.177.0.dist-info/RECORD +152 -0
- onetick_py-1.177.0.dist-info/WHEEL +5 -0
- onetick_py-1.177.0.dist-info/entry_points.txt +2 -0
- onetick_py-1.177.0.dist-info/licenses/LICENSE +21 -0
- onetick_py-1.177.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, Optional
|
|
2
|
+
|
|
3
|
+
from onetick import py as otp
|
|
4
|
+
from onetick.py import types as ott
|
|
5
|
+
from onetick.py.otq import otq
|
|
6
|
+
|
|
7
|
+
from .misc import inplace_operation
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from onetick.py.core.source import Source
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@inplace_operation
|
|
14
|
+
def update_timestamp(
|
|
15
|
+
self: 'Source',
|
|
16
|
+
timestamp_field: str,
|
|
17
|
+
timestamp_psec_field: Optional[str] = None,
|
|
18
|
+
max_delay_of_original_timestamp=0,
|
|
19
|
+
max_delay_of_new_timestamp=0,
|
|
20
|
+
max_out_of_order_interval=0,
|
|
21
|
+
max_delay_handling: str = 'complain',
|
|
22
|
+
out_of_order_timestamp_handling: str = 'complain',
|
|
23
|
+
zero_timestamp_handling: Optional[str] = None,
|
|
24
|
+
log_sequence_violations: bool = False,
|
|
25
|
+
inplace=False,
|
|
26
|
+
) -> Optional['Source']:
|
|
27
|
+
"""
|
|
28
|
+
Assigns alternative timestamps to input ticks.
|
|
29
|
+
|
|
30
|
+
In case resulting timestamps are out of order, time series is sorted in ascending order of those timestamps.
|
|
31
|
+
|
|
32
|
+
Ticks with equal alternative timestamps are sorted in ascending order of the respective original timestamps,
|
|
33
|
+
and equal ones among those are further sorted by ascending values of the **OMDSEQ** field,
|
|
34
|
+
if such a field is present.
|
|
35
|
+
|
|
36
|
+
Note
|
|
37
|
+
----
|
|
38
|
+
|
|
39
|
+
In case parameters ``max_delay_of_original_timestamp`` or ``max_delay_of_new_timestamp`` are specified
|
|
40
|
+
this method automatically uses :py:meth:`~onetick.py.Source.modify_query_times` method, affecting
|
|
41
|
+
the start or end time of the query thus possibly changing some logic of the nodes placed higher in the graph.
|
|
42
|
+
|
|
43
|
+
Also there are some limitations on using this method in the graph, e.g. this method can't be used in the
|
|
44
|
+
"diamond" pattern and can't be used twice in the same graph.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
timestamp_field: str
|
|
49
|
+
Specifies the name of the input field, which is assumed to carry alternative timestamps.
|
|
50
|
+
timestamp_psec_field: str
|
|
51
|
+
Fractional (< 1 millisecond) parts of alternative timestamps will be taken from this field.
|
|
52
|
+
They are assumed to be in picoseconds (0.001 of nanosecond).
|
|
53
|
+
|
|
54
|
+
If this field is specified, then even if alternative timestamps have nanosecond granularity,
|
|
55
|
+
only millisecond parts of them will be taken, nanosecond part will be rewritten.
|
|
56
|
+
|
|
57
|
+
Useful in case ``timestamp_field`` has lower than millisecond granularity.
|
|
58
|
+
|
|
59
|
+
max_delay_of_original_timestamp: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
|
|
60
|
+
Changes the start time of the query to ``original_start_time - max_delay_of_original_timestamp``
|
|
61
|
+
to make sure it processes all possible ticks that might have a new timestamp in the query time range.
|
|
62
|
+
|
|
63
|
+
If integer value is specified, it is assumed to be milliseconds.
|
|
64
|
+
|
|
65
|
+
max_delay_of_new_timestamp: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
|
|
66
|
+
Changes the end time of the query to ``original_end_time + max_delay_of_new_timestamp``
|
|
67
|
+
to make sure it processes all possible ticks that might have a new timestamp in the query time range.
|
|
68
|
+
|
|
69
|
+
If integer value is specified, it is assumed to be milliseconds.
|
|
70
|
+
|
|
71
|
+
max_out_of_order_interval: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
|
|
72
|
+
Specifies the maximum out-of-order interval for alternative timestamps.
|
|
73
|
+
Ticks with new timestamps out of order will be sorted in ascending order.
|
|
74
|
+
|
|
75
|
+
This is the only parameter that leads to accumulation of ticks.
|
|
76
|
+
|
|
77
|
+
If integer value is specified, it is assumed to be milliseconds.
|
|
78
|
+
|
|
79
|
+
max_delay_handling: str ('complain' or 'discard' or 'use_original_timestamp' or 'use_new_timestamp')
|
|
80
|
+
|
|
81
|
+
This parameter how to process ticks that are delayed even more than specified in
|
|
82
|
+
``max_delay_of_original_timestamp`` or ``max_delay_of_new_timestamp`` parameters.
|
|
83
|
+
|
|
84
|
+
- **complain**: raise an exception
|
|
85
|
+
|
|
86
|
+
- **discard**: do not add tick to the output time series
|
|
87
|
+
|
|
88
|
+
- **use_original_timestamp**: assign original timestamp to the tick
|
|
89
|
+
|
|
90
|
+
- **use_new_timestamp**: try to assign new timestamp to the tick. If the new timestamp of previous tick
|
|
91
|
+
is greater than new timestamp of this tick then previous one is used.
|
|
92
|
+
**NOTE: A previously propagated timestamp could be from a heartbeat.**
|
|
93
|
+
When there is no previous propagated tick and the new timestamp falls behind the query start time,
|
|
94
|
+
the latter is preferred, while query end time is preferred if the new timestamp exceeds it.
|
|
95
|
+
|
|
96
|
+
This parameter is processed *before* any action from ``out_of_order_timestamp_handling`` parameter.
|
|
97
|
+
|
|
98
|
+
out_of_order_timestamp_handling: str ('complain' or 'use_previous_value' or 'use_original_timestamp')
|
|
99
|
+
|
|
100
|
+
This parameter how to process ticks that are out of order even more than specified in
|
|
101
|
+
``max_out_of_order_interval`` parameter.
|
|
102
|
+
|
|
103
|
+
- **complain**: raise an exception
|
|
104
|
+
|
|
105
|
+
- **use_previous_value**: assign new timestamp from a previous propagated tick
|
|
106
|
+
|
|
107
|
+
- **use_original_timestamp**: try to use original timestamp. If maximum out-of-order interval is still
|
|
108
|
+
exceeded then exception will be thrown.
|
|
109
|
+
|
|
110
|
+
This parameter is processed *after* any action from ``max_delay_handling`` parameter.
|
|
111
|
+
|
|
112
|
+
zero_timestamp_handling: str ('preserve_sequence'), optional
|
|
113
|
+
|
|
114
|
+
This parameter specifies how to process ticks with zero alternative timestamps.
|
|
115
|
+
|
|
116
|
+
If value is None, actions from ``max_delay_handling``
|
|
117
|
+
and ``out_of_order_timestamp_handling`` are performed on it.
|
|
118
|
+
|
|
119
|
+
If value is **preserve_sequence** then new timestamp will be set to the maximum between
|
|
120
|
+
query start time and the new timestamp of the previous propagated tick.
|
|
121
|
+
|
|
122
|
+
log_sequence_violations: bool
|
|
123
|
+
If set to True, then warnings about actions from parameters ``out_of_order_timestamp_handling``,
|
|
124
|
+
``max_delay_handling`` and ``zero_timestamp_handling`` are logged into the log file.
|
|
125
|
+
inplace: bool
|
|
126
|
+
The flag controls whether operation should be applied inplace or not.
|
|
127
|
+
If ``inplace=True``, then it returns nothing.
|
|
128
|
+
Otherwise method returns a new modified object.
|
|
129
|
+
|
|
130
|
+
Returns
|
|
131
|
+
-------
|
|
132
|
+
:class:`Source` or ``None``
|
|
133
|
+
|
|
134
|
+
See also
|
|
135
|
+
--------
|
|
136
|
+
**UPDATE_TIMESTAMP** OneTick event processor
|
|
137
|
+
|
|
138
|
+
Examples
|
|
139
|
+
--------
|
|
140
|
+
|
|
141
|
+
Data and timestamps from the database:
|
|
142
|
+
|
|
143
|
+
>>> start = otp.dt(2022, 3, 2)
|
|
144
|
+
>>> end = otp.dt(2022, 3, 3)
|
|
145
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
146
|
+
>>> otp.run(data, start=start, end=end)
|
|
147
|
+
Time PRICE SIZE
|
|
148
|
+
0 2022-03-02 00:00:00.000 1.0 100
|
|
149
|
+
1 2022-03-02 00:00:00.001 1.1 101
|
|
150
|
+
2 2022-03-02 00:00:00.002 1.2 102
|
|
151
|
+
|
|
152
|
+
Adding one hour to all ticks.
|
|
153
|
+
Parameter ``max_delay_of_original_timestamp`` must be specified in this case:
|
|
154
|
+
|
|
155
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
156
|
+
>>> data['ORIG_TS'] = data['TIMESTAMP']
|
|
157
|
+
>>> data['NEW_TS'] = data['TIMESTAMP'] + otp.Hour(1)
|
|
158
|
+
>>> data = data.update_timestamp('NEW_TS', max_delay_of_original_timestamp=otp.Hour(1))
|
|
159
|
+
>>> otp.run(data, start=start, end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
|
|
160
|
+
Time PRICE SIZE ORIG_TS
|
|
161
|
+
0 2022-03-02 01:00:00.000 1.0 100 2022-03-02 00:00:00.000
|
|
162
|
+
1 2022-03-02 01:00:00.001 1.1 101 2022-03-02 00:00:00.001
|
|
163
|
+
2 2022-03-02 01:00:00.002 1.2 102 2022-03-02 00:00:00.002
|
|
164
|
+
|
|
165
|
+
Subtracting one day from all ticks.
|
|
166
|
+
Parameter ``max_delay_of_new_timestamp`` must be specified in this case.
|
|
167
|
+
|
|
168
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
169
|
+
>>> data['ORIG_TS'] = data['TIMESTAMP']
|
|
170
|
+
>>> data['NEW_TS'] = data['TIMESTAMP'] - otp.Day(1)
|
|
171
|
+
>>> data = data.update_timestamp('NEW_TS', max_delay_of_new_timestamp=otp.Day(1))
|
|
172
|
+
>>> otp.run(data, start=start - otp.Day(1), end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
|
|
173
|
+
Time PRICE SIZE ORIG_TS
|
|
174
|
+
0 2022-03-01 00:00:00.000 1.0 100 2022-03-02 00:00:00.000
|
|
175
|
+
1 2022-03-01 00:00:00.001 1.1 101 2022-03-02 00:00:00.001
|
|
176
|
+
2 2022-03-01 00:00:00.002 1.2 102 2022-03-02 00:00:00.002
|
|
177
|
+
|
|
178
|
+
Parameter ``max_delay_handling`` can be used to specify how to handle ticks exceeding the maximum:
|
|
179
|
+
|
|
180
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
181
|
+
>>> data['ORIG_TS'] = data['TIMESTAMP']
|
|
182
|
+
>>> data['NEW_TS'] = data.apply(
|
|
183
|
+
... lambda row: row['TIMESTAMP'] + otp.Hour(24)
|
|
184
|
+
... if row['PRICE'] == 1.1
|
|
185
|
+
... else row['TIMESTAMP'] + otp.Hour(1)
|
|
186
|
+
... )
|
|
187
|
+
>>> data = data.update_timestamp('NEW_TS',
|
|
188
|
+
... max_delay_of_original_timestamp=otp.Hour(1),
|
|
189
|
+
... max_delay_handling='discard')
|
|
190
|
+
>>> otp.run(data, start=start, end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
|
|
191
|
+
Time PRICE SIZE ORIG_TS
|
|
192
|
+
0 2022-03-02 01:00:00.000 1.0 100 2022-03-02 00:00:00.000
|
|
193
|
+
1 2022-03-02 01:00:00.002 1.2 102 2022-03-02 00:00:00.002
|
|
194
|
+
|
|
195
|
+
Parameter ``max_out_of_order_interval`` can be used in case new timestamp are out of order:
|
|
196
|
+
|
|
197
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
198
|
+
>>> data['ORIG_TS'] = data['TIMESTAMP']
|
|
199
|
+
>>> data = data.agg({'COUNT': otp.agg.count()}, running=True, all_fields=True)
|
|
200
|
+
>>> data['NEW_TS'] = data['TIMESTAMP'] - otp.Minute(data['COUNT'])
|
|
201
|
+
>>> data = data.update_timestamp('NEW_TS',
|
|
202
|
+
... max_delay_of_new_timestamp=otp.Hour(10),
|
|
203
|
+
... max_out_of_order_interval=otp.Minute(100))
|
|
204
|
+
>>> otp.run(data, start=start - otp.Hour(2), end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS', 'COUNT']]
|
|
205
|
+
Time PRICE SIZE ORIG_TS COUNT
|
|
206
|
+
0 2022-03-01 23:57:00.002 1.2 102 2022-03-02 00:00:00.002 3
|
|
207
|
+
1 2022-03-01 23:58:00.001 1.1 101 2022-03-02 00:00:00.001 2
|
|
208
|
+
2 2022-03-01 23:59:00.000 1.0 100 2022-03-02 00:00:00.000 1
|
|
209
|
+
"""
|
|
210
|
+
if timestamp_field not in self.schema:
|
|
211
|
+
raise ValueError(f"Field '{timestamp_field}' is not in schema")
|
|
212
|
+
if self.schema[timestamp_field] not in {otp.msectime, otp.nsectime}:
|
|
213
|
+
raise ValueError(f"Unsupported type for 'timestamp_field': {self.schema[timestamp_field]}")
|
|
214
|
+
|
|
215
|
+
if timestamp_psec_field is not None:
|
|
216
|
+
if timestamp_psec_field not in self.schema:
|
|
217
|
+
raise ValueError(f"Field '{timestamp_psec_field}' is not in schema")
|
|
218
|
+
if self.schema[timestamp_psec_field] is not int:
|
|
219
|
+
raise ValueError(f"Unsupported type for 'timestamp_psec_field': {self.schema[timestamp_psec_field]}")
|
|
220
|
+
|
|
221
|
+
if max_delay_handling not in {'complain', 'discard', 'use_original_timestamp', 'use_new_timestamp'}:
|
|
222
|
+
raise ValueError(f"Unsupported value for parameter 'max_delay_handling': {max_delay_handling}")
|
|
223
|
+
|
|
224
|
+
if out_of_order_timestamp_handling not in {'complain', 'use_previous_value', 'use_original_timestamp'}:
|
|
225
|
+
raise ValueError(
|
|
226
|
+
f"Unsupported value for parameter 'out_of_order_timestamp_handling': {out_of_order_timestamp_handling}"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
if zero_timestamp_handling not in {None, 'preserve_sequence'}:
|
|
230
|
+
raise ValueError(f"Unsupported value for parameter 'zero_timestamp_handling': {zero_timestamp_handling}")
|
|
231
|
+
|
|
232
|
+
def get_ms(value):
|
|
233
|
+
if isinstance(value, int):
|
|
234
|
+
return value
|
|
235
|
+
try:
|
|
236
|
+
nanos = value.nanos
|
|
237
|
+
except Exception as e:
|
|
238
|
+
raise ValueError(f'Unsupported parameter value: {type(value)}') from e
|
|
239
|
+
if nanos < 1_000_000:
|
|
240
|
+
raise ValueError('Values less than one millisecond are not supported in this parameter')
|
|
241
|
+
return nanos // 1_000_000
|
|
242
|
+
|
|
243
|
+
self.sink(
|
|
244
|
+
otq.UpdateTimestamp(
|
|
245
|
+
timestamp_field_name=timestamp_field,
|
|
246
|
+
timestamp_psec_field_name=timestamp_psec_field or '',
|
|
247
|
+
max_delay_of_original_timestamp=get_ms(max_delay_of_original_timestamp),
|
|
248
|
+
max_delay_of_new_timestamp=get_ms(max_delay_of_new_timestamp),
|
|
249
|
+
max_out_of_order_interval_msec=get_ms(max_out_of_order_interval),
|
|
250
|
+
max_delay_handling=max_delay_handling,
|
|
251
|
+
out_of_order_timestamp_handling=out_of_order_timestamp_handling,
|
|
252
|
+
zero_timestamp_handling=zero_timestamp_handling,
|
|
253
|
+
log_sequence_violations=log_sequence_violations,
|
|
254
|
+
)
|
|
255
|
+
)
|
|
256
|
+
return self
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@inplace_operation
|
|
260
|
+
def modify_query_times(
|
|
261
|
+
self: 'Source', start=None, end=None, output_timestamp=None, propagate_heartbeats=True, inplace=False
|
|
262
|
+
) -> Optional['Source']:
|
|
263
|
+
"""
|
|
264
|
+
Modify ``start`` and ``end`` time of the query.
|
|
265
|
+
|
|
266
|
+
* query times are changed for all operations
|
|
267
|
+
only **before** this method up to the source of the graph.
|
|
268
|
+
* all ticks' timestamps produced by this method
|
|
269
|
+
**must** fall into original time range of the query.
|
|
270
|
+
|
|
271
|
+
It is possible to change ticks' timestamps with parameter ``output_timestamp``,
|
|
272
|
+
so they will stay inside the original time range.
|
|
273
|
+
|
|
274
|
+
Note
|
|
275
|
+
----
|
|
276
|
+
Due to how OneTick works internally, tick generators
|
|
277
|
+
:py:class:`otp.Tick <onetick.py.Tick>` and :py:func:`otp.Ticks <onetick.py.Ticks>`
|
|
278
|
+
are not affected by this method.
|
|
279
|
+
|
|
280
|
+
Parameters
|
|
281
|
+
----------
|
|
282
|
+
start: :py:class:`otp.datetime <onetick.py.datetime>` or \
|
|
283
|
+
:py:class:`~onetick.py.core.source.MetaFields` or :py:class:`~onetick.py.Operation`
|
|
284
|
+
Expression to replace query start time.
|
|
285
|
+
By default, start time is not changed.
|
|
286
|
+
Note that expression in this parameter can't depend on ticks, thus only
|
|
287
|
+
:py:class:`~onetick.py.core.source.MetaFields` and constants can be used.
|
|
288
|
+
end: :py:class:`otp.datetime <onetick.py.datetime>` or \
|
|
289
|
+
:py:class:`~onetick.py.core.source.MetaFields` or :py:class:`~onetick.py.Operation`
|
|
290
|
+
Expression to replace query end time.
|
|
291
|
+
By default, end time is not changed.
|
|
292
|
+
Note that expression in this parameter can't depend on ticks, thus only
|
|
293
|
+
:py:class:`~onetick.py.core.source.MetaFields` and constants can be used.
|
|
294
|
+
output_timestamp: :py:class:`onetick.py.Operation`
|
|
295
|
+
Expression that produces timestamp for each tick.
|
|
296
|
+
By default, the following expression is used: ``orig_start + orig_timestamp - start``
|
|
297
|
+
This expression covers cases when start time of the query is changed and keeps
|
|
298
|
+
timestamp inside original time range.
|
|
299
|
+
Note that it doesn't cover cases, for example, if end time was increased,
|
|
300
|
+
you have to handle such cases yourself.
|
|
301
|
+
propagate_heartbeats: bool
|
|
302
|
+
Controls heartbeat propagation.
|
|
303
|
+
inplace: bool
|
|
304
|
+
The flag controls whether operation should be applied inplace or not.
|
|
305
|
+
If ``inplace=True``, then it returns nothing.
|
|
306
|
+
Otherwise method returns a new modified object.
|
|
307
|
+
|
|
308
|
+
See also
|
|
309
|
+
--------
|
|
310
|
+
| **MODIFY_QUERY_TIMES** OneTick event processor
|
|
311
|
+
| :py:meth:`onetick.py.Source.time_interval_shift`
|
|
312
|
+
|
|
313
|
+
Returns
|
|
314
|
+
-------
|
|
315
|
+
:class:`Source` or ``None``
|
|
316
|
+
|
|
317
|
+
Examples
|
|
318
|
+
--------
|
|
319
|
+
|
|
320
|
+
>>> start = otp.dt(2022, 3, 2)
|
|
321
|
+
>>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
|
|
322
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
323
|
+
|
|
324
|
+
By default, method does nothing:
|
|
325
|
+
|
|
326
|
+
>>> t = data.modify_query_times()
|
|
327
|
+
>>> otp.run(t, start=start, end=end)
|
|
328
|
+
Time PRICE SIZE
|
|
329
|
+
0 2022-03-02 00:00:00.000 1.0 100
|
|
330
|
+
1 2022-03-02 00:00:00.001 1.1 101
|
|
331
|
+
2 2022-03-02 00:00:00.002 1.2 102
|
|
332
|
+
|
|
333
|
+
See how ``_START_TIME`` and ``_END_TIME`` meta fields are changed.
|
|
334
|
+
They are changed *before* ``modify_query_times``:
|
|
335
|
+
|
|
336
|
+
>>> t = data.copy()
|
|
337
|
+
>>> t['S_BEFORE'] = t['_START_TIME']
|
|
338
|
+
>>> t['E_BEFORE'] = t['_END_TIME']
|
|
339
|
+
>>> t = t.modify_query_times(start=t['_START_TIME'] + otp.Milli(1),
|
|
340
|
+
... end=t['_END_TIME'] - otp.Milli(1))
|
|
341
|
+
>>> t['S_AFTER'] = t['_START_TIME']
|
|
342
|
+
>>> t['E_AFTER'] = t['_END_TIME']
|
|
343
|
+
>>> otp.run(t, start=start, end=end)
|
|
344
|
+
Time PRICE SIZE S_BEFORE E_BEFORE S_AFTER E_AFTER
|
|
345
|
+
0 2022-03-02 1.1 101 2022-03-02 00:00:00.001 2022-03-02 00:00:00.002 2022-03-02 2022-03-02 00:00:00.003
|
|
346
|
+
|
|
347
|
+
You can decrease time interval without problems:
|
|
348
|
+
|
|
349
|
+
>>> t = data.modify_query_times(start=data['_START_TIME'] + otp.Milli(1),
|
|
350
|
+
... end=data['_END_TIME'] - otp.Milli(1))
|
|
351
|
+
>>> otp.run(t, start=start, end=end)
|
|
352
|
+
Time PRICE SIZE
|
|
353
|
+
0 2022-03-02 1.1 101
|
|
354
|
+
|
|
355
|
+
Note that the timestamp of the tick was changed with default expression.
|
|
356
|
+
In this case we can output original timestamps,
|
|
357
|
+
because they fall into original time range:
|
|
358
|
+
|
|
359
|
+
>>> t = data.modify_query_times(start=data['_START_TIME'] + otp.Milli(1),
|
|
360
|
+
... end=data['_END_TIME'] - otp.Milli(1),
|
|
361
|
+
... output_timestamp=data['TIMESTAMP'])
|
|
362
|
+
>>> otp.run(t, start=start, end=end)
|
|
363
|
+
Time PRICE SIZE
|
|
364
|
+
0 2022-03-02 00:00:00.001 1.1 101
|
|
365
|
+
|
|
366
|
+
But it will not work if new time range is wider than original:
|
|
367
|
+
|
|
368
|
+
>>> t = data.modify_query_times(start=data['_START_TIME'] - otp.Milli(1),
|
|
369
|
+
... output_timestamp=data['TIMESTAMP'])
|
|
370
|
+
>>> otp.run(t, start=start + otp.Milli(1), end=end + otp.Milli(1)) # doctest: +ELLIPSIS
|
|
371
|
+
Traceback (most recent call last):
|
|
372
|
+
Exception...timestamp is falling out of initial start/end time range...
|
|
373
|
+
|
|
374
|
+
In this case default ``output_timestamp`` expression would work just fine:
|
|
375
|
+
|
|
376
|
+
>>> t = data.modify_query_times(start=data['_START_TIME'] - otp.Milli(1))
|
|
377
|
+
>>> otp.run(t, start=start + otp.Milli(1), end=end + otp.Milli(1))
|
|
378
|
+
Time PRICE SIZE
|
|
379
|
+
0 2022-03-02 00:00:00.001 1.0 100
|
|
380
|
+
1 2022-03-02 00:00:00.002 1.1 101
|
|
381
|
+
2 2022-03-02 00:00:00.003 1.2 102
|
|
382
|
+
|
|
383
|
+
But it doesn't work, for example, if end time has crossed the borders of original time range.
|
|
384
|
+
In this case other ``output_timestamp`` expression must be specified:
|
|
385
|
+
|
|
386
|
+
>>> t = data.modify_query_times(
|
|
387
|
+
... start=data['_START_TIME'] - otp.Milli(2),
|
|
388
|
+
... output_timestamp=otp.math.min(data['TIMESTAMP'] + otp.Milli(2), data['_END_TIME'])
|
|
389
|
+
... )
|
|
390
|
+
>>> otp.run(t, start=start + otp.Milli(2), end=end)
|
|
391
|
+
Time PRICE SIZE
|
|
392
|
+
0 2022-03-02 00:00:00.002 1.0 100
|
|
393
|
+
1 2022-03-02 00:00:00.003 1.1 101
|
|
394
|
+
2 2022-03-02 00:00:00.003 1.2 102
|
|
395
|
+
|
|
396
|
+
Remember that ``start`` and ``end`` parameters can't depend on ticks:
|
|
397
|
+
|
|
398
|
+
>>> t = data.copy()
|
|
399
|
+
>>> t['X'] = 12345
|
|
400
|
+
>>> t = t.modify_query_times(start=t['_START_TIME'] + t['X'] - t['X'],
|
|
401
|
+
... end=t['_END_TIME'] - otp.Milli(1))
|
|
402
|
+
>>> otp.run(t, start=start, end=end) # doctest: +ELLIPSIS
|
|
403
|
+
Traceback (most recent call last):
|
|
404
|
+
Exception...parameter must not depend on ticks...
|
|
405
|
+
|
|
406
|
+
Constant datetime values can be used as parameters too:
|
|
407
|
+
|
|
408
|
+
>>> t = data.modify_query_times(start=start + otp.Milli(1),
|
|
409
|
+
... end=end - otp.Milli(1))
|
|
410
|
+
>>> otp.run(t, start=start, end=end)
|
|
411
|
+
Time PRICE SIZE
|
|
412
|
+
0 2022-03-02 1.1 101
|
|
413
|
+
|
|
414
|
+
Note that some graph patterns are not allowed when using this method.
|
|
415
|
+
For example, modifying query times for a branch that will be merged later:
|
|
416
|
+
|
|
417
|
+
>>> t1, t2 = data[data['PRICE'] > 1.3]
|
|
418
|
+
>>> t2 = t2.modify_query_times(start=start + otp.Milli(1))
|
|
419
|
+
>>> t = otp.merge([t1, t2])
|
|
420
|
+
>>> otp.run(t, start=start, end=end) # doctest: +ELLIPSIS
|
|
421
|
+
Traceback (most recent call last):
|
|
422
|
+
Exception...Invalid graph...time bound to a node...an intermediate node in one of the cycles in graph...
|
|
423
|
+
"""
|
|
424
|
+
start = ott.value2str(start) if start is not None else ''
|
|
425
|
+
end = ott.value2str(end) if end is not None else ''
|
|
426
|
+
output_timestamp = ott.value2str(output_timestamp) if output_timestamp is not None else ''
|
|
427
|
+
self.sink(
|
|
428
|
+
otq.ModifyQueryTimes(
|
|
429
|
+
start_time=start,
|
|
430
|
+
end_time=end,
|
|
431
|
+
output_timestamp=output_timestamp,
|
|
432
|
+
propagate_heartbeats=propagate_heartbeats,
|
|
433
|
+
)
|
|
434
|
+
)
|
|
435
|
+
return self
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def time_interval_shift(self: 'Source', shift, inplace=False) -> Optional['Source']:
|
|
439
|
+
"""
|
|
440
|
+
Shifting time interval for a source.
|
|
441
|
+
|
|
442
|
+
The whole data flow is shifted all the way up to the source of the graph.
|
|
443
|
+
|
|
444
|
+
The start and end times of the query will be changed for all operations before this method,
|
|
445
|
+
and will stay the same after this method.
|
|
446
|
+
|
|
447
|
+
WARNING: The ticks' timestamps *are changed* automatically so they fit into original time range.
|
|
448
|
+
|
|
449
|
+
You will get different set of ticks from the database, but the timestamps of the ticks
|
|
450
|
+
from that database will not be the same as in the database.
|
|
451
|
+
|
|
452
|
+
They need to be changed so they fit into the original query time range.
|
|
453
|
+
See details in :py:meth:`onetick.py.Source.modify_query_times`.
|
|
454
|
+
|
|
455
|
+
Parameters
|
|
456
|
+
----------
|
|
457
|
+
shift: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
|
|
458
|
+
Offset to shift the whole time interval.
|
|
459
|
+
Can be positive or negative.
|
|
460
|
+
Positive value moves time interval into the future, negative -- to the past.
|
|
461
|
+
int values are interpreted as milliseconds.
|
|
462
|
+
|
|
463
|
+
Timestamps of the ticks will be changed so they fit into the original query time range
|
|
464
|
+
by subtracting ``shift`` from each timestamp.
|
|
465
|
+
inplace: bool
|
|
466
|
+
The flag controls whether operation should be applied inplace or not.
|
|
467
|
+
If ``inplace=True``, then it returns nothing.
|
|
468
|
+
Otherwise method returns a new modified object.
|
|
469
|
+
|
|
470
|
+
Returns
|
|
471
|
+
-------
|
|
472
|
+
:class:`Source` or ``None``
|
|
473
|
+
|
|
474
|
+
See also
|
|
475
|
+
--------
|
|
476
|
+
| :py:meth:`onetick.py.Source.modify_query_times`
|
|
477
|
+
| :py:meth:`onetick.py.Source.time_interval_change`
|
|
478
|
+
|
|
479
|
+
Examples
|
|
480
|
+
--------
|
|
481
|
+
|
|
482
|
+
--> Also see use-case using :py:meth:`time_interval_shift` for calculating
|
|
483
|
+
`Markouts <../../static/getting_started/use_cases.html#point-in-time-benchmarks-bbo-at-different-markouts>`_
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
>>> start = otp.dt(2022, 3, 2)
|
|
487
|
+
>>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
|
|
488
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
489
|
+
|
|
490
|
+
Default data:
|
|
491
|
+
|
|
492
|
+
>>> otp.run(data, start=start, end=end)
|
|
493
|
+
Time PRICE SIZE
|
|
494
|
+
0 2022-03-02 00:00:00.000 1.0 100
|
|
495
|
+
1 2022-03-02 00:00:00.001 1.1 101
|
|
496
|
+
2 2022-03-02 00:00:00.002 1.2 102
|
|
497
|
+
|
|
498
|
+
Get window for a third tick:
|
|
499
|
+
|
|
500
|
+
>>> otp.run(data, start=start + otp.Milli(2), end=start + otp.Milli(3))
|
|
501
|
+
Time PRICE SIZE
|
|
502
|
+
0 2022-03-02 00:00:00.002 1.2 102
|
|
503
|
+
|
|
504
|
+
Shifting time window will result in different set of ticks,
|
|
505
|
+
but the ticks will have their timestamps changed to fit into original time range.
|
|
506
|
+
Let's shift time 2 milliseconds back and thus get the first tick:
|
|
507
|
+
|
|
508
|
+
>>> t = data.time_interval_shift(shift=-otp.Milli(2))
|
|
509
|
+
>>> otp.run(t, start=start + otp.Milli(2), end=start + otp.Milli(3))
|
|
510
|
+
Time PRICE SIZE
|
|
511
|
+
0 2022-03-02 00:00:00.002 1.0 100
|
|
512
|
+
|
|
513
|
+
Here we are querying empty time interval, but shifting one second back to get ticks.
|
|
514
|
+
|
|
515
|
+
>>> t = data.time_interval_shift(shift=-otp.Second(1))
|
|
516
|
+
>>> otp.run(t, start=start + otp.Second(1), end=end + otp.Second(1))
|
|
517
|
+
Time PRICE SIZE
|
|
518
|
+
0 2022-03-02 00:00:01.000 1.0 100
|
|
519
|
+
1 2022-03-02 00:00:01.001 1.1 101
|
|
520
|
+
2 2022-03-02 00:00:01.002 1.2 102
|
|
521
|
+
|
|
522
|
+
Note that tick generators
|
|
523
|
+
:py:class:`otp.Tick <onetick.py.Tick>` and :py:func:`otp.Ticks <onetick.py.Ticks>`
|
|
524
|
+
are not really affected by this method, they will have the same timestamps:
|
|
525
|
+
|
|
526
|
+
>>> t = otp.Tick(A=1)
|
|
527
|
+
>>> otp.run(t)
|
|
528
|
+
Time A
|
|
529
|
+
0 2003-12-01 1
|
|
530
|
+
|
|
531
|
+
>>> t = t.time_interval_shift(shift=otp.Second(1))
|
|
532
|
+
>>> otp.run(t)
|
|
533
|
+
Time A
|
|
534
|
+
0 2003-12-01 1
|
|
535
|
+
"""
|
|
536
|
+
start = self['_START_TIME'] + shift
|
|
537
|
+
end = self['_END_TIME'] + shift
|
|
538
|
+
# change timestamps so they fit into original time range
|
|
539
|
+
output_timestamp = self['TIMESTAMP'] - shift
|
|
540
|
+
return self.modify_query_times(start=start, end=end, output_timestamp=output_timestamp, inplace=inplace)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def time_interval_change(self: 'Source', start_change=0, end_change=0, inplace=False) -> Optional['Source']:
|
|
544
|
+
"""
|
|
545
|
+
Changing time interval by making it bigger or smaller.
|
|
546
|
+
|
|
547
|
+
All timestamps of ticks that are crossing the border of original time range
|
|
548
|
+
will be set to original start time or end time depending on their original time.
|
|
549
|
+
|
|
550
|
+
Parameters
|
|
551
|
+
----------
|
|
552
|
+
start_change: int or :ref:`datetime offset <api/datetime/offsets/root:datetime offsets>`
|
|
553
|
+
Offset to shift start time.
|
|
554
|
+
Can be positive or negative.
|
|
555
|
+
Positive value moves start time into the future, negative -- to the past.
|
|
556
|
+
int values are interpreted as milliseconds.
|
|
557
|
+
end_change: int or :ref:`datetime offset <api/datetime/offsets/root:datetime offsets>`
|
|
558
|
+
Offset to shift end time.
|
|
559
|
+
Can be positive or negative.
|
|
560
|
+
Positive value moves end time into the future, negative -- to the past.
|
|
561
|
+
int values are interpreted as milliseconds.
|
|
562
|
+
inplace: bool
|
|
563
|
+
The flag controls whether operation should be applied inplace or not.
|
|
564
|
+
If ``inplace=True``, then it returns nothing.
|
|
565
|
+
Otherwise method returns a new modified object.
|
|
566
|
+
|
|
567
|
+
Returns
|
|
568
|
+
-------
|
|
569
|
+
:class:`Source` or ``None``
|
|
570
|
+
|
|
571
|
+
See also
|
|
572
|
+
--------
|
|
573
|
+
| :py:meth:`onetick.py.Source.modify_query_times`
|
|
574
|
+
| :py:meth:`onetick.py.Source.time_interval_shift`
|
|
575
|
+
|
|
576
|
+
Examples
|
|
577
|
+
--------
|
|
578
|
+
|
|
579
|
+
>>> start = otp.dt(2022, 3, 2)
|
|
580
|
+
>>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
|
|
581
|
+
>>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
|
|
582
|
+
|
|
583
|
+
By default, ``time_interval_change()`` does nothing:
|
|
584
|
+
|
|
585
|
+
>>> t = data.time_interval_change()
|
|
586
|
+
>>> otp.run(t, start=start, end=end)
|
|
587
|
+
Time PRICE SIZE
|
|
588
|
+
0 2022-03-02 00:00:00.000 1.0 100
|
|
589
|
+
1 2022-03-02 00:00:00.001 1.1 101
|
|
590
|
+
2 2022-03-02 00:00:00.002 1.2 102
|
|
591
|
+
|
|
592
|
+
Decreasing time range will not change ticks' timestamps:
|
|
593
|
+
|
|
594
|
+
>>> t = data.time_interval_change(start_change=otp.Milli(1), end_change=-otp.Milli(1))
|
|
595
|
+
>>> otp.run(t, start=start, end=end)
|
|
596
|
+
Time PRICE SIZE
|
|
597
|
+
0 2022-03-02 00:00:00.001 1.1 101
|
|
598
|
+
|
|
599
|
+
Increasing time range will change timestamps of the ticks that crossed the border.
|
|
600
|
+
In this case first tick's timestamp will be set to original start time,
|
|
601
|
+
and third tick's to original end time.
|
|
602
|
+
|
|
603
|
+
>>> t = data.time_interval_change(start_change=-otp.Milli(1), end_change=otp.Milli(1))
|
|
604
|
+
>>> otp.run(t, start=start + otp.Milli(1), end=start + otp.Milli(2))
|
|
605
|
+
Time PRICE SIZE
|
|
606
|
+
0 2022-03-02 00:00:00.001 1.0 100
|
|
607
|
+
1 2022-03-02 00:00:00.001 1.1 101
|
|
608
|
+
2 2022-03-02 00:00:00.002 1.2 102
|
|
609
|
+
|
|
610
|
+
Here we are querying empty time interval, but changing start time one second back to get ticks.
|
|
611
|
+
|
|
612
|
+
>>> t = data.time_interval_change(start_change=-otp.Second(1))
|
|
613
|
+
>>> otp.run(t, start=start + otp.Second(1), end=end + otp.Second(1))
|
|
614
|
+
Time PRICE SIZE
|
|
615
|
+
0 2022-03-02 00:00:01 1.0 100
|
|
616
|
+
1 2022-03-02 00:00:01 1.1 101
|
|
617
|
+
2 2022-03-02 00:00:01 1.2 102
|
|
618
|
+
"""
|
|
619
|
+
start = self['_START_TIME'] + start_change
|
|
620
|
+
end = self['_END_TIME'] + end_change
|
|
621
|
+
|
|
622
|
+
# change ticks' timestamps only if they are out of bounds
|
|
623
|
+
output_timestamp = self['TIMESTAMP']
|
|
624
|
+
output_timestamp = otp.math.min(output_timestamp, self['_END_TIME'])
|
|
625
|
+
output_timestamp = otp.math.max(output_timestamp, self['_START_TIME'])
|
|
626
|
+
|
|
627
|
+
return self.modify_query_times(start=start, end=end, output_timestamp=output_timestamp, inplace=inplace)
|