onetick-py 1.177.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. locator_parser/__init__.py +0 -0
  2. locator_parser/acl.py +73 -0
  3. locator_parser/actions.py +262 -0
  4. locator_parser/common.py +368 -0
  5. locator_parser/io.py +43 -0
  6. locator_parser/locator.py +150 -0
  7. onetick/__init__.py +101 -0
  8. onetick/doc_utilities/__init__.py +3 -0
  9. onetick/doc_utilities/napoleon.py +40 -0
  10. onetick/doc_utilities/ot_doctest.py +140 -0
  11. onetick/doc_utilities/snippets.py +279 -0
  12. onetick/lib/__init__.py +4 -0
  13. onetick/lib/instance.py +141 -0
  14. onetick/py/__init__.py +293 -0
  15. onetick/py/_stack_info.py +89 -0
  16. onetick/py/_version.py +2 -0
  17. onetick/py/aggregations/__init__.py +11 -0
  18. onetick/py/aggregations/_base.py +648 -0
  19. onetick/py/aggregations/_docs.py +948 -0
  20. onetick/py/aggregations/compute.py +286 -0
  21. onetick/py/aggregations/functions.py +2216 -0
  22. onetick/py/aggregations/generic.py +104 -0
  23. onetick/py/aggregations/high_low.py +80 -0
  24. onetick/py/aggregations/num_distinct.py +83 -0
  25. onetick/py/aggregations/order_book.py +501 -0
  26. onetick/py/aggregations/other.py +1014 -0
  27. onetick/py/backports.py +26 -0
  28. onetick/py/cache.py +374 -0
  29. onetick/py/callback/__init__.py +5 -0
  30. onetick/py/callback/callback.py +276 -0
  31. onetick/py/callback/callbacks.py +131 -0
  32. onetick/py/compatibility.py +798 -0
  33. onetick/py/configuration.py +771 -0
  34. onetick/py/core/__init__.py +0 -0
  35. onetick/py/core/_csv_inspector.py +93 -0
  36. onetick/py/core/_internal/__init__.py +0 -0
  37. onetick/py/core/_internal/_manually_bound_value.py +6 -0
  38. onetick/py/core/_internal/_nodes_history.py +250 -0
  39. onetick/py/core/_internal/_op_utils/__init__.py +0 -0
  40. onetick/py/core/_internal/_op_utils/every_operand.py +9 -0
  41. onetick/py/core/_internal/_op_utils/is_const.py +10 -0
  42. onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script +121 -0
  43. onetick/py/core/_internal/_proxy_node.py +140 -0
  44. onetick/py/core/_internal/_state_objects.py +2312 -0
  45. onetick/py/core/_internal/_state_vars.py +93 -0
  46. onetick/py/core/_source/__init__.py +0 -0
  47. onetick/py/core/_source/_symbol_param.py +95 -0
  48. onetick/py/core/_source/schema.py +97 -0
  49. onetick/py/core/_source/source_methods/__init__.py +0 -0
  50. onetick/py/core/_source/source_methods/aggregations.py +809 -0
  51. onetick/py/core/_source/source_methods/applyers.py +296 -0
  52. onetick/py/core/_source/source_methods/columns.py +141 -0
  53. onetick/py/core/_source/source_methods/data_quality.py +301 -0
  54. onetick/py/core/_source/source_methods/debugs.py +272 -0
  55. onetick/py/core/_source/source_methods/drops.py +120 -0
  56. onetick/py/core/_source/source_methods/fields.py +619 -0
  57. onetick/py/core/_source/source_methods/filters.py +1002 -0
  58. onetick/py/core/_source/source_methods/joins.py +1413 -0
  59. onetick/py/core/_source/source_methods/merges.py +605 -0
  60. onetick/py/core/_source/source_methods/misc.py +1455 -0
  61. onetick/py/core/_source/source_methods/pandases.py +155 -0
  62. onetick/py/core/_source/source_methods/renames.py +356 -0
  63. onetick/py/core/_source/source_methods/sorts.py +183 -0
  64. onetick/py/core/_source/source_methods/switches.py +142 -0
  65. onetick/py/core/_source/source_methods/symbols.py +117 -0
  66. onetick/py/core/_source/source_methods/times.py +627 -0
  67. onetick/py/core/_source/source_methods/writes.py +986 -0
  68. onetick/py/core/_source/symbol.py +205 -0
  69. onetick/py/core/_source/tmp_otq.py +222 -0
  70. onetick/py/core/column.py +209 -0
  71. onetick/py/core/column_operations/__init__.py +0 -0
  72. onetick/py/core/column_operations/_methods/__init__.py +4 -0
  73. onetick/py/core/column_operations/_methods/_internal.py +28 -0
  74. onetick/py/core/column_operations/_methods/conversions.py +216 -0
  75. onetick/py/core/column_operations/_methods/methods.py +292 -0
  76. onetick/py/core/column_operations/_methods/op_types.py +160 -0
  77. onetick/py/core/column_operations/accessors/__init__.py +0 -0
  78. onetick/py/core/column_operations/accessors/_accessor.py +28 -0
  79. onetick/py/core/column_operations/accessors/decimal_accessor.py +104 -0
  80. onetick/py/core/column_operations/accessors/dt_accessor.py +537 -0
  81. onetick/py/core/column_operations/accessors/float_accessor.py +184 -0
  82. onetick/py/core/column_operations/accessors/str_accessor.py +1367 -0
  83. onetick/py/core/column_operations/base.py +1121 -0
  84. onetick/py/core/cut_builder.py +150 -0
  85. onetick/py/core/db_constants.py +20 -0
  86. onetick/py/core/eval_query.py +245 -0
  87. onetick/py/core/lambda_object.py +441 -0
  88. onetick/py/core/multi_output_source.py +232 -0
  89. onetick/py/core/per_tick_script.py +2256 -0
  90. onetick/py/core/query_inspector.py +464 -0
  91. onetick/py/core/source.py +1744 -0
  92. onetick/py/db/__init__.py +2 -0
  93. onetick/py/db/_inspection.py +1128 -0
  94. onetick/py/db/db.py +1327 -0
  95. onetick/py/db/utils.py +64 -0
  96. onetick/py/docs/__init__.py +0 -0
  97. onetick/py/docs/docstring_parser.py +112 -0
  98. onetick/py/docs/utils.py +81 -0
  99. onetick/py/functions.py +2398 -0
  100. onetick/py/license.py +190 -0
  101. onetick/py/log.py +88 -0
  102. onetick/py/math.py +935 -0
  103. onetick/py/misc.py +470 -0
  104. onetick/py/oqd/__init__.py +22 -0
  105. onetick/py/oqd/eps.py +1195 -0
  106. onetick/py/oqd/sources.py +325 -0
  107. onetick/py/otq.py +216 -0
  108. onetick/py/pyomd_mock.py +47 -0
  109. onetick/py/run.py +916 -0
  110. onetick/py/servers.py +173 -0
  111. onetick/py/session.py +1347 -0
  112. onetick/py/sources/__init__.py +19 -0
  113. onetick/py/sources/cache.py +167 -0
  114. onetick/py/sources/common.py +128 -0
  115. onetick/py/sources/csv.py +642 -0
  116. onetick/py/sources/custom.py +85 -0
  117. onetick/py/sources/data_file.py +305 -0
  118. onetick/py/sources/data_source.py +1045 -0
  119. onetick/py/sources/empty.py +94 -0
  120. onetick/py/sources/odbc.py +337 -0
  121. onetick/py/sources/order_book.py +271 -0
  122. onetick/py/sources/parquet.py +168 -0
  123. onetick/py/sources/pit.py +191 -0
  124. onetick/py/sources/query.py +495 -0
  125. onetick/py/sources/snapshots.py +419 -0
  126. onetick/py/sources/split_query_output_by_symbol.py +198 -0
  127. onetick/py/sources/symbology_mapping.py +123 -0
  128. onetick/py/sources/symbols.py +374 -0
  129. onetick/py/sources/ticks.py +825 -0
  130. onetick/py/sql.py +70 -0
  131. onetick/py/state.py +251 -0
  132. onetick/py/types.py +2131 -0
  133. onetick/py/utils/__init__.py +70 -0
  134. onetick/py/utils/acl.py +93 -0
  135. onetick/py/utils/config.py +186 -0
  136. onetick/py/utils/default.py +49 -0
  137. onetick/py/utils/file.py +38 -0
  138. onetick/py/utils/helpers.py +76 -0
  139. onetick/py/utils/locator.py +94 -0
  140. onetick/py/utils/perf.py +498 -0
  141. onetick/py/utils/query.py +49 -0
  142. onetick/py/utils/render.py +1374 -0
  143. onetick/py/utils/script.py +244 -0
  144. onetick/py/utils/temp.py +471 -0
  145. onetick/py/utils/types.py +120 -0
  146. onetick/py/utils/tz.py +84 -0
  147. onetick_py-1.177.0.dist-info/METADATA +137 -0
  148. onetick_py-1.177.0.dist-info/RECORD +152 -0
  149. onetick_py-1.177.0.dist-info/WHEEL +5 -0
  150. onetick_py-1.177.0.dist-info/entry_points.txt +2 -0
  151. onetick_py-1.177.0.dist-info/licenses/LICENSE +21 -0
  152. onetick_py-1.177.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,627 @@
1
+ from typing import TYPE_CHECKING, Optional
2
+
3
+ from onetick import py as otp
4
+ from onetick.py import types as ott
5
+ from onetick.py.otq import otq
6
+
7
+ from .misc import inplace_operation
8
+
9
+ if TYPE_CHECKING:
10
+ from onetick.py.core.source import Source
11
+
12
+
13
+ @inplace_operation
14
+ def update_timestamp(
15
+ self: 'Source',
16
+ timestamp_field: str,
17
+ timestamp_psec_field: Optional[str] = None,
18
+ max_delay_of_original_timestamp=0,
19
+ max_delay_of_new_timestamp=0,
20
+ max_out_of_order_interval=0,
21
+ max_delay_handling: str = 'complain',
22
+ out_of_order_timestamp_handling: str = 'complain',
23
+ zero_timestamp_handling: Optional[str] = None,
24
+ log_sequence_violations: bool = False,
25
+ inplace=False,
26
+ ) -> Optional['Source']:
27
+ """
28
+ Assigns alternative timestamps to input ticks.
29
+
30
+ In case resulting timestamps are out of order, time series is sorted in ascending order of those timestamps.
31
+
32
+ Ticks with equal alternative timestamps are sorted in ascending order of the respective original timestamps,
33
+ and equal ones among those are further sorted by ascending values of the **OMDSEQ** field,
34
+ if such a field is present.
35
+
36
+ Note
37
+ ----
38
+
39
+ In case parameters ``max_delay_of_original_timestamp`` or ``max_delay_of_new_timestamp`` are specified
40
+ this method automatically uses :py:meth:`~onetick.py.Source.modify_query_times` method, affecting
41
+ the start or end time of the query thus possibly changing some logic of the nodes placed higher in the graph.
42
+
43
+ Also there are some limitations on using this method in the graph, e.g. this method can't be used in the
44
+ "diamond" pattern and can't be used twice in the same graph.
45
+
46
+ Parameters
47
+ ----------
48
+ timestamp_field: str
49
+ Specifies the name of the input field, which is assumed to carry alternative timestamps.
50
+ timestamp_psec_field: str
51
+ Fractional (< 1 millisecond) parts of alternative timestamps will be taken from this field.
52
+ They are assumed to be in picoseconds (0.001 of nanosecond).
53
+
54
+ If this field is specified, then even if alternative timestamps have nanosecond granularity,
55
+ only millisecond parts of them will be taken, nanosecond part will be rewritten.
56
+
57
+ Useful in case ``timestamp_field`` has lower than millisecond granularity.
58
+
59
+ max_delay_of_original_timestamp: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
60
+ Changes the start time of the query to ``original_start_time - max_delay_of_original_timestamp``
61
+ to make sure it processes all possible ticks that might have a new timestamp in the query time range.
62
+
63
+ If integer value is specified, it is assumed to be milliseconds.
64
+
65
+ max_delay_of_new_timestamp: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
66
+ Changes the end time of the query to ``original_end_time + max_delay_of_new_timestamp``
67
+ to make sure it processes all possible ticks that might have a new timestamp in the query time range.
68
+
69
+ If integer value is specified, it is assumed to be milliseconds.
70
+
71
+ max_out_of_order_interval: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
72
+ Specifies the maximum out-of-order interval for alternative timestamps.
73
+ Ticks with new timestamps out of order will be sorted in ascending order.
74
+
75
+ This is the only parameter that leads to accumulation of ticks.
76
+
77
+ If integer value is specified, it is assumed to be milliseconds.
78
+
79
+ max_delay_handling: str ('complain' or 'discard' or 'use_original_timestamp' or 'use_new_timestamp')
80
+
81
+ This parameter how to process ticks that are delayed even more than specified in
82
+ ``max_delay_of_original_timestamp`` or ``max_delay_of_new_timestamp`` parameters.
83
+
84
+ - **complain**: raise an exception
85
+
86
+ - **discard**: do not add tick to the output time series
87
+
88
+ - **use_original_timestamp**: assign original timestamp to the tick
89
+
90
+ - **use_new_timestamp**: try to assign new timestamp to the tick. If the new timestamp of previous tick
91
+ is greater than new timestamp of this tick then previous one is used.
92
+ **NOTE: A previously propagated timestamp could be from a heartbeat.**
93
+ When there is no previous propagated tick and the new timestamp falls behind the query start time,
94
+ the latter is preferred, while query end time is preferred if the new timestamp exceeds it.
95
+
96
+ This parameter is processed *before* any action from ``out_of_order_timestamp_handling`` parameter.
97
+
98
+ out_of_order_timestamp_handling: str ('complain' or 'use_previous_value' or 'use_original_timestamp')
99
+
100
+ This parameter how to process ticks that are out of order even more than specified in
101
+ ``max_out_of_order_interval`` parameter.
102
+
103
+ - **complain**: raise an exception
104
+
105
+ - **use_previous_value**: assign new timestamp from a previous propagated tick
106
+
107
+ - **use_original_timestamp**: try to use original timestamp. If maximum out-of-order interval is still
108
+ exceeded then exception will be thrown.
109
+
110
+ This parameter is processed *after* any action from ``max_delay_handling`` parameter.
111
+
112
+ zero_timestamp_handling: str ('preserve_sequence'), optional
113
+
114
+ This parameter specifies how to process ticks with zero alternative timestamps.
115
+
116
+ If value is None, actions from ``max_delay_handling``
117
+ and ``out_of_order_timestamp_handling`` are performed on it.
118
+
119
+ If value is **preserve_sequence** then new timestamp will be set to the maximum between
120
+ query start time and the new timestamp of the previous propagated tick.
121
+
122
+ log_sequence_violations: bool
123
+ If set to True, then warnings about actions from parameters ``out_of_order_timestamp_handling``,
124
+ ``max_delay_handling`` and ``zero_timestamp_handling`` are logged into the log file.
125
+ inplace: bool
126
+ The flag controls whether operation should be applied inplace or not.
127
+ If ``inplace=True``, then it returns nothing.
128
+ Otherwise method returns a new modified object.
129
+
130
+ Returns
131
+ -------
132
+ :class:`Source` or ``None``
133
+
134
+ See also
135
+ --------
136
+ **UPDATE_TIMESTAMP** OneTick event processor
137
+
138
+ Examples
139
+ --------
140
+
141
+ Data and timestamps from the database:
142
+
143
+ >>> start = otp.dt(2022, 3, 2)
144
+ >>> end = otp.dt(2022, 3, 3)
145
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
146
+ >>> otp.run(data, start=start, end=end)
147
+ Time PRICE SIZE
148
+ 0 2022-03-02 00:00:00.000 1.0 100
149
+ 1 2022-03-02 00:00:00.001 1.1 101
150
+ 2 2022-03-02 00:00:00.002 1.2 102
151
+
152
+ Adding one hour to all ticks.
153
+ Parameter ``max_delay_of_original_timestamp`` must be specified in this case:
154
+
155
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
156
+ >>> data['ORIG_TS'] = data['TIMESTAMP']
157
+ >>> data['NEW_TS'] = data['TIMESTAMP'] + otp.Hour(1)
158
+ >>> data = data.update_timestamp('NEW_TS', max_delay_of_original_timestamp=otp.Hour(1))
159
+ >>> otp.run(data, start=start, end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
160
+ Time PRICE SIZE ORIG_TS
161
+ 0 2022-03-02 01:00:00.000 1.0 100 2022-03-02 00:00:00.000
162
+ 1 2022-03-02 01:00:00.001 1.1 101 2022-03-02 00:00:00.001
163
+ 2 2022-03-02 01:00:00.002 1.2 102 2022-03-02 00:00:00.002
164
+
165
+ Subtracting one day from all ticks.
166
+ Parameter ``max_delay_of_new_timestamp`` must be specified in this case.
167
+
168
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
169
+ >>> data['ORIG_TS'] = data['TIMESTAMP']
170
+ >>> data['NEW_TS'] = data['TIMESTAMP'] - otp.Day(1)
171
+ >>> data = data.update_timestamp('NEW_TS', max_delay_of_new_timestamp=otp.Day(1))
172
+ >>> otp.run(data, start=start - otp.Day(1), end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
173
+ Time PRICE SIZE ORIG_TS
174
+ 0 2022-03-01 00:00:00.000 1.0 100 2022-03-02 00:00:00.000
175
+ 1 2022-03-01 00:00:00.001 1.1 101 2022-03-02 00:00:00.001
176
+ 2 2022-03-01 00:00:00.002 1.2 102 2022-03-02 00:00:00.002
177
+
178
+ Parameter ``max_delay_handling`` can be used to specify how to handle ticks exceeding the maximum:
179
+
180
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
181
+ >>> data['ORIG_TS'] = data['TIMESTAMP']
182
+ >>> data['NEW_TS'] = data.apply(
183
+ ... lambda row: row['TIMESTAMP'] + otp.Hour(24)
184
+ ... if row['PRICE'] == 1.1
185
+ ... else row['TIMESTAMP'] + otp.Hour(1)
186
+ ... )
187
+ >>> data = data.update_timestamp('NEW_TS',
188
+ ... max_delay_of_original_timestamp=otp.Hour(1),
189
+ ... max_delay_handling='discard')
190
+ >>> otp.run(data, start=start, end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS']]
191
+ Time PRICE SIZE ORIG_TS
192
+ 0 2022-03-02 01:00:00.000 1.0 100 2022-03-02 00:00:00.000
193
+ 1 2022-03-02 01:00:00.002 1.2 102 2022-03-02 00:00:00.002
194
+
195
+ Parameter ``max_out_of_order_interval`` can be used in case new timestamp are out of order:
196
+
197
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
198
+ >>> data['ORIG_TS'] = data['TIMESTAMP']
199
+ >>> data = data.agg({'COUNT': otp.agg.count()}, running=True, all_fields=True)
200
+ >>> data['NEW_TS'] = data['TIMESTAMP'] - otp.Minute(data['COUNT'])
201
+ >>> data = data.update_timestamp('NEW_TS',
202
+ ... max_delay_of_new_timestamp=otp.Hour(10),
203
+ ... max_out_of_order_interval=otp.Minute(100))
204
+ >>> otp.run(data, start=start - otp.Hour(2), end=end)[['Time', 'PRICE', 'SIZE', 'ORIG_TS', 'COUNT']]
205
+ Time PRICE SIZE ORIG_TS COUNT
206
+ 0 2022-03-01 23:57:00.002 1.2 102 2022-03-02 00:00:00.002 3
207
+ 1 2022-03-01 23:58:00.001 1.1 101 2022-03-02 00:00:00.001 2
208
+ 2 2022-03-01 23:59:00.000 1.0 100 2022-03-02 00:00:00.000 1
209
+ """
210
+ if timestamp_field not in self.schema:
211
+ raise ValueError(f"Field '{timestamp_field}' is not in schema")
212
+ if self.schema[timestamp_field] not in {otp.msectime, otp.nsectime}:
213
+ raise ValueError(f"Unsupported type for 'timestamp_field': {self.schema[timestamp_field]}")
214
+
215
+ if timestamp_psec_field is not None:
216
+ if timestamp_psec_field not in self.schema:
217
+ raise ValueError(f"Field '{timestamp_psec_field}' is not in schema")
218
+ if self.schema[timestamp_psec_field] is not int:
219
+ raise ValueError(f"Unsupported type for 'timestamp_psec_field': {self.schema[timestamp_psec_field]}")
220
+
221
+ if max_delay_handling not in {'complain', 'discard', 'use_original_timestamp', 'use_new_timestamp'}:
222
+ raise ValueError(f"Unsupported value for parameter 'max_delay_handling': {max_delay_handling}")
223
+
224
+ if out_of_order_timestamp_handling not in {'complain', 'use_previous_value', 'use_original_timestamp'}:
225
+ raise ValueError(
226
+ f"Unsupported value for parameter 'out_of_order_timestamp_handling': {out_of_order_timestamp_handling}"
227
+ )
228
+
229
+ if zero_timestamp_handling not in {None, 'preserve_sequence'}:
230
+ raise ValueError(f"Unsupported value for parameter 'zero_timestamp_handling': {zero_timestamp_handling}")
231
+
232
+ def get_ms(value):
233
+ if isinstance(value, int):
234
+ return value
235
+ try:
236
+ nanos = value.nanos
237
+ except Exception as e:
238
+ raise ValueError(f'Unsupported parameter value: {type(value)}') from e
239
+ if nanos < 1_000_000:
240
+ raise ValueError('Values less than one millisecond are not supported in this parameter')
241
+ return nanos // 1_000_000
242
+
243
+ self.sink(
244
+ otq.UpdateTimestamp(
245
+ timestamp_field_name=timestamp_field,
246
+ timestamp_psec_field_name=timestamp_psec_field or '',
247
+ max_delay_of_original_timestamp=get_ms(max_delay_of_original_timestamp),
248
+ max_delay_of_new_timestamp=get_ms(max_delay_of_new_timestamp),
249
+ max_out_of_order_interval_msec=get_ms(max_out_of_order_interval),
250
+ max_delay_handling=max_delay_handling,
251
+ out_of_order_timestamp_handling=out_of_order_timestamp_handling,
252
+ zero_timestamp_handling=zero_timestamp_handling,
253
+ log_sequence_violations=log_sequence_violations,
254
+ )
255
+ )
256
+ return self
257
+
258
+
259
+ @inplace_operation
260
+ def modify_query_times(
261
+ self: 'Source', start=None, end=None, output_timestamp=None, propagate_heartbeats=True, inplace=False
262
+ ) -> Optional['Source']:
263
+ """
264
+ Modify ``start`` and ``end`` time of the query.
265
+
266
+ * query times are changed for all operations
267
+ only **before** this method up to the source of the graph.
268
+ * all ticks' timestamps produced by this method
269
+ **must** fall into original time range of the query.
270
+
271
+ It is possible to change ticks' timestamps with parameter ``output_timestamp``,
272
+ so they will stay inside the original time range.
273
+
274
+ Note
275
+ ----
276
+ Due to how OneTick works internally, tick generators
277
+ :py:class:`otp.Tick <onetick.py.Tick>` and :py:func:`otp.Ticks <onetick.py.Ticks>`
278
+ are not affected by this method.
279
+
280
+ Parameters
281
+ ----------
282
+ start: :py:class:`otp.datetime <onetick.py.datetime>` or \
283
+ :py:class:`~onetick.py.core.source.MetaFields` or :py:class:`~onetick.py.Operation`
284
+ Expression to replace query start time.
285
+ By default, start time is not changed.
286
+ Note that expression in this parameter can't depend on ticks, thus only
287
+ :py:class:`~onetick.py.core.source.MetaFields` and constants can be used.
288
+ end: :py:class:`otp.datetime <onetick.py.datetime>` or \
289
+ :py:class:`~onetick.py.core.source.MetaFields` or :py:class:`~onetick.py.Operation`
290
+ Expression to replace query end time.
291
+ By default, end time is not changed.
292
+ Note that expression in this parameter can't depend on ticks, thus only
293
+ :py:class:`~onetick.py.core.source.MetaFields` and constants can be used.
294
+ output_timestamp: :py:class:`onetick.py.Operation`
295
+ Expression that produces timestamp for each tick.
296
+ By default, the following expression is used: ``orig_start + orig_timestamp - start``
297
+ This expression covers cases when start time of the query is changed and keeps
298
+ timestamp inside original time range.
299
+ Note that it doesn't cover cases, for example, if end time was increased,
300
+ you have to handle such cases yourself.
301
+ propagate_heartbeats: bool
302
+ Controls heartbeat propagation.
303
+ inplace: bool
304
+ The flag controls whether operation should be applied inplace or not.
305
+ If ``inplace=True``, then it returns nothing.
306
+ Otherwise method returns a new modified object.
307
+
308
+ See also
309
+ --------
310
+ | **MODIFY_QUERY_TIMES** OneTick event processor
311
+ | :py:meth:`onetick.py.Source.time_interval_shift`
312
+
313
+ Returns
314
+ -------
315
+ :class:`Source` or ``None``
316
+
317
+ Examples
318
+ --------
319
+
320
+ >>> start = otp.dt(2022, 3, 2)
321
+ >>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
322
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
323
+
324
+ By default, method does nothing:
325
+
326
+ >>> t = data.modify_query_times()
327
+ >>> otp.run(t, start=start, end=end)
328
+ Time PRICE SIZE
329
+ 0 2022-03-02 00:00:00.000 1.0 100
330
+ 1 2022-03-02 00:00:00.001 1.1 101
331
+ 2 2022-03-02 00:00:00.002 1.2 102
332
+
333
+ See how ``_START_TIME`` and ``_END_TIME`` meta fields are changed.
334
+ They are changed *before* ``modify_query_times``:
335
+
336
+ >>> t = data.copy()
337
+ >>> t['S_BEFORE'] = t['_START_TIME']
338
+ >>> t['E_BEFORE'] = t['_END_TIME']
339
+ >>> t = t.modify_query_times(start=t['_START_TIME'] + otp.Milli(1),
340
+ ... end=t['_END_TIME'] - otp.Milli(1))
341
+ >>> t['S_AFTER'] = t['_START_TIME']
342
+ >>> t['E_AFTER'] = t['_END_TIME']
343
+ >>> otp.run(t, start=start, end=end)
344
+ Time PRICE SIZE S_BEFORE E_BEFORE S_AFTER E_AFTER
345
+ 0 2022-03-02 1.1 101 2022-03-02 00:00:00.001 2022-03-02 00:00:00.002 2022-03-02 2022-03-02 00:00:00.003
346
+
347
+ You can decrease time interval without problems:
348
+
349
+ >>> t = data.modify_query_times(start=data['_START_TIME'] + otp.Milli(1),
350
+ ... end=data['_END_TIME'] - otp.Milli(1))
351
+ >>> otp.run(t, start=start, end=end)
352
+ Time PRICE SIZE
353
+ 0 2022-03-02 1.1 101
354
+
355
+ Note that the timestamp of the tick was changed with default expression.
356
+ In this case we can output original timestamps,
357
+ because they fall into original time range:
358
+
359
+ >>> t = data.modify_query_times(start=data['_START_TIME'] + otp.Milli(1),
360
+ ... end=data['_END_TIME'] - otp.Milli(1),
361
+ ... output_timestamp=data['TIMESTAMP'])
362
+ >>> otp.run(t, start=start, end=end)
363
+ Time PRICE SIZE
364
+ 0 2022-03-02 00:00:00.001 1.1 101
365
+
366
+ But it will not work if new time range is wider than original:
367
+
368
+ >>> t = data.modify_query_times(start=data['_START_TIME'] - otp.Milli(1),
369
+ ... output_timestamp=data['TIMESTAMP'])
370
+ >>> otp.run(t, start=start + otp.Milli(1), end=end + otp.Milli(1)) # doctest: +ELLIPSIS
371
+ Traceback (most recent call last):
372
+ Exception...timestamp is falling out of initial start/end time range...
373
+
374
+ In this case default ``output_timestamp`` expression would work just fine:
375
+
376
+ >>> t = data.modify_query_times(start=data['_START_TIME'] - otp.Milli(1))
377
+ >>> otp.run(t, start=start + otp.Milli(1), end=end + otp.Milli(1))
378
+ Time PRICE SIZE
379
+ 0 2022-03-02 00:00:00.001 1.0 100
380
+ 1 2022-03-02 00:00:00.002 1.1 101
381
+ 2 2022-03-02 00:00:00.003 1.2 102
382
+
383
+ But it doesn't work, for example, if end time has crossed the borders of original time range.
384
+ In this case other ``output_timestamp`` expression must be specified:
385
+
386
+ >>> t = data.modify_query_times(
387
+ ... start=data['_START_TIME'] - otp.Milli(2),
388
+ ... output_timestamp=otp.math.min(data['TIMESTAMP'] + otp.Milli(2), data['_END_TIME'])
389
+ ... )
390
+ >>> otp.run(t, start=start + otp.Milli(2), end=end)
391
+ Time PRICE SIZE
392
+ 0 2022-03-02 00:00:00.002 1.0 100
393
+ 1 2022-03-02 00:00:00.003 1.1 101
394
+ 2 2022-03-02 00:00:00.003 1.2 102
395
+
396
+ Remember that ``start`` and ``end`` parameters can't depend on ticks:
397
+
398
+ >>> t = data.copy()
399
+ >>> t['X'] = 12345
400
+ >>> t = t.modify_query_times(start=t['_START_TIME'] + t['X'] - t['X'],
401
+ ... end=t['_END_TIME'] - otp.Milli(1))
402
+ >>> otp.run(t, start=start, end=end) # doctest: +ELLIPSIS
403
+ Traceback (most recent call last):
404
+ Exception...parameter must not depend on ticks...
405
+
406
+ Constant datetime values can be used as parameters too:
407
+
408
+ >>> t = data.modify_query_times(start=start + otp.Milli(1),
409
+ ... end=end - otp.Milli(1))
410
+ >>> otp.run(t, start=start, end=end)
411
+ Time PRICE SIZE
412
+ 0 2022-03-02 1.1 101
413
+
414
+ Note that some graph patterns are not allowed when using this method.
415
+ For example, modifying query times for a branch that will be merged later:
416
+
417
+ >>> t1, t2 = data[data['PRICE'] > 1.3]
418
+ >>> t2 = t2.modify_query_times(start=start + otp.Milli(1))
419
+ >>> t = otp.merge([t1, t2])
420
+ >>> otp.run(t, start=start, end=end) # doctest: +ELLIPSIS
421
+ Traceback (most recent call last):
422
+ Exception...Invalid graph...time bound to a node...an intermediate node in one of the cycles in graph...
423
+ """
424
+ start = ott.value2str(start) if start is not None else ''
425
+ end = ott.value2str(end) if end is not None else ''
426
+ output_timestamp = ott.value2str(output_timestamp) if output_timestamp is not None else ''
427
+ self.sink(
428
+ otq.ModifyQueryTimes(
429
+ start_time=start,
430
+ end_time=end,
431
+ output_timestamp=output_timestamp,
432
+ propagate_heartbeats=propagate_heartbeats,
433
+ )
434
+ )
435
+ return self
436
+
437
+
438
+ def time_interval_shift(self: 'Source', shift, inplace=False) -> Optional['Source']:
439
+ """
440
+ Shifting time interval for a source.
441
+
442
+ The whole data flow is shifted all the way up to the source of the graph.
443
+
444
+ The start and end times of the query will be changed for all operations before this method,
445
+ and will stay the same after this method.
446
+
447
+ WARNING: The ticks' timestamps *are changed* automatically so they fit into original time range.
448
+
449
+ You will get different set of ticks from the database, but the timestamps of the ticks
450
+ from that database will not be the same as in the database.
451
+
452
+ They need to be changed so they fit into the original query time range.
453
+ See details in :py:meth:`onetick.py.Source.modify_query_times`.
454
+
455
+ Parameters
456
+ ----------
457
+ shift: int or :ref:`datetime offset<api/datetime/offsets/root:datetime offsets>`
458
+ Offset to shift the whole time interval.
459
+ Can be positive or negative.
460
+ Positive value moves time interval into the future, negative -- to the past.
461
+ int values are interpreted as milliseconds.
462
+
463
+ Timestamps of the ticks will be changed so they fit into the original query time range
464
+ by subtracting ``shift`` from each timestamp.
465
+ inplace: bool
466
+ The flag controls whether operation should be applied inplace or not.
467
+ If ``inplace=True``, then it returns nothing.
468
+ Otherwise method returns a new modified object.
469
+
470
+ Returns
471
+ -------
472
+ :class:`Source` or ``None``
473
+
474
+ See also
475
+ --------
476
+ | :py:meth:`onetick.py.Source.modify_query_times`
477
+ | :py:meth:`onetick.py.Source.time_interval_change`
478
+
479
+ Examples
480
+ --------
481
+
482
+ --> Also see use-case using :py:meth:`time_interval_shift` for calculating
483
+ `Markouts <../../static/getting_started/use_cases.html#point-in-time-benchmarks-bbo-at-different-markouts>`_
484
+
485
+
486
+ >>> start = otp.dt(2022, 3, 2)
487
+ >>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
488
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
489
+
490
+ Default data:
491
+
492
+ >>> otp.run(data, start=start, end=end)
493
+ Time PRICE SIZE
494
+ 0 2022-03-02 00:00:00.000 1.0 100
495
+ 1 2022-03-02 00:00:00.001 1.1 101
496
+ 2 2022-03-02 00:00:00.002 1.2 102
497
+
498
+ Get window for a third tick:
499
+
500
+ >>> otp.run(data, start=start + otp.Milli(2), end=start + otp.Milli(3))
501
+ Time PRICE SIZE
502
+ 0 2022-03-02 00:00:00.002 1.2 102
503
+
504
+ Shifting time window will result in different set of ticks,
505
+ but the ticks will have their timestamps changed to fit into original time range.
506
+ Let's shift time 2 milliseconds back and thus get the first tick:
507
+
508
+ >>> t = data.time_interval_shift(shift=-otp.Milli(2))
509
+ >>> otp.run(t, start=start + otp.Milli(2), end=start + otp.Milli(3))
510
+ Time PRICE SIZE
511
+ 0 2022-03-02 00:00:00.002 1.0 100
512
+
513
+ Here we are querying empty time interval, but shifting one second back to get ticks.
514
+
515
+ >>> t = data.time_interval_shift(shift=-otp.Second(1))
516
+ >>> otp.run(t, start=start + otp.Second(1), end=end + otp.Second(1))
517
+ Time PRICE SIZE
518
+ 0 2022-03-02 00:00:01.000 1.0 100
519
+ 1 2022-03-02 00:00:01.001 1.1 101
520
+ 2 2022-03-02 00:00:01.002 1.2 102
521
+
522
+ Note that tick generators
523
+ :py:class:`otp.Tick <onetick.py.Tick>` and :py:func:`otp.Ticks <onetick.py.Ticks>`
524
+ are not really affected by this method, they will have the same timestamps:
525
+
526
+ >>> t = otp.Tick(A=1)
527
+ >>> otp.run(t)
528
+ Time A
529
+ 0 2003-12-01 1
530
+
531
+ >>> t = t.time_interval_shift(shift=otp.Second(1))
532
+ >>> otp.run(t)
533
+ Time A
534
+ 0 2003-12-01 1
535
+ """
536
+ start = self['_START_TIME'] + shift
537
+ end = self['_END_TIME'] + shift
538
+ # change timestamps so they fit into original time range
539
+ output_timestamp = self['TIMESTAMP'] - shift
540
+ return self.modify_query_times(start=start, end=end, output_timestamp=output_timestamp, inplace=inplace)
541
+
542
+
543
+ def time_interval_change(self: 'Source', start_change=0, end_change=0, inplace=False) -> Optional['Source']:
544
+ """
545
+ Changing time interval by making it bigger or smaller.
546
+
547
+ All timestamps of ticks that are crossing the border of original time range
548
+ will be set to original start time or end time depending on their original time.
549
+
550
+ Parameters
551
+ ----------
552
+ start_change: int or :ref:`datetime offset <api/datetime/offsets/root:datetime offsets>`
553
+ Offset to shift start time.
554
+ Can be positive or negative.
555
+ Positive value moves start time into the future, negative -- to the past.
556
+ int values are interpreted as milliseconds.
557
+ end_change: int or :ref:`datetime offset <api/datetime/offsets/root:datetime offsets>`
558
+ Offset to shift end time.
559
+ Can be positive or negative.
560
+ Positive value moves end time into the future, negative -- to the past.
561
+ int values are interpreted as milliseconds.
562
+ inplace: bool
563
+ The flag controls whether operation should be applied inplace or not.
564
+ If ``inplace=True``, then it returns nothing.
565
+ Otherwise method returns a new modified object.
566
+
567
+ Returns
568
+ -------
569
+ :class:`Source` or ``None``
570
+
571
+ See also
572
+ --------
573
+ | :py:meth:`onetick.py.Source.modify_query_times`
574
+ | :py:meth:`onetick.py.Source.time_interval_shift`
575
+
576
+ Examples
577
+ --------
578
+
579
+ >>> start = otp.dt(2022, 3, 2)
580
+ >>> end = otp.dt(2022, 3, 2) + otp.Milli(3)
581
+ >>> data = otp.DataSource('US_COMP', symbols='AAPL', tick_type='TRD')
582
+
583
+ By default, ``time_interval_change()`` does nothing:
584
+
585
+ >>> t = data.time_interval_change()
586
+ >>> otp.run(t, start=start, end=end)
587
+ Time PRICE SIZE
588
+ 0 2022-03-02 00:00:00.000 1.0 100
589
+ 1 2022-03-02 00:00:00.001 1.1 101
590
+ 2 2022-03-02 00:00:00.002 1.2 102
591
+
592
+ Decreasing time range will not change ticks' timestamps:
593
+
594
+ >>> t = data.time_interval_change(start_change=otp.Milli(1), end_change=-otp.Milli(1))
595
+ >>> otp.run(t, start=start, end=end)
596
+ Time PRICE SIZE
597
+ 0 2022-03-02 00:00:00.001 1.1 101
598
+
599
+ Increasing time range will change timestamps of the ticks that crossed the border.
600
+ In this case first tick's timestamp will be set to original start time,
601
+ and third tick's to original end time.
602
+
603
+ >>> t = data.time_interval_change(start_change=-otp.Milli(1), end_change=otp.Milli(1))
604
+ >>> otp.run(t, start=start + otp.Milli(1), end=start + otp.Milli(2))
605
+ Time PRICE SIZE
606
+ 0 2022-03-02 00:00:00.001 1.0 100
607
+ 1 2022-03-02 00:00:00.001 1.1 101
608
+ 2 2022-03-02 00:00:00.002 1.2 102
609
+
610
+ Here we are querying empty time interval, but changing start time one second back to get ticks.
611
+
612
+ >>> t = data.time_interval_change(start_change=-otp.Second(1))
613
+ >>> otp.run(t, start=start + otp.Second(1), end=end + otp.Second(1))
614
+ Time PRICE SIZE
615
+ 0 2022-03-02 00:00:01 1.0 100
616
+ 1 2022-03-02 00:00:01 1.1 101
617
+ 2 2022-03-02 00:00:01 1.2 102
618
+ """
619
+ start = self['_START_TIME'] + start_change
620
+ end = self['_END_TIME'] + end_change
621
+
622
+ # change ticks' timestamps only if they are out of bounds
623
+ output_timestamp = self['TIMESTAMP']
624
+ output_timestamp = otp.math.min(output_timestamp, self['_END_TIME'])
625
+ output_timestamp = otp.math.max(output_timestamp, self['_START_TIME'])
626
+
627
+ return self.modify_query_times(start=start, end=end, output_timestamp=output_timestamp, inplace=inplace)