onetick-py 1.162.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. locator_parser/__init__.py +0 -0
  2. locator_parser/acl.py +73 -0
  3. locator_parser/actions.py +266 -0
  4. locator_parser/common.py +365 -0
  5. locator_parser/io.py +41 -0
  6. locator_parser/locator.py +150 -0
  7. onetick/__init__.py +101 -0
  8. onetick/doc_utilities/__init__.py +3 -0
  9. onetick/doc_utilities/napoleon.py +40 -0
  10. onetick/doc_utilities/ot_doctest.py +140 -0
  11. onetick/doc_utilities/snippets.py +280 -0
  12. onetick/lib/__init__.py +4 -0
  13. onetick/lib/instance.py +138 -0
  14. onetick/py/__init__.py +290 -0
  15. onetick/py/_stack_info.py +89 -0
  16. onetick/py/_version.py +2 -0
  17. onetick/py/aggregations/__init__.py +11 -0
  18. onetick/py/aggregations/_base.py +645 -0
  19. onetick/py/aggregations/_docs.py +912 -0
  20. onetick/py/aggregations/compute.py +286 -0
  21. onetick/py/aggregations/functions.py +2216 -0
  22. onetick/py/aggregations/generic.py +104 -0
  23. onetick/py/aggregations/high_low.py +80 -0
  24. onetick/py/aggregations/num_distinct.py +83 -0
  25. onetick/py/aggregations/order_book.py +427 -0
  26. onetick/py/aggregations/other.py +1014 -0
  27. onetick/py/backports.py +26 -0
  28. onetick/py/cache.py +373 -0
  29. onetick/py/callback/__init__.py +5 -0
  30. onetick/py/callback/callback.py +275 -0
  31. onetick/py/callback/callbacks.py +131 -0
  32. onetick/py/compatibility.py +752 -0
  33. onetick/py/configuration.py +736 -0
  34. onetick/py/core/__init__.py +0 -0
  35. onetick/py/core/_csv_inspector.py +93 -0
  36. onetick/py/core/_internal/__init__.py +0 -0
  37. onetick/py/core/_internal/_manually_bound_value.py +6 -0
  38. onetick/py/core/_internal/_nodes_history.py +250 -0
  39. onetick/py/core/_internal/_op_utils/__init__.py +0 -0
  40. onetick/py/core/_internal/_op_utils/every_operand.py +9 -0
  41. onetick/py/core/_internal/_op_utils/is_const.py +10 -0
  42. onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script +121 -0
  43. onetick/py/core/_internal/_proxy_node.py +140 -0
  44. onetick/py/core/_internal/_state_objects.py +2307 -0
  45. onetick/py/core/_internal/_state_vars.py +87 -0
  46. onetick/py/core/_source/__init__.py +0 -0
  47. onetick/py/core/_source/_symbol_param.py +95 -0
  48. onetick/py/core/_source/schema.py +97 -0
  49. onetick/py/core/_source/source_methods/__init__.py +0 -0
  50. onetick/py/core/_source/source_methods/aggregations.py +810 -0
  51. onetick/py/core/_source/source_methods/applyers.py +296 -0
  52. onetick/py/core/_source/source_methods/columns.py +141 -0
  53. onetick/py/core/_source/source_methods/data_quality.py +301 -0
  54. onetick/py/core/_source/source_methods/debugs.py +270 -0
  55. onetick/py/core/_source/source_methods/drops.py +120 -0
  56. onetick/py/core/_source/source_methods/fields.py +619 -0
  57. onetick/py/core/_source/source_methods/filters.py +1001 -0
  58. onetick/py/core/_source/source_methods/joins.py +1393 -0
  59. onetick/py/core/_source/source_methods/merges.py +566 -0
  60. onetick/py/core/_source/source_methods/misc.py +1325 -0
  61. onetick/py/core/_source/source_methods/pandases.py +155 -0
  62. onetick/py/core/_source/source_methods/renames.py +356 -0
  63. onetick/py/core/_source/source_methods/sorts.py +183 -0
  64. onetick/py/core/_source/source_methods/switches.py +142 -0
  65. onetick/py/core/_source/source_methods/symbols.py +117 -0
  66. onetick/py/core/_source/source_methods/times.py +627 -0
  67. onetick/py/core/_source/source_methods/writes.py +702 -0
  68. onetick/py/core/_source/symbol.py +202 -0
  69. onetick/py/core/_source/tmp_otq.py +222 -0
  70. onetick/py/core/column.py +209 -0
  71. onetick/py/core/column_operations/__init__.py +0 -0
  72. onetick/py/core/column_operations/_methods/__init__.py +4 -0
  73. onetick/py/core/column_operations/_methods/_internal.py +28 -0
  74. onetick/py/core/column_operations/_methods/conversions.py +215 -0
  75. onetick/py/core/column_operations/_methods/methods.py +294 -0
  76. onetick/py/core/column_operations/_methods/op_types.py +150 -0
  77. onetick/py/core/column_operations/accessors/__init__.py +0 -0
  78. onetick/py/core/column_operations/accessors/_accessor.py +30 -0
  79. onetick/py/core/column_operations/accessors/decimal_accessor.py +92 -0
  80. onetick/py/core/column_operations/accessors/dt_accessor.py +464 -0
  81. onetick/py/core/column_operations/accessors/float_accessor.py +160 -0
  82. onetick/py/core/column_operations/accessors/str_accessor.py +1374 -0
  83. onetick/py/core/column_operations/base.py +1061 -0
  84. onetick/py/core/cut_builder.py +149 -0
  85. onetick/py/core/db_constants.py +20 -0
  86. onetick/py/core/eval_query.py +244 -0
  87. onetick/py/core/lambda_object.py +442 -0
  88. onetick/py/core/multi_output_source.py +193 -0
  89. onetick/py/core/per_tick_script.py +2253 -0
  90. onetick/py/core/query_inspector.py +465 -0
  91. onetick/py/core/source.py +1663 -0
  92. onetick/py/db/__init__.py +2 -0
  93. onetick/py/db/_inspection.py +1042 -0
  94. onetick/py/db/db.py +1423 -0
  95. onetick/py/db/utils.py +64 -0
  96. onetick/py/docs/__init__.py +0 -0
  97. onetick/py/docs/docstring_parser.py +112 -0
  98. onetick/py/docs/utils.py +81 -0
  99. onetick/py/functions.py +2354 -0
  100. onetick/py/license.py +188 -0
  101. onetick/py/log.py +88 -0
  102. onetick/py/math.py +947 -0
  103. onetick/py/misc.py +437 -0
  104. onetick/py/oqd/__init__.py +22 -0
  105. onetick/py/oqd/eps.py +1195 -0
  106. onetick/py/oqd/sources.py +325 -0
  107. onetick/py/otq.py +211 -0
  108. onetick/py/pyomd_mock.py +47 -0
  109. onetick/py/run.py +841 -0
  110. onetick/py/servers.py +173 -0
  111. onetick/py/session.py +1342 -0
  112. onetick/py/sources/__init__.py +19 -0
  113. onetick/py/sources/cache.py +167 -0
  114. onetick/py/sources/common.py +126 -0
  115. onetick/py/sources/csv.py +642 -0
  116. onetick/py/sources/custom.py +85 -0
  117. onetick/py/sources/data_file.py +305 -0
  118. onetick/py/sources/data_source.py +1049 -0
  119. onetick/py/sources/empty.py +94 -0
  120. onetick/py/sources/odbc.py +337 -0
  121. onetick/py/sources/order_book.py +238 -0
  122. onetick/py/sources/parquet.py +168 -0
  123. onetick/py/sources/pit.py +191 -0
  124. onetick/py/sources/query.py +495 -0
  125. onetick/py/sources/snapshots.py +419 -0
  126. onetick/py/sources/split_query_output_by_symbol.py +198 -0
  127. onetick/py/sources/symbology_mapping.py +123 -0
  128. onetick/py/sources/symbols.py +357 -0
  129. onetick/py/sources/ticks.py +825 -0
  130. onetick/py/sql.py +70 -0
  131. onetick/py/state.py +256 -0
  132. onetick/py/types.py +2056 -0
  133. onetick/py/utils/__init__.py +70 -0
  134. onetick/py/utils/acl.py +93 -0
  135. onetick/py/utils/config.py +186 -0
  136. onetick/py/utils/default.py +49 -0
  137. onetick/py/utils/file.py +38 -0
  138. onetick/py/utils/helpers.py +76 -0
  139. onetick/py/utils/locator.py +94 -0
  140. onetick/py/utils/perf.py +499 -0
  141. onetick/py/utils/query.py +49 -0
  142. onetick/py/utils/render.py +1139 -0
  143. onetick/py/utils/script.py +244 -0
  144. onetick/py/utils/temp.py +471 -0
  145. onetick/py/utils/types.py +118 -0
  146. onetick/py/utils/tz.py +82 -0
  147. onetick_py-1.162.2.dist-info/METADATA +148 -0
  148. onetick_py-1.162.2.dist-info/RECORD +152 -0
  149. onetick_py-1.162.2.dist-info/WHEEL +5 -0
  150. onetick_py-1.162.2.dist-info/entry_points.txt +2 -0
  151. onetick_py-1.162.2.dist-info/licenses/LICENSE +21 -0
  152. onetick_py-1.162.2.dist-info/top_level.txt +2 -0
@@ -0,0 +1,1325 @@
1
+ import functools
2
+ import re
3
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
4
+ from onetick.py.backports import Literal
5
+
6
+ import onetick.py as otp
7
+ from onetick.py import types as ott
8
+ from onetick.py import utils
9
+ from onetick.py.core.column import _Column
10
+ from onetick.py.otq import otq
11
+ from onetick.py.compatibility import (
12
+ is_ob_virtual_prl_and_show_full_detail_supported,
13
+ is_per_tick_script_boolean_problem,
14
+ )
15
+ from onetick.py.aggregations._docs import copy_signature
16
+
17
+ if TYPE_CHECKING:
18
+ from onetick.py.core.source import Source
19
+
20
+
21
+ def inplace_operation(method):
22
+ """Decorator that adds the `inplace` parameter and logic according to this
23
+ flag. inplace=True means that method modifies an object, otherwise it copies
24
+ the object firstly, modifies copy and returns the copy.
25
+ """
26
+
27
+ @functools.wraps(method)
28
+ def _inner(self, *args, inplace=False, **kwargs):
29
+ kwargs['inplace'] = inplace
30
+ if inplace:
31
+ method(self, *args, **kwargs)
32
+ else:
33
+ obj = self.copy()
34
+ return method(obj, *args, **kwargs)
35
+
36
+ return _inner
37
+
38
+
39
+ def _columns_names_regex(
40
+ self: 'Source', objs: Tuple[Union[_Column, str]], drop: bool = False
41
+ ) -> Tuple[List[str], bool]:
42
+ """
43
+ We can't be sure python Source has consistent columns cache, because sinking complex event processors
44
+ can change columns unpredictable, so if user will specify regex as a param, we will pass regex
45
+ as an onetick's param, but pass or delete all matched columns from python Source cache.
46
+
47
+ Parameters
48
+ ----------
49
+ objs:
50
+ Tuple of _Columns or string to pass or drop. String can be regex.
51
+ drop: bool
52
+ To drop columns from python schema or not.
53
+
54
+ Returns
55
+ -------
56
+ items_to_passthrough:
57
+ Items to pass to Passthrough as `field` parameter.
58
+ regex:
59
+ Value to pass to Passthrough as `use_regex` parameter.
60
+ """
61
+ def is_regex_string(obj):
62
+ return isinstance(obj, str) and any(c in r"*+?\:[]{}()^$" for c in obj)
63
+
64
+ # if any object from the list is a regexp, then all other objects will be treated like regexps too
65
+ regex = any(is_regex_string(obj) for obj in objs)
66
+
67
+ items_to_passthrough = []
68
+ names_of_columns = []
69
+ for obj in objs:
70
+ if not isinstance(obj, (str, _Column)):
71
+ raise TypeError(f"It is not supported to select or delete item '{obj}' of type '{type(obj)}'")
72
+ if regex:
73
+ # if column object or non-regex string is specified in regex mode,
74
+ # then we assume that the user requested that exact column to be dropped
75
+ if isinstance(obj, _Column):
76
+ names_of_columns.append(obj.name)
77
+ obj = f'^{obj.name}$'
78
+ elif not is_regex_string(obj):
79
+ names_of_columns.append(obj)
80
+ obj = f'^{obj}$'
81
+ else:
82
+ names_of_columns.extend(col for col in self.columns() if re.search(obj, col))
83
+ items_to_passthrough.append(obj)
84
+ else:
85
+ name = obj.name if isinstance(obj, _Column) else obj
86
+ items_to_passthrough.append(name)
87
+ names_of_columns.append(name)
88
+
89
+ # remove duplications and meta_fields
90
+ names_of_columns: set[str] = set(names_of_columns) - set(self.__class__.meta_fields) # type: ignore[no-redef]
91
+ # TODO: we definitely have the same logic of checking columns somewhere else too, need to refactor
92
+ for item in names_of_columns:
93
+ if item not in self.__dict__ or not isinstance(self.__dict__[item], _Column):
94
+ raise AttributeError(f"There is no '{item}' column")
95
+ if drop:
96
+ for item in names_of_columns:
97
+ del self.__dict__[item]
98
+ return items_to_passthrough, regex
99
+
100
+
101
+ @inplace_operation
102
+ def pause(self: 'Source', delay, busy_waiting=False, where=None, inplace=False) -> Optional['Source']:
103
+ """
104
+ Pauses processing of each tick for number of milliseconds specified via ``delay`` expression.
105
+
106
+ Parameters
107
+ ----------
108
+ delay: int or :py:class:`onetick.py.Operation`
109
+ Integer number or OneTick expression used to calculate the delay.
110
+ Delay is in milliseconds.
111
+ Note that number can't be negative.
112
+ busy_waiting: bool
113
+ If True then delay is done via busy loop (consuming CPU time).
114
+ where: :py:class:`onetick.py.Operation`
115
+ Expression to select ticks for which processing will be paused.
116
+ By default, all ticks are selected.
117
+ inplace: bool
118
+ The flag controls whether operation should be applied inplace or not.
119
+ If ``inplace=True``, then it returns nothing.
120
+ Otherwise method returns a new modified object.
121
+
122
+ See also
123
+ --------
124
+ **PAUSE** OneTick event processor
125
+
126
+ Returns
127
+ -------
128
+ :class:`Source` or ``None``
129
+
130
+ Examples
131
+ --------
132
+
133
+ Pause every tick for 100 milliseconds (0.1 second).
134
+
135
+ >>> data = otp.Tick(A=1)
136
+ >>> data = data.pause(100)
137
+
138
+ Set the ``delay`` with expression and use ``where`` parameter to pause conditionally:
139
+
140
+ >>> data = otp.Ticks(A=[-1, 1, 2])
141
+ >>> data = data.pause(data['A'] * 100, where=data['A'] > 0)
142
+ """
143
+ if not isinstance(delay, int):
144
+ delay = str(delay)
145
+ elif delay < 0:
146
+ raise ValueError("Parameter 'delay' can't be negative")
147
+ where = '' if where is None else str(where)
148
+ self.sink(
149
+ otq.Pause(
150
+ delay=delay,
151
+ busy_waiting=busy_waiting,
152
+ where=where,
153
+ )
154
+ )
155
+ return self
156
+
157
+
158
+ @inplace_operation
159
+ def insert_tick(
160
+ self: 'Source',
161
+ fields=None,
162
+ where=None,
163
+ preserve_input_ticks=True,
164
+ num_ticks_to_insert=1,
165
+ insert_before=True,
166
+ inplace=False,
167
+ ) -> Optional['Source']:
168
+ """
169
+ Insert tick.
170
+
171
+ Parameters
172
+ ----------
173
+ fields: dict of str to :py:class:`onetick.py.Operation`
174
+ Mapping of field names to some expressions or values.
175
+ These fields in inserted ticks will be set to corresponding values or results of expressions.
176
+ If field is presented in input tick, but not set in ``fields`` dict,
177
+ then the value of the field will be copied from input tick to inserted tick.
178
+ If parameter ``fields`` is not set at all, then values for inserted ticks' fields
179
+ will be default values for fields' types from input ticks (0 for integers etc.).
180
+ where: :py:class:`onetick.py.Operation`
181
+ Expression to select ticks near which the new ticks will be inserted.
182
+ By default, all ticks are selected.
183
+ preserve_input_ticks: bool
184
+ A switch controlling whether input ticks have to be preserved in output time series or not.
185
+ While the former case results in fields of input ticks to be present in the output time series
186
+ together with those defined by the ``fields`` parameter,
187
+ the latter case results in only defined fields to be present.
188
+ If a field of the input time series is defined in the ``fields`` parameter,
189
+ the defined value takes precedence.
190
+ num_ticks_to_insert: int
191
+ Number of ticks to insert.
192
+ insert_before: bool
193
+ Insert tick before each input tick or after.
194
+ inplace: bool
195
+ The flag controls whether operation should be applied inplace or not.
196
+ If ``inplace=True``, then it returns nothing.
197
+ Otherwise method returns a new modified object.
198
+
199
+ See also
200
+ --------
201
+ **INSERT_TICK** OneTick event processor
202
+
203
+ Returns
204
+ -------
205
+ :class:`Source` or ``None``
206
+
207
+ Examples
208
+ --------
209
+
210
+ Insert tick before each tick with default type values.
211
+
212
+ >>> data = otp.Tick(A=1)
213
+ >>> data = data.insert_tick()
214
+ >>> otp.run(data)
215
+ Time A
216
+ 0 2003-12-01 0
217
+ 1 2003-12-01 1
218
+
219
+ Insert tick before each tick with field `A` copied from input tick
220
+ and field `B` set to specified value.
221
+
222
+ >>> data = otp.Tick(A=1)
223
+ >>> data = data.insert_tick(fields={'B': 'b'})
224
+ >>> otp.run(data)
225
+ Time B A
226
+ 0 2003-12-01 b 1
227
+ 1 2003-12-01 1
228
+
229
+ Insert two ticks only after first tick.
230
+
231
+ >>> data = otp.Ticks(A=[1, 2, 3])
232
+ >>> data = data.insert_tick(where=data['A'] == 1,
233
+ ... insert_before=False,
234
+ ... num_ticks_to_insert=2)
235
+ >>> otp.run(data)
236
+ Time A
237
+ 0 2003-12-01 00:00:00.000 1
238
+ 1 2003-12-01 00:00:00.000 0
239
+ 2 2003-12-01 00:00:00.000 0
240
+ 3 2003-12-01 00:00:00.001 2
241
+ 4 2003-12-01 00:00:00.002 3
242
+ """
243
+ if not isinstance(num_ticks_to_insert, int) or num_ticks_to_insert <= 0:
244
+ raise ValueError("Parameter 'num_ticks_to_insert' must be a positive integer")
245
+ if not preserve_input_ticks and not fields:
246
+ raise ValueError("Parameter 'fields' must be set if 'preserve_input_ticks' is False")
247
+ where = '' if where is None else str(where)
248
+
249
+ fields = fields or {}
250
+ update_schema = {}
251
+ for field, value in fields.items():
252
+ dtype = ott.get_object_type(value)
253
+ if field not in self.schema:
254
+ update_schema[field] = dtype
255
+ elif dtype is not self.schema[field]:
256
+ raise ValueError(f"Incompatible types for field '{field}': {self.schema[field]} --> {dtype}")
257
+ dtype = ott.type2str(dtype)
258
+ if isinstance(value, type):
259
+ value = ott.default_by_type(value)
260
+ value = ott.value2str(value)
261
+ fields[field] = (dtype, value)
262
+ fields = ','.join(
263
+ f'{field} {dtype}={value}' if value else f'{field} {dtype}' for field, (dtype, value) in fields.items()
264
+ )
265
+
266
+ self.sink(
267
+ otq.InsertTick(
268
+ fields=fields,
269
+ where=where,
270
+ preserve_input_ticks=preserve_input_ticks,
271
+ num_ticks_to_insert=num_ticks_to_insert,
272
+ insert_before=insert_before,
273
+ )
274
+ )
275
+ if preserve_input_ticks:
276
+ self.table(inplace=True, strict=False, **update_schema)
277
+ else:
278
+ self.table(inplace=True, strict=True, **update_schema)
279
+ return self
280
+
281
+
282
+ @inplace_operation
283
+ def transpose(
284
+ self: 'Source',
285
+ inplace: bool = False,
286
+ direction: Literal['rows', 'columns'] = 'rows',
287
+ n: Optional[int] = None,
288
+ ) -> Optional['Source']:
289
+ """
290
+ Data transposing.
291
+ The main idea is joining many ticks into one or splitting one tick to many.
292
+
293
+ Parameters
294
+ ----------
295
+ inplace: bool, default=False
296
+ if `True` method will modify current object,
297
+ otherwise it will return modified copy of the object.
298
+ direction: 'rows', 'columns', default='rows'
299
+ - `rows`: join certain input ticks (depending on other parameters) with preceding ones.
300
+ Fields of each tick will be added to the output tick and their names will be suffixed
301
+ with **_K** where **K** is the positional number of tick (starting from 1) in reverse order.
302
+ So fields of current tick will be suffixed with **_1**, fields of previous tick will be
303
+ suffixed with **_2** and so on.
304
+ - `columns`: the operation is opposite to `rows`. It splits each input tick to several
305
+ output ticks. Each input tick must have fields with names suffixed with **_K**
306
+ where **K** is the positional number of tick (starting from 1) in reverse order.
307
+ n: Optional[int], default=None
308
+ must be specified only if ``direction`` is 'rows'.
309
+ Joins every **n** number of ticks with **n-1** preceding ticks.
310
+
311
+ Returns
312
+ -------
313
+ If ``inplace`` parameter is `True` method will return `None`,
314
+ otherwise it will return modified copy of the object.
315
+
316
+ See also
317
+ --------
318
+ **TRANSPOSE** OneTick event processor
319
+
320
+ Examples
321
+ --------
322
+ Merging two ticks into one.
323
+
324
+ >>> data = otp.Ticks(dict(A=[1, 2],
325
+ ... B=[3, 4]))
326
+ >>> data = data.transpose(direction='rows', n=2) # OTdirective: skip-snippet:;
327
+ >>> otp.run(data)
328
+ Time TIMESTAMP_1 A_1 B_1 TIMESTAMP_2 A_2 B_2
329
+ 0 2003-12-01 00:00:00.001 2003-12-01 00:00:00.001 2 4 2003-12-01 1 3
330
+
331
+ And splitting them back into two.
332
+
333
+ >>> data = data.transpose(direction='columns') # OTdirective: skip-snippet:;
334
+ >>> otp.run(data)
335
+ Time A B
336
+ 0 2003-12-01 00:00:00.000 1 3
337
+ 1 2003-12-01 00:00:00.001 2 4
338
+ """
339
+
340
+ direction_map = {'rows': 'ROWS_TO_COLUMNS', 'columns': 'COLUMNS_TO_ROWS'}
341
+ n: Union[str, int] = '' if n is None else n # type: ignore[no-redef]
342
+
343
+ self.sink(otq.Transpose(direction=direction_map[direction], key_constraint_values=n))
344
+ # TODO: we should change source's schema after transposing
345
+ return self
346
+
347
+
348
+ def cache(
349
+ self: 'Source',
350
+ cache_name: str,
351
+ delete_if_exists: bool = True,
352
+ inheritability: bool = True,
353
+ otq_params: Union[dict, None] = None,
354
+ time_granularity: int = 0,
355
+ time_granularity_units: Optional[str] = None,
356
+ timezone: str = "",
357
+ time_intervals_to_cache: Optional[List[tuple]] = None,
358
+ allow_delete_to_everyone: bool = False,
359
+ allow_update_to_everyone: bool = False,
360
+ allow_search_to_everyone: bool = True,
361
+ cache_expiration_interval: int = 0,
362
+ start: Optional[ott.datetime] = None,
363
+ end: Optional[ott.datetime] = None,
364
+ read_mode: str = "automatic",
365
+ update_cache: bool = True,
366
+ tick_type: str = "ANY",
367
+ symbol: Optional[str] = None,
368
+ db: Optional[str] = None,
369
+ ) -> 'Source':
370
+ """
371
+ Create cache from query and :py:class:`onetick.py.ReadCache` for created cache.
372
+
373
+ Cache will be created only for current session.
374
+
375
+ By default, if cache with specified name exists, it will be deleted and recreated.
376
+ You can change this behaviour via ``delete_if_exists`` parameter.
377
+
378
+ Parameters
379
+ ----------
380
+ cache_name: str
381
+ Name of the cache to be deleted.
382
+ delete_if_exists: bool
383
+ If set to ``True`` (default), a check will be made to detect the existence of a cache
384
+ with the specified name. Cache will be deleted and recreated only if it exists.
385
+ If set to ``False``, if cache exists it won't be deleted and recreated.
386
+ inheritability: bool
387
+ Indicates whether results can be obtained by combining time intervals that were cached with intervals
388
+ freshly computed to obtain results for larger intervals.
389
+ otq_params: dict
390
+ OTQ params of the query to be cached.
391
+ time_granularity: int
392
+ Value N for seconds/days/months granularity means that start and end time of the query have to be on N
393
+ second/day/month boundaries relative to start of the day/month/year.
394
+ This doesn't affect the frequency of data within the cache, just the start and end dates.
395
+ time_granularity_units: str, None
396
+ Units used in ``time_granularity`` parameter. Possible values: 'none', 'days', 'months', 'seconds' or None.
397
+ timezone: str
398
+ Timezone of the query to be cached.
399
+ time_intervals_to_cache: List[tuple]
400
+ List of tuples with start and end times in ``[(<start_time_1>, <end_time_1>), ...]`` format,
401
+ where ``<start_time>`` and ``<end_time>`` should be one of these:
402
+
403
+ * string in ``YYYYMMDDhhmmss[.msec]`` format.
404
+ * :py:class:`datetime.datetime`
405
+ * :py:class:`onetick.py.types.datetime`
406
+
407
+ If specified only these time intervals can be cached. Ignored if ``inheritability=True``.
408
+ If you try to make a query outside defined interval, error will be raised.
409
+ allow_delete_to_everyone: bool
410
+ When set to ``True`` everyone is allowed to delete the cache.
411
+ allow_update_to_everyone: bool
412
+ When set to ``True`` everyone is allowed to update the cache.
413
+ allow_search_to_everyone: bool
414
+ When set to ``True`` everyone is allowed to read the cached data.
415
+ cache_expiration_interval: int
416
+ If set to a non-zero value determines the periodicity of cache clearing, in seconds.
417
+ The cache will be cleared every X seconds, triggering new query executions when data is requested.
418
+ start: :py:class:`otp.datetime <onetick.py.datetime>`
419
+ Start time for cache query. By default, the start time of the query will be used.
420
+ end: :py:class:`otp.datetime <onetick.py.datetime>`
421
+ End time for cache query. By default, the end time of the query will be used.
422
+ read_mode: str
423
+ Mode of querying cache. One of these:
424
+
425
+ * ``cache_only`` - only cached results are returned and queries are not performed.
426
+ * ``query_only`` - the query is run irrespective of whether the result is already available in the cache.
427
+ * ``automatic`` (default) - perform the query if the data is not found in the cache.
428
+ update_cache: bool
429
+ If set to ``True``, updates the cached data if ``read_mode=query_only`` or if ``read_mode=automatic`` and
430
+ the result data not found in the cache. Otherwise, the cache remains unchanged.
431
+ tick_type: str
432
+ Tick type.
433
+ symbol: str, list of str, list of otq.Symbol, :py:class:`onetick.py.Source`, pd.DataFrame, optional
434
+ ``symbols`` parameter of ``otp.run()``.
435
+ db: str
436
+ Database.
437
+
438
+ See also
439
+ --------
440
+ | :py:func:`onetick.py.create_cache`
441
+ | :py:class:`onetick.py.ReadCache`
442
+
443
+ Examples
444
+ --------
445
+ Simple usage
446
+
447
+ >>> src = otp.DataSource("COMMON", tick_type="TRD", symbols="AAPL")
448
+ >>> data = src.cache(
449
+ ... cache_name="some_cache",
450
+ ... tick_type="TRD", symbol="SYM", db="LOCAL",
451
+ ... )
452
+ >>> df = otp.run(data) # doctest: +SKIP
453
+ """
454
+ from onetick.py.cache import create_cache, delete_cache, modify_cache_config
455
+ from onetick.py.sources import ReadCache
456
+
457
+ cache_exists = True
458
+
459
+ if delete_if_exists:
460
+ try:
461
+ modify_cache_config(cache_name, "TEST_PARAM", "TEST_VALUE")
462
+ except Exception as exc:
463
+ if "There is no cache" in str(exc):
464
+ cache_exists = False
465
+
466
+ if cache_exists and delete_if_exists:
467
+ delete_cache(cache_name)
468
+ cache_exists = False
469
+
470
+ if not cache_exists:
471
+ create_cache(
472
+ cache_name=cache_name,
473
+ query=self,
474
+ inheritability=inheritability,
475
+ otq_params=otq_params,
476
+ time_granularity=time_granularity,
477
+ time_granularity_units=time_granularity_units,
478
+ timezone=timezone,
479
+ time_intervals_to_cache=time_intervals_to_cache,
480
+ allow_delete_to_everyone=allow_delete_to_everyone,
481
+ allow_update_to_everyone=allow_update_to_everyone,
482
+ allow_search_to_everyone=allow_search_to_everyone,
483
+ cache_expiration_interval=cache_expiration_interval,
484
+ tick_type=tick_type,
485
+ symbol=symbol,
486
+ db=db,
487
+ )
488
+
489
+ return ReadCache(
490
+ cache_name=cache_name,
491
+ start=start if start is not None else utils.adaptive,
492
+ end=end if end is not None else utils.adaptive,
493
+ read_mode=read_mode,
494
+ update_cache=update_cache,
495
+ tick_type=tick_type,
496
+ symbol=symbol if symbol is not None else utils.adaptive,
497
+ db=db if db is not None else utils.adaptive_to_default,
498
+ )
499
+
500
+
501
+ @inplace_operation
502
+ def pnl_realized(
503
+ self: 'Source',
504
+ computation_method: str = 'fifo',
505
+ output_field_name: str = 'PNL_REALIZED',
506
+ size_field: Union[str, _Column] = 'SIZE',
507
+ price_field: Union[str, _Column] = 'PRICE',
508
+ buy_sell_flag_field: Union[str, _Column] = 'BUY_SELL_FLAG',
509
+ inplace=False,
510
+ ) -> Optional['Source']:
511
+ """
512
+ Computes the realized Profit and Loss (**PNL**) on each tick and is applicable to both scenarios,
513
+ whether selling after buying or buying after selling.
514
+
515
+ Parameters
516
+ ----------
517
+ computation_method: str
518
+ This parameter determines the approach used to calculate the realized Profit and Loss (**PnL**).
519
+
520
+ Possible options are:
521
+
522
+ * ``fifo`` - Stands for 'First-In-First-Out,' is used to calculate Profit and Loss (**PnL**)
523
+ based on the principle that the first trading positions bought are the first ones to be sold,
524
+ or conversely, the first trading positions sold are the first ones to be bought.
525
+
526
+ output_field_name: str
527
+ This parameter defines the name of the output field.
528
+
529
+ Default: **PNL_REALIZED**.
530
+
531
+ size_field: str, :py:class:`otp.Column <onetick.py.Column>`
532
+ The name of the field with size, default is **SIZE**.
533
+ price_field: str, :py:class:`otp.Column <onetick.py.Column>`
534
+ The name of the field with price, default is **PRICE**.
535
+ buy_sell_flag_field: str, :py:class:`otp.Column <onetick.py.Column>`
536
+ The name of the field with buy/sell flag, default is **BUY_SELL_FLAG**.
537
+
538
+ See also
539
+ --------
540
+ **PNL_REALIZED** OneTick event processor
541
+
542
+ Examples
543
+ --------
544
+
545
+ Simple usage
546
+
547
+ >>> data = otp.DataSource('SOME_DATABASE', tick_type='TT', symbols='AA') # doctest: +SKIP
548
+ >>> data = data.pnl_realized() # doctest: +SKIP
549
+ >>> otp.run(data) # doctest: +SKIP
550
+ Time BUY_SELL_FLAG PRICE SIZE PNL_REALIZED
551
+ 0 2003-12-01 00:00:00.000 0 5 20 0.0
552
+ 1 2003-12-01 00:00:00.001 0 4 30 0.0
553
+ 2 2003-12-01 00:00:00.002 1 3 15 -30.0
554
+ 3 2003-12-01 00:00:00.003 0 10 50 0.0
555
+ 4 2003-12-01 00:00:00.004 1 7 18 49.0
556
+ 5 2003-12-01 00:00:00.005 1 9 6 30.0
557
+ 6 2003-12-01 00:00:00.006 0 8 40 0.0
558
+ """
559
+ if computation_method not in ['fifo']:
560
+ raise ValueError(
561
+ f"Parameter 'computation_method' has incorrect value: '{computation_method}', "
562
+ f"should be one of these: 'fifo'."
563
+ )
564
+
565
+ for field in (size_field, price_field, buy_sell_flag_field):
566
+ if str(field) not in self.schema:
567
+ raise ValueError(f'Field {field} is not in schema')
568
+
569
+ # PNL_REALIZED doesn't support choosing input fields, so we workaround by renaming fields
570
+ restore_fields = {}
571
+ added_fields = []
572
+ for default_field, field in (('SIZE', size_field), ('PRICE', price_field), ('BUY_SELL_FLAG', buy_sell_flag_field)):
573
+ if field != default_field:
574
+ if default_field in self.schema:
575
+ tmp_field_name = f'__TMP__{field}__TMP__'
576
+ self[tmp_field_name] = self[default_field]
577
+ restore_fields[default_field] = self[tmp_field_name]
578
+ self[default_field] = self[field]
579
+ added_fields.append(default_field)
580
+
581
+ if output_field_name in self.schema:
582
+ raise ValueError(f'Field {output_field_name} is already in schema')
583
+
584
+ computation_method = computation_method.upper()
585
+ self.sink(otq.PnlRealized(computation_method=computation_method, output_field_name=output_field_name))
586
+ self.schema[output_field_name] = float
587
+
588
+ if added_fields:
589
+ self.drop(added_fields, inplace=True)
590
+ if restore_fields:
591
+ # restore fields from temporary fields and delete temporary fields
592
+ self.add_fields(restore_fields)
593
+ self.drop(list(restore_fields.values()), inplace=True)
594
+
595
+ return self
596
+
597
+
598
+ @inplace_operation
599
+ def execute(self: 'Source', *operations, inplace=False) -> Optional['Source']:
600
+ """
601
+ Execute operations without returning their values.
602
+ Some operations in onetick.py can be used to modify the state of some object
603
+ (tick sequences mostly) and in that case user may not want to save the result of the
604
+ operation to column.
605
+
606
+ Parameters
607
+ ----------
608
+ operations : list of :py:class:`~onetick.py.Operation`
609
+ operations to execute.
610
+ inplace : bool
611
+ The flag controls whether operation should be applied inplace or not.
612
+ If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
613
+ object.
614
+
615
+ Returns
616
+ -------
617
+ :class:`Source` or ``None``
618
+
619
+ See Also
620
+ --------
621
+ **EXECUTE_EXPRESSIONS** OneTick event processor
622
+
623
+ Examples
624
+ --------
625
+ >>> data = otp.Tick(A=1)
626
+ >>> data.state_vars['SET'] = otp.state.tick_set('oldest', 'A')
627
+ >>> data = data.execute(data.state_vars['SET'].erase(A=1))
628
+ """
629
+ if not operations:
630
+ raise ValueError('At least one operation must be specified in execute() method')
631
+ op_str = ';'.join(map(str, operations))
632
+ self.sink(otq.ExecuteExpressions(op_str))
633
+ return self
634
+
635
+
636
+ @inplace_operation
637
+ def fillna(self: 'Source', value=None, columns=None, inplace=False) -> Optional['Source']:
638
+ """
639
+ Replace NaN values in floating point type fields with the ``value``.
640
+
641
+ Parameters
642
+ ----------
643
+ value : float, :py:class:`~onetick.py.Operation`
644
+ The value to replace NaN.
645
+ If not specified then the value from the previous tick will be used.
646
+ columns: list
647
+ List of strings with column names or :py:class:`~onetick.py.Column` objects.
648
+ Only the values in specified columns will be replaced.
649
+ By default the values in all floating point type fields will be replaced.
650
+ inplace : bool
651
+ The flag controls whether operation should be applied inplace or not.
652
+ If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
653
+ object.
654
+
655
+ Returns
656
+ -------
657
+ :class:`Source` or ``None``
658
+
659
+ See Also
660
+ --------
661
+ :py:meth:`onetick.py.Operation.fillna`
662
+
663
+ Examples
664
+ --------
665
+
666
+ By default, the value of the previous tick is used as a value to replace NaN
667
+ (for the first tick the previous value do not exist, so it will still be NaN):
668
+
669
+ >>> data = otp.Ticks({'A': [0, 1, 2, 3], 'B': [otp.nan, 2.2, otp.nan, 3.3]})
670
+ >>> data = data.fillna()
671
+ >>> otp.run(data)
672
+ Time A B
673
+ 0 2003-12-01 00:00:00.000 0 NaN
674
+ 1 2003-12-01 00:00:00.001 1 2.2
675
+ 2 2003-12-01 00:00:00.002 2 2.2
676
+ 3 2003-12-01 00:00:00.003 3 3.3
677
+
678
+ The value can also be a constant:
679
+
680
+ >>> data = otp.Ticks({'A': [0, 1, 2, 3], 'B': [otp.nan, 2.2, otp.nan, 3.3]})
681
+ >>> data = data.fillna(777)
682
+ >>> otp.run(data)
683
+ Time A B
684
+ 0 2003-12-01 00:00:00.000 0 777.0
685
+ 1 2003-12-01 00:00:00.001 1 2.2
686
+ 2 2003-12-01 00:00:00.002 2 777.0
687
+ 3 2003-12-01 00:00:00.003 3 3.3
688
+
689
+ :py:class:`~onetick.py.Operation` objects can also be used as a ``value``:
690
+
691
+ >>> data = otp.Ticks({'A': [0, 1, 2, 3], 'B': [otp.nan, 2.2, otp.nan, 3.3]})
692
+ >>> data = data.fillna(data['A'])
693
+ >>> otp.run(data)
694
+ Time A B
695
+ 0 2003-12-01 00:00:00.000 0 0.0
696
+ 1 2003-12-01 00:00:00.001 1 2.2
697
+ 2 2003-12-01 00:00:00.002 2 2.0
698
+ 3 2003-12-01 00:00:00.003 3 3.3
699
+
700
+ Parameter ``columns`` can be used to specify the columns where values will be replaced:
701
+
702
+ .. testcode::
703
+ :skipif: is_per_tick_script_boolean_problem()
704
+
705
+ data = otp.Ticks({'A': [0, 1, 2, 3], 'B': [otp.nan, 2.2, otp.nan, 3.3], 'C': [otp.nan, 2.2, otp.nan, 3.3]})
706
+ data = data.fillna(columns=['B'])
707
+ df = otp.run(data)
708
+ print(df)
709
+
710
+ .. testoutput::
711
+
712
+ Time A B C
713
+ 0 2003-12-01 00:00:00.000 0 NaN NaN
714
+ 1 2003-12-01 00:00:00.001 1 2.2 2.2
715
+ 2 2003-12-01 00:00:00.002 2 2.2 NaN
716
+ 3 2003-12-01 00:00:00.003 3 3.3 3.3
717
+ """
718
+ if columns:
719
+ for column in columns:
720
+ if column not in self.schema:
721
+ raise ValueError(f"Column '{column}' is not in schema")
722
+ if self.schema[column] is not float:
723
+ raise TypeError(f"Column '{column}' doesn't have float type")
724
+ columns = list(map(str, columns))
725
+ no_columns = columns is None
726
+
727
+ if value is not None:
728
+ if isinstance(value, int):
729
+ value = float(value)
730
+ if isinstance(value, otp.Operation) and value.dtype is int:
731
+ value = value.astype(float)
732
+ dtype = ott.get_object_type(value)
733
+ if dtype is not float:
734
+ raise ValueError(f"The type of parameter 'value' must be float, not {dtype}")
735
+
736
+ def fun(tick):
737
+ for field in otp.tick_descriptor_fields():
738
+ if (field.get_type() == 'double'
739
+ and tick.get_double_value(field.get_name()) == otp.nan
740
+ and (no_columns or field.get_name().isin(*columns))):
741
+ tick.set_double_value(field.get_name(), value)
742
+ else:
743
+ # deque may have already been created by previous call of fillna() method
744
+ if '__OTP_FILLNA_DEQUE__' not in self.state_vars:
745
+ self.state_vars['__OTP_FILLNA_DEQUE__'] = otp.state.tick_deque(scope='branch')
746
+
747
+ def fun(tick):
748
+ i = otp.static(0)
749
+ prev_tick = otp.tick_deque_tick()
750
+ # first tick doesn't have the previous tick, so skipping it
751
+ if i > 0:
752
+ # get the previous tick from the deque
753
+ tick.state_vars['__OTP_FILLNA_DEQUE__'].get_tick(0, prev_tick)
754
+ # replace value in all double fields with the value of the previous tick
755
+ for field in otp.tick_descriptor_fields():
756
+ if (field.get_type() == 'double'
757
+ and tick.get_double_value(field.get_name()) == otp.nan
758
+ and (no_columns or field.get_name().isin(*columns))):
759
+ tick.set_double_value(field.get_name(),
760
+ prev_tick.get_double_value(field.get_name()))
761
+ # clear the deque and add the current tick to it
762
+ tick.state_vars['__OTP_FILLNA_DEQUE__'].clear()
763
+ tick.state_vars['__OTP_FILLNA_DEQUE__'].push_back(tick)
764
+ i = i + 1
765
+
766
+ return self.script(fun, inplace=inplace)
767
+
768
+
769
+ @inplace_operation
770
+ def mkt_activity(self: 'Source', calendar_name=None, inplace=False) -> Optional['Source']:
771
+ """
772
+ Adds a string field named **MKT_ACTIVITY** to each tick in the input tick stream.
773
+
774
+ The value of this field is set to the union of session flags that apply for the security at the time of the tick,
775
+ as specified in the calendar sections of the reference database (see Reference Database Guide).
776
+
777
+ Session flags may differ between databases,
778
+ but the following letters have reserved meaning: L - half day, H - holiday, W - weekend, R - regular.
779
+
780
+ The calendar can either be specified explicitly by name
781
+ (the *CALENDAR* sections of the reference database are assumed to contain a calendar with such a name),
782
+ or default to the security- or exchange-level calendars for the queried symbol
783
+ (the *SYMBOL_CALENDAR* or *EXCH_CALENDAR* sections of the reference database).
784
+
785
+ The latter case requires a non-zero symbol date to be specified for queried symbols
786
+ (see parameter ``symbol_date`` in :py:func:`otp.run <onetick.py.run>`).
787
+
788
+ Parameters
789
+ ----------
790
+ calendar_name : str, :py:class:`~onetick.py.Column`
791
+ The calendar name to choose for the respective calendar from the *CALENDAR* sections of the reference database.
792
+ It can be a string constant or the name of the field with per-tick calendar name.
793
+
794
+ When this parameter is not specified,
795
+ default security- or exchange-level calendars configured for the queried database and symbol are used
796
+ (but symbol date must be specified in this case).
797
+ inplace : bool
798
+ The flag controls whether operation should be applied inplace or not.
799
+ If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
800
+ object.
801
+
802
+ Note
803
+ ----
804
+ When applying :py:meth:`mkt_activity` to aggregated data,
805
+ please take into account that session flags may change during the aggregation bucket.
806
+ The field **MKT_ACTIVITY**, in this case, will represent the session flags at the time assigned to the bucket,
807
+ which may be different from the session flags at some times during the bucket.
808
+
809
+ Returns
810
+ -------
811
+ :class:`Source` or ``None``
812
+
813
+ See Also
814
+ --------
815
+ **MKT_ACTIVITY** OneTick event processor
816
+
817
+ Examples
818
+ --------
819
+
820
+ By default, security- or exchange-level calendars configured for the queried database and symbol are used
821
+ (but symbol date must be specified in this case):
822
+
823
+ >>> data = otp.DataSource(...) # doctest: +SKIP
824
+ >>> data = data.mkt_activity() # doctest: +SKIP
825
+ >>> otp.run(data, date=otp.date(2022, 1, 1), symbol_date=otp.date(2022, 1, 1)) # doctest: +SKIP
826
+
827
+ Otherwise, parameter ``calendar_name`` must be specified:
828
+
829
+ >>> data = otp.DataSource(...) # doctest: +SKIP
830
+ >>> data = data.mkt_activity(calendar_name='WNY') # doctest: +SKIP
831
+ >>> otp.run(data, date=otp.date(2022, 1, 1)) # doctest: +SKIP
832
+
833
+ Parameter ``calendar_name`` can also be specified as a column.
834
+ In this case calendar name can be different for each tick:
835
+
836
+ >>> data = otp.DataSource(...) # doctest: +SKIP
837
+ >>> data = data.mkt_activity(calendar_name=data['CALENDAR_NAME']) # doctest: +SKIP
838
+ >>> otp.run(data, date=otp.date(2022, 1, 1)) # doctest: +SKIP
839
+
840
+ In this example you can see how market activity status is changing during the days.
841
+ We are getting first and last tick of the group each time the type of market activity is changed.
842
+ You can see regular trades (R) from 9:30 to 16:00, and a holiday (H) on 2018-02-07.
843
+
844
+ >>> data = otp.DataSource('TRAIN_A_PRL_TRD', tick_type='TRD', symbols='MSFT') # doctest: +SKIP
845
+ >>> data = data.mkt_activity('FRED') # doctest: +SKIP
846
+ >>> data = data[['PRICE', 'SIZE', 'MKT_ACTIVITY']] # doctest: +SKIP
847
+ >>> first = data.first(1, bucket_interval=(data['MKT_ACTIVITY'] != data['MKT_ACTIVITY'][-1])) # doctest: +SKIP
848
+ >>> last = data.last(1, bucket_interval=(data['MKT_ACTIVITY'] != data['MKT_ACTIVITY'][-1])) # doctest: +SKIP
849
+ >>> data = otp.merge([first, last]) # doctest: +SKIP
850
+ >>> df = otp.run(data, # doctest: +SKIP
851
+ ... start=otp.dt(2018, 2, 1), end=otp.dt(2018, 2, 9),
852
+ ... symbol_date=otp.dt(2018, 2, 1), timezone='EST5EDT')
853
+ >>> df[['Time', 'MKT_ACTIVITY']] # doctest: +SKIP
854
+ Time MKT_ACTIVITY
855
+ 0 2018-02-01 01:31:44.466
856
+ 1 2018-02-01 09:29:59.996
857
+ 2 2018-02-01 09:30:00.225 R
858
+ 3 2018-02-01 15:59:58.857 R
859
+ 4 2018-02-01 16:00:01.858
860
+ 5 2018-02-02 09:29:50.366
861
+ 6 2018-02-02 09:30:01.847 R
862
+ 7 2018-02-02 15:59:59.829 R
863
+ 8 2018-02-02 16:00:01.782
864
+ 9 2018-02-05 09:29:43.084
865
+ 10 2018-02-05 09:30:00.301 R
866
+ 11 2018-02-05 15:59:59.974 R
867
+ 12 2018-02-05 16:00:02.438
868
+ 13 2018-02-06 09:29:27.279
869
+ 14 2018-02-06 09:30:00.045 R
870
+ 15 2018-02-06 15:59:59.903 R
871
+ 16 2018-02-06 16:01:03.524
872
+ 17 2018-02-07 09:29:56.739
873
+ 18 2018-02-07 09:30:00.365 H
874
+ 19 2018-02-07 15:59:59.940 H
875
+ 20 2018-02-07 16:00:00.187
876
+ 21 2018-02-08 09:29:28.446
877
+ 22 2018-02-08 09:30:00.658 F
878
+ 23 2018-02-08 15:59:59.564 F
879
+ 24 2018-02-08 16:00:02.355
880
+ 25 2018-02-08 19:59:57.061
881
+ """
882
+ if calendar_name is None:
883
+ calendar_name = ''
884
+ if isinstance(calendar_name, str):
885
+ calendar_field_name = ''
886
+ elif isinstance(calendar_name, _Column):
887
+ calendar_field_name = str(calendar_name)
888
+ calendar_name = ''
889
+ else:
890
+ raise ValueError(f"Unsupported type for parameter 'calendar_name': {type(calendar_name)}")
891
+ self.sink(
892
+ otq.MktActivity(calendar_name=calendar_name, calendar_field_name=calendar_field_name)
893
+ )
894
+ self.schema['MKT_ACTIVITY'] = str
895
+ return self
896
+
897
+
898
+ @inplace_operation
899
+ def book_diff(self: 'Source', include_initial_book: bool = False, inplace=False) -> Optional['Source']:
900
+ """
901
+ Performs book diffing for every pair of consecutive ticks, each representing a flat book of a fixed depth.
902
+
903
+ This method can be thought to be an operation, opposite to :py:meth:`~onetick.py.Source.ob_snapshot_flat`.
904
+ Every input tick, different from the previous one, results in a series of output PRL ticks to be propagated,
905
+ each carrying information about a level deletion, addition or update.
906
+
907
+ The input of this method is a time series of flat book ticks,
908
+ just like the result of :py:meth:`~onetick.py.Source.ob_snapshot_flat`.
909
+ More precisely, for a fixed depth N of the flat book,
910
+ input ticks should carry the fields BID_<FIELD_NAME>K and ASK_<FIELD_NAME>K,
911
+ where 1 <= K <= N and <FIELD_NAME> ranges over a specific set of fields F,
912
+ among which PRICE and SIZE are mandatory.
913
+
914
+ The output of this method is a time series of PRL ticks,
915
+ carrying information about a level deletion, addition, or update.
916
+ Each tick carries the fields from the above-mentioned set F,
917
+ plus BUY_SELL_FLAG, RECORD_TYPE, TICK_STATUS, and DELETED_TIME.
918
+ BUY_SELL_FLAG carries the side (0 - bid, 1 - ask), the rest are for internal use.
919
+
920
+ Parameters
921
+ ----------
922
+ include_initial_book : bool
923
+ This method treats the first tick as an "initial state" of the book.
924
+ If this parameter is set to True, then this initial tick will also be in the output.
925
+ inplace : bool
926
+ The flag controls whether operation should be applied inplace or not.
927
+ If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
928
+ object.
929
+
930
+ Note
931
+ ----
932
+ Tick descriptor changes are not allowed in this event processor.
933
+
934
+ Returns
935
+ -------
936
+ :class:`Source` or ``None``
937
+
938
+ See Also
939
+ --------
940
+ **BOOK_DIFF** OneTick event processor
941
+
942
+ Examples
943
+ --------
944
+
945
+ First let's get flat order book snapshot from the database (top level each 12 hours):
946
+
947
+ >>> data = otp.DataSource('TRAIN_A_PRL_TRD', tick_type='PRL', symbols='MSFT') # doctest: +SKIP
948
+ >>> flat = data.ob_snapshot_flat(max_levels=1, bucket_interval=otp.Hour(12)) # doctest: +SKIP
949
+ >>> otp.run(flat, date=otp.dt(2018, 2, 1)) # doctest: +SKIP
950
+ Time BID_PRICE1 BID_SIZE1 BID_UPDATE_TIME1 ASK_PRICE1 ASK_SIZE1 ASK_UPDATE_TIME1
951
+ 0 2018-02-01 12:00:00 95.53 1400 2018-02-01 11:59:59.797 95.54 200 2018-02-01 11:59:59.978
952
+ 1 2018-02-02 00:00:00 94.52 100 2018-02-01 19:57:02.502 94.90 250 2018-02-01 18:35:38.543
953
+
954
+ Then we can apply ``book_diff`` method to the result to get the PRL ticks again:
955
+
956
+ >>> diff = flat.book_diff() # doctest: +SKIP
957
+ >>> otp.run(diff, date=otp.dt(2018, 2, 1)) # doctest: +SKIP
958
+ Time PRICE SIZE UPDATE_TIME BUY_SELL_FLAG RECORD_TYPE TICK_STATUS DELETED_TIME
959
+ 0 2018-02-02 95.53 0 2018-02-01 11:59:59.797 0 R 0 1969-12-31 19:00:00
960
+ 1 2018-02-02 94.52 100 2018-02-01 19:57:02.502 0 R 0 1969-12-31 19:00:00
961
+ 2 2018-02-02 94.90 250 2018-02-01 18:35:38.543 1 R 0 1969-12-31 19:00:00
962
+ 3 2018-02-02 95.54 0 2018-02-01 11:59:59.978 1 R 0 1969-12-31 19:00:00
963
+
964
+ By default the first tick in the query range is not included,
965
+ use parameter ``include_initial_book`` to include it:
966
+
967
+ >>> diff = flat.book_diff(include_initial_book=True) # doctest: +SKIP
968
+ >>> otp.run(diff, date=otp.dt(2018, 2, 1)) # doctest: +SKIP
969
+ Time PRICE SIZE UPDATE_TIME BUY_SELL_FLAG RECORD_TYPE TICK_STATUS DELETED_TIME
970
+ 0 2018-02-01 12:00:00 95.53 1400 2018-02-01 11:59:59.797 0 R 0 1969-12-31 19:00:00
971
+ 1 2018-02-01 12:00:00 95.54 200 2018-02-01 11:59:59.978 1 R 0 1969-12-31 19:00:00
972
+ 2 2018-02-02 00:00:00 95.53 0 2018-02-01 11:59:59.797 0 R 0 1969-12-31 19:00:00
973
+ 3 2018-02-02 00:00:00 94.52 100 2018-02-01 19:57:02.502 0 R 0 1969-12-31 19:00:00
974
+ 4 2018-02-02 00:00:00 94.90 250 2018-02-01 18:35:38.543 1 R 0 1969-12-31 19:00:00
975
+ 5 2018-02-02 00:00:00 95.54 0 2018-02-01 11:59:59.978 1 R 0 1969-12-31 19:00:00
976
+ """
977
+ self.sink(otq.BookDiff(include_initial_book=include_initial_book))
978
+ return self
979
+
980
+
981
+ @inplace_operation
982
+ def limit(self: 'Source', tick_limit: int, inplace=False) -> Optional['Source']:
983
+ """
984
+ Propagates ticks until the count limit is reached. Once the limit is reached,
985
+ hidden ticks will still continue to propagate until the next regular tick appears.
986
+
987
+ Parameters
988
+ ----------
989
+ tick_limit: int
990
+ The number of regular ticks to propagate. Must be a non-negative integer or -1, which will mean no limit.
991
+ inplace : bool
992
+ The flag controls whether operation should be applied inplace or not.
993
+ If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
994
+ object.
995
+
996
+ Returns
997
+ -------
998
+ :class:`Source` or ``None``
999
+
1000
+ See Also
1001
+ --------
1002
+ **LIMIT** OneTick event processor
1003
+
1004
+ Examples
1005
+ --------
1006
+
1007
+ Basic example
1008
+
1009
+ .. testcode::
1010
+ :skipif: not otp.compatibility.is_limit_ep_supported()
1011
+
1012
+ data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
1013
+ data = data.limit(tick_limit=3)
1014
+ print(otp.run(data))
1015
+
1016
+ .. testoutput::
1017
+
1018
+ Time X
1019
+ 0 2003-12-01 00:00:00.000 1
1020
+ 1 2003-12-01 00:00:00.001 2
1021
+ 2 2003-12-01 00:00:00.002 3
1022
+
1023
+ Disable limit
1024
+
1025
+ .. testcode::
1026
+ :skipif: not otp.compatibility.is_limit_ep_supported()
1027
+
1028
+ data = otp.Ticks(X=[1, 2, 3, 4, 5, 6])
1029
+ data = data.limit(tick_limit=-1)
1030
+ print(otp.run(data))
1031
+
1032
+ .. testoutput::
1033
+
1034
+ Time X
1035
+ 0 2003-12-01 00:00:00.000 1
1036
+ 1 2003-12-01 00:00:00.001 2
1037
+ 2 2003-12-01 00:00:00.002 3
1038
+ 3 2003-12-01 00:00:00.003 4
1039
+ 4 2003-12-01 00:00:00.004 5
1040
+ 5 2003-12-01 00:00:00.005 6
1041
+ """
1042
+ if not hasattr(otq, 'Limit'):
1043
+ raise RuntimeError('LIMIT EP isn\'t supported by the current OneTick version.')
1044
+
1045
+ if tick_limit < 0 and tick_limit != -1:
1046
+ raise ValueError('Negative values, except -1, not allowed as `tick_limit` in `limit` method.')
1047
+
1048
+ self.sink(otq.Limit(tick_limit=tick_limit))
1049
+ return self
1050
+
1051
+
1052
+ def _merge_fields_by_regex(schema: dict, columns_regex: str, match_index: int):
1053
+ new_columns: Dict[str, type] = {}
1054
+ for column, column_type in list(schema.items()):
1055
+ match = re.match(columns_regex, column)
1056
+ if match:
1057
+ new_column = match.group(match_index)
1058
+
1059
+ if new_column in schema:
1060
+ raise KeyError(
1061
+ f'Can\'t apply `show_full_detail` for column `{column}`: '
1062
+ f'column `{new_column}` is already in schema.'
1063
+ )
1064
+
1065
+ if new_column in new_columns:
1066
+ if column_type != new_columns[new_column]:
1067
+ raise TypeError(
1068
+ f'Can\'t apply `show_full_detail`: '
1069
+ f'type mismatch for columns with suffix `{new_column}`.'
1070
+ )
1071
+ else:
1072
+ new_columns[new_column] = column_type
1073
+
1074
+ del schema[column]
1075
+
1076
+ schema.update(**new_columns)
1077
+
1078
+
1079
+ @inplace_operation
1080
+ def virtual_ob(
1081
+ self: 'Source',
1082
+ quote_source_fields: Optional[List[Union[str, _Column]]] = None,
1083
+ quote_timeout: Optional[float] = None,
1084
+ show_full_detail: bool = False,
1085
+ output_book_format: Literal['ob', 'prl'] = 'ob',
1086
+ inplace=False,
1087
+ ) -> Optional['Source']:
1088
+ """
1089
+ Creates a series of fake orders from a time series of best bids and asks. The algorithm used is as follows:
1090
+ For a tick with the best bid or ask, create an order tick adding the new best bid or ask and an order
1091
+ withdrawing the old one. Virtual order books can be created for multiple subgroups at once
1092
+ by using the ``quote_source_fields`` parameter to specify a list of string fields to be used for grouping.
1093
+ A separate book will be created for each combination.
1094
+
1095
+ Parameters
1096
+ ----------
1097
+ quote_source_fields: Optional[List[Union[str, Column]]]
1098
+ Specifies a list of string fields for grouping quotes.
1099
+ The virtual order book is then constructed for each subgroup separately and
1100
+ the ``SOURCE`` field is constructed to contain the description of the group.
1101
+ quote_timeout: Optional[float]
1102
+ Specifies the maximum age of a quote that is not stale.
1103
+ A quote that is not replaced after more than ``quote_timeout`` seconds is considered stale,
1104
+ and delete orders will be generated for it.
1105
+
1106
+ A value of ``quote_timeout`` can be fractional (for example, 3.51)
1107
+ show_full_detail: bool
1108
+ If set to ``True``, **virtual_ob** will attempt to include
1109
+ all fields in the input tick when forming the output tick.
1110
+
1111
+ ``ASK_X/BID_X`` fields will combine under paired field ``X``
1112
+ ``ASK_X`` and ``BID_X`` must have the same type.
1113
+
1114
+ If only ``ASK_X`` or ``BID_X`` exist, output will have ``X`` and the missing field
1115
+ will be assumed to have its default value.
1116
+
1117
+ Paired and non-paired fields must not interfere with each other and the fields originally added by this EP
1118
+ output_book_format: 'prl' or 'ob'
1119
+ Supported values are ``prl`` and ``ob``. When set to ``prl``, field ``SIZE`` of output ticks represents
1120
+ current size for the tick's source, price, and side, and the EP propagates ``PRICE`` and ``SOURCE``
1121
+ as the state keys of its output time series.
1122
+
1123
+ When set to ``ob``, field ``SIZE`` of output ticks represents the delta of size for
1124
+ the tick's source, price, and side, and the state key of the output ticks is empty.
1125
+ inplace : bool
1126
+ The flag controls whether operation should be applied inplace or not.
1127
+ If ``inplace=True``, then it returns nothing. Otherwise, method returns a new modified object.
1128
+
1129
+ Returns
1130
+ -------
1131
+ :class:`Source` or ``None``
1132
+
1133
+ See Also
1134
+ --------
1135
+ **VIRTUAL_OB** OneTick event processor
1136
+
1137
+ Examples
1138
+ --------
1139
+
1140
+ Basic example
1141
+
1142
+ >>> data = otp.DataSource(
1143
+ ... db='US_COMP', symbols='AAPL', tick_type='QTE', date=otp.date(2003, 12, 1)
1144
+ ... ) # doctest: +SKIP
1145
+ >>> data = data[['ASK_PRICE', 'ASK_SIZE', 'BID_PRICE', 'BID_SIZE']] # doctest: +SKIP
1146
+ >>> data = data.virtual_ob() # doctest: +SKIP
1147
+ >>> otp.run(data) # doctest: +SKIP
1148
+ Time PRICE DELETED_TIME SIZE BUY_SELL_FLAG TICK_STATUS SOURCE
1149
+ 0 2003-12-01 00:00:00.000 22.28 1969-12-31 19:00:00 500 1 0 AAPL
1150
+ 1 2003-12-01 00:00:00.000 21.66 1969-12-31 19:00:00 100 0 0 AAPL
1151
+ 2 2003-12-01 00:00:00.001 21.8 1969-12-31 19:00:00 500 1 0 AAPL
1152
+ ...
1153
+
1154
+ Specify columns to group quotes
1155
+
1156
+ >>> data = otp.DataSource(
1157
+ ... db='US_COMP', symbols='AAPL', tick_type='QTE', date=otp.date(2003, 12, 1)
1158
+ ... ) # doctest: +SKIP
1159
+ >>> data = data[['ASK_PRICE', 'ASK_SIZE', 'BID_PRICE', 'BID_SIZE', 'EXCHANGE']] # doctest: +SKIP
1160
+ >>> data = data.virtual_ob(['EXCHANGE']) # doctest: +SKIP
1161
+ >>> otp.run(data) # doctest: +SKIP
1162
+ Time PRICE DELETED_TIME SIZE BUY_SELL_FLAG TICK_STATUS SOURCE
1163
+ 0 2003-12-01 00:00:00.000 22.28 1969-12-31 19:00:00 500 1 0 D
1164
+ 1 2003-12-01 00:00:00.000 21.66 1969-12-31 19:00:00 100 0 0 D
1165
+ 2 2003-12-01 00:00:00.001 21.8 1969-12-31 19:00:00 500 1 0 P
1166
+ ...
1167
+ """
1168
+ kwargs: Dict[str, Any] = {}
1169
+
1170
+ required_fields = {'BID_PRICE', 'BID_SIZE', 'ASK_PRICE', 'ASK_SIZE'}
1171
+ missing_fields = required_fields - set(self.schema.keys())
1172
+
1173
+ if missing_fields:
1174
+ raise ValueError('Missing fields required by `virtual_ob`: ' + ', '.join(missing_fields))
1175
+
1176
+ if output_book_format not in {'ob', 'prl'}:
1177
+ raise ValueError(
1178
+ f'Incorrect value for `output_book_format` parameter: {output_book_format}. '
1179
+ f'Supported values: \'ob\' or \'prl\''
1180
+ )
1181
+
1182
+ if show_full_detail and output_book_format != 'prl':
1183
+ raise ValueError('`output_book_format` should be set to \'prl\' when `show_full_detail` set to `True`')
1184
+
1185
+ is_pnl_and_show_full_detail_supported = is_ob_virtual_prl_and_show_full_detail_supported()
1186
+
1187
+ if show_full_detail and not is_pnl_and_show_full_detail_supported:
1188
+ raise RuntimeError('`virtual_ob` not supports `show_full_detail` parameter on this OneTick version')
1189
+
1190
+ if output_book_format != 'ob' and not is_pnl_and_show_full_detail_supported:
1191
+ raise RuntimeError('`virtual_ob` not supports `output_book_format` parameter on this OneTick version')
1192
+
1193
+ if is_pnl_and_show_full_detail_supported:
1194
+ kwargs['show_full_detail'] = show_full_detail
1195
+ kwargs['output_book_format'] = output_book_format
1196
+
1197
+ if quote_source_fields is None:
1198
+ quote_source_fields = []
1199
+
1200
+ quote_source_fields_list = list(map(str, quote_source_fields))
1201
+ for column in quote_source_fields_list:
1202
+ if column not in self.schema:
1203
+ raise ValueError(f'Column \'{column}\' passed in `quote_source_fields` parameter is missing in the schema')
1204
+
1205
+ self.sink(
1206
+ otq.VirtualOb(
1207
+ quote_source_fields=','.join(quote_source_fields_list),
1208
+ quote_timeout='' if quote_timeout is None else str(quote_timeout),
1209
+ **kwargs,
1210
+ )
1211
+ )
1212
+
1213
+ schema = {
1214
+ 'PRICE': float,
1215
+ 'DELETED_TIME': otp.nsectime,
1216
+ 'SIZE': float,
1217
+ 'BUY_SELL_FLAG': int,
1218
+ 'TICK_STATUS': int,
1219
+ 'SOURCE': str,
1220
+ }
1221
+
1222
+ if show_full_detail:
1223
+ exclude_fields = list(required_fields) + quote_source_fields_list
1224
+ schema.update({
1225
+ k: v for k, v in self.schema.copy().items()
1226
+ if k not in exclude_fields
1227
+ })
1228
+
1229
+ _merge_fields_by_regex(schema, r'^(ASK|BID)_(.*)$', 2)
1230
+
1231
+ self.schema.set(**schema)
1232
+
1233
+ return self
1234
+
1235
+
1236
+ @copy_signature(otp.functions.corp_actions, add_self=True, drop_parameters=['source'])
1237
+ def corp_actions(self: 'Source', *args, **kwargs) -> 'Source':
1238
+ """
1239
+ Adjusts values using corporate actions information loaded into OneTick
1240
+ from the reference data file. To use it, location of reference database must
1241
+ be specified via OneTick configuration.
1242
+
1243
+ Parameters
1244
+ ----------
1245
+ adjustment_date : :py:class:`onetick.py.date`, :py:class:`onetick.py.datetime`, int, str, None, optional
1246
+ The date as of which the values are adjusted.
1247
+ All corporate actions of the types specified in the parameters
1248
+ that lie between the tick timestamp and the adjustment date will be applied to each tick.
1249
+
1250
+ This parameter can be either date or datetime .
1251
+ `int` and `str` format can be *YYYYMMDD* or *YYYYMMDDhhmmss*.
1252
+ When parameter is a date, the time is assumed to be 17:00:00 GMT
1253
+ and parameter ``adjustment_date_tz`` is ignored.
1254
+
1255
+ If it is not set, the values are adjusted as of the end date in the query.
1256
+
1257
+ Notice that the ``adjustment date`` is not affected neither by *_SYMBOL_PARAM._PARAM_END_TIME_NANOS*
1258
+ nor by the *apply_times_daily* setting in :py:func:`onetick.py.run`.
1259
+
1260
+ adjustment_date_tz : str, optional
1261
+ Timezone for ``adjustment date``.
1262
+
1263
+ By default global :py:attr:`tz<onetick.py.configuration.Config.tz>` value is used.
1264
+ Local timezone can't be used so in this case parameter is set to GMT.
1265
+ When ``adjustment_date`` is in YYYYMMDD format, this parameter is set to GMT.
1266
+ fields : str, optional
1267
+ A comma-separated list of fields to be adjusted. If this parameter is not set,
1268
+ some default adjustments will take place if appropriately named fields exist in the tick:
1269
+
1270
+ - If the ``adjust_rule`` parameter is set to PRICE, and the PRICE field is present,
1271
+ it will get adjusted. If the fields ASK_PRICE or BID_PRICE are present, they will get adjusted.
1272
+ If fields ASK_VALUE or BID_VALUE are present, they will get adjusted
1273
+
1274
+ - If the ``adjust_rule`` parameter is set to SIZE, and the SIZE field is present,
1275
+ it will get adjusted. If the fields ASK_SIZE or BID_SIZE are present, they will get adjusted.
1276
+ If fields ASK_VALUE or BID_VALUE are present, they will get adjusted.
1277
+
1278
+ adjust_rule : str, optional
1279
+ When set to PRICE, adjustments are applied under the assumption that fields to be adjusted contain prices
1280
+ (adjustment direction is determined appropriately).
1281
+
1282
+ When set to SIZE, adjustments are applied under the assumption that fields contain sizes
1283
+ (adjustment direction is opposite to that when the parameter's value is PRICE).
1284
+
1285
+ By default the value is PRICE.
1286
+ apply_split : bool, optional
1287
+ If True, adjustments for splits are applied.
1288
+ apply_spinoff : bool, optional
1289
+ If True, adjustments for spin-offs are applied.
1290
+ apply_cash_dividend : bool, optional
1291
+ If True, adjustments for cash dividends are applied.
1292
+ apply_stock_dividend : bool, optional
1293
+ If True, adjustments for stock dividends are applied.
1294
+ apply_security_splice : bool, optional
1295
+ If True, adjustments for security splices are applied.
1296
+ apply_others : str, optional
1297
+ A comma-separated list of names of custom adjustment types to apply.
1298
+ apply_all : bool, optional
1299
+ If True, applies all types of adjustments, both built-in and custom.
1300
+
1301
+ Returns
1302
+ -------
1303
+ :py:class:`onetick.py.Source`
1304
+ A new source object with applied adjustments.
1305
+
1306
+ See also
1307
+ --------
1308
+ **CORP_ACTIONS** OneTick event processor
1309
+
1310
+ Examples
1311
+ --------
1312
+ >>> src = otp.DataSource('US_COMP',
1313
+ ... tick_type='TRD',
1314
+ ... start=otp.dt(2022, 5, 20, 9, 30),
1315
+ ... end=otp.dt(2022, 5, 26, 16))
1316
+ >>> df = otp.run(src, symbols='MKD', symbol_date=otp.date(2022, 5, 22))
1317
+ >>> df["PRICE"][0]
1318
+ 0.0911
1319
+ >>> src = src.corp_actions(adjustment_date=otp.date(2022, 5, 22),
1320
+ ... fields="PRICE")
1321
+ >>> df = otp.run(src, symbols='MKD', symbol_date=otp.date(2022, 5, 22))
1322
+ >>> df["PRICE"][0]
1323
+ 1.36649931675
1324
+ """
1325
+ return otp.functions.corp_actions(self, *args, **kwargs)