onetick-py 1.162.2__py3-none-any.whl → 1.164.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- locator_parser/actions.py +10 -14
- locator_parser/common.py +13 -10
- locator_parser/io.py +6 -4
- locator_parser/locator.py +1 -1
- onetick/doc_utilities/ot_doctest.py +1 -1
- onetick/doc_utilities/snippets.py +1 -2
- onetick/lib/instance.py +7 -4
- onetick/py/__init__.py +5 -9
- onetick/py/_version.py +1 -1
- onetick/py/aggregations/_base.py +7 -4
- onetick/py/aggregations/_docs.py +22 -7
- onetick/py/aggregations/other.py +1 -1
- onetick/py/cache.py +1 -0
- onetick/py/callback/callback.py +1 -0
- onetick/py/core/_internal/_proxy_node.py +1 -1
- onetick/py/core/_internal/_state_objects.py +2 -2
- onetick/py/core/_source/source_methods/aggregations.py +8 -9
- onetick/py/core/_source/source_methods/applyers.py +2 -2
- onetick/py/core/_source/source_methods/debugs.py +16 -14
- onetick/py/core/_source/source_methods/drops.py +1 -1
- onetick/py/core/_source/source_methods/fields.py +5 -5
- onetick/py/core/_source/source_methods/filters.py +4 -3
- onetick/py/core/_source/source_methods/joins.py +6 -6
- onetick/py/core/_source/source_methods/misc.py +84 -0
- onetick/py/core/_source/source_methods/renames.py +3 -3
- onetick/py/core/_source/source_methods/switches.py +3 -3
- onetick/py/core/_source/source_methods/writes.py +279 -10
- onetick/py/core/_source/tmp_otq.py +1 -1
- onetick/py/core/column_operations/_methods/_internal.py +1 -1
- onetick/py/core/column_operations/_methods/methods.py +8 -7
- onetick/py/core/column_operations/_methods/op_types.py +1 -0
- onetick/py/core/column_operations/accessors/dt_accessor.py +4 -0
- onetick/py/core/column_operations/base.py +5 -5
- onetick/py/core/cut_builder.py +1 -0
- onetick/py/core/eval_query.py +1 -0
- onetick/py/core/lambda_object.py +2 -3
- onetick/py/core/per_tick_script.py +6 -5
- onetick/py/core/query_inspector.py +6 -7
- onetick/py/core/source.py +11 -8
- onetick/py/db/_inspection.py +4 -8
- onetick/py/db/db.py +4 -100
- onetick/py/docs/docstring_parser.py +1 -1
- onetick/py/functions.py +48 -11
- onetick/py/license.py +2 -0
- onetick/py/math.py +2 -2
- onetick/py/otq.py +1 -2
- onetick/py/run.py +8 -7
- onetick/py/servers.py +2 -2
- onetick/py/session.py +8 -6
- onetick/py/sources/common.py +6 -4
- onetick/py/sources/data_source.py +25 -35
- onetick/py/sources/query.py +7 -7
- onetick/py/sources/symbols.py +1 -1
- onetick/py/sources/ticks.py +3 -3
- onetick/py/state.py +1 -0
- onetick/py/types.py +27 -25
- onetick/py/utils/config.py +2 -2
- onetick/py/utils/perf.py +2 -3
- onetick/py/utils/temp.py +2 -2
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/METADATA +1 -1
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/RECORD +65 -65
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/WHEEL +0 -0
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/entry_points.txt +0 -0
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/licenses/LICENSE +0 -0
- {onetick_py-1.162.2.dist-info → onetick_py-1.164.0.dist-info}/top_level.txt +0 -0
|
@@ -127,11 +127,12 @@ def table(self: 'Source', inplace=False, strict: bool = True, **schema) -> Optio
|
|
|
127
127
|
|
|
128
128
|
|
|
129
129
|
def __add_field_parse_value(value):
|
|
130
|
-
if
|
|
130
|
+
if isinstance(value, tuple):
|
|
131
131
|
value, dtype = value
|
|
132
132
|
else:
|
|
133
133
|
dtype = ott.get_object_type(value)
|
|
134
134
|
|
|
135
|
+
# pylint: disable-next=unidiomatic-typecheck
|
|
135
136
|
if type(value) is str and len(value) > ott.string.DEFAULT_LENGTH:
|
|
136
137
|
dtype = ott.string[len(value)]
|
|
137
138
|
|
|
@@ -213,6 +214,7 @@ def _replace_positive_lag_operator_with_tmp_column(value):
|
|
|
213
214
|
)
|
|
214
215
|
name = f"__{name}_{operation.index}_NEW__"
|
|
215
216
|
return _Column(name, column.dtype, column.obj_ref, precision=getattr(column, "_precision", None))
|
|
217
|
+
return None
|
|
216
218
|
|
|
217
219
|
op, replace_tuples = value._replace_parameters(fun, return_replace_tuples=True)
|
|
218
220
|
return op, {str(new): old for old, new in replace_tuples}
|
|
@@ -228,7 +230,7 @@ def _update_field(self: 'Source', field, value):
|
|
|
228
230
|
if names_mapping:
|
|
229
231
|
self.add_fields(names_mapping, inplace=True)
|
|
230
232
|
|
|
231
|
-
if
|
|
233
|
+
if isinstance(value, tuple):
|
|
232
234
|
# support to be compatible with adding fields to get rid of some strange problems
|
|
233
235
|
# but really we do not use passed type, because update field does not support it
|
|
234
236
|
value, _ = value
|
|
@@ -327,9 +329,7 @@ def _validate_before_setting(key, value):
|
|
|
327
329
|
value = value.item()
|
|
328
330
|
if not (
|
|
329
331
|
ott.is_type_supported(ott.get_object_type(value))
|
|
330
|
-
or isinstance(value, _Operation)
|
|
331
|
-
or type(value) is tuple
|
|
332
|
-
or isinstance(value, _ColumnAggregation)
|
|
332
|
+
or isinstance(value, (_Operation, tuple, _ColumnAggregation))
|
|
333
333
|
):
|
|
334
334
|
raise TypeError(f'It is not allowed to set objects of "{type(value)}" type')
|
|
335
335
|
return value
|
|
@@ -161,16 +161,16 @@ def where_clause(
|
|
|
161
161
|
condition = condition._make_python_way_bool_expression()
|
|
162
162
|
if isinstance(condition, _QueryEvalWrapper):
|
|
163
163
|
condition = condition.to_eval_string(self._tmp_otq)
|
|
164
|
-
|
|
164
|
+
where_branch = self.copy(
|
|
165
165
|
ep=otq.WhereClause(
|
|
166
166
|
where=str(condition), discard_on_match=discard_on_match, stop_on_first_mismatch=stop_on_first_mismatch
|
|
167
167
|
)
|
|
168
168
|
)
|
|
169
169
|
|
|
170
|
-
if_source =
|
|
170
|
+
if_source = where_branch.copy()
|
|
171
171
|
if_source.node().out_pin("IF")
|
|
172
172
|
|
|
173
|
-
else_source =
|
|
173
|
+
else_source = where_branch.copy()
|
|
174
174
|
else_source.node().out_pin("ELSE")
|
|
175
175
|
# TODO: add ability to remove then this ep, because it is required only for right output
|
|
176
176
|
else_source.sink(otq.Passthrough())
|
|
@@ -263,6 +263,7 @@ def _get_integer_slice(self: 'Source', item: slice) -> Optional['Source']:
|
|
|
263
263
|
raise ValueError("step value can't be negative or zero")
|
|
264
264
|
if stop is not None and stop == 0:
|
|
265
265
|
raise ValueError("stop value can't be zero")
|
|
266
|
+
# pylint: disable=chained-comparison
|
|
266
267
|
if start and stop and start > 0 and stop > 0 and start >= stop:
|
|
267
268
|
raise ValueError("stop value can't be less than start")
|
|
268
269
|
if start and start < 0 and stop and stop > 0:
|
|
@@ -201,14 +201,14 @@ def _fill_time_param_for_jwq(join_params, start_time, end_time, timezone):
|
|
|
201
201
|
|
|
202
202
|
|
|
203
203
|
def _fill_aux_params_for_joins(
|
|
204
|
-
join_params, caching, end_time, prefix, start_time, symbol_name, timezone,
|
|
204
|
+
join_params, caching, end_time, prefix, start_time, symbol_name, timezone, for_join_with_collection=False
|
|
205
205
|
):
|
|
206
|
-
if symbol_name and not
|
|
206
|
+
if symbol_name and not for_join_with_collection:
|
|
207
207
|
join_params["symbol_name"] = symbol_name
|
|
208
208
|
if prefix is not None:
|
|
209
209
|
join_params["prefix_for_output_ticks"] = str(prefix)
|
|
210
210
|
if caching:
|
|
211
|
-
if
|
|
211
|
+
if for_join_with_collection:
|
|
212
212
|
supported = ("per_symbol",)
|
|
213
213
|
else:
|
|
214
214
|
supported = ("cross_symbol", "per_symbol")
|
|
@@ -217,7 +217,7 @@ def _fill_aux_params_for_joins(
|
|
|
217
217
|
else:
|
|
218
218
|
raise ValueError(f"Unknown value for caching param, please use None or any of {supported}.")
|
|
219
219
|
_fill_time_param_for_jwq(join_params, start_time, end_time, timezone)
|
|
220
|
-
if
|
|
220
|
+
if for_join_with_collection:
|
|
221
221
|
del join_params['timezone']
|
|
222
222
|
|
|
223
223
|
|
|
@@ -510,7 +510,7 @@ def join_with_collection(
|
|
|
510
510
|
)
|
|
511
511
|
|
|
512
512
|
_fill_aux_params_for_joins(
|
|
513
|
-
join_params, caching, end, prefix, start, symbol_name=None, timezone=None,
|
|
513
|
+
join_params, caching, end, prefix, start, symbol_name=None, timezone=None, for_join_with_collection=True
|
|
514
514
|
)
|
|
515
515
|
res.sink(otq.JoinWithCollectionSummary(**join_params))
|
|
516
516
|
res._add_table()
|
|
@@ -909,7 +909,7 @@ def join_with_query(
|
|
|
909
909
|
process_query_asynchronously=process_query_async,
|
|
910
910
|
)
|
|
911
911
|
if concurrency is not None:
|
|
912
|
-
if
|
|
912
|
+
if not isinstance(concurrency, int) or concurrency <= 0:
|
|
913
913
|
raise ValueError('Wrong value of concurrency parameter passed! concurrency should be a positive integer')
|
|
914
914
|
join_params['shared_thread_count'] = concurrency
|
|
915
915
|
|
|
@@ -29,6 +29,7 @@ def inplace_operation(method):
|
|
|
29
29
|
kwargs['inplace'] = inplace
|
|
30
30
|
if inplace:
|
|
31
31
|
method(self, *args, **kwargs)
|
|
32
|
+
return None
|
|
32
33
|
else:
|
|
33
34
|
obj = self.copy()
|
|
34
35
|
return method(obj, *args, **kwargs)
|
|
@@ -279,6 +280,89 @@ def insert_tick(
|
|
|
279
280
|
return self
|
|
280
281
|
|
|
281
282
|
|
|
283
|
+
@inplace_operation
|
|
284
|
+
def insert_at_end(
|
|
285
|
+
self: 'Source',
|
|
286
|
+
*,
|
|
287
|
+
propagate_ticks: bool = True,
|
|
288
|
+
delimiter_name: str = 'AT_END',
|
|
289
|
+
inplace: bool = False,
|
|
290
|
+
) -> Optional['Source']:
|
|
291
|
+
"""
|
|
292
|
+
This function adds a field ``delimiter_name``,
|
|
293
|
+
which is set to zero for all inbound ticks
|
|
294
|
+
and set to 1 for an additional tick that is generated when the data ends.
|
|
295
|
+
|
|
296
|
+
The timestamp of the additional tick is set to the query end time.
|
|
297
|
+
The values of all fields from the input schema of additional tick are set to default values for each type.
|
|
298
|
+
|
|
299
|
+
Parameters
|
|
300
|
+
----------
|
|
301
|
+
propagate_ticks: bool
|
|
302
|
+
If True (default) this function returns all input ticks and an additionally generated tick,
|
|
303
|
+
otherwise it returns only the last generated tick.
|
|
304
|
+
delimiter_name: str
|
|
305
|
+
The name of the delimiter field to add.
|
|
306
|
+
inplace: bool
|
|
307
|
+
The flag controls whether operation should be applied inplace or not.
|
|
308
|
+
If ``inplace=True``, then it returns nothing.
|
|
309
|
+
Otherwise method returns a new modified object.
|
|
310
|
+
|
|
311
|
+
See also
|
|
312
|
+
--------
|
|
313
|
+
**INSERT_AT_END** OneTick event processor
|
|
314
|
+
|
|
315
|
+
Returns
|
|
316
|
+
-------
|
|
317
|
+
:class:`Source` or ``None``
|
|
318
|
+
|
|
319
|
+
Examples
|
|
320
|
+
--------
|
|
321
|
+
Insert tick at the end of the stream:
|
|
322
|
+
|
|
323
|
+
>>> data = otp.Ticks(A=[1, 2, 3])
|
|
324
|
+
>>> data = data.insert_at_end()
|
|
325
|
+
>>> otp.run(data)
|
|
326
|
+
Time A AT_END
|
|
327
|
+
0 2003-12-01 00:00:00.000 1 0
|
|
328
|
+
1 2003-12-01 00:00:00.001 2 0
|
|
329
|
+
2 2003-12-01 00:00:00.002 3 0
|
|
330
|
+
3 2003-12-04 00:00:00.000 0 1
|
|
331
|
+
|
|
332
|
+
The name of the added field can be changed:
|
|
333
|
+
|
|
334
|
+
>>> data = otp.Ticks(A=[1, 2, 3])
|
|
335
|
+
>>> data = data.insert_at_end(delimiter_name='LAST_TICK')
|
|
336
|
+
>>> otp.run(data)
|
|
337
|
+
Time A LAST_TICK
|
|
338
|
+
0 2003-12-01 00:00:00.000 1 0
|
|
339
|
+
1 2003-12-01 00:00:00.001 2 0
|
|
340
|
+
2 2003-12-01 00:00:00.002 3 0
|
|
341
|
+
3 2003-12-04 00:00:00.000 0 1
|
|
342
|
+
|
|
343
|
+
If parameter ``propagate_ticks`` is set to False, then only the last tick is returned:
|
|
344
|
+
|
|
345
|
+
>>> data = otp.Ticks(A=[1, 2, 3])
|
|
346
|
+
>>> data = data.insert_at_end(propagate_ticks=False)
|
|
347
|
+
>>> otp.run(data)
|
|
348
|
+
Time A AT_END
|
|
349
|
+
0 2003-12-04 0 1
|
|
350
|
+
"""
|
|
351
|
+
if not hasattr(otq, 'InsertAtEnd'):
|
|
352
|
+
raise RuntimeError("Function insert_at_end() is not available on this OneTick API version")
|
|
353
|
+
if delimiter_name in self.schema:
|
|
354
|
+
raise ValueError(f"Field '{delimiter_name}' is already in schema")
|
|
355
|
+
|
|
356
|
+
self.sink(
|
|
357
|
+
otq.InsertAtEnd(
|
|
358
|
+
propagate_ticks=propagate_ticks,
|
|
359
|
+
delimiter_name=delimiter_name,
|
|
360
|
+
)
|
|
361
|
+
)
|
|
362
|
+
self.schema.update(**{delimiter_name: int})
|
|
363
|
+
return self
|
|
364
|
+
|
|
365
|
+
|
|
282
366
|
@inplace_operation
|
|
283
367
|
def transpose(
|
|
284
368
|
self: 'Source',
|
|
@@ -71,7 +71,7 @@ def _is_excluded(s: str, not_to_rename: List[str]) -> bool:
|
|
|
71
71
|
return False
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def _get_columns_names_renaming(schema,
|
|
74
|
+
def _get_columns_names_renaming(schema, rename_dict: Dict[str, str], not_to_rename: List[str]) -> Dict[str, str]:
|
|
75
75
|
"""
|
|
76
76
|
We can't be sure python Source has consistent columns cache, because sinking complex event processors
|
|
77
77
|
can change columns unpredictable, so if user will specify regex as a param, we will pass regex
|
|
@@ -79,7 +79,7 @@ def _get_columns_names_renaming(schema, rename: Dict[str, str], not_to_rename: L
|
|
|
79
79
|
|
|
80
80
|
Parameters
|
|
81
81
|
----------
|
|
82
|
-
|
|
82
|
+
rename_dict:
|
|
83
83
|
Dict old_name -> new_name. Some of the dictionary's items use regex.
|
|
84
84
|
|
|
85
85
|
Returns
|
|
@@ -88,7 +88,7 @@ def _get_columns_names_renaming(schema, rename: Dict[str, str], not_to_rename: L
|
|
|
88
88
|
Dict old_name -> new_name used for renaming columns in Source. None of this dictionary's items use regex.
|
|
89
89
|
"""
|
|
90
90
|
output_dict = {}
|
|
91
|
-
for old_name, new_name in
|
|
91
|
+
for old_name, new_name in rename_dict.items():
|
|
92
92
|
matching_columns = [col for col in schema if re.match(old_name, col) and not _is_excluded(col, not_to_rename)]
|
|
93
93
|
for col in matching_columns:
|
|
94
94
|
output_dict[col] = re.sub(old_name, new_name, col)
|
|
@@ -72,20 +72,20 @@ def split(self: 'Source', expr, cases, default=False) -> Tuple['Source', ...]:
|
|
|
72
72
|
if default:
|
|
73
73
|
params["default_output"] = "DEF_OUT"
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
switch_branch = self.copy(ep=otq.SwitchEp(**params))
|
|
76
76
|
|
|
77
77
|
# construct results
|
|
78
78
|
result = []
|
|
79
79
|
|
|
80
80
|
for inx in range(output_num):
|
|
81
|
-
res =
|
|
81
|
+
res = switch_branch.copy()
|
|
82
82
|
res.node().out_pin(f"OUT{inx}")
|
|
83
83
|
res.sink(otq.Passthrough())
|
|
84
84
|
|
|
85
85
|
result.append(res)
|
|
86
86
|
|
|
87
87
|
if default:
|
|
88
|
-
res =
|
|
88
|
+
res = switch_branch.copy()
|
|
89
89
|
res.node().out_pin("DEF_OUT")
|
|
90
90
|
res.sink(otq.Passthrough())
|
|
91
91
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import warnings
|
|
2
|
-
|
|
2
|
+
import datetime
|
|
3
3
|
from typing import TYPE_CHECKING, Optional, Set, Type, Union
|
|
4
4
|
from onetick.py.backports import Literal
|
|
5
5
|
|
|
@@ -22,9 +22,9 @@ def write(
|
|
|
22
22
|
db: Union[str, 'otp.DB'],
|
|
23
23
|
symbol: Union[str, 'otp.Column', None] = None,
|
|
24
24
|
tick_type: Union[str, 'otp.Column', None] = None,
|
|
25
|
-
date: Union[date, Type[adaptive], None] = adaptive,
|
|
26
|
-
start: Optional[date] = None,
|
|
27
|
-
end: Optional[date] = None,
|
|
25
|
+
date: Union[datetime.date, Type[adaptive], None] = adaptive,
|
|
26
|
+
start: Optional[datetime.date] = None,
|
|
27
|
+
end: Optional[datetime.date] = None,
|
|
28
28
|
append: bool = False,
|
|
29
29
|
keep_symbol_and_tick_type: Union[bool, Type[adaptive]] = adaptive,
|
|
30
30
|
propagate: bool = True,
|
|
@@ -292,10 +292,6 @@ def write(
|
|
|
292
292
|
' Possible values are: "ignore", "exception"'
|
|
293
293
|
)
|
|
294
294
|
|
|
295
|
-
branches = []
|
|
296
|
-
if propagate:
|
|
297
|
-
branches = [self]
|
|
298
|
-
|
|
299
295
|
kwargs = dict(
|
|
300
296
|
**kwargs,
|
|
301
297
|
database=str(db),
|
|
@@ -308,13 +304,18 @@ def write(
|
|
|
308
304
|
)
|
|
309
305
|
|
|
310
306
|
if start and end:
|
|
311
|
-
days = (end - start).days
|
|
307
|
+
days = (end - start).days
|
|
308
|
+
if days < 0:
|
|
309
|
+
raise ValueError("Parameter 'start' must be less than parameter 'end'")
|
|
310
|
+
if days == 0:
|
|
311
|
+
raise ValueError("Parameters 'start' and 'end' must specify different dates")
|
|
312
|
+
branches = []
|
|
312
313
|
for i in range(days):
|
|
313
314
|
branch = self.copy()
|
|
314
315
|
branch.sink(
|
|
315
316
|
otq.WriteToOnetickDb(
|
|
316
317
|
date=(start + otp.Day(i)).strftime('%Y%m%d'),
|
|
317
|
-
propagate_ticks=
|
|
318
|
+
propagate_ticks=propagate,
|
|
318
319
|
out_of_range_tick_action='IGNORE',
|
|
319
320
|
**kwargs,
|
|
320
321
|
)
|
|
@@ -700,3 +701,271 @@ def save_snapshot(
|
|
|
700
701
|
)
|
|
701
702
|
|
|
702
703
|
return self
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
@inplace_operation
|
|
707
|
+
def write_text(
|
|
708
|
+
self: 'Source',
|
|
709
|
+
*,
|
|
710
|
+
propagate_ticks=True,
|
|
711
|
+
output_headers=True,
|
|
712
|
+
output_types_in_headers=False,
|
|
713
|
+
order=None,
|
|
714
|
+
prepend_symbol_name=True,
|
|
715
|
+
prepended_symbol_name_size=0,
|
|
716
|
+
prepend_timestamp=True,
|
|
717
|
+
separator=',',
|
|
718
|
+
formats_of_fields=None,
|
|
719
|
+
double_format='%f',
|
|
720
|
+
output_dir=None,
|
|
721
|
+
output_file=None,
|
|
722
|
+
error_file=None,
|
|
723
|
+
warning_file=None,
|
|
724
|
+
data_quality_file=None,
|
|
725
|
+
treat_input_as_binary=False,
|
|
726
|
+
flush=True,
|
|
727
|
+
append=False,
|
|
728
|
+
allow_concurrent_write=False,
|
|
729
|
+
inplace=False,
|
|
730
|
+
):
|
|
731
|
+
r"""
|
|
732
|
+
Writes the input tick series to a text file or standard output.
|
|
733
|
+
|
|
734
|
+
Parameters
|
|
735
|
+
----------
|
|
736
|
+
propagate_ticks: bool
|
|
737
|
+
If True (default) then ticks will be propagated after this method, otherwise this method won't return ticks.
|
|
738
|
+
output_headers: bool
|
|
739
|
+
Switches the output of the headers.
|
|
740
|
+
If True (default), a tick descriptor line appears in the output before the very first tick for that query.
|
|
741
|
+
If the structure of the output tick changes, another tick descriptor line appears before the first changed tick.
|
|
742
|
+
The header line starts with **#**.
|
|
743
|
+
The field names are ordered as mandated by the ``order`` parameter or,
|
|
744
|
+
if it is empty, in the order of appearance in the tick descriptor.
|
|
745
|
+
Fields that are not specified in the ``order`` parameter
|
|
746
|
+
will appear after specified ones in the order of their appearance in the tick descriptor.
|
|
747
|
+
output_types_in_headers: bool
|
|
748
|
+
Switches the output of field types in the header lines.
|
|
749
|
+
``output_types_in_headers`` can be set only when ``output_headers`` is set too.
|
|
750
|
+
order: list
|
|
751
|
+
The field appearance order in the output.
|
|
752
|
+
If all or some fields are not specified,
|
|
753
|
+
those fields will be written in the order of their appearance in the tick descriptor.
|
|
754
|
+
|
|
755
|
+
Field **SYMBOL_NAME** may be specified if parameter ``prepend_symbol_name`` is set.
|
|
756
|
+
|
|
757
|
+
Field **TIMESTAMP** may be specified if parameter ``prepend_timestamp`` is set.
|
|
758
|
+
prepend_symbol_name: bool
|
|
759
|
+
If True (default), prepends symbol name before other fields as a new field named **SYMBOL_NAME** in the header
|
|
760
|
+
(if ``output_headers`` is set).
|
|
761
|
+
prepended_symbol_name_size: int
|
|
762
|
+
When ``prepend_symbol_name`` is set, symbol will be adjusted to this size.
|
|
763
|
+
If set to 0 (default), no adjustment will be done.
|
|
764
|
+
prepend_timestamp: bool
|
|
765
|
+
If set (default), tick timestamps, formatted as *YYYYMMDDhhmmss.qqqqqq* in the GMT time zone,
|
|
766
|
+
will be prepended to the output lines.
|
|
767
|
+
Header lines, if present, will have **TIMESTAMP** as the first field name.
|
|
768
|
+
The default output format for tick timestamps can be specified in the ``formats_of_fields`` parameter.
|
|
769
|
+
separator: str
|
|
770
|
+
The delimiter string. This doesn't have to be a single character.
|
|
771
|
+
Escape sequences are allowed for **\\t** (tab), **\\\\** (\\ character) and **\\xHH** (hex codes).
|
|
772
|
+
By default "," (comma) will be used.
|
|
773
|
+
formats_of_fields: dict
|
|
774
|
+
The dictionary of field names and their formatting specifications.
|
|
775
|
+
The formatting specification is the same as in the standard C
|
|
776
|
+
`printf <https://pubs.opengroup.org/onlinepubs/009695399/functions/printf.html>`_ function.
|
|
777
|
+
|
|
778
|
+
For float and decimal fields **%f** and **%.[<precision>]f** formats are only supported,
|
|
779
|
+
first one being the default an outputting 6 decimal digits.
|
|
780
|
+
|
|
781
|
+
Also if the field format starts with **%|**,
|
|
782
|
+
it means that this is a timestamp field and should be in the format **%|tz|time_format_spec**,
|
|
783
|
+
where the *tz* is the time zone name (if not specified GMT will be used),
|
|
784
|
+
and *time_format_spec* is a custom time format specification,
|
|
785
|
+
which is the same as the one used by the
|
|
786
|
+
`strftime <https://pubs.opengroup.org/onlinepubs/009695399/functions/strftime.html>`_ function.
|
|
787
|
+
|
|
788
|
+
In addition, you can also use **%q** , **%Q** , **%k** and **%J** placeholders,
|
|
789
|
+
which will be replaced by 3 and 2 sign milliseconds, 6 sign microseconds and 9 sign nanoseconds, respectively.
|
|
790
|
+
|
|
791
|
+
**%#**, **%-**, **%U**, **%N** placeholders will be replaced by Unix timestamp, Unix timestamp in milliseconds,
|
|
792
|
+
microseconds and nanoseconds, respectively.
|
|
793
|
+
|
|
794
|
+
**%+** and **%~** placeholders will be replaced by milliseconds and nanoseconds passed since midnight.
|
|
795
|
+
double_format: str
|
|
796
|
+
This format will be used for fields that are holding double values
|
|
797
|
+
if they are not specified in ``formats_of_fields``.
|
|
798
|
+
output_dir: str
|
|
799
|
+
If specified, all output (output, warning, error, and data quality) files will be redirected to it.
|
|
800
|
+
If this directory does not exist, it will get created.
|
|
801
|
+
By default, the current directory is used.
|
|
802
|
+
output_file: str
|
|
803
|
+
The output file name for generated text data.
|
|
804
|
+
If not set, the standard output will be used.
|
|
805
|
+
It is also possible to add symbol name, database name, tick type,
|
|
806
|
+
date of tick and query start time to the file name.
|
|
807
|
+
For this special placeholders should be used, which will be replaced with the appropriate values:
|
|
808
|
+
|
|
809
|
+
* **%SYMBOL%** - will be replaced with symbol name,
|
|
810
|
+
* **%DBNAME%** - with database name,
|
|
811
|
+
* **%TICKTYPE%** - with tick type,
|
|
812
|
+
* **%DATE%** - with date of tick,
|
|
813
|
+
* **%STARTTIME%** - with start time of the query.
|
|
814
|
+
|
|
815
|
+
.. note::
|
|
816
|
+
In case of using placeholders the output of the data may be split into different files.
|
|
817
|
+
For example when querying several days of data and using **%DATE%** placeholder,
|
|
818
|
+
the file will be created for every day of the interval.
|
|
819
|
+
|
|
820
|
+
This format is also available for ``error_file``, ``warning_file`` and ``data_quality_file`` input parameters.
|
|
821
|
+
error_file: str
|
|
822
|
+
The file name where all error messages are directed.
|
|
823
|
+
If not set the standard error will be used.
|
|
824
|
+
warning_file: str
|
|
825
|
+
The file name where all warning messages are directed.
|
|
826
|
+
If not set the standard error will be used.
|
|
827
|
+
data_quality_file: str
|
|
828
|
+
The file name where all data quality messages are directed.
|
|
829
|
+
If not set the standard error will be used.
|
|
830
|
+
treat_input_as_binary: bool
|
|
831
|
+
Opens output file in binary mode to not modify content of ticks when printing them to the file.
|
|
832
|
+
Also in this mode method prints no new line to the file after every tick write.
|
|
833
|
+
flush: bool
|
|
834
|
+
If True (default) then the output will be flushed to disk after every tick.
|
|
835
|
+
|
|
836
|
+
.. note::
|
|
837
|
+
Notice that while this setting makes results of the query recorded into a file without delay,
|
|
838
|
+
making them immediately available to applications that read this file,
|
|
839
|
+
it may slow down the query significantly.
|
|
840
|
+
|
|
841
|
+
append: bool
|
|
842
|
+
If set to True, will try to append data to files (output, error, warning, data_quality), instead of overwriting.
|
|
843
|
+
allow_concurrent_write: bool
|
|
844
|
+
Allows different queries running on the same server to write concurrently to the same files
|
|
845
|
+
(output, error, warning, data_quality).
|
|
846
|
+
inplace: bool
|
|
847
|
+
A flag controls whether operation should be applied inplace.
|
|
848
|
+
If ``inplace=True``, then it returns nothing. Otherwise method
|
|
849
|
+
returns a new modified object.
|
|
850
|
+
|
|
851
|
+
See also
|
|
852
|
+
--------
|
|
853
|
+
| **WRITE_TEXT** OneTick event processor
|
|
854
|
+
| :py:meth:`onetick.py.Source.dump`
|
|
855
|
+
|
|
856
|
+
Examples
|
|
857
|
+
--------
|
|
858
|
+
|
|
859
|
+
By default the text is written to the standard output:
|
|
860
|
+
|
|
861
|
+
>>> data = otp.Ticks(A=[1, 2, 3])
|
|
862
|
+
>>> write = data.write_text()
|
|
863
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
864
|
+
#SYMBOL_NAME,TIMESTAMP,A
|
|
865
|
+
AAPL,20031201050000.000000,1
|
|
866
|
+
AAPL,20031201050000.001000,2
|
|
867
|
+
AAPL,20031201050000.002000,3
|
|
868
|
+
|
|
869
|
+
Output file can also be specified:
|
|
870
|
+
|
|
871
|
+
>>> write = data.write_text(output_file='result.csv')
|
|
872
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
873
|
+
>>> with open('result.csv') as f: # doctest: +SKIP
|
|
874
|
+
... print(f.read()) # doctest: +SKIP
|
|
875
|
+
#SYMBOL_NAME,TIMESTAMP,A
|
|
876
|
+
AAPL,20031201050000.000000,1
|
|
877
|
+
AAPL,20031201050000.001000,2
|
|
878
|
+
AAPL,20031201050000.002000,3
|
|
879
|
+
|
|
880
|
+
Symbol name, timestamp of the tick and can be removed from the output:
|
|
881
|
+
|
|
882
|
+
>>> write = data.write_text(prepend_timestamp=False,
|
|
883
|
+
... prepend_symbol_name=False)
|
|
884
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
885
|
+
#A
|
|
886
|
+
1
|
|
887
|
+
2
|
|
888
|
+
3
|
|
889
|
+
|
|
890
|
+
The header can also be removed from the output:
|
|
891
|
+
|
|
892
|
+
>>> write = data.write_text(output_headers=False)
|
|
893
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
894
|
+
AAPL,20031201050000.000000,1
|
|
895
|
+
AAPL,20031201050000.001000,2
|
|
896
|
+
AAPL,20031201050000.002000,3
|
|
897
|
+
|
|
898
|
+
The order of fields and separator character can be specified:
|
|
899
|
+
|
|
900
|
+
>>> write = data.write_text(order=['A', 'TIMESTAMP'],
|
|
901
|
+
... separator='\t',
|
|
902
|
+
... prepend_symbol_name=False)
|
|
903
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
904
|
+
#A TIMESTAMP
|
|
905
|
+
1 20031201050000.000000
|
|
906
|
+
2 20031201050000.001000
|
|
907
|
+
3 20031201050000.002000
|
|
908
|
+
|
|
909
|
+
The formatting can be specified for each field:
|
|
910
|
+
|
|
911
|
+
>>> write = data.write_text(formats_of_fields={
|
|
912
|
+
... 'TIMESTAMP': '%|GMT|%Y-%m-%d %H:%M:%S.%q',
|
|
913
|
+
... 'A': '%3d'
|
|
914
|
+
... })
|
|
915
|
+
>>> _ = otp.run(write) # doctest: +SKIP
|
|
916
|
+
#SYMBOL_NAME,TIMESTAMP,A
|
|
917
|
+
AAPL,2003-12-01 05:00:00.000, 1
|
|
918
|
+
AAPL,2003-12-01 05:00:00.001, 2
|
|
919
|
+
AAPL,2003-12-01 05:00:00.002, 3
|
|
920
|
+
"""
|
|
921
|
+
if output_types_in_headers and not output_headers:
|
|
922
|
+
raise ValueError("Parameter 'output_types_in_headers' can only be set together with 'output_headers'")
|
|
923
|
+
|
|
924
|
+
order = order or []
|
|
925
|
+
formats_of_fields = formats_of_fields or {}
|
|
926
|
+
for field in list(order) + list(formats_of_fields):
|
|
927
|
+
if prepend_symbol_name and field == 'SYMBOL_NAME':
|
|
928
|
+
continue
|
|
929
|
+
if not prepend_symbol_name and field == 'SYMBOL_NAME' and field not in self.schema:
|
|
930
|
+
raise ValueError(
|
|
931
|
+
"Field 'SYMBOL_NAME' can't be specified in 'order' parameter if 'prepend_symbol_name' is not set"
|
|
932
|
+
)
|
|
933
|
+
if not prepend_timestamp and field == 'TIMESTAMP':
|
|
934
|
+
raise ValueError(
|
|
935
|
+
"Field 'TIMESTAMP' can't be specified in 'order' parameter if 'prepend_timestamp' is not set"
|
|
936
|
+
)
|
|
937
|
+
if field not in self.schema:
|
|
938
|
+
raise ValueError(f"Field '{field}' is not in schema")
|
|
939
|
+
|
|
940
|
+
kwargs = dict(
|
|
941
|
+
propagate_ticks=propagate_ticks,
|
|
942
|
+
output_headers=output_headers,
|
|
943
|
+
output_types_in_headers=output_types_in_headers,
|
|
944
|
+
order='|'.join(order),
|
|
945
|
+
prepend_symbol_name=prepend_symbol_name,
|
|
946
|
+
prepended_symbol_name_size=prepended_symbol_name_size,
|
|
947
|
+
prepend_timestamp=prepend_timestamp,
|
|
948
|
+
# OneTick uses \ as an escape character,
|
|
949
|
+
# so replacing a single \ character with two \\ characters to escape it in OneTick
|
|
950
|
+
separator=separator.replace('\\', r'\\'),
|
|
951
|
+
formats_of_fields='\n'.join(f'{k}={v}' for k, v in formats_of_fields.items()),
|
|
952
|
+
double_format=double_format,
|
|
953
|
+
output_dir=output_dir,
|
|
954
|
+
output_file=output_file,
|
|
955
|
+
error_file=error_file,
|
|
956
|
+
warning_file=warning_file,
|
|
957
|
+
data_quality_file=data_quality_file,
|
|
958
|
+
treat_input_as_binary=treat_input_as_binary,
|
|
959
|
+
flush=flush,
|
|
960
|
+
append=append,
|
|
961
|
+
allow_concurrent_write=allow_concurrent_write,
|
|
962
|
+
)
|
|
963
|
+
for k, v in kwargs.items():
|
|
964
|
+
if v is None:
|
|
965
|
+
# None values may not be supported by onetick.query
|
|
966
|
+
kwargs[k] = ''
|
|
967
|
+
|
|
968
|
+
self.sink(otq.WriteText(**kwargs))
|
|
969
|
+
if not propagate_ticks:
|
|
970
|
+
self.schema.set(**{})
|
|
971
|
+
return self
|
|
@@ -21,7 +21,7 @@ def is_millisecond_precision(dt):
|
|
|
21
21
|
"""
|
|
22
22
|
if (dt.microsecond % 1000) != 0:
|
|
23
23
|
return False
|
|
24
|
-
if isinstance(dt, pd_Timestamp
|
|
24
|
+
if isinstance(dt, (pd_Timestamp, otp_datetime)):
|
|
25
25
|
if dt.nanosecond != 0:
|
|
26
26
|
return False
|
|
27
27
|
return True
|
|
@@ -15,7 +15,7 @@ def _wrap_object(o):
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def _type_error_for_op(op, types):
|
|
18
|
-
|
|
18
|
+
return TypeError(f"Unsupported operand type(s) for {op} operation: {types}")
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
def _init_binary_op(prev_op, other):
|
|
@@ -70,6 +70,7 @@ def _return_dateadd_command(prev_op, other, left, right, left_t, right_t, op_sig
|
|
|
70
70
|
return _form_method_result_for_dateadd(right, right_t, op_sign, prev_op)
|
|
71
71
|
elif issubclass(right_t, ott.OTPBaseTimeOffset) and are_time(left_t):
|
|
72
72
|
return _form_method_result_for_dateadd(left, left_t, op_sign, other)
|
|
73
|
+
return None
|
|
73
74
|
|
|
74
75
|
|
|
75
76
|
def _form_method_result_for_dateadd(op_str, dtype, op_sign, datepart):
|
|
@@ -102,7 +103,7 @@ def _plus(prev_op, other, left, right, left_t, right_t):
|
|
|
102
103
|
if dtype:
|
|
103
104
|
return MethodResult(op_str, dtype)
|
|
104
105
|
else:
|
|
105
|
-
_type_error_for_op("+", f"'{left_t}' and '{right_t}'")
|
|
106
|
+
raise _type_error_for_op("+", f"'{left_t}' and '{right_t}'")
|
|
106
107
|
|
|
107
108
|
|
|
108
109
|
def _minus(prev_op, other, left, right, left_t, right_t): # noqa # NOSONAR
|
|
@@ -122,7 +123,7 @@ def _minus(prev_op, other, left, right, left_t, right_t): # noqa # NOSONAR
|
|
|
122
123
|
if dtype:
|
|
123
124
|
return MethodResult(op_str, dtype)
|
|
124
125
|
else:
|
|
125
|
-
_type_error_for_op("-", f"'{left_t}' and '{right_t}'")
|
|
126
|
+
raise _type_error_for_op("-", f"'{left_t}' and '{right_t}'")
|
|
126
127
|
|
|
127
128
|
|
|
128
129
|
def _get_dtype_for_plus_or_minus(left_t, right_t):
|
|
@@ -165,13 +166,13 @@ def mul(prev_op, other):
|
|
|
165
166
|
if dtype and op_str:
|
|
166
167
|
return MethodResult(op_str, dtype)
|
|
167
168
|
else:
|
|
168
|
-
_type_error_for_op("*", f"'{left_t}' and '{right_t}'")
|
|
169
|
+
raise _type_error_for_op("*", f"'{left_t}' and '{right_t}'")
|
|
169
170
|
|
|
170
171
|
|
|
171
172
|
def div(prev_op, other):
|
|
172
173
|
left, right, left_t, right_t, op_str, dtype = _init_binary_op(prev_op, other)
|
|
173
174
|
if not are_numerics(left_t, right_t):
|
|
174
|
-
_type_error_for_op("/", f"'{left_t}' and '{right_t}'")
|
|
175
|
+
raise _type_error_for_op("/", f"'{left_t}' and '{right_t}'")
|
|
175
176
|
dtype = _get_widest_type(_get_widest_type(left_t, right_t), float)
|
|
176
177
|
op_str = f"{left} / {right}"
|
|
177
178
|
return MethodResult(op_str, dtype)
|
|
@@ -180,7 +181,7 @@ def div(prev_op, other):
|
|
|
180
181
|
def mod(prev_op, other):
|
|
181
182
|
left, right, left_t, right_t, op_str, dtype = _init_binary_op(prev_op, other)
|
|
182
183
|
if not are_ints_not_time(left_t, right_t):
|
|
183
|
-
_type_error_for_op("mod", f"'{left_t}' and '{right_t}'")
|
|
184
|
+
raise _type_error_for_op("mod", f"'{left_t}' and '{right_t}'")
|
|
184
185
|
dtype = int
|
|
185
186
|
op_str = f"mod({left}, {right})"
|
|
186
187
|
return MethodResult(op_str, dtype)
|
|
@@ -264,7 +265,7 @@ def _compare(prev_op, other, op_sign, op_sign_python_level=None):
|
|
|
264
265
|
return MethodResult(op_str, bool)
|
|
265
266
|
else:
|
|
266
267
|
# replace = with == for comparisions
|
|
267
|
-
_type_error_for_op(f"{op_sign_python_level or op_sign}", f"'{left_t}' and '{right_t}'")
|
|
268
|
+
raise _type_error_for_op(f"{op_sign_python_level or op_sign}", f"'{left_t}' and '{right_t}'")
|
|
268
269
|
|
|
269
270
|
|
|
270
271
|
def and_(prev_op, other):
|
|
@@ -281,7 +282,7 @@ def _and_or(prev_op, other, op_sign):
|
|
|
281
282
|
op_str = f"{left} {op_sign} {right}"
|
|
282
283
|
return MethodResult(op_str, prev_op.dtype)
|
|
283
284
|
else:
|
|
284
|
-
_type_error_for_op(f"{op_sign}", f"'{left_t}' and '{right_t}'")
|
|
285
|
+
raise _type_error_for_op(f"{op_sign}", f"'{left_t}' and '{right_t}'")
|
|
285
286
|
|
|
286
287
|
|
|
287
288
|
def is_arithmetical(op):
|