onetick-py 1.177.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- locator_parser/__init__.py +0 -0
- locator_parser/acl.py +73 -0
- locator_parser/actions.py +262 -0
- locator_parser/common.py +368 -0
- locator_parser/io.py +43 -0
- locator_parser/locator.py +150 -0
- onetick/__init__.py +101 -0
- onetick/doc_utilities/__init__.py +3 -0
- onetick/doc_utilities/napoleon.py +40 -0
- onetick/doc_utilities/ot_doctest.py +140 -0
- onetick/doc_utilities/snippets.py +279 -0
- onetick/lib/__init__.py +4 -0
- onetick/lib/instance.py +141 -0
- onetick/py/__init__.py +293 -0
- onetick/py/_stack_info.py +89 -0
- onetick/py/_version.py +2 -0
- onetick/py/aggregations/__init__.py +11 -0
- onetick/py/aggregations/_base.py +648 -0
- onetick/py/aggregations/_docs.py +948 -0
- onetick/py/aggregations/compute.py +286 -0
- onetick/py/aggregations/functions.py +2216 -0
- onetick/py/aggregations/generic.py +104 -0
- onetick/py/aggregations/high_low.py +80 -0
- onetick/py/aggregations/num_distinct.py +83 -0
- onetick/py/aggregations/order_book.py +501 -0
- onetick/py/aggregations/other.py +1014 -0
- onetick/py/backports.py +26 -0
- onetick/py/cache.py +374 -0
- onetick/py/callback/__init__.py +5 -0
- onetick/py/callback/callback.py +276 -0
- onetick/py/callback/callbacks.py +131 -0
- onetick/py/compatibility.py +798 -0
- onetick/py/configuration.py +771 -0
- onetick/py/core/__init__.py +0 -0
- onetick/py/core/_csv_inspector.py +93 -0
- onetick/py/core/_internal/__init__.py +0 -0
- onetick/py/core/_internal/_manually_bound_value.py +6 -0
- onetick/py/core/_internal/_nodes_history.py +250 -0
- onetick/py/core/_internal/_op_utils/__init__.py +0 -0
- onetick/py/core/_internal/_op_utils/every_operand.py +9 -0
- onetick/py/core/_internal/_op_utils/is_const.py +10 -0
- onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script +121 -0
- onetick/py/core/_internal/_proxy_node.py +140 -0
- onetick/py/core/_internal/_state_objects.py +2312 -0
- onetick/py/core/_internal/_state_vars.py +93 -0
- onetick/py/core/_source/__init__.py +0 -0
- onetick/py/core/_source/_symbol_param.py +95 -0
- onetick/py/core/_source/schema.py +97 -0
- onetick/py/core/_source/source_methods/__init__.py +0 -0
- onetick/py/core/_source/source_methods/aggregations.py +809 -0
- onetick/py/core/_source/source_methods/applyers.py +296 -0
- onetick/py/core/_source/source_methods/columns.py +141 -0
- onetick/py/core/_source/source_methods/data_quality.py +301 -0
- onetick/py/core/_source/source_methods/debugs.py +272 -0
- onetick/py/core/_source/source_methods/drops.py +120 -0
- onetick/py/core/_source/source_methods/fields.py +619 -0
- onetick/py/core/_source/source_methods/filters.py +1002 -0
- onetick/py/core/_source/source_methods/joins.py +1413 -0
- onetick/py/core/_source/source_methods/merges.py +605 -0
- onetick/py/core/_source/source_methods/misc.py +1455 -0
- onetick/py/core/_source/source_methods/pandases.py +155 -0
- onetick/py/core/_source/source_methods/renames.py +356 -0
- onetick/py/core/_source/source_methods/sorts.py +183 -0
- onetick/py/core/_source/source_methods/switches.py +142 -0
- onetick/py/core/_source/source_methods/symbols.py +117 -0
- onetick/py/core/_source/source_methods/times.py +627 -0
- onetick/py/core/_source/source_methods/writes.py +986 -0
- onetick/py/core/_source/symbol.py +205 -0
- onetick/py/core/_source/tmp_otq.py +222 -0
- onetick/py/core/column.py +209 -0
- onetick/py/core/column_operations/__init__.py +0 -0
- onetick/py/core/column_operations/_methods/__init__.py +4 -0
- onetick/py/core/column_operations/_methods/_internal.py +28 -0
- onetick/py/core/column_operations/_methods/conversions.py +216 -0
- onetick/py/core/column_operations/_methods/methods.py +292 -0
- onetick/py/core/column_operations/_methods/op_types.py +160 -0
- onetick/py/core/column_operations/accessors/__init__.py +0 -0
- onetick/py/core/column_operations/accessors/_accessor.py +28 -0
- onetick/py/core/column_operations/accessors/decimal_accessor.py +104 -0
- onetick/py/core/column_operations/accessors/dt_accessor.py +537 -0
- onetick/py/core/column_operations/accessors/float_accessor.py +184 -0
- onetick/py/core/column_operations/accessors/str_accessor.py +1367 -0
- onetick/py/core/column_operations/base.py +1121 -0
- onetick/py/core/cut_builder.py +150 -0
- onetick/py/core/db_constants.py +20 -0
- onetick/py/core/eval_query.py +245 -0
- onetick/py/core/lambda_object.py +441 -0
- onetick/py/core/multi_output_source.py +232 -0
- onetick/py/core/per_tick_script.py +2256 -0
- onetick/py/core/query_inspector.py +464 -0
- onetick/py/core/source.py +1744 -0
- onetick/py/db/__init__.py +2 -0
- onetick/py/db/_inspection.py +1128 -0
- onetick/py/db/db.py +1327 -0
- onetick/py/db/utils.py +64 -0
- onetick/py/docs/__init__.py +0 -0
- onetick/py/docs/docstring_parser.py +112 -0
- onetick/py/docs/utils.py +81 -0
- onetick/py/functions.py +2398 -0
- onetick/py/license.py +190 -0
- onetick/py/log.py +88 -0
- onetick/py/math.py +935 -0
- onetick/py/misc.py +470 -0
- onetick/py/oqd/__init__.py +22 -0
- onetick/py/oqd/eps.py +1195 -0
- onetick/py/oqd/sources.py +325 -0
- onetick/py/otq.py +216 -0
- onetick/py/pyomd_mock.py +47 -0
- onetick/py/run.py +916 -0
- onetick/py/servers.py +173 -0
- onetick/py/session.py +1347 -0
- onetick/py/sources/__init__.py +19 -0
- onetick/py/sources/cache.py +167 -0
- onetick/py/sources/common.py +128 -0
- onetick/py/sources/csv.py +642 -0
- onetick/py/sources/custom.py +85 -0
- onetick/py/sources/data_file.py +305 -0
- onetick/py/sources/data_source.py +1045 -0
- onetick/py/sources/empty.py +94 -0
- onetick/py/sources/odbc.py +337 -0
- onetick/py/sources/order_book.py +271 -0
- onetick/py/sources/parquet.py +168 -0
- onetick/py/sources/pit.py +191 -0
- onetick/py/sources/query.py +495 -0
- onetick/py/sources/snapshots.py +419 -0
- onetick/py/sources/split_query_output_by_symbol.py +198 -0
- onetick/py/sources/symbology_mapping.py +123 -0
- onetick/py/sources/symbols.py +374 -0
- onetick/py/sources/ticks.py +825 -0
- onetick/py/sql.py +70 -0
- onetick/py/state.py +251 -0
- onetick/py/types.py +2131 -0
- onetick/py/utils/__init__.py +70 -0
- onetick/py/utils/acl.py +93 -0
- onetick/py/utils/config.py +186 -0
- onetick/py/utils/default.py +49 -0
- onetick/py/utils/file.py +38 -0
- onetick/py/utils/helpers.py +76 -0
- onetick/py/utils/locator.py +94 -0
- onetick/py/utils/perf.py +498 -0
- onetick/py/utils/query.py +49 -0
- onetick/py/utils/render.py +1374 -0
- onetick/py/utils/script.py +244 -0
- onetick/py/utils/temp.py +471 -0
- onetick/py/utils/types.py +120 -0
- onetick/py/utils/tz.py +84 -0
- onetick_py-1.177.0.dist-info/METADATA +137 -0
- onetick_py-1.177.0.dist-info/RECORD +152 -0
- onetick_py-1.177.0.dist-info/WHEEL +5 -0
- onetick_py-1.177.0.dist-info/entry_points.txt +2 -0
- onetick_py-1.177.0.dist-info/licenses/LICENSE +21 -0
- onetick_py-1.177.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
from onetick import py as otp
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
import pandas
|
|
7
|
+
from onetick.py.core.source import Source
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def plot(self: 'Source', y, x='Time', kind='line', **kwargs):
|
|
11
|
+
"""
|
|
12
|
+
Executes the query with known properties and builds a plot resulting dataframe.
|
|
13
|
+
|
|
14
|
+
Uses the :pandas:`pandas.DataFrame.plot` method to plot data.
|
|
15
|
+
Other parameters could be specified through the ``kwargs``.
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
x: str
|
|
20
|
+
Column name for the X axis
|
|
21
|
+
y: str
|
|
22
|
+
Column name for the Y axis
|
|
23
|
+
kind: str
|
|
24
|
+
The kind of plot
|
|
25
|
+
|
|
26
|
+
Examples
|
|
27
|
+
--------
|
|
28
|
+
>>> data = otp.Ticks(X=[1, 2, 3])
|
|
29
|
+
>>> data.plot(y='X', kind='bar') # doctest: +SKIP
|
|
30
|
+
"""
|
|
31
|
+
result = self.copy()
|
|
32
|
+
return result[[y, x]]().plot(x=x, y=y, kind=kind, **kwargs)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def count(self: 'Source', **kwargs) -> int:
|
|
36
|
+
"""
|
|
37
|
+
Returns the number of ticks in the query.
|
|
38
|
+
|
|
39
|
+
Adds an aggregation that calculate total ticks count, and *executes a query*.
|
|
40
|
+
Result is a single value -- number of ticks. Possible application is the Jupyter when
|
|
41
|
+
a developer wants to check data presences for example.
|
|
42
|
+
|
|
43
|
+
Parameters
|
|
44
|
+
----------
|
|
45
|
+
kwargs
|
|
46
|
+
parameters that will be passed to :py:func:`otp.run <onetick.py.run>`
|
|
47
|
+
|
|
48
|
+
Returns
|
|
49
|
+
-------
|
|
50
|
+
int
|
|
51
|
+
|
|
52
|
+
See Also
|
|
53
|
+
--------
|
|
54
|
+
| :py:func:`onetick.py.agg.count`
|
|
55
|
+
| :py:func:`otp.run <onetick.py.run>`
|
|
56
|
+
| :py:meth:`onetick.py.Source.head`
|
|
57
|
+
| :py:meth:`onetick.py.Source.tail`
|
|
58
|
+
|
|
59
|
+
Examples
|
|
60
|
+
--------
|
|
61
|
+
|
|
62
|
+
>>> data = otp.Ticks(X=[1, 2, 3])
|
|
63
|
+
>>> data.count()
|
|
64
|
+
3
|
|
65
|
+
|
|
66
|
+
>>> data = otp.Empty()
|
|
67
|
+
>>> data.count()
|
|
68
|
+
0
|
|
69
|
+
"""
|
|
70
|
+
result = self.copy()
|
|
71
|
+
df = otp.run(result.agg({'__num_rows': otp.agg.count()}), **kwargs)
|
|
72
|
+
if df.empty:
|
|
73
|
+
return 0
|
|
74
|
+
return int(df['__num_rows'][0])
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def head(self: 'Source', n=5, **kwargs) -> 'pandas.DataFrame':
|
|
78
|
+
"""
|
|
79
|
+
*Executes the query* and returns first ``n`` ticks as a pandas dataframe.
|
|
80
|
+
|
|
81
|
+
It is useful in the Jupyter case when you want to observe first ``n`` values.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
n: int, default=5
|
|
86
|
+
number of ticks to return
|
|
87
|
+
kwargs:
|
|
88
|
+
parameters will be passed to :py:func:`otp.run <onetick.py.run>`
|
|
89
|
+
|
|
90
|
+
Returns
|
|
91
|
+
-------
|
|
92
|
+
:pandas:`DataFrame <pandas.DataFrame>`
|
|
93
|
+
|
|
94
|
+
See Also
|
|
95
|
+
--------
|
|
96
|
+
| :py:func:`onetick.py.agg.first`
|
|
97
|
+
| :py:func:`otp.run <onetick.py.run>`
|
|
98
|
+
| :py:meth:`onetick.py.Source.tail`
|
|
99
|
+
| :py:meth:`onetick.py.Source.count`
|
|
100
|
+
|
|
101
|
+
Examples
|
|
102
|
+
--------
|
|
103
|
+
|
|
104
|
+
>>> data = otp.Ticks(X=list('abcdefgik'))
|
|
105
|
+
>>> data.head()[['X']]
|
|
106
|
+
X
|
|
107
|
+
0 a
|
|
108
|
+
1 b
|
|
109
|
+
2 c
|
|
110
|
+
3 d
|
|
111
|
+
4 e
|
|
112
|
+
"""
|
|
113
|
+
result = self.copy()
|
|
114
|
+
result = result.first(n=n) # pylint: disable=E1123
|
|
115
|
+
return otp.run(result, **kwargs)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def tail(self: 'Source', n=5, **kwargs) -> 'pandas.DataFrame':
|
|
119
|
+
"""
|
|
120
|
+
*Executes the query* and returns last ``n`` ticks as a pandas dataframe.
|
|
121
|
+
|
|
122
|
+
It is useful in the Jupyter case when you want to observe last ``n`` values.
|
|
123
|
+
|
|
124
|
+
Parameters
|
|
125
|
+
----------
|
|
126
|
+
n: int
|
|
127
|
+
number of ticks to return
|
|
128
|
+
kwargs:
|
|
129
|
+
parameters will be passed to :py:func:`otp.run <onetick.py.run>`
|
|
130
|
+
|
|
131
|
+
Returns
|
|
132
|
+
-------
|
|
133
|
+
:pandas:`DataFrame <pandas.DataFrame>`
|
|
134
|
+
|
|
135
|
+
See Also
|
|
136
|
+
--------
|
|
137
|
+
| :py:func:`onetick.py.agg.last`
|
|
138
|
+
| :py:func:`otp.run <onetick.py.run>`
|
|
139
|
+
| :py:meth:`onetick.py.Source.head`
|
|
140
|
+
| :py:meth:`onetick.py.Source.count`
|
|
141
|
+
|
|
142
|
+
Examples
|
|
143
|
+
--------
|
|
144
|
+
>>> data = otp.Ticks(X=list('abcdefgik'))
|
|
145
|
+
>>> data.tail()[['X']]
|
|
146
|
+
X
|
|
147
|
+
0 e
|
|
148
|
+
1 f
|
|
149
|
+
2 g
|
|
150
|
+
3 i
|
|
151
|
+
4 k
|
|
152
|
+
"""
|
|
153
|
+
result = self.copy()
|
|
154
|
+
result = result.last(n=n) # pylint: disable=E1123
|
|
155
|
+
return otp.run(result, **kwargs)
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
from onetick.py.core.column import _Column
|
|
5
|
+
from onetick.py.otq import otq
|
|
6
|
+
|
|
7
|
+
from .misc import inplace_operation
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from onetick.py.core.source import Source
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _add_prefix_and_suffix(
|
|
14
|
+
self: 'Source',
|
|
15
|
+
prefix='',
|
|
16
|
+
suffix='',
|
|
17
|
+
columns=None,
|
|
18
|
+
ignore_columns=None,
|
|
19
|
+
) -> Optional['Source']:
|
|
20
|
+
if not prefix and not suffix:
|
|
21
|
+
raise ValueError('Both suffix and prefix are empty')
|
|
22
|
+
if ' ' in prefix:
|
|
23
|
+
raise ValueError(f'There is space in prefix: {prefix}')
|
|
24
|
+
if ' ' in suffix:
|
|
25
|
+
raise ValueError(f'There is space in suffix: {prefix}')
|
|
26
|
+
columns = columns or []
|
|
27
|
+
ignore_columns = ignore_columns or []
|
|
28
|
+
if columns and ignore_columns:
|
|
29
|
+
raise ValueError('It is allowed to use only one of `columns` or `ignore_columns` parameters at a time')
|
|
30
|
+
schema = self.schema
|
|
31
|
+
for column in columns:
|
|
32
|
+
if column not in schema:
|
|
33
|
+
raise AttributeError(f'Column `{column}` does not exist in the schema')
|
|
34
|
+
for column_name in columns or schema:
|
|
35
|
+
if column_name in ignore_columns:
|
|
36
|
+
continue
|
|
37
|
+
new_column_name = f'{prefix}{column_name}{suffix}'
|
|
38
|
+
if new_column_name in self.__dict__:
|
|
39
|
+
if (not columns and not ignore_columns
|
|
40
|
+
or columns and new_column_name in columns
|
|
41
|
+
or ignore_columns and new_column_name not in ignore_columns):
|
|
42
|
+
# if the column with the same name already exists, but will be renamed too,
|
|
43
|
+
# then we don't need to raise exception, as it will have a different name after renaming
|
|
44
|
+
continue
|
|
45
|
+
if prefix:
|
|
46
|
+
raise AttributeError(f'Column {new_column_name} already exists, please, use another prefix')
|
|
47
|
+
else:
|
|
48
|
+
raise AttributeError(f'Column {new_column_name} already exists, please, use another suffix')
|
|
49
|
+
for column_name in columns or schema:
|
|
50
|
+
if column_name in ignore_columns:
|
|
51
|
+
continue
|
|
52
|
+
new_column_name = f'{prefix}{column_name}{suffix}'
|
|
53
|
+
self.__dict__[column_name].rename(new_column_name, update_parent_object=False)
|
|
54
|
+
self.__dict__[new_column_name] = self.__dict__[column_name]
|
|
55
|
+
del self.__dict__[column_name]
|
|
56
|
+
if not columns and not ignore_columns:
|
|
57
|
+
self.sink(otq.RenameFieldsEp(rename_fields=f'(.*)={prefix}\\1{suffix}', use_regex=True))
|
|
58
|
+
elif columns:
|
|
59
|
+
renames = [f'{column}={prefix}{column}{suffix}' for column in columns]
|
|
60
|
+
self.sink(otq.RenameFieldsEp(rename_fields=','.join(renames)))
|
|
61
|
+
else:
|
|
62
|
+
renames = [f'{column}={prefix}{column}{suffix}' for column in schema if column not in ignore_columns]
|
|
63
|
+
self.sink(otq.RenameFieldsEp(rename_fields=','.join(renames)))
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_excluded(s: str, not_to_rename: List[str]) -> bool:
|
|
68
|
+
for excluded in not_to_rename:
|
|
69
|
+
if re.match(excluded, s):
|
|
70
|
+
return True
|
|
71
|
+
return False
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _get_columns_names_renaming(schema, rename_dict: Dict[str, str], not_to_rename: List[str]) -> Dict[str, str]:
|
|
75
|
+
"""
|
|
76
|
+
We can't be sure python Source has consistent columns cache, because sinking complex event processors
|
|
77
|
+
can change columns unpredictable, so if user will specify regex as a param, we will pass regex
|
|
78
|
+
as an onetick's param, but rename all matched columns from python Source cache.
|
|
79
|
+
|
|
80
|
+
Parameters
|
|
81
|
+
----------
|
|
82
|
+
rename_dict:
|
|
83
|
+
Dict old_name -> new_name. Some of the dictionary's items use regex.
|
|
84
|
+
|
|
85
|
+
Returns
|
|
86
|
+
-------
|
|
87
|
+
output_dict:
|
|
88
|
+
Dict old_name -> new_name used for renaming columns in Source. None of this dictionary's items use regex.
|
|
89
|
+
"""
|
|
90
|
+
output_dict = {}
|
|
91
|
+
for old_name, new_name in rename_dict.items():
|
|
92
|
+
matching_columns = [col for col in schema if re.match(old_name, col) and not _is_excluded(col, not_to_rename)]
|
|
93
|
+
for col in matching_columns:
|
|
94
|
+
output_dict[col] = re.sub(old_name, new_name, col)
|
|
95
|
+
return output_dict
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@inplace_operation
|
|
99
|
+
def add_prefix(self: 'Source', prefix, inplace=False, columns=None, ignore_columns=None) -> Optional['Source']:
|
|
100
|
+
"""
|
|
101
|
+
Adds prefix to all column names (except **TIMESTAMP** (or **Time**) special column).
|
|
102
|
+
|
|
103
|
+
Parameters
|
|
104
|
+
----------
|
|
105
|
+
prefix : str
|
|
106
|
+
String prefix to add to all columns.
|
|
107
|
+
inplace : bool
|
|
108
|
+
The flag controls whether operation should be applied inplace or not.
|
|
109
|
+
If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
|
|
110
|
+
object.
|
|
111
|
+
columns: List[str], optional
|
|
112
|
+
If set, only selected columns will be updated with prefix. Can't be used with ``ignore_columns`` parameter.
|
|
113
|
+
ignore_columns: List[str], optional
|
|
114
|
+
If set, selected columns won't be updated with prefix. Can't be used with ``columns`` parameter.
|
|
115
|
+
|
|
116
|
+
Returns
|
|
117
|
+
-------
|
|
118
|
+
:class:`Source` or ``None``
|
|
119
|
+
|
|
120
|
+
Examples
|
|
121
|
+
--------
|
|
122
|
+
|
|
123
|
+
Add prefix *test_* to all columns (note that column **Time** is not renamed):
|
|
124
|
+
|
|
125
|
+
>>> data = otp.DataSource(db='US_COMP', tick_type='TRD', symbols='AAPL')
|
|
126
|
+
>>> data = data.add_prefix('test_')
|
|
127
|
+
>>> otp.run(data, start=otp.dt(2022, 3, 1), end=otp.dt(2022, 3, 2))
|
|
128
|
+
Time test_PRICE test_SIZE
|
|
129
|
+
0 2022-03-01 00:00:00.000 1.3 100
|
|
130
|
+
1 2022-03-01 00:00:00.001 1.4 10
|
|
131
|
+
2 2022-03-01 00:00:00.002 1.4 50
|
|
132
|
+
|
|
133
|
+
Parameter ``columns`` specifies columns to be updated with prefix:
|
|
134
|
+
|
|
135
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
136
|
+
>>> data = data.add_prefix('test_', columns=['A', 'B', 'C'])
|
|
137
|
+
>>> otp.run(data)
|
|
138
|
+
Time test_A test_B test_C D E
|
|
139
|
+
0 2003-12-01 1 2 3 4 5
|
|
140
|
+
|
|
141
|
+
Parameter ``ignore_columns`` specifies columns to ignore:
|
|
142
|
+
|
|
143
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
144
|
+
>>> data = data.add_prefix('test_', ignore_columns=['A', 'B', 'C'])
|
|
145
|
+
>>> otp.run(data)
|
|
146
|
+
Time A B C test_D test_E
|
|
147
|
+
0 2003-12-01 1 2 3 4 5
|
|
148
|
+
|
|
149
|
+
Parameters ``columns`` and ``ignore_columns`` can't be used at the same time:
|
|
150
|
+
|
|
151
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
152
|
+
>>> data.add_prefix('test_', columns=['B', 'C'], ignore_columns=['A'])
|
|
153
|
+
Traceback (most recent call last):
|
|
154
|
+
...
|
|
155
|
+
ValueError: It is allowed to use only one of `columns` or `ignore_columns` parameters at a time
|
|
156
|
+
|
|
157
|
+
Columns can't be renamed if their resulting name will be equal to existing column name:
|
|
158
|
+
|
|
159
|
+
>>> data = otp.Tick(X=1, XX=2)
|
|
160
|
+
>>> data.add_prefix('X', columns=['X'])
|
|
161
|
+
Traceback (most recent call last):
|
|
162
|
+
...
|
|
163
|
+
AttributeError: Column XX already exists, please, use another prefix
|
|
164
|
+
|
|
165
|
+
"""
|
|
166
|
+
return self._add_prefix_and_suffix(
|
|
167
|
+
prefix=prefix,
|
|
168
|
+
columns=columns,
|
|
169
|
+
ignore_columns=ignore_columns,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@inplace_operation
|
|
174
|
+
def add_suffix(self: 'Source', suffix, inplace=False, columns=None, ignore_columns=None) -> Optional['Source']:
|
|
175
|
+
"""
|
|
176
|
+
Adds suffix to all column names (except **TIMESTAMP** (or **Time**) special column).
|
|
177
|
+
|
|
178
|
+
Parameters
|
|
179
|
+
----------
|
|
180
|
+
suffix : str
|
|
181
|
+
String suffix to add to all columns.
|
|
182
|
+
inplace : bool
|
|
183
|
+
The flag controls whether operation should be applied inplace or not.
|
|
184
|
+
If ``inplace=True``, then it returns nothing. Otherwise method returns a new modified
|
|
185
|
+
object.
|
|
186
|
+
columns: List[str], optional
|
|
187
|
+
If set, only selected columns will be updated with suffix. Can't be used with ``ignore_columns`` parameter.
|
|
188
|
+
ignore_columns: List[str], optional
|
|
189
|
+
If set, selected columns won't be updated with suffix. Can't be used with ``columns`` parameter.
|
|
190
|
+
|
|
191
|
+
Returns
|
|
192
|
+
-------
|
|
193
|
+
:class:`Source` or ``None``
|
|
194
|
+
|
|
195
|
+
Examples
|
|
196
|
+
--------
|
|
197
|
+
|
|
198
|
+
Add suffix *_test* to all columns (note that column **Time** is not renamed):
|
|
199
|
+
|
|
200
|
+
>>> data = otp.DataSource(db='US_COMP', tick_type='TRD', symbols='AAPL')
|
|
201
|
+
>>> data = data.add_suffix('_test')
|
|
202
|
+
>>> otp.run(data, start=otp.dt(2022, 3, 1), end=otp.dt(2022, 3, 2))
|
|
203
|
+
Time PRICE_test SIZE_test
|
|
204
|
+
0 2022-03-01 00:00:00.000 1.3 100
|
|
205
|
+
1 2022-03-01 00:00:00.001 1.4 10
|
|
206
|
+
2 2022-03-01 00:00:00.002 1.4 50
|
|
207
|
+
|
|
208
|
+
Parameter ``columns`` specifies columns to be updated with suffix:
|
|
209
|
+
|
|
210
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
211
|
+
>>> data = data.add_suffix('_test', columns=['A', 'B', 'C'])
|
|
212
|
+
>>> otp.run(data)
|
|
213
|
+
Time A_test B_test C_test D E
|
|
214
|
+
0 2003-12-01 1 2 3 4 5
|
|
215
|
+
|
|
216
|
+
Parameter ``ignore_columns`` specifies columns to ignore:
|
|
217
|
+
|
|
218
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
219
|
+
>>> data = data.add_suffix('_test', ignore_columns=['A', 'B', 'C'])
|
|
220
|
+
>>> otp.run(data)
|
|
221
|
+
Time A B C D_test E_test
|
|
222
|
+
0 2003-12-01 1 2 3 4 5
|
|
223
|
+
|
|
224
|
+
Parameters ``columns`` and ``ignore_columns`` can't be used at the same time:
|
|
225
|
+
|
|
226
|
+
>>> data = otp.Tick(A=1, B=2, C=3, D=4, E=5)
|
|
227
|
+
>>> data.add_suffix('_test', columns=['B', 'C'], ignore_columns=['A'])
|
|
228
|
+
Traceback (most recent call last):
|
|
229
|
+
...
|
|
230
|
+
ValueError: It is allowed to use only one of `columns` or `ignore_columns` parameters at a time
|
|
231
|
+
|
|
232
|
+
Columns can't be renamed if their resulting name will be equal to existing column name:
|
|
233
|
+
|
|
234
|
+
>>> data = otp.Tick(X=1, XX=2)
|
|
235
|
+
>>> data.add_suffix('X', columns=['X'])
|
|
236
|
+
Traceback (most recent call last):
|
|
237
|
+
...
|
|
238
|
+
AttributeError: Column XX already exists, please, use another suffix
|
|
239
|
+
"""
|
|
240
|
+
return self._add_prefix_and_suffix(
|
|
241
|
+
suffix=suffix,
|
|
242
|
+
columns=columns,
|
|
243
|
+
ignore_columns=ignore_columns,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@inplace_operation
|
|
248
|
+
def rename(self: 'Source', columns=None, use_regex=False, fields_to_skip=None, inplace=False) -> Optional['Source']:
|
|
249
|
+
r"""
|
|
250
|
+
Rename columns
|
|
251
|
+
|
|
252
|
+
Parameters
|
|
253
|
+
----------
|
|
254
|
+
columns : dict
|
|
255
|
+
Rules how to rename in the following format: {<column> : <new-column-name>},
|
|
256
|
+
where <column> is either existing column name of str type or reference to a column,
|
|
257
|
+
and <new-column-name> a new column name of str type.
|
|
258
|
+
use_regex: bool
|
|
259
|
+
If true, then old-name=new-name pairs in the `columns` parameter are treated as regular expressions.
|
|
260
|
+
This allows bulk renaming for field names. Notice that regular expressions for old names are treated as
|
|
261
|
+
if both their prefix and their suffix are .*, i.e. the prefix and suffix match any substring.
|
|
262
|
+
As a result, old-name *XX* will match all of *aXX*, *aXXB*, and *XXb*, when `use_regex=true`.
|
|
263
|
+
You can have old-name begin from ^ to indicate that .* prefix does not apply,
|
|
264
|
+
and you can have old name end at $ to indicate that .* suffix does not apply.
|
|
265
|
+
Default: false
|
|
266
|
+
fields_to_skip: list of str
|
|
267
|
+
A list of regular expressions for specifying fields that should be skipped
|
|
268
|
+
(i.e., not be renamed). If a field is matched by one of the specified regular expressions,
|
|
269
|
+
it won't be considered for renaming.
|
|
270
|
+
Default: None
|
|
271
|
+
inplace : bool
|
|
272
|
+
The flag controls whether operation should be applied inplace or not.
|
|
273
|
+
If ``inplace=True``, then it returns nothing. Otherwise, method returns a new modified
|
|
274
|
+
object.
|
|
275
|
+
|
|
276
|
+
Returns
|
|
277
|
+
-------
|
|
278
|
+
:class:`Source` or ``None``
|
|
279
|
+
|
|
280
|
+
See also
|
|
281
|
+
--------
|
|
282
|
+
**RENAME** OneTick event processor
|
|
283
|
+
|
|
284
|
+
Examples
|
|
285
|
+
--------
|
|
286
|
+
>>> data = otp.Ticks(X=[1], Y=[2])
|
|
287
|
+
>>> data = data.rename({'X': 'XX',
|
|
288
|
+
... data['Y']: 'YY'})
|
|
289
|
+
>>> otp.run(data)
|
|
290
|
+
Time XX YY
|
|
291
|
+
0 2003-12-01 1 2
|
|
292
|
+
|
|
293
|
+
>>> data = otp.Tick(**{'X.X': 1, 'X.Y': 2})
|
|
294
|
+
>>> data = data.rename({r'X\.(.*)': r'\1'}, use_regex=True)
|
|
295
|
+
>>> otp.run(data)
|
|
296
|
+
Time X Y
|
|
297
|
+
0 2003-12-01 1 2
|
|
298
|
+
|
|
299
|
+
>>> data = otp.Tick(**{'X.X': 1, 'X.Y': 2})
|
|
300
|
+
>>> data = data.rename({r'X\.(.*)': r'\1'}, use_regex=True, fields_to_skip=['X.Y'])
|
|
301
|
+
>>> otp.run(data)
|
|
302
|
+
Time X X.Y
|
|
303
|
+
0 2003-12-01 1 2
|
|
304
|
+
"""
|
|
305
|
+
if columns is None:
|
|
306
|
+
columns = {}
|
|
307
|
+
|
|
308
|
+
# prepare
|
|
309
|
+
items = {}
|
|
310
|
+
out_names = set()
|
|
311
|
+
fields_to_skip = fields_to_skip or []
|
|
312
|
+
|
|
313
|
+
for in_obj, out_obj in columns.items():
|
|
314
|
+
if isinstance(in_obj, _Column):
|
|
315
|
+
items[in_obj.name.strip()] = out_obj.strip()
|
|
316
|
+
elif isinstance(in_obj, str):
|
|
317
|
+
items[in_obj.strip()] = out_obj.strip()
|
|
318
|
+
else:
|
|
319
|
+
raise TypeError(f"It is not supported to rename item '{in_obj}' of type {type(in_obj)}'")
|
|
320
|
+
|
|
321
|
+
if out_obj in out_names:
|
|
322
|
+
raise AttributeError(
|
|
323
|
+
f"You want to rename '{in_obj}' into '{out_obj}', "
|
|
324
|
+
f"but also want to rename another column into '{out_obj}'"
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
out_names.add(out_obj)
|
|
328
|
+
|
|
329
|
+
schema_update_dict = items
|
|
330
|
+
if use_regex:
|
|
331
|
+
schema_update_dict = _get_columns_names_renaming(self.schema, items, fields_to_skip)
|
|
332
|
+
|
|
333
|
+
# validate
|
|
334
|
+
for in_key, out_key in schema_update_dict.items():
|
|
335
|
+
if in_key not in self.__dict__ or not isinstance(self.__dict__[in_key], _Column):
|
|
336
|
+
raise AttributeError(f"There is no '{in_key}' column to rename")
|
|
337
|
+
|
|
338
|
+
if out_key in self.__dict__ and isinstance(self.__dict__[out_key], _Column):
|
|
339
|
+
raise AttributeError(f"Column '{out_key}' already exists")
|
|
340
|
+
|
|
341
|
+
# apply
|
|
342
|
+
for in_key, out_key in schema_update_dict.items():
|
|
343
|
+
self.__dict__[in_key].rename(out_key, update_parent_object=False)
|
|
344
|
+
self.__dict__[out_key] = self.__dict__[in_key]
|
|
345
|
+
del self.__dict__[in_key]
|
|
346
|
+
|
|
347
|
+
rename_rules = [key + "=" + value for key, value in items.items()]
|
|
348
|
+
kwargs: Dict[str, Any] = dict(rename_fields=",".join(rename_rules))
|
|
349
|
+
if use_regex:
|
|
350
|
+
kwargs['use_regex'] = True
|
|
351
|
+
if fields_to_skip:
|
|
352
|
+
kwargs['fields_to_skip'] = ",".join(fields_to_skip)
|
|
353
|
+
|
|
354
|
+
self.sink(otq.RenameFieldsEp(**kwargs))
|
|
355
|
+
|
|
356
|
+
return self
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, Collection, Optional, Union
|
|
2
|
+
|
|
3
|
+
from onetick.py.core.column import _Column
|
|
4
|
+
from onetick.py.otq import otq
|
|
5
|
+
|
|
6
|
+
from .misc import inplace_operation
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from onetick.py.core.source import Source
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@inplace_operation
|
|
13
|
+
def sort(
|
|
14
|
+
self: 'Source', by: Union[str, Collection[Union[str, '_Column']]], ascending=True, inplace=False
|
|
15
|
+
) -> Optional['Source']:
|
|
16
|
+
"""Sort ticks by columns.
|
|
17
|
+
|
|
18
|
+
Parameters
|
|
19
|
+
----------
|
|
20
|
+
by: str, Column or list of them
|
|
21
|
+
Column(s) to sort by. It is possible to pass a list of column, where is the order is important:
|
|
22
|
+
from the left to the right.
|
|
23
|
+
ascending: bool or list
|
|
24
|
+
Order to sort by. If list of columns is specified, then list of ascending values per column is expected.
|
|
25
|
+
(the :class:`nan` is the smallest for ``float`` type fields)
|
|
26
|
+
inplace: bool
|
|
27
|
+
A flag controls whether operation should be applied inplace.
|
|
28
|
+
If ``inplace=True``, then it returns nothing. Otherwise method
|
|
29
|
+
returns a new modified object
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
:class:`Source`
|
|
34
|
+
|
|
35
|
+
See also
|
|
36
|
+
--------
|
|
37
|
+
**ORDER_BY** OneTick event processor
|
|
38
|
+
|
|
39
|
+
Examples
|
|
40
|
+
--------
|
|
41
|
+
|
|
42
|
+
Single column examples
|
|
43
|
+
|
|
44
|
+
>>> data = otp.Ticks({'X':[ 94, 5, 34],
|
|
45
|
+
... 'Y':[otp.nan, 3.1, -0.3]})
|
|
46
|
+
>>> data = data.sort(data['X'])
|
|
47
|
+
>>> otp.run(data)
|
|
48
|
+
Time X Y
|
|
49
|
+
0 2003-12-01 00:00:00.001 5 3.1
|
|
50
|
+
1 2003-12-01 00:00:00.002 34 -0.3
|
|
51
|
+
2 2003-12-01 00:00:00.000 94 NaN
|
|
52
|
+
|
|
53
|
+
>>> data = otp.Ticks({'X':[ 94, 5, 34],
|
|
54
|
+
... 'Y':[otp.nan, 3.1, -0.3]})
|
|
55
|
+
>>> data = data.sort(data['Y'])
|
|
56
|
+
>>> otp.run(data)
|
|
57
|
+
Time X Y
|
|
58
|
+
0 2003-12-01 00:00:00.000 94 NaN
|
|
59
|
+
1 2003-12-01 00:00:00.002 34 -0.3
|
|
60
|
+
2 2003-12-01 00:00:00.001 5 3.1
|
|
61
|
+
|
|
62
|
+
Inplace
|
|
63
|
+
|
|
64
|
+
>>> data = otp.Ticks({'X':[ 94, 5, 34],
|
|
65
|
+
... 'Y':[otp.nan, 3.1, -0.3]})
|
|
66
|
+
>>> data.sort(data['Y'], inplace=True) # OTdirective: snippet-name: Arrange.sort.inplace;
|
|
67
|
+
>>> otp.run(data)
|
|
68
|
+
Time X Y
|
|
69
|
+
0 2003-12-01 00:00:00.000 94 NaN
|
|
70
|
+
1 2003-12-01 00:00:00.002 34 -0.3
|
|
71
|
+
2 2003-12-01 00:00:00.001 5 3.1
|
|
72
|
+
|
|
73
|
+
Multiple columns
|
|
74
|
+
|
|
75
|
+
>>> data = otp.Ticks({'X':[ 5, 6, 3, 6],
|
|
76
|
+
... 'Y':[1.4, 3.1, 9.1, 5.5]})
|
|
77
|
+
>>> data = data.sort([data['X'], data['Y']])
|
|
78
|
+
>>> otp.run(data)
|
|
79
|
+
Time X Y
|
|
80
|
+
0 2003-12-01 00:00:00.002 3 9.1
|
|
81
|
+
1 2003-12-01 00:00:00.000 5 1.4
|
|
82
|
+
2 2003-12-01 00:00:00.001 6 3.1
|
|
83
|
+
3 2003-12-01 00:00:00.003 6 5.5
|
|
84
|
+
|
|
85
|
+
Ascending/descending control
|
|
86
|
+
|
|
87
|
+
>>> data = otp.Ticks({'X':[ 94, 5, 34],
|
|
88
|
+
... 'Y':[otp.nan, 3.1, -0.3]})
|
|
89
|
+
>>> data = data.sort(data['X'], ascending=False)
|
|
90
|
+
>>> otp.run(data)
|
|
91
|
+
Time X Y
|
|
92
|
+
0 2003-12-01 00:00:00.000 94 NaN
|
|
93
|
+
1 2003-12-01 00:00:00.002 34 -0.3
|
|
94
|
+
2 2003-12-01 00:00:00.001 5 3.1
|
|
95
|
+
|
|
96
|
+
>>> # OTdirective: snippet-name: Arrange.sort.sort;
|
|
97
|
+
>>> data = otp.Ticks({'X':[ 5, 6, 3, 6],
|
|
98
|
+
... 'Y':[1.4, 3.1, 9.1, 5.5]})
|
|
99
|
+
>>> data = data.sort([data['X'], data['Y']], ascending=[True, False])
|
|
100
|
+
>>> otp.run(data)
|
|
101
|
+
Time X Y
|
|
102
|
+
0 2003-12-01 00:00:00.002 3 9.1
|
|
103
|
+
1 2003-12-01 00:00:00.000 5 1.4
|
|
104
|
+
2 2003-12-01 00:00:00.003 6 5.5
|
|
105
|
+
3 2003-12-01 00:00:00.001 6 3.1
|
|
106
|
+
"""
|
|
107
|
+
columns = by
|
|
108
|
+
|
|
109
|
+
if isinstance(columns, list):
|
|
110
|
+
objs = columns
|
|
111
|
+
else:
|
|
112
|
+
objs = [columns]
|
|
113
|
+
|
|
114
|
+
if isinstance(ascending, list):
|
|
115
|
+
asc_objs = ascending
|
|
116
|
+
else:
|
|
117
|
+
asc_objs = [ascending]
|
|
118
|
+
|
|
119
|
+
items = []
|
|
120
|
+
|
|
121
|
+
# -------------------------------
|
|
122
|
+
# Columns processing
|
|
123
|
+
# -------------------------------
|
|
124
|
+
# convert to strings
|
|
125
|
+
# TODO: it seems as a common code, need to move to a separate function
|
|
126
|
+
for obj in objs:
|
|
127
|
+
if isinstance(obj, _Column):
|
|
128
|
+
items.append(obj.name)
|
|
129
|
+
elif isinstance(obj, str):
|
|
130
|
+
items.append(obj)
|
|
131
|
+
else:
|
|
132
|
+
# TODO: cover with tests
|
|
133
|
+
raise TypeError(f"It is not supported to order by '{obj}' of type '{type(obj)}'")
|
|
134
|
+
|
|
135
|
+
# validate
|
|
136
|
+
for item in items:
|
|
137
|
+
if item in self.__dict__:
|
|
138
|
+
if not isinstance(self.__dict__[item], _Column):
|
|
139
|
+
# TODO: cover with tests
|
|
140
|
+
raise AttributeError(f"There is no '{item}' column")
|
|
141
|
+
# if
|
|
142
|
+
else:
|
|
143
|
+
# TODO: covert with tests
|
|
144
|
+
raise AttributeError(f"There is no '{item}' column")
|
|
145
|
+
|
|
146
|
+
# -------------------------------
|
|
147
|
+
# Asc processing
|
|
148
|
+
# -------------------------------
|
|
149
|
+
asc_items = [True] * len(items)
|
|
150
|
+
|
|
151
|
+
def asc_convert(v):
|
|
152
|
+
return "ASC" if v else "DESC"
|
|
153
|
+
|
|
154
|
+
for inx in range(len(items)):
|
|
155
|
+
if inx >= len(asc_objs):
|
|
156
|
+
asc_obj = asc_items[inx]
|
|
157
|
+
else:
|
|
158
|
+
asc_obj = asc_objs[inx]
|
|
159
|
+
|
|
160
|
+
if isinstance(asc_obj, bool):
|
|
161
|
+
asc_items[inx] = asc_convert(asc_obj)
|
|
162
|
+
elif isinstance(asc_obj, int):
|
|
163
|
+
asc_items[inx] = asc_convert(asc_obj)
|
|
164
|
+
else:
|
|
165
|
+
raise TypeError(f"asc can not be '{asc_obj}' of type '{type(asc_obj)}'")
|
|
166
|
+
|
|
167
|
+
# ---------------
|
|
168
|
+
# combine together
|
|
169
|
+
order_by = [column_name + " " + asc for column_name, asc in zip(items, asc_items)]
|
|
170
|
+
|
|
171
|
+
self.sink(otq.OrderByEp(order_by=",".join(order_by)))
|
|
172
|
+
return self
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def sort_values(self: 'Source', *args, **kwargs):
|
|
176
|
+
"""
|
|
177
|
+
alias of sort
|
|
178
|
+
|
|
179
|
+
See Also
|
|
180
|
+
--------
|
|
181
|
+
:meth:`Source.sort`
|
|
182
|
+
"""
|
|
183
|
+
return self.sort(*args, **kwargs)
|