onetick-py 1.177.0__py3-none-any.whl

onetick/py/core/_csv_inspector.py

@@ -0,0 +1,93 @@
+import pandas as pd
+import numpy as np
+
+from .. import types as ott
+
+
+def _convert_pandas_types(dtype):
+    if pd.api.types.is_integer_dtype(dtype):
+        return int
+    elif pd.api.types.is_float_dtype(dtype):
+        return float
+    elif pd.api.types.is_string_dtype(dtype):
+        return str
+    elif pd.api.types.is_bool_dtype(dtype):
+        return bool
+    return None
+
+
+def inspect_by_pandas(path_to_csv, first_line_is_title=True, names=None, field_delimiter=',', quote_char='"'):
+    # read CHUNK_SIZE first lines to determine column types
+    CHUNK_SIZE = 300
+
+    header = None
+
+    if first_line_is_title:
+        header = 0 if names else "infer"
+
+    with pd.read_csv(
+        path_to_csv,
+        engine="python",
+        iterator=True,
+        header=header,
+        names=names,
+        sep=field_delimiter,
+        escapechar='\\',
+        quotechar=quote_char,
+    ) as reader:
+        df = reader.read(CHUNK_SIZE)
+        prefix = None if first_line_is_title or names else "COLUMN_"
+        if prefix:
+            df.columns = [f'{prefix}{col}' for col in df.columns]
+
+    if not first_line_is_title:
+        first_column = "COLUMN_0"
+        if names:
+            first_column = names[0]
+        if len(df) > 0 and len(df.columns) > 0 and df.dtypes[first_column] == np.dtype("O"):
+            if df[first_column][0].startswith("#"):
+                raise ValueError(
+                    "If first line of CSV starts with #, you must set first_line_is_title=True, "
+                    "because OneTick will forcefully use first line as header.")
+
+    # CSV_FILE_LISTING will ignore FIRST_LINE_IS_TITLE, if first line starts with hash sign #
+    forced_title = df.columns.values[0][0] == '#'
+    if forced_title:
+        # remove hash sign from first column name
+        df.rename(columns={df.columns.values[0]: df.columns.values[0][1:]}, inplace=True)
+
+    # check for default types in OneTick format ("columnname type")
+    default_types = {}
+    rename = {}
+
+    for column in df.columns.values:
+        c = column.split()
+
+        if column.startswith('Unnamed: '):
+            # OneTick doesn't allow to have empty column name, so we explicitly set it here
+            # and in CSV_FILE_LISTING EP later in CSV.base_ep()
+            rename[column] = "COLUMN_" + column[9:]
+
+        elif len(c) == 2:
+            # format: "type COLUMNNAME"
+            # http://solutions.pages.soltest.onetick.com/iac/onetick-server/ep_guide/EP/FieldTypeDeclarations.htm#supported_field_types
+            dtype = ott.str2type(c[0])
+            if dtype is not None:
+                default_types[c[1]] = dtype
+                rename[column] = c[1]
+
+    if rename:
+        df.rename(columns=rename, inplace=True)
+
+    # convert pandas types to otp
+    columns = dict(map(lambda x: (x[0], _convert_pandas_types(x[1])), dict(df.dtypes).items()))
+
+    # explicitly set types for columns having format "type COLUMNNAME"
+    for column_name, dtype in default_types.items():
+        columns[column_name] = dtype
+
+    # reset default_types if # is not in first line
+    if not forced_title:
+        default_types = {}
+
+    return columns, default_types, forced_title

onetick/py/core/_internal/_manually_bound_value.py

@@ -0,0 +1,6 @@
+class _ManuallyBoundValue:
+    def __init__(self, value):
+        if isinstance(value, _ManuallyBoundValue):
+            self.value = value.value
+        else:
+            self.value = value

onetick/py/core/_internal/_nodes_history.py

@@ -0,0 +1,250 @@
+from abc import ABC, abstractmethod
+from typing import List
+from copy import deepcopy
+
+
+class _NodeRule(ABC):
+    key_params: List[str] = []
+
+    @abstractmethod
+    def build(self, eps, print_out=False):
+        raise NotImplementedError
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({id(self)})'
+
+
+class _Assign(_NodeRule):
+    key_params = ['key']
+
+    def __init__(self, ep, key):
+        self.ep = ep
+        self.key = key
+
+    def build(self, eps, print_out=False):
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        if print_out:
+            print(f'[API] Create "{str(eps[self.key]).strip()}" (id={id(eps[self.key])})')
+
+        return eps[self.key]
+
+
+class _TickType(_NodeRule):
+    key_params = ['key']
+
+    def __init__(self, ep, key, tt):
+        self.ep = ep
+        self.key = key
+        self.tt = tt
+
+    def build(self, eps, print_out=False):
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        if print_out:
+            print(f'[API] Set tick type "{self.tt}" for "{str(eps[self.key]).strip()}" (id={id(eps[self.key])})')
+
+        eps[self.key].tick_type(self.tt)
+        return eps[self.key]
+
+
+class _Symbol(_NodeRule):
+    key_params = ['key']
+
+    def __init__(self, ep, key, symbol):
+        self.ep = ep
+        self.key = key
+        self.symbol = symbol
+
+    def build(self, eps, print_out=False):
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        if print_out:
+            print(f'[API] Set symbol "{self.symbol}" for "{str(eps[self.key]).strip()}" (id={id(eps[self.key])})')
+
+        eps[self.key].set_symbol(self.symbol)
+        return eps[self.key]
+
+
+class _NodeName(_NodeRule):
+    key_params = ['key']
+
+    def __init__(self, key, name):
+        self.key = key
+        self.name = name
+
+    def build(self, eps, print_out=False):
+        eps[self.key].node_name(self.name)
+
+        if print_out:
+            print(f'[API] Set name "{self.name}" for "{str(eps[self.key]).strip()}" (id={id(eps[self.key])})')
+
+        return eps[self.key]
+
+
+class _Sink(_NodeRule):
+    key_params = ['p_key', 'key']
+
+    def __init__(self, p_ep, p_key, p_out_pin, ep, key, in_pin):
+        self.p_ep = p_ep
+        self.p_key = p_key
+        self.p_out_pin = p_out_pin
+        self.ep = ep
+        self.key = key
+        self.in_pin = in_pin
+
+    def build(self, eps, print_out=False):
+        if self.p_key not in eps:
+            eps[self.p_key] = self.p_ep.copy()
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        t_p_ep = eps[self.p_key]
+        t_ep = eps[self.key]
+
+        if self.p_out_pin:
+            t_p_ep = eps[self.p_key][self.p_out_pin]
+        if self.in_pin:
+            t_ep = eps[self.key](self.in_pin)
+
+        if print_out:
+            print(f'[API] Connect "{str(t_p_ep).strip()}"[out_pin={self.p_out_pin}] -> '
+                  f'"{str(t_ep).strip()}"[self.in_pin={self.in_pin}] (id={id(t_p_ep)} and id={id(t_ep)})')
+
+        t_p_ep.sink(t_ep)
+        return t_ep
+
+
+class _Source(_NodeRule):
+    key_params = ['p_key', 'key']
+
+    def __init__(self, p_ep, p_key, p_in_pin, ep, key, out_pin):
+        self.p_ep = p_ep
+        self.p_key = p_key
+        self.p_in_pin = p_in_pin
+        self.ep = ep
+        self.key = key
+        self.out_pin = out_pin
+
+    def build(self, eps, print_out=False):
+        if self.p_key not in eps:
+            eps[self.p_key] = self.p_ep.copy()
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        t_p_ep = eps[self.p_key]
+        t_ep = eps[self.key]
+
+        if self.p_in_pin:
+            t_p_ep = eps[self.p_key](self.p_in_pin)
+        if self.out_pin:
+            t_ep = eps[self.key][self.out_pin]
+
+        if print_out:
+            print(f'[API] Connect "{str(t_ep).strip()}"[out_pin={self.out_pin}] -> '
+                  f'"{str(t_p_ep).strip()}"[in_pin={self.p_in_pin}] (id={id(t_ep)} and id={id(t_p_ep)})')
+
+        t_p_ep.source(t_ep)
+        return t_p_ep
+
+
+class _SourceByKey(_NodeRule):
+    key_params = ['p_key', 'key']
+
+    def __init__(self, p_key, ep, key, out_pin):
+        self.p_key = p_key
+        self.ep = ep
+        self.key = key
+        self.out_pin = out_pin
+
+    def build(self, eps, print_out=False):
+        if self.key not in eps:
+            eps[self.key] = self.ep.copy()
+
+        t_p_ep = eps[self.p_key]
+        t_ep = eps[self.key]
+
+        if self.out_pin:
+            t_ep = eps[self.key][self.out_pin]
+
+        if print_out:
+            print(f'[API] Connect "{str(t_ep).strip()}" -> "{str(t_p_ep).strip()}" (id={id(t_ep)} and id={id(t_p_ep)})')
+
+        t_p_ep.source(t_ep)
+        return t_p_ep
+
+
+class _NodesHistory:
+    def __init__(self):
+        self._rules: List[_NodeRule] = []
+
+    def assign(self, ep, key):
+        self._rules.append(_Assign(ep, key))
+
+    def tick_type(self, ep, key, tt):
+        self._rules.append(_TickType(ep, key, tt))
+
+    def symbol(self, ep, key, symbol):
+        self._rules.append(_Symbol(ep, key, symbol))
+
+    def node_name(self, key, name):
+        self._rules.append(_NodeName(key, name))
+
+    def sink(self, p_ep, p_key, p_out_pin, ep, key, in_pin):
+        self._rules.append(_Sink(p_ep, p_key, p_out_pin, ep, key, in_pin))
+
+    def source(self, p_ep, p_key, p_in_pin, ep, key, out_pin):
+        self._rules.append(_Source(p_ep, p_key, p_in_pin, ep, key, out_pin))
+
+    def source_by_key(self, p_key, ep, key, out_pin):
+        self._rules.append(_SourceByKey(p_key, ep, key, out_pin))
+
+    def rebuild(self, keys: dict):
+        """Rebuild history, change uuid for each node"""
+        processed = set()
+        for rule in self._rules:
+            if id(rule) in processed:
+                # this can happen when we are merging the branch of the source with this source.
+                # In this case some rules in the source and the branch will be the same.
+                # (e.g. in dump() function)
+                continue
+            processed.add(id(rule))
+            for key_param in rule.key_params:
+                key = getattr(rule, key_param)
+                setattr(rule, key_param, keys[key])
+
+    def build(self, eps, root_key, print_out=False):
+        if print_out:
+            print("")
+            print("[[API OUTPUT STARTS]]")
+
+        if print_out:
+            print('number of rules:', len(self._rules))
+        for rule in self._rules:
+            rule.build(eps, print_out=print_out)
+
+        if print_out:
+            print("[[API OUTPUT ENDS]]")
+
+        res = eps[root_key]
+
+        assert res is not None
+
+        return res
+
+    def copy(self, deep=False):
+        return list(self._rules) if not deep else deepcopy(self._rules)
+
+    def add(self, other_rules):
+        processed = set(id(rule) for rule in self._rules)
+        for rule in other_rules:
+            if id(rule) in processed:
+                # this can happen when we are merging the branch of the source with this source.
+                # In this case some rules in the source and the branch will be the same.
+                # (e.g. in dump() function)
+                continue
+            processed.add(id(rule))
+            self._rules.append(rule)

onetick/py/core/_internal/_op_utils/every_operand.py

@@ -0,0 +1,9 @@
+def every_operand(op):
+    operations = [op]
+    while operations:
+        operation = operations.pop()
+        operands = getattr(operation, "_op_params", None)
+        if operands:
+            operations.extend(operands)
+        else:
+            yield operation

onetick/py/core/_internal/_op_utils/is_const.py

@@ -0,0 +1,10 @@
+from onetick.py.core._internal._op_utils.every_operand import every_operand
+from onetick.py.core._source._symbol_param import _SymbolParamColumn
+from onetick.py.core.column import _Column
+
+
+def is_const(operation):
+    for op in every_operand(operation):
+        if isinstance(op, _Column) and not isinstance(op, _SymbolParamColumn):
+            return False
+    return True

onetick/py/core/_internal/_per_tick_scripts/tick_list_sort_template.script

@@ -0,0 +1,121 @@
+/* this file is a python template string that, upon substitution, produces a per tick script */
+
+long main()
+{
+    /* merge sort implementation for tick lists */
+    /*
+    We iterate over powers of 2:  N=1, 2, 4, 8, etc.
+    For each N we do the following:
+        - take a new empty list
+        - iteratively take two consecutive sequences of size N from the initial list
+          and merge them (with ordering) into the new list
+        - sequences taken at step i will have items:
+            - for sequence 1, [2N(i-1), 2N(i-1) + N)
+            - for sequence 2, [2N(i-1) + N, 2Ni)
+        - note that both sequences may turn out incomplete or even empty, that's fine
+        - do this until the whole input list was split into those consecutive sequences and merged into the new list
+        - copy new list into the old one and clean the new list
+    We do this until N is greater than the size of the initial list, at which point we will have a sorted list
+    */
+	static TICK_LIST LOCAL::TARGET_TICKS;
+	long LOCAL::I = 0;
+	/* 2^K is the length of the sub-lists that we're going to merge on each iteration */
+	long LOCAL::K = 0;
+	long LOCAL::N = 0;
+	static TICK_LIST_TICK LOCAL::SUB_LIST_1;
+	static TICK_LIST_TICK LOCAL::SUB_LIST_2;
+	long LOCAL::SUB_LIST_1_CNT = 0;
+	long LOCAL::SUB_LIST_2_CNT = 0;
+	long LOCAL::SUB_LIST_1_ENDED = 0;
+	long LOCAL::SUB_LIST_2_ENDED = 0;
+	/*
+	due to implementation, it won't hurt to do an extra iteration just to be sure that log calculation rounding
+	did not lead to any funny things
+	*/
+	long LOCAL::K_MAX = LOG(${tick_list_var}.GET_SIZE())/LOG(2) + 1;
+	for ( LOCAL::K=0; LOCAL::K <= LOCAL::K_MAX; LOCAL::K+=1 )
+	{
+		/* this is the main loop, here we select sub-lists and merge them */
+		LOCAL::N = POWER(2,LOCAL::K);
+		/* first, we set first iterator to the beginning */
+		${tick_list_var}.ITERATOR(LOCAL::SUB_LIST_1);
+		/* then we continue setting the second iterator to the required amount of ticks past that */
+		while(true)
+		{
+			LOCAL::SUB_LIST_1.COPY(LOCAL::SUB_LIST_2);
+			for (LOCAL::I=0; LOCAL::I < LOCAL::N; LOCAL::I+=1)
+			{
+				LOCAL::SUB_LIST_2.NEXT();
+			}
+			/*
+			now, we start from SUB_LIST_1 and SUB_LIST_2 and iterate over both
+			and merge them with proper order into the LOCAL::TARGET_TICKS list
+			while doing this, we take care to check if any of those lists already ended
+			*/
+			LOCAL::SUB_LIST_1_CNT = 0;
+			LOCAL::SUB_LIST_2_CNT = 0;
+			LOCAL::SUB_LIST_1_ENDED = 0;
+			LOCAL::SUB_LIST_2_ENDED = 0;
+			while(true)
+			{
+				/* first, setting flags for end of sub-lists */
+				if ((LOCAL::SUB_LIST_1_CNT >= LOCAL::N) or (LOCAL.SUB_LIST_1.IS_END() == 1))
+				{
+					LOCAL::SUB_LIST_1_ENDED = 1;
+				}
+				if ((LOCAL::SUB_LIST_2_CNT >= LOCAL::N) or (LOCAL.SUB_LIST_2.IS_END() == 1))
+				{
+					LOCAL::SUB_LIST_2_ENDED = 1;
+				}
+				/* now taking next element, or breaking, depending on the flags */
+				if ((LOCAL::SUB_LIST_1_ENDED == 0) and (LOCAL::SUB_LIST_2_ENDED == 0))
+				{
+				    /* <= is important for the stability of the sort */
+					if (LOCAL::SUB_LIST_1.${field_access_function}("${field_name}") <= LOCAL::SUB_LIST_2.${field_access_function}("${field_name}"))
+					{
+						LOCAL::TARGET_TICKS.PUSH_BACK(LOCAL::SUB_LIST_1);
+						LOCAL::SUB_LIST_1_CNT += 1;
+						LOCAL::SUB_LIST_1.NEXT();
+					}
+					else
+					{
+						LOCAL::TARGET_TICKS.PUSH_BACK(LOCAL::SUB_LIST_2);
+						LOCAL::SUB_LIST_2_CNT += 1;
+						LOCAL::SUB_LIST_2.NEXT();
+					}
+					continue;
+				}
+				if (LOCAL::SUB_LIST_1_ENDED == 0)
+				{
+					LOCAL::TARGET_TICKS.PUSH_BACK(LOCAL::SUB_LIST_1);
+					LOCAL::SUB_LIST_1_CNT += 1;
+					LOCAL::SUB_LIST_1.NEXT();
+					continue;
+				}
+				if (LOCAL::SUB_LIST_2_ENDED == 0)
+				{
+					LOCAL::TARGET_TICKS.PUSH_BACK(LOCAL::SUB_LIST_2);
+					LOCAL::SUB_LIST_2_CNT += 1;
+					LOCAL::SUB_LIST_2.NEXT();
+					continue;
+				}
+				/* we fell into none of the continues above, this means that both sub-lists ended */
+				break;
+			}
+			/* for the next iteration, we start new sub-list 1 from where old sub-list 2 ended */
+			LOCAL::SUB_LIST_2.COPY(LOCAL::SUB_LIST_1);
+			if (LOCAL::SUB_LIST_1.IS_END() == 1)
+			{
+				break;
+			}
+		}
+		/* copying newer tick list into the original one */
+		/* implementation can be optimized to avoid this step but well */
+		${tick_list_var}.CLEAR();
+		for (TICK_LIST_TICK LOCAL::T : LOCAL::TARGET_TICKS)
+		{
+			${tick_list_var}.PUSH_BACK(LOCAL::T);
+		}
+		LOCAL::TARGET_TICKS.CLEAR();
+	}
+}

onetick/py/core/_internal/_proxy_node.py

@@ -0,0 +1,140 @@
+from collections import defaultdict
+
+from onetick.py.core._internal._nodes_history import _NodesHistory
+
+
+class _ProxyNode:
+    """
+    This class wraps node in _Source with goal to track changes
+    """
+
+    def __init__(self, ep, key, _, out_pin, refresh_func=None):
+        self._ep = ep
+        self._key = key
+        self._out_pin = out_pin
+        self._name = ""
+        self._refresh_func = refresh_func
+
+        self._hist = _NodesHistory()
+        self._hist.assign(ep, key)
+
+    def sink(self, p_out_pin, ep, key, in_pin, _, move_node=True):
+        """
+        Connect self._ep[p_out_pin|self._out_pin] -> ep[in_pin]
+        """
+        self._refresh_func()
+
+        if not p_out_pin:
+            p_out_pin = self._out_pin
+
+        self._hist.sink(self._ep, self._key, p_out_pin, ep, key, in_pin)
+
+        t_p_ep = self._ep
+        if p_out_pin:
+            t_p_ep = t_p_ep[p_out_pin]
+
+        t_ep = ep
+        if in_pin:
+            t_ep = ep
+
+        if move_node:
+            t_p_ep.sink(t_ep)
+            self._ep = ep
+            self._key = key
+            #  it is not the first, then no need to store this pin
+            self._out_pin = None
+            self._name = ""
+
+        return self._ep
+
+    def source(self, p_in_pin, ep, key, _, out_pin):
+        """
+        Connect self._ep[p_in_pin] <- ep[out_pin]
+        """
+        self._refresh_func()
+
+        self._hist.source(self._ep, self._key, p_in_pin, ep, key, out_pin)
+
+        t_p_ep = self._ep
+        if p_in_pin:
+            t_p_ep = t_p_ep(p_in_pin)
+
+        t_ep = ep
+        if out_pin:
+            t_ep = t_ep[out_pin]
+
+        t_p_ep.source(t_ep)
+
+        return self._ep
+
+    def source_by_key(self, to_key, ep, key, _, out_pin):
+        """
+        Connect node with key=to_key <- ep[out_pin]
+        """
+        self._refresh_func()
+
+        self._hist.source_by_key(to_key, ep, key, out_pin)
+
+        return self._ep
+
+    def node_name(self, name=None, key=None):
+        if name is not None:
+            if key:
+                # set node_name by key
+                self._hist.node_name(key, name)
+            else:
+                self._hist.node_name(self._key, name)
+                self._name = name
+            return None
+        else:
+            return self._name
+
+    def tick_type(self, tt):
+        self._refresh_func()
+        self._hist.tick_type(self._ep, self._key, tt)
+        self._ep.tick_type(tt)
+
+    def symbol(self, symbol):
+        self._refresh_func()
+        self._hist.symbol(self._ep, self._key, symbol)
+        self._ep.set_symbol(symbol)
+
+    def get(self):
+        return self._ep
+
+    def key(self, _key=None):
+        if _key:
+            self._key = _key
+
+        return self._key
+
+    def out_pin(self, _out_pin=None):
+        if _out_pin:
+            self._out_pin = _out_pin
+
+        return self._out_pin
+
+    def copy_graph(self, eps=None, print_out=False):
+        if eps is None:
+            eps = defaultdict()
+
+        return self._hist.build(eps, self._key, print_out=print_out), self._key, None, self._out_pin
+
+    def rebuild_graph(self, keys: dict):
+        """
+        Changing all uuids in this node and it's history.
+        Need it for making a deep copy of node.
+
+        Parameters
+        ----------
+        keys: dict
+            Mapping from old key to new key
+        """
+        self._key = keys[self._key]
+        self._hist.rebuild(keys)
+
+    def copy_rules(self, deep=False):
+        return self._hist.copy(deep=deep)
+
+    def add_rules(self, rules):
+        self._hist.add(rules)