ccxt 4.2.77__py2.py3-none-any.whl → 4.4.48__py2.py3-none-any.whl

ccxt/static_dependencies/lark/tree_matcher.py

@@ -0,0 +1,186 @@
+"""Tree matcher based on Lark grammar"""
+
+import re
+from collections import defaultdict
+
+from . import Tree, Token
+from .common import ParserConf
+from .parsers import earley
+from .grammar import Rule, Terminal, NonTerminal
+
+
+def is_discarded_terminal(t):
+    return t.is_term and t.filter_out
+
+
+class _MakeTreeMatch:
+    def __init__(self, name, expansion):
+        self.name = name
+        self.expansion = expansion
+
+    def __call__(self, args):
+        t = Tree(self.name, args)
+        t.meta.match_tree = True
+        t.meta.orig_expansion = self.expansion
+        return t
+
+
+def _best_from_group(seq, group_key, cmp_key):
+    d = {}
+    for item in seq:
+        key = group_key(item)
+        if key in d:
+            v1 = cmp_key(item)
+            v2 = cmp_key(d[key])
+            if v2 > v1:
+                d[key] = item
+        else:
+            d[key] = item
+    return list(d.values())
+
+
+def _best_rules_from_group(rules):
+    rules = _best_from_group(rules, lambda r: r, lambda r: -len(r.expansion))
+    rules.sort(key=lambda r: len(r.expansion))
+    return rules
+
+
+def _match(term, token):
+    if isinstance(token, Tree):
+        name, _args = parse_rulename(term.name)
+        return token.data == name
+    elif isinstance(token, Token):
+        return term == Terminal(token.type)
+    assert False, (term, token)
+
+
+def make_recons_rule(origin, expansion, old_expansion):
+    return Rule(origin, expansion, alias=_MakeTreeMatch(origin.name, old_expansion))
+
+
+def make_recons_rule_to_term(origin, term):
+    return make_recons_rule(origin, [Terminal(term.name)], [term])
+
+
+def parse_rulename(s):
+    "Parse rule names that may contain a template syntax (like rule{a, b, ...})"
+    name, args_str = re.match(r'(\w+)(?:{(.+)})?', s).groups()
+    args = args_str and [a.strip() for a in args_str.split(',')]
+    return name, args
+
+
+
+class ChildrenLexer:
+    def __init__(self, children):
+        self.children = children
+
+    def lex(self, parser_state):
+        return self.children
+
+class TreeMatcher:
+    """Match the elements of a tree node, based on an ontology
+    provided by a Lark grammar.
+
+    Supports templates and inlined rules (`rule{a, b,..}` and `_rule`)
+
+    Initialize with an instance of Lark.
+    """
+
+    def __init__(self, parser):
+        # XXX TODO calling compile twice returns different results!
+        assert not parser.options.maybe_placeholders
+        # XXX TODO: we just ignore the potential existence of a postlexer
+        self.tokens, rules, _extra = parser.grammar.compile(parser.options.start, set())
+
+        self.rules_for_root = defaultdict(list)
+
+        self.rules = list(self._build_recons_rules(rules))
+        self.rules.reverse()
+
+        # Choose the best rule from each group of {rule => [rule.alias]}, since we only really need one derivation.
+        self.rules = _best_rules_from_group(self.rules)
+
+        self.parser = parser
+        self._parser_cache = {}
+
+    def _build_recons_rules(self, rules):
+        "Convert tree-parsing/construction rules to tree-matching rules"
+        expand1s = {r.origin for r in rules if r.options.expand1}
+
+        aliases = defaultdict(list)
+        for r in rules:
+            if r.alias:
+                aliases[r.origin].append(r.alias)
+
+        rule_names = {r.origin for r in rules}
+        nonterminals = {sym for sym in rule_names
+                        if sym.name.startswith('_') or sym in expand1s or sym in aliases}
+
+        seen = set()
+        for r in rules:
+            recons_exp = [sym if sym in nonterminals else Terminal(sym.name)
+                          for sym in r.expansion if not is_discarded_terminal(sym)]
+
+            # Skip self-recursive constructs
+            if recons_exp == [r.origin] and r.alias is None:
+                continue
+
+            sym = NonTerminal(r.alias) if r.alias else r.origin
+            rule = make_recons_rule(sym, recons_exp, r.expansion)
+
+            if sym in expand1s and len(recons_exp) != 1:
+                self.rules_for_root[sym.name].append(rule)
+
+                if sym.name not in seen:
+                    yield make_recons_rule_to_term(sym, sym)
+                    seen.add(sym.name)
+            else:
+                if sym.name.startswith('_') or sym in expand1s:
+                    yield rule
+                else:
+                    self.rules_for_root[sym.name].append(rule)
+
+        for origin, rule_aliases in aliases.items():
+            for alias in rule_aliases:
+                yield make_recons_rule_to_term(origin, NonTerminal(alias))
+            yield make_recons_rule_to_term(origin, origin)
+
+    def match_tree(self, tree, rulename):
+        """Match the elements of `tree` to the symbols of rule `rulename`.
+
+        Parameters:
+            tree (Tree): the tree node to match
+            rulename (str): The expected full rule name (including template args)
+
+        Returns:
+            Tree: an unreduced tree that matches `rulename`
+
+        Raises:
+            UnexpectedToken: If no match was found.
+
+        Note:
+            It's the callers' responsibility match the tree recursively.
+        """
+        if rulename:
+            # validate
+            name, _args = parse_rulename(rulename)
+            assert tree.data == name
+        else:
+            rulename = tree.data
+
+        # TODO: ambiguity?
+        try:
+            parser = self._parser_cache[rulename]
+        except KeyError:
+            rules = self.rules + _best_rules_from_group(self.rules_for_root[rulename])
+
+            # TODO pass callbacks through dict, instead of alias?
+            callbacks = {rule: rule.alias for rule in rules}
+            conf = ParserConf(rules, callbacks, [rulename])
+            parser = earley.Parser(self.parser.lexer_conf, conf, _match, resolve_ambiguity=True)
+            self._parser_cache[rulename] = parser
+
+        # find a full derivation
+        unreduced_tree = parser.parse(ChildrenLexer(tree.children), rulename)
+        assert unreduced_tree.data == rulename
+        return unreduced_tree

ccxt/static_dependencies/lark/tree_templates.py

@@ -0,0 +1,180 @@
+"""This module defines utilities for matching and translation tree templates.
+
+A tree templates is a tree that contains nodes that are template variables.
+
+"""
+
+from typing import Union, Optional, Mapping, Dict, Tuple, Iterator
+
+from lark import Tree, Transformer
+from lark.exceptions import MissingVariableError
+
+Branch = Union[Tree[str], str]
+TreeOrCode = Union[Tree[str], str]
+MatchResult = Dict[str, Tree]
+_TEMPLATE_MARKER = '$'
+
+
+class TemplateConf:
+    """Template Configuration
+
+    Allows customization for different uses of Template
+
+    parse() must return a Tree instance.
+    """
+
+    def __init__(self, parse=None):
+        self._parse = parse
+
+    def test_var(self, var: Union[Tree[str], str]) -> Optional[str]:
+        """Given a tree node, if it is a template variable return its name. Otherwise, return None.
+
+        This method may be overridden for customization
+
+        Parameters:
+            var: Tree | str - The tree node to test
+
+        """
+        if isinstance(var, str):
+            return _get_template_name(var)
+
+        if (
+            isinstance(var, Tree)
+            and var.data == "var"
+            and len(var.children) > 0
+            and isinstance(var.children[0], str)
+        ):
+            return _get_template_name(var.children[0])
+
+        return None
+
+    def _get_tree(self, template: TreeOrCode) -> Tree[str]:
+        if isinstance(template, str):
+            assert self._parse
+            template = self._parse(template)
+
+        if not isinstance(template, Tree):
+            raise TypeError("template parser must return a Tree instance")
+
+        return template
+
+    def __call__(self, template: Tree[str]) -> 'Template':
+        return Template(template, conf=self)
+
+    def _match_tree_template(self, template: TreeOrCode, tree: Branch) -> Optional[MatchResult]:
+        """Returns dict of {var: match} if found a match, else None
+        """
+        template_var = self.test_var(template)
+        if template_var:
+            if not isinstance(tree, Tree):
+                raise TypeError(f"Template variables can only match Tree instances. Not {tree!r}")
+            return {template_var: tree}
+
+        if isinstance(template, str):
+            if template == tree:
+                return {}
+            return None
+
+        assert isinstance(template, Tree) and isinstance(tree, Tree), f"template={template} tree={tree}"
+
+        if template.data == tree.data and len(template.children) == len(tree.children):
+            res = {}
+            for t1, t2 in zip(template.children, tree.children):
+                matches = self._match_tree_template(t1, t2)
+                if matches is None:
+                    return None
+
+                res.update(matches)
+
+            return res
+
+        return None
+
+
+class _ReplaceVars(Transformer[str, Tree[str]]):
+    def __init__(self, conf: TemplateConf, vars: Mapping[str, Tree[str]]) -> None:
+        super().__init__()
+        self._conf = conf
+        self._vars = vars
+
+    def __default__(self, data, children, meta) -> Tree[str]:
+        tree = super().__default__(data, children, meta)
+
+        var = self._conf.test_var(tree)
+        if var:
+            try:
+                return self._vars[var]
+            except KeyError:
+                raise MissingVariableError(f"No mapping for template variable ({var})")
+        return tree
+
+
+class Template:
+    """Represents a tree template, tied to a specific configuration
+
+    A tree template is a tree that contains nodes that are template variables.
+    Those variables will match any tree.
+    (future versions may support annotations on the variables, to allow more complex templates)
+    """
+
+    def __init__(self, tree: Tree[str], conf: TemplateConf = TemplateConf()):
+        self.conf = conf
+        self.tree = conf._get_tree(tree)
+
+    def match(self, tree: TreeOrCode) -> Optional[MatchResult]:
+        """Match a tree template to a tree.
+
+        A tree template without variables will only match ``tree`` if it is equal to the template.
+
+        Parameters:
+            tree (Tree): The tree to match to the template
+
+        Returns:
+            Optional[Dict[str, Tree]]: If match is found, returns a dictionary mapping
+                template variable names to their matching tree nodes.
+                If no match was found, returns None.
+        """
+        tree = self.conf._get_tree(tree)
+        return self.conf._match_tree_template(self.tree, tree)
+
+    def search(self, tree: TreeOrCode) -> Iterator[Tuple[Tree[str], MatchResult]]:
+        """Search for all occurrences of the tree template inside ``tree``.
+        """
+        tree = self.conf._get_tree(tree)
+        for subtree in tree.iter_subtrees():
+            res = self.match(subtree)
+            if res:
+                yield subtree, res
+
+    def apply_vars(self, vars: Mapping[str, Tree[str]]) -> Tree[str]:
+        """Apply vars to the template tree
+        """
+        return _ReplaceVars(self.conf, vars).transform(self.tree)
+
+
+def translate(t1: Template, t2: Template, tree: TreeOrCode):
+    """Search tree and translate each occurrence of t1 into t2.
+    """
+    tree = t1.conf._get_tree(tree)      # ensure it's a tree, parse if necessary and possible
+    for subtree, vars in t1.search(tree):
+        res = t2.apply_vars(vars)
+        subtree.set(res.data, res.children)
+    return tree
+
+
+class TemplateTranslator:
+    """Utility class for translating a collection of patterns
+    """
+
+    def __init__(self, translations: Mapping[Template, Template]):
+        assert all(isinstance(k, Template) and isinstance(v, Template) for k, v in translations.items())
+        self.translations = translations
+
+    def translate(self, tree: Tree[str]):
+        for k, v in self.translations.items():
+            tree = translate(k, v, tree)
+        return tree
+
+
+def _get_template_name(value: str) -> Optional[str]:
+    return value.lstrip(_TEMPLATE_MARKER) if value.startswith(_TEMPLATE_MARKER) else None

ccxt/static_dependencies/lark/utils.py

@@ -0,0 +1,343 @@
+import unicodedata
+import os
+from itertools import product
+from collections import deque
+from typing import Callable, Iterator, List, Optional, Tuple, Type, TypeVar, Union, Dict, Any, Sequence, Iterable, AbstractSet
+
+###{standalone
+import sys, re
+import logging
+
+logger: logging.Logger = logging.getLogger("lark")
+logger.addHandler(logging.StreamHandler())
+# Set to highest level, since we have some warnings amongst the code
+# By default, we should not output any log messages
+logger.setLevel(logging.CRITICAL)
+
+
+NO_VALUE = object()
+
+T = TypeVar("T")
+
+
+def classify(seq: Iterable, key: Optional[Callable] = None, value: Optional[Callable] = None) -> Dict:
+    d: Dict[Any, Any] = {}
+    for item in seq:
+        k = key(item) if (key is not None) else item
+        v = value(item) if (value is not None) else item
+        try:
+            d[k].append(v)
+        except KeyError:
+            d[k] = [v]
+    return d
+
+
+def _deserialize(data: Any, namespace: Dict[str, Any], memo: Dict) -> Any:
+    if isinstance(data, dict):
+        if '__type__' in data:  # Object
+            class_ = namespace[data['__type__']]
+            return class_.deserialize(data, memo)
+        elif '@' in data:
+            return memo[data['@']]
+        return {key:_deserialize(value, namespace, memo) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [_deserialize(value, namespace, memo) for value in data]
+    return data
+
+
+_T = TypeVar("_T", bound="Serialize")
+
+class Serialize:
+    """Safe-ish serialization interface that doesn't rely on Pickle
+
+    Attributes:
+        __serialize_fields__ (List[str]): Fields (aka attributes) to serialize.
+        __serialize_namespace__ (list): List of classes that deserialization is allowed to instantiate.
+                                        Should include all field types that aren't builtin types.
+    """
+
+    def memo_serialize(self, types_to_memoize: List) -> Any:
+        memo = SerializeMemoizer(types_to_memoize)
+        return self.serialize(memo), memo.serialize()
+
+    def serialize(self, memo = None) -> Dict[str, Any]:
+        if memo and memo.in_types(self):
+            return {'@': memo.memoized.get(self)}
+
+        fields = getattr(self, '__serialize_fields__')
+        res = {f: _serialize(getattr(self, f), memo) for f in fields}
+        res['__type__'] = type(self).__name__
+        if hasattr(self, '_serialize'):
+            self._serialize(res, memo)
+        return res
+
+    @classmethod
+    def deserialize(cls: Type[_T], data: Dict[str, Any], memo: Dict[int, Any]) -> _T:
+        namespace = getattr(cls, '__serialize_namespace__', [])
+        namespace = {c.__name__:c for c in namespace}
+
+        fields = getattr(cls, '__serialize_fields__')
+
+        if '@' in data:
+            return memo[data['@']]
+
+        inst = cls.__new__(cls)
+        for f in fields:
+            try:
+                setattr(inst, f, _deserialize(data[f], namespace, memo))
+            except KeyError as e:
+                raise KeyError("Cannot find key for class", cls, e)
+
+        if hasattr(inst, '_deserialize'):
+            inst._deserialize()
+
+        return inst
+
+
+class SerializeMemoizer(Serialize):
+    "A version of serialize that memoizes objects to reduce space"
+
+    __serialize_fields__ = 'memoized',
+
+    def __init__(self, types_to_memoize: List) -> None:
+        self.types_to_memoize = tuple(types_to_memoize)
+        self.memoized = Enumerator()
+
+    def in_types(self, value: Serialize) -> bool:
+        return isinstance(value, self.types_to_memoize)
+
+    def serialize(self) -> Dict[int, Any]:  # type: ignore[override]
+        return _serialize(self.memoized.reversed(), None)
+
+    @classmethod
+    def deserialize(cls, data: Dict[int, Any], namespace: Dict[str, Any], memo: Dict[Any, Any]) -> Dict[int, Any]:  # type: ignore[override]
+        return _deserialize(data, namespace, memo)
+
+
+try:
+    import regex
+    _has_regex = True
+except ImportError:
+    _has_regex = False
+
+if sys.version_info >= (3, 11):
+    import re._parser as sre_parse
+    import re._constants as sre_constants
+else:
+    import sre_parse
+    import sre_constants
+
+categ_pattern = re.compile(r'\\p{[A-Za-z_]+}')
+
+def get_regexp_width(expr: str) -> Union[Tuple[int, int], List[int]]:
+    if _has_regex:
+        # Since `sre_parse` cannot deal with Unicode categories of the form `\p{Mn}`, we replace these with
+        # a simple letter, which makes no difference as we are only trying to get the possible lengths of the regex
+        # match here below.
+        regexp_final = re.sub(categ_pattern, 'A', expr)
+    else:
+        if re.search(categ_pattern, expr):
+            raise ImportError('`regex` module must be installed in order to use Unicode categories.', expr)
+        regexp_final = expr
+    try:
+        # Fixed in next version (past 0.960) of typeshed
+        return [int(x) for x in sre_parse.parse(regexp_final).getwidth()]
+    except sre_constants.error:
+        if not _has_regex:
+            raise ValueError(expr)
+        else:
+            # sre_parse does not support the new features in regex. To not completely fail in that case,
+            # we manually test for the most important info (whether the empty string is matched)
+            c = regex.compile(regexp_final)
+            # Python 3.11.7 introducded sre_parse.MAXWIDTH that is used instead of MAXREPEAT
+            # See lark-parser/lark#1376 and python/cpython#109859
+            MAXWIDTH = getattr(sre_parse, "MAXWIDTH", sre_constants.MAXREPEAT)
+            if c.match('') is None:
+                # MAXREPEAT is a none pickable subclass of int, therefore needs to be converted to enable caching
+                return 1, int(MAXWIDTH)
+            else:
+                return 0, int(MAXWIDTH)
+
+###}
+
+
+_ID_START =    'Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Mn', 'Mc', 'Pc'
+_ID_CONTINUE = _ID_START + ('Nd', 'Nl',)
+
+def _test_unicode_category(s: str, categories: Sequence[str]) -> bool:
+    if len(s) != 1:
+        return all(_test_unicode_category(char, categories) for char in s)
+    return s == '_' or unicodedata.category(s) in categories
+
+def is_id_continue(s: str) -> bool:
+    """
+    Checks if all characters in `s` are alphanumeric characters (Unicode standard, so diacritics, indian vowels, non-latin
+    numbers, etc. all pass). Synonymous with a Python `ID_CONTINUE` identifier. See PEP 3131 for details.
+    """
+    return _test_unicode_category(s, _ID_CONTINUE)
+
+def is_id_start(s: str) -> bool:
+    """
+    Checks if all characters in `s` are alphabetic characters (Unicode standard, so diacritics, indian vowels, non-latin
+    numbers, etc. all pass). Synonymous with a Python `ID_START` identifier. See PEP 3131 for details.
+    """
+    return _test_unicode_category(s, _ID_START)
+
+
+def dedup_list(l: Sequence[T]) -> List[T]:
+    """Given a list (l) will removing duplicates from the list,
+       preserving the original order of the list. Assumes that
+       the list entries are hashable."""
+    return list(dict.fromkeys(l))
+
+
+class Enumerator(Serialize):
+    def __init__(self) -> None:
+        self.enums: Dict[Any, int] = {}
+
+    def get(self, item) -> int:
+        if item not in self.enums:
+            self.enums[item] = len(self.enums)
+        return self.enums[item]
+
+    def __len__(self):
+        return len(self.enums)
+
+    def reversed(self) -> Dict[int, Any]:
+        r = {v: k for k, v in self.enums.items()}
+        assert len(r) == len(self.enums)
+        return r
+
+
+
+def combine_alternatives(lists):
+    """
+    Accepts a list of alternatives, and enumerates all their possible concatenations.
+
+    Examples:
+        >>> combine_alternatives([range(2), [4,5]])
+        [[0, 4], [0, 5], [1, 4], [1, 5]]
+
+        >>> combine_alternatives(["abc", "xy", '$'])
+        [['a', 'x', '$'], ['a', 'y', '$'], ['b', 'x', '$'], ['b', 'y', '$'], ['c', 'x', '$'], ['c', 'y', '$']]
+
+        >>> combine_alternatives([])
+        [[]]
+    """
+    if not lists:
+        return [[]]
+    assert all(l for l in lists), lists
+    return list(product(*lists))
+
+try:
+    import atomicwrites
+    _has_atomicwrites = True
+except ImportError:
+    _has_atomicwrites = False
+
+class FS:
+    exists = staticmethod(os.path.exists)
+
+    @staticmethod
+    def open(name, mode="r", **kwargs):
+        if _has_atomicwrites and "w" in mode:
+            return atomicwrites.atomic_write(name, mode=mode, overwrite=True, **kwargs)
+        else:
+            return open(name, mode, **kwargs)
+
+
+class fzset(frozenset):
+    def __repr__(self):
+        return '{%s}' % ', '.join(map(repr, self))
+
+
+def classify_bool(seq: Iterable, pred: Callable) -> Any:
+    false_elems = []
+    true_elems = [elem for elem in seq if pred(elem) or false_elems.append(elem)]  # type: ignore[func-returns-value]
+    return true_elems, false_elems
+
+
+def bfs(initial: Iterable, expand: Callable) -> Iterator:
+    open_q = deque(list(initial))
+    visited = set(open_q)
+    while open_q:
+        node = open_q.popleft()
+        yield node
+        for next_node in expand(node):
+            if next_node not in visited:
+                visited.add(next_node)
+                open_q.append(next_node)
+
+def bfs_all_unique(initial, expand):
+    "bfs, but doesn't keep track of visited (aka seen), because there can be no repetitions"
+    open_q = deque(list(initial))
+    while open_q:
+        node = open_q.popleft()
+        yield node
+        open_q += expand(node)
+
+
+def _serialize(value: Any, memo: Optional[SerializeMemoizer]) -> Any:
+    if isinstance(value, Serialize):
+        return value.serialize(memo)
+    elif isinstance(value, list):
+        return [_serialize(elem, memo) for elem in value]
+    elif isinstance(value, frozenset):
+        return list(value)  # TODO reversible?
+    elif isinstance(value, dict):
+        return {key:_serialize(elem, memo) for key, elem in value.items()}
+    # assert value is None or isinstance(value, (int, float, str, tuple)), value
+    return value
+
+
+
+
+def small_factors(n: int, max_factor: int) -> List[Tuple[int, int]]:
+    """
+    Splits n up into smaller factors and summands <= max_factor.
+    Returns a list of [(a, b), ...]
+    so that the following code returns n:
+
+    n = 1
+    for a, b in values:
+        n = n * a + b
+
+    Currently, we also keep a + b <= max_factor, but that might change
+    """
+    assert n >= 0
+    assert max_factor > 2
+    if n <= max_factor:
+        return [(n, 0)]
+
+    for a in range(max_factor, 1, -1):
+        r, b = divmod(n, a)
+        if a + b <= max_factor:
+            return small_factors(r, max_factor) + [(a, b)]
+    assert False, "Failed to factorize %s" % n
+
+
+class OrderedSet(AbstractSet[T]):
+    """A minimal OrderedSet implementation, using a dictionary.
+
+    (relies on the dictionary being ordered)
+    """
+    def __init__(self, items: Iterable[T] =()):
+        self.d = dict.fromkeys(items)
+
+    def __contains__(self, item: Any) -> bool:
+        return item in self.d
+
+    def add(self, item: T):
+        self.d[item] = None
+
+    def __iter__(self) -> Iterator[T]:
+        return iter(self.d)
+
+    def remove(self, item: T):
+        del self.d[item]
+
+    def __bool__(self):
+        return bool(self.d)
+
+    def __len__(self) -> int:
+        return len(self.d)