maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/dataframe/misc/eval.py

@@ -0,0 +1,730 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ast
+import binascii
+import operator
+import sys
+import textwrap
+import tokenize
+from collections import OrderedDict
+from functools import reduce
+from io import StringIO
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, OutputType, get_output_types
+from ...serialization.serializables import BoolField, DictField, StringField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index
+
+LOCAL_TAG = "_local_var_"
+BACKTICK_TAG = "_backtick_var_"
+
+
+def _tokenize_str(reader):
+    token_generator = tokenize.generate_tokens(reader)
+
+    def _iter_backtick_string(gen, line, back_start):
+        for _, tokval, start, _, _ in gen:
+            if tokval == "`":
+                return (
+                    BACKTICK_TAG
+                    + binascii.b2a_hex(
+                        line[back_start[1] + 1 : start[1]].encode()
+                    ).decode()
+                )
+        else:
+            raise SyntaxError(f"backtick quote at {back_start} does not match")
+
+    for toknum, tokval, start, _, line in token_generator:
+        if toknum == tokenize.OP:
+            if tokval == "@":
+                tokval = LOCAL_TAG
+            if tokval == "&":
+                toknum = tokenize.NAME
+                tokval = "and"
+            elif tokval == "|":
+                toknum = tokenize.NAME
+                tokval = "or"
+        elif tokval == "`":
+            yield tokenize.NAME, _iter_backtick_string(token_generator, line, start)
+            continue
+        yield toknum, tokval
+
+
+class CollectionVisitor(ast.NodeVisitor):
+    _op_handlers = {
+        ast.Add: operator.add,
+        ast.Sub: operator.sub,
+        ast.Mult: operator.mul,
+        ast.Div: operator.truediv,
+        ast.FloorDiv: operator.floordiv,
+        ast.mod: operator.mod,
+        ast.Pow: operator.pow,
+        ast.Eq: operator.eq,
+        ast.NotEq: operator.ne,
+        ast.Lt: operator.lt,
+        ast.LtE: operator.le,
+        ast.Gt: operator.gt,
+        ast.GtE: operator.ge,
+        ast.In: lambda x, y: y.isin(x),
+        ast.NotIn: lambda x, y: ~y.isin(x),
+        ast.UAdd: operator.pos,
+        ast.USub: operator.neg,
+        ast.Invert: operator.invert,
+        ast.And: operator.and_,
+        ast.Or: operator.or_,
+    }
+
+    def __init__(self, resolvers, target, env):
+        self.env = env
+        self.target = target
+        self.resolvers = resolvers
+
+        self.referenced_vars = set()
+        self.assigned = False
+        self.entity_subscribe = False
+
+    def _preparse(self, expr):
+        reader = StringIO(expr).readline
+        return tokenize.untokenize(list(_tokenize_str(reader)))
+
+    def eval(self, expr, rewrite=True):
+        if rewrite:
+            expr = self._preparse(expr)
+        node = ast.fix_missing_locations(ast.parse(expr))
+        return self.visit(node)
+
+    def get_named_object(self, obj_name):
+        for resolver in self.resolvers:
+            try:
+                return resolver[obj_name]
+            except (IndexError, KeyError):
+                continue
+        if obj_name in self.env:
+            self.referenced_vars.add(obj_name)
+            return self.env[obj_name]
+        try:
+            return self.target[obj_name]
+        except KeyError:
+            pass
+        raise KeyError(f"name {obj_name} is not defined")
+
+    def visit(self, node):
+        if isinstance(node, ENTITY_TYPE):
+            return node
+        node_name = node.__class__.__name__
+        method = "visit_" + node_name
+        try:
+            visitor = getattr(self, method)
+        except AttributeError:
+            raise SyntaxError(
+                "Query string contains unsupported syntax: {}".format(node_name)
+            )
+        return visitor(node)
+
+    def visit_Module(self, node):
+        if self.target is None and len(node.body) != 1:
+            raise SyntaxError("Only a single expression is allowed")
+        result = None
+        for expr in node.body:
+            result = self.visit(expr)
+        return result
+
+    def visit_Expr(self, node):
+        return self.visit(node.value)
+
+    def visit_BinOp(self, node):
+        left = self.visit(node.left)
+        right = self.visit(node.right)
+        return self._op_handlers[type(node.op)](left, right)
+
+    def visit_Call(self, node):
+        func = self.visit(node.func)
+        args = [self.visit(n) for n in node.args]
+        kwargs = OrderedDict([(kw.arg, self.visit(kw.value)) for kw in node.keywords])
+        return func(*args, **kwargs)
+
+    def visit_Compare(self, node):
+        ops = node.ops
+        comps = node.comparators
+
+        if len(comps) == 1:
+            binop = ast.BinOp(op=ops[0], left=node.left, right=comps[0])
+            return self.visit(binop)
+
+        left = node.left
+        values = []
+        for op, comp in zip(ops, comps):
+            new_node = ast.Compare(comparators=[comp], left=left, ops=[op])
+            left = comp
+            values.append(new_node)
+        return self.visit(ast.BoolOp(op=ast.And(), values=values))
+
+    def visit_BoolOp(self, node):
+        def func(lhs, rhs):
+            binop = ast.BinOp(op=node.op, left=lhs, right=rhs)
+            return self.visit(binop)
+
+        return reduce(func, node.values)
+
+    def visit_UnaryOp(self, node):
+        op = self.visit(node.operand)
+        return self._op_handlers[type(node.op)](op)
+
+    def visit_Name(self, node):
+        if node.id.startswith(LOCAL_TAG):
+            local_name = node.id.replace(LOCAL_TAG, "")
+            self.referenced_vars.add(local_name)
+            return self.env[local_name]
+        if node.id.startswith(BACKTICK_TAG):
+            local_name = binascii.a2b_hex(
+                node.id.replace(BACKTICK_TAG, "").encode()
+            ).decode()
+            return self.get_named_object(local_name)
+        return self.get_named_object(node.id)
+
+    def visit_NameConstant(self, node):  # pragma: no cover
+        return node.value
+
+    def visit_Num(self, node):  # pragma: no cover
+        return node.n
+
+    def visit_Str(self, node):  # pragma: no cover
+        return node.s
+
+    def visit_Constant(self, node):
+        return node.value
+
+    def visit_List(self, node):
+        return [self.visit(e) for e in node.elts]
+
+    def visit_Assign(self, node):
+        if self.target is None:
+            raise ValueError("Target not specified for assignment")
+        if isinstance(node.targets[0], ast.Tuple):
+            raise ValueError("Does not support assigning to multiple objects")
+
+        target = node.targets[0].id
+        value = self.visit(node.value)
+        self.target[target] = value
+        self.assigned = True
+
+    visit_Tuple = visit_List
+
+    def visit_Attribute(self, node):
+        attr = node.attr
+        value = node.value
+
+        ctx = node.ctx
+        if isinstance(ctx, ast.Load):
+            resolved = self.visit(value)
+            return getattr(resolved, attr)
+
+        raise ValueError("Invalid Attribute context {0}".format(ctx.__name__))
+
+    def visit_Subscript(self, node):
+        value = self.visit(node.value)
+        sub = self.visit(node.slice)
+        if isinstance(value, ENTITY_TYPE):
+            self.entity_subscribe = True
+        return value[sub]
+
+    def visit_Index(self, node):
+        return self.visit(node.value)
+
+    def visit_Slice(self, node):
+        lower = node.lower
+        if lower is not None:
+            lower = self.visit(lower)
+        upper = node.upper
+        if upper is not None:
+            upper = self.visit(upper)
+        step = node.step
+        if step is not None:
+            step = self.visit(step)
+
+        return slice(lower, upper, step)
+
+
+class DataFrameEval(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.DATAFRAME_EVAL
+
+    expr = StringField("expr", default=None)
+    parser = StringField("parser", default=None)
+    engine = StringField("engine", default=None)
+    variables = DictField("variables", default=None)
+    self_target = BoolField("self_target", default=None)
+    is_query = BoolField("is_query", default=None)
+
+    def __call__(self, df, output_type, shape, dtypes):
+        self._output_types = [output_type]
+        params = df.params
+        new_index_value = (
+            df.index_value if not np.isnan(shape[0]) else parse_index(pd.RangeIndex(-1))
+        )
+        if output_type == OutputType.dataframe:
+            params.update(
+                dict(
+                    dtypes=dtypes,
+                    shape=shape,
+                    columns_value=parse_index(dtypes.index, store_data=True),
+                    index_value=new_index_value,
+                )
+            )
+        else:
+            name, dtype = dtypes
+            params = dict(
+                name=name,
+                dtype=dtype,
+                shape=shape,
+                index_value=new_index_value,
+            )
+        return self.new_tileable([df], **params)
+
+    def convert_to_query(self, df, output_type, shape, dtypes):
+        new_op = self.copy().reset_key()
+        new_op.is_query = True
+        new_op.self_target = False
+        return new_op(df, output_type, shape, dtypes)
+
+
+def maxframe_eval(
+    expr,
+    parser="maxframe",
+    engine=None,
+    local_dict=None,
+    global_dict=None,
+    resolvers=(),
+    level=0,
+    target=None,
+    inplace=False,
+):
+    """
+
+    Evaluate a Python expression as a string using various backends.
+
+    The following arithmetic operations are supported: ``+``, ``-``, ``*``,
+    ``/``, ``**``, ``%``, ``//`` (python engine only) along with the following
+    boolean operations: ``|`` (or), ``&`` (and), and ``~`` (not).
+    Additionally, the ``'pandas'`` parser allows the use of :keyword:`and`,
+    :keyword:`or`, and :keyword:`not` with the same semantics as the
+    corresponding bitwise operators.  :class:`~pandas.Series` and
+    :class:`~pandas.DataFrame` objects are supported and behave as they would
+    with plain ol' Python evaluation.
+
+    Parameters
+    ----------
+    expr : str
+        The expression to evaluate. This string cannot contain any Python
+        `statements
+        <https://docs.python.org/3/reference/simple_stmts.html#simple-statements>`__,
+        only Python `expressions
+        <https://docs.python.org/3/reference/simple_stmts.html#expression-statements>`__.
+    local_dict : dict or None, optional
+        A dictionary of local variables, taken from locals() by default.
+    global_dict : dict or None, optional
+        A dictionary of global variables, taken from globals() by default.
+    resolvers : list of dict-like or None, optional
+        A list of objects implementing the ``__getitem__`` special method that
+        you can use to inject an additional collection of namespaces to use for
+        variable lookup. For example, this is used in the
+        :meth:`~DataFrame.query` method to inject the
+        ``DataFrame.index`` and ``DataFrame.columns``
+        variables that refer to their respective :class:`~pandas.DataFrame`
+        instance attributes.
+    level : int, optional
+        The number of prior stack frames to traverse and add to the current
+        scope. Most users will **not** need to change this parameter.
+    target : object, optional, default None
+        This is the target object for assignment. It is used when there is
+        variable assignment in the expression. If so, then `target` must
+        support item assignment with string keys, and if a copy is being
+        returned, it must also support `.copy()`.
+    inplace : bool, default False
+        If `target` is provided, and the expression mutates `target`, whether
+        to modify `target` inplace. Otherwise, return a copy of `target` with
+        the mutation.
+
+    Returns
+    -------
+    ndarray, numeric scalar, DataFrame, Series
+
+    Raises
+    ------
+    ValueError
+        There are many instances where such an error can be raised:
+
+        - `target=None`, but the expression is multiline.
+        - The expression is multiline, but not all them have item assignment.
+          An example of such an arrangement is this:
+
+          a = b + 1
+          a + 2
+
+          Here, there are expressions on different lines, making it multiline,
+          but the last line has no variable assigned to the output of `a + 2`.
+        - `inplace=True`, but the expression is missing item assignment.
+        - Item assignment is provided, but the `target` does not support
+          string item assignment.
+        - Item assignment is provided and `inplace=False`, but the `target`
+          does not support the `.copy()` method
+
+    See Also
+    --------
+    DataFrame.query : Evaluates a boolean expression to query the columns
+            of a frame.
+    DataFrame.eval : Evaluate a string describing operations on
+            DataFrame columns.
+
+    Notes
+    -----
+    The ``dtype`` of any objects involved in an arithmetic ``%`` operation are
+    recursively cast to ``float64``.
+
+    See the :ref:`enhancing performance <enhancingperf.eval>` documentation for
+    more details.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({"animal": ["dog", "pig"], "age": [10, 20]})
+    >>> df.execute()
+      animal  age
+    0    dog   10
+    1    pig   20
+
+    We can add a new column using ``pd.eval``:
+
+    >>> md.eval("double_age = df.age * 2", target=df).execute()
+      animal  age  double_age
+    0    dog   10          20
+    1    pig   20          40
+    """
+    if not isinstance(expr, str):
+        raise TypeError("expr must be a string")
+
+    expr = textwrap.dedent(expr)
+
+    try:
+        frame = sys._getframe(level + 1)
+        local_dict = local_dict or dict()
+        local_dict.update(frame.f_locals)
+        global_dict = global_dict or dict()
+        global_dict.update(frame.f_globals)
+    finally:
+        del frame
+
+    env = dict()
+    env.update(global_dict)
+    env.update(local_dict)
+
+    ref_frames = set(resolvers) | set([target] if target is not None else [])
+    self_target = len(resolvers) > 0 and resolvers[0] is target
+
+    if target is not None and not inplace:
+        target = target.copy()
+
+    visitor = CollectionVisitor(resolvers, target, env)
+    result = visitor.eval(expr)
+    result = result if result is not None else target
+    has_var_frame = any(
+        isinstance(env[k], ENTITY_TYPE) for k in visitor.referenced_vars
+    )
+    if len(ref_frames) != 1 or visitor.entity_subscribe or has_var_frame:
+        if parser != "maxframe":
+            raise NotImplementedError(
+                "Does not support parser names other than maxframe"
+            )
+        if engine is not None:
+            raise NotImplementedError("Does not support specifying engine names")
+        return result
+    else:
+        parser = "pandas" if parser == "maxframe" else parser
+        referenced_env = {k: env[k] for k in visitor.referenced_vars}
+        op = DataFrameEval(
+            expr=expr,
+            parser=parser,
+            engine=engine,
+            variables=referenced_env,
+            self_target=visitor.assigned and self_target,
+            is_query=False,
+        )
+        output_type = get_output_types(result)[0]
+        dtypes = result.dtypes if result.ndim == 2 else (result.name, result.dtype)
+        return op(resolvers[0], output_type, result.shape, dtypes)
+
+
+def df_eval(df, expr, inplace=False, **kwargs):
+    """
+    Evaluate a string describing operations on DataFrame columns.
+
+    Operates on columns only, not specific rows or elements.  This allows
+    `eval` to run arbitrary code, which can make you vulnerable to code
+    injection if you pass user input to this function.
+
+    Parameters
+    ----------
+    expr : str
+        The expression string to evaluate.
+    inplace : bool, default False
+        If the expression contains an assignment, whether to perform the
+        operation inplace and mutate the existing DataFrame. Otherwise,
+        a new DataFrame is returned.
+    **kwargs
+        See the documentation for :func:`eval` for complete details
+        on the keyword arguments accepted by
+        :meth:`~pandas.DataFrame.query`.
+
+    Returns
+    -------
+    ndarray, scalar, or pandas object
+        The result of the evaluation.
+
+    See Also
+    --------
+    DataFrame.query : Evaluates a boolean expression to query the columns
+        of a frame.
+    DataFrame.assign : Can evaluate an expression or function to create new
+        values for a column.
+    eval : Evaluate a Python expression as a string using various
+        backends.
+
+    Notes
+    -----
+    For more details see the API documentation for :func:`~eval`.
+    For detailed examples see :ref:`enhancing performance with eval
+    <enhancingperf.eval>`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'A': range(1, 6), 'B': range(10, 0, -2)})
+    >>> df.execute()
+       A   B
+    0  1  10
+    1  2   8
+    2  3   6
+    3  4   4
+    4  5   2
+    >>> df.eval('A + B').execute()
+    0    11
+    1    10
+    2     9
+    3     8
+    4     7
+    dtype: int64
+
+    Assignment is allowed though by default the original DataFrame is not
+    modified.
+
+    >>> df.eval('C = A + B').execute()
+       A   B   C
+    0  1  10  11
+    1  2   8  10
+    2  3   6   9
+    3  4   4   8
+    4  5   2   7
+    >>> df.execute()
+       A   B
+    0  1  10
+    1  2   8
+    2  3   6
+    3  4   4
+    4  5   2
+
+    Use ``inplace=True`` to modify the original DataFrame.
+
+    >>> df.eval('C = A + B', inplace=True)
+    >>> df.execute()
+       A   B   C
+    0  1  10  11
+    1  2   8  10
+    2  3   6   9
+    3  4   4   8
+    4  5   2   7
+
+    Multiple columns can be assigned to using multi-line expressions:
+
+    >>> df.eval('''
+    ... C = A + B
+    ... D = A - B
+    ... ''').execute()
+       A   B   C  D
+    0  1  10  11 -9
+    1  2   8  10 -6
+    2  3   6   9 -3
+    3  4   4   8  0
+    4  5   2   7  3
+    """
+    level = kwargs.pop("level", None) or 0
+    kwargs["inplace"] = inplace
+    val = maxframe_eval(expr, resolvers=(df,), target=df, level=level + 1, **kwargs)
+    if not inplace:
+        return val
+
+
+def df_query(df, expr, inplace=False, **kwargs):
+    """
+    Query the columns of a DataFrame with a boolean expression.
+
+    Parameters
+    ----------
+    expr : str
+        The query string to evaluate.
+
+        You can refer to variables
+        in the environment by prefixing them with an '@' character like
+        ``@a + b``.
+
+        You can refer to column names that contain spaces or operators by
+        surrounding them in backticks. This way you can also escape
+        names that start with a digit, or those that  are a Python keyword.
+        Basically when it is not valid Python identifier. See notes down
+        for more details.
+
+        For example, if one of your columns is called ``a a`` and you want
+        to sum it with ``b``, your query should be ```a a` + b``.
+
+    inplace : bool
+        Whether the query should modify the data in place or return
+        a modified copy.
+    **kwargs
+        See the documentation for :func:`eval` for complete details
+        on the keyword arguments accepted by :meth:`DataFrame.query`.
+
+    Returns
+    -------
+    DataFrame
+        DataFrame resulting from the provided query expression.
+
+    See Also
+    --------
+    eval : Evaluate a string describing operations on
+        DataFrame columns.
+    DataFrame.eval : Evaluate a string describing operations on
+        DataFrame columns.
+
+    Notes
+    -----
+    The result of the evaluation of this expression is first passed to
+    :attr:`DataFrame.loc` and if that fails because of a
+    multidimensional key (e.g., a DataFrame) then the result will be passed
+    to :meth:`DataFrame.__getitem__`.
+
+    This method uses the top-level :func:`eval` function to
+    evaluate the passed query.
+
+    The :meth:`~pandas.DataFrame.query` method uses a slightly
+    modified Python syntax by default. For example, the ``&`` and ``|``
+    (bitwise) operators have the precedence of their boolean cousins,
+    :keyword:`and` and :keyword:`or`. This *is* syntactically valid Python,
+    however the semantics are different.
+
+    You can change the semantics of the expression by passing the keyword
+    argument ``parser='python'``. This enforces the same semantics as
+    evaluation in Python space. Likewise, you can pass ``engine='python'``
+    to evaluate an expression using Python itself as a backend. This is not
+    recommended as it is inefficient compared to using ``numexpr`` as the
+    engine.
+
+    The :attr:`DataFrame.index` and
+    :attr:`DataFrame.columns` attributes of the
+    :class:`~pandas.DataFrame` instance are placed in the query namespace
+    by default, which allows you to treat both the index and columns of the
+    frame as a column in the frame.
+    The identifier ``index`` is used for the frame index; you can also
+    use the name of the index to identify it in a query. Please note that
+    Python keywords may not be used as identifiers.
+
+    For further details and examples see the ``query`` documentation in
+    :ref:`indexing <indexing.query>`.
+
+    *Backtick quoted variables*
+
+    Backtick quoted variables are parsed as literal Python code and
+    are converted internally to a Python valid identifier.
+    This can lead to the following problems.
+
+    During parsing a number of disallowed characters inside the backtick
+    quoted string are replaced by strings that are allowed as a Python identifier.
+    These characters include all operators in Python, the space character, the
+    question mark, the exclamation mark, the dollar sign, and the euro sign.
+    For other characters that fall outside the ASCII range (U+0001..U+007F)
+    and those that are not further specified in PEP 3131,
+    the query parser will raise an error.
+    This excludes whitespace different than the space character,
+    but also the hashtag (as it is used for comments) and the backtick
+    itself (backtick can also not be escaped).
+
+    In a special case, quotes that make a pair around a backtick can
+    confuse the parser.
+    For example, ```it's` > `that's``` will raise an error,
+    as it forms a quoted string (``'s > `that'``) with a backtick inside.
+
+    See also the Python documentation about lexical analysis
+    (https://docs.python.org/3/reference/lexical_analysis.html)
+    in combination with the source code in :mod:`pandas.core.computation.parsing`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'A': range(1, 6),
+    ...                    'B': range(10, 0, -2),
+    ...                    'C C': range(10, 5, -1)})
+    >>> df.execute()
+       A   B  C C
+    0  1  10   10
+    1  2   8    9
+    2  3   6    8
+    3  4   4    7
+    4  5   2    6
+    >>> df.query('A > B').execute()
+       A  B  C C
+    4  5  2    6
+
+    The previous expression is equivalent to
+
+    >>> df[df.A > df.B].execute()
+       A  B  C C
+    4  5  2    6
+
+    For columns with spaces in their name, you can use backtick quoting.
+
+    >>> df.query('B == `C C`').execute()
+       A   B  C C
+    0  1  10   10
+
+    The previous expression is equivalent to
+
+    >>> df[df.B == df['C C']].execute()
+       A   B  C C
+    0  1  10   10
+    """
+    level = kwargs.pop("level", None) or 0
+    predicate = maxframe_eval(expr, resolvers=(df,), level=level + 1, **kwargs)
+    result = df[predicate]
+
+    if isinstance(predicate.op, DataFrameEval):
+        output_type = get_output_types(result)[0]
+        dtypes = result.dtypes if result.ndim == 2 else (result.name, result.dtype)
+        result = predicate.op.convert_to_query(df, output_type, result.shape, dtypes)
+
+    if inplace:
+        df.data = result.data
+    else:
+        return result