maxframe 0.1.0b3__cp37-cp37m-win_amd64.whl → 0.1.0b5__cp37-cp37m-win_amd64.whl

maxframe/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from . import dataframe, learn, remote, tensor
+from .config import options
 from .session import execute, fetch, new_session, stop_server
 
 

maxframe/codegen.py

@@ -16,6 +16,7 @@ import abc
 import base64
 import dataclasses
 import logging
+from collections import defaultdict
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
@@ -32,7 +33,7 @@ from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
 from .serialization.serializables import Serializable, StringField
 from .typing_ import PandasObjectTypes
-from .udf import MarkedFunction
+from .udf import MarkedFunction, PythonPackOptions
 
 if TYPE_CHECKING:
     from odpsctx import ODPSSessionContext
@@ -75,6 +76,14 @@ class AbstractUDF(Serializable):
     def unregister(self, odps: "ODPSSessionContext"):
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def collect_pythonpack(self) -> List[PythonPackOptions]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load_pythonpack_resources(self, odps_ctx: "ODPSSessionContext") -> None:
+        raise NotImplementedError
+
 
 class UserCodeMixin:
     @classmethod
@@ -469,6 +478,42 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
             output_key_to_result_infos=self._context.get_tileable_result_infos(),
         )
 
+    def run_pythonpacks(
+        self,
+        odps_ctx: "ODPSSessionContext",
+        python_tag: str,
+        is_production: bool = False,
+        schedule_id: Optional[str] = None,
+        hints: Optional[dict] = None,
+        priority: Optional[int] = None,
+    ) -> Dict[str, PythonPackOptions]:
+        key_to_packs = defaultdict(list)
+        for udf in self._context.get_udfs():
+            for pack in udf.collect_pythonpack():
+                key_to_packs[pack.key].append(pack)
+        distinct_packs = []
+        for packs in key_to_packs.values():
+            distinct_packs.append(packs[0])
+
+        inst_id_to_req = {}
+        for pack in distinct_packs:
+            inst = odps_ctx.run_pythonpack(
+                requirements=pack.requirements,
+                prefer_binary=pack.prefer_binary,
+                pre_release=pack.pre_release,
+                force_rebuild=pack.force_rebuild,
+                python_tag=python_tag,
+                is_production=is_production,
+                schedule_id=schedule_id,
+                hints=hints,
+                priority=priority,
+            )
+            # fulfill instance id of pythonpacks with same keys
+            for same_pack in key_to_packs[pack.key]:
+                same_pack.pack_instance_id = inst.id
+            inst_id_to_req[inst.id] = pack
+        return inst_id_to_req
+
     def register_udfs(self, odps_ctx: "ODPSSessionContext"):
         for udf in self._context.get_udfs():
             logger.info("[Session %s] Registering UDF %s", self._session_id, udf.name)

maxframe/config/config.py

@@ -40,6 +40,7 @@ _DEFAULT_SPE_OPERATION_TIMEOUT_SECONDS = 120
 _DEFAULT_UPLOAD_BATCH_SIZE = 4096
 _DEFAULT_TEMP_LIFECYCLE = 1
 _DEFAULT_TASK_START_TIMEOUT = 60
+_DEFAULT_LOGVIEW_HOURS = 24 * 60
 
 
 class OptionError(Exception):
@@ -296,13 +297,15 @@ class Config:
 
 
 default_options = Config()
-
 default_options.register_option(
     "execution_mode", "trigger", validator=is_in(["trigger", "eager"])
 )
 default_options.register_option(
     "python_tag", get_python_tag(), validator=is_string, remote=True
 )
+default_options.register_option(
+    "session.logview_hours", _DEFAULT_LOGVIEW_HOURS, validator=is_integer, remote=True
+)
 default_options.register_option(
     "client.task_start_timeout", _DEFAULT_TASK_START_TIMEOUT, validator=is_integer
 )
@@ -312,6 +315,9 @@ default_options.register_option(
 )
 default_options.register_option("sql.settings", {}, validator=is_dict, remote=True)
 
+default_options.register_option("is_production", False, validator=is_bool, remote=True)
+default_options.register_option("schedule_id", "", validator=is_string, remote=True)
+
 default_options.register_option(
     "session.max_alive_seconds",
     _DEFAULT_MAX_ALIVE_SECONDS,
@@ -358,6 +364,9 @@ default_options.register_option(
 default_options.register_option(
     "show_progress", "auto", validator=any_validator(is_bool, is_string)
 )
+default_options.register_option(
+    "dag.settings", value=dict(), validator=is_dict, remote=True
+)
 
 ################
 # SPE Settings #
@@ -373,6 +382,10 @@ default_options.register_option(
     "spe.task.settings", dict(), validator=is_dict, remote=True
 )
 
+default_options.register_option(
+    "pythonpack.task.settings", {}, validator=is_dict, remote=True
+)
+
 _options_ctx_var = contextvars.ContextVar("_options_ctx_var")
 
 

maxframe/dataframe/__init__.py

@@ -46,6 +46,7 @@ from .misc.cut import cut
 from .misc.eval import maxframe_eval as eval  # pylint: disable=redefined-builtin
 from .misc.get_dummies import get_dummies
 from .misc.melt import melt
+from .misc.pivot_table import pivot_table
 from .misc.qcut import qcut
 from .misc.to_numeric import to_numeric
 from .missing import isna, isnull, notna, notnull
@@ -57,6 +58,11 @@ try:
 except ImportError:  # pragma: no cover
     pass
 
+try:
+    from . import _internal
+except ImportError:  # pragma: no cover
+    pass
+
 del (
     arithmetic,
     datasource,

maxframe/dataframe/core.py

@@ -35,6 +35,7 @@ from ..core import (
     register_output_types,
 )
 from ..core.entity.utils import refresh_tileable_shape
+from ..protocol import DataFrameTableMeta
 from ..serialization.serializables import (
     AnyField,
     BoolField,
@@ -59,7 +60,13 @@ from ..utils import (
     on_serialize_numpy_type,
     tokenize,
 )
-from .utils import ReprSeries, fetch_corner_data, merge_index_value, parse_index
+from .utils import (
+    ReprSeries,
+    apply_if_callable,
+    fetch_corner_data,
+    merge_index_value,
+    parse_index,
+)
 
 
 class IndexValue(Serializable):
@@ -616,6 +623,9 @@ class IndexData(HasShapeTileableData, _ToPandasMixin):
         if self._name is None:
             self._name = self.chunks[0].name
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -945,6 +955,9 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
         if self._name is None:
             self._name = self.chunks[0].name
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -960,7 +973,9 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
             buf = StringIO()
             max_rows = pd.get_option("display.max_rows")
             corner_max_rows = (
-                max_rows if self.shape[0] <= max_rows else corner_data.shape[0] - 1
+                max_rows
+                if self.shape[0] <= max_rows or corner_data.shape[0] == 0
+                else corner_data.shape[0] - 1
             )  # make sure max_rows < corner_data
 
             with pd.option_context("display.max_rows", corner_max_rows):
@@ -976,7 +991,7 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
         return self._to_str(representation=False)
 
     def __repr__(self):
-        return self._to_str(representation=False)
+        return self._to_str(representation=True)
 
     @property
     def dtype(self):
@@ -1499,6 +1514,15 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         refresh_index_value(self)
         refresh_dtypes(self)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        dtypes = table_meta.pd_column_dtypes
+        self._dtypes = dtypes
+        self._columns_value = parse_index(dtypes.index, store_data=True)
+        self._dtypes_value = DtypesValue(key=tokenize(dtypes), value=dtypes)
+        new_shape = list(self._shape)
+        new_shape[0] = len(dtypes)
+        self._shape = tuple(new_shape)
+
     @property
     def dtypes(self):
         dt = getattr(self, "_dtypes", None)
@@ -1605,7 +1629,7 @@ class DataFrameData(_BatchedFetcher, BaseDataFrameData):
             buf = StringIO()
             max_rows = pd.get_option("display.max_rows")
 
-            if self.shape[0] <= max_rows:
+            if self.shape[0] <= max_rows or corner_data.shape[0] == 0:
                 buf.write(repr(corner_data) if representation else str(corner_data))
             else:
                 # remember we cannot directly call repr(df),
@@ -1995,12 +2019,6 @@ class DataFrame(HasShapeTileable, _ToPandasMixin):
         Berkeley    25.0    77.0  298.15
         """
 
-        def apply_if_callable(maybe_callable, obj, **kwargs):
-            if callable(maybe_callable):
-                return maybe_callable(obj, **kwargs)
-
-            return maybe_callable
-
         data = self.copy()
 
         for k, v in kwargs.items():
@@ -2195,6 +2213,9 @@ class CategoricalData(HasShapeTileableData, _ToPandasMixin):
                 pd.Categorical(categories).categories, store_data=True
             )
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -2345,6 +2366,9 @@ class DataFrameOrSeriesData(HasShapeTileableData, _ToPandasMixin):
             data_params["name"] = self.chunks[0].name
         self._data_params.update(data_params)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def ensure_data(self):
         from .fetch.core import DataFrameFetch
 

maxframe/dataframe/datasource/read_odps_query.py

@@ -216,7 +216,9 @@ class DataFrameReadODPSQuery(
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(
-                pd.Index([], name=self.index_columns[0]).astype(self.index_dtypes[0])
+                pd.Index([], name=self.index_columns[0]).astype(
+                    self.index_dtypes.iloc[0]
+                )
             )
         else:
             idx = pd.MultiIndex.from_frame(
@@ -263,7 +265,9 @@ def read_odps_query(
     result: DataFrame
         DataFrame read from MaxCompute (ODPS) table
     """
-    odps_entry = odps_entry or ODPS.from_environments()
+    odps_entry = odps_entry or ODPS.from_global() or ODPS.from_environments()
+    if odps_entry is None:
+        raise ValueError("Missing odps_entry parameter")
     inst = odps_entry.execute_sql(f"EXPLAIN {query}")
     explain_str = list(inst.get_task_results().values())[0]
 

maxframe/dataframe/datasource/read_odps_table.py

@@ -82,7 +82,9 @@ class DataFrameReadODPSTable(
                 index_value = parse_index(pd.RangeIndex(shape[0]))
         elif len(self.index_columns) == 1:
             index_value = parse_index(
-                pd.Index([], name=self.index_columns[0]).astype(self.index_dtypes[0])
+                pd.Index([], name=self.index_columns[0]).astype(
+                    self.index_dtypes.iloc[0]
+                )
             )
         else:
             idx = pd.MultiIndex.from_frame(
@@ -164,6 +166,8 @@ def read_odps_table(
         DataFrame read from MaxCompute (ODPS) table
     """
     odps_entry = odps_entry or ODPS.from_global() or ODPS.from_environments()
+    if odps_entry is None:
+        raise ValueError("Missing odps_entry parameter")
     if isinstance(table_name, Table):
         table = table_name
     else:

maxframe/dataframe/datastore/core.py

@@ -0,0 +1,19 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameDataStore(DataFrameOperator, DataFrameOperatorMixin):
+    pass

maxframe/dataframe/datastore/to_csv.py

@@ -23,11 +23,11 @@ from ...serialization.serializables import (
     ListField,
     StringField,
 )
-from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import parse_index
+from .core import DataFrameDataStore
 
 
-class DataFrameToCSV(DataFrameOperator, DataFrameOperatorMixin):
+class DataFrameToCSV(DataFrameDataStore):
     _op_type_ = opcodes.TO_CSV
 
     input = KeyField("input")

maxframe/dataframe/datastore/to_odps.py

@@ -32,13 +32,13 @@ from ...serialization.serializables import (
 )
 from ...typing_ import TileableType
 from ..core import DataFrame  # noqa: F401
-from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import parse_index
+from .core import DataFrameDataStore
 
 logger = logging.getLogger(__name__)
 
 
-class DataFrameToODPSTable(DataFrameOperator, DataFrameOperatorMixin):
+class DataFrameToODPSTable(DataFrameDataStore):
     _op_type_ = opcodes.TO_ODPS_TABLE
 
     dtypes = SeriesField("dtypes")

maxframe/dataframe/indexing/reset_index.py

@@ -107,7 +107,6 @@ def df_reset_index(
     inplace=False,
     col_level=0,
     col_fill="",
-    incremental_index=False,
 ):
     """
     Reset the index, or a level of it.
@@ -133,12 +132,6 @@ def df_reset_index(
     col_fill : object, default ''
         If the columns have multiple levels, determines how the other
         levels are named. If None then the index name is repeated.
-    incremental_index: bool, default False
-        Ensure RangeIndex incremental, when output DataFrame has multiple chunks,
-        ensuring index incremental costs more computation,
-        so by default, each chunk will have index which starts from 0,
-        setting incremental_index=True，reset_index will guarantee that
-        output DataFrame's index is from 0 to n - 1.
 
     Returns
     -------
@@ -264,7 +257,6 @@ def df_reset_index(
         drop=drop,
         col_level=col_level,
         col_fill=col_fill,
-        incremental_index=incremental_index,
         output_types=[OutputType.dataframe],
     )
     ret = op(df)
@@ -280,7 +272,6 @@ def series_reset_index(
     drop=False,
     name=no_default,
     inplace=False,
-    incremental_index=False,
 ):
     """
     Generate a new DataFrame or Series with the index reset.
@@ -303,12 +294,6 @@ def series_reset_index(
         when `drop` is True.
     inplace : bool, default False
         Modify the Series in place (do not create a new object).
-    incremental_index: bool, default False
-        Ensure RangeIndex incremental, when output Series has multiple chunks,
-        ensuring index incremental costs more computation,
-        so by default, each chunk will have index which starts from 0,
-        setting incremental_index=True，reset_index will guarantee that
-        output Series's index is from 0 to n - 1.
 
     Returns
     -------
@@ -406,8 +391,7 @@ def series_reset_index(
         level=level,
         drop=drop,
         name=name,
-        incremental_index=incremental_index,
-        output_types=[OutputType.series],
+        output_types=[OutputType.series if drop else OutputType.dataframe],
     )
     ret = op(series)
     if not inplace:

maxframe/dataframe/misc/__init__.py

@@ -14,6 +14,7 @@
 
 from .apply import df_apply, series_apply
 from .astype import astype, index_astype
+from .case_when import case_when
 from .check_monotonic import (
     check_monotonic,
     is_monotonic,
@@ -37,6 +38,7 @@ from .map import index_map, series_map
 from .melt import melt
 from .memory_usage import df_memory_usage, index_memory_usage, series_memory_usage
 from .pct_change import pct_change
+from .pivot_table import pivot_table
 from .qcut import qcut
 from .select_dtypes import select_dtypes
 from .shift import shift, tshift
@@ -69,6 +71,7 @@ def _install():
         setattr(t, "melt", melt)
         setattr(t, "memory_usage", df_memory_usage)
         setattr(t, "pct_change", pct_change)
+        setattr(t, "pivot_table", pivot_table)
         setattr(t, "pop", df_pop)
         setattr(t, "query", df_query)
         setattr(t, "select_dtypes", select_dtypes)
@@ -81,6 +84,7 @@ def _install():
     for t in SERIES_TYPE:
         setattr(t, "apply", series_apply)
         setattr(t, "astype", astype)
+        setattr(t, "case_when", case_when)
         setattr(t, "check_monotonic", check_monotonic)
         setattr(t, "describe", describe)
         setattr(t, "diff", series_diff)

maxframe/dataframe/misc/apply.py

@@ -225,7 +225,7 @@ class ApplyOperator(
                 else:  # pragma: no cover
                     index_value = parse_index(infer_series.index)
             else:
-                index_value = parse_index(None, series)
+                index_value = parse_index(series.index_value)
 
             if output_type == OutputType.dataframe:
                 if dtypes is None:

maxframe/dataframe/misc/case_when.py

@@ -0,0 +1,141 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+from pandas.core.dtypes.cast import find_common_type
+
+from ... import opcodes
+from ...core import TILEABLE_TYPE
+from ...serialization.serializables import FieldTypes, ListField
+from ..core import SERIES_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import apply_if_callable
+
+
+class DataFrameCaseWhen(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CASE_WHEN
+
+    conditions = ListField("conditions", FieldTypes.reference, default=None)
+    replacements = ListField("replacements", FieldTypes.reference, default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        it = iter(inputs)
+        next(it)
+        self.conditions = [
+            next(it) if isinstance(t, TILEABLE_TYPE) else t for t in self.conditions
+        ]
+        self.replacements = [
+            next(it) if isinstance(t, TILEABLE_TYPE) else t for t in self.replacements
+        ]
+
+    def __call__(self, series):
+        replacement_dtypes = [
+            it.dtype if isinstance(it, SERIES_TYPE) else np.array(it).dtype
+            for it in self.replacements
+        ]
+        dtype = find_common_type([series.dtype] + replacement_dtypes)
+
+        condition_tileables = [
+            it for it in self.conditions if isinstance(it, TILEABLE_TYPE)
+        ]
+        replacement_tileables = [
+            it for it in self.replacements if isinstance(it, TILEABLE_TYPE)
+        ]
+        inputs = [series] + condition_tileables + replacement_tileables
+
+        params = series.params
+        params["dtype"] = dtype
+        return self.new_series(inputs, **params)
+
+
+def case_when(series, caselist):
+    """
+    Replace values where the conditions are True.
+
+    Parameters
+    ----------
+    caselist : A list of tuples of conditions and expected replacements
+        Takes the form:  ``(condition0, replacement0)``,
+        ``(condition1, replacement1)``, ... .
+        ``condition`` should be a 1-D boolean array-like object
+        or a callable. If ``condition`` is a callable,
+        it is computed on the Series
+        and should return a boolean Series or array.
+        The callable must not change the input Series
+        (though pandas doesn`t check it). ``replacement`` should be a
+        1-D array-like object, a scalar or a callable.
+        If ``replacement`` is a callable, it is computed on the Series
+        and should return a scalar or Series. The callable
+        must not change the input Series.
+
+    Returns
+    -------
+    Series
+
+    See Also
+    --------
+    Series.mask : Replace values where the condition is True.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> c = md.Series([6, 7, 8, 9], name='c')
+    >>> a = md.Series([0, 0, 1, 2])
+    >>> b = md.Series([0, 3, 4, 5])
+
+    >>> c.case_when(caselist=[(a.gt(0), a),  # condition, replacement
+    ...                       (b.gt(0), b)])
+    0    6
+    1    3
+    2    1
+    3    2
+    Name: c, dtype: int64
+    """
+    if not isinstance(caselist, list):
+        raise TypeError(
+            f"The caselist argument should be a list; instead got {type(caselist)}"
+        )
+
+    if not caselist:
+        raise ValueError(
+            "provide at least one boolean condition, "
+            "with a corresponding replacement."
+        )
+
+    for num, entry in enumerate(caselist):
+        if not isinstance(entry, tuple):
+            raise TypeError(
+                f"Argument {num} must be a tuple; instead got {type(entry)}."
+            )
+        if len(entry) != 2:
+            raise ValueError(
+                f"Argument {num} must have length 2; "
+                "a condition and replacement; "
+                f"instead got length {len(entry)}."
+            )
+    caselist = [
+        (
+            apply_if_callable(condition, series),
+            apply_if_callable(replacement, series),
+        )
+        for condition, replacement in caselist
+    ]
+    conditions = [case[0] for case in caselist]
+    replacements = [case[1] for case in caselist]
+    op = DataFrameCaseWhen(conditions=conditions, replacements=replacements)
+    return op(series)