maxframe 0.1.0b4__tar.gz → 1.0.0rc1__tar.gz

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/MANIFEST.in

@@ -5,4 +5,7 @@ global-exclude *.c
 global-exclude *.cpp
 include pyproject.toml
 include maxframe/lib/mmh3_src/*
+exclude **/tests/**/*
+exclude **/tests/*
+global-exclude conftest.py
 global-exclude .DS_Store

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b4
+Version: 1.0.0rc1
 Summary: MaxFrame operator-based data analyze framework
 Provides-Extra: dev
 Provides-Extra: test

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from . import dataframe, learn, remote, tensor
+from .config import options
 from .session import execute, fetch, new_session, stop_server
 
 

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/codegen.py

@@ -16,6 +16,7 @@ import abc
 import base64
 import dataclasses
 import logging
+from collections import defaultdict
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
@@ -32,7 +33,7 @@ from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
 from .serialization.serializables import Serializable, StringField
 from .typing_ import PandasObjectTypes
-from .udf import MarkedFunction
+from .udf import MarkedFunction, PythonPackOptions
 
 if TYPE_CHECKING:
     from odpsctx import ODPSSessionContext
@@ -75,8 +76,18 @@ class AbstractUDF(Serializable):
     def unregister(self, odps: "ODPSSessionContext"):
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def collect_pythonpack(self) -> List[PythonPackOptions]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load_pythonpack_resources(self, odps_ctx: "ODPSSessionContext") -> None:
+        raise NotImplementedError
+
 
 class UserCodeMixin:
+    __slots__ = ()
+
     @classmethod
     def obj_to_python_expr(cls, obj: Any = None) -> str:
         """
@@ -194,8 +205,12 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
         return self._session_id
 
     def register_udf(self, udf: AbstractUDF):
+        from maxframe_framedriver.services.session import SessionManager
+
         udf.session_id = self._session_id
         self._udfs[udf.name] = udf
+        if self._session_id and SessionManager.initialized():
+            SessionManager.instance().register_udf(self._session_id, udf)
 
     def get_udfs(self) -> List[AbstractUDF]:
         return list(self._udfs.values())
@@ -335,6 +350,8 @@ def register_engine_codegen(type_: Type["BigDagCodeGenerator"]):
 BUILTIN_ENGINE_SPE = "SPE"
 BUILTIN_ENGINE_MCSQL = "MCSQL"
 
+FAST_RANGE_INDEX_ENABLED = "codegen.fast_range_index_enabled"
+
 
 class BigDagCodeGenerator(metaclass=abc.ABCMeta):
     _context: BigDagCodeContext
@@ -469,14 +486,50 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
             output_key_to_result_infos=self._context.get_tileable_result_infos(),
         )
 
+    def run_pythonpacks(
+        self,
+        odps_ctx: "ODPSSessionContext",
+        python_tag: str,
+        is_production: bool = False,
+        schedule_id: Optional[str] = None,
+        hints: Optional[dict] = None,
+        priority: Optional[int] = None,
+    ) -> Dict[str, PythonPackOptions]:
+        key_to_packs = defaultdict(list)
+        for udf in self._context.get_udfs():
+            for pack in udf.collect_pythonpack():
+                key_to_packs[pack.key].append(pack)
+        distinct_packs = []
+        for packs in key_to_packs.values():
+            distinct_packs.append(packs[0])
+
+        inst_id_to_req = {}
+        for pack in distinct_packs:
+            inst = odps_ctx.run_pythonpack(
+                requirements=pack.requirements,
+                prefer_binary=pack.prefer_binary,
+                pre_release=pack.pre_release,
+                force_rebuild=pack.force_rebuild,
+                python_tag=python_tag,
+                is_production=is_production,
+                schedule_id=schedule_id,
+                hints=hints,
+                priority=priority,
+            )
+            # fulfill instance id of pythonpacks with same keys
+            for same_pack in key_to_packs[pack.key]:
+                same_pack.pack_instance_id = inst.id
+            inst_id_to_req[inst.id] = pack
+        return inst_id_to_req
+
     def register_udfs(self, odps_ctx: "ODPSSessionContext"):
         for udf in self._context.get_udfs():
-            logger.info("[Session %s] Registering UDF %s", self._session_id, udf.name)
+            logger.info("[Session=%s] Registering UDF %s", self._session_id, udf.name)
             udf.register(odps_ctx, True)
 
     def unregister_udfs(self, odps_ctx: "ODPSSessionContext"):
         for udf in self._context.get_udfs():
-            logger.info("[Session %s] Unregistering UDF %s", self._session_id, udf.name)
+            logger.info("[Session=%s] Unregistering UDF %s", self._session_id, udf.name)
             udf.unregister(odps_ctx)
 
     def get_udfs(self) -> List[AbstractUDF]:

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/config/config.py

@@ -40,6 +40,8 @@ _DEFAULT_SPE_OPERATION_TIMEOUT_SECONDS = 120
 _DEFAULT_UPLOAD_BATCH_SIZE = 4096
 _DEFAULT_TEMP_LIFECYCLE = 1
 _DEFAULT_TASK_START_TIMEOUT = 60
+_DEFAULT_TASK_RESTART_TIMEOUT = 300
+_DEFAULT_LOGVIEW_HOURS = 24 * 60
 
 
 class OptionError(Exception):
@@ -296,22 +298,30 @@ class Config:
 
 
 default_options = Config()
-
 default_options.register_option(
     "execution_mode", "trigger", validator=is_in(["trigger", "eager"])
 )
 default_options.register_option(
     "python_tag", get_python_tag(), validator=is_string, remote=True
 )
+default_options.register_option(
+    "session.logview_hours", _DEFAULT_LOGVIEW_HOURS, validator=is_integer, remote=True
+)
 default_options.register_option(
     "client.task_start_timeout", _DEFAULT_TASK_START_TIMEOUT, validator=is_integer
 )
+default_options.register_option(
+    "client.task_restart_timeout", _DEFAULT_TASK_RESTART_TIMEOUT, validator=is_integer
+)
 default_options.register_option("sql.enable_mcqa", True, validator=is_bool, remote=True)
 default_options.register_option(
     "sql.generate_comments", True, validator=is_bool, remote=True
 )
 default_options.register_option("sql.settings", {}, validator=is_dict, remote=True)
 
+default_options.register_option("is_production", False, validator=is_bool, remote=True)
+default_options.register_option("schedule_id", "", validator=is_string, remote=True)
+
 default_options.register_option(
     "session.max_alive_seconds",
     _DEFAULT_MAX_ALIVE_SECONDS,
@@ -376,6 +386,10 @@ default_options.register_option(
     "spe.task.settings", dict(), validator=is_dict, remote=True
 )
 
+default_options.register_option(
+    "pythonpack.task.settings", {}, validator=is_dict, remote=True
+)
+
 _options_ctx_var = contextvars.ContextVar("_options_ctx_var")
 
 

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/core/__init__.py

@@ -19,7 +19,6 @@ from .entity import (
     CHUNK_TYPE,
     ENTITY_TYPE,
     FUSE_CHUNK_TYPE,
-    OBJECT_CHUNK_TYPE,
     OBJECT_TYPE,
     TILEABLE_TYPE,
     Chunk,
@@ -33,8 +32,6 @@ from .entity import (
     HasShapeTileableData,
     NotSupportTile,
     Object,
-    ObjectChunk,
-    ObjectChunkData,
     ObjectData,
     OutputType,
     Tileable,

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/core/entity/__init__.py

@@ -16,14 +16,7 @@ from .chunks import CHUNK_TYPE, Chunk, ChunkData
 from .core import ENTITY_TYPE, Entity, EntityData
 from .executable import ExecutableTuple, _ExecuteAndFetchMixin
 from .fuse import FUSE_CHUNK_TYPE, FuseChunk, FuseChunkData
-from .objects import (
-    OBJECT_CHUNK_TYPE,
-    OBJECT_TYPE,
-    Object,
-    ObjectChunk,
-    ObjectChunkData,
-    ObjectData,
-)
+from .objects import OBJECT_TYPE, Object, ObjectData
 from .output_types import (
     OutputType,
     get_fetch_class,

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/core/entity/objects.py

@@ -14,58 +14,17 @@
 
 from typing import Any, Dict
 
-from ...serialization.serializables import FieldTypes, ListField
-from ...utils import skip_na_call
-from .chunks import Chunk, ChunkData
 from .core import Entity
 from .executable import _ToObjectMixin
 from .tileables import TileableData
 
 
-class ObjectChunkData(ChunkData):
-    # chunk whose data could be any serializable
-    __slots__ = ()
-    type_name = "Object"
-
-    def __init__(self, op=None, index=None, **kw):
-        super().__init__(_op=op, _index=index, **kw)
-
-    @property
-    def params(self) -> Dict[str, Any]:
-        # params return the properties which useful to rebuild a new chunk
-        return {
-            "index": self.index,
-        }
-
-    @params.setter
-    def params(self, new_params: Dict[str, Any]):
-        params = new_params.copy()
-        params.pop("index", None)  # index not needed to update
-        if params:  # pragma: no cover
-            raise TypeError(f"Unknown params: {list(params)}")
-
-    @classmethod
-    def get_params_from_data(cls, data: Any) -> Dict[str, Any]:
-        return dict()
-
-
-class ObjectChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (ObjectChunkData,)
-    type_name = "Object"
-
-
 class ObjectData(TileableData, _ToObjectMixin):
     __slots__ = ()
     type_name = "Object"
-
-    # optional fields
-    _chunks = ListField(
-        "chunks",
-        FieldTypes.reference(ObjectChunkData),
-        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
-        on_deserialize=skip_na_call(lambda x: [ObjectChunk(it) for it in x]),
-    )
+    # workaround for removed field since v0.1.0b5
+    # todo remove this when all versions below v0.1.0b5 is eliminated
+    _legacy_deprecated_non_primitives = ["_chunks"]
 
     def __init__(self, op=None, nsplits=None, **kw):
         super().__init__(_op=op, _nsplits=nsplits, **kw)
@@ -97,4 +56,3 @@ class Object(Entity, _ToObjectMixin):
 
 
 OBJECT_TYPE = (Object, ObjectData)
-OBJECT_CHUNK_TYPE = (ObjectChunk, ObjectChunkData)

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/core/graph/core.pyx

@@ -354,10 +354,10 @@ cdef class DirectedGraph:
                         sio.write(f'"Chunk:{self._gen_chunk_key(input_chunk, trunc_key)}" {chunk_style}\n')
                         visited.add(input_chunk.key)
                     if op.key not in visited:
-                        sio.write(f'"{op_name}:{op.key[:trunc_key]}" {operator_style}\n')
+                        sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" {operator_style}\n')
                         visited.add(op.key)
                     sio.write(f'"Chunk:{self._gen_chunk_key(input_chunk, trunc_key)}" -> '
-                              f'"{op_name}:{op.key[:trunc_key]}"\n')
+                              f'"{op_name}:{op.key[:trunc_key]}_{id(op)}"\n')
 
                 for output_chunk in (op.outputs or []):
                     if output_chunk.key not in visited:
@@ -367,9 +367,9 @@ cdef class DirectedGraph:
                         sio.write(f'"Chunk:{self._gen_chunk_key(output_chunk, trunc_key)}" {tmp_chunk_style}\n')
                         visited.add(output_chunk.key)
                     if op.key not in visited:
-                        sio.write(f'"{op_name}:{op.key[:trunc_key]}" {operator_style}\n')
+                        sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" {operator_style}\n')
                         visited.add(op.key)
-                    sio.write(f'"{op_name}:{op.key[:trunc_key]}" -> '
+                    sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" -> '
                               f'"Chunk:{self._gen_chunk_key(output_chunk, trunc_key)}"')
                     if show_columns:
                         sio.write(f' [ label={get_col_names(output_chunk)} ]')

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/__init__.py

@@ -46,6 +46,7 @@ from .misc.cut import cut
 from .misc.eval import maxframe_eval as eval  # pylint: disable=redefined-builtin
 from .misc.get_dummies import get_dummies
 from .misc.melt import melt
+from .misc.pivot_table import pivot_table
 from .misc.qcut import qcut
 from .misc.to_numeric import to_numeric
 from .missing import isna, isnull, notna, notnull

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/core.py

@@ -35,6 +35,7 @@ from ..core import (
     register_output_types,
 )
 from ..core.entity.utils import refresh_tileable_shape
+from ..protocol import DataFrameTableMeta
 from ..serialization.serializables import (
     AnyField,
     BoolField,
@@ -59,7 +60,13 @@ from ..utils import (
     on_serialize_numpy_type,
     tokenize,
 )
-from .utils import ReprSeries, fetch_corner_data, merge_index_value, parse_index
+from .utils import (
+    ReprSeries,
+    apply_if_callable,
+    fetch_corner_data,
+    merge_index_value,
+    parse_index,
+)
 
 
 class IndexValue(Serializable):
@@ -616,6 +623,9 @@ class IndexData(HasShapeTileableData, _ToPandasMixin):
         if self._name is None:
             self._name = self.chunks[0].name
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -945,6 +955,9 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
         if self._name is None:
             self._name = self.chunks[0].name
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -978,7 +991,7 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
         return self._to_str(representation=False)
 
     def __repr__(self):
-        return self._to_str(representation=False)
+        return self._to_str(representation=True)
 
     @property
     def dtype(self):
@@ -1501,6 +1514,15 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         refresh_index_value(self)
         refresh_dtypes(self)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        dtypes = table_meta.pd_column_dtypes
+        self._dtypes = dtypes
+        self._columns_value = parse_index(dtypes.index, store_data=True)
+        self._dtypes_value = DtypesValue(key=tokenize(dtypes), value=dtypes)
+        new_shape = list(self._shape)
+        new_shape[0] = len(dtypes)
+        self._shape = tuple(new_shape)
+
     @property
     def dtypes(self):
         dt = getattr(self, "_dtypes", None)
@@ -1997,12 +2019,6 @@ class DataFrame(HasShapeTileable, _ToPandasMixin):
         Berkeley    25.0    77.0  298.15
         """
 
-        def apply_if_callable(maybe_callable, obj, **kwargs):
-            if callable(maybe_callable):
-                return maybe_callable(obj, **kwargs)
-
-            return maybe_callable
-
         data = self.copy()
 
         for k, v in kwargs.items():
@@ -2197,6 +2213,9 @@ class CategoricalData(HasShapeTileableData, _ToPandasMixin):
                 pd.Categorical(categories).categories, store_data=True
             )
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def _to_str(self, representation=False):
         if is_build_mode() or len(self._executed_sessions) == 0:
             # in build mode, or not executed, just return representation
@@ -2347,6 +2366,9 @@ class DataFrameOrSeriesData(HasShapeTileableData, _ToPandasMixin):
             data_params["name"] = self.chunks[0].name
         self._data_params.update(data_params)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        pass
+
     def ensure_data(self):
         from .fetch.core import DataFrameFetch
 

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/datasource/read_odps_query.py

@@ -216,7 +216,9 @@ class DataFrameReadODPSQuery(
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(
-                pd.Index([], name=self.index_columns[0]).astype(self.index_dtypes[0])
+                pd.Index([], name=self.index_columns[0]).astype(
+                    self.index_dtypes.iloc[0]
+                )
             )
         else:
             idx = pd.MultiIndex.from_frame(

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/datasource/read_odps_table.py

@@ -82,7 +82,9 @@ class DataFrameReadODPSTable(
                 index_value = parse_index(pd.RangeIndex(shape[0]))
         elif len(self.index_columns) == 1:
             index_value = parse_index(
-                pd.Index([], name=self.index_columns[0]).astype(self.index_dtypes[0])
+                pd.Index([], name=self.index_columns[0]).astype(
+                    self.index_dtypes.iloc[0]
+                )
             )
         else:
             idx = pd.MultiIndex.from_frame(

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/datastore/to_odps.py

@@ -18,10 +18,12 @@ import logging
 from typing import List, Optional, Union
 
 from odps.models import Table as ODPSTable
+from odps.types import PartitionSpec
 
 from ... import opcodes
 from ...config import options
 from ...core import OutputType
+from ...odpsio import build_dataframe_table_meta
 from ...serialization.serializables import (
     BoolField,
     FieldTypes,
@@ -147,6 +149,25 @@ def to_odps_table(
             f"index_label needs {len(df.index.nlevels)} labels "
             f"but it only have {len(index_label)}"
         )
+    table_cols = set(build_dataframe_table_meta(df).table_column_names)
+    if partition:
+        partition_intersect = (
+            set(x.lower() for x in PartitionSpec(partition).keys()) & table_cols
+        )
+        if partition_intersect:
+            raise ValueError(
+                f"Data column(s) {partition_intersect} in the dataframe"
+                " cannot be used in parameter 'partition'."
+                " Use 'partition_col' instead."
+            )
+
+    if partition_col:
+        partition_diff = set(x.lower() for x in partition_col) - table_cols
+        if partition_diff:
+            raise ValueError(
+                f"Partition column(s) {partition_diff}"
+                " is not the data column(s) of the input dataframe."
+            )
 
     op = DataFrameToODPSTable(
         dtypes=df.dtypes,

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/indexing/align.py

@@ -138,7 +138,7 @@ class DataFrameAlign(DataFrameOperator, DataFrameOperatorMixin):
                 series_index = rhs.index_value.to_pandas()
                 dtypes = lhs.dtypes.reindex(
                     lhs.dtypes.index.join(series_index, how=self.join)
-                ).fillna(np.dtype(np.float_))
+                ).fillna(np.dtype(float))
                 l_shape[1] = r_size = len(dtypes)
                 col_val = r_idx_val = parse_index(dtypes.index, store_data=True)
 

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/misc/__init__.py

@@ -14,6 +14,7 @@
 
 from .apply import df_apply, series_apply
 from .astype import astype, index_astype
+from .case_when import case_when
 from .check_monotonic import (
     check_monotonic,
     is_monotonic,
@@ -37,6 +38,7 @@ from .map import index_map, series_map
 from .melt import melt
 from .memory_usage import df_memory_usage, index_memory_usage, series_memory_usage
 from .pct_change import pct_change
+from .pivot_table import pivot_table
 from .qcut import qcut
 from .select_dtypes import select_dtypes
 from .shift import shift, tshift
@@ -69,6 +71,7 @@ def _install():
         setattr(t, "melt", melt)
         setattr(t, "memory_usage", df_memory_usage)
         setattr(t, "pct_change", pct_change)
+        setattr(t, "pivot_table", pivot_table)
         setattr(t, "pop", df_pop)
         setattr(t, "query", df_query)
         setattr(t, "select_dtypes", select_dtypes)
@@ -81,6 +84,7 @@ def _install():
     for t in SERIES_TYPE:
         setattr(t, "apply", series_apply)
         setattr(t, "astype", astype)
+        setattr(t, "case_when", case_when)
         setattr(t, "check_monotonic", check_monotonic)
         setattr(t, "describe", describe)
         setattr(t, "diff", series_diff)

{maxframe-0.1.0b4 → maxframe-1.0.0rc1}/maxframe/dataframe/misc/apply.py

@@ -170,6 +170,8 @@ class ApplyOperator(
         elif self.output_types[0] == OutputType.dataframe:
             shape = [np.nan, np.nan]
             shape[1 - self.axis] = df.shape[1 - self.axis]
+            if self.axis == 1:
+                shape[1] = len(dtypes)
             shape = tuple(shape)
         else:
             shape = (df.shape[1 - self.axis],)
@@ -225,7 +227,7 @@ class ApplyOperator(
                 else:  # pragma: no cover
                     index_value = parse_index(infer_series.index)
             else:
-                index_value = parse_index(None, series)
+                index_value = parse_index(series.index_value)
 
             if output_type == OutputType.dataframe:
                 if dtypes is None: