maxframe 1.0.0rc4__tar.gz → 1.1.1__tar.gz

{maxframe-1.0.0rc4 → maxframe-1.1.1}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.0.0rc4
+Version: 1.1.1
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy<2.0.0,>=1.19.0
 Requires-Dist: pandas>=1.0.0
-Requires-Dist: pyodps>=0.11.6.1
+Requires-Dist: pyodps>=0.12.0
 Requires-Dist: scipy>=1.0
 Requires-Dist: pyarrow>=1.0.0
 Requires-Dist: msgpack>=1.0.0

maxframe-1.1.1/maxframe/config/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .config import AttributeDict, option_context, options, update_wlm_quota_settings

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/config/config.py

@@ -28,6 +28,8 @@ except ImportError:
 
     available_timezones = lambda: all_timezones
 
+import logging
+
 from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
@@ -43,6 +45,8 @@ from .validators import (
     is_valid_cache_path,
 )
 
+logger = logging.getLogger(__name__)
+
 _DEFAULT_REDIRECT_WARN = "Option {source} has been replaced by {target} and might be removed in a future release."
 _DEFAULT_MAX_ALIVE_SECONDS = 3 * 24 * 3600
 _DEFAULT_MAX_IDLE_SECONDS = 3600
@@ -380,6 +384,9 @@ default_options.register_option(
 default_options.register_option(
     "session.enable_schema", None, validator=is_null | is_bool, remote=True
 )
+default_options.register_option(
+    "session.enable_high_availability", None, validator=is_null | is_bool, remote=True
+)
 default_options.register_option(
     "session.default_schema", None, validator=is_null | is_string, remote=True
 )
@@ -496,3 +503,22 @@ class OptionsProxy:
 
 
 options = OptionsProxy()
+
+
+def update_wlm_quota_settings(session_id: str, engine_settings: Dict[str, Any]):
+    engine_quota = engine_settings.get("odps.task.wlm.quota", None)
+    session_quota = options.session.quota_name or None
+    if engine_quota != session_quota and engine_quota:
+        logger.warning(
+            "[Session=%s] Session quota (%s) is different to SubDag engine quota (%s)",
+            session_id,
+            session_quota,
+            engine_quota,
+        )
+        # TODO(renxiang): overwrite or not overwrite
+        return
+
+    if session_quota:
+        engine_settings["odps.task.wlm.quota"] = session_quota
+    elif "odps.task.wlm.quota" in engine_settings:
+        engine_settings.pop("odps.task.wlm.quota")

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/core/operator/base.py

@@ -86,6 +86,8 @@ class SchedulingHint(Serializable):
     # `gpu` indicates that if the operator should be executed on the GPU.
     gpu = BoolField("gpu", default=None)
     priority = Int32Field("priority", default=None)
+    expect_engine = StringField("expect_engine", default=None)
+    expect_resources = DictField("expect_resources", FieldTypes.string, default=None)
 
     @classproperty
     @lru_cache(1)

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/dataframe/core.py

@@ -142,6 +142,14 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
+        @property
+        def inferred_type(self):
+            return "floating" if self.dtype.kind == "f" else "integer"
+
     class RangeIndex(IndexBase):
         _name = AnyField("name")
         _slice = SliceField("slice")
@@ -243,6 +251,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "integer"
@@ -254,6 +266,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "integer"
@@ -265,6 +281,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "floating"
@@ -1514,8 +1534,7 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         refresh_index_value(self)
         refresh_dtypes(self)
 
-    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
-        dtypes = table_meta.pd_column_dtypes
+    def refresh_from_dtypes(self, dtypes: pd.Series) -> None:
         self._dtypes = dtypes
         self._columns_value = parse_index(dtypes.index, store_data=True)
         self._dtypes_value = DtypesValue(key=tokenize(dtypes), value=dtypes)
@@ -1523,6 +1542,9 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         new_shape[-1] = len(dtypes)
         self._shape = tuple(new_shape)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        self.refresh_from_dtypes(table_meta.pd_column_dtypes)
+
     @property
     def dtypes(self):
         dt = getattr(self, "_dtypes", None)

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/dataframe/datasource/read_odps_query.py

@@ -37,6 +37,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
 
@@ -57,7 +58,7 @@ _EXPLAIN_COLUMN_REGEX = re.compile(r"([^\(]+) \(([^\n]+)\)(?:| AS ([^ ]+))(?:\n|
 _ANONYMOUS_COL_REGEX = re.compile(r"^_c(\d+)$")
 
 _SIMPLE_SCHEMA_COLS_REGEX = re.compile(r"SELECT (([^:]+:[^, ]+[, ]*)+)FROM")
-_SIMPLE_SCHEMA_COL_REGEX = re.compile(r"([^\.]+):([^, ]+)")
+_SIMPLE_SCHEMA_COL_REGEX = re.compile(r"([^ \.\)]+):([^ ]+)")
 
 
 @dataclasses.dataclass
@@ -180,23 +181,30 @@ def _parse_full_explain(explain_string: str) -> OdpsSchema:
 
     job_dag = jobs_sector.build_dag()
     indep_job_names = list(job_dag.iter_indep(reverse=True))
-    if len(indep_job_names) > 1:  # pragma: no cover
-        raise ValueError("Only one final job is allowed in SQL statement")
-
-    tasks_sector = jobs_sector.jobs[indep_job_names[0]]
-    task_dag = tasks_sector.build_dag()
-    indep_task_names = list(task_dag.iter_indep(reverse=True))
-    if len(indep_task_names) > 1:  # pragma: no cover
+    schema_signatures = dict()
+    for job_name in indep_job_names:
+        tasks_sector = jobs_sector.jobs[job_name]
+        task_dag = tasks_sector.build_dag()
+        indep_task_names = list(task_dag.iter_indep(reverse=True))
+        for task_name in indep_task_names:
+            task_sector = tasks_sector.tasks[task_name]
+            if not task_sector.schema:  # pragma: no cover
+                raise ValueError("Cannot detect output schema")
+            if task_sector.output_target != "Screen":
+                raise ValueError("The SQL statement should be an instant query")
+            sig_tuples = sorted(
+                [
+                    (c.column_alias or c.column_name, c.column_type)
+                    for c in task_sector.schema
+                ]
+            )
+            schema_signatures[hash(tuple(sig_tuples))] = task_sector.schema
+    if len(schema_signatures) != 1:
         raise ValueError("Only one final task is allowed in SQL statement")
-
-    task_sector = tasks_sector.tasks[indep_task_names[0]]
-    if not task_sector.schema:  # pragma: no cover
-        raise ValueError("Cannot detect output schema")
-    if task_sector.output_target != "Screen":
-        raise ValueError("The SQL statement should be an instant query")
+    schema = list(schema_signatures.values())[0]
     cols = [
         Column(c.column_alias or c.column_name, validate_data_type(c.column_type))
-        for c in task_sector.schema
+        for c in schema
     ]
     return OdpsSchema(cols)
 
@@ -209,7 +217,7 @@ def _parse_simple_explain(explain_string: str) -> OdpsSchema:
     fields_str = fields_match.group(1)
     cols = []
     for field, type_name in _SIMPLE_SCHEMA_COL_REGEX.findall(fields_str):
-        cols.append(Column(field, validate_data_type(type_name)))
+        cols.append(Column(field, validate_data_type(type_name.rstrip(","))))
     return OdpsSchema(cols)
 
 
@@ -243,7 +251,7 @@ class DataFrameReadODPSQuery(
         self.columns = columns
 
     def __call__(self, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(
@@ -257,12 +265,18 @@ class DataFrameReadODPSQuery(
             )
             index_value = parse_index(idx)
 
-        columns_value = parse_index(self.dtypes.index, store_data=True)
+        if self.dtypes is not None:
+            columns_value = parse_index(self.dtypes.index, store_data=True)
+            shape = (np.nan, len(self.dtypes))
+        else:
+            columns_value = None
+            shape = (np.nan, np.nan)
+
         self.output_types = [OutputType.dataframe]
         return self.new_tileable(
             [],
             None,
-            shape=(len(self.dtypes), np.nan),
+            shape=shape,
             dtypes=self.dtypes,
             index_value=index_value,
             columns_value=columns_value,
@@ -278,6 +292,7 @@ def read_odps_query(
     string_as_binary: bool = None,
     sql_hints: Dict[str, str] = None,
     anonymous_col_prefix: str = _DEFAULT_ANONYMOUS_COL_PREFIX,
+    skip_schema: bool = False,
     **kw,
 ):
     """
@@ -298,6 +313,10 @@ def read_odps_query(
         User specified SQL hints.
     anonymous_col_prefix: str, optional
         Prefix for anonymous columns, '_anon_col_' by default.
+    skip_schema: bool, optional
+        Skip resolving output schema before execution. Once this is configured,
+        the output DataFrame cannot be inputs of other DataFrame operators
+        before execution.
 
     Returns
     -------
@@ -319,28 +338,39 @@ def read_odps_query(
 
     if odps_entry is None:
         raise ValueError("Missing odps_entry parameter")
-    inst = odps_entry.execute_sql(f"EXPLAIN {query}", hints=hints)
-    logger.debug("Explain instance ID: %s", inst.id)
-    explain_str = list(inst.get_task_results().values())[0]
 
-    odps_schema = _parse_explained_schema(explain_str)
-
-    new_columns = []
     col_renames = {}
-    for col in odps_schema.columns:
-        anon_match = _ANONYMOUS_COL_REGEX.match(col.name)
-        if anon_match and col.name not in query:
-            new_name = anonymous_col_prefix + anon_match.group(1)
-            col_renames[col.name] = new_name
-            new_columns.append(Column(new_name, col.type))
-        else:
-            new_columns.append(col)
-
-    dtypes = odps_schema_to_pandas_dtypes(OdpsSchema(new_columns))
+    if not skip_schema:
+        inst = odps_entry.execute_sql(f"EXPLAIN {query}", hints=hints)
+        logger.debug("Explain instance ID: %s", inst.id)
+        explain_str = list(inst.get_task_results().values())[0]
+
+        try:
+            odps_schema = _parse_explained_schema(explain_str)
+        except ValueError as ex:
+            exc = ValueError(str(ex) + "\nExplain instance ID: " + inst.id)
+            raise exc.with_traceback(ex.__traceback__) from None
+
+        new_columns = []
+        for col in odps_schema.columns:
+            anon_match = _ANONYMOUS_COL_REGEX.match(col.name)
+            if anon_match and col.name not in query:
+                new_name = anonymous_col_prefix + anon_match.group(1)
+                col_renames[col.name] = new_name
+                new_columns.append(Column(new_name, col.type))
+            else:
+                new_columns.append(col)
+
+        dtypes = odps_schema_to_pandas_dtypes(OdpsSchema(new_columns))
+    else:
+        dtypes = None
 
     if not index_col:
         index_dtypes = None
     else:
+        if dtypes is None:
+            raise ValueError("Cannot configure index_col when skip_schema is True")
+
         if isinstance(index_col, str):
             index_col = [index_col]
         index_col_set = set(index_col)

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/dataframe/datasource/read_odps_table.py

@@ -34,6 +34,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..core import DataFrame  # noqa: F401
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
@@ -76,7 +77,7 @@ class DataFrameReadODPSTable(
         self.columns = columns
 
     def __call__(self, shape, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             if np.isnan(shape[0]):
                 index_value = parse_index(pd.RangeIndex(0))
             else:
@@ -238,7 +239,8 @@ def read_odps_table(
         partitions = [partitions]
 
     append_partitions = append_partitions or any(
-        pt.name in (columns or ()) for pt in (table.table_schema.partitions or ())
+        pt.name in (columns if not is_empty(columns) else ())
+        for pt in (table.table_schema.partitions or ())
     )
     op = DataFrameReadODPSTable(
         table_name=table.full_table_name,

{maxframe-1.0.0rc4 → maxframe-1.1.1}/maxframe/dataframe/extensions/__init__.py

@@ -18,6 +18,8 @@ from .accessor import (
     IndexMaxFrameAccessor,
     SeriesMaxFrameAccessor,
 )
+from .apply_chunk import df_apply_chunk, series_apply_chunk
+from .flatjson import series_flatjson
 from .flatmap import df_flatmap, series_flatmap
 from .reshuffle import DataFrameReshuffle, df_reshuffle
 
@@ -27,7 +29,10 @@ def _install():
 
     DataFrameMaxFrameAccessor._register("reshuffle", df_reshuffle)
     DataFrameMaxFrameAccessor._register("flatmap", df_flatmap)
+    DataFrameMaxFrameAccessor._register("apply_chunk", df_apply_chunk)
     SeriesMaxFrameAccessor._register("flatmap", series_flatmap)
+    SeriesMaxFrameAccessor._register("flatjson", series_flatjson)
+    SeriesMaxFrameAccessor._register("apply_chunk", series_apply_chunk)
 
     if DataFrameMaxFrameAccessor._api_count:
         for t in DATAFRAME_TYPE: