maxframe 1.3.1__cp310-cp310-macosx_10_9_universal2.whl → 2.0.0b1__cp310-cp310-macosx_10_9_universal2.whl

maxframe/dataframe/datasource/from_records.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import MutableMapping, Union
+
 import numpy as np
 import pandas as pd
 
@@ -58,6 +58,13 @@ class DataFrameFromRecords(DataFrameOperator, DataFrameOperatorMixin):
             columns_value=columns_value,
         )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameFromRecords"
+    ):  # pragma: no cover
+        # todo implement this to facilitate local computation
+        ctx[op.outputs[0].key] = float("inf")
+
 
 def from_records(
     data,

maxframe/dataframe/datasource/from_tensor.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,13 +13,13 @@
 # limitations under the License.
 
 from collections import OrderedDict
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, MutableMapping, Union
 
 import numpy as np
 import pandas as pd
 
 from ... import opcodes
-from ...core import ENTITY_TYPE, OutputType
+from ...core import ENTITY_TYPE, EntityData, OutputType
 from ...serialization.serializables import AnyField, KeyField
 from ...tensor.core import Tensor
 from ...tensor.datasource import tensor as astensor
@@ -46,24 +44,25 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
         kwargs["_output_types"] = [OutputType.dataframe]
         super().__init__(*args, **kwargs)
 
-    def _set_inputs(self, inputs: List[EntityType]):
-        super()._set_inputs(inputs)
-        inputs_iter = iter(self._inputs)
-        if self.input is not None:
-            if not isinstance(self.input, dict):
-                self.input = next(inputs_iter)
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameFromTensor", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        inputs_iter = iter(op._inputs)
+        if op.input is not None:
+            if not isinstance(op.input, dict):
+                op.input = next(inputs_iter)
             else:
                 # check each value for input
                 new_input = OrderedDict()
-                for k, v in self.input.items():
+                for k, v in op.input.items():
                     if isinstance(v, ENTITY_TYPE):
                         new_input[k] = next(inputs_iter)
                     else:
                         new_input[k] = v
-                self.input = new_input
+                op.input = new_input
 
-        if isinstance(self.index, ENTITY_TYPE):
-            self.index = next(inputs_iter)
+        if isinstance(op.index, ENTITY_TYPE):
+            op.index = next(inputs_iter)
 
     def __call__(
         self,
@@ -138,7 +137,11 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
                 )
             index_value = self._process_index(index, tileables)
         else:
-            self.index = index = pd.RangeIndex(0, tileables[0].shape[0])
+            if np.isnan(tileables[0].shape[0]):
+                index = pd.RangeIndex(0)
+            else:
+                index = pd.RangeIndex(0, tileables[0].shape[0])
+            self.index = index
             index_value = parse_index(index)
 
         if columns is not None:
@@ -260,6 +263,13 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
             columns_value=columns_value,
         )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameFromTensor"
+    ):  # pragma: no cover
+        # todo implement this to facilitate local computation
+        ctx[op.outputs[0].key] = float("inf")
+
 
 def dataframe_from_tensor(
     tensor: Tensor,
@@ -340,12 +350,13 @@ class SeriesFromTensor(DataFrameOperator, DataFrameOperatorMixin):
     input = KeyField("input")
     index = AnyField("index")
 
-    def _set_inputs(self, inputs: List[EntityType]):
-        super()._set_inputs(inputs)
-        if self.input is not None:
-            self.input = self.inputs[0]
-        if self.index is not None and hasattr(self.index, "key"):
-            self.index = self.inputs[-1]
+    @classmethod
+    def _set_inputs(cls, op: "SeriesFromTensor", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.input is not None:
+            op.input = op.inputs[0]
+        if op.index is not None and hasattr(op.index, "key"):
+            op.index = op.inputs[-1]
 
     def __call__(
         self,

maxframe/dataframe/datasource/read_csv.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +13,7 @@
 # limitations under the License.
 
 from io import BytesIO
+from typing import MutableMapping, Union
 from urllib.parse import urlparse
 
 import numpy as np
@@ -129,6 +128,13 @@ class DataFrameReadCSV(
             chunk_bytes=chunk_bytes,
         )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameReadCSV"
+    ):  # pragma: no cover
+        # todo implement this to facilitate local computation
+        ctx[op.outputs[0].key] = float("inf")
+
 
 def read_csv(
     path,

maxframe/dataframe/datasource/read_odps_query.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import dataclasses
+import io
 import logging
 import re
-from typing import Dict, List, Optional, Tuple, Union
+import tokenize
+from typing import Dict, List, MutableMapping, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -110,7 +112,15 @@ def _split_explain_string(explain_string: str) -> List[str]:
     grouped = []
     for part in parts:
         part = part.strip("\n")
-        if grouped and not part.startswith(" "):
+        part_line1 = part.split("\n", 1)[0]
+        # initial line of part should not start with spaces (Statistics row)
+        #  or with quote marks
+        if (
+            grouped
+            and not part.startswith(" ")
+            and "'" not in part_line1
+            and '"' not in part_line1
+        ):
             final_parts.append("\n\n".join(grouped).strip())
             grouped = []
         grouped.append(part)
@@ -267,6 +277,15 @@ class DataFrameReadODPSQuery(
     def set_pruned_columns(self, columns, *, keep_order=None):  # pragma: no cover
         self.columns = columns
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameReadODPSQuery"
+    ):  # pragma: no cover
+        # use infinity to show that the size cannot be inferred
+        # todo when local catalyst is implemented,
+        #  a more precise estimation here can be useful then.
+        ctx[op.outputs[0].key] = float("inf")
+
     def __call__(self, chunk_bytes=None, chunk_size=None):
         if is_empty(self.index_columns):
             index_value = parse_index(pd.RangeIndex(0))
@@ -302,6 +321,17 @@ class DataFrameReadODPSQuery(
         )
 
 
+def _check_token_in_sql(token: str, sql: str) -> bool:
+    try:
+        names = set()
+        for tk_info in tokenize.tokenize(io.BytesIO(sql.encode()).readline):
+            if tk_info.type == tokenize.NAME:
+                names.add(tk_info.string)
+        return token in names
+    except:  # pragma: no cover
+        return False
+
+
 def read_odps_query(
     query: str,
     odps_entry: ODPS = None,
@@ -382,7 +412,7 @@ def read_odps_query(
         new_columns = []
         for col in odps_schema.columns:
             anon_match = _ANONYMOUS_COL_REGEX.match(col.name)
-            if anon_match and col.name not in query:
+            if anon_match and not _check_token_in_sql(col.name, query):
                 new_name = anonymous_col_prefix + anon_match.group(1)
                 col_renames[col.name] = new_name
                 new_columns.append(Column(new_name, col.type))

maxframe/dataframe/datasource/read_odps_table.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import List, Optional, Union
+from typing import List, MutableMapping, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -34,7 +34,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
-from ...utils import is_empty
+from ...utils import estimate_table_size, is_empty
 from ..core import DataFrame  # noqa: F401
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
@@ -46,6 +46,7 @@ class DataFrameReadODPSTable(
     IncrementalIndexDatasource,
     ColumnPruneSupportedDataSourceMixin,
 ):
+    __slots__ = ("_odps_entry",)
     _op_type_ = opcodes.READ_ODPS_TABLE
 
     table_name = StringField("table_name")
@@ -61,7 +62,8 @@ class DataFrameReadODPSTable(
     index_dtypes = SeriesField("index_dtypes", default=None)
 
     def __init__(self, memory_scale=None, **kw):
-        output_type = kw.get("output_type", OutputType.dataframe)
+        output_type = kw.pop("output_type", OutputType.dataframe)
+        self._odps_entry = kw.pop("odps_entry", None)
         super(DataFrameReadODPSTable, self).__init__(
             memory_scale=memory_scale, _output_types=[output_type], **kw
         )
@@ -130,6 +132,18 @@ class DataFrameReadODPSTable(
                 chunk_size=chunk_size,
             )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameReadODPSTable"
+    ) -> None:
+        odps_entry = op._odps_entry or ODPS.from_global() or ODPS.from_environments()
+        if not odps_entry:  # pragma: no cover
+            ctx[op.outputs[0].key] = float("inf")
+            return
+        ctx[op.outputs[0].key] = estimate_table_size(
+            odps_entry, op.table_name, op.partitions
+        )
+
 
 def read_odps_table(
     table_name: Union[str, Table],
@@ -212,7 +226,8 @@ def read_odps_table(
         index_dtypes = pd.Series(table_index_types, index=index_col)
 
     if columns is not None:
-        table_col_set = set([c.lower() for c in columns])
+        new_columns = [c.lower() for c in columns]
+        table_col_set = set(new_columns)
         col_diff = sorted(table_col_set - set(table_columns))
         if col_diff:
             raise ValueError(
@@ -223,7 +238,6 @@ def read_odps_table(
             raise ValueError("Index columns and columns shall not overlap.")
 
         # reorder columns
-        new_columns = [c for c in table_columns if c in table_col_set]
         df_types = [df_types[table_columns.index(col)] for col in new_columns]
         table_columns = new_columns
         columns = new_columns
@@ -253,6 +267,7 @@ def read_odps_table(
         last_modified_time=to_timestamp(table.last_data_modified_time),
         index_columns=index_col,
         index_dtypes=index_dtypes,
+        odps_entry=odps_entry,
         **kw,
     )
     return op(shape, chunk_bytes=chunk_bytes, chunk_size=chunk_size)

maxframe/dataframe/datasource/read_parquet.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +13,7 @@
 # limitations under the License.
 
 import os
-from typing import Dict
+from typing import Dict, MutableMapping, Union
 from urllib.parse import urlparse
 
 import numpy as np
@@ -308,6 +306,13 @@ class DataFrameReadParquet(
             columns_value=columns_value,
         )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameReadParquet"
+    ):  # pragma: no cover
+        # todo implement this to facilitate local computation
+        ctx[op.outputs[0].key] = float("inf")
+
 
 def read_parquet(
     path,

maxframe/dataframe/datasource/tests/test_datasource.py

@@ -15,6 +15,7 @@
 import os
 import uuid
 from collections import OrderedDict
+from math import isinf
 
 import numpy as np
 import pandas as pd
@@ -24,6 +25,7 @@ from odps import types as odps_types
 
 from .... import tensor as mt
 from ....core import OutputType
+from ....core.operator import estimate_size
 from ....tests.utils import tn
 from ....utils import lazy_import
 from ... import read_odps_query, read_odps_table
@@ -71,6 +73,10 @@ def test_from_pandas_dataframe():
     assert df.index_value.max_val == 9
     np.testing.assert_equal(df.columns_value._index_value._data, data.columns.values)
 
+    result_ctx = dict()
+    estimate_size(result_ctx, df.op)
+    assert result_ctx[df.key] > 0 and not isinf(result_ctx[df.key])
+
     data2 = data[::2]
     df2 = from_pandas_df(data2, chunk_size=4)
 
@@ -258,6 +264,10 @@ def test_from_odps_table():
         ),
     )
 
+    result_ctx = dict()
+    estimate_size(result_ctx, df.op)
+    assert result_ctx[df.key] >= 0 and not isinf(result_ctx[df.key])
+
     with pytest.raises(ValueError):
         read_odps_table(test_table, columns=["col3", "col4"])
     with pytest.raises(ValueError):
@@ -300,6 +310,7 @@ def test_from_odps_table():
         ),
     )
 
+    test_parted_table.create_partition("pt=20240103")
     df = read_odps_table(
         test_parted_table, columns=["col1", "col2", "pt"], partitions="pt=20240103"
     )
@@ -314,6 +325,10 @@ def test_from_odps_table():
         ),
     )
 
+    result_ctx = dict()
+    estimate_size(result_ctx, df.op)
+    assert result_ctx[df.key] >= 0 and not isinf(result_ctx[df.key])
+
     out_idx = read_odps_table(
         test_table,
         columns=[],
@@ -545,3 +560,21 @@ def test_resolve_multi_join():
     for col, (exp_nm, exp_tp) in zip(schema.columns, expected_col_types.items()):
         assert col.name == exp_nm
         assert col.type == odps_types.validate_data_type(exp_tp)
+
+
+def test_resolve_break_lines():
+    input_path = os.path.join(
+        os.path.dirname(__file__), "test-data", "task-input-with-break-line.txt"
+    )
+    with open(input_path, "r") as f:
+        sector = f.read()
+
+    expected_col_types = {
+        "key": "string",
+        "value": "string",
+    }
+
+    schema = _parse_full_explain(sector)
+    for col, (exp_nm, exp_tp) in zip(schema.columns, expected_col_types.items()):
+        assert col.name == exp_nm
+        assert col.type == odps_types.validate_data_type(exp_tp)

maxframe/dataframe/datastore/tests/test_to_odps.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pandas as pd
 import pytest
 
-from ... import DataFrame
+from ... import DataFrame, Index
 from ..to_odps import to_odps_table
 
 
@@ -23,6 +24,25 @@ def df():
     return DataFrame({"A": [1, 2], "B": [3, 4]})
 
 
+@pytest.fixture
+def df_with_named_index():
+    return DataFrame({"A": [1, 2], "B": [3, 4]}, index=Index([1, 2], name="A"))
+
+
+@pytest.fixture
+def df_with_named_multi_indexes():
+    arrays = [
+        ["c1", "c2"],
+        ["d1", "d2"],
+        ["e1", "e2"],
+    ]
+    multi_index = pd.MultiIndex.from_arrays(arrays, names=("C", "D", "E"))
+    return DataFrame(
+        {"A": [1, 2], "B": [3, 4]},
+        index=multi_index,
+    )
+
+
 @pytest.mark.parametrize(
     "kwargs",
     [
@@ -46,3 +66,34 @@ def test_to_odps_table_validation(df, kwargs):
 )
 def test_to_odps_table_vaild(df, kwargs):
     to_odps_table(df, "test_table", **kwargs)
+
+
+def test_to_odps_table_column_conflicts(
+    df, df_with_named_index, df_with_named_multi_indexes
+):
+    to_odps_table(df.reset_index(), "test_table", index=False)
+    to_odps_table(df.reset_index(), "test_table", index_label="C")
+    with pytest.raises(ValueError):
+        to_odps_table(df.reset_index(), "test_table")
+
+    to_odps_table(df_with_named_index, "test_table", index=False)
+    to_odps_table(df_with_named_index, "test_table", index_label="C")
+    with pytest.raises(ValueError):
+        to_odps_table(df_with_named_index, "test_table")
+
+    to_odps_table(df, "test_table", partition="C='1'")
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", partition="A='1'")
+
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", partition="A='1'")
+
+    to_odps_table(df_with_named_multi_indexes, "test_table")
+    to_odps_table(
+        df_with_named_multi_indexes, "test_table", partition="C='1'", index=False
+    )
+    with pytest.raises(ValueError):
+        to_odps_table(df_with_named_multi_indexes, "test_table", partition="C='1'")
+
+    df_with_named_multi_indexes.index.names = ["C1", "D1", "E1"]
+    to_odps_table(df_with_named_multi_indexes, "test_table", partition="C='1'")

maxframe/dataframe/datastore/to_csv.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
+
 from ... import opcodes
+from ...core import EntityData
 from ...serialization.serializables import (
     AnyField,
     BoolField,
@@ -68,9 +71,10 @@ class DataFrameToCSV(DataFrameDataStore):
     def output_limit(self):
         return 1 if not self.output_stat else 2
 
-    def _set_inputs(self, inputs):
-        super()._set_inputs(inputs)
-        self._input = self._inputs[0]
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameToCSV", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        op._input = op._inputs[0]
 
     def __call__(self, df):
         index_value = parse_index(df.index_value.to_pandas()[:0], df)

maxframe/dataframe/datastore/to_odps.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import logging
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 from odps import ODPS
 from odps.models import Table as ODPSTable
@@ -75,6 +74,25 @@ class DataFrameToODPSTable(DataFrameDataStore):
             columns_value=columns_value,
         )
 
+    @classmethod
+    def get_index_mapping(
+        cls,
+        index_label: Optional[List[str]],
+        raw_index_levels: List[Any],
+    ) -> List[Any]:
+        def_labels = index_label or itertools.repeat(None)
+        def_labels = itertools.chain(def_labels, itertools.repeat(None))
+        names = raw_index_levels
+        if len(names) == 1:
+            default_labels = ["index"]
+        else:
+            default_labels = [f"level_{i}" for i in range(len(names))]
+        indexes = [
+            def_label or name or label
+            for def_label, name, label in zip(def_labels, names, default_labels)
+        ]
+        return [x.lower() for x in indexes]
+
 
 def to_odps_table(
     df: TileableType,
@@ -161,11 +179,14 @@ def to_odps_table(
             f"index_label needs {len(df.index.nlevels)} labels "
             f"but it only have {len(index_label)}"
         )
+
+    # check if table partition columns conflicts with dataframe columns
     table_cols = set(build_dataframe_table_meta(df).table_column_names)
+    partition_col_set = (
+        set(x.lower() for x in PartitionSpec(partition).keys()) if partition else set()
+    )
     if partition:
-        partition_intersect = (
-            set(x.lower() for x in PartitionSpec(partition).keys()) & table_cols
-        )
+        partition_intersect = partition_col_set & table_cols
         if partition_intersect:
             raise ValueError(
                 f"Data column(s) {partition_intersect} in the dataframe"
@@ -173,6 +194,21 @@ def to_odps_table(
                 " Use 'partition_col' instead."
             )
 
+    if index:
+        index_cols = set(
+            DataFrameToODPSTable.get_index_mapping(index_label, df.index.names)
+        )
+        index_table_intersect = index_cols & table_cols
+        if index_table_intersect:
+            raise ValueError(
+                f"Index column(s) {index_table_intersect} conflict with column(s) of the input dataframe."
+            )
+        index_partition_intersect = index_cols & partition_col_set
+        if index_partition_intersect:
+            raise ValueError(
+                f"Index column(s) {index_partition_intersect} conflict with partition column(s)."
+            )
+
     if partition_col:
         partition_diff = set(x.lower() for x in partition_col) - table_cols
         if partition_diff:

maxframe/dataframe/extensions/__init__.py

@@ -18,7 +18,12 @@ from .accessor import (
     IndexMaxFrameAccessor,
     SeriesMaxFrameAccessor,
 )
-from .apply_chunk import df_apply_chunk, series_apply_chunk
+from .apply_chunk import (
+    DataFrameApplyChunk,
+    DataFrameApplyChunkOperator,
+    df_apply_chunk,
+    series_apply_chunk,
+)
 from .flatjson import series_flatjson
 from .flatmap import df_flatmap, series_flatmap
 from .reshuffle import DataFrameReshuffle, df_reshuffle