maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/datasource/from_dict.py

@@ -0,0 +1,124 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...core import ENTITY_TYPE
+from ...utils import find_objects, no_default
+from ..utils import validate_axis
+
+
+def dataframe_from_dict(data, orient="columns", dtype=None, columns=None):
+    """
+    Construct DataFrame from dict of array-like or dicts.
+
+    Creates DataFrame object from dictionary by columns or by index
+    allowing dtype specification.
+
+    Parameters
+    ----------
+    data : dict
+        Of the form {field : array-like} or {field : dict}.
+    orient : {'columns', 'index', 'tight'}, default 'columns'
+        The "orientation" of the data. If the keys of the passed dict
+        should be the columns of the resulting DataFrame, pass 'columns'
+        (default). Otherwise if the keys should be rows, pass 'index'.
+        If 'tight', assume a dict with keys ['index', 'columns', 'data',
+        'index_names', 'column_names'].
+
+    dtype : dtype, default None
+        Data type to force after DataFrame construction, otherwise infer.
+    columns : list, default None
+        Column labels to use when ``orient='index'``. Raises a ValueError
+        if used with ``orient='columns'`` or ``orient='tight'``.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.from_records : DataFrame from structured ndarray, sequence
+        of tuples or dicts, or DataFrame.
+    DataFrame : DataFrame object creation using constructor.
+    DataFrame.to_dict : Convert the DataFrame to a dictionary.
+
+    Examples
+    --------
+    By default the keys of the dict become the DataFrame columns:
+
+    >>> import maxframe.dataframe as md
+    >>> data = {'col_1': [3, 2, 1, 0], 'col_2': ['a', 'b', 'c', 'd']}
+    >>> md.DataFrame.from_dict(data).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+
+    Specify ``orient='index'`` to create the DataFrame using dictionary
+    keys as rows:
+
+    >>> data = {'row_1': [3, 2, 1, 0], 'row_2': ['a', 'b', 'c', 'd']}
+    >>> md.DataFrame.from_dict(data, orient='index').execute()
+           0  1  2  3
+    row_1  3  2  1  0
+    row_2  a  b  c  d
+
+    When using the 'index' orientation, the column names can be
+    specified manually:
+
+    >>> md.DataFrame.from_dict(data, orient='index',
+    ...                        columns=['A', 'B', 'C', 'D']).execute()
+           A  B  C  D
+    row_1  3  2  1  0
+    row_2  a  b  c  d
+
+    Specify ``orient='tight'`` to create the DataFrame using a 'tight'
+    format:
+
+    >>> data = {'index': [('a', 'b'), ('a', 'c')],
+    ...         'columns': [('x', 1), ('y', 2)],
+    ...         'data': [[1, 3], [2, 4]],
+    ...         'index_names': ['n1', 'n2'],
+    ...         'column_names': ['z1', 'z2']}
+    >>> md.DataFrame.from_dict(data, orient='tight').execute()
+    z1     x  y
+    z2     1  2
+    n1 n2
+    a  b   1  3
+       c   2  4
+    """
+    from ..initializer import DataFrame as DataFrameInit
+    from .from_tensor import dataframe_from_1d_tileables
+
+    if orient != "tight" and not find_objects(data, ENTITY_TYPE):
+        res = DataFrameInit(data)
+    elif orient == "tight":
+        # init directly
+        init_kw = {
+            "index": data.get("index"),
+            "columns": data.get("columns"),
+        }
+        df = DataFrameInit(data["data"], **init_kw)
+        rename_kw = {
+            "index": data.get("index_names", no_default),
+            "columns": data.get("column_names", no_default),
+        }
+        res = df.rename_axis(**rename_kw)
+    else:
+        axis = validate_axis(orient)
+        res = dataframe_from_1d_tileables(data, columns=columns, axis=axis)
+
+    if dtype is not None:
+        res = res.astype(dtype)
+    return res

maxframe/dataframe/datasource/from_index.py

@@ -51,7 +51,7 @@ class SeriesFromIndex(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def series_from_index(ind, index=None, name=None):
-    name = name or ind.name or 0
+    name = name or ind.name
     if index is not None:
         index = Index(index)
     op = SeriesFromIndex(input_=ind, index=index, name=name)

maxframe/dataframe/datasource/from_records.py

@@ -77,6 +77,83 @@ def from_records(
     sparse=False,
     **kw
 ):
+    """
+    Convert structured or record ndarray to DataFrame.
+
+    Creates a DataFrame object from a structured ndarray, sequence of
+    tuples or dicts, or DataFrame.
+
+    Parameters
+    ----------
+    data : structured ndarray, sequence of tuples or dicts, or DataFrame
+        Structured input data.
+
+        .. deprecated:: 2.1.0
+            Passing a DataFrame is deprecated.
+    index : str, list of fields, array-like
+        Field of array to use as the index, alternately a specific set of
+        input labels to use.
+    exclude : sequence, default None
+        Columns or fields to exclude.
+    columns : sequence, default None
+        Column names to use. If the passed data do not have names
+        associated with them, this argument provides names for the
+        columns. Otherwise this argument indicates the order of the columns
+        in the result (any names not found in the data will become all-NA
+        columns).
+    coerce_float : bool, default False
+        Attempt to convert values of non-string, non-numeric objects (like
+        decimal.Decimal) to floating point, useful for SQL result sets.
+    nrows : int, default None
+        Number of rows to read if data is an iterator.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.from_dict : DataFrame from dict of array-like or dicts.
+    DataFrame : DataFrame object creation using constructor.
+
+    Examples
+    --------
+    Data can be provided as a structured ndarray:
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> data = mt.array([(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')],
+    ...                 dtype=[('col_1', 'i4'), ('col_2', 'U1')])
+    >>> md.DataFrame.from_records(data).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+
+    Data can be provided as a list of dicts:
+
+    >>> data = [{'col_1': 3, 'col_2': 'a'},
+    ...         {'col_1': 2, 'col_2': 'b'},
+    ...         {'col_1': 1, 'col_2': 'c'},
+    ...         {'col_1': 0, 'col_2': 'd'}]
+    >>> md.DataFrame.from_records(data).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+
+    Data can be provided as a list of tuples with corresponding columns:
+
+    >>> data = [(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')]
+    >>> md.DataFrame.from_records(data, columns=['col_1', 'col_2']).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+    """
     if isinstance(data, np.ndarray):
         from .dataframe import from_pandas
 

maxframe/dataframe/datasource/from_tensor.py

@@ -39,6 +39,7 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
     input = AnyField("input")
     index = AnyField("index")
     columns = AnyField("columns")
+    axis = AnyField("axis")
 
     def __init__(self, *args, **kwargs):
         kwargs["_output_types"] = [OutputType.dataframe]
@@ -120,46 +121,82 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
             if isinstance(tileable, ENTITY_TYPE):
                 tileables.append(tileable)
 
-        if index is not None:
-            tileable_size = tileables[0].shape[0]
-            if hasattr(index, "shape"):
-                index_size = index.shape[0]
+        if self.axis == 0:
+            if index is not None:
+                raise NotImplementedError("Cannot accept index when axis=0")
             else:
-                index_size = len(index)
-            if (
-                not pd.isna(tileable_size)
-                and not pd.isna(index_size)
-                and tileable_size != index_size
-            ):
-                raise ValueError(
-                    f"index {index} should have the same shape "
-                    f"with tensor: {tileable_size}"
-                )
-            index_value = self._process_index(index, tileables)
+                index = pd.Index(list(input_1d_tileables.keys()))
+                index_value = parse_index(index, store_data=True)
+                self.index = index
+
+            if columns is not None:
+                tileable_size = tileables[0].shape[0] if tileables else 0
+                if not isinstance(columns, pd.Index):
+                    columns = self.columns = pd.Index(columns)
+                column_size = columns.shape[0]
+                if (
+                    not pd.isna(tileable_size)
+                    and not pd.isna(column_size)
+                    and tileable_size != column_size
+                ):
+                    raise ValueError(
+                        f"columns {columns} should have the same shape "
+                        f"with tensor: {tileable_size}"
+                    )
+                columns_value = self._process_index(columns, tileables)
+            else:
+                if not tileables or np.isnan(tileables[0].shape[0]):
+                    columns = columns_value = None
+                else:
+                    columns = pd.RangeIndex(0, tileables[0].shape[0])
+                    columns_value = parse_index(columns, store_data=True)
+                self.columns = columns
+
+            shape = (len(input_1d_tileables), shape[0] if shape else 0)
         else:
-            if np.isnan(tileables[0].shape[0]):
-                index = pd.RangeIndex(0)
+            if index is not None:
+                tileable_size = tileables[0].shape[0] if tileables else 0
+                if hasattr(index, "shape"):
+                    index_size = index.shape[0]
+                else:
+                    index_size = len(index)
+                if (
+                    not pd.isna(tileable_size)
+                    and not pd.isna(index_size)
+                    and tileable_size != index_size
+                ):
+                    raise ValueError(
+                        f"index {index} should have the same shape "
+                        f"with tensor: {tileable_size}"
+                    )
+                index_value = self._process_index(index, tileables)
             else:
-                index = pd.RangeIndex(0, tileables[0].shape[0])
-            self.index = index
-            index_value = parse_index(index)
+                if not tileables or np.isnan(tileables[0].shape[0]):
+                    index = pd.RangeIndex(0)
+                else:
+                    index = pd.RangeIndex(0, tileables[0].shape[0])
+                self.index = index
+                index_value = parse_index(index)
 
-        if columns is not None:
-            if len(input_1d_tileables) != len(columns):
-                raise ValueError(
-                    f"columns {columns} should have size {len(input_1d_tileables)}"
+            if columns is not None:
+                if len(input_1d_tileables) != len(columns):
+                    raise ValueError(
+                        f"columns {columns} should have size {len(input_1d_tileables)}"
+                    )
+                if not isinstance(columns, pd.Index):
+                    if isinstance(columns, ENTITY_TYPE):
+                        raise NotImplementedError(
+                            "The columns value cannot be a tileable"
+                        )
+                    columns = pd.Index(columns)
+                columns_value = parse_index(columns, store_data=True)
+            else:
+                columns_value = parse_index(
+                    pd.RangeIndex(0, len(input_1d_tileables)), store_data=True
                 )
-            if not isinstance(columns, pd.Index):
-                if isinstance(columns, ENTITY_TYPE):
-                    raise NotImplementedError("The columns value cannot be a tileable")
-                columns = pd.Index(columns)
-            columns_value = parse_index(columns, store_data=True)
-        else:
-            columns_value = parse_index(
-                pd.RangeIndex(0, len(input_1d_tileables)), store_data=True
-            )
 
-        shape = (shape[0], len(input_1d_tileables))
+            shape = (shape[0] if shape else 0, len(input_1d_tileables))
+
         return self.new_dataframe(
             tileables,
             shape,
@@ -278,6 +315,9 @@ def dataframe_from_tensor(
     gpu: bool = None,
     sparse: bool = False,
 ):
+    if isinstance(columns, list) and columns and isinstance(columns[0], tuple):
+        columns = pd.MultiIndex.from_tuples(columns)
+
     if tensor is not None:
         if tensor.ndim > 2 or tensor.ndim <= 0:
             raise TypeError(
@@ -299,6 +339,8 @@ def dataframe_from_tensor(
             dtypes = pd.Series([], index=pd.Index([], dtype=object))
     if index is not None and not isinstance(index, ENTITY_TYPE):
         index = pd.Index(index)
+        if isinstance(index[0], tuple):
+            index = pd.MultiIndex.from_tuples(index)
     op = DataFrameFromTensor(
         input=tensor, index=index, columns=columns, gpu=gpu, sparse=sparse
     )
@@ -311,7 +353,10 @@ def dataframe_from_1d_tileables(
     columns: Union[pd.Index, list] = None,
     gpu: bool = None,
     sparse: bool = False,
+    axis: int = 1,
 ):
+    from pandas.core.dtypes.cast import find_common_type
+
     data = dict()
     for k, v in d.items():
         if isinstance(v, (list, tuple)) and any(
@@ -322,9 +367,9 @@ def dataframe_from_1d_tileables(
             data[k] = v
     d = data
     if columns is not None:
-        tileables = [d.get(c) for c in columns]
+        tileables = [d.get(c) for c in columns] if axis == 1 else list(d.values())
     else:
-        columns = list(d.keys())
+        columns = list(d.keys()) if axis == 1 else None
         tileables = list(d.values())
 
     gpu = (
@@ -332,14 +377,37 @@ def dataframe_from_1d_tileables(
         if gpu is None
         else gpu
     )
-    dtypes = pd.Series(
-        [t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype for t in tileables],
-        index=columns,
-    )
+
+    if axis == 0:
+        col_num = (
+            tileables[0].shape[0]
+            if hasattr(tileables[0], "shape")
+            else len(tileables[0])
+        )
+        if pd.isna(col_num):
+            dtypes = None
+        else:
+            common_dtype = find_common_type(
+                [
+                    t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype
+                    for t in tileables
+                ]
+            )
+            dtypes = pd.Series(
+                [common_dtype] * col_num,
+                index=columns if columns is not None else pd.RangeIndex(col_num),
+            )
+    else:
+        dtypes = pd.Series(
+            [t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype for t in tileables],
+            index=columns,
+        )
+
     if index is not None and not isinstance(index, ENTITY_TYPE):
         index = pd.Index(index)
+
     op = DataFrameFromTensor(
-        input=d, index=index, columns=columns, gpu=gpu, sparse=sparse
+        input=d, index=index, columns=columns, gpu=gpu, sparse=sparse, axis=axis
     )
     return op(d, index, columns, dtypes)
 

maxframe/dataframe/datasource/read_csv.py

@@ -38,8 +38,12 @@ from ...serialization.serializables import (
     StringField,
 )
 from ...utils import lazy_import, parse_readable_size
-from ..utils import parse_index, to_arrow_dtypes
-from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
+from ..utils import parse_index, to_arrow_dtypes, validate_dtype_backend
+from .core import (
+    ColumnPruneSupportedDataSourceMixin,
+    DtypeBackendCompatibleMixin,
+    IncrementalIndexDatasource,
+)
 
 cudf = lazy_import("cudf")
 
@@ -88,6 +92,7 @@ def _find_chunk_start_end(f, offset, size):
 class DataFrameReadCSV(
     IncrementalIndexDatasource,
     ColumnPruneSupportedDataSourceMixin,
+    DtypeBackendCompatibleMixin,
 ):
     _op_type_ = opcodes.READ_CSV
 
@@ -101,7 +106,7 @@ class DataFrameReadCSV(
     offset = Int64Field("offset")
     size = Int64Field("size")
     incremental_index = BoolField("incremental_index")
-    use_arrow_dtype = BoolField("use_arrow_dtype")
+    dtype_backend = StringField("dtype_backend", default=None)
     keep_usecols_order = BoolField("keep_usecols_order", default=None)
     storage_options = DictField("storage_options")
     merge_small_files = BoolField("merge_small_files")
@@ -151,7 +156,7 @@ def read_csv(
     head_bytes="100k",
     head_lines=None,
     incremental_index: bool = True,
-    use_arrow_dtype: bool = None,
+    dtype_backend: str = None,
     storage_options: dict = None,
     memory_scale: int = None,
     merge_small_files: bool = True,
@@ -419,8 +424,8 @@ def read_csv(
     incremental_index: bool, default True
         If index_col not specified, ensure range index incremental,
         gain a slightly better performance if setting False.
-    use_arrow_dtype: bool, default None
-        If True, use arrow dtype to store columns.
+    dtype_backend: {'numpy', 'pyarrow'}, default 'numpy'
+        Back-end data type applied to the resultant DataFrame (still experimental).
     storage_options: dict, optional
         Options for storage connection.
     merge_small_files: bool, default True
@@ -441,13 +446,12 @@ def read_csv(
     Examples
     --------
     >>> import maxframe.dataframe as md
-    >>> from maxframe.lib.filesystem.oss import build_oss_path
     >>> md.read_csv('data.csv')  # doctest: +SKIP
     >>> # read from HDFS
     >>> md.read_csv('hdfs://localhost:8020/test.csv')  # doctest: +SKIP
     >>> # read from OSS
-    >>> auth_path = build_oss_path(file_path, access_key_id, access_key_secret, end_point)
-    >>> md.read_csv(auth_path)
+    >>> md.read_csv('oss://oss-cn-hangzhou-internal.aliyuncs.com/bucket/test.csv',
+    >>>             storage_options={'role_arn': 'acs:ram::xxxxxx:role/aliyunodpsdefaultrole'})
     """
     # infer dtypes and columns
     if isinstance(path, (list, tuple)):
@@ -510,7 +514,7 @@ def read_csv(
         compression=compression,
         gpu=gpu,
         incremental_index=incremental_index,
-        use_arrow_dtype=use_arrow_dtype,
+        dtype_backend=dtype_backend,
         storage_options=storage_options,
         memory_scale=memory_scale,
         merge_small_files=merge_small_files,
@@ -519,10 +523,13 @@ def read_csv(
     )
     chunk_bytes = chunk_bytes or options.chunk_store_limit
     dtypes = mini_df.dtypes
-    if use_arrow_dtype is None:
-        use_arrow_dtype = options.dataframe.use_arrow_dtype
-    if not gpu and use_arrow_dtype:
-        dtypes = to_arrow_dtypes(dtypes, test_df=mini_df)
+
+    dtype_backend = validate_dtype_backend(
+        dtype_backend or options.dataframe.dtype_backend
+    )
+
+    if not gpu and dtype_backend == "pyarrow":
+        dtypes = to_arrow_dtypes(dtypes)
     ret = op(
         index_value=index_value,
         columns_value=columns_value,

maxframe/dataframe/datasource/read_odps_query.py

@@ -29,7 +29,7 @@ from odps.types import Column, OdpsSchema, validate_data_type
 from odps.utils import split_sql_by_semicolon
 
 from ... import opcodes
-from ...config import options
+from ...config import option_context, options
 from ...core import OutputType
 from ...core.graph import DAG
 from ...io.odpsio import odps_schema_to_pandas_dtypes
@@ -44,8 +44,12 @@ from ...serialization.serializables import (
     StringField,
 )
 from ...utils import is_empty
-from ..utils import parse_index
-from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
+from ..utils import parse_index, validate_dtype_backend
+from .core import (
+    ColumnPruneSupportedDataSourceMixin,
+    DtypeBackendCompatibleMixin,
+    IncrementalIndexDatasource,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -266,6 +270,7 @@ def _build_explain_sql(
 class DataFrameReadODPSQuery(
     IncrementalIndexDatasource,
     ColumnPruneSupportedDataSourceMixin,
+    DtypeBackendCompatibleMixin,
 ):
     _op_type_ = opcodes.READ_ODPS_QUERY
 
@@ -273,12 +278,16 @@ class DataFrameReadODPSQuery(
     dtypes = SeriesField("dtypes", default=None)
     columns = AnyField("columns", default=None)
     nrows = Int64Field("nrows", default=None)
-    use_arrow_dtype = BoolField("use_arrow_dtype", default=None)
+    dtype_backend = StringField("dtype_backend", default=None)
     string_as_binary = BoolField("string_as_binary", default=None)
     index_columns = ListField("index_columns", FieldTypes.string, default=None)
     index_dtypes = SeriesField("index_dtypes", default=None)
     column_renames = DictField("column_renames", default=None)
 
+    def __init__(self, dtype_backend=None, **kw):
+        dtype_backend = validate_dtype_backend(dtype_backend)
+        super().__init__(dtype_backend=dtype_backend, **kw)
+
     def get_columns(self):
         return self.columns or list(self.dtypes.index)
 
@@ -404,6 +413,7 @@ def read_odps_query(
     sql_hints: Dict[str, str] = None,
     anonymous_col_prefix: str = _DEFAULT_ANONYMOUS_COL_PREFIX,
     skip_schema: bool = False,
+    dtype_backend: str = None,
     **kw,
 ):
     """
@@ -428,6 +438,8 @@ def read_odps_query(
         Skip resolving output schema before execution. Once this is configured,
         the output DataFrame cannot be inputs of other DataFrame operators
         before execution.
+    dtype_backend: {'numpy', 'pyarrow'}, default 'numpy'
+        Back-end data type applied to the resultant DataFrame (still experimental).
 
     Returns
     -------
@@ -459,6 +471,14 @@ def read_odps_query(
     if odps_entry is None:
         raise ValueError("Missing odps_entry parameter")
 
+    if "use_arrow_dtype" in kw:
+        dtype_backend = dtype_backend or validate_dtype_backend(
+            kw.pop("use_arrow_dtype")
+        )
+    dtype_backend = validate_dtype_backend(
+        dtype_backend or options.dataframe.dtype_backend
+    )
+
     col_renames = {}
     if not skip_schema:
         odps_schema = _resolve_query_schema(
@@ -479,7 +499,9 @@ def read_odps_query(
             else:
                 new_columns.append(col)
 
-        dtypes = odps_schema_to_pandas_dtypes(OdpsSchema(new_columns))
+        with option_context():
+            options.dataframe.dtype_backend = dtype_backend
+            dtypes = odps_schema_to_pandas_dtypes(OdpsSchema(new_columns))
     else:
         dtypes = None
 
@@ -500,10 +522,11 @@ def read_odps_query(
 
     chunk_bytes = kw.pop("chunk_bytes", None)
     chunk_size = kw.pop("chunk_size", None)
+
     op = DataFrameReadODPSQuery(
         query=query,
         dtypes=dtypes,
-        use_arrow_dtype=kw.pop("use_arrow_dtype", True),
+        dtype_backend=dtype_backend,
         string_as_binary=string_as_binary,
         index_columns=index_col,
         index_dtypes=index_dtypes,