maxframe 2.0.0b1__cp37-cp37m-win_amd64.whl → 2.2.0__cp37-cp37m-win_amd64.whl

maxframe/dataframe/datasource/from_index.py

@@ -51,7 +51,7 @@ class SeriesFromIndex(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def series_from_index(ind, index=None, name=None):
-    name = name or ind.name or 0
+    name = name or ind.name
     if index is not None:
         index = Index(index)
     op = SeriesFromIndex(input_=ind, index=index, name=name)

maxframe/dataframe/datasource/from_records.py

@@ -77,6 +77,83 @@ def from_records(
     sparse=False,
     **kw
 ):
+    """
+    Convert structured or record ndarray to DataFrame.
+
+    Creates a DataFrame object from a structured ndarray, sequence of
+    tuples or dicts, or DataFrame.
+
+    Parameters
+    ----------
+    data : structured ndarray, sequence of tuples or dicts, or DataFrame
+        Structured input data.
+
+        .. deprecated:: 2.1.0
+            Passing a DataFrame is deprecated.
+    index : str, list of fields, array-like
+        Field of array to use as the index, alternately a specific set of
+        input labels to use.
+    exclude : sequence, default None
+        Columns or fields to exclude.
+    columns : sequence, default None
+        Column names to use. If the passed data do not have names
+        associated with them, this argument provides names for the
+        columns. Otherwise this argument indicates the order of the columns
+        in the result (any names not found in the data will become all-NA
+        columns).
+    coerce_float : bool, default False
+        Attempt to convert values of non-string, non-numeric objects (like
+        decimal.Decimal) to floating point, useful for SQL result sets.
+    nrows : int, default None
+        Number of rows to read if data is an iterator.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.from_dict : DataFrame from dict of array-like or dicts.
+    DataFrame : DataFrame object creation using constructor.
+
+    Examples
+    --------
+    Data can be provided as a structured ndarray:
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> data = mt.array([(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')],
+    ...                 dtype=[('col_1', 'i4'), ('col_2', 'U1')])
+    >>> md.DataFrame.from_records(data).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+
+    Data can be provided as a list of dicts:
+
+    >>> data = [{'col_1': 3, 'col_2': 'a'},
+    ...         {'col_1': 2, 'col_2': 'b'},
+    ...         {'col_1': 1, 'col_2': 'c'},
+    ...         {'col_1': 0, 'col_2': 'd'}]
+    >>> md.DataFrame.from_records(data).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+
+    Data can be provided as a list of tuples with corresponding columns:
+
+    >>> data = [(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')]
+    >>> md.DataFrame.from_records(data, columns=['col_1', 'col_2']).execute()
+       col_1 col_2
+    0      3     a
+    1      2     b
+    2      1     c
+    3      0     d
+    """
     if isinstance(data, np.ndarray):
         from .dataframe import from_pandas
 

maxframe/dataframe/datasource/from_tensor.py

@@ -39,6 +39,7 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
     input = AnyField("input")
     index = AnyField("index")
     columns = AnyField("columns")
+    axis = AnyField("axis")
 
     def __init__(self, *args, **kwargs):
         kwargs["_output_types"] = [OutputType.dataframe]
@@ -120,46 +121,82 @@ class DataFrameFromTensor(DataFrameOperator, DataFrameOperatorMixin):
             if isinstance(tileable, ENTITY_TYPE):
                 tileables.append(tileable)
 
-        if index is not None:
-            tileable_size = tileables[0].shape[0]
-            if hasattr(index, "shape"):
-                index_size = index.shape[0]
+        if self.axis == 0:
+            if index is not None:
+                raise NotImplementedError("Cannot accept index when axis=0")
             else:
-                index_size = len(index)
-            if (
-                not pd.isna(tileable_size)
-                and not pd.isna(index_size)
-                and tileable_size != index_size
-            ):
-                raise ValueError(
-                    f"index {index} should have the same shape "
-                    f"with tensor: {tileable_size}"
-                )
-            index_value = self._process_index(index, tileables)
+                index = pd.Index(list(input_1d_tileables.keys()))
+                index_value = parse_index(index, store_data=True)
+                self.index = index
+
+            if columns is not None:
+                tileable_size = tileables[0].shape[0] if tileables else 0
+                if not isinstance(columns, pd.Index):
+                    columns = self.columns = pd.Index(columns)
+                column_size = columns.shape[0]
+                if (
+                    not pd.isna(tileable_size)
+                    and not pd.isna(column_size)
+                    and tileable_size != column_size
+                ):
+                    raise ValueError(
+                        f"columns {columns} should have the same shape "
+                        f"with tensor: {tileable_size}"
+                    )
+                columns_value = self._process_index(columns, tileables)
+            else:
+                if not tileables or np.isnan(tileables[0].shape[0]):
+                    columns = columns_value = None
+                else:
+                    columns = pd.RangeIndex(0, tileables[0].shape[0])
+                    columns_value = parse_index(columns, store_data=True)
+                self.columns = columns
+
+            shape = (len(input_1d_tileables), shape[0] if shape else 0)
         else:
-            if np.isnan(tileables[0].shape[0]):
-                index = pd.RangeIndex(0)
+            if index is not None:
+                tileable_size = tileables[0].shape[0] if tileables else 0
+                if hasattr(index, "shape"):
+                    index_size = index.shape[0]
+                else:
+                    index_size = len(index)
+                if (
+                    not pd.isna(tileable_size)
+                    and not pd.isna(index_size)
+                    and tileable_size != index_size
+                ):
+                    raise ValueError(
+                        f"index {index} should have the same shape "
+                        f"with tensor: {tileable_size}"
+                    )
+                index_value = self._process_index(index, tileables)
             else:
-                index = pd.RangeIndex(0, tileables[0].shape[0])
-            self.index = index
-            index_value = parse_index(index)
+                if not tileables or np.isnan(tileables[0].shape[0]):
+                    index = pd.RangeIndex(0)
+                else:
+                    index = pd.RangeIndex(0, tileables[0].shape[0])
+                self.index = index
+                index_value = parse_index(index)
 
-        if columns is not None:
-            if len(input_1d_tileables) != len(columns):
-                raise ValueError(
-                    f"columns {columns} should have size {len(input_1d_tileables)}"
+            if columns is not None:
+                if len(input_1d_tileables) != len(columns):
+                    raise ValueError(
+                        f"columns {columns} should have size {len(input_1d_tileables)}"
+                    )
+                if not isinstance(columns, pd.Index):
+                    if isinstance(columns, ENTITY_TYPE):
+                        raise NotImplementedError(
+                            "The columns value cannot be a tileable"
+                        )
+                    columns = pd.Index(columns)
+                columns_value = parse_index(columns, store_data=True)
+            else:
+                columns_value = parse_index(
+                    pd.RangeIndex(0, len(input_1d_tileables)), store_data=True
                 )
-            if not isinstance(columns, pd.Index):
-                if isinstance(columns, ENTITY_TYPE):
-                    raise NotImplementedError("The columns value cannot be a tileable")
-                columns = pd.Index(columns)
-            columns_value = parse_index(columns, store_data=True)
-        else:
-            columns_value = parse_index(
-                pd.RangeIndex(0, len(input_1d_tileables)), store_data=True
-            )
 
-        shape = (shape[0], len(input_1d_tileables))
+            shape = (shape[0] if shape else 0, len(input_1d_tileables))
+
         return self.new_dataframe(
             tileables,
             shape,
@@ -278,6 +315,9 @@ def dataframe_from_tensor(
     gpu: bool = None,
     sparse: bool = False,
 ):
+    if isinstance(columns, list) and columns and isinstance(columns[0], tuple):
+        columns = pd.MultiIndex.from_tuples(columns)
+
     if tensor is not None:
         if tensor.ndim > 2 or tensor.ndim <= 0:
             raise TypeError(
@@ -299,6 +339,8 @@ def dataframe_from_tensor(
             dtypes = pd.Series([], index=pd.Index([], dtype=object))
     if index is not None and not isinstance(index, ENTITY_TYPE):
         index = pd.Index(index)
+        if isinstance(index[0], tuple):
+            index = pd.MultiIndex.from_tuples(index)
     op = DataFrameFromTensor(
         input=tensor, index=index, columns=columns, gpu=gpu, sparse=sparse
     )
@@ -311,7 +353,10 @@ def dataframe_from_1d_tileables(
     columns: Union[pd.Index, list] = None,
     gpu: bool = None,
     sparse: bool = False,
+    axis: int = 1,
 ):
+    from pandas.core.dtypes.cast import find_common_type
+
     data = dict()
     for k, v in d.items():
         if isinstance(v, (list, tuple)) and any(
@@ -322,9 +367,9 @@ def dataframe_from_1d_tileables(
             data[k] = v
     d = data
     if columns is not None:
-        tileables = [d.get(c) for c in columns]
+        tileables = [d.get(c) for c in columns] if axis == 1 else list(d.values())
     else:
-        columns = list(d.keys())
+        columns = list(d.keys()) if axis == 1 else None
         tileables = list(d.values())
 
     gpu = (
@@ -332,14 +377,37 @@ def dataframe_from_1d_tileables(
         if gpu is None
         else gpu
     )
-    dtypes = pd.Series(
-        [t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype for t in tileables],
-        index=columns,
-    )
+
+    if axis == 0:
+        col_num = (
+            tileables[0].shape[0]
+            if hasattr(tileables[0], "shape")
+            else len(tileables[0])
+        )
+        if pd.isna(col_num):
+            dtypes = None
+        else:
+            common_dtype = find_common_type(
+                [
+                    t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype
+                    for t in tileables
+                ]
+            )
+            dtypes = pd.Series(
+                [common_dtype] * col_num,
+                index=columns if columns is not None else pd.RangeIndex(col_num),
+            )
+    else:
+        dtypes = pd.Series(
+            [t.dtype if hasattr(t, "dtype") else pd.Series(t).dtype for t in tileables],
+            index=columns,
+        )
+
     if index is not None and not isinstance(index, ENTITY_TYPE):
         index = pd.Index(index)
+
     op = DataFrameFromTensor(
-        input=d, index=index, columns=columns, gpu=gpu, sparse=sparse
+        input=d, index=index, columns=columns, gpu=gpu, sparse=sparse, axis=axis
     )
     return op(d, index, columns, dtypes)
 

maxframe/dataframe/datasource/read_csv.py

@@ -441,13 +441,12 @@ def read_csv(
     Examples
     --------
     >>> import maxframe.dataframe as md
-    >>> from maxframe.lib.filesystem.oss import build_oss_path
     >>> md.read_csv('data.csv')  # doctest: +SKIP
     >>> # read from HDFS
     >>> md.read_csv('hdfs://localhost:8020/test.csv')  # doctest: +SKIP
     >>> # read from OSS
-    >>> auth_path = build_oss_path(file_path, access_key_id, access_key_secret, end_point)
-    >>> md.read_csv(auth_path)
+    >>> md.read_csv('oss://oss-cn-hangzhou-internal.aliyuncs.com/bucket/test.csv',
+    >>>             storage_options={'role_arn': 'acs:ram::xxxxxx:role/aliyunodpsdefaultrole'})
     """
     # infer dtypes and columns
     if isinstance(path, (list, tuple)):

maxframe/dataframe/datasource/read_odps_query.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import dataclasses
+import functools
 import io
 import logging
 import re
@@ -22,6 +23,8 @@ from typing import Dict, List, MutableMapping, Optional, Tuple, Union
 import numpy as np
 import pandas as pd
 from odps import ODPS
+from odps.errors import ODPSError
+from odps.models import TableSchema
 from odps.types import Column, OdpsSchema, validate_data_type
 from odps.utils import split_sql_by_semicolon
 
@@ -245,13 +248,18 @@ def _parse_explained_schema(explain_string: str) -> OdpsSchema:
         return _parse_full_explain(explain_string)
 
 
-def _build_explain_sql(sql_stmt: str, no_split: bool = False) -> str:
+def _build_explain_sql(
+    sql_stmt: str, no_split: bool = False, use_output: bool = False
+) -> str:
+    clause = "EXPLAIN "
+    if use_output:
+        clause += "OUTPUT "
     if no_split:
-        return "EXPLAIN " + sql_stmt
+        return clause + sql_stmt
     sql_parts = split_sql_by_semicolon(sql_stmt)
     if not sql_parts:
         raise ValueError(f"Cannot explain SQL statement {sql_stmt}")
-    sql_parts[-1] = "EXPLAIN " + sql_parts[-1]
+    sql_parts[-1] = clause + sql_parts[-1]
     return "\n".join(sql_parts)
 
 
@@ -332,6 +340,62 @@ def _check_token_in_sql(token: str, sql: str) -> bool:
         return False
 
 
+def _resolve_schema_by_explain(
+    odps_entry: ODPS,
+    query: str,
+    no_split_sql: bool = False,
+    hints: Dict[str, str] = None,
+    use_explain_output: bool = True,
+) -> OdpsSchema:
+    hints = (hints or dict()).copy()
+    hints["odps.sql.select.output.format"] = "json"
+    explain_stmt = _build_explain_sql(
+        query, no_split=no_split_sql, use_output=use_explain_output
+    )
+    inst = odps_entry.execute_sql(explain_stmt, hints=hints)
+    logger.debug("Explain output instance ID: %s", inst.id)
+    explain_str = list(inst.get_task_results().values())[0]
+    if use_explain_output:
+        if not explain_str or "nothing to explain" in explain_str:
+            raise ValueError("The SQL statement should be an instant query")
+        return TableSchema.parse(None, explain_str)
+    else:
+        return _parse_explained_schema(explain_str)
+
+
+def _resolve_query_schema(
+    odps_entry: ODPS,
+    query: str,
+    no_split_sql: bool = False,
+    hints: Dict[str, str] = None,
+    use_explain_output: Optional[bool] = None,
+) -> OdpsSchema:
+    methods = []
+    if use_explain_output is not False:
+        # None or True
+        methods.append(_resolve_schema_by_explain)
+    if use_explain_output is not True:
+        # None or False
+        methods.append(
+            functools.partial(_resolve_schema_by_explain, use_explain_output=False)
+        )
+    for idx, resolve_method in enumerate(methods):
+        try:
+            return resolve_method(
+                odps_entry, query, no_split_sql=no_split_sql, hints=hints
+            )
+        except ODPSError as ex:
+            msg = (
+                f"Failed to obtain schema from SQL explain: {ex!r}\n"
+                f"Explain instance ID: {ex.instance_id}"
+            )
+            if idx + 1 == len(methods) or "ODPS-0130161" not in str(ex):
+                exc = ValueError(msg)
+                raise exc.with_traceback(ex.__traceback__) from None
+    # will this happen?
+    raise ValueError("Failed to obtain schema from SQL explain")  # pragma: no cover
+
+
 def read_odps_query(
     query: str,
     odps_entry: ODPS = None,
@@ -371,6 +435,8 @@ def read_odps_query(
         DataFrame read from MaxCompute (ODPS) table
     """
     no_split_sql = kw.pop("no_split_sql", False)
+    # if use_explain_output is None, will try two methods.
+    use_explain_output = kw.pop("use_explain_output", None)
 
     hints = options.sql.settings.copy() or {}
     if sql_hints:
@@ -395,19 +461,13 @@ def read_odps_query(
 
     col_renames = {}
     if not skip_schema:
-        explain_stmt = _build_explain_sql(query, no_split=no_split_sql)
-        inst = odps_entry.execute_sql(explain_stmt, hints=hints)
-        logger.debug("Explain instance ID: %s", inst.id)
-        explain_str = list(inst.get_task_results().values())[0]
-
-        try:
-            odps_schema = _parse_explained_schema(explain_str)
-        except BaseException as ex:
-            exc = ValueError(
-                f"Failed to obtain schema from SQL explain: {ex!r}"
-                f"\nExplain instance ID: {inst.id}"
-            )
-            raise exc.with_traceback(ex.__traceback__) from None
+        odps_schema = _resolve_query_schema(
+            odps_entry,
+            query,
+            no_split_sql=no_split_sql,
+            hints=hints,
+            use_explain_output=use_explain_output,
+        )
 
         new_columns = []
         for col in odps_schema.columns:

maxframe/dataframe/datasource/tests/test_datasource.py

@@ -17,11 +17,13 @@ import uuid
 from collections import OrderedDict
 from math import isinf
 
+import mock
 import numpy as np
 import pandas as pd
 import pytest
 from odps import ODPS
 from odps import types as odps_types
+from odps.errors import ODPSError
 
 from .... import tensor as mt
 from ....core import OutputType
@@ -50,6 +52,7 @@ from ..read_odps_query import (
     ColumnSchema,
     _parse_full_explain,
     _parse_simple_explain,
+    _resolve_query_schema,
     _resolve_task_sector,
 )
 from ..series import from_pandas as from_pandas_series
@@ -181,6 +184,23 @@ def test_from_tensor():
     df = dataframe_from_1d_tileables(d)
     pd.testing.assert_index_equal(df.columns_value.to_pandas(), pd.RangeIndex(2))
 
+    # test axis parameter for dataframe_from_1d_tileables
+    d = OrderedDict(
+        [("a", mt.tensor(np.random.rand(4))), ("b", mt.tensor(np.random.rand(4)))]
+    )
+
+    # axis=1 (default behavior) - keys become columns
+    df = dataframe_from_1d_tileables(d, axis=1)
+    assert df.shape == (4, 2)
+    pd.testing.assert_index_equal(df.columns_value.to_pandas(), pd.Index(["a", "b"]))
+    pd.testing.assert_index_equal(df.index_value.to_pandas(), pd.RangeIndex(4))
+
+    # axis=0 - keys become index (rows)
+    df = dataframe_from_1d_tileables(d, axis=0)
+    assert df.shape == (2, 4)
+    pd.testing.assert_index_equal(df.index_value.to_pandas(), pd.Index(["a", "b"]))
+    pd.testing.assert_index_equal(df.columns_value.to_pandas(), pd.RangeIndex(4))
+
     series = series_from_tensor(mt.random.rand(4))
     pd.testing.assert_index_equal(series.index_value.to_pandas(), pd.RangeIndex(4))
 
@@ -204,6 +224,26 @@ def test_from_tensor():
     with pytest.raises(ValueError):
         dataframe_from_tensor(mt.random.rand(4, 3), columns=["a", "b"])
 
+    # 1-d tensors should have same shape
+    with pytest.raises(ValueError):
+        dataframe_from_1d_tileables(
+            OrderedDict(
+                [(0, mt.tensor(np.random.rand(3))), (1, mt.tensor(np.random.rand(2)))]
+            )
+        )
+
+    # index has wrong shape
+    with pytest.raises(ValueError):
+        dataframe_from_1d_tileables(
+            {0: mt.tensor(np.random.rand(3))}, index=mt.tensor(np.random.rand(2))
+        )
+
+    # columns have wrong shape
+    with pytest.raises(ValueError):
+        dataframe_from_1d_tileables(
+            {0: mt.tensor(np.random.rand(3))}, columns=["a", "b"]
+        )
+
     # index should be 1-d
     with pytest.raises(ValueError):
         dataframe_from_tensor(
@@ -360,7 +400,7 @@ def test_from_odps_query():
 
     with pytest.raises(ValueError) as err_info:
         read_odps_query(
-            f"CREATE TABLE dummy_table_{uuid.uuid4().hex} "
+            f"CREATE TABLE dummy_table_{uuid.uuid4().hex} LIFECYCLE 1 "
             f"AS SELECT * FROM {table1_name}"
         )
     assert "instant query" in err_info.value.args[0]
@@ -578,3 +618,46 @@ def test_resolve_break_lines():
     for col, (exp_nm, exp_tp) in zip(schema.columns, expected_col_types.items()):
         assert col.name == exp_nm
         assert col.type == odps_types.validate_data_type(exp_tp)
+
+
+@pytest.mark.parametrize("use_explain_output", [None, False, True])
+def test_explain_use_explain_output(use_explain_output):
+    class MockInstance:
+        @property
+        def id(self):
+            return "mock_id"
+
+        def get_task_results(self):
+            return {"pot": """{"columns":[{"name":"a_bigint","type":"BIGINT"}]}"""}
+
+    old_execute_sql = ODPS.execute_sql
+    exec_count = 0
+
+    def new_execute_sql(self, sql, *args, **kw):
+        nonlocal exec_count
+        exec_count += 1
+
+        if use_explain_output and sql.lower().startswith("explain output select"):
+            return MockInstance()
+        elif use_explain_output is None and sql.lower().startswith("explain output"):
+            raise ODPSError("ODPS-0130161: mock error")
+        return old_execute_sql(self, sql, *args, **kw)
+
+    odps_entry = ODPS.from_environments()
+
+    with mock.patch("odps.core.ODPS.execute_sql", new=new_execute_sql):
+        with pytest.raises(ValueError):
+            _resolve_query_schema(
+                odps_entry, "not_a_sql", use_explain_output=use_explain_output
+            )
+        assert exec_count == (2 if use_explain_output is None else 1)
+
+        exec_count = 0
+        schema = _resolve_query_schema(
+            odps_entry,
+            "select cast(1 as bigint) as a_bigint",
+            use_explain_output=use_explain_output,
+        )
+        assert schema.columns[0].name == "a_bigint"
+        assert schema.columns[0].type == odps_types.bigint
+        assert exec_count == (2 if use_explain_output is None else 1)

maxframe/dataframe/datastore/__init__.py

@@ -12,14 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .to_csv import to_csv
 from .to_odps import to_odps_table
 
 
 def _install():
-    from ..core import DATAFRAME_TYPE
+    from ..core import DATAFRAME_TYPE, SERIES_TYPE
 
     for t in DATAFRAME_TYPE:
+        t.to_csv = to_csv
         t.to_odps_table = to_odps_table
+    for t in SERIES_TYPE:
+        t.to_csv = to_csv
 
 
 _install()