maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/dataframe/datastore/core.py

@@ -0,0 +1,28 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...serialization.serializables import AnyField, DictField, ListField, StringField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameDataStore(DataFrameOperator, DataFrameOperatorMixin):
+    pass
+
+
+class LakeDataStore(DataFrameDataStore):
+    path = AnyField("path")
+    compression = AnyField("compression", default=None)
+    partition_cols = ListField("partition_cols", default=None)
+    storage_options = DictField("storage_options", default=None)
+    write_stage = StringField("write_stage", default=None)

maxframe/dataframe/datastore/direct.py

@@ -0,0 +1,268 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...utils import pd_release_version
+
+_to_dict_has_index = pd_release_version[0] >= 2
+
+
+def df_to_dict(
+    df, orient="dict", into=dict, index=True, batch_size=10000, session=None
+):
+    """
+    Convert the DataFrame to a dictionary.
+
+    The type of the key-value pairs can be customized with the parameters
+    (see below).
+
+    Parameters
+    ----------
+    orient : str {'dict', 'list', 'series', 'split', 'tight', 'records', 'index'}
+        Determines the type of the values of the dictionary.
+
+        - 'dict' (default) : dict like {column -> {index -> value}}
+        - 'list' : dict like {column -> [values]}
+        - 'series' : dict like {column -> Series(values)}
+        - 'split' : dict like
+          {'index' -> [index], 'columns' -> [columns], 'data' -> [values]}
+        - 'tight' : dict like
+          {'index' -> [index], 'columns' -> [columns], 'data' -> [values],
+          'index_names' -> [index.names], 'column_names' -> [column.names]}
+        - 'records' : list like
+          [{column -> value}, ... , {column -> value}]
+        - 'index' : dict like {index -> {column -> value}}
+
+    into : class, default dict
+        The collections.abc.MutableMapping subclass used for all Mappings
+        in the return value.  Can be the actual class or an empty
+        instance of the mapping type you want.  If you want a
+        collections.defaultdict, you must pass it initialized.
+
+    index : bool, default True
+        Whether to include the index item (and index_names item if `orient`
+        is 'tight') in the returned dictionary. Can only be ``False``
+        when `orient` is 'split' or 'tight'.
+
+    Returns
+    -------
+    dict, list or collections.abc.MutableMapping
+        Return a collections.abc.MutableMapping object representing the
+        DataFrame. The resulting transformation depends on the `orient`
+        parameter.
+
+    See Also
+    --------
+    DataFrame.from_dict: Create a DataFrame from a dictionary.
+    DataFrame.to_json: Convert a DataFrame to JSON format.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'col1': [1, 2],
+    ...                    'col2': [0.5, 0.75]},
+    ...                   index=['row1', 'row2'])
+    >>> df.execute()
+          col1  col2
+    row1     1  0.50
+    row2     2  0.75
+    >>> df.to_dict()
+    {'col1': {'row1': 1, 'row2': 2}, 'col2': {'row1': 0.5, 'row2': 0.75}}
+
+    You can specify the return orientation.
+
+    >>> df.to_dict('series')
+    {'col1': row1    1
+             row2    2
+    Name: col1, dtype: int64,
+    'col2': row1    0.50
+            row2    0.75
+    Name: col2, dtype: float64}
+
+    >>> df.to_dict('split')
+    {'index': ['row1', 'row2'], 'columns': ['col1', 'col2'],
+     'data': [[1, 0.5], [2, 0.75]]}
+
+    >>> df.to_dict('records')
+    [{'col1': 1, 'col2': 0.5}, {'col1': 2, 'col2': 0.75}]
+
+    >>> df.to_dict('index')
+    {'row1': {'col1': 1, 'col2': 0.5}, 'row2': {'col1': 2, 'col2': 0.75}}
+
+    >>> df.to_dict('tight')
+    {'index': ['row1', 'row2'], 'columns': ['col1', 'col2'],
+     'data': [[1, 0.5], [2, 0.75]], 'index_names': [None], 'column_names': [None]}
+
+    You can also specify the mapping type.
+
+    >>> from collections import OrderedDict, defaultdict
+    >>> df.to_dict(into=OrderedDict)
+    OrderedDict([('col1', OrderedDict([('row1', 1), ('row2', 2)])),
+                 ('col2', OrderedDict([('row1', 0.5), ('row2', 0.75)]))])
+
+    If you want a `defaultdict`, you need to initialize it:
+
+    >>> dd = defaultdict(list)
+    >>> df.to_dict('records', into=dd)
+    [defaultdict(<class 'list'>, {'col1': 1, 'col2': 0.5}),
+     defaultdict(<class 'list'>, {'col1': 2, 'col2': 0.75})]
+    """
+    fetch_kwargs = dict(batch_size=batch_size)
+    to_dict_kw = dict(orient=orient, into=into)
+    if _to_dict_has_index:
+        to_dict_kw["index"] = index
+    return df.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_dict(
+        **to_dict_kw
+    )
+
+
+def series_to_dict(series, into=dict, batch_size=10000, session=None):
+    """
+    Convert Series to {label -> value} dict or dict-like object.
+
+    Parameters
+    ----------
+    into : class, default dict
+        The collections.abc.Mapping subclass to use as the return
+        object. Can be the actual class or an empty
+        instance of the mapping type you want.  If you want a
+        collections.defaultdict, you must pass it initialized.
+
+    Returns
+    -------
+    collections.abc.Mapping
+        Key-value representation of Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([1, 2, 3, 4])
+    >>> s.to_dict()
+    {0: 1, 1: 2, 2: 3, 3: 4}
+    >>> from collections import OrderedDict, defaultdict
+    >>> s.to_dict(OrderedDict)
+    OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)])
+    >>> dd = defaultdict(list)
+    >>> s.to_dict(dd)
+    defaultdict(<class 'list'>, {0: 1, 1: 2, 2: 3, 3: 4})
+    """
+    fetch_kwargs = dict(batch_size=batch_size)
+    return series.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_dict(
+        into=into
+    )
+
+
+def series_to_list(series, batch_size=10000, session=None):
+    """
+    Return a list of the values.
+
+    These are each a scalar type, which is a Python scalar
+    (for str, int, float) or a pandas scalar
+    (for Timestamp/Timedelta/Interval/Period)
+
+    Returns
+    -------
+    list
+
+    See Also
+    --------
+    numpy.ndarray.tolist : Return the array as an a.ndim-levels deep
+        nested list of Python scalars.
+
+    Examples
+    --------
+    For Series
+
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([1, 2, 3])
+    >>> s.to_list()
+    [1, 2, 3]
+
+    For Index:
+
+    >>> idx = md.Index([1, 2, 3])
+    >>> idx.execute()
+    Index([1, 2, 3], dtype='int64')
+
+    >>> idx.to_list()
+    [1, 2, 3]
+    """
+    fetch_kwargs = dict(batch_size=batch_size)
+    return series.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_list()
+
+
+def to_clipboard(
+    obj, *, excel=True, sep=None, batch_size=10000, session=None, **kwargs
+):
+    """
+    Copy object to the system clipboard.
+
+    Write a text representation of object to the system clipboard.
+    This can be pasted into Excel, for example.
+
+    Parameters
+    ----------
+    excel : bool, default True
+        Produce output in a csv format for easy pasting into excel.
+
+        - True, use the provided separator for csv pasting.
+        - False, write a string representation of the object to the clipboard.
+
+    sep : str, default ``'\t'``
+        Field delimiter.
+    **kwargs
+        These parameters will be passed to DataFrame.to_csv.
+
+    See Also
+    --------
+    DataFrame.to_csv : Write a DataFrame to a comma-separated values
+        (csv) file.
+    read_clipboard : Read text from clipboard and pass to read_csv.
+
+    Notes
+    -----
+    Requirements for your platform.
+
+      - Linux : `xclip`, or `xsel` (with `PyQt4` modules)
+      - Windows : none
+      - macOS : none
+
+    This method uses the processes developed for the package `pyperclip`. A
+    solution to render any output string format is given in the examples.
+
+    Examples
+    --------
+    Copy the contents of a DataFrame to the clipboard.
+
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])
+
+    >>> df.to_clipboard(sep=',')  # doctest: +SKIP
+    ... # Wrote the following to the system clipboard:
+    ... # ,A,B,C
+    ... # 0,1,2,3
+    ... # 1,4,5,6
+
+    We can omit the index by passing the keyword `index` and setting
+    it to false.
+
+    >>> df.to_clipboard(sep=',', index=False)  # doctest: +SKIP
+    ... # Wrote the following to the system clipboard:
+    ... # A,B,C
+    ... # 1,2,3
+    ... # 4,5,6
+    """
+    fetch_kwargs = dict(batch_size=batch_size)
+    return obj.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_clipboard(
+        excel=excel, sep=sep, **kwargs
+    )

maxframe/dataframe/datastore/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/datastore/tests/test_to_odps.py

@@ -0,0 +1,99 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+import pytest
+
+from ... import DataFrame, Index
+from ..to_odps import to_odps_table
+
+
+@pytest.fixture
+def df():
+    return DataFrame({"A": [1, 2], "B": [3, 4]})
+
+
+@pytest.fixture
+def df_with_named_index():
+    return DataFrame({"A": [1, 2], "B": [3, 4]}, index=Index([1, 2], name="A"))
+
+
+@pytest.fixture
+def df_with_named_multi_indexes():
+    arrays = [
+        ["c1", "c2"],
+        ["d1", "d2"],
+        ["e1", "e2"],
+    ]
+    multi_index = pd.MultiIndex.from_arrays(arrays, names=("C", "D", "E"))
+    return DataFrame(
+        {"A": [1, 2], "B": [3, 4]},
+        index=multi_index,
+    )
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"partition_col": ["A", "C"]},
+        {"partition_col": "C"},
+        {"partition": "a=1,C=2"},
+    ],
+)
+def test_to_odps_table_validation(df, kwargs):
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", **kwargs)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"partition_col": ["a", "B"]},
+        {"partition_col": "a"},
+        {"partition": "C=1,d=2"},
+    ],
+)
+def test_to_odps_table_vaild(df, kwargs):
+    to_odps_table(df, "test_table", **kwargs)
+
+
+def test_to_odps_table_column_conflicts(
+    df, df_with_named_index, df_with_named_multi_indexes
+):
+    to_odps_table(df.reset_index(), "test_table", index=False)
+    to_odps_table(df.reset_index(), "test_table", index_label="C")
+    with pytest.raises(ValueError):
+        to_odps_table(df.reset_index(), "test_table")
+
+    to_odps_table(df_with_named_index, "test_table", index=False)
+    to_odps_table(df_with_named_index, "test_table", index_label="C")
+    with pytest.raises(ValueError):
+        to_odps_table(df_with_named_index, "test_table")
+
+    to_odps_table(df, "test_table", partition="C='1'")
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", partition="A='1'")
+
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", partition="A='1'")
+
+    to_odps_table(df_with_named_multi_indexes, "test_table")
+    to_odps_table(
+        df_with_named_multi_indexes, "test_table", partition="C='1'", index=False
+    )
+    with pytest.raises(ValueError):
+        to_odps_table(df_with_named_multi_indexes, "test_table", partition="C='1'")
+
+    df_with_named_multi_indexes.index.names = ["C1", "D1", "E1"]
+    to_odps_table(df_with_named_multi_indexes, "test_table", partition="C='1'")

maxframe/dataframe/datastore/to_csv.py

@@ -0,0 +1,219 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int64Field,
+    ListField,
+    StringField,
+)
+from ..utils import parse_index
+from .core import LakeDataStore
+
+
+class DataFrameToCSV(LakeDataStore):
+    _op_type_ = opcodes.TO_CSV
+
+    sep = StringField("sep", default=None)
+    na_rep = StringField("na_rep", default=None)
+    columns = ListField("columns", default=None)
+    header = AnyField("header", default=None)
+    index = BoolField("index", default=None)
+    index_label = AnyField("index_label", default=None)
+    mode = StringField("mode", default=None)
+    encoding = StringField("encoding", default=None)
+    chunksize = Int64Field("chunksize", default=None)
+    csv_kwargs = DictField("csv_kwargs", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @property
+    def one_file(self):
+        # if wildcard in path, write csv into multiple files
+        return "*" not in self.path and not self.partition_cols
+
+    def __call__(self, df):
+        index_value = parse_index(df.index_value.to_pandas()[:0], df)
+        if df.ndim == 2:
+            columns_value = parse_index(
+                df.columns_value.to_pandas()[:0], store_data=True
+            )
+            return self.new_dataframe(
+                [df],
+                shape=(0, 0),
+                dtypes=df.dtypes[:0],
+                index_value=index_value,
+                columns_value=columns_value,
+            )
+        else:
+            return self.new_series(
+                [df], shape=(0,), dtype=df.dtype, index_value=index_value
+            )
+
+
+def to_csv(
+    df,
+    path,
+    sep=",",
+    na_rep="",
+    float_format=None,
+    columns=None,
+    header=True,
+    index=True,
+    index_label=None,
+    mode="w",
+    encoding=None,
+    compression="infer",
+    quoting=None,
+    quotechar='"',
+    lineterminator=None,
+    chunksize=None,
+    date_format=None,
+    doublequote=True,
+    escapechar=None,
+    decimal=".",
+    partition_cols=None,
+    storage_options=None,
+    **kw,
+):
+    r"""
+    Write object to a comma-separated values (csv) file.
+
+    Parameters
+    ----------
+    path : str
+        File path.
+        If path is a string with wildcard e.g. '/to/path/out-*.csv',
+        to_csv will try to write multiple files, for instance,
+        chunk (0, 0) will write data into '/to/path/out-0.csv'.
+        If path is a string without wildcard,
+        all data will be written into a single file.
+    sep : str, default ','
+        String of length 1. Field delimiter for the output file.
+    na_rep : str, default ''
+        Missing data representation.
+    float_format : str, default None
+        Format string for floating point numbers.
+    columns : sequence, optional
+        Columns to write.
+    header : bool or list of str, default True
+        Write out the column names. If a list of strings is given it is
+        assumed to be aliases for the column names.
+    index : bool, default True
+        Write row names (index).
+    index_label : str or sequence, or False, default None
+        Column label for index column(s) if desired. If None is given, and
+        `header` and `index` are True, then the index names are used. A
+        sequence should be given if the object uses MultiIndex. If
+        False do not print fields for index names. Use index_label=False
+        for easier importing in R.
+    mode : str
+        Python write mode, default 'w'.
+    encoding : str, optional
+        A string representing the encoding to use in the output file,
+        defaults to 'utf-8'.
+    compression : str or dict, default 'infer'
+        If str, represents compression mode. If dict, value at 'method' is
+        the compression mode. Compression mode may be any of the following
+        possible values: {'infer', 'gzip', 'bz2', 'zip', 'xz', None}. If
+        compression mode is 'infer' and `path_or_buf` is path-like, then
+        detect compression mode from the following extensions: '.gz',
+        '.bz2', '.zip' or '.xz'. (otherwise no compression). If dict given
+        and mode is 'zip' or inferred as 'zip', other entries passed as
+        additional compression options.
+    quoting : optional constant from csv module
+        Defaults to csv.QUOTE_MINIMAL. If you have set a `float_format`
+        then floats are converted to strings and thus csv.QUOTE_NONNUMERIC
+        will treat them as non-numeric.
+    quotechar : str, default '\"'
+        String of length 1. Character used to quote fields.
+    lineterminator : str, optional
+        The newline character or character sequence to use in the output
+        file. Defaults to `os.linesep`, which depends on the OS in which
+        this method is called ('\n' for linux, '\r\n' for Windows, i.e.).
+    chunksize : int or None
+        Rows to write at a time.
+    date_format : str, default None
+        Format string for datetime objects.
+    doublequote : bool, default True
+        Control quoting of `quotechar` inside a field.
+    escapechar : str, default None
+        String of length 1. Character used to escape `sep` and `quotechar`
+        when appropriate.
+    decimal : str, default '.'
+        Character recognized as decimal separator. E.g. use ',' for
+        European data.
+    partition_cols : list, optional, default None
+        Column names by which to partition the dataset.
+        Columns are partitioned in the order they are given.
+    Returns
+    -------
+    None or str
+        If path_or_buf is None, returns the resulting csv format as a
+        string. Otherwise returns None.
+
+    See Also
+    --------
+    read_csv : Load a CSV file into a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'name': ['Raphael', 'Donatello'],
+    ...                    'mask': ['red', 'purple'],
+    ...                    'weapon': ['sai', 'bo staff']})
+    >>> df.to_csv('out.csv', index=False).execute()
+    >>> # Write partitioned dataset
+    >>> df.to_csv('dataset', partition_cols=['mask']).execute()  # doctest: +SKIP
+    """
+    lineterminator = lineterminator or kw.pop("line_terminator", None)
+    if kw:
+        raise TypeError(
+            f"to_csv() got an unexpected keyword argument '{next(iter(kw))}'"
+        )
+
+    if mode != "w":  # pragma: no cover
+        raise NotImplementedError("only support to_csv with mode 'w' for now")
+
+    csv_kwargs = dict(
+        float_format=float_format,
+        quoting=quoting,
+        quotechar=quotechar,
+        line_terminator=lineterminator,
+        date_format=date_format,
+        doublequote=doublequote,
+        escapechar=escapechar,
+        decimal=decimal,
+    )
+    op = DataFrameToCSV(
+        path=path,
+        sep=sep,
+        na_rep=na_rep,
+        columns=columns,
+        header=header,
+        index=index,
+        index_label=index_label,
+        mode=mode,
+        encoding=encoding,
+        compression=compression,
+        chunksize=chunksize,
+        storage_options=storage_options,
+        partition_cols=partition_cols,
+        csv_kwargs=csv_kwargs,
+    )
+    return op(df)