maxframe 1.0.0rc1__cp38-cp38-macosx_10_9_universal2.whl → 1.0.0rc3__cp38-cp38-macosx_10_9_universal2.whl

maxframe_client/fetcher.py

@@ -12,23 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
-import json
 from abc import ABC, abstractmethod
 from numbers import Integral
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
 import pandas as pd
 import pyarrow as pa
 from odps import ODPS
-from odps.models import ExternalVolume, PartedVolume
+from odps.models import ExternalVolume
 from odps.tunnel import TableTunnel
-from tornado import httpclient
 
 from maxframe.core import OBJECT_TYPE
 from maxframe.dataframe.core import DATAFRAME_TYPE
-from maxframe.lib import wrapped_pickle as pickle
-from maxframe.odpsio import HaloTableIO, arrow_to_pandas, build_dataframe_table_meta
+from maxframe.io.objects import get_object_io_handler
+from maxframe.io.odpsio import (
+    ODPSTableIO,
+    ODPSVolumeReader,
+    arrow_to_pandas,
+    build_dataframe_table_meta,
+)
 from maxframe.protocol import (
     DataFrameTableMeta,
     ODPSTableResultInfo,
@@ -38,7 +40,7 @@ from maxframe.protocol import (
 )
 from maxframe.tensor.core import TENSOR_TYPE
 from maxframe.typing_ import PandasObjectTypes, TileableType
-from maxframe.utils import ToThreadMixin, deserialize_serializable
+from maxframe.utils import ToThreadMixin
 
 _result_fetchers: Dict[ResultType, Type["ResultFetcher"]] = dict()
 
@@ -109,17 +111,12 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
         tileable: TileableType,
         info: ODPSTableResultInfo,
     ) -> None:
-        if isinstance(tileable, DATAFRAME_TYPE) and tileable.dtypes is None:
-            tb_comment = await self.to_thread(
-                self._get_table_comment, info.full_table_name
-            )
-            if tb_comment:  # pragma: no branch
-                comment_data = json.loads(tb_comment)
-
-                table_meta: DataFrameTableMeta = deserialize_serializable(
-                    base64.b64decode(comment_data["table_meta"])
-                )
-                tileable.refresh_from_table_meta(table_meta)
+        if (
+            isinstance(tileable, DATAFRAME_TYPE)
+            and tileable.dtypes is None
+            and info.table_meta is not None
+        ):
+            tileable.refresh_from_table_meta(info.table_meta)
 
         if tileable.shape and any(pd.isna(x) for x in tileable.shape):
             part_specs = [None] if not info.partition_specs else info.partition_specs
@@ -131,16 +128,39 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
                 )
                 total_records += session.count
             new_shape_list = list(tileable.shape)
-            new_shape_list[-1] = total_records
+            new_shape_list[0] = total_records
             tileable.params = {"shape": tuple(new_shape_list)}
 
+    @staticmethod
+    def _align_selection_with_shape(
+        row_sel: slice, shape: Tuple[Optional[int], ...]
+    ) -> dict:
+        size = shape[0]
+        if not row_sel.start and not row_sel.stop:
+            return {}
+        is_reversed = row_sel.step is not None and row_sel.step < 0
+        read_kw = {
+            "start": row_sel.start,
+            "stop": row_sel.stop,
+            "reverse_range": is_reversed,
+        }
+        if pd.isna(size):
+            return read_kw
+
+        if is_reversed and row_sel.start is not None:
+            read_kw["start"] = min(size - 1, row_sel.start)
+        if not is_reversed and row_sel.stop is not None:
+            read_kw["stop"] = min(size, row_sel.stop)
+        return read_kw
+
     def _read_single_source(
         self,
         table_meta: DataFrameTableMeta,
         info: ODPSTableResultInfo,
         indexes: List[Union[None, Integral, slice]],
+        shape: Tuple[Optional[int], ...],
     ):
-        table_io = HaloTableIO(self._odps_entry)
+        table_io = ODPSTableIO(self._odps_entry)
         read_kw = {}
         row_step = None
         if indexes:
@@ -148,13 +168,8 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
                 indexes += [None]
             row_sel, col_sel = indexes
             if isinstance(row_sel, slice):
-                if row_sel.start or row_sel.stop:
-                    read_kw["start"] = row_sel.start
-                    read_kw["stop"] = row_sel.stop
-                    read_kw["reverse_range"] = (
-                        row_sel.step is not None and row_sel.step < 0
-                    )
-                    row_step = row_sel.step
+                row_step = row_sel.step
+                read_kw = self._align_selection_with_shape(row_sel, shape)
             elif isinstance(row_sel, int):
                 read_kw["start"] = row_sel
                 read_kw["stop"] = row_sel + 1
@@ -173,8 +188,8 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
         with table_io.open_reader(
             info.full_table_name, info.partition_specs, **read_kw
         ) as reader:
-            reader_count = reader.count
             result = reader.read_all()
+            reader_count = result.num_rows
 
         if not row_step:
             return result
@@ -195,7 +210,7 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
     ) -> PandasObjectTypes:
         table_meta = build_dataframe_table_meta(tileable)
         arrow_table: pa.Table = await self.to_thread(
-            self._read_single_source, table_meta, info, indexes
+            self._read_single_source, table_meta, info, indexes, tileable.shape
         )
         return arrow_to_pandas(arrow_table, table_meta)
 
@@ -211,47 +226,24 @@ class ODPSVolumeFetcher(ToThreadMixin, ResultFetcher):
     ) -> None:
         return
 
-    async def _read_parted_volume_data(
-        self, volume: PartedVolume, partition: str, file_name: str
-    ) -> bytes:
-        def sync_read():
-            with volume.open_reader(partition, file_name) as reader:
-                return reader.read()
-
-        return await self.to_thread(sync_read)
-
-    async def _read_external_volume_data(
-        self, volume: ExternalVolume, path: str, file_name: str
-    ) -> bytes:
-        signed_url = await self.to_thread(
-            volume.get_sign_url, path + "/" + file_name, "GET"
-        )
-        http_client = httpclient.AsyncHTTPClient()
-
-        resp = await http_client.fetch(signed_url)
-        if hasattr(resp, "status_code") and resp.code >= 400:
-            try:
-                import oss2.exceptions
-
-                oss_exc = oss2.exceptions.make_exception(resp.body)
-                raise oss_exc
-            except ImportError:
-                raise SystemError(resp.body)
-        return resp.body
+    async def _fetch_object(
+        self,
+        tileable: TileableType,
+        info: ODPSVolumeResultInfo,
+        indexes: List[Union[Integral, slice]],
+    ) -> Any:
+        def volume_fetch_func():
+            reader = ODPSVolumeReader(
+                self._odps_entry, info.volume_name, info.volume_path
+            )
+            io_handler = get_object_io_handler(tileable)()
+            return io_handler.read_object(reader, tileable, indexes)
 
-    async def _fetch_object(self, info: ODPSVolumeResultInfo) -> Any:
         volume = await self.to_thread(self._odps_entry.get_volume, info.volume_name)
-        if isinstance(volume, PartedVolume):
-            byte_data = await self._read_parted_volume_data(
-                volume, info.volume_path, "data"
-            )
-        elif isinstance(volume, ExternalVolume):
-            byte_data = await self._read_external_volume_data(
-                volume, info.volume_path, "data"
-            )
+        if isinstance(volume, ExternalVolume):
+            return await self.to_thread(volume_fetch_func)
         else:
             raise NotImplementedError(f"Volume type {type(volume)} not supported")
-        return pickle.loads(byte_data)
 
     async def fetch(
         self,
@@ -260,5 +252,5 @@ class ODPSVolumeFetcher(ToThreadMixin, ResultFetcher):
         indexes: List[Union[Integral, slice]],
     ) -> Any:
         if isinstance(tileable, (OBJECT_TYPE, TENSOR_TYPE)):
-            return await self._fetch_object(info)
+            return await self._fetch_object(tileable, info, indexes)
         raise NotImplementedError(f"Fetching {type(tileable)} not implemented")

maxframe_client/session/graph.py

@@ -19,10 +19,16 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Tuple, Union
 from weakref import WeakSet
 
-from maxframe.core import ChunkType, TileableGraph, TileableType, enter_mode
+from maxframe.core import (
+    ChunkType,
+    TileableGraph,
+    TileableType,
+    build_fetch,
+    enter_mode,
+)
 from maxframe.core.operator import Fetch
 from maxframe.session import AbstractSession
-from maxframe.utils import build_fetch, copy_tileables
+from maxframe.utils import copy_tileables
 
 logger = logging.getLogger(__name__)
 

maxframe_client/session/odps.py

@@ -26,7 +26,8 @@ import pandas as pd
 from odps import ODPS
 
 from maxframe.config import options
-from maxframe.core import Entity, TileableGraph, enter_mode
+from maxframe.core import Entity, TileableGraph, build_fetch, enter_mode
+from maxframe.core.operator import Fetch
 from maxframe.dataframe import read_odps_table
 from maxframe.dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
 from maxframe.dataframe.datasource import PandasDataSourceOperator
@@ -36,11 +37,18 @@ from maxframe.errors import (
     NoTaskServerResponseError,
     SessionAlreadyClosedError,
 )
-from maxframe.odpsio import HaloTableIO, pandas_to_arrow, pandas_to_odps_schema
+from maxframe.io.objects import get_object_io_handler
+from maxframe.io.odpsio import (
+    ODPSTableIO,
+    ODPSVolumeWriter,
+    pandas_to_arrow,
+    pandas_to_odps_schema,
+)
 from maxframe.protocol import (
     DagInfo,
     DagStatus,
     ODPSTableResultInfo,
+    ODPSVolumeResultInfo,
     ResultInfo,
     SessionInfo,
 )
@@ -51,8 +59,13 @@ from maxframe.session import (
     Profiling,
     Progress,
 )
+from maxframe.tensor.datasource import ArrayDataSource
 from maxframe.typing_ import TileableType
-from maxframe.utils import ToThreadMixin, build_temp_table_name
+from maxframe.utils import (
+    ToThreadMixin,
+    build_session_volume_name,
+    build_temp_table_name,
+)
 
 from ..clients.framedriver import FrameDriverClient
 from ..fetcher import get_fetcher_cls
@@ -139,14 +152,9 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         self._session_id = session_info.session_id
         await self._show_logview_address()
 
-    def _upload_and_get_read_tileable(self, t: TileableType) -> Optional[TileableType]:
-        if (
-            not isinstance(t.op, PandasDataSourceOperator)
-            or t.op.get_data() is None
-            or t.inputs
-        ):
-            return None
-
+    def _upload_and_get_table_read_tileable(
+        self, t: TileableType
+    ) -> Optional[TileableType]:
         schema, table_meta = pandas_to_odps_schema(t, unknown_as_string=True)
         if self._odps_entry.exist_table(table_meta.table_name):
             self._odps_entry.delete_table(
@@ -164,8 +172,8 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         batch_size = options.session.upload_batch_size
 
         if len(data):
-            halo_client = HaloTableIO(self._odps_entry)
-            with halo_client.open_writer(table_obj.full_table_name) as writer:
+            table_client = ODPSTableIO(self._odps_entry)
+            with table_client.open_writer(table_obj.full_table_name) as writer:
                 for batch_start in range(0, len(data), batch_size):
                     if isinstance(data, pd.Index):
                         batch = data[batch_start : batch_start + batch_size]
@@ -188,13 +196,34 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
                 read_tileable.name = t.name
         else:  # INDEX_TYPE
             if list(read_tileable.names) != list(t.names):
-                read_tileable.names = t.names
+                read_tileable.rename(t.names, inplace=True)
         read_tileable._key = t.key
         read_tileable.params = t.params
         return read_tileable.data
 
+    def _upload_and_get_vol_read_tileable(
+        self, t: TileableType
+    ) -> Optional[TileableType]:
+        vol_name = build_session_volume_name(self.session_id)
+        writer = ODPSVolumeWriter(self._odps_entry, vol_name, t.key)
+        io_handler = get_object_io_handler(t)
+        io_handler().write_object(writer, t, t.op.data)
+        return build_fetch(t).data
+
+    def _upload_and_get_read_tileable(self, t: TileableType) -> Optional[TileableType]:
+        if (
+            not isinstance(t.op, (ArrayDataSource, PandasDataSourceOperator))
+            or t.op.get_data() is None
+            or t.inputs
+        ):
+            return None
+        if isinstance(t.op, PandasDataSourceOperator):
+            return self._upload_and_get_table_read_tileable(t)
+        else:
+            return self._upload_and_get_vol_read_tileable(t)
+
     @enter_mode(kernel=True, build=True)
-    def _scan_and_replace_pandas_sources(
+    def _scan_and_replace_local_sources(
         self, graph: TileableGraph
     ) -> Dict[TileableType, TileableType]:
         """Replaces Pandas data sources with temp table sources in the graph"""
@@ -223,14 +252,21 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
     @enter_mode(kernel=True, build=True)
     def _get_input_infos(self, tileables: List[TileableType]) -> Dict[str, ResultInfo]:
         """Generate ResultInfo structs from generated temp tables"""
+        vol_name = build_session_volume_name(self.session_id)
+
         infos = dict()
         for t in tileables:
             key = t.key
-            if not isinstance(t.op, DataFrameReadODPSTable):
-                if not isinstance(t.inputs[0].op, DataFrameReadODPSTable):
-                    continue
-                t = t.inputs[0]
-            infos[key] = ODPSTableResultInfo(full_table_name=t.op.table_name)
+            if isinstance(t.op, DataFrameReadODPSTable):
+                infos[key] = ODPSTableResultInfo(full_table_name=t.op.table_name)
+            else:
+                if isinstance(t.op, Fetch):
+                    infos[key] = ODPSVolumeResultInfo(
+                        volume_name=vol_name, volume_path=t.key
+                    )
+                elif t.inputs and isinstance(t.inputs[0].op, DataFrameReadODPSTable):
+                    t = t.inputs[0]
+                    infos[key] = ODPSTableResultInfo(full_table_name=t.op.table_name)
         return infos
 
     async def execute(self, *tileables, **kwargs) -> ExecutionInfo:
@@ -242,7 +278,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         tileable_graph, to_execute_tileables = gen_submit_tileable_graph(
             self, tileables, tileable_to_copied
         )
-        source_replacements = self._scan_and_replace_pandas_sources(tileable_graph)
+        source_replacements = self._scan_and_replace_local_sources(tileable_graph)
 
         # we need to manage uploaded data sources with refcounting mechanism
         # as nodes in tileable_graph are copied, we need to use original nodes
@@ -384,7 +420,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
                 data_tileable, indexes = self._get_data_tileable_and_indexes(tileable)
                 info = self._tileable_to_infos[data_tileable]
                 fetcher = get_fetcher_cls(info.result_type)(self._odps_entry)
-                results.append(await fetcher.fetch(tileable, info, indexes))
+                results.append(await fetcher.fetch(data_tileable, info, indexes))
         return results
 
     async def decref(self, *tileable_keys):

maxframe_client/tests/test_fetcher.py

@@ -17,19 +17,32 @@ import uuid
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pytest
 from odps import ODPS
 
 import maxframe.dataframe as md
-from maxframe.odpsio import HaloTableIO
+from maxframe.config import options
+from maxframe.io.odpsio import ODPSTableIO
 from maxframe.protocol import ODPSTableResultInfo, ResultType
 from maxframe.tests.utils import tn
 
 from ..fetcher import ODPSTableFetcher
 
 
-async def test_table_fetcher():
+@pytest.fixture
+def switch_table_io(request):
+    old_use_common_table = options.use_common_table
+    try:
+        options.use_common_table = request.param
+        yield
+    finally:
+        options.use_common_table = old_use_common_table
+
+
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+async def test_table_fetcher(switch_table_io):
     odps_entry = ODPS.from_environments()
-    halo_table_io = HaloTableIO(odps_entry)
+    halo_table_io = ODPSTableIO(odps_entry)
     fetcher = ODPSTableFetcher(odps_entry)
 
     data = pd.DataFrame(
@@ -58,6 +71,11 @@ async def test_table_fetcher():
     assert len(fetched) == 1000
     pd.testing.assert_frame_equal(raw_data, fetched)
 
+    result_info = ODPSTableResultInfo(ResultType.ODPS_TABLE, full_table_name=table_name)
+    fetched = await fetcher.fetch(tileable, result_info, [slice(None, 2000), None])
+    assert len(fetched) == 1000
+    pd.testing.assert_frame_equal(raw_data, fetched)
+
     result_info = ODPSTableResultInfo(ResultType.ODPS_TABLE, full_table_name=table_name)
     fetched = await fetcher.fetch(tileable, result_info, [2, None])
     assert len(fetched) == 1

maxframe_client/tests/test_session.py

@@ -195,7 +195,8 @@ def test_run_dataframe_from_to_odps_table(start_mock_session):
         assert len(result_df) == 10
         assert len(result_df.columns) == 6
 
-        df = md.read_odps_table(table_obj, index_col="index").head(10).execute().fetch()
+        df = md.read_odps_table(table_obj, index_col="index").head(10).execute()
+        assert df.shape == (10, 5)
         assert len(df) == 10
         assert len(df.columns) == 5
     finally:
@@ -246,7 +247,19 @@ def test_run_and_fetch_series(start_mock_session):
         )
 
 
-def test_run_remote_success(start_mock_session):
+def test_execute_with_tensor(oss_config, start_mock_session):
+    pd_df = pd.DataFrame(
+        {"angles": [0, 3, 4], "degrees": [360, 180, 360]},
+        index=["circle", "triangle", "rectangle"],
+    )
+    df = md.DataFrame(pd_df)
+
+    result = (df - [1, 2]).execute().fetch()
+    expected = pd_df - [1, 2]
+    pd.testing.assert_frame_equal(result, expected)
+
+
+def test_run_remote_success(oss_config, start_mock_session):
     def func(a, b):
         return a + b
 
@@ -257,7 +270,7 @@ def test_run_remote_success(start_mock_session):
     assert result == 21
 
 
-def test_run_remote_error(start_mock_session):
+def test_run_remote_error(oss_config, start_mock_session):
     def func():
         raise ValueError
 
@@ -280,7 +293,7 @@ def test_pivot_dataframe(start_mock_session):
     df = md.DataFrame(pd_df)
     pivot = df.pivot_table(values="D", index=["A", "B"], columns=["C"], aggfunc="sum")
     executed = pivot.execute()
-    assert pivot.shape == (2, 4)
+    assert pivot.shape == (4, 2)
     pd.testing.assert_index_equal(
         pivot.dtypes.index, pd.Index(["large", "small"], name="C")
     )
@@ -289,3 +302,13 @@ def test_pivot_dataframe(start_mock_session):
         values="D", index=["A", "B"], columns=["C"], aggfunc="sum"
     )
     pd.testing.assert_frame_equal(executed.to_pandas(), expected)
+
+
+def test_index_drop_duplicates(start_mock_session):
+    pd_idx = pd.Index(["lame", "cow", "lame", "beetle", "lame", "hippo"])
+    idx = md.Index(pd_idx)
+    executed = idx.drop_duplicates(keep="first").execute()
+    expected = pd_idx.drop_duplicates(keep="first")
+    pd.testing.assert_index_equal(
+        executed.to_pandas().sort_values(), expected.sort_values()
+    )

maxframe/core/entity/chunks.py

@@ -1,68 +0,0 @@
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from ...serialization.serializables import BoolField, FieldTypes, TupleField
-from ...utils import tokenize
-from .core import Entity, EntityData
-
-
-class ChunkData(EntityData):
-    __slots__ = ()
-
-    is_broadcaster = BoolField("is_broadcaster", default=False)
-    # If the operator is a shuffle mapper, this flag indicates whether the current chunk is mapper chunk when
-    # the operator produce multiple chunks such as TensorUnique.
-    is_mapper = BoolField("is_mapper", default=None)
-    # optional fields
-    _index = TupleField("index", FieldTypes.uint32)
-
-    def __repr__(self):
-        if self.op.stage is None:
-            return (
-                f"{type(self).__name__} <op={type(self.op).__name__}, "
-                f"key={self.key}>"
-            )
-        else:
-            return (
-                f"{type(self).__name__} <op={type(self.op).__name__}, "
-                f"stage={self.op.stage.name}, key={self.key}>"
-            )
-
-    @property
-    def index(self):
-        return getattr(self, "_index", None)
-
-    @property
-    def device(self):
-        return self.op.device
-
-    def _update_key(self):
-        object.__setattr__(
-            self,
-            "_key",
-            tokenize(
-                type(self).__name__,
-                *(getattr(self, k, None) for k in self._keys_ if k != "_index"),
-            ),
-        )
-
-
-class Chunk(Entity):
-    _allow_data_type_ = (ChunkData,)
-
-    def __repr__(self):
-        return f"{type(self).__name__}({self._data.__repr__()})"
-
-
-CHUNK_TYPE = (Chunk, ChunkData)

maxframe/core/entity/fuse.py

@@ -1,73 +0,0 @@
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-
-from ...serialization.serializables import ReferenceField
-from .chunks import CHUNK_TYPE, Chunk, ChunkData
-
-
-class FuseChunkData(ChunkData):
-    __slots__ = ("_inited",)
-
-    _chunk = ReferenceField(
-        "chunk", CHUNK_TYPE, on_serialize=lambda x: x.data if hasattr(x, "data") else x
-    )
-
-    def __init__(self, *args, **kwargs):
-        self._inited = False
-        super().__init__(*args, **kwargs)
-        self._extra_params = {}
-        self._inited = True
-
-    @property
-    def chunk(self):
-        return self._chunk
-
-    @property
-    def composed(self):
-        # for compatibility, just return the topological ordering,
-        # once we apply optimization on the subgraph,
-        # `composed` is not needed any more and should be removed then.
-        assert getattr(self._op, "fuse_graph", None) is not None
-        fuse_graph = self._op.fuse_graph
-        return list(fuse_graph.topological_iter())
-
-    def __getattr__(self, attr):
-        if not self._inited:
-            return object.__getattribute__(self, attr)
-        if attr in self._extra_params:
-            return self._extra_params[attr]
-        try:
-            return getattr(self._chunk, attr)
-        except AttributeError:
-            return object.__getattribute__(self, attr)
-
-    def __setattr__(self, attr, value):
-        if attr == "params":
-            self._chunk.params = value
-        else:
-            super().__setattr__(attr, value)
-
-    @property
-    def nbytes(self):
-        return np.prod(self.shape) * self.dtype.itemsize
-
-
-class FuseChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (FuseChunkData,)
-
-
-FUSE_CHUNK_TYPE = (FuseChunkData, FuseChunk)